682 files changed, 52133 insertions, 20231 deletions

diff --git a/Documentation/DocBook/80211.tmpl b/Documentation/DocBook/80211.tmpl
index 5f7c55999c77..800fe7a9024c 100644
--- a/Documentation/DocBook/80211.tmpl
+++ b/Documentation/DocBook/80211.tmpl
@@ -136,6 +136,7 @@
 !Finclude/net/cfg80211.h cfg80211_ibss_joined
 !Finclude/net/cfg80211.h cfg80211_connect_result
 !Finclude/net/cfg80211.h cfg80211_connect_bss
+!Finclude/net/cfg80211.h cfg80211_connect_timeout
 !Finclude/net/cfg80211.h cfg80211_roamed
 !Finclude/net/cfg80211.h cfg80211_disconnected
 !Finclude/net/cfg80211.h cfg80211_ready_on_channel
diff --git a/Documentation/devicetree/bindings/net/brcm,mdio-mux-iproc.txt b/Documentation/devicetree/bindings/net/brcm,mdio-mux-iproc.txt
new file mode 100644
index 000000000000..dfe287a5d6f2
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/brcm,mdio-mux-iproc.txt
@@ -0,0 +1,59 @@
+Properties for an MDIO bus multiplexer found in Broadcom iProc based SoCs.
+
+This MDIO bus multiplexer defines buses that could be internal as well as
+external to SoCs and could accept MDIO transaction compatible to C-22 or
+C-45 Clause. When child bus is selected, one needs to select these two
+properties as well to generate desired MDIO transaction on appropriate bus.
+
+Required properties in addition to the generic multiplexer properties:
+
+MDIO multiplexer node:
+- compatible: brcm,mdio-mux-iproc.
+
+Every non-ethernet PHY requires a compatible so that it could be probed based
+on this compatible string.
+
+Additional information regarding generic multiplexer properties can be found
+at- Documentation/devicetree/bindings/net/mdio-mux.txt
+
+
+for example:
+                mdio_mux_iproc: mdio-mux@6602023c {
+                        compatible = "brcm,mdio-mux-iproc";
+                        reg = <0x6602023c 0x14>;
+                        #address-cells = <1>;
+                        #size-cells = <0>;
+
+                        mdio@0 {
+                                reg = <0x0>;
+                                #address-cells = <1>;
+                                #size-cells = <0>;
+
+                                pci_phy0: pci-phy@0 {
+                                        compatible = "brcm,ns2-pcie-phy";
+                                        reg = <0x0>;
+                                        #phy-cells = <0>;
+                                };
+                        };
+
+                        mdio@7 {
+                                reg = <0x7>;
+                                #address-cells = <1>;
+                                #size-cells = <0>;
+
+                                pci_phy1: pci-phy@0 {
+                                        compatible = "brcm,ns2-pcie-phy";
+                                        reg = <0x0>;
+                                        #phy-cells = <0>;
+                                };
+                        };
+                        mdio@10 {
+                                reg = <0x10>;
+                                #address-cells = <1>;
+                                #size-cells = <0>;
+
+                                gphy0: eth-phy@10 {
+                                        reg = <0x10>;
+                                };
+                        };
+                };
diff --git a/Documentation/devicetree/bindings/net/can/rcar_canfd.txt b/Documentation/devicetree/bindings/net/can/rcar_canfd.txt
new file mode 100644
index 000000000000..22a6f10bab05
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/can/rcar_canfd.txt
@@ -0,0 +1,96 @@
+Renesas R-Car CAN FD controller Device Tree Bindings
+----------------------------------------------------
+
+Required properties:
+- compatible: Must contain one or more of the following:
+  - "renesas,rcar-gen3-canfd" for R-Car Gen3 compatible controller.
+  - "renesas,r8a7795-canfd" for R8A7795 (R-Car H3) compatible controller.
+
+  When compatible with the generic version, nodes must list the
+  SoC-specific version corresponding to the platform first, followed by the
+  family-specific and/or generic versions.
+
+- reg: physical base address and size of the R-Car CAN FD register map.
+- interrupts: interrupt specifier for the Global & Channel interrupts
+- clocks: phandles and clock specifiers for 3 clock inputs.
+- clock-names: 3 clock input name strings: "fck", "canfd", "can_clk".
+- pinctrl-0: pin control group to be used for this controller.
+- pinctrl-names: must be "default".
+
+Required child nodes:
+The controller supports two channels and each is represented as a child node.
+The name of the child nodes are "channel0" and "channel1" respectively. Each
+child node supports the "status" property only, which is used to
+enable/disable the respective channel.
+
+Required properties for "renesas,r8a7795-canfd" compatible:
+In R8A7795 SoC, canfd clock is a div6 clock and can be used by both CAN
+and CAN FD controller at the same time. It needs to be scaled to maximum
+frequency if any of these controllers use it. This is done using the
+below properties.
+
+- assigned-clocks: phandle of canfd clock.
+- assigned-clock-rates: maximum frequency of this clock.
+
+Optional property:
+The controller can operate in either CAN FD only mode (default) or
+Classical CAN only mode. The mode is global to both the channels. In order to
+enable the later, define the following optional property.
+ - renesas,no-can-fd: puts the controller in Classical CAN only mode.
+
+Example
+-------
+
+SoC common .dtsi file:
+
+                canfd: can@e66c0000 {
+                        compatible = "renesas,r8a7795-canfd",
+                                     "renesas,rcar-gen3-canfd";
+                        reg = <0 0xe66c0000 0 0x8000>;
+                        interrupts = <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>,
+                                   <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
+                        clocks = <&cpg CPG_MOD 914>,
+                               <&cpg CPG_CORE R8A7795_CLK_CANFD>,
+                               <&can_clk>;
+                        clock-names = "fck", "canfd", "can_clk";
+                        assigned-clocks = <&cpg CPG_CORE R8A7795_CLK_CANFD>;
+                        assigned-clock-rates = <40000000>;
+                        power-domains = <&cpg>;
+                        status = "disabled";
+
+                        channel0 {
+                                status = "disabled";
+                        };
+
+                        channel1 {
+                                status = "disabled";
+                        };
+                };
+
+Board specific .dts file:
+
+E.g. below enables Channel 1 alone in the board in Classical CAN only mode.
+
+&canfd {
+        pinctrl-0 = <&canfd1_pins>;
+        pinctrl-names = "default";
+        renesas,no-can-fd;
+        status = "okay";
+
+        channel1 {
+                status = "okay";
+        };
+};
+
+E.g. below enables Channel 0 alone in the board using External clock
+as fCAN clock.
+
+&canfd {
+        pinctrl-0 = <&canfd0_pins &can_clk_pins>;
+        pinctrl-names = "default";
+        status = "okay";
+
+        channel0 {
+                status = "okay";
+        };
+};
diff --git a/Documentation/devicetree/bindings/net/cirrus,cs89x0.txt b/Documentation/devicetree/bindings/net/cirrus,cs89x0.txt
new file mode 100644
index 000000000000..c070076bacb9
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/cirrus,cs89x0.txt
@@ -0,0 +1,13 @@
+* Cirrus Logic CS8900/CS8920 Network Controller
+
+Required properties:
+- compatible    : Should be "cirrus,cs8900" or "cirrus,cs8920".
+- reg           : Address and length of the IO space.
+- interrupts    : Should contain the controller interrupt line.
+
+Examples:
+        eth0: eth@10000000 {
+                compatible = "cirrus,cs8900";
+                reg = <0x10000000 0x400>;
+                interrupts = <10>;
+        };
diff --git a/Documentation/devicetree/bindings/net/cpsw.txt b/Documentation/devicetree/bindings/net/cpsw.txt
index 0ae06491b430..5ad439f30135 100644
--- a/Documentation/devicetree/bindings/net/cpsw.txt
+++ b/Documentation/devicetree/bindings/net/cpsw.txt
@@ -15,7 +15,6 @@ Required properties:
 - cpdma_channels        : Specifies number of channels in CPDMA
 - ale_entries           : Specifies No of entries ALE can hold
 - bd_ram_size           : Specifies internal descriptor RAM size
-- rx_descs              : Specifies number of Rx descriptors
 - mac_control           : Specifies Default MAC control register content
                           for the specific platform
 - slaves                : Specifies number for slaves
diff --git a/Documentation/devicetree/bindings/net/davinci-mdio.txt b/Documentation/devicetree/bindings/net/davinci-mdio.txt
index 0369e25aabd2..621156ca4ffd 100644
--- a/Documentation/devicetree/bindings/net/davinci-mdio.txt
+++ b/Documentation/devicetree/bindings/net/davinci-mdio.txt
@@ -2,7 +2,10 @@ TI SoC Davinci/Keystone2 MDIO Controller Device Tree Bindings
 ---------------------------------------------------
 
 Required properties:
-- compatible            : Should be "ti,davinci_mdio" or "ti,keystone_mdio"
+- compatible            : Should be "ti,davinci_mdio"
+                          and "ti,keystone_mdio" for Keystone 2 SoCs
+                          and "ti,cpsw-mdio" for am335x, am472x, am57xx/dra7, dm814x SoCs
+                          and "ti,am4372-mdio" for am472x SoC
 - reg                   : physical base address and size of the davinci mdio
                           registers map
 - bus_freq              : Mdio Bus frequency
diff --git a/Documentation/devicetree/bindings/net/dsa/b53.txt b/Documentation/devicetree/bindings/net/dsa/b53.txt
new file mode 100644
index 000000000000..ca752db14dff
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/dsa/b53.txt
@@ -0,0 +1,88 @@
+Broadcom BCM53xx Ethernet switches
+==================================
+
+Required properties:
+
+- compatible: For external switch chips, compatible string must be exactly one
+  of: "brcm,bcm5325"
+      "brcm,bcm53115"
+      "brcm,bcm53125"
+      "brcm,bcm53128"
+      "brcm,bcm5365"
+      "brcm,bcm5395"
+      "brcm,bcm5397"
+      "brcm,bcm5398"
+
+  For the BCM5310x SoCs with an integrated switch, must be one of:
+      "brcm,bcm53010-srab"
+      "brcm,bcm53011-srab"
+      "brcm,bcm53012-srab"
+      "brcm,bcm53018-srab"
+      "brcm,bcm53019-srab" and the mandatory "brcm,bcm5301x-srab" string
+
+  For the BCM63xx/33xx SoCs with an integrated switch, must be one of:
+      "brcm,bcm3384-switch"
+      "brcm,bcm6328-switch"
+      "brcm,bcm6368-switch" and the mandatory "brcm,bcm63xx-switch"
+
+See Documentation/devicetree/bindings/dsa/dsa.txt for a list of additional
+required and optional properties.
+
+Examples:
+
+Ethernet switch connected via MDIO to the host, CPU port wired to eth0:
+
+        eth0: ethernet@10001000 {
+                compatible = "brcm,unimac";
+                reg = <0x10001000 0x1000>;
+
+                fixed-link {
+                        speed = <1000>;
+                        duplex-full;
+                };
+        };
+
+        mdio0: mdio@10000000 {
+                compatible = "brcm,unimac-mdio";
+                #address-cells = <1>;
+                #size-cells = <0>;
+
+                switch0: ethernet-switch@30 {
+                        compatible = "brcm,bcm53125";
+                        #address-cells = <1>;
+                        #size-cells = <0>;
+
+                        ports {
+                                port0@0 {
+                                        reg = <0>;
+                                        label = "lan1";
+                                };
+
+                                port1@1 {
+                                        reg = <1>;
+                                        label = "lan2";
+                                };
+
+                                port5@5 {
+                                        reg = <5>;
+                                        label = "cable-modem";
+                                        fixed-link {
+                                                speed = <1000>;
+                                                duplex-full;
+                                        };
+                                        phy-mode = "rgmii-txid";
+                                };
+
+                                port8@8 {
+                                        reg = <8>;
+                                        label = "cpu";
+                                        fixed-link {
+                                                speed = <1000>;
+                                                duplex-full;
+                                        };
+                                        phy-mode = "rgmii-txid";
+                                        ethernet = <&eth0>;
+                                };
+                        };
+                };
+        };
diff --git a/Documentation/devicetree/bindings/net/dsa/dsa.txt b/Documentation/devicetree/bindings/net/dsa/dsa.txt
index 9f4807f90c31..9bbbe7f87d67 100644
--- a/Documentation/devicetree/bindings/net/dsa/dsa.txt
+++ b/Documentation/devicetree/bindings/net/dsa/dsa.txt
@@ -1,5 +1,279 @@
-Marvell Distributed Switch Architecture Device Tree Bindings
-------------------------------------------------------------
+Distributed Switch Architecture Device Tree Bindings
+----------------------------------------------------
+
+Two bindings exist, one of which has been deprecated due to
+limitations.
+
+Current Binding
+---------------
+
+Switches are true Linux devices and can be probes by any means. Once
+probed, they register to the DSA framework, passing a node
+pointer. This node is expected to fulfil the following binding, and
+may contain additional properties as required by the device it is
+embedded within.
+
+Required properties:
+
+- ports         : A container for child nodes representing switch ports.
+
+Optional properties:
+
+- dsa,member    : A two element list indicates which DSA cluster, and position
+                  within the cluster a switch takes. <0 0> is cluster 0,
+                  switch 0. <0 1> is cluster 0, switch 1. <1 0> is cluster 1,
+                  switch 0. A switch not part of any cluster (single device
+                  hanging off a CPU port) must not specify this property
+
+The ports container has the following properties
+
+Required properties:
+
+- #address-cells        : Must be 1
+- #size-cells           : Must be 0
+
+Each port children node must have the following mandatory properties:
+- reg                   : Describes the port address in the switch
+- label                 : Describes the label associated with this port, which
+                          will become the netdev name. Special labels are
+                          "cpu" to indicate a CPU port and "dsa" to
+                          indicate an uplink/downlink port between switches in
+                          the cluster.
+
+A port labelled "dsa" has the following mandatory property:
+
+- link                  : Should be a list of phandles to other switch's DSA
+                          port. This port is used as the outgoing port
+                          towards the phandle ports. The full routing
+                          information must be given, not just the one hop
+                          routes to neighbouring switches.
+
+A port labelled "cpu" has the following mandatory property:
+
+- ethernet              : Should be a phandle to a valid Ethernet device node.
+                          This host device is what the switch port is
+                          connected to.
+
+Port child nodes may also contain the following optional standardised
+properties, described in binding documents:
+
+- phy-handle            : Phandle to a PHY on an MDIO bus. See
+                          Documentation/devicetree/bindings/net/ethernet.txt
+                          for details.
+
+- phy-mode              : See
+                          Documentation/devicetree/bindings/net/ethernet.txt
+                          for details.
+
+- fixed-link            : Fixed-link subnode describing a link to a non-MDIO
+                          managed entity. See
+                          Documentation/devicetree/bindings/net/fixed-link.txt
+                          for details.
+
+Example
+
+The following example shows three switches on three MDIO busses,
+linked into one DSA cluster.
+
+&mdio1 {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        switch0: switch0@0 {
+                compatible = "marvell,mv88e6085";
+                #address-cells = <1>;
+                #size-cells = <0>;
+                reg = <0>;
+
+                dsa,member = <0 0>;
+
+                ports {
+                        #address-cells = <1>;
+                        #size-cells = <0>;
+                        port@0 {
+                                reg = <0>;
+                                label = "lan0";
+                        };
+
+                        port@1 {
+                                reg = <1>;
+                                label = "lan1";
+                        };
+
+                        port@2 {
+                                reg = <2>;
+                                label = "lan2";
+                        };
+
+                        switch0port5: port@5 {
+                                reg = <5>;
+                                label = "dsa";
+                                phy-mode = "rgmii-txid";
+                                link = <&switch1port6
+                                        &switch2port9>;
+                                fixed-link {
+                                        speed = <1000>;
+                                        full-duplex;
+                                };
+                        };
+
+                        port@6 {
+                                reg = <6>;
+                                label = "cpu";
+                                ethernet = <&fec1>;
+                                fixed-link {
+                                        speed = <100>;
+                                        full-duplex;
+                                };
+                        };
+                };
+        };
+};
+
+&mdio2 {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        switch1: switch1@0 {
+                compatible = "marvell,mv88e6085";
+                #address-cells = <1>;
+                #size-cells = <0>;
+                reg = <0>;
+
+                dsa,member = <0 1>;
+
+                ports {
+                        #address-cells = <1>;
+                        #size-cells = <0>;
+                        port@0 {
+                                reg = <0>;
+                                label = "lan3";
+                                phy-handle = <&switch1phy0>;
+                        };
+
+                        port@1 {
+                                reg = <1>;
+                                label = "lan4";
+                                phy-handle = <&switch1phy1>;
+                        };
+
+                        port@2 {
+                                reg = <2>;
+                                label = "lan5";
+                                phy-handle = <&switch1phy2>;
+                        };
+
+                        switch1port5: port@5 {
+                                reg = <5>;
+                                label = "dsa";
+                                link = <&switch2port9>;
+                                phy-mode = "rgmii-txid";
+                                fixed-link {
+                                        speed = <1000>;
+                                        full-duplex;
+                                };
+                        };
+
+                        switch1port6: port@6 {
+                                reg = <6>;
+                                label = "dsa";
+                                phy-mode = "rgmii-txid";
+                                link = <&switch0port5>;
+                                fixed-link {
+                                        speed = <1000>;
+                                        full-duplex;
+                                };
+                        };
+                };
+                mdio-bus {
+                        #address-cells = <1>;
+                        #size-cells = <0>;
+                        switch1phy0: switch1phy0@0 {
+                                reg = <0>;
+                        };
+                        switch1phy1: switch1phy0@1 {
+                                reg = <1>;
+                        };
+                        switch1phy2: switch1phy0@2 {
+                                reg = <2>;
+                        };
+                };
+         };
+};
+
+&mdio4 {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        switch2: switch2@0 {
+                compatible = "marvell,mv88e6085";
+                #address-cells = <1>;
+                #size-cells = <0>;
+                reg = <0>;
+
+                dsa,member = <0 2>;
+
+                ports {
+                        #address-cells = <1>;
+                        #size-cells = <0>;
+                        port@0 {
+                                reg = <0>;
+                                label = "lan6";
+                        };
+
+                        port@1 {
+                                reg = <1>;
+                                label = "lan7";
+                        };
+
+                        port@2 {
+                                reg = <2>;
+                                label = "lan8";
+                        };
+
+                        port@3 {
+                                reg = <3>;
+                                label = "optical3";
+                                fixed-link {
+                                        speed = <1000>;
+                                        full-duplex;
+                                        link-gpios = <&gpio6 2
+                                              GPIO_ACTIVE_HIGH>;
+                                };
+                        };
+
+                        port@4 {
+                                reg = <4>;
+                                label = "optical4";
+                                fixed-link {
+                                        speed = <1000>;
+                                        full-duplex;
+                                        link-gpios = <&gpio6 3
+                                              GPIO_ACTIVE_HIGH>;
+                                };
+                        };
+
+                        switch2port9: port@9 {
+                                reg = <9>;
+                                label = "dsa";
+                                phy-mode = "rgmii-txid";
+                                link = <&switch1port5
+                                        &switch0port5>;
+                                fixed-link {
+                                        speed = <1000>;
+                                        full-duplex;
+                                };
+                        };
+                };
+        };
+};
+
+Deprecated Binding
+------------------
+
+The deprecated binding makes use of a platform device to represent the
+switches. The switches themselves are not Linux devices, and make use
+of an MDIO bus for management.
 
 Required properties:
 - compatible            : Should be "marvell,dsa"
diff --git a/Documentation/devicetree/bindings/net/mdio-mux.txt b/Documentation/devicetree/bindings/net/mdio-mux.txt
index 491f5bd55203..f58571f36570 100644
--- a/Documentation/devicetree/bindings/net/mdio-mux.txt
+++ b/Documentation/devicetree/bindings/net/mdio-mux.txt
@@ -5,11 +5,12 @@ numbered uniquely in a device dependent manner.  The nodes for an MDIO
 bus multiplexer/switch will have one child node for each child bus.
 
 Required properties:
-- mdio-parent-bus : phandle to the parent MDIO bus.
 - #address-cells = <1>;
 - #size-cells = <0>;
 
 Optional properties:
+- mdio-parent-bus : phandle to the parent MDIO bus.
+
 - Other properties specific to the multiplexer/switch hardware.
 
 Required properties for child nodes:
diff --git a/Documentation/devicetree/bindings/net/rockchip-dwmac.txt b/Documentation/devicetree/bindings/net/rockchip-dwmac.txt
index 93eac7ce1446..cccd945fc45b 100644
--- a/Documentation/devicetree/bindings/net/rockchip-dwmac.txt
+++ b/Documentation/devicetree/bindings/net/rockchip-dwmac.txt
@@ -3,7 +3,8 @@ Rockchip SoC RK3288 10/100/1000 Ethernet driver(GMAC)
 The device node has following properties.
 
 Required properties:
- - compatible: Can be one of "rockchip,rk3288-gmac", "rockchip,rk3368-gmac"
+ - compatible: Can be one of "rockchip,rk3228-gmac", "rockchip,rk3288-gmac",
+                             "rockchip,rk3368-gmac"
  - reg: addresses and length of the register sets for the device.
  - interrupts: Should contain the GMAC interrupts.
  - interrupt-names: Should contain the interrupt names "macirq".
diff --git a/Documentation/devicetree/bindings/net/stmmac.txt b/Documentation/devicetree/bindings/net/stmmac.txt
index 95816c5fc589..41b49e6075f5 100644
--- a/Documentation/devicetree/bindings/net/stmmac.txt
+++ b/Documentation/devicetree/bindings/net/stmmac.txt
@@ -47,6 +47,9 @@ Optional properties:
                                 supported by this device instance
 - snps,perfect-filter-entries:  Number of perfect filter entries supported
                                 by this device instance
+- snps,ps-speed: port selection speed that can be passed to the core when
+                 PCS is supported. For example, this is used in case of SGMII
+                 and MAC2MAC connection.
 - AXI BUS Mode parameters: below the list of all the parameters to program the
                            AXI register inside the DMA module:
         - snps,lpi_en: enable Low Power Interface
diff --git a/Documentation/devicetree/bindings/phy/brcm,mdio-mux-bus-pci.txt b/Documentation/devicetree/bindings/phy/brcm,mdio-mux-bus-pci.txt
new file mode 100644
index 000000000000..5b51007c6f24
--- /dev/null
+++ b/Documentation/devicetree/bindings/phy/brcm,mdio-mux-bus-pci.txt
@@ -0,0 +1,27 @@
+* Broadcom NS2 PCIe PHY binding document
+
+Required bus properties:
+- reg: MDIO Bus number for the MDIO interface
+- #address-cells: must be 1
+- #size-cells: must be 0
+
+Required PHY properties:
+- compatible: should be "brcm,ns2-pcie-phy"
+- reg: MDIO Phy ID for the MDIO interface
+- #phy-cells: must be 0
+
+This is a child bus node of "brcm,mdio-mux-iproc" node.
+
+Example:
+
+mdio@0 {
+        reg = <0x0>;
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        pci_phy0: pci-phy@0 {
+                compatible = "brcm,ns2-pcie-phy";
+                reg = <0x0>;
+                #phy-cells = <0>;
+        };
+};
diff --git a/Documentation/networking/can.txt b/Documentation/networking/can.txt
index d58ff8467953..aa15b9ee2e70 100644
--- a/Documentation/networking/can.txt
+++ b/Documentation/networking/can.txt
@@ -31,6 +31,7 @@ This file contains
       4.2.4 Broadcast Manager message sequence transmission
       4.2.5 Broadcast Manager receive filter timers
       4.2.6 Broadcast Manager multiplex message receive filter
+      4.2.7 Broadcast Manager CAN FD support
     4.3 connected transport protocols (SOCK_SEQPACKET)
     4.4 unconnected transport protocols (SOCK_DGRAM)
 
@@ -799,7 +800,7 @@ solution for a couple of reasons:
     } mytxmsg;
 
     (..)
-    mytxmsg.nframes = 4;
+    mytxmsg.msg_head.nframes = 4;
     (..)
 
     write(s, &mytxmsg, sizeof(mytxmsg));
@@ -852,6 +853,28 @@ solution for a couple of reasons:
 
     write(s, &msg, sizeof(msg));
 
+  4.2.7 Broadcast Manager CAN FD support
+
+  The programming API of the CAN_BCM depends on struct can_frame which is
+  given as array directly behind the bcm_msg_head structure. To follow this
+  schema for the CAN FD frames a new flag 'CAN_FD_FRAME' in the bcm_msg_head
+  flags indicates that the concatenated CAN frame structures behind the
+  bcm_msg_head are defined as struct canfd_frame.
+
+    struct {
+            struct bcm_msg_head msg_head;
+            struct canfd_frame frame[5];
+    } msg;
+
+    msg.msg_head.opcode  = RX_SETUP;
+    msg.msg_head.can_id  = 0x42;
+    msg.msg_head.flags   = CAN_FD_FRAME;
+    msg.msg_head.nframes = 5;
+    (..)
+
+  When using CAN FD frames for multiplex filtering the MUX mask is still
+  expected in the first 64 bit of the struct canfd_frame data section.
+
   4.3 connected transport protocols (SOCK_SEQPACKET)
   4.4 unconnected transport protocols (SOCK_DGRAM)
 
diff --git a/Documentation/networking/gen_stats.txt b/Documentation/networking/gen_stats.txt
index ff630a87b511..179b18ce45ff 100644
--- a/Documentation/networking/gen_stats.txt
+++ b/Documentation/networking/gen_stats.txt
@@ -21,7 +21,7 @@ struct mystruct {
         ...
 };
 
-Update statistics:
+Update statistics, in dequeue() methods only, (while owning qdisc->running)
 mystruct->tstats.packet++;
 mystruct->qstats.backlog += skb->pkt_len;
 
diff --git a/Documentation/networking/stmmac.txt b/Documentation/networking/stmmac.txt
index 671fe3dd56d3..e226f8925c9e 100644
--- a/Documentation/networking/stmmac.txt
+++ b/Documentation/networking/stmmac.txt
@@ -285,6 +285,7 @@ Please see the following document:
  o mmc_core.c/mmc.h: Management MAC Counters;
  o stmmac_hwtstamp.c: HW timestamp support for PTP;
  o stmmac_ptp.c: PTP 1588 clock;
+ o stmmac_pcs.h: Physical Coding Sublayer common implementation;
  o dwmac-<XXX>.c: these are for the platform glue-logic file; e.g. dwmac-sti.c
    for STMicroelectronics SoCs.
 
diff --git a/MAINTAINERS b/MAINTAINERS
index 1209323b7e43..d8c078491e55 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -2460,6 +2460,14 @@ L:	netdev@vger.kernel.org
 S:      Supported
 F:      drivers/net/ethernet/broadcom/b44.*
 
+BROADCOM B53 ETHERNET SWITCH DRIVER
+M:      Florian Fainelli <f.fainelli@gmail.com>
+L:      netdev@vger.kernel.org
+L:      openwrt-devel@lists.openwrt.org (subscribers-only)
+S:      Supported
+F:      drivers/net/dsa/b53/*
+F:      include/linux/platform_data/b53.h
+
 BROADCOM GENET ETHERNET DRIVER
 M:      Florian Fainelli <f.fainelli@gmail.com>
 L:      netdev@vger.kernel.org
@@ -2576,12 +2584,11 @@ S:	Supported
 F:      drivers/net/ethernet/broadcom/tg3.*
 
 BROADCOM BRCM80211 IEEE802.11n WIRELESS DRIVER
-M:      Brett Rudley <brudley@broadcom.com>
-M:      Arend van Spriel <arend@broadcom.com>
-M:      Franky (Zhenhui) Lin <frankyl@broadcom.com>
-M:      Hante Meuleman <meuleman@broadcom.com>
+M:      Arend van Spriel <arend.vanspriel@broadcom.com>
+M:      Franky Lin <franky.lin@broadcom.com>
+M:      Hante Meuleman <hante.meuleman@broadcom.com>
 L:      linux-wireless@vger.kernel.org
-L:      brcm80211-dev-list@broadcom.com
+L:      brcm80211-dev-list.pdl@broadcom.com
 S:      Supported
 F:      drivers/net/wireless/broadcom/brcm80211/
 
@@ -2813,6 +2820,7 @@ W:	https://github.com/linux-can
 T:      git git://git.kernel.org/pub/scm/linux/kernel/git/mkl/linux-can.git
 T:      git git://git.kernel.org/pub/scm/linux/kernel/git/mkl/linux-can-next.git
 S:      Maintained
+F:      Documentation/devicetree/bindings/net/can/
 F:      drivers/net/can/
 F:      include/linux/can/dev.h
 F:      include/linux/can/platform/
@@ -4887,6 +4895,13 @@ F:	drivers/net/ethernet/freescale/gianfar*
 X:      drivers/net/ethernet/freescale/gianfar_ptp.c
 F:      Documentation/devicetree/bindings/net/fsl-tsec-phy.txt
 
+FREESCALE QUICC ENGINE UCC HDLC DRIVER
+M:      Zhao Qiang <qiang.zhao@nxp.com>
+L:      netdev@vger.kernel.org
+L:      linuxppc-dev@lists.ozlabs.org
+S:      Maintained
+F:      drivers/net/wan/fsl_ucc_hdlc*
+
 FREESCALE QUICC ENGINE UCC UART DRIVER
 M:      Timur Tabi <timur@tabi.org>
 L:      linuxppc-dev@lists.ozlabs.org
@@ -7173,6 +7188,12 @@ W:	http://www.kernel.org/doc/man-pages
 L:      linux-man@vger.kernel.org
 S:      Maintained
 
+MARVELL 88E6XXX ETHERNET SWITCH FABRIC DRIVER
+M:      Andrew Lunn <andrew@lunn.ch>
+M:      Vivien Didelot <vivien.didelot@savoirfairelinux.com>
+S:      Maintained
+F:      drivers/net/dsa/mv88e6xxx/
+
 MARVELL ARMADA DRM SUPPORT
 M:      Russell King <rmk+kernel@armlinux.org.uk>
 S:      Maintained
@@ -7180,11 +7201,6 @@ F:	drivers/gpu/drm/armada/
 F:      include/uapi/drm/armada_drm.h
 F:      Documentation/devicetree/bindings/display/armada/
 
-MARVELL 88E6352 DSA support
-M:      Guenter Roeck <linux@roeck-us.net>
-S:      Maintained
-F:      drivers/net/dsa/mv88e6352.c
-
 MARVELL CRYPTO DRIVER
 M:      Boris Brezillon <boris.brezillon@free-electrons.com>
 M:      Arnaud Ebalard <arno@natisbad.org>
@@ -10281,10 +10297,9 @@ W:	http://www.avagotech.com
 S:      Supported
 F:      drivers/scsi/be2iscsi/
 
-Emulex 10Gbps NIC BE2, BE3-R, Lancer, Skyhawk-R DRIVER
+Emulex 10Gbps NIC BE2, BE3-R, Lancer, Skyhawk-R DRIVER (be2net)
 M:      Sathya Perla <sathya.perla@broadcom.com>
 M:      Ajit Khaparde <ajit.khaparde@broadcom.com>
-M:      Padmanabh Ratnakar <padmanabh.ratnakar@broadcom.com>
 M:      Sriharsha Basavapatna <sriharsha.basavapatna@broadcom.com>
 M:      Somnath Kotur <somnath.kotur@broadcom.com>
 L:      netdev@vger.kernel.org
diff --git a/arch/arm/boot/dts/am33xx.dtsi b/arch/arm/boot/dts/am33xx.dtsi
index 52be48bbd2dd..7fa295155543 100644
--- a/arch/arm/boot/dts/am33xx.dtsi
+++ b/arch/arm/boot/dts/am33xx.dtsi
@@ -766,7 +766,6 @@
                         ale_entries = <1024>;
                         bd_ram_size = <0x2000>;
                         no_bd_ram = <0>;
-                        rx_descs = <64>;
                         mac_control = <0x20>;
                         slaves = <2>;
                         active_slave = <0>;
@@ -789,7 +788,7 @@
                         status = "disabled";
 
                         davinci_mdio: mdio@4a101000 {
-                                compatible = "ti,davinci_mdio";
+                                compatible = "ti,cpsw-mdio","ti,davinci_mdio";
                                 #address-cells = <1>;
                                 #size-cells = <0>;
                                 ti,hwmods = "davinci_mdio";
diff --git a/arch/arm/boot/dts/am4372.dtsi b/arch/arm/boot/dts/am4372.dtsi
index 12fcde4d4d2e..cd81ecf12731 100644
--- a/arch/arm/boot/dts/am4372.dtsi
+++ b/arch/arm/boot/dts/am4372.dtsi
@@ -626,7 +626,6 @@
                         ale_entries = <1024>;
                         bd_ram_size = <0x2000>;
                         no_bd_ram = <0>;
-                        rx_descs = <64>;
                         mac_control = <0x20>;
                         slaves = <2>;
                         active_slave = <0>;
@@ -636,7 +635,7 @@
                         syscon = <&scm_conf>;
 
                         davinci_mdio: mdio@4a101000 {
-                                compatible = "ti,am4372-mdio","ti,davinci_mdio";
+                                compatible = "ti,am4372-mdio","ti,cpsw-mdio","ti,davinci_mdio";
                                 reg = <0x4a101000 0x100>;
                                 #address-cells = <1>;
                                 #size-cells = <0>;
diff --git a/arch/arm/boot/dts/dm814x.dtsi b/arch/arm/boot/dts/dm814x.dtsi
index d4537dc61497..f23cae0c2179 100644
--- a/arch/arm/boot/dts/dm814x.dtsi
+++ b/arch/arm/boot/dts/dm814x.dtsi
@@ -509,7 +509,6 @@
                         ale_entries = <1024>;
                         bd_ram_size = <0x2000>;
                         no_bd_ram = <0>;
-                        rx_descs = <64>;
                         mac_control = <0x20>;
                         slaves = <2>;
                         active_slave = <0>;
diff --git a/arch/arm/boot/dts/dra7.dtsi b/arch/arm/boot/dts/dra7.dtsi
index 3a8f3976f6f9..de559f6e4fee 100644
--- a/arch/arm/boot/dts/dra7.dtsi
+++ b/arch/arm/boot/dts/dra7.dtsi
@@ -1628,7 +1628,6 @@
                         ale_entries = <1024>;
                         bd_ram_size = <0x2000>;
                         no_bd_ram = <0>;
-                        rx_descs = <64>;
                         mac_control = <0x20>;
                         slaves = <2>;
                         active_slave = <0>;
@@ -1663,7 +1662,7 @@
                         status = "disabled";
 
                         davinci_mdio: mdio@48485000 {
-                                compatible = "ti,davinci_mdio";
+                                compatible = "ti,cpsw-mdio","ti,davinci_mdio";
                                 #address-cells = <1>;
                                 #size-cells = <0>;
                                 ti,hwmods = "davinci_mdio";
diff --git a/arch/arm/boot/dts/rk3288.dtsi b/arch/arm/boot/dts/rk3288.dtsi
index 3b44ef3cff12..3ebee530f2b0 100644
--- a/arch/arm/boot/dts/rk3288.dtsi
+++ b/arch/arm/boot/dts/rk3288.dtsi
@@ -539,8 +539,9 @@
         gmac: ethernet@ff290000 {
                 compatible = "rockchip,rk3288-gmac";
                 reg = <0xff290000 0x10000>;
-                interrupts = <GIC_SPI 27 IRQ_TYPE_LEVEL_HIGH>;
-                interrupt-names = "macirq";
+                interrupts = <GIC_SPI 27 IRQ_TYPE_LEVEL_HIGH>,
+                                <GIC_SPI 28 IRQ_TYPE_LEVEL_HIGH>;
+                interrupt-names = "macirq", "eth_wake_irq";
                 rockchip,grf = <&grf>;
                 clocks = <&cru SCLK_MAC>,
                         <&cru SCLK_MAC_RX>, <&cru SCLK_MAC_TX>,
diff --git a/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts b/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts
index 6c60b7f91104..5c1fcab4a6f7 100644
--- a/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts
+++ b/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts
@@ -85,187 +85,199 @@
                         reg = <1>;
                         #address-cells = <1>;
                         #size-cells = <0>;
+
+                        switch0: switch0@0 {
+                                compatible = "marvell,mv88e6085";
+                                #address-cells = <1>;
+                                #size-cells = <0>;
+                                reg = <0>;
+                                dsa,member = <0 0>;
+
+                                ports {
+                                        #address-cells = <1>;
+                                        #size-cells = <0>;
+                                        port@0 {
+                                                reg = <0>;
+                                                label = "lan0";
+                                        };
+
+                                        port@1 {
+                                                reg = <1>;
+                                                label = "lan1";
+                                        };
+
+                                        port@2 {
+                                                reg = <2>;
+                                                label = "lan2";
+                                        };
+
+                                        switch0port5: port@5 {
+                                                reg = <5>;
+                                                label = "dsa";
+                                                phy-mode = "rgmii-txid";
+                                                link = <&switch1port6
+                                                        &switch2port9>;
+                                                fixed-link {
+                                                        speed = <1000>;
+                                                        full-duplex;
+                                                };
+                                        };
+
+                                        port@6 {
+                                                reg = <6>;
+                                                label = "cpu";
+                                                ethernet = <&fec1>;
+                                                fixed-link {
+                                                        speed = <100>;
+                                                        full-duplex;
+                                                };
+                                        };
+                                };
+                        };
                 };
 
                 mdio_mux_2: mdio@2 {
                         reg = <2>;
                         #address-cells = <1>;
                         #size-cells = <0>;
-                };
-
-                mdio_mux_4: mdio@4 {
-                        reg = <4>;
-                        #address-cells = <1>;
-                        #size-cells = <0>;
-                };
-
-                mdio_mux_8: mdio@8 {
-                        reg = <8>;
-                        #address-cells = <1>;
-                        #size-cells = <0>;
-                };
-        };
-
-        dsa {
-                compatible = "marvell,dsa";
-                #address-cells = <2>;
-                #size-cells = <0>;
-                dsa,ethernet = <&fec1>;
-                dsa,mii-bus = <&mdio_mux_1>;
-
-                /* 6352 - Primary - 7 ports */
-                switch0: switch@0-0 {
-                        #address-cells = <1>;
-                        #size-cells = <0>;
-                        reg = <0x00 0>;
-                        eeprom-length = <512>;
 
-                        port@0 {
+                        switch1: switch1@0 {
+                                compatible = "marvell,mv88e6085";
+                                #address-cells = <1>;
+                                #size-cells = <0>;
                                 reg = <0>;
-                                label = "lan0";
-                        };
-
-                        port@1 {
-                                reg = <1>;
-                                label = "lan1";
-                        };
-
-                        port@2 {
-                                reg = <2>;
-                                label = "lan2";
-                        };
-
-                        switch0port5: port@5 {
-                                reg = <5>;
-                                label = "dsa";
-                                phy-mode = "rgmii-txid";
-                                link = <&switch1port6
-                                        &switch2port9>;
-
-                                fixed-link {
-                                        speed = <1000>;
-                                        full-duplex;
+                                dsa,member = <0 1>;
+
+                                ports {
+                                        #address-cells = <1>;
+                                        #size-cells = <0>;
+                                        port@0 {
+                                                reg = <0>;
+                                                label = "lan3";
+                                                phy-handle = <&switch1phy0>;
+                                        };
+
+                                        port@1 {
+                                                reg = <1>;
+                                                label = "lan4";
+                                                phy-handle = <&switch1phy1>;
+                                        };
+
+                                        port@2 {
+                                                reg = <2>;
+                                                label = "lan5";
+                                                phy-handle = <&switch1phy2>;
+                                        };
+
+                                        switch1port5: port@5 {
+                                                reg = <5>;
+                                                label = "dsa";
+                                                link = <&switch2port9>;
+                                                phy-mode = "rgmii-txid";
+                                                fixed-link {
+                                                        speed = <1000>;
+                                                        full-duplex;
+                                                };
+                                        };
+
+                                        switch1port6: port@6 {
+                                                reg = <6>;
+                                                label = "dsa";
+                                                phy-mode = "rgmii-txid";
+                                                link = <&switch0port5>;
+                                                fixed-link {
+                                                        speed = <1000>;
+                                                        full-duplex;
+                                                };
+                                        };
                                 };
-                        };
-
-                        port@6 {
-                                reg = <6>;
-                                label = "cpu";
-
-                                fixed-link {
-                                        speed = <100>;
-                                        full-duplex;
+                                mdio {
+                                        #address-cells = <1>;
+                                        #size-cells = <0>;
+                                        switch1phy0: switch1phy0@0 {
+                                                reg = <0>;
+                                        };
+                                        switch1phy1: switch1phy0@1 {
+                                                reg = <1>;
+                                        };
+                                        switch1phy2: switch1phy0@2 {
+                                                reg = <2>;
+                                        };
                                 };
                         };
-
                 };
 
-                /* 6352 - Secondary - 7 ports */
-                switch1: switch@0-1 {
+                mdio_mux_4: mdio@4 {
                         #address-cells = <1>;
                         #size-cells = <0>;
-                        reg = <0x00 1>;
-                        eeprom-length = <512>;
-                        mii-bus = <&mdio_mux_2>;
+                        reg = <4>;
 
-                        port@0 {
+                        switch2: switch2@0 {
+                                compatible = "marvell,mv88e6085";
+                                #address-cells = <1>;
+                                #size-cells = <0>;
                                 reg = <0>;
-                                label = "lan3";
-                        };
-
-                        port@1 {
-                                reg = <1>;
-                                label = "lan4";
-                        };
-
-                        port@2 {
-                                reg = <2>;
-                                label = "lan5";
-                        };
-
-                        switch1port5: port@5 {
-                                reg = <5>;
-                                label = "dsa";
-                                link = <&switch2port9>;
-                                phy-mode = "rgmii-txid";
-
-                                fixed-link {
-                                        speed = <1000>;
-                                        full-duplex;
-                                };
-                        };
-
-                        switch1port6: port@6 {
-                                reg = <6>;
-                                label = "dsa";
-                                phy-mode = "rgmii-txid";
-                                link = <&switch0port5>;
-
-                                fixed-link {
-                                        speed = <1000>;
-                                        full-duplex;
+                                dsa,member = <0 2>;
+
+                                ports {
+                                        #address-cells = <1>;
+                                        #size-cells = <0>;
+                                        port@0 {
+                                                reg = <0>;
+                                                label = "lan6";
+                                        };
+
+                                        port@1 {
+                                                reg = <1>;
+                                                label = "lan7";
+                                        };
+
+                                        port@2 {
+                                                reg = <2>;
+                                                label = "lan8";
+                                        };
+
+                                        port@3 {
+                                                reg = <3>;
+                                                label = "optical3";
+                                                fixed-link {
+                                                        speed = <1000>;
+                                                        full-duplex;
+                                                        link-gpios = <&gpio6 2
+                                                              GPIO_ACTIVE_HIGH>;
+                                                };
+                                        };
+
+                                        port@4 {
+                                                reg = <4>;
+                                                label = "optical4";
+                                                fixed-link {
+                                                        speed = <1000>;
+                                                        full-duplex;
+                                                        link-gpios = <&gpio6 3
+                                                              GPIO_ACTIVE_HIGH>;
+                                                };
+                                        };
+
+                                        switch2port9: port@9 {
+                                                reg = <9>;
+                                                label = "dsa";
+                                                phy-mode = "rgmii-txid";
+                                                link = <&switch1port5
+                                                        &switch0port5>;
+                                                fixed-link {
+                                                        speed = <1000>;
+                                                        full-duplex;
+                                                };
+                                        };
                                 };
                         };
                 };
 
-                /* 6185 - 10 ports */
-                switch2: switch@0-2 {
+                mdio_mux_8: mdio@8 {
+                        reg = <8>;
                         #address-cells = <1>;
                         #size-cells = <0>;
-                        reg = <0x00 2>;
-                        mii-bus = <&mdio_mux_4>;
-
-                        port@0 {
-                                reg = <0>;
-                                label = "lan6";
-                        };
-
-                        port@1 {
-                                reg = <1>;
-                                label = "lan7";
-                        };
-
-                        port@2 {
-                                reg = <2>;
-                                label = "lan8";
-                        };
-
-                        port@3 {
-                                reg = <3>;
-                                label = "optical3";
-
-                                fixed-link {
-                                        speed = <1000>;
-                                        full-duplex;
-                                        link-gpios = <&gpio6 2
-                                                      GPIO_ACTIVE_HIGH>;
-                                };
-                        };
-
-                        port@4 {
-                                reg = <4>;
-                                label = "optical4";
-
-                                fixed-link {
-                                        speed = <1000>;
-                                        full-duplex;
-                                        link-gpios = <&gpio6 3
-                                                      GPIO_ACTIVE_HIGH>;
-                                };
-                        };
-
-                        switch2port9: port@9 {
-                                reg = <9>;
-                                label = "dsa";
-                                phy-mode = "rgmii-txid";
-                                link = <&switch1port5
-                                        &switch0port5>;
-
-                                fixed-link {
-                                        speed = <1000>;
-                                        full-duplex;
-                                };
-                        };
                 };
         };
 
diff --git a/arch/arm64/boot/dts/broadcom/ns2-svk.dts b/arch/arm64/boot/dts/broadcom/ns2-svk.dts
index 54ca40c9f711..ea5603fd106a 100644
--- a/arch/arm64/boot/dts/broadcom/ns2-svk.dts
+++ b/arch/arm64/boot/dts/broadcom/ns2-svk.dts
@@ -52,6 +52,14 @@
         };
 };
 
+&pci_phy0 {
+        status = "ok";
+};
+
+&pci_phy1 {
+        status = "ok";
+};
+
 &pcie0 {
         status = "ok";
 };
@@ -132,3 +140,11 @@
                 #size-cells = <1>;
         };
 };
+
+&mdio_mux_iproc {
+        mdio@10 {
+                gphy0: eth-phy@10 {
+                        reg = <0x10>;
+                };
+        };
+};
diff --git a/arch/arm64/boot/dts/broadcom/ns2.dtsi b/arch/arm64/boot/dts/broadcom/ns2.dtsi
index ec68ec1a80c8..46b78fa89f4c 100644
--- a/arch/arm64/boot/dts/broadcom/ns2.dtsi
+++ b/arch/arm64/boot/dts/broadcom/ns2.dtsi
@@ -263,6 +263,45 @@
                                       IRQ_TYPE_LEVEL_HIGH)>;
                 };
 
+                mdio_mux_iproc: mdio-mux@6602023c {
+                        compatible = "brcm,mdio-mux-iproc";
+                        reg = <0x6602023c 0x14>;
+                        #address-cells = <1>;
+                        #size-cells = <0>;
+
+                        mdio@0 {
+                                reg = <0x0>;
+                                #address-cells = <1>;
+                                #size-cells = <0>;
+
+                                pci_phy0: pci-phy@0 {
+                                        compatible = "brcm,ns2-pcie-phy";
+                                        reg = <0x0>;
+                                        #phy-cells = <0>;
+                                        status = "disabled";
+                                };
+                        };
+
+                        mdio@7 {
+                                reg = <0x7>;
+                                #address-cells = <1>;
+                                #size-cells = <0>;
+
+                                pci_phy1: pci-phy@0 {
+                                        compatible = "brcm,ns2-pcie-phy";
+                                        reg = <0x0>;
+                                        #phy-cells = <0>;
+                                        status = "disabled";
+                                };
+                        };
+
+                        mdio@10 {
+                                reg = <0x10>;
+                                #address-cells = <1>;
+                                #size-cells = <0>;
+                        };
+                };
+
                 timer0: timer@66030000 {
                         compatible = "arm,sp804", "arm,primecell";
                         reg = <0x66030000 0x1000>;
diff --git a/arch/arm64/net/bpf_jit.h b/arch/arm64/net/bpf_jit.h
index aee5637ea436..7c16e547ccb2 100644
--- a/arch/arm64/net/bpf_jit.h
+++ b/arch/arm64/net/bpf_jit.h
@@ -1,7 +1,7 @@
 /*
  * BPF JIT compiler for ARM64
  *
- * Copyright (C) 2014-2015 Zi Shen Lim <zlim.lnx@gmail.com>
+ * Copyright (C) 2014-2016 Zi Shen Lim <zlim.lnx@gmail.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -55,6 +55,7 @@
 #define A64_BL(imm26) A64_BRANCH((imm26) << 2, LINK)
 
 /* Unconditional branch (register) */
+#define A64_BR(Rn)  aarch64_insn_gen_branch_reg(Rn, AARCH64_INSN_BRANCH_NOLINK)
 #define A64_BLR(Rn) aarch64_insn_gen_branch_reg(Rn, AARCH64_INSN_BRANCH_LINK)
 #define A64_RET(Rn) aarch64_insn_gen_branch_reg(Rn, AARCH64_INSN_BRANCH_RETURN)
 
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index 49ba37e4bfc0..b2fc97a2c56c 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -18,6 +18,7 @@
 
 #define pr_fmt(fmt) "bpf_jit: " fmt
 
+#include <linux/bpf.h>
 #include <linux/filter.h>
 #include <linux/printk.h>
 #include <linux/skbuff.h>
@@ -33,6 +34,7 @@ int bpf_jit_enable __read_mostly;
 
 #define TMP_REG_1 (MAX_BPF_JIT_REG + 0)
 #define TMP_REG_2 (MAX_BPF_JIT_REG + 1)
+#define TCALL_CNT (MAX_BPF_JIT_REG + 2)
 
 /* Map BPF registers to A64 registers */
 static const int bpf2a64[] = {
@@ -54,6 +56,8 @@ static const int bpf2a64[] = {
         /* temporary registers for internal BPF JIT */
         [TMP_REG_1] = A64_R(10),
         [TMP_REG_2] = A64_R(11),
+        /* tail_call_cnt */
+        [TCALL_CNT] = A64_R(26),
         /* temporary register for blinding constants */
         [BPF_REG_AX] = A64_R(9),
 };
@@ -146,13 +150,18 @@ static inline int epilogue_offset(const struct jit_ctx *ctx)
 
 #define STACK_SIZE STACK_ALIGN(_STACK_SIZE)
 
-static void build_prologue(struct jit_ctx *ctx)
+#define PROLOGUE_OFFSET 8
+
+static int build_prologue(struct jit_ctx *ctx)
 {
         const u8 r6 = bpf2a64[BPF_REG_6];
         const u8 r7 = bpf2a64[BPF_REG_7];
         const u8 r8 = bpf2a64[BPF_REG_8];
         const u8 r9 = bpf2a64[BPF_REG_9];
         const u8 fp = bpf2a64[BPF_REG_FP];
+        const u8 tcc = bpf2a64[TCALL_CNT];
+        const int idx0 = ctx->idx;
+        int cur_offset;
 
         /*
          * BPF prog stack layout
@@ -162,8 +171,6 @@ static void build_prologue(struct jit_ctx *ctx)
          *                        |FP/LR|
          * current A64_FP =>  -16:+-----+
          *                        | ... | callee saved registers
-         *                        +-----+
-         *                        |     | x25/x26
          * BPF fp register => -64:+-----+ <= (BPF_FP)
          *                        |     |
          *                        | ... | BPF prog stack
@@ -183,18 +190,90 @@ static void build_prologue(struct jit_ctx *ctx)
         emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx);
         emit(A64_MOV(1, A64_FP, A64_SP), ctx);
 
-        /* Save callee-saved register */
+        /* Save callee-saved registers */
         emit(A64_PUSH(r6, r7, A64_SP), ctx);
         emit(A64_PUSH(r8, r9, A64_SP), ctx);
+        emit(A64_PUSH(fp, tcc, A64_SP), ctx);
 
-        /* Save fp (x25) and x26. SP requires 16 bytes alignment */
-        emit(A64_PUSH(fp, A64_R(26), A64_SP), ctx);
-
-        /* Set up BPF prog stack base register (x25) */
+        /* Set up BPF prog stack base register */
         emit(A64_MOV(1, fp, A64_SP), ctx);
 
+        /* Initialize tail_call_cnt */
+        emit(A64_MOVZ(1, tcc, 0, 0), ctx);
+
         /* Set up function call stack */
         emit(A64_SUB_I(1, A64_SP, A64_SP, STACK_SIZE), ctx);
+
+        cur_offset = ctx->idx - idx0;
+        if (cur_offset != PROLOGUE_OFFSET) {
+                pr_err_once("PROLOGUE_OFFSET = %d, expected %d!\n",
+                            cur_offset, PROLOGUE_OFFSET);
+                return -1;
+        }
+        return 0;
+}
+
+static int out_offset = -1; /* initialized on the first pass of build_body() */
+static int emit_bpf_tail_call(struct jit_ctx *ctx)
+{
+        /* bpf_tail_call(void *prog_ctx, struct bpf_array *array, u64 index) */
+        const u8 r2 = bpf2a64[BPF_REG_2];
+        const u8 r3 = bpf2a64[BPF_REG_3];
+
+        const u8 tmp = bpf2a64[TMP_REG_1];
+        const u8 prg = bpf2a64[TMP_REG_2];
+        const u8 tcc = bpf2a64[TCALL_CNT];
+        const int idx0 = ctx->idx;
+#define cur_offset (ctx->idx - idx0)
+#define jmp_offset (out_offset - (cur_offset))
+        size_t off;
+
+        /* if (index >= array->map.max_entries)
+         *     goto out;
+         */
+        off = offsetof(struct bpf_array, map.max_entries);
+        emit_a64_mov_i64(tmp, off, ctx);
+        emit(A64_LDR32(tmp, r2, tmp), ctx);
+        emit(A64_CMP(0, r3, tmp), ctx);
+        emit(A64_B_(A64_COND_GE, jmp_offset), ctx);
+
+        /* if (tail_call_cnt > MAX_TAIL_CALL_CNT)
+         *     goto out;
+         * tail_call_cnt++;
+         */
+        emit_a64_mov_i64(tmp, MAX_TAIL_CALL_CNT, ctx);
+        emit(A64_CMP(1, tcc, tmp), ctx);
+        emit(A64_B_(A64_COND_GT, jmp_offset), ctx);
+        emit(A64_ADD_I(1, tcc, tcc, 1), ctx);
+
+        /* prog = array->ptrs[index];
+         * if (prog == NULL)
+         *     goto out;
+         */
+        off = offsetof(struct bpf_array, ptrs);
+        emit_a64_mov_i64(tmp, off, ctx);
+        emit(A64_LDR64(tmp, r2, tmp), ctx);
+        emit(A64_LDR64(prg, tmp, r3), ctx);
+        emit(A64_CBZ(1, prg, jmp_offset), ctx);
+
+        /* goto *(prog->bpf_func + prologue_size); */
+        off = offsetof(struct bpf_prog, bpf_func);
+        emit_a64_mov_i64(tmp, off, ctx);
+        emit(A64_LDR64(tmp, prg, tmp), ctx);
+        emit(A64_ADD_I(1, tmp, tmp, sizeof(u32) * PROLOGUE_OFFSET), ctx);
+        emit(A64_BR(tmp), ctx);
+
+        /* out: */
+        if (out_offset == -1)
+                out_offset = cur_offset;
+        if (cur_offset != out_offset) {
+                pr_err_once("tail_call out_offset = %d, expected %d!\n",
+                            cur_offset, out_offset);
+                return -1;
+        }
+        return 0;
+#undef cur_offset
+#undef jmp_offset
 }
 
 static void build_epilogue(struct jit_ctx *ctx)
@@ -499,13 +578,15 @@ emit_cond_jmp:
                 const u64 func = (u64)__bpf_call_base + imm;
 
                 emit_a64_mov_i64(tmp, func, ctx);
-                emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx);
-                emit(A64_MOV(1, A64_FP, A64_SP), ctx);
                 emit(A64_BLR(tmp), ctx);
                 emit(A64_MOV(1, r0, A64_R(0)), ctx);
-                emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx);
                 break;
         }
+        /* tail call */
+        case BPF_JMP | BPF_CALL | BPF_X:
+                if (emit_bpf_tail_call(ctx))
+                        return -EFAULT;
+                break;
         /* function return */
         case BPF_JMP | BPF_EXIT:
                 /* Optimization: when last instruction is EXIT,
@@ -650,11 +731,8 @@ emit_cond_jmp:
                 emit_a64_mov_i64(r3, size, ctx);
                 emit(A64_SUB_I(1, r4, fp, STACK_SIZE), ctx);
                 emit_a64_mov_i64(r5, (unsigned long)bpf_load_pointer, ctx);
-                emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx);
-                emit(A64_MOV(1, A64_FP, A64_SP), ctx);
                 emit(A64_BLR(r5), ctx);
                 emit(A64_MOV(1, r0, A64_R(0)), ctx);
-                emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx);
 
                 jmp_offset = epilogue_offset(ctx);
                 check_imm19(jmp_offset);
@@ -780,7 +858,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
                 goto out_off;
         }
 
-        build_prologue(&ctx);
+        if (build_prologue(&ctx)) {
+                prog = orig_prog;
+                goto out_off;
+        }
 
         ctx.epilogue_offset = ctx.idx;
         build_epilogue(&ctx);
diff --git a/drivers/isdn/hardware/eicon/divasmain.c b/drivers/isdn/hardware/eicon/divasmain.c
index a2e0ed6c9a4d..32f34511c416 100644
--- a/drivers/isdn/hardware/eicon/divasmain.c
+++ b/drivers/isdn/hardware/eicon/divasmain.c
@@ -445,32 +445,32 @@ void divasa_unmap_pci_bar(void __iomem *bar)
 /*********************************************************
  ** I/O port access
  *********************************************************/
-byte __inline__ inpp(void __iomem *addr)
+inline byte inpp(void __iomem *addr)
 {
         return (inb((unsigned long) addr));
 }
 
-word __inline__ inppw(void __iomem *addr)
+inline word inppw(void __iomem *addr)
 {
         return (inw((unsigned long) addr));
 }
 
-void __inline__ inppw_buffer(void __iomem *addr, void *P, int length)
+inline void inppw_buffer(void __iomem *addr, void *P, int length)
 {
         insw((unsigned long) addr, (word *) P, length >> 1);
 }
 
-void __inline__ outppw_buffer(void __iomem *addr, void *P, int length)
+inline void outppw_buffer(void __iomem *addr, void *P, int length)
 {
         outsw((unsigned long) addr, (word *) P, length >> 1);
 }
 
-void __inline__ outppw(void __iomem *addr, word w)
+inline void outppw(void __iomem *addr, word w)
 {
         outw(w, (unsigned long) addr);
 }
 
-void __inline__ outpp(void __iomem *addr, word p)
+inline void outpp(void __iomem *addr, word p)
 {
         outb(p, (unsigned long) addr);
 }
diff --git a/drivers/isdn/hardware/eicon/platform.h b/drivers/isdn/hardware/eicon/platform.h
index b2edb7590dda..62e2073c3690 100644
--- a/drivers/isdn/hardware/eicon/platform.h
+++ b/drivers/isdn/hardware/eicon/platform.h
@@ -203,7 +203,7 @@ void PCIread(byte bus, byte func, int offset, void *data, int length, void *pci_
 /*
 **  I/O Port utilities
 */
-int diva_os_register_io_port(void *adapter, int register, unsigned long port,
+int diva_os_register_io_port(void *adapter, int reg, unsigned long port,
                              unsigned long length, const char *name, int id);
 /*
 **  I/O port access abstraction
@@ -271,13 +271,13 @@ void diva_os_get_time(dword *sec, dword *usec);
 **  atomic operation, fake because we use threads
 */
 typedef int diva_os_atomic_t;
-static diva_os_atomic_t __inline__
+static inline diva_os_atomic_t
 diva_os_atomic_increment(diva_os_atomic_t *pv)
 {
         *pv += 1;
         return (*pv);
 }
-static diva_os_atomic_t __inline__
+static inline diva_os_atomic_t
 diva_os_atomic_decrement(diva_os_atomic_t *pv)
 {
         *pv -= 1;
diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c
index 941ec99cd3b6..90157e20357e 100644
--- a/drivers/net/bonding/bond_main.c
+++ b/drivers/net/bonding/bond_main.c
@@ -4607,26 +4607,6 @@ static int bond_check_params(struct bond_params *params)
         return 0;
 }
 
-static struct lock_class_key bonding_netdev_xmit_lock_key;
-static struct lock_class_key bonding_netdev_addr_lock_key;
-static struct lock_class_key bonding_tx_busylock_key;
-
-static void bond_set_lockdep_class_one(struct net_device *dev,
-                                       struct netdev_queue *txq,
-                                       void *_unused)
-{
-        lockdep_set_class(&txq->_xmit_lock,
-                          &bonding_netdev_xmit_lock_key);
-}
-
-static void bond_set_lockdep_class(struct net_device *dev)
-{
-        lockdep_set_class(&dev->addr_list_lock,
-                          &bonding_netdev_addr_lock_key);
-        netdev_for_each_tx_queue(dev, bond_set_lockdep_class_one, NULL);
-        dev->qdisc_tx_busylock = &bonding_tx_busylock_key;
-}
-
 /* Called from registration process */
 static int bond_init(struct net_device *bond_dev)
 {
@@ -4639,7 +4619,7 @@ static int bond_init(struct net_device *bond_dev)
         if (!bond->wq)
                 return -ENOMEM;
 
-        bond_set_lockdep_class(bond_dev);
+        netdev_lockdep_set_classes(bond_dev);
 
         list_add_tail(&bond->bond_list, &bn->dev_list);
 
diff --git a/drivers/net/can/Kconfig b/drivers/net/can/Kconfig
index 0d40aef928e2..22570ea3a8d2 100644
--- a/drivers/net/can/Kconfig
+++ b/drivers/net/can/Kconfig
@@ -104,16 +104,6 @@ config CAN_JANZ_ICAN3
           This driver can also be built as a module. If so, the module will be
           called janz-ican3.ko.
 
-config CAN_RCAR
-        tristate "Renesas R-Car CAN controller"
-        depends on ARCH_RENESAS || ARM
-        ---help---
-          Say Y here if you want to use CAN controller found on Renesas R-Car
-          SoCs.
-
-          To compile this driver as a module, choose M here: the module will
-          be called rcar_can.
-
 config CAN_SUN4I
         tristate "Allwinner A10 CAN controller"
         depends on MACH_SUN4I || MACH_SUN7I || COMPILE_TEST
@@ -152,6 +142,7 @@ source "drivers/net/can/cc770/Kconfig"
 source "drivers/net/can/ifi_canfd/Kconfig"
 source "drivers/net/can/m_can/Kconfig"
 source "drivers/net/can/mscan/Kconfig"
+source "drivers/net/can/rcar/Kconfig"
 source "drivers/net/can/sja1000/Kconfig"
 source "drivers/net/can/softing/Kconfig"
 source "drivers/net/can/spi/Kconfig"
diff --git a/drivers/net/can/Makefile b/drivers/net/can/Makefile
index e3db0c807f55..26ba4b794a0b 100644
--- a/drivers/net/can/Makefile
+++ b/drivers/net/can/Makefile
@@ -10,6 +10,7 @@ can-dev-y			:= dev.o
 
 can-dev-$(CONFIG_CAN_LEDS)      += led.o
 
+obj-y                           += rcar/
 obj-y                           += spi/
 obj-y                           += usb/
 obj-y                           += softing/
@@ -24,7 +25,6 @@ obj-$(CONFIG_CAN_IFI_CANFD)	+= ifi_canfd/
 obj-$(CONFIG_CAN_JANZ_ICAN3)    += janz-ican3.o
 obj-$(CONFIG_CAN_MSCAN)         += mscan/
 obj-$(CONFIG_CAN_M_CAN)         += m_can/
-obj-$(CONFIG_CAN_RCAR)          += rcar_can.o
 obj-$(CONFIG_CAN_SJA1000)       += sja1000/
 obj-$(CONFIG_CAN_SUN4I)         += sun4i_can.o
 obj-$(CONFIG_CAN_TI_HECC)       += ti_hecc.o
diff --git a/drivers/net/can/dev.c b/drivers/net/can/dev.c
index ad535a854e5c..e21f7cc5ae4d 100644
--- a/drivers/net/can/dev.c
+++ b/drivers/net/can/dev.c
@@ -69,6 +69,7 @@ EXPORT_SYMBOL_GPL(can_len2dlc);
 
 #ifdef CONFIG_CAN_CALC_BITTIMING
 #define CAN_CALC_MAX_ERROR 50 /* in one-tenth of a percent */
+#define CAN_CALC_SYNC_SEG 1
 
 /*
  * Bit-timing calculation derived from:
@@ -83,98 +84,126 @@ EXPORT_SYMBOL_GPL(can_len2dlc);
  * registers of the CAN controller. You can find more information
  * in the header file linux/can/netlink.h.
  */
-static int can_update_spt(const struct can_bittiming_const *btc,
-                          int sampl_pt, int tseg, int *tseg1, int *tseg2)
+static int can_update_sample_point(const struct can_bittiming_const *btc,
+                          unsigned int sample_point_nominal, unsigned int tseg,
+                          unsigned int *tseg1_ptr, unsigned int *tseg2_ptr,
+                          unsigned int *sample_point_error_ptr)
 {
-        *tseg2 = tseg + 1 - (sampl_pt * (tseg + 1)) / 1000;
-        if (*tseg2 < btc->tseg2_min)
-                *tseg2 = btc->tseg2_min;
-        if (*tseg2 > btc->tseg2_max)
-                *tseg2 = btc->tseg2_max;
-        *tseg1 = tseg - *tseg2;
-        if (*tseg1 > btc->tseg1_max) {
-                *tseg1 = btc->tseg1_max;
-                *tseg2 = tseg - *tseg1;
+        unsigned int sample_point_error, best_sample_point_error = UINT_MAX;
+        unsigned int sample_point, best_sample_point = 0;
+        unsigned int tseg1, tseg2;
+        int i;
+
+        for (i = 0; i <= 1; i++) {
+                tseg2 = tseg + CAN_CALC_SYNC_SEG - (sample_point_nominal * (tseg + CAN_CALC_SYNC_SEG)) / 1000 - i;
+                tseg2 = clamp(tseg2, btc->tseg2_min, btc->tseg2_max);
+                tseg1 = tseg - tseg2;
+                if (tseg1 > btc->tseg1_max) {
+                        tseg1 = btc->tseg1_max;
+                        tseg2 = tseg - tseg1;
+                }
+
+                sample_point = 1000 * (tseg + CAN_CALC_SYNC_SEG - tseg2) / (tseg + CAN_CALC_SYNC_SEG);
+                sample_point_error = abs(sample_point_nominal - sample_point);
+
+                if ((sample_point <= sample_point_nominal) && (sample_point_error < best_sample_point_error)) {
+                        best_sample_point = sample_point;
+                        best_sample_point_error = sample_point_error;
+                        *tseg1_ptr = tseg1;
+                        *tseg2_ptr = tseg2;
+                }
         }
-        return 1000 * (tseg + 1 - *tseg2) / (tseg + 1);
+
+        if (sample_point_error_ptr)
+                *sample_point_error_ptr = best_sample_point_error;
+
+        return best_sample_point;
 }
 
 static int can_calc_bittiming(struct net_device *dev, struct can_bittiming *bt,
                               const struct can_bittiming_const *btc)
 {
         struct can_priv *priv = netdev_priv(dev);
-        long best_error = 1000000000, error = 0;
-        int best_tseg = 0, best_brp = 0, brp = 0;
-        int tsegall, tseg = 0, tseg1 = 0, tseg2 = 0;
-        int spt_error = 1000, spt = 0, sampl_pt;
-        long rate;
+        unsigned int bitrate;                   /* current bitrate */
+        unsigned int bitrate_error;             /* difference between current and nominal value */
+        unsigned int best_bitrate_error = UINT_MAX;
+        unsigned int sample_point_error;        /* difference between current and nominal value */
+        unsigned int best_sample_point_error = UINT_MAX;
+        unsigned int sample_point_nominal;      /* nominal sample point */
+        unsigned int best_tseg = 0;             /* current best value for tseg */
+        unsigned int best_brp = 0;              /* current best value for brp */
+        unsigned int brp, tsegall, tseg, tseg1 = 0, tseg2 = 0;
         u64 v64;
 
         /* Use CiA recommended sample points */
         if (bt->sample_point) {
-                sampl_pt = bt->sample_point;
+                sample_point_nominal = bt->sample_point;
         } else {
                 if (bt->bitrate > 800000)
-                        sampl_pt = 750;
+                        sample_point_nominal = 750;
                 else if (bt->bitrate > 500000)
-                        sampl_pt = 800;
+                        sample_point_nominal = 800;
                 else
-                        sampl_pt = 875;
+                        sample_point_nominal = 875;
         }
 
         /* tseg even = round down, odd = round up */
         for (tseg = (btc->tseg1_max + btc->tseg2_max) * 2 + 1;
              tseg >= (btc->tseg1_min + btc->tseg2_min) * 2; tseg--) {
-                tsegall = 1 + tseg / 2;
+                tsegall = CAN_CALC_SYNC_SEG + tseg / 2;
+
                 /* Compute all possible tseg choices (tseg=tseg1+tseg2) */
                 brp = priv->clock.freq / (tsegall * bt->bitrate) + tseg % 2;
-                /* chose brp step which is possible in system */
+
+                /* choose brp step which is possible in system */
                 brp = (brp / btc->brp_inc) * btc->brp_inc;
                 if ((brp < btc->brp_min) || (brp > btc->brp_max))
                         continue;
-                rate = priv->clock.freq / (brp * tsegall);
-                error = bt->bitrate - rate;
+
+                bitrate = priv->clock.freq / (brp * tsegall);
+                bitrate_error = abs(bt->bitrate - bitrate);
+
                 /* tseg brp biterror */
-                if (error < 0)
-                        error = -error;
-                if (error > best_error)
+                if (bitrate_error > best_bitrate_error)
                         continue;
-                best_error = error;
-                if (error == 0) {
-                        spt = can_update_spt(btc, sampl_pt, tseg / 2,
-                                             &tseg1, &tseg2);
-                        error = sampl_pt - spt;
-                        if (error < 0)
-                                error = -error;
-                        if (error > spt_error)
-                                continue;
-                        spt_error = error;
-                }
+
+                /* reset sample point error if we have a better bitrate */
+                if (bitrate_error < best_bitrate_error)
+                        best_sample_point_error = UINT_MAX;
+
+                can_update_sample_point(btc, sample_point_nominal, tseg / 2, &tseg1, &tseg2, &sample_point_error);
+                if (sample_point_error > best_sample_point_error)
+                        continue;
+
+                best_sample_point_error = sample_point_error;
+                best_bitrate_error = bitrate_error;
                 best_tseg = tseg / 2;
                 best_brp = brp;
-                if (error == 0)
+
+                if (bitrate_error == 0 && sample_point_error == 0)
                         break;
         }
 
-        if (best_error) {
+        if (best_bitrate_error) {
                 /* Error in one-tenth of a percent */
-                error = (best_error * 1000) / bt->bitrate;
-                if (error > CAN_CALC_MAX_ERROR) {
+                v64 = (u64)best_bitrate_error * 1000;
+                do_div(v64, bt->bitrate);
+                bitrate_error = (u32)v64;
+                if (bitrate_error > CAN_CALC_MAX_ERROR) {
                         netdev_err(dev,
-                                   "bitrate error %ld.%ld%% too high\n",
-                                   error / 10, error % 10);
+                                   "bitrate error %d.%d%% too high\n",
+                                   bitrate_error / 10, bitrate_error % 10);
                         return -EDOM;
-                } else {
-                        netdev_warn(dev, "bitrate error %ld.%ld%%\n",
-                                    error / 10, error % 10);
                 }
+                netdev_warn(dev, "bitrate error %d.%d%%\n",
+                            bitrate_error / 10, bitrate_error % 10);
         }
 
         /* real sample point */
-        bt->sample_point = can_update_spt(btc, sampl_pt, best_tseg,
-                                          &tseg1, &tseg2);
+        bt->sample_point = can_update_sample_point(btc, sample_point_nominal, best_tseg,
+                                          &tseg1, &tseg2, NULL);
 
-        v64 = (u64)best_brp * 1000000000UL;
+        v64 = (u64)best_brp * 1000 * 1000 * 1000;
         do_div(v64, priv->clock.freq);
         bt->tq = (u32)v64;
         bt->prop_seg = tseg1 / 2;
@@ -182,9 +211,9 @@ static int can_calc_bittiming(struct net_device *dev, struct can_bittiming *bt,
         bt->phase_seg2 = tseg2;
 
         /* check for sjw user settings */
-        if (!bt->sjw || !btc->sjw_max)
+        if (!bt->sjw || !btc->sjw_max) {
                 bt->sjw = 1;
-        else {
+        } else {
                 /* bt->sjw is at least 1 -> sanitize upper bound to sjw_max */
                 if (bt->sjw > btc->sjw_max)
                         bt->sjw = btc->sjw_max;
@@ -194,8 +223,9 @@ static int can_calc_bittiming(struct net_device *dev, struct can_bittiming *bt,
         }
 
         bt->brp = best_brp;
-        /* real bit-rate */
-        bt->bitrate = priv->clock.freq / (bt->brp * (tseg1 + tseg2 + 1));
+
+        /* real bitrate */
+        bt->bitrate = priv->clock.freq / (bt->brp * (CAN_CALC_SYNC_SEG + tseg1 + tseg2));
 
         return 0;
 }
diff --git a/drivers/net/can/rcar/Kconfig b/drivers/net/can/rcar/Kconfig
new file mode 100644
index 000000000000..7b03a3a37db7
--- /dev/null
+++ b/drivers/net/can/rcar/Kconfig
@@ -0,0 +1,21 @@
+config CAN_RCAR
+        tristate "Renesas R-Car CAN controller"
+        depends on ARCH_RENESAS || ARM
+        ---help---
+          Say Y here if you want to use CAN controller found on Renesas R-Car
+          SoCs.
+
+          To compile this driver as a module, choose M here: the module will
+          be called rcar_can.
+
+config CAN_RCAR_CANFD
+        tristate "Renesas R-Car CAN FD controller"
+        depends on ARCH_RENESAS || ARM
+        ---help---
+          Say Y here if you want to use CAN FD controller found on
+          Renesas R-Car SoCs. The driver puts the controller in CAN FD only
+          mode, which can interoperate with CAN2.0 nodes but does not support
+          dedicated CAN 2.0 mode.
+
+          To compile this driver as a module, choose M here: the module will
+          be called rcar_canfd.
diff --git a/drivers/net/can/rcar/Makefile b/drivers/net/can/rcar/Makefile
new file mode 100644
index 000000000000..08de36a4cfcc
--- /dev/null
+++ b/drivers/net/can/rcar/Makefile
@@ -0,0 +1,6 @@
+#
+#  Makefile for the Renesas R-Car CAN & CAN FD controller drivers
+#
+
+obj-$(CONFIG_CAN_RCAR)          += rcar_can.o
+obj-$(CONFIG_CAN_RCAR_CANFD)    += rcar_canfd.o
diff --git a/drivers/net/can/rcar_can.c b/drivers/net/can/rcar/rcar_can.c
index 788459f6bf5c..788459f6bf5c 100644
--- a/drivers/net/can/rcar_can.c
+++ b/drivers/net/can/rcar/rcar_can.c
diff --git a/drivers/net/can/rcar/rcar_canfd.c b/drivers/net/can/rcar/rcar_canfd.c
new file mode 100644
index 000000000000..43cdd5544b0c
--- /dev/null
+++ b/drivers/net/can/rcar/rcar_canfd.c
@@ -0,0 +1,1858 @@
+/* Renesas R-Car CAN FD device driver
+ *
+ * Copyright (C) 2015 Renesas Electronics Corp.
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+
+/* The R-Car CAN FD controller can operate in either one of the below two modes
+ *  - CAN FD only mode
+ *  - Classical CAN (CAN 2.0) only mode
+ *
+ * This driver puts the controller in CAN FD only mode by default. In this
+ * mode, the controller acts as a CAN FD node that can also interoperate with
+ * CAN 2.0 nodes.
+ *
+ * To switch the controller to Classical CAN (CAN 2.0) only mode, add
+ * "renesas,no-can-fd" optional property to the device tree node. A h/w reset is
+ * also required to switch modes.
+ *
+ * Note: The h/w manual register naming convention is clumsy and not acceptable
+ * to use as it is in the driver. However, those names are added as comments
+ * wherever it is modified to a readable name.
+ */
+
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/interrupt.h>
+#include <linux/errno.h>
+#include <linux/netdevice.h>
+#include <linux/platform_device.h>
+#include <linux/can/led.h>
+#include <linux/can/dev.h>
+#include <linux/clk.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/bitmap.h>
+#include <linux/bitops.h>
+#include <linux/iopoll.h>
+
+#define RCANFD_DRV_NAME                 "rcar_canfd"
+
+/* Global register bits */
+
+/* RSCFDnCFDGRMCFG */
+#define RCANFD_GRMCFG_RCMC              BIT(0)
+
+/* RSCFDnCFDGCFG / RSCFDnGCFG */
+#define RCANFD_GCFG_EEFE                BIT(6)
+#define RCANFD_GCFG_CMPOC               BIT(5)  /* CAN FD only */
+#define RCANFD_GCFG_DCS                 BIT(4)
+#define RCANFD_GCFG_DCE                 BIT(1)
+#define RCANFD_GCFG_TPRI                BIT(0)
+
+/* RSCFDnCFDGCTR / RSCFDnGCTR */
+#define RCANFD_GCTR_TSRST               BIT(16)
+#define RCANFD_GCTR_CFMPOFIE            BIT(11) /* CAN FD only */
+#define RCANFD_GCTR_THLEIE              BIT(10)
+#define RCANFD_GCTR_MEIE                BIT(9)
+#define RCANFD_GCTR_DEIE                BIT(8)
+#define RCANFD_GCTR_GSLPR               BIT(2)
+#define RCANFD_GCTR_GMDC_MASK           (0x3)
+#define RCANFD_GCTR_GMDC_GOPM           (0x0)
+#define RCANFD_GCTR_GMDC_GRESET         (0x1)
+#define RCANFD_GCTR_GMDC_GTEST          (0x2)
+
+/* RSCFDnCFDGSTS / RSCFDnGSTS */
+#define RCANFD_GSTS_GRAMINIT            BIT(3)
+#define RCANFD_GSTS_GSLPSTS             BIT(2)
+#define RCANFD_GSTS_GHLTSTS             BIT(1)
+#define RCANFD_GSTS_GRSTSTS             BIT(0)
+/* Non-operational status */
+#define RCANFD_GSTS_GNOPM               (BIT(0) | BIT(1) | BIT(2) | BIT(3))
+
+/* RSCFDnCFDGERFL / RSCFDnGERFL */
+#define RCANFD_GERFL_EEF1               BIT(17)
+#define RCANFD_GERFL_EEF0               BIT(16)
+#define RCANFD_GERFL_CMPOF              BIT(3)  /* CAN FD only */
+#define RCANFD_GERFL_THLES              BIT(2)
+#define RCANFD_GERFL_MES                BIT(1)
+#define RCANFD_GERFL_DEF                BIT(0)
+
+#define RCANFD_GERFL_ERR(gpriv, x)      ((x) & (RCANFD_GERFL_EEF1 |\
+                                        RCANFD_GERFL_EEF0 | RCANFD_GERFL_MES |\
+                                        (gpriv->fdmode ?\
+                                         RCANFD_GERFL_CMPOF : 0)))
+
+/* AFL Rx rules registers */
+
+/* RSCFDnCFDGAFLCFG0 / RSCFDnGAFLCFG0 */
+#define RCANFD_GAFLCFG_SETRNC(n, x)     (((x) & 0xff) << (24 - n * 8))
+#define RCANFD_GAFLCFG_GETRNC(n, x)     (((x) >> (24 - n * 8)) & 0xff)
+
+/* RSCFDnCFDGAFLECTR / RSCFDnGAFLECTR */
+#define RCANFD_GAFLECTR_AFLDAE          BIT(8)
+#define RCANFD_GAFLECTR_AFLPN(x)        ((x) & 0x1f)
+
+/* RSCFDnCFDGAFLIDj / RSCFDnGAFLIDj */
+#define RCANFD_GAFLID_GAFLLB            BIT(29)
+
+/* RSCFDnCFDGAFLP1_j / RSCFDnGAFLP1_j */
+#define RCANFD_GAFLP1_GAFLFDP(x)        (1 << (x))
+
+/* Channel register bits */
+
+/* RSCFDnCmCFG - Classical CAN only */
+#define RCANFD_CFG_SJW(x)               (((x) & 0x3) << 24)
+#define RCANFD_CFG_TSEG2(x)             (((x) & 0x7) << 20)
+#define RCANFD_CFG_TSEG1(x)             (((x) & 0xf) << 16)
+#define RCANFD_CFG_BRP(x)               (((x) & 0x3ff) << 0)
+
+/* RSCFDnCFDCmNCFG - CAN FD only */
+#define RCANFD_NCFG_NTSEG2(x)           (((x) & 0x1f) << 24)
+#define RCANFD_NCFG_NTSEG1(x)           (((x) & 0x7f) << 16)
+#define RCANFD_NCFG_NSJW(x)             (((x) & 0x1f) << 11)
+#define RCANFD_NCFG_NBRP(x)             (((x) & 0x3ff) << 0)
+
+/* RSCFDnCFDCmCTR / RSCFDnCmCTR */
+#define RCANFD_CCTR_CTME                BIT(24)
+#define RCANFD_CCTR_ERRD                BIT(23)
+#define RCANFD_CCTR_BOM_MASK            (0x3 << 21)
+#define RCANFD_CCTR_BOM_ISO             (0x0 << 21)
+#define RCANFD_CCTR_BOM_BENTRY          (0x1 << 21)
+#define RCANFD_CCTR_BOM_BEND            (0x2 << 21)
+#define RCANFD_CCTR_TDCVFIE             BIT(19)
+#define RCANFD_CCTR_SOCOIE              BIT(18)
+#define RCANFD_CCTR_EOCOIE              BIT(17)
+#define RCANFD_CCTR_TAIE                BIT(16)
+#define RCANFD_CCTR_ALIE                BIT(15)
+#define RCANFD_CCTR_BLIE                BIT(14)
+#define RCANFD_CCTR_OLIE                BIT(13)
+#define RCANFD_CCTR_BORIE               BIT(12)
+#define RCANFD_CCTR_BOEIE               BIT(11)
+#define RCANFD_CCTR_EPIE                BIT(10)
+#define RCANFD_CCTR_EWIE                BIT(9)
+#define RCANFD_CCTR_BEIE                BIT(8)
+#define RCANFD_CCTR_CSLPR               BIT(2)
+#define RCANFD_CCTR_CHMDC_MASK          (0x3)
+#define RCANFD_CCTR_CHDMC_COPM          (0x0)
+#define RCANFD_CCTR_CHDMC_CRESET        (0x1)
+#define RCANFD_CCTR_CHDMC_CHLT          (0x2)
+
+/* RSCFDnCFDCmSTS / RSCFDnCmSTS */
+#define RCANFD_CSTS_COMSTS              BIT(7)
+#define RCANFD_CSTS_RECSTS              BIT(6)
+#define RCANFD_CSTS_TRMSTS              BIT(5)
+#define RCANFD_CSTS_BOSTS               BIT(4)
+#define RCANFD_CSTS_EPSTS               BIT(3)
+#define RCANFD_CSTS_SLPSTS              BIT(2)
+#define RCANFD_CSTS_HLTSTS              BIT(1)
+#define RCANFD_CSTS_CRSTSTS             BIT(0)
+
+#define RCANFD_CSTS_TECCNT(x)           (((x) >> 24) & 0xff)
+#define RCANFD_CSTS_RECCNT(x)           (((x) >> 16) & 0xff)
+
+/* RSCFDnCFDCmERFL / RSCFDnCmERFL */
+#define RCANFD_CERFL_ADERR              BIT(14)
+#define RCANFD_CERFL_B0ERR              BIT(13)
+#define RCANFD_CERFL_B1ERR              BIT(12)
+#define RCANFD_CERFL_CERR               BIT(11)
+#define RCANFD_CERFL_AERR               BIT(10)
+#define RCANFD_CERFL_FERR               BIT(9)
+#define RCANFD_CERFL_SERR               BIT(8)
+#define RCANFD_CERFL_ALF                BIT(7)
+#define RCANFD_CERFL_BLF                BIT(6)
+#define RCANFD_CERFL_OVLF               BIT(5)
+#define RCANFD_CERFL_BORF               BIT(4)
+#define RCANFD_CERFL_BOEF               BIT(3)
+#define RCANFD_CERFL_EPF                BIT(2)
+#define RCANFD_CERFL_EWF                BIT(1)
+#define RCANFD_CERFL_BEF                BIT(0)
+
+#define RCANFD_CERFL_ERR(x)             ((x) & (0x7fff)) /* above bits 14:0 */
+
+/* RSCFDnCFDCmDCFG */
+#define RCANFD_DCFG_DSJW(x)             (((x) & 0x7) << 24)
+#define RCANFD_DCFG_DTSEG2(x)           (((x) & 0x7) << 20)
+#define RCANFD_DCFG_DTSEG1(x)           (((x) & 0xf) << 16)
+#define RCANFD_DCFG_DBRP(x)             (((x) & 0xff) << 0)
+
+/* RSCFDnCFDCmFDCFG */
+#define RCANFD_FDCFG_TDCE               BIT(9)
+#define RCANFD_FDCFG_TDCOC              BIT(8)
+#define RCANFD_FDCFG_TDCO(x)            (((x) & 0x7f) >> 16)
+
+/* RSCFDnCFDRFCCx */
+#define RCANFD_RFCC_RFIM                BIT(12)
+#define RCANFD_RFCC_RFDC(x)             (((x) & 0x7) << 8)
+#define RCANFD_RFCC_RFPLS(x)            (((x) & 0x7) << 4)
+#define RCANFD_RFCC_RFIE                BIT(1)
+#define RCANFD_RFCC_RFE                 BIT(0)
+
+/* RSCFDnCFDRFSTSx */
+#define RCANFD_RFSTS_RFIF               BIT(3)
+#define RCANFD_RFSTS_RFMLT              BIT(2)
+#define RCANFD_RFSTS_RFFLL              BIT(1)
+#define RCANFD_RFSTS_RFEMP              BIT(0)
+
+/* RSCFDnCFDRFIDx */
+#define RCANFD_RFID_RFIDE               BIT(31)
+#define RCANFD_RFID_RFRTR               BIT(30)
+
+/* RSCFDnCFDRFPTRx */
+#define RCANFD_RFPTR_RFDLC(x)           (((x) >> 28) & 0xf)
+#define RCANFD_RFPTR_RFPTR(x)           (((x) >> 16) & 0xfff)
+#define RCANFD_RFPTR_RFTS(x)            (((x) >> 0) & 0xffff)
+
+/* RSCFDnCFDRFFDSTSx */
+#define RCANFD_RFFDSTS_RFFDF            BIT(2)
+#define RCANFD_RFFDSTS_RFBRS            BIT(1)
+#define RCANFD_RFFDSTS_RFESI            BIT(0)
+
+/* Common FIFO bits */
+
+/* RSCFDnCFDCFCCk */
+#define RCANFD_CFCC_CFTML(x)            (((x) & 0xf) << 20)
+#define RCANFD_CFCC_CFM(x)              (((x) & 0x3) << 16)
+#define RCANFD_CFCC_CFIM                BIT(12)
+#define RCANFD_CFCC_CFDC(x)             (((x) & 0x7) << 8)
+#define RCANFD_CFCC_CFPLS(x)            (((x) & 0x7) << 4)
+#define RCANFD_CFCC_CFTXIE              BIT(2)
+#define RCANFD_CFCC_CFE                 BIT(0)
+
+/* RSCFDnCFDCFSTSk */
+#define RCANFD_CFSTS_CFMC(x)            (((x) >> 8) & 0xff)
+#define RCANFD_CFSTS_CFTXIF             BIT(4)
+#define RCANFD_CFSTS_CFMLT              BIT(2)
+#define RCANFD_CFSTS_CFFLL              BIT(1)
+#define RCANFD_CFSTS_CFEMP              BIT(0)
+
+/* RSCFDnCFDCFIDk */
+#define RCANFD_CFID_CFIDE               BIT(31)
+#define RCANFD_CFID_CFRTR               BIT(30)
+#define RCANFD_CFID_CFID_MASK(x)        ((x) & 0x1fffffff)
+
+/* RSCFDnCFDCFPTRk */
+#define RCANFD_CFPTR_CFDLC(x)           (((x) & 0xf) << 28)
+#define RCANFD_CFPTR_CFPTR(x)           (((x) & 0xfff) << 16)
+#define RCANFD_CFPTR_CFTS(x)            (((x) & 0xff) << 0)
+
+/* RSCFDnCFDCFFDCSTSk */
+#define RCANFD_CFFDCSTS_CFFDF           BIT(2)
+#define RCANFD_CFFDCSTS_CFBRS           BIT(1)
+#define RCANFD_CFFDCSTS_CFESI           BIT(0)
+
+/* This controller supports either Classical CAN only mode or CAN FD only mode.
+ * These modes are supported in two separate set of register maps & names.
+ * However, some of the register offsets are common for both modes. Those
+ * offsets are listed below as Common registers.
+ *
+ * The CAN FD only mode specific registers & Classical CAN only mode specific
+ * registers are listed separately. Their register names starts with
+ * RCANFD_F_xxx & RCANFD_C_xxx respectively.
+ */
+
+/* Common registers */
+
+/* RSCFDnCFDCmNCFG / RSCFDnCmCFG */
+#define RCANFD_CCFG(m)                  (0x0000 + (0x10 * (m)))
+/* RSCFDnCFDCmCTR / RSCFDnCmCTR */
+#define RCANFD_CCTR(m)                  (0x0004 + (0x10 * (m)))
+/* RSCFDnCFDCmSTS / RSCFDnCmSTS */
+#define RCANFD_CSTS(m)                  (0x0008 + (0x10 * (m)))
+/* RSCFDnCFDCmERFL / RSCFDnCmERFL */
+#define RCANFD_CERFL(m)                 (0x000C + (0x10 * (m)))
+
+/* RSCFDnCFDGCFG / RSCFDnGCFG */
+#define RCANFD_GCFG                     (0x0084)
+/* RSCFDnCFDGCTR / RSCFDnGCTR */
+#define RCANFD_GCTR                     (0x0088)
+/* RSCFDnCFDGCTS / RSCFDnGCTS */
+#define RCANFD_GSTS                     (0x008c)
+/* RSCFDnCFDGERFL / RSCFDnGERFL */
+#define RCANFD_GERFL                    (0x0090)
+/* RSCFDnCFDGTSC / RSCFDnGTSC */
+#define RCANFD_GTSC                     (0x0094)
+/* RSCFDnCFDGAFLECTR / RSCFDnGAFLECTR */
+#define RCANFD_GAFLECTR                 (0x0098)
+/* RSCFDnCFDGAFLCFG0 / RSCFDnGAFLCFG0 */
+#define RCANFD_GAFLCFG0                 (0x009c)
+/* RSCFDnCFDGAFLCFG1 / RSCFDnGAFLCFG1 */
+#define RCANFD_GAFLCFG1                 (0x00a0)
+/* RSCFDnCFDRMNB / RSCFDnRMNB */
+#define RCANFD_RMNB                     (0x00a4)
+/* RSCFDnCFDRMND / RSCFDnRMND */
+#define RCANFD_RMND(y)                  (0x00a8 + (0x04 * (y)))
+
+/* RSCFDnCFDRFCCx / RSCFDnRFCCx */
+#define RCANFD_RFCC(x)                  (0x00b8 + (0x04 * (x)))
+/* RSCFDnCFDRFSTSx / RSCFDnRFSTSx */
+#define RCANFD_RFSTS(x)                 (0x00d8 + (0x04 * (x)))
+/* RSCFDnCFDRFPCTRx / RSCFDnRFPCTRx */
+#define RCANFD_RFPCTR(x)                (0x00f8 + (0x04 * (x)))
+
+/* Common FIFO Control registers */
+
+/* RSCFDnCFDCFCCx / RSCFDnCFCCx */
+#define RCANFD_CFCC(ch, idx)            (0x0118 + (0x0c * (ch)) + \
+                                         (0x04 * (idx)))
+/* RSCFDnCFDCFSTSx / RSCFDnCFSTSx */
+#define RCANFD_CFSTS(ch, idx)           (0x0178 + (0x0c * (ch)) + \
+                                         (0x04 * (idx)))
+/* RSCFDnCFDCFPCTRx / RSCFDnCFPCTRx */
+#define RCANFD_CFPCTR(ch, idx)          (0x01d8 + (0x0c * (ch)) + \
+                                         (0x04 * (idx)))
+
+/* RSCFDnCFDFESTS / RSCFDnFESTS */
+#define RCANFD_FESTS                    (0x0238)
+/* RSCFDnCFDFFSTS / RSCFDnFFSTS */
+#define RCANFD_FFSTS                    (0x023c)
+/* RSCFDnCFDFMSTS / RSCFDnFMSTS */
+#define RCANFD_FMSTS                    (0x0240)
+/* RSCFDnCFDRFISTS / RSCFDnRFISTS */
+#define RCANFD_RFISTS                   (0x0244)
+/* RSCFDnCFDCFRISTS / RSCFDnCFRISTS */
+#define RCANFD_CFRISTS                  (0x0248)
+/* RSCFDnCFDCFTISTS / RSCFDnCFTISTS */
+#define RCANFD_CFTISTS                  (0x024c)
+
+/* RSCFDnCFDTMCp / RSCFDnTMCp */
+#define RCANFD_TMC(p)                   (0x0250 + (0x01 * (p)))
+/* RSCFDnCFDTMSTSp / RSCFDnTMSTSp */
+#define RCANFD_TMSTS(p)                 (0x02d0 + (0x01 * (p)))
+
+/* RSCFDnCFDTMTRSTSp / RSCFDnTMTRSTSp */
+#define RCANFD_TMTRSTS(y)               (0x0350 + (0x04 * (y)))
+/* RSCFDnCFDTMTARSTSp / RSCFDnTMTARSTSp */
+#define RCANFD_TMTARSTS(y)              (0x0360 + (0x04 * (y)))
+/* RSCFDnCFDTMTCSTSp / RSCFDnTMTCSTSp */
+#define RCANFD_TMTCSTS(y)               (0x0370 + (0x04 * (y)))
+/* RSCFDnCFDTMTASTSp / RSCFDnTMTASTSp */
+#define RCANFD_TMTASTS(y)               (0x0380 + (0x04 * (y)))
+/* RSCFDnCFDTMIECy / RSCFDnTMIECy */
+#define RCANFD_TMIEC(y)                 (0x0390 + (0x04 * (y)))
+
+/* RSCFDnCFDTXQCCm / RSCFDnTXQCCm */
+#define RCANFD_TXQCC(m)                 (0x03a0 + (0x04 * (m)))
+/* RSCFDnCFDTXQSTSm / RSCFDnTXQSTSm */
+#define RCANFD_TXQSTS(m)                (0x03c0 + (0x04 * (m)))
+/* RSCFDnCFDTXQPCTRm / RSCFDnTXQPCTRm */
+#define RCANFD_TXQPCTR(m)               (0x03e0 + (0x04 * (m)))
+
+/* RSCFDnCFDTHLCCm / RSCFDnTHLCCm */
+#define RCANFD_THLCC(m)                 (0x0400 + (0x04 * (m)))
+/* RSCFDnCFDTHLSTSm / RSCFDnTHLSTSm */
+#define RCANFD_THLSTS(m)                (0x0420 + (0x04 * (m)))
+/* RSCFDnCFDTHLPCTRm / RSCFDnTHLPCTRm */
+#define RCANFD_THLPCTR(m)               (0x0440 + (0x04 * (m)))
+
+/* RSCFDnCFDGTINTSTS0 / RSCFDnGTINTSTS0 */
+#define RCANFD_GTINTSTS0                (0x0460)
+/* RSCFDnCFDGTINTSTS1 / RSCFDnGTINTSTS1 */
+#define RCANFD_GTINTSTS1                (0x0464)
+/* RSCFDnCFDGTSTCFG / RSCFDnGTSTCFG */
+#define RCANFD_GTSTCFG                  (0x0468)
+/* RSCFDnCFDGTSTCTR / RSCFDnGTSTCTR */
+#define RCANFD_GTSTCTR                  (0x046c)
+/* RSCFDnCFDGLOCKK / RSCFDnGLOCKK */
+#define RCANFD_GLOCKK                   (0x047c)
+/* RSCFDnCFDGRMCFG */
+#define RCANFD_GRMCFG                   (0x04fc)
+
+/* RSCFDnCFDGAFLIDj / RSCFDnGAFLIDj */
+#define RCANFD_GAFLID(offset, j)        ((offset) + (0x10 * (j)))
+/* RSCFDnCFDGAFLMj / RSCFDnGAFLMj */
+#define RCANFD_GAFLM(offset, j)         ((offset) + 0x04 + (0x10 * (j)))
+/* RSCFDnCFDGAFLP0j / RSCFDnGAFLP0j */
+#define RCANFD_GAFLP0(offset, j)        ((offset) + 0x08 + (0x10 * (j)))
+/* RSCFDnCFDGAFLP1j / RSCFDnGAFLP1j */
+#define RCANFD_GAFLP1(offset, j)        ((offset) + 0x0c + (0x10 * (j)))
+
+/* Classical CAN only mode register map */
+
+/* RSCFDnGAFLXXXj offset */
+#define RCANFD_C_GAFL_OFFSET            (0x0500)
+
+/* RSCFDnRMXXXq -> RCANFD_C_RMXXX(q) */
+#define RCANFD_C_RMID(q)                (0x0600 + (0x10 * (q)))
+#define RCANFD_C_RMPTR(q)               (0x0604 + (0x10 * (q)))
+#define RCANFD_C_RMDF0(q)               (0x0608 + (0x10 * (q)))
+#define RCANFD_C_RMDF1(q)               (0x060c + (0x10 * (q)))
+
+/* RSCFDnRFXXx -> RCANFD_C_RFXX(x) */
+#define RCANFD_C_RFOFFSET               (0x0e00)
+#define RCANFD_C_RFID(x)                (RCANFD_C_RFOFFSET + (0x10 * (x)))
+#define RCANFD_C_RFPTR(x)               (RCANFD_C_RFOFFSET + 0x04 + \
+                                         (0x10 * (x)))
+#define RCANFD_C_RFDF(x, df)            (RCANFD_C_RFOFFSET + 0x08 + \
+                                         (0x10 * (x)) + (0x04 * (df)))
+
+/* RSCFDnCFXXk -> RCANFD_C_CFXX(ch, k) */
+#define RCANFD_C_CFOFFSET               (0x0e80)
+#define RCANFD_C_CFID(ch, idx)          (RCANFD_C_CFOFFSET + (0x30 * (ch)) + \
+                                         (0x10 * (idx)))
+#define RCANFD_C_CFPTR(ch, idx)         (RCANFD_C_CFOFFSET + 0x04 + \
+                                         (0x30 * (ch)) + (0x10 * (idx)))
+#define RCANFD_C_CFDF(ch, idx, df)      (RCANFD_C_CFOFFSET + 0x08 + \
+                                         (0x30 * (ch)) + (0x10 * (idx)) + \
+                                         (0x04 * (df)))
+
+/* RSCFDnTMXXp -> RCANFD_C_TMXX(p) */
+#define RCANFD_C_TMID(p)                (0x1000 + (0x10 * (p)))
+#define RCANFD_C_TMPTR(p)               (0x1004 + (0x10 * (p)))
+#define RCANFD_C_TMDF0(p)               (0x1008 + (0x10 * (p)))
+#define RCANFD_C_TMDF1(p)               (0x100c + (0x10 * (p)))
+
+/* RSCFDnTHLACCm */
+#define RCANFD_C_THLACC(m)              (0x1800 + (0x04 * (m)))
+/* RSCFDnRPGACCr */
+#define RCANFD_C_RPGACC(r)              (0x1900 + (0x04 * (r)))
+
+/* CAN FD mode specific regsiter map */
+
+/* RSCFDnCFDCmXXX -> RCANFD_F_XXX(m) */
+#define RCANFD_F_DCFG(m)                (0x0500 + (0x20 * (m)))
+#define RCANFD_F_CFDCFG(m)              (0x0504 + (0x20 * (m)))
+#define RCANFD_F_CFDCTR(m)              (0x0508 + (0x20 * (m)))
+#define RCANFD_F_CFDSTS(m)              (0x050c + (0x20 * (m)))
+#define RCANFD_F_CFDCRC(m)              (0x0510 + (0x20 * (m)))
+
+/* RSCFDnCFDGAFLXXXj offset */
+#define RCANFD_F_GAFL_OFFSET            (0x1000)
+
+/* RSCFDnCFDRMXXXq -> RCANFD_F_RMXXX(q) */
+#define RCANFD_F_RMID(q)                (0x2000 + (0x20 * (q)))
+#define RCANFD_F_RMPTR(q)               (0x2004 + (0x20 * (q)))
+#define RCANFD_F_RMFDSTS(q)             (0x2008 + (0x20 * (q)))
+#define RCANFD_F_RMDF(q, b)             (0x200c + (0x04 * (b)) + (0x20 * (q)))
+
+/* RSCFDnCFDRFXXx -> RCANFD_F_RFXX(x) */
+#define RCANFD_F_RFOFFSET               (0x3000)
+#define RCANFD_F_RFID(x)                (RCANFD_F_RFOFFSET + (0x80 * (x)))
+#define RCANFD_F_RFPTR(x)               (RCANFD_F_RFOFFSET + 0x04 + \
+                                         (0x80 * (x)))
+#define RCANFD_F_RFFDSTS(x)             (RCANFD_F_RFOFFSET + 0x08 + \
+                                         (0x80 * (x)))
+#define RCANFD_F_RFDF(x, df)            (RCANFD_F_RFOFFSET + 0x0c + \
+                                         (0x80 * (x)) + (0x04 * (df)))
+
+/* RSCFDnCFDCFXXk -> RCANFD_F_CFXX(ch, k) */
+#define RCANFD_F_CFOFFSET               (0x3400)
+#define RCANFD_F_CFID(ch, idx)          (RCANFD_F_CFOFFSET + (0x180 * (ch)) + \
+                                         (0x80 * (idx)))
+#define RCANFD_F_CFPTR(ch, idx)         (RCANFD_F_CFOFFSET + 0x04 + \
+                                         (0x180 * (ch)) + (0x80 * (idx)))
+#define RCANFD_F_CFFDCSTS(ch, idx)      (RCANFD_F_CFOFFSET + 0x08 + \
+                                         (0x180 * (ch)) + (0x80 * (idx)))
+#define RCANFD_F_CFDF(ch, idx, df)      (RCANFD_F_CFOFFSET + 0x0c + \
+                                         (0x180 * (ch)) + (0x80 * (idx)) + \
+                                         (0x04 * (df)))
+
+/* RSCFDnCFDTMXXp -> RCANFD_F_TMXX(p) */
+#define RCANFD_F_TMID(p)                (0x4000 + (0x20 * (p)))
+#define RCANFD_F_TMPTR(p)               (0x4004 + (0x20 * (p)))
+#define RCANFD_F_TMFDCTR(p)             (0x4008 + (0x20 * (p)))
+#define RCANFD_F_TMDF(p, b)             (0x400c + (0x20 * (p)) + (0x04 * (b)))
+
+/* RSCFDnCFDTHLACCm */
+#define RCANFD_F_THLACC(m)              (0x6000 + (0x04 * (m)))
+/* RSCFDnCFDRPGACCr */
+#define RCANFD_F_RPGACC(r)              (0x6400 + (0x04 * (r)))
+
+/* Constants */
+#define RCANFD_FIFO_DEPTH               8       /* Tx FIFO depth */
+#define RCANFD_NAPI_WEIGHT              8       /* Rx poll quota */
+
+#define RCANFD_NUM_CHANNELS             2       /* Two channels max */
+#define RCANFD_CHANNELS_MASK            BIT((RCANFD_NUM_CHANNELS) - 1)
+
+#define RCANFD_GAFL_PAGENUM(entry)      ((entry) / 16)
+#define RCANFD_CHANNEL_NUMRULES         1       /* only one rule per channel */
+
+/* Rx FIFO is a global resource of the controller. There are 8 such FIFOs
+ * available. Each channel gets a dedicated Rx FIFO (i.e.) the channel
+ * number is added to RFFIFO index.
+ */
+#define RCANFD_RFFIFO_IDX               0
+
+/* Tx/Rx or Common FIFO is a per channel resource. Each channel has 3 Common
+ * FIFOs dedicated to them. Use the first (index 0) FIFO out of the 3 for Tx.
+ */
+#define RCANFD_CFFIFO_IDX               0
+
+/* fCAN clock select register settings */
+enum rcar_canfd_fcanclk {
+        RCANFD_CANFDCLK = 0,            /* CANFD clock */
+        RCANFD_EXTCLK,                  /* Externally input clock */
+};
+
+struct rcar_canfd_global;
+
+/* Channel priv data */
+struct rcar_canfd_channel {
+        struct can_priv can;                    /* Must be the first member */
+        struct net_device *ndev;
+        struct rcar_canfd_global *gpriv;        /* Controller reference */
+        void __iomem *base;                     /* Register base address */
+        struct napi_struct napi;
+        u8  tx_len[RCANFD_FIFO_DEPTH];          /* For net stats */
+        u32 tx_head;                            /* Incremented on xmit */
+        u32 tx_tail;                            /* Incremented on xmit done */
+        u32 channel;                            /* Channel number */
+        spinlock_t tx_lock;                     /* To protect tx path */
+};
+
+/* Global priv data */
+struct rcar_canfd_global {
+        struct rcar_canfd_channel *ch[RCANFD_NUM_CHANNELS];
+        void __iomem *base;             /* Register base address */
+        struct platform_device *pdev;   /* Respective platform device */
+        struct clk *clkp;               /* Peripheral clock */
+        struct clk *can_clk;            /* fCAN clock */
+        enum rcar_canfd_fcanclk fcan;   /* CANFD or Ext clock */
+        unsigned long channels_mask;    /* Enabled channels mask */
+        bool fdmode;                    /* CAN FD or Classical CAN only mode */
+};
+
+/* CAN FD mode nominal rate constants */
+static const struct can_bittiming_const rcar_canfd_nom_bittiming_const = {
+        .name = RCANFD_DRV_NAME,
+        .tseg1_min = 2,
+        .tseg1_max = 128,
+        .tseg2_min = 2,
+        .tseg2_max = 32,
+        .sjw_max = 32,
+        .brp_min = 1,
+        .brp_max = 1024,
+        .brp_inc = 1,
+};
+
+/* CAN FD mode data rate constants */
+static const struct can_bittiming_const rcar_canfd_data_bittiming_const = {
+        .name = RCANFD_DRV_NAME,
+        .tseg1_min = 2,
+        .tseg1_max = 16,
+        .tseg2_min = 2,
+        .tseg2_max = 8,
+        .sjw_max = 8,
+        .brp_min = 1,
+        .brp_max = 256,
+        .brp_inc = 1,
+};
+
+/* Classical CAN mode bitrate constants */
+static const struct can_bittiming_const rcar_canfd_bittiming_const = {
+        .name = RCANFD_DRV_NAME,
+        .tseg1_min = 4,
+        .tseg1_max = 16,
+        .tseg2_min = 2,
+        .tseg2_max = 8,
+        .sjw_max = 4,
+        .brp_min = 1,
+        .brp_max = 1024,
+        .brp_inc = 1,
+};
+
+/* Helper functions */
+static inline void rcar_canfd_update(u32 mask, u32 val, u32 __iomem *reg)
+{
+        u32 data = readl(reg);
+
+        data &= ~mask;
+        data |= (val & mask);
+        writel(data, reg);
+}
+
+static inline u32 rcar_canfd_read(void __iomem *base, u32 offset)
+{
+        return readl(base + (offset));
+}
+
+static inline void rcar_canfd_write(void __iomem *base, u32 offset, u32 val)
+{
+        writel(val, base + (offset));
+}
+
+static void rcar_canfd_set_bit(void __iomem *base, u32 reg, u32 val)
+{
+        rcar_canfd_update(val, val, base + (reg));
+}
+
+static void rcar_canfd_clear_bit(void __iomem *base, u32 reg, u32 val)
+{
+        rcar_canfd_update(val, 0, base + (reg));
+}
+
+static void rcar_canfd_update_bit(void __iomem *base, u32 reg,
+                                  u32 mask, u32 val)
+{
+        rcar_canfd_update(mask, val, base + (reg));
+}
+
+static void rcar_canfd_get_data(struct rcar_canfd_channel *priv,
+                                struct canfd_frame *cf, u32 off)
+{
+        u32 i, lwords;
+
+        lwords = DIV_ROUND_UP(cf->len, sizeof(u32));
+        for (i = 0; i < lwords; i++)
+                *((u32 *)cf->data + i) =
+                        rcar_canfd_read(priv->base, off + (i * sizeof(u32)));
+}
+
+static void rcar_canfd_put_data(struct rcar_canfd_channel *priv,
+                                struct canfd_frame *cf, u32 off)
+{
+        u32 i, lwords;
+
+        lwords = DIV_ROUND_UP(cf->len, sizeof(u32));
+        for (i = 0; i < lwords; i++)
+                rcar_canfd_write(priv->base, off + (i * sizeof(u32)),
+                                 *((u32 *)cf->data + i));
+}
+
+static void rcar_canfd_tx_failure_cleanup(struct net_device *ndev)
+{
+        u32 i;
+
+        for (i = 0; i < RCANFD_FIFO_DEPTH; i++)
+                can_free_echo_skb(ndev, i);
+}
+
+static int rcar_canfd_reset_controller(struct rcar_canfd_global *gpriv)
+{
+        u32 sts, ch;
+        int err;
+
+        /* Check RAMINIT flag as CAN RAM initialization takes place
+         * after the MCU reset
+         */
+        err = readl_poll_timeout((gpriv->base + RCANFD_GSTS), sts,
+                                 !(sts & RCANFD_GSTS_GRAMINIT), 2, 500000);
+        if (err) {
+                dev_dbg(&gpriv->pdev->dev, "global raminit failed\n");
+                return err;
+        }
+
+        /* Transition to Global Reset mode */
+        rcar_canfd_clear_bit(gpriv->base, RCANFD_GCTR, RCANFD_GCTR_GSLPR);
+        rcar_canfd_update_bit(gpriv->base, RCANFD_GCTR,
+                              RCANFD_GCTR_GMDC_MASK, RCANFD_GCTR_GMDC_GRESET);
+
+        /* Ensure Global reset mode */
+        err = readl_poll_timeout((gpriv->base + RCANFD_GSTS), sts,
+                                 (sts & RCANFD_GSTS_GRSTSTS), 2, 500000);
+        if (err) {
+                dev_dbg(&gpriv->pdev->dev, "global reset failed\n");
+                return err;
+        }
+
+        /* Reset Global error flags */
+        rcar_canfd_write(gpriv->base, RCANFD_GERFL, 0x0);
+
+        /* Set the controller into appropriate mode */
+        if (gpriv->fdmode)
+                rcar_canfd_set_bit(gpriv->base, RCANFD_GRMCFG,
+                                   RCANFD_GRMCFG_RCMC);
+        else
+                rcar_canfd_clear_bit(gpriv->base, RCANFD_GRMCFG,
+                                     RCANFD_GRMCFG_RCMC);
+
+        /* Transition all Channels to reset mode */
+        for_each_set_bit(ch, &gpriv->channels_mask, RCANFD_NUM_CHANNELS) {
+                rcar_canfd_clear_bit(gpriv->base,
+                                     RCANFD_CCTR(ch), RCANFD_CCTR_CSLPR);
+
+                rcar_canfd_update_bit(gpriv->base, RCANFD_CCTR(ch),
+                                      RCANFD_CCTR_CHMDC_MASK,
+                                      RCANFD_CCTR_CHDMC_CRESET);
+
+                /* Ensure Channel reset mode */
+                err = readl_poll_timeout((gpriv->base + RCANFD_CSTS(ch)), sts,
+                                         (sts & RCANFD_CSTS_CRSTSTS),
+                                         2, 500000);
+                if (err) {
+                        dev_dbg(&gpriv->pdev->dev,
+                                "channel %u reset failed\n", ch);
+                        return err;
+                }
+        }
+        return 0;
+}
+
+static void rcar_canfd_configure_controller(struct rcar_canfd_global *gpriv)
+{
+        u32 cfg, ch;
+
+        /* Global configuration settings */
+
+        /* ECC Error flag Enable */
+        cfg = RCANFD_GCFG_EEFE;
+
+        if (gpriv->fdmode)
+                /* Truncate payload to configured message size RFPLS */
+                cfg |= RCANFD_GCFG_CMPOC;
+
+        /* Set External Clock if selected */
+        if (gpriv->fcan != RCANFD_CANFDCLK)
+                cfg |= RCANFD_GCFG_DCS;
+
+        rcar_canfd_set_bit(gpriv->base, RCANFD_GCFG, cfg);
+
+        /* Channel configuration settings */
+        for_each_set_bit(ch, &gpriv->channels_mask, RCANFD_NUM_CHANNELS) {
+                rcar_canfd_set_bit(gpriv->base, RCANFD_CCTR(ch),
+                                   RCANFD_CCTR_ERRD);
+                rcar_canfd_update_bit(gpriv->base, RCANFD_CCTR(ch),
+                                      RCANFD_CCTR_BOM_MASK,
+                                      RCANFD_CCTR_BOM_BENTRY);
+        }
+}
+
+static void rcar_canfd_configure_afl_rules(struct rcar_canfd_global *gpriv,
+                                           u32 ch)
+{
+        u32 cfg;
+        int offset, start, page, num_rules = RCANFD_CHANNEL_NUMRULES;
+        u32 ridx = ch + RCANFD_RFFIFO_IDX;
+
+        if (ch == 0) {
+                start = 0; /* Channel 0 always starts from 0th rule */
+        } else {
+                /* Get number of Channel 0 rules and adjust */
+                cfg = rcar_canfd_read(gpriv->base, RCANFD_GAFLCFG0);
+                start = RCANFD_GAFLCFG_GETRNC(0, cfg);
+        }
+
+        /* Enable write access to entry */
+        page = RCANFD_GAFL_PAGENUM(start);
+        rcar_canfd_set_bit(gpriv->base, RCANFD_GAFLECTR,
+                           (RCANFD_GAFLECTR_AFLPN(page) |
+                            RCANFD_GAFLECTR_AFLDAE));
+
+        /* Write number of rules for channel */
+        rcar_canfd_set_bit(gpriv->base, RCANFD_GAFLCFG0,
+                           RCANFD_GAFLCFG_SETRNC(ch, num_rules));
+        if (gpriv->fdmode)
+                offset = RCANFD_F_GAFL_OFFSET;
+        else
+                offset = RCANFD_C_GAFL_OFFSET;
+
+        /* Accept all IDs */
+        rcar_canfd_write(gpriv->base, RCANFD_GAFLID(offset, start), 0);
+        /* IDE or RTR is not considered for matching */
+        rcar_canfd_write(gpriv->base, RCANFD_GAFLM(offset, start), 0);
+        /* Any data length accepted */
+        rcar_canfd_write(gpriv->base, RCANFD_GAFLP0(offset, start), 0);
+        /* Place the msg in corresponding Rx FIFO entry */
+        rcar_canfd_write(gpriv->base, RCANFD_GAFLP1(offset, start),
+                         RCANFD_GAFLP1_GAFLFDP(ridx));
+
+        /* Disable write access to page */
+        rcar_canfd_clear_bit(gpriv->base,
+                             RCANFD_GAFLECTR, RCANFD_GAFLECTR_AFLDAE);
+}
+
+static void rcar_canfd_configure_rx(struct rcar_canfd_global *gpriv, u32 ch)
+{
+        /* Rx FIFO is used for reception */
+        u32 cfg;
+        u16 rfdc, rfpls;
+
+        /* Select Rx FIFO based on channel */
+        u32 ridx = ch + RCANFD_RFFIFO_IDX;
+
+        rfdc = 2;               /* b010 - 8 messages Rx FIFO depth */
+        if (gpriv->fdmode)
+                rfpls = 7;      /* b111 - Max 64 bytes payload */
+        else
+                rfpls = 0;      /* b000 - Max 8 bytes payload */
+
+        cfg = (RCANFD_RFCC_RFIM | RCANFD_RFCC_RFDC(rfdc) |
+                RCANFD_RFCC_RFPLS(rfpls) | RCANFD_RFCC_RFIE);
+        rcar_canfd_write(gpriv->base, RCANFD_RFCC(ridx), cfg);
+}
+
+static void rcar_canfd_configure_tx(struct rcar_canfd_global *gpriv, u32 ch)
+{
+        /* Tx/Rx(Common) FIFO configured in Tx mode is
+         * used for transmission
+         *
+         * Each channel has 3 Common FIFO dedicated to them.
+         * Use the 1st (index 0) out of 3
+         */
+        u32 cfg;
+        u16 cftml, cfm, cfdc, cfpls;
+
+        cftml = 0;              /* 0th buffer */
+        cfm = 1;                /* b01 - Transmit mode */
+        cfdc = 2;               /* b010 - 8 messages Tx FIFO depth */
+        if (gpriv->fdmode)
+                cfpls = 7;      /* b111 - Max 64 bytes payload */
+        else
+                cfpls = 0;      /* b000 - Max 8 bytes payload */
+
+        cfg = (RCANFD_CFCC_CFTML(cftml) | RCANFD_CFCC_CFM(cfm) |
+                RCANFD_CFCC_CFIM | RCANFD_CFCC_CFDC(cfdc) |
+                RCANFD_CFCC_CFPLS(cfpls) | RCANFD_CFCC_CFTXIE);
+        rcar_canfd_write(gpriv->base, RCANFD_CFCC(ch, RCANFD_CFFIFO_IDX), cfg);
+
+        if (gpriv->fdmode)
+                /* Clear FD mode specific control/status register */
+                rcar_canfd_write(gpriv->base,
+                                 RCANFD_F_CFFDCSTS(ch, RCANFD_CFFIFO_IDX), 0);
+}
+
+static void rcar_canfd_enable_global_interrupts(struct rcar_canfd_global *gpriv)
+{
+        u32 ctr;
+
+        /* Clear any stray error interrupt flags */
+        rcar_canfd_write(gpriv->base, RCANFD_GERFL, 0);
+
+        /* Global interrupts setup */
+        ctr = RCANFD_GCTR_MEIE;
+        if (gpriv->fdmode)
+                ctr |= RCANFD_GCTR_CFMPOFIE;
+
+        rcar_canfd_set_bit(gpriv->base, RCANFD_GCTR, ctr);
+}
+
+static void rcar_canfd_disable_global_interrupts(struct rcar_canfd_global
+                                                 *gpriv)
+{
+        /* Disable all interrupts */
+        rcar_canfd_write(gpriv->base, RCANFD_GCTR, 0);
+
+        /* Clear any stray error interrupt flags */
+        rcar_canfd_write(gpriv->base, RCANFD_GERFL, 0);
+}
+
+static void rcar_canfd_enable_channel_interrupts(struct rcar_canfd_channel
+                                                 *priv)
+{
+        u32 ctr, ch = priv->channel;
+
+        /* Clear any stray error flags */
+        rcar_canfd_write(priv->base, RCANFD_CERFL(ch), 0);
+
+        /* Channel interrupts setup */
+        ctr = (RCANFD_CCTR_TAIE |
+               RCANFD_CCTR_ALIE | RCANFD_CCTR_BLIE |
+               RCANFD_CCTR_OLIE | RCANFD_CCTR_BORIE |
+               RCANFD_CCTR_BOEIE | RCANFD_CCTR_EPIE |
+               RCANFD_CCTR_EWIE | RCANFD_CCTR_BEIE);
+        rcar_canfd_set_bit(priv->base, RCANFD_CCTR(ch), ctr);
+}
+
+static void rcar_canfd_disable_channel_interrupts(struct rcar_canfd_channel
+                                                  *priv)
+{
+        u32 ctr, ch = priv->channel;
+
+        ctr = (RCANFD_CCTR_TAIE |
+               RCANFD_CCTR_ALIE | RCANFD_CCTR_BLIE |
+               RCANFD_CCTR_OLIE | RCANFD_CCTR_BORIE |
+               RCANFD_CCTR_BOEIE | RCANFD_CCTR_EPIE |
+               RCANFD_CCTR_EWIE | RCANFD_CCTR_BEIE);
+        rcar_canfd_clear_bit(priv->base, RCANFD_CCTR(ch), ctr);
+
+        /* Clear any stray error flags */
+        rcar_canfd_write(priv->base, RCANFD_CERFL(ch), 0);
+}
+
+static void rcar_canfd_global_error(struct net_device *ndev)
+{
+        struct rcar_canfd_channel *priv = netdev_priv(ndev);
+        struct rcar_canfd_global *gpriv = priv->gpriv;
+        struct net_device_stats *stats = &ndev->stats;
+        u32 ch = priv->channel;
+        u32 gerfl, sts;
+        u32 ridx = ch + RCANFD_RFFIFO_IDX;
+
+        gerfl = rcar_canfd_read(priv->base, RCANFD_GERFL);
+        if ((gerfl & RCANFD_GERFL_EEF0) && (ch == 0)) {
+                netdev_dbg(ndev, "Ch0: ECC Error flag\n");
+                stats->tx_dropped++;
+        }
+        if ((gerfl & RCANFD_GERFL_EEF1) && (ch == 1)) {
+                netdev_dbg(ndev, "Ch1: ECC Error flag\n");
+                stats->tx_dropped++;
+        }
+        if (gerfl & RCANFD_GERFL_MES) {
+                sts = rcar_canfd_read(priv->base,
+                                      RCANFD_CFSTS(ch, RCANFD_CFFIFO_IDX));
+                if (sts & RCANFD_CFSTS_CFMLT) {
+                        netdev_dbg(ndev, "Tx Message Lost flag\n");
+                        stats->tx_dropped++;
+                        rcar_canfd_write(priv->base,
+                                         RCANFD_CFSTS(ch, RCANFD_CFFIFO_IDX),
+                                         sts & ~RCANFD_CFSTS_CFMLT);
+                }
+
+                sts = rcar_canfd_read(priv->base, RCANFD_RFSTS(ridx));
+                if (sts & RCANFD_RFSTS_RFMLT) {
+                        netdev_dbg(ndev, "Rx Message Lost flag\n");
+                        stats->rx_dropped++;
+                        rcar_canfd_write(priv->base, RCANFD_RFSTS(ridx),
+                                         sts & ~RCANFD_RFSTS_RFMLT);
+                }
+        }
+        if (gpriv->fdmode && gerfl & RCANFD_GERFL_CMPOF) {
+                /* Message Lost flag will be set for respective channel
+                 * when this condition happens with counters and flags
+                 * already updated.
+                 */
+                netdev_dbg(ndev, "global payload overflow interrupt\n");
+        }
+
+        /* Clear all global error interrupts. Only affected channels bits
+         * get cleared
+         */
+        rcar_canfd_write(priv->base, RCANFD_GERFL, 0);
+}
+
+static void rcar_canfd_error(struct net_device *ndev, u32 cerfl,
+                             u16 txerr, u16 rxerr)
+{
+        struct rcar_canfd_channel *priv = netdev_priv(ndev);
+        struct net_device_stats *stats = &ndev->stats;
+        struct can_frame *cf;
+        struct sk_buff *skb;
+        u32 ch = priv->channel;
+
+        netdev_dbg(ndev, "ch erfl %x txerr %u rxerr %u\n", cerfl, txerr, rxerr);
+
+        /* Propagate the error condition to the CAN stack */
+        skb = alloc_can_err_skb(ndev, &cf);
+        if (!skb) {
+                stats->rx_dropped++;
+                return;
+        }
+
+        /* Channel error interrupts */
+        if (cerfl & RCANFD_CERFL_BEF) {
+                netdev_dbg(ndev, "Bus error\n");
+                cf->can_id |= CAN_ERR_BUSERROR | CAN_ERR_PROT;
+                cf->data[2] = CAN_ERR_PROT_UNSPEC;
+                priv->can.can_stats.bus_error++;
+        }
+        if (cerfl & RCANFD_CERFL_ADERR) {
+                netdev_dbg(ndev, "ACK Delimiter Error\n");
+                stats->tx_errors++;
+                cf->data[3] |= CAN_ERR_PROT_LOC_ACK_DEL;
+        }
+        if (cerfl & RCANFD_CERFL_B0ERR) {
+                netdev_dbg(ndev, "Bit Error (dominant)\n");
+                stats->tx_errors++;
+                cf->data[2] |= CAN_ERR_PROT_BIT0;
+        }
+        if (cerfl & RCANFD_CERFL_B1ERR) {
+                netdev_dbg(ndev, "Bit Error (recessive)\n");
+                stats->tx_errors++;
+                cf->data[2] |= CAN_ERR_PROT_BIT1;
+        }
+        if (cerfl & RCANFD_CERFL_CERR) {
+                netdev_dbg(ndev, "CRC Error\n");
+                stats->rx_errors++;
+                cf->data[3] |= CAN_ERR_PROT_LOC_CRC_SEQ;
+        }
+        if (cerfl & RCANFD_CERFL_AERR) {
+                netdev_dbg(ndev, "ACK Error\n");
+                stats->tx_errors++;
+                cf->can_id |= CAN_ERR_ACK;
+                cf->data[3] |= CAN_ERR_PROT_LOC_ACK;
+        }
+        if (cerfl & RCANFD_CERFL_FERR) {
+                netdev_dbg(ndev, "Form Error\n");
+                stats->rx_errors++;
+                cf->data[2] |= CAN_ERR_PROT_FORM;
+        }
+        if (cerfl & RCANFD_CERFL_SERR) {
+                netdev_dbg(ndev, "Stuff Error\n");
+                stats->rx_errors++;
+                cf->data[2] |= CAN_ERR_PROT_STUFF;
+        }
+        if (cerfl & RCANFD_CERFL_ALF) {
+                netdev_dbg(ndev, "Arbitration lost Error\n");
+                priv->can.can_stats.arbitration_lost++;
+                cf->can_id |= CAN_ERR_LOSTARB;
+                cf->data[0] |= CAN_ERR_LOSTARB_UNSPEC;
+        }
+        if (cerfl & RCANFD_CERFL_BLF) {
+                netdev_dbg(ndev, "Bus Lock Error\n");
+                stats->rx_errors++;
+                cf->can_id |= CAN_ERR_BUSERROR;
+        }
+        if (cerfl & RCANFD_CERFL_EWF) {
+                netdev_dbg(ndev, "Error warning interrupt\n");
+                priv->can.state = CAN_STATE_ERROR_WARNING;
+                priv->can.can_stats.error_warning++;
+                cf->can_id |= CAN_ERR_CRTL;
+                cf->data[1] = txerr > rxerr ? CAN_ERR_CRTL_TX_WARNING :
+                        CAN_ERR_CRTL_RX_WARNING;
+                cf->data[6] = txerr;
+                cf->data[7] = rxerr;
+        }
+        if (cerfl & RCANFD_CERFL_EPF) {
+                netdev_dbg(ndev, "Error passive interrupt\n");
+                priv->can.state = CAN_STATE_ERROR_PASSIVE;
+                priv->can.can_stats.error_passive++;
+                cf->can_id |= CAN_ERR_CRTL;
+                cf->data[1] = txerr > rxerr ? CAN_ERR_CRTL_TX_PASSIVE :
+                        CAN_ERR_CRTL_RX_PASSIVE;
+                cf->data[6] = txerr;
+                cf->data[7] = rxerr;
+        }
+        if (cerfl & RCANFD_CERFL_BOEF) {
+                netdev_dbg(ndev, "Bus-off entry interrupt\n");
+                rcar_canfd_tx_failure_cleanup(ndev);
+                priv->can.state = CAN_STATE_BUS_OFF;
+                priv->can.can_stats.bus_off++;
+                can_bus_off(ndev);
+                cf->can_id |= CAN_ERR_BUSOFF;
+        }
+        if (cerfl & RCANFD_CERFL_OVLF) {
+                netdev_dbg(ndev,
+                           "Overload Frame Transmission error interrupt\n");
+                stats->tx_errors++;
+                cf->can_id |= CAN_ERR_PROT;
+                cf->data[2] |= CAN_ERR_PROT_OVERLOAD;
+        }
+
+        /* Clear channel error interrupts that are handled */
+        rcar_canfd_write(priv->base, RCANFD_CERFL(ch),
+                         RCANFD_CERFL_ERR(~cerfl));
+        stats->rx_packets++;
+        stats->rx_bytes += cf->can_dlc;
+        netif_rx(skb);
+}
+
+static void rcar_canfd_tx_done(struct net_device *ndev)
+{
+        struct rcar_canfd_channel *priv = netdev_priv(ndev);
+        struct net_device_stats *stats = &ndev->stats;
+        u32 sts;
+        unsigned long flags;
+        u32 ch = priv->channel;
+
+        do {
+                u8 unsent, sent;
+
+                sent = priv->tx_tail % RCANFD_FIFO_DEPTH;
+                stats->tx_packets++;
+                stats->tx_bytes += priv->tx_len[sent];
+                priv->tx_len[sent] = 0;
+                can_get_echo_skb(ndev, sent);
+
+                spin_lock_irqsave(&priv->tx_lock, flags);
+                priv->tx_tail++;
+                sts = rcar_canfd_read(priv->base,
+                                      RCANFD_CFSTS(ch, RCANFD_CFFIFO_IDX));
+                unsent = RCANFD_CFSTS_CFMC(sts);
+
+                /* Wake producer only when there is room */
+                if (unsent != RCANFD_FIFO_DEPTH)
+                        netif_wake_queue(ndev);
+
+                if (priv->tx_head - priv->tx_tail <= unsent) {
+                        spin_unlock_irqrestore(&priv->tx_lock, flags);
+                        break;
+                }
+                spin_unlock_irqrestore(&priv->tx_lock, flags);
+
+        } while (1);
+
+        /* Clear interrupt */
+        rcar_canfd_write(priv->base, RCANFD_CFSTS(ch, RCANFD_CFFIFO_IDX),
+                         sts & ~RCANFD_CFSTS_CFTXIF);
+        can_led_event(ndev, CAN_LED_EVENT_TX);
+}
+
+static irqreturn_t rcar_canfd_global_interrupt(int irq, void *dev_id)
+{
+        struct rcar_canfd_global *gpriv = dev_id;
+        struct net_device *ndev;
+        struct rcar_canfd_channel *priv;
+        u32 sts, gerfl;
+        u32 ch, ridx;
+
+        /* Global error interrupts still indicate a condition specific
+         * to a channel. RxFIFO interrupt is a global interrupt.
+         */
+        for_each_set_bit(ch, &gpriv->channels_mask, RCANFD_NUM_CHANNELS) {
+                priv = gpriv->ch[ch];
+                ndev = priv->ndev;
+                ridx = ch + RCANFD_RFFIFO_IDX;
+
+                /* Global error interrupts */
+                gerfl = rcar_canfd_read(priv->base, RCANFD_GERFL);
+                if (unlikely(RCANFD_GERFL_ERR(gpriv, gerfl)))
+                        rcar_canfd_global_error(ndev);
+
+                /* Handle Rx interrupts */
+                sts = rcar_canfd_read(priv->base, RCANFD_RFSTS(ridx));
+                if (likely(sts & RCANFD_RFSTS_RFIF)) {
+                        if (napi_schedule_prep(&priv->napi)) {
+                                /* Disable Rx FIFO interrupts */
+                                rcar_canfd_clear_bit(priv->base,
+                                                     RCANFD_RFCC(ridx),
+                                                     RCANFD_RFCC_RFIE);
+                                __napi_schedule(&priv->napi);
+                        }
+                }
+        }
+        return IRQ_HANDLED;
+}
+
+static void rcar_canfd_state_change(struct net_device *ndev,
+                                    u16 txerr, u16 rxerr)
+{
+        struct rcar_canfd_channel *priv = netdev_priv(ndev);
+        struct net_device_stats *stats = &ndev->stats;
+        enum can_state rx_state, tx_state, state = priv->can.state;
+        struct can_frame *cf;
+        struct sk_buff *skb;
+
+        /* Handle transition from error to normal states */
+        if (txerr < 96 && rxerr < 96)
+                state = CAN_STATE_ERROR_ACTIVE;
+        else if (txerr < 128 && rxerr < 128)
+                state = CAN_STATE_ERROR_WARNING;
+
+        if (state != priv->can.state) {
+                netdev_dbg(ndev, "state: new %d, old %d: txerr %u, rxerr %u\n",
+                           state, priv->can.state, txerr, rxerr);
+                skb = alloc_can_err_skb(ndev, &cf);
+                if (!skb) {
+                        stats->rx_dropped++;
+                        return;
+                }
+                tx_state = txerr >= rxerr ? state : 0;
+                rx_state = txerr <= rxerr ? state : 0;
+
+                can_change_state(ndev, cf, tx_state, rx_state);
+                stats->rx_packets++;
+                stats->rx_bytes += cf->can_dlc;
+                netif_rx(skb);
+        }
+}
+
+static irqreturn_t rcar_canfd_channel_interrupt(int irq, void *dev_id)
+{
+        struct rcar_canfd_global *gpriv = dev_id;
+        struct net_device *ndev;
+        struct rcar_canfd_channel *priv;
+        u32 sts, ch, cerfl;
+        u16 txerr, rxerr;
+
+        /* Common FIFO is a per channel resource */
+        for_each_set_bit(ch, &gpriv->channels_mask, RCANFD_NUM_CHANNELS) {
+                priv = gpriv->ch[ch];
+                ndev = priv->ndev;
+
+                /* Channel error interrupts */
+                cerfl = rcar_canfd_read(priv->base, RCANFD_CERFL(ch));
+                sts = rcar_canfd_read(priv->base, RCANFD_CSTS(ch));
+                txerr = RCANFD_CSTS_TECCNT(sts);
+                rxerr = RCANFD_CSTS_RECCNT(sts);
+                if (unlikely(RCANFD_CERFL_ERR(cerfl)))
+                        rcar_canfd_error(ndev, cerfl, txerr, rxerr);
+
+                /* Handle state change to lower states */
+                if (unlikely((priv->can.state != CAN_STATE_ERROR_ACTIVE) &&
+                             (priv->can.state != CAN_STATE_BUS_OFF)))
+                        rcar_canfd_state_change(ndev, txerr, rxerr);
+
+                /* Handle Tx interrupts */
+                sts = rcar_canfd_read(priv->base,
+                                      RCANFD_CFSTS(ch, RCANFD_CFFIFO_IDX));
+                if (likely(sts & RCANFD_CFSTS_CFTXIF))
+                        rcar_canfd_tx_done(ndev);
+        }
+        return IRQ_HANDLED;
+}
+
+static void rcar_canfd_set_bittiming(struct net_device *dev)
+{
+        struct rcar_canfd_channel *priv = netdev_priv(dev);
+        const struct can_bittiming *bt = &priv->can.bittiming;
+        const struct can_bittiming *dbt = &priv->can.data_bittiming;
+        u16 brp, sjw, tseg1, tseg2;
+        u32 cfg;
+        u32 ch = priv->channel;
+
+        /* Nominal bit timing settings */
+        brp = bt->brp - 1;
+        sjw = bt->sjw - 1;
+        tseg1 = bt->prop_seg + bt->phase_seg1 - 1;
+        tseg2 = bt->phase_seg2 - 1;
+
+        if (priv->can.ctrlmode & CAN_CTRLMODE_FD) {
+                /* CAN FD only mode */
+                cfg = (RCANFD_NCFG_NTSEG1(tseg1) | RCANFD_NCFG_NBRP(brp) |
+                       RCANFD_NCFG_NSJW(sjw) | RCANFD_NCFG_NTSEG2(tseg2));
+
+                rcar_canfd_write(priv->base, RCANFD_CCFG(ch), cfg);
+                netdev_dbg(priv->ndev, "nrate: brp %u, sjw %u, tseg1 %u, tseg2 %u\n",
+                           brp, sjw, tseg1, tseg2);
+
+                /* Data bit timing settings */
+                brp = dbt->brp - 1;
+                sjw = dbt->sjw - 1;
+                tseg1 = dbt->prop_seg + dbt->phase_seg1 - 1;
+                tseg2 = dbt->phase_seg2 - 1;
+
+                cfg = (RCANFD_DCFG_DTSEG1(tseg1) | RCANFD_DCFG_DBRP(brp) |
+                       RCANFD_DCFG_DSJW(sjw) | RCANFD_DCFG_DTSEG2(tseg2));
+
+                rcar_canfd_write(priv->base, RCANFD_F_DCFG(ch), cfg);
+                netdev_dbg(priv->ndev, "drate: brp %u, sjw %u, tseg1 %u, tseg2 %u\n",
+                           brp, sjw, tseg1, tseg2);
+        } else {
+                /* Classical CAN only mode */
+                cfg = (RCANFD_CFG_TSEG1(tseg1) | RCANFD_CFG_BRP(brp) |
+                        RCANFD_CFG_SJW(sjw) | RCANFD_CFG_TSEG2(tseg2));
+
+                rcar_canfd_write(priv->base, RCANFD_CCFG(ch), cfg);
+                netdev_dbg(priv->ndev,
+                           "rate: brp %u, sjw %u, tseg1 %u, tseg2 %u\n",
+                           brp, sjw, tseg1, tseg2);
+        }
+}
+
+static int rcar_canfd_start(struct net_device *ndev)
+{
+        struct rcar_canfd_channel *priv = netdev_priv(ndev);
+        int err = -EOPNOTSUPP;
+        u32 sts, ch = priv->channel;
+        u32 ridx = ch + RCANFD_RFFIFO_IDX;
+
+        rcar_canfd_set_bittiming(ndev);
+
+        rcar_canfd_enable_channel_interrupts(priv);
+
+        /* Set channel to Operational mode */
+        rcar_canfd_update_bit(priv->base, RCANFD_CCTR(ch),
+                              RCANFD_CCTR_CHMDC_MASK, RCANFD_CCTR_CHDMC_COPM);
+
+        /* Verify channel mode change */
+        err = readl_poll_timeout((priv->base + RCANFD_CSTS(ch)), sts,
+                                 (sts & RCANFD_CSTS_COMSTS), 2, 500000);
+        if (err) {
+                netdev_err(ndev, "channel %u communication state failed\n", ch);
+                goto fail_mode_change;
+        }
+
+        /* Enable Common & Rx FIFO */
+        rcar_canfd_set_bit(priv->base, RCANFD_CFCC(ch, RCANFD_CFFIFO_IDX),
+                           RCANFD_CFCC_CFE);
+        rcar_canfd_set_bit(priv->base, RCANFD_RFCC(ridx), RCANFD_RFCC_RFE);
+
+        priv->can.state = CAN_STATE_ERROR_ACTIVE;
+        return 0;
+
+fail_mode_change:
+        rcar_canfd_disable_channel_interrupts(priv);
+        return err;
+}
+
+static int rcar_canfd_open(struct net_device *ndev)
+{
+        struct rcar_canfd_channel *priv = netdev_priv(ndev);
+        struct rcar_canfd_global *gpriv = priv->gpriv;
+        int err;
+
+        /* Peripheral clock is already enabled in probe */
+        err = clk_prepare_enable(gpriv->can_clk);
+        if (err) {
+                netdev_err(ndev, "failed to enable CAN clock, error %d\n", err);
+                goto out_clock;
+        }
+
+        err = open_candev(ndev);
+        if (err) {
+                netdev_err(ndev, "open_candev() failed, error %d\n", err);
+                goto out_can_clock;
+        }
+
+        napi_enable(&priv->napi);
+        err = rcar_canfd_start(ndev);
+        if (err)
+                goto out_close;
+        netif_start_queue(ndev);
+        can_led_event(ndev, CAN_LED_EVENT_OPEN);
+        return 0;
+out_close:
+        napi_disable(&priv->napi);
+        close_candev(ndev);
+out_can_clock:
+        clk_disable_unprepare(gpriv->can_clk);
+out_clock:
+        return err;
+}
+
+static void rcar_canfd_stop(struct net_device *ndev)
+{
+        struct rcar_canfd_channel *priv = netdev_priv(ndev);
+        int err;
+        u32 sts, ch = priv->channel;
+        u32 ridx = ch + RCANFD_RFFIFO_IDX;
+
+        /* Transition to channel reset mode  */
+        rcar_canfd_update_bit(priv->base, RCANFD_CCTR(ch),
+                              RCANFD_CCTR_CHMDC_MASK, RCANFD_CCTR_CHDMC_CRESET);
+
+        /* Check Channel reset mode */
+        err = readl_poll_timeout((priv->base + RCANFD_CSTS(ch)), sts,
+                                 (sts & RCANFD_CSTS_CRSTSTS), 2, 500000);
+        if (err)
+                netdev_err(ndev, "channel %u reset failed\n", ch);
+
+        rcar_canfd_disable_channel_interrupts(priv);
+
+        /* Disable Common & Rx FIFO */
+        rcar_canfd_clear_bit(priv->base, RCANFD_CFCC(ch, RCANFD_CFFIFO_IDX),
+                             RCANFD_CFCC_CFE);
+        rcar_canfd_clear_bit(priv->base, RCANFD_RFCC(ridx), RCANFD_RFCC_RFE);
+
+        /* Set the state as STOPPED */
+        priv->can.state = CAN_STATE_STOPPED;
+}
+
+static int rcar_canfd_close(struct net_device *ndev)
+{
+        struct rcar_canfd_channel *priv = netdev_priv(ndev);
+        struct rcar_canfd_global *gpriv = priv->gpriv;
+
+        netif_stop_queue(ndev);
+        rcar_canfd_stop(ndev);
+        napi_disable(&priv->napi);
+        clk_disable_unprepare(gpriv->can_clk);
+        close_candev(ndev);
+        can_led_event(ndev, CAN_LED_EVENT_STOP);
+        return 0;
+}
+
+static netdev_tx_t rcar_canfd_start_xmit(struct sk_buff *skb,
+                                         struct net_device *ndev)
+{
+        struct rcar_canfd_channel *priv = netdev_priv(ndev);
+        struct canfd_frame *cf = (struct canfd_frame *)skb->data;
+        u32 sts = 0, id, dlc;
+        unsigned long flags;
+        u32 ch = priv->channel;
+
+        if (can_dropped_invalid_skb(ndev, skb))
+                return NETDEV_TX_OK;
+
+        if (cf->can_id & CAN_EFF_FLAG) {
+                id = cf->can_id & CAN_EFF_MASK;
+                id |= RCANFD_CFID_CFIDE;
+        } else {
+                id = cf->can_id & CAN_SFF_MASK;
+        }
+
+        if (cf->can_id & CAN_RTR_FLAG)
+                id |= RCANFD_CFID_CFRTR;
+
+        dlc = RCANFD_CFPTR_CFDLC(can_len2dlc(cf->len));
+
+        if (priv->can.ctrlmode & CAN_CTRLMODE_FD) {
+                rcar_canfd_write(priv->base,
+                                 RCANFD_F_CFID(ch, RCANFD_CFFIFO_IDX), id);
+                rcar_canfd_write(priv->base,
+                                 RCANFD_F_CFPTR(ch, RCANFD_CFFIFO_IDX), dlc);
+
+                if (can_is_canfd_skb(skb)) {
+                        /* CAN FD frame format */
+                        sts |= RCANFD_CFFDCSTS_CFFDF;
+                        if (cf->flags & CANFD_BRS)
+                                sts |= RCANFD_CFFDCSTS_CFBRS;
+
+                        if (priv->can.state == CAN_STATE_ERROR_PASSIVE)
+                                sts |= RCANFD_CFFDCSTS_CFESI;
+                }
+
+                rcar_canfd_write(priv->base,
+                                 RCANFD_F_CFFDCSTS(ch, RCANFD_CFFIFO_IDX), sts);
+
+                rcar_canfd_put_data(priv, cf,
+                                    RCANFD_F_CFDF(ch, RCANFD_CFFIFO_IDX, 0));
+        } else {
+                rcar_canfd_write(priv->base,
+                                 RCANFD_C_CFID(ch, RCANFD_CFFIFO_IDX), id);
+                rcar_canfd_write(priv->base,
+                                 RCANFD_C_CFPTR(ch, RCANFD_CFFIFO_IDX), dlc);
+                rcar_canfd_put_data(priv, cf,
+                                    RCANFD_C_CFDF(ch, RCANFD_CFFIFO_IDX, 0));
+        }
+
+        priv->tx_len[priv->tx_head % RCANFD_FIFO_DEPTH] = cf->len;
+        can_put_echo_skb(skb, ndev, priv->tx_head % RCANFD_FIFO_DEPTH);
+
+        spin_lock_irqsave(&priv->tx_lock, flags);
+        priv->tx_head++;
+
+        /* Stop the queue if we've filled all FIFO entries */
+        if (priv->tx_head - priv->tx_tail >= RCANFD_FIFO_DEPTH)
+                netif_stop_queue(ndev);
+
+        /* Start Tx: Write 0xff to CFPC to increment the CPU-side
+         * pointer for the Common FIFO
+         */
+        rcar_canfd_write(priv->base,
+                         RCANFD_CFPCTR(ch, RCANFD_CFFIFO_IDX), 0xff);
+
+        spin_unlock_irqrestore(&priv->tx_lock, flags);
+        return NETDEV_TX_OK;
+}
+
+static void rcar_canfd_rx_pkt(struct rcar_canfd_channel *priv)
+{
+        struct net_device_stats *stats = &priv->ndev->stats;
+        struct canfd_frame *cf;
+        struct sk_buff *skb;
+        u32 sts = 0, id, dlc;
+        u32 ch = priv->channel;
+        u32 ridx = ch + RCANFD_RFFIFO_IDX;
+
+        if (priv->can.ctrlmode & CAN_CTRLMODE_FD) {
+                id = rcar_canfd_read(priv->base, RCANFD_F_RFID(ridx));
+                dlc = rcar_canfd_read(priv->base, RCANFD_F_RFPTR(ridx));
+
+                sts = rcar_canfd_read(priv->base, RCANFD_F_RFFDSTS(ridx));
+                if (sts & RCANFD_RFFDSTS_RFFDF)
+                        skb = alloc_canfd_skb(priv->ndev, &cf);
+                else
+                        skb = alloc_can_skb(priv->ndev,
+                                            (struct can_frame **)&cf);
+        } else {
+                id = rcar_canfd_read(priv->base, RCANFD_C_RFID(ridx));
+                dlc = rcar_canfd_read(priv->base, RCANFD_C_RFPTR(ridx));
+                skb = alloc_can_skb(priv->ndev, (struct can_frame **)&cf);
+        }
+
+        if (!skb) {
+                stats->rx_dropped++;
+                return;
+        }
+
+        if (id & RCANFD_RFID_RFIDE)
+                cf->can_id = (id & CAN_EFF_MASK) | CAN_EFF_FLAG;
+        else
+                cf->can_id = id & CAN_SFF_MASK;
+
+        if (priv->can.ctrlmode & CAN_CTRLMODE_FD) {
+                if (sts & RCANFD_RFFDSTS_RFFDF)
+                        cf->len = can_dlc2len(RCANFD_RFPTR_RFDLC(dlc));
+                else
+                        cf->len = get_can_dlc(RCANFD_RFPTR_RFDLC(dlc));
+
+                if (sts & RCANFD_RFFDSTS_RFESI) {
+                        cf->flags |= CANFD_ESI;
+                        netdev_dbg(priv->ndev, "ESI Error\n");
+                }
+
+                if (!(sts & RCANFD_RFFDSTS_RFFDF) && (id & RCANFD_RFID_RFRTR)) {
+                        cf->can_id |= CAN_RTR_FLAG;
+                } else {
+                        if (sts & RCANFD_RFFDSTS_RFBRS)
+                                cf->flags |= CANFD_BRS;
+
+                        rcar_canfd_get_data(priv, cf, RCANFD_F_RFDF(ridx, 0));
+                }
+        } else {
+                cf->len = get_can_dlc(RCANFD_RFPTR_RFDLC(dlc));
+                if (id & RCANFD_RFID_RFRTR)
+                        cf->can_id |= CAN_RTR_FLAG;
+                else
+                        rcar_canfd_get_data(priv, cf, RCANFD_C_RFDF(ridx, 0));
+        }
+
+        /* Write 0xff to RFPC to increment the CPU-side
+         * pointer of the Rx FIFO
+         */
+        rcar_canfd_write(priv->base, RCANFD_RFPCTR(ridx), 0xff);
+
+        can_led_event(priv->ndev, CAN_LED_EVENT_RX);
+
+        stats->rx_bytes += cf->len;
+        stats->rx_packets++;
+        netif_receive_skb(skb);
+}
+
+static int rcar_canfd_rx_poll(struct napi_struct *napi, int quota)
+{
+        struct rcar_canfd_channel *priv =
+                container_of(napi, struct rcar_canfd_channel, napi);
+        int num_pkts;
+        u32 sts;
+        u32 ch = priv->channel;
+        u32 ridx = ch + RCANFD_RFFIFO_IDX;
+
+        for (num_pkts = 0; num_pkts < quota; num_pkts++) {
+                sts = rcar_canfd_read(priv->base, RCANFD_RFSTS(ridx));
+                /* Check FIFO empty condition */
+                if (sts & RCANFD_RFSTS_RFEMP)
+                        break;
+
+                rcar_canfd_rx_pkt(priv);
+
+                /* Clear interrupt bit */
+                if (sts & RCANFD_RFSTS_RFIF)
+                        rcar_canfd_write(priv->base, RCANFD_RFSTS(ridx),
+                                         sts & ~RCANFD_RFSTS_RFIF);
+        }
+
+        /* All packets processed */
+        if (num_pkts < quota) {
+                napi_complete(napi);
+                /* Enable Rx FIFO interrupts */
+                rcar_canfd_set_bit(priv->base, RCANFD_RFCC(ridx),
+                                   RCANFD_RFCC_RFIE);
+        }
+        return num_pkts;
+}
+
+static int rcar_canfd_do_set_mode(struct net_device *ndev, enum can_mode mode)
+{
+        int err;
+
+        switch (mode) {
+        case CAN_MODE_START:
+                err = rcar_canfd_start(ndev);
+                if (err)
+                        return err;
+                netif_wake_queue(ndev);
+                return 0;
+        default:
+                return -EOPNOTSUPP;
+        }
+}
+
+static int rcar_canfd_get_berr_counter(const struct net_device *dev,
+                                       struct can_berr_counter *bec)
+{
+        struct rcar_canfd_channel *priv = netdev_priv(dev);
+        u32 val, ch = priv->channel;
+
+        /* Peripheral clock is already enabled in probe */
+        val = rcar_canfd_read(priv->base, RCANFD_CSTS(ch));
+        bec->txerr = RCANFD_CSTS_TECCNT(val);
+        bec->rxerr = RCANFD_CSTS_RECCNT(val);
+        return 0;
+}
+
+static const struct net_device_ops rcar_canfd_netdev_ops = {
+        .ndo_open = rcar_canfd_open,
+        .ndo_stop = rcar_canfd_close,
+        .ndo_start_xmit = rcar_canfd_start_xmit,
+        .ndo_change_mtu = can_change_mtu,
+};
+
+static int rcar_canfd_channel_probe(struct rcar_canfd_global *gpriv, u32 ch,
+                                    u32 fcan_freq)
+{
+        struct platform_device *pdev = gpriv->pdev;
+        struct rcar_canfd_channel *priv;
+        struct net_device *ndev;
+        int err = -ENODEV;
+
+        ndev = alloc_candev(sizeof(*priv), RCANFD_FIFO_DEPTH);
+        if (!ndev) {
+                dev_err(&pdev->dev, "alloc_candev() failed\n");
+                err = -ENOMEM;
+                goto fail;
+        }
+        priv = netdev_priv(ndev);
+
+        ndev->netdev_ops = &rcar_canfd_netdev_ops;
+        ndev->flags |= IFF_ECHO;
+        priv->ndev = ndev;
+        priv->base = gpriv->base;
+        priv->channel = ch;
+        priv->can.clock.freq = fcan_freq;
+        dev_info(&pdev->dev, "can_clk rate is %u\n", priv->can.clock.freq);
+
+        if (gpriv->fdmode) {
+                priv->can.bittiming_const = &rcar_canfd_nom_bittiming_const;
+                priv->can.data_bittiming_const =
+                        &rcar_canfd_data_bittiming_const;
+
+                /* Controller starts in CAN FD only mode */
+                can_set_static_ctrlmode(ndev, CAN_CTRLMODE_FD);
+                priv->can.ctrlmode_supported = CAN_CTRLMODE_BERR_REPORTING;
+        } else {
+                /* Controller starts in Classical CAN only mode */
+                priv->can.bittiming_const = &rcar_canfd_bittiming_const;
+                priv->can.ctrlmode_supported = CAN_CTRLMODE_BERR_REPORTING;
+        }
+
+        priv->can.do_set_mode = rcar_canfd_do_set_mode;
+        priv->can.do_get_berr_counter = rcar_canfd_get_berr_counter;
+        priv->gpriv = gpriv;
+        SET_NETDEV_DEV(ndev, &pdev->dev);
+
+        netif_napi_add(ndev, &priv->napi, rcar_canfd_rx_poll,
+                       RCANFD_NAPI_WEIGHT);
+        err = register_candev(ndev);
+        if (err) {
+                dev_err(&pdev->dev,
+                        "register_candev() failed, error %d\n", err);
+                goto fail_candev;
+        }
+        spin_lock_init(&priv->tx_lock);
+        devm_can_led_init(ndev);
+        gpriv->ch[priv->channel] = priv;
+        dev_info(&pdev->dev, "device registered (channel %u)\n", priv->channel);
+        return 0;
+
+fail_candev:
+        netif_napi_del(&priv->napi);
+        free_candev(ndev);
+fail:
+        return err;
+}
+
+static void rcar_canfd_channel_remove(struct rcar_canfd_global *gpriv, u32 ch)
+{
+        struct rcar_canfd_channel *priv = gpriv->ch[ch];
+
+        if (priv) {
+                unregister_candev(priv->ndev);
+                netif_napi_del(&priv->napi);
+                free_candev(priv->ndev);
+        }
+}
+
+static int rcar_canfd_probe(struct platform_device *pdev)
+{
+        struct resource *mem;
+        void __iomem *addr;
+        u32 sts, ch, fcan_freq;
+        struct rcar_canfd_global *gpriv;
+        struct device_node *of_child;
+        unsigned long channels_mask = 0;
+        int err, ch_irq, g_irq;
+        bool fdmode = true;                     /* CAN FD only mode - default */
+
+        if (of_property_read_bool(pdev->dev.of_node, "renesas,no-can-fd"))
+                fdmode = false;                 /* Classical CAN only mode */
+
+        of_child = of_get_child_by_name(pdev->dev.of_node, "channel0");
+        if (of_child && of_device_is_available(of_child))
+                channels_mask |= BIT(0);        /* Channel 0 */
+
+        of_child = of_get_child_by_name(pdev->dev.of_node, "channel1");
+        if (of_child && of_device_is_available(of_child))
+                channels_mask |= BIT(1);        /* Channel 1 */
+
+        ch_irq = platform_get_irq(pdev, 0);
+        if (ch_irq < 0) {
+                dev_err(&pdev->dev, "no Channel IRQ resource\n");
+                err = ch_irq;
+                goto fail_dev;
+        }
+
+        g_irq = platform_get_irq(pdev, 1);
+        if (g_irq < 0) {
+                dev_err(&pdev->dev, "no Global IRQ resource\n");
+                err = g_irq;
+                goto fail_dev;
+        }
+
+        /* Global controller context */
+        gpriv = devm_kzalloc(&pdev->dev, sizeof(*gpriv), GFP_KERNEL);
+        if (!gpriv) {
+                err = -ENOMEM;
+                goto fail_dev;
+        }
+        gpriv->pdev = pdev;
+        gpriv->channels_mask = channels_mask;
+        gpriv->fdmode = fdmode;
+
+        /* Peripheral clock */
+        gpriv->clkp = devm_clk_get(&pdev->dev, "fck");
+        if (IS_ERR(gpriv->clkp)) {
+                err = PTR_ERR(gpriv->clkp);
+                dev_err(&pdev->dev, "cannot get peripheral clock, error %d\n",
+                        err);
+                goto fail_dev;
+        }
+
+        /* fCAN clock: Pick External clock. If not available fallback to
+         * CANFD clock
+         */
+        gpriv->can_clk = devm_clk_get(&pdev->dev, "can_clk");
+        if (IS_ERR(gpriv->can_clk) || (clk_get_rate(gpriv->can_clk) == 0)) {
+                gpriv->can_clk = devm_clk_get(&pdev->dev, "canfd");
+                if (IS_ERR(gpriv->can_clk)) {
+                        err = PTR_ERR(gpriv->can_clk);
+                        dev_err(&pdev->dev,
+                                "cannot get canfd clock, error %d\n", err);
+                        goto fail_dev;
+                }
+                gpriv->fcan = RCANFD_CANFDCLK;
+
+        } else {
+                gpriv->fcan = RCANFD_EXTCLK;
+        }
+        fcan_freq = clk_get_rate(gpriv->can_clk);
+
+        if (gpriv->fcan == RCANFD_CANFDCLK)
+                /* CANFD clock is further divided by (1/2) within the IP */
+                fcan_freq /= 2;
+
+        mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+        addr = devm_ioremap_resource(&pdev->dev, mem);
+        if (IS_ERR(addr)) {
+                err = PTR_ERR(addr);
+                goto fail_dev;
+        }
+        gpriv->base = addr;
+
+        /* Request IRQ that's common for both channels */
+        err = devm_request_irq(&pdev->dev, ch_irq,
+                               rcar_canfd_channel_interrupt, 0,
+                               "canfd.chn", gpriv);
+        if (err) {
+                dev_err(&pdev->dev, "devm_request_irq(%d) failed, error %d\n",
+                        ch_irq, err);
+                goto fail_dev;
+        }
+        err = devm_request_irq(&pdev->dev, g_irq,
+                               rcar_canfd_global_interrupt, 0,
+                               "canfd.gbl", gpriv);
+        if (err) {
+                dev_err(&pdev->dev, "devm_request_irq(%d) failed, error %d\n",
+                        g_irq, err);
+                goto fail_dev;
+        }
+
+        /* Enable peripheral clock for register access */
+        err = clk_prepare_enable(gpriv->clkp);
+        if (err) {
+                dev_err(&pdev->dev,
+                        "failed to enable peripheral clock, error %d\n", err);
+                goto fail_dev;
+        }
+
+        err = rcar_canfd_reset_controller(gpriv);
+        if (err) {
+                dev_err(&pdev->dev, "reset controller failed\n");
+                goto fail_clk;
+        }
+
+        /* Controller in Global reset & Channel reset mode */
+        rcar_canfd_configure_controller(gpriv);
+
+        /* Configure per channel attributes */
+        for_each_set_bit(ch, &gpriv->channels_mask, RCANFD_NUM_CHANNELS) {
+                /* Configure Channel's Rx fifo */
+                rcar_canfd_configure_rx(gpriv, ch);
+
+                /* Configure Channel's Tx (Common) fifo */
+                rcar_canfd_configure_tx(gpriv, ch);
+
+                /* Configure receive rules */
+                rcar_canfd_configure_afl_rules(gpriv, ch);
+        }
+
+        /* Configure common interrupts */
+        rcar_canfd_enable_global_interrupts(gpriv);
+
+        /* Start Global operation mode */
+        rcar_canfd_update_bit(gpriv->base, RCANFD_GCTR, RCANFD_GCTR_GMDC_MASK,
+                              RCANFD_GCTR_GMDC_GOPM);
+
+        /* Verify mode change */
+        err = readl_poll_timeout((gpriv->base + RCANFD_GSTS), sts,
+                                 !(sts & RCANFD_GSTS_GNOPM), 2, 500000);
+        if (err) {
+                dev_err(&pdev->dev, "global operational mode failed\n");
+                goto fail_mode;
+        }
+
+        for_each_set_bit(ch, &gpriv->channels_mask, RCANFD_NUM_CHANNELS) {
+                err = rcar_canfd_channel_probe(gpriv, ch, fcan_freq);
+                if (err)
+                        goto fail_channel;
+        }
+
+        platform_set_drvdata(pdev, gpriv);
+        dev_info(&pdev->dev, "global operational state (clk %d, fdmode %d)\n",
+                 gpriv->fcan, gpriv->fdmode);
+        return 0;
+
+fail_channel:
+        for_each_set_bit(ch, &gpriv->channels_mask, RCANFD_NUM_CHANNELS)
+                rcar_canfd_channel_remove(gpriv, ch);
+fail_mode:
+        rcar_canfd_disable_global_interrupts(gpriv);
+fail_clk:
+        clk_disable_unprepare(gpriv->clkp);
+fail_dev:
+        return err;
+}
+
+static int rcar_canfd_remove(struct platform_device *pdev)
+{
+        struct rcar_canfd_global *gpriv = platform_get_drvdata(pdev);
+        u32 ch;
+
+        rcar_canfd_reset_controller(gpriv);
+        rcar_canfd_disable_global_interrupts(gpriv);
+
+        for_each_set_bit(ch, &gpriv->channels_mask, RCANFD_NUM_CHANNELS) {
+                rcar_canfd_disable_channel_interrupts(gpriv->ch[ch]);
+                rcar_canfd_channel_remove(gpriv, ch);
+        }
+
+        /* Enter global sleep mode */
+        rcar_canfd_set_bit(gpriv->base, RCANFD_GCTR, RCANFD_GCTR_GSLPR);
+        clk_disable_unprepare(gpriv->clkp);
+        return 0;
+}
+
+static int __maybe_unused rcar_canfd_suspend(struct device *dev)
+{
+        return 0;
+}
+
+static int __maybe_unused rcar_canfd_resume(struct device *dev)
+{
+        return 0;
+}
+
+static SIMPLE_DEV_PM_OPS(rcar_canfd_pm_ops, rcar_canfd_suspend,
+                         rcar_canfd_resume);
+
+static const struct of_device_id rcar_canfd_of_table[] = {
+        { .compatible = "renesas,rcar-gen3-canfd" },
+        { }
+};
+
+MODULE_DEVICE_TABLE(of, rcar_canfd_of_table);
+
+static struct platform_driver rcar_canfd_driver = {
+        .driver = {
+                .name = RCANFD_DRV_NAME,
+                .of_match_table = of_match_ptr(rcar_canfd_of_table),
+                .pm = &rcar_canfd_pm_ops,
+        },
+        .probe = rcar_canfd_probe,
+        .remove = rcar_canfd_remove,
+};
+
+module_platform_driver(rcar_canfd_driver);
+
+MODULE_AUTHOR("Ramesh Shanmugasundaram <ramesh.shanmugasundaram@bp.renesas.com>");
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("CAN FD driver for Renesas R-Car SoC");
+MODULE_ALIAS("platform:" RCANFD_DRV_NAME);
diff --git a/drivers/net/can/sja1000/tscan1.c b/drivers/net/can/sja1000/tscan1.c
index 76513dd780c7..79572457a2d6 100644
--- a/drivers/net/can/sja1000/tscan1.c
+++ b/drivers/net/can/sja1000/tscan1.c
@@ -203,14 +203,4 @@ static struct isa_driver tscan1_isa_driver = {
         },
 };
 
-static int __init tscan1_init(void)
-{
-        return isa_register_driver(&tscan1_isa_driver, TSCAN1_MAXDEV);
-}
-module_init(tscan1_init);
-
-static void __exit tscan1_exit(void)
-{
-        isa_unregister_driver(&tscan1_isa_driver);
-}
-module_exit(tscan1_exit);
+module_isa_driver(tscan1_isa_driver, TSCAN1_MAXDEV);
diff --git a/drivers/net/can/slcan.c b/drivers/net/can/slcan.c
index 9a3f15cb7ef4..eb7173713bbc 100644
--- a/drivers/net/can/slcan.c
+++ b/drivers/net/can/slcan.c
@@ -354,7 +354,7 @@ static netdev_tx_t slc_xmit(struct sk_buff *skb, struct net_device *dev)
 {
         struct slcan *sl = netdev_priv(dev);
 
-        if (skb->len != sizeof(struct can_frame))
+        if (skb->len != CAN_MTU)
                 goto out;
 
         spin_lock(&sl->lock);
@@ -442,7 +442,7 @@ static void slc_setup(struct net_device *dev)
         dev->addr_len           = 0;
         dev->tx_queue_len       = 10;
 
-        dev->mtu                = sizeof(struct can_frame);
+        dev->mtu                = CAN_MTU;
         dev->type               = ARPHRD_CAN;
 
         /* New-style flags. */
diff --git a/drivers/net/can/spi/mcp251x.c b/drivers/net/can/spi/mcp251x.c
index cf36d26ef002..f3f05fea8e1f 100644
--- a/drivers/net/can/spi/mcp251x.c
+++ b/drivers/net/can/spi/mcp251x.c
@@ -1145,8 +1145,11 @@ static int mcp251x_can_probe(struct spi_device *spi)
 
         /* Here is OK to not lock the MCP, no one knows about it yet */
         ret = mcp251x_hw_probe(spi);
-        if (ret)
+        if (ret) {
+                if (ret == -ENODEV)
+                        dev_err(&spi->dev, "Cannot initialize MCP%x. Wrong wiring?\n", priv->model);
                 goto error_probe;
+        }
 
         mcp251x_hw_sleep(spi);
 
@@ -1156,6 +1159,7 @@ static int mcp251x_can_probe(struct spi_device *spi)
 
         devm_can_led_init(net);
 
+        netdev_info(net, "MCP%x successfully initialized.\n", priv->model);
         return 0;
 
 error_probe:
@@ -1168,6 +1172,7 @@ out_clk:
 out_free:
         free_candev(net);
 
+        dev_err(&spi->dev, "Probe failed, err=%d\n", -ret);
         return ret;
 }
 
diff --git a/drivers/net/can/usb/gs_usb.c b/drivers/net/can/usb/gs_usb.c
index acb0c8490673..6f0cbc38782e 100644
--- a/drivers/net/can/usb/gs_usb.c
+++ b/drivers/net/can/usb/gs_usb.c
@@ -44,7 +44,9 @@ enum gs_usb_breq {
         GS_USB_BREQ_MODE,
         GS_USB_BREQ_BERR,
         GS_USB_BREQ_BT_CONST,
-        GS_USB_BREQ_DEVICE_CONFIG
+        GS_USB_BREQ_DEVICE_CONFIG,
+        GS_USB_BREQ_TIMESTAMP,
+        GS_USB_BREQ_IDENTIFY,
 };
 
 enum gs_can_mode {
@@ -63,6 +65,11 @@ enum gs_can_state {
         GS_CAN_STATE_SLEEPING
 };
 
+enum gs_can_identify_mode {
+        GS_CAN_IDENTIFY_OFF = 0,
+        GS_CAN_IDENTIFY_ON
+};
+
 /* data types passed between host and device */
 struct gs_host_config {
         u32 byte_order;
@@ -82,10 +89,10 @@ struct gs_device_config {
 } __packed;
 
 #define GS_CAN_MODE_NORMAL               0
-#define GS_CAN_MODE_LISTEN_ONLY          (1<<0)
-#define GS_CAN_MODE_LOOP_BACK            (1<<1)
-#define GS_CAN_MODE_TRIPLE_SAMPLE        (1<<2)
-#define GS_CAN_MODE_ONE_SHOT             (1<<3)
+#define GS_CAN_MODE_LISTEN_ONLY          BIT(0)
+#define GS_CAN_MODE_LOOP_BACK            BIT(1)
+#define GS_CAN_MODE_TRIPLE_SAMPLE        BIT(2)
+#define GS_CAN_MODE_ONE_SHOT             BIT(3)
 
 struct gs_device_mode {
         u32 mode;
@@ -106,10 +113,16 @@ struct gs_device_bittiming {
         u32 brp;
 } __packed;
 
-#define GS_CAN_FEATURE_LISTEN_ONLY      (1<<0)
-#define GS_CAN_FEATURE_LOOP_BACK        (1<<1)
-#define GS_CAN_FEATURE_TRIPLE_SAMPLE    (1<<2)
-#define GS_CAN_FEATURE_ONE_SHOT         (1<<3)
+struct gs_identify_mode {
+        u32 mode;
+} __packed;
+
+#define GS_CAN_FEATURE_LISTEN_ONLY      BIT(0)
+#define GS_CAN_FEATURE_LOOP_BACK        BIT(1)
+#define GS_CAN_FEATURE_TRIPLE_SAMPLE    BIT(2)
+#define GS_CAN_FEATURE_ONE_SHOT         BIT(3)
+#define GS_CAN_FEATURE_HW_TIMESTAMP     BIT(4)
+#define GS_CAN_FEATURE_IDENTIFY         BIT(5)
 
 struct gs_device_bt_const {
         u32 feature;
@@ -214,7 +227,8 @@ static void gs_free_tx_context(struct gs_tx_context *txc)
 
 /* Get a tx context by id.
  */
-static struct gs_tx_context *gs_get_tx_context(struct gs_can *dev, unsigned int id)
+static struct gs_tx_context *gs_get_tx_context(struct gs_can *dev,
+                                               unsigned int id)
 {
         unsigned long flags;
 
@@ -457,7 +471,8 @@ static void gs_usb_xmit_callback(struct urb *urb)
                 netif_wake_queue(netdev);
 }
 
-static netdev_tx_t gs_can_start_xmit(struct sk_buff *skb, struct net_device *netdev)
+static netdev_tx_t gs_can_start_xmit(struct sk_buff *skb,
+                                     struct net_device *netdev)
 {
         struct gs_can *dev = netdev_priv(netdev);
         struct net_device_stats *stats = &dev->netdev->stats;
@@ -663,7 +678,8 @@ static int gs_can_open(struct net_device *netdev)
         rc = usb_control_msg(interface_to_usbdev(dev->iface),
                              usb_sndctrlpipe(interface_to_usbdev(dev->iface), 0),
                              GS_USB_BREQ_MODE,
-                             USB_DIR_OUT|USB_TYPE_VENDOR|USB_RECIP_INTERFACE,
+                             USB_DIR_OUT | USB_TYPE_VENDOR |
+                             USB_RECIP_INTERFACE,
                              dev->channel,
                              0,
                              dm,
@@ -726,7 +742,59 @@ static const struct net_device_ops gs_usb_netdev_ops = {
         .ndo_change_mtu = can_change_mtu,
 };
 
-static struct gs_can *gs_make_candev(unsigned int channel, struct usb_interface *intf)
+static int gs_usb_set_identify(struct net_device *netdev, bool do_identify)
+{
+        struct gs_can *dev = netdev_priv(netdev);
+        struct gs_identify_mode imode;
+        int rc;
+
+        if (do_identify)
+                imode.mode = GS_CAN_IDENTIFY_ON;
+        else
+                imode.mode = GS_CAN_IDENTIFY_OFF;
+
+        rc = usb_control_msg(interface_to_usbdev(dev->iface),
+                             usb_sndctrlpipe(interface_to_usbdev(dev->iface),
+                                             0),
+                             GS_USB_BREQ_IDENTIFY,
+                             USB_DIR_OUT | USB_TYPE_VENDOR |
+                             USB_RECIP_INTERFACE,
+                             dev->channel,
+                             0,
+                             &imode,
+                             sizeof(imode),
+                             100);
+
+        return (rc > 0) ? 0 : rc;
+}
+
+/* blink LED's for finding the this interface */
+static int gs_usb_set_phys_id(struct net_device *dev,
+                              enum ethtool_phys_id_state state)
+{
+        int rc = 0;
+
+        switch (state) {
+        case ETHTOOL_ID_ACTIVE:
+                rc = gs_usb_set_identify(dev, GS_CAN_IDENTIFY_ON);
+                break;
+        case ETHTOOL_ID_INACTIVE:
+                rc = gs_usb_set_identify(dev, GS_CAN_IDENTIFY_OFF);
+                break;
+        default:
+                break;
+        }
+
+        return rc;
+}
+
+static const struct ethtool_ops gs_usb_ethtool_ops = {
+        .set_phys_id = gs_usb_set_phys_id,
+};
+
+static struct gs_can *gs_make_candev(unsigned int channel,
+                                     struct usb_interface *intf,
+                                     struct gs_device_config *dconf)
 {
         struct gs_can *dev;
         struct net_device *netdev;
@@ -814,10 +882,14 @@ static struct gs_can *gs_make_candev(unsigned int channel, struct usb_interface
         if (bt_const->feature & GS_CAN_FEATURE_ONE_SHOT)
                 dev->can.ctrlmode_supported |= CAN_CTRLMODE_ONE_SHOT;
 
-        kfree(bt_const);
-
         SET_NETDEV_DEV(netdev, &intf->dev);
 
+        if (dconf->sw_version > 1)
+                if (bt_const->feature & GS_CAN_FEATURE_IDENTIFY)
+                        netdev->ethtool_ops = &gs_usb_ethtool_ops;
+
+        kfree(bt_const);
+
         rc = register_candev(dev->netdev);
         if (rc) {
                 free_candev(dev->netdev);
@@ -835,19 +907,16 @@ static void gs_destroy_candev(struct gs_can *dev)
         free_candev(dev->netdev);
 }
 
-static int gs_usb_probe(struct usb_interface *intf, const struct usb_device_id *id)
+static int gs_usb_probe(struct usb_interface *intf,
+                        const struct usb_device_id *id)
 {
         struct gs_usb *dev;
         int rc = -ENOMEM;
         unsigned int icount, i;
-        struct gs_host_config *hconf;
-        struct gs_device_config *dconf;
-
-        hconf = kmalloc(sizeof(*hconf), GFP_KERNEL);
-        if (!hconf)
-                return -ENOMEM;
-
-        hconf->byte_order = 0x0000beef;
+        struct gs_host_config hconf = {
+                .byte_order = 0x0000beef,
+        };
+        struct gs_device_config dconf;
 
         /* send host config */
         rc = usb_control_msg(interface_to_usbdev(intf),
@@ -856,22 +925,16 @@ static int gs_usb_probe(struct usb_interface *intf, const struct usb_device_id *
                              USB_DIR_OUT|USB_TYPE_VENDOR|USB_RECIP_INTERFACE,
                              1,
                              intf->altsetting[0].desc.bInterfaceNumber,
-                             hconf,
-                             sizeof(*hconf),
+                             &hconf,
+                             sizeof(hconf),
                              1000);
 
-        kfree(hconf);
-
         if (rc < 0) {
                 dev_err(&intf->dev, "Couldn't send data format (err=%d)\n",
                         rc);
                 return rc;
         }
 
-        dconf = kmalloc(sizeof(*dconf), GFP_KERNEL);
-        if (!dconf)
-                return -ENOMEM;
-
         /* read device config */
         rc = usb_control_msg(interface_to_usbdev(intf),
                              usb_rcvctrlpipe(interface_to_usbdev(intf), 0),
@@ -879,22 +942,16 @@ static int gs_usb_probe(struct usb_interface *intf, const struct usb_device_id *
                              USB_DIR_IN|USB_TYPE_VENDOR|USB_RECIP_INTERFACE,
                              1,
                              intf->altsetting[0].desc.bInterfaceNumber,
-                             dconf,
-                             sizeof(*dconf),
+                             &dconf,
+                             sizeof(dconf),
                              1000);
         if (rc < 0) {
                 dev_err(&intf->dev, "Couldn't get device config: (err=%d)\n",
                         rc);
-
-                kfree(dconf);
-
                 return rc;
         }
 
-        icount = dconf->icount+1;
-
-        kfree(dconf);
-
+        icount = dconf.icount + 1;
         dev_info(&intf->dev, "Configuring for %d interfaces\n", icount);
 
         if (icount > GS_MAX_INTF) {
@@ -915,7 +972,7 @@ static int gs_usb_probe(struct usb_interface *intf, const struct usb_device_id *
         dev->udev = interface_to_usbdev(intf);
 
         for (i = 0; i < icount; i++) {
-                dev->canch[i] = gs_make_candev(i, intf);
+                dev->canch[i] = gs_make_candev(i, intf, &dconf);
                 if (IS_ERR_OR_NULL(dev->canch[i])) {
                         /* save error code to return later */
                         rc = PTR_ERR(dev->canch[i]);
diff --git a/drivers/net/dsa/Kconfig b/drivers/net/dsa/Kconfig
index 200663c43ce9..8f4544394f44 100644
--- a/drivers/net/dsa/Kconfig
+++ b/drivers/net/dsa/Kconfig
@@ -9,14 +9,6 @@ config NET_DSA_MV88E6060
           This enables support for the Marvell 88E6060 ethernet switch
           chip.
 
-config NET_DSA_MV88E6XXX
-        tristate "Marvell 88E6xxx Ethernet switch chip support"
-        depends on NET_DSA
-        select NET_DSA_TAG_EDSA
-        ---help---
-          This enables support for most of the Marvell 88E6xxx models of
-          Ethernet switch chips, except 88E6060.
-
 config NET_DSA_BCM_SF2
         tristate "Broadcom Starfighter 2 Ethernet switch support"
         depends on HAS_IOMEM && NET_DSA
@@ -28,4 +20,8 @@ config NET_DSA_BCM_SF2
           This enables support for the Broadcom Starfighter 2 Ethernet
           switch chips.
 
+source "drivers/net/dsa/b53/Kconfig"
+
+source "drivers/net/dsa/mv88e6xxx/Kconfig"
+
 endmenu
diff --git a/drivers/net/dsa/Makefile b/drivers/net/dsa/Makefile
index 76b751dd9efd..ca1e71b853a6 100644
--- a/drivers/net/dsa/Makefile
+++ b/drivers/net/dsa/Makefile
@@ -1,3 +1,5 @@
 obj-$(CONFIG_NET_DSA_MV88E6060) += mv88e6060.o
-obj-$(CONFIG_NET_DSA_MV88E6XXX) += mv88e6xxx.o
 obj-$(CONFIG_NET_DSA_BCM_SF2)   += bcm_sf2.o
+
+obj-y                           += b53/
+obj-y                           += mv88e6xxx/
diff --git a/drivers/net/dsa/b53/Kconfig b/drivers/net/dsa/b53/Kconfig
new file mode 100644
index 000000000000..27f32a50df57
--- /dev/null
+++ b/drivers/net/dsa/b53/Kconfig
@@ -0,0 +1,33 @@
+menuconfig B53
+        tristate "Broadcom BCM53xx managed switch support"
+        depends on NET_DSA
+        help
+          This driver adds support for Broadcom managed switch chips. It supports
+          BCM5325E, BCM5365, BCM539x, BCM53115 and BCM53125 as well as BCM63XX
+          integrated switches.
+
+config B53_SPI_DRIVER
+        tristate "B53 SPI connected switch driver"
+        depends on B53 && SPI
+        help
+          Select to enable support for registering switches configured through SPI.
+
+config B53_MDIO_DRIVER
+        tristate "B53 MDIO connected switch driver"
+        depends on B53
+        help
+          Select to enable support for registering switches configured through MDIO.
+
+config B53_MMAP_DRIVER
+        tristate "B53 MMAP connected switch driver"
+        depends on B53 && HAS_IOMEM
+        help
+          Select to enable support for memory-mapped switches like the BCM63XX
+          integrated switches.
+
+config B53_SRAB_DRIVER
+        tristate "B53 SRAB connected switch driver"
+        depends on B53 && HAS_IOMEM
+        help
+          Select to enable support for memory-mapped Switch Register Access
+          Bridge Registers (SRAB) like it is found on the BCM53010
diff --git a/drivers/net/dsa/b53/Makefile b/drivers/net/dsa/b53/Makefile
new file mode 100644
index 000000000000..7e6f9a8bfd75
--- /dev/null
+++ b/drivers/net/dsa/b53/Makefile
@@ -0,0 +1,6 @@
+obj-$(CONFIG_B53)               += b53_common.o
+
+obj-$(CONFIG_B53_SPI_DRIVER)    += b53_spi.o
+obj-$(CONFIG_B53_MDIO_DRIVER)   += b53_mdio.o
+obj-$(CONFIG_B53_MMAP_DRIVER)   += b53_mmap.o
+obj-$(CONFIG_B53_SRAB_DRIVER)   += b53_srab.o
diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c
new file mode 100644
index 000000000000..444de66667b9
--- /dev/null
+++ b/drivers/net/dsa/b53/b53_common.c
@@ -0,0 +1,1787 @@
+/*
+ * B53 switch driver main logic
+ *
+ * Copyright (C) 2011-2013 Jonas Gorski <jogo@openwrt.org>
+ * Copyright (C) 2016 Florian Fainelli <f.fainelli@gmail.com>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/delay.h>
+#include <linux/export.h>
+#include <linux/gpio.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/platform_data/b53.h>
+#include <linux/phy.h>
+#include <linux/etherdevice.h>
+#include <linux/if_bridge.h>
+#include <net/dsa.h>
+#include <net/switchdev.h>
+
+#include "b53_regs.h"
+#include "b53_priv.h"
+
+struct b53_mib_desc {
+        u8 size;
+        u8 offset;
+        const char *name;
+};
+
+/* BCM5365 MIB counters */
+static const struct b53_mib_desc b53_mibs_65[] = {
+        { 8, 0x00, "TxOctets" },
+        { 4, 0x08, "TxDropPkts" },
+        { 4, 0x10, "TxBroadcastPkts" },
+        { 4, 0x14, "TxMulticastPkts" },
+        { 4, 0x18, "TxUnicastPkts" },
+        { 4, 0x1c, "TxCollisions" },
+        { 4, 0x20, "TxSingleCollision" },
+        { 4, 0x24, "TxMultipleCollision" },
+        { 4, 0x28, "TxDeferredTransmit" },
+        { 4, 0x2c, "TxLateCollision" },
+        { 4, 0x30, "TxExcessiveCollision" },
+        { 4, 0x38, "TxPausePkts" },
+        { 8, 0x44, "RxOctets" },
+        { 4, 0x4c, "RxUndersizePkts" },
+        { 4, 0x50, "RxPausePkts" },
+        { 4, 0x54, "Pkts64Octets" },
+        { 4, 0x58, "Pkts65to127Octets" },
+        { 4, 0x5c, "Pkts128to255Octets" },
+        { 4, 0x60, "Pkts256to511Octets" },
+        { 4, 0x64, "Pkts512to1023Octets" },
+        { 4, 0x68, "Pkts1024to1522Octets" },
+        { 4, 0x6c, "RxOversizePkts" },
+        { 4, 0x70, "RxJabbers" },
+        { 4, 0x74, "RxAlignmentErrors" },
+        { 4, 0x78, "RxFCSErrors" },
+        { 8, 0x7c, "RxGoodOctets" },
+        { 4, 0x84, "RxDropPkts" },
+        { 4, 0x88, "RxUnicastPkts" },
+        { 4, 0x8c, "RxMulticastPkts" },
+        { 4, 0x90, "RxBroadcastPkts" },
+        { 4, 0x94, "RxSAChanges" },
+        { 4, 0x98, "RxFragments" },
+};
+
+#define B53_MIBS_65_SIZE        ARRAY_SIZE(b53_mibs_65)
+
+/* BCM63xx MIB counters */
+static const struct b53_mib_desc b53_mibs_63xx[] = {
+        { 8, 0x00, "TxOctets" },
+        { 4, 0x08, "TxDropPkts" },
+        { 4, 0x0c, "TxQoSPkts" },
+        { 4, 0x10, "TxBroadcastPkts" },
+        { 4, 0x14, "TxMulticastPkts" },
+        { 4, 0x18, "TxUnicastPkts" },
+        { 4, 0x1c, "TxCollisions" },
+        { 4, 0x20, "TxSingleCollision" },
+        { 4, 0x24, "TxMultipleCollision" },
+        { 4, 0x28, "TxDeferredTransmit" },
+        { 4, 0x2c, "TxLateCollision" },
+        { 4, 0x30, "TxExcessiveCollision" },
+        { 4, 0x38, "TxPausePkts" },
+        { 8, 0x3c, "TxQoSOctets" },
+        { 8, 0x44, "RxOctets" },
+        { 4, 0x4c, "RxUndersizePkts" },
+        { 4, 0x50, "RxPausePkts" },
+        { 4, 0x54, "Pkts64Octets" },
+        { 4, 0x58, "Pkts65to127Octets" },
+        { 4, 0x5c, "Pkts128to255Octets" },
+        { 4, 0x60, "Pkts256to511Octets" },
+        { 4, 0x64, "Pkts512to1023Octets" },
+        { 4, 0x68, "Pkts1024to1522Octets" },
+        { 4, 0x6c, "RxOversizePkts" },
+        { 4, 0x70, "RxJabbers" },
+        { 4, 0x74, "RxAlignmentErrors" },
+        { 4, 0x78, "RxFCSErrors" },
+        { 8, 0x7c, "RxGoodOctets" },
+        { 4, 0x84, "RxDropPkts" },
+        { 4, 0x88, "RxUnicastPkts" },
+        { 4, 0x8c, "RxMulticastPkts" },
+        { 4, 0x90, "RxBroadcastPkts" },
+        { 4, 0x94, "RxSAChanges" },
+        { 4, 0x98, "RxFragments" },
+        { 4, 0xa0, "RxSymbolErrors" },
+        { 4, 0xa4, "RxQoSPkts" },
+        { 8, 0xa8, "RxQoSOctets" },
+        { 4, 0xb0, "Pkts1523to2047Octets" },
+        { 4, 0xb4, "Pkts2048to4095Octets" },
+        { 4, 0xb8, "Pkts4096to8191Octets" },
+        { 4, 0xbc, "Pkts8192to9728Octets" },
+        { 4, 0xc0, "RxDiscarded" },
+};
+
+#define B53_MIBS_63XX_SIZE      ARRAY_SIZE(b53_mibs_63xx)
+
+/* MIB counters */
+static const struct b53_mib_desc b53_mibs[] = {
+        { 8, 0x00, "TxOctets" },
+        { 4, 0x08, "TxDropPkts" },
+        { 4, 0x10, "TxBroadcastPkts" },
+        { 4, 0x14, "TxMulticastPkts" },
+        { 4, 0x18, "TxUnicastPkts" },
+        { 4, 0x1c, "TxCollisions" },
+        { 4, 0x20, "TxSingleCollision" },
+        { 4, 0x24, "TxMultipleCollision" },
+        { 4, 0x28, "TxDeferredTransmit" },
+        { 4, 0x2c, "TxLateCollision" },
+        { 4, 0x30, "TxExcessiveCollision" },
+        { 4, 0x38, "TxPausePkts" },
+        { 8, 0x50, "RxOctets" },
+        { 4, 0x58, "RxUndersizePkts" },
+        { 4, 0x5c, "RxPausePkts" },
+        { 4, 0x60, "Pkts64Octets" },
+        { 4, 0x64, "Pkts65to127Octets" },
+        { 4, 0x68, "Pkts128to255Octets" },
+        { 4, 0x6c, "Pkts256to511Octets" },
+        { 4, 0x70, "Pkts512to1023Octets" },
+        { 4, 0x74, "Pkts1024to1522Octets" },
+        { 4, 0x78, "RxOversizePkts" },
+        { 4, 0x7c, "RxJabbers" },
+        { 4, 0x80, "RxAlignmentErrors" },
+        { 4, 0x84, "RxFCSErrors" },
+        { 8, 0x88, "RxGoodOctets" },
+        { 4, 0x90, "RxDropPkts" },
+        { 4, 0x94, "RxUnicastPkts" },
+        { 4, 0x98, "RxMulticastPkts" },
+        { 4, 0x9c, "RxBroadcastPkts" },
+        { 4, 0xa0, "RxSAChanges" },
+        { 4, 0xa4, "RxFragments" },
+        { 4, 0xa8, "RxJumboPkts" },
+        { 4, 0xac, "RxSymbolErrors" },
+        { 4, 0xc0, "RxDiscarded" },
+};
+
+#define B53_MIBS_SIZE   ARRAY_SIZE(b53_mibs)
+
+static int b53_do_vlan_op(struct b53_device *dev, u8 op)
+{
+        unsigned int i;
+
+        b53_write8(dev, B53_ARLIO_PAGE, dev->vta_regs[0], VTA_START_CMD | op);
+
+        for (i = 0; i < 10; i++) {
+                u8 vta;
+
+                b53_read8(dev, B53_ARLIO_PAGE, dev->vta_regs[0], &vta);
+                if (!(vta & VTA_START_CMD))
+                        return 0;
+
+                usleep_range(100, 200);
+        }
+
+        return -EIO;
+}
+
+static void b53_set_vlan_entry(struct b53_device *dev, u16 vid,
+                               struct b53_vlan *vlan)
+{
+        if (is5325(dev)) {
+                u32 entry = 0;
+
+                if (vlan->members) {
+                        entry = ((vlan->untag & VA_UNTAG_MASK_25) <<
+                                 VA_UNTAG_S_25) | vlan->members;
+                        if (dev->core_rev >= 3)
+                                entry |= VA_VALID_25_R4 | vid << VA_VID_HIGH_S;
+                        else
+                                entry |= VA_VALID_25;
+                }
+
+                b53_write32(dev, B53_VLAN_PAGE, B53_VLAN_WRITE_25, entry);
+                b53_write16(dev, B53_VLAN_PAGE, B53_VLAN_TABLE_ACCESS_25, vid |
+                            VTA_RW_STATE_WR | VTA_RW_OP_EN);
+        } else if (is5365(dev)) {
+                u16 entry = 0;
+
+                if (vlan->members)
+                        entry = ((vlan->untag & VA_UNTAG_MASK_65) <<
+                                 VA_UNTAG_S_65) | vlan->members | VA_VALID_65;
+
+                b53_write16(dev, B53_VLAN_PAGE, B53_VLAN_WRITE_65, entry);
+                b53_write16(dev, B53_VLAN_PAGE, B53_VLAN_TABLE_ACCESS_65, vid |
+                            VTA_RW_STATE_WR | VTA_RW_OP_EN);
+        } else {
+                b53_write16(dev, B53_ARLIO_PAGE, dev->vta_regs[1], vid);
+                b53_write32(dev, B53_ARLIO_PAGE, dev->vta_regs[2],
+                            (vlan->untag << VTE_UNTAG_S) | vlan->members);
+
+                b53_do_vlan_op(dev, VTA_CMD_WRITE);
+        }
+
+        dev_dbg(dev->ds->dev, "VID: %d, members: 0x%04x, untag: 0x%04x\n",
+                vid, vlan->members, vlan->untag);
+}
+
+static void b53_get_vlan_entry(struct b53_device *dev, u16 vid,
+                               struct b53_vlan *vlan)
+{
+        if (is5325(dev)) {
+                u32 entry = 0;
+
+                b53_write16(dev, B53_VLAN_PAGE, B53_VLAN_TABLE_ACCESS_25, vid |
+                            VTA_RW_STATE_RD | VTA_RW_OP_EN);
+                b53_read32(dev, B53_VLAN_PAGE, B53_VLAN_WRITE_25, &entry);
+
+                if (dev->core_rev >= 3)
+                        vlan->valid = !!(entry & VA_VALID_25_R4);
+                else
+                        vlan->valid = !!(entry & VA_VALID_25);
+                vlan->members = entry & VA_MEMBER_MASK;
+                vlan->untag = (entry >> VA_UNTAG_S_25) & VA_UNTAG_MASK_25;
+
+        } else if (is5365(dev)) {
+                u16 entry = 0;
+
+                b53_write16(dev, B53_VLAN_PAGE, B53_VLAN_TABLE_ACCESS_65, vid |
+                            VTA_RW_STATE_WR | VTA_RW_OP_EN);
+                b53_read16(dev, B53_VLAN_PAGE, B53_VLAN_WRITE_65, &entry);
+
+                vlan->valid = !!(entry & VA_VALID_65);
+                vlan->members = entry & VA_MEMBER_MASK;
+                vlan->untag = (entry >> VA_UNTAG_S_65) & VA_UNTAG_MASK_65;
+        } else {
+                u32 entry = 0;
+
+                b53_write16(dev, B53_ARLIO_PAGE, dev->vta_regs[1], vid);
+                b53_do_vlan_op(dev, VTA_CMD_READ);
+                b53_read32(dev, B53_ARLIO_PAGE, dev->vta_regs[2], &entry);
+                vlan->members = entry & VTE_MEMBERS;
+                vlan->untag = (entry >> VTE_UNTAG_S) & VTE_MEMBERS;
+                vlan->valid = true;
+        }
+}
+
+static void b53_set_forwarding(struct b53_device *dev, int enable)
+{
+        u8 mgmt;
+
+        b53_read8(dev, B53_CTRL_PAGE, B53_SWITCH_MODE, &mgmt);
+
+        if (enable)
+                mgmt |= SM_SW_FWD_EN;
+        else
+                mgmt &= ~SM_SW_FWD_EN;
+
+        b53_write8(dev, B53_CTRL_PAGE, B53_SWITCH_MODE, mgmt);
+}
+
+static void b53_enable_vlan(struct b53_device *dev, bool enable)
+{
+        u8 mgmt, vc0, vc1, vc4 = 0, vc5;
+
+        b53_read8(dev, B53_CTRL_PAGE, B53_SWITCH_MODE, &mgmt);
+        b53_read8(dev, B53_VLAN_PAGE, B53_VLAN_CTRL0, &vc0);
+        b53_read8(dev, B53_VLAN_PAGE, B53_VLAN_CTRL1, &vc1);
+
+        if (is5325(dev) || is5365(dev)) {
+                b53_read8(dev, B53_VLAN_PAGE, B53_VLAN_CTRL4_25, &vc4);
+                b53_read8(dev, B53_VLAN_PAGE, B53_VLAN_CTRL5_25, &vc5);
+        } else if (is63xx(dev)) {
+                b53_read8(dev, B53_VLAN_PAGE, B53_VLAN_CTRL4_63XX, &vc4);
+                b53_read8(dev, B53_VLAN_PAGE, B53_VLAN_CTRL5_63XX, &vc5);
+        } else {
+                b53_read8(dev, B53_VLAN_PAGE, B53_VLAN_CTRL4, &vc4);
+                b53_read8(dev, B53_VLAN_PAGE, B53_VLAN_CTRL5, &vc5);
+        }
+
+        mgmt &= ~SM_SW_FWD_MODE;
+
+        if (enable) {
+                vc0 |= VC0_VLAN_EN | VC0_VID_CHK_EN | VC0_VID_HASH_VID;
+                vc1 |= VC1_RX_MCST_UNTAG_EN | VC1_RX_MCST_FWD_EN;
+                vc4 &= ~VC4_ING_VID_CHECK_MASK;
+                vc4 |= VC4_ING_VID_VIO_DROP << VC4_ING_VID_CHECK_S;
+                vc5 |= VC5_DROP_VTABLE_MISS;
+
+                if (is5325(dev))
+                        vc0 &= ~VC0_RESERVED_1;
+
+                if (is5325(dev) || is5365(dev))
+                        vc1 |= VC1_RX_MCST_TAG_EN;
+
+        } else {
+                vc0 &= ~(VC0_VLAN_EN | VC0_VID_CHK_EN | VC0_VID_HASH_VID);
+                vc1 &= ~(VC1_RX_MCST_UNTAG_EN | VC1_RX_MCST_FWD_EN);
+                vc4 &= ~VC4_ING_VID_CHECK_MASK;
+                vc5 &= ~VC5_DROP_VTABLE_MISS;
+
+                if (is5325(dev) || is5365(dev))
+                        vc4 |= VC4_ING_VID_VIO_FWD << VC4_ING_VID_CHECK_S;
+                else
+                        vc4 |= VC4_ING_VID_VIO_TO_IMP << VC4_ING_VID_CHECK_S;
+
+                if (is5325(dev) || is5365(dev))
+                        vc1 &= ~VC1_RX_MCST_TAG_EN;
+        }
+
+        if (!is5325(dev) && !is5365(dev))
+                vc5 &= ~VC5_VID_FFF_EN;
+
+        b53_write8(dev, B53_VLAN_PAGE, B53_VLAN_CTRL0, vc0);
+        b53_write8(dev, B53_VLAN_PAGE, B53_VLAN_CTRL1, vc1);
+
+        if (is5325(dev) || is5365(dev)) {
+                /* enable the high 8 bit vid check on 5325 */
+                if (is5325(dev) && enable)
+                        b53_write8(dev, B53_VLAN_PAGE, B53_VLAN_CTRL3,
+                                   VC3_HIGH_8BIT_EN);
+                else
+                        b53_write8(dev, B53_VLAN_PAGE, B53_VLAN_CTRL3, 0);
+
+                b53_write8(dev, B53_VLAN_PAGE, B53_VLAN_CTRL4_25, vc4);
+                b53_write8(dev, B53_VLAN_PAGE, B53_VLAN_CTRL5_25, vc5);
+        } else if (is63xx(dev)) {
+                b53_write16(dev, B53_VLAN_PAGE, B53_VLAN_CTRL3_63XX, 0);
+                b53_write8(dev, B53_VLAN_PAGE, B53_VLAN_CTRL4_63XX, vc4);
+                b53_write8(dev, B53_VLAN_PAGE, B53_VLAN_CTRL5_63XX, vc5);
+        } else {
+                b53_write16(dev, B53_VLAN_PAGE, B53_VLAN_CTRL3, 0);
+                b53_write8(dev, B53_VLAN_PAGE, B53_VLAN_CTRL4, vc4);
+                b53_write8(dev, B53_VLAN_PAGE, B53_VLAN_CTRL5, vc5);
+        }
+
+        b53_write8(dev, B53_CTRL_PAGE, B53_SWITCH_MODE, mgmt);
+}
+
+static int b53_set_jumbo(struct b53_device *dev, bool enable, bool allow_10_100)
+{
+        u32 port_mask = 0;
+        u16 max_size = JMS_MIN_SIZE;
+
+        if (is5325(dev) || is5365(dev))
+                return -EINVAL;
+
+        if (enable) {
+                port_mask = dev->enabled_ports;
+                max_size = JMS_MAX_SIZE;
+                if (allow_10_100)
+                        port_mask |= JPM_10_100_JUMBO_EN;
+        }
+
+        b53_write32(dev, B53_JUMBO_PAGE, dev->jumbo_pm_reg, port_mask);
+        return b53_write16(dev, B53_JUMBO_PAGE, dev->jumbo_size_reg, max_size);
+}
+
+static int b53_flush_arl(struct b53_device *dev, u8 mask)
+{
+        unsigned int i;
+
+        b53_write8(dev, B53_CTRL_PAGE, B53_FAST_AGE_CTRL,
+                   FAST_AGE_DONE | FAST_AGE_DYNAMIC | mask);
+
+        for (i = 0; i < 10; i++) {
+                u8 fast_age_ctrl;
+
+                b53_read8(dev, B53_CTRL_PAGE, B53_FAST_AGE_CTRL,
+                          &fast_age_ctrl);
+
+                if (!(fast_age_ctrl & FAST_AGE_DONE))
+                        goto out;
+
+                msleep(1);
+        }
+
+        return -ETIMEDOUT;
+out:
+        /* Only age dynamic entries (default behavior) */
+        b53_write8(dev, B53_CTRL_PAGE, B53_FAST_AGE_CTRL, FAST_AGE_DYNAMIC);
+        return 0;
+}
+
+static int b53_fast_age_port(struct b53_device *dev, int port)
+{
+        b53_write8(dev, B53_CTRL_PAGE, B53_FAST_AGE_PORT_CTRL, port);
+
+        return b53_flush_arl(dev, FAST_AGE_PORT);
+}
+
+static int b53_fast_age_vlan(struct b53_device *dev, u16 vid)
+{
+        b53_write16(dev, B53_CTRL_PAGE, B53_FAST_AGE_VID_CTRL, vid);
+
+        return b53_flush_arl(dev, FAST_AGE_VLAN);
+}
+
+static void b53_imp_vlan_setup(struct dsa_switch *ds, int cpu_port)
+{
+        struct b53_device *dev = ds_to_priv(ds);
+        unsigned int i;
+        u16 pvlan;
+
+        /* Enable the IMP port to be in the same VLAN as the other ports
+         * on a per-port basis such that we only have Port i and IMP in
+         * the same VLAN.
+         */
+        b53_for_each_port(dev, i) {
+                b53_read16(dev, B53_PVLAN_PAGE, B53_PVLAN_PORT_MASK(i), &pvlan);
+                pvlan |= BIT(cpu_port);
+                b53_write16(dev, B53_PVLAN_PAGE, B53_PVLAN_PORT_MASK(i), pvlan);
+        }
+}
+
+static int b53_enable_port(struct dsa_switch *ds, int port,
+                           struct phy_device *phy)
+{
+        struct b53_device *dev = ds_to_priv(ds);
+        unsigned int cpu_port = dev->cpu_port;
+        u16 pvlan;
+
+        /* Clear the Rx and Tx disable bits and set to no spanning tree */
+        b53_write8(dev, B53_CTRL_PAGE, B53_PORT_CTRL(port), 0);
+
+        /* Set this port, and only this one to be in the default VLAN,
+         * if member of a bridge, restore its membership prior to
+         * bringing down this port.
+         */
+        b53_read16(dev, B53_PVLAN_PAGE, B53_PVLAN_PORT_MASK(port), &pvlan);
+        pvlan &= ~0x1ff;
+        pvlan |= BIT(port);
+        pvlan |= dev->ports[port].vlan_ctl_mask;
+        b53_write16(dev, B53_PVLAN_PAGE, B53_PVLAN_PORT_MASK(port), pvlan);
+
+        b53_imp_vlan_setup(ds, cpu_port);
+
+        return 0;
+}
+
+static void b53_disable_port(struct dsa_switch *ds, int port,
+                             struct phy_device *phy)
+{
+        struct b53_device *dev = ds_to_priv(ds);
+        u8 reg;
+
+        /* Disable Tx/Rx for the port */
+        b53_read8(dev, B53_CTRL_PAGE, B53_PORT_CTRL(port), &reg);
+        reg |= PORT_CTRL_RX_DISABLE | PORT_CTRL_TX_DISABLE;
+        b53_write8(dev, B53_CTRL_PAGE, B53_PORT_CTRL(port), reg);
+}
+
+static void b53_enable_cpu_port(struct b53_device *dev)
+{
+        unsigned int cpu_port = dev->cpu_port;
+        u8 port_ctrl;
+
+        /* BCM5325 CPU port is at 8 */
+        if ((is5325(dev) || is5365(dev)) && cpu_port == B53_CPU_PORT_25)
+                cpu_port = B53_CPU_PORT;
+
+        port_ctrl = PORT_CTRL_RX_BCST_EN |
+                    PORT_CTRL_RX_MCST_EN |
+                    PORT_CTRL_RX_UCST_EN;
+        b53_write8(dev, B53_CTRL_PAGE, B53_PORT_CTRL(cpu_port), port_ctrl);
+}
+
+static void b53_enable_mib(struct b53_device *dev)
+{
+        u8 gc;
+
+        b53_read8(dev, B53_MGMT_PAGE, B53_GLOBAL_CONFIG, &gc);
+        gc &= ~(GC_RESET_MIB | GC_MIB_AC_EN);
+        b53_write8(dev, B53_MGMT_PAGE, B53_GLOBAL_CONFIG, gc);
+}
+
+static int b53_configure_vlan(struct b53_device *dev)
+{
+        struct b53_vlan vl = { 0 };
+        int i;
+
+        /* clear all vlan entries */
+        if (is5325(dev) || is5365(dev)) {
+                for (i = 1; i < dev->num_vlans; i++)
+                        b53_set_vlan_entry(dev, i, &vl);
+        } else {
+                b53_do_vlan_op(dev, VTA_CMD_CLEAR);
+        }
+
+        b53_enable_vlan(dev, false);
+
+        b53_for_each_port(dev, i)
+                b53_write16(dev, B53_VLAN_PAGE,
+                            B53_VLAN_PORT_DEF_TAG(i), 1);
+
+        if (!is5325(dev) && !is5365(dev))
+                b53_set_jumbo(dev, dev->enable_jumbo, false);
+
+        return 0;
+}
+
+static void b53_switch_reset_gpio(struct b53_device *dev)
+{
+        int gpio = dev->reset_gpio;
+
+        if (gpio < 0)
+                return;
+
+        /* Reset sequence: RESET low(50ms)->high(20ms)
+         */
+        gpio_set_value(gpio, 0);
+        mdelay(50);
+
+        gpio_set_value(gpio, 1);
+        mdelay(20);
+
+        dev->current_page = 0xff;
+}
+
+static int b53_switch_reset(struct b53_device *dev)
+{
+        u8 mgmt;
+
+        b53_switch_reset_gpio(dev);
+
+        if (is539x(dev)) {
+                b53_write8(dev, B53_CTRL_PAGE, B53_SOFTRESET, 0x83);
+                b53_write8(dev, B53_CTRL_PAGE, B53_SOFTRESET, 0x00);
+        }
+
+        b53_read8(dev, B53_CTRL_PAGE, B53_SWITCH_MODE, &mgmt);
+
+        if (!(mgmt & SM_SW_FWD_EN)) {
+                mgmt &= ~SM_SW_FWD_MODE;
+                mgmt |= SM_SW_FWD_EN;
+
+                b53_write8(dev, B53_CTRL_PAGE, B53_SWITCH_MODE, mgmt);
+                b53_read8(dev, B53_CTRL_PAGE, B53_SWITCH_MODE, &mgmt);
+
+                if (!(mgmt & SM_SW_FWD_EN)) {
+                        dev_err(dev->dev, "Failed to enable switch!\n");
+                        return -EINVAL;
+                }
+        }
+
+        b53_enable_mib(dev);
+
+        return b53_flush_arl(dev, FAST_AGE_STATIC);
+}
+
+static int b53_phy_read16(struct dsa_switch *ds, int addr, int reg)
+{
+        struct b53_device *priv = ds_to_priv(ds);
+        u16 value = 0;
+        int ret;
+
+        if (priv->ops->phy_read16)
+                ret = priv->ops->phy_read16(priv, addr, reg, &value);
+        else
+                ret = b53_read16(priv, B53_PORT_MII_PAGE(addr),
+                                 reg * 2, &value);
+
+        return ret ? ret : value;
+}
+
+static int b53_phy_write16(struct dsa_switch *ds, int addr, int reg, u16 val)
+{
+        struct b53_device *priv = ds_to_priv(ds);
+
+        if (priv->ops->phy_write16)
+                return priv->ops->phy_write16(priv, addr, reg, val);
+
+        return b53_write16(priv, B53_PORT_MII_PAGE(addr), reg * 2, val);
+}
+
+static int b53_reset_switch(struct b53_device *priv)
+{
+        /* reset vlans */
+        priv->enable_jumbo = false;
+
+        memset(priv->vlans, 0, sizeof(*priv->vlans) * priv->num_vlans);
+        memset(priv->ports, 0, sizeof(*priv->ports) * priv->num_ports);
+
+        return b53_switch_reset(priv);
+}
+
+static int b53_apply_config(struct b53_device *priv)
+{
+        /* disable switching */
+        b53_set_forwarding(priv, 0);
+
+        b53_configure_vlan(priv);
+
+        /* enable switching */
+        b53_set_forwarding(priv, 1);
+
+        return 0;
+}
+
+static void b53_reset_mib(struct b53_device *priv)
+{
+        u8 gc;
+
+        b53_read8(priv, B53_MGMT_PAGE, B53_GLOBAL_CONFIG, &gc);
+
+        b53_write8(priv, B53_MGMT_PAGE, B53_GLOBAL_CONFIG, gc | GC_RESET_MIB);
+        msleep(1);
+        b53_write8(priv, B53_MGMT_PAGE, B53_GLOBAL_CONFIG, gc & ~GC_RESET_MIB);
+        msleep(1);
+}
+
+static const struct b53_mib_desc *b53_get_mib(struct b53_device *dev)
+{
+        if (is5365(dev))
+                return b53_mibs_65;
+        else if (is63xx(dev))
+                return b53_mibs_63xx;
+        else
+                return b53_mibs;
+}
+
+static unsigned int b53_get_mib_size(struct b53_device *dev)
+{
+        if (is5365(dev))
+                return B53_MIBS_65_SIZE;
+        else if (is63xx(dev))
+                return B53_MIBS_63XX_SIZE;
+        else
+                return B53_MIBS_SIZE;
+}
+
+static void b53_get_strings(struct dsa_switch *ds, int port, uint8_t *data)
+{
+        struct b53_device *dev = ds_to_priv(ds);
+        const struct b53_mib_desc *mibs = b53_get_mib(dev);
+        unsigned int mib_size = b53_get_mib_size(dev);
+        unsigned int i;
+
+        for (i = 0; i < mib_size; i++)
+                memcpy(data + i * ETH_GSTRING_LEN,
+                       mibs[i].name, ETH_GSTRING_LEN);
+}
+
+static void b53_get_ethtool_stats(struct dsa_switch *ds, int port,
+                                  uint64_t *data)
+{
+        struct b53_device *dev = ds_to_priv(ds);
+        const struct b53_mib_desc *mibs = b53_get_mib(dev);
+        unsigned int mib_size = b53_get_mib_size(dev);
+        const struct b53_mib_desc *s;
+        unsigned int i;
+        u64 val = 0;
+
+        if (is5365(dev) && port == 5)
+                port = 8;
+
+        mutex_lock(&dev->stats_mutex);
+
+        for (i = 0; i < mib_size; i++) {
+                s = &mibs[i];
+
+                if (s->size == 8) {
+                        b53_read64(dev, B53_MIB_PAGE(port), s->offset, &val);
+                } else {
+                        u32 val32;
+
+                        b53_read32(dev, B53_MIB_PAGE(port), s->offset,
+                                   &val32);
+                        val = val32;
+                }
+                data[i] = (u64)val;
+        }
+
+        mutex_unlock(&dev->stats_mutex);
+}
+
+static int b53_get_sset_count(struct dsa_switch *ds)
+{
+        struct b53_device *dev = ds_to_priv(ds);
+
+        return b53_get_mib_size(dev);
+}
+
+static int b53_set_addr(struct dsa_switch *ds, u8 *addr)
+{
+        return 0;
+}
+
+static int b53_setup(struct dsa_switch *ds)
+{
+        struct b53_device *dev = ds_to_priv(ds);
+        unsigned int port;
+        int ret;
+
+        ret = b53_reset_switch(dev);
+        if (ret) {
+                dev_err(ds->dev, "failed to reset switch\n");
+                return ret;
+        }
+
+        b53_reset_mib(dev);
+
+        ret = b53_apply_config(dev);
+        if (ret)
+                dev_err(ds->dev, "failed to apply configuration\n");
+
+        for (port = 0; port < dev->num_ports; port++) {
+                if (BIT(port) & ds->enabled_port_mask)
+                        b53_enable_port(ds, port, NULL);
+                else if (dsa_is_cpu_port(ds, port))
+                        b53_enable_cpu_port(dev);
+                else
+                        b53_disable_port(ds, port, NULL);
+        }
+
+        return ret;
+}
+
+static void b53_adjust_link(struct dsa_switch *ds, int port,
+                            struct phy_device *phydev)
+{
+        struct b53_device *dev = ds_to_priv(ds);
+        u8 rgmii_ctrl = 0, reg = 0, off;
+
+        if (!phy_is_pseudo_fixed_link(phydev))
+                return;
+
+        /* Override the port settings */
+        if (port == dev->cpu_port) {
+                off = B53_PORT_OVERRIDE_CTRL;
+                reg = PORT_OVERRIDE_EN;
+        } else {
+                off = B53_GMII_PORT_OVERRIDE_CTRL(port);
+                reg = GMII_PO_EN;
+        }
+
+        /* Set the link UP */
+        if (phydev->link)
+                reg |= PORT_OVERRIDE_LINK;
+
+        if (phydev->duplex == DUPLEX_FULL)
+                reg |= PORT_OVERRIDE_FULL_DUPLEX;
+
+        switch (phydev->speed) {
+        case 2000:
+                reg |= PORT_OVERRIDE_SPEED_2000M;
+                /* fallthrough */
+        case SPEED_1000:
+                reg |= PORT_OVERRIDE_SPEED_1000M;
+                break;
+        case SPEED_100:
+                reg |= PORT_OVERRIDE_SPEED_100M;
+                break;
+        case SPEED_10:
+                reg |= PORT_OVERRIDE_SPEED_10M;
+                break;
+        default:
+                dev_err(ds->dev, "unknown speed: %d\n", phydev->speed);
+                return;
+        }
+
+        /* Enable flow control on BCM5301x's CPU port */
+        if (is5301x(dev) && port == dev->cpu_port)
+                reg |= PORT_OVERRIDE_RX_FLOW | PORT_OVERRIDE_TX_FLOW;
+
+        if (phydev->pause) {
+                if (phydev->asym_pause)
+                        reg |= PORT_OVERRIDE_TX_FLOW;
+                reg |= PORT_OVERRIDE_RX_FLOW;
+        }
+
+        b53_write8(dev, B53_CTRL_PAGE, off, reg);
+
+        if (is531x5(dev) && phy_interface_is_rgmii(phydev)) {
+                if (port == 8)
+                        off = B53_RGMII_CTRL_IMP;
+                else
+                        off = B53_RGMII_CTRL_P(port);
+
+                /* Configure the port RGMII clock delay by DLL disabled and
+                 * tx_clk aligned timing (restoring to reset defaults)
+                 */
+                b53_read8(dev, B53_CTRL_PAGE, off, &rgmii_ctrl);
+                rgmii_ctrl &= ~(RGMII_CTRL_DLL_RXC | RGMII_CTRL_DLL_TXC |
+                                RGMII_CTRL_TIMING_SEL);
+
+                /* PHY_INTERFACE_MODE_RGMII_TXID means TX internal delay, make
+                 * sure that we enable the port TX clock internal delay to
+                 * account for this internal delay that is inserted, otherwise
+                 * the switch won't be able to receive correctly.
+                 *
+                 * PHY_INTERFACE_MODE_RGMII means that we are not introducing
+                 * any delay neither on transmission nor reception, so the
+                 * BCM53125 must also be configured accordingly to account for
+                 * the lack of delay and introduce
+                 *
+                 * The BCM53125 switch has its RX clock and TX clock control
+                 * swapped, hence the reason why we modify the TX clock path in
+                 * the "RGMII" case
+                 */
+                if (phydev->interface == PHY_INTERFACE_MODE_RGMII_TXID)
+                        rgmii_ctrl |= RGMII_CTRL_DLL_TXC;
+                if (phydev->interface == PHY_INTERFACE_MODE_RGMII)
+                        rgmii_ctrl |= RGMII_CTRL_DLL_TXC | RGMII_CTRL_DLL_RXC;
+                rgmii_ctrl |= RGMII_CTRL_TIMING_SEL;
+                b53_write8(dev, B53_CTRL_PAGE, off, rgmii_ctrl);
+
+                dev_info(ds->dev, "Configured port %d for %s\n", port,
+                         phy_modes(phydev->interface));
+        }
+
+        /* configure MII port if necessary */
+        if (is5325(dev)) {
+                b53_read8(dev, B53_CTRL_PAGE, B53_PORT_OVERRIDE_CTRL,
+                          &reg);
+
+                /* reverse mii needs to be enabled */
+                if (!(reg & PORT_OVERRIDE_RV_MII_25)) {
+                        b53_write8(dev, B53_CTRL_PAGE, B53_PORT_OVERRIDE_CTRL,
+                                   reg | PORT_OVERRIDE_RV_MII_25);
+                        b53_read8(dev, B53_CTRL_PAGE, B53_PORT_OVERRIDE_CTRL,
+                                  &reg);
+
+                        if (!(reg & PORT_OVERRIDE_RV_MII_25)) {
+                                dev_err(ds->dev,
+                                        "Failed to enable reverse MII mode\n");
+                                return;
+                        }
+                }
+        } else if (is5301x(dev)) {
+                if (port != dev->cpu_port) {
+                        u8 po_reg = B53_GMII_PORT_OVERRIDE_CTRL(dev->cpu_port);
+                        u8 gmii_po;
+
+                        b53_read8(dev, B53_CTRL_PAGE, po_reg, &gmii_po);
+                        gmii_po |= GMII_PO_LINK |
+                                   GMII_PO_RX_FLOW |
+                                   GMII_PO_TX_FLOW |
+                                   GMII_PO_EN |
+                                   GMII_PO_SPEED_2000M;
+                        b53_write8(dev, B53_CTRL_PAGE, po_reg, gmii_po);
+                }
+        }
+}
+
+static int b53_vlan_filtering(struct dsa_switch *ds, int port,
+                              bool vlan_filtering)
+{
+        return 0;
+}
+
+static int b53_vlan_prepare(struct dsa_switch *ds, int port,
+                            const struct switchdev_obj_port_vlan *vlan,
+                            struct switchdev_trans *trans)
+{
+        struct b53_device *dev = ds_to_priv(ds);
+
+        if ((is5325(dev) || is5365(dev)) && vlan->vid_begin == 0)
+                return -EOPNOTSUPP;
+
+        if (vlan->vid_end > dev->num_vlans)
+                return -ERANGE;
+
+        b53_enable_vlan(dev, true);
+
+        return 0;
+}
+
+static void b53_vlan_add(struct dsa_switch *ds, int port,
+                         const struct switchdev_obj_port_vlan *vlan,
+                         struct switchdev_trans *trans)
+{
+        struct b53_device *dev = ds_to_priv(ds);
+        bool untagged = vlan->flags & BRIDGE_VLAN_INFO_UNTAGGED;
+        bool pvid = vlan->flags & BRIDGE_VLAN_INFO_PVID;
+        unsigned int cpu_port = dev->cpu_port;
+        struct b53_vlan *vl;
+        u16 vid;
+
+        for (vid = vlan->vid_begin; vid <= vlan->vid_end; ++vid) {
+                vl = &dev->vlans[vid];
+
+                b53_get_vlan_entry(dev, vid, vl);
+
+                vl->members |= BIT(port) | BIT(cpu_port);
+                if (untagged)
+                        vl->untag |= BIT(port) | BIT(cpu_port);
+                else
+                        vl->untag &= ~(BIT(port) | BIT(cpu_port));
+
+                b53_set_vlan_entry(dev, vid, vl);
+                b53_fast_age_vlan(dev, vid);
+        }
+
+        if (pvid) {
+                b53_write16(dev, B53_VLAN_PAGE, B53_VLAN_PORT_DEF_TAG(port),
+                            vlan->vid_end);
+                b53_write16(dev, B53_VLAN_PAGE, B53_VLAN_PORT_DEF_TAG(cpu_port),
+                            vlan->vid_end);
+                b53_fast_age_vlan(dev, vid);
+        }
+}
+
+static int b53_vlan_del(struct dsa_switch *ds, int port,
+                        const struct switchdev_obj_port_vlan *vlan)
+{
+        struct b53_device *dev = ds_to_priv(ds);
+        bool untagged = vlan->flags & BRIDGE_VLAN_INFO_UNTAGGED;
+        unsigned int cpu_port = dev->cpu_port;
+        struct b53_vlan *vl;
+        u16 vid;
+        u16 pvid;
+
+        b53_read16(dev, B53_VLAN_PAGE, B53_VLAN_PORT_DEF_TAG(port), &pvid);
+
+        for (vid = vlan->vid_begin; vid <= vlan->vid_end; ++vid) {
+                vl = &dev->vlans[vid];
+
+                b53_get_vlan_entry(dev, vid, vl);
+
+                vl->members &= ~BIT(port);
+                if ((vl->members & BIT(cpu_port)) == BIT(cpu_port))
+                        vl->members = 0;
+
+                if (pvid == vid) {
+                        if (is5325(dev) || is5365(dev))
+                                pvid = 1;
+                        else
+                                pvid = 0;
+                }
+
+                if (untagged) {
+                        vl->untag &= ~(BIT(port));
+                        if ((vl->untag & BIT(cpu_port)) == BIT(cpu_port))
+                                vl->untag = 0;
+                }
+
+                b53_set_vlan_entry(dev, vid, vl);
+                b53_fast_age_vlan(dev, vid);
+        }
+
+        b53_write16(dev, B53_VLAN_PAGE, B53_VLAN_PORT_DEF_TAG(port), pvid);
+        b53_write16(dev, B53_VLAN_PAGE, B53_VLAN_PORT_DEF_TAG(cpu_port), pvid);
+        b53_fast_age_vlan(dev, pvid);
+
+        return 0;
+}
+
+static int b53_vlan_dump(struct dsa_switch *ds, int port,
+                         struct switchdev_obj_port_vlan *vlan,
+                         int (*cb)(struct switchdev_obj *obj))
+{
+        struct b53_device *dev = ds_to_priv(ds);
+        u16 vid, vid_start = 0, pvid;
+        struct b53_vlan *vl;
+        int err = 0;
+
+        if (is5325(dev) || is5365(dev))
+                vid_start = 1;
+
+        b53_read16(dev, B53_VLAN_PAGE, B53_VLAN_PORT_DEF_TAG(port), &pvid);
+
+        /* Use our software cache for dumps, since we do not have any HW
+         * operation returning only the used/valid VLANs
+         */
+        for (vid = vid_start; vid < dev->num_vlans; vid++) {
+                vl = &dev->vlans[vid];
+
+                if (!vl->valid)
+                        continue;
+
+                if (!(vl->members & BIT(port)))
+                        continue;
+
+                vlan->vid_begin = vlan->vid_end = vid;
+                vlan->flags = 0;
+
+                if (vl->untag & BIT(port))
+                        vlan->flags |= BRIDGE_VLAN_INFO_UNTAGGED;
+                if (pvid == vid)
+                        vlan->flags |= BRIDGE_VLAN_INFO_PVID;
+
+                err = cb(&vlan->obj);
+                if (err)
+                        break;
+        }
+
+        return err;
+}
+
+/* Address Resolution Logic routines */
+static int b53_arl_op_wait(struct b53_device *dev)
+{
+        unsigned int timeout = 10;
+        u8 reg;
+
+        do {
+                b53_read8(dev, B53_ARLIO_PAGE, B53_ARLTBL_RW_CTRL, &reg);
+                if (!(reg & ARLTBL_START_DONE))
+                        return 0;
+
+                usleep_range(1000, 2000);
+        } while (timeout--);
+
+        dev_warn(dev->dev, "timeout waiting for ARL to finish: 0x%02x\n", reg);
+
+        return -ETIMEDOUT;
+}
+
+static int b53_arl_rw_op(struct b53_device *dev, unsigned int op)
+{
+        u8 reg;
+
+        if (op > ARLTBL_RW)
+                return -EINVAL;
+
+        b53_read8(dev, B53_ARLIO_PAGE, B53_ARLTBL_RW_CTRL, &reg);
+        reg |= ARLTBL_START_DONE;
+        if (op)
+                reg |= ARLTBL_RW;
+        else
+                reg &= ~ARLTBL_RW;
+        b53_write8(dev, B53_ARLIO_PAGE, B53_ARLTBL_RW_CTRL, reg);
+
+        return b53_arl_op_wait(dev);
+}
+
+static int b53_arl_read(struct b53_device *dev, u64 mac,
+                        u16 vid, struct b53_arl_entry *ent, u8 *idx,
+                        bool is_valid)
+{
+        unsigned int i;
+        int ret;
+
+        ret = b53_arl_op_wait(dev);
+        if (ret)
+                return ret;
+
+        /* Read the bins */
+        for (i = 0; i < dev->num_arl_entries; i++) {
+                u64 mac_vid;
+                u32 fwd_entry;
+
+                b53_read64(dev, B53_ARLIO_PAGE,
+                           B53_ARLTBL_MAC_VID_ENTRY(i), &mac_vid);
+                b53_read32(dev, B53_ARLIO_PAGE,
+                           B53_ARLTBL_DATA_ENTRY(i), &fwd_entry);
+                b53_arl_to_entry(ent, mac_vid, fwd_entry);
+
+                if (!(fwd_entry & ARLTBL_VALID))
+                        continue;
+                if ((mac_vid & ARLTBL_MAC_MASK) != mac)
+                        continue;
+                *idx = i;
+        }
+
+        return -ENOENT;
+}
+
+static int b53_arl_op(struct b53_device *dev, int op, int port,
+                      const unsigned char *addr, u16 vid, bool is_valid)
+{
+        struct b53_arl_entry ent;
+        u32 fwd_entry;
+        u64 mac, mac_vid = 0;
+        u8 idx = 0;
+        int ret;
+
+        /* Convert the array into a 64-bit MAC */
+        mac = b53_mac_to_u64(addr);
+
+        /* Perform a read for the given MAC and VID */
+        b53_write48(dev, B53_ARLIO_PAGE, B53_MAC_ADDR_IDX, mac);
+        b53_write16(dev, B53_ARLIO_PAGE, B53_VLAN_ID_IDX, vid);
+
+        /* Issue a read operation for this MAC */
+        ret = b53_arl_rw_op(dev, 1);
+        if (ret)
+                return ret;
+
+        ret = b53_arl_read(dev, mac, vid, &ent, &idx, is_valid);
+        /* If this is a read, just finish now */
+        if (op)
+                return ret;
+
+        /* We could not find a matching MAC, so reset to a new entry */
+        if (ret) {
+                fwd_entry = 0;
+                idx = 1;
+        }
+
+        memset(&ent, 0, sizeof(ent));
+        ent.port = port;
+        ent.is_valid = is_valid;
+        ent.vid = vid;
+        ent.is_static = true;
+        memcpy(ent.mac, addr, ETH_ALEN);
+        b53_arl_from_entry(&mac_vid, &fwd_entry, &ent);
+
+        b53_write64(dev, B53_ARLIO_PAGE,
+                    B53_ARLTBL_MAC_VID_ENTRY(idx), mac_vid);
+        b53_write32(dev, B53_ARLIO_PAGE,
+                    B53_ARLTBL_DATA_ENTRY(idx), fwd_entry);
+
+        return b53_arl_rw_op(dev, 0);
+}
+
+static int b53_fdb_prepare(struct dsa_switch *ds, int port,
+                           const struct switchdev_obj_port_fdb *fdb,
+                           struct switchdev_trans *trans)
+{
+        struct b53_device *priv = ds_to_priv(ds);
+
+        /* 5325 and 5365 require some more massaging, but could
+         * be supported eventually
+         */
+        if (is5325(priv) || is5365(priv))
+                return -EOPNOTSUPP;
+
+        return 0;
+}
+
+static void b53_fdb_add(struct dsa_switch *ds, int port,
+                        const struct switchdev_obj_port_fdb *fdb,
+                        struct switchdev_trans *trans)
+{
+        struct b53_device *priv = ds_to_priv(ds);
+
+        if (b53_arl_op(priv, 0, port, fdb->addr, fdb->vid, true))
+                pr_err("%s: failed to add MAC address\n", __func__);
+}
+
+static int b53_fdb_del(struct dsa_switch *ds, int port,
+                       const struct switchdev_obj_port_fdb *fdb)
+{
+        struct b53_device *priv = ds_to_priv(ds);
+
+        return b53_arl_op(priv, 0, port, fdb->addr, fdb->vid, false);
+}
+
+static int b53_arl_search_wait(struct b53_device *dev)
+{
+        unsigned int timeout = 1000;
+        u8 reg;
+
+        do {
+                b53_read8(dev, B53_ARLIO_PAGE, B53_ARL_SRCH_CTL, &reg);
+                if (!(reg & ARL_SRCH_STDN))
+                        return 0;
+
+                if (reg & ARL_SRCH_VLID)
+                        return 0;
+
+                usleep_range(1000, 2000);
+        } while (timeout--);
+
+        return -ETIMEDOUT;
+}
+
+static void b53_arl_search_rd(struct b53_device *dev, u8 idx,
+                              struct b53_arl_entry *ent)
+{
+        u64 mac_vid;
+        u32 fwd_entry;
+
+        b53_read64(dev, B53_ARLIO_PAGE,
+                   B53_ARL_SRCH_RSTL_MACVID(idx), &mac_vid);
+        b53_read32(dev, B53_ARLIO_PAGE,
+                   B53_ARL_SRCH_RSTL(idx), &fwd_entry);
+        b53_arl_to_entry(ent, mac_vid, fwd_entry);
+}
+
+static int b53_fdb_copy(struct net_device *dev, int port,
+                        const struct b53_arl_entry *ent,
+                        struct switchdev_obj_port_fdb *fdb,
+                        int (*cb)(struct switchdev_obj *obj))
+{
+        if (!ent->is_valid)
+                return 0;
+
+        if (port != ent->port)
+                return 0;
+
+        ether_addr_copy(fdb->addr, ent->mac);
+        fdb->vid = ent->vid;
+        fdb->ndm_state = ent->is_static ? NUD_NOARP : NUD_REACHABLE;
+
+        return cb(&fdb->obj);
+}
+
+static int b53_fdb_dump(struct dsa_switch *ds, int port,
+                        struct switchdev_obj_port_fdb *fdb,
+                        int (*cb)(struct switchdev_obj *obj))
+{
+        struct b53_device *priv = ds_to_priv(ds);
+        struct net_device *dev = ds->ports[port].netdev;
+        struct b53_arl_entry results[2];
+        unsigned int count = 0;
+        int ret;
+        u8 reg;
+
+        /* Start search operation */
+        reg = ARL_SRCH_STDN;
+        b53_write8(priv, B53_ARLIO_PAGE, B53_ARL_SRCH_CTL, reg);
+
+        do {
+                ret = b53_arl_search_wait(priv);
+                if (ret)
+                        return ret;
+
+                b53_arl_search_rd(priv, 0, &results[0]);
+                ret = b53_fdb_copy(dev, port, &results[0], fdb, cb);
+                if (ret)
+                        return ret;
+
+                if (priv->num_arl_entries > 2) {
+                        b53_arl_search_rd(priv, 1, &results[1]);
+                        ret = b53_fdb_copy(dev, port, &results[1], fdb, cb);
+                        if (ret)
+                                return ret;
+
+                        if (!results[0].is_valid && !results[1].is_valid)
+                                break;
+                }
+
+        } while (count++ < 1024);
+
+        return 0;
+}
+
+static int b53_br_join(struct dsa_switch *ds, int port,
+                       struct net_device *bridge)
+{
+        struct b53_device *dev = ds_to_priv(ds);
+        u16 pvlan, reg;
+        unsigned int i;
+
+        dev->ports[port].bridge_dev = bridge;
+        b53_read16(dev, B53_PVLAN_PAGE, B53_PVLAN_PORT_MASK(port), &pvlan);
+
+        b53_for_each_port(dev, i) {
+                if (dev->ports[i].bridge_dev != bridge)
+                        continue;
+
+                /* Add this local port to the remote port VLAN control
+                 * membership and update the remote port bitmask
+                 */
+                b53_read16(dev, B53_PVLAN_PAGE, B53_PVLAN_PORT_MASK(i), &reg);
+                reg |= BIT(port);
+                b53_write16(dev, B53_PVLAN_PAGE, B53_PVLAN_PORT_MASK(i), reg);
+                dev->ports[i].vlan_ctl_mask = reg;
+
+                pvlan |= BIT(i);
+        }
+
+        /* Configure the local port VLAN control membership to include
+         * remote ports and update the local port bitmask
+         */
+        b53_write16(dev, B53_PVLAN_PAGE, B53_PVLAN_PORT_MASK(port), pvlan);
+        dev->ports[port].vlan_ctl_mask = pvlan;
+
+        return 0;
+}
+
+static void b53_br_leave(struct dsa_switch *ds, int port)
+{
+        struct b53_device *dev = ds_to_priv(ds);
+        struct net_device *bridge = dev->ports[port].bridge_dev;
+        struct b53_vlan *vl = &dev->vlans[0];
+        unsigned int i;
+        u16 pvlan, reg, pvid;
+
+        b53_read16(dev, B53_PVLAN_PAGE, B53_PVLAN_PORT_MASK(port), &pvlan);
+
+        b53_for_each_port(dev, i) {
+                /* Don't touch the remaining ports */
+                if (dev->ports[i].bridge_dev != bridge)
+                        continue;
+
+                b53_read16(dev, B53_PVLAN_PAGE, B53_PVLAN_PORT_MASK(i), &reg);
+                reg &= ~BIT(port);
+                b53_write16(dev, B53_PVLAN_PAGE, B53_PVLAN_PORT_MASK(i), reg);
+                dev->ports[port].vlan_ctl_mask = reg;
+
+                /* Prevent self removal to preserve isolation */
+                if (port != i)
+                        pvlan &= ~BIT(i);
+        }
+
+        b53_write16(dev, B53_PVLAN_PAGE, B53_PVLAN_PORT_MASK(port), pvlan);
+        dev->ports[port].vlan_ctl_mask = pvlan;
+        dev->ports[port].bridge_dev = NULL;
+
+        if (is5325(dev) || is5365(dev))
+                pvid = 1;
+        else
+                pvid = 0;
+
+        b53_get_vlan_entry(dev, pvid, vl);
+        vl->members |= BIT(port) | BIT(dev->cpu_port);
+        vl->untag |= BIT(port) | BIT(dev->cpu_port);
+        b53_set_vlan_entry(dev, pvid, vl);
+}
+
+static void b53_br_set_stp_state(struct dsa_switch *ds, int port,
+                                 u8 state)
+{
+        struct b53_device *dev = ds_to_priv(ds);
+        u8 hw_state, cur_hw_state;
+        u8 reg;
+
+        b53_read8(dev, B53_CTRL_PAGE, B53_PORT_CTRL(port), &reg);
+        cur_hw_state = reg & PORT_CTRL_STP_STATE_MASK;
+
+        switch (state) {
+        case BR_STATE_DISABLED:
+                hw_state = PORT_CTRL_DIS_STATE;
+                break;
+        case BR_STATE_LISTENING:
+                hw_state = PORT_CTRL_LISTEN_STATE;
+                break;
+        case BR_STATE_LEARNING:
+                hw_state = PORT_CTRL_LEARN_STATE;
+                break;
+        case BR_STATE_FORWARDING:
+                hw_state = PORT_CTRL_FWD_STATE;
+                break;
+        case BR_STATE_BLOCKING:
+                hw_state = PORT_CTRL_BLOCK_STATE;
+                break;
+        default:
+                dev_err(ds->dev, "invalid STP state: %d\n", state);
+                return;
+        }
+
+        /* Fast-age ARL entries if we are moving a port from Learning or
+         * Forwarding (cur_hw_state) state to Disabled, Blocking or Listening
+         * state (hw_state)
+         */
+        if (cur_hw_state != hw_state) {
+                if (cur_hw_state >= PORT_CTRL_LEARN_STATE &&
+                    hw_state <= PORT_CTRL_LISTEN_STATE) {
+                        if (b53_fast_age_port(dev, port)) {
+                                dev_err(ds->dev, "fast ageing failed\n");
+                                return;
+                        }
+                }
+        }
+
+        b53_read8(dev, B53_CTRL_PAGE, B53_PORT_CTRL(port), &reg);
+        reg &= ~PORT_CTRL_STP_STATE_MASK;
+        reg |= hw_state;
+        b53_write8(dev, B53_CTRL_PAGE, B53_PORT_CTRL(port), reg);
+}
+
+static struct dsa_switch_driver b53_switch_ops = {
+        .tag_protocol           = DSA_TAG_PROTO_NONE,
+        .setup                  = b53_setup,
+        .set_addr               = b53_set_addr,
+        .get_strings            = b53_get_strings,
+        .get_ethtool_stats      = b53_get_ethtool_stats,
+        .get_sset_count         = b53_get_sset_count,
+        .phy_read               = b53_phy_read16,
+        .phy_write              = b53_phy_write16,
+        .adjust_link            = b53_adjust_link,
+        .port_enable            = b53_enable_port,
+        .port_disable           = b53_disable_port,
+        .port_bridge_join       = b53_br_join,
+        .port_bridge_leave      = b53_br_leave,
+        .port_stp_state_set     = b53_br_set_stp_state,
+        .port_vlan_filtering    = b53_vlan_filtering,
+        .port_vlan_prepare      = b53_vlan_prepare,
+        .port_vlan_add          = b53_vlan_add,
+        .port_vlan_del          = b53_vlan_del,
+        .port_vlan_dump         = b53_vlan_dump,
+        .port_fdb_prepare       = b53_fdb_prepare,
+        .port_fdb_dump          = b53_fdb_dump,
+        .port_fdb_add           = b53_fdb_add,
+        .port_fdb_del           = b53_fdb_del,
+};
+
+struct b53_chip_data {
+        u32 chip_id;
+        const char *dev_name;
+        u16 vlans;
+        u16 enabled_ports;
+        u8 cpu_port;
+        u8 vta_regs[3];
+        u8 arl_entries;
+        u8 duplex_reg;
+        u8 jumbo_pm_reg;
+        u8 jumbo_size_reg;
+};
+
+#define B53_VTA_REGS    \
+        { B53_VT_ACCESS, B53_VT_INDEX, B53_VT_ENTRY }
+#define B53_VTA_REGS_9798 \
+        { B53_VT_ACCESS_9798, B53_VT_INDEX_9798, B53_VT_ENTRY_9798 }
+#define B53_VTA_REGS_63XX \
+        { B53_VT_ACCESS_63XX, B53_VT_INDEX_63XX, B53_VT_ENTRY_63XX }
+
+static const struct b53_chip_data b53_switch_chips[] = {
+        {
+                .chip_id = BCM5325_DEVICE_ID,
+                .dev_name = "BCM5325",
+                .vlans = 16,
+                .enabled_ports = 0x1f,
+                .arl_entries = 2,
+                .cpu_port = B53_CPU_PORT_25,
+                .duplex_reg = B53_DUPLEX_STAT_FE,
+        },
+        {
+                .chip_id = BCM5365_DEVICE_ID,
+                .dev_name = "BCM5365",
+                .vlans = 256,
+                .enabled_ports = 0x1f,
+                .arl_entries = 2,
+                .cpu_port = B53_CPU_PORT_25,
+                .duplex_reg = B53_DUPLEX_STAT_FE,
+        },
+        {
+                .chip_id = BCM5395_DEVICE_ID,
+                .dev_name = "BCM5395",
+                .vlans = 4096,
+                .enabled_ports = 0x1f,
+                .arl_entries = 4,
+                .cpu_port = B53_CPU_PORT,
+                .vta_regs = B53_VTA_REGS,
+                .duplex_reg = B53_DUPLEX_STAT_GE,
+                .jumbo_pm_reg = B53_JUMBO_PORT_MASK,
+                .jumbo_size_reg = B53_JUMBO_MAX_SIZE,
+        },
+        {
+                .chip_id = BCM5397_DEVICE_ID,
+                .dev_name = "BCM5397",
+                .vlans = 4096,
+                .enabled_ports = 0x1f,
+                .arl_entries = 4,
+                .cpu_port = B53_CPU_PORT,
+                .vta_regs = B53_VTA_REGS_9798,
+                .duplex_reg = B53_DUPLEX_STAT_GE,
+                .jumbo_pm_reg = B53_JUMBO_PORT_MASK,
+                .jumbo_size_reg = B53_JUMBO_MAX_SIZE,
+        },
+        {
+                .chip_id = BCM5398_DEVICE_ID,
+                .dev_name = "BCM5398",
+                .vlans = 4096,
+                .enabled_ports = 0x7f,
+                .arl_entries = 4,
+                .cpu_port = B53_CPU_PORT,
+                .vta_regs = B53_VTA_REGS_9798,
+                .duplex_reg = B53_DUPLEX_STAT_GE,
+                .jumbo_pm_reg = B53_JUMBO_PORT_MASK,
+                .jumbo_size_reg = B53_JUMBO_MAX_SIZE,
+        },
+        {
+                .chip_id = BCM53115_DEVICE_ID,
+                .dev_name = "BCM53115",
+                .vlans = 4096,
+                .enabled_ports = 0x1f,
+                .arl_entries = 4,
+                .vta_regs = B53_VTA_REGS,
+                .cpu_port = B53_CPU_PORT,
+                .duplex_reg = B53_DUPLEX_STAT_GE,
+                .jumbo_pm_reg = B53_JUMBO_PORT_MASK,
+                .jumbo_size_reg = B53_JUMBO_MAX_SIZE,
+        },
+        {
+                .chip_id = BCM53125_DEVICE_ID,
+                .dev_name = "BCM53125",
+                .vlans = 4096,
+                .enabled_ports = 0xff,
+                .cpu_port = B53_CPU_PORT,
+                .vta_regs = B53_VTA_REGS,
+                .duplex_reg = B53_DUPLEX_STAT_GE,
+                .jumbo_pm_reg = B53_JUMBO_PORT_MASK,
+                .jumbo_size_reg = B53_JUMBO_MAX_SIZE,
+        },
+        {
+                .chip_id = BCM53128_DEVICE_ID,
+                .dev_name = "BCM53128",
+                .vlans = 4096,
+                .enabled_ports = 0x1ff,
+                .arl_entries = 4,
+                .cpu_port = B53_CPU_PORT,
+                .vta_regs = B53_VTA_REGS,
+                .duplex_reg = B53_DUPLEX_STAT_GE,
+                .jumbo_pm_reg = B53_JUMBO_PORT_MASK,
+                .jumbo_size_reg = B53_JUMBO_MAX_SIZE,
+        },
+        {
+                .chip_id = BCM63XX_DEVICE_ID,
+                .dev_name = "BCM63xx",
+                .vlans = 4096,
+                .enabled_ports = 0, /* pdata must provide them */
+                .arl_entries = 4,
+                .cpu_port = B53_CPU_PORT,
+                .vta_regs = B53_VTA_REGS_63XX,
+                .duplex_reg = B53_DUPLEX_STAT_63XX,
+                .jumbo_pm_reg = B53_JUMBO_PORT_MASK_63XX,
+                .jumbo_size_reg = B53_JUMBO_MAX_SIZE_63XX,
+        },
+        {
+                .chip_id = BCM53010_DEVICE_ID,
+                .dev_name = "BCM53010",
+                .vlans = 4096,
+                .enabled_ports = 0x1f,
+                .arl_entries = 4,
+                .cpu_port = B53_CPU_PORT_25, /* TODO: auto detect */
+                .vta_regs = B53_VTA_REGS,
+                .duplex_reg = B53_DUPLEX_STAT_GE,
+                .jumbo_pm_reg = B53_JUMBO_PORT_MASK,
+                .jumbo_size_reg = B53_JUMBO_MAX_SIZE,
+        },
+        {
+                .chip_id = BCM53011_DEVICE_ID,
+                .dev_name = "BCM53011",
+                .vlans = 4096,
+                .enabled_ports = 0x1bf,
+                .arl_entries = 4,
+                .cpu_port = B53_CPU_PORT_25, /* TODO: auto detect */
+                .vta_regs = B53_VTA_REGS,
+                .duplex_reg = B53_DUPLEX_STAT_GE,
+                .jumbo_pm_reg = B53_JUMBO_PORT_MASK,
+                .jumbo_size_reg = B53_JUMBO_MAX_SIZE,
+        },
+        {
+                .chip_id = BCM53012_DEVICE_ID,
+                .dev_name = "BCM53012",
+                .vlans = 4096,
+                .enabled_ports = 0x1bf,
+                .arl_entries = 4,
+                .cpu_port = B53_CPU_PORT_25, /* TODO: auto detect */
+                .vta_regs = B53_VTA_REGS,
+                .duplex_reg = B53_DUPLEX_STAT_GE,
+                .jumbo_pm_reg = B53_JUMBO_PORT_MASK,
+                .jumbo_size_reg = B53_JUMBO_MAX_SIZE,
+        },
+        {
+                .chip_id = BCM53018_DEVICE_ID,
+                .dev_name = "BCM53018",
+                .vlans = 4096,
+                .enabled_ports = 0x1f,
+                .arl_entries = 4,
+                .cpu_port = B53_CPU_PORT_25, /* TODO: auto detect */
+                .vta_regs = B53_VTA_REGS,
+                .duplex_reg = B53_DUPLEX_STAT_GE,
+                .jumbo_pm_reg = B53_JUMBO_PORT_MASK,
+                .jumbo_size_reg = B53_JUMBO_MAX_SIZE,
+        },
+        {
+                .chip_id = BCM53019_DEVICE_ID,
+                .dev_name = "BCM53019",
+                .vlans = 4096,
+                .enabled_ports = 0x1f,
+                .arl_entries = 4,
+                .cpu_port = B53_CPU_PORT_25, /* TODO: auto detect */
+                .vta_regs = B53_VTA_REGS,
+                .duplex_reg = B53_DUPLEX_STAT_GE,
+                .jumbo_pm_reg = B53_JUMBO_PORT_MASK,
+                .jumbo_size_reg = B53_JUMBO_MAX_SIZE,
+        },
+};
+
+static int b53_switch_init(struct b53_device *dev)
+{
+        struct dsa_switch *ds = dev->ds;
+        unsigned int i;
+        int ret;
+
+        for (i = 0; i < ARRAY_SIZE(b53_switch_chips); i++) {
+                const struct b53_chip_data *chip = &b53_switch_chips[i];
+
+                if (chip->chip_id == dev->chip_id) {
+                        if (!dev->enabled_ports)
+                                dev->enabled_ports = chip->enabled_ports;
+                        dev->name = chip->dev_name;
+                        dev->duplex_reg = chip->duplex_reg;
+                        dev->vta_regs[0] = chip->vta_regs[0];
+                        dev->vta_regs[1] = chip->vta_regs[1];
+                        dev->vta_regs[2] = chip->vta_regs[2];
+                        dev->jumbo_pm_reg = chip->jumbo_pm_reg;
+                        ds->drv = &b53_switch_ops;
+                        dev->cpu_port = chip->cpu_port;
+                        dev->num_vlans = chip->vlans;
+                        dev->num_arl_entries = chip->arl_entries;
+                        break;
+                }
+        }
+
+        /* check which BCM5325x version we have */
+        if (is5325(dev)) {
+                u8 vc4;
+
+                b53_read8(dev, B53_VLAN_PAGE, B53_VLAN_CTRL4_25, &vc4);
+
+                /* check reserved bits */
+                switch (vc4 & 3) {
+                case 1:
+                        /* BCM5325E */
+                        break;
+                case 3:
+                        /* BCM5325F - do not use port 4 */
+                        dev->enabled_ports &= ~BIT(4);
+                        break;
+                default:
+/* On the BCM47XX SoCs this is the supported internal switch.*/
+#ifndef CONFIG_BCM47XX
+                        /* BCM5325M */
+                        return -EINVAL;
+#else
+                        break;
+#endif
+                }
+        } else if (dev->chip_id == BCM53115_DEVICE_ID) {
+                u64 strap_value;
+
+                b53_read48(dev, B53_STAT_PAGE, B53_STRAP_VALUE, &strap_value);
+                /* use second IMP port if GMII is enabled */
+                if (strap_value & SV_GMII_CTRL_115)
+                        dev->cpu_port = 5;
+        }
+
+        /* cpu port is always last */
+        dev->num_ports = dev->cpu_port + 1;
+        dev->enabled_ports |= BIT(dev->cpu_port);
+
+        dev->ports = devm_kzalloc(dev->dev,
+                                  sizeof(struct b53_port) * dev->num_ports,
+                                  GFP_KERNEL);
+        if (!dev->ports)
+                return -ENOMEM;
+
+        dev->vlans = devm_kzalloc(dev->dev,
+                                  sizeof(struct b53_vlan) * dev->num_vlans,
+                                  GFP_KERNEL);
+        if (!dev->vlans)
+                return -ENOMEM;
+
+        dev->reset_gpio = b53_switch_get_reset_gpio(dev);
+        if (dev->reset_gpio >= 0) {
+                ret = devm_gpio_request_one(dev->dev, dev->reset_gpio,
+                                            GPIOF_OUT_INIT_HIGH, "robo_reset");
+                if (ret)
+                        return ret;
+        }
+
+        return 0;
+}
+
+struct b53_device *b53_switch_alloc(struct device *base, struct b53_io_ops *ops,
+                                    void *priv)
+{
+        struct dsa_switch *ds;
+        struct b53_device *dev;
+
+        ds = devm_kzalloc(base, sizeof(*ds) + sizeof(*dev), GFP_KERNEL);
+        if (!ds)
+                return NULL;
+
+        dev = (struct b53_device *)(ds + 1);
+
+        ds->priv = dev;
+        ds->dev = base;
+        dev->dev = base;
+
+        dev->ds = ds;
+        dev->priv = priv;
+        dev->ops = ops;
+        mutex_init(&dev->reg_mutex);
+        mutex_init(&dev->stats_mutex);
+
+        return dev;
+}
+EXPORT_SYMBOL(b53_switch_alloc);
+
+int b53_switch_detect(struct b53_device *dev)
+{
+        u32 id32;
+        u16 tmp;
+        u8 id8;
+        int ret;
+
+        ret = b53_read8(dev, B53_MGMT_PAGE, B53_DEVICE_ID, &id8);
+        if (ret)
+                return ret;
+
+        switch (id8) {
+        case 0:
+                /* BCM5325 and BCM5365 do not have this register so reads
+                 * return 0. But the read operation did succeed, so assume this
+                 * is one of them.
+                 *
+                 * Next check if we can write to the 5325's VTA register; for
+                 * 5365 it is read only.
+                 */
+                b53_write16(dev, B53_VLAN_PAGE, B53_VLAN_TABLE_ACCESS_25, 0xf);
+                b53_read16(dev, B53_VLAN_PAGE, B53_VLAN_TABLE_ACCESS_25, &tmp);
+
+                if (tmp == 0xf)
+                        dev->chip_id = BCM5325_DEVICE_ID;
+                else
+                        dev->chip_id = BCM5365_DEVICE_ID;
+                break;
+        case BCM5395_DEVICE_ID:
+        case BCM5397_DEVICE_ID:
+        case BCM5398_DEVICE_ID:
+                dev->chip_id = id8;
+                break;
+        default:
+                ret = b53_read32(dev, B53_MGMT_PAGE, B53_DEVICE_ID, &id32);
+                if (ret)
+                        return ret;
+
+                switch (id32) {
+                case BCM53115_DEVICE_ID:
+                case BCM53125_DEVICE_ID:
+                case BCM53128_DEVICE_ID:
+                case BCM53010_DEVICE_ID:
+                case BCM53011_DEVICE_ID:
+                case BCM53012_DEVICE_ID:
+                case BCM53018_DEVICE_ID:
+                case BCM53019_DEVICE_ID:
+                        dev->chip_id = id32;
+                        break;
+                default:
+                        pr_err("unsupported switch detected (BCM53%02x/BCM%x)\n",
+                               id8, id32);
+                        return -ENODEV;
+                }
+        }
+
+        if (dev->chip_id == BCM5325_DEVICE_ID)
+                return b53_read8(dev, B53_STAT_PAGE, B53_REV_ID_25,
+                                 &dev->core_rev);
+        else
+                return b53_read8(dev, B53_MGMT_PAGE, B53_REV_ID,
+                                 &dev->core_rev);
+}
+EXPORT_SYMBOL(b53_switch_detect);
+
+int b53_switch_register(struct b53_device *dev)
+{
+        int ret;
+
+        if (dev->pdata) {
+                dev->chip_id = dev->pdata->chip_id;
+                dev->enabled_ports = dev->pdata->enabled_ports;
+        }
+
+        if (!dev->chip_id && b53_switch_detect(dev))
+                return -EINVAL;
+
+        ret = b53_switch_init(dev);
+        if (ret)
+                return ret;
+
+        pr_info("found switch: %s, rev %i\n", dev->name, dev->core_rev);
+
+        return dsa_register_switch(dev->ds, dev->ds->dev->of_node);
+}
+EXPORT_SYMBOL(b53_switch_register);
+
+MODULE_AUTHOR("Jonas Gorski <jogo@openwrt.org>");
+MODULE_DESCRIPTION("B53 switch library");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/net/dsa/b53/b53_mdio.c b/drivers/net/dsa/b53/b53_mdio.c
new file mode 100644
index 000000000000..aa87c3fffdac
--- /dev/null
+++ b/drivers/net/dsa/b53/b53_mdio.c
@@ -0,0 +1,392 @@
+/*
+ * B53 register access through MII registers
+ *
+ * Copyright (C) 2011-2013 Jonas Gorski <jogo@openwrt.org>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/phy.h>
+#include <linux/module.h>
+#include <linux/delay.h>
+#include <linux/brcmphy.h>
+#include <linux/rtnetlink.h>
+#include <net/dsa.h>
+
+#include "b53_priv.h"
+
+/* MII registers */
+#define REG_MII_PAGE    0x10    /* MII Page register */
+#define REG_MII_ADDR    0x11    /* MII Address register */
+#define REG_MII_DATA0   0x18    /* MII Data register 0 */
+#define REG_MII_DATA1   0x19    /* MII Data register 1 */
+#define REG_MII_DATA2   0x1a    /* MII Data register 2 */
+#define REG_MII_DATA3   0x1b    /* MII Data register 3 */
+
+#define REG_MII_PAGE_ENABLE     BIT(0)
+#define REG_MII_ADDR_WRITE      BIT(0)
+#define REG_MII_ADDR_READ       BIT(1)
+
+static int b53_mdio_op(struct b53_device *dev, u8 page, u8 reg, u16 op)
+{
+        int i;
+        u16 v;
+        int ret;
+        struct mii_bus *bus = dev->priv;
+
+        if (dev->current_page != page) {
+                /* set page number */
+                v = (page << 8) | REG_MII_PAGE_ENABLE;
+                ret = mdiobus_write_nested(bus, BRCM_PSEUDO_PHY_ADDR,
+                                           REG_MII_PAGE, v);
+                if (ret)
+                        return ret;
+                dev->current_page = page;
+        }
+
+        /* set register address */
+        v = (reg << 8) | op;
+        ret = mdiobus_write_nested(bus, BRCM_PSEUDO_PHY_ADDR, REG_MII_ADDR, v);
+        if (ret)
+                return ret;
+
+        /* check if operation completed */
+        for (i = 0; i < 5; ++i) {
+                v = mdiobus_read_nested(bus, BRCM_PSEUDO_PHY_ADDR,
+                                        REG_MII_ADDR);
+                if (!(v & (REG_MII_ADDR_WRITE | REG_MII_ADDR_READ)))
+                        break;
+                usleep_range(10, 100);
+        }
+
+        if (WARN_ON(i == 5))
+                return -EIO;
+
+        return 0;
+}
+
+static int b53_mdio_read8(struct b53_device *dev, u8 page, u8 reg, u8 *val)
+{
+        struct mii_bus *bus = dev->priv;
+        int ret;
+
+        ret = b53_mdio_op(dev, page, reg, REG_MII_ADDR_READ);
+        if (ret)
+                return ret;
+
+        *val = mdiobus_read_nested(bus, BRCM_PSEUDO_PHY_ADDR,
+                                   REG_MII_DATA0) & 0xff;
+
+        return 0;
+}
+
+static int b53_mdio_read16(struct b53_device *dev, u8 page, u8 reg, u16 *val)
+{
+        struct mii_bus *bus = dev->priv;
+        int ret;
+
+        ret = b53_mdio_op(dev, page, reg, REG_MII_ADDR_READ);
+        if (ret)
+                return ret;
+
+        *val = mdiobus_read_nested(bus, BRCM_PSEUDO_PHY_ADDR, REG_MII_DATA0);
+
+        return 0;
+}
+
+static int b53_mdio_read32(struct b53_device *dev, u8 page, u8 reg, u32 *val)
+{
+        struct mii_bus *bus = dev->priv;
+        int ret;
+
+        ret = b53_mdio_op(dev, page, reg, REG_MII_ADDR_READ);
+        if (ret)
+                return ret;
+
+        *val = mdiobus_read_nested(bus, BRCM_PSEUDO_PHY_ADDR, REG_MII_DATA0);
+        *val |= mdiobus_read_nested(bus, BRCM_PSEUDO_PHY_ADDR,
+                                    REG_MII_DATA1) << 16;
+
+        return 0;
+}
+
+static int b53_mdio_read48(struct b53_device *dev, u8 page, u8 reg, u64 *val)
+{
+        struct mii_bus *bus = dev->priv;
+        u64 temp = 0;
+        int i;
+        int ret;
+
+        ret = b53_mdio_op(dev, page, reg, REG_MII_ADDR_READ);
+        if (ret)
+                return ret;
+
+        for (i = 2; i >= 0; i--) {
+                temp <<= 16;
+                temp |= mdiobus_read_nested(bus, BRCM_PSEUDO_PHY_ADDR,
+                                     REG_MII_DATA0 + i);
+        }
+
+        *val = temp;
+
+        return 0;
+}
+
+static int b53_mdio_read64(struct b53_device *dev, u8 page, u8 reg, u64 *val)
+{
+        struct mii_bus *bus = dev->priv;
+        u64 temp = 0;
+        int i;
+        int ret;
+
+        ret = b53_mdio_op(dev, page, reg, REG_MII_ADDR_READ);
+        if (ret)
+                return ret;
+
+        for (i = 3; i >= 0; i--) {
+                temp <<= 16;
+                temp |= mdiobus_read_nested(bus, BRCM_PSEUDO_PHY_ADDR,
+                                            REG_MII_DATA0 + i);
+        }
+
+        *val = temp;
+
+        return 0;
+}
+
+static int b53_mdio_write8(struct b53_device *dev, u8 page, u8 reg, u8 value)
+{
+        struct mii_bus *bus = dev->priv;
+        int ret;
+
+        ret = mdiobus_write_nested(bus, BRCM_PSEUDO_PHY_ADDR,
+                                   REG_MII_DATA0, value);
+        if (ret)
+                return ret;
+
+        return b53_mdio_op(dev, page, reg, REG_MII_ADDR_WRITE);
+}
+
+static int b53_mdio_write16(struct b53_device *dev, u8 page, u8 reg,
+                            u16 value)
+{
+        struct mii_bus *bus = dev->priv;
+        int ret;
+
+        ret = mdiobus_write_nested(bus, BRCM_PSEUDO_PHY_ADDR,
+                                   REG_MII_DATA0, value);
+        if (ret)
+                return ret;
+
+        return b53_mdio_op(dev, page, reg, REG_MII_ADDR_WRITE);
+}
+
+static int b53_mdio_write32(struct b53_device *dev, u8 page, u8 reg,
+                            u32 value)
+{
+        struct mii_bus *bus = dev->priv;
+        unsigned int i;
+        u32 temp = value;
+
+        for (i = 0; i < 2; i++) {
+                int ret = mdiobus_write_nested(bus, BRCM_PSEUDO_PHY_ADDR,
+                                               REG_MII_DATA0 + i,
+                                               temp & 0xffff);
+                if (ret)
+                        return ret;
+                temp >>= 16;
+        }
+
+        return b53_mdio_op(dev, page, reg, REG_MII_ADDR_WRITE);
+}
+
+static int b53_mdio_write48(struct b53_device *dev, u8 page, u8 reg,
+                            u64 value)
+{
+        struct mii_bus *bus = dev->priv;
+        unsigned int i;
+        u64 temp = value;
+
+        for (i = 0; i < 3; i++) {
+                int ret = mdiobus_write_nested(bus, BRCM_PSEUDO_PHY_ADDR,
+                                               REG_MII_DATA0 + i,
+                                               temp & 0xffff);
+                if (ret)
+                        return ret;
+                temp >>= 16;
+        }
+
+        return b53_mdio_op(dev, page, reg, REG_MII_ADDR_WRITE);
+}
+
+static int b53_mdio_write64(struct b53_device *dev, u8 page, u8 reg,
+                            u64 value)
+{
+        struct mii_bus *bus = dev->priv;
+        unsigned int i;
+        u64 temp = value;
+
+        for (i = 0; i < 4; i++) {
+                int ret = mdiobus_write_nested(bus, BRCM_PSEUDO_PHY_ADDR,
+                                               REG_MII_DATA0 + i,
+                                               temp & 0xffff);
+                if (ret)
+                        return ret;
+                temp >>= 16;
+        }
+
+        return b53_mdio_op(dev, page, reg, REG_MII_ADDR_WRITE);
+}
+
+static int b53_mdio_phy_read16(struct b53_device *dev, int addr, int reg,
+                               u16 *value)
+{
+        struct mii_bus *bus = dev->priv;
+
+        *value = mdiobus_read_nested(bus, addr, reg);
+
+        return 0;
+}
+
+static int b53_mdio_phy_write16(struct b53_device *dev, int addr, int reg,
+                                u16 value)
+{
+        struct mii_bus *bus = dev->bus;
+
+        return mdiobus_write_nested(bus, addr, reg, value);
+}
+
+static struct b53_io_ops b53_mdio_ops = {
+        .read8 = b53_mdio_read8,
+        .read16 = b53_mdio_read16,
+        .read32 = b53_mdio_read32,
+        .read48 = b53_mdio_read48,
+        .read64 = b53_mdio_read64,
+        .write8 = b53_mdio_write8,
+        .write16 = b53_mdio_write16,
+        .write32 = b53_mdio_write32,
+        .write48 = b53_mdio_write48,
+        .write64 = b53_mdio_write64,
+        .phy_read16 = b53_mdio_phy_read16,
+        .phy_write16 = b53_mdio_phy_write16,
+};
+
+#define B53_BRCM_OUI_1  0x0143bc00
+#define B53_BRCM_OUI_2  0x03625c00
+#define B53_BRCM_OUI_3  0x00406000
+
+static int b53_mdio_probe(struct mdio_device *mdiodev)
+{
+        struct b53_device *dev;
+        u32 phy_id;
+        int ret;
+
+        /* allow the generic PHY driver to take over the non-management MDIO
+         * addresses
+         */
+        if (mdiodev->addr != BRCM_PSEUDO_PHY_ADDR && mdiodev->addr != 0) {
+                dev_err(&mdiodev->dev, "leaving address %d to PHY\n",
+                        mdiodev->addr);
+                return -ENODEV;
+        }
+
+        /* read the first port's id */
+        phy_id = mdiobus_read(mdiodev->bus, 0, 2) << 16;
+        phy_id |= mdiobus_read(mdiodev->bus, 0, 3);
+
+        /* BCM5325, BCM539x (OUI_1)
+         * BCM53125, BCM53128 (OUI_2)
+         * BCM5365 (OUI_3)
+         */
+        if ((phy_id & 0xfffffc00) != B53_BRCM_OUI_1 &&
+            (phy_id & 0xfffffc00) != B53_BRCM_OUI_2 &&
+            (phy_id & 0xfffffc00) != B53_BRCM_OUI_3) {
+                dev_err(&mdiodev->dev, "Unsupported device: 0x%08x\n", phy_id);
+                return -ENODEV;
+        }
+
+        /* First probe will come from SWITCH_MDIO controller on the 7445D0
+         * switch, which will conflict with the 7445 integrated switch
+         * pseudo-phy (we end-up programming both). In that case, we return
+         * -EPROBE_DEFER for the first time we get here, and wait until we come
+         * back with the slave MDIO bus which has the correct indirection
+         * layer setup
+         */
+        if (of_machine_is_compatible("brcm,bcm7445d0") &&
+            strcmp(mdiodev->bus->name, "sf2 slave mii"))
+                return -EPROBE_DEFER;
+
+        dev = b53_switch_alloc(&mdiodev->dev, &b53_mdio_ops, mdiodev->bus);
+        if (!dev)
+                return -ENOMEM;
+
+        /* we don't use page 0xff, so force a page set */
+        dev->current_page = 0xff;
+        dev->bus = mdiodev->bus;
+
+        dev_set_drvdata(&mdiodev->dev, dev);
+
+        ret = b53_switch_register(dev);
+        if (ret) {
+                dev_err(&mdiodev->dev, "failed to register switch: %i\n", ret);
+                return ret;
+        }
+
+        return ret;
+}
+
+static void b53_mdio_remove(struct mdio_device *mdiodev)
+{
+        struct b53_device *dev = dev_get_drvdata(&mdiodev->dev);
+        struct dsa_switch *ds = dev->ds;
+
+        dsa_unregister_switch(ds);
+}
+
+static const struct of_device_id b53_of_match[] = {
+        { .compatible = "brcm,bcm5325" },
+        { .compatible = "brcm,bcm53115" },
+        { .compatible = "brcm,bcm53125" },
+        { .compatible = "brcm,bcm53128" },
+        { .compatible = "brcm,bcm5365" },
+        { .compatible = "brcm,bcm5395" },
+        { .compatible = "brcm,bcm5397" },
+        { .compatible = "brcm,bcm5398" },
+        { /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, b53_of_match);
+
+static struct mdio_driver b53_mdio_driver = {
+        .probe  = b53_mdio_probe,
+        .remove = b53_mdio_remove,
+        .mdiodrv.driver = {
+                .name = "bcm53xx",
+                .of_match_table = b53_of_match,
+        },
+};
+
+static int __init b53_mdio_driver_register(void)
+{
+        return mdio_driver_register(&b53_mdio_driver);
+}
+module_init(b53_mdio_driver_register);
+
+static void __exit b53_mdio_driver_unregister(void)
+{
+        mdio_driver_unregister(&b53_mdio_driver);
+}
+module_exit(b53_mdio_driver_unregister);
+
+MODULE_DESCRIPTION("B53 MDIO access driver");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/net/dsa/b53/b53_mmap.c b/drivers/net/dsa/b53/b53_mmap.c
new file mode 100644
index 000000000000..21f1068b0804
--- /dev/null
+++ b/drivers/net/dsa/b53/b53_mmap.c
@@ -0,0 +1,274 @@
+/*
+ * B53 register access through memory mapped registers
+ *
+ * Copyright (C) 2012-2013 Jonas Gorski <jogo@openwrt.org>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/kconfig.h>
+#include <linux/module.h>
+#include <linux/io.h>
+#include <linux/platform_device.h>
+#include <linux/platform_data/b53.h>
+
+#include "b53_priv.h"
+
+struct b53_mmap_priv {
+        void __iomem *regs;
+};
+
+static int b53_mmap_read8(struct b53_device *dev, u8 page, u8 reg, u8 *val)
+{
+        u8 __iomem *regs = dev->priv;
+
+        *val = readb(regs + (page << 8) + reg);
+
+        return 0;
+}
+
+static int b53_mmap_read16(struct b53_device *dev, u8 page, u8 reg, u16 *val)
+{
+        u8 __iomem *regs = dev->priv;
+
+        if (WARN_ON(reg % 2))
+                return -EINVAL;
+
+        if (dev->pdata && dev->pdata->big_endian)
+                *val = ioread16be(regs + (page << 8) + reg);
+        else
+                *val = readw(regs + (page << 8) + reg);
+
+        return 0;
+}
+
+static int b53_mmap_read32(struct b53_device *dev, u8 page, u8 reg, u32 *val)
+{
+        u8 __iomem *regs = dev->priv;
+
+        if (WARN_ON(reg % 4))
+                return -EINVAL;
+
+        if (dev->pdata && dev->pdata->big_endian)
+                *val = ioread32be(regs + (page << 8) + reg);
+        else
+                *val = readl(regs + (page << 8) + reg);
+
+        return 0;
+}
+
+static int b53_mmap_read48(struct b53_device *dev, u8 page, u8 reg, u64 *val)
+{
+        u8 __iomem *regs = dev->priv;
+
+        if (WARN_ON(reg % 2))
+                return -EINVAL;
+
+        if (reg % 4) {
+                u16 lo;
+                u32 hi;
+
+                if (dev->pdata && dev->pdata->big_endian) {
+                        lo = ioread16be(regs + (page << 8) + reg);
+                        hi = ioread32be(regs + (page << 8) + reg + 2);
+                } else {
+                        lo = readw(regs + (page << 8) + reg);
+                        hi = readl(regs + (page << 8) + reg + 2);
+                }
+
+                *val = ((u64)hi << 16) | lo;
+        } else {
+                u32 lo;
+                u16 hi;
+
+                if (dev->pdata && dev->pdata->big_endian) {
+                        lo = ioread32be(regs + (page << 8) + reg);
+                        hi = ioread16be(regs + (page << 8) + reg + 4);
+                } else {
+                        lo = readl(regs + (page << 8) + reg);
+                        hi = readw(regs + (page << 8) + reg + 4);
+                }
+
+                *val = ((u64)hi << 32) | lo;
+        }
+
+        return 0;
+}
+
+static int b53_mmap_read64(struct b53_device *dev, u8 page, u8 reg, u64 *val)
+{
+        u8 __iomem *regs = dev->priv;
+        u32 hi, lo;
+
+        if (WARN_ON(reg % 4))
+                return -EINVAL;
+
+        if (dev->pdata && dev->pdata->big_endian) {
+                lo = ioread32be(regs + (page << 8) + reg);
+                hi = ioread32be(regs + (page << 8) + reg + 4);
+        } else {
+                lo = readl(regs + (page << 8) + reg);
+                hi = readl(regs + (page << 8) + reg + 4);
+        }
+
+        *val = ((u64)hi << 32) | lo;
+
+        return 0;
+}
+
+static int b53_mmap_write8(struct b53_device *dev, u8 page, u8 reg, u8 value)
+{
+        u8 __iomem *regs = dev->priv;
+
+        writeb(value, regs + (page << 8) + reg);
+
+        return 0;
+}
+
+static int b53_mmap_write16(struct b53_device *dev, u8 page, u8 reg,
+                            u16 value)
+{
+        u8 __iomem *regs = dev->priv;
+
+        if (WARN_ON(reg % 2))
+                return -EINVAL;
+
+        if (dev->pdata && dev->pdata->big_endian)
+                iowrite16be(value, regs + (page << 8) + reg);
+        else
+                writew(value, regs + (page << 8) + reg);
+
+        return 0;
+}
+
+static int b53_mmap_write32(struct b53_device *dev, u8 page, u8 reg,
+                            u32 value)
+{
+        u8 __iomem *regs = dev->priv;
+
+        if (WARN_ON(reg % 4))
+                return -EINVAL;
+
+        if (dev->pdata && dev->pdata->big_endian)
+                iowrite32be(value, regs + (page << 8) + reg);
+        else
+                writel(value, regs + (page << 8) + reg);
+
+        return 0;
+}
+
+static int b53_mmap_write48(struct b53_device *dev, u8 page, u8 reg,
+                            u64 value)
+{
+        if (WARN_ON(reg % 2))
+                return -EINVAL;
+
+        if (reg % 4) {
+                u32 hi = (u32)(value >> 16);
+                u16 lo = (u16)value;
+
+                b53_mmap_write16(dev, page, reg, lo);
+                b53_mmap_write32(dev, page, reg + 2, hi);
+        } else {
+                u16 hi = (u16)(value >> 32);
+                u32 lo = (u32)value;
+
+                b53_mmap_write32(dev, page, reg, lo);
+                b53_mmap_write16(dev, page, reg + 4, hi);
+        }
+
+        return 0;
+}
+
+static int b53_mmap_write64(struct b53_device *dev, u8 page, u8 reg,
+                            u64 value)
+{
+        u32 hi, lo;
+
+        hi = upper_32_bits(value);
+        lo = lower_32_bits(value);
+
+        if (WARN_ON(reg % 4))
+                return -EINVAL;
+
+        b53_mmap_write32(dev, page, reg, lo);
+        b53_mmap_write32(dev, page, reg + 4, hi);
+
+        return 0;
+}
+
+static struct b53_io_ops b53_mmap_ops = {
+        .read8 = b53_mmap_read8,
+        .read16 = b53_mmap_read16,
+        .read32 = b53_mmap_read32,
+        .read48 = b53_mmap_read48,
+        .read64 = b53_mmap_read64,
+        .write8 = b53_mmap_write8,
+        .write16 = b53_mmap_write16,
+        .write32 = b53_mmap_write32,
+        .write48 = b53_mmap_write48,
+        .write64 = b53_mmap_write64,
+};
+
+static int b53_mmap_probe(struct platform_device *pdev)
+{
+        struct b53_platform_data *pdata = pdev->dev.platform_data;
+        struct b53_device *dev;
+
+        if (!pdata)
+                return -EINVAL;
+
+        dev = b53_switch_alloc(&pdev->dev, &b53_mmap_ops, pdata->regs);
+        if (!dev)
+                return -ENOMEM;
+
+        if (pdata)
+                dev->pdata = pdata;
+
+        platform_set_drvdata(pdev, dev);
+
+        return b53_switch_register(dev);
+}
+
+static int b53_mmap_remove(struct platform_device *pdev)
+{
+        struct b53_device *dev = platform_get_drvdata(pdev);
+
+        if (dev)
+                b53_switch_remove(dev);
+
+        return 0;
+}
+
+static const struct of_device_id b53_mmap_of_table[] = {
+        { .compatible = "brcm,bcm3384-switch" },
+        { .compatible = "brcm,bcm6328-switch" },
+        { .compatible = "brcm,bcm6368-switch" },
+        { .compatible = "brcm,bcm63xx-switch" },
+        { /* sentinel */ },
+};
+
+static struct platform_driver b53_mmap_driver = {
+        .probe = b53_mmap_probe,
+        .remove = b53_mmap_remove,
+        .driver = {
+                .name = "b53-switch",
+                .of_match_table = b53_mmap_of_table,
+        },
+};
+
+module_platform_driver(b53_mmap_driver);
+MODULE_AUTHOR("Jonas Gorski <jogo@openwrt.org>");
+MODULE_DESCRIPTION("B53 MMAP access driver");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/net/dsa/b53/b53_priv.h b/drivers/net/dsa/b53/b53_priv.h
new file mode 100644
index 000000000000..5d8c602fb877
--- /dev/null
+++ b/drivers/net/dsa/b53/b53_priv.h
@@ -0,0 +1,387 @@
+/*
+ * B53 common definitions
+ *
+ * Copyright (C) 2011-2013 Jonas Gorski <jogo@openwrt.org>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef __B53_PRIV_H
+#define __B53_PRIV_H
+
+#include <linux/kernel.h>
+#include <linux/mutex.h>
+#include <linux/phy.h>
+#include <net/dsa.h>
+
+#include "b53_regs.h"
+
+struct b53_device;
+struct net_device;
+
+struct b53_io_ops {
+        int (*read8)(struct b53_device *dev, u8 page, u8 reg, u8 *value);
+        int (*read16)(struct b53_device *dev, u8 page, u8 reg, u16 *value);
+        int (*read32)(struct b53_device *dev, u8 page, u8 reg, u32 *value);
+        int (*read48)(struct b53_device *dev, u8 page, u8 reg, u64 *value);
+        int (*read64)(struct b53_device *dev, u8 page, u8 reg, u64 *value);
+        int (*write8)(struct b53_device *dev, u8 page, u8 reg, u8 value);
+        int (*write16)(struct b53_device *dev, u8 page, u8 reg, u16 value);
+        int (*write32)(struct b53_device *dev, u8 page, u8 reg, u32 value);
+        int (*write48)(struct b53_device *dev, u8 page, u8 reg, u64 value);
+        int (*write64)(struct b53_device *dev, u8 page, u8 reg, u64 value);
+        int (*phy_read16)(struct b53_device *dev, int addr, int reg, u16 *value);
+        int (*phy_write16)(struct b53_device *dev, int addr, int reg, u16 value);
+};
+
+enum {
+        BCM5325_DEVICE_ID = 0x25,
+        BCM5365_DEVICE_ID = 0x65,
+        BCM5395_DEVICE_ID = 0x95,
+        BCM5397_DEVICE_ID = 0x97,
+        BCM5398_DEVICE_ID = 0x98,
+        BCM53115_DEVICE_ID = 0x53115,
+        BCM53125_DEVICE_ID = 0x53125,
+        BCM53128_DEVICE_ID = 0x53128,
+        BCM63XX_DEVICE_ID = 0x6300,
+        BCM53010_DEVICE_ID = 0x53010,
+        BCM53011_DEVICE_ID = 0x53011,
+        BCM53012_DEVICE_ID = 0x53012,
+        BCM53018_DEVICE_ID = 0x53018,
+        BCM53019_DEVICE_ID = 0x53019,
+};
+
+#define B53_N_PORTS     9
+#define B53_N_PORTS_25  6
+
+struct b53_port {
+        u16             vlan_ctl_mask;
+        struct net_device *bridge_dev;
+};
+
+struct b53_vlan {
+        u16 members;
+        u16 untag;
+        bool valid;
+};
+
+struct b53_device {
+        struct dsa_switch *ds;
+        struct b53_platform_data *pdata;
+        const char *name;
+
+        struct mutex reg_mutex;
+        struct mutex stats_mutex;
+        const struct b53_io_ops *ops;
+
+        /* chip specific data */
+        u32 chip_id;
+        u8 core_rev;
+        u8 vta_regs[3];
+        u8 duplex_reg;
+        u8 jumbo_pm_reg;
+        u8 jumbo_size_reg;
+        int reset_gpio;
+        u8 num_arl_entries;
+
+        /* used ports mask */
+        u16 enabled_ports;
+        unsigned int cpu_port;
+
+        /* connect specific data */
+        u8 current_page;
+        struct device *dev;
+
+        /* Master MDIO bus we got probed from */
+        struct mii_bus *bus;
+
+        void *priv;
+
+        /* run time configuration */
+        bool enable_jumbo;
+
+        unsigned int num_vlans;
+        struct b53_vlan *vlans;
+        unsigned int num_ports;
+        struct b53_port *ports;
+};
+
+#define b53_for_each_port(dev, i) \
+        for (i = 0; i < B53_N_PORTS; i++) \
+                if (dev->enabled_ports & BIT(i))
+
+
+static inline int is5325(struct b53_device *dev)
+{
+        return dev->chip_id == BCM5325_DEVICE_ID;
+}
+
+static inline int is5365(struct b53_device *dev)
+{
+#ifdef CONFIG_BCM47XX
+        return dev->chip_id == BCM5365_DEVICE_ID;
+#else
+        return 0;
+#endif
+}
+
+static inline int is5397_98(struct b53_device *dev)
+{
+        return dev->chip_id == BCM5397_DEVICE_ID ||
+                dev->chip_id == BCM5398_DEVICE_ID;
+}
+
+static inline int is539x(struct b53_device *dev)
+{
+        return dev->chip_id == BCM5395_DEVICE_ID ||
+                dev->chip_id == BCM5397_DEVICE_ID ||
+                dev->chip_id == BCM5398_DEVICE_ID;
+}
+
+static inline int is531x5(struct b53_device *dev)
+{
+        return dev->chip_id == BCM53115_DEVICE_ID ||
+                dev->chip_id == BCM53125_DEVICE_ID ||
+                dev->chip_id == BCM53128_DEVICE_ID;
+}
+
+static inline int is63xx(struct b53_device *dev)
+{
+#ifdef CONFIG_BCM63XX
+        return dev->chip_id == BCM63XX_DEVICE_ID;
+#else
+        return 0;
+#endif
+}
+
+static inline int is5301x(struct b53_device *dev)
+{
+        return dev->chip_id == BCM53010_DEVICE_ID ||
+                dev->chip_id == BCM53011_DEVICE_ID ||
+                dev->chip_id == BCM53012_DEVICE_ID ||
+                dev->chip_id == BCM53018_DEVICE_ID ||
+                dev->chip_id == BCM53019_DEVICE_ID;
+}
+
+#define B53_CPU_PORT_25 5
+#define B53_CPU_PORT    8
+
+static inline int is_cpu_port(struct b53_device *dev, int port)
+{
+        return dev->cpu_port;
+}
+
+struct b53_device *b53_switch_alloc(struct device *base, struct b53_io_ops *ops,
+                                    void *priv);
+
+int b53_switch_detect(struct b53_device *dev);
+
+int b53_switch_register(struct b53_device *dev);
+
+static inline void b53_switch_remove(struct b53_device *dev)
+{
+        dsa_unregister_switch(dev->ds);
+}
+
+static inline int b53_read8(struct b53_device *dev, u8 page, u8 reg, u8 *val)
+{
+        int ret;
+
+        mutex_lock(&dev->reg_mutex);
+        ret = dev->ops->read8(dev, page, reg, val);
+        mutex_unlock(&dev->reg_mutex);
+
+        return ret;
+}
+
+static inline int b53_read16(struct b53_device *dev, u8 page, u8 reg, u16 *val)
+{
+        int ret;
+
+        mutex_lock(&dev->reg_mutex);
+        ret = dev->ops->read16(dev, page, reg, val);
+        mutex_unlock(&dev->reg_mutex);
+
+        return ret;
+}
+
+static inline int b53_read32(struct b53_device *dev, u8 page, u8 reg, u32 *val)
+{
+        int ret;
+
+        mutex_lock(&dev->reg_mutex);
+        ret = dev->ops->read32(dev, page, reg, val);
+        mutex_unlock(&dev->reg_mutex);
+
+        return ret;
+}
+
+static inline int b53_read48(struct b53_device *dev, u8 page, u8 reg, u64 *val)
+{
+        int ret;
+
+        mutex_lock(&dev->reg_mutex);
+        ret = dev->ops->read48(dev, page, reg, val);
+        mutex_unlock(&dev->reg_mutex);
+
+        return ret;
+}
+
+static inline int b53_read64(struct b53_device *dev, u8 page, u8 reg, u64 *val)
+{
+        int ret;
+
+        mutex_lock(&dev->reg_mutex);
+        ret = dev->ops->read64(dev, page, reg, val);
+        mutex_unlock(&dev->reg_mutex);
+
+        return ret;
+}
+
+static inline int b53_write8(struct b53_device *dev, u8 page, u8 reg, u8 value)
+{
+        int ret;
+
+        mutex_lock(&dev->reg_mutex);
+        ret = dev->ops->write8(dev, page, reg, value);
+        mutex_unlock(&dev->reg_mutex);
+
+        return ret;
+}
+
+static inline int b53_write16(struct b53_device *dev, u8 page, u8 reg,
+                              u16 value)
+{
+        int ret;
+
+        mutex_lock(&dev->reg_mutex);
+        ret = dev->ops->write16(dev, page, reg, value);
+        mutex_unlock(&dev->reg_mutex);
+
+        return ret;
+}
+
+static inline int b53_write32(struct b53_device *dev, u8 page, u8 reg,
+                              u32 value)
+{
+        int ret;
+
+        mutex_lock(&dev->reg_mutex);
+        ret = dev->ops->write32(dev, page, reg, value);
+        mutex_unlock(&dev->reg_mutex);
+
+        return ret;
+}
+
+static inline int b53_write48(struct b53_device *dev, u8 page, u8 reg,
+                              u64 value)
+{
+        int ret;
+
+        mutex_lock(&dev->reg_mutex);
+        ret = dev->ops->write48(dev, page, reg, value);
+        mutex_unlock(&dev->reg_mutex);
+
+        return ret;
+}
+
+static inline int b53_write64(struct b53_device *dev, u8 page, u8 reg,
+                               u64 value)
+{
+        int ret;
+
+        mutex_lock(&dev->reg_mutex);
+        ret = dev->ops->write64(dev, page, reg, value);
+        mutex_unlock(&dev->reg_mutex);
+
+        return ret;
+}
+
+struct b53_arl_entry {
+        u8 port;
+        u8 mac[ETH_ALEN];
+        u16 vid;
+        u8 is_valid:1;
+        u8 is_age:1;
+        u8 is_static:1;
+};
+
+static inline void b53_mac_from_u64(u64 src, u8 *dst)
+{
+        unsigned int i;
+
+        for (i = 0; i < ETH_ALEN; i++)
+                dst[ETH_ALEN - 1 - i] = (src >> (8 * i)) & 0xff;
+}
+
+static inline u64 b53_mac_to_u64(const u8 *src)
+{
+        unsigned int i;
+        u64 dst = 0;
+
+        for (i = 0; i < ETH_ALEN; i++)
+                dst |= (u64)src[ETH_ALEN - 1 - i] << (8 * i);
+
+        return dst;
+}
+
+static inline void b53_arl_to_entry(struct b53_arl_entry *ent,
+                                    u64 mac_vid, u32 fwd_entry)
+{
+        memset(ent, 0, sizeof(*ent));
+        ent->port = fwd_entry & ARLTBL_DATA_PORT_ID_MASK;
+        ent->is_valid = !!(fwd_entry & ARLTBL_VALID);
+        ent->is_age = !!(fwd_entry & ARLTBL_AGE);
+        ent->is_static = !!(fwd_entry & ARLTBL_STATIC);
+        b53_mac_from_u64(mac_vid, ent->mac);
+        ent->vid = mac_vid >> ARLTBL_VID_S;
+}
+
+static inline void b53_arl_from_entry(u64 *mac_vid, u32 *fwd_entry,
+                                      const struct b53_arl_entry *ent)
+{
+        *mac_vid = b53_mac_to_u64(ent->mac);
+        *mac_vid |= (u64)(ent->vid & ARLTBL_VID_MASK) << ARLTBL_VID_S;
+        *fwd_entry = ent->port & ARLTBL_DATA_PORT_ID_MASK;
+        if (ent->is_valid)
+                *fwd_entry |= ARLTBL_VALID;
+        if (ent->is_static)
+                *fwd_entry |= ARLTBL_STATIC;
+        if (ent->is_age)
+                *fwd_entry |= ARLTBL_AGE;
+}
+
+#ifdef CONFIG_BCM47XX
+
+#include <linux/version.h>
+#include <linux/bcm47xx_nvram.h>
+#include <bcm47xx_board.h>
+static inline int b53_switch_get_reset_gpio(struct b53_device *dev)
+{
+        enum bcm47xx_board board = bcm47xx_board_get();
+
+        switch (board) {
+        case BCM47XX_BOARD_LINKSYS_WRT300NV11:
+        case BCM47XX_BOARD_LINKSYS_WRT310NV1:
+                return 8;
+        default:
+                return bcm47xx_nvram_gpio_pin("robo_reset");
+        }
+}
+#else
+static inline int b53_switch_get_reset_gpio(struct b53_device *dev)
+{
+        return -ENOENT;
+}
+#endif
+#endif
diff --git a/drivers/net/dsa/b53/b53_regs.h b/drivers/net/dsa/b53/b53_regs.h
new file mode 100644
index 000000000000..8f12bddd5dc9
--- /dev/null
+++ b/drivers/net/dsa/b53/b53_regs.h
@@ -0,0 +1,434 @@
+/*
+ * B53 register definitions
+ *
+ * Copyright (C) 2004 Broadcom Corporation
+ * Copyright (C) 2011-2013 Jonas Gorski <jogo@openwrt.org>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef __B53_REGS_H
+#define __B53_REGS_H
+
+/* Management Port (SMP) Page offsets */
+#define B53_CTRL_PAGE                   0x00 /* Control */
+#define B53_STAT_PAGE                   0x01 /* Status */
+#define B53_MGMT_PAGE                   0x02 /* Management Mode */
+#define B53_MIB_AC_PAGE                 0x03 /* MIB Autocast */
+#define B53_ARLCTRL_PAGE                0x04 /* ARL Control */
+#define B53_ARLIO_PAGE                  0x05 /* ARL Access */
+#define B53_FRAMEBUF_PAGE               0x06 /* Management frame access */
+#define B53_MEM_ACCESS_PAGE             0x08 /* Memory access */
+
+/* PHY Registers */
+#define B53_PORT_MII_PAGE(i)            (0x10 + (i)) /* Port i MII Registers */
+#define B53_IM_PORT_PAGE                0x18 /* Inverse MII Port (to EMAC) */
+#define B53_ALL_PORT_PAGE               0x19 /* All ports MII (broadcast) */
+
+/* MIB registers */
+#define B53_MIB_PAGE(i)                 (0x20 + (i))
+
+/* Quality of Service (QoS) Registers */
+#define B53_QOS_PAGE                    0x30
+
+/* Port VLAN Page */
+#define B53_PVLAN_PAGE                  0x31
+
+/* VLAN Registers */
+#define B53_VLAN_PAGE                   0x34
+
+/* Jumbo Frame Registers */
+#define B53_JUMBO_PAGE                  0x40
+
+/* CFP Configuration Registers Page */
+#define B53_CFP_PAGE                    0xa1
+
+/*************************************************************************
+ * Control Page registers
+ *************************************************************************/
+
+/* Port Control Register (8 bit) */
+#define B53_PORT_CTRL(i)                (0x00 + (i))
+#define   PORT_CTRL_RX_DISABLE          BIT(0)
+#define   PORT_CTRL_TX_DISABLE          BIT(1)
+#define   PORT_CTRL_RX_BCST_EN          BIT(2) /* Broadcast RX (P8 only) */
+#define   PORT_CTRL_RX_MCST_EN          BIT(3) /* Multicast RX (P8 only) */
+#define   PORT_CTRL_RX_UCST_EN          BIT(4) /* Unicast RX (P8 only) */
+#define   PORT_CTRL_STP_STATE_S         5
+#define   PORT_CTRL_NO_STP              (0 << PORT_CTRL_STP_STATE_S)
+#define   PORT_CTRL_DIS_STATE           (1 << PORT_CTRL_STP_STATE_S)
+#define   PORT_CTRL_BLOCK_STATE         (2 << PORT_CTRL_STP_STATE_S)
+#define   PORT_CTRL_LISTEN_STATE        (3 << PORT_CTRL_STP_STATE_S)
+#define   PORT_CTRL_LEARN_STATE         (4 << PORT_CTRL_STP_STATE_S)
+#define   PORT_CTRL_FWD_STATE           (5 << PORT_CTRL_STP_STATE_S)
+#define   PORT_CTRL_STP_STATE_MASK      (0x7 << PORT_CTRL_STP_STATE_S)
+
+/* SMP Control Register (8 bit) */
+#define B53_SMP_CTRL                    0x0a
+
+/* Switch Mode Control Register (8 bit) */
+#define B53_SWITCH_MODE                 0x0b
+#define   SM_SW_FWD_MODE                BIT(0)  /* 1 = Managed Mode */
+#define   SM_SW_FWD_EN                  BIT(1)  /* Forwarding Enable */
+
+/* IMP Port state override register (8 bit) */
+#define B53_PORT_OVERRIDE_CTRL          0x0e
+#define   PORT_OVERRIDE_LINK            BIT(0)
+#define   PORT_OVERRIDE_FULL_DUPLEX     BIT(1) /* 0 = Half Duplex */
+#define   PORT_OVERRIDE_SPEED_S         2
+#define   PORT_OVERRIDE_SPEED_10M       (0 << PORT_OVERRIDE_SPEED_S)
+#define   PORT_OVERRIDE_SPEED_100M      (1 << PORT_OVERRIDE_SPEED_S)
+#define   PORT_OVERRIDE_SPEED_1000M     (2 << PORT_OVERRIDE_SPEED_S)
+#define   PORT_OVERRIDE_RV_MII_25       BIT(4) /* BCM5325 only */
+#define   PORT_OVERRIDE_RX_FLOW         BIT(4)
+#define   PORT_OVERRIDE_TX_FLOW         BIT(5)
+#define   PORT_OVERRIDE_SPEED_2000M     BIT(6) /* BCM5301X only, requires setting 1000M */
+#define   PORT_OVERRIDE_EN              BIT(7) /* Use the register contents */
+
+/* Power-down mode control */
+#define B53_PD_MODE_CTRL_25             0x0f
+
+/* IP Multicast control (8 bit) */
+#define B53_IP_MULTICAST_CTRL           0x21
+#define  B53_IPMC_FWD_EN                BIT(1)
+#define  B53_UC_FWD_EN                  BIT(6)
+#define  B53_MC_FWD_EN                  BIT(7)
+
+/* (16 bit) */
+#define B53_UC_FLOOD_MASK               0x32
+#define B53_MC_FLOOD_MASK               0x34
+#define B53_IPMC_FLOOD_MASK             0x36
+
+/*
+ * Override Ports 0-7 State on devices with xMII interfaces (8 bit)
+ *
+ * For port 8 still use B53_PORT_OVERRIDE_CTRL
+ * Please note that not all ports are available on every hardware, e.g. BCM5301X
+ * don't include overriding port 6, BCM63xx also have some limitations.
+ */
+#define B53_GMII_PORT_OVERRIDE_CTRL(i)  (0x58 + (i))
+#define   GMII_PO_LINK                  BIT(0)
+#define   GMII_PO_FULL_DUPLEX           BIT(1) /* 0 = Half Duplex */
+#define   GMII_PO_SPEED_S               2
+#define   GMII_PO_SPEED_10M             (0 << GMII_PO_SPEED_S)
+#define   GMII_PO_SPEED_100M            (1 << GMII_PO_SPEED_S)
+#define   GMII_PO_SPEED_1000M           (2 << GMII_PO_SPEED_S)
+#define   GMII_PO_RX_FLOW               BIT(4)
+#define   GMII_PO_TX_FLOW               BIT(5)
+#define   GMII_PO_EN                    BIT(6) /* Use the register contents */
+#define   GMII_PO_SPEED_2000M           BIT(7) /* BCM5301X only, requires setting 1000M */
+
+#define B53_RGMII_CTRL_IMP              0x60
+#define   RGMII_CTRL_ENABLE_GMII        BIT(7)
+#define   RGMII_CTRL_TIMING_SEL         BIT(2)
+#define   RGMII_CTRL_DLL_RXC            BIT(1)
+#define   RGMII_CTRL_DLL_TXC            BIT(0)
+
+#define B53_RGMII_CTRL_P(i)             (B53_RGMII_CTRL_IMP + (i))
+
+/* Software reset register (8 bit) */
+#define B53_SOFTRESET                   0x79
+#define   SW_RST                        BIT(7)
+#define   EN_SW_RST                     BIT(4)
+
+/* Fast Aging Control register (8 bit) */
+#define B53_FAST_AGE_CTRL               0x88
+#define   FAST_AGE_STATIC               BIT(0)
+#define   FAST_AGE_DYNAMIC              BIT(1)
+#define   FAST_AGE_PORT                 BIT(2)
+#define   FAST_AGE_VLAN                 BIT(3)
+#define   FAST_AGE_STP                  BIT(4)
+#define   FAST_AGE_MC                   BIT(5)
+#define   FAST_AGE_DONE                 BIT(7)
+
+/* Fast Aging Port Control register (8 bit) */
+#define B53_FAST_AGE_PORT_CTRL          0x89
+
+/* Fast Aging VID Control register (16 bit) */
+#define B53_FAST_AGE_VID_CTRL           0x8a
+
+/*************************************************************************
+ * Status Page registers
+ *************************************************************************/
+
+/* Link Status Summary Register (16bit) */
+#define B53_LINK_STAT                   0x00
+
+/* Link Status Change Register (16 bit) */
+#define B53_LINK_STAT_CHANGE            0x02
+
+/* Port Speed Summary Register (16 bit for FE, 32 bit for GE) */
+#define B53_SPEED_STAT                  0x04
+#define  SPEED_PORT_FE(reg, port)       (((reg) >> (port)) & 1)
+#define  SPEED_PORT_GE(reg, port)       (((reg) >> 2 * (port)) & 3)
+#define  SPEED_STAT_10M                 0
+#define  SPEED_STAT_100M                1
+#define  SPEED_STAT_1000M               2
+
+/* Duplex Status Summary (16 bit) */
+#define B53_DUPLEX_STAT_FE              0x06
+#define B53_DUPLEX_STAT_GE              0x08
+#define B53_DUPLEX_STAT_63XX            0x0c
+
+/* Revision ID register for BCM5325 */
+#define B53_REV_ID_25                   0x50
+
+/* Strap Value (48 bit) */
+#define B53_STRAP_VALUE                 0x70
+#define   SV_GMII_CTRL_115              BIT(27)
+
+/*************************************************************************
+ * Management Mode Page Registers
+ *************************************************************************/
+
+/* Global Management Config Register (8 bit) */
+#define B53_GLOBAL_CONFIG               0x00
+#define   GC_RESET_MIB                  0x01
+#define   GC_RX_BPDU_EN                 0x02
+#define   GC_MIB_AC_HDR_EN              0x10
+#define   GC_MIB_AC_EN                  0x20
+#define   GC_FRM_MGMT_PORT_M            0xC0
+#define   GC_FRM_MGMT_PORT_04           0x00
+#define   GC_FRM_MGMT_PORT_MII          0x80
+
+/* Broadcom Header control register (8 bit) */
+#define B53_BRCM_HDR                    0x03
+#define   BRCM_HDR_P8_EN                BIT(0) /* Enable tagging on port 8 */
+#define   BRCM_HDR_P5_EN                BIT(1) /* Enable tagging on port 5 */
+
+/* Device ID register (8 or 32 bit) */
+#define B53_DEVICE_ID                   0x30
+
+/* Revision ID register (8 bit) */
+#define B53_REV_ID                      0x40
+
+/*************************************************************************
+ * ARL Access Page Registers
+ *************************************************************************/
+
+/* VLAN Table Access Register (8 bit) */
+#define B53_VT_ACCESS                   0x80
+#define B53_VT_ACCESS_9798              0x60 /* for BCM5397/BCM5398 */
+#define B53_VT_ACCESS_63XX              0x60 /* for BCM6328/62/68 */
+#define   VTA_CMD_WRITE                 0
+#define   VTA_CMD_READ                  1
+#define   VTA_CMD_CLEAR                 2
+#define   VTA_START_CMD                 BIT(7)
+
+/* VLAN Table Index Register (16 bit) */
+#define B53_VT_INDEX                    0x81
+#define B53_VT_INDEX_9798               0x61
+#define B53_VT_INDEX_63XX               0x62
+
+/* VLAN Table Entry Register (32 bit) */
+#define B53_VT_ENTRY                    0x83
+#define B53_VT_ENTRY_9798               0x63
+#define B53_VT_ENTRY_63XX               0x64
+#define   VTE_MEMBERS                   0x1ff
+#define   VTE_UNTAG_S                   9
+#define   VTE_UNTAG                     (0x1ff << 9)
+
+/*************************************************************************
+ * ARL I/O Registers
+ *************************************************************************/
+
+/* ARL Table Read/Write Register (8 bit) */
+#define B53_ARLTBL_RW_CTRL              0x00
+#define    ARLTBL_RW                    BIT(0)
+#define    ARLTBL_START_DONE            BIT(7)
+
+/* MAC Address Index Register (48 bit) */
+#define B53_MAC_ADDR_IDX                0x02
+
+/* VLAN ID Index Register (16 bit) */
+#define B53_VLAN_ID_IDX                 0x08
+
+/* ARL Table MAC/VID Entry N Registers (64 bit)
+ *
+ * BCM5325 and BCM5365 share most definitions below
+ */
+#define B53_ARLTBL_MAC_VID_ENTRY(n)     (0x10 * (n))
+#define   ARLTBL_MAC_MASK               0xffffffffffff
+#define   ARLTBL_VID_S                  48
+#define   ARLTBL_VID_MASK_25            0xff
+#define   ARLTBL_VID_MASK               0xfff
+#define   ARLTBL_DATA_PORT_ID_S_25      48
+#define   ARLTBL_DATA_PORT_ID_MASK_25   0xf
+#define   ARLTBL_AGE_25                 BIT(61)
+#define   ARLTBL_STATIC_25              BIT(62)
+#define   ARLTBL_VALID_25               BIT(63)
+
+/* ARL Table Data Entry N Registers (32 bit) */
+#define B53_ARLTBL_DATA_ENTRY(n)        ((0x10 * (n)) + 0x08)
+#define   ARLTBL_DATA_PORT_ID_MASK      0x1ff
+#define   ARLTBL_TC(tc)                 ((3 & tc) << 11)
+#define   ARLTBL_AGE                    BIT(14)
+#define   ARLTBL_STATIC                 BIT(15)
+#define   ARLTBL_VALID                  BIT(16)
+
+/* ARL Search Control Register (8 bit) */
+#define B53_ARL_SRCH_CTL                0x50
+#define B53_ARL_SRCH_CTL_25             0x20
+#define   ARL_SRCH_VLID                 BIT(0)
+#define   ARL_SRCH_STDN                 BIT(7)
+
+/* ARL Search Address Register (16 bit) */
+#define B53_ARL_SRCH_ADDR               0x51
+#define B53_ARL_SRCH_ADDR_25            0x22
+#define B53_ARL_SRCH_ADDR_65            0x24
+#define  ARL_ADDR_MASK                  GENMASK(14, 0)
+
+/* ARL Search MAC/VID Result (64 bit) */
+#define B53_ARL_SRCH_RSTL_0_MACVID      0x60
+
+/* Single register search result on 5325 */
+#define B53_ARL_SRCH_RSTL_0_MACVID_25   0x24
+/* Single register search result on 5365 */
+#define B53_ARL_SRCH_RSTL_0_MACVID_65   0x30
+
+/* ARL Search Data Result (32 bit) */
+#define B53_ARL_SRCH_RSTL_0             0x68
+
+#define B53_ARL_SRCH_RSTL_MACVID(x)     (B53_ARL_SRCH_RSTL_0_MACVID + ((x) * 0x10))
+#define B53_ARL_SRCH_RSTL(x)            (B53_ARL_SRCH_RSTL_0 + ((x) * 0x10))
+
+/*************************************************************************
+ * Port VLAN Registers
+ *************************************************************************/
+
+/* Port VLAN mask (16 bit) IMP port is always 8, also on 5325 & co */
+#define B53_PVLAN_PORT_MASK(i)          ((i) * 2)
+
+/*************************************************************************
+ * 802.1Q Page Registers
+ *************************************************************************/
+
+/* Global QoS Control (8 bit) */
+#define B53_QOS_GLOBAL_CTL              0x00
+
+/* Enable 802.1Q for individual Ports (16 bit) */
+#define B53_802_1P_EN                   0x04
+
+/*************************************************************************
+ * VLAN Page Registers
+ *************************************************************************/
+
+/* VLAN Control 0 (8 bit) */
+#define B53_VLAN_CTRL0                  0x00
+#define   VC0_8021PF_CTRL_MASK          0x3
+#define   VC0_8021PF_CTRL_NONE          0x0
+#define   VC0_8021PF_CTRL_CHANGE_PRI    0x1
+#define   VC0_8021PF_CTRL_CHANGE_VID    0x2
+#define   VC0_8021PF_CTRL_CHANGE_BOTH   0x3
+#define   VC0_8021QF_CTRL_MASK          0xc
+#define   VC0_8021QF_CTRL_CHANGE_PRI    0x1
+#define   VC0_8021QF_CTRL_CHANGE_VID    0x2
+#define   VC0_8021QF_CTRL_CHANGE_BOTH   0x3
+#define   VC0_RESERVED_1                BIT(1)
+#define   VC0_DROP_VID_MISS             BIT(4)
+#define   VC0_VID_HASH_VID              BIT(5)
+#define   VC0_VID_CHK_EN                BIT(6)  /* Use VID,DA or VID,SA */
+#define   VC0_VLAN_EN                   BIT(7)  /* 802.1Q VLAN Enabled */
+
+/* VLAN Control 1 (8 bit) */
+#define B53_VLAN_CTRL1                  0x01
+#define   VC1_RX_MCST_TAG_EN            BIT(1)
+#define   VC1_RX_MCST_FWD_EN            BIT(2)
+#define   VC1_RX_MCST_UNTAG_EN          BIT(3)
+
+/* VLAN Control 2 (8 bit) */
+#define B53_VLAN_CTRL2                  0x02
+
+/* VLAN Control 3 (8 bit when BCM5325, 16 bit else) */
+#define B53_VLAN_CTRL3                  0x03
+#define B53_VLAN_CTRL3_63XX             0x04
+#define   VC3_MAXSIZE_1532              BIT(6) /* 5325 only */
+#define   VC3_HIGH_8BIT_EN              BIT(7) /* 5325 only */
+
+/* VLAN Control 4 (8 bit) */
+#define B53_VLAN_CTRL4                  0x05
+#define B53_VLAN_CTRL4_25               0x04
+#define B53_VLAN_CTRL4_63XX             0x06
+#define   VC4_ING_VID_CHECK_S           6
+#define   VC4_ING_VID_CHECK_MASK        (0x3 << VC4_ING_VID_CHECK_S)
+#define   VC4_ING_VID_VIO_FWD           0 /* forward, but do not learn */
+#define   VC4_ING_VID_VIO_DROP          1 /* drop VID violations */
+#define   VC4_NO_ING_VID_CHK            2 /* do not check */
+#define   VC4_ING_VID_VIO_TO_IMP        3 /* redirect to MII port */
+
+/* VLAN Control 5 (8 bit) */
+#define B53_VLAN_CTRL5                  0x06
+#define B53_VLAN_CTRL5_25               0x05
+#define B53_VLAN_CTRL5_63XX             0x07
+#define   VC5_VID_FFF_EN                BIT(2)
+#define   VC5_DROP_VTABLE_MISS          BIT(3)
+
+/* VLAN Control 6 (8 bit) */
+#define B53_VLAN_CTRL6                  0x07
+#define B53_VLAN_CTRL6_63XX             0x08
+
+/* VLAN Table Access Register (16 bit) */
+#define B53_VLAN_TABLE_ACCESS_25        0x06    /* BCM5325E/5350 */
+#define B53_VLAN_TABLE_ACCESS_65        0x08    /* BCM5365 */
+#define   VTA_VID_LOW_MASK_25           0xf
+#define   VTA_VID_LOW_MASK_65           0xff
+#define   VTA_VID_HIGH_S_25             4
+#define   VTA_VID_HIGH_S_65             8
+#define   VTA_VID_HIGH_MASK_25          (0xff << VTA_VID_HIGH_S_25E)
+#define   VTA_VID_HIGH_MASK_65          (0xf << VTA_VID_HIGH_S_65)
+#define   VTA_RW_STATE                  BIT(12)
+#define   VTA_RW_STATE_RD               0
+#define   VTA_RW_STATE_WR               BIT(12)
+#define   VTA_RW_OP_EN                  BIT(13)
+
+/* VLAN Read/Write Registers for (16/32 bit) */
+#define B53_VLAN_WRITE_25               0x08
+#define B53_VLAN_WRITE_65               0x0a
+#define B53_VLAN_READ                   0x0c
+#define   VA_MEMBER_MASK                0x3f
+#define   VA_UNTAG_S_25                 6
+#define   VA_UNTAG_MASK_25              0x3f
+#define   VA_UNTAG_S_65                 7
+#define   VA_UNTAG_MASK_65              0x1f
+#define   VA_VID_HIGH_S                 12
+#define   VA_VID_HIGH_MASK              (0xffff << VA_VID_HIGH_S)
+#define   VA_VALID_25                   BIT(20)
+#define   VA_VALID_25_R4                BIT(24)
+#define   VA_VALID_65                   BIT(14)
+
+/* VLAN Port Default Tag (16 bit) */
+#define B53_VLAN_PORT_DEF_TAG(i)        (0x10 + 2 * (i))
+
+/*************************************************************************
+ * Jumbo Frame Page Registers
+ *************************************************************************/
+
+/* Jumbo Enable Port Mask (bit i == port i enabled) (32 bit) */
+#define B53_JUMBO_PORT_MASK             0x01
+#define B53_JUMBO_PORT_MASK_63XX        0x04
+#define   JPM_10_100_JUMBO_EN           BIT(24) /* GigE always enabled */
+
+/* Good Frame Max Size without 802.1Q TAG (16 bit) */
+#define B53_JUMBO_MAX_SIZE              0x05
+#define B53_JUMBO_MAX_SIZE_63XX         0x08
+#define   JMS_MIN_SIZE                  1518
+#define   JMS_MAX_SIZE                  9724
+
+/*************************************************************************
+ * CFP Configuration Page Registers
+ *************************************************************************/
+
+/* CFP Control Register with ports map (8 bit) */
+#define B53_CFP_CTRL                    0x00
+
+#endif /* !__B53_REGS_H */
diff --git a/drivers/net/dsa/b53/b53_spi.c b/drivers/net/dsa/b53/b53_spi.c
new file mode 100644
index 000000000000..2bda0b5f1578
--- /dev/null
+++ b/drivers/net/dsa/b53/b53_spi.c
@@ -0,0 +1,331 @@
+/*
+ * B53 register access through SPI
+ *
+ * Copyright (C) 2011-2013 Jonas Gorski <jogo@openwrt.org>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <asm/unaligned.h>
+
+#include <linux/delay.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/spi/spi.h>
+#include <linux/platform_data/b53.h>
+
+#include "b53_priv.h"
+
+#define B53_SPI_DATA            0xf0
+
+#define B53_SPI_STATUS          0xfe
+#define B53_SPI_CMD_SPIF        BIT(7)
+#define B53_SPI_CMD_RACK        BIT(5)
+
+#define B53_SPI_CMD_READ        0x00
+#define B53_SPI_CMD_WRITE       0x01
+#define B53_SPI_CMD_NORMAL      0x60
+#define B53_SPI_CMD_FAST        0x10
+
+#define B53_SPI_PAGE_SELECT     0xff
+
+static inline int b53_spi_read_reg(struct spi_device *spi, u8 reg, u8 *val,
+                                   unsigned int len)
+{
+        u8 txbuf[2];
+
+        txbuf[0] = B53_SPI_CMD_NORMAL | B53_SPI_CMD_READ;
+        txbuf[1] = reg;
+
+        return spi_write_then_read(spi, txbuf, 2, val, len);
+}
+
+static inline int b53_spi_clear_status(struct spi_device *spi)
+{
+        unsigned int i;
+        u8 rxbuf;
+        int ret;
+
+        for (i = 0; i < 10; i++) {
+                ret = b53_spi_read_reg(spi, B53_SPI_STATUS, &rxbuf, 1);
+                if (ret)
+                        return ret;
+
+                if (!(rxbuf & B53_SPI_CMD_SPIF))
+                        break;
+
+                mdelay(1);
+        }
+
+        if (i == 10)
+                return -EIO;
+
+        return 0;
+}
+
+static inline int b53_spi_set_page(struct spi_device *spi, u8 page)
+{
+        u8 txbuf[3];
+
+        txbuf[0] = B53_SPI_CMD_NORMAL | B53_SPI_CMD_WRITE;
+        txbuf[1] = B53_SPI_PAGE_SELECT;
+        txbuf[2] = page;
+
+        return spi_write(spi, txbuf, sizeof(txbuf));
+}
+
+static inline int b53_prepare_reg_access(struct spi_device *spi, u8 page)
+{
+        int ret = b53_spi_clear_status(spi);
+
+        if (ret)
+                return ret;
+
+        return b53_spi_set_page(spi, page);
+}
+
+static int b53_spi_prepare_reg_read(struct spi_device *spi, u8 reg)
+{
+        u8 rxbuf;
+        int retry_count;
+        int ret;
+
+        ret = b53_spi_read_reg(spi, reg, &rxbuf, 1);
+        if (ret)
+                return ret;
+
+        for (retry_count = 0; retry_count < 10; retry_count++) {
+                ret = b53_spi_read_reg(spi, B53_SPI_STATUS, &rxbuf, 1);
+                if (ret)
+                        return ret;
+
+                if (rxbuf & B53_SPI_CMD_RACK)
+                        break;
+
+                mdelay(1);
+        }
+
+        if (retry_count == 10)
+                return -EIO;
+
+        return 0;
+}
+
+static int b53_spi_read(struct b53_device *dev, u8 page, u8 reg, u8 *data,
+                        unsigned int len)
+{
+        struct spi_device *spi = dev->priv;
+        int ret;
+
+        ret = b53_prepare_reg_access(spi, page);
+        if (ret)
+                return ret;
+
+        ret = b53_spi_prepare_reg_read(spi, reg);
+        if (ret)
+                return ret;
+
+        return b53_spi_read_reg(spi, B53_SPI_DATA, data, len);
+}
+
+static int b53_spi_read8(struct b53_device *dev, u8 page, u8 reg, u8 *val)
+{
+        return b53_spi_read(dev, page, reg, val, 1);
+}
+
+static int b53_spi_read16(struct b53_device *dev, u8 page, u8 reg, u16 *val)
+{
+        int ret = b53_spi_read(dev, page, reg, (u8 *)val, 2);
+
+        if (!ret)
+                *val = le16_to_cpu(*val);
+
+        return ret;
+}
+
+static int b53_spi_read32(struct b53_device *dev, u8 page, u8 reg, u32 *val)
+{
+        int ret = b53_spi_read(dev, page, reg, (u8 *)val, 4);
+
+        if (!ret)
+                *val = le32_to_cpu(*val);
+
+        return ret;
+}
+
+static int b53_spi_read48(struct b53_device *dev, u8 page, u8 reg, u64 *val)
+{
+        int ret;
+
+        *val = 0;
+        ret = b53_spi_read(dev, page, reg, (u8 *)val, 6);
+        if (!ret)
+                *val = le64_to_cpu(*val);
+
+        return ret;
+}
+
+static int b53_spi_read64(struct b53_device *dev, u8 page, u8 reg, u64 *val)
+{
+        int ret = b53_spi_read(dev, page, reg, (u8 *)val, 8);
+
+        if (!ret)
+                *val = le64_to_cpu(*val);
+
+        return ret;
+}
+
+static int b53_spi_write8(struct b53_device *dev, u8 page, u8 reg, u8 value)
+{
+        struct spi_device *spi = dev->priv;
+        int ret;
+        u8 txbuf[3];
+
+        ret = b53_prepare_reg_access(spi, page);
+        if (ret)
+                return ret;
+
+        txbuf[0] = B53_SPI_CMD_NORMAL | B53_SPI_CMD_WRITE;
+        txbuf[1] = reg;
+        txbuf[2] = value;
+
+        return spi_write(spi, txbuf, sizeof(txbuf));
+}
+
+static int b53_spi_write16(struct b53_device *dev, u8 page, u8 reg, u16 value)
+{
+        struct spi_device *spi = dev->priv;
+        int ret;
+        u8 txbuf[4];
+
+        ret = b53_prepare_reg_access(spi, page);
+        if (ret)
+                return ret;
+
+        txbuf[0] = B53_SPI_CMD_NORMAL | B53_SPI_CMD_WRITE;
+        txbuf[1] = reg;
+        put_unaligned_le16(value, &txbuf[2]);
+
+        return spi_write(spi, txbuf, sizeof(txbuf));
+}
+
+static int b53_spi_write32(struct b53_device *dev, u8 page, u8 reg, u32 value)
+{
+        struct spi_device *spi = dev->priv;
+        int ret;
+        u8 txbuf[6];
+
+        ret = b53_prepare_reg_access(spi, page);
+        if (ret)
+                return ret;
+
+        txbuf[0] = B53_SPI_CMD_NORMAL | B53_SPI_CMD_WRITE;
+        txbuf[1] = reg;
+        put_unaligned_le32(value, &txbuf[2]);
+
+        return spi_write(spi, txbuf, sizeof(txbuf));
+}
+
+static int b53_spi_write48(struct b53_device *dev, u8 page, u8 reg, u64 value)
+{
+        struct spi_device *spi = dev->priv;
+        int ret;
+        u8 txbuf[10];
+
+        ret = b53_prepare_reg_access(spi, page);
+        if (ret)
+                return ret;
+
+        txbuf[0] = B53_SPI_CMD_NORMAL | B53_SPI_CMD_WRITE;
+        txbuf[1] = reg;
+        put_unaligned_le64(value, &txbuf[2]);
+
+        return spi_write(spi, txbuf, sizeof(txbuf) - 2);
+}
+
+static int b53_spi_write64(struct b53_device *dev, u8 page, u8 reg, u64 value)
+{
+        struct spi_device *spi = dev->priv;
+        int ret;
+        u8 txbuf[10];
+
+        ret = b53_prepare_reg_access(spi, page);
+        if (ret)
+                return ret;
+
+        txbuf[0] = B53_SPI_CMD_NORMAL | B53_SPI_CMD_WRITE;
+        txbuf[1] = reg;
+        put_unaligned_le64(value, &txbuf[2]);
+
+        return spi_write(spi, txbuf, sizeof(txbuf));
+}
+
+static struct b53_io_ops b53_spi_ops = {
+        .read8 = b53_spi_read8,
+        .read16 = b53_spi_read16,
+        .read32 = b53_spi_read32,
+        .read48 = b53_spi_read48,
+        .read64 = b53_spi_read64,
+        .write8 = b53_spi_write8,
+        .write16 = b53_spi_write16,
+        .write32 = b53_spi_write32,
+        .write48 = b53_spi_write48,
+        .write64 = b53_spi_write64,
+};
+
+static int b53_spi_probe(struct spi_device *spi)
+{
+        struct b53_device *dev;
+        int ret;
+
+        dev = b53_switch_alloc(&spi->dev, &b53_spi_ops, spi);
+        if (!dev)
+                return -ENOMEM;
+
+        if (spi->dev.platform_data)
+                dev->pdata = spi->dev.platform_data;
+
+        ret = b53_switch_register(dev);
+        if (ret)
+                return ret;
+
+        spi_set_drvdata(spi, dev);
+
+        return 0;
+}
+
+static int b53_spi_remove(struct spi_device *spi)
+{
+        struct b53_device *dev = spi_get_drvdata(spi);
+
+        if (dev)
+                b53_switch_remove(dev);
+
+        return 0;
+}
+
+static struct spi_driver b53_spi_driver = {
+        .driver = {
+                .name   = "b53-switch",
+                .bus    = &spi_bus_type,
+                .owner  = THIS_MODULE,
+        },
+        .probe  = b53_spi_probe,
+        .remove = b53_spi_remove,
+};
+
+module_spi_driver(b53_spi_driver);
+
+MODULE_AUTHOR("Jonas Gorski <jogo@openwrt.org>");
+MODULE_DESCRIPTION("B53 SPI access driver");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/net/dsa/b53/b53_srab.c b/drivers/net/dsa/b53/b53_srab.c
new file mode 100644
index 000000000000..70fd47284535
--- /dev/null
+++ b/drivers/net/dsa/b53/b53_srab.c
@@ -0,0 +1,415 @@
+/*
+ * B53 register access through Switch Register Access Bridge Registers
+ *
+ * Copyright (C) 2013 Hauke Mehrtens <hauke@hauke-m.de>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/delay.h>
+#include <linux/platform_device.h>
+#include <linux/platform_data/b53.h>
+
+#include "b53_priv.h"
+
+/* command and status register of the SRAB */
+#define B53_SRAB_CMDSTAT                0x2c
+#define  B53_SRAB_CMDSTAT_RST           BIT(2)
+#define  B53_SRAB_CMDSTAT_WRITE         BIT(1)
+#define  B53_SRAB_CMDSTAT_GORDYN        BIT(0)
+#define  B53_SRAB_CMDSTAT_PAGE          24
+#define  B53_SRAB_CMDSTAT_REG           16
+
+/* high order word of write data to switch registe */
+#define B53_SRAB_WD_H                   0x30
+
+/* low order word of write data to switch registe */
+#define B53_SRAB_WD_L                   0x34
+
+/* high order word of read data from switch register */
+#define B53_SRAB_RD_H                   0x38
+
+/* low order word of read data from switch register */
+#define B53_SRAB_RD_L                   0x3c
+
+/* command and status register of the SRAB */
+#define B53_SRAB_CTRLS                  0x40
+#define  B53_SRAB_CTRLS_RCAREQ          BIT(3)
+#define  B53_SRAB_CTRLS_RCAGNT          BIT(4)
+#define  B53_SRAB_CTRLS_SW_INIT_DONE    BIT(6)
+
+/* the register captures interrupt pulses from the switch */
+#define B53_SRAB_INTR                   0x44
+#define  B53_SRAB_INTR_P(x)             BIT(x)
+#define  B53_SRAB_SWITCH_PHY            BIT(8)
+#define  B53_SRAB_1588_SYNC             BIT(9)
+#define  B53_SRAB_IMP1_SLEEP_TIMER      BIT(10)
+#define  B53_SRAB_P7_SLEEP_TIMER        BIT(11)
+#define  B53_SRAB_IMP0_SLEEP_TIMER      BIT(12)
+
+struct b53_srab_priv {
+        void __iomem *regs;
+};
+
+static int b53_srab_request_grant(struct b53_device *dev)
+{
+        struct b53_srab_priv *priv = dev->priv;
+        u8 __iomem *regs = priv->regs;
+        u32 ctrls;
+        int i;
+
+        ctrls = readl(regs + B53_SRAB_CTRLS);
+        ctrls |= B53_SRAB_CTRLS_RCAREQ;
+        writel(ctrls, regs + B53_SRAB_CTRLS);
+
+        for (i = 0; i < 20; i++) {
+                ctrls = readl(regs + B53_SRAB_CTRLS);
+                if (ctrls & B53_SRAB_CTRLS_RCAGNT)
+                        break;
+                usleep_range(10, 100);
+        }
+        if (WARN_ON(i == 5))
+                return -EIO;
+
+        return 0;
+}
+
+static void b53_srab_release_grant(struct b53_device *dev)
+{
+        struct b53_srab_priv *priv = dev->priv;
+        u8 __iomem *regs = priv->regs;
+        u32 ctrls;
+
+        ctrls = readl(regs + B53_SRAB_CTRLS);
+        ctrls &= ~B53_SRAB_CTRLS_RCAREQ;
+        writel(ctrls, regs + B53_SRAB_CTRLS);
+}
+
+static int b53_srab_op(struct b53_device *dev, u8 page, u8 reg, u32 op)
+{
+        struct b53_srab_priv *priv = dev->priv;
+        u8 __iomem *regs = priv->regs;
+        int i;
+        u32 cmdstat;
+
+        /* set register address */
+        cmdstat = (page << B53_SRAB_CMDSTAT_PAGE) |
+                  (reg << B53_SRAB_CMDSTAT_REG) |
+                  B53_SRAB_CMDSTAT_GORDYN |
+                  op;
+        writel(cmdstat, regs + B53_SRAB_CMDSTAT);
+
+        /* check if operation completed */
+        for (i = 0; i < 5; ++i) {
+                cmdstat = readl(regs + B53_SRAB_CMDSTAT);
+                if (!(cmdstat & B53_SRAB_CMDSTAT_GORDYN))
+                        break;
+                usleep_range(10, 100);
+        }
+
+        if (WARN_ON(i == 5))
+                return -EIO;
+
+        return 0;
+}
+
+static int b53_srab_read8(struct b53_device *dev, u8 page, u8 reg, u8 *val)
+{
+        struct b53_srab_priv *priv = dev->priv;
+        u8 __iomem *regs = priv->regs;
+        int ret = 0;
+
+        ret = b53_srab_request_grant(dev);
+        if (ret)
+                goto err;
+
+        ret = b53_srab_op(dev, page, reg, 0);
+        if (ret)
+                goto err;
+
+        *val = readl(regs + B53_SRAB_RD_L) & 0xff;
+
+err:
+        b53_srab_release_grant(dev);
+
+        return ret;
+}
+
+static int b53_srab_read16(struct b53_device *dev, u8 page, u8 reg, u16 *val)
+{
+        struct b53_srab_priv *priv = dev->priv;
+        u8 __iomem *regs = priv->regs;
+        int ret = 0;
+
+        ret = b53_srab_request_grant(dev);
+        if (ret)
+                goto err;
+
+        ret = b53_srab_op(dev, page, reg, 0);
+        if (ret)
+                goto err;
+
+        *val = readl(regs + B53_SRAB_RD_L) & 0xffff;
+
+err:
+        b53_srab_release_grant(dev);
+
+        return ret;
+}
+
+static int b53_srab_read32(struct b53_device *dev, u8 page, u8 reg, u32 *val)
+{
+        struct b53_srab_priv *priv = dev->priv;
+        u8 __iomem *regs = priv->regs;
+        int ret = 0;
+
+        ret = b53_srab_request_grant(dev);
+        if (ret)
+                goto err;
+
+        ret = b53_srab_op(dev, page, reg, 0);
+        if (ret)
+                goto err;
+
+        *val = readl(regs + B53_SRAB_RD_L);
+
+err:
+        b53_srab_release_grant(dev);
+
+        return ret;
+}
+
+static int b53_srab_read48(struct b53_device *dev, u8 page, u8 reg, u64 *val)
+{
+        struct b53_srab_priv *priv = dev->priv;
+        u8 __iomem *regs = priv->regs;
+        int ret = 0;
+
+        ret = b53_srab_request_grant(dev);
+        if (ret)
+                goto err;
+
+        ret = b53_srab_op(dev, page, reg, 0);
+        if (ret)
+                goto err;
+
+        *val = readl(regs + B53_SRAB_RD_L);
+        *val += ((u64)readl(regs + B53_SRAB_RD_H) & 0xffff) << 32;
+
+err:
+        b53_srab_release_grant(dev);
+
+        return ret;
+}
+
+static int b53_srab_read64(struct b53_device *dev, u8 page, u8 reg, u64 *val)
+{
+        struct b53_srab_priv *priv = dev->priv;
+        u8 __iomem *regs = priv->regs;
+        int ret = 0;
+
+        ret = b53_srab_request_grant(dev);
+        if (ret)
+                goto err;
+
+        ret = b53_srab_op(dev, page, reg, 0);
+        if (ret)
+                goto err;
+
+        *val = readl(regs + B53_SRAB_RD_L);
+        *val += (u64)readl(regs + B53_SRAB_RD_H) << 32;
+
+err:
+        b53_srab_release_grant(dev);
+
+        return ret;
+}
+
+static int b53_srab_write8(struct b53_device *dev, u8 page, u8 reg, u8 value)
+{
+        struct b53_srab_priv *priv = dev->priv;
+        u8 __iomem *regs = priv->regs;
+        int ret = 0;
+
+        ret = b53_srab_request_grant(dev);
+        if (ret)
+                goto err;
+
+        writel(value, regs + B53_SRAB_WD_L);
+
+        ret = b53_srab_op(dev, page, reg, B53_SRAB_CMDSTAT_WRITE);
+
+err:
+        b53_srab_release_grant(dev);
+
+        return ret;
+}
+
+static int b53_srab_write16(struct b53_device *dev, u8 page, u8 reg,
+                            u16 value)
+{
+        struct b53_srab_priv *priv = dev->priv;
+        u8 __iomem *regs = priv->regs;
+        int ret = 0;
+
+        ret = b53_srab_request_grant(dev);
+        if (ret)
+                goto err;
+
+        writel(value, regs + B53_SRAB_WD_L);
+
+        ret = b53_srab_op(dev, page, reg, B53_SRAB_CMDSTAT_WRITE);
+
+err:
+        b53_srab_release_grant(dev);
+
+        return ret;
+}
+
+static int b53_srab_write32(struct b53_device *dev, u8 page, u8 reg,
+                            u32 value)
+{
+        struct b53_srab_priv *priv = dev->priv;
+        u8 __iomem *regs = priv->regs;
+        int ret = 0;
+
+        ret = b53_srab_request_grant(dev);
+        if (ret)
+                goto err;
+
+        writel(value, regs + B53_SRAB_WD_L);
+
+        ret = b53_srab_op(dev, page, reg, B53_SRAB_CMDSTAT_WRITE);
+
+err:
+        b53_srab_release_grant(dev);
+
+        return ret;
+}
+
+static int b53_srab_write48(struct b53_device *dev, u8 page, u8 reg,
+                            u64 value)
+{
+        struct b53_srab_priv *priv = dev->priv;
+        u8 __iomem *regs = priv->regs;
+        int ret = 0;
+
+        ret = b53_srab_request_grant(dev);
+        if (ret)
+                goto err;
+
+        writel((u32)value, regs + B53_SRAB_WD_L);
+        writel((u16)(value >> 32), regs + B53_SRAB_WD_H);
+
+        ret = b53_srab_op(dev, page, reg, B53_SRAB_CMDSTAT_WRITE);
+
+err:
+        b53_srab_release_grant(dev);
+
+        return ret;
+}
+
+static int b53_srab_write64(struct b53_device *dev, u8 page, u8 reg,
+                            u64 value)
+{
+        struct b53_srab_priv *priv = dev->priv;
+        u8 __iomem *regs = priv->regs;
+        int ret = 0;
+
+        ret = b53_srab_request_grant(dev);
+        if (ret)
+                goto err;
+
+        writel((u32)value, regs + B53_SRAB_WD_L);
+        writel((u32)(value >> 32), regs + B53_SRAB_WD_H);
+
+        ret = b53_srab_op(dev, page, reg, B53_SRAB_CMDSTAT_WRITE);
+
+err:
+        b53_srab_release_grant(dev);
+
+        return ret;
+}
+
+static struct b53_io_ops b53_srab_ops = {
+        .read8 = b53_srab_read8,
+        .read16 = b53_srab_read16,
+        .read32 = b53_srab_read32,
+        .read48 = b53_srab_read48,
+        .read64 = b53_srab_read64,
+        .write8 = b53_srab_write8,
+        .write16 = b53_srab_write16,
+        .write32 = b53_srab_write32,
+        .write48 = b53_srab_write48,
+        .write64 = b53_srab_write64,
+};
+
+static int b53_srab_probe(struct platform_device *pdev)
+{
+        struct b53_srab_priv *priv;
+        struct b53_device *dev;
+        struct resource *r;
+
+        priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+        if (!priv)
+                return -ENOMEM;
+
+        r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+        priv->regs = devm_ioremap_resource(&pdev->dev, r);
+        if (IS_ERR(priv->regs))
+                return -ENOMEM;
+
+        dev = b53_switch_alloc(&pdev->dev, &b53_srab_ops, priv);
+        if (!dev)
+                return -ENOMEM;
+
+        platform_set_drvdata(pdev, dev);
+
+        return b53_switch_register(dev);
+}
+
+static int b53_srab_remove(struct platform_device *pdev)
+{
+        struct b53_device *dev = platform_get_drvdata(pdev);
+
+        if (dev)
+                b53_switch_remove(dev);
+
+        return 0;
+}
+
+static const struct of_device_id b53_srab_of_match[] = {
+        { .compatible = "brcm,bcm53010-srab" },
+        { .compatible = "brcm,bcm53011-srab" },
+        { .compatible = "brcm,bcm53012-srab" },
+        { .compatible = "brcm,bcm53018-srab" },
+        { .compatible = "brcm,bcm53019-srab" },
+        { .compatible = "brcm,bcm5301x-srab" },
+        { /* sentinel */ },
+};
+
+static struct platform_driver b53_srab_driver = {
+        .probe = b53_srab_probe,
+        .remove = b53_srab_remove,
+        .driver = {
+                .name = "b53-srab-switch",
+                .of_match_table = b53_srab_of_match,
+        },
+};
+
+module_platform_driver(b53_srab_driver);
+MODULE_AUTHOR("Hauke Mehrtens <hauke@hauke-m.de>");
+MODULE_DESCRIPTION("B53 Switch Register Access Bridge Registers (SRAB) access driver");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/net/dsa/bcm_sf2.c b/drivers/net/dsa/bcm_sf2.c
index 10ddd5a5dfb6..cd1d630ae3a9 100644
--- a/drivers/net/dsa/bcm_sf2.c
+++ b/drivers/net/dsa/bcm_sf2.c
@@ -22,6 +22,7 @@
 #include <linux/of_irq.h>
 #include <linux/of_address.h>
 #include <linux/of_net.h>
+#include <linux/of_mdio.h>
 #include <net/dsa.h>
 #include <linux/ethtool.h>
 #include <linux/if_bridge.h>
@@ -460,19 +461,13 @@ static int bcm_sf2_sw_set_eee(struct dsa_switch *ds, int port,
         return 0;
 }
 
-/* Fast-ageing of ARL entries for a given port, equivalent to an ARL
- * flush for that port.
- */
-static int bcm_sf2_sw_fast_age_port(struct dsa_switch  *ds, int port)
+static int bcm_sf2_fast_age_op(struct bcm_sf2_priv *priv)
 {
-        struct bcm_sf2_priv *priv = ds_to_priv(ds);
         unsigned int timeout = 1000;
         u32 reg;
 
-        core_writel(priv, port, CORE_FAST_AGE_PORT);
-
         reg = core_readl(priv, CORE_FAST_AGE_CTRL);
-        reg |= EN_AGE_PORT | EN_AGE_DYNAMIC | FAST_AGE_STR_DONE;
+        reg |= EN_AGE_PORT | EN_AGE_VLAN | EN_AGE_DYNAMIC | FAST_AGE_STR_DONE;
         core_writel(priv, reg, CORE_FAST_AGE_CTRL);
 
         do {
@@ -491,13 +486,98 @@ static int bcm_sf2_sw_fast_age_port(struct dsa_switch  *ds, int port)
         return 0;
 }
 
+/* Fast-ageing of ARL entries for a given port, equivalent to an ARL
+ * flush for that port.
+ */
+static int bcm_sf2_sw_fast_age_port(struct dsa_switch *ds, int port)
+{
+        struct bcm_sf2_priv *priv = ds_to_priv(ds);
+
+        core_writel(priv, port, CORE_FAST_AGE_PORT);
+
+        return bcm_sf2_fast_age_op(priv);
+}
+
+static int bcm_sf2_sw_fast_age_vlan(struct bcm_sf2_priv *priv, u16 vid)
+{
+        core_writel(priv, vid, CORE_FAST_AGE_VID);
+
+        return bcm_sf2_fast_age_op(priv);
+}
+
+static int bcm_sf2_vlan_op_wait(struct bcm_sf2_priv *priv)
+{
+        unsigned int timeout = 10;
+        u32 reg;
+
+        do {
+                reg = core_readl(priv, CORE_ARLA_VTBL_RWCTRL);
+                if (!(reg & ARLA_VTBL_STDN))
+                        return 0;
+
+                usleep_range(1000, 2000);
+        } while (timeout--);
+
+        return -ETIMEDOUT;
+}
+
+static int bcm_sf2_vlan_op(struct bcm_sf2_priv *priv, u8 op)
+{
+        core_writel(priv, ARLA_VTBL_STDN | op, CORE_ARLA_VTBL_RWCTRL);
+
+        return bcm_sf2_vlan_op_wait(priv);
+}
+
+static void bcm_sf2_set_vlan_entry(struct bcm_sf2_priv *priv, u16 vid,
+                                   struct bcm_sf2_vlan *vlan)
+{
+        int ret;
+
+        core_writel(priv, vid & VTBL_ADDR_INDEX_MASK, CORE_ARLA_VTBL_ADDR);
+        core_writel(priv, vlan->untag << UNTAG_MAP_SHIFT | vlan->members,
+                    CORE_ARLA_VTBL_ENTRY);
+
+        ret = bcm_sf2_vlan_op(priv, ARLA_VTBL_CMD_WRITE);
+        if (ret)
+                pr_err("failed to write VLAN entry\n");
+}
+
+static int bcm_sf2_get_vlan_entry(struct bcm_sf2_priv *priv, u16 vid,
+                                  struct bcm_sf2_vlan *vlan)
+{
+        u32 entry;
+        int ret;
+
+        core_writel(priv, vid & VTBL_ADDR_INDEX_MASK, CORE_ARLA_VTBL_ADDR);
+
+        ret = bcm_sf2_vlan_op(priv, ARLA_VTBL_CMD_READ);
+        if (ret)
+                return ret;
+
+        entry = core_readl(priv, CORE_ARLA_VTBL_ENTRY);
+        vlan->members = entry & FWD_MAP_MASK;
+        vlan->untag = (entry >> UNTAG_MAP_SHIFT) & UNTAG_MAP_MASK;
+
+        return 0;
+}
+
 static int bcm_sf2_sw_br_join(struct dsa_switch *ds, int port,
                               struct net_device *bridge)
 {
         struct bcm_sf2_priv *priv = ds_to_priv(ds);
+        s8 cpu_port = ds->dst->cpu_port;
         unsigned int i;
         u32 reg, p_ctl;
 
+        /* Make this port leave the all VLANs join since we will have proper
+         * VLAN entries from now on
+         */
+        reg = core_readl(priv, CORE_JOIN_ALL_VLAN_EN);
+        reg &= ~BIT(port);
+        if ((reg & BIT(cpu_port)) == BIT(cpu_port))
+                reg &= ~BIT(cpu_port);
+        core_writel(priv, reg, CORE_JOIN_ALL_VLAN_EN);
+
         priv->port_sts[port].bridge_dev = bridge;
         p_ctl = core_readl(priv, CORE_PORT_VLAN_CTL_PORT(port));
 
@@ -529,6 +609,7 @@ static void bcm_sf2_sw_br_leave(struct dsa_switch *ds, int port)
 {
         struct bcm_sf2_priv *priv = ds_to_priv(ds);
         struct net_device *bridge = priv->port_sts[port].bridge_dev;
+        s8 cpu_port = ds->dst->cpu_port;
         unsigned int i;
         u32 reg, p_ctl;
 
@@ -552,6 +633,13 @@ static void bcm_sf2_sw_br_leave(struct dsa_switch *ds, int port)
         core_writel(priv, p_ctl, CORE_PORT_VLAN_CTL_PORT(port));
         priv->port_sts[port].vlan_ctl_mask = p_ctl;
         priv->port_sts[port].bridge_dev = NULL;
+
+        /* Make this port join all VLANs without VLAN entries */
+        reg = core_readl(priv, CORE_JOIN_ALL_VLAN_EN);
+        reg |= BIT(port);
+        if (!(reg & BIT(cpu_port)))
+                reg |= BIT(cpu_port);
+        core_writel(priv, reg, CORE_JOIN_ALL_VLAN_EN);
 }
 
 static void bcm_sf2_sw_br_set_stp_state(struct dsa_switch *ds, int port,
@@ -804,7 +892,7 @@ static int bcm_sf2_sw_fdb_dump(struct dsa_switch *ds, int port,
                                int (*cb)(struct switchdev_obj *obj))
 {
         struct bcm_sf2_priv *priv = ds_to_priv(ds);
-        struct net_device *dev = ds->ports[port];
+        struct net_device *dev = ds->ports[port].netdev;
         struct bcm_sf2_arl_entry results[2];
         unsigned int count = 0;
         int ret;
@@ -836,6 +924,66 @@ static int bcm_sf2_sw_fdb_dump(struct dsa_switch *ds, int port,
         return 0;
 }
 
+static int bcm_sf2_sw_indir_rw(struct bcm_sf2_priv *priv, int op, int addr,
+                               int regnum, u16 val)
+{
+        int ret = 0;
+        u32 reg;
+
+        reg = reg_readl(priv, REG_SWITCH_CNTRL);
+        reg |= MDIO_MASTER_SEL;
+        reg_writel(priv, reg, REG_SWITCH_CNTRL);
+
+        /* Page << 8 | offset */
+        reg = 0x70;
+        reg <<= 2;
+        core_writel(priv, addr, reg);
+
+        /* Page << 8 | offset */
+        reg = 0x80 << 8 | regnum << 1;
+        reg <<= 2;
+
+        if (op)
+                ret = core_readl(priv, reg);
+        else
+                core_writel(priv, val, reg);
+
+        reg = reg_readl(priv, REG_SWITCH_CNTRL);
+        reg &= ~MDIO_MASTER_SEL;
+        reg_writel(priv, reg, REG_SWITCH_CNTRL);
+
+        return ret & 0xffff;
+}
+
+static int bcm_sf2_sw_mdio_read(struct mii_bus *bus, int addr, int regnum)
+{
+        struct bcm_sf2_priv *priv = bus->priv;
+
+        /* Intercept reads from Broadcom pseudo-PHY address, else, send
+         * them to our master MDIO bus controller
+         */
+        if (addr == BRCM_PSEUDO_PHY_ADDR && priv->indir_phy_mask & BIT(addr))
+                return bcm_sf2_sw_indir_rw(priv, 1, addr, regnum, 0);
+        else
+                return mdiobus_read(priv->master_mii_bus, addr, regnum);
+}
+
+static int bcm_sf2_sw_mdio_write(struct mii_bus *bus, int addr, int regnum,
+                                 u16 val)
+{
+        struct bcm_sf2_priv *priv = bus->priv;
+
+        /* Intercept writes to the Broadcom pseudo-PHY address, else,
+         * send them to our master MDIO bus controller
+         */
+        if (addr == BRCM_PSEUDO_PHY_ADDR && priv->indir_phy_mask & BIT(addr))
+                bcm_sf2_sw_indir_rw(priv, 0, addr, regnum, val);
+        else
+                mdiobus_write(priv->master_mii_bus, addr, regnum, val);
+
+        return 0;
+}
+
 static irqreturn_t bcm_sf2_switch_0_isr(int irq, void *dev_id)
 {
         struct bcm_sf2_priv *priv = dev_id;
@@ -932,133 +1080,70 @@ static void bcm_sf2_identify_ports(struct bcm_sf2_priv *priv,
         }
 }
 
-static int bcm_sf2_sw_setup(struct dsa_switch *ds)
+static int bcm_sf2_mdio_register(struct dsa_switch *ds)
 {
-        const char *reg_names[BCM_SF2_REGS_NUM] = BCM_SF2_REGS_NAME;
         struct bcm_sf2_priv *priv = ds_to_priv(ds);
         struct device_node *dn;
-        void __iomem **base;
-        unsigned int port;
-        unsigned int i;
-        u32 reg, rev;
-        int ret;
-
-        spin_lock_init(&priv->indir_lock);
-        mutex_init(&priv->stats_mutex);
-
-        /* All the interesting properties are at the parent device_node
-         * level
-         */
-        dn = ds->cd->of_node->parent;
-        bcm_sf2_identify_ports(priv, ds->cd->of_node);
-
-        priv->irq0 = irq_of_parse_and_map(dn, 0);
-        priv->irq1 = irq_of_parse_and_map(dn, 1);
-
-        base = &priv->core;
-        for (i = 0; i < BCM_SF2_REGS_NUM; i++) {
-                *base = of_iomap(dn, i);
-                if (*base == NULL) {
-                        pr_err("unable to find register: %s\n", reg_names[i]);
-                        ret = -ENOMEM;
-                        goto out_unmap;
-                }
-                base++;
-        }
-
-        ret = bcm_sf2_sw_rst(priv);
-        if (ret) {
-                pr_err("unable to software reset switch: %d\n", ret);
-                goto out_unmap;
-        }
-
-        /* Disable all interrupts and request them */
-        bcm_sf2_intr_disable(priv);
-
-        ret = request_irq(priv->irq0, bcm_sf2_switch_0_isr, 0,
-                          "switch_0", priv);
-        if (ret < 0) {
-                pr_err("failed to request switch_0 IRQ\n");
-                goto out_unmap;
-        }
-
-        ret = request_irq(priv->irq1, bcm_sf2_switch_1_isr, 0,
-                          "switch_1", priv);
-        if (ret < 0) {
-                pr_err("failed to request switch_1 IRQ\n");
-                goto out_free_irq0;
-        }
-
-        /* Reset the MIB counters */
-        reg = core_readl(priv, CORE_GMNCFGCFG);
-        reg |= RST_MIB_CNT;
-        core_writel(priv, reg, CORE_GMNCFGCFG);
-        reg &= ~RST_MIB_CNT;
-        core_writel(priv, reg, CORE_GMNCFGCFG);
-
-        /* Get the maximum number of ports for this switch */
-        priv->hw_params.num_ports = core_readl(priv, CORE_IMP0_PRT_ID) + 1;
-        if (priv->hw_params.num_ports > DSA_MAX_PORTS)
-                priv->hw_params.num_ports = DSA_MAX_PORTS;
-
-        /* Assume a single GPHY setup if we can't read that property */
-        if (of_property_read_u32(dn, "brcm,num-gphy",
-                                 &priv->hw_params.num_gphy))
-                priv->hw_params.num_gphy = 1;
-
-        /* Enable all valid ports and disable those unused */
-        for (port = 0; port < priv->hw_params.num_ports; port++) {
-                /* IMP port receives special treatment */
-                if ((1 << port) & ds->enabled_port_mask)
-                        bcm_sf2_port_setup(ds, port, NULL);
-                else if (dsa_is_cpu_port(ds, port))
-                        bcm_sf2_imp_setup(ds, port);
-                else
-                        bcm_sf2_port_disable(ds, port, NULL);
-        }
-
-        /* Include the pseudo-PHY address and the broadcast PHY address to
-         * divert reads towards our workaround. This is only required for
-         * 7445D0, since 7445E0 disconnects the internal switch pseudo-PHY such
-         * that we can use the regular SWITCH_MDIO master controller instead.
+        static int index;
+        int err;
+
+        /* Find our integrated MDIO bus node */
+        dn = of_find_compatible_node(NULL, NULL, "brcm,unimac-mdio");
+        priv->master_mii_bus = of_mdio_find_bus(dn);
+        if (!priv->master_mii_bus)
+                return -EPROBE_DEFER;
+
+        get_device(&priv->master_mii_bus->dev);
+        priv->master_mii_dn = dn;
+
+        priv->slave_mii_bus = devm_mdiobus_alloc(ds->dev);
+        if (!priv->slave_mii_bus)
+                return -ENOMEM;
+
+        priv->slave_mii_bus->priv = priv;
+        priv->slave_mii_bus->name = "sf2 slave mii";
+        priv->slave_mii_bus->read = bcm_sf2_sw_mdio_read;
+        priv->slave_mii_bus->write = bcm_sf2_sw_mdio_write;
+        snprintf(priv->slave_mii_bus->id, MII_BUS_ID_SIZE, "sf2-%d",
+                 index++);
+        priv->slave_mii_bus->dev.of_node = dn;
+
+        /* Include the pseudo-PHY address to divert reads towards our
+         * workaround. This is only required for 7445D0, since 7445E0
+         * disconnects the internal switch pseudo-PHY such that we can use the
+         * regular SWITCH_MDIO master controller instead.
          *
-         * By default, DSA initializes ds->phys_mii_mask to
-         * ds->enabled_port_mask to have a 1:1 mapping between Port address
-         * and PHY address in order to utilize the slave_mii_bus instance to
-         * read from Port PHYs. This is not what we want here, so we
-         * initialize phys_mii_mask 0 to always utilize the "master" MDIO
-         * bus backed by the "mdio-unimac" driver.
+         * Here we flag the pseudo PHY as needing special treatment and would
+         * otherwise make all other PHY read/writes go to the master MDIO bus
+         * controller that comes with this switch backed by the "mdio-unimac"
+         * driver.
          */
         if (of_machine_is_compatible("brcm,bcm7445d0"))
-                ds->phys_mii_mask |= ((1 << BRCM_PSEUDO_PHY_ADDR) | (1 << 0));
+                priv->indir_phy_mask |= (1 << BRCM_PSEUDO_PHY_ADDR);
         else
-                ds->phys_mii_mask = 0;
+                priv->indir_phy_mask = 0;
 
-        rev = reg_readl(priv, REG_SWITCH_REVISION);
-        priv->hw_params.top_rev = (rev >> SWITCH_TOP_REV_SHIFT) &
-                                        SWITCH_TOP_REV_MASK;
-        priv->hw_params.core_rev = (rev & SF2_REV_MASK);
+        ds->phys_mii_mask = priv->indir_phy_mask;
+        ds->slave_mii_bus = priv->slave_mii_bus;
+        priv->slave_mii_bus->parent = ds->dev->parent;
+        priv->slave_mii_bus->phy_mask = ~priv->indir_phy_mask;
 
-        rev = reg_readl(priv, REG_PHY_REVISION);
-        priv->hw_params.gphy_rev = rev & PHY_REVISION_MASK;
+        if (dn)
+                err = of_mdiobus_register(priv->slave_mii_bus, dn);
+        else
+                err = mdiobus_register(priv->slave_mii_bus);
 
-        pr_info("Starfighter 2 top: %x.%02x, core: %x.%02x base: 0x%p, IRQs: %d, %d\n",
-                priv->hw_params.top_rev >> 8, priv->hw_params.top_rev & 0xff,
-                priv->hw_params.core_rev >> 8, priv->hw_params.core_rev & 0xff,
-                priv->core, priv->irq0, priv->irq1);
+        if (err)
+                of_node_put(dn);
 
-        return 0;
+        return err;
+}
 
-out_free_irq0:
-        free_irq(priv->irq0, priv);
-out_unmap:
-        base = &priv->core;
-        for (i = 0; i < BCM_SF2_REGS_NUM; i++) {
-                if (*base)
-                        iounmap(*base);
-                base++;
-        }
-        return ret;
+static void bcm_sf2_mdio_unregister(struct bcm_sf2_priv *priv)
+{
+        mdiobus_unregister(priv->slave_mii_bus);
+        if (priv->master_mii_dn)
+                of_node_put(priv->master_mii_dn);
 }
 
 static int bcm_sf2_sw_set_addr(struct dsa_switch *ds, u8 *addr)
@@ -1078,68 +1163,6 @@ static u32 bcm_sf2_sw_get_phy_flags(struct dsa_switch *ds, int port)
         return priv->hw_params.gphy_rev;
 }
 
-static int bcm_sf2_sw_indir_rw(struct dsa_switch *ds, int op, int addr,
-                               int regnum, u16 val)
-{
-        struct bcm_sf2_priv *priv = ds_to_priv(ds);
-        int ret = 0;
-        u32 reg;
-
-        reg = reg_readl(priv, REG_SWITCH_CNTRL);
-        reg |= MDIO_MASTER_SEL;
-        reg_writel(priv, reg, REG_SWITCH_CNTRL);
-
-        /* Page << 8 | offset */
-        reg = 0x70;
-        reg <<= 2;
-        core_writel(priv, addr, reg);
-
-        /* Page << 8 | offset */
-        reg = 0x80 << 8 | regnum << 1;
-        reg <<= 2;
-
-        if (op)
-                ret = core_readl(priv, reg);
-        else
-                core_writel(priv, val, reg);
-
-        reg = reg_readl(priv, REG_SWITCH_CNTRL);
-        reg &= ~MDIO_MASTER_SEL;
-        reg_writel(priv, reg, REG_SWITCH_CNTRL);
-
-        return ret & 0xffff;
-}
-
-static int bcm_sf2_sw_phy_read(struct dsa_switch *ds, int addr, int regnum)
-{
-        /* Intercept reads from the MDIO broadcast address or Broadcom
-         * pseudo-PHY address
-         */
-        switch (addr) {
-        case 0:
-        case BRCM_PSEUDO_PHY_ADDR:
-                return bcm_sf2_sw_indir_rw(ds, 1, addr, regnum, 0);
-        default:
-                return 0xffff;
-        }
-}
-
-static int bcm_sf2_sw_phy_write(struct dsa_switch *ds, int addr, int regnum,
-                                u16 val)
-{
-        /* Intercept writes to the MDIO broadcast address or Broadcom
-         * pseudo-PHY address
-         */
-        switch (addr) {
-        case 0:
-        case BRCM_PSEUDO_PHY_ADDR:
-                bcm_sf2_sw_indir_rw(ds, 0, addr, regnum, val);
-                break;
-        }
-
-        return 0;
-}
-
 static void bcm_sf2_sw_adjust_link(struct dsa_switch *ds, int port,
                                    struct phy_device *phydev)
 {
@@ -1248,7 +1271,7 @@ static void bcm_sf2_sw_fixed_link_update(struct dsa_switch *ds, int port,
                  * state machine and make it go in PHY_FORCING state instead.
                  */
                 if (!status->link)
-                        netif_carrier_off(ds->ports[port]);
+                        netif_carrier_off(ds->ports[port].netdev);
                 status->duplex = 1;
         } else {
                 status->link = 1;
@@ -1370,14 +1393,309 @@ static int bcm_sf2_sw_set_wol(struct dsa_switch *ds, int port,
         return p->ethtool_ops->set_wol(p, wol);
 }
 
+static void bcm_sf2_enable_vlan(struct bcm_sf2_priv *priv, bool enable)
+{
+        u32 mgmt, vc0, vc1, vc4, vc5;
+
+        mgmt = core_readl(priv, CORE_SWMODE);
+        vc0 = core_readl(priv, CORE_VLAN_CTRL0);
+        vc1 = core_readl(priv, CORE_VLAN_CTRL1);
+        vc4 = core_readl(priv, CORE_VLAN_CTRL4);
+        vc5 = core_readl(priv, CORE_VLAN_CTRL5);
+
+        mgmt &= ~SW_FWDG_MODE;
+
+        if (enable) {
+                vc0 |= VLAN_EN | VLAN_LEARN_MODE_IVL;
+                vc1 |= EN_RSV_MCAST_UNTAG | EN_RSV_MCAST_FWDMAP;
+                vc4 &= ~(INGR_VID_CHK_MASK << INGR_VID_CHK_SHIFT);
+                vc4 |= INGR_VID_CHK_DROP;
+                vc5 |= DROP_VTABLE_MISS | EN_VID_FFF_FWD;
+        } else {
+                vc0 &= ~(VLAN_EN | VLAN_LEARN_MODE_IVL);
+                vc1 &= ~(EN_RSV_MCAST_UNTAG | EN_RSV_MCAST_FWDMAP);
+                vc4 &= ~(INGR_VID_CHK_MASK << INGR_VID_CHK_SHIFT);
+                vc5 &= ~(DROP_VTABLE_MISS | EN_VID_FFF_FWD);
+                vc4 |= INGR_VID_CHK_VID_VIOL_IMP;
+        }
+
+        core_writel(priv, vc0, CORE_VLAN_CTRL0);
+        core_writel(priv, vc1, CORE_VLAN_CTRL1);
+        core_writel(priv, 0, CORE_VLAN_CTRL3);
+        core_writel(priv, vc4, CORE_VLAN_CTRL4);
+        core_writel(priv, vc5, CORE_VLAN_CTRL5);
+        core_writel(priv, mgmt, CORE_SWMODE);
+}
+
+static void bcm_sf2_sw_configure_vlan(struct dsa_switch *ds)
+{
+        struct bcm_sf2_priv *priv = ds_to_priv(ds);
+        unsigned int port;
+
+        /* Clear all VLANs */
+        bcm_sf2_vlan_op(priv, ARLA_VTBL_CMD_CLEAR);
+
+        for (port = 0; port < priv->hw_params.num_ports; port++) {
+                if (!((1 << port) & ds->enabled_port_mask))
+                        continue;
+
+                core_writel(priv, 1, CORE_DEFAULT_1Q_TAG_P(port));
+        }
+}
+
+static int bcm_sf2_sw_vlan_filtering(struct dsa_switch *ds, int port,
+                                     bool vlan_filtering)
+{
+        return 0;
+}
+
+static int bcm_sf2_sw_vlan_prepare(struct dsa_switch *ds, int port,
+                                   const struct switchdev_obj_port_vlan *vlan,
+                                   struct switchdev_trans *trans)
+{
+        struct bcm_sf2_priv *priv = ds_to_priv(ds);
+
+        bcm_sf2_enable_vlan(priv, true);
+
+        return 0;
+}
+
+static void bcm_sf2_sw_vlan_add(struct dsa_switch *ds, int port,
+                                const struct switchdev_obj_port_vlan *vlan,
+                                struct switchdev_trans *trans)
+{
+        struct bcm_sf2_priv *priv = ds_to_priv(ds);
+        bool untagged = vlan->flags & BRIDGE_VLAN_INFO_UNTAGGED;
+        bool pvid = vlan->flags & BRIDGE_VLAN_INFO_PVID;
+        s8 cpu_port = ds->dst->cpu_port;
+        struct bcm_sf2_vlan *vl;
+        u16 vid;
+
+        for (vid = vlan->vid_begin; vid <= vlan->vid_end; ++vid) {
+                vl = &priv->vlans[vid];
+
+                bcm_sf2_get_vlan_entry(priv, vid, vl);
+
+                vl->members |= BIT(port) | BIT(cpu_port);
+                if (untagged)
+                        vl->untag |= BIT(port) | BIT(cpu_port);
+                else
+                        vl->untag &= ~(BIT(port) | BIT(cpu_port));
+
+                bcm_sf2_set_vlan_entry(priv, vid, vl);
+                bcm_sf2_sw_fast_age_vlan(priv, vid);
+        }
+
+        if (pvid) {
+                core_writel(priv, vlan->vid_end, CORE_DEFAULT_1Q_TAG_P(port));
+                core_writel(priv, vlan->vid_end,
+                            CORE_DEFAULT_1Q_TAG_P(cpu_port));
+                bcm_sf2_sw_fast_age_vlan(priv, vid);
+        }
+}
+
+static int bcm_sf2_sw_vlan_del(struct dsa_switch *ds, int port,
+                               const struct switchdev_obj_port_vlan *vlan)
+{
+        struct bcm_sf2_priv *priv = ds_to_priv(ds);
+        bool untagged = vlan->flags & BRIDGE_VLAN_INFO_UNTAGGED;
+        s8 cpu_port = ds->dst->cpu_port;
+        struct bcm_sf2_vlan *vl;
+        u16 vid, pvid;
+        int ret;
+
+        pvid = core_readl(priv, CORE_DEFAULT_1Q_TAG_P(port));
+
+        for (vid = vlan->vid_begin; vid <= vlan->vid_end; ++vid) {
+                vl = &priv->vlans[vid];
+
+                ret = bcm_sf2_get_vlan_entry(priv, vid, vl);
+                if (ret)
+                        return ret;
+
+                vl->members &= ~BIT(port);
+                if ((vl->members & BIT(cpu_port)) == BIT(cpu_port))
+                        vl->members = 0;
+                if (pvid == vid)
+                        pvid = 0;
+                if (untagged) {
+                        vl->untag &= ~BIT(port);
+                        if ((vl->untag & BIT(port)) == BIT(cpu_port))
+                                vl->untag = 0;
+                }
+
+                bcm_sf2_set_vlan_entry(priv, vid, vl);
+                bcm_sf2_sw_fast_age_vlan(priv, vid);
+        }
+
+        core_writel(priv, pvid, CORE_DEFAULT_1Q_TAG_P(port));
+        core_writel(priv, pvid, CORE_DEFAULT_1Q_TAG_P(cpu_port));
+        bcm_sf2_sw_fast_age_vlan(priv, vid);
+
+        return 0;
+}
+
+static int bcm_sf2_sw_vlan_dump(struct dsa_switch *ds, int port,
+                                struct switchdev_obj_port_vlan *vlan,
+                                int (*cb)(struct switchdev_obj *obj))
+{
+        struct bcm_sf2_priv *priv = ds_to_priv(ds);
+        struct bcm_sf2_port_status *p = &priv->port_sts[port];
+        struct bcm_sf2_vlan *vl;
+        u16 vid, pvid;
+        int err = 0;
+
+        pvid = core_readl(priv, CORE_DEFAULT_1Q_TAG_P(port));
+
+        for (vid = 0; vid < VLAN_N_VID; vid++) {
+                vl = &priv->vlans[vid];
+
+                if (!(vl->members & BIT(port)))
+                        continue;
+
+                vlan->vid_begin = vlan->vid_end = vid;
+                vlan->flags = 0;
+
+                if (vl->untag & BIT(port))
+                        vlan->flags |= BRIDGE_VLAN_INFO_UNTAGGED;
+                if (p->pvid == vid)
+                        vlan->flags |= BRIDGE_VLAN_INFO_PVID;
+
+                err = cb(&vlan->obj);
+                if (err)
+                        break;
+        }
+
+        return err;
+}
+
+static int bcm_sf2_sw_setup(struct dsa_switch *ds)
+{
+        const char *reg_names[BCM_SF2_REGS_NUM] = BCM_SF2_REGS_NAME;
+        struct bcm_sf2_priv *priv = ds_to_priv(ds);
+        struct device_node *dn;
+        void __iomem **base;
+        unsigned int port;
+        unsigned int i;
+        u32 reg, rev;
+        int ret;
+
+        spin_lock_init(&priv->indir_lock);
+        mutex_init(&priv->stats_mutex);
+
+        /* All the interesting properties are at the parent device_node
+         * level
+         */
+        dn = ds->cd->of_node->parent;
+        bcm_sf2_identify_ports(priv, ds->cd->of_node);
+
+        priv->irq0 = irq_of_parse_and_map(dn, 0);
+        priv->irq1 = irq_of_parse_and_map(dn, 1);
+
+        base = &priv->core;
+        for (i = 0; i < BCM_SF2_REGS_NUM; i++) {
+                *base = of_iomap(dn, i);
+                if (*base == NULL) {
+                        pr_err("unable to find register: %s\n", reg_names[i]);
+                        ret = -ENOMEM;
+                        goto out_unmap;
+                }
+                base++;
+        }
+
+        ret = bcm_sf2_sw_rst(priv);
+        if (ret) {
+                pr_err("unable to software reset switch: %d\n", ret);
+                goto out_unmap;
+        }
+
+        ret = bcm_sf2_mdio_register(ds);
+        if (ret) {
+                pr_err("failed to register MDIO bus\n");
+                goto out_unmap;
+        }
+
+        /* Disable all interrupts and request them */
+        bcm_sf2_intr_disable(priv);
+
+        ret = request_irq(priv->irq0, bcm_sf2_switch_0_isr, 0,
+                          "switch_0", priv);
+        if (ret < 0) {
+                pr_err("failed to request switch_0 IRQ\n");
+                goto out_unmap;
+        }
+
+        ret = request_irq(priv->irq1, bcm_sf2_switch_1_isr, 0,
+                          "switch_1", priv);
+        if (ret < 0) {
+                pr_err("failed to request switch_1 IRQ\n");
+                goto out_free_irq0;
+        }
+
+        /* Reset the MIB counters */
+        reg = core_readl(priv, CORE_GMNCFGCFG);
+        reg |= RST_MIB_CNT;
+        core_writel(priv, reg, CORE_GMNCFGCFG);
+        reg &= ~RST_MIB_CNT;
+        core_writel(priv, reg, CORE_GMNCFGCFG);
+
+        /* Get the maximum number of ports for this switch */
+        priv->hw_params.num_ports = core_readl(priv, CORE_IMP0_PRT_ID) + 1;
+        if (priv->hw_params.num_ports > DSA_MAX_PORTS)
+                priv->hw_params.num_ports = DSA_MAX_PORTS;
+
+        /* Assume a single GPHY setup if we can't read that property */
+        if (of_property_read_u32(dn, "brcm,num-gphy",
+                                 &priv->hw_params.num_gphy))
+                priv->hw_params.num_gphy = 1;
+
+        /* Enable all valid ports and disable those unused */
+        for (port = 0; port < priv->hw_params.num_ports; port++) {
+                /* IMP port receives special treatment */
+                if ((1 << port) & ds->enabled_port_mask)
+                        bcm_sf2_port_setup(ds, port, NULL);
+                else if (dsa_is_cpu_port(ds, port))
+                        bcm_sf2_imp_setup(ds, port);
+                else
+                        bcm_sf2_port_disable(ds, port, NULL);
+        }
+
+        bcm_sf2_sw_configure_vlan(ds);
+
+        rev = reg_readl(priv, REG_SWITCH_REVISION);
+        priv->hw_params.top_rev = (rev >> SWITCH_TOP_REV_SHIFT) &
+                                        SWITCH_TOP_REV_MASK;
+        priv->hw_params.core_rev = (rev & SF2_REV_MASK);
+
+        rev = reg_readl(priv, REG_PHY_REVISION);
+        priv->hw_params.gphy_rev = rev & PHY_REVISION_MASK;
+
+        pr_info("Starfighter 2 top: %x.%02x, core: %x.%02x base: 0x%p, IRQs: %d, %d\n",
+                priv->hw_params.top_rev >> 8, priv->hw_params.top_rev & 0xff,
+                priv->hw_params.core_rev >> 8, priv->hw_params.core_rev & 0xff,
+                priv->core, priv->irq0, priv->irq1);
+
+        return 0;
+
+out_free_irq0:
+        free_irq(priv->irq0, priv);
+out_unmap:
+        base = &priv->core;
+        for (i = 0; i < BCM_SF2_REGS_NUM; i++) {
+                if (*base)
+                        iounmap(*base);
+                base++;
+        }
+        bcm_sf2_mdio_unregister(priv);
+        return ret;
+}
+
 static struct dsa_switch_driver bcm_sf2_switch_driver = {
         .tag_protocol           = DSA_TAG_PROTO_BRCM,
         .probe                  = bcm_sf2_sw_drv_probe,
         .setup                  = bcm_sf2_sw_setup,
         .set_addr               = bcm_sf2_sw_set_addr,
         .get_phy_flags          = bcm_sf2_sw_get_phy_flags,
-        .phy_read               = bcm_sf2_sw_phy_read,
-        .phy_write              = bcm_sf2_sw_phy_write,
         .get_strings            = bcm_sf2_sw_get_strings,
         .get_ethtool_stats      = bcm_sf2_sw_get_ethtool_stats,
         .get_sset_count         = bcm_sf2_sw_get_sset_count,
@@ -1398,6 +1716,11 @@ static struct dsa_switch_driver bcm_sf2_switch_driver = {
         .port_fdb_add           = bcm_sf2_sw_fdb_add,
         .port_fdb_del           = bcm_sf2_sw_fdb_del,
         .port_fdb_dump          = bcm_sf2_sw_fdb_dump,
+        .port_vlan_filtering    = bcm_sf2_sw_vlan_filtering,
+        .port_vlan_prepare      = bcm_sf2_sw_vlan_prepare,
+        .port_vlan_add          = bcm_sf2_sw_vlan_add,
+        .port_vlan_del          = bcm_sf2_sw_vlan_del,
+        .port_vlan_dump         = bcm_sf2_sw_vlan_dump,
 };
 
 static int __init bcm_sf2_init(void)
diff --git a/drivers/net/dsa/bcm_sf2.h b/drivers/net/dsa/bcm_sf2.h
index 200b1f5fdb56..463bed8cbe4c 100644
--- a/drivers/net/dsa/bcm_sf2.h
+++ b/drivers/net/dsa/bcm_sf2.h
@@ -21,6 +21,7 @@
 #include <linux/ethtool.h>
 #include <linux/types.h>
 #include <linux/bitops.h>
+#include <linux/if_vlan.h>
 
 #include <net/dsa.h>
 
@@ -50,6 +51,7 @@ struct bcm_sf2_port_status {
         struct ethtool_eee eee;
 
         u32 vlan_ctl_mask;
+        u16 pvid;
 
         struct net_device *bridge_dev;
 };
@@ -63,6 +65,11 @@ struct bcm_sf2_arl_entry {
         u8 is_static:1;
 };
 
+struct bcm_sf2_vlan {
+        u16 members;
+        u16 untag;
+};
+
 static inline void bcm_sf2_mac_from_u64(u64 src, u8 *dst)
 {
         unsigned int i;
@@ -142,6 +149,15 @@ struct bcm_sf2_priv {
 
         /* Bitmask of ports having an integrated PHY */
         unsigned int                    int_phy_mask;
+
+        /* Master and slave MDIO bus controller */
+        unsigned int                    indir_phy_mask;
+        struct device_node              *master_mii_dn;
+        struct mii_bus                  *slave_mii_bus;
+        struct mii_bus                  *master_mii_bus;
+
+        /* Cache of programmed VLANs */
+        struct bcm_sf2_vlan             vlans[VLAN_N_VID];
 };
 
 struct bcm_sf2_hw_stats {
diff --git a/drivers/net/dsa/bcm_sf2_regs.h b/drivers/net/dsa/bcm_sf2_regs.h
index 97780d43b5c0..9f2a9cb42074 100644
--- a/drivers/net/dsa/bcm_sf2_regs.h
+++ b/drivers/net/dsa/bcm_sf2_regs.h
@@ -274,6 +274,23 @@
 #define CORE_ARLA_SRCH_RSLT_MACVID(x)   (CORE_ARLA_SRCH_RSLT_0_MACVID + ((x) * 0x40))
 #define CORE_ARLA_SRCH_RSLT(x)          (CORE_ARLA_SRCH_RSLT_0 + ((x) * 0x40))
 
+#define CORE_ARLA_VTBL_RWCTRL           0x1600
+#define  ARLA_VTBL_CMD_WRITE            0
+#define  ARLA_VTBL_CMD_READ             1
+#define  ARLA_VTBL_CMD_CLEAR            2
+#define  ARLA_VTBL_STDN                 (1 << 7)
+
+#define CORE_ARLA_VTBL_ADDR             0x1604
+#define  VTBL_ADDR_INDEX_MASK           0xfff
+
+#define CORE_ARLA_VTBL_ENTRY            0x160c
+#define  FWD_MAP_MASK                   0x1ff
+#define  UNTAG_MAP_MASK                 0x1ff
+#define  UNTAG_MAP_SHIFT                9
+#define  MSTP_INDEX_MASK                0x7
+#define  MSTP_INDEX_SHIFT               18
+#define  FWD_MODE                       (1 << 21)
+
 #define CORE_MEM_PSM_VDD_CTRL           0x2380
 #define  P_TXQ_PSM_VDD_SHIFT            2
 #define  P_TXQ_PSM_VDD_MASK             0x3
@@ -287,6 +304,59 @@
 #define CORE_PORT_VLAN_CTL_PORT(x)      (0xc400 + ((x) * 0x8))
 #define  PORT_VLAN_CTRL_MASK            0x1ff
 
+#define CORE_VLAN_CTRL0                 0xd000
+#define  CHANGE_1P_VID_INNER            (1 << 0)
+#define  CHANGE_1P_VID_OUTER            (1 << 1)
+#define  CHANGE_1Q_VID                  (1 << 3)
+#define  VLAN_LEARN_MODE_SVL            (0 << 5)
+#define  VLAN_LEARN_MODE_IVL            (3 << 5)
+#define  VLAN_EN                        (1 << 7)
+
+#define CORE_VLAN_CTRL1                 0xd004
+#define  EN_RSV_MCAST_FWDMAP            (1 << 2)
+#define  EN_RSV_MCAST_UNTAG             (1 << 3)
+#define  EN_IPMC_BYPASS_FWDMAP          (1 << 5)
+#define  EN_IPMC_BYPASS_UNTAG           (1 << 6)
+
+#define CORE_VLAN_CTRL2                 0xd008
+#define  EN_MIIM_BYPASS_V_FWDMAP        (1 << 2)
+#define  EN_GMRP_GVRP_V_FWDMAP          (1 << 5)
+#define  EN_GMRP_GVRP_UNTAG_MAP         (1 << 6)
+
+#define CORE_VLAN_CTRL3                 0xd00c
+#define  EN_DROP_NON1Q_MASK             0x1ff
+
+#define CORE_VLAN_CTRL4                 0xd014
+#define  RESV_MCAST_FLOOD               (1 << 1)
+#define  EN_DOUBLE_TAG_MASK             0x3
+#define  EN_DOUBLE_TAG_SHIFT            2
+#define  EN_MGE_REV_GMRP                (1 << 4)
+#define  EN_MGE_REV_GVRP                (1 << 5)
+#define  INGR_VID_CHK_SHIFT             6
+#define  INGR_VID_CHK_MASK              0x3
+#define  INGR_VID_CHK_FWD               (0 << INGR_VID_CHK_SHIFT)
+#define  INGR_VID_CHK_DROP              (1 << INGR_VID_CHK_SHIFT)
+#define  INGR_VID_CHK_NO_CHK            (2 << INGR_VID_CHK_SHIFT)
+#define  INGR_VID_CHK_VID_VIOL_IMP      (3 << INGR_VID_CHK_SHIFT)
+
+#define CORE_VLAN_CTRL5                 0xd018
+#define  EN_CPU_RX_BYP_INNER_CRCCHCK    (1 << 0)
+#define  EN_VID_FFF_FWD                 (1 << 2)
+#define  DROP_VTABLE_MISS               (1 << 3)
+#define  EGRESS_DIR_FRM_BYP_TRUNK_EN    (1 << 4)
+#define  PRESV_NON1Q                    (1 << 6)
+
+#define CORE_VLAN_CTRL6                 0xd01c
+#define  STRICT_SFD_DETECT              (1 << 0)
+#define  DIS_ARL_BUST_LMIT              (1 << 4)
+
+#define CORE_DEFAULT_1Q_TAG_P(x)        (0xd040 + ((x) * 8))
+#define  CFI_SHIFT                      12
+#define  PRI_SHIFT                      13
+#define  PRI_MASK                       0x7
+
+#define CORE_JOIN_ALL_VLAN_EN           0xd140
+
 #define CORE_EEE_EN_CTRL                0x24800
 #define CORE_EEE_LPI_INDICATE           0x24810
 
diff --git a/drivers/net/dsa/mv88e6xxx/Kconfig b/drivers/net/dsa/mv88e6xxx/Kconfig
new file mode 100644
index 000000000000..490bc06f993e
--- /dev/null
+++ b/drivers/net/dsa/mv88e6xxx/Kconfig
@@ -0,0 +1,7 @@
+config NET_DSA_MV88E6XXX
+        tristate "Marvell 88E6xxx Ethernet switch fabric support"
+        depends on NET_DSA
+        select NET_DSA_TAG_EDSA
+        help
+          This driver adds support for most of the Marvell 88E6xxx models of
+          Ethernet switch chips, except 88E6060.
diff --git a/drivers/net/dsa/mv88e6xxx/Makefile b/drivers/net/dsa/mv88e6xxx/Makefile
new file mode 100644
index 000000000000..6e29a75ee2f7
--- /dev/null
+++ b/drivers/net/dsa/mv88e6xxx/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_NET_DSA_MV88E6XXX) += chip.o
diff --git a/drivers/net/dsa/mv88e6xxx.c b/drivers/net/dsa/mv88e6xxx/chip.c
index ba9dfc9421ef..5cb06f7673af 100644
--- a/drivers/net/dsa/mv88e6xxx.c
+++ b/drivers/net/dsa/mv88e6xxx/chip.c
@@ -1,5 +1,6 @@
 /*
- * net/dsa/mv88e6xxx.c - Marvell 88e6xxx switch chip support
+ * Marvell 88e6xxx Ethernet switch single-chip support
+ *
  * Copyright (c) 2008 Marvell Semiconductor
  *
  * Copyright (c) 2015 CMC Electronics, Inc.
@@ -21,6 +22,8 @@
 #include <linux/list.h>
 #include <linux/mdio.h>
 #include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/of_mdio.h>
 #include <linux/netdevice.h>
 #include <linux/gpio/consumer.h>
 #include <linux/phy.h>
@@ -28,29 +31,82 @@
 #include <net/switchdev.h>
 #include "mv88e6xxx.h"
 
-static void assert_smi_lock(struct mv88e6xxx_priv_state *ps)
+static void assert_reg_lock(struct mv88e6xxx_chip *chip)
 {
-        if (unlikely(!mutex_is_locked(&ps->smi_mutex))) {
-                dev_err(ps->dev, "SMI lock not held!\n");
+        if (unlikely(!mutex_is_locked(&chip->reg_lock))) {
+                dev_err(chip->dev, "Switch registers lock not held!\n");
                 dump_stack();
         }
 }
 
-/* If the switch's ADDR[4:0] strap pins are strapped to zero, it will
- * use all 32 SMI bus addresses on its SMI bus, and all switch registers
- * will be directly accessible on some {device address,register address}
- * pair.  If the ADDR[4:0] pins are not strapped to zero, the switch
- * will only respond to SMI transactions to that specific address, and
- * an indirect addressing mechanism needs to be used to access its
- * registers.
+/* The switch ADDR[4:1] configuration pins define the chip SMI device address
+ * (ADDR[0] is always zero, thus only even SMI addresses can be strapped).
+ *
+ * When ADDR is all zero, the chip uses Single-chip Addressing Mode, assuming it
+ * is the only device connected to the SMI master. In this mode it responds to
+ * all 32 possible SMI addresses, and thus maps directly the internal devices.
+ *
+ * When ADDR is non-zero, the chip uses Multi-chip Addressing Mode, allowing
+ * multiple devices to share the SMI interface. In this mode it responds to only
+ * 2 registers, used to indirectly access the internal SMI devices.
  */
-static int mv88e6xxx_reg_wait_ready(struct mii_bus *bus, int sw_addr)
+
+static int mv88e6xxx_smi_read(struct mv88e6xxx_chip *chip,
+                              int addr, int reg, u16 *val)
+{
+        if (!chip->smi_ops)
+                return -EOPNOTSUPP;
+
+        return chip->smi_ops->read(chip, addr, reg, val);
+}
+
+static int mv88e6xxx_smi_write(struct mv88e6xxx_chip *chip,
+                               int addr, int reg, u16 val)
+{
+        if (!chip->smi_ops)
+                return -EOPNOTSUPP;
+
+        return chip->smi_ops->write(chip, addr, reg, val);
+}
+
+static int mv88e6xxx_smi_single_chip_read(struct mv88e6xxx_chip *chip,
+                                          int addr, int reg, u16 *val)
+{
+        int ret;
+
+        ret = mdiobus_read_nested(chip->bus, addr, reg);
+        if (ret < 0)
+                return ret;
+
+        *val = ret & 0xffff;
+
+        return 0;
+}
+
+static int mv88e6xxx_smi_single_chip_write(struct mv88e6xxx_chip *chip,
+                                           int addr, int reg, u16 val)
+{
+        int ret;
+
+        ret = mdiobus_write_nested(chip->bus, addr, reg, val);
+        if (ret < 0)
+                return ret;
+
+        return 0;
+}
+
+static const struct mv88e6xxx_ops mv88e6xxx_smi_single_chip_ops = {
+        .read = mv88e6xxx_smi_single_chip_read,
+        .write = mv88e6xxx_smi_single_chip_write,
+};
+
+static int mv88e6xxx_smi_multi_chip_wait(struct mv88e6xxx_chip *chip)
 {
         int ret;
         int i;
 
         for (i = 0; i < 16; i++) {
-                ret = mdiobus_read_nested(bus, sw_addr, SMI_CMD);
+                ret = mdiobus_read_nested(chip->bus, chip->sw_addr, SMI_CMD);
                 if (ret < 0)
                         return ret;
 
@@ -61,143 +117,168 @@ static int mv88e6xxx_reg_wait_ready(struct mii_bus *bus, int sw_addr)
         return -ETIMEDOUT;
 }
 
-static int __mv88e6xxx_reg_read(struct mii_bus *bus, int sw_addr, int addr,
-                                int reg)
+static int mv88e6xxx_smi_multi_chip_read(struct mv88e6xxx_chip *chip,
+                                         int addr, int reg, u16 *val)
 {
         int ret;
 
-        if (sw_addr == 0)
-                return mdiobus_read_nested(bus, addr, reg);
-
         /* Wait for the bus to become free. */
-        ret = mv88e6xxx_reg_wait_ready(bus, sw_addr);
+        ret = mv88e6xxx_smi_multi_chip_wait(chip);
         if (ret < 0)
                 return ret;
 
         /* Transmit the read command. */
-        ret = mdiobus_write_nested(bus, sw_addr, SMI_CMD,
+        ret = mdiobus_write_nested(chip->bus, chip->sw_addr, SMI_CMD,
                                    SMI_CMD_OP_22_READ | (addr << 5) | reg);
         if (ret < 0)
                 return ret;
 
         /* Wait for the read command to complete. */
-        ret = mv88e6xxx_reg_wait_ready(bus, sw_addr);
+        ret = mv88e6xxx_smi_multi_chip_wait(chip);
         if (ret < 0)
                 return ret;
 
         /* Read the data. */
-        ret = mdiobus_read_nested(bus, sw_addr, SMI_DATA);
-        if (ret < 0)
-                return ret;
-
-        return ret & 0xffff;
-}
-
-static int _mv88e6xxx_reg_read(struct mv88e6xxx_priv_state *ps,
-                               int addr, int reg)
-{
-        int ret;
-
-        assert_smi_lock(ps);
-
-        ret = __mv88e6xxx_reg_read(ps->bus, ps->sw_addr, addr, reg);
+        ret = mdiobus_read_nested(chip->bus, chip->sw_addr, SMI_DATA);
         if (ret < 0)
                 return ret;
 
-        dev_dbg(ps->dev, "<- addr: 0x%.2x reg: 0x%.2x val: 0x%.4x\n",
-                addr, reg, ret);
-
-        return ret;
-}
-
-int mv88e6xxx_reg_read(struct mv88e6xxx_priv_state *ps, int addr, int reg)
-{
-        int ret;
+        *val = ret & 0xffff;
 
-        mutex_lock(&ps->smi_mutex);
-        ret = _mv88e6xxx_reg_read(ps, addr, reg);
-        mutex_unlock(&ps->smi_mutex);
-
-        return ret;
+        return 0;
 }
 
-static int __mv88e6xxx_reg_write(struct mii_bus *bus, int sw_addr, int addr,
-                                 int reg, u16 val)
+static int mv88e6xxx_smi_multi_chip_write(struct mv88e6xxx_chip *chip,
+                                          int addr, int reg, u16 val)
 {
         int ret;
 
-        if (sw_addr == 0)
-                return mdiobus_write_nested(bus, addr, reg, val);
-
         /* Wait for the bus to become free. */
-        ret = mv88e6xxx_reg_wait_ready(bus, sw_addr);
+        ret = mv88e6xxx_smi_multi_chip_wait(chip);
         if (ret < 0)
                 return ret;
 
         /* Transmit the data to write. */
-        ret = mdiobus_write_nested(bus, sw_addr, SMI_DATA, val);
+        ret = mdiobus_write_nested(chip->bus, chip->sw_addr, SMI_DATA, val);
         if (ret < 0)
                 return ret;
 
         /* Transmit the write command. */
-        ret = mdiobus_write_nested(bus, sw_addr, SMI_CMD,
+        ret = mdiobus_write_nested(chip->bus, chip->sw_addr, SMI_CMD,
                                    SMI_CMD_OP_22_WRITE | (addr << 5) | reg);
         if (ret < 0)
                 return ret;
 
         /* Wait for the write command to complete. */
-        ret = mv88e6xxx_reg_wait_ready(bus, sw_addr);
+        ret = mv88e6xxx_smi_multi_chip_wait(chip);
         if (ret < 0)
                 return ret;
 
         return 0;
 }
 
-static int _mv88e6xxx_reg_write(struct mv88e6xxx_priv_state *ps, int addr,
-                                int reg, u16 val)
+static const struct mv88e6xxx_ops mv88e6xxx_smi_multi_chip_ops = {
+        .read = mv88e6xxx_smi_multi_chip_read,
+        .write = mv88e6xxx_smi_multi_chip_write,
+};
+
+static int mv88e6xxx_read(struct mv88e6xxx_chip *chip,
+                          int addr, int reg, u16 *val)
 {
-        assert_smi_lock(ps);
+        int err;
+
+        assert_reg_lock(chip);
+
+        err = mv88e6xxx_smi_read(chip, addr, reg, val);
+        if (err)
+                return err;
+
+        dev_dbg(chip->dev, "<- addr: 0x%.2x reg: 0x%.2x val: 0x%.4x\n",
+                addr, reg, *val);
+
+        return 0;
+}
+
+static int mv88e6xxx_write(struct mv88e6xxx_chip *chip,
+                           int addr, int reg, u16 val)
+{
+        int err;
 
-        dev_dbg(ps->dev, "-> addr: 0x%.2x reg: 0x%.2x val: 0x%.4x\n",
+        assert_reg_lock(chip);
+
+        err = mv88e6xxx_smi_write(chip, addr, reg, val);
+        if (err)
+                return err;
+
+        dev_dbg(chip->dev, "-> addr: 0x%.2x reg: 0x%.2x val: 0x%.4x\n",
                 addr, reg, val);
 
-        return __mv88e6xxx_reg_write(ps->bus, ps->sw_addr, addr, reg, val);
+        return 0;
+}
+
+static int _mv88e6xxx_reg_read(struct mv88e6xxx_chip *chip, int addr, int reg)
+{
+        u16 val;
+        int err;
+
+        err = mv88e6xxx_read(chip, addr, reg, &val);
+        if (err)
+                return err;
+
+        return val;
 }
 
-int mv88e6xxx_reg_write(struct mv88e6xxx_priv_state *ps, int addr,
-                        int reg, u16 val)
+static int mv88e6xxx_reg_read(struct mv88e6xxx_chip *chip, int addr, int reg)
 {
         int ret;
 
-        mutex_lock(&ps->smi_mutex);
-        ret = _mv88e6xxx_reg_write(ps, addr, reg, val);
-        mutex_unlock(&ps->smi_mutex);
+        mutex_lock(&chip->reg_lock);
+        ret = _mv88e6xxx_reg_read(chip, addr, reg);
+        mutex_unlock(&chip->reg_lock);
+
+        return ret;
+}
+
+static int _mv88e6xxx_reg_write(struct mv88e6xxx_chip *chip, int addr,
+                                int reg, u16 val)
+{
+        return mv88e6xxx_write(chip, addr, reg, val);
+}
+
+static int mv88e6xxx_reg_write(struct mv88e6xxx_chip *chip, int addr,
+                               int reg, u16 val)
+{
+        int ret;
+
+        mutex_lock(&chip->reg_lock);
+        ret = _mv88e6xxx_reg_write(chip, addr, reg, val);
+        mutex_unlock(&chip->reg_lock);
 
         return ret;
 }
 
 static int mv88e6xxx_set_addr_direct(struct dsa_switch *ds, u8 *addr)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         int err;
 
-        err = mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_MAC_01,
+        err = mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_MAC_01,
                                   (addr[0] << 8) | addr[1]);
         if (err)
                 return err;
 
-        err = mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_MAC_23,
+        err = mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_MAC_23,
                                   (addr[2] << 8) | addr[3]);
         if (err)
                 return err;
 
-        return mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_MAC_45,
+        return mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_MAC_45,
                                    (addr[4] << 8) | addr[5]);
 }
 
 static int mv88e6xxx_set_addr_indirect(struct dsa_switch *ds, u8 *addr)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         int ret;
         int i;
 
@@ -205,7 +286,7 @@ static int mv88e6xxx_set_addr_indirect(struct dsa_switch *ds, u8 *addr)
                 int j;
 
                 /* Write the MAC address byte. */
-                ret = mv88e6xxx_reg_write(ps, REG_GLOBAL2, GLOBAL2_SWITCH_MAC,
+                ret = mv88e6xxx_reg_write(chip, REG_GLOBAL2, GLOBAL2_SWITCH_MAC,
                                           GLOBAL2_SWITCH_MAC_BUSY |
                                           (i << 8) | addr[i]);
                 if (ret)
@@ -213,7 +294,7 @@ static int mv88e6xxx_set_addr_indirect(struct dsa_switch *ds, u8 *addr)
 
                 /* Wait for the write to complete. */
                 for (j = 0; j < 16; j++) {
-                        ret = mv88e6xxx_reg_read(ps, REG_GLOBAL2,
+                        ret = mv88e6xxx_reg_read(chip, REG_GLOBAL2,
                                                  GLOBAL2_SWITCH_MAC);
                         if (ret < 0)
                                 return ret;
@@ -228,49 +309,49 @@ static int mv88e6xxx_set_addr_indirect(struct dsa_switch *ds, u8 *addr)
         return 0;
 }
 
-int mv88e6xxx_set_addr(struct dsa_switch *ds, u8 *addr)
+static int mv88e6xxx_set_addr(struct dsa_switch *ds, u8 *addr)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
 
-        if (mv88e6xxx_has(ps, MV88E6XXX_FLAG_SWITCH_MAC))
+        if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_SWITCH_MAC))
                 return mv88e6xxx_set_addr_indirect(ds, addr);
         else
                 return mv88e6xxx_set_addr_direct(ds, addr);
 }
 
-static int _mv88e6xxx_phy_read(struct mv88e6xxx_priv_state *ps, int addr,
-                               int regnum)
+static int mv88e6xxx_mdio_read_direct(struct mv88e6xxx_chip *chip,
+                                      int addr, int regnum)
 {
         if (addr >= 0)
-                return _mv88e6xxx_reg_read(ps, addr, regnum);
+                return _mv88e6xxx_reg_read(chip, addr, regnum);
         return 0xffff;
 }
 
-static int _mv88e6xxx_phy_write(struct mv88e6xxx_priv_state *ps, int addr,
-                                int regnum, u16 val)
+static int mv88e6xxx_mdio_write_direct(struct mv88e6xxx_chip *chip,
+                                       int addr, int regnum, u16 val)
 {
         if (addr >= 0)
-                return _mv88e6xxx_reg_write(ps, addr, regnum, val);
+                return _mv88e6xxx_reg_write(chip, addr, regnum, val);
         return 0;
 }
 
-static int mv88e6xxx_ppu_disable(struct mv88e6xxx_priv_state *ps)
+static int mv88e6xxx_ppu_disable(struct mv88e6xxx_chip *chip)
 {
         int ret;
         unsigned long timeout;
 
-        ret = _mv88e6xxx_reg_read(ps, REG_GLOBAL, GLOBAL_CONTROL);
+        ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, GLOBAL_CONTROL);
         if (ret < 0)
                 return ret;
 
-        ret = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_CONTROL,
+        ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_CONTROL,
                                    ret & ~GLOBAL_CONTROL_PPU_ENABLE);
         if (ret)
                 return ret;
 
         timeout = jiffies + 1 * HZ;
         while (time_before(jiffies, timeout)) {
-                ret = _mv88e6xxx_reg_read(ps, REG_GLOBAL, GLOBAL_STATUS);
+                ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, GLOBAL_STATUS);
                 if (ret < 0)
                         return ret;
 
@@ -283,23 +364,23 @@ static int mv88e6xxx_ppu_disable(struct mv88e6xxx_priv_state *ps)
         return -ETIMEDOUT;
 }
 
-static int mv88e6xxx_ppu_enable(struct mv88e6xxx_priv_state *ps)
+static int mv88e6xxx_ppu_enable(struct mv88e6xxx_chip *chip)
 {
         int ret, err;
         unsigned long timeout;
 
-        ret = mv88e6xxx_reg_read(ps, REG_GLOBAL, GLOBAL_CONTROL);
+        ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, GLOBAL_CONTROL);
         if (ret < 0)
                 return ret;
 
-        err = mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_CONTROL,
-                                  ret | GLOBAL_CONTROL_PPU_ENABLE);
+        err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_CONTROL,
+                                   ret | GLOBAL_CONTROL_PPU_ENABLE);
         if (err)
                 return err;
 
         timeout = jiffies + 1 * HZ;
         while (time_before(jiffies, timeout)) {
-                ret = mv88e6xxx_reg_read(ps, REG_GLOBAL, GLOBAL_STATUS);
+                ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, GLOBAL_STATUS);
                 if (ret < 0)
                         return ret;
 
@@ -314,143 +395,148 @@ static int mv88e6xxx_ppu_enable(struct mv88e6xxx_priv_state *ps)
 
 static void mv88e6xxx_ppu_reenable_work(struct work_struct *ugly)
 {
-        struct mv88e6xxx_priv_state *ps;
+        struct mv88e6xxx_chip *chip;
+
+        chip = container_of(ugly, struct mv88e6xxx_chip, ppu_work);
 
-        ps = container_of(ugly, struct mv88e6xxx_priv_state, ppu_work);
-        if (mutex_trylock(&ps->ppu_mutex)) {
-                if (mv88e6xxx_ppu_enable(ps) == 0)
-                        ps->ppu_disabled = 0;
-                mutex_unlock(&ps->ppu_mutex);
+        mutex_lock(&chip->reg_lock);
+
+        if (mutex_trylock(&chip->ppu_mutex)) {
+                if (mv88e6xxx_ppu_enable(chip) == 0)
+                        chip->ppu_disabled = 0;
+                mutex_unlock(&chip->ppu_mutex);
         }
+
+        mutex_unlock(&chip->reg_lock);
 }
 
 static void mv88e6xxx_ppu_reenable_timer(unsigned long _ps)
 {
-        struct mv88e6xxx_priv_state *ps = (void *)_ps;
+        struct mv88e6xxx_chip *chip = (void *)_ps;
 
-        schedule_work(&ps->ppu_work);
+        schedule_work(&chip->ppu_work);
 }
 
-static int mv88e6xxx_ppu_access_get(struct mv88e6xxx_priv_state *ps)
+static int mv88e6xxx_ppu_access_get(struct mv88e6xxx_chip *chip)
 {
         int ret;
 
-        mutex_lock(&ps->ppu_mutex);
+        mutex_lock(&chip->ppu_mutex);
 
         /* If the PHY polling unit is enabled, disable it so that
          * we can access the PHY registers.  If it was already
          * disabled, cancel the timer that is going to re-enable
          * it.
          */
-        if (!ps->ppu_disabled) {
-                ret = mv88e6xxx_ppu_disable(ps);
+        if (!chip->ppu_disabled) {
+                ret = mv88e6xxx_ppu_disable(chip);
                 if (ret < 0) {
-                        mutex_unlock(&ps->ppu_mutex);
+                        mutex_unlock(&chip->ppu_mutex);
                         return ret;
                 }
-                ps->ppu_disabled = 1;
+                chip->ppu_disabled = 1;
         } else {
-                del_timer(&ps->ppu_timer);
+                del_timer(&chip->ppu_timer);
                 ret = 0;
         }
 
         return ret;
 }
 
-static void mv88e6xxx_ppu_access_put(struct mv88e6xxx_priv_state *ps)
+static void mv88e6xxx_ppu_access_put(struct mv88e6xxx_chip *chip)
 {
         /* Schedule a timer to re-enable the PHY polling unit. */
-        mod_timer(&ps->ppu_timer, jiffies + msecs_to_jiffies(10));
-        mutex_unlock(&ps->ppu_mutex);
+        mod_timer(&chip->ppu_timer, jiffies + msecs_to_jiffies(10));
+        mutex_unlock(&chip->ppu_mutex);
 }
 
-void mv88e6xxx_ppu_state_init(struct mv88e6xxx_priv_state *ps)
+static void mv88e6xxx_ppu_state_init(struct mv88e6xxx_chip *chip)
 {
-        mutex_init(&ps->ppu_mutex);
-        INIT_WORK(&ps->ppu_work, mv88e6xxx_ppu_reenable_work);
-        init_timer(&ps->ppu_timer);
-        ps->ppu_timer.data = (unsigned long)ps;
-        ps->ppu_timer.function = mv88e6xxx_ppu_reenable_timer;
+        mutex_init(&chip->ppu_mutex);
+        INIT_WORK(&chip->ppu_work, mv88e6xxx_ppu_reenable_work);
+        init_timer(&chip->ppu_timer);
+        chip->ppu_timer.data = (unsigned long)chip;
+        chip->ppu_timer.function = mv88e6xxx_ppu_reenable_timer;
 }
 
-static int mv88e6xxx_phy_read_ppu(struct mv88e6xxx_priv_state *ps, int addr,
-                                  int regnum)
+static int mv88e6xxx_mdio_read_ppu(struct mv88e6xxx_chip *chip, int addr,
+                                   int regnum)
 {
         int ret;
 
-        ret = mv88e6xxx_ppu_access_get(ps);
+        ret = mv88e6xxx_ppu_access_get(chip);
         if (ret >= 0) {
-                ret = _mv88e6xxx_reg_read(ps, addr, regnum);
-                mv88e6xxx_ppu_access_put(ps);
+                ret = _mv88e6xxx_reg_read(chip, addr, regnum);
+                mv88e6xxx_ppu_access_put(chip);
         }
 
         return ret;
 }
 
-static int mv88e6xxx_phy_write_ppu(struct mv88e6xxx_priv_state *ps, int addr,
-                                   int regnum, u16 val)
+static int mv88e6xxx_mdio_write_ppu(struct mv88e6xxx_chip *chip, int addr,
+                                    int regnum, u16 val)
 {
         int ret;
 
-        ret = mv88e6xxx_ppu_access_get(ps);
+        ret = mv88e6xxx_ppu_access_get(chip);
         if (ret >= 0) {
-                ret = _mv88e6xxx_reg_write(ps, addr, regnum, val);
-                mv88e6xxx_ppu_access_put(ps);
+                ret = _mv88e6xxx_reg_write(chip, addr, regnum, val);
+                mv88e6xxx_ppu_access_put(chip);
         }
 
         return ret;
 }
 
-static bool mv88e6xxx_6065_family(struct mv88e6xxx_priv_state *ps)
+static bool mv88e6xxx_6065_family(struct mv88e6xxx_chip *chip)
 {
-        return ps->info->family == MV88E6XXX_FAMILY_6065;
+        return chip->info->family == MV88E6XXX_FAMILY_6065;
 }
 
-static bool mv88e6xxx_6095_family(struct mv88e6xxx_priv_state *ps)
+static bool mv88e6xxx_6095_family(struct mv88e6xxx_chip *chip)
 {
-        return ps->info->family == MV88E6XXX_FAMILY_6095;
+        return chip->info->family == MV88E6XXX_FAMILY_6095;
 }
 
-static bool mv88e6xxx_6097_family(struct mv88e6xxx_priv_state *ps)
+static bool mv88e6xxx_6097_family(struct mv88e6xxx_chip *chip)
 {
-        return ps->info->family == MV88E6XXX_FAMILY_6097;
+        return chip->info->family == MV88E6XXX_FAMILY_6097;
 }
 
-static bool mv88e6xxx_6165_family(struct mv88e6xxx_priv_state *ps)
+static bool mv88e6xxx_6165_family(struct mv88e6xxx_chip *chip)
 {
-        return ps->info->family == MV88E6XXX_FAMILY_6165;
+        return chip->info->family == MV88E6XXX_FAMILY_6165;
 }
 
-static bool mv88e6xxx_6185_family(struct mv88e6xxx_priv_state *ps)
+static bool mv88e6xxx_6185_family(struct mv88e6xxx_chip *chip)
 {
-        return ps->info->family == MV88E6XXX_FAMILY_6185;
+        return chip->info->family == MV88E6XXX_FAMILY_6185;
 }
 
-static bool mv88e6xxx_6320_family(struct mv88e6xxx_priv_state *ps)
+static bool mv88e6xxx_6320_family(struct mv88e6xxx_chip *chip)
 {
-        return ps->info->family == MV88E6XXX_FAMILY_6320;
+        return chip->info->family == MV88E6XXX_FAMILY_6320;
 }
 
-static bool mv88e6xxx_6351_family(struct mv88e6xxx_priv_state *ps)
+static bool mv88e6xxx_6351_family(struct mv88e6xxx_chip *chip)
 {
-        return ps->info->family == MV88E6XXX_FAMILY_6351;
+        return chip->info->family == MV88E6XXX_FAMILY_6351;
 }
 
-static bool mv88e6xxx_6352_family(struct mv88e6xxx_priv_state *ps)
+static bool mv88e6xxx_6352_family(struct mv88e6xxx_chip *chip)
 {
-        return ps->info->family == MV88E6XXX_FAMILY_6352;
+        return chip->info->family == MV88E6XXX_FAMILY_6352;
 }
 
-static unsigned int mv88e6xxx_num_databases(struct mv88e6xxx_priv_state *ps)
+static unsigned int mv88e6xxx_num_databases(struct mv88e6xxx_chip *chip)
 {
-        return ps->info->num_databases;
+        return chip->info->num_databases;
 }
 
-static bool mv88e6xxx_has_fid_reg(struct mv88e6xxx_priv_state *ps)
+static bool mv88e6xxx_has_fid_reg(struct mv88e6xxx_chip *chip)
 {
         /* Does the device have dedicated FID registers for ATU and VTU ops? */
-        if (mv88e6xxx_6097_family(ps) || mv88e6xxx_6165_family(ps) ||
-            mv88e6xxx_6351_family(ps) || mv88e6xxx_6352_family(ps))
+        if (mv88e6xxx_6097_family(chip) || mv88e6xxx_6165_family(chip) ||
+            mv88e6xxx_6351_family(chip) || mv88e6xxx_6352_family(chip))
                 return true;
 
         return false;
@@ -463,16 +549,16 @@ static bool mv88e6xxx_has_fid_reg(struct mv88e6xxx_priv_state *ps)
 static void mv88e6xxx_adjust_link(struct dsa_switch *ds, int port,
                                   struct phy_device *phydev)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         u32 reg;
         int ret;
 
         if (!phy_is_pseudo_fixed_link(phydev))
                 return;
 
-        mutex_lock(&ps->smi_mutex);
+        mutex_lock(&chip->reg_lock);
 
-        ret = _mv88e6xxx_reg_read(ps, REG_PORT(port), PORT_PCS_CTRL);
+        ret = _mv88e6xxx_reg_read(chip, REG_PORT(port), PORT_PCS_CTRL);
         if (ret < 0)
                 goto out;
 
@@ -484,9 +570,9 @@ static void mv88e6xxx_adjust_link(struct dsa_switch *ds, int port,
 
         reg |= PORT_PCS_CTRL_FORCE_LINK;
         if (phydev->link)
-                        reg |= PORT_PCS_CTRL_LINK_UP;
+                reg |= PORT_PCS_CTRL_LINK_UP;
 
-        if (mv88e6xxx_6065_family(ps) && phydev->speed > SPEED_100)
+        if (mv88e6xxx_6065_family(chip) && phydev->speed > SPEED_100)
                 goto out;
 
         switch (phydev->speed) {
@@ -508,8 +594,8 @@ static void mv88e6xxx_adjust_link(struct dsa_switch *ds, int port,
         if (phydev->duplex == DUPLEX_FULL)
                 reg |= PORT_PCS_CTRL_DUPLEX_FULL;
 
-        if ((mv88e6xxx_6352_family(ps) || mv88e6xxx_6351_family(ps)) &&
-            (port >= ps->info->num_ports - 2)) {
+        if ((mv88e6xxx_6352_family(chip) || mv88e6xxx_6351_family(chip)) &&
+            (port >= chip->info->num_ports - 2)) {
                 if (phydev->interface == PHY_INTERFACE_MODE_RGMII_RXID)
                         reg |= PORT_PCS_CTRL_RGMII_DELAY_RXCLK;
                 if (phydev->interface == PHY_INTERFACE_MODE_RGMII_TXID)
@@ -518,19 +604,19 @@ static void mv88e6xxx_adjust_link(struct dsa_switch *ds, int port,
                         reg |= (PORT_PCS_CTRL_RGMII_DELAY_RXCLK |
                                 PORT_PCS_CTRL_RGMII_DELAY_TXCLK);
         }
-        _mv88e6xxx_reg_write(ps, REG_PORT(port), PORT_PCS_CTRL, reg);
+        _mv88e6xxx_reg_write(chip, REG_PORT(port), PORT_PCS_CTRL, reg);
 
 out:
-        mutex_unlock(&ps->smi_mutex);
+        mutex_unlock(&chip->reg_lock);
 }
 
-static int _mv88e6xxx_stats_wait(struct mv88e6xxx_priv_state *ps)
+static int _mv88e6xxx_stats_wait(struct mv88e6xxx_chip *chip)
 {
         int ret;
         int i;
 
         for (i = 0; i < 10; i++) {
-                ret = _mv88e6xxx_reg_read(ps, REG_GLOBAL, GLOBAL_STATS_OP);
+                ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, GLOBAL_STATS_OP);
                 if ((ret & GLOBAL_STATS_OP_BUSY) == 0)
                         return 0;
         }
@@ -538,30 +624,29 @@ static int _mv88e6xxx_stats_wait(struct mv88e6xxx_priv_state *ps)
         return -ETIMEDOUT;
 }
 
-static int _mv88e6xxx_stats_snapshot(struct mv88e6xxx_priv_state *ps,
-                                     int port)
+static int _mv88e6xxx_stats_snapshot(struct mv88e6xxx_chip *chip, int port)
 {
         int ret;
 
-        if (mv88e6xxx_6320_family(ps) || mv88e6xxx_6352_family(ps))
+        if (mv88e6xxx_6320_family(chip) || mv88e6xxx_6352_family(chip))
                 port = (port + 1) << 5;
 
         /* Snapshot the hardware statistics counters for this port. */
-        ret = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_STATS_OP,
+        ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_STATS_OP,
                                    GLOBAL_STATS_OP_CAPTURE_PORT |
                                    GLOBAL_STATS_OP_HIST_RX_TX | port);
         if (ret < 0)
                 return ret;
 
         /* Wait for the snapshotting to complete. */
-        ret = _mv88e6xxx_stats_wait(ps);
+        ret = _mv88e6xxx_stats_wait(chip);
         if (ret < 0)
                 return ret;
 
         return 0;
 }
 
-static void _mv88e6xxx_stats_read(struct mv88e6xxx_priv_state *ps,
+static void _mv88e6xxx_stats_read(struct mv88e6xxx_chip *chip,
                                   int stat, u32 *val)
 {
         u32 _val;
@@ -569,23 +654,23 @@ static void _mv88e6xxx_stats_read(struct mv88e6xxx_priv_state *ps,
 
         *val = 0;
 
-        ret = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_STATS_OP,
+        ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_STATS_OP,
                                    GLOBAL_STATS_OP_READ_CAPTURED |
                                    GLOBAL_STATS_OP_HIST_RX_TX | stat);
         if (ret < 0)
                 return;
 
-        ret = _mv88e6xxx_stats_wait(ps);
+        ret = _mv88e6xxx_stats_wait(chip);
         if (ret < 0)
                 return;
 
-        ret = _mv88e6xxx_reg_read(ps, REG_GLOBAL, GLOBAL_STATS_COUNTER_32);
+        ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, GLOBAL_STATS_COUNTER_32);
         if (ret < 0)
                 return;
 
         _val = ret << 16;
 
-        ret = _mv88e6xxx_reg_read(ps, REG_GLOBAL, GLOBAL_STATS_COUNTER_01);
+        ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, GLOBAL_STATS_COUNTER_01);
         if (ret < 0)
                 return;
 
@@ -654,26 +739,26 @@ static struct mv88e6xxx_hw_stat mv88e6xxx_hw_stats[] = {
         { "out_management",     4, 0x1f | GLOBAL_STATS_OP_BANK_1, BANK1, },
 };
 
-static bool mv88e6xxx_has_stat(struct mv88e6xxx_priv_state *ps,
+static bool mv88e6xxx_has_stat(struct mv88e6xxx_chip *chip,
                                struct mv88e6xxx_hw_stat *stat)
 {
         switch (stat->type) {
         case BANK0:
                 return true;
         case BANK1:
-                return mv88e6xxx_6320_family(ps);
+                return mv88e6xxx_6320_family(chip);
         case PORT:
-                return mv88e6xxx_6095_family(ps) ||
-                        mv88e6xxx_6185_family(ps) ||
-                        mv88e6xxx_6097_family(ps) ||
-                        mv88e6xxx_6165_family(ps) ||
-                        mv88e6xxx_6351_family(ps) ||
-                        mv88e6xxx_6352_family(ps);
+                return mv88e6xxx_6095_family(chip) ||
+                        mv88e6xxx_6185_family(chip) ||
+                        mv88e6xxx_6097_family(chip) ||
+                        mv88e6xxx_6165_family(chip) ||
+                        mv88e6xxx_6351_family(chip) ||
+                        mv88e6xxx_6352_family(chip);
         }
         return false;
 }
 
-static uint64_t _mv88e6xxx_get_ethtool_stat(struct mv88e6xxx_priv_state *ps,
+static uint64_t _mv88e6xxx_get_ethtool_stat(struct mv88e6xxx_chip *chip,
                                             struct mv88e6xxx_hw_stat *s,
                                             int port)
 {
@@ -684,13 +769,13 @@ static uint64_t _mv88e6xxx_get_ethtool_stat(struct mv88e6xxx_priv_state *ps,
 
         switch (s->type) {
         case PORT:
-                ret = _mv88e6xxx_reg_read(ps, REG_PORT(port), s->reg);
+                ret = _mv88e6xxx_reg_read(chip, REG_PORT(port), s->reg);
                 if (ret < 0)
                         return UINT64_MAX;
 
                 low = ret;
                 if (s->sizeof_stat == 4) {
-                        ret = _mv88e6xxx_reg_read(ps, REG_PORT(port),
+                        ret = _mv88e6xxx_reg_read(chip, REG_PORT(port),
                                                   s->reg + 1);
                         if (ret < 0)
                                 return UINT64_MAX;
@@ -699,9 +784,9 @@ static uint64_t _mv88e6xxx_get_ethtool_stat(struct mv88e6xxx_priv_state *ps,
                 break;
         case BANK0:
         case BANK1:
-                _mv88e6xxx_stats_read(ps, s->reg, &low);
+                _mv88e6xxx_stats_read(chip, s->reg, &low);
                 if (s->sizeof_stat == 8)
-                        _mv88e6xxx_stats_read(ps, s->reg + 1, &high);
+                        _mv88e6xxx_stats_read(chip, s->reg + 1, &high);
         }
         value = (((u64)high) << 16) | low;
         return value;
@@ -710,13 +795,13 @@ static uint64_t _mv88e6xxx_get_ethtool_stat(struct mv88e6xxx_priv_state *ps,
 static void mv88e6xxx_get_strings(struct dsa_switch *ds, int port,
                                   uint8_t *data)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         struct mv88e6xxx_hw_stat *stat;
         int i, j;
 
         for (i = 0, j = 0; i < ARRAY_SIZE(mv88e6xxx_hw_stats); i++) {
                 stat = &mv88e6xxx_hw_stats[i];
-                if (mv88e6xxx_has_stat(ps, stat)) {
+                if (mv88e6xxx_has_stat(chip, stat)) {
                         memcpy(data + j * ETH_GSTRING_LEN, stat->string,
                                ETH_GSTRING_LEN);
                         j++;
@@ -726,13 +811,13 @@ static void mv88e6xxx_get_strings(struct dsa_switch *ds, int port,
 
 static int mv88e6xxx_get_sset_count(struct dsa_switch *ds)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         struct mv88e6xxx_hw_stat *stat;
         int i, j;
 
         for (i = 0, j = 0; i < ARRAY_SIZE(mv88e6xxx_hw_stats); i++) {
                 stat = &mv88e6xxx_hw_stats[i];
-                if (mv88e6xxx_has_stat(ps, stat))
+                if (mv88e6xxx_has_stat(chip, stat))
                         j++;
         }
         return j;
@@ -741,27 +826,27 @@ static int mv88e6xxx_get_sset_count(struct dsa_switch *ds)
 static void mv88e6xxx_get_ethtool_stats(struct dsa_switch *ds, int port,
                                         uint64_t *data)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         struct mv88e6xxx_hw_stat *stat;
         int ret;
         int i, j;
 
-        mutex_lock(&ps->smi_mutex);
+        mutex_lock(&chip->reg_lock);
 
-        ret = _mv88e6xxx_stats_snapshot(ps, port);
+        ret = _mv88e6xxx_stats_snapshot(chip, port);
         if (ret < 0) {
-                mutex_unlock(&ps->smi_mutex);
+                mutex_unlock(&chip->reg_lock);
                 return;
         }
         for (i = 0, j = 0; i < ARRAY_SIZE(mv88e6xxx_hw_stats); i++) {
                 stat = &mv88e6xxx_hw_stats[i];
-                if (mv88e6xxx_has_stat(ps, stat)) {
-                        data[j] = _mv88e6xxx_get_ethtool_stat(ps, stat, port);
+                if (mv88e6xxx_has_stat(chip, stat)) {
+                        data[j] = _mv88e6xxx_get_ethtool_stat(chip, stat, port);
                         j++;
                 }
         }
 
-        mutex_unlock(&ps->smi_mutex);
+        mutex_unlock(&chip->reg_lock);
 }
 
 static int mv88e6xxx_get_regs_len(struct dsa_switch *ds, int port)
@@ -772,7 +857,7 @@ static int mv88e6xxx_get_regs_len(struct dsa_switch *ds, int port)
 static void mv88e6xxx_get_regs(struct dsa_switch *ds, int port,
                                struct ethtool_regs *regs, void *_p)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         u16 *p = _p;
         int i;
 
@@ -780,20 +865,20 @@ static void mv88e6xxx_get_regs(struct dsa_switch *ds, int port,
 
         memset(p, 0xff, 32 * sizeof(u16));
 
-        mutex_lock(&ps->smi_mutex);
+        mutex_lock(&chip->reg_lock);
 
         for (i = 0; i < 32; i++) {
                 int ret;
 
-                ret = _mv88e6xxx_reg_read(ps, REG_PORT(port), i);
+                ret = _mv88e6xxx_reg_read(chip, REG_PORT(port), i);
                 if (ret >= 0)
                         p[i] = ret;
         }
 
-        mutex_unlock(&ps->smi_mutex);
+        mutex_unlock(&chip->reg_lock);
 }
 
-static int _mv88e6xxx_wait(struct mv88e6xxx_priv_state *ps, int reg, int offset,
+static int _mv88e6xxx_wait(struct mv88e6xxx_chip *chip, int reg, int offset,
                            u16 mask)
 {
         unsigned long timeout = jiffies + HZ / 10;
@@ -801,7 +886,7 @@ static int _mv88e6xxx_wait(struct mv88e6xxx_priv_state *ps, int reg, int offset,
         while (time_before(jiffies, timeout)) {
                 int ret;
 
-                ret = _mv88e6xxx_reg_read(ps, reg, offset);
+                ret = _mv88e6xxx_reg_read(chip, reg, offset);
                 if (ret < 0)
                         return ret;
                 if (!(ret & mask))
@@ -812,48 +897,48 @@ static int _mv88e6xxx_wait(struct mv88e6xxx_priv_state *ps, int reg, int offset,
         return -ETIMEDOUT;
 }
 
-static int mv88e6xxx_wait(struct mv88e6xxx_priv_state *ps, int reg,
+static int mv88e6xxx_wait(struct mv88e6xxx_chip *chip, int reg,
                           int offset, u16 mask)
 {
         int ret;
 
-        mutex_lock(&ps->smi_mutex);
-        ret = _mv88e6xxx_wait(ps, reg, offset, mask);
-        mutex_unlock(&ps->smi_mutex);
+        mutex_lock(&chip->reg_lock);
+        ret = _mv88e6xxx_wait(chip, reg, offset, mask);
+        mutex_unlock(&chip->reg_lock);
 
         return ret;
 }
 
-static int _mv88e6xxx_phy_wait(struct mv88e6xxx_priv_state *ps)
+static int mv88e6xxx_mdio_wait(struct mv88e6xxx_chip *chip)
 {
-        return _mv88e6xxx_wait(ps, REG_GLOBAL2, GLOBAL2_SMI_OP,
+        return _mv88e6xxx_wait(chip, REG_GLOBAL2, GLOBAL2_SMI_OP,
                                GLOBAL2_SMI_OP_BUSY);
 }
 
 static int mv88e6xxx_eeprom_load_wait(struct dsa_switch *ds)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
 
-        return mv88e6xxx_wait(ps, REG_GLOBAL2, GLOBAL2_EEPROM_OP,
+        return mv88e6xxx_wait(chip, REG_GLOBAL2, GLOBAL2_EEPROM_OP,
                               GLOBAL2_EEPROM_OP_LOAD);
 }
 
 static int mv88e6xxx_eeprom_busy_wait(struct dsa_switch *ds)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
 
-        return mv88e6xxx_wait(ps, REG_GLOBAL2, GLOBAL2_EEPROM_OP,
+        return mv88e6xxx_wait(chip, REG_GLOBAL2, GLOBAL2_EEPROM_OP,
                               GLOBAL2_EEPROM_OP_BUSY);
 }
 
 static int mv88e6xxx_read_eeprom_word(struct dsa_switch *ds, int addr)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         int ret;
 
-        mutex_lock(&ps->eeprom_mutex);
+        mutex_lock(&chip->eeprom_mutex);
 
-        ret = mv88e6xxx_reg_write(ps, REG_GLOBAL2, GLOBAL2_EEPROM_OP,
+        ret = mv88e6xxx_reg_write(chip, REG_GLOBAL2, GLOBAL2_EEPROM_OP,
                                   GLOBAL2_EEPROM_OP_READ |
                                   (addr & GLOBAL2_EEPROM_OP_ADDR_MASK));
         if (ret < 0)
@@ -863,18 +948,18 @@ static int mv88e6xxx_read_eeprom_word(struct dsa_switch *ds, int addr)
         if (ret < 0)
                 goto error;
 
-        ret = mv88e6xxx_reg_read(ps, REG_GLOBAL2, GLOBAL2_EEPROM_DATA);
+        ret = mv88e6xxx_reg_read(chip, REG_GLOBAL2, GLOBAL2_EEPROM_DATA);
 error:
-        mutex_unlock(&ps->eeprom_mutex);
+        mutex_unlock(&chip->eeprom_mutex);
         return ret;
 }
 
 static int mv88e6xxx_get_eeprom_len(struct dsa_switch *ds)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
 
-        if (mv88e6xxx_has(ps, MV88E6XXX_FLAG_EEPROM))
-                return ps->eeprom_len;
+        if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_EEPROM))
+                return chip->eeprom_len;
 
         return 0;
 }
@@ -882,12 +967,12 @@ static int mv88e6xxx_get_eeprom_len(struct dsa_switch *ds)
 static int mv88e6xxx_get_eeprom(struct dsa_switch *ds,
                                 struct ethtool_eeprom *eeprom, u8 *data)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         int offset;
         int len;
         int ret;
 
-        if (!mv88e6xxx_has(ps, MV88E6XXX_FLAG_EEPROM))
+        if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_EEPROM))
                 return -EOPNOTSUPP;
 
         offset = eeprom->offset;
@@ -948,10 +1033,10 @@ static int mv88e6xxx_get_eeprom(struct dsa_switch *ds,
 
 static int mv88e6xxx_eeprom_is_readonly(struct dsa_switch *ds)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         int ret;
 
-        ret = mv88e6xxx_reg_read(ps, REG_GLOBAL2, GLOBAL2_EEPROM_OP);
+        ret = mv88e6xxx_reg_read(chip, REG_GLOBAL2, GLOBAL2_EEPROM_OP);
         if (ret < 0)
                 return ret;
 
@@ -964,16 +1049,16 @@ static int mv88e6xxx_eeprom_is_readonly(struct dsa_switch *ds)
 static int mv88e6xxx_write_eeprom_word(struct dsa_switch *ds, int addr,
                                        u16 data)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         int ret;
 
-        mutex_lock(&ps->eeprom_mutex);
+        mutex_lock(&chip->eeprom_mutex);
 
-        ret = mv88e6xxx_reg_write(ps, REG_GLOBAL2, GLOBAL2_EEPROM_DATA, data);
+        ret = mv88e6xxx_reg_write(chip, REG_GLOBAL2, GLOBAL2_EEPROM_DATA, data);
         if (ret < 0)
                 goto error;
 
-        ret = mv88e6xxx_reg_write(ps, REG_GLOBAL2, GLOBAL2_EEPROM_OP,
+        ret = mv88e6xxx_reg_write(chip, REG_GLOBAL2, GLOBAL2_EEPROM_OP,
                                   GLOBAL2_EEPROM_OP_WRITE |
                                   (addr & GLOBAL2_EEPROM_OP_ADDR_MASK));
         if (ret < 0)
@@ -981,19 +1066,19 @@ static int mv88e6xxx_write_eeprom_word(struct dsa_switch *ds, int addr,
 
         ret = mv88e6xxx_eeprom_busy_wait(ds);
 error:
-        mutex_unlock(&ps->eeprom_mutex);
+        mutex_unlock(&chip->eeprom_mutex);
         return ret;
 }
 
 static int mv88e6xxx_set_eeprom(struct dsa_switch *ds,
                                 struct ethtool_eeprom *eeprom, u8 *data)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         int offset;
         int ret;
         int len;
 
-        if (!mv88e6xxx_has(ps, MV88E6XXX_FLAG_EEPROM))
+        if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_EEPROM))
                 return -EOPNOTSUPP;
 
         if (eeprom->magic != 0xc3ec4951)
@@ -1065,67 +1150,67 @@ static int mv88e6xxx_set_eeprom(struct dsa_switch *ds,
         return 0;
 }
 
-static int _mv88e6xxx_atu_wait(struct mv88e6xxx_priv_state *ps)
+static int _mv88e6xxx_atu_wait(struct mv88e6xxx_chip *chip)
 {
-        return _mv88e6xxx_wait(ps, REG_GLOBAL, GLOBAL_ATU_OP,
+        return _mv88e6xxx_wait(chip, REG_GLOBAL, GLOBAL_ATU_OP,
                                GLOBAL_ATU_OP_BUSY);
 }
 
-static int _mv88e6xxx_phy_read_indirect(struct mv88e6xxx_priv_state *ps,
+static int mv88e6xxx_mdio_read_indirect(struct mv88e6xxx_chip *chip,
                                         int addr, int regnum)
 {
         int ret;
 
-        ret = _mv88e6xxx_reg_write(ps, REG_GLOBAL2, GLOBAL2_SMI_OP,
+        ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL2, GLOBAL2_SMI_OP,
                                    GLOBAL2_SMI_OP_22_READ | (addr << 5) |
                                    regnum);
         if (ret < 0)
                 return ret;
 
-        ret = _mv88e6xxx_phy_wait(ps);
+        ret = mv88e6xxx_mdio_wait(chip);
         if (ret < 0)
                 return ret;
 
-        ret = _mv88e6xxx_reg_read(ps, REG_GLOBAL2, GLOBAL2_SMI_DATA);
+        ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL2, GLOBAL2_SMI_DATA);
 
         return ret;
 }
 
-static int _mv88e6xxx_phy_write_indirect(struct mv88e6xxx_priv_state *ps,
+static int mv88e6xxx_mdio_write_indirect(struct mv88e6xxx_chip *chip,
                                          int addr, int regnum, u16 val)
 {
         int ret;
 
-        ret = _mv88e6xxx_reg_write(ps, REG_GLOBAL2, GLOBAL2_SMI_DATA, val);
+        ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL2, GLOBAL2_SMI_DATA, val);
         if (ret < 0)
                 return ret;
 
-        ret = _mv88e6xxx_reg_write(ps, REG_GLOBAL2, GLOBAL2_SMI_OP,
+        ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL2, GLOBAL2_SMI_OP,
                                    GLOBAL2_SMI_OP_22_WRITE | (addr << 5) |
                                    regnum);
 
-        return _mv88e6xxx_phy_wait(ps);
+        return mv88e6xxx_mdio_wait(chip);
 }
 
 static int mv88e6xxx_get_eee(struct dsa_switch *ds, int port,
                              struct ethtool_eee *e)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         int reg;
 
-        if (!mv88e6xxx_has(ps, MV88E6XXX_FLAG_EEE))
+        if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_EEE))
                 return -EOPNOTSUPP;
 
-        mutex_lock(&ps->smi_mutex);
+        mutex_lock(&chip->reg_lock);
 
-        reg = _mv88e6xxx_phy_read_indirect(ps, port, 16);
+        reg = mv88e6xxx_mdio_read_indirect(chip, port, 16);
         if (reg < 0)
                 goto out;
 
         e->eee_enabled = !!(reg & 0x0200);
         e->tx_lpi_enabled = !!(reg & 0x0100);
 
-        reg = _mv88e6xxx_reg_read(ps, REG_PORT(port), PORT_STATUS);
+        reg = _mv88e6xxx_reg_read(chip, REG_PORT(port), PORT_STATUS);
         if (reg < 0)
                 goto out;
 
@@ -1133,23 +1218,23 @@ static int mv88e6xxx_get_eee(struct dsa_switch *ds, int port,
         reg = 0;
 
 out:
-        mutex_unlock(&ps->smi_mutex);
+        mutex_unlock(&chip->reg_lock);
         return reg;
 }
 
 static int mv88e6xxx_set_eee(struct dsa_switch *ds, int port,
                              struct phy_device *phydev, struct ethtool_eee *e)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         int reg;
         int ret;
 
-        if (!mv88e6xxx_has(ps, MV88E6XXX_FLAG_EEE))
+        if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_EEE))
                 return -EOPNOTSUPP;
 
-        mutex_lock(&ps->smi_mutex);
+        mutex_lock(&chip->reg_lock);
 
-        ret = _mv88e6xxx_phy_read_indirect(ps, port, 16);
+        ret = mv88e6xxx_mdio_read_indirect(chip, port, 16);
         if (ret < 0)
                 goto out;
 
@@ -1159,28 +1244,29 @@ static int mv88e6xxx_set_eee(struct dsa_switch *ds, int port,
         if (e->tx_lpi_enabled)
                 reg |= 0x0100;
 
-        ret = _mv88e6xxx_phy_write_indirect(ps, port, 16, reg);
+        ret = mv88e6xxx_mdio_write_indirect(chip, port, 16, reg);
 out:
-        mutex_unlock(&ps->smi_mutex);
+        mutex_unlock(&chip->reg_lock);
 
         return ret;
 }
 
-static int _mv88e6xxx_atu_cmd(struct mv88e6xxx_priv_state *ps, u16 fid, u16 cmd)
+static int _mv88e6xxx_atu_cmd(struct mv88e6xxx_chip *chip, u16 fid, u16 cmd)
 {
         int ret;
 
-        if (mv88e6xxx_has_fid_reg(ps)) {
-                ret = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_ATU_FID, fid);
+        if (mv88e6xxx_has_fid_reg(chip)) {
+                ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_ATU_FID,
+                                           fid);
                 if (ret < 0)
                         return ret;
-        } else if (mv88e6xxx_num_databases(ps) == 256) {
+        } else if (mv88e6xxx_num_databases(chip) == 256) {
                 /* ATU DBNum[7:4] are located in ATU Control 15:12 */
-                ret = _mv88e6xxx_reg_read(ps, REG_GLOBAL, GLOBAL_ATU_CONTROL);
+                ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, GLOBAL_ATU_CONTROL);
                 if (ret < 0)
                         return ret;
 
-                ret = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_ATU_CONTROL,
+                ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_ATU_CONTROL,
                                            (ret & 0xfff) |
                                            ((fid << 8) & 0xf000));
                 if (ret < 0)
@@ -1190,14 +1276,14 @@ static int _mv88e6xxx_atu_cmd(struct mv88e6xxx_priv_state *ps, u16 fid, u16 cmd)
                 cmd |= fid & 0xf;
         }
 
-        ret = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_ATU_OP, cmd);
+        ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_ATU_OP, cmd);
         if (ret < 0)
                 return ret;
 
-        return _mv88e6xxx_atu_wait(ps);
+        return _mv88e6xxx_atu_wait(chip);
 }
 
-static int _mv88e6xxx_atu_data_write(struct mv88e6xxx_priv_state *ps,
+static int _mv88e6xxx_atu_data_write(struct mv88e6xxx_chip *chip,
                                      struct mv88e6xxx_atu_entry *entry)
 {
         u16 data = entry->state & GLOBAL_ATU_DATA_STATE_MASK;
@@ -1217,21 +1303,21 @@ static int _mv88e6xxx_atu_data_write(struct mv88e6xxx_priv_state *ps,
                 data |= (entry->portv_trunkid << shift) & mask;
         }
 
-        return _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_ATU_DATA, data);
+        return _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_ATU_DATA, data);
 }
 
-static int _mv88e6xxx_atu_flush_move(struct mv88e6xxx_priv_state *ps,
+static int _mv88e6xxx_atu_flush_move(struct mv88e6xxx_chip *chip,
                                      struct mv88e6xxx_atu_entry *entry,
                                      bool static_too)
 {
         int op;
         int err;
 
-        err = _mv88e6xxx_atu_wait(ps);
+        err = _mv88e6xxx_atu_wait(chip);
         if (err)
                 return err;
 
-        err = _mv88e6xxx_atu_data_write(ps, entry);
+        err = _mv88e6xxx_atu_data_write(chip, entry);
         if (err)
                 return err;
 
@@ -1243,10 +1329,10 @@ static int _mv88e6xxx_atu_flush_move(struct mv88e6xxx_priv_state *ps,
                         GLOBAL_ATU_OP_FLUSH_MOVE_NON_STATIC;
         }
 
-        return _mv88e6xxx_atu_cmd(ps, entry->fid, op);
+        return _mv88e6xxx_atu_cmd(chip, entry->fid, op);
 }
 
-static int _mv88e6xxx_atu_flush(struct mv88e6xxx_priv_state *ps,
+static int _mv88e6xxx_atu_flush(struct mv88e6xxx_chip *chip,
                                 u16 fid, bool static_too)
 {
         struct mv88e6xxx_atu_entry entry = {
@@ -1254,10 +1340,10 @@ static int _mv88e6xxx_atu_flush(struct mv88e6xxx_priv_state *ps,
                 .state = 0, /* EntryState bits must be 0 */
         };
 
-        return _mv88e6xxx_atu_flush_move(ps, &entry, static_too);
+        return _mv88e6xxx_atu_flush_move(chip, &entry, static_too);
 }
 
-static int _mv88e6xxx_atu_move(struct mv88e6xxx_priv_state *ps, u16 fid,
+static int _mv88e6xxx_atu_move(struct mv88e6xxx_chip *chip, u16 fid,
                                int from_port, int to_port, bool static_too)
 {
         struct mv88e6xxx_atu_entry entry = {
@@ -1272,14 +1358,14 @@ static int _mv88e6xxx_atu_move(struct mv88e6xxx_priv_state *ps, u16 fid,
         entry.portv_trunkid = (to_port & 0x0f) << 4;
         entry.portv_trunkid |= from_port & 0x0f;
 
-        return _mv88e6xxx_atu_flush_move(ps, &entry, static_too);
+        return _mv88e6xxx_atu_flush_move(chip, &entry, static_too);
 }
 
-static int _mv88e6xxx_atu_remove(struct mv88e6xxx_priv_state *ps, u16 fid,
+static int _mv88e6xxx_atu_remove(struct mv88e6xxx_chip *chip, u16 fid,
                                  int port, bool static_too)
 {
         /* Destination port 0xF means remove the entries */
-        return _mv88e6xxx_atu_move(ps, fid, port, 0x0f, static_too);
+        return _mv88e6xxx_atu_move(chip, fid, port, 0x0f, static_too);
 }
 
 static const char * const mv88e6xxx_port_state_names[] = {
@@ -1289,14 +1375,14 @@ static const char * const mv88e6xxx_port_state_names[] = {
         [PORT_CONTROL_STATE_FORWARDING] = "Forwarding",
 };
 
-static int _mv88e6xxx_port_state(struct mv88e6xxx_priv_state *ps, int port,
+static int _mv88e6xxx_port_state(struct mv88e6xxx_chip *chip, int port,
                                  u8 state)
 {
-        struct dsa_switch *ds = ps->ds;
+        struct dsa_switch *ds = chip->ds;
         int reg, ret = 0;
         u8 oldstate;
 
-        reg = _mv88e6xxx_reg_read(ps, REG_PORT(port), PORT_CONTROL);
+        reg = _mv88e6xxx_reg_read(chip, REG_PORT(port), PORT_CONTROL);
         if (reg < 0)
                 return reg;
 
@@ -1308,21 +1394,21 @@ static int _mv88e6xxx_port_state(struct mv88e6xxx_priv_state *ps, int port,
                  * Blocking or Listening state.
                  */
                 if ((oldstate == PORT_CONTROL_STATE_LEARNING ||
-                     oldstate == PORT_CONTROL_STATE_FORWARDING)
-                    && (state == PORT_CONTROL_STATE_DISABLED ||
-                        state == PORT_CONTROL_STATE_BLOCKING)) {
-                        ret = _mv88e6xxx_atu_remove(ps, 0, port, false);
+                     oldstate == PORT_CONTROL_STATE_FORWARDING) &&
+                    (state == PORT_CONTROL_STATE_DISABLED ||
+                     state == PORT_CONTROL_STATE_BLOCKING)) {
+                        ret = _mv88e6xxx_atu_remove(chip, 0, port, false);
                         if (ret)
                                 return ret;
                 }
 
                 reg = (reg & ~PORT_CONTROL_STATE_MASK) | state;
-                ret = _mv88e6xxx_reg_write(ps, REG_PORT(port), PORT_CONTROL,
+                ret = _mv88e6xxx_reg_write(chip, REG_PORT(port), PORT_CONTROL,
                                            reg);
                 if (ret)
                         return ret;
 
-                netdev_dbg(ds->ports[port], "PortState %s (was %s)\n",
+                netdev_dbg(ds->ports[port].netdev, "PortState %s (was %s)\n",
                            mv88e6xxx_port_state_names[state],
                            mv88e6xxx_port_state_names[oldstate]);
         }
@@ -1330,12 +1416,11 @@ static int _mv88e6xxx_port_state(struct mv88e6xxx_priv_state *ps, int port,
         return ret;
 }
 
-static int _mv88e6xxx_port_based_vlan_map(struct mv88e6xxx_priv_state *ps,
-                                          int port)
+static int _mv88e6xxx_port_based_vlan_map(struct mv88e6xxx_chip *chip, int port)
 {
-        struct net_device *bridge = ps->ports[port].bridge_dev;
-        const u16 mask = (1 << ps->info->num_ports) - 1;
-        struct dsa_switch *ds = ps->ds;
+        struct net_device *bridge = chip->ports[port].bridge_dev;
+        const u16 mask = (1 << chip->info->num_ports) - 1;
+        struct dsa_switch *ds = chip->ds;
         u16 output_ports = 0;
         int reg;
         int i;
@@ -1344,9 +1429,9 @@ static int _mv88e6xxx_port_based_vlan_map(struct mv88e6xxx_priv_state *ps,
         if (dsa_is_cpu_port(ds, port) || dsa_is_dsa_port(ds, port)) {
                 output_ports = mask;
         } else {
-                for (i = 0; i < ps->info->num_ports; ++i) {
+                for (i = 0; i < chip->info->num_ports; ++i) {
                         /* allow sending frames to every group member */
-                        if (bridge && ps->ports[i].bridge_dev == bridge)
+                        if (bridge && chip->ports[i].bridge_dev == bridge)
                                 output_ports |= BIT(i);
 
                         /* allow sending frames to CPU port and DSA link(s) */
@@ -1358,24 +1443,24 @@ static int _mv88e6xxx_port_based_vlan_map(struct mv88e6xxx_priv_state *ps,
         /* prevent frames from going back out of the port they came in on */
         output_ports &= ~BIT(port);
 
-        reg = _mv88e6xxx_reg_read(ps, REG_PORT(port), PORT_BASE_VLAN);
+        reg = _mv88e6xxx_reg_read(chip, REG_PORT(port), PORT_BASE_VLAN);
         if (reg < 0)
                 return reg;
 
         reg &= ~mask;
         reg |= output_ports & mask;
 
-        return _mv88e6xxx_reg_write(ps, REG_PORT(port), PORT_BASE_VLAN, reg);
+        return _mv88e6xxx_reg_write(chip, REG_PORT(port), PORT_BASE_VLAN, reg);
 }
 
 static void mv88e6xxx_port_stp_state_set(struct dsa_switch *ds, int port,
                                          u8 state)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         int stp_state;
         int err;
 
-        if (!mv88e6xxx_has(ps, MV88E6XXX_FLAG_PORTSTATE))
+        if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_PORTSTATE))
                 return;
 
         switch (state) {
@@ -1395,23 +1480,24 @@ static void mv88e6xxx_port_stp_state_set(struct dsa_switch *ds, int port,
                 break;
         }
 
-        mutex_lock(&ps->smi_mutex);
-        err = _mv88e6xxx_port_state(ps, port, stp_state);
-        mutex_unlock(&ps->smi_mutex);
+        mutex_lock(&chip->reg_lock);
+        err = _mv88e6xxx_port_state(chip, port, stp_state);
+        mutex_unlock(&chip->reg_lock);
 
         if (err)
-                netdev_err(ds->ports[port], "failed to update state to %s\n",
+                netdev_err(ds->ports[port].netdev,
+                           "failed to update state to %s\n",
                            mv88e6xxx_port_state_names[stp_state]);
 }
 
-static int _mv88e6xxx_port_pvid(struct mv88e6xxx_priv_state *ps, int port,
+static int _mv88e6xxx_port_pvid(struct mv88e6xxx_chip *chip, int port,
                                 u16 *new, u16 *old)
 {
-        struct dsa_switch *ds = ps->ds;
+        struct dsa_switch *ds = chip->ds;
         u16 pvid;
         int ret;
 
-        ret = _mv88e6xxx_reg_read(ps, REG_PORT(port), PORT_DEFAULT_VLAN);
+        ret = _mv88e6xxx_reg_read(chip, REG_PORT(port), PORT_DEFAULT_VLAN);
         if (ret < 0)
                 return ret;
 
@@ -1421,13 +1507,13 @@ static int _mv88e6xxx_port_pvid(struct mv88e6xxx_priv_state *ps, int port,
                 ret &= ~PORT_DEFAULT_VLAN_MASK;
                 ret |= *new & PORT_DEFAULT_VLAN_MASK;
 
-                ret = _mv88e6xxx_reg_write(ps, REG_PORT(port),
+                ret = _mv88e6xxx_reg_write(chip, REG_PORT(port),
                                            PORT_DEFAULT_VLAN, ret);
                 if (ret < 0)
                         return ret;
 
-                netdev_dbg(ds->ports[port], "DefaultVID %d (was %d)\n", *new,
-                           pvid);
+                netdev_dbg(ds->ports[port].netdev,
+                           "DefaultVID %d (was %d)\n", *new, pvid);
         }
 
         if (old)
@@ -1436,47 +1522,47 @@ static int _mv88e6xxx_port_pvid(struct mv88e6xxx_priv_state *ps, int port,
         return 0;
 }
 
-static int _mv88e6xxx_port_pvid_get(struct mv88e6xxx_priv_state *ps,
+static int _mv88e6xxx_port_pvid_get(struct mv88e6xxx_chip *chip,
                                     int port, u16 *pvid)
 {
-        return _mv88e6xxx_port_pvid(ps, port, NULL, pvid);
+        return _mv88e6xxx_port_pvid(chip, port, NULL, pvid);
 }
 
-static int _mv88e6xxx_port_pvid_set(struct mv88e6xxx_priv_state *ps,
+static int _mv88e6xxx_port_pvid_set(struct mv88e6xxx_chip *chip,
                                     int port, u16 pvid)
 {
-        return _mv88e6xxx_port_pvid(ps, port, &pvid, NULL);
+        return _mv88e6xxx_port_pvid(chip, port, &pvid, NULL);
 }
 
-static int _mv88e6xxx_vtu_wait(struct mv88e6xxx_priv_state *ps)
+static int _mv88e6xxx_vtu_wait(struct mv88e6xxx_chip *chip)
 {
-        return _mv88e6xxx_wait(ps, REG_GLOBAL, GLOBAL_VTU_OP,
+        return _mv88e6xxx_wait(chip, REG_GLOBAL, GLOBAL_VTU_OP,
                                GLOBAL_VTU_OP_BUSY);
 }
 
-static int _mv88e6xxx_vtu_cmd(struct mv88e6xxx_priv_state *ps, u16 op)
+static int _mv88e6xxx_vtu_cmd(struct mv88e6xxx_chip *chip, u16 op)
 {
         int ret;
 
-        ret = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_VTU_OP, op);
+        ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_VTU_OP, op);
         if (ret < 0)
                 return ret;
 
-        return _mv88e6xxx_vtu_wait(ps);
+        return _mv88e6xxx_vtu_wait(chip);
 }
 
-static int _mv88e6xxx_vtu_stu_flush(struct mv88e6xxx_priv_state *ps)
+static int _mv88e6xxx_vtu_stu_flush(struct mv88e6xxx_chip *chip)
 {
         int ret;
 
-        ret = _mv88e6xxx_vtu_wait(ps);
+        ret = _mv88e6xxx_vtu_wait(chip);
         if (ret < 0)
                 return ret;
 
-        return _mv88e6xxx_vtu_cmd(ps, GLOBAL_VTU_OP_FLUSH_ALL);
+        return _mv88e6xxx_vtu_cmd(chip, GLOBAL_VTU_OP_FLUSH_ALL);
 }
 
-static int _mv88e6xxx_vtu_stu_data_read(struct mv88e6xxx_priv_state *ps,
+static int _mv88e6xxx_vtu_stu_data_read(struct mv88e6xxx_chip *chip,
                                         struct mv88e6xxx_vtu_stu_entry *entry,
                                         unsigned int nibble_offset)
 {
@@ -1485,7 +1571,7 @@ static int _mv88e6xxx_vtu_stu_data_read(struct mv88e6xxx_priv_state *ps,
         int ret;
 
         for (i = 0; i < 3; ++i) {
-                ret = _mv88e6xxx_reg_read(ps, REG_GLOBAL,
+                ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL,
                                           GLOBAL_VTU_DATA_0_3 + i);
                 if (ret < 0)
                         return ret;
@@ -1493,7 +1579,7 @@ static int _mv88e6xxx_vtu_stu_data_read(struct mv88e6xxx_priv_state *ps,
                 regs[i] = ret;
         }
 
-        for (i = 0; i < ps->info->num_ports; ++i) {
+        for (i = 0; i < chip->info->num_ports; ++i) {
                 unsigned int shift = (i % 4) * 4 + nibble_offset;
                 u16 reg = regs[i / 4];
 
@@ -1503,19 +1589,19 @@ static int _mv88e6xxx_vtu_stu_data_read(struct mv88e6xxx_priv_state *ps,
         return 0;
 }
 
-static int mv88e6xxx_vtu_data_read(struct mv88e6xxx_priv_state *ps,
+static int mv88e6xxx_vtu_data_read(struct mv88e6xxx_chip *chip,
                                    struct mv88e6xxx_vtu_stu_entry *entry)
 {
-        return _mv88e6xxx_vtu_stu_data_read(ps, entry, 0);
+        return _mv88e6xxx_vtu_stu_data_read(chip, entry, 0);
 }
 
-static int mv88e6xxx_stu_data_read(struct mv88e6xxx_priv_state *ps,
+static int mv88e6xxx_stu_data_read(struct mv88e6xxx_chip *chip,
                                    struct mv88e6xxx_vtu_stu_entry *entry)
 {
-        return _mv88e6xxx_vtu_stu_data_read(ps, entry, 2);
+        return _mv88e6xxx_vtu_stu_data_read(chip, entry, 2);
 }
 
-static int _mv88e6xxx_vtu_stu_data_write(struct mv88e6xxx_priv_state *ps,
+static int _mv88e6xxx_vtu_stu_data_write(struct mv88e6xxx_chip *chip,
                                          struct mv88e6xxx_vtu_stu_entry *entry,
                                          unsigned int nibble_offset)
 {
@@ -1523,7 +1609,7 @@ static int _mv88e6xxx_vtu_stu_data_write(struct mv88e6xxx_priv_state *ps,
         int i;
         int ret;
 
-        for (i = 0; i < ps->info->num_ports; ++i) {
+        for (i = 0; i < chip->info->num_ports; ++i) {
                 unsigned int shift = (i % 4) * 4 + nibble_offset;
                 u8 data = entry->data[i];
 
@@ -1531,7 +1617,7 @@ static int _mv88e6xxx_vtu_stu_data_write(struct mv88e6xxx_priv_state *ps,
         }
 
         for (i = 0; i < 3; ++i) {
-                ret = _mv88e6xxx_reg_write(ps, REG_GLOBAL,
+                ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL,
                                            GLOBAL_VTU_DATA_0_3 + i, regs[i]);
                 if (ret < 0)
                         return ret;
@@ -1540,39 +1626,39 @@ static int _mv88e6xxx_vtu_stu_data_write(struct mv88e6xxx_priv_state *ps,
         return 0;
 }
 
-static int mv88e6xxx_vtu_data_write(struct mv88e6xxx_priv_state *ps,
+static int mv88e6xxx_vtu_data_write(struct mv88e6xxx_chip *chip,
                                     struct mv88e6xxx_vtu_stu_entry *entry)
 {
-        return _mv88e6xxx_vtu_stu_data_write(ps, entry, 0);
+        return _mv88e6xxx_vtu_stu_data_write(chip, entry, 0);
 }
 
-static int mv88e6xxx_stu_data_write(struct mv88e6xxx_priv_state *ps,
+static int mv88e6xxx_stu_data_write(struct mv88e6xxx_chip *chip,
                                     struct mv88e6xxx_vtu_stu_entry *entry)
 {
-        return _mv88e6xxx_vtu_stu_data_write(ps, entry, 2);
+        return _mv88e6xxx_vtu_stu_data_write(chip, entry, 2);
 }
 
-static int _mv88e6xxx_vtu_vid_write(struct mv88e6xxx_priv_state *ps, u16 vid)
+static int _mv88e6xxx_vtu_vid_write(struct mv88e6xxx_chip *chip, u16 vid)
 {
-        return _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_VTU_VID,
+        return _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_VTU_VID,
                                     vid & GLOBAL_VTU_VID_MASK);
 }
 
-static int _mv88e6xxx_vtu_getnext(struct mv88e6xxx_priv_state *ps,
+static int _mv88e6xxx_vtu_getnext(struct mv88e6xxx_chip *chip,
                                   struct mv88e6xxx_vtu_stu_entry *entry)
 {
         struct mv88e6xxx_vtu_stu_entry next = { 0 };
         int ret;
 
-        ret = _mv88e6xxx_vtu_wait(ps);
+        ret = _mv88e6xxx_vtu_wait(chip);
         if (ret < 0)
                 return ret;
 
-        ret = _mv88e6xxx_vtu_cmd(ps, GLOBAL_VTU_OP_VTU_GET_NEXT);
+        ret = _mv88e6xxx_vtu_cmd(chip, GLOBAL_VTU_OP_VTU_GET_NEXT);
         if (ret < 0)
                 return ret;
 
-        ret = _mv88e6xxx_reg_read(ps, REG_GLOBAL, GLOBAL_VTU_VID);
+        ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, GLOBAL_VTU_VID);
         if (ret < 0)
                 return ret;
 
@@ -1580,22 +1666,22 @@ static int _mv88e6xxx_vtu_getnext(struct mv88e6xxx_priv_state *ps,
         next.valid = !!(ret & GLOBAL_VTU_VID_VALID);
 
         if (next.valid) {
-                ret = mv88e6xxx_vtu_data_read(ps, &next);
+                ret = mv88e6xxx_vtu_data_read(chip, &next);
                 if (ret < 0)
                         return ret;
 
-                if (mv88e6xxx_has_fid_reg(ps)) {
-                        ret = _mv88e6xxx_reg_read(ps, REG_GLOBAL,
+                if (mv88e6xxx_has_fid_reg(chip)) {
+                        ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL,
                                                   GLOBAL_VTU_FID);
                         if (ret < 0)
                                 return ret;
 
                         next.fid = ret & GLOBAL_VTU_FID_MASK;
-                } else if (mv88e6xxx_num_databases(ps) == 256) {
+                } else if (mv88e6xxx_num_databases(chip) == 256) {
                         /* VTU DBNum[7:4] are located in VTU Operation 11:8, and
                          * VTU DBNum[3:0] are located in VTU Operation 3:0
                          */
-                        ret = _mv88e6xxx_reg_read(ps, REG_GLOBAL,
+                        ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL,
                                                   GLOBAL_VTU_OP);
                         if (ret < 0)
                                 return ret;
@@ -1604,8 +1690,8 @@ static int _mv88e6xxx_vtu_getnext(struct mv88e6xxx_priv_state *ps,
                         next.fid |= ret & 0xf;
                 }
 
-                if (mv88e6xxx_has(ps, MV88E6XXX_FLAG_STU)) {
-                        ret = _mv88e6xxx_reg_read(ps, REG_GLOBAL,
+                if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_STU)) {
+                        ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL,
                                                   GLOBAL_VTU_SID);
                         if (ret < 0)
                                 return ret;
@@ -1622,26 +1708,26 @@ static int mv88e6xxx_port_vlan_dump(struct dsa_switch *ds, int port,
                                     struct switchdev_obj_port_vlan *vlan,
                                     int (*cb)(struct switchdev_obj *obj))
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         struct mv88e6xxx_vtu_stu_entry next;
         u16 pvid;
         int err;
 
-        if (!mv88e6xxx_has(ps, MV88E6XXX_FLAG_VTU))
+        if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_VTU))
                 return -EOPNOTSUPP;
 
-        mutex_lock(&ps->smi_mutex);
+        mutex_lock(&chip->reg_lock);
 
-        err = _mv88e6xxx_port_pvid_get(ps, port, &pvid);
+        err = _mv88e6xxx_port_pvid_get(chip, port, &pvid);
         if (err)
                 goto unlock;
 
-        err = _mv88e6xxx_vtu_vid_write(ps, GLOBAL_VTU_VID_MASK);
+        err = _mv88e6xxx_vtu_vid_write(chip, GLOBAL_VTU_VID_MASK);
         if (err)
                 goto unlock;
 
         do {
-                err = _mv88e6xxx_vtu_getnext(ps, &next);
+                err = _mv88e6xxx_vtu_getnext(chip, &next);
                 if (err)
                         break;
 
@@ -1652,7 +1738,8 @@ static int mv88e6xxx_port_vlan_dump(struct dsa_switch *ds, int port,
                         continue;
 
                 /* reinit and dump this VLAN obj */
-                vlan->vid_begin = vlan->vid_end = next.vid;
+                vlan->vid_begin = next.vid;
+                vlan->vid_end = next.vid;
                 vlan->flags = 0;
 
                 if (next.data[port] == GLOBAL_VTU_DATA_MEMBER_TAG_UNTAGGED)
@@ -1667,19 +1754,19 @@ static int mv88e6xxx_port_vlan_dump(struct dsa_switch *ds, int port,
         } while (next.vid < GLOBAL_VTU_VID_MASK);
 
 unlock:
-        mutex_unlock(&ps->smi_mutex);
+        mutex_unlock(&chip->reg_lock);
 
         return err;
 }
 
-static int _mv88e6xxx_vtu_loadpurge(struct mv88e6xxx_priv_state *ps,
+static int _mv88e6xxx_vtu_loadpurge(struct mv88e6xxx_chip *chip,
                                     struct mv88e6xxx_vtu_stu_entry *entry)
 {
         u16 op = GLOBAL_VTU_OP_VTU_LOAD_PURGE;
         u16 reg = 0;
         int ret;
 
-        ret = _mv88e6xxx_vtu_wait(ps);
+        ret = _mv88e6xxx_vtu_wait(chip);
         if (ret < 0)
                 return ret;
 
@@ -1687,23 +1774,25 @@ static int _mv88e6xxx_vtu_loadpurge(struct mv88e6xxx_priv_state *ps,
                 goto loadpurge;
 
         /* Write port member tags */
-        ret = mv88e6xxx_vtu_data_write(ps, entry);
+        ret = mv88e6xxx_vtu_data_write(chip, entry);
         if (ret < 0)
                 return ret;
 
-        if (mv88e6xxx_has(ps, MV88E6XXX_FLAG_STU)) {
+        if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_STU)) {
                 reg = entry->sid & GLOBAL_VTU_SID_MASK;
-                ret = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_VTU_SID, reg);
+                ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_VTU_SID,
+                                           reg);
                 if (ret < 0)
                         return ret;
         }
 
-        if (mv88e6xxx_has_fid_reg(ps)) {
+        if (mv88e6xxx_has_fid_reg(chip)) {
                 reg = entry->fid & GLOBAL_VTU_FID_MASK;
-                ret = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_VTU_FID, reg);
+                ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_VTU_FID,
+                                           reg);
                 if (ret < 0)
                         return ret;
-        } else if (mv88e6xxx_num_databases(ps) == 256) {
+        } else if (mv88e6xxx_num_databases(chip) == 256) {
                 /* VTU DBNum[7:4] are located in VTU Operation 11:8, and
                  * VTU DBNum[3:0] are located in VTU Operation 3:0
                  */
@@ -1714,46 +1803,46 @@ static int _mv88e6xxx_vtu_loadpurge(struct mv88e6xxx_priv_state *ps,
         reg = GLOBAL_VTU_VID_VALID;
 loadpurge:
         reg |= entry->vid & GLOBAL_VTU_VID_MASK;
-        ret = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_VTU_VID, reg);
+        ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_VTU_VID, reg);
         if (ret < 0)
                 return ret;
 
-        return _mv88e6xxx_vtu_cmd(ps, op);
+        return _mv88e6xxx_vtu_cmd(chip, op);
 }
 
-static int _mv88e6xxx_stu_getnext(struct mv88e6xxx_priv_state *ps, u8 sid,
+static int _mv88e6xxx_stu_getnext(struct mv88e6xxx_chip *chip, u8 sid,
                                   struct mv88e6xxx_vtu_stu_entry *entry)
 {
         struct mv88e6xxx_vtu_stu_entry next = { 0 };
         int ret;
 
-        ret = _mv88e6xxx_vtu_wait(ps);
+        ret = _mv88e6xxx_vtu_wait(chip);
         if (ret < 0)
                 return ret;
 
-        ret = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_VTU_SID,
+        ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_VTU_SID,
                                    sid & GLOBAL_VTU_SID_MASK);
         if (ret < 0)
                 return ret;
 
-        ret = _mv88e6xxx_vtu_cmd(ps, GLOBAL_VTU_OP_STU_GET_NEXT);
+        ret = _mv88e6xxx_vtu_cmd(chip, GLOBAL_VTU_OP_STU_GET_NEXT);
         if (ret < 0)
                 return ret;
 
-        ret = _mv88e6xxx_reg_read(ps, REG_GLOBAL, GLOBAL_VTU_SID);
+        ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, GLOBAL_VTU_SID);
         if (ret < 0)
                 return ret;
 
         next.sid = ret & GLOBAL_VTU_SID_MASK;
 
-        ret = _mv88e6xxx_reg_read(ps, REG_GLOBAL, GLOBAL_VTU_VID);
+        ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, GLOBAL_VTU_VID);
         if (ret < 0)
                 return ret;
 
         next.valid = !!(ret & GLOBAL_VTU_VID_VALID);
 
         if (next.valid) {
-                ret = mv88e6xxx_stu_data_read(ps, &next);
+                ret = mv88e6xxx_stu_data_read(chip, &next);
                 if (ret < 0)
                         return ret;
         }
@@ -1762,13 +1851,13 @@ static int _mv88e6xxx_stu_getnext(struct mv88e6xxx_priv_state *ps, u8 sid,
         return 0;
 }
 
-static int _mv88e6xxx_stu_loadpurge(struct mv88e6xxx_priv_state *ps,
+static int _mv88e6xxx_stu_loadpurge(struct mv88e6xxx_chip *chip,
                                     struct mv88e6xxx_vtu_stu_entry *entry)
 {
         u16 reg = 0;
         int ret;
 
-        ret = _mv88e6xxx_vtu_wait(ps);
+        ret = _mv88e6xxx_vtu_wait(chip);
         if (ret < 0)
                 return ret;
 
@@ -1776,41 +1865,41 @@ static int _mv88e6xxx_stu_loadpurge(struct mv88e6xxx_priv_state *ps,
                 goto loadpurge;
 
         /* Write port states */
-        ret = mv88e6xxx_stu_data_write(ps, entry);
+        ret = mv88e6xxx_stu_data_write(chip, entry);
         if (ret < 0)
                 return ret;
 
         reg = GLOBAL_VTU_VID_VALID;
 loadpurge:
-        ret = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_VTU_VID, reg);
+        ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_VTU_VID, reg);
         if (ret < 0)
                 return ret;
 
         reg = entry->sid & GLOBAL_VTU_SID_MASK;
-        ret = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_VTU_SID, reg);
+        ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_VTU_SID, reg);
         if (ret < 0)
                 return ret;
 
-        return _mv88e6xxx_vtu_cmd(ps, GLOBAL_VTU_OP_STU_LOAD_PURGE);
+        return _mv88e6xxx_vtu_cmd(chip, GLOBAL_VTU_OP_STU_LOAD_PURGE);
 }
 
-static int _mv88e6xxx_port_fid(struct mv88e6xxx_priv_state *ps, int port,
+static int _mv88e6xxx_port_fid(struct mv88e6xxx_chip *chip, int port,
                                u16 *new, u16 *old)
 {
-        struct dsa_switch *ds = ps->ds;
+        struct dsa_switch *ds = chip->ds;
         u16 upper_mask;
         u16 fid;
         int ret;
 
-        if (mv88e6xxx_num_databases(ps) == 4096)
+        if (mv88e6xxx_num_databases(chip) == 4096)
                 upper_mask = 0xff;
-        else if (mv88e6xxx_num_databases(ps) == 256)
+        else if (mv88e6xxx_num_databases(chip) == 256)
                 upper_mask = 0xf;
         else
                 return -EOPNOTSUPP;
 
         /* Port's default FID bits 3:0 are located in reg 0x06, offset 12 */
-        ret = _mv88e6xxx_reg_read(ps, REG_PORT(port), PORT_BASE_VLAN);
+        ret = _mv88e6xxx_reg_read(chip, REG_PORT(port), PORT_BASE_VLAN);
         if (ret < 0)
                 return ret;
 
@@ -1820,14 +1909,14 @@ static int _mv88e6xxx_port_fid(struct mv88e6xxx_priv_state *ps, int port,
                 ret &= ~PORT_BASE_VLAN_FID_3_0_MASK;
                 ret |= (*new << 12) & PORT_BASE_VLAN_FID_3_0_MASK;
 
-                ret = _mv88e6xxx_reg_write(ps, REG_PORT(port), PORT_BASE_VLAN,
+                ret = _mv88e6xxx_reg_write(chip, REG_PORT(port), PORT_BASE_VLAN,
                                            ret);
                 if (ret < 0)
                         return ret;
         }
 
         /* Port's default FID bits 11:4 are located in reg 0x05, offset 0 */
-        ret = _mv88e6xxx_reg_read(ps, REG_PORT(port), PORT_CONTROL_1);
+        ret = _mv88e6xxx_reg_read(chip, REG_PORT(port), PORT_CONTROL_1);
         if (ret < 0)
                 return ret;
 
@@ -1837,12 +1926,13 @@ static int _mv88e6xxx_port_fid(struct mv88e6xxx_priv_state *ps, int port,
                 ret &= ~upper_mask;
                 ret |= (*new >> 4) & upper_mask;
 
-                ret = _mv88e6xxx_reg_write(ps, REG_PORT(port), PORT_CONTROL_1,
+                ret = _mv88e6xxx_reg_write(chip, REG_PORT(port), PORT_CONTROL_1,
                                            ret);
                 if (ret < 0)
                         return ret;
 
-                netdev_dbg(ds->ports[port], "FID %d (was %d)\n", *new, fid);
+                netdev_dbg(ds->ports[port].netdev,
+                           "FID %d (was %d)\n", *new, fid);
         }
 
         if (old)
@@ -1851,19 +1941,19 @@ static int _mv88e6xxx_port_fid(struct mv88e6xxx_priv_state *ps, int port,
         return 0;
 }
 
-static int _mv88e6xxx_port_fid_get(struct mv88e6xxx_priv_state *ps,
+static int _mv88e6xxx_port_fid_get(struct mv88e6xxx_chip *chip,
                                    int port, u16 *fid)
 {
-        return _mv88e6xxx_port_fid(ps, port, NULL, fid);
+        return _mv88e6xxx_port_fid(chip, port, NULL, fid);
 }
 
-static int _mv88e6xxx_port_fid_set(struct mv88e6xxx_priv_state *ps,
+static int _mv88e6xxx_port_fid_set(struct mv88e6xxx_chip *chip,
                                    int port, u16 fid)
 {
-        return _mv88e6xxx_port_fid(ps, port, &fid, NULL);
+        return _mv88e6xxx_port_fid(chip, port, &fid, NULL);
 }
 
-static int _mv88e6xxx_fid_new(struct mv88e6xxx_priv_state *ps, u16 *fid)
+static int _mv88e6xxx_fid_new(struct mv88e6xxx_chip *chip, u16 *fid)
 {
         DECLARE_BITMAP(fid_bitmap, MV88E6XXX_N_FID);
         struct mv88e6xxx_vtu_stu_entry vlan;
@@ -1872,8 +1962,8 @@ static int _mv88e6xxx_fid_new(struct mv88e6xxx_priv_state *ps, u16 *fid)
         bitmap_zero(fid_bitmap, MV88E6XXX_N_FID);
 
         /* Set every FID bit used by the (un)bridged ports */
-        for (i = 0; i < ps->info->num_ports; ++i) {
-                err = _mv88e6xxx_port_fid_get(ps, i, fid);
+        for (i = 0; i < chip->info->num_ports; ++i) {
+                err = _mv88e6xxx_port_fid_get(chip, i, fid);
                 if (err)
                         return err;
 
@@ -1881,12 +1971,12 @@ static int _mv88e6xxx_fid_new(struct mv88e6xxx_priv_state *ps, u16 *fid)
         }
 
         /* Set every FID bit used by the VLAN entries */
-        err = _mv88e6xxx_vtu_vid_write(ps, GLOBAL_VTU_VID_MASK);
+        err = _mv88e6xxx_vtu_vid_write(chip, GLOBAL_VTU_VID_MASK);
         if (err)
                 return err;
 
         do {
-                err = _mv88e6xxx_vtu_getnext(ps, &vlan);
+                err = _mv88e6xxx_vtu_getnext(chip, &vlan);
                 if (err)
                         return err;
 
@@ -1900,35 +1990,35 @@ static int _mv88e6xxx_fid_new(struct mv88e6xxx_priv_state *ps, u16 *fid)
          * databases are not needed. Return the next positive available.
          */
         *fid = find_next_zero_bit(fid_bitmap, MV88E6XXX_N_FID, 1);
-        if (unlikely(*fid >= mv88e6xxx_num_databases(ps)))
+        if (unlikely(*fid >= mv88e6xxx_num_databases(chip)))
                 return -ENOSPC;
 
         /* Clear the database */
-        return _mv88e6xxx_atu_flush(ps, *fid, true);
+        return _mv88e6xxx_atu_flush(chip, *fid, true);
 }
 
-static int _mv88e6xxx_vtu_new(struct mv88e6xxx_priv_state *ps, u16 vid,
+static int _mv88e6xxx_vtu_new(struct mv88e6xxx_chip *chip, u16 vid,
                               struct mv88e6xxx_vtu_stu_entry *entry)
 {
-        struct dsa_switch *ds = ps->ds;
+        struct dsa_switch *ds = chip->ds;
         struct mv88e6xxx_vtu_stu_entry vlan = {
                 .valid = true,
                 .vid = vid,
         };
         int i, err;
 
-        err = _mv88e6xxx_fid_new(ps, &vlan.fid);
+        err = _mv88e6xxx_fid_new(chip, &vlan.fid);
         if (err)
                 return err;
 
         /* exclude all ports except the CPU and DSA ports */
-        for (i = 0; i < ps->info->num_ports; ++i)
+        for (i = 0; i < chip->info->num_ports; ++i)
                 vlan.data[i] = dsa_is_cpu_port(ds, i) || dsa_is_dsa_port(ds, i)
                         ? GLOBAL_VTU_DATA_MEMBER_TAG_UNMODIFIED
                         : GLOBAL_VTU_DATA_MEMBER_TAG_NON_MEMBER;
 
-        if (mv88e6xxx_6097_family(ps) || mv88e6xxx_6165_family(ps) ||
-            mv88e6xxx_6351_family(ps) || mv88e6xxx_6352_family(ps)) {
+        if (mv88e6xxx_6097_family(chip) || mv88e6xxx_6165_family(chip) ||
+            mv88e6xxx_6351_family(chip) || mv88e6xxx_6352_family(chip)) {
                 struct mv88e6xxx_vtu_stu_entry vstp;
 
                 /* Adding a VTU entry requires a valid STU entry. As VSTP is not
@@ -1936,7 +2026,7 @@ static int _mv88e6xxx_vtu_new(struct mv88e6xxx_priv_state *ps, u16 vid,
                  * entries. Thus, validate the SID 0.
                  */
                 vlan.sid = 0;
-                err = _mv88e6xxx_stu_getnext(ps, GLOBAL_VTU_SID_MASK, &vstp);
+                err = _mv88e6xxx_stu_getnext(chip, GLOBAL_VTU_SID_MASK, &vstp);
                 if (err)
                         return err;
 
@@ -1945,7 +2035,7 @@ static int _mv88e6xxx_vtu_new(struct mv88e6xxx_priv_state *ps, u16 vid,
                         vstp.valid = true;
                         vstp.sid = vlan.sid;
 
-                        err = _mv88e6xxx_stu_loadpurge(ps, &vstp);
+                        err = _mv88e6xxx_stu_loadpurge(chip, &vstp);
                         if (err)
                                 return err;
                 }
@@ -1955,7 +2045,7 @@ static int _mv88e6xxx_vtu_new(struct mv88e6xxx_priv_state *ps, u16 vid,
         return 0;
 }
 
-static int _mv88e6xxx_vtu_get(struct mv88e6xxx_priv_state *ps, u16 vid,
+static int _mv88e6xxx_vtu_get(struct mv88e6xxx_chip *chip, u16 vid,
                               struct mv88e6xxx_vtu_stu_entry *entry, bool creat)
 {
         int err;
@@ -1963,11 +2053,11 @@ static int _mv88e6xxx_vtu_get(struct mv88e6xxx_priv_state *ps, u16 vid,
         if (!vid)
                 return -EINVAL;
 
-        err = _mv88e6xxx_vtu_vid_write(ps, vid - 1);
+        err = _mv88e6xxx_vtu_vid_write(chip, vid - 1);
         if (err)
                 return err;
 
-        err = _mv88e6xxx_vtu_getnext(ps, entry);
+        err = _mv88e6xxx_vtu_getnext(chip, entry);
         if (err)
                 return err;
 
@@ -1978,7 +2068,7 @@ static int _mv88e6xxx_vtu_get(struct mv88e6xxx_priv_state *ps, u16 vid,
                  * -EOPNOTSUPP to inform bridge about an eventual software VLAN.
                  */
 
-                err = _mv88e6xxx_vtu_new(ps, vid, entry);
+                err = _mv88e6xxx_vtu_new(chip, vid, entry);
         }
 
         return err;
@@ -1987,21 +2077,21 @@ static int _mv88e6xxx_vtu_get(struct mv88e6xxx_priv_state *ps, u16 vid,
 static int mv88e6xxx_port_check_hw_vlan(struct dsa_switch *ds, int port,
                                         u16 vid_begin, u16 vid_end)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         struct mv88e6xxx_vtu_stu_entry vlan;
         int i, err;
 
         if (!vid_begin)
                 return -EOPNOTSUPP;
 
-        mutex_lock(&ps->smi_mutex);
+        mutex_lock(&chip->reg_lock);
 
-        err = _mv88e6xxx_vtu_vid_write(ps, vid_begin - 1);
+        err = _mv88e6xxx_vtu_vid_write(chip, vid_begin - 1);
         if (err)
                 goto unlock;
 
         do {
-                err = _mv88e6xxx_vtu_getnext(ps, &vlan);
+                err = _mv88e6xxx_vtu_getnext(chip, &vlan);
                 if (err)
                         goto unlock;
 
@@ -2011,7 +2101,7 @@ static int mv88e6xxx_port_check_hw_vlan(struct dsa_switch *ds, int port,
                 if (vlan.vid > vid_end)
                         break;
 
-                for (i = 0; i < ps->info->num_ports; ++i) {
+                for (i = 0; i < chip->info->num_ports; ++i) {
                         if (dsa_is_dsa_port(ds, i) || dsa_is_cpu_port(ds, i))
                                 continue;
 
@@ -2019,21 +2109,21 @@ static int mv88e6xxx_port_check_hw_vlan(struct dsa_switch *ds, int port,
                             GLOBAL_VTU_DATA_MEMBER_TAG_NON_MEMBER)
                                 continue;
 
-                        if (ps->ports[i].bridge_dev ==
-                            ps->ports[port].bridge_dev)
+                        if (chip->ports[i].bridge_dev ==
+                            chip->ports[port].bridge_dev)
                                 break; /* same bridge, check next VLAN */
 
-                        netdev_warn(ds->ports[port],
+                        netdev_warn(ds->ports[port].netdev,
                                     "hardware VLAN %d already used by %s\n",
                                     vlan.vid,
-                                    netdev_name(ps->ports[i].bridge_dev));
+                                    netdev_name(chip->ports[i].bridge_dev));
                         err = -EOPNOTSUPP;
                         goto unlock;
                 }
         } while (vlan.vid < vid_end);
 
 unlock:
-        mutex_unlock(&ps->smi_mutex);
+        mutex_unlock(&chip->reg_lock);
 
         return err;
 }
@@ -2048,17 +2138,17 @@ static const char * const mv88e6xxx_port_8021q_mode_names[] = {
 static int mv88e6xxx_port_vlan_filtering(struct dsa_switch *ds, int port,
                                          bool vlan_filtering)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         u16 old, new = vlan_filtering ? PORT_CONTROL_2_8021Q_SECURE :
                 PORT_CONTROL_2_8021Q_DISABLED;
         int ret;
 
-        if (!mv88e6xxx_has(ps, MV88E6XXX_FLAG_VTU))
+        if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_VTU))
                 return -EOPNOTSUPP;
 
-        mutex_lock(&ps->smi_mutex);
+        mutex_lock(&chip->reg_lock);
 
-        ret = _mv88e6xxx_reg_read(ps, REG_PORT(port), PORT_CONTROL_2);
+        ret = _mv88e6xxx_reg_read(chip, REG_PORT(port), PORT_CONTROL_2);
         if (ret < 0)
                 goto unlock;
 
@@ -2068,31 +2158,32 @@ static int mv88e6xxx_port_vlan_filtering(struct dsa_switch *ds, int port,
                 ret &= ~PORT_CONTROL_2_8021Q_MASK;
                 ret |= new & PORT_CONTROL_2_8021Q_MASK;
 
-                ret = _mv88e6xxx_reg_write(ps, REG_PORT(port), PORT_CONTROL_2,
+                ret = _mv88e6xxx_reg_write(chip, REG_PORT(port), PORT_CONTROL_2,
                                            ret);
                 if (ret < 0)
                         goto unlock;
 
-                netdev_dbg(ds->ports[port], "802.1Q Mode %s (was %s)\n",
+                netdev_dbg(ds->ports[port].netdev, "802.1Q Mode %s (was %s)\n",
                            mv88e6xxx_port_8021q_mode_names[new],
                            mv88e6xxx_port_8021q_mode_names[old]);
         }
 
         ret = 0;
 unlock:
-        mutex_unlock(&ps->smi_mutex);
+        mutex_unlock(&chip->reg_lock);
 
         return ret;
 }
 
-static int mv88e6xxx_port_vlan_prepare(struct dsa_switch *ds, int port,
-                                       const struct switchdev_obj_port_vlan *vlan,
-                                       struct switchdev_trans *trans)
+static int
+mv88e6xxx_port_vlan_prepare(struct dsa_switch *ds, int port,
+                            const struct switchdev_obj_port_vlan *vlan,
+                            struct switchdev_trans *trans)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         int err;
 
-        if (!mv88e6xxx_has(ps, MV88E6XXX_FLAG_VTU))
+        if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_VTU))
                 return -EOPNOTSUPP;
 
         /* If the requested port doesn't belong to the same bridge as the VLAN
@@ -2109,13 +2200,13 @@ static int mv88e6xxx_port_vlan_prepare(struct dsa_switch *ds, int port,
         return 0;
 }
 
-static int _mv88e6xxx_port_vlan_add(struct mv88e6xxx_priv_state *ps, int port,
+static int _mv88e6xxx_port_vlan_add(struct mv88e6xxx_chip *chip, int port,
                                     u16 vid, bool untagged)
 {
         struct mv88e6xxx_vtu_stu_entry vlan;
         int err;
 
-        err = _mv88e6xxx_vtu_get(ps, vid, &vlan, true);
+        err = _mv88e6xxx_vtu_get(chip, vid, &vlan, true);
         if (err)
                 return err;
 
@@ -2123,43 +2214,44 @@ static int _mv88e6xxx_port_vlan_add(struct mv88e6xxx_priv_state *ps, int port,
                 GLOBAL_VTU_DATA_MEMBER_TAG_UNTAGGED :
                 GLOBAL_VTU_DATA_MEMBER_TAG_TAGGED;
 
-        return _mv88e6xxx_vtu_loadpurge(ps, &vlan);
+        return _mv88e6xxx_vtu_loadpurge(chip, &vlan);
 }
 
 static void mv88e6xxx_port_vlan_add(struct dsa_switch *ds, int port,
                                     const struct switchdev_obj_port_vlan *vlan,
                                     struct switchdev_trans *trans)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         bool untagged = vlan->flags & BRIDGE_VLAN_INFO_UNTAGGED;
         bool pvid = vlan->flags & BRIDGE_VLAN_INFO_PVID;
         u16 vid;
 
-        if (!mv88e6xxx_has(ps, MV88E6XXX_FLAG_VTU))
+        if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_VTU))
                 return;
 
-        mutex_lock(&ps->smi_mutex);
+        mutex_lock(&chip->reg_lock);
 
         for (vid = vlan->vid_begin; vid <= vlan->vid_end; ++vid)
-                if (_mv88e6xxx_port_vlan_add(ps, port, vid, untagged))
-                        netdev_err(ds->ports[port], "failed to add VLAN %d%c\n",
+                if (_mv88e6xxx_port_vlan_add(chip, port, vid, untagged))
+                        netdev_err(ds->ports[port].netdev,
+                                   "failed to add VLAN %d%c\n",
                                    vid, untagged ? 'u' : 't');
 
-        if (pvid && _mv88e6xxx_port_pvid_set(ps, port, vlan->vid_end))
-                netdev_err(ds->ports[port], "failed to set PVID %d\n",
+        if (pvid && _mv88e6xxx_port_pvid_set(chip, port, vlan->vid_end))
+                netdev_err(ds->ports[port].netdev, "failed to set PVID %d\n",
                            vlan->vid_end);
 
-        mutex_unlock(&ps->smi_mutex);
+        mutex_unlock(&chip->reg_lock);
 }
 
-static int _mv88e6xxx_port_vlan_del(struct mv88e6xxx_priv_state *ps,
+static int _mv88e6xxx_port_vlan_del(struct mv88e6xxx_chip *chip,
                                     int port, u16 vid)
 {
-        struct dsa_switch *ds = ps->ds;
+        struct dsa_switch *ds = chip->ds;
         struct mv88e6xxx_vtu_stu_entry vlan;
         int i, err;
 
-        err = _mv88e6xxx_vtu_get(ps, vid, &vlan, false);
+        err = _mv88e6xxx_vtu_get(chip, vid, &vlan, false);
         if (err)
                 return err;
 
@@ -2171,7 +2263,7 @@ static int _mv88e6xxx_port_vlan_del(struct mv88e6xxx_priv_state *ps,
 
         /* keep the VLAN unless all ports are excluded */
         vlan.valid = false;
-        for (i = 0; i < ps->info->num_ports; ++i) {
+        for (i = 0; i < chip->info->num_ports; ++i) {
                 if (dsa_is_cpu_port(ds, i) || dsa_is_dsa_port(ds, i))
                         continue;
 
@@ -2181,55 +2273,55 @@ static int _mv88e6xxx_port_vlan_del(struct mv88e6xxx_priv_state *ps,
                 }
         }
 
-        err = _mv88e6xxx_vtu_loadpurge(ps, &vlan);
+        err = _mv88e6xxx_vtu_loadpurge(chip, &vlan);
         if (err)
                 return err;
 
-        return _mv88e6xxx_atu_remove(ps, vlan.fid, port, false);
+        return _mv88e6xxx_atu_remove(chip, vlan.fid, port, false);
 }
 
 static int mv88e6xxx_port_vlan_del(struct dsa_switch *ds, int port,
                                    const struct switchdev_obj_port_vlan *vlan)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         u16 pvid, vid;
         int err = 0;
 
-        if (!mv88e6xxx_has(ps, MV88E6XXX_FLAG_VTU))
+        if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_VTU))
                 return -EOPNOTSUPP;
 
-        mutex_lock(&ps->smi_mutex);
+        mutex_lock(&chip->reg_lock);
 
-        err = _mv88e6xxx_port_pvid_get(ps, port, &pvid);
+        err = _mv88e6xxx_port_pvid_get(chip, port, &pvid);
         if (err)
                 goto unlock;
 
         for (vid = vlan->vid_begin; vid <= vlan->vid_end; ++vid) {
-                err = _mv88e6xxx_port_vlan_del(ps, port, vid);
+                err = _mv88e6xxx_port_vlan_del(chip, port, vid);
                 if (err)
                         goto unlock;
 
                 if (vid == pvid) {
-                        err = _mv88e6xxx_port_pvid_set(ps, port, 0);
+                        err = _mv88e6xxx_port_pvid_set(chip, port, 0);
                         if (err)
                                 goto unlock;
                 }
         }
 
 unlock:
-        mutex_unlock(&ps->smi_mutex);
+        mutex_unlock(&chip->reg_lock);
 
         return err;
 }
 
-static int _mv88e6xxx_atu_mac_write(struct mv88e6xxx_priv_state *ps,
+static int _mv88e6xxx_atu_mac_write(struct mv88e6xxx_chip *chip,
                                     const unsigned char *addr)
 {
         int i, ret;
 
         for (i = 0; i < 3; i++) {
                 ret = _mv88e6xxx_reg_write(
-                        ps, REG_GLOBAL, GLOBAL_ATU_MAC_01 + i,
+                        chip, REG_GLOBAL, GLOBAL_ATU_MAC_01 + i,
                         (addr[i * 2] << 8) | addr[i * 2 + 1]);
                 if (ret < 0)
                         return ret;
@@ -2238,13 +2330,13 @@ static int _mv88e6xxx_atu_mac_write(struct mv88e6xxx_priv_state *ps,
         return 0;
 }
 
-static int _mv88e6xxx_atu_mac_read(struct mv88e6xxx_priv_state *ps,
+static int _mv88e6xxx_atu_mac_read(struct mv88e6xxx_chip *chip,
                                    unsigned char *addr)
 {
         int i, ret;
 
         for (i = 0; i < 3; i++) {
-                ret = _mv88e6xxx_reg_read(ps, REG_GLOBAL,
+                ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL,
                                           GLOBAL_ATU_MAC_01 + i);
                 if (ret < 0)
                         return ret;
@@ -2255,27 +2347,27 @@ static int _mv88e6xxx_atu_mac_read(struct mv88e6xxx_priv_state *ps,
         return 0;
 }
 
-static int _mv88e6xxx_atu_load(struct mv88e6xxx_priv_state *ps,
+static int _mv88e6xxx_atu_load(struct mv88e6xxx_chip *chip,
                                struct mv88e6xxx_atu_entry *entry)
 {
         int ret;
 
-        ret = _mv88e6xxx_atu_wait(ps);
+        ret = _mv88e6xxx_atu_wait(chip);
         if (ret < 0)
                 return ret;
 
-        ret = _mv88e6xxx_atu_mac_write(ps, entry->mac);
+        ret = _mv88e6xxx_atu_mac_write(chip, entry->mac);
         if (ret < 0)
                 return ret;
 
-        ret = _mv88e6xxx_atu_data_write(ps, entry);
+        ret = _mv88e6xxx_atu_data_write(chip, entry);
         if (ret < 0)
                 return ret;
 
-        return _mv88e6xxx_atu_cmd(ps, entry->fid, GLOBAL_ATU_OP_LOAD_DB);
+        return _mv88e6xxx_atu_cmd(chip, entry->fid, GLOBAL_ATU_OP_LOAD_DB);
 }
 
-static int _mv88e6xxx_port_fdb_load(struct mv88e6xxx_priv_state *ps, int port,
+static int _mv88e6xxx_port_fdb_load(struct mv88e6xxx_chip *chip, int port,
                                     const unsigned char *addr, u16 vid,
                                     u8 state)
 {
@@ -2285,9 +2377,9 @@ static int _mv88e6xxx_port_fdb_load(struct mv88e6xxx_priv_state *ps, int port,
 
         /* Null VLAN ID corresponds to the port private database */
         if (vid == 0)
-                err = _mv88e6xxx_port_fid_get(ps, port, &vlan.fid);
+                err = _mv88e6xxx_port_fid_get(chip, port, &vlan.fid);
         else
-                err = _mv88e6xxx_vtu_get(ps, vid, &vlan, false);
+                err = _mv88e6xxx_vtu_get(chip, vid, &vlan, false);
         if (err)
                 return err;
 
@@ -2299,16 +2391,16 @@ static int _mv88e6xxx_port_fdb_load(struct mv88e6xxx_priv_state *ps, int port,
                 entry.portv_trunkid = BIT(port);
         }
 
-        return _mv88e6xxx_atu_load(ps, &entry);
+        return _mv88e6xxx_atu_load(chip, &entry);
 }
 
 static int mv88e6xxx_port_fdb_prepare(struct dsa_switch *ds, int port,
                                       const struct switchdev_obj_port_fdb *fdb,
                                       struct switchdev_trans *trans)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
 
-        if (!mv88e6xxx_has(ps, MV88E6XXX_FLAG_ATU))
+        if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_ATU))
                 return -EOPNOTSUPP;
 
         /* We don't need any dynamic resource from the kernel (yet),
@@ -2324,35 +2416,36 @@ static void mv88e6xxx_port_fdb_add(struct dsa_switch *ds, int port,
         int state = is_multicast_ether_addr(fdb->addr) ?
                 GLOBAL_ATU_DATA_STATE_MC_STATIC :
                 GLOBAL_ATU_DATA_STATE_UC_STATIC;
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
 
-        if (!mv88e6xxx_has(ps, MV88E6XXX_FLAG_ATU))
+        if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_ATU))
                 return;
 
-        mutex_lock(&ps->smi_mutex);
-        if (_mv88e6xxx_port_fdb_load(ps, port, fdb->addr, fdb->vid, state))
-                netdev_err(ds->ports[port], "failed to load MAC address\n");
-        mutex_unlock(&ps->smi_mutex);
+        mutex_lock(&chip->reg_lock);
+        if (_mv88e6xxx_port_fdb_load(chip, port, fdb->addr, fdb->vid, state))
+                netdev_err(ds->ports[port].netdev,
+                           "failed to load MAC address\n");
+        mutex_unlock(&chip->reg_lock);
 }
 
 static int mv88e6xxx_port_fdb_del(struct dsa_switch *ds, int port,
                                   const struct switchdev_obj_port_fdb *fdb)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         int ret;
 
-        if (!mv88e6xxx_has(ps, MV88E6XXX_FLAG_ATU))
+        if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_ATU))
                 return -EOPNOTSUPP;
 
-        mutex_lock(&ps->smi_mutex);
-        ret = _mv88e6xxx_port_fdb_load(ps, port, fdb->addr, fdb->vid,
+        mutex_lock(&chip->reg_lock);
+        ret = _mv88e6xxx_port_fdb_load(chip, port, fdb->addr, fdb->vid,
                                        GLOBAL_ATU_DATA_STATE_UNUSED);
-        mutex_unlock(&ps->smi_mutex);
+        mutex_unlock(&chip->reg_lock);
 
         return ret;
 }
 
-static int _mv88e6xxx_atu_getnext(struct mv88e6xxx_priv_state *ps, u16 fid,
+static int _mv88e6xxx_atu_getnext(struct mv88e6xxx_chip *chip, u16 fid,
                                   struct mv88e6xxx_atu_entry *entry)
 {
         struct mv88e6xxx_atu_entry next = { 0 };
@@ -2360,19 +2453,19 @@ static int _mv88e6xxx_atu_getnext(struct mv88e6xxx_priv_state *ps, u16 fid,
 
         next.fid = fid;
 
-        ret = _mv88e6xxx_atu_wait(ps);
+        ret = _mv88e6xxx_atu_wait(chip);
         if (ret < 0)
                 return ret;
 
-        ret = _mv88e6xxx_atu_cmd(ps, fid, GLOBAL_ATU_OP_GET_NEXT_DB);
+        ret = _mv88e6xxx_atu_cmd(chip, fid, GLOBAL_ATU_OP_GET_NEXT_DB);
         if (ret < 0)
                 return ret;
 
-        ret = _mv88e6xxx_atu_mac_read(ps, next.mac);
+        ret = _mv88e6xxx_atu_mac_read(chip, next.mac);
         if (ret < 0)
                 return ret;
 
-        ret = _mv88e6xxx_reg_read(ps, REG_GLOBAL, GLOBAL_ATU_DATA);
+        ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, GLOBAL_ATU_DATA);
         if (ret < 0)
                 return ret;
 
@@ -2397,7 +2490,7 @@ static int _mv88e6xxx_atu_getnext(struct mv88e6xxx_priv_state *ps, u16 fid,
         return 0;
 }
 
-static int _mv88e6xxx_port_fdb_dump_one(struct mv88e6xxx_priv_state *ps,
+static int _mv88e6xxx_port_fdb_dump_one(struct mv88e6xxx_chip *chip,
                                         u16 fid, u16 vid, int port,
                                         struct switchdev_obj_port_fdb *fdb,
                                         int (*cb)(struct switchdev_obj *obj))
@@ -2407,12 +2500,12 @@ static int _mv88e6xxx_port_fdb_dump_one(struct mv88e6xxx_priv_state *ps,
         };
         int err;
 
-        err = _mv88e6xxx_atu_mac_write(ps, addr.mac);
+        err = _mv88e6xxx_atu_mac_write(chip, addr.mac);
         if (err)
                 return err;
 
         do {
-                err = _mv88e6xxx_atu_getnext(ps, fid, &addr);
+                err = _mv88e6xxx_atu_getnext(chip, fid, &addr);
                 if (err)
                         break;
 
@@ -2442,48 +2535,48 @@ static int mv88e6xxx_port_fdb_dump(struct dsa_switch *ds, int port,
                                    struct switchdev_obj_port_fdb *fdb,
                                    int (*cb)(struct switchdev_obj *obj))
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         struct mv88e6xxx_vtu_stu_entry vlan = {
                 .vid = GLOBAL_VTU_VID_MASK, /* all ones */
         };
         u16 fid;
         int err;
 
-        if (!mv88e6xxx_has(ps, MV88E6XXX_FLAG_ATU))
+        if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_ATU))
                 return -EOPNOTSUPP;
 
-        mutex_lock(&ps->smi_mutex);
+        mutex_lock(&chip->reg_lock);
 
         /* Dump port's default Filtering Information Database (VLAN ID 0) */
-        err = _mv88e6xxx_port_fid_get(ps, port, &fid);
+        err = _mv88e6xxx_port_fid_get(chip, port, &fid);
         if (err)
                 goto unlock;
 
-        err = _mv88e6xxx_port_fdb_dump_one(ps, fid, 0, port, fdb, cb);
+        err = _mv88e6xxx_port_fdb_dump_one(chip, fid, 0, port, fdb, cb);
         if (err)
                 goto unlock;
 
         /* Dump VLANs' Filtering Information Databases */
-        err = _mv88e6xxx_vtu_vid_write(ps, vlan.vid);
+        err = _mv88e6xxx_vtu_vid_write(chip, vlan.vid);
         if (err)
                 goto unlock;
 
         do {
-                err = _mv88e6xxx_vtu_getnext(ps, &vlan);
+                err = _mv88e6xxx_vtu_getnext(chip, &vlan);
                 if (err)
                         break;
 
                 if (!vlan.valid)
                         break;
 
-                err = _mv88e6xxx_port_fdb_dump_one(ps, vlan.fid, vlan.vid, port,
-                                                   fdb, cb);
+                err = _mv88e6xxx_port_fdb_dump_one(chip, vlan.fid, vlan.vid,
+                                                   port, fdb, cb);
                 if (err)
                         break;
         } while (vlan.vid < GLOBAL_VTU_VID_MASK);
 
 unlock:
-        mutex_unlock(&ps->smi_mutex);
+        mutex_unlock(&chip->reg_lock);
 
         return err;
 }
@@ -2491,100 +2584,101 @@ unlock:
 static int mv88e6xxx_port_bridge_join(struct dsa_switch *ds, int port,
                                       struct net_device *bridge)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         int i, err = 0;
 
-        if (!mv88e6xxx_has(ps, MV88E6XXX_FLAG_VLANTABLE))
+        if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_VLANTABLE))
                 return -EOPNOTSUPP;
 
-        mutex_lock(&ps->smi_mutex);
+        mutex_lock(&chip->reg_lock);
 
         /* Assign the bridge and remap each port's VLANTable */
-        ps->ports[port].bridge_dev = bridge;
+        chip->ports[port].bridge_dev = bridge;
 
-        for (i = 0; i < ps->info->num_ports; ++i) {
-                if (ps->ports[i].bridge_dev == bridge) {
-                        err = _mv88e6xxx_port_based_vlan_map(ps, i);
+        for (i = 0; i < chip->info->num_ports; ++i) {
+                if (chip->ports[i].bridge_dev == bridge) {
+                        err = _mv88e6xxx_port_based_vlan_map(chip, i);
                         if (err)
                                 break;
                 }
         }
 
-        mutex_unlock(&ps->smi_mutex);
+        mutex_unlock(&chip->reg_lock);
 
         return err;
 }
 
 static void mv88e6xxx_port_bridge_leave(struct dsa_switch *ds, int port)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
-        struct net_device *bridge = ps->ports[port].bridge_dev;
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+        struct net_device *bridge = chip->ports[port].bridge_dev;
         int i;
 
-        if (!mv88e6xxx_has(ps, MV88E6XXX_FLAG_VLANTABLE))
+        if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_VLANTABLE))
                 return;
 
-        mutex_lock(&ps->smi_mutex);
+        mutex_lock(&chip->reg_lock);
 
         /* Unassign the bridge and remap each port's VLANTable */
-        ps->ports[port].bridge_dev = NULL;
+        chip->ports[port].bridge_dev = NULL;
 
-        for (i = 0; i < ps->info->num_ports; ++i)
-                if (i == port || ps->ports[i].bridge_dev == bridge)
-                        if (_mv88e6xxx_port_based_vlan_map(ps, i))
-                                netdev_warn(ds->ports[i], "failed to remap\n");
+        for (i = 0; i < chip->info->num_ports; ++i)
+                if (i == port || chip->ports[i].bridge_dev == bridge)
+                        if (_mv88e6xxx_port_based_vlan_map(chip, i))
+                                netdev_warn(ds->ports[i].netdev,
+                                            "failed to remap\n");
 
-        mutex_unlock(&ps->smi_mutex);
+        mutex_unlock(&chip->reg_lock);
 }
 
-static int _mv88e6xxx_phy_page_write(struct mv88e6xxx_priv_state *ps,
-                                     int port, int page, int reg, int val)
+static int _mv88e6xxx_mdio_page_write(struct mv88e6xxx_chip *chip,
+                                      int port, int page, int reg, int val)
 {
         int ret;
 
-        ret = _mv88e6xxx_phy_write_indirect(ps, port, 0x16, page);
+        ret = mv88e6xxx_mdio_write_indirect(chip, port, 0x16, page);
         if (ret < 0)
                 goto restore_page_0;
 
-        ret = _mv88e6xxx_phy_write_indirect(ps, port, reg, val);
+        ret = mv88e6xxx_mdio_write_indirect(chip, port, reg, val);
 restore_page_0:
-        _mv88e6xxx_phy_write_indirect(ps, port, 0x16, 0x0);
+        mv88e6xxx_mdio_write_indirect(chip, port, 0x16, 0x0);
 
         return ret;
 }
 
-static int _mv88e6xxx_phy_page_read(struct mv88e6xxx_priv_state *ps,
-                                    int port, int page, int reg)
+static int _mv88e6xxx_mdio_page_read(struct mv88e6xxx_chip *chip,
+                                     int port, int page, int reg)
 {
         int ret;
 
-        ret = _mv88e6xxx_phy_write_indirect(ps, port, 0x16, page);
+        ret = mv88e6xxx_mdio_write_indirect(chip, port, 0x16, page);
         if (ret < 0)
                 goto restore_page_0;
 
-        ret = _mv88e6xxx_phy_read_indirect(ps, port, reg);
+        ret = mv88e6xxx_mdio_read_indirect(chip, port, reg);
 restore_page_0:
-        _mv88e6xxx_phy_write_indirect(ps, port, 0x16, 0x0);
+        mv88e6xxx_mdio_write_indirect(chip, port, 0x16, 0x0);
 
         return ret;
 }
 
-static int mv88e6xxx_switch_reset(struct mv88e6xxx_priv_state *ps)
+static int mv88e6xxx_switch_reset(struct mv88e6xxx_chip *chip)
 {
-        bool ppu_active = mv88e6xxx_has(ps, MV88E6XXX_FLAG_PPU_ACTIVE);
+        bool ppu_active = mv88e6xxx_has(chip, MV88E6XXX_FLAG_PPU_ACTIVE);
         u16 is_reset = (ppu_active ? 0x8800 : 0xc800);
-        struct gpio_desc *gpiod = ps->reset;
+        struct gpio_desc *gpiod = chip->reset;
         unsigned long timeout;
         int ret;
         int i;
 
         /* Set all ports to the disabled state. */
-        for (i = 0; i < ps->info->num_ports; i++) {
-                ret = _mv88e6xxx_reg_read(ps, REG_PORT(i), PORT_CONTROL);
+        for (i = 0; i < chip->info->num_ports; i++) {
+                ret = _mv88e6xxx_reg_read(chip, REG_PORT(i), PORT_CONTROL);
                 if (ret < 0)
                         return ret;
 
-                ret = _mv88e6xxx_reg_write(ps, REG_PORT(i), PORT_CONTROL,
+                ret = _mv88e6xxx_reg_write(chip, REG_PORT(i), PORT_CONTROL,
                                            ret & 0xfffc);
                 if (ret)
                         return ret;
@@ -2606,16 +2700,16 @@ static int mv88e6xxx_switch_reset(struct mv88e6xxx_priv_state *ps)
          * through global registers 0x18 and 0x19.
          */
         if (ppu_active)
-                ret = _mv88e6xxx_reg_write(ps, REG_GLOBAL, 0x04, 0xc000);
+                ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, 0x04, 0xc000);
         else
-                ret = _mv88e6xxx_reg_write(ps, REG_GLOBAL, 0x04, 0xc400);
+                ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, 0x04, 0xc400);
         if (ret)
                 return ret;
 
         /* Wait up to one second for reset to complete. */
         timeout = jiffies + 1 * HZ;
         while (time_before(jiffies, timeout)) {
-                ret = _mv88e6xxx_reg_read(ps, REG_GLOBAL, 0x00);
+                ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, 0x00);
                 if (ret < 0)
                         return ret;
 
@@ -2631,49 +2725,49 @@ static int mv88e6xxx_switch_reset(struct mv88e6xxx_priv_state *ps)
         return ret;
 }
 
-static int mv88e6xxx_power_on_serdes(struct mv88e6xxx_priv_state *ps)
+static int mv88e6xxx_power_on_serdes(struct mv88e6xxx_chip *chip)
 {
         int ret;
 
-        ret = _mv88e6xxx_phy_page_read(ps, REG_FIBER_SERDES, PAGE_FIBER_SERDES,
-                                       MII_BMCR);
+        ret = _mv88e6xxx_mdio_page_read(chip, REG_FIBER_SERDES,
+                                        PAGE_FIBER_SERDES, MII_BMCR);
         if (ret < 0)
                 return ret;
 
         if (ret & BMCR_PDOWN) {
                 ret &= ~BMCR_PDOWN;
-                ret = _mv88e6xxx_phy_page_write(ps, REG_FIBER_SERDES,
-                                                PAGE_FIBER_SERDES, MII_BMCR,
-                                                ret);
+                ret = _mv88e6xxx_mdio_page_write(chip, REG_FIBER_SERDES,
+                                                 PAGE_FIBER_SERDES, MII_BMCR,
+                                                 ret);
         }
 
         return ret;
 }
 
-static int mv88e6xxx_setup_port(struct mv88e6xxx_priv_state *ps, int port)
+static int mv88e6xxx_setup_port(struct mv88e6xxx_chip *chip, int port)
 {
-        struct dsa_switch *ds = ps->ds;
+        struct dsa_switch *ds = chip->ds;
         int ret;
         u16 reg;
 
-        if (mv88e6xxx_6352_family(ps) || mv88e6xxx_6351_family(ps) ||
-            mv88e6xxx_6165_family(ps) || mv88e6xxx_6097_family(ps) ||
-            mv88e6xxx_6185_family(ps) || mv88e6xxx_6095_family(ps) ||
-            mv88e6xxx_6065_family(ps) || mv88e6xxx_6320_family(ps)) {
+        if (mv88e6xxx_6352_family(chip) || mv88e6xxx_6351_family(chip) ||
+            mv88e6xxx_6165_family(chip) || mv88e6xxx_6097_family(chip) ||
+            mv88e6xxx_6185_family(chip) || mv88e6xxx_6095_family(chip) ||
+            mv88e6xxx_6065_family(chip) || mv88e6xxx_6320_family(chip)) {
                 /* MAC Forcing register: don't force link, speed,
                  * duplex or flow control state to any particular
                  * values on physical ports, but force the CPU port
                  * and all DSA ports to their maximum bandwidth and
                  * full duplex.
                  */
-                reg = _mv88e6xxx_reg_read(ps, REG_PORT(port), PORT_PCS_CTRL);
+                reg = _mv88e6xxx_reg_read(chip, REG_PORT(port), PORT_PCS_CTRL);
                 if (dsa_is_cpu_port(ds, port) || dsa_is_dsa_port(ds, port)) {
                         reg &= ~PORT_PCS_CTRL_UNFORCED;
                         reg |= PORT_PCS_CTRL_FORCE_LINK |
                                 PORT_PCS_CTRL_LINK_UP |
                                 PORT_PCS_CTRL_DUPLEX_FULL |
                                 PORT_PCS_CTRL_FORCE_DUPLEX;
-                        if (mv88e6xxx_6065_family(ps))
+                        if (mv88e6xxx_6065_family(chip))
                                 reg |= PORT_PCS_CTRL_100;
                         else
                                 reg |= PORT_PCS_CTRL_1000;
@@ -2681,7 +2775,7 @@ static int mv88e6xxx_setup_port(struct mv88e6xxx_priv_state *ps, int port)
                         reg |= PORT_PCS_CTRL_UNFORCED;
                 }
 
-                ret = _mv88e6xxx_reg_write(ps, REG_PORT(port),
+                ret = _mv88e6xxx_reg_write(chip, REG_PORT(port),
                                            PORT_PCS_CTRL, reg);
                 if (ret)
                         return ret;
@@ -2702,41 +2796,46 @@ static int mv88e6xxx_setup_port(struct mv88e6xxx_priv_state *ps, int port)
          * forwarding of unknown unicasts and multicasts.
          */
         reg = 0;
-        if (mv88e6xxx_6352_family(ps) || mv88e6xxx_6351_family(ps) ||
-            mv88e6xxx_6165_family(ps) || mv88e6xxx_6097_family(ps) ||
-            mv88e6xxx_6095_family(ps) || mv88e6xxx_6065_family(ps) ||
-            mv88e6xxx_6185_family(ps) || mv88e6xxx_6320_family(ps))
+        if (mv88e6xxx_6352_family(chip) || mv88e6xxx_6351_family(chip) ||
+            mv88e6xxx_6165_family(chip) || mv88e6xxx_6097_family(chip) ||
+            mv88e6xxx_6095_family(chip) || mv88e6xxx_6065_family(chip) ||
+            mv88e6xxx_6185_family(chip) || mv88e6xxx_6320_family(chip))
                 reg = PORT_CONTROL_IGMP_MLD_SNOOP |
                 PORT_CONTROL_USE_TAG | PORT_CONTROL_USE_IP |
                 PORT_CONTROL_STATE_FORWARDING;
         if (dsa_is_cpu_port(ds, port)) {
-                if (mv88e6xxx_6095_family(ps) || mv88e6xxx_6185_family(ps))
+                if (mv88e6xxx_6095_family(chip) || mv88e6xxx_6185_family(chip))
                         reg |= PORT_CONTROL_DSA_TAG;
-                if (mv88e6xxx_6352_family(ps) || mv88e6xxx_6351_family(ps) ||
-                    mv88e6xxx_6165_family(ps) || mv88e6xxx_6097_family(ps) ||
-                    mv88e6xxx_6320_family(ps)) {
-                        if (ds->dst->tag_protocol == DSA_TAG_PROTO_EDSA)
-                                reg |= PORT_CONTROL_FRAME_ETHER_TYPE_DSA;
-                        else
-                                reg |= PORT_CONTROL_FRAME_MODE_DSA;
-                        reg |= PORT_CONTROL_FORWARD_UNKNOWN |
+                if (mv88e6xxx_6352_family(chip) ||
+                    mv88e6xxx_6351_family(chip) ||
+                    mv88e6xxx_6165_family(chip) ||
+                    mv88e6xxx_6097_family(chip) ||
+                    mv88e6xxx_6320_family(chip)) {
+                        reg |= PORT_CONTROL_FRAME_ETHER_TYPE_DSA |
+                                PORT_CONTROL_FORWARD_UNKNOWN |
                                 PORT_CONTROL_FORWARD_UNKNOWN_MC;
                 }
 
-                if (mv88e6xxx_6352_family(ps) || mv88e6xxx_6351_family(ps) ||
-                    mv88e6xxx_6165_family(ps) || mv88e6xxx_6097_family(ps) ||
-                    mv88e6xxx_6095_family(ps) || mv88e6xxx_6065_family(ps) ||
-                    mv88e6xxx_6185_family(ps) || mv88e6xxx_6320_family(ps)) {
-                        if (ds->dst->tag_protocol == DSA_TAG_PROTO_EDSA)
-                                reg |= PORT_CONTROL_EGRESS_ADD_TAG;
+                if (mv88e6xxx_6352_family(chip) ||
+                    mv88e6xxx_6351_family(chip) ||
+                    mv88e6xxx_6165_family(chip) ||
+                    mv88e6xxx_6097_family(chip) ||
+                    mv88e6xxx_6095_family(chip) ||
+                    mv88e6xxx_6065_family(chip) ||
+                    mv88e6xxx_6185_family(chip) ||
+                    mv88e6xxx_6320_family(chip)) {
+                        reg |= PORT_CONTROL_EGRESS_ADD_TAG;
                 }
         }
         if (dsa_is_dsa_port(ds, port)) {
-                if (mv88e6xxx_6095_family(ps) || mv88e6xxx_6185_family(ps))
+                if (mv88e6xxx_6095_family(chip) ||
+                    mv88e6xxx_6185_family(chip))
                         reg |= PORT_CONTROL_DSA_TAG;
-                if (mv88e6xxx_6352_family(ps) || mv88e6xxx_6351_family(ps) ||
-                    mv88e6xxx_6165_family(ps) || mv88e6xxx_6097_family(ps) ||
-                    mv88e6xxx_6320_family(ps)) {
+                if (mv88e6xxx_6352_family(chip) ||
+                    mv88e6xxx_6351_family(chip) ||
+                    mv88e6xxx_6165_family(chip) ||
+                    mv88e6xxx_6097_family(chip) ||
+                    mv88e6xxx_6320_family(chip)) {
                         reg |= PORT_CONTROL_FRAME_MODE_DSA;
                 }
 
@@ -2745,7 +2844,7 @@ static int mv88e6xxx_setup_port(struct mv88e6xxx_priv_state *ps, int port)
                                 PORT_CONTROL_FORWARD_UNKNOWN_MC;
         }
         if (reg) {
-                ret = _mv88e6xxx_reg_write(ps, REG_PORT(port),
+                ret = _mv88e6xxx_reg_write(chip, REG_PORT(port),
                                            PORT_CONTROL, reg);
                 if (ret)
                         return ret;
@@ -2754,15 +2853,15 @@ static int mv88e6xxx_setup_port(struct mv88e6xxx_priv_state *ps, int port)
         /* If this port is connected to a SerDes, make sure the SerDes is not
          * powered down.
          */
-        if (mv88e6xxx_6352_family(ps)) {
-                ret = _mv88e6xxx_reg_read(ps, REG_PORT(port), PORT_STATUS);
+        if (mv88e6xxx_6352_family(chip)) {
+                ret = _mv88e6xxx_reg_read(chip, REG_PORT(port), PORT_STATUS);
                 if (ret < 0)
                         return ret;
                 ret &= PORT_STATUS_CMODE_MASK;
                 if ((ret == PORT_STATUS_CMODE_100BASE_X) ||
                     (ret == PORT_STATUS_CMODE_1000BASE_X) ||
                     (ret == PORT_STATUS_CMODE_SGMII)) {
-                        ret = mv88e6xxx_power_on_serdes(ps);
+                        ret = mv88e6xxx_power_on_serdes(chip);
                         if (ret < 0)
                                 return ret;
                 }
@@ -2775,17 +2874,17 @@ static int mv88e6xxx_setup_port(struct mv88e6xxx_priv_state *ps, int port)
          * copy of all transmitted/received frames on this port to the CPU.
          */
         reg = 0;
-        if (mv88e6xxx_6352_family(ps) || mv88e6xxx_6351_family(ps) ||
-            mv88e6xxx_6165_family(ps) || mv88e6xxx_6097_family(ps) ||
-            mv88e6xxx_6095_family(ps) || mv88e6xxx_6320_family(ps) ||
-            mv88e6xxx_6185_family(ps))
+        if (mv88e6xxx_6352_family(chip) || mv88e6xxx_6351_family(chip) ||
+            mv88e6xxx_6165_family(chip) || mv88e6xxx_6097_family(chip) ||
+            mv88e6xxx_6095_family(chip) || mv88e6xxx_6320_family(chip) ||
+            mv88e6xxx_6185_family(chip))
                 reg = PORT_CONTROL_2_MAP_DA;
 
-        if (mv88e6xxx_6352_family(ps) || mv88e6xxx_6351_family(ps) ||
-            mv88e6xxx_6165_family(ps) || mv88e6xxx_6320_family(ps))
+        if (mv88e6xxx_6352_family(chip) || mv88e6xxx_6351_family(chip) ||
+            mv88e6xxx_6165_family(chip) || mv88e6xxx_6320_family(chip))
                 reg |= PORT_CONTROL_2_JUMBO_10240;
 
-        if (mv88e6xxx_6095_family(ps) || mv88e6xxx_6185_family(ps)) {
+        if (mv88e6xxx_6095_family(chip) || mv88e6xxx_6185_family(chip)) {
                 /* Set the upstream port this port should use */
                 reg |= dsa_upstream_port(ds);
                 /* enable forwarding of unknown multicast addresses to
@@ -2798,7 +2897,7 @@ static int mv88e6xxx_setup_port(struct mv88e6xxx_priv_state *ps, int port)
         reg |= PORT_CONTROL_2_8021Q_DISABLED;
 
         if (reg) {
-                ret = _mv88e6xxx_reg_write(ps, REG_PORT(port),
+                ret = _mv88e6xxx_reg_write(chip, REG_PORT(port),
                                            PORT_CONTROL_2, reg);
                 if (ret)
                         return ret;
@@ -2814,24 +2913,25 @@ static int mv88e6xxx_setup_port(struct mv88e6xxx_priv_state *ps, int port)
         if (dsa_is_cpu_port(ds, port))
                 reg = 0;
 
-        ret = _mv88e6xxx_reg_write(ps, REG_PORT(port), PORT_ASSOC_VECTOR, reg);
+        ret = _mv88e6xxx_reg_write(chip, REG_PORT(port), PORT_ASSOC_VECTOR,
+                                   reg);
         if (ret)
                 return ret;
 
         /* Egress rate control 2: disable egress rate control. */
-        ret = _mv88e6xxx_reg_write(ps, REG_PORT(port), PORT_RATE_CONTROL_2,
+        ret = _mv88e6xxx_reg_write(chip, REG_PORT(port), PORT_RATE_CONTROL_2,
                                    0x0000);
         if (ret)
                 return ret;
 
-        if (mv88e6xxx_6352_family(ps) || mv88e6xxx_6351_family(ps) ||
-            mv88e6xxx_6165_family(ps) || mv88e6xxx_6097_family(ps) ||
-            mv88e6xxx_6320_family(ps)) {
+        if (mv88e6xxx_6352_family(chip) || mv88e6xxx_6351_family(chip) ||
+            mv88e6xxx_6165_family(chip) || mv88e6xxx_6097_family(chip) ||
+            mv88e6xxx_6320_family(chip)) {
                 /* Do not limit the period of time that this port can
                  * be paused for by the remote end or the period of
                  * time that this port can pause the remote end.
                  */
-                ret = _mv88e6xxx_reg_write(ps, REG_PORT(port),
+                ret = _mv88e6xxx_reg_write(chip, REG_PORT(port),
                                            PORT_PAUSE_CTRL, 0x0000);
                 if (ret)
                         return ret;
@@ -2840,12 +2940,12 @@ static int mv88e6xxx_setup_port(struct mv88e6xxx_priv_state *ps, int port)
                  * address database entries that this port is allowed
                  * to use.
                  */
-                ret = _mv88e6xxx_reg_write(ps, REG_PORT(port),
+                ret = _mv88e6xxx_reg_write(chip, REG_PORT(port),
                                            PORT_ATU_CONTROL, 0x0000);
                 /* Priority Override: disable DA, SA and VTU priority
                  * override.
                  */
-                ret = _mv88e6xxx_reg_write(ps, REG_PORT(port),
+                ret = _mv88e6xxx_reg_write(chip, REG_PORT(port),
                                            PORT_PRI_OVERRIDE, 0x0000);
                 if (ret)
                         return ret;
@@ -2853,14 +2953,14 @@ static int mv88e6xxx_setup_port(struct mv88e6xxx_priv_state *ps, int port)
                 /* Port Ethertype: use the Ethertype DSA Ethertype
                  * value.
                  */
-                ret = _mv88e6xxx_reg_write(ps, REG_PORT(port),
+                ret = _mv88e6xxx_reg_write(chip, REG_PORT(port),
                                            PORT_ETH_TYPE, ETH_P_EDSA);
                 if (ret)
                         return ret;
                 /* Tag Remap: use an identity 802.1p prio -> switch
                  * prio mapping.
                  */
-                ret = _mv88e6xxx_reg_write(ps, REG_PORT(port),
+                ret = _mv88e6xxx_reg_write(chip, REG_PORT(port),
                                            PORT_TAG_REGMAP_0123, 0x3210);
                 if (ret)
                         return ret;
@@ -2868,18 +2968,18 @@ static int mv88e6xxx_setup_port(struct mv88e6xxx_priv_state *ps, int port)
                 /* Tag Remap 2: use an identity 802.1p prio -> switch
                  * prio mapping.
                  */
-                ret = _mv88e6xxx_reg_write(ps, REG_PORT(port),
+                ret = _mv88e6xxx_reg_write(chip, REG_PORT(port),
                                            PORT_TAG_REGMAP_4567, 0x7654);
                 if (ret)
                         return ret;
         }
 
-        if (mv88e6xxx_6352_family(ps) || mv88e6xxx_6351_family(ps) ||
-            mv88e6xxx_6165_family(ps) || mv88e6xxx_6097_family(ps) ||
-            mv88e6xxx_6185_family(ps) || mv88e6xxx_6095_family(ps) ||
-            mv88e6xxx_6320_family(ps)) {
+        if (mv88e6xxx_6352_family(chip) || mv88e6xxx_6351_family(chip) ||
+            mv88e6xxx_6165_family(chip) || mv88e6xxx_6097_family(chip) ||
+            mv88e6xxx_6185_family(chip) || mv88e6xxx_6095_family(chip) ||
+            mv88e6xxx_6320_family(chip)) {
                 /* Rate Control: disable ingress rate limiting. */
-                ret = _mv88e6xxx_reg_write(ps, REG_PORT(port),
+                ret = _mv88e6xxx_reg_write(chip, REG_PORT(port),
                                            PORT_RATE_CONTROL, 0x0001);
                 if (ret)
                         return ret;
@@ -2888,7 +2988,8 @@ static int mv88e6xxx_setup_port(struct mv88e6xxx_priv_state *ps, int port)
         /* Port Control 1: disable trunking, disable sending
          * learning messages to this port.
          */
-        ret = _mv88e6xxx_reg_write(ps, REG_PORT(port), PORT_CONTROL_1, 0x0000);
+        ret = _mv88e6xxx_reg_write(chip, REG_PORT(port), PORT_CONTROL_1,
+                                   0x0000);
         if (ret)
                 return ret;
 
@@ -2896,18 +2997,18 @@ static int mv88e6xxx_setup_port(struct mv88e6xxx_priv_state *ps, int port)
          * database, and allow bidirectional communication between the
          * CPU and DSA port(s), and the other ports.
          */
-        ret = _mv88e6xxx_port_fid_set(ps, port, 0);
+        ret = _mv88e6xxx_port_fid_set(chip, port, 0);
         if (ret)
                 return ret;
 
-        ret = _mv88e6xxx_port_based_vlan_map(ps, port);
+        ret = _mv88e6xxx_port_based_vlan_map(chip, port);
         if (ret)
                 return ret;
 
         /* Default VLAN ID and priority: don't set a default VLAN
          * ID, and set the default packet priority to zero.
          */
-        ret = _mv88e6xxx_reg_write(ps, REG_PORT(port), PORT_DEFAULT_VLAN,
+        ret = _mv88e6xxx_reg_write(chip, REG_PORT(port), PORT_DEFAULT_VLAN,
                                    0x0000);
         if (ret)
                 return ret;
@@ -2915,9 +3016,9 @@ static int mv88e6xxx_setup_port(struct mv88e6xxx_priv_state *ps, int port)
         return 0;
 }
 
-static int mv88e6xxx_setup_global(struct mv88e6xxx_priv_state *ps)
+static int mv88e6xxx_setup_global(struct mv88e6xxx_chip *chip)
 {
-        struct dsa_switch *ds = ps->ds;
+        struct dsa_switch *ds = chip->ds;
         u32 upstream_port = dsa_upstream_port(ds);
         u16 reg;
         int err;
@@ -2927,11 +3028,11 @@ static int mv88e6xxx_setup_global(struct mv88e6xxx_priv_state *ps)
          * and mask all interrupt sources.
          */
         reg = 0;
-        if (mv88e6xxx_has(ps, MV88E6XXX_FLAG_PPU) ||
-            mv88e6xxx_has(ps, MV88E6XXX_FLAG_PPU_ACTIVE))
+        if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_PPU) ||
+            mv88e6xxx_has(chip, MV88E6XXX_FLAG_PPU_ACTIVE))
                 reg |= GLOBAL_CONTROL_PPU_ENABLE;
 
-        err = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_CONTROL, reg);
+        err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_CONTROL, reg);
         if (err)
                 return err;
 
@@ -2941,12 +3042,13 @@ static int mv88e6xxx_setup_global(struct mv88e6xxx_priv_state *ps)
         reg = upstream_port << GLOBAL_MONITOR_CONTROL_INGRESS_SHIFT |
                 upstream_port << GLOBAL_MONITOR_CONTROL_EGRESS_SHIFT |
                 upstream_port << GLOBAL_MONITOR_CONTROL_ARP_SHIFT;
-        err = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_MONITOR_CONTROL, reg);
+        err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_MONITOR_CONTROL,
+                                   reg);
         if (err)
                 return err;
 
         /* Disable remote management, and set the switch's DSA device number. */
-        err = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_CONTROL_2,
+        err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_CONTROL_2,
                                    GLOBAL_CONTROL_2_MULTIPLE_CASCADE |
                                    (ds->index & 0x1f));
         if (err)
@@ -2956,46 +3058,47 @@ static int mv88e6xxx_setup_global(struct mv88e6xxx_priv_state *ps)
          * enable address learn messages to be sent to all message
          * ports.
          */
-        err = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_ATU_CONTROL,
+        err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_ATU_CONTROL,
                                    0x0140 | GLOBAL_ATU_CONTROL_LEARN2ALL);
         if (err)
                 return err;
 
         /* Configure the IP ToS mapping registers. */
-        err = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_IP_PRI_0, 0x0000);
+        err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_IP_PRI_0, 0x0000);
         if (err)
                 return err;
-        err = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_IP_PRI_1, 0x0000);
+        err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_IP_PRI_1, 0x0000);
         if (err)
                 return err;
-        err = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_IP_PRI_2, 0x5555);
+        err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_IP_PRI_2, 0x5555);
         if (err)
                 return err;
-        err = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_IP_PRI_3, 0x5555);
+        err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_IP_PRI_3, 0x5555);
         if (err)
                 return err;
-        err = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_IP_PRI_4, 0xaaaa);
+        err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_IP_PRI_4, 0xaaaa);
         if (err)
                 return err;
-        err = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_IP_PRI_5, 0xaaaa);
+        err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_IP_PRI_5, 0xaaaa);
         if (err)
                 return err;
-        err = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_IP_PRI_6, 0xffff);
+        err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_IP_PRI_6, 0xffff);
         if (err)
                 return err;
-        err = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_IP_PRI_7, 0xffff);
+        err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_IP_PRI_7, 0xffff);
         if (err)
                 return err;
 
         /* Configure the IEEE 802.1p priority mapping register. */
-        err = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_IEEE_PRI, 0xfa41);
+        err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_IEEE_PRI, 0xfa41);
         if (err)
                 return err;
 
         /* Send all frames with destination addresses matching
          * 01:80:c2:00:00:0x to the CPU port.
          */
-        err = _mv88e6xxx_reg_write(ps, REG_GLOBAL2, GLOBAL2_MGMT_EN_0X, 0xffff);
+        err = _mv88e6xxx_reg_write(chip, REG_GLOBAL2, GLOBAL2_MGMT_EN_0X,
+                                   0xffff);
         if (err)
                 return err;
 
@@ -3004,7 +3107,7 @@ static int mv88e6xxx_setup_global(struct mv88e6xxx_priv_state *ps)
          * highest, and send all special multicast frames to the CPU
          * port at the highest priority.
          */
-        err = _mv88e6xxx_reg_write(ps, REG_GLOBAL2, GLOBAL2_SWITCH_MGMT,
+        err = _mv88e6xxx_reg_write(chip, REG_GLOBAL2, GLOBAL2_SWITCH_MGMT,
                                    0x7 | GLOBAL2_SWITCH_MGMT_RSVD2CPU | 0x70 |
                                    GLOBAL2_SWITCH_MGMT_FORCE_FLOW_CTRL_PRI);
         if (err)
@@ -3014,12 +3117,11 @@ static int mv88e6xxx_setup_global(struct mv88e6xxx_priv_state *ps)
         for (i = 0; i < 32; i++) {
                 int nexthop = 0x1f;
 
-                if (ps->ds->cd->rtable &&
-                    i != ps->ds->index && i < ps->ds->dst->pd->nr_chips)
-                        nexthop = ps->ds->cd->rtable[i] & 0x1f;
+                if (i != ds->index && i < DSA_MAX_SWITCHES)
+                        nexthop = ds->rtable[i] & 0x1f;
 
                 err = _mv88e6xxx_reg_write(
-                        ps, REG_GLOBAL2,
+                        chip, REG_GLOBAL2,
                         GLOBAL2_DEVICE_MAPPING,
                         GLOBAL2_DEVICE_MAPPING_UPDATE |
                         (i << GLOBAL2_DEVICE_MAPPING_TARGET_SHIFT) | nexthop);
@@ -3029,10 +3131,11 @@ static int mv88e6xxx_setup_global(struct mv88e6xxx_priv_state *ps)
 
         /* Clear all trunk masks. */
         for (i = 0; i < 8; i++) {
-                err = _mv88e6xxx_reg_write(ps, REG_GLOBAL2, GLOBAL2_TRUNK_MASK,
+                err = _mv88e6xxx_reg_write(chip, REG_GLOBAL2,
+                                           GLOBAL2_TRUNK_MASK,
                                            0x8000 |
                                            (i << GLOBAL2_TRUNK_MASK_NUM_SHIFT) |
-                                           ((1 << ps->info->num_ports) - 1));
+                                           ((1 << chip->info->num_ports) - 1));
                 if (err)
                         return err;
         }
@@ -3040,7 +3143,7 @@ static int mv88e6xxx_setup_global(struct mv88e6xxx_priv_state *ps)
         /* Clear all trunk mappings. */
         for (i = 0; i < 16; i++) {
                 err = _mv88e6xxx_reg_write(
-                        ps, REG_GLOBAL2,
+                        chip, REG_GLOBAL2,
                         GLOBAL2_TRUNK_MAPPING,
                         GLOBAL2_TRUNK_MAPPING_UPDATE |
                         (i << GLOBAL2_TRUNK_MAPPING_ID_SHIFT));
@@ -3048,13 +3151,13 @@ static int mv88e6xxx_setup_global(struct mv88e6xxx_priv_state *ps)
                         return err;
         }
 
-        if (mv88e6xxx_6352_family(ps) || mv88e6xxx_6351_family(ps) ||
-            mv88e6xxx_6165_family(ps) || mv88e6xxx_6097_family(ps) ||
-            mv88e6xxx_6320_family(ps)) {
+        if (mv88e6xxx_6352_family(chip) || mv88e6xxx_6351_family(chip) ||
+            mv88e6xxx_6165_family(chip) || mv88e6xxx_6097_family(chip) ||
+            mv88e6xxx_6320_family(chip)) {
                 /* Send all frames with destination addresses matching
                  * 01:80:c2:00:00:2x to the CPU port.
                  */
-                err = _mv88e6xxx_reg_write(ps, REG_GLOBAL2,
+                err = _mv88e6xxx_reg_write(chip, REG_GLOBAL2,
                                            GLOBAL2_MGMT_EN_2X, 0xffff);
                 if (err)
                         return err;
@@ -3062,14 +3165,14 @@ static int mv88e6xxx_setup_global(struct mv88e6xxx_priv_state *ps)
                 /* Initialise cross-chip port VLAN table to reset
                  * defaults.
                  */
-                err = _mv88e6xxx_reg_write(ps, REG_GLOBAL2,
+                err = _mv88e6xxx_reg_write(chip, REG_GLOBAL2,
                                            GLOBAL2_PVT_ADDR, 0x9000);
                 if (err)
                         return err;
 
                 /* Clear the priority override table. */
                 for (i = 0; i < 16; i++) {
-                        err = _mv88e6xxx_reg_write(ps, REG_GLOBAL2,
+                        err = _mv88e6xxx_reg_write(chip, REG_GLOBAL2,
                                                    GLOBAL2_PRIO_OVERRIDE,
                                                    0x8000 | (i << 8));
                         if (err)
@@ -3077,16 +3180,16 @@ static int mv88e6xxx_setup_global(struct mv88e6xxx_priv_state *ps)
                 }
         }
 
-        if (mv88e6xxx_6352_family(ps) || mv88e6xxx_6351_family(ps) ||
-            mv88e6xxx_6165_family(ps) || mv88e6xxx_6097_family(ps) ||
-            mv88e6xxx_6185_family(ps) || mv88e6xxx_6095_family(ps) ||
-            mv88e6xxx_6320_family(ps)) {
+        if (mv88e6xxx_6352_family(chip) || mv88e6xxx_6351_family(chip) ||
+            mv88e6xxx_6165_family(chip) || mv88e6xxx_6097_family(chip) ||
+            mv88e6xxx_6185_family(chip) || mv88e6xxx_6095_family(chip) ||
+            mv88e6xxx_6320_family(chip)) {
                 /* Disable ingress rate limiting by resetting all
                  * ingress rate limit registers to their initial
                  * state.
                  */
-                for (i = 0; i < ps->info->num_ports; i++) {
-                        err = _mv88e6xxx_reg_write(ps, REG_GLOBAL2,
+                for (i = 0; i < chip->info->num_ports; i++) {
+                        err = _mv88e6xxx_reg_write(chip, REG_GLOBAL2,
                                                    GLOBAL2_INGRESS_OP,
                                                    0x9000 | (i << 8));
                         if (err)
@@ -3095,23 +3198,23 @@ static int mv88e6xxx_setup_global(struct mv88e6xxx_priv_state *ps)
         }
 
         /* Clear the statistics counters for all ports */
-        err = _mv88e6xxx_reg_write(ps, REG_GLOBAL, GLOBAL_STATS_OP,
+        err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_STATS_OP,
                                    GLOBAL_STATS_OP_FLUSH_ALL);
         if (err)
                 return err;
 
         /* Wait for the flush to complete. */
-        err = _mv88e6xxx_stats_wait(ps);
+        err = _mv88e6xxx_stats_wait(chip);
         if (err)
                 return err;
 
         /* Clear all ATU entries */
-        err = _mv88e6xxx_atu_flush(ps, 0, true);
+        err = _mv88e6xxx_atu_flush(chip, 0, true);
         if (err)
                 return err;
 
         /* Clear all the VTU and STU entries */
-        err = _mv88e6xxx_vtu_stu_flush(ps);
+        err = _mv88e6xxx_vtu_stu_flush(chip);
         if (err < 0)
                 return err;
 
@@ -3120,174 +3223,232 @@ static int mv88e6xxx_setup_global(struct mv88e6xxx_priv_state *ps)
 
 static int mv88e6xxx_setup(struct dsa_switch *ds)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         int err;
         int i;
 
-        ps->ds = ds;
+        chip->ds = ds;
+        ds->slave_mii_bus = chip->mdio_bus;
 
-        if (mv88e6xxx_has(ps, MV88E6XXX_FLAG_EEPROM))
-                mutex_init(&ps->eeprom_mutex);
+        if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_EEPROM))
+                mutex_init(&chip->eeprom_mutex);
 
-        if (mv88e6xxx_has(ps, MV88E6XXX_FLAG_PPU))
-                mv88e6xxx_ppu_state_init(ps);
+        mutex_lock(&chip->reg_lock);
 
-        mutex_lock(&ps->smi_mutex);
-
-        err = mv88e6xxx_switch_reset(ps);
+        err = mv88e6xxx_switch_reset(chip);
         if (err)
                 goto unlock;
 
-        err = mv88e6xxx_setup_global(ps);
+        err = mv88e6xxx_setup_global(chip);
         if (err)
                 goto unlock;
 
-        for (i = 0; i < ps->info->num_ports; i++) {
-                err = mv88e6xxx_setup_port(ps, i);
+        for (i = 0; i < chip->info->num_ports; i++) {
+                err = mv88e6xxx_setup_port(chip, i);
                 if (err)
                         goto unlock;
         }
 
 unlock:
-        mutex_unlock(&ps->smi_mutex);
+        mutex_unlock(&chip->reg_lock);
 
         return err;
 }
 
-int mv88e6xxx_phy_page_read(struct dsa_switch *ds, int port, int page, int reg)
+static int mv88e6xxx_mdio_page_read(struct dsa_switch *ds, int port, int page,
+                                    int reg)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         int ret;
 
-        mutex_lock(&ps->smi_mutex);
-        ret = _mv88e6xxx_phy_page_read(ps, port, page, reg);
-        mutex_unlock(&ps->smi_mutex);
+        mutex_lock(&chip->reg_lock);
+        ret = _mv88e6xxx_mdio_page_read(chip, port, page, reg);
+        mutex_unlock(&chip->reg_lock);
 
         return ret;
 }
 
-int mv88e6xxx_phy_page_write(struct dsa_switch *ds, int port, int page,
-                             int reg, int val)
+static int mv88e6xxx_mdio_page_write(struct dsa_switch *ds, int port, int page,
+                                     int reg, int val)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         int ret;
 
-        mutex_lock(&ps->smi_mutex);
-        ret = _mv88e6xxx_phy_page_write(ps, port, page, reg, val);
-        mutex_unlock(&ps->smi_mutex);
+        mutex_lock(&chip->reg_lock);
+        ret = _mv88e6xxx_mdio_page_write(chip, port, page, reg, val);
+        mutex_unlock(&chip->reg_lock);
 
         return ret;
 }
 
-static int mv88e6xxx_port_to_phy_addr(struct mv88e6xxx_priv_state *ps,
-                                      int port)
+static int mv88e6xxx_port_to_mdio_addr(struct mv88e6xxx_chip *chip, int port)
 {
-        if (port >= 0 && port < ps->info->num_ports)
+        if (port >= 0 && port < chip->info->num_ports)
                 return port;
         return -EINVAL;
 }
 
-static int mv88e6xxx_phy_read(struct dsa_switch *ds, int port, int regnum)
+static int mv88e6xxx_mdio_read(struct mii_bus *bus, int port, int regnum)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
-        int addr = mv88e6xxx_port_to_phy_addr(ps, port);
+        struct mv88e6xxx_chip *chip = bus->priv;
+        int addr = mv88e6xxx_port_to_mdio_addr(chip, port);
         int ret;
 
         if (addr < 0)
                 return 0xffff;
 
-        mutex_lock(&ps->smi_mutex);
+        mutex_lock(&chip->reg_lock);
 
-        if (mv88e6xxx_has(ps, MV88E6XXX_FLAG_PPU))
-                ret = mv88e6xxx_phy_read_ppu(ps, addr, regnum);
-        else if (mv88e6xxx_has(ps, MV88E6XXX_FLAG_SMI_PHY))
-                ret = _mv88e6xxx_phy_read_indirect(ps, addr, regnum);
+        if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_PPU))
+                ret = mv88e6xxx_mdio_read_ppu(chip, addr, regnum);
+        else if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_SMI_PHY))
+                ret = mv88e6xxx_mdio_read_indirect(chip, addr, regnum);
         else
-                ret = _mv88e6xxx_phy_read(ps, addr, regnum);
+                ret = mv88e6xxx_mdio_read_direct(chip, addr, regnum);
 
-        mutex_unlock(&ps->smi_mutex);
+        mutex_unlock(&chip->reg_lock);
         return ret;
 }
 
-static int mv88e6xxx_phy_write(struct dsa_switch *ds, int port, int regnum,
-                               u16 val)
+static int mv88e6xxx_mdio_write(struct mii_bus *bus, int port, int regnum,
+                                u16 val)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
-        int addr = mv88e6xxx_port_to_phy_addr(ps, port);
+        struct mv88e6xxx_chip *chip = bus->priv;
+        int addr = mv88e6xxx_port_to_mdio_addr(chip, port);
         int ret;
 
         if (addr < 0)
                 return 0xffff;
 
-        mutex_lock(&ps->smi_mutex);
+        mutex_lock(&chip->reg_lock);
 
-        if (mv88e6xxx_has(ps, MV88E6XXX_FLAG_PPU))
-                ret = mv88e6xxx_phy_write_ppu(ps, addr, regnum, val);
-        else if (mv88e6xxx_has(ps, MV88E6XXX_FLAG_SMI_PHY))
-                ret = _mv88e6xxx_phy_write_indirect(ps, addr, regnum, val);
+        if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_PPU))
+                ret = mv88e6xxx_mdio_write_ppu(chip, addr, regnum, val);
+        else if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_SMI_PHY))
+                ret = mv88e6xxx_mdio_write_indirect(chip, addr, regnum, val);
         else
-                ret = _mv88e6xxx_phy_write(ps, addr, regnum, val);
+                ret = mv88e6xxx_mdio_write_direct(chip, addr, regnum, val);
 
-        mutex_unlock(&ps->smi_mutex);
+        mutex_unlock(&chip->reg_lock);
         return ret;
 }
 
+static int mv88e6xxx_mdio_register(struct mv88e6xxx_chip *chip,
+                                   struct device_node *np)
+{
+        static int index;
+        struct mii_bus *bus;
+        int err;
+
+        if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_PPU))
+                mv88e6xxx_ppu_state_init(chip);
+
+        if (np)
+                chip->mdio_np = of_get_child_by_name(np, "mdio");
+
+        bus = devm_mdiobus_alloc(chip->dev);
+        if (!bus)
+                return -ENOMEM;
+
+        bus->priv = (void *)chip;
+        if (np) {
+                bus->name = np->full_name;
+                snprintf(bus->id, MII_BUS_ID_SIZE, "%s", np->full_name);
+        } else {
+                bus->name = "mv88e6xxx SMI";
+                snprintf(bus->id, MII_BUS_ID_SIZE, "mv88e6xxx-%d", index++);
+        }
+
+        bus->read = mv88e6xxx_mdio_read;
+        bus->write = mv88e6xxx_mdio_write;
+        bus->parent = chip->dev;
+
+        if (chip->mdio_np)
+                err = of_mdiobus_register(bus, chip->mdio_np);
+        else
+                err = mdiobus_register(bus);
+        if (err) {
+                dev_err(chip->dev, "Cannot register MDIO bus (%d)\n", err);
+                goto out;
+        }
+        chip->mdio_bus = bus;
+
+        return 0;
+
+out:
+        if (chip->mdio_np)
+                of_node_put(chip->mdio_np);
+
+        return err;
+}
+
+static void mv88e6xxx_mdio_unregister(struct mv88e6xxx_chip *chip)
+
+{
+        struct mii_bus *bus = chip->mdio_bus;
+
+        mdiobus_unregister(bus);
+
+        if (chip->mdio_np)
+                of_node_put(chip->mdio_np);
+}
+
 #ifdef CONFIG_NET_DSA_HWMON
 
 static int mv88e61xx_get_temp(struct dsa_switch *ds, int *temp)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
         int ret;
         int val;
 
         *temp = 0;
 
-        mutex_lock(&ps->smi_mutex);
+        mutex_lock(&chip->reg_lock);
 
-        ret = _mv88e6xxx_phy_write(ps, 0x0, 0x16, 0x6);
+        ret = mv88e6xxx_mdio_write_direct(chip, 0x0, 0x16, 0x6);
         if (ret < 0)
                 goto error;
 
         /* Enable temperature sensor */
-        ret = _mv88e6xxx_phy_read(ps, 0x0, 0x1a);
+        ret = mv88e6xxx_mdio_read_direct(chip, 0x0, 0x1a);
         if (ret < 0)
                 goto error;
 
-        ret = _mv88e6xxx_phy_write(ps, 0x0, 0x1a, ret | (1 << 5));
+        ret = mv88e6xxx_mdio_write_direct(chip, 0x0, 0x1a, ret | (1 << 5));
         if (ret < 0)
                 goto error;
 
         /* Wait for temperature to stabilize */
         usleep_range(10000, 12000);
 
-        val = _mv88e6xxx_phy_read(ps, 0x0, 0x1a);
+        val = mv88e6xxx_mdio_read_direct(chip, 0x0, 0x1a);
         if (val < 0) {
                 ret = val;
                 goto error;
         }
 
         /* Disable temperature sensor */
-        ret = _mv88e6xxx_phy_write(ps, 0x0, 0x1a, ret & ~(1 << 5));
+        ret = mv88e6xxx_mdio_write_direct(chip, 0x0, 0x1a, ret & ~(1 << 5));
         if (ret < 0)
                 goto error;
 
         *temp = ((val & 0x1f) - 5) * 5;
 
 error:
-        _mv88e6xxx_phy_write(ps, 0x0, 0x16, 0x0);
-        mutex_unlock(&ps->smi_mutex);
+        mv88e6xxx_mdio_write_direct(chip, 0x0, 0x16, 0x0);
+        mutex_unlock(&chip->reg_lock);
         return ret;
 }
 
 static int mv88e63xx_get_temp(struct dsa_switch *ds, int *temp)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
-        int phy = mv88e6xxx_6320_family(ps) ? 3 : 0;
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+        int phy = mv88e6xxx_6320_family(chip) ? 3 : 0;
         int ret;
 
         *temp = 0;
 
-        ret = mv88e6xxx_phy_page_read(ds, phy, 6, 27);
+        ret = mv88e6xxx_mdio_page_read(ds, phy, 6, 27);
         if (ret < 0)
                 return ret;
 
@@ -3298,12 +3459,12 @@ static int mv88e63xx_get_temp(struct dsa_switch *ds, int *temp)
 
 static int mv88e6xxx_get_temp(struct dsa_switch *ds, int *temp)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
 
-        if (!mv88e6xxx_has(ps, MV88E6XXX_FLAG_TEMP))
+        if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_TEMP))
                 return -EOPNOTSUPP;
 
-        if (mv88e6xxx_6320_family(ps) || mv88e6xxx_6352_family(ps))
+        if (mv88e6xxx_6320_family(chip) || mv88e6xxx_6352_family(chip))
                 return mv88e63xx_get_temp(ds, temp);
 
         return mv88e61xx_get_temp(ds, temp);
@@ -3311,16 +3472,16 @@ static int mv88e6xxx_get_temp(struct dsa_switch *ds, int *temp)
 
 static int mv88e6xxx_get_temp_limit(struct dsa_switch *ds, int *temp)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
-        int phy = mv88e6xxx_6320_family(ps) ? 3 : 0;
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+        int phy = mv88e6xxx_6320_family(chip) ? 3 : 0;
         int ret;
 
-        if (!mv88e6xxx_has(ps, MV88E6XXX_FLAG_TEMP_LIMIT))
+        if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_TEMP_LIMIT))
                 return -EOPNOTSUPP;
 
         *temp = 0;
 
-        ret = mv88e6xxx_phy_page_read(ds, phy, 6, 26);
+        ret = mv88e6xxx_mdio_page_read(ds, phy, 6, 26);
         if (ret < 0)
                 return ret;
 
@@ -3331,33 +3492,33 @@ static int mv88e6xxx_get_temp_limit(struct dsa_switch *ds, int *temp)
 
 static int mv88e6xxx_set_temp_limit(struct dsa_switch *ds, int temp)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
-        int phy = mv88e6xxx_6320_family(ps) ? 3 : 0;
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+        int phy = mv88e6xxx_6320_family(chip) ? 3 : 0;
         int ret;
 
-        if (!mv88e6xxx_has(ps, MV88E6XXX_FLAG_TEMP_LIMIT))
+        if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_TEMP_LIMIT))
                 return -EOPNOTSUPP;
 
-        ret = mv88e6xxx_phy_page_read(ds, phy, 6, 26);
+        ret = mv88e6xxx_mdio_page_read(ds, phy, 6, 26);
         if (ret < 0)
                 return ret;
         temp = clamp_val(DIV_ROUND_CLOSEST(temp, 5) + 5, 0, 0x1f);
-        return mv88e6xxx_phy_page_write(ds, phy, 6, 26,
-                                        (ret & 0xe0ff) | (temp << 8));
+        return mv88e6xxx_mdio_page_write(ds, phy, 6, 26,
+                                         (ret & 0xe0ff) | (temp << 8));
 }
 
 static int mv88e6xxx_get_temp_alarm(struct dsa_switch *ds, bool *alarm)
 {
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
-        int phy = mv88e6xxx_6320_family(ps) ? 3 : 0;
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+        int phy = mv88e6xxx_6320_family(chip) ? 3 : 0;
         int ret;
 
-        if (!mv88e6xxx_has(ps, MV88E6XXX_FLAG_TEMP_LIMIT))
+        if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_TEMP_LIMIT))
                 return -EOPNOTSUPP;
 
         *alarm = false;
 
-        ret = mv88e6xxx_phy_page_read(ds, phy, 6, 26);
+        ret = mv88e6xxx_mdio_page_read(ds, phy, 6, 26);
         if (ret < 0)
                 return ret;
 
@@ -3374,6 +3535,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
                 .name = "Marvell 88E6085",
                 .num_databases = 4096,
                 .num_ports = 10,
+                .port_base_addr = 0x10,
                 .flags = MV88E6XXX_FLAGS_FAMILY_6097,
         },
 
@@ -3383,6 +3545,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
                 .name = "Marvell 88E6095/88E6095F",
                 .num_databases = 256,
                 .num_ports = 11,
+                .port_base_addr = 0x10,
                 .flags = MV88E6XXX_FLAGS_FAMILY_6095,
         },
 
@@ -3392,6 +3555,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
                 .name = "Marvell 88E6123",
                 .num_databases = 4096,
                 .num_ports = 3,
+                .port_base_addr = 0x10,
                 .flags = MV88E6XXX_FLAGS_FAMILY_6165,
         },
 
@@ -3401,6 +3565,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
                 .name = "Marvell 88E6131",
                 .num_databases = 256,
                 .num_ports = 8,
+                .port_base_addr = 0x10,
                 .flags = MV88E6XXX_FLAGS_FAMILY_6185,
         },
 
@@ -3410,6 +3575,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
                 .name = "Marvell 88E6161",
                 .num_databases = 4096,
                 .num_ports = 6,
+                .port_base_addr = 0x10,
                 .flags = MV88E6XXX_FLAGS_FAMILY_6165,
         },
 
@@ -3419,6 +3585,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
                 .name = "Marvell 88E6165",
                 .num_databases = 4096,
                 .num_ports = 6,
+                .port_base_addr = 0x10,
                 .flags = MV88E6XXX_FLAGS_FAMILY_6165,
         },
 
@@ -3428,6 +3595,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
                 .name = "Marvell 88E6171",
                 .num_databases = 4096,
                 .num_ports = 7,
+                .port_base_addr = 0x10,
                 .flags = MV88E6XXX_FLAGS_FAMILY_6351,
         },
 
@@ -3437,6 +3605,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
                 .name = "Marvell 88E6172",
                 .num_databases = 4096,
                 .num_ports = 7,
+                .port_base_addr = 0x10,
                 .flags = MV88E6XXX_FLAGS_FAMILY_6352,
         },
 
@@ -3446,6 +3615,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
                 .name = "Marvell 88E6175",
                 .num_databases = 4096,
                 .num_ports = 7,
+                .port_base_addr = 0x10,
                 .flags = MV88E6XXX_FLAGS_FAMILY_6351,
         },
 
@@ -3455,6 +3625,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
                 .name = "Marvell 88E6176",
                 .num_databases = 4096,
                 .num_ports = 7,
+                .port_base_addr = 0x10,
                 .flags = MV88E6XXX_FLAGS_FAMILY_6352,
         },
 
@@ -3464,6 +3635,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
                 .name = "Marvell 88E6185",
                 .num_databases = 256,
                 .num_ports = 10,
+                .port_base_addr = 0x10,
                 .flags = MV88E6XXX_FLAGS_FAMILY_6185,
         },
 
@@ -3473,6 +3645,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
                 .name = "Marvell 88E6240",
                 .num_databases = 4096,
                 .num_ports = 7,
+                .port_base_addr = 0x10,
                 .flags = MV88E6XXX_FLAGS_FAMILY_6352,
         },
 
@@ -3482,6 +3655,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
                 .name = "Marvell 88E6320",
                 .num_databases = 4096,
                 .num_ports = 7,
+                .port_base_addr = 0x10,
                 .flags = MV88E6XXX_FLAGS_FAMILY_6320,
         },
 
@@ -3491,6 +3665,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
                 .name = "Marvell 88E6321",
                 .num_databases = 4096,
                 .num_ports = 7,
+                .port_base_addr = 0x10,
                 .flags = MV88E6XXX_FLAGS_FAMILY_6320,
         },
 
@@ -3500,6 +3675,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
                 .name = "Marvell 88E6350",
                 .num_databases = 4096,
                 .num_ports = 7,
+                .port_base_addr = 0x10,
                 .flags = MV88E6XXX_FLAGS_FAMILY_6351,
         },
 
@@ -3509,6 +3685,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
                 .name = "Marvell 88E6351",
                 .num_databases = 4096,
                 .num_ports = 7,
+                .port_base_addr = 0x10,
                 .flags = MV88E6XXX_FLAGS_FAMILY_6351,
         },
 
@@ -3518,75 +3695,128 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
                 .name = "Marvell 88E6352",
                 .num_databases = 4096,
                 .num_ports = 7,
+                .port_base_addr = 0x10,
                 .flags = MV88E6XXX_FLAGS_FAMILY_6352,
         },
 };
 
-static const struct mv88e6xxx_info *
-mv88e6xxx_lookup_info(unsigned int prod_num, const struct mv88e6xxx_info *table,
-                      unsigned int num)
+static const struct mv88e6xxx_info *mv88e6xxx_lookup_info(unsigned int prod_num)
 {
         int i;
 
-        for (i = 0; i < num; ++i)
-                if (table[i].prod_num == prod_num)
-                        return &table[i];
+        for (i = 0; i < ARRAY_SIZE(mv88e6xxx_table); ++i)
+                if (mv88e6xxx_table[i].prod_num == prod_num)
+                        return &mv88e6xxx_table[i];
 
         return NULL;
 }
 
-static const char *mv88e6xxx_drv_probe(struct device *dsa_dev,
-                                       struct device *host_dev, int sw_addr,
-                                       void **priv)
+static int mv88e6xxx_detect(struct mv88e6xxx_chip *chip)
 {
         const struct mv88e6xxx_info *info;
-        struct mv88e6xxx_priv_state *ps;
-        struct mii_bus *bus;
-        const char *name;
         int id, prod_num, rev;
 
-        bus = dsa_host_dev_to_mii_bus(host_dev);
-        if (!bus)
-                return NULL;
-
-        id = __mv88e6xxx_reg_read(bus, sw_addr, REG_PORT(0), PORT_SWITCH_ID);
+        id = mv88e6xxx_reg_read(chip, chip->info->port_base_addr,
+                                PORT_SWITCH_ID);
         if (id < 0)
-                return NULL;
+                return id;
 
         prod_num = (id & 0xfff0) >> 4;
         rev = id & 0x000f;
 
-        info = mv88e6xxx_lookup_info(prod_num, mv88e6xxx_table,
-                                     ARRAY_SIZE(mv88e6xxx_table));
+        info = mv88e6xxx_lookup_info(prod_num);
         if (!info)
+                return -ENODEV;
+
+        /* Update the compatible info with the probed one */
+        chip->info = info;
+
+        dev_info(chip->dev, "switch 0x%x detected: %s, revision %u\n",
+                 chip->info->prod_num, chip->info->name, rev);
+
+        return 0;
+}
+
+static struct mv88e6xxx_chip *mv88e6xxx_alloc_chip(struct device *dev)
+{
+        struct mv88e6xxx_chip *chip;
+
+        chip = devm_kzalloc(dev, sizeof(*chip), GFP_KERNEL);
+        if (!chip)
                 return NULL;
 
-        name = info->name;
+        chip->dev = dev;
+
+        mutex_init(&chip->reg_lock);
+
+        return chip;
+}
+
+static int mv88e6xxx_smi_init(struct mv88e6xxx_chip *chip,
+                              struct mii_bus *bus, int sw_addr)
+{
+        /* ADDR[0] pin is unavailable externally and considered zero */
+        if (sw_addr & 0x1)
+                return -EINVAL;
+
+        if (sw_addr == 0)
+                chip->smi_ops = &mv88e6xxx_smi_single_chip_ops;
+        else if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_MULTI_CHIP))
+                chip->smi_ops = &mv88e6xxx_smi_multi_chip_ops;
+        else
+                return -EINVAL;
+
+        chip->bus = bus;
+        chip->sw_addr = sw_addr;
+
+        return 0;
+}
 
-        ps = devm_kzalloc(dsa_dev, sizeof(*ps), GFP_KERNEL);
-        if (!ps)
+static const char *mv88e6xxx_drv_probe(struct device *dsa_dev,
+                                       struct device *host_dev, int sw_addr,
+                                       void **priv)
+{
+        struct mv88e6xxx_chip *chip;
+        struct mii_bus *bus;
+        int err;
+
+        bus = dsa_host_dev_to_mii_bus(host_dev);
+        if (!bus)
+                return NULL;
+
+        chip = mv88e6xxx_alloc_chip(dsa_dev);
+        if (!chip)
                 return NULL;
 
-        ps->bus = bus;
-        ps->sw_addr = sw_addr;
-        ps->info = info;
-        mutex_init(&ps->smi_mutex);
+        /* Legacy SMI probing will only support chips similar to 88E6085 */
+        chip->info = &mv88e6xxx_table[MV88E6085];
+
+        err = mv88e6xxx_smi_init(chip, bus, sw_addr);
+        if (err)
+                goto free;
+
+        err = mv88e6xxx_detect(chip);
+        if (err)
+                goto free;
+
+        err = mv88e6xxx_mdio_register(chip, NULL);
+        if (err)
+                goto free;
 
-        *priv = ps;
+        *priv = chip;
 
-        dev_info(&ps->bus->dev, "switch 0x%x probed: %s, revision %u\n",
-                 prod_num, name, rev);
+        return chip->info->name;
+free:
+        devm_kfree(dsa_dev, chip);
 
-        return name;
+        return NULL;
 }
 
-struct dsa_switch_driver mv88e6xxx_switch_driver = {
+static struct dsa_switch_driver mv88e6xxx_switch_driver = {
         .tag_protocol           = DSA_TAG_PROTO_EDSA,
         .probe                  = mv88e6xxx_drv_probe,
         .setup                  = mv88e6xxx_setup,
         .set_addr               = mv88e6xxx_set_addr,
-        .phy_read               = mv88e6xxx_phy_read,
-        .phy_write              = mv88e6xxx_phy_write,
         .adjust_link            = mv88e6xxx_adjust_link,
         .get_strings            = mv88e6xxx_get_strings,
         .get_ethtool_stats      = mv88e6xxx_get_ethtool_stats,
@@ -3618,64 +3848,74 @@ struct dsa_switch_driver mv88e6xxx_switch_driver = {
         .port_fdb_dump          = mv88e6xxx_port_fdb_dump,
 };
 
-int mv88e6xxx_probe(struct mdio_device *mdiodev)
+static int mv88e6xxx_register_switch(struct mv88e6xxx_chip *chip,
+                                     struct device_node *np)
 {
-        struct device *dev = &mdiodev->dev;
-        struct device_node *np = dev->of_node;
-        struct mv88e6xxx_priv_state *ps;
-        int id, prod_num, rev;
+        struct device *dev = chip->dev;
         struct dsa_switch *ds;
-        u32 eeprom_len;
-        int err;
 
-        ds = devm_kzalloc(dev, sizeof(*ds) + sizeof(*ps), GFP_KERNEL);
+        ds = devm_kzalloc(dev, sizeof(*ds), GFP_KERNEL);
         if (!ds)
                 return -ENOMEM;
 
-        ps = (struct mv88e6xxx_priv_state *)(ds + 1);
-        ds->priv = ps;
         ds->dev = dev;
-        ps->dev = dev;
-        ps->ds = ds;
-        ps->bus = mdiodev->bus;
-        ps->sw_addr = mdiodev->addr;
-        mutex_init(&ps->smi_mutex);
+        ds->priv = chip;
+        ds->drv = &mv88e6xxx_switch_driver;
 
-        get_device(&ps->bus->dev);
+        dev_set_drvdata(dev, ds);
 
-        ds->drv = &mv88e6xxx_switch_driver;
+        return dsa_register_switch(ds, np);
+}
 
-        id = mv88e6xxx_reg_read(ps, REG_PORT(0), PORT_SWITCH_ID);
-        if (id < 0)
-                return id;
+static void mv88e6xxx_unregister_switch(struct mv88e6xxx_chip *chip)
+{
+        dsa_unregister_switch(chip->ds);
+}
 
-        prod_num = (id & 0xfff0) >> 4;
-        rev = id & 0x000f;
+static int mv88e6xxx_probe(struct mdio_device *mdiodev)
+{
+        struct device *dev = &mdiodev->dev;
+        struct device_node *np = dev->of_node;
+        const struct mv88e6xxx_info *compat_info;
+        struct mv88e6xxx_chip *chip;
+        u32 eeprom_len;
+        int err;
 
-        ps->info = mv88e6xxx_lookup_info(prod_num, mv88e6xxx_table,
-                                         ARRAY_SIZE(mv88e6xxx_table));
-        if (!ps->info)
-                return -ENODEV;
+        compat_info = of_device_get_match_data(dev);
+        if (!compat_info)
+                return -EINVAL;
 
-        ps->reset = devm_gpiod_get(&mdiodev->dev, "reset", GPIOD_ASIS);
-        if (IS_ERR(ps->reset)) {
-                err = PTR_ERR(ps->reset);
-                if (err == -ENOENT) {
-                        /* Optional, so not an error */
-                        ps->reset = NULL;
-                } else {
-                        return err;
-                }
-        }
+        chip = mv88e6xxx_alloc_chip(dev);
+        if (!chip)
+                return -ENOMEM;
 
-        if (mv88e6xxx_has(ps, MV88E6XXX_FLAG_EEPROM) &&
+        chip->info = compat_info;
+
+        err = mv88e6xxx_smi_init(chip, mdiodev->bus, mdiodev->addr);
+        if (err)
+                return err;
+
+        err = mv88e6xxx_detect(chip);
+        if (err)
+                return err;
+
+        chip->reset = devm_gpiod_get_optional(dev, "reset", GPIOD_ASIS);
+        if (IS_ERR(chip->reset))
+                return PTR_ERR(chip->reset);
+
+        if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_EEPROM) &&
             !of_property_read_u32(np, "eeprom-length", &eeprom_len))
-                ps->eeprom_len = eeprom_len;
+                chip->eeprom_len = eeprom_len;
 
-        dev_set_drvdata(dev, ds);
+        err = mv88e6xxx_mdio_register(chip, np);
+        if (err)
+                return err;
 
-        dev_info(dev, "switch 0x%x probed: %s, revision %u\n",
-                 prod_num, ps->info->name, rev);
+        err = mv88e6xxx_register_switch(chip, np);
+        if (err) {
+                mv88e6xxx_mdio_unregister(chip);
+                return err;
+        }
 
         return 0;
 }
@@ -3683,13 +3923,17 @@ int mv88e6xxx_probe(struct mdio_device *mdiodev)
 static void mv88e6xxx_remove(struct mdio_device *mdiodev)
 {
         struct dsa_switch *ds = dev_get_drvdata(&mdiodev->dev);
-        struct mv88e6xxx_priv_state *ps = ds_to_priv(ds);
+        struct mv88e6xxx_chip *chip = ds_to_priv(ds);
 
-        put_device(&ps->bus->dev);
+        mv88e6xxx_unregister_switch(chip);
+        mv88e6xxx_mdio_unregister(chip);
 }
 
 static const struct of_device_id mv88e6xxx_of_match[] = {
-        { .compatible = "marvell,mv88e6085" },
+        {
+                .compatible = "marvell,mv88e6085",
+                .data = &mv88e6xxx_table[MV88E6085],
+        },
         { /* sentinel */ },
 };
 
diff --git a/drivers/net/dsa/mv88e6xxx.h b/drivers/net/dsa/mv88e6xxx/mv88e6xxx.h
index 36d0e1504de1..83f06620133d 100644
--- a/drivers/net/dsa/mv88e6xxx.h
+++ b/drivers/net/dsa/mv88e6xxx/mv88e6xxx.h
@@ -1,5 +1,6 @@
 /*
- * net/dsa/mv88e6xxx.h - Marvell 88e6xxx switch chip support
+ * Marvell 88e6xxx common definitions
+ *
  * Copyright (c) 2008 Marvell Semiconductor
  *
  * This program is free software; you can redistribute it and/or modify
@@ -387,6 +388,12 @@ enum mv88e6xxx_cap {
          */
         MV88E6XXX_CAP_EEPROM,
 
+        /* Multi-chip Addressing Mode.
+         * Some chips require an indirect SMI access when their SMI device
+         * address is not zero. See SMI_CMD and SMI_DATA.
+         */
+        MV88E6XXX_CAP_MULTI_CHIP,
+
         /* Port State Filtering for 802.1D Spanning Tree.
          * See PORT_CONTROL_STATE_* values in the PORT_CONTROL register.
          */
@@ -439,6 +446,7 @@ enum mv88e6xxx_cap {
 #define MV88E6XXX_FLAG_ATU              BIT(MV88E6XXX_CAP_ATU)
 #define MV88E6XXX_FLAG_EEE              BIT(MV88E6XXX_CAP_EEE)
 #define MV88E6XXX_FLAG_EEPROM           BIT(MV88E6XXX_CAP_EEPROM)
+#define MV88E6XXX_FLAG_MULTI_CHIP       BIT(MV88E6XXX_CAP_MULTI_CHIP)
 #define MV88E6XXX_FLAG_PORTSTATE        BIT(MV88E6XXX_CAP_PORTSTATE)
 #define MV88E6XXX_FLAG_PPU              BIT(MV88E6XXX_CAP_PPU)
 #define MV88E6XXX_FLAG_PPU_ACTIVE       BIT(MV88E6XXX_CAP_PPU_ACTIVE)
@@ -452,25 +460,29 @@ enum mv88e6xxx_cap {
 
 #define MV88E6XXX_FLAGS_FAMILY_6095     \
         (MV88E6XXX_FLAG_ATU |           \
+         MV88E6XXX_FLAG_MULTI_CHIP |    \
          MV88E6XXX_FLAG_PPU |           \
          MV88E6XXX_FLAG_VLANTABLE |     \
          MV88E6XXX_FLAG_VTU)
 
 #define MV88E6XXX_FLAGS_FAMILY_6097     \
         (MV88E6XXX_FLAG_ATU |           \
+         MV88E6XXX_FLAG_MULTI_CHIP |    \
          MV88E6XXX_FLAG_PPU |           \
          MV88E6XXX_FLAG_STU |           \
          MV88E6XXX_FLAG_VLANTABLE |     \
          MV88E6XXX_FLAG_VTU)
 
 #define MV88E6XXX_FLAGS_FAMILY_6165     \
-        (MV88E6XXX_FLAG_STU |           \
+        (MV88E6XXX_FLAG_MULTI_CHIP |    \
+         MV88E6XXX_FLAG_STU |           \
          MV88E6XXX_FLAG_SWITCH_MAC |    \
          MV88E6XXX_FLAG_TEMP |          \
          MV88E6XXX_FLAG_VTU)
 
 #define MV88E6XXX_FLAGS_FAMILY_6185     \
         (MV88E6XXX_FLAG_ATU |           \
+         MV88E6XXX_FLAG_MULTI_CHIP |    \
          MV88E6XXX_FLAG_PPU |           \
          MV88E6XXX_FLAG_VLANTABLE |     \
          MV88E6XXX_FLAG_VTU)
@@ -479,6 +491,7 @@ enum mv88e6xxx_cap {
         (MV88E6XXX_FLAG_ATU |           \
          MV88E6XXX_FLAG_EEE |           \
          MV88E6XXX_FLAG_EEPROM |        \
+         MV88E6XXX_FLAG_MULTI_CHIP |    \
          MV88E6XXX_FLAG_PORTSTATE |     \
          MV88E6XXX_FLAG_PPU_ACTIVE |    \
          MV88E6XXX_FLAG_SMI_PHY |       \
@@ -490,6 +503,7 @@ enum mv88e6xxx_cap {
 
 #define MV88E6XXX_FLAGS_FAMILY_6351     \
         (MV88E6XXX_FLAG_ATU |           \
+         MV88E6XXX_FLAG_MULTI_CHIP |    \
          MV88E6XXX_FLAG_PORTSTATE |     \
          MV88E6XXX_FLAG_PPU_ACTIVE |    \
          MV88E6XXX_FLAG_SMI_PHY |       \
@@ -503,6 +517,7 @@ enum mv88e6xxx_cap {
         (MV88E6XXX_FLAG_ATU |           \
          MV88E6XXX_FLAG_EEE |           \
          MV88E6XXX_FLAG_EEPROM |        \
+         MV88E6XXX_FLAG_MULTI_CHIP |    \
          MV88E6XXX_FLAG_PORTSTATE |     \
          MV88E6XXX_FLAG_PPU_ACTIVE |    \
          MV88E6XXX_FLAG_SMI_PHY |       \
@@ -519,6 +534,7 @@ struct mv88e6xxx_info {
         const char *name;
         unsigned int num_databases;
         unsigned int num_ports;
+        unsigned int port_base_addr;
         unsigned long flags;
 };
 
@@ -541,11 +557,13 @@ struct mv88e6xxx_vtu_stu_entry {
         u8      data[DSA_MAX_PORTS];
 };
 
+struct mv88e6xxx_ops;
+
 struct mv88e6xxx_priv_port {
         struct net_device *bridge_dev;
 };
 
-struct mv88e6xxx_priv_state {
+struct mv88e6xxx_chip {
         const struct mv88e6xxx_info *info;
 
         /* The dsa_switch this private structure is related to */
@@ -554,15 +572,13 @@ struct mv88e6xxx_priv_state {
         /* The device this structure is associated to */
         struct device *dev;
 
-        /* When using multi-chip addressing, this mutex protects
-         * access to the indirect access registers.  (In single-chip
-         * mode, this mutex is effectively useless.)
-         */
-        struct mutex    smi_mutex;
+        /* This mutex protects the access to the switch registers */
+        struct mutex reg_lock;
 
         /* The MII bus and the address on the bus that is used to
          * communication with the switch
          */
+        const struct mv88e6xxx_ops *smi_ops;
         struct mii_bus *bus;
         int sw_addr;
 
@@ -600,6 +616,17 @@ struct mv88e6xxx_priv_state {
 
         /* set to size of eeprom if supported by the switch */
         int             eeprom_len;
+
+        /* Device node for the MDIO bus */
+        struct device_node *mdio_np;
+
+        /* And the MDIO bus itself */
+        struct mii_bus *mdio_bus;
+};
+
+struct mv88e6xxx_ops {
+        int (*read)(struct mv88e6xxx_chip *chip, int addr, int reg, u16 *val);
+        int (*write)(struct mv88e6xxx_chip *chip, int addr, int reg, u16 val);
 };
 
 enum stat_type {
@@ -615,10 +642,10 @@ struct mv88e6xxx_hw_stat {
         enum stat_type type;
 };
 
-static inline bool mv88e6xxx_has(struct mv88e6xxx_priv_state *ps,
+static inline bool mv88e6xxx_has(struct mv88e6xxx_chip *chip,
                                  unsigned long flags)
 {
-        return (ps->info->flags & flags) == flags;
+        return (chip->info->flags & flags) == flags;
 }
 
 #endif
diff --git a/drivers/net/ethernet/8390/ax88796.c b/drivers/net/ethernet/8390/ax88796.c
index c89b9aeeceb6..5698f5354c0b 100644
--- a/drivers/net/ethernet/8390/ax88796.c
+++ b/drivers/net/ethernet/8390/ax88796.c
@@ -84,7 +84,6 @@ static u32 ax_msg_enable;
 struct ax_device {
         struct mii_bus *mii_bus;
         struct mdiobb_ctrl bb_ctrl;
-        struct phy_device *phy_dev;
         void __iomem *addr_memr;
         u8 reg_memr;
         int link;
@@ -320,7 +319,7 @@ static void ax_block_output(struct net_device *dev, int count,
 static void ax_handle_link_change(struct net_device *dev)
 {
         struct ax_device  *ax = to_ax_dev(dev);
-        struct phy_device *phy_dev = ax->phy_dev;
+        struct phy_device *phy_dev = dev->phydev;
         int status_change = 0;
 
         if (phy_dev->link && ((ax->speed != phy_dev->speed) ||
@@ -369,8 +368,6 @@ static int ax_mii_probe(struct net_device *dev)
         phy_dev->supported &= PHY_BASIC_FEATURES;
         phy_dev->advertising = phy_dev->supported;
 
-        ax->phy_dev = phy_dev;
-
         netdev_info(dev, "PHY driver [%s] (mii_bus:phy_addr=%s, irq=%d)\n",
                     phy_dev->drv->name, phydev_name(phy_dev), phy_dev->irq);
 
@@ -410,7 +407,7 @@ static int ax_open(struct net_device *dev)
         ret = ax_mii_probe(dev);
         if (ret)
                 goto failed_mii_probe;
-        phy_start(ax->phy_dev);
+        phy_start(dev->phydev);
 
         ret = ax_ei_open(dev);
         if (ret)
@@ -421,7 +418,7 @@ static int ax_open(struct net_device *dev)
         return 0;
 
  failed_ax_ei_open:
-        phy_disconnect(ax->phy_dev);
+        phy_disconnect(dev->phydev);
  failed_mii_probe:
         ax_phy_switch(dev, 0);
         free_irq(dev->irq, dev);
@@ -442,7 +439,7 @@ static int ax_close(struct net_device *dev)
 
         /* turn the phy off */
         ax_phy_switch(dev, 0);
-        phy_disconnect(ax->phy_dev);
+        phy_disconnect(dev->phydev);
 
         free_irq(dev->irq, dev);
         return 0;
@@ -450,8 +447,7 @@ static int ax_close(struct net_device *dev)
 
 static int ax_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
 {
-        struct ax_device *ax = to_ax_dev(dev);
-        struct phy_device *phy_dev = ax->phy_dev;
+        struct phy_device *phy_dev = dev->phydev;
 
         if (!netif_running(dev))
                 return -EINVAL;
@@ -474,28 +470,6 @@ static void ax_get_drvinfo(struct net_device *dev,
         strlcpy(info->bus_info, pdev->name, sizeof(info->bus_info));
 }
 
-static int ax_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
-{
-        struct ax_device *ax = to_ax_dev(dev);
-        struct phy_device *phy_dev = ax->phy_dev;
-
-        if (!phy_dev)
-                return -ENODEV;
-
-        return phy_ethtool_gset(phy_dev, cmd);
-}
-
-static int ax_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
-{
-        struct ax_device *ax = to_ax_dev(dev);
-        struct phy_device *phy_dev = ax->phy_dev;
-
-        if (!phy_dev)
-                return -ENODEV;
-
-        return phy_ethtool_sset(phy_dev, cmd);
-}
-
 static u32 ax_get_msglevel(struct net_device *dev)
 {
         struct ei_device *ei_local = netdev_priv(dev);
@@ -512,12 +486,12 @@ static void ax_set_msglevel(struct net_device *dev, u32 v)
 
 static const struct ethtool_ops ax_ethtool_ops = {
         .get_drvinfo            = ax_get_drvinfo,
-        .get_settings           = ax_get_settings,
-        .set_settings           = ax_set_settings,
         .get_link               = ethtool_op_get_link,
         .get_ts_info            = ethtool_op_get_ts_info,
         .get_msglevel           = ax_get_msglevel,
         .set_msglevel           = ax_set_msglevel,
+        .get_link_ksettings     = phy_ethtool_get_link_ksettings,
+        .set_link_ksettings     = phy_ethtool_set_link_ksettings,
 };
 
 #ifdef CONFIG_AX88796_93CX6
diff --git a/drivers/net/ethernet/agere/et131x.c b/drivers/net/ethernet/agere/et131x.c
index 30defe6c81f2..cd7e2e5f496b 100644
--- a/drivers/net/ethernet/agere/et131x.c
+++ b/drivers/net/ethernet/agere/et131x.c
@@ -440,7 +440,6 @@ struct et131x_adapter {
         struct net_device *netdev;
         struct pci_dev *pdev;
         struct mii_bus *mii_bus;
-        struct phy_device *phydev;
         struct napi_struct napi;
 
         /* Flags that indicate current state of the adapter */
@@ -864,7 +863,7 @@ static void et1310_config_mac_regs2(struct et131x_adapter *adapter)
 {
         int32_t delay = 0;
         struct mac_regs __iomem *mac = &adapter->regs->mac;
-        struct phy_device *phydev = adapter->phydev;
+        struct phy_device *phydev = adapter->netdev->phydev;
         u32 cfg1;
         u32 cfg2;
         u32 ifctrl;
@@ -1035,7 +1034,7 @@ static void et1310_setup_device_for_unicast(struct et131x_adapter *adapter)
 static void et1310_config_rxmac_regs(struct et131x_adapter *adapter)
 {
         struct rxmac_regs __iomem *rxmac = &adapter->regs->rxmac;
-        struct phy_device *phydev = adapter->phydev;
+        struct phy_device *phydev = adapter->netdev->phydev;
         u32 sa_lo;
         u32 sa_hi = 0;
         u32 pf_ctrl = 0;
@@ -1230,7 +1229,7 @@ out:
 
 static int et131x_mii_read(struct et131x_adapter *adapter, u8 reg, u16 *value)
 {
-        struct phy_device *phydev = adapter->phydev;
+        struct phy_device *phydev = adapter->netdev->phydev;
 
         if (!phydev)
                 return -EIO;
@@ -1311,7 +1310,7 @@ static void et1310_phy_read_mii_bit(struct et131x_adapter *adapter,
 
 static void et1310_config_flow_control(struct et131x_adapter *adapter)
 {
-        struct phy_device *phydev = adapter->phydev;
+        struct phy_device *phydev = adapter->netdev->phydev;
 
         if (phydev->duplex == DUPLEX_HALF) {
                 adapter->flow = FLOW_NONE;
@@ -1456,7 +1455,7 @@ static int et131x_mdio_write(struct mii_bus *bus, int phy_addr,
 static void et1310_phy_power_switch(struct et131x_adapter *adapter, bool down)
 {
         u16 data;
-        struct  phy_device *phydev = adapter->phydev;
+        struct  phy_device *phydev = adapter->netdev->phydev;
 
         et131x_mii_read(adapter, MII_BMCR, &data);
         data &= ~BMCR_PDOWN;
@@ -1469,7 +1468,7 @@ static void et1310_phy_power_switch(struct et131x_adapter *adapter, bool down)
 static void et131x_xcvr_init(struct et131x_adapter *adapter)
 {
         u16 lcr2;
-        struct  phy_device *phydev = adapter->phydev;
+        struct  phy_device *phydev = adapter->netdev->phydev;
 
         /* Set the LED behavior such that LED 1 indicates speed (off =
          * 10Mbits, blink = 100Mbits, on = 1000Mbits) and LED 2 indicates
@@ -2111,7 +2110,7 @@ static int et131x_init_recv(struct et131x_adapter *adapter)
 /* et131x_set_rx_dma_timer - Set the heartbeat timer according to line rate */
 static void et131x_set_rx_dma_timer(struct et131x_adapter *adapter)
 {
-        struct phy_device *phydev = adapter->phydev;
+        struct phy_device *phydev = adapter->netdev->phydev;
 
         /* For version B silicon, we do not use the RxDMA timer for 10 and 100
          * Mbits/s line rates. We do not enable and RxDMA interrupt coalescing.
@@ -2426,7 +2425,7 @@ static int nic_send_packet(struct et131x_adapter *adapter, struct tcb *tcb)
         struct sk_buff *skb = tcb->skb;
         u32 nr_frags = skb_shinfo(skb)->nr_frags + 1;
         struct skb_frag_struct *frags = &skb_shinfo(skb)->frags[0];
-        struct phy_device *phydev = adapter->phydev;
+        struct phy_device *phydev = adapter->netdev->phydev;
         dma_addr_t dma_addr;
         struct tx_ring *tx_ring = &adapter->tx_ring;
 
@@ -2791,22 +2790,6 @@ static void et131x_handle_send_pkts(struct et131x_adapter *adapter)
         spin_unlock_irqrestore(&adapter->tcb_send_qlock, flags);
 }
 
-static int et131x_get_settings(struct net_device *netdev,
-                               struct ethtool_cmd *cmd)
-{
-        struct et131x_adapter *adapter = netdev_priv(netdev);
-
-        return phy_ethtool_gset(adapter->phydev, cmd);
-}
-
-static int et131x_set_settings(struct net_device *netdev,
-                               struct ethtool_cmd *cmd)
-{
-        struct et131x_adapter *adapter = netdev_priv(netdev);
-
-        return phy_ethtool_sset(adapter->phydev, cmd);
-}
-
 static int et131x_get_regs_len(struct net_device *netdev)
 {
 #define ET131X_REGS_LEN 256
@@ -2979,12 +2962,12 @@ static void et131x_get_drvinfo(struct net_device *netdev,
 }
 
 static struct ethtool_ops et131x_ethtool_ops = {
-        .get_settings   = et131x_get_settings,
-        .set_settings   = et131x_set_settings,
         .get_drvinfo    = et131x_get_drvinfo,
         .get_regs_len   = et131x_get_regs_len,
         .get_regs       = et131x_get_regs,
         .get_link       = ethtool_op_get_link,
+        .get_link_ksettings = phy_ethtool_get_link_ksettings,
+        .set_link_ksettings = phy_ethtool_set_link_ksettings,
 };
 
 /* et131x_hwaddr_init - set up the MAC Address */
@@ -3098,7 +3081,7 @@ err_out:
 static void et131x_error_timer_handler(unsigned long data)
 {
         struct et131x_adapter *adapter = (struct et131x_adapter *)data;
-        struct phy_device *phydev = adapter->phydev;
+        struct phy_device *phydev = adapter->netdev->phydev;
 
         if (et1310_in_phy_coma(adapter)) {
                 /* Bring the device immediately out of coma, to
@@ -3168,7 +3151,7 @@ static int et131x_adapter_memory_alloc(struct et131x_adapter *adapter)
 static void et131x_adjust_link(struct net_device *netdev)
 {
         struct et131x_adapter *adapter = netdev_priv(netdev);
-        struct  phy_device *phydev = adapter->phydev;
+        struct  phy_device *phydev = netdev->phydev;
 
         if (!phydev)
                 return;
@@ -3287,7 +3270,6 @@ static int et131x_mii_probe(struct net_device *netdev)
 
         phydev->advertising = phydev->supported;
         phydev->autoneg = AUTONEG_ENABLE;
-        adapter->phydev = phydev;
 
         phy_attached_info(phydev);
 
@@ -3323,7 +3305,7 @@ static void et131x_pci_remove(struct pci_dev *pdev)
 
         unregister_netdev(netdev);
         netif_napi_del(&adapter->napi);
-        phy_disconnect(adapter->phydev);
+        phy_disconnect(netdev->phydev);
         mdiobus_unregister(adapter->mii_bus);
         mdiobus_free(adapter->mii_bus);
 
@@ -3338,20 +3320,16 @@ static void et131x_pci_remove(struct pci_dev *pdev)
 
 static void et131x_up(struct net_device *netdev)
 {
-        struct et131x_adapter *adapter = netdev_priv(netdev);
-
         et131x_enable_txrx(netdev);
-        phy_start(adapter->phydev);
+        phy_start(netdev->phydev);
 }
 
 static void et131x_down(struct net_device *netdev)
 {
-        struct et131x_adapter *adapter = netdev_priv(netdev);
-
         /* Save the timestamp for the TX watchdog, prevent a timeout */
         netif_trans_update(netdev);
 
-        phy_stop(adapter->phydev);
+        phy_stop(netdev->phydev);
         et131x_disable_txrx(netdev);
 }
 
@@ -3684,12 +3662,10 @@ static int et131x_close(struct net_device *netdev)
 static int et131x_ioctl(struct net_device *netdev, struct ifreq *reqbuf,
                         int cmd)
 {
-        struct et131x_adapter *adapter = netdev_priv(netdev);
-
-        if (!adapter->phydev)
+        if (!netdev->phydev)
                 return -EINVAL;
 
-        return phy_mii_ioctl(adapter->phydev, reqbuf, cmd);
+        return phy_mii_ioctl(netdev->phydev, reqbuf, cmd);
 }
 
 /* et131x_set_packet_filter - Configures the Rx Packet filtering */
@@ -4073,7 +4049,7 @@ out:
         return rc;
 
 err_phy_disconnect:
-        phy_disconnect(adapter->phydev);
+        phy_disconnect(netdev->phydev);
 err_mdio_unregister:
         mdiobus_unregister(adapter->mii_bus);
 err_mdio_free:
diff --git a/drivers/net/ethernet/allwinner/sun4i-emac.c b/drivers/net/ethernet/allwinner/sun4i-emac.c
index de2c4bf5fac4..6ffdff68bfc4 100644
--- a/drivers/net/ethernet/allwinner/sun4i-emac.c
+++ b/drivers/net/ethernet/allwinner/sun4i-emac.c
@@ -77,7 +77,6 @@ struct emac_board_info {
 
         int                     emacrx_completed_flag;
 
-        struct phy_device       *phy_dev;
         struct device_node      *phy_node;
         unsigned int            link;
         unsigned int            speed;
@@ -115,7 +114,7 @@ static void emac_update_duplex(struct net_device *dev)
 static void emac_handle_link_change(struct net_device *dev)
 {
         struct emac_board_info *db = netdev_priv(dev);
-        struct phy_device *phydev = db->phy_dev;
+        struct phy_device *phydev = dev->phydev;
         unsigned long flags;
         int status_change = 0;
 
@@ -154,21 +153,22 @@ static void emac_handle_link_change(struct net_device *dev)
 static int emac_mdio_probe(struct net_device *dev)
 {
         struct emac_board_info *db = netdev_priv(dev);
+        struct phy_device *phydev;
 
         /* to-do: PHY interrupts are currently not supported */
 
         /* attach the mac to the phy */
-        db->phy_dev = of_phy_connect(db->ndev, db->phy_node,
-                                     &emac_handle_link_change, 0,
-                                     db->phy_interface);
-        if (!db->phy_dev) {
+        phydev = of_phy_connect(db->ndev, db->phy_node,
+                                &emac_handle_link_change, 0,
+                                db->phy_interface);
+        if (!phydev) {
                 netdev_err(db->ndev, "could not find the PHY\n");
                 return -ENODEV;
         }
 
         /* mask with MAC supported features */
-        db->phy_dev->supported &= PHY_BASIC_FEATURES;
-        db->phy_dev->advertising = db->phy_dev->supported;
+        phydev->supported &= PHY_BASIC_FEATURES;
+        phydev->advertising = phydev->supported;
 
         db->link = 0;
         db->speed = 0;
@@ -179,10 +179,7 @@ static int emac_mdio_probe(struct net_device *dev)
 
 static void emac_mdio_remove(struct net_device *dev)
 {
-        struct emac_board_info *db = netdev_priv(dev);
-
-        phy_disconnect(db->phy_dev);
-        db->phy_dev = NULL;
+        phy_disconnect(dev->phydev);
 }
 
 static void emac_reset(struct emac_board_info *db)
@@ -208,8 +205,7 @@ static void emac_inblk_32bit(void __iomem *reg, void *data, int count)
 
 static int emac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
 {
-        struct emac_board_info *dm = netdev_priv(dev);
-        struct phy_device *phydev = dm->phy_dev;
+        struct phy_device *phydev = dev->phydev;
 
         if (!netif_running(dev))
                 return -EINVAL;
@@ -229,33 +225,11 @@ static void emac_get_drvinfo(struct net_device *dev,
         strlcpy(info->bus_info, dev_name(&dev->dev), sizeof(info->bus_info));
 }
 
-static int emac_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
-{
-        struct emac_board_info *dm = netdev_priv(dev);
-        struct phy_device *phydev = dm->phy_dev;
-
-        if (!phydev)
-                return -ENODEV;
-
-        return phy_ethtool_gset(phydev, cmd);
-}
-
-static int emac_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
-{
-        struct emac_board_info *dm = netdev_priv(dev);
-        struct phy_device *phydev = dm->phy_dev;
-
-        if (!phydev)
-                return -ENODEV;
-
-        return phy_ethtool_sset(phydev, cmd);
-}
-
 static const struct ethtool_ops emac_ethtool_ops = {
         .get_drvinfo    = emac_get_drvinfo,
-        .get_settings   = emac_get_settings,
-        .set_settings   = emac_set_settings,
         .get_link       = ethtool_op_get_link,
+        .get_link_ksettings = phy_ethtool_get_link_ksettings,
+        .set_link_ksettings = phy_ethtool_set_link_ksettings,
 };
 
 static unsigned int emac_setup(struct net_device *ndev)
@@ -744,7 +718,7 @@ static int emac_open(struct net_device *dev)
                 return ret;
         }
 
-        phy_start(db->phy_dev);
+        phy_start(dev->phydev);
         netif_start_queue(dev);
 
         return 0;
@@ -781,7 +755,7 @@ static int emac_stop(struct net_device *ndev)
         netif_stop_queue(ndev);
         netif_carrier_off(ndev);
 
-        phy_stop(db->phy_dev);
+        phy_stop(ndev->phydev);
 
         emac_mdio_remove(ndev);
 
diff --git a/drivers/net/ethernet/altera/altera_tse.h b/drivers/net/ethernet/altera/altera_tse.h
index 103c30ddddf7..e0052003d16f 100644
--- a/drivers/net/ethernet/altera/altera_tse.h
+++ b/drivers/net/ethernet/altera/altera_tse.h
@@ -473,7 +473,6 @@ struct altera_tse_private {
         int phy_addr;           /* PHY's MDIO address, -1 for autodetection */
         phy_interface_t phy_iface;
         struct mii_bus *mdio;
-        struct phy_device *phydev;
         int oldspeed;
         int oldduplex;
         int oldlink;
diff --git a/drivers/net/ethernet/altera/altera_tse_ethtool.c b/drivers/net/ethernet/altera/altera_tse_ethtool.c
index be72e1e64525..7c367713c3e6 100644
--- a/drivers/net/ethernet/altera/altera_tse_ethtool.c
+++ b/drivers/net/ethernet/altera/altera_tse_ethtool.c
@@ -233,40 +233,18 @@ static void tse_get_regs(struct net_device *dev, struct ethtool_regs *regs,
                 buf[i] = csrrd32(priv->mac_dev, i * 4);
 }
 
-static int tse_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
-{
-        struct altera_tse_private *priv = netdev_priv(dev);
-        struct phy_device *phydev = priv->phydev;
-
-        if (phydev == NULL)
-                return -ENODEV;
-
-        return phy_ethtool_gset(phydev, cmd);
-}
-
-static int tse_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
-{
-        struct altera_tse_private *priv = netdev_priv(dev);
-        struct phy_device *phydev = priv->phydev;
-
-        if (phydev == NULL)
-                return -ENODEV;
-
-        return phy_ethtool_sset(phydev, cmd);
-}
-
 static const struct ethtool_ops tse_ethtool_ops = {
         .get_drvinfo = tse_get_drvinfo,
         .get_regs_len = tse_reglen,
         .get_regs = tse_get_regs,
         .get_link = ethtool_op_get_link,
-        .get_settings = tse_get_settings,
-        .set_settings = tse_set_settings,
         .get_strings = tse_gstrings,
         .get_sset_count = tse_sset_count,
         .get_ethtool_stats = tse_fill_stats,
         .get_msglevel = tse_get_msglevel,
         .set_msglevel = tse_set_msglevel,
+        .get_link_ksettings = phy_ethtool_get_link_ksettings,
+        .set_link_ksettings = phy_ethtool_set_link_ksettings,
 };
 
 void altera_tse_set_ethtool_ops(struct net_device *netdev)
diff --git a/drivers/net/ethernet/altera/altera_tse_main.c b/drivers/net/ethernet/altera/altera_tse_main.c
index f749e4d389eb..49025e99fb0e 100644
--- a/drivers/net/ethernet/altera/altera_tse_main.c
+++ b/drivers/net/ethernet/altera/altera_tse_main.c
@@ -625,7 +625,7 @@ out:
 static void altera_tse_adjust_link(struct net_device *dev)
 {
         struct altera_tse_private *priv = netdev_priv(dev);
-        struct phy_device *phydev = priv->phydev;
+        struct phy_device *phydev = dev->phydev;
         int new_state = 0;
 
         /* only change config if there is a link */
@@ -845,7 +845,6 @@ static int init_phy(struct net_device *dev)
         netdev_dbg(dev, "attached to PHY %d UID 0x%08x Link = %d\n",
                    phydev->mdio.addr, phydev->phy_id, phydev->link);
 
-        priv->phydev = phydev;
         return 0;
 }
 
@@ -1172,8 +1171,8 @@ static int tse_open(struct net_device *dev)
 
         spin_unlock_irqrestore(&priv->rxdma_irq_lock, flags);
 
-        if (priv->phydev)
-                phy_start(priv->phydev);
+        if (dev->phydev)
+                phy_start(dev->phydev);
 
         napi_enable(&priv->napi);
         netif_start_queue(dev);
@@ -1205,8 +1204,8 @@ static int tse_shutdown(struct net_device *dev)
         unsigned long int flags;
 
         /* Stop the PHY */
-        if (priv->phydev)
-                phy_stop(priv->phydev);
+        if (dev->phydev)
+                phy_stop(dev->phydev);
 
         netif_stop_queue(dev);
         napi_disable(&priv->napi);
@@ -1545,10 +1544,9 @@ err_free_netdev:
 static int altera_tse_remove(struct platform_device *pdev)
 {
         struct net_device *ndev = platform_get_drvdata(pdev);
-        struct altera_tse_private *priv = netdev_priv(ndev);
 
-        if (priv->phydev)
-                phy_disconnect(priv->phydev);
+        if (ndev->phydev)
+                phy_disconnect(ndev->phydev);
 
         platform_set_drvdata(pdev, NULL);
         altera_tse_mdio_destroy(ndev);
diff --git a/drivers/net/ethernet/aurora/nb8800.c b/drivers/net/ethernet/aurora/nb8800.c
index 08a23e6b60e9..dc2c35dce216 100644
--- a/drivers/net/ethernet/aurora/nb8800.c
+++ b/drivers/net/ethernet/aurora/nb8800.c
@@ -631,7 +631,7 @@ static void nb8800_mac_config(struct net_device *dev)
 static void nb8800_pause_config(struct net_device *dev)
 {
         struct nb8800_priv *priv = netdev_priv(dev);
-        struct phy_device *phydev = priv->phydev;
+        struct phy_device *phydev = dev->phydev;
         u32 rxcr;
 
         if (priv->pause_aneg) {
@@ -664,7 +664,7 @@ static void nb8800_pause_config(struct net_device *dev)
 static void nb8800_link_reconfigure(struct net_device *dev)
 {
         struct nb8800_priv *priv = netdev_priv(dev);
-        struct phy_device *phydev = priv->phydev;
+        struct phy_device *phydev = dev->phydev;
         int change = 0;
 
         if (phydev->link) {
@@ -690,7 +690,7 @@ static void nb8800_link_reconfigure(struct net_device *dev)
         }
 
         if (change)
-                phy_print_status(priv->phydev);
+                phy_print_status(phydev);
 }
 
 static void nb8800_update_mac_addr(struct net_device *dev)
@@ -935,9 +935,10 @@ static int nb8800_dma_stop(struct net_device *dev)
 static void nb8800_pause_adv(struct net_device *dev)
 {
         struct nb8800_priv *priv = netdev_priv(dev);
+        struct phy_device *phydev = dev->phydev;
         u32 adv = 0;
 
-        if (!priv->phydev)
+        if (!phydev)
                 return;
 
         if (priv->pause_rx)
@@ -945,13 +946,14 @@ static void nb8800_pause_adv(struct net_device *dev)
         if (priv->pause_tx)
                 adv ^= ADVERTISED_Asym_Pause;
 
-        priv->phydev->supported |= adv;
-        priv->phydev->advertising |= adv;
+        phydev->supported |= adv;
+        phydev->advertising |= adv;
 }
 
 static int nb8800_open(struct net_device *dev)
 {
         struct nb8800_priv *priv = netdev_priv(dev);
+        struct phy_device *phydev;
         int err;
 
         /* clear any pending interrupts */
@@ -969,10 +971,10 @@ static int nb8800_open(struct net_device *dev)
         nb8800_mac_rx(dev, true);
         nb8800_mac_tx(dev, true);
 
-        priv->phydev = of_phy_connect(dev, priv->phy_node,
-                                      nb8800_link_reconfigure, 0,
-                                      priv->phy_mode);
-        if (!priv->phydev)
+        phydev = of_phy_connect(dev, priv->phy_node,
+                                nb8800_link_reconfigure, 0,
+                                priv->phy_mode);
+        if (!phydev)
                 goto err_free_irq;
 
         nb8800_pause_adv(dev);
@@ -982,7 +984,7 @@ static int nb8800_open(struct net_device *dev)
         netif_start_queue(dev);
 
         nb8800_start_rx(dev);
-        phy_start(priv->phydev);
+        phy_start(phydev);
 
         return 0;
 
@@ -997,8 +999,9 @@ err_free_dma:
 static int nb8800_stop(struct net_device *dev)
 {
         struct nb8800_priv *priv = netdev_priv(dev);
+        struct phy_device *phydev = dev->phydev;
 
-        phy_stop(priv->phydev);
+        phy_stop(phydev);
 
         netif_stop_queue(dev);
         napi_disable(&priv->napi);
@@ -1007,8 +1010,7 @@ static int nb8800_stop(struct net_device *dev)
         nb8800_mac_rx(dev, false);
         nb8800_mac_tx(dev, false);
 
-        phy_disconnect(priv->phydev);
-        priv->phydev = NULL;
+        phy_disconnect(phydev);
 
         free_irq(dev->irq, dev);
 
@@ -1019,9 +1021,7 @@ static int nb8800_stop(struct net_device *dev)
 
 static int nb8800_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
 {
-        struct nb8800_priv *priv = netdev_priv(dev);
-
-        return phy_mii_ioctl(priv->phydev, rq, cmd);
+        return phy_mii_ioctl(dev->phydev, rq, cmd);
 }
 
 static const struct net_device_ops nb8800_netdev_ops = {
@@ -1035,34 +1035,14 @@ static const struct net_device_ops nb8800_netdev_ops = {
         .ndo_validate_addr      = eth_validate_addr,
 };
 
-static int nb8800_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
-{
-        struct nb8800_priv *priv = netdev_priv(dev);
-
-        if (!priv->phydev)
-                return -ENODEV;
-
-        return phy_ethtool_gset(priv->phydev, cmd);
-}
-
-static int nb8800_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
-{
-        struct nb8800_priv *priv = netdev_priv(dev);
-
-        if (!priv->phydev)
-                return -ENODEV;
-
-        return phy_ethtool_sset(priv->phydev, cmd);
-}
-
 static int nb8800_nway_reset(struct net_device *dev)
 {
-        struct nb8800_priv *priv = netdev_priv(dev);
+        struct phy_device *phydev = dev->phydev;
 
-        if (!priv->phydev)
+        if (!phydev)
                 return -ENODEV;
 
-        return genphy_restart_aneg(priv->phydev);
+        return genphy_restart_aneg(phydev);
 }
 
 static void nb8800_get_pauseparam(struct net_device *dev,
@@ -1079,6 +1059,7 @@ static int nb8800_set_pauseparam(struct net_device *dev,
                                  struct ethtool_pauseparam *pp)
 {
         struct nb8800_priv *priv = netdev_priv(dev);
+        struct phy_device *phydev = dev->phydev;
 
         priv->pause_aneg = pp->autoneg;
         priv->pause_rx = pp->rx_pause;
@@ -1088,8 +1069,8 @@ static int nb8800_set_pauseparam(struct net_device *dev,
 
         if (!priv->pause_aneg)
                 nb8800_pause_config(dev);
-        else if (priv->phydev)
-                phy_start_aneg(priv->phydev);
+        else if (phydev)
+                phy_start_aneg(phydev);
 
         return 0;
 }
@@ -1182,8 +1163,6 @@ static void nb8800_get_ethtool_stats(struct net_device *dev,
 }
 
 static const struct ethtool_ops nb8800_ethtool_ops = {
-        .get_settings           = nb8800_get_settings,
-        .set_settings           = nb8800_set_settings,
         .nway_reset             = nb8800_nway_reset,
         .get_link               = ethtool_op_get_link,
         .get_pauseparam         = nb8800_get_pauseparam,
@@ -1191,6 +1170,8 @@ static const struct ethtool_ops nb8800_ethtool_ops = {
         .get_sset_count         = nb8800_get_sset_count,
         .get_strings            = nb8800_get_strings,
         .get_ethtool_stats      = nb8800_get_ethtool_stats,
+        .get_link_ksettings     = phy_ethtool_get_link_ksettings,
+        .set_link_ksettings     = phy_ethtool_set_link_ksettings,
 };
 
 static int nb8800_hw_init(struct net_device *dev)
diff --git a/drivers/net/ethernet/aurora/nb8800.h b/drivers/net/ethernet/aurora/nb8800.h
index e5adbc2aac9f..6ec4a956e1e5 100644
--- a/drivers/net/ethernet/aurora/nb8800.h
+++ b/drivers/net/ethernet/aurora/nb8800.h
@@ -284,7 +284,6 @@ struct nb8800_priv {
 
         struct mii_bus                  *mii_bus;
         struct device_node              *phy_node;
-        struct phy_device               *phydev;
 
         /* PHY connection type from DT */
         int                             phy_mode;
diff --git a/drivers/net/ethernet/broadcom/Kconfig b/drivers/net/ethernet/broadcom/Kconfig
index 18042c2460bd..d74a92e1c27d 100644
--- a/drivers/net/ethernet/broadcom/Kconfig
+++ b/drivers/net/ethernet/broadcom/Kconfig
@@ -139,26 +139,6 @@ config BNX2X_SRIOV
           Virtualization support in the 578xx and 57712 products. This
           allows for virtual function acceleration in virtual environments.
 
-config BNX2X_VXLAN
-        bool "Virtual eXtensible Local Area Network support"
-        default n
-        depends on BNX2X && VXLAN && !(BNX2X=y && VXLAN=m)
-        ---help---
-          This enables hardward offload support for VXLAN protocol over the
-          NetXtremeII series adapters.
-          Say Y here if you want to enable hardware offload support for
-          Virtual eXtensible Local Area Network (VXLAN) in the driver.
-
-config BNX2X_GENEVE
-        bool "Generic Network Virtualization Encapsulation (GENEVE) support"
-        depends on BNX2X && GENEVE && !(BNX2X=y && GENEVE=m)
-        ---help---
-          This allows one to create GENEVE virtual interfaces that provide
-          Layer 2 Networks over Layer 3 Networks. GENEVE is often used
-          to tunnel virtual network infrastructure in virtualized environments.
-          Say Y here if you want to enable hardware offload support for
-          Generic Network Virtualization Encapsulation (GENEVE) in the driver.
-
 config BGMAC
         tristate "BCMA bus GBit core support"
         depends on BCMA && BCMA_HOST_SOC
@@ -186,7 +166,6 @@ config SYSTEMPORT
 config BNXT
         tristate "Broadcom NetXtreme-C/E support"
         depends on PCI
-        depends on VXLAN || VXLAN=n
         select FW_LOADER
         select LIBCRC32C
         ---help---
diff --git a/drivers/net/ethernet/broadcom/bcmsysport.c b/drivers/net/ethernet/broadcom/bcmsysport.c
index 543bf38105c9..834afbb51aff 100644
--- a/drivers/net/ethernet/broadcom/bcmsysport.c
+++ b/drivers/net/ethernet/broadcom/bcmsysport.c
@@ -96,28 +96,6 @@ static inline void tdma_port_write_desc_addr(struct bcm_sysport_priv *priv,
 }
 
 /* Ethtool operations */
-static int bcm_sysport_set_settings(struct net_device *dev,
-                                    struct ethtool_cmd *cmd)
-{
-        struct bcm_sysport_priv *priv = netdev_priv(dev);
-
-        if (!netif_running(dev))
-                return -EINVAL;
-
-        return phy_ethtool_sset(priv->phydev, cmd);
-}
-
-static int bcm_sysport_get_settings(struct net_device *dev,
-                                    struct ethtool_cmd *cmd)
-{
-        struct bcm_sysport_priv *priv = netdev_priv(dev);
-
-        if (!netif_running(dev))
-                return -EINVAL;
-
-        return phy_ethtool_gset(priv->phydev, cmd);
-}
-
 static int bcm_sysport_set_rx_csum(struct net_device *dev,
                                    netdev_features_t wanted)
 {
@@ -1127,7 +1105,7 @@ static void bcm_sysport_tx_timeout(struct net_device *dev)
 static void bcm_sysport_adj_link(struct net_device *dev)
 {
         struct bcm_sysport_priv *priv = netdev_priv(dev);
-        struct phy_device *phydev = priv->phydev;
+        struct phy_device *phydev = dev->phydev;
         unsigned int changed = 0;
         u32 cmd_bits = 0, reg;
 
@@ -1182,7 +1160,7 @@ static void bcm_sysport_adj_link(struct net_device *dev)
                 umac_writel(priv, reg, UMAC_CMD);
         }
 
-        phy_print_status(priv->phydev);
+        phy_print_status(phydev);
 }
 
 static int bcm_sysport_init_tx_ring(struct bcm_sysport_priv *priv,
@@ -1525,7 +1503,7 @@ static void bcm_sysport_netif_start(struct net_device *dev)
         /* Enable RX interrupt and TX ring full interrupt */
         intrl2_0_mask_clear(priv, INTRL2_0_RDMA_MBDONE | INTRL2_0_TX_RING_FULL);
 
-        phy_start(priv->phydev);
+        phy_start(dev->phydev);
 
         /* Enable TX interrupts for the 32 TXQs */
         intrl2_1_mask_clear(priv, 0xffffffff);
@@ -1546,6 +1524,7 @@ static void rbuf_init(struct bcm_sysport_priv *priv)
 static int bcm_sysport_open(struct net_device *dev)
 {
         struct bcm_sysport_priv *priv = netdev_priv(dev);
+        struct phy_device *phydev;
         unsigned int i;
         int ret;
 
@@ -1570,9 +1549,9 @@ static int bcm_sysport_open(struct net_device *dev)
         /* Read CRC forward */
         priv->crc_fwd = !!(umac_readl(priv, UMAC_CMD) & CMD_CRC_FWD);
 
-        priv->phydev = of_phy_connect(dev, priv->phy_dn, bcm_sysport_adj_link,
-                                        0, priv->phy_interface);
-        if (!priv->phydev) {
+        phydev = of_phy_connect(dev, priv->phy_dn, bcm_sysport_adj_link,
+                                0, priv->phy_interface);
+        if (!phydev) {
                 netdev_err(dev, "could not attach to PHY\n");
                 return -ENODEV;
         }
@@ -1650,7 +1629,7 @@ out_free_tx_ring:
 out_free_irq0:
         free_irq(priv->irq0, dev);
 out_phy_disconnect:
-        phy_disconnect(priv->phydev);
+        phy_disconnect(phydev);
         return ret;
 }
 
@@ -1661,7 +1640,7 @@ static void bcm_sysport_netif_stop(struct net_device *dev)
         /* stop all software from updating hardware */
         netif_tx_stop_all_queues(dev);
         napi_disable(&priv->napi);
-        phy_stop(priv->phydev);
+        phy_stop(dev->phydev);
 
         /* mask all interrupts */
         intrl2_0_mask_set(priv, 0xffffffff);
@@ -1708,14 +1687,12 @@ static int bcm_sysport_stop(struct net_device *dev)
         free_irq(priv->irq1, dev);
 
         /* Disconnect from PHY */
-        phy_disconnect(priv->phydev);
+        phy_disconnect(dev->phydev);
 
         return 0;
 }
 
 static struct ethtool_ops bcm_sysport_ethtool_ops = {
-        .get_settings           = bcm_sysport_get_settings,
-        .set_settings           = bcm_sysport_set_settings,
         .get_drvinfo            = bcm_sysport_get_drvinfo,
         .get_msglevel           = bcm_sysport_get_msglvl,
         .set_msglevel           = bcm_sysport_set_msglvl,
@@ -1727,6 +1704,8 @@ static struct ethtool_ops bcm_sysport_ethtool_ops = {
         .set_wol                = bcm_sysport_set_wol,
         .get_coalesce           = bcm_sysport_get_coalesce,
         .set_coalesce           = bcm_sysport_set_coalesce,
+        .get_link_ksettings     = phy_ethtool_get_link_ksettings,
+        .set_link_ksettings     = phy_ethtool_set_link_ksettings,
 };
 
 static const struct net_device_ops bcm_sysport_netdev_ops = {
@@ -1929,7 +1908,7 @@ static int bcm_sysport_suspend(struct device *d)
 
         bcm_sysport_netif_stop(dev);
 
-        phy_suspend(priv->phydev);
+        phy_suspend(dev->phydev);
 
         netif_device_detach(dev);
 
@@ -2055,7 +2034,7 @@ static int bcm_sysport_resume(struct device *d)
                 goto out_free_rx_ring;
         }
 
-        phy_resume(priv->phydev);
+        phy_resume(dev->phydev);
 
         bcm_sysport_netif_start(dev);
 
diff --git a/drivers/net/ethernet/broadcom/bcmsysport.h b/drivers/net/ethernet/broadcom/bcmsysport.h
index f28bf545d7f4..1c82e3da69a7 100644
--- a/drivers/net/ethernet/broadcom/bcmsysport.h
+++ b/drivers/net/ethernet/broadcom/bcmsysport.h
@@ -670,7 +670,6 @@ struct bcm_sysport_priv {
 
         /* PHY device */
         struct device_node      *phy_dn;
-        struct phy_device       *phydev;
         phy_interface_t         phy_interface;
         int                     old_pause;
         int                     old_link;
diff --git a/drivers/net/ethernet/broadcom/bgmac.c b/drivers/net/ethernet/broadcom/bgmac.c
index a6333d38ecc0..b045dc072c40 100644
--- a/drivers/net/ethernet/broadcom/bgmac.c
+++ b/drivers/net/ethernet/broadcom/bgmac.c
@@ -246,6 +246,8 @@ err_dma_head:
 
 err_drop:
         dev_kfree_skb(skb);
+        net_dev->stats.tx_dropped++;
+        net_dev->stats.tx_errors++;
         return NETDEV_TX_OK;
 }
 
@@ -285,6 +287,8 @@ static void bgmac_dma_tx_free(struct bgmac *bgmac, struct bgmac_dma_ring *ring)
                                        DMA_TO_DEVICE);
 
                 if (slot->skb) {
+                        bgmac->net_dev->stats.tx_bytes += slot->skb->len;
+                        bgmac->net_dev->stats.tx_packets++;
                         bytes_compl += slot->skb->len;
                         pkts_compl++;
 
@@ -465,6 +469,7 @@ static int bgmac_dma_rx_read(struct bgmac *bgmac, struct bgmac_dma_ring *ring,
                                 bgmac_err(bgmac, "Found poisoned packet at slot %d, DMA issue!\n",
                                           ring->start);
                                 put_page(virt_to_head_page(buf));
+                                bgmac->net_dev->stats.rx_errors++;
                                 break;
                         }
 
@@ -472,6 +477,8 @@ static int bgmac_dma_rx_read(struct bgmac *bgmac, struct bgmac_dma_ring *ring,
                                 bgmac_err(bgmac, "Found oversized packet at slot %d, DMA issue!\n",
                                           ring->start);
                                 put_page(virt_to_head_page(buf));
+                                bgmac->net_dev->stats.rx_length_errors++;
+                                bgmac->net_dev->stats.rx_errors++;
                                 break;
                         }
 
@@ -482,6 +489,7 @@ static int bgmac_dma_rx_read(struct bgmac *bgmac, struct bgmac_dma_ring *ring,
                         if (unlikely(!skb)) {
                                 bgmac_err(bgmac, "build_skb failed\n");
                                 put_page(virt_to_head_page(buf));
+                                bgmac->net_dev->stats.rx_errors++;
                                 break;
                         }
                         skb_put(skb, BGMAC_RX_FRAME_OFFSET +
@@ -491,6 +499,8 @@ static int bgmac_dma_rx_read(struct bgmac *bgmac, struct bgmac_dma_ring *ring,
 
                         skb_checksum_none_assert(skb);
                         skb->protocol = eth_type_trans(skb, bgmac->net_dev);
+                        bgmac->net_dev->stats.rx_bytes += len;
+                        bgmac->net_dev->stats.rx_packets++;
                         napi_gro_receive(&bgmac->napi, skb);
                         handled++;
                 } while (0);
@@ -1311,7 +1321,7 @@ static int bgmac_open(struct net_device *net_dev)
         }
         napi_enable(&bgmac->napi);
 
-        phy_start(bgmac->phy_dev);
+        phy_start(net_dev->phydev);
 
         netif_start_queue(net_dev);
 
@@ -1324,7 +1334,7 @@ static int bgmac_stop(struct net_device *net_dev)
 
         netif_carrier_off(net_dev);
 
-        phy_stop(bgmac->phy_dev);
+        phy_stop(net_dev->phydev);
 
         napi_disable(&bgmac->napi);
         bgmac_chip_intrs_off(bgmac);
@@ -1362,12 +1372,10 @@ static int bgmac_set_mac_address(struct net_device *net_dev, void *addr)
 
 static int bgmac_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd)
 {
-        struct bgmac *bgmac = netdev_priv(net_dev);
-
         if (!netif_running(net_dev))
                 return -EINVAL;
 
-        return phy_mii_ioctl(bgmac->phy_dev, ifr, cmd);
+        return phy_mii_ioctl(net_dev->phydev, ifr, cmd);
 }
 
 static const struct net_device_ops bgmac_netdev_ops = {
@@ -1384,20 +1392,125 @@ static const struct net_device_ops bgmac_netdev_ops = {
  * ethtool_ops
  **************************************************/
 
-static int bgmac_get_settings(struct net_device *net_dev,
-                              struct ethtool_cmd *cmd)
+struct bgmac_stat {
+        u8 size;
+        u32 offset;
+        const char *name;
+};
+
+static struct bgmac_stat bgmac_get_strings_stats[] = {
+        { 8, BGMAC_TX_GOOD_OCTETS, "tx_good_octets" },
+        { 4, BGMAC_TX_GOOD_PKTS, "tx_good" },
+        { 8, BGMAC_TX_OCTETS, "tx_octets" },
+        { 4, BGMAC_TX_PKTS, "tx_pkts" },
+        { 4, BGMAC_TX_BROADCAST_PKTS, "tx_broadcast" },
+        { 4, BGMAC_TX_MULTICAST_PKTS, "tx_multicast" },
+        { 4, BGMAC_TX_LEN_64, "tx_64" },
+        { 4, BGMAC_TX_LEN_65_TO_127, "tx_65_127" },
+        { 4, BGMAC_TX_LEN_128_TO_255, "tx_128_255" },
+        { 4, BGMAC_TX_LEN_256_TO_511, "tx_256_511" },
+        { 4, BGMAC_TX_LEN_512_TO_1023, "tx_512_1023" },
+        { 4, BGMAC_TX_LEN_1024_TO_1522, "tx_1024_1522" },
+        { 4, BGMAC_TX_LEN_1523_TO_2047, "tx_1523_2047" },
+        { 4, BGMAC_TX_LEN_2048_TO_4095, "tx_2048_4095" },
+        { 4, BGMAC_TX_LEN_4096_TO_8191, "tx_4096_8191" },
+        { 4, BGMAC_TX_LEN_8192_TO_MAX, "tx_8192_max" },
+        { 4, BGMAC_TX_JABBER_PKTS, "tx_jabber" },
+        { 4, BGMAC_TX_OVERSIZE_PKTS, "tx_oversize" },
+        { 4, BGMAC_TX_FRAGMENT_PKTS, "tx_fragment" },
+        { 4, BGMAC_TX_UNDERRUNS, "tx_underruns" },
+        { 4, BGMAC_TX_TOTAL_COLS, "tx_total_cols" },
+        { 4, BGMAC_TX_SINGLE_COLS, "tx_single_cols" },
+        { 4, BGMAC_TX_MULTIPLE_COLS, "tx_multiple_cols" },
+        { 4, BGMAC_TX_EXCESSIVE_COLS, "tx_excessive_cols" },
+        { 4, BGMAC_TX_LATE_COLS, "tx_late_cols" },
+        { 4, BGMAC_TX_DEFERED, "tx_defered" },
+        { 4, BGMAC_TX_CARRIER_LOST, "tx_carrier_lost" },
+        { 4, BGMAC_TX_PAUSE_PKTS, "tx_pause" },
+        { 4, BGMAC_TX_UNI_PKTS, "tx_unicast" },
+        { 4, BGMAC_TX_Q0_PKTS, "tx_q0" },
+        { 8, BGMAC_TX_Q0_OCTETS, "tx_q0_octets" },
+        { 4, BGMAC_TX_Q1_PKTS, "tx_q1" },
+        { 8, BGMAC_TX_Q1_OCTETS, "tx_q1_octets" },
+        { 4, BGMAC_TX_Q2_PKTS, "tx_q2" },
+        { 8, BGMAC_TX_Q2_OCTETS, "tx_q2_octets" },
+        { 4, BGMAC_TX_Q3_PKTS, "tx_q3" },
+        { 8, BGMAC_TX_Q3_OCTETS, "tx_q3_octets" },
+        { 8, BGMAC_RX_GOOD_OCTETS, "rx_good_octets" },
+        { 4, BGMAC_RX_GOOD_PKTS, "rx_good" },
+        { 8, BGMAC_RX_OCTETS, "rx_octets" },
+        { 4, BGMAC_RX_PKTS, "rx_pkts" },
+        { 4, BGMAC_RX_BROADCAST_PKTS, "rx_broadcast" },
+        { 4, BGMAC_RX_MULTICAST_PKTS, "rx_multicast" },
+        { 4, BGMAC_RX_LEN_64, "rx_64" },
+        { 4, BGMAC_RX_LEN_65_TO_127, "rx_65_127" },
+        { 4, BGMAC_RX_LEN_128_TO_255, "rx_128_255" },
+        { 4, BGMAC_RX_LEN_256_TO_511, "rx_256_511" },
+        { 4, BGMAC_RX_LEN_512_TO_1023, "rx_512_1023" },
+        { 4, BGMAC_RX_LEN_1024_TO_1522, "rx_1024_1522" },
+        { 4, BGMAC_RX_LEN_1523_TO_2047, "rx_1523_2047" },
+        { 4, BGMAC_RX_LEN_2048_TO_4095, "rx_2048_4095" },
+        { 4, BGMAC_RX_LEN_4096_TO_8191, "rx_4096_8191" },
+        { 4, BGMAC_RX_LEN_8192_TO_MAX, "rx_8192_max" },
+        { 4, BGMAC_RX_JABBER_PKTS, "rx_jabber" },
+        { 4, BGMAC_RX_OVERSIZE_PKTS, "rx_oversize" },
+        { 4, BGMAC_RX_FRAGMENT_PKTS, "rx_fragment" },
+        { 4, BGMAC_RX_MISSED_PKTS, "rx_missed" },
+        { 4, BGMAC_RX_CRC_ALIGN_ERRS, "rx_crc_align" },
+        { 4, BGMAC_RX_UNDERSIZE, "rx_undersize" },
+        { 4, BGMAC_RX_CRC_ERRS, "rx_crc" },
+        { 4, BGMAC_RX_ALIGN_ERRS, "rx_align" },
+        { 4, BGMAC_RX_SYMBOL_ERRS, "rx_symbol" },
+        { 4, BGMAC_RX_PAUSE_PKTS, "rx_pause" },
+        { 4, BGMAC_RX_NONPAUSE_PKTS, "rx_nonpause" },
+        { 4, BGMAC_RX_SACHANGES, "rx_sa_changes" },
+        { 4, BGMAC_RX_UNI_PKTS, "rx_unicast" },
+};
+
+#define BGMAC_STATS_LEN ARRAY_SIZE(bgmac_get_strings_stats)
+
+static int bgmac_get_sset_count(struct net_device *dev, int string_set)
 {
-        struct bgmac *bgmac = netdev_priv(net_dev);
+        switch (string_set) {
+        case ETH_SS_STATS:
+                return BGMAC_STATS_LEN;
+        }
 
-        return phy_ethtool_gset(bgmac->phy_dev, cmd);
+        return -EOPNOTSUPP;
 }
 
-static int bgmac_set_settings(struct net_device *net_dev,
-                              struct ethtool_cmd *cmd)
+static void bgmac_get_strings(struct net_device *dev, u32 stringset,
+                              u8 *data)
 {
-        struct bgmac *bgmac = netdev_priv(net_dev);
+        int i;
 
-        return phy_ethtool_sset(bgmac->phy_dev, cmd);
+        if (stringset != ETH_SS_STATS)
+                return;
+
+        for (i = 0; i < BGMAC_STATS_LEN; i++)
+                strlcpy(data + i * ETH_GSTRING_LEN,
+                        bgmac_get_strings_stats[i].name, ETH_GSTRING_LEN);
+}
+
+static void bgmac_get_ethtool_stats(struct net_device *dev,
+                                    struct ethtool_stats *ss, uint64_t *data)
+{
+        struct bgmac *bgmac = netdev_priv(dev);
+        const struct bgmac_stat *s;
+        unsigned int i;
+        u64 val;
+
+        if (!netif_running(dev))
+                return;
+
+        for (i = 0; i < BGMAC_STATS_LEN; i++) {
+                s = &bgmac_get_strings_stats[i];
+                val = 0;
+                if (s->size == 8)
+                        val = (u64)bgmac_read(bgmac, s->offset + 4) << 32;
+                val |= bgmac_read(bgmac, s->offset);
+                data[i] = val;
+        }
 }
 
 static void bgmac_get_drvinfo(struct net_device *net_dev,
@@ -1408,9 +1521,12 @@ static void bgmac_get_drvinfo(struct net_device *net_dev,
 }
 
 static const struct ethtool_ops bgmac_ethtool_ops = {
-        .get_settings           = bgmac_get_settings,
-        .set_settings           = bgmac_set_settings,
+        .get_strings            = bgmac_get_strings,
+        .get_sset_count         = bgmac_get_sset_count,
+        .get_ethtool_stats      = bgmac_get_ethtool_stats,
         .get_drvinfo            = bgmac_get_drvinfo,
+        .get_link_ksettings     = phy_ethtool_get_link_ksettings,
+        .set_link_ksettings     = phy_ethtool_set_link_ksettings,
 };
 
 /**************************************************
@@ -1431,7 +1547,7 @@ static int bgmac_mii_write(struct mii_bus *bus, int mii_id, int regnum,
 static void bgmac_adjust_link(struct net_device *net_dev)
 {
         struct bgmac *bgmac = netdev_priv(net_dev);
-        struct phy_device *phy_dev = bgmac->phy_dev;
+        struct phy_device *phy_dev = net_dev->phydev;
         bool update = false;
 
         if (phy_dev->link) {
@@ -1475,8 +1591,6 @@ static int bgmac_fixed_phy_register(struct bgmac *bgmac)
                 return err;
         }
 
-        bgmac->phy_dev = phy_dev;
-
         return err;
 }
 
@@ -1521,7 +1635,6 @@ static int bgmac_mii_register(struct bgmac *bgmac)
                 err = PTR_ERR(phy_dev);
                 goto err_unregister_bus;
         }
-        bgmac->phy_dev = phy_dev;
 
         return err;
 
@@ -1590,6 +1703,7 @@ static int bgmac_probe(struct bcma_device *core)
         bgmac->net_dev = net_dev;
         bgmac->core = core;
         bcma_set_drvdata(core, bgmac);
+        SET_NETDEV_DEV(net_dev, &core->dev);
 
         /* Defaults */
         memcpy(bgmac->net_dev->dev_addr, mac, ETH_ALEN);
diff --git a/drivers/net/ethernet/broadcom/bgmac.h b/drivers/net/ethernet/broadcom/bgmac.h
index 9a03c142b742..99beb181f577 100644
--- a/drivers/net/ethernet/broadcom/bgmac.h
+++ b/drivers/net/ethernet/broadcom/bgmac.h
@@ -123,7 +123,7 @@
 #define BGMAC_TX_LEN_1024_TO_1522               0x334
 #define BGMAC_TX_LEN_1523_TO_2047               0x338
 #define BGMAC_TX_LEN_2048_TO_4095               0x33c
-#define BGMAC_TX_LEN_4095_TO_8191               0x340
+#define BGMAC_TX_LEN_4096_TO_8191               0x340
 #define BGMAC_TX_LEN_8192_TO_MAX                0x344
 #define BGMAC_TX_JABBER_PKTS                    0x348           /* Error */
 #define BGMAC_TX_OVERSIZE_PKTS                  0x34c           /* Error */
@@ -166,7 +166,7 @@
 #define BGMAC_RX_LEN_1024_TO_1522               0x3e4
 #define BGMAC_RX_LEN_1523_TO_2047               0x3e8
 #define BGMAC_RX_LEN_2048_TO_4095               0x3ec
-#define BGMAC_RX_LEN_4095_TO_8191               0x3f0
+#define BGMAC_RX_LEN_4096_TO_8191               0x3f0
 #define BGMAC_RX_LEN_8192_TO_MAX                0x3f4
 #define BGMAC_RX_JABBER_PKTS                    0x3f8           /* Error */
 #define BGMAC_RX_OVERSIZE_PKTS                  0x3fc           /* Error */
@@ -441,7 +441,6 @@ struct bgmac {
         struct net_device *net_dev;
         struct napi_struct napi;
         struct mii_bus *mii_bus;
-        struct phy_device *phy_dev;
 
         /* DMA */
         struct bgmac_dma_ring tx_ring[BGMAC_MAX_TX_RINGS];
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
index a59d55e25d5f..97e892511666 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
@@ -59,9 +59,6 @@
 #include <linux/semaphore.h>
 #include <linux/stringify.h>
 #include <linux/vmalloc.h>
-#if IS_ENABLED(CONFIG_BNX2X_GENEVE)
-#include <net/geneve.h>
-#endif
 #include "bnx2x.h"
 #include "bnx2x_init.h"
 #include "bnx2x_init_ops.h"
@@ -10076,7 +10073,6 @@ static void bnx2x_parity_recover(struct bnx2x *bp)
         }
 }
 
-#if defined(CONFIG_BNX2X_VXLAN) || IS_ENABLED(CONFIG_BNX2X_GENEVE)
 static int bnx2x_udp_port_update(struct bnx2x *bp)
 {
         struct bnx2x_func_switch_update_params *switch_update_params;
@@ -10177,47 +10173,42 @@ static void __bnx2x_del_udp_port(struct bnx2x *bp, u16 port,
                 DP(BNX2X_MSG_SP, "Deleted UDP tunnel [%d] port %d\n",
                    type, port);
 }
-#endif
-
-#ifdef CONFIG_BNX2X_VXLAN
-static void bnx2x_add_vxlan_port(struct net_device *netdev,
-                                 sa_family_t sa_family, __be16 port)
-{
-        struct bnx2x *bp = netdev_priv(netdev);
-        u16 t_port = ntohs(port);
-
-        __bnx2x_add_udp_port(bp, t_port, BNX2X_UDP_PORT_VXLAN);
-}
-
-static void bnx2x_del_vxlan_port(struct net_device *netdev,
-                                 sa_family_t sa_family, __be16 port)
-{
-        struct bnx2x *bp = netdev_priv(netdev);
-        u16 t_port = ntohs(port);
-
-        __bnx2x_del_udp_port(bp, t_port, BNX2X_UDP_PORT_VXLAN);
-}
-#endif
 
-#if IS_ENABLED(CONFIG_BNX2X_GENEVE)
-static void bnx2x_add_geneve_port(struct net_device *netdev,
-                                  sa_family_t sa_family, __be16 port)
+static void bnx2x_udp_tunnel_add(struct net_device *netdev,
+                                 struct udp_tunnel_info *ti)
 {
         struct bnx2x *bp = netdev_priv(netdev);
-        u16 t_port = ntohs(port);
+        u16 t_port = ntohs(ti->port);
 
-        __bnx2x_add_udp_port(bp, t_port, BNX2X_UDP_PORT_GENEVE);
+        switch (ti->type) {
+        case UDP_TUNNEL_TYPE_VXLAN:
+                __bnx2x_add_udp_port(bp, t_port, BNX2X_UDP_PORT_VXLAN);
+                break;
+        case UDP_TUNNEL_TYPE_GENEVE:
+                __bnx2x_add_udp_port(bp, t_port, BNX2X_UDP_PORT_GENEVE);
+                break;
+        default:
+                break;
+        }
 }
 
-static void bnx2x_del_geneve_port(struct net_device *netdev,
-                                  sa_family_t sa_family, __be16 port)
+static void bnx2x_udp_tunnel_del(struct net_device *netdev,
+                                 struct udp_tunnel_info *ti)
 {
         struct bnx2x *bp = netdev_priv(netdev);
-        u16 t_port = ntohs(port);
+        u16 t_port = ntohs(ti->port);
 
-        __bnx2x_del_udp_port(bp, t_port, BNX2X_UDP_PORT_GENEVE);
+        switch (ti->type) {
+        case UDP_TUNNEL_TYPE_VXLAN:
+                __bnx2x_del_udp_port(bp, t_port, BNX2X_UDP_PORT_VXLAN);
+                break;
+        case UDP_TUNNEL_TYPE_GENEVE:
+                __bnx2x_del_udp_port(bp, t_port, BNX2X_UDP_PORT_GENEVE);
+                break;
+        default:
+                break;
+        }
 }
-#endif
 
 static int bnx2x_close(struct net_device *dev);
 
@@ -10325,7 +10316,6 @@ sp_rtnl_not_reset:
                                &bp->sp_rtnl_state))
                 bnx2x_update_mng_version(bp);
 
-#if defined(CONFIG_BNX2X_VXLAN) || IS_ENABLED(CONFIG_BNX2X_GENEVE)
         if (test_and_clear_bit(BNX2X_SP_RTNL_CHANGE_UDP_PORT,
                                &bp->sp_rtnl_state)) {
                 if (bnx2x_udp_port_update(bp)) {
@@ -10335,20 +10325,14 @@ sp_rtnl_not_reset:
                                BNX2X_UDP_PORT_MAX);
                 } else {
                         /* Since we don't store additional port information,
-                         * if no port is configured for any feature ask for
+                         * if no ports are configured for any feature ask for
                          * information about currently configured ports.
                          */
-#ifdef CONFIG_BNX2X_VXLAN
-                        if (!bp->udp_tunnel_ports[BNX2X_UDP_PORT_VXLAN].count)
-                                vxlan_get_rx_port(bp->dev);
-#endif
-#if IS_ENABLED(CONFIG_BNX2X_GENEVE)
-                        if (!bp->udp_tunnel_ports[BNX2X_UDP_PORT_GENEVE].count)
-                                geneve_get_rx_port(bp->dev);
-#endif
+                        if (!bp->udp_tunnel_ports[BNX2X_UDP_PORT_VXLAN].count &&
+                            !bp->udp_tunnel_ports[BNX2X_UDP_PORT_GENEVE].count)
+                                udp_tunnel_get_rx_info(bp->dev);
                 }
         }
-#endif
 
         /* work which needs rtnl lock not-taken (as it takes the lock itself and
          * can be called from other contexts as well)
@@ -12551,14 +12535,8 @@ static int bnx2x_open(struct net_device *dev)
         if (rc)
                 return rc;
 
-#ifdef CONFIG_BNX2X_VXLAN
-        if (IS_PF(bp))
-                vxlan_get_rx_port(dev);
-#endif
-#if IS_ENABLED(CONFIG_BNX2X_GENEVE)
         if (IS_PF(bp))
-                geneve_get_rx_port(dev);
-#endif
+                udp_tunnel_get_rx_info(dev);
 
         return 0;
 }
@@ -13045,14 +13023,8 @@ static const struct net_device_ops bnx2x_netdev_ops = {
         .ndo_get_phys_port_id   = bnx2x_get_phys_port_id,
         .ndo_set_vf_link_state  = bnx2x_set_vf_link_state,
         .ndo_features_check     = bnx2x_features_check,
-#ifdef CONFIG_BNX2X_VXLAN
-        .ndo_add_vxlan_port     = bnx2x_add_vxlan_port,
-        .ndo_del_vxlan_port     = bnx2x_del_vxlan_port,
-#endif
-#if IS_ENABLED(CONFIG_BNX2X_GENEVE)
-        .ndo_add_geneve_port    = bnx2x_add_geneve_port,
-        .ndo_del_geneve_port    = bnx2x_del_geneve_port,
-#endif
+        .ndo_udp_tunnel_add     = bnx2x_udp_tunnel_add,
+        .ndo_udp_tunnel_del     = bnx2x_udp_tunnel_del,
 };
 
 static int bnx2x_set_coherency_mask(struct bnx2x *bp)
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
index c777cde85ce4..673f4d62e73e 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
@@ -37,9 +37,7 @@
 #include <net/udp.h>
 #include <net/checksum.h>
 #include <net/ip6_checksum.h>
-#if defined(CONFIG_VXLAN) || defined(CONFIG_VXLAN_MODULE)
-#include <net/vxlan.h>
-#endif
+#include <net/udp_tunnel.h>
 #ifdef CONFIG_NET_RX_BUSY_POLL
 #include <net/busy_poll.h>
 #endif
@@ -75,12 +73,22 @@ enum board_idx {
         BCM57301,
         BCM57302,
         BCM57304,
+        BCM57311,
+        BCM57312,
         BCM57402,
         BCM57404,
         BCM57406,
+        BCM57404_NPAR,
+        BCM57412,
+        BCM57414,
+        BCM57416,
+        BCM57417,
+        BCM57414_NPAR,
         BCM57314,
         BCM57304_VF,
         BCM57404_VF,
+        BCM57414_VF,
+        BCM57314_VF,
 };
 
 /* indexed by enum above */
@@ -90,25 +98,45 @@ static const struct {
         { "Broadcom BCM57301 NetXtreme-C Single-port 10Gb Ethernet" },
         { "Broadcom BCM57302 NetXtreme-C Dual-port 10Gb/25Gb Ethernet" },
         { "Broadcom BCM57304 NetXtreme-C Dual-port 10Gb/25Gb/40Gb/50Gb Ethernet" },
+        { "Broadcom BCM57311 NetXtreme-C Single-port 10Gb Ethernet" },
+        { "Broadcom BCM57312 NetXtreme-C Dual-port 10Gb/25Gb Ethernet" },
         { "Broadcom BCM57402 NetXtreme-E Dual-port 10Gb Ethernet" },
         { "Broadcom BCM57404 NetXtreme-E Dual-port 10Gb/25Gb Ethernet" },
         { "Broadcom BCM57406 NetXtreme-E Dual-port 10GBase-T Ethernet" },
+        { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
+        { "Broadcom BCM57412 NetXtreme-E Dual-port 10Gb Ethernet" },
+        { "Broadcom BCM57414 NetXtreme-E Dual-port 10Gb/25Gb Ethernet" },
+        { "Broadcom BCM57416 NetXtreme-E Dual-port 10GBase-T Ethernet" },
+        { "Broadcom BCM57417 NetXtreme-E Dual-port 10GBase-T Ethernet" },
+        { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
         { "Broadcom BCM57314 NetXtreme-C Dual-port 10Gb/25Gb/40Gb/50Gb Ethernet" },
         { "Broadcom BCM57304 NetXtreme-C Ethernet Virtual Function" },
         { "Broadcom BCM57404 NetXtreme-E Ethernet Virtual Function" },
+        { "Broadcom BCM57414 NetXtreme-E Ethernet Virtual Function" },
+        { "Broadcom BCM57314 NetXtreme-E Ethernet Virtual Function" },
 };
 
 static const struct pci_device_id bnxt_pci_tbl[] = {
         { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
         { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
         { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
+        { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
+        { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
         { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
         { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
         { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
+        { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57404_NPAR },
+        { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
+        { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
+        { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
+        { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
+        { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57414_NPAR },
         { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
 #ifdef CONFIG_BNXT_SRIOV
         { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = BCM57304_VF },
         { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = BCM57404_VF },
+        { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = BCM57414_VF },
+        { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = BCM57314_VF },
 #endif
         { 0 }
 };
@@ -125,12 +153,14 @@ static const u16 bnxt_async_events_arr[] = {
         HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
         HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
         HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
+        HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
         HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
 };
 
 static bool bnxt_vf_pciid(enum board_idx idx)
 {
-        return (idx == BCM57304_VF || idx == BCM57404_VF);
+        return (idx == BCM57304_VF || idx == BCM57404_VF ||
+                idx == BCM57314_VF || idx == BCM57414_VF);
 }
 
 #define DB_CP_REARM_FLAGS       (DB_KEY_CP | DB_IDX_VALID)
@@ -920,6 +950,7 @@ static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
         }
         tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
         tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
+        tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
 
         rxr->rx_prod = NEXT_RX(prod);
         cons = NEXT_RX(cons);
@@ -938,32 +969,102 @@ static void bnxt_abort_tpa(struct bnxt *bp, struct bnxt_napi *bnapi,
                 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs);
 }
 
+static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
+                                           int payload_off, int tcp_ts,
+                                           struct sk_buff *skb)
+{
+#ifdef CONFIG_INET
+        struct tcphdr *th;
+        int len, nw_off;
+        u16 outer_ip_off, inner_ip_off, inner_mac_off;
+        u32 hdr_info = tpa_info->hdr_info;
+        bool loopback = false;
+
+        inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
+        inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
+        outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
+
+        /* If the packet is an internal loopback packet, the offsets will
+         * have an extra 4 bytes.
+         */
+        if (inner_mac_off == 4) {
+                loopback = true;
+        } else if (inner_mac_off > 4) {
+                __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
+                                            ETH_HLEN - 2));
+
+                /* We only support inner iPv4/ipv6.  If we don't see the
+                 * correct protocol ID, it must be a loopback packet where
+                 * the offsets are off by 4.
+                 */
+                if (proto != htons(ETH_P_IP) && proto && htons(ETH_P_IPV6))
+                        loopback = true;
+        }
+        if (loopback) {
+                /* internal loopback packet, subtract all offsets by 4 */
+                inner_ip_off -= 4;
+                inner_mac_off -= 4;
+                outer_ip_off -= 4;
+        }
+
+        nw_off = inner_ip_off - ETH_HLEN;
+        skb_set_network_header(skb, nw_off);
+        if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
+                struct ipv6hdr *iph = ipv6_hdr(skb);
+
+                skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
+                len = skb->len - skb_transport_offset(skb);
+                th = tcp_hdr(skb);
+                th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
+        } else {
+                struct iphdr *iph = ip_hdr(skb);
+
+                skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
+                len = skb->len - skb_transport_offset(skb);
+                th = tcp_hdr(skb);
+                th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
+        }
+
+        if (inner_mac_off) { /* tunnel */
+                struct udphdr *uh = NULL;
+                __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
+                                            ETH_HLEN - 2));
+
+                if (proto == htons(ETH_P_IP)) {
+                        struct iphdr *iph = (struct iphdr *)skb->data;
+
+                        if (iph->protocol == IPPROTO_UDP)
+                                uh = (struct udphdr *)(iph + 1);
+                } else {
+                        struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
+
+                        if (iph->nexthdr == IPPROTO_UDP)
+                                uh = (struct udphdr *)(iph + 1);
+                }
+                if (uh) {
+                        if (uh->check)
+                                skb_shinfo(skb)->gso_type |=
+                                        SKB_GSO_UDP_TUNNEL_CSUM;
+                        else
+                                skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
+                }
+        }
+#endif
+        return skb;
+}
+
 #define BNXT_IPV4_HDR_SIZE      (sizeof(struct iphdr) + sizeof(struct tcphdr))
 #define BNXT_IPV6_HDR_SIZE      (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
 
-static inline struct sk_buff *bnxt_gro_skb(struct bnxt_tpa_info *tpa_info,
-                                           struct rx_tpa_end_cmp *tpa_end,
-                                           struct rx_tpa_end_cmp_ext *tpa_end1,
+static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
+                                           int payload_off, int tcp_ts,
                                            struct sk_buff *skb)
 {
 #ifdef CONFIG_INET
         struct tcphdr *th;
-        int payload_off, tcp_opt_len = 0;
-        int len, nw_off;
-        u16 segs;
-
-        segs = TPA_END_TPA_SEGS(tpa_end);
-        if (segs == 1)
-                return skb;
+        int len, nw_off, tcp_opt_len;
 
-        NAPI_GRO_CB(skb)->count = segs;
-        skb_shinfo(skb)->gso_size =
-                le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
-        skb_shinfo(skb)->gso_type = tpa_info->gso_type;
-        payload_off = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
-                       RX_TPA_END_CMP_PAYLOAD_OFFSET) >>
-                      RX_TPA_END_CMP_PAYLOAD_OFFSET_SHIFT;
-        if (TPA_END_GRO_TS(tpa_end))
+        if (tcp_ts)
                 tcp_opt_len = 12;
 
         if (tpa_info->gso_type == SKB_GSO_TCPV4) {
@@ -1020,6 +1121,32 @@ static inline struct sk_buff *bnxt_gro_skb(struct bnxt_tpa_info *tpa_info,
         return skb;
 }
 
+static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
+                                           struct bnxt_tpa_info *tpa_info,
+                                           struct rx_tpa_end_cmp *tpa_end,
+                                           struct rx_tpa_end_cmp_ext *tpa_end1,
+                                           struct sk_buff *skb)
+{
+#ifdef CONFIG_INET
+        int payload_off;
+        u16 segs;
+
+        segs = TPA_END_TPA_SEGS(tpa_end);
+        if (segs == 1)
+                return skb;
+
+        NAPI_GRO_CB(skb)->count = segs;
+        skb_shinfo(skb)->gso_size =
+                le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
+        skb_shinfo(skb)->gso_type = tpa_info->gso_type;
+        payload_off = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
+                       RX_TPA_END_CMP_PAYLOAD_OFFSET) >>
+                      RX_TPA_END_CMP_PAYLOAD_OFFSET_SHIFT;
+        skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
+#endif
+        return skb;
+}
+
 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
                                            struct bnxt_napi *bnapi,
                                            u32 *raw_cons,
@@ -1130,7 +1257,7 @@ static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
         }
 
         if (TPA_END_GRO(tpa_end))
-                skb = bnxt_gro_skb(tpa_info, tpa_end, tpa_end1, skb);
+                skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
 
         return skb;
 }
@@ -1358,6 +1485,11 @@ static int bnxt_async_event_process(struct bnxt *bp,
                 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
                 break;
         }
+        case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
+                if (BNXT_PF(bp))
+                        goto async_event_process_exit;
+                set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
+                break;
         default:
                 netdev_err(bp->dev, "unhandled ASYNC event (id 0x%x)\n",
                            event_id);
@@ -2262,7 +2394,7 @@ static void bnxt_set_tpa_flags(struct bnxt *bp)
         bp->flags &= ~BNXT_FLAG_TPA;
         if (bp->dev->features & NETIF_F_LRO)
                 bp->flags |= BNXT_FLAG_LRO;
-        if ((bp->dev->features & NETIF_F_GRO) && (bp->pdev->revision > 0))
+        if (bp->dev->features & NETIF_F_GRO)
                 bp->flags |= BNXT_FLAG_GRO;
 }
 
@@ -3277,6 +3409,7 @@ static int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
         unsigned int ring = 0, grp_idx;
         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
         struct hwrm_vnic_cfg_input req = {0};
+        u16 def_vlan = 0;
 
         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
         /* Only RSS support for now TBD: COS & LB */
@@ -3297,7 +3430,11 @@ static int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
         req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + ETH_FCS_LEN +
                               VLAN_HLEN);
 
-        if (bp->flags & BNXT_FLAG_STRIP_VLAN)
+#ifdef CONFIG_BNXT_SRIOV
+        if (BNXT_VF(bp))
+                def_vlan = bp->vf.vlan;
+#endif
+        if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
 
         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
@@ -3836,6 +3973,39 @@ static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
         return 0;
 }
 
+static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
+{
+        struct hwrm_func_qcfg_input req = {0};
+        struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
+        int rc;
+
+        bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
+        req.fid = cpu_to_le16(0xffff);
+        mutex_lock(&bp->hwrm_cmd_lock);
+        rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
+        if (rc)
+                goto func_qcfg_exit;
+
+#ifdef CONFIG_BNXT_SRIOV
+        if (BNXT_VF(bp)) {
+                struct bnxt_vf_info *vf = &bp->vf;
+
+                vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
+        }
+#endif
+        switch (resp->port_partition_type) {
+        case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
+        case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
+        case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
+                bp->port_partition_type = resp->port_partition_type;
+                break;
+        }
+
+func_qcfg_exit:
+        mutex_unlock(&bp->hwrm_cmd_lock);
+        return rc;
+}
+
 int bnxt_hwrm_func_qcaps(struct bnxt *bp)
 {
         int rc = 0;
@@ -3990,6 +4160,8 @@ static int bnxt_hwrm_ver_get(struct bnxt *bp)
         if (resp->hwrm_intf_maj >= 1)
                 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
 
+        bp->chip_num = le16_to_cpu(resp->chip_num);
+
 hwrm_ver_get_exit:
         mutex_unlock(&bp->hwrm_cmd_lock);
         return rc;
@@ -4230,6 +4402,11 @@ static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
                 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
                             rc);
 
+        if (BNXT_VF(bp)) {
+                bnxt_hwrm_func_qcfg(bp);
+                netdev_update_features(bp->dev);
+        }
+
         return 0;
 
 err_out:
@@ -4644,6 +4821,7 @@ static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
         int rc = 0;
         struct hwrm_port_phy_qcaps_input req = {0};
         struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
+        struct bnxt_link_info *link_info = &bp->link_info;
 
         if (bp->hwrm_spec_code < 0x10201)
                 return 0;
@@ -4666,6 +4844,8 @@ static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
                 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
         }
+        link_info->support_auto_speeds =
+                le16_to_cpu(resp->supported_speeds_auto_mode);
 
 hwrm_phy_qcaps_exit:
         mutex_unlock(&bp->hwrm_cmd_lock);
@@ -4923,7 +5103,7 @@ static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
 {
         struct hwrm_port_phy_cfg_input req = {0};
 
-        if (BNXT_VF(bp))
+        if (!BNXT_SINGLE_PF(bp))
                 return 0;
 
         if (pci_num_vf(bp->pdev))
@@ -5073,15 +5253,8 @@ static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
                         netdev_warn(bp->dev, "failed to update phy settings\n");
         }
 
-        if (irq_re_init) {
-#if defined(CONFIG_VXLAN) || defined(CONFIG_VXLAN_MODULE)
-                vxlan_get_rx_port(bp->dev);
-#endif
-                if (!bnxt_hwrm_tunnel_dst_port_alloc(
-                                bp, htons(0x17c1),
-                                TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE))
-                        bp->nge_port_cnt = 1;
-        }
+        if (irq_re_init)
+                udp_tunnel_get_rx_info(bp->dev);
 
         set_bit(BNXT_STATE_OPEN, &bp->state);
         bnxt_enable_int(bp);
@@ -5469,7 +5642,14 @@ static netdev_features_t bnxt_fix_features(struct net_device *dev,
                         features |= NETIF_F_HW_VLAN_CTAG_RX |
                                     NETIF_F_HW_VLAN_STAG_RX;
         }
-
+#ifdef CONFIG_BNXT_SRIOV
+        if (BNXT_VF(bp)) {
+                if (bp->vf.vlan) {
+                        features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
+                                      NETIF_F_HW_VLAN_STAG_RX);
+                }
+        }
+#endif
         return features;
 }
 
@@ -5585,9 +5765,10 @@ static void bnxt_dbg_dump_states(struct bnxt *bp)
         }
 }
 
-static void bnxt_reset_task(struct bnxt *bp)
+static void bnxt_reset_task(struct bnxt *bp, bool silent)
 {
-        bnxt_dbg_dump_states(bp);
+        if (!silent)
+                bnxt_dbg_dump_states(bp);
         if (netif_running(bp->dev)) {
                 bnxt_close_nic(bp, false, false);
                 bnxt_open_nic(bp, false, false);
@@ -5638,6 +5819,23 @@ bnxt_restart_timer:
         mod_timer(&bp->timer, jiffies + bp->current_interval);
 }
 
+/* Only called from bnxt_sp_task() */
+static void bnxt_reset(struct bnxt *bp, bool silent)
+{
+        /* bnxt_reset_task() calls bnxt_close_nic() which waits
+         * for BNXT_STATE_IN_SP_TASK to clear.
+         * If there is a parallel dev_close(), bnxt_close() may be holding
+         * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear.  So we
+         * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
+         */
+        clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
+        rtnl_lock();
+        if (test_bit(BNXT_STATE_OPEN, &bp->state))
+                bnxt_reset_task(bp, silent);
+        set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
+        rtnl_unlock();
+}
+
 static void bnxt_cfg_ntp_filters(struct bnxt *);
 
 static void bnxt_sp_task(struct work_struct *work)
@@ -5674,16 +5872,20 @@ static void bnxt_sp_task(struct work_struct *work)
                 bnxt_hwrm_tunnel_dst_port_free(
                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
         }
-        if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event)) {
-                /* bnxt_reset_task() calls bnxt_close_nic() which waits
-                 * for BNXT_STATE_IN_SP_TASK to clear.
-                 */
-                clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
-                rtnl_lock();
-                bnxt_reset_task(bp);
-                set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
-                rtnl_unlock();
+        if (test_and_clear_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event)) {
+                bnxt_hwrm_tunnel_dst_port_alloc(
+                        bp, bp->nge_port,
+                        TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
         }
+        if (test_and_clear_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event)) {
+                bnxt_hwrm_tunnel_dst_port_free(
+                        bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
+        }
+        if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
+                bnxt_reset(bp, false);
+
+        if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
+                bnxt_reset(bp, true);
 
         if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event))
                 bnxt_get_port_module_status(bp);
@@ -6048,47 +6250,83 @@ static void bnxt_cfg_ntp_filters(struct bnxt *bp)
 
 #endif /* CONFIG_RFS_ACCEL */
 
-static void bnxt_add_vxlan_port(struct net_device *dev, sa_family_t sa_family,
-                                __be16 port)
+static void bnxt_udp_tunnel_add(struct net_device *dev,
+                                struct udp_tunnel_info *ti)
 {
         struct bnxt *bp = netdev_priv(dev);
 
-        if (!netif_running(dev))
+        if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET)
                 return;
 
-        if (sa_family != AF_INET6 && sa_family != AF_INET)
+        if (!netif_running(dev))
                 return;
 
-        if (bp->vxlan_port_cnt && bp->vxlan_port != port)
-                return;
+        switch (ti->type) {
+        case UDP_TUNNEL_TYPE_VXLAN:
+                if (bp->vxlan_port_cnt && bp->vxlan_port != ti->port)
+                        return;
 
-        bp->vxlan_port_cnt++;
-        if (bp->vxlan_port_cnt == 1) {
-                bp->vxlan_port = port;
-                set_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event);
-                schedule_work(&bp->sp_task);
+                bp->vxlan_port_cnt++;
+                if (bp->vxlan_port_cnt == 1) {
+                        bp->vxlan_port = ti->port;
+                        set_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event);
+                        schedule_work(&bp->sp_task);
+                }
+                break;
+        case UDP_TUNNEL_TYPE_GENEVE:
+                if (bp->nge_port_cnt && bp->nge_port != ti->port)
+                        return;
+
+                bp->nge_port_cnt++;
+                if (bp->nge_port_cnt == 1) {
+                        bp->nge_port = ti->port;
+                        set_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event);
+                }
+                break;
+        default:
+                return;
         }
+
+        schedule_work(&bp->sp_task);
 }
 
-static void bnxt_del_vxlan_port(struct net_device *dev, sa_family_t sa_family,
-                                __be16 port)
+static void bnxt_udp_tunnel_del(struct net_device *dev,
+                                struct udp_tunnel_info *ti)
 {
         struct bnxt *bp = netdev_priv(dev);
 
-        if (!netif_running(dev))
+        if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET)
                 return;
 
-        if (sa_family != AF_INET6 && sa_family != AF_INET)
+        if (!netif_running(dev))
                 return;
 
-        if (bp->vxlan_port_cnt && bp->vxlan_port == port) {
+        switch (ti->type) {
+        case UDP_TUNNEL_TYPE_VXLAN:
+                if (!bp->vxlan_port_cnt || bp->vxlan_port != ti->port)
+                        return;
                 bp->vxlan_port_cnt--;
 
-                if (bp->vxlan_port_cnt == 0) {
-                        set_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event);
-                        schedule_work(&bp->sp_task);
-                }
+                if (bp->vxlan_port_cnt != 0)
+                        return;
+
+                set_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event);
+                break;
+        case UDP_TUNNEL_TYPE_GENEVE:
+                if (!bp->nge_port_cnt || bp->nge_port != ti->port)
+                        return;
+                bp->nge_port_cnt--;
+
+                if (bp->nge_port_cnt != 0)
+                        return;
+
+                set_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event);
+                break;
+        default:
+                return;
         }
+
+        schedule_work(&bp->sp_task);
 }
 
 static const struct net_device_ops bnxt_netdev_ops = {
@@ -6119,8 +6357,8 @@ static const struct net_device_ops bnxt_netdev_ops = {
 #ifdef CONFIG_RFS_ACCEL
         .ndo_rx_flow_steer      = bnxt_rx_flow_steer,
 #endif
-        .ndo_add_vxlan_port     = bnxt_add_vxlan_port,
-        .ndo_del_vxlan_port     = bnxt_del_vxlan_port,
+        .ndo_udp_tunnel_add     = bnxt_udp_tunnel_add,
+        .ndo_udp_tunnel_del     = bnxt_udp_tunnel_del,
 #ifdef CONFIG_NET_RX_BUSY_POLL
         .ndo_busy_poll          = bnxt_busy_poll,
 #endif
@@ -6169,6 +6407,12 @@ static int bnxt_probe_phy(struct bnxt *bp)
                 return rc;
         }
 
+        /* Older firmware does not have supported_auto_speeds, so assume
+         * that all supported speeds can be autonegotiated.
+         */
+        if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
+                link_info->support_auto_speeds = link_info->support_speeds;
+
         /*initialize the ethool setting copy with NVM settings */
         if (BNXT_AUTO_MODE(link_info->auto_mode)) {
                 link_info->autoneg = BNXT_AUTONEG_SPEED;
@@ -6342,7 +6586,13 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
                 goto init_err;
 
         mutex_init(&bp->hwrm_cmd_lock);
-        bnxt_hwrm_ver_get(bp);
+        rc = bnxt_hwrm_ver_get(bp);
+        if (rc)
+                goto init_err;
+
+        bp->gro_func = bnxt_gro_func_5730x;
+        if (BNXT_CHIP_NUM_57X1X(bp->chip_num))
+                bp->gro_func = bnxt_gro_func_5731x;
 
         rc = bnxt_hwrm_func_drv_rgtr(bp);
         if (rc)
@@ -6365,6 +6615,8 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
                 goto init_err;
         }
 
+        bnxt_hwrm_func_qcfg(bp);
+
         bnxt_set_tpa_flags(bp);
         bnxt_set_ring_params(bp);
         if (BNXT_PF(bp))
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
index 2824d65b2e35..927ece9c408a 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
@@ -298,13 +298,14 @@ struct rx_tpa_start_cmp_ext {
         #define RX_TPA_START_CMP_FLAGS2_L4_CS_CALC              (0x1 << 1)
         #define RX_TPA_START_CMP_FLAGS2_T_IP_CS_CALC            (0x1 << 2)
         #define RX_TPA_START_CMP_FLAGS2_T_L4_CS_CALC            (0x1 << 3)
+        #define RX_TPA_START_CMP_FLAGS2_IP_TYPE                 (0x1 << 8)
 
         __le32 rx_tpa_start_cmp_metadata;
         __le32 rx_tpa_start_cmp_cfa_code_v2;
         #define RX_TPA_START_CMP_V2                             (0x1 << 0)
         #define RX_TPA_START_CMP_CFA_CODE                       (0xffff << 16)
          #define RX_TPA_START_CMPL_CFA_CODE_SHIFT                16
-        __le32 rx_tpa_start_cmp_unused5;
+        __le32 rx_tpa_start_cmp_hdr_info;
 };
 
 struct rx_tpa_end_cmp {
@@ -584,6 +585,19 @@ struct bnxt_tpa_info {
         u32                     metadata;
         enum pkt_hash_types     hash_type;
         u32                     rss_hash;
+        u32                     hdr_info;
+
+#define BNXT_TPA_L4_SIZE(hdr_info)      \
+        (((hdr_info) & 0xf8000000) ? ((hdr_info) >> 27) : 32)
+
+#define BNXT_TPA_INNER_L3_OFF(hdr_info) \
+        (((hdr_info) >> 18) & 0x1ff)
+
+#define BNXT_TPA_INNER_L2_OFF(hdr_info) \
+        (((hdr_info) >> 9) & 0x1ff)
+
+#define BNXT_TPA_OUTER_L3_OFF(hdr_info) \
+        ((hdr_info) & 0x1ff)
 };
 
 struct bnxt_rx_ring_info {
@@ -835,6 +849,7 @@ struct bnxt_link_info {
 #define BNXT_LINK_SPEED_MSK_25GB PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_25GB
 #define BNXT_LINK_SPEED_MSK_40GB PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_40GB
 #define BNXT_LINK_SPEED_MSK_50GB PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_50GB
+        u16                     support_auto_speeds;
         u16                     lp_auto_link_speeds;
         u16                     force_link_speed;
         u32                     preemphasis;
@@ -873,6 +888,44 @@ struct bnxt {
         void __iomem            *bar2;
 
         u32                     reg_base;
+        u16                     chip_num;
+#define CHIP_NUM_57301          0x16c8
+#define CHIP_NUM_57302          0x16c9
+#define CHIP_NUM_57304          0x16ca
+#define CHIP_NUM_57402          0x16d0
+#define CHIP_NUM_57404          0x16d1
+#define CHIP_NUM_57406          0x16d2
+
+#define CHIP_NUM_57311          0x16ce
+#define CHIP_NUM_57312          0x16cf
+#define CHIP_NUM_57314          0x16df
+#define CHIP_NUM_57412          0x16d6
+#define CHIP_NUM_57414          0x16d7
+#define CHIP_NUM_57416          0x16d8
+#define CHIP_NUM_57417          0x16d9
+
+#define BNXT_CHIP_NUM_5730X(chip_num)           \
+        ((chip_num) >= CHIP_NUM_57301 &&        \
+         (chip_num) <= CHIP_NUM_57304)
+
+#define BNXT_CHIP_NUM_5740X(chip_num)           \
+        ((chip_num) >= CHIP_NUM_57402 &&        \
+         (chip_num) <= CHIP_NUM_57406)
+
+#define BNXT_CHIP_NUM_5731X(chip_num)           \
+        ((chip_num) == CHIP_NUM_57311 ||        \
+         (chip_num) == CHIP_NUM_57312 ||        \
+         (chip_num) == CHIP_NUM_57314)
+
+#define BNXT_CHIP_NUM_5741X(chip_num)           \
+        ((chip_num) >= CHIP_NUM_57412 &&        \
+         (chip_num) <= CHIP_NUM_57417)
+
+#define BNXT_CHIP_NUM_57X0X(chip_num)           \
+        (BNXT_CHIP_NUM_5730X(chip_num) || BNXT_CHIP_NUM_5740X(chip_num))
+
+#define BNXT_CHIP_NUM_57X1X(chip_num)           \
+        (BNXT_CHIP_NUM_5731X(chip_num) || BNXT_CHIP_NUM_5741X(chip_num))
 
         struct net_device       *dev;
         struct pci_dev          *pdev;
@@ -907,12 +960,17 @@ struct bnxt {
 
 #define BNXT_PF(bp)             (!((bp)->flags & BNXT_FLAG_VF))
 #define BNXT_VF(bp)             ((bp)->flags & BNXT_FLAG_VF)
+#define BNXT_NPAR(bp)           ((bp)->port_partition_type)
+#define BNXT_SINGLE_PF(bp)      (BNXT_PF(bp) && !BNXT_NPAR(bp))
 
         struct bnxt_napi        **bnapi;
 
         struct bnxt_rx_ring_info        *rx_ring;
         struct bnxt_tx_ring_info        *tx_ring;
 
+        struct sk_buff *        (*gro_func)(struct bnxt_tpa_info *, int, int,
+                                            struct sk_buff *);
+
         u32                     rx_buf_size;
         u32                     rx_buf_use_size;        /* useable size */
         u32                     rx_ring_size;
@@ -991,8 +1049,10 @@ struct bnxt {
         __be16                  vxlan_port;
         u8                      vxlan_port_cnt;
         __le16                  vxlan_fw_dst_port_id;
+        __be16                  nge_port;
         u8                      nge_port_cnt;
         __le16                  nge_fw_dst_port_id;
+        u8                      port_partition_type;
 
         u16                     rx_coal_ticks;
         u16                     rx_coal_ticks_irq;
@@ -1018,6 +1078,9 @@ struct bnxt {
 #define BNXT_HWRM_PF_UNLOAD_SP_EVENT    8
 #define BNXT_PERIODIC_STATS_SP_EVENT    9
 #define BNXT_HWRM_PORT_MODULE_SP_EVENT  10
+#define BNXT_RESET_TASK_SILENT_SP_EVENT 11
+#define BNXT_GENEVE_ADD_PORT_SP_EVENT   12
+#define BNXT_GENEVE_DEL_PORT_SP_EVENT   13
 
         struct bnxt_pf_info     pf;
 #ifdef CONFIG_BNXT_SRIOV
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
index a38cb047b540..d7ab2d7982c2 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
@@ -628,7 +628,66 @@ u32 _bnxt_fw_to_ethtool_adv_spds(u16 fw_speeds, u8 fw_pause)
         return speed_mask;
 }
 
-static u32 bnxt_fw_to_ethtool_advertised_spds(struct bnxt_link_info *link_info)
+#define BNXT_FW_TO_ETHTOOL_SPDS(fw_speeds, fw_pause, lk_ksettings, name)\
+{                                                                       \
+        if ((fw_speeds) & BNXT_LINK_SPEED_MSK_100MB)                    \
+                ethtool_link_ksettings_add_link_mode(lk_ksettings, name,\
+                                                     100baseT_Full);    \
+        if ((fw_speeds) & BNXT_LINK_SPEED_MSK_1GB)                      \
+                ethtool_link_ksettings_add_link_mode(lk_ksettings, name,\
+                                                     1000baseT_Full);   \
+        if ((fw_speeds) & BNXT_LINK_SPEED_MSK_10GB)                     \
+                ethtool_link_ksettings_add_link_mode(lk_ksettings, name,\
+                                                     10000baseT_Full);  \
+        if ((fw_speeds) & BNXT_LINK_SPEED_MSK_25GB)                     \
+                ethtool_link_ksettings_add_link_mode(lk_ksettings, name,\
+                                                     25000baseCR_Full); \
+        if ((fw_speeds) & BNXT_LINK_SPEED_MSK_40GB)                     \
+                ethtool_link_ksettings_add_link_mode(lk_ksettings, name,\
+                                                     40000baseCR4_Full);\
+        if ((fw_speeds) & BNXT_LINK_SPEED_MSK_50GB)                     \
+                ethtool_link_ksettings_add_link_mode(lk_ksettings, name,\
+                                                     50000baseCR2_Full);\
+        if ((fw_pause) & BNXT_LINK_PAUSE_RX) {                          \
+                ethtool_link_ksettings_add_link_mode(lk_ksettings, name,\
+                                                     Pause);            \
+                if (!((fw_pause) & BNXT_LINK_PAUSE_TX))                 \
+                        ethtool_link_ksettings_add_link_mode(           \
+                                        lk_ksettings, name, Asym_Pause);\
+        } else if ((fw_pause) & BNXT_LINK_PAUSE_TX) {                   \
+                ethtool_link_ksettings_add_link_mode(lk_ksettings, name,\
+                                                     Asym_Pause);       \
+        }                                                               \
+}
+
+#define BNXT_ETHTOOL_TO_FW_SPDS(fw_speeds, lk_ksettings, name)          \
+{                                                                       \
+        if (ethtool_link_ksettings_test_link_mode(lk_ksettings, name,   \
+                                                  100baseT_Full) ||     \
+            ethtool_link_ksettings_test_link_mode(lk_ksettings, name,   \
+                                                  100baseT_Half))       \
+                (fw_speeds) |= BNXT_LINK_SPEED_MSK_100MB;               \
+        if (ethtool_link_ksettings_test_link_mode(lk_ksettings, name,   \
+                                                  1000baseT_Full) ||    \
+            ethtool_link_ksettings_test_link_mode(lk_ksettings, name,   \
+                                                  1000baseT_Half))      \
+                (fw_speeds) |= BNXT_LINK_SPEED_MSK_1GB;                 \
+        if (ethtool_link_ksettings_test_link_mode(lk_ksettings, name,   \
+                                                  10000baseT_Full))     \
+                (fw_speeds) |= BNXT_LINK_SPEED_MSK_10GB;                \
+        if (ethtool_link_ksettings_test_link_mode(lk_ksettings, name,   \
+                                                  25000baseCR_Full))    \
+                (fw_speeds) |= BNXT_LINK_SPEED_MSK_25GB;                \
+        if (ethtool_link_ksettings_test_link_mode(lk_ksettings, name,   \
+                                                  40000baseCR4_Full))   \
+                (fw_speeds) |= BNXT_LINK_SPEED_MSK_40GB;                \
+        if (ethtool_link_ksettings_test_link_mode(lk_ksettings, name,   \
+                                                  50000baseCR2_Full))   \
+                (fw_speeds) |= BNXT_LINK_SPEED_MSK_50GB;                \
+}
+
+static void bnxt_fw_to_ethtool_advertised_spds(struct bnxt_link_info *link_info,
+                                struct ethtool_link_ksettings *lk_ksettings)
 {
         u16 fw_speeds = link_info->auto_link_speeds;
         u8 fw_pause = 0;
@@ -636,10 +695,11 @@ static u32 bnxt_fw_to_ethtool_advertised_spds(struct bnxt_link_info *link_info)
         if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
                 fw_pause = link_info->auto_pause_setting;
 
-        return _bnxt_fw_to_ethtool_adv_spds(fw_speeds, fw_pause);
+        BNXT_FW_TO_ETHTOOL_SPDS(fw_speeds, fw_pause, lk_ksettings, advertising);
 }
 
-static u32 bnxt_fw_to_ethtool_lp_adv(struct bnxt_link_info *link_info)
+static void bnxt_fw_to_ethtool_lp_adv(struct bnxt_link_info *link_info,
+                                struct ethtool_link_ksettings *lk_ksettings)
 {
         u16 fw_speeds = link_info->lp_auto_link_speeds;
         u8 fw_pause = 0;
@@ -647,16 +707,24 @@ static u32 bnxt_fw_to_ethtool_lp_adv(struct bnxt_link_info *link_info)
         if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
                 fw_pause = link_info->lp_pause;
 
-        return _bnxt_fw_to_ethtool_adv_spds(fw_speeds, fw_pause);
+        BNXT_FW_TO_ETHTOOL_SPDS(fw_speeds, fw_pause, lk_ksettings,
+                                lp_advertising);
 }
 
-static u32 bnxt_fw_to_ethtool_support_spds(struct bnxt_link_info *link_info)
+static void bnxt_fw_to_ethtool_support_spds(struct bnxt_link_info *link_info,
+                                struct ethtool_link_ksettings *lk_ksettings)
 {
         u16 fw_speeds = link_info->support_speeds;
-        u32 supported;
 
-        supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
-        return supported | SUPPORTED_Pause | SUPPORTED_Asym_Pause;
+        BNXT_FW_TO_ETHTOOL_SPDS(fw_speeds, 0, lk_ksettings, supported);
+
+        ethtool_link_ksettings_add_link_mode(lk_ksettings, supported, Pause);
+        ethtool_link_ksettings_add_link_mode(lk_ksettings, supported,
+                                             Asym_Pause);
+
+        if (link_info->support_auto_speeds)
+                ethtool_link_ksettings_add_link_mode(lk_ksettings, supported,
+                                                     Autoneg);
 }
 
 u32 bnxt_fw_to_ethtool_speed(u16 fw_link_speed)
@@ -683,65 +751,62 @@ u32 bnxt_fw_to_ethtool_speed(u16 fw_link_speed)
         }
 }
 
-static int bnxt_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
+static int bnxt_get_link_ksettings(struct net_device *dev,
+                                   struct ethtool_link_ksettings *lk_ksettings)
 {
         struct bnxt *bp = netdev_priv(dev);
         struct bnxt_link_info *link_info = &bp->link_info;
-        u16 ethtool_speed;
+        struct ethtool_link_settings *base = &lk_ksettings->base;
+        u32 ethtool_speed;
 
-        cmd->supported = bnxt_fw_to_ethtool_support_spds(link_info);
-
-        if (link_info->auto_link_speeds)
-                cmd->supported |= SUPPORTED_Autoneg;
+        ethtool_link_ksettings_zero_link_mode(lk_ksettings, supported);
+        bnxt_fw_to_ethtool_support_spds(link_info, lk_ksettings);
 
+        ethtool_link_ksettings_zero_link_mode(lk_ksettings, advertising);
         if (link_info->autoneg) {
-                cmd->advertising =
-                        bnxt_fw_to_ethtool_advertised_spds(link_info);
-                cmd->advertising |= ADVERTISED_Autoneg;
-                cmd->autoneg = AUTONEG_ENABLE;
+                bnxt_fw_to_ethtool_advertised_spds(link_info, lk_ksettings);
+                ethtool_link_ksettings_add_link_mode(lk_ksettings,
+                                                     advertising, Autoneg);
+                base->autoneg = AUTONEG_ENABLE;
                 if (link_info->phy_link_status == BNXT_LINK_LINK)
-                        cmd->lp_advertising =
-                                bnxt_fw_to_ethtool_lp_adv(link_info);
+                        bnxt_fw_to_ethtool_lp_adv(link_info, lk_ksettings);
                 ethtool_speed = bnxt_fw_to_ethtool_speed(link_info->link_speed);
                 if (!netif_carrier_ok(dev))
-                        cmd->duplex = DUPLEX_UNKNOWN;
+                        base->duplex = DUPLEX_UNKNOWN;
                 else if (link_info->duplex & BNXT_LINK_DUPLEX_FULL)
-                        cmd->duplex = DUPLEX_FULL;
+                        base->duplex = DUPLEX_FULL;
                 else
-                        cmd->duplex = DUPLEX_HALF;
+                        base->duplex = DUPLEX_HALF;
         } else {
-                cmd->autoneg = AUTONEG_DISABLE;
-                cmd->advertising = 0;
+                base->autoneg = AUTONEG_DISABLE;
                 ethtool_speed =
                         bnxt_fw_to_ethtool_speed(link_info->req_link_speed);
-                cmd->duplex = DUPLEX_HALF;
+                base->duplex = DUPLEX_HALF;
                 if (link_info->req_duplex == BNXT_LINK_DUPLEX_FULL)
-                        cmd->duplex = DUPLEX_FULL;
+                        base->duplex = DUPLEX_FULL;
         }
-        ethtool_cmd_speed_set(cmd, ethtool_speed);
+        base->speed = ethtool_speed;
 
-        cmd->port = PORT_NONE;
+        base->port = PORT_NONE;
         if (link_info->media_type == PORT_PHY_QCFG_RESP_MEDIA_TYPE_TP) {
-                cmd->port = PORT_TP;
-                cmd->supported |= SUPPORTED_TP;
-                cmd->advertising |= ADVERTISED_TP;
+                base->port = PORT_TP;
+                ethtool_link_ksettings_add_link_mode(lk_ksettings, supported,
+                                                     TP);
+                ethtool_link_ksettings_add_link_mode(lk_ksettings, advertising,
+                                                     TP);
         } else {
-                cmd->supported |= SUPPORTED_FIBRE;
-                cmd->advertising |= ADVERTISED_FIBRE;
+                ethtool_link_ksettings_add_link_mode(lk_ksettings, supported,
+                                                     FIBRE);
+                ethtool_link_ksettings_add_link_mode(lk_ksettings, advertising,
+                                                     FIBRE);
 
                 if (link_info->media_type == PORT_PHY_QCFG_RESP_MEDIA_TYPE_DAC)
-                        cmd->port = PORT_DA;
+                        base->port = PORT_DA;
                 else if (link_info->media_type ==
                          PORT_PHY_QCFG_RESP_MEDIA_TYPE_FIBRE)
-                        cmd->port = PORT_FIBRE;
+                        base->port = PORT_FIBRE;
         }
-
-        if (link_info->transceiver ==
-            PORT_PHY_QCFG_RESP_XCVR_PKG_TYPE_XCVR_INTERNAL)
-                cmd->transceiver = XCVR_INTERNAL;
-        else
-                cmd->transceiver = XCVR_EXTERNAL;
-        cmd->phy_address = link_info->phy_addr;
+        base->phy_address = link_info->phy_addr;
 
         return 0;
 }
@@ -815,37 +880,25 @@ u16 bnxt_get_fw_auto_link_speeds(u32 advertising)
         return fw_speed_mask;
 }
 
-static int bnxt_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
+static int bnxt_set_link_ksettings(struct net_device *dev,
+                           const struct ethtool_link_ksettings *lk_ksettings)
 {
-        int rc = 0;
         struct bnxt *bp = netdev_priv(dev);
         struct bnxt_link_info *link_info = &bp->link_info;
+        const struct ethtool_link_settings *base = &lk_ksettings->base;
         u32 speed, fw_advertising = 0;
         bool set_pause = false;
+        int rc = 0;
 
-        if (BNXT_VF(bp))
-                return rc;
-
-        if (cmd->autoneg == AUTONEG_ENABLE) {
-                u32 supported_spds = bnxt_fw_to_ethtool_support_spds(link_info);
+        if (!BNXT_SINGLE_PF(bp))
+                return -EOPNOTSUPP;
 
-                if (cmd->advertising & ~(supported_spds | ADVERTISED_Autoneg |
-                                         ADVERTISED_TP | ADVERTISED_FIBRE)) {
-                        netdev_err(dev, "Unsupported advertising mask (adv: 0x%x)\n",
-                                   cmd->advertising);
-                        rc = -EINVAL;
-                        goto set_setting_exit;
-                }
-                fw_advertising = bnxt_get_fw_auto_link_speeds(cmd->advertising);
-                if (fw_advertising & ~link_info->support_speeds) {
-                        netdev_err(dev, "Advertising parameters are not supported! (adv: 0x%x)\n",
-                                   cmd->advertising);
-                        rc = -EINVAL;
-                        goto set_setting_exit;
-                }
+        if (base->autoneg == AUTONEG_ENABLE) {
+                BNXT_ETHTOOL_TO_FW_SPDS(fw_advertising, lk_ksettings,
+                                        advertising);
                 link_info->autoneg |= BNXT_AUTONEG_SPEED;
                 if (!fw_advertising)
-                        link_info->advertising = link_info->support_speeds;
+                        link_info->advertising = link_info->support_auto_speeds;
                 else
                         link_info->advertising = fw_advertising;
                 /* any change to autoneg will cause link change, therefore the
@@ -863,16 +916,12 @@ static int bnxt_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
                         rc = -EINVAL;
                         goto set_setting_exit;
                 }
-                /* TODO: currently don't support half duplex */
-                if (cmd->duplex == DUPLEX_HALF) {
+                if (base->duplex == DUPLEX_HALF) {
                         netdev_err(dev, "HALF DUPLEX is not supported!\n");
                         rc = -EINVAL;
                         goto set_setting_exit;
                 }
-                /* If received a request for an unknown duplex, assume full*/
-                if (cmd->duplex == DUPLEX_UNKNOWN)
-                        cmd->duplex = DUPLEX_FULL;
-                speed = ethtool_cmd_speed(cmd);
+                speed = base->speed;
                 fw_speed = bnxt_get_fw_speed(dev, speed);
                 if (!fw_speed) {
                         rc = -EINVAL;
@@ -911,7 +960,7 @@ static int bnxt_set_pauseparam(struct net_device *dev,
         struct bnxt *bp = netdev_priv(dev);
         struct bnxt_link_info *link_info = &bp->link_info;
 
-        if (BNXT_VF(bp))
+        if (!BNXT_SINGLE_PF(bp))
                 return rc;
 
         if (epause->autoneg) {
@@ -1433,7 +1482,7 @@ static int bnxt_set_eee(struct net_device *dev, struct ethtool_eee *edata)
                  _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
         int rc = 0;
 
-        if (BNXT_VF(bp))
+        if (!BNXT_SINGLE_PF(bp))
                 return 0;
 
         if (!(bp->flags & BNXT_FLAG_EEE_CAP))
@@ -1618,8 +1667,8 @@ static int bnxt_get_module_eeprom(struct net_device *dev,
 }
 
 const struct ethtool_ops bnxt_ethtool_ops = {
-        .get_settings           = bnxt_get_settings,
-        .set_settings           = bnxt_set_settings,
+        .get_link_ksettings     = bnxt_get_link_ksettings,
+        .set_link_ksettings     = bnxt_set_link_ksettings,
         .get_pauseparam         = bnxt_get_pauseparam,
         .set_pauseparam         = bnxt_set_pauseparam,
         .get_drvinfo            = bnxt_get_drvinfo,
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c
index 363884dd9e8a..50d2007a2640 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c
@@ -143,6 +143,9 @@ int bnxt_set_vf_vlan(struct net_device *dev, int vf_id, u16 vlan_id, u8 qos)
         u16 vlan_tag;
         int rc;
 
+        if (bp->hwrm_spec_code < 0x10201)
+                return -ENOTSUPP;
+
         rc = bnxt_vf_ndo_prep(bp, vf_id);
         if (rc)
                 return rc;
diff --git a/drivers/net/ethernet/cadence/macb.c b/drivers/net/ethernet/cadence/macb.c
index cb07d95e3dd9..89c0cfa9719f 100644
--- a/drivers/net/ethernet/cadence/macb.c
+++ b/drivers/net/ethernet/cadence/macb.c
@@ -304,7 +304,7 @@ static void macb_set_tx_clk(struct clk *clk, int speed, struct net_device *dev)
 static void macb_handle_link_change(struct net_device *dev)
 {
         struct macb *bp = netdev_priv(dev);
-        struct phy_device *phydev = bp->phy_dev;
+        struct phy_device *phydev = dev->phydev;
         unsigned long flags;
         int status_change = 0;
 
@@ -414,7 +414,6 @@ static int macb_mii_probe(struct net_device *dev)
         bp->link = 0;
         bp->speed = 0;
         bp->duplex = -1;
-        bp->phy_dev = phydev;
 
         return 0;
 }
@@ -1886,7 +1885,7 @@ static int macb_open(struct net_device *dev)
         netif_carrier_off(dev);
 
         /* if the phy is not yet register, retry later*/
-        if (!bp->phy_dev)
+        if (!dev->phydev)
                 return -EAGAIN;
 
         /* RX buffers initialization */
@@ -1905,7 +1904,7 @@ static int macb_open(struct net_device *dev)
         macb_init_hw(bp);
 
         /* schedule a link state check */
-        phy_start(bp->phy_dev);
+        phy_start(dev->phydev);
 
         netif_tx_start_all_queues(dev);
 
@@ -1920,8 +1919,8 @@ static int macb_close(struct net_device *dev)
         netif_tx_stop_all_queues(dev);
         napi_disable(&bp->napi);
 
-        if (bp->phy_dev)
-                phy_stop(bp->phy_dev);
+        if (dev->phydev)
+                phy_stop(dev->phydev);
 
         spin_lock_irqsave(&bp->lock, flags);
         macb_reset_hw(bp);
@@ -2092,28 +2091,6 @@ static struct net_device_stats *macb_get_stats(struct net_device *dev)
         return nstat;
 }
 
-static int macb_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
-{
-        struct macb *bp = netdev_priv(dev);
-        struct phy_device *phydev = bp->phy_dev;
-
-        if (!phydev)
-                return -ENODEV;
-
-        return phy_ethtool_gset(phydev, cmd);
-}
-
-static int macb_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
-{
-        struct macb *bp = netdev_priv(dev);
-        struct phy_device *phydev = bp->phy_dev;
-
-        if (!phydev)
-                return -ENODEV;
-
-        return phy_ethtool_sset(phydev, cmd);
-}
-
 static int macb_get_regs_len(struct net_device *netdev)
 {
         return MACB_GREGS_NBR * sizeof(u32);
@@ -2186,19 +2163,17 @@ static int macb_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol)
 }
 
 static const struct ethtool_ops macb_ethtool_ops = {
-        .get_settings           = macb_get_settings,
-        .set_settings           = macb_set_settings,
         .get_regs_len           = macb_get_regs_len,
         .get_regs               = macb_get_regs,
         .get_link               = ethtool_op_get_link,
         .get_ts_info            = ethtool_op_get_ts_info,
         .get_wol                = macb_get_wol,
         .set_wol                = macb_set_wol,
+        .get_link_ksettings     = phy_ethtool_get_link_ksettings,
+        .set_link_ksettings     = phy_ethtool_set_link_ksettings,
 };
 
 static const struct ethtool_ops gem_ethtool_ops = {
-        .get_settings           = macb_get_settings,
-        .set_settings           = macb_set_settings,
         .get_regs_len           = macb_get_regs_len,
         .get_regs               = macb_get_regs,
         .get_link               = ethtool_op_get_link,
@@ -2206,12 +2181,13 @@ static const struct ethtool_ops gem_ethtool_ops = {
         .get_ethtool_stats      = gem_get_ethtool_stats,
         .get_strings            = gem_get_ethtool_strings,
         .get_sset_count         = gem_get_sset_count,
+        .get_link_ksettings     = phy_ethtool_get_link_ksettings,
+        .set_link_ksettings     = phy_ethtool_set_link_ksettings,
 };
 
 static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
 {
-        struct macb *bp = netdev_priv(dev);
-        struct phy_device *phydev = bp->phy_dev;
+        struct phy_device *phydev = dev->phydev;
 
         if (!netif_running(dev))
                 return -EINVAL;
@@ -2570,7 +2546,7 @@ static int at91ether_open(struct net_device *dev)
                              MACB_BIT(HRESP));
 
         /* schedule a link state check */
-        phy_start(lp->phy_dev);
+        phy_start(dev->phydev);
 
         netif_start_queue(dev);
 
@@ -3010,7 +2986,7 @@ static int macb_probe(struct platform_device *pdev)
         if (err)
                 goto err_out_free_netdev;
 
-        phydev = bp->phy_dev;
+        phydev = dev->phydev;
 
         netif_carrier_off(dev);
 
@@ -3029,7 +3005,7 @@ static int macb_probe(struct platform_device *pdev)
         return 0;
 
 err_out_unregister_mdio:
-        phy_disconnect(bp->phy_dev);
+        phy_disconnect(dev->phydev);
         mdiobus_unregister(bp->mii_bus);
         mdiobus_free(bp->mii_bus);
 
@@ -3057,8 +3033,8 @@ static int macb_remove(struct platform_device *pdev)
 
         if (dev) {
                 bp = netdev_priv(dev);
-                if (bp->phy_dev)
-                        phy_disconnect(bp->phy_dev);
+                if (dev->phydev)
+                        phy_disconnect(dev->phydev);
                 mdiobus_unregister(bp->mii_bus);
                 mdiobus_free(bp->mii_bus);
 
diff --git a/drivers/net/ethernet/cadence/macb.h b/drivers/net/ethernet/cadence/macb.h
index 8a13824ef802..36893d8958d4 100644
--- a/drivers/net/ethernet/cadence/macb.h
+++ b/drivers/net/ethernet/cadence/macb.h
@@ -823,7 +823,6 @@ struct macb {
         struct macb_or_gem_ops  macbgem_ops;
 
         struct mii_bus          *mii_bus;
-        struct phy_device       *phy_dev;
         int                     link;
         int                     speed;
         int                     duplex;
diff --git a/drivers/net/ethernet/cavium/liquidio/cn66xx_device.c b/drivers/net/ethernet/cavium/liquidio/cn66xx_device.c
index 8ad7425f89bf..d35864ada9a3 100644
--- a/drivers/net/ethernet/cavium/liquidio/cn66xx_device.c
+++ b/drivers/net/ethernet/cavium/liquidio/cn66xx_device.c
@@ -367,7 +367,8 @@ void lio_cn6xxx_enable_io_queues(struct octeon_device *oct)
 
 void lio_cn6xxx_disable_io_queues(struct octeon_device *oct)
 {
-        u32 mask, i, loop = HZ;
+        int i;
+        u32 mask, loop = HZ;
         u32 d32;
 
         /* Reset the Enable bits for Input Queues. */
@@ -376,7 +377,7 @@ void lio_cn6xxx_disable_io_queues(struct octeon_device *oct)
         octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, mask);
 
         /* Wait until hardware indicates that the queues are out of reset. */
-        mask = oct->io_qmask.iq;
+        mask = (u32)oct->io_qmask.iq;
         d32 = octeon_read_csr(oct, CN6XXX_SLI_PORT_IN_RST_IQ);
         while (((d32 & mask) != mask) && loop--) {
                 d32 = octeon_read_csr(oct, CN6XXX_SLI_PORT_IN_RST_IQ);
@@ -384,8 +385,8 @@ void lio_cn6xxx_disable_io_queues(struct octeon_device *oct)
         }
 
         /* Reset the doorbell register for each Input queue. */
-        for (i = 0; i < MAX_OCTEON_INSTR_QUEUES; i++) {
-                if (!(oct->io_qmask.iq & (1UL << i)))
+        for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
+                if (!(oct->io_qmask.iq & (1ULL << i)))
                         continue;
                 octeon_write_csr(oct, CN6XXX_SLI_IQ_DOORBELL(i), 0xFFFFFFFF);
                 d32 = octeon_read_csr(oct, CN6XXX_SLI_IQ_DOORBELL(i));
@@ -398,7 +399,7 @@ void lio_cn6xxx_disable_io_queues(struct octeon_device *oct)
 
         /* Wait until hardware indicates that the queues are out of reset. */
         loop = HZ;
-        mask = oct->io_qmask.oq;
+        mask = (u32)oct->io_qmask.oq;
         d32 = octeon_read_csr(oct, CN6XXX_SLI_PORT_IN_RST_OQ);
         while (((d32 & mask) != mask) && loop--) {
                 d32 = octeon_read_csr(oct, CN6XXX_SLI_PORT_IN_RST_OQ);
@@ -408,8 +409,8 @@ void lio_cn6xxx_disable_io_queues(struct octeon_device *oct)
 
         /* Reset the doorbell register for each Output queue. */
         /* for (i = 0; i < oct->num_oqs; i++) { */
-        for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES; i++) {
-                if (!(oct->io_qmask.oq & (1UL << i)))
+        for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
+                if (!(oct->io_qmask.oq & (1ULL << i)))
                         continue;
                 octeon_write_csr(oct, CN6XXX_SLI_OQ_PKTS_CREDIT(i), 0xFFFFFFFF);
                 d32 = octeon_read_csr(oct, CN6XXX_SLI_OQ_PKTS_CREDIT(i));
@@ -429,16 +430,16 @@ void lio_cn6xxx_disable_io_queues(struct octeon_device *oct)
 
 void lio_cn6xxx_reinit_regs(struct octeon_device *oct)
 {
-        u32 i;
+        int i;
 
-        for (i = 0; i < MAX_OCTEON_INSTR_QUEUES; i++) {
-                if (!(oct->io_qmask.iq & (1UL << i)))
+        for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
+                if (!(oct->io_qmask.iq & (1ULL << i)))
                         continue;
                 oct->fn_list.setup_iq_regs(oct, i);
         }
 
-        for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES; i++) {
-                if (!(oct->io_qmask.oq & (1UL << i)))
+        for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
+                if (!(oct->io_qmask.oq & (1ULL << i)))
                         continue;
                 oct->fn_list.setup_oq_regs(oct, i);
         }
@@ -450,8 +451,8 @@ void lio_cn6xxx_reinit_regs(struct octeon_device *oct)
         oct->fn_list.enable_io_queues(oct);
 
         /* for (i = 0; i < oct->num_oqs; i++) { */
-        for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES; i++) {
-                if (!(oct->io_qmask.oq & (1UL << i)))
+        for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
+                if (!(oct->io_qmask.oq & (1ULL << i)))
                         continue;
                 writel(oct->droq[i]->max_count, oct->droq[i]->pkts_credit_reg);
         }
@@ -495,8 +496,7 @@ u32 lio_cn6xxx_bar1_idx_read(struct octeon_device *oct, u32 idx)
 }
 
 u32
-lio_cn6xxx_update_read_index(struct octeon_device *oct __attribute__((unused)),
-                             struct octeon_instr_queue *iq)
+lio_cn6xxx_update_read_index(struct octeon_instr_queue *iq)
 {
         u32 new_idx = readl(iq->inst_cnt_reg);
 
@@ -557,7 +557,8 @@ lio_cn6xxx_process_pcie_error_intr(struct octeon_device *oct, u64 intr64)
 int lio_cn6xxx_process_droq_intr_regs(struct octeon_device *oct)
 {
         struct octeon_droq *droq;
-        u32 oq_no, pkt_count, droq_time_mask, droq_mask, droq_int_enb;
+        int oq_no;
+        u32 pkt_count, droq_time_mask, droq_mask, droq_int_enb;
         u32 droq_cnt_enb, droq_cnt_mask;
 
         droq_cnt_enb = octeon_read_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB);
@@ -573,8 +574,8 @@ int lio_cn6xxx_process_droq_intr_regs(struct octeon_device *oct)
         oct->droq_intr = 0;
 
         /* for (oq_no = 0; oq_no < oct->num_oqs; oq_no++) { */
-        for (oq_no = 0; oq_no < MAX_OCTEON_OUTPUT_QUEUES; oq_no++) {
-                if (!(droq_mask & (1 << oq_no)))
+        for (oq_no = 0; oq_no < MAX_OCTEON_OUTPUT_QUEUES(oct); oq_no++) {
+                if (!(droq_mask & (1ULL << oq_no)))
                         continue;
 
                 droq = oct->droq[oq_no];
diff --git a/drivers/net/ethernet/cavium/liquidio/cn66xx_device.h b/drivers/net/ethernet/cavium/liquidio/cn66xx_device.h
index f77918779355..fe2932cb7ed8 100644
--- a/drivers/net/ethernet/cavium/liquidio/cn66xx_device.h
+++ b/drivers/net/ethernet/cavium/liquidio/cn66xx_device.h
@@ -91,8 +91,7 @@ void lio_cn6xxx_bar1_idx_setup(struct octeon_device *oct, u64 core_addr,
 void lio_cn6xxx_bar1_idx_write(struct octeon_device *oct, u32 idx, u32 mask);
 u32 lio_cn6xxx_bar1_idx_read(struct octeon_device *oct, u32 idx);
 u32
-lio_cn6xxx_update_read_index(struct octeon_device *oct __attribute__((unused)),
-                             struct octeon_instr_queue *iq);
+lio_cn6xxx_update_read_index(struct octeon_instr_queue *iq);
 void lio_cn6xxx_enable_interrupt(void *chip);
 void lio_cn6xxx_disable_interrupt(void *chip);
 void cn6xxx_get_pcie_qlmport(struct octeon_device *oct);
diff --git a/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c b/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c
index 245c063ed4db..03bfa9771e4d 100644
--- a/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c
+++ b/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c
@@ -40,6 +40,8 @@
 #include "cn68xx_device.h"
 #include "liquidio_image.h"
 
+static int octnet_get_link_stats(struct net_device *netdev);
+
 struct oct_mdio_cmd_context {
         int octeon_id;
         wait_queue_head_t wc;
@@ -71,34 +73,120 @@ enum {
         INTERFACE_MODE_RXAUI,
         INTERFACE_MODE_QSGMII,
         INTERFACE_MODE_AGL,
+        INTERFACE_MODE_XLAUI,
+        INTERFACE_MODE_XFI,
+        INTERFACE_MODE_10G_KR,
+        INTERFACE_MODE_40G_KR4,
+        INTERFACE_MODE_MIXED,
 };
 
 #define ARRAY_LENGTH(a) (sizeof(a) / sizeof((a)[0]))
 #define OCT_ETHTOOL_REGDUMP_LEN  4096
 #define OCT_ETHTOOL_REGSVER  1
 
+/* statistics of PF */
+static const char oct_stats_strings[][ETH_GSTRING_LEN] = {
+        "rx_packets",
+        "tx_packets",
+        "rx_bytes",
+        "tx_bytes",
+        "rx_errors",    /*jabber_err+l2_err+frame_err */
+        "tx_errors",    /*fw_err_pko+fw_err_link+fw_err_drop */
+        "rx_dropped",   /*st->fromwire.total_rcvd - st->fromwire.fw_total_rcvd
+                        *+st->fromwire.dmac_drop + st->fromwire.fw_err_drop
+                        */
+        "tx_dropped",
+
+        "tx_total_sent",
+        "tx_total_fwd",
+        "tx_err_pko",
+        "tx_err_link",
+        "tx_err_drop",
+
+        "tx_tso",
+        "tx_tso_packets",
+        "tx_tso_err",
+
+        "mac_tx_total_pkts",
+        "mac_tx_total_bytes",
+        "mac_tx_mcast_pkts",
+        "mac_tx_bcast_pkts",
+        "mac_tx_ctl_packets",   /*oct->link_stats.fromhost.ctl_sent */
+        "mac_tx_total_collisions",
+        "mac_tx_one_collision",
+        "mac_tx_multi_collison",
+        "mac_tx_max_collision_fail",
+        "mac_tx_max_deferal_fail",
+        "mac_tx_fifo_err",
+        "mac_tx_runts",
+
+        "rx_total_rcvd",
+        "rx_total_fwd",
+        "rx_jabber_err",
+        "rx_l2_err",
+        "rx_frame_err",
+        "rx_err_pko",
+        "rx_err_link",
+        "rx_err_drop",
+
+        "rx_lro_pkts",
+        "rx_lro_bytes",
+        "rx_total_lro",
+
+        "rx_lro_aborts",
+        "rx_lro_aborts_port",
+        "rx_lro_aborts_seq",
+        "rx_lro_aborts_tsval",
+        "rx_lro_aborts_timer",
+        "rx_fwd_rate",
+
+        "mac_rx_total_rcvd",
+        "mac_rx_bytes",
+        "mac_rx_total_bcst",
+        "mac_rx_total_mcst",
+        "mac_rx_runts",
+        "mac_rx_ctl_packets",
+        "mac_rx_fifo_err",
+        "mac_rx_dma_drop",
+        "mac_rx_fcs_err",
+
+        "link_state_changes",
+};
+
+/* statistics of host tx queue */
 static const char oct_iq_stats_strings[][ETH_GSTRING_LEN] = {
-        "Instr posted",
-        "Instr processed",
-        "Instr dropped",
-        "Bytes Sent",
-        "Sgentry_sent",
-        "Inst cntreg",
-        "Tx done",
-        "Tx Iq busy",
-        "Tx dropped",
-        "Tx bytes",
+        "packets",              /*oct->instr_queue[iq_no]->stats.tx_done*/
+        "bytes",                /*oct->instr_queue[iq_no]->stats.tx_tot_bytes*/
+        "dropped",
+        "iq_busy",
+        "sgentry_sent",
+
+        "fw_instr_posted",
+        "fw_instr_processed",
+        "fw_instr_dropped",
+        "fw_bytes_sent",
+
+        "tso",
+        "txq_restart",
 };
 
+/* statistics of host rx queue */
 static const char oct_droq_stats_strings[][ETH_GSTRING_LEN] = {
-        "OQ Pkts Received",
-        "OQ Bytes Received",
-        "Dropped no dispatch",
-        "Dropped nomem",
-        "Dropped toomany",
-        "Stack RX cnt",
-        "Stack RX Bytes",
-        "RX dropped",
+        "packets",              /*oct->droq[oq_no]->stats.rx_pkts_received */
+        "bytes",                /*oct->droq[oq_no]->stats.rx_bytes_received */
+        "dropped",              /*oct->droq[oq_no]->stats.rx_dropped+
+                                 *oct->droq[oq_no]->stats.dropped_nodispatch+
+                                 *oct->droq[oq_no]->stats.dropped_toomany+
+                                 *oct->droq[oq_no]->stats.dropped_nomem
+                                 */
+        "dropped_nomem",
+        "dropped_toomany",
+        "fw_dropped",
+        "fw_pkts_received",
+        "fw_bytes_received",
+        "fw_dropped_nodispatch",
+
+        "buffer_alloc_failure",
 };
 
 #define OCTNIC_NCMD_AUTONEG_ON  0x1
@@ -112,8 +200,9 @@ static int lio_get_settings(struct net_device *netdev, struct ethtool_cmd *ecmd)
 
         linfo = &lio->linfo;
 
-        if (linfo->link.s.interface == INTERFACE_MODE_XAUI ||
-            linfo->link.s.interface == INTERFACE_MODE_RXAUI) {
+        if (linfo->link.s.if_mode == INTERFACE_MODE_XAUI ||
+            linfo->link.s.if_mode == INTERFACE_MODE_RXAUI ||
+            linfo->link.s.if_mode == INTERFACE_MODE_XFI) {
                 ecmd->port = PORT_FIBRE;
                 ecmd->supported =
                         (SUPPORTED_10000baseT_Full | SUPPORTED_FIBRE |
@@ -124,10 +213,11 @@ static int lio_get_settings(struct net_device *netdev, struct ethtool_cmd *ecmd)
                 ecmd->autoneg = AUTONEG_DISABLE;
 
         } else {
-                dev_err(&oct->pci_dev->dev, "Unknown link interface reported\n");
+                dev_err(&oct->pci_dev->dev, "Unknown link interface reported %d\n",
+                        linfo->link.s.if_mode);
         }
 
-        if (linfo->link.s.status) {
+        if (linfo->link.s.link_up) {
                 ethtool_cmd_speed_set(ecmd, linfo->link.s.speed);
                 ecmd->duplex = linfo->link.s.duplex;
         } else {
@@ -222,23 +312,20 @@ static int octnet_gpio_access(struct net_device *netdev, int addr, int val)
         struct lio *lio = GET_LIO(netdev);
         struct octeon_device *oct = lio->oct_dev;
         struct octnic_ctrl_pkt nctrl;
-        struct octnic_ctrl_params nparams;
         int ret = 0;
 
         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
 
         nctrl.ncmd.u64 = 0;
         nctrl.ncmd.s.cmd = OCTNET_CMD_GPIO_ACCESS;
-        nctrl.ncmd.s.param1 = lio->linfo.ifidx;
-        nctrl.ncmd.s.param2 = addr;
-        nctrl.ncmd.s.param3 = val;
+        nctrl.ncmd.s.param1 = addr;
+        nctrl.ncmd.s.param2 = val;
+        nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
         nctrl.wait_time = 100;
         nctrl.netpndev = (u64)netdev;
         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
 
-        nparams.resp_order = OCTEON_RESP_ORDERED;
-
-        ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams);
+        ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
         if (ret < 0) {
                 dev_err(&oct->pci_dev->dev, "Failed to configure gpio value\n");
                 return -EINVAL;
@@ -303,9 +390,10 @@ octnet_mdio45_access(struct lio *lio, int op, int loc, int *value)
         mdio_cmd->mdio_addr = loc;
         if (op)
                 mdio_cmd->value1 = *value;
-        mdio_cmd->value2 = lio->linfo.ifidx;
         octeon_swap_8B_data((u64 *)mdio_cmd, sizeof(struct oct_mdio_cmd) / 8);
 
+        sc->iq_no = lio->linfo.txpciq[0].s.q_no;
+
         octeon_prepare_soft_command(oct_dev, sc, OPCODE_NIC, OPCODE_NIC_MDIO45,
                                     0, 0, 0);
 
@@ -317,7 +405,7 @@ octnet_mdio45_access(struct lio *lio, int op, int loc, int *value)
 
         retval = octeon_send_soft_command(oct_dev, sc);
 
-        if (retval) {
+        if (retval == IQ_SEND_FAILED) {
                 dev_err(&oct_dev->pci_dev->dev,
                         "octnet_mdio45_access instruction failed status: %x\n",
                         retval);
@@ -503,10 +591,10 @@ static void lio_set_msglevel(struct net_device *netdev, u32 msglvl)
         if ((msglvl ^ lio->msg_enable) & NETIF_MSG_HW) {
                 if (msglvl & NETIF_MSG_HW)
                         liquidio_set_feature(netdev,
-                                             OCTNET_CMD_VERBOSE_ENABLE);
+                                             OCTNET_CMD_VERBOSE_ENABLE, 0);
                 else
                         liquidio_set_feature(netdev,
-                                             OCTNET_CMD_VERBOSE_DISABLE);
+                                             OCTNET_CMD_VERBOSE_DISABLE, 0);
         }
 
         lio->msg_enable = msglvl;
@@ -518,8 +606,13 @@ lio_get_pauseparam(struct net_device *netdev, struct ethtool_pauseparam *pause)
         /* Notes: Not supporting any auto negotiation in these
          * drivers. Just report pause frame support.
          */
-        pause->tx_pause = 1;
-        pause->rx_pause = 1;    /* TODO: Need to support RX pause frame!!. */
+        struct lio *lio = GET_LIO(netdev);
+        struct octeon_device *oct = lio->oct_dev;
+
+        pause->autoneg = 0;
+
+        pause->tx_pause = oct->tx_pause;
+        pause->rx_pause = oct->rx_pause;
 }
 
 static void
@@ -528,51 +621,245 @@ lio_get_ethtool_stats(struct net_device *netdev,
 {
         struct lio *lio = GET_LIO(netdev);
         struct octeon_device *oct_dev = lio->oct_dev;
+        struct net_device_stats *netstats = &netdev->stats;
         int i = 0, j;
 
-        for (j = 0; j < MAX_OCTEON_INSTR_QUEUES; j++) {
-                if (!(oct_dev->io_qmask.iq & (1UL << j)))
+        netdev->netdev_ops->ndo_get_stats(netdev);
+        octnet_get_link_stats(netdev);
+
+        /*sum of oct->droq[oq_no]->stats->rx_pkts_received */
+        data[i++] = CVM_CAST64(netstats->rx_packets);
+        /*sum of oct->instr_queue[iq_no]->stats.tx_done */
+        data[i++] = CVM_CAST64(netstats->tx_packets);
+        /*sum of oct->droq[oq_no]->stats->rx_bytes_received */
+        data[i++] = CVM_CAST64(netstats->rx_bytes);
+        /*sum of oct->instr_queue[iq_no]->stats.tx_tot_bytes */
+        data[i++] = CVM_CAST64(netstats->tx_bytes);
+        data[i++] = CVM_CAST64(netstats->rx_errors);
+        data[i++] = CVM_CAST64(netstats->tx_errors);
+        /*sum of oct->droq[oq_no]->stats->rx_dropped +
+         *oct->droq[oq_no]->stats->dropped_nodispatch +
+         *oct->droq[oq_no]->stats->dropped_toomany +
+         *oct->droq[oq_no]->stats->dropped_nomem
+         */
+        data[i++] = CVM_CAST64(netstats->rx_dropped);
+        /*sum of oct->instr_queue[iq_no]->stats.tx_dropped */
+        data[i++] = CVM_CAST64(netstats->tx_dropped);
+
+        /*data[i++] = CVM_CAST64(stats->multicast); */
+        /*data[i++] = CVM_CAST64(stats->collisions); */
+
+        /* firmware tx stats */
+        /*per_core_stats[cvmx_get_core_num()].link_stats[mdata->from_ifidx].
+         *fromhost.fw_total_sent
+         */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromhost.fw_total_sent);
+        /*per_core_stats[i].link_stats[port].fromwire.fw_total_fwd */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromhost.fw_total_fwd);
+        /*per_core_stats[j].link_stats[i].fromhost.fw_err_pko */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromhost.fw_err_pko);
+        /*per_core_stats[j].link_stats[i].fromhost.fw_err_link */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromhost.fw_err_link);
+        /*per_core_stats[cvmx_get_core_num()].link_stats[idx].fromhost.
+         *fw_err_drop
+         */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromhost.fw_err_drop);
+
+        /*per_core_stats[cvmx_get_core_num()].link_stats[idx].fromhost.fw_tso */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromhost.fw_tso);
+        /*per_core_stats[cvmx_get_core_num()].link_stats[idx].fromhost.
+         *fw_tso_fwd
+         */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromhost.fw_tso_fwd);
+        /*per_core_stats[cvmx_get_core_num()].link_stats[idx].fromhost.
+         *fw_err_tso
+         */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromhost.fw_err_tso);
+
+        /* mac tx statistics */
+        /*CVMX_BGXX_CMRX_TX_STAT5 */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromhost.total_pkts_sent);
+        /*CVMX_BGXX_CMRX_TX_STAT4 */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromhost.total_bytes_sent);
+        /*CVMX_BGXX_CMRX_TX_STAT15 */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromhost.mcast_pkts_sent);
+        /*CVMX_BGXX_CMRX_TX_STAT14 */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromhost.bcast_pkts_sent);
+        /*CVMX_BGXX_CMRX_TX_STAT17 */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromhost.ctl_sent);
+        /*CVMX_BGXX_CMRX_TX_STAT0 */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromhost.total_collisions);
+        /*CVMX_BGXX_CMRX_TX_STAT3 */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromhost.one_collision_sent);
+        /*CVMX_BGXX_CMRX_TX_STAT2 */
+        data[i++] =
+                CVM_CAST64(oct_dev->link_stats.fromhost.multi_collision_sent);
+        /*CVMX_BGXX_CMRX_TX_STAT0 */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromhost.max_collision_fail);
+        /*CVMX_BGXX_CMRX_TX_STAT1 */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromhost.max_deferral_fail);
+        /*CVMX_BGXX_CMRX_TX_STAT16 */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromhost.fifo_err);
+        /*CVMX_BGXX_CMRX_TX_STAT6 */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromhost.runts);
+
+        /* RX firmware stats */
+        /*per_core_stats[cvmx_get_core_num()].link_stats[ifidx].fromwire.
+         *fw_total_rcvd
+         */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.fw_total_rcvd);
+        /*per_core_stats[cvmx_get_core_num()].link_stats[ifidx].fromwire.
+         *fw_total_fwd
+         */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.fw_total_fwd);
+        /*per_core_stats[core_id].link_stats[ifidx].fromwire.jabber_err */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.jabber_err);
+        /*per_core_stats[core_id].link_stats[ifidx].fromwire.l2_err */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.l2_err);
+        /*per_core_stats[core_id].link_stats[ifidx].fromwire.frame_err */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.frame_err);
+        /*per_core_stats[cvmx_get_core_num()].link_stats[ifidx].fromwire.
+         *fw_err_pko
+         */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.fw_err_pko);
+        /*per_core_stats[j].link_stats[i].fromwire.fw_err_link */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.fw_err_link);
+        /*per_core_stats[cvmx_get_core_num()].link_stats[lro_ctx->ifidx].
+         *fromwire.fw_err_drop
+         */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.fw_err_drop);
+
+        /* LRO */
+        /*per_core_stats[cvmx_get_core_num()].link_stats[ifidx].fromwire.
+         *fw_lro_pkts
+         */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.fw_lro_pkts);
+        /*per_core_stats[cvmx_get_core_num()].link_stats[ifidx].fromwire.
+         *fw_lro_octs
+         */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.fw_lro_octs);
+        /*per_core_stats[j].link_stats[i].fromwire.fw_total_lro */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.fw_total_lro);
+        /*per_core_stats[j].link_stats[i].fromwire.fw_lro_aborts */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.fw_lro_aborts);
+        /*per_core_stats[cvmx_get_core_num()].link_stats[ifidx].fromwire.
+         *fw_lro_aborts_port
+         */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.fw_lro_aborts_port);
+        /*per_core_stats[cvmx_get_core_num()].link_stats[ifidx].fromwire.
+         *fw_lro_aborts_seq
+         */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.fw_lro_aborts_seq);
+        /*per_core_stats[cvmx_get_core_num()].link_stats[ifidx].fromwire.
+         *fw_lro_aborts_tsval
+         */
+        data[i++] =
+                CVM_CAST64(oct_dev->link_stats.fromwire.fw_lro_aborts_tsval);
+        /*per_core_stats[cvmx_get_core_num()].link_stats[ifidx].fromwire.
+         *fw_lro_aborts_timer
+         */
+        /* intrmod: packet forward rate */
+        data[i++] =
+                CVM_CAST64(oct_dev->link_stats.fromwire.fw_lro_aborts_timer);
+        /*per_core_stats[j].link_stats[i].fromwire.fw_lro_aborts */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.fwd_rate);
+
+        /* mac: link-level stats */
+        /*CVMX_BGXX_CMRX_RX_STAT0 */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.total_rcvd);
+        /*CVMX_BGXX_CMRX_RX_STAT1 */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.bytes_rcvd);
+        /*CVMX_PKI_STATX_STAT5 */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.total_bcst);
+        /*CVMX_PKI_STATX_STAT5 */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.total_mcst);
+        /*wqe->word2.err_code or wqe->word2.err_level */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.runts);
+        /*CVMX_BGXX_CMRX_RX_STAT2 */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.ctl_rcvd);
+        /*CVMX_BGXX_CMRX_RX_STAT6 */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.fifo_err);
+        /*CVMX_BGXX_CMRX_RX_STAT4 */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.dmac_drop);
+        /*wqe->word2.err_code or wqe->word2.err_level */
+        data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.fcs_err);
+        /*lio->link_changes*/
+        data[i++] = CVM_CAST64(lio->link_changes);
+
+        /* TX  -- lio_update_stats(lio); */
+        for (j = 0; j < MAX_OCTEON_INSTR_QUEUES(oct_dev); j++) {
+                if (!(oct_dev->io_qmask.iq & (1ULL << j)))
                         continue;
+                /*packets to network port*/
+                /*# of packets tx to network */
+                data[i++] = CVM_CAST64(oct_dev->instr_queue[j]->stats.tx_done);
+                /*# of bytes tx to network */
                 data[i++] =
-                        CVM_CAST64(oct_dev->instr_queue[j]->stats.instr_posted);
-                data[i++] =
-                        CVM_CAST64(
-                                oct_dev->instr_queue[j]->stats.instr_processed);
+                        CVM_CAST64(oct_dev->instr_queue[j]->stats.tx_tot_bytes);
+                /*# of packets dropped */
                 data[i++] =
-                        CVM_CAST64(
-                                oct_dev->instr_queue[j]->stats.instr_dropped);
+                        CVM_CAST64(oct_dev->instr_queue[j]->stats.tx_dropped);
+                /*# of tx fails due to queue full */
                 data[i++] =
-                        CVM_CAST64(oct_dev->instr_queue[j]->stats.bytes_sent);
+                        CVM_CAST64(oct_dev->instr_queue[j]->stats.tx_iq_busy);
+                /*XXX gather entries sent */
                 data[i++] =
                         CVM_CAST64(oct_dev->instr_queue[j]->stats.sgentry_sent);
+
+                /*instruction to firmware: data and control */
+                /*# of instructions to the queue */
                 data[i++] =
-                        readl(oct_dev->instr_queue[j]->inst_cnt_reg);
-                data[i++] =
-                        CVM_CAST64(oct_dev->instr_queue[j]->stats.tx_done);
-                data[i++] =
-                        CVM_CAST64(oct_dev->instr_queue[j]->stats.tx_iq_busy);
+                        CVM_CAST64(oct_dev->instr_queue[j]->stats.instr_posted);
+                /*# of instructions processed */
+                data[i++] = CVM_CAST64(oct_dev->instr_queue[j]->
+                                       stats.instr_processed);
+                /*# of instructions could not be processed */
+                data[i++] = CVM_CAST64(oct_dev->instr_queue[j]->
+                                       stats.instr_dropped);
+                /*bytes sent through the queue */
                 data[i++] =
-                        CVM_CAST64(oct_dev->instr_queue[j]->stats.tx_dropped);
+                        CVM_CAST64(oct_dev->instr_queue[j]->stats.bytes_sent);
+
+                /*tso request*/
+                data[i++] = CVM_CAST64(oct_dev->instr_queue[j]->stats.tx_gso);
+                /*txq restart*/
                 data[i++] =
-                        CVM_CAST64(oct_dev->instr_queue[j]->stats.tx_tot_bytes);
+                        CVM_CAST64(oct_dev->instr_queue[j]->stats.tx_restart);
         }
 
-        /* for (j = 0; j < oct_dev->num_oqs; j++){ */
-        for (j = 0; j < MAX_OCTEON_OUTPUT_QUEUES; j++) {
-                if (!(oct_dev->io_qmask.oq & (1UL << j)))
+        /* RX */
+        /* for (j = 0; j < oct_dev->num_oqs; j++) { */
+        for (j = 0; j < MAX_OCTEON_OUTPUT_QUEUES(oct_dev); j++) {
+                if (!(oct_dev->io_qmask.oq & (1ULL << j)))
                         continue;
-                data[i++] = CVM_CAST64(oct_dev->droq[j]->stats.pkts_received);
-                data[i++] = CVM_CAST64(oct_dev->droq[j]->stats.bytes_received);
-                data[i++] =
-                        CVM_CAST64(oct_dev->droq[j]->stats.dropped_nodispatch);
-                data[i++] = CVM_CAST64(oct_dev->droq[j]->stats.dropped_nomem);
-                data[i++] = CVM_CAST64(oct_dev->droq[j]->stats.dropped_toomany);
+
+                /*packets send to TCP/IP network stack */
+                /*# of packets to network stack */
                 data[i++] =
                         CVM_CAST64(oct_dev->droq[j]->stats.rx_pkts_received);
+                /*# of bytes to network stack */
                 data[i++] =
                         CVM_CAST64(oct_dev->droq[j]->stats.rx_bytes_received);
+                /*# of packets dropped */
+                data[i++] = CVM_CAST64(oct_dev->droq[j]->stats.dropped_nomem +
+                                       oct_dev->droq[j]->stats.dropped_toomany +
+                                       oct_dev->droq[j]->stats.rx_dropped);
+                data[i++] =
+                        CVM_CAST64(oct_dev->droq[j]->stats.dropped_nomem);
+                data[i++] =
+                        CVM_CAST64(oct_dev->droq[j]->stats.dropped_toomany);
                 data[i++] =
                         CVM_CAST64(oct_dev->droq[j]->stats.rx_dropped);
+
+                /*control and data path*/
+                data[i++] =
+                        CVM_CAST64(oct_dev->droq[j]->stats.pkts_received);
+                data[i++] =
+                        CVM_CAST64(oct_dev->droq[j]->stats.bytes_received);
+                data[i++] =
+                        CVM_CAST64(oct_dev->droq[j]->stats.dropped_nodispatch);
+                data[i++] =
+                        CVM_CAST64(oct_dev->droq[j]->stats.rx_alloc_failure);
         }
 }
 
@@ -581,26 +868,43 @@ static void lio_get_strings(struct net_device *netdev, u32 stringset, u8 *data)
         struct lio *lio = GET_LIO(netdev);
         struct octeon_device *oct_dev = lio->oct_dev;
         int num_iq_stats, num_oq_stats, i, j;
+        int num_stats;
 
-        num_iq_stats = ARRAY_SIZE(oct_iq_stats_strings);
-        for (i = 0; i < MAX_OCTEON_INSTR_QUEUES; i++) {
-                if (!(oct_dev->io_qmask.iq & (1UL << i)))
-                        continue;
-                for (j = 0; j < num_iq_stats; j++) {
-                        sprintf(data, "IQ%d %s", i, oct_iq_stats_strings[j]);
+        switch (stringset) {
+        case ETH_SS_STATS:
+                num_stats = ARRAY_SIZE(oct_stats_strings);
+                for (j = 0; j < num_stats; j++) {
+                        sprintf(data, "%s", oct_stats_strings[j]);
                         data += ETH_GSTRING_LEN;
                 }
-        }
 
-        num_oq_stats = ARRAY_SIZE(oct_droq_stats_strings);
-        /* for (i = 0; i < oct_dev->num_oqs; i++) { */
-        for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES; i++) {
-                if (!(oct_dev->io_qmask.oq & (1UL << i)))
-                        continue;
-                for (j = 0; j < num_oq_stats; j++) {
-                        sprintf(data, "OQ%d %s", i, oct_droq_stats_strings[j]);
-                        data += ETH_GSTRING_LEN;
+                num_iq_stats = ARRAY_SIZE(oct_iq_stats_strings);
+                for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct_dev); i++) {
+                        if (!(oct_dev->io_qmask.iq & (1ULL << i)))
+                                continue;
+                        for (j = 0; j < num_iq_stats; j++) {
+                                sprintf(data, "tx-%d-%s", i,
+                                        oct_iq_stats_strings[j]);
+                                data += ETH_GSTRING_LEN;
+                        }
                 }
+
+                num_oq_stats = ARRAY_SIZE(oct_droq_stats_strings);
+                /* for (i = 0; i < oct_dev->num_oqs; i++) { */
+                for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct_dev); i++) {
+                        if (!(oct_dev->io_qmask.oq & (1ULL << i)))
+                                continue;
+                        for (j = 0; j < num_oq_stats; j++) {
+                                sprintf(data, "rx-%d-%s", i,
+                                        oct_droq_stats_strings[j]);
+                                data += ETH_GSTRING_LEN;
+                        }
+                }
+                break;
+
+        default:
+                netif_info(lio, drv, lio->netdev, "Unknown Stringset !!\n");
+                break;
         }
 }
 
@@ -609,8 +913,14 @@ static int lio_get_sset_count(struct net_device *netdev, int sset)
         struct lio *lio = GET_LIO(netdev);
         struct octeon_device *oct_dev = lio->oct_dev;
 
-        return (ARRAY_SIZE(oct_iq_stats_strings) * oct_dev->num_iqs) +
-               (ARRAY_SIZE(oct_droq_stats_strings) * oct_dev->num_oqs);
+        switch (sset) {
+        case ETH_SS_STATS:
+                return (ARRAY_SIZE(oct_stats_strings) +
+                        ARRAY_SIZE(oct_iq_stats_strings) * oct_dev->num_iqs +
+                        ARRAY_SIZE(oct_droq_stats_strings) * oct_dev->num_oqs);
+        default:
+                return -EOPNOTSUPP;
+        }
 }
 
 static int lio_get_intr_coalesce(struct net_device *netdev,
@@ -618,50 +928,50 @@ static int lio_get_intr_coalesce(struct net_device *netdev,
 {
         struct lio *lio = GET_LIO(netdev);
         struct octeon_device *oct = lio->oct_dev;
-        struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)oct->chip;
         struct octeon_instr_queue *iq;
         struct oct_intrmod_cfg *intrmod_cfg;
 
         intrmod_cfg = &oct->intrmod;
 
         switch (oct->chip_id) {
-        /* case OCTEON_CN73XX: Todo */
-        /*      break; */
         case OCTEON_CN68XX:
-        case OCTEON_CN66XX:
-                if (!intrmod_cfg->intrmod_enable) {
+        case OCTEON_CN66XX: {
+                struct octeon_cn6xxx *cn6xxx =
+                        (struct octeon_cn6xxx *)oct->chip;
+
+                if (!intrmod_cfg->rx_enable) {
                         intr_coal->rx_coalesce_usecs =
                                 CFG_GET_OQ_INTR_TIME(cn6xxx->conf);
                         intr_coal->rx_max_coalesced_frames =
                                 CFG_GET_OQ_INTR_PKT(cn6xxx->conf);
-                } else {
-                        intr_coal->use_adaptive_rx_coalesce =
-                                intrmod_cfg->intrmod_enable;
-                        intr_coal->rate_sample_interval =
-                                intrmod_cfg->intrmod_check_intrvl;
-                        intr_coal->pkt_rate_high =
-                                intrmod_cfg->intrmod_maxpkt_ratethr;
-                        intr_coal->pkt_rate_low =
-                                intrmod_cfg->intrmod_minpkt_ratethr;
-                        intr_coal->rx_max_coalesced_frames_high =
-                                intrmod_cfg->intrmod_maxcnt_trigger;
-                        intr_coal->rx_coalesce_usecs_high =
-                                intrmod_cfg->intrmod_maxtmr_trigger;
-                        intr_coal->rx_coalesce_usecs_low =
-                                intrmod_cfg->intrmod_mintmr_trigger;
-                        intr_coal->rx_max_coalesced_frames_low =
-                                intrmod_cfg->intrmod_mincnt_trigger;
                 }
 
-                iq = oct->instr_queue[lio->linfo.txpciq[0]];
+                iq = oct->instr_queue[lio->linfo.txpciq[0].s.q_no];
                 intr_coal->tx_max_coalesced_frames = iq->fill_threshold;
                 break;
-
+        }
         default:
                 netif_info(lio, drv, lio->netdev, "Unknown Chip !!\n");
                 return -EINVAL;
         }
-
+        if (intrmod_cfg->rx_enable) {
+                intr_coal->use_adaptive_rx_coalesce =
+                        intrmod_cfg->rx_enable;
+                intr_coal->rate_sample_interval =
+                        intrmod_cfg->check_intrvl;
+                intr_coal->pkt_rate_high =
+                        intrmod_cfg->maxpkt_ratethr;
+                intr_coal->pkt_rate_low =
+                        intrmod_cfg->minpkt_ratethr;
+                intr_coal->rx_max_coalesced_frames_high =
+                        intrmod_cfg->rx_maxcnt_trigger;
+                intr_coal->rx_coalesce_usecs_high =
+                        intrmod_cfg->rx_maxtmr_trigger;
+                intr_coal->rx_coalesce_usecs_low =
+                        intrmod_cfg->rx_mintmr_trigger;
+                intr_coal->rx_max_coalesced_frames_low =
+                        intrmod_cfg->rx_mincnt_trigger;
+        }
         return 0;
 }
 
@@ -681,19 +991,20 @@ static void octnet_intrmod_callback(struct octeon_device *oct_dev,
         else
                 dev_info(&oct_dev->pci_dev->dev,
                          "Rx-Adaptive Interrupt moderation enabled:%llx\n",
-                         oct_dev->intrmod.intrmod_enable);
+                         oct_dev->intrmod.rx_enable);
 
         octeon_free_soft_command(oct_dev, sc);
 }
 
 /*  Configure interrupt moderation parameters */
-static int octnet_set_intrmod_cfg(void *oct, struct oct_intrmod_cfg *intr_cfg)
+static int octnet_set_intrmod_cfg(struct lio *lio,
+                                  struct oct_intrmod_cfg *intr_cfg)
 {
         struct octeon_soft_command *sc;
         struct oct_intrmod_cmd *cmd;
         struct oct_intrmod_cfg *cfg;
         int retval;
-        struct octeon_device *oct_dev = (struct octeon_device *)oct;
+        struct octeon_device *oct_dev = lio->oct_dev;
 
         /* Alloc soft command */
         sc = (struct octeon_soft_command *)
@@ -714,6 +1025,8 @@ static int octnet_set_intrmod_cfg(void *oct, struct oct_intrmod_cfg *intr_cfg)
         cmd->cfg = cfg;
         cmd->oct_dev = oct_dev;
 
+        sc->iq_no = lio->linfo.txpciq[0].s.q_no;
+
         octeon_prepare_soft_command(oct_dev, sc, OPCODE_NIC,
                                     OPCODE_NIC_INTRMOD_CFG, 0, 0, 0);
 
@@ -722,7 +1035,7 @@ static int octnet_set_intrmod_cfg(void *oct, struct oct_intrmod_cfg *intr_cfg)
         sc->wait_time = 1000;
 
         retval = octeon_send_soft_command(oct_dev, sc);
-        if (retval) {
+        if (retval == IQ_SEND_FAILED) {
                 octeon_free_soft_command(oct_dev, sc);
                 return -EINVAL;
         }
@@ -730,9 +1043,158 @@ static int octnet_set_intrmod_cfg(void *oct, struct oct_intrmod_cfg *intr_cfg)
         return 0;
 }
 
+void
+octnet_nic_stats_callback(struct octeon_device *oct_dev,
+                          u32 status, void *ptr)
+{
+        struct octeon_soft_command  *sc = (struct octeon_soft_command  *)ptr;
+        struct oct_nic_stats_resp *resp = (struct oct_nic_stats_resp *)
+                sc->virtrptr;
+        struct oct_nic_stats_ctrl *ctrl = (struct oct_nic_stats_ctrl *)
+                sc->ctxptr;
+        struct nic_rx_stats *rsp_rstats = &resp->stats.fromwire;
+        struct nic_tx_stats *rsp_tstats = &resp->stats.fromhost;
+
+        struct nic_rx_stats *rstats = &oct_dev->link_stats.fromwire;
+        struct nic_tx_stats *tstats = &oct_dev->link_stats.fromhost;
+
+        if ((status != OCTEON_REQUEST_TIMEOUT) && !resp->status) {
+                octeon_swap_8B_data((u64 *)&resp->stats,
+                                    (sizeof(struct oct_link_stats)) >> 3);
+
+                /* RX link-level stats */
+                rstats->total_rcvd = rsp_rstats->total_rcvd;
+                rstats->bytes_rcvd = rsp_rstats->bytes_rcvd;
+                rstats->total_bcst = rsp_rstats->total_bcst;
+                rstats->total_mcst = rsp_rstats->total_mcst;
+                rstats->runts      = rsp_rstats->runts;
+                rstats->ctl_rcvd   = rsp_rstats->ctl_rcvd;
+                /* Accounts for over/under-run of buffers */
+                rstats->fifo_err  = rsp_rstats->fifo_err;
+                rstats->dmac_drop = rsp_rstats->dmac_drop;
+                rstats->fcs_err   = rsp_rstats->fcs_err;
+                rstats->jabber_err = rsp_rstats->jabber_err;
+                rstats->l2_err    = rsp_rstats->l2_err;
+                rstats->frame_err = rsp_rstats->frame_err;
+
+                /* RX firmware stats */
+                rstats->fw_total_rcvd = rsp_rstats->fw_total_rcvd;
+                rstats->fw_total_fwd = rsp_rstats->fw_total_fwd;
+                rstats->fw_err_pko = rsp_rstats->fw_err_pko;
+                rstats->fw_err_link = rsp_rstats->fw_err_link;
+                rstats->fw_err_drop = rsp_rstats->fw_err_drop;
+                /* Number of packets that are LROed      */
+                rstats->fw_lro_pkts = rsp_rstats->fw_lro_pkts;
+                /* Number of octets that are LROed       */
+                rstats->fw_lro_octs = rsp_rstats->fw_lro_octs;
+                /* Number of LRO packets formed          */
+                rstats->fw_total_lro = rsp_rstats->fw_total_lro;
+                /* Number of times lRO of packet aborted */
+                rstats->fw_lro_aborts = rsp_rstats->fw_lro_aborts;
+                rstats->fw_lro_aborts_port = rsp_rstats->fw_lro_aborts_port;
+                rstats->fw_lro_aborts_seq = rsp_rstats->fw_lro_aborts_seq;
+                rstats->fw_lro_aborts_tsval = rsp_rstats->fw_lro_aborts_tsval;
+                rstats->fw_lro_aborts_timer = rsp_rstats->fw_lro_aborts_timer;
+                /* intrmod: packet forward rate */
+                rstats->fwd_rate = rsp_rstats->fwd_rate;
+
+                /* TX link-level stats */
+                tstats->total_pkts_sent = rsp_tstats->total_pkts_sent;
+                tstats->total_bytes_sent = rsp_tstats->total_bytes_sent;
+                tstats->mcast_pkts_sent = rsp_tstats->mcast_pkts_sent;
+                tstats->bcast_pkts_sent = rsp_tstats->bcast_pkts_sent;
+                tstats->ctl_sent = rsp_tstats->ctl_sent;
+                /* Packets sent after one collision*/
+                tstats->one_collision_sent = rsp_tstats->one_collision_sent;
+                /* Packets sent after multiple collision*/
+                tstats->multi_collision_sent = rsp_tstats->multi_collision_sent;
+                /* Packets not sent due to max collisions */
+                tstats->max_collision_fail = rsp_tstats->max_collision_fail;
+                /* Packets not sent due to max deferrals */
+                tstats->max_deferral_fail = rsp_tstats->max_deferral_fail;
+                /* Accounts for over/under-run of buffers */
+                tstats->fifo_err = rsp_tstats->fifo_err;
+                tstats->runts = rsp_tstats->runts;
+                /* Total number of collisions detected */
+                tstats->total_collisions = rsp_tstats->total_collisions;
+
+                /* firmware stats */
+                tstats->fw_total_sent = rsp_tstats->fw_total_sent;
+                tstats->fw_total_fwd = rsp_tstats->fw_total_fwd;
+                tstats->fw_err_pko = rsp_tstats->fw_err_pko;
+                tstats->fw_err_link = rsp_tstats->fw_err_link;
+                tstats->fw_err_drop = rsp_tstats->fw_err_drop;
+                tstats->fw_tso = rsp_tstats->fw_tso;
+                tstats->fw_tso_fwd = rsp_tstats->fw_tso_fwd;
+                tstats->fw_err_tso = rsp_tstats->fw_err_tso;
+                resp->status = 1;
+        } else {
+                resp->status = -1;
+        }
+        complete(&ctrl->complete);
+}
+
+/*  Configure interrupt moderation parameters */
+static int octnet_get_link_stats(struct net_device *netdev)
+{
+        struct lio *lio = GET_LIO(netdev);
+        struct octeon_device *oct_dev = lio->oct_dev;
+
+        struct octeon_soft_command *sc;
+        struct oct_nic_stats_ctrl *ctrl;
+        struct oct_nic_stats_resp *resp;
+
+        int retval;
+
+        /* Alloc soft command */
+        sc = (struct octeon_soft_command *)
+                octeon_alloc_soft_command(oct_dev,
+                                          0,
+                                          sizeof(struct oct_nic_stats_resp),
+                                          sizeof(struct octnic_ctrl_pkt));
+
+        if (!sc)
+                return -ENOMEM;
+
+        resp = (struct oct_nic_stats_resp *)sc->virtrptr;
+        memset(resp, 0, sizeof(struct oct_nic_stats_resp));
+
+        ctrl = (struct oct_nic_stats_ctrl *)sc->ctxptr;
+        memset(ctrl, 0, sizeof(struct oct_nic_stats_ctrl));
+        ctrl->netdev = netdev;
+        init_completion(&ctrl->complete);
+
+        sc->iq_no = lio->linfo.txpciq[0].s.q_no;
+
+        octeon_prepare_soft_command(oct_dev, sc, OPCODE_NIC,
+                                    OPCODE_NIC_PORT_STATS, 0, 0, 0);
+
+        sc->callback = octnet_nic_stats_callback;
+        sc->callback_arg = sc;
+        sc->wait_time = 500;    /*in milli seconds*/
+
+        retval = octeon_send_soft_command(oct_dev, sc);
+        if (retval == IQ_SEND_FAILED) {
+                octeon_free_soft_command(oct_dev, sc);
+                return -EINVAL;
+        }
+
+        wait_for_completion_timeout(&ctrl->complete, msecs_to_jiffies(1000));
+
+        if (resp->status != 1) {
+                octeon_free_soft_command(oct_dev, sc);
+
+                return -EINVAL;
+        }
+
+        octeon_free_soft_command(oct_dev, sc);
+
+        return 0;
+}
+
 /* Enable/Disable auto interrupt Moderation */
 static int oct_cfg_adaptive_intr(struct lio *lio, struct ethtool_coalesce
-                                 *intr_coal, int adaptive)
+                                 *intr_coal)
 {
         int ret = 0;
         struct octeon_device *oct = lio->oct_dev;
@@ -740,59 +1202,73 @@ static int oct_cfg_adaptive_intr(struct lio *lio, struct ethtool_coalesce
 
         intrmod_cfg = &oct->intrmod;
 
-        if (adaptive) {
+        if (oct->intrmod.rx_enable || oct->intrmod.tx_enable) {
                 if (intr_coal->rate_sample_interval)
-                        intrmod_cfg->intrmod_check_intrvl =
+                        intrmod_cfg->check_intrvl =
                                 intr_coal->rate_sample_interval;
                 else
-                        intrmod_cfg->intrmod_check_intrvl =
+                        intrmod_cfg->check_intrvl =
                                 LIO_INTRMOD_CHECK_INTERVAL;
 
                 if (intr_coal->pkt_rate_high)
-                        intrmod_cfg->intrmod_maxpkt_ratethr =
+                        intrmod_cfg->maxpkt_ratethr =
                                 intr_coal->pkt_rate_high;
                 else
-                        intrmod_cfg->intrmod_maxpkt_ratethr =
+                        intrmod_cfg->maxpkt_ratethr =
                                 LIO_INTRMOD_MAXPKT_RATETHR;
 
                 if (intr_coal->pkt_rate_low)
-                        intrmod_cfg->intrmod_minpkt_ratethr =
+                        intrmod_cfg->minpkt_ratethr =
                                 intr_coal->pkt_rate_low;
                 else
-                        intrmod_cfg->intrmod_minpkt_ratethr =
+                        intrmod_cfg->minpkt_ratethr =
                                 LIO_INTRMOD_MINPKT_RATETHR;
-
+        }
+        if (oct->intrmod.rx_enable) {
                 if (intr_coal->rx_max_coalesced_frames_high)
-                        intrmod_cfg->intrmod_maxcnt_trigger =
+                        intrmod_cfg->rx_maxcnt_trigger =
                                 intr_coal->rx_max_coalesced_frames_high;
                 else
-                        intrmod_cfg->intrmod_maxcnt_trigger =
-                                LIO_INTRMOD_MAXCNT_TRIGGER;
+                        intrmod_cfg->rx_maxcnt_trigger =
+                                LIO_INTRMOD_RXMAXCNT_TRIGGER;
 
                 if (intr_coal->rx_coalesce_usecs_high)
-                        intrmod_cfg->intrmod_maxtmr_trigger =
+                        intrmod_cfg->rx_maxtmr_trigger =
                                 intr_coal->rx_coalesce_usecs_high;
                 else
-                        intrmod_cfg->intrmod_maxtmr_trigger =
-                                LIO_INTRMOD_MAXTMR_TRIGGER;
+                        intrmod_cfg->rx_maxtmr_trigger =
+                                LIO_INTRMOD_RXMAXTMR_TRIGGER;
 
                 if (intr_coal->rx_coalesce_usecs_low)
-                        intrmod_cfg->intrmod_mintmr_trigger =
+                        intrmod_cfg->rx_mintmr_trigger =
                                 intr_coal->rx_coalesce_usecs_low;
                 else
-                        intrmod_cfg->intrmod_mintmr_trigger =
-                                LIO_INTRMOD_MINTMR_TRIGGER;
+                        intrmod_cfg->rx_mintmr_trigger =
+                                LIO_INTRMOD_RXMINTMR_TRIGGER;
 
                 if (intr_coal->rx_max_coalesced_frames_low)
-                        intrmod_cfg->intrmod_mincnt_trigger =
+                        intrmod_cfg->rx_mincnt_trigger =
                                 intr_coal->rx_max_coalesced_frames_low;
                 else
-                        intrmod_cfg->intrmod_mincnt_trigger =
-                                LIO_INTRMOD_MINCNT_TRIGGER;
+                        intrmod_cfg->rx_mincnt_trigger =
+                                LIO_INTRMOD_RXMINCNT_TRIGGER;
+        }
+        if (oct->intrmod.tx_enable) {
+                if (intr_coal->tx_max_coalesced_frames_high)
+                        intrmod_cfg->tx_maxcnt_trigger =
+                                intr_coal->tx_max_coalesced_frames_high;
+                else
+                        intrmod_cfg->tx_maxcnt_trigger =
+                                LIO_INTRMOD_TXMAXCNT_TRIGGER;
+                if (intr_coal->tx_max_coalesced_frames_low)
+                        intrmod_cfg->tx_mincnt_trigger =
+                                intr_coal->tx_max_coalesced_frames_low;
+                else
+                        intrmod_cfg->tx_mincnt_trigger =
+                                LIO_INTRMOD_TXMINCNT_TRIGGER;
         }
 
-        intrmod_cfg->intrmod_enable = adaptive;
-        ret = octnet_set_intrmod_cfg(oct, intrmod_cfg);
+        ret = octnet_set_intrmod_cfg(lio, intrmod_cfg);
 
         return ret;
 }
@@ -800,51 +1276,79 @@ static int oct_cfg_adaptive_intr(struct lio *lio, struct ethtool_coalesce
 static int
 oct_cfg_rx_intrcnt(struct lio *lio, struct ethtool_coalesce *intr_coal)
 {
-        int ret;
         struct octeon_device *oct = lio->oct_dev;
-        struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)oct->chip;
         u32 rx_max_coalesced_frames;
 
-        if (!intr_coal->rx_max_coalesced_frames)
-                rx_max_coalesced_frames = CN6XXX_OQ_INTR_PKT;
-        else
-                rx_max_coalesced_frames = intr_coal->rx_max_coalesced_frames;
-
-        /* Disable adaptive interrupt modulation */
-        ret = oct_cfg_adaptive_intr(lio, intr_coal, 0);
-        if (ret)
-                return ret;
-
         /* Config Cnt based interrupt values */
-        octeon_write_csr(oct, CN6XXX_SLI_OQ_INT_LEVEL_PKTS,
-                         rx_max_coalesced_frames);
-        CFG_SET_OQ_INTR_PKT(cn6xxx->conf, rx_max_coalesced_frames);
+        switch (oct->chip_id) {
+        case OCTEON_CN68XX:
+        case OCTEON_CN66XX: {
+                struct octeon_cn6xxx *cn6xxx =
+                        (struct octeon_cn6xxx *)oct->chip;
+
+                if (!intr_coal->rx_max_coalesced_frames)
+                        rx_max_coalesced_frames = CN6XXX_OQ_INTR_PKT;
+                else
+                        rx_max_coalesced_frames =
+                                intr_coal->rx_max_coalesced_frames;
+                octeon_write_csr(oct, CN6XXX_SLI_OQ_INT_LEVEL_PKTS,
+                                 rx_max_coalesced_frames);
+                CFG_SET_OQ_INTR_PKT(cn6xxx->conf, rx_max_coalesced_frames);
+                break;
+        }
+        default:
+                return -EINVAL;
+        }
         return 0;
 }
 
 static int oct_cfg_rx_intrtime(struct lio *lio, struct ethtool_coalesce
                                *intr_coal)
 {
-        int ret;
         struct octeon_device *oct = lio->oct_dev;
-        struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)oct->chip;
         u32 time_threshold, rx_coalesce_usecs;
 
-        if (!intr_coal->rx_coalesce_usecs)
-                rx_coalesce_usecs = CN6XXX_OQ_INTR_TIME;
-        else
-                rx_coalesce_usecs = intr_coal->rx_coalesce_usecs;
+        /* Config Time based interrupt values */
+        switch (oct->chip_id) {
+        case OCTEON_CN68XX:
+        case OCTEON_CN66XX: {
+                struct octeon_cn6xxx *cn6xxx =
+                        (struct octeon_cn6xxx *)oct->chip;
+                if (!intr_coal->rx_coalesce_usecs)
+                        rx_coalesce_usecs = CN6XXX_OQ_INTR_TIME;
+                else
+                        rx_coalesce_usecs = intr_coal->rx_coalesce_usecs;
 
-        /* Disable adaptive interrupt modulation */
-        ret = oct_cfg_adaptive_intr(lio, intr_coal, 0);
-        if (ret)
-                return ret;
+                time_threshold = lio_cn6xxx_get_oq_ticks(oct,
+                                                         rx_coalesce_usecs);
+                octeon_write_csr(oct,
+                                 CN6XXX_SLI_OQ_INT_LEVEL_TIME,
+                                 time_threshold);
 
-        /* Config Time based interrupt values */
-        time_threshold = lio_cn6xxx_get_oq_ticks(oct, rx_coalesce_usecs);
-        octeon_write_csr(oct, CN6XXX_SLI_OQ_INT_LEVEL_TIME, time_threshold);
-        CFG_SET_OQ_INTR_TIME(cn6xxx->conf, rx_coalesce_usecs);
+                CFG_SET_OQ_INTR_TIME(cn6xxx->conf, rx_coalesce_usecs);
+                break;
+        }
+        default:
+                return -EINVAL;
+        }
+
+        return 0;
+}
 
+static int
+oct_cfg_tx_intrcnt(struct lio *lio, struct ethtool_coalesce *intr_coal
+                   __attribute__((unused)))
+{
+        struct octeon_device *oct = lio->oct_dev;
+
+        /* Config Cnt based interrupt values */
+        switch (oct->chip_id) {
+        case OCTEON_CN68XX:
+        case OCTEON_CN66XX:
+                break;
+        default:
+                return -EINVAL;
+        }
         return 0;
 }
 
@@ -855,59 +1359,48 @@ static int lio_set_intr_coalesce(struct net_device *netdev,
         int ret;
         struct octeon_device *oct = lio->oct_dev;
         u32 j, q_no;
+        int db_max, db_min;
 
-        if ((intr_coal->tx_max_coalesced_frames >= CN6XXX_DB_MIN) &&
-            (intr_coal->tx_max_coalesced_frames <= CN6XXX_DB_MAX)) {
-                for (j = 0; j < lio->linfo.num_txpciq; j++) {
-                        q_no = lio->linfo.txpciq[j];
-                        oct->instr_queue[q_no]->fill_threshold =
-                                intr_coal->tx_max_coalesced_frames;
+        switch (oct->chip_id) {
+        case OCTEON_CN68XX:
+        case OCTEON_CN66XX:
+                db_min = CN6XXX_DB_MIN;
+                db_max = CN6XXX_DB_MAX;
+                if ((intr_coal->tx_max_coalesced_frames >= db_min) &&
+                    (intr_coal->tx_max_coalesced_frames <= db_max)) {
+                        for (j = 0; j < lio->linfo.num_txpciq; j++) {
+                                q_no = lio->linfo.txpciq[j].s.q_no;
+                                oct->instr_queue[q_no]->fill_threshold =
+                                        intr_coal->tx_max_coalesced_frames;
+                        }
+                } else {
+                        dev_err(&oct->pci_dev->dev,
+                                "LIQUIDIO: Invalid tx-frames:%d. Range is min:%d max:%d\n",
+                                intr_coal->tx_max_coalesced_frames, db_min,
+                                db_max);
+                        return -EINVAL;
                 }
-        } else {
-                dev_err(&oct->pci_dev->dev,
-                        "LIQUIDIO: Invalid tx-frames:%d. Range is min:%d max:%d\n",
-                        intr_coal->tx_max_coalesced_frames, CN6XXX_DB_MIN,
-                        CN6XXX_DB_MAX);
+                break;
+        default:
                 return -EINVAL;
         }
 
-        /* User requested adaptive-rx on */
-        if (intr_coal->use_adaptive_rx_coalesce) {
-                ret = oct_cfg_adaptive_intr(lio, intr_coal, 1);
-                if (ret)
-                        goto ret_intrmod;
-        }
+        oct->intrmod.rx_enable = intr_coal->use_adaptive_rx_coalesce ? 1 : 0;
+        oct->intrmod.tx_enable = intr_coal->use_adaptive_tx_coalesce ? 1 : 0;
 
-        /* User requested adaptive-rx off and rx coalesce */
-        if ((intr_coal->rx_coalesce_usecs) &&
-            (!intr_coal->use_adaptive_rx_coalesce)) {
+        ret = oct_cfg_adaptive_intr(lio, intr_coal);
+
+        if (!intr_coal->use_adaptive_rx_coalesce) {
                 ret = oct_cfg_rx_intrtime(lio, intr_coal);
                 if (ret)
                         goto ret_intrmod;
-        }
 
-        /* User requested adaptive-rx off and rx coalesce */
-        if ((intr_coal->rx_max_coalesced_frames) &&
-            (!intr_coal->use_adaptive_rx_coalesce)) {
                 ret = oct_cfg_rx_intrcnt(lio, intr_coal);
                 if (ret)
                         goto ret_intrmod;
         }
-
-        /* User requested adaptive-rx off, so use default coalesce params */
-        if ((!intr_coal->rx_max_coalesced_frames) &&
-            (!intr_coal->use_adaptive_rx_coalesce) &&
-            (!intr_coal->rx_coalesce_usecs)) {
-                dev_info(&oct->pci_dev->dev,
-                         "Turning off adaptive-rx interrupt moderation\n");
-                dev_info(&oct->pci_dev->dev,
-                         "Using RX Coalesce Default values rx_coalesce_usecs:%d rx_max_coalesced_frames:%d\n",
-                         CN6XXX_OQ_INTR_TIME, CN6XXX_OQ_INTR_PKT);
-                ret = oct_cfg_rx_intrtime(lio, intr_coal);
-                if (ret)
-                        goto ret_intrmod;
-
-                ret = oct_cfg_rx_intrcnt(lio, intr_coal);
+        if (!intr_coal->use_adaptive_tx_coalesce) {
+                ret = oct_cfg_tx_intrcnt(lio, intr_coal);
                 if (ret)
                         goto ret_intrmod;
         }
@@ -923,23 +1416,28 @@ static int lio_get_ts_info(struct net_device *netdev,
         struct lio *lio = GET_LIO(netdev);
 
         info->so_timestamping =
+#ifdef PTP_HARDWARE_TIMESTAMPING
                 SOF_TIMESTAMPING_TX_HARDWARE |
-                SOF_TIMESTAMPING_TX_SOFTWARE |
                 SOF_TIMESTAMPING_RX_HARDWARE |
+                SOF_TIMESTAMPING_RAW_HARDWARE |
+                SOF_TIMESTAMPING_TX_SOFTWARE |
+#endif
                 SOF_TIMESTAMPING_RX_SOFTWARE |
-                SOF_TIMESTAMPING_SOFTWARE | SOF_TIMESTAMPING_RAW_HARDWARE;
+                SOF_TIMESTAMPING_SOFTWARE;
 
         if (lio->ptp_clock)
                 info->phc_index = ptp_clock_index(lio->ptp_clock);
         else
                 info->phc_index = -1;
 
+#ifdef PTP_HARDWARE_TIMESTAMPING
         info->tx_types = (1 << HWTSTAMP_TX_OFF) | (1 << HWTSTAMP_TX_ON);
 
         info->rx_filters = (1 << HWTSTAMP_FILTER_NONE) |
                            (1 << HWTSTAMP_FILTER_PTP_V1_L4_EVENT) |
                            (1 << HWTSTAMP_FILTER_PTP_V2_L2_EVENT) |
                            (1 << HWTSTAMP_FILTER_PTP_V2_L4_EVENT);
+#endif
 
         return 0;
 }
@@ -950,7 +1448,6 @@ static int lio_set_settings(struct net_device *netdev, struct ethtool_cmd *ecmd)
         struct octeon_device *oct = lio->oct_dev;
         struct oct_link_info *linfo;
         struct octnic_ctrl_pkt nctrl;
-        struct octnic_ctrl_params nparams;
         int ret = 0;
 
         /* get the link info */
@@ -965,12 +1462,14 @@ static int lio_set_settings(struct net_device *netdev, struct ethtool_cmd *ecmd)
                                                   ecmd->duplex != DUPLEX_FULL)))
                 return -EINVAL;
 
-        /* Ethtool Support is not provided for XAUI and RXAUI Interfaces
+        /* Ethtool Support is not provided for XAUI, RXAUI, and XFI Interfaces
          * as they operate at fixed Speed and Duplex settings
          */
-        if (linfo->link.s.interface == INTERFACE_MODE_XAUI ||
-            linfo->link.s.interface == INTERFACE_MODE_RXAUI) {
-                dev_info(&oct->pci_dev->dev, "XAUI IFs settings cannot be modified.\n");
+        if (linfo->link.s.if_mode == INTERFACE_MODE_XAUI ||
+            linfo->link.s.if_mode == INTERFACE_MODE_RXAUI ||
+            linfo->link.s.if_mode == INTERFACE_MODE_XFI) {
+                dev_info(&oct->pci_dev->dev,
+                         "Autonegotiation, duplex and speed settings cannot be modified.\n");
                 return -EINVAL;
         }
 
@@ -978,9 +1477,9 @@ static int lio_set_settings(struct net_device *netdev, struct ethtool_cmd *ecmd)
 
         nctrl.ncmd.u64 = 0;
         nctrl.ncmd.s.cmd = OCTNET_CMD_SET_SETTINGS;
+        nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
         nctrl.wait_time = 1000;
         nctrl.netpndev = (u64)netdev;
-        nctrl.ncmd.s.param1 = lio->linfo.ifidx;
         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
 
         /* Passing the parameters sent by ethtool like Speed, Autoneg & Duplex
@@ -990,19 +1489,17 @@ static int lio_set_settings(struct net_device *netdev, struct ethtool_cmd *ecmd)
                 /* Autoneg ON */
                 nctrl.ncmd.s.more = OCTNIC_NCMD_PHY_ON |
                                      OCTNIC_NCMD_AUTONEG_ON;
-                nctrl.ncmd.s.param2 = ecmd->advertising;
+                nctrl.ncmd.s.param1 = ecmd->advertising;
         } else {
                 /* Autoneg OFF */
                 nctrl.ncmd.s.more = OCTNIC_NCMD_PHY_ON;
 
-                nctrl.ncmd.s.param3 = ecmd->duplex;
+                nctrl.ncmd.s.param2 = ecmd->duplex;
 
-                nctrl.ncmd.s.param2 = ecmd->speed;
+                nctrl.ncmd.s.param1 = ecmd->speed;
         }
 
-        nparams.resp_order = OCTEON_RESP_ORDERED;
-
-        ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams);
+        ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
         if (ret < 0) {
                 dev_err(&oct->pci_dev->dev, "Failed to set settings\n");
                 return -1;
@@ -1186,6 +1683,23 @@ static void lio_get_regs(struct net_device *dev,
         }
 }
 
+static u32 lio_get_priv_flags(struct net_device *netdev)
+{
+        struct lio *lio = GET_LIO(netdev);
+
+        return lio->oct_dev->priv_flags;
+}
+
+static int lio_set_priv_flags(struct net_device *netdev, u32 flags)
+{
+        struct lio *lio = GET_LIO(netdev);
+        bool intr_by_tx_bytes = !!(flags & (0x1 << OCT_PRIV_FLAG_TX_BYTES));
+
+        lio_set_priv_flag(lio->oct_dev, OCT_PRIV_FLAG_TX_BYTES,
+                          intr_by_tx_bytes);
+        return 0;
+}
+
 static const struct ethtool_ops lio_ethtool_ops = {
         .get_settings           = lio_get_settings,
         .get_link               = ethtool_op_get_link,
@@ -1207,6 +1721,8 @@ static const struct ethtool_ops lio_ethtool_ops = {
         .set_settings           = lio_set_settings,
         .get_coalesce           = lio_get_intr_coalesce,
         .set_coalesce           = lio_set_intr_coalesce,
+        .get_priv_flags         = lio_get_priv_flags,
+        .set_priv_flags         = lio_set_priv_flags,
         .get_ts_info            = lio_get_ts_info,
 };
 
diff --git a/drivers/net/ethernet/cavium/liquidio/lio_main.c b/drivers/net/ethernet/cavium/liquidio/lio_main.c
index 8de79ae63231..1a584ebde42c 100644
--- a/drivers/net/ethernet/cavium/liquidio/lio_main.c
+++ b/drivers/net/ethernet/cavium/liquidio/lio_main.c
@@ -72,6 +72,9 @@ MODULE_PARM_DESC(console_bitmask,
 
 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
 
+#define INCR_INSTRQUEUE_PKT_COUNT(octeon_dev_ptr, iq_no, field, count)  \
+        (octeon_dev_ptr->instr_queue[iq_no]->stats.field += count)
+
 static int debug = -1;
 module_param(debug, int, 0644);
 MODULE_PARM_DESC(debug, "NETIF_MSG debug bits");
@@ -84,6 +87,8 @@ static int conf_type;
 module_param(conf_type, int, 0);
 MODULE_PARM_DESC(conf_type, "select octeon configuration 0 default 1 ovs");
 
+static int ptp_enable = 1;
+
 /* Bit mask values for lio->ifstate */
 #define   LIO_IFSTATE_DROQ_OPS             0x01
 #define   LIO_IFSTATE_REGISTERED           0x02
@@ -166,6 +171,8 @@ struct octnic_gather {
          *  received from the IP layer.
          */
         struct octeon_sg_entry *sg;
+
+        u64 sg_dma_ptr;
 };
 
 /** This structure is used by NIC driver to store information required
@@ -220,8 +227,8 @@ static void octeon_droq_bh(unsigned long pdev)
                 (struct octeon_device_priv *)oct->priv;
 
         /* for (q_no = 0; q_no < oct->num_oqs; q_no++) { */
-        for (q_no = 0; q_no < MAX_OCTEON_OUTPUT_QUEUES; q_no++) {
-                if (!(oct->io_qmask.oq & (1UL << q_no)))
+        for (q_no = 0; q_no < MAX_OCTEON_OUTPUT_QUEUES(oct); q_no++) {
+                if (!(oct->io_qmask.oq & (1ULL << q_no)))
                         continue;
                 reschedule |= octeon_droq_process_packets(oct, oct->droq[q_no],
                                                           MAX_PACKET_BUDGET);
@@ -241,8 +248,8 @@ static int lio_wait_for_oq_pkts(struct octeon_device *oct)
         do {
                 pending_pkts = 0;
 
-                for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES; i++) {
-                        if (!(oct->io_qmask.oq & (1UL << i)))
+                for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
+                        if (!(oct->io_qmask.oq & (1ULL << i)))
                                 continue;
                         pkt_cnt += octeon_droq_check_hw_for_pkts(oct,
                                                                  oct->droq[i]);
@@ -361,7 +368,7 @@ static int wait_for_pending_requests(struct octeon_device *oct)
                                 [OCTEON_ORDERED_SC_LIST].pending_req_count);
                 if (pcount)
                         schedule_timeout_uninterruptible(HZ / 10);
-                 else
+                else
                         break;
         }
 
@@ -392,10 +399,10 @@ static inline void pcierror_quiesce_device(struct octeon_device *oct)
                 dev_err(&oct->pci_dev->dev, "There were pending requests\n");
 
         /* Force all requests waiting to be fetched by OCTEON to complete. */
-        for (i = 0; i < MAX_OCTEON_INSTR_QUEUES; i++) {
+        for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
                 struct octeon_instr_queue *iq;
 
-                if (!(oct->io_qmask.iq & (1UL << i)))
+                if (!(oct->io_qmask.iq & (1ULL << i)))
                         continue;
                 iq = oct->instr_queue[i];
 
@@ -405,7 +412,7 @@ static inline void pcierror_quiesce_device(struct octeon_device *oct)
                         iq->octeon_read_index = iq->host_write_index;
                         iq->stats.instr_processed +=
                                 atomic_read(&iq->instr_pending);
-                        lio_process_iq_request_list(oct, iq);
+                        lio_process_iq_request_list(oct, iq, 0);
                         spin_unlock_bh(&iq->lock);
                 }
         }
@@ -678,12 +685,24 @@ static inline void txqs_start(struct net_device *netdev)
  */
 static inline void txqs_wake(struct net_device *netdev)
 {
+        struct lio *lio = GET_LIO(netdev);
+
         if (netif_is_multiqueue(netdev)) {
                 int i;
 
-                for (i = 0; i < netdev->num_tx_queues; i++)
-                        netif_wake_subqueue(netdev, i);
+                for (i = 0; i < netdev->num_tx_queues; i++) {
+                        int qno = lio->linfo.txpciq[i %
+                                (lio->linfo.num_txpciq)].s.q_no;
+
+                        if (__netif_subqueue_stopped(netdev, i)) {
+                                INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, qno,
+                                                          tx_restart, 1);
+                                netif_wake_subqueue(netdev, i);
+                        }
+                }
         } else {
+                INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, lio->txq,
+                                          tx_restart, 1);
                 netif_wake_queue(netdev);
         }
 }
@@ -705,7 +724,7 @@ static void start_txq(struct net_device *netdev)
 {
         struct lio *lio = GET_LIO(netdev);
 
-        if (lio->linfo.link.s.status) {
+        if (lio->linfo.link.s.link_up) {
                 txqs_start(netdev);
                 return;
         }
@@ -752,16 +771,23 @@ static inline int check_txq_status(struct lio *lio)
 
                 /* check each sub-queue state */
                 for (q = 0; q < numqs; q++) {
-                        iq = lio->linfo.txpciq[q & (lio->linfo.num_txpciq - 1)];
+                        iq = lio->linfo.txpciq[q %
+                                (lio->linfo.num_txpciq)].s.q_no;
                         if (octnet_iq_is_full(lio->oct_dev, iq))
                                 continue;
-                        wake_q(lio->netdev, q);
-                        ret_val++;
+                        if (__netif_subqueue_stopped(lio->netdev, q)) {
+                                wake_q(lio->netdev, q);
+                                INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq,
+                                                          tx_restart, 1);
+                                ret_val++;
+                        }
                 }
         } else {
                 if (octnet_iq_is_full(lio->oct_dev, lio->txq))
                         return 0;
                 wake_q(lio->netdev, lio->txq);
+                INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, lio->txq,
+                                          tx_restart, 1);
                 ret_val = 1;
         }
         return ret_val;
@@ -787,64 +813,116 @@ static inline struct list_head *list_delete_head(struct list_head *root)
 }
 
 /**
- * \brief Delete gather list
+ * \brief Delete gather lists
  * @param lio per-network private data
  */
-static void delete_glist(struct lio *lio)
+static void delete_glists(struct lio *lio)
 {
         struct octnic_gather *g;
+        int i;
 
-        do {
-                g = (struct octnic_gather *)
-                    list_delete_head(&lio->glist);
-                if (g) {
-                        if (g->sg)
-                                kfree((void *)((unsigned long)g->sg -
-                                                g->adjust));
-                        kfree(g);
-                }
-        } while (g);
+        if (!lio->glist)
+                return;
+
+        for (i = 0; i < lio->linfo.num_txpciq; i++) {
+                do {
+                        g = (struct octnic_gather *)
+                                list_delete_head(&lio->glist[i]);
+                        if (g) {
+                                if (g->sg) {
+                                        dma_unmap_single(&lio->oct_dev->
+                                                         pci_dev->dev,
+                                                         g->sg_dma_ptr,
+                                                         g->sg_size,
+                                                         DMA_TO_DEVICE);
+                                        kfree((void *)((unsigned long)g->sg -
+                                                       g->adjust));
+                                }
+                                kfree(g);
+                        }
+                } while (g);
+        }
+
+        kfree((void *)lio->glist);
 }
 
 /**
- * \brief Setup gather list
+ * \brief Setup gather lists
  * @param lio per-network private data
  */
-static int setup_glist(struct lio *lio)
+static int setup_glists(struct octeon_device *oct, struct lio *lio, int num_iqs)
 {
-        int i;
+        int i, j;
         struct octnic_gather *g;
 
-        INIT_LIST_HEAD(&lio->glist);
+        lio->glist_lock = kcalloc(num_iqs, sizeof(*lio->glist_lock),
+                                  GFP_KERNEL);
+        if (!lio->glist_lock)
+                return 1;
+
+        lio->glist = kcalloc(num_iqs, sizeof(*lio->glist),
+                             GFP_KERNEL);
+        if (!lio->glist) {
+                kfree((void *)lio->glist_lock);
+                return 1;
+        }
 
-        for (i = 0; i < lio->tx_qsize; i++) {
-                g = kzalloc(sizeof(*g), GFP_KERNEL);
-                if (!g)
-                        break;
+        for (i = 0; i < num_iqs; i++) {
+                int numa_node = cpu_to_node(i % num_online_cpus());
 
-                g->sg_size =
-                        ((ROUNDUP4(OCTNIC_MAX_SG) >> 2) * OCT_SG_ENTRY_SIZE);
+                spin_lock_init(&lio->glist_lock[i]);
 
-                g->sg = kmalloc(g->sg_size + 8, GFP_KERNEL);
-                if (!g->sg) {
-                        kfree(g);
-                        break;
+                INIT_LIST_HEAD(&lio->glist[i]);
+
+                for (j = 0; j < lio->tx_qsize; j++) {
+                        g = kzalloc_node(sizeof(*g), GFP_KERNEL,
+                                         numa_node);
+                        if (!g)
+                                g = kzalloc(sizeof(*g), GFP_KERNEL);
+                        if (!g)
+                                break;
+
+                        g->sg_size = ((ROUNDUP4(OCTNIC_MAX_SG) >> 2) *
+                                      OCT_SG_ENTRY_SIZE);
+
+                        g->sg = kmalloc_node(g->sg_size + 8,
+                                             GFP_KERNEL, numa_node);
+                        if (!g->sg)
+                                g->sg = kmalloc(g->sg_size + 8, GFP_KERNEL);
+                        if (!g->sg) {
+                                kfree(g);
+                                break;
+                        }
+
+                        /* The gather component should be aligned on 64-bit
+                         * boundary
+                         */
+                        if (((unsigned long)g->sg) & 7) {
+                                g->adjust = 8 - (((unsigned long)g->sg) & 7);
+                                g->sg = (struct octeon_sg_entry *)
+                                        ((unsigned long)g->sg + g->adjust);
+                        }
+                        g->sg_dma_ptr = dma_map_single(&oct->pci_dev->dev,
+                                                       g->sg, g->sg_size,
+                                                       DMA_TO_DEVICE);
+                        if (dma_mapping_error(&oct->pci_dev->dev,
+                                              g->sg_dma_ptr)) {
+                                kfree((void *)((unsigned long)g->sg -
+                                               g->adjust));
+                                kfree(g);
+                                break;
+                        }
+
+                        list_add_tail(&g->list, &lio->glist[i]);
                 }
 
-                /* The gather component should be aligned on 64-bit boundary */
-                if (((unsigned long)g->sg) & 7) {
-                        g->adjust = 8 - (((unsigned long)g->sg) & 7);
-                        g->sg = (struct octeon_sg_entry *)
-                                ((unsigned long)g->sg + g->adjust);
+                if (j != lio->tx_qsize) {
+                        delete_glists(lio);
+                        return 1;
                 }
-                list_add_tail(&g->list, &lio->glist);
         }
 
-        if (i == lio->tx_qsize)
-                return 0;
-
-        delete_glist(lio);
-        return 1;
+        return 0;
 }
 
 /**
@@ -858,7 +936,7 @@ static void print_link_info(struct net_device *netdev)
         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED) {
                 struct oct_link_info *linfo = &lio->linfo;
 
-                if (linfo->link.s.status) {
+                if (linfo->link.s.link_up) {
                         netif_info(lio, link, lio->netdev, "%d Mbps %s Duplex UP\n",
                                    linfo->link.s.speed,
                                    (linfo->link.s.duplex) ? "Full" : "Half");
@@ -880,13 +958,15 @@ static inline void update_link_status(struct net_device *netdev,
                                       union oct_link_status *ls)
 {
         struct lio *lio = GET_LIO(netdev);
+        int changed = (lio->linfo.link.u64 != ls->u64);
 
-        if ((lio->intf_open) && (lio->linfo.link.u64 != ls->u64)) {
-                lio->linfo.link.u64 = ls->u64;
+        lio->linfo.link.u64 = ls->u64;
 
+        if ((lio->intf_open) && (changed)) {
                 print_link_info(netdev);
+                lio->link_changes++;
 
-                if (lio->linfo.link.s.status) {
+                if (lio->linfo.link.s.link_up) {
                         netif_carrier_on(netdev);
                         /* start_txq(netdev); */
                         txqs_wake(netdev);
@@ -897,6 +977,42 @@ static inline void update_link_status(struct net_device *netdev,
         }
 }
 
+/* Runs in interrupt context. */
+static void update_txq_status(struct octeon_device *oct, int iq_num)
+{
+        struct net_device *netdev;
+        struct lio *lio;
+        struct octeon_instr_queue *iq = oct->instr_queue[iq_num];
+
+        /*octeon_update_iq_read_idx(oct, iq);*/
+
+        netdev = oct->props[iq->ifidx].netdev;
+
+        /* This is needed because the first IQ does not have
+         * a netdev associated with it.
+         */
+        if (!netdev)
+                return;
+
+        lio = GET_LIO(netdev);
+        if (netif_is_multiqueue(netdev)) {
+                if (__netif_subqueue_stopped(netdev, iq->q_index) &&
+                    lio->linfo.link.s.link_up &&
+                    (!octnet_iq_is_full(oct, iq_num))) {
+                        INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq_num,
+                                                  tx_restart, 1);
+                        netif_wake_subqueue(netdev, iq->q_index);
+                } else {
+                        if (!octnet_iq_is_full(oct, lio->txq)) {
+                                INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev,
+                                                          lio->txq,
+                                                          tx_restart, 1);
+                                wake_q(netdev, lio->txq);
+                        }
+                }
+        }
+}
+
 /**
  * \brief Droq packet processor sceduler
  * @param oct octeon device
@@ -910,8 +1026,9 @@ void liquidio_schedule_droq_pkt_handlers(struct octeon_device *oct)
         struct octeon_droq *droq;
 
         if (oct->int_status & OCT_DEV_INTR_PKT_DATA) {
-                for (oq_no = 0; oq_no < MAX_OCTEON_OUTPUT_QUEUES; oq_no++) {
-                        if (!(oct->droq_intr & (1 << oq_no)))
+                for (oq_no = 0; oq_no < MAX_OCTEON_OUTPUT_QUEUES(oct);
+                     oq_no++) {
+                        if (!(oct->droq_intr & (1ULL << oq_no)))
                                 continue;
 
                         droq = oct->droq[oq_no];
@@ -1022,6 +1139,9 @@ static int liquidio_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
                 return -ENOMEM;
         }
 
+        oct_dev->rx_pause = 1;
+        oct_dev->tx_pause = 1;
+
         dev_dbg(&oct_dev->pci_dev->dev, "Device is ready\n");
 
         return 0;
@@ -1087,19 +1207,13 @@ static void octeon_destroy_resources(struct octeon_device *oct)
                 if (oct->flags & LIO_FLAG_MSI_ENABLED)
                         pci_disable_msi(oct->pci_dev);
 
-                /* Soft reset the octeon device before exiting */
-                oct->fn_list.soft_reset(oct);
-
-                /* Disable the device, releasing the PCI INT */
-                pci_disable_device(oct->pci_dev);
-
                 /* fallthrough */
         case OCT_DEV_IN_RESET:
         case OCT_DEV_DROQ_INIT_DONE:
                 /*atomic_set(&oct->status, OCT_DEV_DROQ_INIT_DONE);*/
                 mdelay(100);
-                for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES; i++) {
-                        if (!(oct->io_qmask.oq & (1UL << i)))
+                for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
+                        if (!(oct->io_qmask.oq & (1ULL << i)))
                                 continue;
                         octeon_delete_droq(oct, i);
                 }
@@ -1126,8 +1240,8 @@ static void octeon_destroy_resources(struct octeon_device *oct)
 
                 /* fallthrough */
         case OCT_DEV_INSTR_QUEUE_INIT_DONE:
-                for (i = 0; i < MAX_OCTEON_INSTR_QUEUES; i++) {
-                        if (!(oct->io_qmask.iq & (1UL << i)))
+                for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
+                        if (!(oct->io_qmask.iq & (1ULL << i)))
                                 continue;
                         octeon_delete_instr_queue(oct, i);
                 }
@@ -1139,11 +1253,18 @@ static void octeon_destroy_resources(struct octeon_device *oct)
 
                 /* fallthrough */
         case OCT_DEV_PCI_MAP_DONE:
+
+                /* Soft reset the octeon device before exiting */
+                oct->fn_list.soft_reset(oct);
+
                 octeon_unmap_pci_barx(oct, 0);
                 octeon_unmap_pci_barx(oct, 1);
 
                 /* fallthrough */
         case OCT_DEV_BEGIN_STATE:
+                /* Disable the device, releasing the PCI INT */
+                pci_disable_device(oct->pci_dev);
+
                 /* Nothing to be done here either */
                 break;
         }                       /* end switch(oct->status) */
@@ -1159,18 +1280,15 @@ static void octeon_destroy_resources(struct octeon_device *oct)
 static void send_rx_ctrl_cmd(struct lio *lio, int start_stop)
 {
         struct octnic_ctrl_pkt nctrl;
-        struct octnic_ctrl_params nparams;
 
         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
 
         nctrl.ncmd.s.cmd = OCTNET_CMD_RX_CTL;
-        nctrl.ncmd.s.param1 = lio->linfo.ifidx;
-        nctrl.ncmd.s.param2 = start_stop;
+        nctrl.ncmd.s.param1 = start_stop;
+        nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
         nctrl.netpndev = (u64)lio->netdev;
 
-        nparams.resp_order = OCTEON_RESP_NORESPONSE;
-
-        if (octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams) < 0)
+        if (octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl) < 0)
                 netif_info(lio, rx_err, lio->netdev, "Failed to send RX Control message\n");
 }
 
@@ -1186,6 +1304,7 @@ static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)
 {
         struct net_device *netdev = oct->props[ifidx].netdev;
         struct lio *lio;
+        struct napi_struct *napi, *n;
 
         if (!netdev) {
                 dev_err(&oct->pci_dev->dev, "%s No netdevice ptr for index %d\n",
@@ -1202,13 +1321,22 @@ static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)
         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING)
                 txqs_stop(netdev);
 
+        if (oct->props[lio->ifidx].napi_enabled == 1) {
+                list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
+                        napi_disable(napi);
+
+                oct->props[lio->ifidx].napi_enabled = 0;
+        }
+
         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED)
                 unregister_netdev(netdev);
 
-        delete_glist(lio);
+        delete_glists(lio);
 
         free_netdev(netdev);
 
+        oct->props[ifidx].gmxport = -1;
+
         oct->props[ifidx].netdev = NULL;
 }
 
@@ -1227,10 +1355,15 @@ static int liquidio_stop_nic_module(struct octeon_device *oct)
                 return 1;
         }
 
+        spin_lock_bh(&oct->cmd_resp_wqlock);
+        oct->cmd_resp_state = OCT_DRV_OFFLINE;
+        spin_unlock_bh(&oct->cmd_resp_wqlock);
+
         for (i = 0; i < oct->ifcount; i++) {
                 lio = GET_LIO(oct->props[i].netdev);
                 for (j = 0; j < lio->linfo.num_rxpciq; j++)
-                        octeon_unregister_droq_ops(oct, lio->linfo.rxpciq[j]);
+                        octeon_unregister_droq_ops(oct,
+                                                   lio->linfo.rxpciq[j].s.q_no);
         }
 
         for (i = 0; i < oct->ifcount; i++)
@@ -1274,6 +1407,7 @@ static int octeon_chip_specific_setup(struct octeon_device *oct)
 {
         u32 dev_id, rev_id;
         int ret = 1;
+        char *s;
 
         pci_read_config_dword(oct->pci_dev, 0, &dev_id);
         pci_read_config_dword(oct->pci_dev, 8, &rev_id);
@@ -1283,22 +1417,27 @@ static int octeon_chip_specific_setup(struct octeon_device *oct)
         case OCTEON_CN68XX_PCIID:
                 oct->chip_id = OCTEON_CN68XX;
                 ret = lio_setup_cn68xx_octeon_device(oct);
+                s = "CN68XX";
                 break;
 
         case OCTEON_CN66XX_PCIID:
                 oct->chip_id = OCTEON_CN66XX;
                 ret = lio_setup_cn66xx_octeon_device(oct);
+                s = "CN66XX";
                 break;
+
         default:
+                s = "?";
                 dev_err(&oct->pci_dev->dev, "Unknown device found (dev_id: %x)\n",
                         dev_id);
         }
 
         if (!ret)
-                dev_info(&oct->pci_dev->dev, "CN68XX PASS%d.%d %s\n",
+                dev_info(&oct->pci_dev->dev, "%s PASS%d.%d %s Version: %s\n", s,
                          OCTEON_MAJOR_REV(oct),
                          OCTEON_MINOR_REV(oct),
-                         octeon_get_conf(oct)->card_name);
+                         octeon_get_conf(oct)->card_name,
+                         LIQUIDIO_VERSION);
 
         return ret;
 }
@@ -1326,6 +1465,16 @@ static int octeon_pci_os_setup(struct octeon_device *oct)
         return 0;
 }
 
+static inline int skb_iq(struct lio *lio, struct sk_buff *skb)
+{
+        int q = 0;
+
+        if (netif_is_multiqueue(lio->netdev))
+                q = skb->queue_mapping % lio->linfo.num_txpciq;
+
+        return q;
+}
+
 /**
  * \brief Check Tx queue state for a given network buffer
  * @param lio per-network private data
@@ -1337,14 +1486,19 @@ static inline int check_txq_state(struct lio *lio, struct sk_buff *skb)
 
         if (netif_is_multiqueue(lio->netdev)) {
                 q = skb->queue_mapping;
-                iq = lio->linfo.txpciq[(q & (lio->linfo.num_txpciq - 1))];
+                iq = lio->linfo.txpciq[(q % (lio->linfo.num_txpciq))].s.q_no;
         } else {
                 iq = lio->txq;
+                q = iq;
         }
 
         if (octnet_iq_is_full(lio->oct_dev, iq))
                 return 0;
-        wake_q(lio->netdev, q);
+
+        if (__netif_subqueue_stopped(lio->netdev, q)) {
+                INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq, tx_restart, 1);
+                wake_q(lio->netdev, q);
+        }
         return 1;
 }
 
@@ -1367,7 +1521,7 @@ static void free_netbuf(void *buf)
 
         check_txq_state(lio, skb);
 
-        recv_buffer_free((struct sk_buff *)skb);
+        tx_buffer_free(skb);
 }
 
 /**
@@ -1380,7 +1534,7 @@ static void free_netsgbuf(void *buf)
         struct sk_buff *skb;
         struct lio *lio;
         struct octnic_gather *g;
-        int i, frags;
+        int i, frags, iq;
 
         finfo = (struct octnet_buf_free_info *)buf;
         skb = finfo->skb;
@@ -1402,17 +1556,17 @@ static void free_netsgbuf(void *buf)
                 i++;
         }
 
-        dma_unmap_single(&lio->oct_dev->pci_dev->dev,
-                         finfo->dptr, g->sg_size,
-                         DMA_TO_DEVICE);
+        dma_sync_single_for_cpu(&lio->oct_dev->pci_dev->dev,
+                                g->sg_dma_ptr, g->sg_size, DMA_TO_DEVICE);
 
-        spin_lock(&lio->lock);
-        list_add_tail(&g->list, &lio->glist);
-        spin_unlock(&lio->lock);
+        iq = skb_iq(lio, skb);
+        spin_lock(&lio->glist_lock[iq]);
+        list_add_tail(&g->list, &lio->glist[iq]);
+        spin_unlock(&lio->glist_lock[iq]);
 
         check_txq_state(lio, skb);     /* mq support: sub-queue state check */
 
-        recv_buffer_free((struct sk_buff *)skb);
+        tx_buffer_free(skb);
 }
 
 /**
@@ -1426,7 +1580,7 @@ static void free_netsgbuf_with_resp(void *buf)
         struct sk_buff *skb;
         struct lio *lio;
         struct octnic_gather *g;
-        int i, frags;
+        int i, frags, iq;
 
         sc = (struct octeon_soft_command *)buf;
         skb = (struct sk_buff *)sc->callback_arg;
@@ -1450,13 +1604,14 @@ static void free_netsgbuf_with_resp(void *buf)
                 i++;
         }
 
-        dma_unmap_single(&lio->oct_dev->pci_dev->dev,
-                         finfo->dptr, g->sg_size,
-                         DMA_TO_DEVICE);
+        dma_sync_single_for_cpu(&lio->oct_dev->pci_dev->dev,
+                                g->sg_dma_ptr, g->sg_size, DMA_TO_DEVICE);
+
+        iq = skb_iq(lio, skb);
 
-        spin_lock(&lio->lock);
-        list_add_tail(&g->list, &lio->glist);
-        spin_unlock(&lio->lock);
+        spin_lock(&lio->glist_lock[iq]);
+        list_add_tail(&g->list, &lio->glist[iq]);
+        spin_unlock(&lio->glist_lock[iq]);
 
         /* Don't free the skb yet */
 
@@ -1657,6 +1812,7 @@ static int load_firmware(struct octeon_device *oct)
         if (ret) {
                 dev_err(&oct->pci_dev->dev, "Request firmware failed. Could not find file %s.\n.",
                         fw_name);
+                release_firmware(fw);
                 return ret;
         }
 
@@ -1726,6 +1882,9 @@ static void if_cfg_callback(struct octeon_device *oct,
                         CVM_CAST64(resp->status));
         ACCESS_ONCE(ctx->cond) = 1;
 
+        snprintf(oct->fw_info.liquidio_firmware_version, 32, "%s",
+                 resp->cfg_info.liquidio_firmware_version);
+
         /* This barrier is required to be sure that the response has been
          * written fully before waking up the handler
          */
@@ -1743,14 +1902,13 @@ static void if_cfg_callback(struct octeon_device *oct,
 static u16 select_q(struct net_device *dev, struct sk_buff *skb,
                     void *accel_priv, select_queue_fallback_t fallback)
 {
-        int qindex;
+        u32 qindex = 0;
         struct lio *lio;
 
         lio = GET_LIO(dev);
-        /* select queue on chosen queue_mapping or core */
-        qindex = skb_rx_queue_recorded(skb) ?
-                 skb_get_rx_queue(skb) : smp_processor_id();
-        return (u16)(qindex & (lio->linfo.num_txpciq - 1));
+        qindex = skb_tx_hash(dev, skb);
+
+        return (u16)(qindex % (lio->linfo.num_txpciq));
 }
 
 /** Routine to push packets arriving on Octeon interface upto network layer.
@@ -1759,26 +1917,28 @@ static u16 select_q(struct net_device *dev, struct sk_buff *skb,
  * @param len      - size of total data received.
  * @param rh       - Control header associated with the packet
  * @param param    - additional control data with the packet
+ * @param arg      - farg registered in droq_ops
  */
 static void
 liquidio_push_packet(u32 octeon_id,
                      void *skbuff,
                      u32 len,
                      union octeon_rh *rh,
-                     void *param)
+                     void *param,
+                     void *arg)
 {
         struct napi_struct *napi = param;
-        struct octeon_device *oct = lio_get_device(octeon_id);
         struct sk_buff *skb = (struct sk_buff *)skbuff;
         struct skb_shared_hwtstamps *shhwtstamps;
         u64 ns;
-        struct net_device *netdev =
-                (struct net_device *)oct->props[rh->r_dh.link].netdev;
+        u16 vtag = 0;
+        struct net_device *netdev = (struct net_device *)arg;
         struct octeon_droq *droq = container_of(param, struct octeon_droq,
                                                 napi);
         if (netdev) {
                 int packet_was_received;
                 struct lio *lio = GET_LIO(netdev);
+                struct octeon_device *oct = lio->oct_dev;
 
                 /* Do not proceed if the interface is not in RUNNING state. */
                 if (!ifstate_check(lio, LIO_IFSTATE_RUNNING)) {
@@ -1789,21 +1949,54 @@ liquidio_push_packet(u32 octeon_id,
 
                 skb->dev = netdev;
 
-                if (rh->r_dh.has_hwtstamp) {
-                        /* timestamp is included from the hardware at the
-                         * beginning of the packet.
-                         */
-                        if (ifstate_check(lio,
-                                          LIO_IFSTATE_RX_TIMESTAMP_ENABLED)) {
-                                /* Nanoseconds are in the first 64-bits
-                                 * of the packet.
+                skb_record_rx_queue(skb, droq->q_no);
+                if (likely(len > MIN_SKB_SIZE)) {
+                        struct octeon_skb_page_info *pg_info;
+                        unsigned char *va;
+
+                        pg_info = ((struct octeon_skb_page_info *)(skb->cb));
+                        if (pg_info->page) {
+                                /* For Paged allocation use the frags */
+                                va = page_address(pg_info->page) +
+                                        pg_info->page_offset;
+                                memcpy(skb->data, va, MIN_SKB_SIZE);
+                                skb_put(skb, MIN_SKB_SIZE);
+                                skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
+                                                pg_info->page,
+                                                pg_info->page_offset +
+                                                MIN_SKB_SIZE,
+                                                len - MIN_SKB_SIZE,
+                                                LIO_RXBUFFER_SZ);
+                        }
+                } else {
+                        struct octeon_skb_page_info *pg_info =
+                                ((struct octeon_skb_page_info *)(skb->cb));
+                        skb_copy_to_linear_data(skb, page_address(pg_info->page)
+                                                + pg_info->page_offset, len);
+                        skb_put(skb, len);
+                        put_page(pg_info->page);
+                }
+
+                if (((oct->chip_id == OCTEON_CN66XX) ||
+                     (oct->chip_id == OCTEON_CN68XX)) &&
+                    ptp_enable) {
+                        if (rh->r_dh.has_hwtstamp) {
+                                /* timestamp is included from the hardware at
+                                 * the beginning of the packet.
                                  */
-                                memcpy(&ns, (skb->data), sizeof(ns));
-                                shhwtstamps = skb_hwtstamps(skb);
-                                shhwtstamps->hwtstamp =
-                                        ns_to_ktime(ns + lio->ptp_adjust);
+                                if (ifstate_check
+                                    (lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED)) {
+                                        /* Nanoseconds are in the first 64-bits
+                                         * of the packet.
+                                         */
+                                        memcpy(&ns, (skb->data), sizeof(ns));
+                                        shhwtstamps = skb_hwtstamps(skb);
+                                        shhwtstamps->hwtstamp =
+                                                ns_to_ktime(ns +
+                                                            lio->ptp_adjust);
+                                }
+                                skb_pull(skb, sizeof(ns));
                         }
-                        skb_pull(skb, sizeof(ns));
                 }
 
                 skb->protocol = eth_type_trans(skb, skb->dev);
@@ -1815,6 +2008,16 @@ liquidio_push_packet(u32 octeon_id,
                 else
                         skb->ip_summed = CHECKSUM_NONE;
 
+                /* inbound VLAN tag */
+                if ((netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
+                    (rh->r_dh.vlan != 0)) {
+                        u16 vid = rh->r_dh.vlan;
+                        u16 priority = rh->r_dh.priority;
+
+                        vtag = priority << 13 | vid;
+                        __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vtag);
+                }
+
                 packet_was_received = napi_gro_receive(napi, skb) != GRO_DROP;
 
                 if (packet_was_received) {
@@ -1869,39 +2072,6 @@ static void liquidio_napi_drv_callback(void *arg)
 }
 
 /**
- * \brief Main NAPI poll function
- * @param droq octeon output queue
- * @param budget maximum number of items to process
- */
-static int liquidio_napi_do_rx(struct octeon_droq *droq, int budget)
-{
-        int work_done;
-        struct lio *lio = GET_LIO(droq->napi.dev);
-        struct octeon_device *oct = lio->oct_dev;
-
-        work_done = octeon_process_droq_poll_cmd(oct, droq->q_no,
-                                                 POLL_EVENT_PROCESS_PKTS,
-                                                 budget);
-        if (work_done < 0) {
-                netif_info(lio, rx_err, lio->netdev,
-                           "Receive work_done < 0, rxq:%d\n", droq->q_no);
-                goto octnet_napi_finish;
-        }
-
-        if (work_done > budget)
-                dev_err(&oct->pci_dev->dev, ">>>> %s work_done: %d budget: %d\n",
-                        __func__, work_done, budget);
-
-        return work_done;
-
-octnet_napi_finish:
-        napi_complete(&droq->napi);
-        octeon_process_droq_poll_cmd(oct, droq->q_no, POLL_EVENT_ENABLE_INTR,
-                                     0);
-        return 0;
-}
-
-/**
  * \brief Entry point for NAPI polling
  * @param napi NAPI structure
  * @param budget maximum number of items to process
@@ -1910,19 +2080,41 @@ static int liquidio_napi_poll(struct napi_struct *napi, int budget)
 {
         struct octeon_droq *droq;
         int work_done;
+        int tx_done = 0, iq_no;
+        struct octeon_instr_queue *iq;
+        struct octeon_device *oct;
 
         droq = container_of(napi, struct octeon_droq, napi);
+        oct = droq->oct_dev;
+        iq_no = droq->q_no;
+        /* Handle Droq descriptors */
+        work_done = octeon_process_droq_poll_cmd(oct, droq->q_no,
+                                                 POLL_EVENT_PROCESS_PKTS,
+                                                 budget);
 
-        work_done = liquidio_napi_do_rx(droq, budget);
+        /* Flush the instruction queue */
+        iq = oct->instr_queue[iq_no];
+        if (iq) {
+                /* Process iq buffers with in the budget limits */
+                tx_done = octeon_flush_iq(oct, iq, 1, budget);
+                /* Update iq read-index rather than waiting for next interrupt.
+                 * Return back if tx_done is false.
+                 */
+                update_txq_status(oct, iq_no);
+                /*tx_done = (iq->flush_index == iq->octeon_read_index);*/
+        } else {
+                dev_err(&oct->pci_dev->dev, "%s:  iq (%d) num invalid\n",
+                        __func__, iq_no);
+        }
 
-        if (work_done < budget) {
+        if ((work_done < budget) && (tx_done)) {
                 napi_complete(napi);
                 octeon_process_droq_poll_cmd(droq->oct_dev, droq->q_no,
                                              POLL_EVENT_ENABLE_INTR, 0);
                 return 0;
         }
 
-        return work_done;
+        return (!tx_done) ? (budget) : (work_done);
 }
 
 /**
@@ -1935,10 +2127,10 @@ static int liquidio_napi_poll(struct napi_struct *napi, int budget)
  * are for ingress packets.
  */
 static inline int setup_io_queues(struct octeon_device *octeon_dev,
-                                  struct net_device *net_device)
+                                  int ifidx)
 {
-        static int first_time = 1;
-        static struct octeon_droq_ops droq_ops;
+        struct octeon_droq_ops droq_ops;
+        struct net_device *netdev;
         static int cpu_id;
         static int cpu_id_modulus;
         struct octeon_droq *droq;
@@ -1947,23 +2139,26 @@ static inline int setup_io_queues(struct octeon_device *octeon_dev,
         struct lio *lio;
         int num_tx_descs;
 
-        lio = GET_LIO(net_device);
-        if (first_time) {
-                first_time = 0;
-                memset(&droq_ops, 0, sizeof(struct octeon_droq_ops));
+        netdev = octeon_dev->props[ifidx].netdev;
+
+        lio = GET_LIO(netdev);
 
-                droq_ops.fptr = liquidio_push_packet;
+        memset(&droq_ops, 0, sizeof(struct octeon_droq_ops));
 
-                droq_ops.poll_mode = 1;
-                droq_ops.napi_fn = liquidio_napi_drv_callback;
-                cpu_id = 0;
-                cpu_id_modulus = num_present_cpus();
-        }
+        droq_ops.fptr = liquidio_push_packet;
+        droq_ops.farg = (void *)netdev;
+
+        droq_ops.poll_mode = 1;
+        droq_ops.napi_fn = liquidio_napi_drv_callback;
+        cpu_id = 0;
+        cpu_id_modulus = num_present_cpus();
 
         /* set up DROQs. */
         for (q = 0; q < lio->linfo.num_rxpciq; q++) {
-                q_no = lio->linfo.rxpciq[q];
-
+                q_no = lio->linfo.rxpciq[q].s.q_no;
+                dev_dbg(&octeon_dev->pci_dev->dev,
+                        "setup_io_queues index:%d linfo.rxpciq.s.q_no:%d\n",
+                        q, q_no);
                 retval = octeon_setup_droq(octeon_dev, q_no,
                                            CFG_GET_NUM_RX_DESCS_NIC_IF
                                                    (octeon_get_conf(octeon_dev),
@@ -1980,7 +2175,11 @@ static inline int setup_io_queues(struct octeon_device *octeon_dev,
 
                 droq = octeon_dev->droq[q_no];
                 napi = &droq->napi;
-                netif_napi_add(net_device, napi, liquidio_napi_poll, 64);
+                dev_dbg(&octeon_dev->pci_dev->dev,
+                        "netif_napi_add netdev:%llx oct:%llx\n",
+                        (u64)netdev,
+                        (u64)octeon_dev);
+                netif_napi_add(netdev, napi, liquidio_napi_poll, 64);
 
                 /* designate a CPU for this droq */
                 droq->cpu_id = cpu_id;
@@ -1996,9 +2195,9 @@ static inline int setup_io_queues(struct octeon_device *octeon_dev,
                 num_tx_descs = CFG_GET_NUM_TX_DESCS_NIC_IF(octeon_get_conf
                                                            (octeon_dev),
                                                            lio->ifidx);
-                retval = octeon_setup_iq(octeon_dev, lio->linfo.txpciq[q],
-                                         num_tx_descs,
-                                         netdev_get_tx_queue(net_device, q));
+                retval = octeon_setup_iq(octeon_dev, ifidx, q,
+                                         lio->linfo.txpciq[q], num_tx_descs,
+                                         netdev_get_tx_queue(netdev, q));
                 if (retval) {
                         dev_err(&octeon_dev->pci_dev->dev,
                                 " %s : Runtime IQ(TxQ) creation failed.\n",
@@ -2036,7 +2235,8 @@ static inline void setup_tx_poll_fn(struct net_device *netdev)
         struct lio *lio = GET_LIO(netdev);
         struct octeon_device *oct = lio->oct_dev;
 
-        lio->txq_status_wq.wq = create_workqueue("txq-status");
+        lio->txq_status_wq.wq = alloc_workqueue("txq-status",
+                                                WQ_MEM_RECLAIM, 0);
         if (!lio->txq_status_wq.wq) {
                 dev_err(&oct->pci_dev->dev, "unable to create cavium txq status wq\n");
                 return;
@@ -2048,6 +2248,14 @@ static inline void setup_tx_poll_fn(struct net_device *netdev)
                            &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
 }
 
+static inline void cleanup_tx_poll_fn(struct net_device *netdev)
+{
+        struct lio *lio = GET_LIO(netdev);
+
+        cancel_delayed_work_sync(&lio->txq_status_wq.wk.work);
+        destroy_workqueue(lio->txq_status_wq.wq);
+}
+
 /**
  * \brief Net device open for LiquidIO
  * @param netdev network device
@@ -2058,17 +2266,22 @@ static int liquidio_open(struct net_device *netdev)
         struct octeon_device *oct = lio->oct_dev;
         struct napi_struct *napi, *n;
 
-        list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
-                napi_enable(napi);
+        if (oct->props[lio->ifidx].napi_enabled == 0) {
+                list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
+                        napi_enable(napi);
+
+                oct->props[lio->ifidx].napi_enabled = 1;
+        }
 
         oct_ptp_open(netdev);
 
         ifstate_set(lio, LIO_IFSTATE_RUNNING);
+
         setup_tx_poll_fn(netdev);
+
         start_txq(netdev);
 
         netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n");
-        try_module_get(THIS_MODULE);
 
         /* tell Octeon to start forwarding packets to host */
         send_rx_ctrl_cmd(lio, 1);
@@ -2088,39 +2301,35 @@ static int liquidio_open(struct net_device *netdev)
  */
 static int liquidio_stop(struct net_device *netdev)
 {
-        struct napi_struct *napi, *n;
         struct lio *lio = GET_LIO(netdev);
         struct octeon_device *oct = lio->oct_dev;
 
-        netif_info(lio, ifdown, lio->netdev, "Stopping interface!\n");
+        ifstate_reset(lio, LIO_IFSTATE_RUNNING);
+
+        netif_tx_disable(netdev);
+
         /* Inform that netif carrier is down */
+        netif_carrier_off(netdev);
         lio->intf_open = 0;
-        lio->linfo.link.s.status = 0;
+        lio->linfo.link.s.link_up = 0;
+        lio->link_changes++;
 
-        netif_carrier_off(netdev);
+        /* Pause for a moment and wait for Octeon to flush out (to the wire) any
+         * egress packets that are in-flight.
+         */
+        set_current_state(TASK_INTERRUPTIBLE);
+        schedule_timeout(msecs_to_jiffies(100));
 
-        /* tell Octeon to stop forwarding packets to host */
+        /* Now it should be safe to tell Octeon that nic interface is down. */
         send_rx_ctrl_cmd(lio, 0);
 
-        cancel_delayed_work_sync(&lio->txq_status_wq.wk.work);
-        flush_workqueue(lio->txq_status_wq.wq);
-        destroy_workqueue(lio->txq_status_wq.wq);
+        cleanup_tx_poll_fn(netdev);
 
         if (lio->ptp_clock) {
                 ptp_clock_unregister(lio->ptp_clock);
                 lio->ptp_clock = NULL;
         }
 
-        ifstate_reset(lio, LIO_IFSTATE_RUNNING);
-
-        /* This is a hack that allows DHCP to continue working. */
-        set_bit(__LINK_STATE_START, &lio->netdev->state);
-
-        list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
-                napi_disable(napi);
-
-        txqs_stop(netdev);
-
         dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name);
         module_put(THIS_MODULE);
 
@@ -2181,12 +2390,31 @@ void liquidio_link_ctrl_cmd_completion(void *nctrl_ptr)
                          netdev->name);
                 break;
 
+        case OCTNET_CMD_ENABLE_VLAN_FILTER:
+                dev_info(&oct->pci_dev->dev, "%s VLAN filter enabled\n",
+                         netdev->name);
+                break;
+
+        case OCTNET_CMD_ADD_VLAN_FILTER:
+                dev_info(&oct->pci_dev->dev, "%s VLAN filter %d added\n",
+                         netdev->name, nctrl->ncmd.s.param1);
+                break;
+
+        case OCTNET_CMD_DEL_VLAN_FILTER:
+                dev_info(&oct->pci_dev->dev, "%s VLAN filter %d removed\n",
+                         netdev->name, nctrl->ncmd.s.param1);
+                break;
+
         case OCTNET_CMD_SET_SETTINGS:
                 dev_info(&oct->pci_dev->dev, "%s settings changed\n",
                          netdev->name);
 
                 break;
 
+        case OCTNET_CMD_SET_FLOW_CTL:
+                netif_info(lio, probe, lio->netdev, "Set RX/TX flow control parameters\n");
+                break;
+
         default:
                 dev_err(&oct->pci_dev->dev, "%s Unknown cmd %d\n", __func__,
                         nctrl->ncmd.s.cmd);
@@ -2235,7 +2463,6 @@ static void liquidio_set_mcast_list(struct net_device *netdev)
         struct lio *lio = GET_LIO(netdev);
         struct octeon_device *oct = lio->oct_dev;
         struct octnic_ctrl_pkt nctrl;
-        struct octnic_ctrl_params nparams;
         struct netdev_hw_addr *ha;
         u64 *mc;
         int ret, i;
@@ -2246,10 +2473,10 @@ static void liquidio_set_mcast_list(struct net_device *netdev)
         /* Create a ctrl pkt command to be sent to core app. */
         nctrl.ncmd.u64 = 0;
         nctrl.ncmd.s.cmd = OCTNET_CMD_SET_MULTI_LIST;
-        nctrl.ncmd.s.param1 = lio->linfo.ifidx;
-        nctrl.ncmd.s.param2 = get_new_flags(netdev);
-        nctrl.ncmd.s.param3 = mc_count;
+        nctrl.ncmd.s.param1 = get_new_flags(netdev);
+        nctrl.ncmd.s.param2 = mc_count;
         nctrl.ncmd.s.more = mc_count;
+        nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
         nctrl.netpndev = (u64)netdev;
         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
 
@@ -2270,9 +2497,7 @@ static void liquidio_set_mcast_list(struct net_device *netdev)
          */
         nctrl.wait_time = 0;
 
-        nparams.resp_order = OCTEON_RESP_NORESPONSE;
-
-        ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams);
+        ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
         if (ret < 0) {
                 dev_err(&oct->pci_dev->dev, "DEVFLAGS change failed in core (ret: 0x%x)\n",
                         ret);
@@ -2290,19 +2515,17 @@ static int liquidio_set_mac(struct net_device *netdev, void *p)
         struct octeon_device *oct = lio->oct_dev;
         struct sockaddr *addr = (struct sockaddr *)p;
         struct octnic_ctrl_pkt nctrl;
-        struct octnic_ctrl_params nparams;
 
-        if ((!is_valid_ether_addr(addr->sa_data)) ||
-            (ifstate_check(lio, LIO_IFSTATE_RUNNING)))
+        if (!is_valid_ether_addr(addr->sa_data))
                 return -EADDRNOTAVAIL;
 
         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
 
         nctrl.ncmd.u64 = 0;
         nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;
-        nctrl.ncmd.s.param1 = lio->linfo.ifidx;
-        nctrl.ncmd.s.param2 = 0;
+        nctrl.ncmd.s.param1 = 0;
         nctrl.ncmd.s.more = 1;
+        nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
         nctrl.netpndev = (u64)netdev;
         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
         nctrl.wait_time = 100;
@@ -2311,9 +2534,7 @@ static int liquidio_set_mac(struct net_device *netdev, void *p)
         /* The MAC Address is presented in network byte order. */
         memcpy((u8 *)&nctrl.udd[0] + 2, addr->sa_data, ETH_ALEN);
 
-        nparams.resp_order = OCTEON_RESP_ORDERED;
-
-        ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams);
+        ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
         if (ret < 0) {
                 dev_err(&oct->pci_dev->dev, "MAC Address change failed\n");
                 return -ENOMEM;
@@ -2341,7 +2562,7 @@ static struct net_device_stats *liquidio_get_stats(struct net_device *netdev)
         oct = lio->oct_dev;
 
         for (i = 0; i < lio->linfo.num_txpciq; i++) {
-                iq_no = lio->linfo.txpciq[i];
+                iq_no = lio->linfo.txpciq[i].s.q_no;
                 iq_stats = &oct->instr_queue[iq_no]->stats;
                 pkts += iq_stats->tx_done;
                 drop += iq_stats->tx_dropped;
@@ -2357,7 +2578,7 @@ static struct net_device_stats *liquidio_get_stats(struct net_device *netdev)
         bytes = 0;
 
         for (i = 0; i < lio->linfo.num_rxpciq; i++) {
-                oq_no = lio->linfo.rxpciq[i];
+                oq_no = lio->linfo.rxpciq[i].s.q_no;
                 oq_stats = &oct->droq[oq_no]->stats;
                 pkts += oq_stats->rx_pkts_received;
                 drop += (oq_stats->rx_dropped +
@@ -2383,7 +2604,6 @@ static int liquidio_change_mtu(struct net_device *netdev, int new_mtu)
         struct lio *lio = GET_LIO(netdev);
         struct octeon_device *oct = lio->oct_dev;
         struct octnic_ctrl_pkt nctrl;
-        struct octnic_ctrl_params nparams;
         int max_frm_size = new_mtu + OCTNET_FRM_HEADER_SIZE;
         int ret = 0;
 
@@ -2403,15 +2623,13 @@ static int liquidio_change_mtu(struct net_device *netdev, int new_mtu)
 
         nctrl.ncmd.u64 = 0;
         nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MTU;
-        nctrl.ncmd.s.param1 = lio->linfo.ifidx;
-        nctrl.ncmd.s.param2 = new_mtu;
+        nctrl.ncmd.s.param1 = new_mtu;
+        nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
         nctrl.wait_time = 100;
         nctrl.netpndev = (u64)netdev;
         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
 
-        nparams.resp_order = OCTEON_RESP_ORDERED;
-
-        ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams);
+        ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
         if (ret < 0) {
                 dev_err(&oct->pci_dev->dev, "Failed to set MTU\n");
                 return -1;
@@ -2536,7 +2754,7 @@ static void handle_timestamp(struct octeon_device *oct,
         }
 
         octeon_free_soft_command(oct, sc);
-        recv_buffer_free(skb);
+        tx_buffer_free(skb);
 }
 
 /* \brief Send a data packet that will be timestamped
@@ -2551,10 +2769,9 @@ static inline int send_nic_timestamp_pkt(struct octeon_device *oct,
 {
         int retval;
         struct octeon_soft_command *sc;
-        struct octeon_instr_ih *ih;
-        struct octeon_instr_rdp *rdp;
         struct lio *lio;
         int ring_doorbell;
+        u32 len;
 
         lio = finfo->lio;
 
@@ -2576,14 +2793,13 @@ static inline int send_nic_timestamp_pkt(struct octeon_device *oct,
         sc->callback_arg = finfo->skb;
         sc->iq_no = ndata->q_no;
 
-        ih = (struct octeon_instr_ih *)&sc->cmd.ih;
-        rdp = (struct octeon_instr_rdp *)&sc->cmd.rdp;
+        len = (u32)((struct octeon_instr_ih2 *)(&sc->cmd.cmd2.ih2))->dlengsz;
 
         ring_doorbell = !xmit_more;
         retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd,
-                                     sc, ih->dlengsz, ndata->reqtype);
+                                     sc, len, ndata->reqtype);
 
-        if (retval) {
+        if (retval == IQ_SEND_FAILED) {
                 dev_err(&oct->pci_dev->dev, "timestamp data packet failed status: %x\n",
                         retval);
                 octeon_free_soft_command(oct, sc);
@@ -2594,68 +2810,6 @@ static inline int send_nic_timestamp_pkt(struct octeon_device *oct,
         return retval;
 }
 
-static inline int is_ipv4(struct sk_buff *skb)
-{
-        return (skb->protocol == htons(ETH_P_IP)) &&
-               (ip_hdr(skb)->version == 4);
-}
-
-static inline int is_vlan(struct sk_buff *skb)
-{
-        return skb->protocol == htons(ETH_P_8021Q);
-}
-
-static inline int is_ip_fragmented(struct sk_buff *skb)
-{
-        /* The Don't fragment and Reserved flag fields are ignored.
-         * IP is fragmented if
-         * -  the More fragments bit is set (indicating this IP is a fragment
-         * with more to follow; the current offset could be 0 ).
-         * -  ths offset field is non-zero.
-         */
-        return (ip_hdr(skb)->frag_off & htons(IP_MF | IP_OFFSET)) ? 1 : 0;
-}
-
-static inline int is_ipv6(struct sk_buff *skb)
-{
-        return (skb->protocol == htons(ETH_P_IPV6)) &&
-               (ipv6_hdr(skb)->version == 6);
-}
-
-static inline int is_with_extn_hdr(struct sk_buff *skb)
-{
-        return (ipv6_hdr(skb)->nexthdr != IPPROTO_TCP) &&
-               (ipv6_hdr(skb)->nexthdr != IPPROTO_UDP);
-}
-
-static inline int is_tcpudp(struct sk_buff *skb)
-{
-        return (ip_hdr(skb)->protocol == IPPROTO_TCP) ||
-               (ip_hdr(skb)->protocol == IPPROTO_UDP);
-}
-
-static inline u32 get_ipv4_5tuple_tag(struct sk_buff *skb)
-{
-        u32 tag;
-        struct iphdr *iphdr = ip_hdr(skb);
-
-        tag = crc32(0, &iphdr->protocol, 1);
-        tag = crc32(tag, (u8 *)&iphdr->saddr, 8);
-        tag = crc32(tag, skb_transport_header(skb), 4);
-        return tag;
-}
-
-static inline u32 get_ipv6_5tuple_tag(struct sk_buff *skb)
-{
-        u32 tag;
-        struct ipv6hdr *ipv6hdr = ipv6_hdr(skb);
-
-        tag = crc32(0, &ipv6hdr->nexthdr, 1);
-        tag = crc32(tag, (u8 *)&ipv6hdr->saddr, 32);
-        tag = crc32(tag, skb_transport_header(skb), 4);
-        return tag;
-}
-
 /** \brief Transmit networks packets to the Octeon interface
  * @param skbuff   skbuff struct to be passed to network layer.
  * @param netdev    pointer to network device
@@ -2670,18 +2824,22 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
         struct octnic_data_pkt ndata;
         struct octeon_device *oct;
         struct oct_iq_stats *stats;
-        int cpu = 0, status = 0;
+        struct octeon_instr_irh *irh;
+        union tx_info *tx_info;
+        int status = 0;
         int q_idx = 0, iq_no = 0;
-        int xmit_more;
+        int xmit_more, j;
+        u64 dptr = 0;
         u32 tag = 0;
 
         lio = GET_LIO(netdev);
         oct = lio->oct_dev;
 
         if (netif_is_multiqueue(netdev)) {
-                cpu = skb->queue_mapping;
-                q_idx = (cpu & (lio->linfo.num_txpciq - 1));
-                iq_no = lio->linfo.txpciq[q_idx];
+                q_idx = skb->queue_mapping;
+                q_idx = (q_idx % (lio->linfo.num_txpciq));
+                tag = q_idx;
+                iq_no = lio->linfo.txpciq[q_idx].s.q_no;
         } else {
                 iq_no = lio->txq;
         }
@@ -2692,11 +2850,11 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
          * transmitted.
          */
         if (!(atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) ||
-            (!lio->linfo.link.s.status) ||
+            (!lio->linfo.link.s.link_up) ||
             (skb->len <= 0)) {
                 netif_info(lio, tx_err, lio->netdev,
                            "Transmit failed link_status : %d\n",
-                           lio->linfo.link.s.status);
+                           lio->linfo.link.s.link_up);
                 goto lio_xmit_failed;
         }
 
@@ -2739,53 +2897,11 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
         ndata.datasize = skb->len;
 
         cmdsetup.u64 = 0;
-        cmdsetup.s.ifidx = lio->linfo.ifidx;
-
-        if (skb->ip_summed == CHECKSUM_PARTIAL) {
-                if (is_ipv4(skb) && !is_ip_fragmented(skb) && is_tcpudp(skb)) {
-                        tag = get_ipv4_5tuple_tag(skb);
-
-                        cmdsetup.s.cksum_offset = sizeof(struct ethhdr) + 1;
-
-                        if (ip_hdr(skb)->ihl > 5)
-                                cmdsetup.s.ipv4opts_ipv6exthdr =
-                                                OCT_PKT_PARAM_IPV4OPTS;
-
-                } else if (is_ipv6(skb)) {
-                        tag = get_ipv6_5tuple_tag(skb);
-
-                        cmdsetup.s.cksum_offset = sizeof(struct ethhdr) + 1;
-
-                        if (is_with_extn_hdr(skb))
-                                cmdsetup.s.ipv4opts_ipv6exthdr =
-                                                OCT_PKT_PARAM_IPV6EXTHDR;
+        cmdsetup.s.iq_no = iq_no;
 
-                } else if (is_vlan(skb)) {
-                        if (vlan_eth_hdr(skb)->h_vlan_encapsulated_proto
-                                == htons(ETH_P_IP) &&
-                                !is_ip_fragmented(skb) && is_tcpudp(skb)) {
-                                tag = get_ipv4_5tuple_tag(skb);
+        if (skb->ip_summed == CHECKSUM_PARTIAL)
+                cmdsetup.s.transport_csum = 1;
 
-                                cmdsetup.s.cksum_offset =
-                                        sizeof(struct vlan_ethhdr) + 1;
-
-                                if (ip_hdr(skb)->ihl > 5)
-                                        cmdsetup.s.ipv4opts_ipv6exthdr =
-                                                OCT_PKT_PARAM_IPV4OPTS;
-
-                        } else if (vlan_eth_hdr(skb)->h_vlan_encapsulated_proto
-                                == htons(ETH_P_IPV6)) {
-                                tag = get_ipv6_5tuple_tag(skb);
-
-                                cmdsetup.s.cksum_offset =
-                                        sizeof(struct vlan_ethhdr) + 1;
-
-                                if (is_with_extn_hdr(skb))
-                                        cmdsetup.s.ipv4opts_ipv6exthdr =
-                                                OCT_PKT_PARAM_IPV6EXTHDR;
-                        }
-                }
-        }
         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
                 cmdsetup.s.timestamp = 1;
@@ -2793,20 +2909,20 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
 
         if (skb_shinfo(skb)->nr_frags == 0) {
                 cmdsetup.s.u.datasize = skb->len;
-                octnet_prepare_pci_cmd(&ndata.cmd, &cmdsetup, tag);
+                octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
                 /* Offload checksum calculation for TCP/UDP packets */
-                ndata.cmd.dptr = dma_map_single(&oct->pci_dev->dev,
-                                                skb->data,
-                                                skb->len,
-                                                DMA_TO_DEVICE);
-                if (dma_mapping_error(&oct->pci_dev->dev, ndata.cmd.dptr)) {
+                dptr = dma_map_single(&oct->pci_dev->dev,
+                                      skb->data,
+                                      skb->len,
+                                      DMA_TO_DEVICE);
+                if (dma_mapping_error(&oct->pci_dev->dev, dptr)) {
                         dev_err(&oct->pci_dev->dev, "%s DMA mapping error 1\n",
                                 __func__);
                         return NETDEV_TX_BUSY;
                 }
 
-                finfo->dptr = ndata.cmd.dptr;
-
+                ndata.cmd.cmd2.dptr = dptr;
+                finfo->dptr = dptr;
                 ndata.reqtype = REQTYPE_NORESP_NET;
 
         } else {
@@ -2814,9 +2930,10 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
                 struct skb_frag_struct *frag;
                 struct octnic_gather *g;
 
-                spin_lock(&lio->lock);
-                g = (struct octnic_gather *)list_delete_head(&lio->glist);
-                spin_unlock(&lio->lock);
+                spin_lock(&lio->glist_lock[q_idx]);
+                g = (struct octnic_gather *)
+                        list_delete_head(&lio->glist[q_idx]);
+                spin_unlock(&lio->glist_lock[q_idx]);
 
                 if (!g) {
                         netif_info(lio, tx_err, lio->netdev,
@@ -2826,7 +2943,7 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
 
                 cmdsetup.s.gather = 1;
                 cmdsetup.s.u.gatherptrs = (skb_shinfo(skb)->nr_frags + 1);
-                octnet_prepare_pci_cmd(&ndata.cmd, &cmdsetup, tag);
+                octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
 
                 memset(g->sg, 0, g->sg_size);
 
@@ -2853,36 +2970,52 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
                                              frag->size,
                                              DMA_TO_DEVICE);
 
+                        if (dma_mapping_error(&oct->pci_dev->dev,
+                                              g->sg[i >> 2].ptr[i & 3])) {
+                                dma_unmap_single(&oct->pci_dev->dev,
+                                                 g->sg[0].ptr[0],
+                                                 skb->len - skb->data_len,
+                                                 DMA_TO_DEVICE);
+                                for (j = 1; j < i; j++) {
+                                        frag = &skb_shinfo(skb)->frags[j - 1];
+                                        dma_unmap_page(&oct->pci_dev->dev,
+                                                       g->sg[j >> 2].ptr[j & 3],
+                                                       frag->size,
+                                                       DMA_TO_DEVICE);
+                                }
+                                dev_err(&oct->pci_dev->dev, "%s DMA mapping error 3\n",
+                                        __func__);
+                                return NETDEV_TX_BUSY;
+                        }
+
                         add_sg_size(&g->sg[(i >> 2)], frag->size, (i & 3));
                         i++;
                 }
 
-                ndata.cmd.dptr = dma_map_single(&oct->pci_dev->dev,
-                                                g->sg, g->sg_size,
-                                                DMA_TO_DEVICE);
-                if (dma_mapping_error(&oct->pci_dev->dev, ndata.cmd.dptr)) {
-                        dev_err(&oct->pci_dev->dev, "%s DMA mapping error 3\n",
-                                __func__);
-                        dma_unmap_single(&oct->pci_dev->dev, g->sg[0].ptr[0],
-                                         skb->len - skb->data_len,
-                                         DMA_TO_DEVICE);
-                        return NETDEV_TX_BUSY;
-                }
+                dma_sync_single_for_device(&oct->pci_dev->dev, g->sg_dma_ptr,
+                                           g->sg_size, DMA_TO_DEVICE);
+                dptr = g->sg_dma_ptr;
 
-                finfo->dptr = ndata.cmd.dptr;
+                ndata.cmd.cmd2.dptr = dptr;
+                finfo->dptr = dptr;
                 finfo->g = g;
 
                 ndata.reqtype = REQTYPE_NORESP_NET_SG;
         }
 
-        if (skb_shinfo(skb)->gso_size) {
-                struct octeon_instr_irh *irh =
-                        (struct octeon_instr_irh *)&ndata.cmd.irh;
-                union tx_info *tx_info = (union tx_info *)&ndata.cmd.ossp[0];
+        irh = (struct octeon_instr_irh *)&ndata.cmd.cmd2.irh;
+        tx_info = (union tx_info *)&ndata.cmd.cmd2.ossp[0];
 
-                irh->len = 1;   /* to indicate that ossp[0] contains tx_info */
+        if (skb_shinfo(skb)->gso_size) {
                 tx_info->s.gso_size = skb_shinfo(skb)->gso_size;
                 tx_info->s.gso_segs = skb_shinfo(skb)->gso_segs;
+                stats->tx_gso++;
+        }
+
+        /* HW insert VLAN tag */
+        if (skb_vlan_tag_present(skb)) {
+                irh->priority = skb_vlan_tag_get(skb) >> 13;
+                irh->vlan = skb_vlan_tag_get(skb) & 0xfff;
         }
 
         xmit_more = skb->xmit_more;
@@ -2901,7 +3034,10 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
 
         netif_trans_update(netdev);
 
-        stats->tx_done++;
+        if (skb_shinfo(skb)->gso_size)
+                stats->tx_done += skb_shinfo(skb)->gso_segs;
+        else
+                stats->tx_done++;
         stats->tx_tot_bytes += skb->len;
 
         return NETDEV_TX_OK;
@@ -2910,9 +3046,10 @@ lio_xmit_failed:
         stats->tx_dropped++;
         netif_info(lio, tx_err, lio->netdev, "IQ%d Transmit dropped:%llu\n",
                    iq_no, stats->tx_dropped);
-        dma_unmap_single(&oct->pci_dev->dev, ndata.cmd.dptr,
-                         ndata.datasize, DMA_TO_DEVICE);
-        recv_buffer_free(skb);
+        if (dptr)
+                dma_unmap_single(&oct->pci_dev->dev, dptr,
+                                 ndata.datasize, DMA_TO_DEVICE);
+        tx_buffer_free(skb);
         return NETDEV_TX_OK;
 }
 
@@ -2932,27 +3069,79 @@ static void liquidio_tx_timeout(struct net_device *netdev)
         txqs_wake(netdev);
 }
 
-int liquidio_set_feature(struct net_device *netdev, int cmd)
+static int liquidio_vlan_rx_add_vid(struct net_device *netdev,
+                                    __be16 proto __attribute__((unused)),
+                                    u16 vid)
 {
         struct lio *lio = GET_LIO(netdev);
         struct octeon_device *oct = lio->oct_dev;
         struct octnic_ctrl_pkt nctrl;
-        struct octnic_ctrl_params nparams;
         int ret = 0;
 
         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
 
         nctrl.ncmd.u64 = 0;
-        nctrl.ncmd.s.cmd = cmd;
-        nctrl.ncmd.s.param1 = lio->linfo.ifidx;
-        nctrl.ncmd.s.param2 = OCTNIC_LROIPV4 | OCTNIC_LROIPV6;
+        nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER;
+        nctrl.ncmd.s.param1 = vid;
+        nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
+        nctrl.wait_time = 100;
+        nctrl.netpndev = (u64)netdev;
+        nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
+
+        ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
+        if (ret < 0) {
+                dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n",
+                        ret);
+        }
+
+        return ret;
+}
+
+static int liquidio_vlan_rx_kill_vid(struct net_device *netdev,
+                                     __be16 proto __attribute__((unused)),
+                                     u16 vid)
+{
+        struct lio *lio = GET_LIO(netdev);
+        struct octeon_device *oct = lio->oct_dev;
+        struct octnic_ctrl_pkt nctrl;
+        int ret = 0;
+
+        memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
+
+        nctrl.ncmd.u64 = 0;
+        nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER;
+        nctrl.ncmd.s.param1 = vid;
+        nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
         nctrl.wait_time = 100;
         nctrl.netpndev = (u64)netdev;
         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
 
-        nparams.resp_order = OCTEON_RESP_NORESPONSE;
+        ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
+        if (ret < 0) {
+                dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n",
+                        ret);
+        }
+        return ret;
+}
+
+int liquidio_set_feature(struct net_device *netdev, int cmd, u16 param1)
+{
+        struct lio *lio = GET_LIO(netdev);
+        struct octeon_device *oct = lio->oct_dev;
+        struct octnic_ctrl_pkt nctrl;
+        int ret = 0;
+
+        memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
+
+        nctrl.ncmd.u64 = 0;
+        nctrl.ncmd.s.cmd = cmd;
+        nctrl.ncmd.s.param1 = param1;
+        nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
+        nctrl.wait_time = 100;
+        nctrl.netpndev = (u64)netdev;
+        nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
 
-        ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams);
+        ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
         if (ret < 0) {
                 dev_err(&oct->pci_dev->dev, "Feature change failed in core (ret: 0x%x)\n",
                         ret);
@@ -3008,10 +3197,12 @@ static int liquidio_set_features(struct net_device *netdev,
                 return 0;
 
         if ((features & NETIF_F_LRO) && (lio->dev_capability & NETIF_F_LRO))
-                liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE);
+                liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
+                                     OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
         else if (!(features & NETIF_F_LRO) &&
                  (lio->dev_capability & NETIF_F_LRO))
-                liquidio_set_feature(netdev, OCTNET_CMD_LRO_DISABLE);
+                liquidio_set_feature(netdev, OCTNET_CMD_LRO_DISABLE,
+                                     OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
 
         return 0;
 }
@@ -3024,6 +3215,9 @@ static struct net_device_ops lionetdevops = {
         .ndo_set_mac_address    = liquidio_set_mac,
         .ndo_set_rx_mode        = liquidio_set_mcast_list,
         .ndo_tx_timeout         = liquidio_tx_timeout,
+
+        .ndo_vlan_rx_add_vid    = liquidio_vlan_rx_add_vid,
+        .ndo_vlan_rx_kill_vid   = liquidio_vlan_rx_kill_vid,
         .ndo_change_mtu         = liquidio_change_mtu,
         .ndo_do_ioctl           = liquidio_ioctl,
         .ndo_fix_features       = liquidio_fix_features,
@@ -3082,24 +3276,27 @@ static int lio_nic_info(struct octeon_recv_info *recv_info, void *buf)
 {
         struct octeon_device *oct = (struct octeon_device *)buf;
         struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;
-        int ifidx = 0;
+        int gmxport = 0;
         union oct_link_status *ls;
         int i;
 
-        if ((recv_pkt->buffer_size[0] != sizeof(*ls)) ||
-            (recv_pkt->rh.r_nic_info.ifidx > oct->ifcount)) {
+        if (recv_pkt->buffer_size[0] != sizeof(*ls)) {
                 dev_err(&oct->pci_dev->dev, "Malformed NIC_INFO, len=%d, ifidx=%d\n",
                         recv_pkt->buffer_size[0],
-                        recv_pkt->rh.r_nic_info.ifidx);
+                        recv_pkt->rh.r_nic_info.gmxport);
                 goto nic_info_err;
         }
 
-        ifidx = recv_pkt->rh.r_nic_info.ifidx;
+        gmxport = recv_pkt->rh.r_nic_info.gmxport;
         ls = (union oct_link_status *)get_rbd(recv_pkt->buffer_ptr[0]);
 
         octeon_swap_8B_data((u64 *)ls, (sizeof(union oct_link_status)) >> 3);
-
-        update_link_status(oct->props[ifidx].netdev, ls);
+        for (i = 0; i < oct->ifcount; i++) {
+                if (oct->props[i].gmxport == gmxport) {
+                        update_link_status(oct->props[i].netdev, ls);
+                        break;
+                }
+        }
 
 nic_info_err:
         for (i = 0; i < recv_pkt->buffer_count; i++)
@@ -3125,13 +3322,13 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
         struct liquidio_if_cfg_context *ctx;
         struct liquidio_if_cfg_resp *resp;
         struct octdev_props *props;
-        int retval, num_iqueues, num_oqueues, q_no;
-        u64 q_mask;
+        int retval, num_iqueues, num_oqueues;
         int num_cpus = num_online_cpus();
         union oct_nic_if_cfg if_cfg;
         unsigned int base_queue;
         unsigned int gmx_port_id;
         u32 resp_size, ctx_size;
+        u32 ifidx_or_pfnum;
 
         /* This is to handle link status changes */
         octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
@@ -3167,13 +3364,14 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
                         CFG_GET_BASE_QUE_NIC_IF(octeon_get_conf(octeon_dev), i);
                 gmx_port_id =
                         CFG_GET_GMXID_NIC_IF(octeon_get_conf(octeon_dev), i);
+                ifidx_or_pfnum = i;
                 if (num_iqueues > num_cpus)
                         num_iqueues = num_cpus;
                 if (num_oqueues > num_cpus)
                         num_oqueues = num_cpus;
                 dev_dbg(&octeon_dev->pci_dev->dev,
                         "requesting config for interface %d, iqs %d, oqs %d\n",
-                        i, num_iqueues, num_oqueues);
+                        ifidx_or_pfnum, num_iqueues, num_oqueues);
                 ACCESS_ONCE(ctx->cond) = 0;
                 ctx->octeon_id = lio_get_device_id(octeon_dev);
                 init_waitqueue_head(&ctx->wc);
@@ -3183,8 +3381,11 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
                 if_cfg.s.num_oqueues = num_oqueues;
                 if_cfg.s.base_queue = base_queue;
                 if_cfg.s.gmx_port_id = gmx_port_id;
+
+                sc->iq_no = 0;
+
                 octeon_prepare_soft_command(octeon_dev, sc, OPCODE_NIC,
-                                            OPCODE_NIC_IF_CFG, i,
+                                            OPCODE_NIC_IF_CFG, 0,
                                             if_cfg.u64, 0);
 
                 sc->callback = if_cfg_callback;
@@ -3192,7 +3393,7 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
                 sc->wait_time = 1000;
 
                 retval = octeon_send_soft_command(octeon_dev, sc);
-                if (retval) {
+                if (retval == IQ_SEND_FAILED) {
                         dev_err(&octeon_dev->pci_dev->dev,
                                 "iq/oq config failed status: %x\n",
                                 retval);
@@ -3234,8 +3435,7 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
                         goto setup_nic_dev_fail;
                 }
 
-                props = &octeon_dev->props[i];
-                props->netdev = netdev;
+                SET_NETDEV_DEV(netdev, &octeon_dev->pci_dev->dev);
 
                 if (num_iqueues > 1)
                         lionetdevops.ndo_select_queue = select_q;
@@ -3249,23 +3449,21 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
 
                 memset(lio, 0, sizeof(struct lio));
 
-                lio->linfo.ifidx = resp->cfg_info.ifidx;
-                lio->ifidx = resp->cfg_info.ifidx;
+                lio->ifidx = ifidx_or_pfnum;
+
+                props = &octeon_dev->props[i];
+                props->gmxport = resp->cfg_info.linfo.gmxport;
+                props->netdev = netdev;
 
                 lio->linfo.num_rxpciq = num_oqueues;
                 lio->linfo.num_txpciq = num_iqueues;
-                q_mask = resp->cfg_info.oqmask;
-                /* q_mask is 0-based and already verified mask is nonzero */
                 for (j = 0; j < num_oqueues; j++) {
-                        q_no = __ffs64(q_mask);
-                        q_mask &= (~(1UL << q_no));
-                        lio->linfo.rxpciq[j] = q_no;
+                        lio->linfo.rxpciq[j].u64 =
+                                resp->cfg_info.linfo.rxpciq[j].u64;
                 }
-                q_mask = resp->cfg_info.iqmask;
                 for (j = 0; j < num_iqueues; j++) {
-                        q_no = __ffs64(q_mask);
-                        q_mask &= (~(1UL << q_no));
-                        lio->linfo.txpciq[j] = q_no;
+                        lio->linfo.txpciq[j].u64 =
+                                resp->cfg_info.linfo.txpciq[j].u64;
                 }
                 lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr;
                 lio->linfo.gmxport = resp->cfg_info.linfo.gmxport;
@@ -3274,16 +3472,25 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
                 lio->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
 
                 lio->dev_capability = NETIF_F_HIGHDMA
-                                      | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM
-                                      | NETIF_F_SG | NETIF_F_RXCSUM
-                                      | NETIF_F_TSO | NETIF_F_TSO6
-                                      | NETIF_F_LRO;
+                                | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM
+                                | NETIF_F_SG | NETIF_F_RXCSUM
+                                | NETIF_F_GRO
+                                | NETIF_F_TSO | NETIF_F_TSO6
+                                | NETIF_F_LRO;
                 netif_set_gso_max_size(netdev, OCTNIC_GSO_MAX_SIZE);
 
-                netdev->features = lio->dev_capability;
                 netdev->vlan_features = lio->dev_capability;
+                /* Add any unchangeable hw features */
+                lio->dev_capability |=  NETIF_F_HW_VLAN_CTAG_FILTER |
+                                        NETIF_F_HW_VLAN_CTAG_RX |
+                                        NETIF_F_HW_VLAN_CTAG_TX;
+
+                netdev->features = (lio->dev_capability & ~NETIF_F_LRO);
 
                 netdev->hw_features = lio->dev_capability;
+                /*HW_VLAN_RX and HW_VLAN_FILTER is always on*/
+                netdev->hw_features = netdev->hw_features &
+                        ~NETIF_F_HW_VLAN_CTAG_RX;
 
                 /* Point to the  properties for octeon device to which this
                  * interface belongs.
@@ -3291,7 +3498,6 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
                 lio->oct_dev = octeon_dev;
                 lio->octprops = props;
                 lio->netdev = netdev;
-                spin_lock_init(&lio->lock);
 
                 dev_dbg(&octeon_dev->pci_dev->dev,
                         "if%d gmx: %d hw_addr: 0x%llx\n", i,
@@ -3306,23 +3512,22 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
 
                 ether_addr_copy(netdev->dev_addr, mac);
 
-                if (setup_io_queues(octeon_dev, netdev)) {
+                /* By default all interfaces on a single Octeon uses the same
+                 * tx and rx queues
+                 */
+                lio->txq = lio->linfo.txpciq[0].s.q_no;
+                lio->rxq = lio->linfo.rxpciq[0].s.q_no;
+                if (setup_io_queues(octeon_dev, i)) {
                         dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n");
                         goto setup_nic_dev_fail;
                 }
 
                 ifstate_set(lio, LIO_IFSTATE_DROQ_OPS);
 
-                /* By default all interfaces on a single Octeon uses the same
-                 * tx and rx queues
-                 */
-                lio->txq = lio->linfo.txpciq[0];
-                lio->rxq = lio->linfo.rxpciq[0];
-
                 lio->tx_qsize = octeon_get_tx_qsize(octeon_dev, lio->txq);
                 lio->rx_qsize = octeon_get_rx_qsize(octeon_dev, lio->rxq);
 
-                if (setup_glist(lio)) {
+                if (setup_glists(octeon_dev, lio, num_iqueues)) {
                         dev_err(&octeon_dev->pci_dev->dev,
                                 "Gather list allocation failed\n");
                         goto setup_nic_dev_fail;
@@ -3330,11 +3535,17 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
 
                 /* Register ethtool support */
                 liquidio_set_ethtool_ops(netdev);
+                octeon_dev->priv_flags = 0x0;
+
+                if (netdev->features & NETIF_F_LRO)
+                liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
+                                     OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
 
-                liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE);
+                liquidio_set_feature(netdev, OCTNET_CMD_ENABLE_VLAN_FILTER, 0);
 
                 if ((debug != -1) && (debug & NETIF_MSG_HW))
-                        liquidio_set_feature(netdev, OCTNET_CMD_VERBOSE_ENABLE);
+                        liquidio_set_feature(netdev,
+                                             OCTNET_CMD_VERBOSE_ENABLE, 0);
 
                 /* Register the network device with the OS */
                 if (register_netdev(netdev)) {
@@ -3346,13 +3557,7 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
                         "Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n",
                         i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
                 netif_carrier_off(netdev);
-
-                if (lio->linfo.link.s.status) {
-                        netif_carrier_on(netdev);
-                        start_txq(netdev);
-                } else {
-                        netif_carrier_off(netdev);
-                }
+                lio->link_changes++;
 
                 ifstate_set(lio, LIO_IFSTATE_REGISTERED);
 
@@ -3386,7 +3591,7 @@ setup_nic_dev_fail:
 static int liquidio_init_nic_module(struct octeon_device *oct)
 {
         struct oct_intrmod_cfg *intrmod_cfg;
-        int retval = 0;
+        int i, retval = 0;
         int num_nic_ports = CFG_GET_NUM_NIC_PORTS(octeon_get_conf(oct));
 
         dev_dbg(&oct->pci_dev->dev, "Initializing network interfaces\n");
@@ -3400,6 +3605,9 @@ static int liquidio_init_nic_module(struct octeon_device *oct)
         memset(oct->props, 0,
                sizeof(struct octdev_props) * num_nic_ports);
 
+        for (i = 0; i < MAX_OCTEON_LINKS; i++)
+                oct->props[i].gmxport = -1;
+
         retval = setup_nic_devices(oct);
         if (retval) {
                 dev_err(&oct->pci_dev->dev, "Setup NIC devices failed\n");
@@ -3410,15 +3618,19 @@ static int liquidio_init_nic_module(struct octeon_device *oct)
 
         /* Initialize interrupt moderation params */
         intrmod_cfg = &((struct octeon_device *)oct)->intrmod;
-        intrmod_cfg->intrmod_enable = 1;
-        intrmod_cfg->intrmod_check_intrvl = LIO_INTRMOD_CHECK_INTERVAL;
-        intrmod_cfg->intrmod_maxpkt_ratethr = LIO_INTRMOD_MAXPKT_RATETHR;
-        intrmod_cfg->intrmod_minpkt_ratethr = LIO_INTRMOD_MINPKT_RATETHR;
-        intrmod_cfg->intrmod_maxcnt_trigger = LIO_INTRMOD_MAXCNT_TRIGGER;
-        intrmod_cfg->intrmod_maxtmr_trigger = LIO_INTRMOD_MAXTMR_TRIGGER;
-        intrmod_cfg->intrmod_mintmr_trigger = LIO_INTRMOD_MINTMR_TRIGGER;
-        intrmod_cfg->intrmod_mincnt_trigger = LIO_INTRMOD_MINCNT_TRIGGER;
-
+        intrmod_cfg->rx_enable = 1;
+        intrmod_cfg->check_intrvl =   LIO_INTRMOD_CHECK_INTERVAL;
+        intrmod_cfg->maxpkt_ratethr = LIO_INTRMOD_MAXPKT_RATETHR;
+        intrmod_cfg->minpkt_ratethr = LIO_INTRMOD_MINPKT_RATETHR;
+        intrmod_cfg->rx_maxcnt_trigger = LIO_INTRMOD_RXMAXCNT_TRIGGER;
+        intrmod_cfg->rx_maxtmr_trigger = LIO_INTRMOD_RXMAXTMR_TRIGGER;
+        intrmod_cfg->rx_mintmr_trigger = LIO_INTRMOD_RXMINTMR_TRIGGER;
+        intrmod_cfg->rx_mincnt_trigger = LIO_INTRMOD_RXMINCNT_TRIGGER;
+        intrmod_cfg->tx_enable = 1;
+        intrmod_cfg->tx_maxcnt_trigger = LIO_INTRMOD_TXMAXCNT_TRIGGER;
+        intrmod_cfg->tx_mincnt_trigger = LIO_INTRMOD_TXMINCNT_TRIGGER;
+        intrmod_cfg->rx_frames = CFG_GET_OQ_INTR_PKT(octeon_get_conf(oct));
+        intrmod_cfg->rx_usecs = CFG_GET_OQ_INTR_TIME(octeon_get_conf(oct));
         dev_dbg(&oct->pci_dev->dev, "Network interfaces ready\n");
 
         return retval;
@@ -3481,6 +3693,7 @@ static void nic_starter(struct work_struct *work)
 static int octeon_device_init(struct octeon_device *octeon_dev)
 {
         int j, ret;
+        char bootcmd[] = "\n";
         struct octeon_device_priv *oct_priv =
                 (struct octeon_device_priv *)octeon_dev->priv;
         atomic_set(&octeon_dev->status, OCT_DEV_BEGIN_STATE);
@@ -3592,14 +3805,19 @@ static int octeon_device_init(struct octeon_device *octeon_dev)
 
         dev_dbg(&octeon_dev->pci_dev->dev, "Waiting for DDR initialization...\n");
 
-        if (ddr_timeout == 0) {
-                dev_info(&octeon_dev->pci_dev->dev,
-                         "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n");
-        }
+        if (ddr_timeout == 0)
+                dev_info(&octeon_dev->pci_dev->dev, "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n");
 
         schedule_timeout_uninterruptible(HZ * LIO_RESET_SECS);
 
         /* Wait for the octeon to initialize DDR after the soft-reset. */
+        while (ddr_timeout == 0) {
+                set_current_state(TASK_INTERRUPTIBLE);
+                if (schedule_timeout(HZ / 10)) {
+                        /* user probably pressed Control-C */
+                        return 1;
+                }
+        }
         ret = octeon_wait_for_ddr_init(octeon_dev, &ddr_timeout);
         if (ret) {
                 dev_err(&octeon_dev->pci_dev->dev,
@@ -3613,6 +3831,9 @@ static int octeon_device_init(struct octeon_device *octeon_dev)
                 return 1;
         }
 
+        /* Divert uboot to take commands from host instead. */
+        ret = octeon_console_send_cmd(octeon_dev, bootcmd, 50);
+
         dev_dbg(&octeon_dev->pci_dev->dev, "Initializing consoles\n");
         ret = octeon_init_consoles(octeon_dev);
         if (ret) {
diff --git a/drivers/net/ethernet/cavium/liquidio/liquidio_common.h b/drivers/net/ethernet/cavium/liquidio/liquidio_common.h
index 0ac347ccc8ba..5aa01f427d4a 100644
--- a/drivers/net/ethernet/cavium/liquidio/liquidio_common.h
+++ b/drivers/net/ethernet/cavium/liquidio/liquidio_common.h
@@ -30,11 +30,10 @@
 
 #include "octeon_config.h"
 
-#define LIQUIDIO_VERSION        "1.1.9"
-#define LIQUIDIO_MAJOR_VERSION  1
-#define LIQUIDIO_MINOR_VERSION  1
-#define LIQUIDIO_MICRO_VERSION  9
-
+#define LIQUIDIO_BASE_VERSION   "1.4"
+#define LIQUIDIO_MICRO_VERSION  ".1"
+#define LIQUIDIO_PACKAGE ""
+#define LIQUIDIO_VERSION  "1.4.1"
 #define CONTROL_IQ 0
 /** Tag types used by Octeon cores in its work. */
 enum octeon_tag_type {
@@ -174,9 +173,11 @@ static inline void add_sg_size(struct octeon_sg_entry *sg_entry,
 /*------------------------- End Scatter/Gather ---------------------------*/
 
 #define   OCTNET_FRM_PTP_HEADER_SIZE  8
-#define   OCTNET_FRM_HEADER_SIZE     30 /* PTP timestamp + VLAN + Ethernet */
 
-#define   OCTNET_MIN_FRM_SIZE        (64  + OCTNET_FRM_PTP_HEADER_SIZE)
+#define   OCTNET_FRM_HEADER_SIZE     22 /* VLAN + Ethernet */
+
+#define   OCTNET_MIN_FRM_SIZE        64
+
 #define   OCTNET_MAX_FRM_SIZE        (16000 + OCTNET_FRM_HEADER_SIZE)
 
 #define   OCTNET_DEFAULT_FRM_SIZE    (1500 + OCTNET_FRM_HEADER_SIZE)
@@ -212,6 +213,10 @@ static inline void add_sg_size(struct octeon_sg_entry *sg_entry,
 #define   OCTNET_CMD_VERBOSE_ENABLE   0x14
 #define   OCTNET_CMD_VERBOSE_DISABLE  0x15
 
+#define   OCTNET_CMD_ENABLE_VLAN_FILTER 0x16
+#define   OCTNET_CMD_ADD_VLAN_FILTER  0x17
+#define   OCTNET_CMD_DEL_VLAN_FILTER  0x18
+
 /* RX(packets coming from wire) Checksum verification flags */
 /* TCP/UDP csum */
 #define   CNNIC_L4SUM_VERIFIED             0x1
@@ -258,19 +263,19 @@ union octnet_cmd {
 
                 u64 more:6; /* How many udd words follow the command */
 
-                u64 param1:29;
+                u64 reserved:29;
 
-                u64 param2:16;
+                u64 param1:16;
 
-                u64 param3:8;
+                u64 param2:8;
 
 #else
 
-                u64 param3:8;
+                u64 param2:8;
 
-                u64 param2:16;
+                u64 param1:16;
 
-                u64 param1:29;
+                u64 reserved:29;
 
                 u64 more:6;
 
@@ -283,8 +288,140 @@ union octnet_cmd {
 
 #define   OCTNET_CMD_SIZE     (sizeof(union octnet_cmd))
 
+/* Instruction Header (DPI - CN23xx) - for OCTEON-III models */
+struct  octeon_instr_ih3 {
+#ifdef __BIG_ENDIAN_BITFIELD
+
+        /** Reserved3 */
+        u64     reserved3:1;
+
+        /** Gather indicator 1=gather*/
+        u64     gather:1;
+
+        /** Data length OR no. of entries in gather list */
+        u64     dlengsz:14;
+
+        /** Front Data size */
+        u64     fsz:6;
+
+        /** Reserved2 */
+        u64     reserved2:4;
+
+        /** PKI port kind - PKIND */
+        u64     pkind:6;
+
+        /** Reserved1 */
+        u64     reserved1:32;
+
+#else
+        /** Reserved1 */
+        u64     reserved1:32;
+
+        /** PKI port kind - PKIND */
+        u64     pkind:6;
+
+        /** Reserved2 */
+        u64     reserved2:4;
+
+        /** Front Data size */
+        u64     fsz:6;
+
+        /** Data length OR no. of entries in gather list */
+        u64     dlengsz:14;
+
+        /** Gather indicator 1=gather*/
+        u64     gather:1;
+
+        /** Reserved3 */
+        u64     reserved3:1;
+
+#endif
+};
+
+/* Optional PKI Instruction Header(PKI IH) - for OCTEON CN23XX models */
+/** BIG ENDIAN format.   */
+struct  octeon_instr_pki_ih3 {
+#ifdef __BIG_ENDIAN_BITFIELD
+
+        /** Wider bit */
+        u64     w:1;
+
+        /** Raw mode indicator 1 = RAW */
+        u64     raw:1;
+
+        /** Use Tag */
+        u64     utag:1;
+
+        /** Use QPG */
+        u64     uqpg:1;
+
+        /** Reserved2 */
+        u64     reserved2:1;
+
+        /** Parse Mode */
+        u64     pm:3;
+
+        /** Skip Length */
+        u64     sl:8;
+
+        /** Use Tag Type */
+        u64     utt:1;
+
+        /** Tag type */
+        u64     tagtype:2;
+
+        /** Reserved1 */
+        u64     reserved1:2;
+
+        /** QPG Value */
+        u64     qpg:11;
+
+        /** Tag Value */
+        u64     tag:32;
+
+#else
+
+        /** Tag Value */
+        u64     tag:32;
+
+        /** QPG Value */
+        u64     qpg:11;
+
+        /** Reserved1 */
+        u64     reserved1:2;
+
+        /** Tag type */
+        u64     tagtype:2;
+
+        /** Use Tag Type */
+        u64     utt:1;
+
+        /** Skip Length */
+        u64     sl:8;
+
+        /** Parse Mode */
+        u64     pm:3;
+
+        /** Reserved2 */
+        u64     reserved2:1;
+
+        /** Use QPG */
+        u64     uqpg:1;
+
+        /** Use Tag */
+        u64     utag:1;
+
+        /** Raw mode indicator 1 = RAW */
+        u64     raw:1;
+
+        /** Wider bit */
+        u64     w:1;
+#endif
+
+};
+
 /** Instruction Header */
-struct octeon_instr_ih {
+struct octeon_instr_ih2 {
 #ifdef __BIG_ENDIAN_BITFIELD
         /** Raw mode indicator 1 = RAW */
         u64 raw:1;
@@ -348,15 +485,15 @@ struct octeon_instr_irh {
         u64 opcode:4;
         u64 rflag:1;
         u64 subcode:7;
-        u64 len:3;
-        u64 rid:13;
-        u64 reserved:4;
+        u64 vlan:12;
+        u64 priority:3;
+        u64 reserved:5;
         u64 ossp:32;             /* opcode/subcode specific parameters */
 #else
         u64 ossp:32;             /* opcode/subcode specific parameters */
-        u64 reserved:4;
-        u64 rid:13;
-        u64 len:3;
+        u64 reserved:5;
+        u64 priority:3;
+        u64 vlan:12;
         u64 subcode:7;
         u64 rflag:1;
         u64 opcode:4;
@@ -383,28 +520,27 @@ union octeon_rh {
         struct {
                 u64 opcode:4;
                 u64 subcode:8;
-                u64 len:3;       /** additional 64-bit words */
-                u64 rid:13;      /** request id in response to pkt sent by host */
-                u64 reserved:4;
-                u64 ossp:32;     /** opcode/subcode specific parameters */
+                u64 len:3;     /** additional 64-bit words */
+                u64 reserved:17;
+                u64 ossp:32;   /** opcode/subcode specific parameters */
         } r;
         struct {
                 u64 opcode:4;
                 u64 subcode:8;
-                u64 len:3;       /** additional 64-bit words */
-                u64 rid:13;      /** request id in response to pkt sent by host */
-                u64 extra:24;
-                u64 link:8;
+                u64 len:3;     /** additional 64-bit words */
+                u64 extra:28;
+                u64 vlan:12;
+                u64 priority:3;
                 u64 csum_verified:3;     /** checksum verified. */
                 u64 has_hwtstamp:1;      /** Has hardware timestamp. 1 = yes. */
         } r_dh;
         struct {
                 u64 opcode:4;
                 u64 subcode:8;
-                u64 len:3;       /** additional 64-bit words */
-                u64 rid:13;      /** request id in response to pkt sent by host */
+                u64 len:3;     /** additional 64-bit words */
+                u64 reserved:11;
                 u64 num_gmx_ports:8;
-                u64 max_nic_ports:8;
+                u64 max_nic_ports:10;
                 u64 app_cap_flags:4;
                 u64 app_mode:16;
         } r_core_drv_init;
@@ -412,17 +548,15 @@ union octeon_rh {
                 u64 opcode:4;
                 u64 subcode:8;
                 u64 len:3;       /** additional 64-bit words */
-                u64 rid:13;
-                u64 reserved:4;
+                u64 reserved:8;
                 u64 extra:25;
-                u64 ifidx:7;
+                u64 gmxport:16;
         } r_nic_info;
 #else
         u64 u64;
         struct {
                 u64 ossp:32;  /** opcode/subcode specific parameters */
-                u64 reserved:4;
-                u64 rid:13;   /** req id in response to pkt sent by host */
+                u64 reserved:17;
                 u64 len:3;    /** additional 64-bit words */
                 u64 subcode:8;
                 u64 opcode:4;
@@ -430,9 +564,9 @@ union octeon_rh {
         struct {
                 u64 has_hwtstamp:1;      /** 1 = has hwtstamp */
                 u64 csum_verified:3;     /** checksum verified. */
-                u64 link:8;
-                u64 extra:24;
-                u64 rid:13;   /** req id in response to pkt sent by host */
+                u64 priority:3;
+                u64 vlan:12;
+                u64 extra:28;
                 u64 len:3;    /** additional 64-bit words */
                 u64 subcode:8;
                 u64 opcode:4;
@@ -440,18 +574,17 @@ union octeon_rh {
         struct {
                 u64 app_mode:16;
                 u64 app_cap_flags:4;
-                u64 max_nic_ports:8;
+                u64 max_nic_ports:10;
                 u64 num_gmx_ports:8;
-                u64 rid:13;
+                u64 reserved:11;
                 u64 len:3;       /** additional 64-bit words */
                 u64 subcode:8;
                 u64 opcode:4;
         } r_core_drv_init;
         struct {
-                u64 ifidx:7;
+                u64 gmxport:16;
                 u64 extra:25;
-                u64 reserved:4;
-                u64 rid:13;
+                u64 reserved:8;
                 u64 len:3;       /** additional 64-bit words */
                 u64 subcode:8;
                 u64 opcode:4;
@@ -461,30 +594,25 @@ union octeon_rh {
 
 #define  OCT_RH_SIZE   (sizeof(union  octeon_rh))
 
-#define OCT_PKT_PARAM_IPV4OPTS   1
-#define OCT_PKT_PARAM_IPV6EXTHDR 2
-
 union octnic_packet_params {
         u32 u32;
         struct {
 #ifdef __BIG_ENDIAN_BITFIELD
-                u32 reserved:6;
+                u32 reserved:24;
+                u32 ip_csum:1;          /* Perform IP header checksum(s) */
+                /* Perform Outer transport header checksum */
+                u32 transport_csum:1;
+                /* Find tunnel, and perform transport csum. */
                 u32 tnl_csum:1;
-                u32 ip_csum:1;
-                u32 ipv4opts_ipv6exthdr:2;
-                u32 ipsec_ops:4;
-                u32 tsflag:1;
-                u32 csoffset:9;
-                u32 ifidx:8;
+                u32 tsflag:1;           /* Timestamp this packet */
+                u32 ipsec_ops:4;        /* IPsec operation */
 #else
-                u32 ifidx:8;
-                u32 csoffset:9;
-                u32 tsflag:1;
                 u32 ipsec_ops:4;
-                u32 ipv4opts_ipv6exthdr:2;
-                u32 ip_csum:1;
+                u32 tsflag:1;
                 u32 tnl_csum:1;
-                u32 reserved:6;
+                u32 transport_csum:1;
+                u32 ip_csum:1;
+                u32 reserved:24;
 #endif
         } s;
 };
@@ -496,56 +624,94 @@ union oct_link_status {
         struct {
 #ifdef __BIG_ENDIAN_BITFIELD
                 u64 duplex:8;
-                u64 status:8;
                 u64 mtu:16;
                 u64 speed:16;
+                u64 link_up:1;
                 u64 autoneg:1;
-                u64 interface:4;
+                u64 if_mode:5;
                 u64 pause:1;
-                u64 reserved:10;
+                u64 reserved:16;
 #else
-                u64 reserved:10;
+                u64 reserved:16;
                 u64 pause:1;
-                u64 interface:4;
+                u64 if_mode:5;
                 u64 autoneg:1;
+                u64 link_up:1;
                 u64 speed:16;
                 u64 mtu:16;
-                u64 status:8;
                 u64 duplex:8;
 #endif
         } s;
 };
 
+/** The txpciq info passed to host from the firmware */
+
+union oct_txpciq {
+        u64 u64;
+
+        struct {
+#ifdef __BIG_ENDIAN_BITFIELD
+                u64 q_no:8;
+                u64 port:8;
+                u64 pkind:6;
+                u64 use_qpg:1;
+                u64 qpg:11;
+                u64 reserved:30;
+#else
+                u64 reserved:30;
+                u64 qpg:11;
+                u64 use_qpg:1;
+                u64 pkind:6;
+                u64 port:8;
+                u64 q_no:8;
+#endif
+        } s;
+};
+
+/** The rxpciq info passed to host from the firmware */
+
+union oct_rxpciq {
+        u64 u64;
+
+        struct {
+#ifdef __BIG_ENDIAN_BITFIELD
+                u64 q_no:8;
+                u64 reserved:56;
+#else
+                u64 reserved:56;
+                u64 q_no:8;
+#endif
+        } s;
+};
+
 /** Information for a OCTEON ethernet interface shared between core & host. */
 struct oct_link_info {
         union oct_link_status link;
         u64 hw_addr;
 
 #ifdef __BIG_ENDIAN_BITFIELD
-        u16 gmxport;
-        u8 rsvd[3];
-        u8 num_txpciq;
-        u8 num_rxpciq;
-        u8 ifidx;
+        u64 gmxport:16;
+        u64 rsvd:32;
+        u64 num_txpciq:8;
+        u64 num_rxpciq:8;
 #else
-        u8 ifidx;
-        u8 num_rxpciq;
-        u8 num_txpciq;
-        u8 rsvd[3];
-        u16 gmxport;
+        u64 num_rxpciq:8;
+        u64 num_txpciq:8;
+        u64 rsvd:32;
+        u64 gmxport:16;
 #endif
 
-        u8 txpciq[MAX_IOQS_PER_NICIF];
-        u8 rxpciq[MAX_IOQS_PER_NICIF];
+        union oct_txpciq txpciq[MAX_IOQS_PER_NICIF];
+        union oct_rxpciq rxpciq[MAX_IOQS_PER_NICIF];
 };
 
 #define OCT_LINK_INFO_SIZE   (sizeof(struct oct_link_info))
 
 struct liquidio_if_cfg_info {
-        u64 ifidx;
         u64 iqmask; /** mask for IQs enabled for  the port */
         u64 oqmask; /** mask for OQs enabled for the port */
         struct oct_link_info linfo; /** initial link information */
+        char   liquidio_firmware_version[32];
 };
 
 /** Stats for each NIC port in RX direction. */
@@ -570,10 +736,16 @@ struct nic_rx_stats {
         u64 fw_err_pko;
         u64 fw_err_link;
         u64 fw_err_drop;
+
+        /* LRO */
         u64 fw_lro_pkts;   /* Number of packets that are LROed      */
         u64 fw_lro_octs;   /* Number of octets that are LROed       */
         u64 fw_total_lro;  /* Number of LRO packets formed          */
         u64 fw_lro_aborts; /* Number of times lRO of packet aborted */
+        u64 fw_lro_aborts_port;
+        u64 fw_lro_aborts_seq;
+        u64 fw_lro_aborts_tsval;
+        u64 fw_lro_aborts_timer;
         /* intrmod: packet forward rate */
         u64 fwd_rate;
 };
@@ -597,9 +769,13 @@ struct nic_tx_stats {
         /* firmware stats */
         u64 fw_total_sent;
         u64 fw_total_fwd;
+        u64 fw_total_fwd_bytes;
         u64 fw_err_pko;
         u64 fw_err_link;
         u64 fw_err_drop;
+        u64 fw_err_tso;
+        u64 fw_tso;             /* number of tso requests */
+        u64 fw_tso_fwd;         /* number of packets segmented in tso */
 };
 
 struct oct_link_stats {
@@ -630,23 +806,44 @@ struct oct_mdio_cmd {
 
 #define OCT_LINK_STATS_SIZE   (sizeof(struct oct_link_stats))
 
+/* intrmod: max. packet rate threshold */
+#define LIO_INTRMOD_MAXPKT_RATETHR      196608
+/* intrmod: min. packet rate threshold */
+#define LIO_INTRMOD_MINPKT_RATETHR      9216
+/* intrmod: max. packets to trigger interrupt */
+#define LIO_INTRMOD_RXMAXCNT_TRIGGER    384
+/* intrmod: min. packets to trigger interrupt */
+#define LIO_INTRMOD_RXMINCNT_TRIGGER    1
+/* intrmod: max. time to trigger interrupt */
+#define LIO_INTRMOD_RXMAXTMR_TRIGGER    128
+/* 66xx:intrmod: min. time to trigger interrupt
+ * (value of 1 is optimum for TCP_RR)
+ */
+#define LIO_INTRMOD_RXMINTMR_TRIGGER    1
+
+/* intrmod: max. packets to trigger interrupt */
+#define LIO_INTRMOD_TXMAXCNT_TRIGGER    64
+/* intrmod: min. packets to trigger interrupt */
+#define LIO_INTRMOD_TXMINCNT_TRIGGER    0
+
+/* intrmod: poll interval in seconds */
 #define LIO_INTRMOD_CHECK_INTERVAL  1
-#define LIO_INTRMOD_MAXPKT_RATETHR  196608 /* max pkt rate threshold */
-#define LIO_INTRMOD_MINPKT_RATETHR  9216   /* min pkt rate threshold */
-#define LIO_INTRMOD_MAXCNT_TRIGGER  384    /* max pkts to trigger interrupt */
-#define LIO_INTRMOD_MINCNT_TRIGGER  1      /* min pkts to trigger interrupt */
-#define LIO_INTRMOD_MAXTMR_TRIGGER  128    /* max time to trigger interrupt */
-#define LIO_INTRMOD_MINTMR_TRIGGER  32     /* min time to trigger interrupt */
 
 struct oct_intrmod_cfg {
-        u64 intrmod_enable;
-        u64 intrmod_check_intrvl;
-        u64 intrmod_maxpkt_ratethr;
-        u64 intrmod_minpkt_ratethr;
-        u64 intrmod_maxcnt_trigger;
-        u64 intrmod_maxtmr_trigger;
-        u64 intrmod_mincnt_trigger;
-        u64 intrmod_mintmr_trigger;
+        u64 rx_enable;
+        u64 tx_enable;
+        u64 check_intrvl;
+        u64 maxpkt_ratethr;
+        u64 minpkt_ratethr;
+        u64 rx_maxcnt_trigger;
+        u64 rx_mincnt_trigger;
+        u64 rx_maxtmr_trigger;
+        u64 rx_mintmr_trigger;
+        u64 tx_mincnt_trigger;
+        u64 tx_maxcnt_trigger;
+        u64 rx_frames;
+        u64 tx_frames;
+        u64 rx_usecs;
 };
 
 #define BASE_QUEUE_NOT_REQUESTED 65535
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_config.h b/drivers/net/ethernet/cavium/liquidio/octeon_config.h
index 62a8dd5cd3dc..4b8c948400be 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_config.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_config.h
@@ -37,7 +37,7 @@
 /* Maximum octeon devices defined as MAX_OCTEON_NICIF to support
  * multiple(<= MAX_OCTEON_NICIF) Miniports
  */
-#define   MAX_OCTEON_NICIF             32
+#define   MAX_OCTEON_NICIF             128
 #define   MAX_OCTEON_DEVICES           MAX_OCTEON_NICIF
 #define   MAX_OCTEON_LINKS             MAX_OCTEON_NICIF
 #define   MAX_OCTEON_MULTICAST_ADDR    32
@@ -135,7 +135,7 @@
 #define CFG_GET_IS_SLI_BP_ON(cfg)                ((cfg)->misc.enable_sli_oq_bp)
 
 /* Max IOQs per OCTEON Link */
-#define MAX_IOQS_PER_NICIF              32
+#define MAX_IOQS_PER_NICIF              64
 
 enum lio_card_type {
         LIO_210SV = 0, /* Two port, 66xx */
@@ -416,9 +416,11 @@ struct octeon_config {
 #define DISPATCH_LIST_SIZE                      BIT(OPCODE_MASK_BITS)
 
 /* Maximum number of Octeon Instruction (command) queues */
-#define MAX_OCTEON_INSTR_QUEUES         CN6XXX_MAX_INPUT_QUEUES
+#define MAX_OCTEON_INSTR_QUEUES(oct)         CN6XXX_MAX_INPUT_QUEUES
+/* Maximum number of Octeon Output queues */
+#define MAX_OCTEON_OUTPUT_QUEUES(oct)         CN6XXX_MAX_OUTPUT_QUEUES
 
-/* Maximum number of Octeon Instruction (command) queues */
-#define MAX_OCTEON_OUTPUT_QUEUES        CN6XXX_MAX_OUTPUT_QUEUES
+#define MAX_POSSIBLE_OCTEON_INSTR_QUEUES       CN6XXX_MAX_INPUT_QUEUES
+#define MAX_POSSIBLE_OCTEON_OUTPUT_QUEUES      CN6XXX_MAX_OUTPUT_QUEUES
 
 #endif /* __OCTEON_CONFIG_H__  */
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_device.c b/drivers/net/ethernet/cavium/liquidio/octeon_device.c
index 8e23e3fad662..337220721632 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_device.c
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_device.c
@@ -19,7 +19,6 @@
 * This file may also be available under a different license from Cavium.
 * Contact Cavium, Inc. for more information
 **********************************************************************/
-#include <linux/version.h>
 #include <linux/types.h>
 #include <linux/list.h>
 #include <linux/interrupt.h>
@@ -550,17 +549,19 @@ static char *get_oct_app_string(u32 app_mode)
         return oct_dev_app_str[CVM_DRV_INVALID_APP - CVM_DRV_APP_START];
 }
 
+u8 fbuf[4 * 1024 * 1024];
+
 int octeon_download_firmware(struct octeon_device *oct, const u8 *data,
                              size_t size)
 {
         int ret = 0;
-        u8 *p;
-        u8 *buffer;
+        u8 *p = fbuf;
         u32 crc32_result;
         u64 load_addr;
         u32 image_len;
         struct octeon_firmware_file_header *h;
-        u32 i;
+        u32 i, rem, base_len = strlen(LIQUIDIO_BASE_VERSION);
+        char *base;
 
         if (size < sizeof(struct octeon_firmware_file_header)) {
                 dev_err(&oct->pci_dev->dev, "Firmware file too small (%d < %d).\n",
@@ -576,19 +577,26 @@ int octeon_download_firmware(struct octeon_device *oct, const u8 *data,
                 return -EINVAL;
         }
 
-        crc32_result =
-                crc32(~0, data,
-                      sizeof(struct octeon_firmware_file_header) -
-                      sizeof(u32)) ^ ~0U;
+        crc32_result = crc32((unsigned int)~0, data,
+                             sizeof(struct octeon_firmware_file_header) -
+                             sizeof(u32)) ^ ~0U;
         if (crc32_result != be32_to_cpu(h->crc32)) {
                 dev_err(&oct->pci_dev->dev, "Firmware CRC mismatch (0x%08x != 0x%08x).\n",
                         crc32_result, be32_to_cpu(h->crc32));
                 return -EINVAL;
         }
 
-        if (memcmp(LIQUIDIO_VERSION, h->version, strlen(LIQUIDIO_VERSION))) {
-                dev_err(&oct->pci_dev->dev, "Unmatched firmware version. Expected %s, got %s.\n",
-                        LIQUIDIO_VERSION, h->version);
+        if (strncmp(LIQUIDIO_PACKAGE, h->version, strlen(LIQUIDIO_PACKAGE))) {
+                dev_err(&oct->pci_dev->dev, "Unmatched firmware package type. Expected %s, got %s.\n",
+                        LIQUIDIO_PACKAGE, h->version);
+                return -EINVAL;
+        }
+
+        base = h->version + strlen(LIQUIDIO_PACKAGE);
+        ret = memcmp(LIQUIDIO_BASE_VERSION, base, base_len);
+        if (ret) {
+                dev_err(&oct->pci_dev->dev, "Unmatched firmware version. Expected %s.x, got %s.\n",
+                        LIQUIDIO_BASE_VERSION, base);
                 return -EINVAL;
         }
 
@@ -602,56 +610,56 @@ int octeon_download_firmware(struct octeon_device *oct, const u8 *data,
         snprintf(oct->fw_info.liquidio_firmware_version, 32, "LIQUIDIO: %s",
                  h->version);
 
-        buffer = kmemdup(data, size, GFP_KERNEL);
-        if (!buffer)
-                return -ENOMEM;
-
-        p = buffer + sizeof(struct octeon_firmware_file_header);
+        data += sizeof(struct octeon_firmware_file_header);
 
+        dev_info(&oct->pci_dev->dev, "%s: Loading %d images\n", __func__,
+                 be32_to_cpu(h->num_images));
         /* load all images */
         for (i = 0; i < be32_to_cpu(h->num_images); i++) {
                 load_addr = be64_to_cpu(h->desc[i].addr);
                 image_len = be32_to_cpu(h->desc[i].len);
 
-                /* validate the image */
-                crc32_result = crc32(~0, p, image_len) ^ ~0U;
-                if (crc32_result != be32_to_cpu(h->desc[i].crc32)) {
-                        dev_err(&oct->pci_dev->dev,
-                                "Firmware CRC mismatch in image %d (0x%08x != 0x%08x).\n",
-                                i, crc32_result,
-                                be32_to_cpu(h->desc[i].crc32));
-                        ret = -EINVAL;
-                        goto done_downloading;
-                }
+                dev_info(&oct->pci_dev->dev, "Loading firmware %d at %llx\n",
+                         image_len, load_addr);
 
-                /* download the image */
-                octeon_pci_write_core_mem(oct, load_addr, p, image_len);
+                /* Write in 4MB chunks*/
+                rem = image_len;
 
-                p += image_len;
-                dev_dbg(&oct->pci_dev->dev,
-                        "Downloaded image %d (%d bytes) to address 0x%016llx\n",
-                        i, image_len, load_addr);
+                while (rem) {
+                        if (rem < (4 * 1024 * 1024))
+                                size = rem;
+                        else
+                                size = 4 * 1024 * 1024;
+
+                        memcpy(p, data, size);
+
+                        /* download the image */
+                        octeon_pci_write_core_mem(oct, load_addr, p, (u32)size);
+
+                        data += size;
+                        rem -= (u32)size;
+                        load_addr += size;
+                }
         }
+        dev_info(&oct->pci_dev->dev, "Writing boot command: %s\n",
+                 h->bootcmd);
 
         /* Invoke the bootcmd */
         ret = octeon_console_send_cmd(oct, h->bootcmd, 50);
 
-done_downloading:
-        kfree(buffer);
-
-        return ret;
+        return 0;
 }
 
 void octeon_free_device_mem(struct octeon_device *oct)
 {
         u32 i;
 
-        for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES; i++) {
+        for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
                 /* could check  mask as well */
                 vfree(oct->droq[i]);
         }
 
-        for (i = 0; i < MAX_OCTEON_INSTR_QUEUES; i++) {
+        for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
                 /* could check mask as well */
                 vfree(oct->instr_queue[i]);
         }
@@ -735,55 +743,65 @@ struct octeon_device *octeon_allocate_device(u32 pci_id,
         octeon_device[oct_idx] = oct;
 
         oct->octeon_id = oct_idx;
-        snprintf((oct->device_name), sizeof(oct->device_name),
+        snprintf(oct->device_name, sizeof(oct->device_name),
                  "LiquidIO%d", (oct->octeon_id));
 
         return oct;
 }
 
+/* this function is only for setting up the first queue */
 int octeon_setup_instr_queues(struct octeon_device *oct)
 {
-        u32 i, num_iqs = 0;
+        u32 num_iqs = 0;
         u32 num_descs = 0;
+        u32 iq_no = 0;
+        union oct_txpciq txpciq;
+        int numa_node = cpu_to_node(iq_no % num_online_cpus());
 
+        num_iqs = 1;
         /* this causes queue 0 to be default queue */
-        if (OCTEON_CN6XXX(oct)) {
-                num_iqs = 1;
+        if (OCTEON_CN6XXX(oct))
                 num_descs =
                         CFG_GET_NUM_DEF_TX_DESCS(CHIP_FIELD(oct, cn6xxx, conf));
-        }
 
         oct->num_iqs = 0;
 
-        for (i = 0; i < num_iqs; i++) {
-                oct->instr_queue[i] =
+        oct->instr_queue[0] = vmalloc_node(sizeof(*oct->instr_queue[0]),
+                                numa_node);
+        if (!oct->instr_queue[0])
+                oct->instr_queue[0] =
                         vmalloc(sizeof(struct octeon_instr_queue));
-                if (!oct->instr_queue[i])
-                        return 1;
-
-                memset(oct->instr_queue[i], 0,
-                       sizeof(struct octeon_instr_queue));
-
-                oct->instr_queue[i]->app_ctx = (void *)(size_t)i;
-                if (octeon_init_instr_queue(oct, i, num_descs))
-                        return 1;
-
-                oct->num_iqs++;
+        if (!oct->instr_queue[0])
+                return 1;
+        memset(oct->instr_queue[0], 0, sizeof(struct octeon_instr_queue));
+        oct->instr_queue[0]->q_index = 0;
+        oct->instr_queue[0]->app_ctx = (void *)(size_t)0;
+        oct->instr_queue[0]->ifidx = 0;
+        txpciq.u64 = 0;
+        txpciq.s.q_no = iq_no;
+        txpciq.s.use_qpg = 0;
+        txpciq.s.qpg = 0;
+        if (octeon_init_instr_queue(oct, txpciq, num_descs)) {
+                /* prevent memory leak */
+                vfree(oct->instr_queue[0]);
+                return 1;
         }
 
+        oct->num_iqs++;
         return 0;
 }
 
 int octeon_setup_output_queues(struct octeon_device *oct)
 {
-        u32 i, num_oqs = 0;
+        u32 num_oqs = 0;
         u32 num_descs = 0;
         u32 desc_size = 0;
+        u32 oq_no = 0;
+        int numa_node = cpu_to_node(oq_no % num_online_cpus());
 
+        num_oqs = 1;
         /* this causes queue 0 to be default queue */
         if (OCTEON_CN6XXX(oct)) {
-                /* CFG_GET_OQ_MAX_BASE_Q(CHIP_FIELD(oct, cn6xxx, conf)); */
-                num_oqs = 1;
                 num_descs =
                         CFG_GET_NUM_DEF_RX_DESCS(CHIP_FIELD(oct, cn6xxx, conf));
                 desc_size =
@@ -791,19 +809,15 @@ int octeon_setup_output_queues(struct octeon_device *oct)
         }
 
         oct->num_oqs = 0;
+        oct->droq[0] = vmalloc_node(sizeof(*oct->droq[0]), numa_node);
+        if (!oct->droq[0])
+                oct->droq[0] = vmalloc(sizeof(*oct->droq[0]));
+        if (!oct->droq[0])
+                return 1;
 
-        for (i = 0; i < num_oqs; i++) {
-                oct->droq[i] = vmalloc(sizeof(*oct->droq[i]));
-                if (!oct->droq[i])
-                        return 1;
-
-                memset(oct->droq[i], 0, sizeof(struct octeon_droq));
-
-                if (octeon_init_droq(oct, i, num_descs, desc_size, NULL))
-                        return 1;
-
-                oct->num_oqs++;
-        }
+        if (octeon_init_droq(oct, oq_no, num_descs, desc_size, NULL))
+                return 1;
+        oct->num_oqs++;
 
         return 0;
 }
@@ -1152,8 +1166,8 @@ core_drv_init_err:
 int octeon_get_tx_qsize(struct octeon_device *oct, u32 q_no)
 
 {
-        if (oct && (q_no < MAX_OCTEON_INSTR_QUEUES) &&
-            (oct->io_qmask.iq & (1UL << q_no)))
+        if (oct && (q_no < MAX_OCTEON_INSTR_QUEUES(oct)) &&
+            (oct->io_qmask.iq & (1ULL << q_no)))
                 return oct->instr_queue[q_no]->max_count;
 
         return -1;
@@ -1161,8 +1175,8 @@ int octeon_get_tx_qsize(struct octeon_device *oct, u32 q_no)
 
 int octeon_get_rx_qsize(struct octeon_device *oct, u32 q_no)
 {
-        if (oct && (q_no < MAX_OCTEON_OUTPUT_QUEUES) &&
-            (oct->io_qmask.oq & (1UL << q_no)))
+        if (oct && (q_no < MAX_OCTEON_OUTPUT_QUEUES(oct)) &&
+            (oct->io_qmask.oq & (1ULL << q_no)))
                 return oct->droq[q_no]->max_count;
         return -1;
 }
@@ -1253,10 +1267,10 @@ void lio_pci_writeq(struct octeon_device *oct,
 int octeon_mem_access_ok(struct octeon_device *oct)
 {
         u64 access_okay = 0;
+        u64 lmc0_reset_ctl;
 
         /* Check to make sure a DDR interface is enabled */
-        u64 lmc0_reset_ctl = lio_pci_readq(oct, CN6XXX_LMC0_RESET_CTL);
-
+        lmc0_reset_ctl = lio_pci_readq(oct, CN6XXX_LMC0_RESET_CTL);
         access_okay = (lmc0_reset_ctl & CN6XXX_LMC0_RESET_CTL_DDR3RST_MASK);
 
         return access_okay ? 0 : 1;
@@ -1270,9 +1284,6 @@ int octeon_wait_for_ddr_init(struct octeon_device *oct, u32 *timeout)
         if (!timeout)
                 return ret;
 
-        while (*timeout == 0)
-                schedule_timeout_uninterruptible(HZ / 10);
-
         for (ms = 0; (ret != 0) && ((*timeout == 0) || (ms <= *timeout));
              ms += HZ / 10) {
                 ret = octeon_mem_access_ok(oct);
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_device.h b/drivers/net/ethernet/cavium/liquidio/octeon_device.h
index 36e1f85df8c4..b4e566dea008 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_device.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_device.h
@@ -152,9 +152,9 @@ struct octeon_mmio {
 #define   MAX_OCTEON_MAPS    32
 
 struct octeon_io_enable {
-        u32 iq;
-        u32 oq;
-        u32 iq64B;
+        u64 iq;
+        u64 oq;
+        u64 iq64B;
 };
 
 struct octeon_reg_list {
@@ -204,8 +204,7 @@ struct octeon_fn_list {
         void (*bar1_idx_setup)(struct octeon_device *, u64, u32, int);
         void (*bar1_idx_write)(struct octeon_device *, u32, u32);
         u32 (*bar1_idx_read)(struct octeon_device *, u32);
-        u32 (*update_iq_read_idx)(struct octeon_device *,
-                                  struct octeon_instr_queue *);
+        u32 (*update_iq_read_idx)(struct octeon_instr_queue *);
 
         void (*enable_oq_pkt_time_intr)(struct octeon_device *, u32);
         void (*disable_oq_pkt_time_intr)(struct octeon_device *, u32);
@@ -267,6 +266,8 @@ struct octdev_props {
         /* Each interface in the Octeon device has a network
          * device pointer (used for OS specific calls).
          */
+        int    napi_enabled;
+        int    gmxport;
         struct net_device *netdev;
 };
 
@@ -324,7 +325,8 @@ struct octeon_device {
         struct octeon_sc_buffer_pool    sc_buf_pool;
 
         /** The input instruction queues */
-        struct octeon_instr_queue *instr_queue[MAX_OCTEON_INSTR_QUEUES];
+        struct octeon_instr_queue *instr_queue
+                [MAX_POSSIBLE_OCTEON_INSTR_QUEUES];
 
         /** The doubly-linked list of instruction response */
         struct octeon_response_list response_list[MAX_RESPONSE_LISTS];
@@ -332,7 +334,7 @@ struct octeon_device {
         u32 num_oqs;
 
         /** The DROQ output queues  */
-        struct octeon_droq *droq[MAX_OCTEON_OUTPUT_QUEUES];
+        struct octeon_droq *droq[MAX_POSSIBLE_OCTEON_OUTPUT_QUEUES];
 
         struct octeon_io_enable io_qmask;
 
@@ -381,15 +383,29 @@ struct octeon_device {
 
         struct cavium_wq dma_comp_wq;
 
-        struct cavium_wq check_db_wq[MAX_OCTEON_INSTR_QUEUES];
+        /** Lock for dma response list */
+        spinlock_t cmd_resp_wqlock;
+        u32 cmd_resp_state;
+
+        struct cavium_wq check_db_wq[MAX_POSSIBLE_OCTEON_INSTR_QUEUES];
 
         struct cavium_wk nic_poll_work;
 
         struct cavium_wk console_poll_work[MAX_OCTEON_MAPS];
 
         void *priv;
+
+        int rx_pause;
+        int tx_pause;
+
+        struct oct_link_stats link_stats; /*stastics from firmware*/
+
+        /* private flags to control driver-specific features through ethtool */
+        u32 priv_flags;
 };
 
+#define  OCT_DRV_ONLINE 1
+#define  OCT_DRV_OFFLINE 2
 #define  OCTEON_CN6XXX(oct)           ((oct->chip_id == OCTEON_CN66XX) || \
                                        (oct->chip_id == OCTEON_CN68XX))
 #define CHIP_FIELD(oct, TYPE, field)             \
@@ -646,4 +662,17 @@ void *oct_get_config_info(struct octeon_device *oct, u16 card_type);
  */
 struct octeon_config *octeon_get_conf(struct octeon_device *oct);
 
+/* LiquidIO driver pivate flags */
+enum {
+        OCT_PRIV_FLAG_TX_BYTES = 0, /* Tx interrupts by pending byte count */
+};
+
+static inline void lio_set_priv_flag(struct octeon_device *octdev, u32 flag,
+                                     u32 val)
+{
+        if (val)
+                octdev->priv_flags |= (0x1 << flag);
+        else
+                octdev->priv_flags &= ~(0x1 << flag);
+}
 #endif
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_droq.c b/drivers/net/ethernet/cavium/liquidio/octeon_droq.c
index 174072b3740b..d9bb2f7e0836 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_droq.c
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_droq.c
@@ -151,22 +151,26 @@ octeon_droq_destroy_ring_buffers(struct octeon_device *oct,
                                  struct octeon_droq *droq)
 {
         u32 i;
+        struct octeon_skb_page_info *pg_info;
 
         for (i = 0; i < droq->max_count; i++) {
-                if (droq->recv_buf_list[i].buffer) {
-                        if (droq->desc_ring) {
-                                lio_unmap_ring_info(oct->pci_dev,
-                                                    (u64)droq->
-                                                    desc_ring[i].info_ptr,
-                                                    OCT_DROQ_INFO_SIZE);
-                                lio_unmap_ring(oct->pci_dev,
-                                               (u64)droq->desc_ring[i].
-                                               buffer_ptr,
-                                               droq->buffer_size);
-                        }
-                        recv_buffer_free(droq->recv_buf_list[i].buffer);
-                        droq->recv_buf_list[i].buffer = NULL;
-                }
+                pg_info = &droq->recv_buf_list[i].pg_info;
+
+                if (pg_info->dma)
+                        lio_unmap_ring(oct->pci_dev,
+                                       (u64)pg_info->dma);
+                pg_info->dma = 0;
+
+                if (pg_info->page)
+                        recv_buffer_destroy(droq->recv_buf_list[i].buffer,
+                                            pg_info);
+
+                if (droq->desc_ring && droq->desc_ring[i].info_ptr)
+                        lio_unmap_ring_info(oct->pci_dev,
+                                            (u64)droq->
+                                            desc_ring[i].info_ptr,
+                                            OCT_DROQ_INFO_SIZE);
+                droq->recv_buf_list[i].buffer = NULL;
         }
 
         octeon_droq_reset_indices(droq);
@@ -181,11 +185,12 @@ octeon_droq_setup_ring_buffers(struct octeon_device *oct,
         struct octeon_droq_desc *desc_ring = droq->desc_ring;
 
         for (i = 0; i < droq->max_count; i++) {
-                buf = recv_buffer_alloc(oct, droq->q_no, droq->buffer_size);
+                buf = recv_buffer_alloc(oct, &droq->recv_buf_list[i].pg_info);
 
                 if (!buf) {
                         dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n",
                                 __func__);
+                        droq->stats.rx_alloc_failure++;
                         return -ENOMEM;
                 }
 
@@ -197,9 +202,7 @@ octeon_droq_setup_ring_buffers(struct octeon_device *oct,
                 /* map ring buffers into memory */
                 desc_ring[i].info_ptr = lio_map_ring_info(droq, i);
                 desc_ring[i].buffer_ptr =
-                        lio_map_ring(oct->pci_dev,
-                                     droq->recv_buf_list[i].buffer,
-                                     droq->buffer_size);
+                        lio_map_ring(droq->recv_buf_list[i].buffer);
         }
 
         octeon_droq_reset_indices(droq);
@@ -242,6 +245,8 @@ int octeon_init_droq(struct octeon_device *oct,
         struct octeon_droq *droq;
         u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0;
         u32 c_pkts_per_intr = 0, c_refill_threshold = 0;
+        int orig_node = dev_to_node(&oct->pci_dev->dev);
+        int numa_node = cpu_to_node(q_no % num_online_cpus());
 
         dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no);
 
@@ -261,15 +266,23 @@ int octeon_init_droq(struct octeon_device *oct,
                 struct octeon_config *conf6x = CHIP_FIELD(oct, cn6xxx, conf);
 
                 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x);
-                c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x);
+                c_refill_threshold =
+                        (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x);
+        } else {
+                return 1;
         }
 
         droq->max_count = c_num_descs;
         droq->buffer_size = c_buf_size;
 
         desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE;
+        set_dev_node(&oct->pci_dev->dev, numa_node);
         droq->desc_ring = lio_dma_alloc(oct, desc_ring_size,
                                         (dma_addr_t *)&droq->desc_ring_dma);
+        set_dev_node(&oct->pci_dev->dev, orig_node);
+        if (!droq->desc_ring)
+                droq->desc_ring = lio_dma_alloc(oct, desc_ring_size,
+                                        (dma_addr_t *)&droq->desc_ring_dma);
 
         if (!droq->desc_ring) {
                 dev_err(&oct->pci_dev->dev,
@@ -283,12 +296,11 @@ int octeon_init_droq(struct octeon_device *oct,
                 droq->max_count);
 
         droq->info_list =
-                cnnic_alloc_aligned_dma(oct->pci_dev,
-                                        (droq->max_count * OCT_DROQ_INFO_SIZE),
-                                        &droq->info_alloc_size,
-                                        &droq->info_base_addr,
-                                        &droq->info_list_dma);
-
+                cnnic_numa_alloc_aligned_dma((droq->max_count *
+                                              OCT_DROQ_INFO_SIZE),
+                                             &droq->info_alloc_size,
+                                             &droq->info_base_addr,
+                                             numa_node);
         if (!droq->info_list) {
                 dev_err(&oct->pci_dev->dev, "Cannot allocate memory for info list.\n");
                 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE),
@@ -297,7 +309,12 @@ int octeon_init_droq(struct octeon_device *oct,
         }
 
         droq->recv_buf_list = (struct octeon_recv_buffer *)
-                              vmalloc(droq->max_count *
+                              vmalloc_node(droq->max_count *
+                                                OCT_DROQ_RECVBUF_SIZE,
+                                                numa_node);
+        if (!droq->recv_buf_list)
+                droq->recv_buf_list = (struct octeon_recv_buffer *)
+                                      vmalloc(droq->max_count *
                                                 OCT_DROQ_RECVBUF_SIZE);
         if (!droq->recv_buf_list) {
                 dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n");
@@ -320,7 +337,7 @@ int octeon_init_droq(struct octeon_device *oct,
         /* For 56xx Pass1, this function won't be called, so no checks. */
         oct->fn_list.setup_oq_regs(oct, q_no);
 
-        oct->io_qmask.oq |= (1 << q_no);
+        oct->io_qmask.oq |= (1ULL << q_no);
 
         return 0;
 
@@ -358,6 +375,7 @@ static inline struct octeon_recv_info *octeon_create_recv_info(
         struct octeon_recv_pkt *recv_pkt;
         struct octeon_recv_info *recv_info;
         u32 i, bytes_left;
+        struct octeon_skb_page_info *pg_info;
 
         info = &droq->info_list[idx];
 
@@ -375,9 +393,14 @@ static inline struct octeon_recv_info *octeon_create_recv_info(
         bytes_left = (u32)info->length;
 
         while (buf_cnt) {
-                lio_unmap_ring(octeon_dev->pci_dev,
-                               (u64)droq->desc_ring[idx].buffer_ptr,
-                               droq->buffer_size);
+                {
+                        pg_info = &droq->recv_buf_list[idx].pg_info;
+
+                        lio_unmap_ring(octeon_dev->pci_dev,
+                                       (u64)pg_info->dma);
+                        pg_info->page = NULL;
+                        pg_info->dma = 0;
+                }
 
                 recv_pkt->buffer_size[i] =
                         (bytes_left >=
@@ -449,6 +472,7 @@ octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq)
         void *buf = NULL;
         u8 *data;
         u32 desc_refilled = 0;
+        struct octeon_skb_page_info *pg_info;
 
         desc_ring = droq->desc_ring;
 
@@ -458,13 +482,22 @@ octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq)
                  * the buffer, else allocate.
                  */
                 if (!droq->recv_buf_list[droq->refill_idx].buffer) {
-                        buf = recv_buffer_alloc(octeon_dev, droq->q_no,
-                                                droq->buffer_size);
+                        pg_info =
+                                &droq->recv_buf_list[droq->refill_idx].pg_info;
+                        /* Either recycle the existing pages or go for
+                         * new page alloc
+                         */
+                        if (pg_info->page)
+                                buf = recv_buffer_reuse(octeon_dev, pg_info);
+                        else
+                                buf = recv_buffer_alloc(octeon_dev, pg_info);
                         /* If a buffer could not be allocated, no point in
                          * continuing
                          */
-                        if (!buf)
+                        if (!buf) {
+                                droq->stats.rx_alloc_failure++;
                                 break;
+                        }
                         droq->recv_buf_list[droq->refill_idx].buffer =
                                 buf;
                         data = get_rbd(buf);
@@ -476,11 +509,8 @@ octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq)
                 droq->recv_buf_list[droq->refill_idx].data = data;
 
                 desc_ring[droq->refill_idx].buffer_ptr =
-                        lio_map_ring(octeon_dev->pci_dev,
-                                     droq->recv_buf_list[droq->
-                                     refill_idx].buffer,
-                                     droq->buffer_size);
-
+                        lio_map_ring(droq->recv_buf_list[droq->
+                                     refill_idx].buffer);
                 /* Reset any previous values in the length field. */
                 droq->info_list[droq->refill_idx].length = 0;
 
@@ -586,6 +616,8 @@ octeon_droq_fast_process_packets(struct octeon_device *oct,
         for (pkt = 0; pkt < pkt_count; pkt++) {
                 u32 pkt_len = 0;
                 struct sk_buff *nicbuf = NULL;
+                struct octeon_skb_page_info *pg_info;
+                void *buf;
 
                 info = &droq->info_list[droq->read_idx];
                 octeon_swap_8B_data((u64 *)info, 2);
@@ -605,7 +637,6 @@ octeon_droq_fast_process_packets(struct octeon_device *oct,
                 rh = &info->rh;
 
                 total_len += (u32)info->length;
-
                 if (OPCODE_SLOW_PATH(rh)) {
                         u32 buf_cnt;
 
@@ -614,50 +645,44 @@ octeon_droq_fast_process_packets(struct octeon_device *oct,
                         droq->refill_count += buf_cnt;
                 } else {
                         if (info->length <= droq->buffer_size) {
-                                lio_unmap_ring(oct->pci_dev,
-                                               (u64)droq->desc_ring[
-                                               droq->read_idx].buffer_ptr,
-                                               droq->buffer_size);
                                 pkt_len = (u32)info->length;
                                 nicbuf = droq->recv_buf_list[
                                         droq->read_idx].buffer;
+                                pg_info = &droq->recv_buf_list[
+                                        droq->read_idx].pg_info;
+                                if (recv_buffer_recycle(oct, pg_info))
+                                        pg_info->page = NULL;
                                 droq->recv_buf_list[droq->read_idx].buffer =
                                         NULL;
                                 INCR_INDEX_BY1(droq->read_idx, droq->max_count);
-                                skb_put(nicbuf, pkt_len);
                                 droq->refill_count++;
                         } else {
-                                nicbuf = octeon_fast_packet_alloc(oct, droq,
-                                                                  droq->q_no,
-                                                                  (u32)
+                                nicbuf = octeon_fast_packet_alloc((u32)
                                                                   info->length);
                                 pkt_len = 0;
                                 /* nicbuf allocation can fail. We'll handle it
                                  * inside the loop.
                                  */
                                 while (pkt_len < info->length) {
-                                        int cpy_len;
+                                        int cpy_len, idx = droq->read_idx;
 
-                                        cpy_len = ((pkt_len +
-                                                droq->buffer_size) >
-                                                info->length) ?
+                                        cpy_len = ((pkt_len + droq->buffer_size)
+                                                   > info->length) ?
                                                 ((u32)info->length - pkt_len) :
                                                 droq->buffer_size;
 
                                         if (nicbuf) {
-                                                lio_unmap_ring(oct->pci_dev,
-                                                               (u64)
-                                                               droq->desc_ring
-                                                               [droq->read_idx].
-                                                               buffer_ptr,
-                                                               droq->
-                                                               buffer_size);
                                                 octeon_fast_packet_next(droq,
                                                                         nicbuf,
                                                                         cpy_len,
-                                                                        droq->
-                                                                        read_idx
-                                                                        );
+                                                                        idx);
+                                                buf = droq->recv_buf_list[idx].
+                                                        buffer;
+                                                recv_buffer_fast_free(buf);
+                                                droq->recv_buf_list[idx].buffer
+                                                        = NULL;
+                                        } else {
+                                                droq->stats.rx_alloc_failure++;
                                         }
 
                                         pkt_len += cpy_len;
@@ -668,12 +693,14 @@ octeon_droq_fast_process_packets(struct octeon_device *oct,
                         }
 
                         if (nicbuf) {
-                                if (droq->ops.fptr)
+                                if (droq->ops.fptr) {
                                         droq->ops.fptr(oct->octeon_id,
-                                        nicbuf, pkt_len,
-                                        rh, &droq->napi);
-                                else
+                                                       nicbuf, pkt_len,
+                                                       rh, &droq->napi,
+                                                       droq->ops.farg);
+                                } else {
                                         recv_buffer_free(nicbuf);
+                                }
                         }
                 }
 
@@ -681,16 +708,16 @@ octeon_droq_fast_process_packets(struct octeon_device *oct,
                         int desc_refilled = octeon_droq_refill(oct, droq);
 
                         /* Flush the droq descriptor data to memory to be sure
-                        * that when we update the credits the data in memory
-                        * is accurate.
-                        */
+                         * that when we update the credits the data in memory
+                         * is accurate.
+                         */
                         wmb();
                         writel((desc_refilled), droq->pkts_credit_reg);
                         /* make sure mmio write completes */
                         mmiowb();
                 }
 
-        }                       /* for ( each packet )... */
+        }                       /* for (each packet)... */
 
         /* Increment refill_count by the number of buffers processed. */
         droq->stats.pkts_received += pkt;
@@ -937,6 +964,7 @@ int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no)
         spin_lock_irqsave(&droq->lock, flags);
 
         droq->ops.fptr = NULL;
+        droq->ops.farg = NULL;
         droq->ops.drop_on_max = 0;
 
         spin_unlock_irqrestore(&droq->lock, flags);
@@ -949,6 +977,7 @@ int octeon_create_droq(struct octeon_device *oct,
                        u32 desc_size, void *app_ctx)
 {
         struct octeon_droq *droq;
+        int numa_node = cpu_to_node(q_no % num_online_cpus());
 
         if (oct->droq[q_no]) {
                 dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n",
@@ -957,7 +986,9 @@ int octeon_create_droq(struct octeon_device *oct,
         }
 
         /* Allocate the DS for the new droq. */
-        droq = vmalloc(sizeof(*droq));
+        droq = vmalloc_node(sizeof(*droq), numa_node);
+        if (!droq)
+                droq = vmalloc(sizeof(*droq));
         if (!droq)
                 goto create_droq_fail;
         memset(droq, 0, sizeof(struct octeon_droq));
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_droq.h b/drivers/net/ethernet/cavium/liquidio/octeon_droq.h
index 7940ccee12d9..1ca9c4f05702 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_droq.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_droq.h
@@ -65,6 +65,17 @@ struct octeon_droq_info {
 
 #define OCT_DROQ_INFO_SIZE   (sizeof(struct octeon_droq_info))
 
+struct octeon_skb_page_info {
+        /* DMA address for the page */
+        dma_addr_t dma;
+
+        /* Page for the rx dma  **/
+        struct page *page;
+
+        /** which offset into page */
+        unsigned int page_offset;
+};
+
 /** Pointer to data buffer.
  *  Driver keeps a pointer to the data buffer that it made available to
  *  the Octeon device. Since the descriptor ring keeps physical (bus)
@@ -77,6 +88,9 @@ struct octeon_recv_buffer {
 
         /** Data in the packet buffer.  */
         u8 *data;
+
+        /** pg_info **/
+        struct octeon_skb_page_info pg_info;
 };
 
 #define OCT_DROQ_RECVBUF_SIZE    (sizeof(struct octeon_recv_buffer))
@@ -106,6 +120,10 @@ struct oct_droq_stats {
 
         /** Num of Packets dropped due to receive path failures. */
         u64 rx_dropped;
+
+        /** Num of failures of recv_buffer_alloc() */
+        u64 rx_alloc_failure;
+
 };
 
 #define POLL_EVENT_INTR_ARRIVED  1
@@ -213,7 +231,8 @@ struct octeon_droq_ops {
          *  data in the buffer. The receive header gives the port
          *  number to the caller.  Function pointer is set by caller.
          */
-        void (*fptr)(u32, void *, u32, union octeon_rh *, void *);
+        void (*fptr)(u32, void *, u32, union octeon_rh *, void *, void *);
+        void *farg;
 
         /* This function will be called by the driver for all NAPI related
          * events. The first param is the octeon id. The second param is the
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_iq.h b/drivers/net/ethernet/cavium/liquidio/octeon_iq.h
index 592fe49b589d..caa2b4f30717 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_iq.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_iq.h
@@ -65,6 +65,10 @@ struct oct_iq_stats {
         u64 tx_iq_busy;/**< Numof times this iq was found to be full. */
         u64 tx_dropped;/**< Numof pkts dropped dueto xmitpath errors. */
         u64 tx_tot_bytes;/**< Total count of bytes sento to network. */
+        u64 tx_gso;  /* count of tso */
+        u64 tx_dmamap_fail;
+        u64 tx_restart;
+        /*u64 tx_timeout_count;*/
 };
 
 #define OCT_IQ_STATS_SIZE   (sizeof(struct oct_iq_stats))
@@ -75,14 +79,22 @@ struct oct_iq_stats {
  *  a Octeon device has one such structure to represent it.
 */
 struct octeon_instr_queue {
+        struct octeon_device *oct_dev;
+
         /** A spinlock to protect access to the input ring.  */
         spinlock_t lock;
 
+        /** A spinlock to protect while posting on the ring.  */
+        spinlock_t post_lock;
+
+        /** A spinlock to protect access to the input ring.*/
+        spinlock_t iq_flush_running_lock;
+
         /** Flag that indicates if the queue uses 64 byte commands. */
         u32 iqcmd_64B:1;
 
-        /** Queue Number. */
-        u32 iq_no:5;
+        /** Queue info. */
+        union oct_txpciq txpciq;
 
         u32 rsvd:17;
 
@@ -147,6 +159,13 @@ struct octeon_instr_queue {
 
         /** Application context */
         void *app_ctx;
+
+        /* network stack queue index */
+        int q_index;
+
+        /*os ifidx associated with this queue */
+        int ifidx;
+
 };
 
 /*----------------------  INSTRUCTION FORMAT ----------------------------*/
@@ -176,12 +195,12 @@ struct octeon_instr_32B {
 /** 64-byte instruction format.
  *  Format of instruction for a 64-byte mode input queue.
  */
-struct octeon_instr_64B {
+struct octeon_instr2_64B {
         /** Pointer where the input data is available. */
         u64 dptr;
 
         /** Instruction Header. */
-        u64 ih;
+        u64 ih2;
 
         /** Input Request Header. */
         u64 irh;
@@ -198,14 +217,44 @@ struct octeon_instr_64B {
         u64 rptr;
 
         u64 reserved;
+};
+
+struct octeon_instr3_64B {
+        /** Pointer where the input data is available. */
+        u64 dptr;
+
+        /** Instruction Header. */
+        u64 ih3;
+
+        /** Instruction Header. */
+        u64 pki_ih3;
+
+        /** Input Request Header. */
+        u64 irh;
 
+        /** opcode/subcode specific parameters */
+        u64 ossp[2];
+
+        /** Return Data Parameters */
+        u64 rdp;
+
+        /** Pointer where the response for a RAW mode packet will be written
+         * by Octeon.
+         */
+        u64 rptr;
+
+};
+
+union octeon_instr_64B {
+        struct octeon_instr2_64B cmd2;
+        struct octeon_instr3_64B cmd3;
 };
 
-#define OCT_64B_INSTR_SIZE     (sizeof(struct octeon_instr_64B))
+#define OCT_64B_INSTR_SIZE     (sizeof(union octeon_instr_64B))
 
 /** The size of each buffer in soft command buffer pool
  */
-#define  SOFT_COMMAND_BUFFER_SIZE       1024
+#define  SOFT_COMMAND_BUFFER_SIZE       1536
 
 struct octeon_soft_command {
         /** Soft command buffer info. */
@@ -214,7 +263,8 @@ struct octeon_soft_command {
         u32 size;
 
         /** Command and return status */
-        struct octeon_instr_64B cmd;
+        union octeon_instr_64B cmd;
+
 #define COMPLETION_WORD_INIT    0xffffffffffffffffULL
         u64 *status_word;
 
@@ -242,7 +292,7 @@ struct octeon_soft_command {
 
 /** Maximum number of buffers to allocate into soft command buffer pool
  */
-#define  MAX_SOFT_COMMAND_BUFFERS       16
+#define  MAX_SOFT_COMMAND_BUFFERS       256
 
 /** Head of a soft command buffer pool.
  */
@@ -268,14 +318,15 @@ void octeon_free_soft_command(struct octeon_device *oct,
 /**
  *  octeon_init_instr_queue()
  *  @param octeon_dev      - pointer to the octeon device structure.
- *  @param iq_no           - queue to be initialized (0 <= q_no <= 3).
+ *  @param txpciq          - queue to be initialized (0 <= q_no <= 3).
  *
  *  Called at driver init time for each input queue. iq_conf has the
  *  configuration parameters for the queue.
  *
  *  @return  Success: 0   Failure: 1
  */
-int octeon_init_instr_queue(struct octeon_device *octeon_dev, u32 iq_no,
+int octeon_init_instr_queue(struct octeon_device *octeon_dev,
+                            union oct_txpciq txpciq,
                             u32 num_descs);
 
 /**
@@ -298,7 +349,7 @@ octeon_register_reqtype_free_fn(struct octeon_device *oct, int reqtype,
 
 int
 lio_process_iq_request_list(struct octeon_device *oct,
-                            struct octeon_instr_queue *iq);
+                            struct octeon_instr_queue *iq, u32 napi_budget);
 
 int octeon_send_command(struct octeon_device *oct, u32 iq_no,
                         u32 force_db, void *cmd, void *buf,
@@ -313,7 +364,10 @@ void octeon_prepare_soft_command(struct octeon_device *oct,
 int octeon_send_soft_command(struct octeon_device *oct,
                              struct octeon_soft_command *sc);
 
-int octeon_setup_iq(struct octeon_device *oct, u32 iq_no,
-                    u32 num_descs, void *app_ctx);
-
+int octeon_setup_iq(struct octeon_device *oct, int ifidx,
+                    int q_index, union oct_txpciq iq_no, u32 num_descs,
+                    void *app_ctx);
+int
+octeon_flush_iq(struct octeon_device *oct, struct octeon_instr_queue *iq,
+                u32 pending_thresh, u32 napi_budget);
 #endif                          /* __OCTEON_IQ_H__ */
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_main.h b/drivers/net/ethernet/cavium/liquidio/octeon_main.h
index cbd081981180..0ff3efc67b84 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_main.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_main.h
@@ -126,22 +126,27 @@ static inline int octeon_map_pci_barx(struct octeon_device *oct,
 }
 
 static inline void *
-cnnic_alloc_aligned_dma(struct pci_dev *pci_dev,
-                        u32 size,
-                        u32 *alloc_size,
-                        size_t *orig_ptr,
-                        size_t *dma_addr __attribute__((unused)))
+cnnic_numa_alloc_aligned_dma(u32 size,
+                             u32 *alloc_size,
+                             size_t *orig_ptr,
+                             int numa_node)
 {
         int retries = 0;
         void *ptr = NULL;
 
 #define OCTEON_MAX_ALLOC_RETRIES     1
         do {
-                ptr =
-                    (void *)__get_free_pages(GFP_KERNEL,
-                                             get_order(size));
+                struct page *page = NULL;
+
+                page = alloc_pages_node(numa_node,
+                                        GFP_KERNEL,
+                                        get_order(size));
+                if (!page)
+                        page = alloc_pages(GFP_KERNEL,
+                                           get_order(size));
+                ptr = (void *)page_address(page);
                 if ((unsigned long)ptr & 0x07) {
-                        free_pages((unsigned long)ptr, get_order(size));
+                        __free_pages(page, get_order(size));
                         ptr = NULL;
                         /* Increment the size required if the first
                          * attempt failed.
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_network.h b/drivers/net/ethernet/cavium/liquidio/octeon_network.h
index b3abe5818fd3..b481edc56c6e 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_network.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_network.h
@@ -30,6 +30,17 @@
 #include <linux/dma-mapping.h>
 #include <linux/ptp_clock_kernel.h>
 
+struct oct_nic_stats_resp {
+        u64     rh;
+        struct oct_link_stats stats;
+        u64     status;
+};
+
+struct oct_nic_stats_ctrl {
+        struct completion complete;
+        struct net_device *netdev;
+};
+
 /** LiquidIO per-interface network private data */
 struct lio {
         /** State of the interface. Rx/Tx happens only in the RUNNING state.  */
@@ -48,11 +59,11 @@ struct lio {
          */
         int rxq;
 
-        /** Guards the glist */
-        spinlock_t lock;
+        /** Guards each glist */
+        spinlock_t *glist_lock;
 
-        /** Linked list of gather components */
-        struct list_head glist;
+        /** Array of gather component linked lists */
+        struct list_head *glist;
 
         /** Pointer to the NIC properties for the Octeon device this network
          *  interface is associated with.
@@ -67,6 +78,9 @@ struct lio {
         /** Link information sent by the core application for this interface. */
         struct oct_link_info linfo;
 
+        /** counter of link changes */
+        u64 link_changes;
+
         /** Size of Tx queue for this octeon device. */
         u32 tx_qsize;
 
@@ -111,8 +125,9 @@ struct lio {
  * \brief Enable or disable feature
  * @param netdev    pointer to network device
  * @param cmd       Command that just requires acknowledgment
+ * @param param1    Parameter to command
  */
-int liquidio_set_feature(struct net_device *netdev, int cmd);
+int liquidio_set_feature(struct net_device *netdev, int cmd, u16 param1);
 
 /**
  * \brief Link control command completion callback
@@ -131,14 +146,30 @@ void liquidio_link_ctrl_cmd_completion(void *nctrl_ptr);
  */
 void liquidio_set_ethtool_ops(struct net_device *netdev);
 
-static inline void
-*recv_buffer_alloc(struct octeon_device *oct __attribute__((unused)),
-                   u32 q_no __attribute__((unused)), u32 size)
-{
 #define SKB_ADJ_MASK  0x3F
 #define SKB_ADJ       (SKB_ADJ_MASK + 1)
 
-        struct sk_buff *skb = dev_alloc_skb(size + SKB_ADJ);
+#define MIN_SKB_SIZE       256 /* 8 bytes and more - 8 bytes for PTP */
+#define LIO_RXBUFFER_SZ    2048
+
+static inline void
+*recv_buffer_alloc(struct octeon_device *oct,
+                   struct octeon_skb_page_info *pg_info)
+{
+        struct page *page;
+        struct sk_buff *skb;
+        struct octeon_skb_page_info *skb_pg_info;
+
+        page = alloc_page(GFP_ATOMIC | __GFP_COLD);
+        if (unlikely(!page))
+                return NULL;
+
+        skb = dev_alloc_skb(MIN_SKB_SIZE + SKB_ADJ);
+        if (unlikely(!skb)) {
+                __free_page(page);
+                pg_info->page = NULL;
+                return NULL;
+        }
 
         if ((unsigned long)skb->data & SKB_ADJ_MASK) {
                 u32 r = SKB_ADJ - ((unsigned long)skb->data & SKB_ADJ_MASK);
@@ -146,11 +177,151 @@ static inline void
                 skb_reserve(skb, r);
         }
 
+        skb_pg_info = ((struct octeon_skb_page_info *)(skb->cb));
+        /* Get DMA info */
+        pg_info->dma = dma_map_page(&oct->pci_dev->dev, page, 0,
+                                    PAGE_SIZE, DMA_FROM_DEVICE);
+
+        /* Mapping failed!! */
+        if (dma_mapping_error(&oct->pci_dev->dev, pg_info->dma)) {
+                __free_page(page);
+                dev_kfree_skb_any((struct sk_buff *)skb);
+                pg_info->page = NULL;
+                return NULL;
+        }
+
+        pg_info->page = page;
+        pg_info->page_offset = 0;
+        skb_pg_info->page = page;
+        skb_pg_info->page_offset = 0;
+        skb_pg_info->dma = pg_info->dma;
+
         return (void *)skb;
 }
 
+static inline void
+*recv_buffer_fast_alloc(u32 size)
+{
+        struct sk_buff *skb;
+        struct octeon_skb_page_info *skb_pg_info;
+
+        skb = dev_alloc_skb(size + SKB_ADJ);
+        if (unlikely(!skb))
+                return NULL;
+
+        if ((unsigned long)skb->data & SKB_ADJ_MASK) {
+                u32 r = SKB_ADJ - ((unsigned long)skb->data & SKB_ADJ_MASK);
+
+                skb_reserve(skb, r);
+        }
+
+        skb_pg_info = ((struct octeon_skb_page_info *)(skb->cb));
+        skb_pg_info->page = NULL;
+        skb_pg_info->page_offset = 0;
+        skb_pg_info->dma = 0;
+
+        return skb;
+}
+
+static inline int
+recv_buffer_recycle(struct octeon_device *oct, void *buf)
+{
+        struct octeon_skb_page_info *pg_info = buf;
+
+        if (!pg_info->page) {
+                dev_err(&oct->pci_dev->dev, "%s: pg_info->page NULL\n",
+                        __func__);
+                return -ENOMEM;
+        }
+
+        if (unlikely(page_count(pg_info->page) != 1) ||
+            unlikely(page_to_nid(pg_info->page) != numa_node_id())) {
+                dma_unmap_page(&oct->pci_dev->dev,
+                               pg_info->dma, (PAGE_SIZE << 0),
+                               DMA_FROM_DEVICE);
+                pg_info->dma = 0;
+                pg_info->page = NULL;
+                pg_info->page_offset = 0;
+                return -ENOMEM;
+        }
+
+        /* Flip to other half of the buffer */
+        if (pg_info->page_offset == 0)
+                pg_info->page_offset = LIO_RXBUFFER_SZ;
+        else
+                pg_info->page_offset = 0;
+        page_ref_inc(pg_info->page);
+
+        return 0;
+}
+
+static inline void
+*recv_buffer_reuse(struct octeon_device *oct, void *buf)
+{
+        struct octeon_skb_page_info *pg_info = buf, *skb_pg_info;
+        struct sk_buff *skb;
+
+        skb = dev_alloc_skb(MIN_SKB_SIZE + SKB_ADJ);
+        if (unlikely(!skb)) {
+                dma_unmap_page(&oct->pci_dev->dev,
+                               pg_info->dma, (PAGE_SIZE << 0),
+                               DMA_FROM_DEVICE);
+                return NULL;
+        }
+
+        if ((unsigned long)skb->data & SKB_ADJ_MASK) {
+                u32 r = SKB_ADJ - ((unsigned long)skb->data & SKB_ADJ_MASK);
+
+                skb_reserve(skb, r);
+        }
+
+        skb_pg_info = ((struct octeon_skb_page_info *)(skb->cb));
+        skb_pg_info->page = pg_info->page;
+        skb_pg_info->page_offset = pg_info->page_offset;
+        skb_pg_info->dma = pg_info->dma;
+
+        return skb;
+}
+
+static inline void
+recv_buffer_destroy(void *buffer, struct octeon_skb_page_info *pg_info)
+{
+        struct sk_buff *skb = (struct sk_buff *)buffer;
+
+        put_page(pg_info->page);
+        pg_info->dma = 0;
+        pg_info->page = NULL;
+        pg_info->page_offset = 0;
+
+        if (skb)
+                dev_kfree_skb_any(skb);
+}
+
 static inline void recv_buffer_free(void *buffer)
 {
+        struct sk_buff *skb = (struct sk_buff *)buffer;
+        struct octeon_skb_page_info *pg_info;
+
+        pg_info = ((struct octeon_skb_page_info *)(skb->cb));
+
+        if (pg_info->page) {
+                put_page(pg_info->page);
+                pg_info->dma = 0;
+                pg_info->page = NULL;
+                pg_info->page_offset = 0;
+        }
+
+        dev_kfree_skb_any((struct sk_buff *)buffer);
+}
+
+static inline void
+recv_buffer_fast_free(void *buffer)
+{
+        dev_kfree_skb_any((struct sk_buff *)buffer);
+}
+
+static inline void tx_buffer_free(void *buffer)
+{
         dev_kfree_skb_any((struct sk_buff *)buffer);
 }
 
@@ -159,7 +330,17 @@ static inline void recv_buffer_free(void *buffer)
 #define lio_dma_free(oct, size, virt_addr, dma_addr) \
         dma_free_coherent(&oct->pci_dev->dev, size, virt_addr, dma_addr)
 
-#define   get_rbd(ptr)      (((struct sk_buff *)(ptr))->data)
+static inline
+void *get_rbd(struct sk_buff *skb)
+{
+        struct octeon_skb_page_info *pg_info;
+        unsigned char *va;
+
+        pg_info = ((struct octeon_skb_page_info *)(skb->cb));
+        va = page_address(pg_info->page) + pg_info->page_offset;
+
+        return va;
+}
 
 static inline u64
 lio_map_ring_info(struct octeon_droq *droq, u32 i)
@@ -183,33 +364,44 @@ lio_unmap_ring_info(struct pci_dev *pci_dev,
 }
 
 static inline u64
-lio_map_ring(struct pci_dev *pci_dev,
-             void *buf, u32 size)
+lio_map_ring(void *buf)
 {
         dma_addr_t dma_addr;
 
-        dma_addr = dma_map_single(&pci_dev->dev, get_rbd(buf), size,
-                                  DMA_FROM_DEVICE);
+        struct sk_buff *skb = (struct sk_buff *)buf;
+        struct octeon_skb_page_info *pg_info;
 
-        BUG_ON(dma_mapping_error(&pci_dev->dev, dma_addr));
+        pg_info = ((struct octeon_skb_page_info *)(skb->cb));
+        if (!pg_info->page) {
+                pr_err("%s: pg_info->page NULL\n", __func__);
+                WARN_ON(1);
+        }
+
+        /* Get DMA info */
+        dma_addr = pg_info->dma;
+        if (!pg_info->dma) {
+                pr_err("%s: ERROR it should be already available\n",
+                       __func__);
+                WARN_ON(1);
+        }
+        dma_addr += pg_info->page_offset;
 
         return (u64)dma_addr;
 }
 
 static inline void
 lio_unmap_ring(struct pci_dev *pci_dev,
-               u64 buf_ptr, u32 size)
+               u64 buf_ptr)
+
 {
-        dma_unmap_single(&pci_dev->dev,
-                         buf_ptr, size,
-                         DMA_FROM_DEVICE);
+        dma_unmap_page(&pci_dev->dev,
+                       buf_ptr, (PAGE_SIZE << 0),
+                       DMA_FROM_DEVICE);
 }
 
-static inline void *octeon_fast_packet_alloc(struct octeon_device *oct,
-                                             struct octeon_droq *droq,
-                                             u32 q_no, u32 size)
+static inline void *octeon_fast_packet_alloc(u32 size)
 {
-        return recv_buffer_alloc(oct, q_no, size);
+        return recv_buffer_fast_alloc(size);
 }
 
 static inline void octeon_fast_packet_next(struct octeon_droq *droq,
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_nic.c b/drivers/net/ethernet/cavium/liquidio/octeon_nic.c
index 1a0191549cb3..36f1970a860e 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_nic.c
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_nic.c
@@ -44,11 +44,11 @@
 
 void *
 octeon_alloc_soft_command_resp(struct octeon_device    *oct,
-                               struct octeon_instr_64B *cmd,
-                               size_t                  rdatasize)
+                               union octeon_instr_64B *cmd,
+                               u32                     rdatasize)
 {
         struct octeon_soft_command *sc;
-        struct octeon_instr_ih  *ih;
+        struct octeon_instr_ih2  *ih2;
         struct octeon_instr_irh *irh;
         struct octeon_instr_rdp *rdp;
 
@@ -59,24 +59,25 @@ octeon_alloc_soft_command_resp(struct octeon_device    *oct,
                 return NULL;
 
         /* Copy existing command structure into the soft command */
-        memcpy(&sc->cmd, cmd, sizeof(struct octeon_instr_64B));
+        memcpy(&sc->cmd, cmd, sizeof(union octeon_instr_64B));
 
         /* Add in the response related fields. Opcode and Param are already
          * there.
          */
-        ih      = (struct octeon_instr_ih *)&sc->cmd.ih;
-        ih->fsz = 40; /* irh + ossp[0] + ossp[1] + rdp + rptr = 40 bytes */
+        ih2      = (struct octeon_instr_ih2 *)&sc->cmd.cmd2.ih2;
+        rdp     = (struct octeon_instr_rdp *)&sc->cmd.cmd2.rdp;
+        irh     = (struct octeon_instr_irh *)&sc->cmd.cmd2.irh;
+        ih2->fsz = 40; /* irh + ossp[0] + ossp[1] + rdp + rptr = 40 bytes */
 
-        irh        = (struct octeon_instr_irh *)&sc->cmd.irh;
         irh->rflag = 1; /* a response is required */
-        irh->len   = 4; /* means four 64-bit words immediately follow irh */
 
-        rdp            = (struct octeon_instr_rdp *)&sc->cmd.rdp;
         rdp->pcie_port = oct->pcie_port;
         rdp->rlen      = rdatasize;
 
         *sc->status_word = COMPLETION_WORD_INIT;
 
+        sc->cmd.cmd2.rptr =  sc->dmarptr;
+
         sc->wait_time = 1000;
         sc->timeout = jiffies + sc->wait_time;
 
@@ -119,12 +120,11 @@ static void octnet_link_ctrl_callback(struct octeon_device *oct,
 
 static inline struct octeon_soft_command
 *octnic_alloc_ctrl_pkt_sc(struct octeon_device *oct,
-                          struct octnic_ctrl_pkt *nctrl,
-                          struct octnic_ctrl_params nparams)
+                          struct octnic_ctrl_pkt *nctrl)
 {
         struct octeon_soft_command *sc = NULL;
         u8 *data;
-        size_t rdatasize;
+        u32 rdatasize;
         u32 uddsize = 0, datasize = 0;
 
         uddsize = (u32)(nctrl->ncmd.s.more * 8);
@@ -143,7 +143,7 @@ static inline struct octeon_soft_command
 
         data = (u8 *)sc->virtdptr;
 
-        memcpy(data, &nctrl->ncmd,  OCTNET_CMD_SIZE);
+        memcpy(data, &nctrl->ncmd, OCTNET_CMD_SIZE);
 
         octeon_swap_8B_data((u64 *)data, (OCTNET_CMD_SIZE >> 3));
 
@@ -152,6 +152,8 @@ static inline struct octeon_soft_command
                 memcpy(data + OCTNET_CMD_SIZE, nctrl->udd, uddsize);
         }
 
+        sc->iq_no = (u32)nctrl->iq_no;
+
         octeon_prepare_soft_command(oct, sc, OPCODE_NIC, OPCODE_NIC_CMD,
                                     0, 0, 0);
 
@@ -164,26 +166,41 @@ static inline struct octeon_soft_command
 
 int
 octnet_send_nic_ctrl_pkt(struct octeon_device *oct,
-                         struct octnic_ctrl_pkt *nctrl,
-                         struct octnic_ctrl_params nparams)
+                         struct octnic_ctrl_pkt *nctrl)
 {
         int retval;
         struct octeon_soft_command *sc = NULL;
 
-        sc = octnic_alloc_ctrl_pkt_sc(oct, nctrl, nparams);
+        spin_lock_bh(&oct->cmd_resp_wqlock);
+        /* Allow only rx ctrl command to stop traffic on the chip
+         * during offline operations
+         */
+        if ((oct->cmd_resp_state == OCT_DRV_OFFLINE) &&
+            (nctrl->ncmd.s.cmd != OCTNET_CMD_RX_CTL)) {
+                spin_unlock_bh(&oct->cmd_resp_wqlock);
+                dev_err(&oct->pci_dev->dev,
+                        "%s cmd:%d not processed since driver offline\n",
+                        __func__, nctrl->ncmd.s.cmd);
+                return -1;
+        }
+
+        sc = octnic_alloc_ctrl_pkt_sc(oct, nctrl);
         if (!sc) {
                 dev_err(&oct->pci_dev->dev, "%s soft command alloc failed\n",
                         __func__);
+                spin_unlock_bh(&oct->cmd_resp_wqlock);
                 return -1;
         }
 
         retval = octeon_send_soft_command(oct, sc);
-        if (retval) {
+        if (retval == IQ_SEND_FAILED) {
                 octeon_free_soft_command(oct, sc);
-                dev_err(&oct->pci_dev->dev, "%s soft command send failed status: %x\n",
-                        __func__, retval);
+                dev_err(&oct->pci_dev->dev, "%s soft command:%d send failed status: %x\n",
+                        __func__, nctrl->ncmd.s.cmd, retval);
+                spin_unlock_bh(&oct->cmd_resp_wqlock);
                 return -1;
         }
 
+        spin_unlock_bh(&oct->cmd_resp_wqlock);
         return retval;
 }
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_nic.h b/drivers/net/ethernet/cavium/liquidio/octeon_nic.h
index 0238857c8105..b71a2bbe4bee 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_nic.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_nic.h
@@ -52,6 +52,9 @@ struct octnic_ctrl_pkt {
         /** Additional data that may be needed by some commands. */
         u64 udd[MAX_NCTRL_UDD];
 
+        /** Input queue to use to send this command. */
+        u64 iq_no;
+
         /** Time to wait for Octeon software to respond to this control command.
          *  If wait_time is 0, OSI assumes no response is expected.
          */
@@ -82,7 +85,7 @@ struct octnic_data_pkt {
         u32 datasize;
 
         /** Command to be passed to the Octeon device software. */
-        struct octeon_instr_64B cmd;
+        union octeon_instr_64B cmd;
 
         /** Input queue to use to send this command. */
         u32 q_no;
@@ -94,15 +97,14 @@ struct octnic_data_pkt {
  */
 union octnic_cmd_setup {
         struct {
-                u32 ifidx:8;
-                u32 cksum_offset:7;
+                u32 iq_no:8;
                 u32 gather:1;
                 u32 timestamp:1;
-                u32 ipv4opts_ipv6exthdr:2;
                 u32 ip_csum:1;
+                u32 transport_csum:1;
                 u32 tnl_csum:1;
+                u32 rsvd:19;
 
-                u32 rsvd:11;
                 union {
                         u32 datasize;
                         u32 gatherptrs;
@@ -113,79 +115,146 @@ union octnic_cmd_setup {
 
 };
 
-struct octnic_ctrl_params {
-        u32 resp_order;
-};
-
 static inline int octnet_iq_is_full(struct octeon_device *oct, u32 q_no)
 {
         return ((u32)atomic_read(&oct->instr_queue[q_no]->instr_pending)
                 >= (oct->instr_queue[q_no]->max_count - 2));
 }
 
-/** Utility function to prepare a 64B NIC instruction based on a setup command
- * @param cmd - pointer to instruction to be filled in.
- * @param setup - pointer to the setup structure
- * @param q_no - which queue for back pressure
- *
- * Assumes the cmd instruction is pre-allocated, but no fields are filled in.
- */
 static inline void
-octnet_prepare_pci_cmd(struct octeon_instr_64B *cmd,
-                       union octnic_cmd_setup *setup, u32 tag)
+octnet_prepare_pci_cmd_o2(struct octeon_device *oct,
+                          union octeon_instr_64B *cmd,
+                          union octnic_cmd_setup *setup, u32 tag)
 {
-        struct octeon_instr_ih *ih;
+        struct octeon_instr_ih2 *ih2;
         struct octeon_instr_irh *irh;
         union octnic_packet_params packet_params;
+        int port;
 
-        memset(cmd, 0, sizeof(struct octeon_instr_64B));
+        memset(cmd, 0, sizeof(union octeon_instr_64B));
 
-        ih = (struct octeon_instr_ih *)&cmd->ih;
+        ih2 = (struct octeon_instr_ih2 *)&cmd->cmd2.ih2;
 
         /* assume that rflag is cleared so therefore front data will only have
-         * irh and ossp[1] and ossp[2] for a total of 24 bytes
+         * irh and ossp[0], ossp[1] for a total of 32 bytes
          */
-        ih->fsz = 24;
+        ih2->fsz = 24;
+
+        ih2->tagtype = ORDERED_TAG;
+        ih2->grp = DEFAULT_POW_GRP;
 
-        ih->tagtype = ORDERED_TAG;
-        ih->grp = DEFAULT_POW_GRP;
+        port = (int)oct->instr_queue[setup->s.iq_no]->txpciq.s.port;
 
         if (tag)
-                ih->tag = tag;
+                ih2->tag = tag;
         else
-                ih->tag = LIO_DATA(setup->s.ifidx);
+                ih2->tag = LIO_DATA(port);
 
-        ih->raw = 1;
-        ih->qos = (setup->s.ifidx & 3) + 4;     /* map qos based on interface */
+        ih2->raw = 1;
+        ih2->qos = (port & 3) + 4;      /* map qos based on interface */
 
         if (!setup->s.gather) {
-                ih->dlengsz = setup->s.u.datasize;
+                ih2->dlengsz = setup->s.u.datasize;
         } else {
-                ih->gather = 1;
-                ih->dlengsz = setup->s.u.gatherptrs;
+                ih2->gather = 1;
+                ih2->dlengsz = setup->s.u.gatherptrs;
         }
 
-        irh = (struct octeon_instr_irh *)&cmd->irh;
+        irh = (struct octeon_instr_irh *)&cmd->cmd2.irh;
 
         irh->opcode = OPCODE_NIC;
         irh->subcode = OPCODE_NIC_NW_DATA;
 
         packet_params.u32 = 0;
 
-        if (setup->s.cksum_offset) {
-                packet_params.s.csoffset = setup->s.cksum_offset;
-                packet_params.s.ipv4opts_ipv6exthdr =
-                                                setup->s.ipv4opts_ipv6exthdr;
+        packet_params.s.ip_csum = setup->s.ip_csum;
+        packet_params.s.transport_csum = setup->s.transport_csum;
+        packet_params.s.tnl_csum = setup->s.tnl_csum;
+        packet_params.s.tsflag = setup->s.timestamp;
+
+        irh->ossp = packet_params.u32;
+}
+
+static inline void
+octnet_prepare_pci_cmd_o3(struct octeon_device *oct,
+                          union octeon_instr_64B *cmd,
+                          union octnic_cmd_setup *setup, u32 tag)
+{
+        struct octeon_instr_irh *irh;
+        struct octeon_instr_ih3     *ih3;
+        struct octeon_instr_pki_ih3 *pki_ih3;
+        union octnic_packet_params packet_params;
+        int port;
+
+        memset(cmd, 0, sizeof(union octeon_instr_64B));
+
+        ih3 = (struct octeon_instr_ih3 *)&cmd->cmd3.ih3;
+        pki_ih3 = (struct octeon_instr_pki_ih3 *)&cmd->cmd3.pki_ih3;
+
+        /* assume that rflag is cleared so therefore front data will only have
+         * irh and ossp[1] and ossp[2] for a total of 24 bytes
+         */
+        ih3->pkind       = oct->instr_queue[setup->s.iq_no]->txpciq.s.pkind;
+        /*PKI IH*/
+        ih3->fsz = 24 + 8;
+
+        if (!setup->s.gather) {
+                ih3->dlengsz = setup->s.u.datasize;
+        } else {
+                ih3->gather = 1;
+                ih3->dlengsz = setup->s.u.gatherptrs;
         }
 
+        pki_ih3->w       = 1;
+        pki_ih3->raw     = 1;
+        pki_ih3->utag    = 1;
+        pki_ih3->utt     = 1;
+        pki_ih3->uqpg    = oct->instr_queue[setup->s.iq_no]->txpciq.s.use_qpg;
+
+        port = (int)oct->instr_queue[setup->s.iq_no]->txpciq.s.port;
+
+        if (tag)
+                pki_ih3->tag = tag;
+        else
+                pki_ih3->tag     = LIO_DATA(port);
+
+        pki_ih3->tagtype = ORDERED_TAG;
+        pki_ih3->qpg     = oct->instr_queue[setup->s.iq_no]->txpciq.s.qpg;
+        pki_ih3->pm      = 0x7; /*0x7 - meant for Parse nothing, uninterpreted*/
+        pki_ih3->sl      = 8;   /* sl will be sizeof(pki_ih3)*/
+
+        irh = (struct octeon_instr_irh *)&cmd->cmd3.irh;
+
+        irh->opcode = OPCODE_NIC;
+        irh->subcode = OPCODE_NIC_NW_DATA;
+
+        packet_params.u32 = 0;
+
         packet_params.s.ip_csum = setup->s.ip_csum;
+        packet_params.s.transport_csum = setup->s.transport_csum;
         packet_params.s.tnl_csum = setup->s.tnl_csum;
-        packet_params.s.ifidx = setup->s.ifidx;
         packet_params.s.tsflag = setup->s.timestamp;
 
         irh->ossp = packet_params.u32;
 }
 
+/** Utility function to prepare a 64B NIC instruction based on a setup command
+ * @param cmd - pointer to instruction to be filled in.
+ * @param setup - pointer to the setup structure
+ * @param q_no - which queue for back pressure
+ *
+ * Assumes the cmd instruction is pre-allocated, but no fields are filled in.
+ */
+static inline void
+octnet_prepare_pci_cmd(struct octeon_device *oct, union octeon_instr_64B *cmd,
+                       union octnic_cmd_setup *setup, u32 tag)
+{
+        if (OCTEON_CN6XXX(oct))
+                octnet_prepare_pci_cmd_o2(oct, cmd, setup, tag);
+        else
+                octnet_prepare_pci_cmd_o3(oct, cmd, setup, tag);
+}
+
 /** Allocate and a soft command with space for a response immediately following
  * the commnad.
  * @param oct - octeon device pointer
@@ -198,8 +267,8 @@ octnet_prepare_pci_cmd(struct octeon_instr_64B *cmd,
  */
 void *
 octeon_alloc_soft_command_resp(struct octeon_device    *oct,
-                               struct octeon_instr_64B *cmd,
-                               size_t                  rdatasize);
+                               union octeon_instr_64B *cmd,
+                               u32                     rdatasize);
 
 /** Send a NIC data packet to the device
  * @param oct - octeon device pointer
@@ -214,14 +283,11 @@ int octnet_send_nic_data_pkt(struct octeon_device *oct,
 /** Send a NIC control packet to the device
  * @param oct - octeon device pointer
  * @param nctrl - control structure with command, timout, and callback info
- * @param nparams - response control structure
- *
  * @returns IQ_FAILED if it failed to add to the input queue. IQ_STOP if it the
  * queue should be stopped, and IQ_SEND_OK if it sent okay.
  */
 int
 octnet_send_nic_ctrl_pkt(struct octeon_device *oct,
-                         struct octnic_ctrl_pkt *nctrl,
-                         struct octnic_ctrl_params nparams);
+                         struct octnic_ctrl_pkt *nctrl);
 
 #endif
diff --git a/drivers/net/ethernet/cavium/liquidio/request_manager.c b/drivers/net/ethernet/cavium/liquidio/request_manager.c
index a2a24652c8f3..7eafa75ac095 100644
--- a/drivers/net/ethernet/cavium/liquidio/request_manager.c
+++ b/drivers/net/ethernet/cavium/liquidio/request_manager.c
@@ -51,7 +51,7 @@ struct iq_post_status {
 };
 
 static void check_db_timeout(struct work_struct *work);
-static void  __check_db_timeout(struct octeon_device *oct, unsigned long iq_no);
+static void  __check_db_timeout(struct octeon_device *oct, u64 iq_no);
 
 static void (*reqtype_free_fn[MAX_OCTEON_DEVICES][REQTYPE_LAST + 1]) (void *);
 
@@ -69,12 +69,16 @@ static inline int IQ_INSTR_MODE_64B(struct octeon_device *oct, int iq_no)
 
 /* Return 0 on success, 1 on failure */
 int octeon_init_instr_queue(struct octeon_device *oct,
-                            u32 iq_no, u32 num_descs)
+                            union oct_txpciq txpciq,
+                            u32 num_descs)
 {
         struct octeon_instr_queue *iq;
         struct octeon_iq_config *conf = NULL;
+        u32 iq_no = (u32)txpciq.s.q_no;
         u32 q_size;
         struct cavium_wq *db_wq;
+        int orig_node = dev_to_node(&oct->pci_dev->dev);
+        int numa_node = cpu_to_node(iq_no % num_online_cpus());
 
         if (OCTEON_CN6XXX(oct))
                 conf = &(CFG_GET_IQ_CFG(CHIP_FIELD(oct, cn6xxx, conf)));
@@ -95,9 +99,15 @@ int octeon_init_instr_queue(struct octeon_device *oct,
         q_size = (u32)conf->instr_type * num_descs;
 
         iq = oct->instr_queue[iq_no];
+        iq->oct_dev = oct;
 
+        set_dev_node(&oct->pci_dev->dev, numa_node);
         iq->base_addr = lio_dma_alloc(oct, q_size,
                                       (dma_addr_t *)&iq->base_addr_dma);
+        set_dev_node(&oct->pci_dev->dev, orig_node);
+        if (!iq->base_addr)
+                iq->base_addr = lio_dma_alloc(oct, q_size,
+                                              (dma_addr_t *)&iq->base_addr_dma);
         if (!iq->base_addr) {
                 dev_err(&oct->pci_dev->dev, "Cannot allocate memory for instr queue %d\n",
                         iq_no);
@@ -109,7 +119,11 @@ int octeon_init_instr_queue(struct octeon_device *oct,
         /* Initialize a list to holds requests that have been posted to Octeon
          * but has yet to be fetched by octeon
          */
-        iq->request_list = vmalloc(sizeof(*iq->request_list) * num_descs);
+        iq->request_list = vmalloc_node((sizeof(*iq->request_list) * num_descs),
+                                               numa_node);
+        if (!iq->request_list)
+                iq->request_list = vmalloc(sizeof(*iq->request_list) *
+                                                  num_descs);
         if (!iq->request_list) {
                 lio_dma_free(oct, q_size, iq->base_addr, iq->base_addr_dma);
                 dev_err(&oct->pci_dev->dev, "Alloc failed for IQ[%d] nr free list\n",
@@ -122,7 +136,7 @@ int octeon_init_instr_queue(struct octeon_device *oct,
         dev_dbg(&oct->pci_dev->dev, "IQ[%d]: base: %p basedma: %llx count: %d\n",
                 iq_no, iq->base_addr, iq->base_addr_dma, iq->max_count);
 
-        iq->iq_no = iq_no;
+        iq->txpciq.u64 = txpciq.u64;
         iq->fill_threshold = (u32)conf->db_min;
         iq->fill_cnt = 0;
         iq->host_write_index = 0;
@@ -135,8 +149,11 @@ int octeon_init_instr_queue(struct octeon_device *oct,
 
         /* Initialize the spinlock for this instruction queue */
         spin_lock_init(&iq->lock);
+        spin_lock_init(&iq->post_lock);
 
-        oct->io_qmask.iq |= (1 << iq_no);
+        spin_lock_init(&iq->iq_flush_running_lock);
+
+        oct->io_qmask.iq |= (1ULL << iq_no);
 
         /* Set the 32B/64B mode for each input queue */
         oct->io_qmask.iq64B |= ((conf->instr_type == 64) << iq_no);
@@ -144,7 +161,9 @@ int octeon_init_instr_queue(struct octeon_device *oct,
 
         oct->fn_list.setup_iq_regs(oct, iq_no);
 
-        oct->check_db_wq[iq_no].wq = create_workqueue("check_iq_db");
+        oct->check_db_wq[iq_no].wq = alloc_workqueue("check_iq_db",
+                                                     WQ_MEM_RECLAIM,
+                                                     0);
         if (!oct->check_db_wq[iq_no].wq) {
                 lio_dma_free(oct, q_size, iq->base_addr, iq->base_addr_dma);
                 dev_err(&oct->pci_dev->dev, "check db wq create failed for iq %d\n",
@@ -168,7 +187,6 @@ int octeon_delete_instr_queue(struct octeon_device *oct, u32 iq_no)
         struct octeon_instr_queue *iq = oct->instr_queue[iq_no];
 
         cancel_delayed_work_sync(&oct->check_db_wq[iq_no].wk.work);
-        flush_workqueue(oct->check_db_wq[iq_no].wq);
         destroy_workqueue(oct->check_db_wq[iq_no].wq);
 
         if (OCTEON_CN6XXX(oct))
@@ -188,26 +206,38 @@ int octeon_delete_instr_queue(struct octeon_device *oct, u32 iq_no)
 
 /* Return 0 on success, 1 on failure */
 int octeon_setup_iq(struct octeon_device *oct,
-                    u32 iq_no,
+                    int ifidx,
+                    int q_index,
+                    union oct_txpciq txpciq,
                     u32 num_descs,
                     void *app_ctx)
 {
+        u32 iq_no = (u32)txpciq.s.q_no;
+        int numa_node = cpu_to_node(iq_no % num_online_cpus());
+
         if (oct->instr_queue[iq_no]) {
                 dev_dbg(&oct->pci_dev->dev, "IQ is in use. Cannot create the IQ: %d again\n",
                         iq_no);
+                oct->instr_queue[iq_no]->txpciq.u64 = txpciq.u64;
                 oct->instr_queue[iq_no]->app_ctx = app_ctx;
                 return 0;
         }
         oct->instr_queue[iq_no] =
-            vmalloc(sizeof(struct octeon_instr_queue));
+            vmalloc_node(sizeof(struct octeon_instr_queue), numa_node);
+        if (!oct->instr_queue[iq_no])
+                oct->instr_queue[iq_no] =
+                    vmalloc(sizeof(struct octeon_instr_queue));
         if (!oct->instr_queue[iq_no])
                 return 1;
 
         memset(oct->instr_queue[iq_no], 0,
                sizeof(struct octeon_instr_queue));
 
+        oct->instr_queue[iq_no]->q_index = q_index;
         oct->instr_queue[iq_no]->app_ctx = app_ctx;
-        if (octeon_init_instr_queue(oct, iq_no, num_descs)) {
+        oct->instr_queue[iq_no]->ifidx = ifidx;
+
+        if (octeon_init_instr_queue(oct, txpciq, num_descs)) {
                 vfree(oct->instr_queue[iq_no]);
                 oct->instr_queue[iq_no] = NULL;
                 return 1;
@@ -226,8 +256,8 @@ int lio_wait_for_instr_fetch(struct octeon_device *oct)
                 instr_cnt = 0;
 
                 /*for (i = 0; i < oct->num_iqs; i++) {*/
-                for (i = 0; i < MAX_OCTEON_INSTR_QUEUES; i++) {
-                        if (!(oct->io_qmask.iq & (1UL << i)))
+                for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
+                        if (!(oct->io_qmask.iq & (1ULL << i)))
                                 continue;
                         pending =
                             atomic_read(&oct->
@@ -364,13 +394,13 @@ __add_to_request_list(struct octeon_instr_queue *iq,
 
 int
 lio_process_iq_request_list(struct octeon_device *oct,
-                            struct octeon_instr_queue *iq)
+                            struct octeon_instr_queue *iq, u32 napi_budget)
 {
         int reqtype;
         void *buf;
         u32 old = iq->flush_index;
         u32 inst_count = 0;
-        unsigned pkts_compl = 0, bytes_compl = 0;
+        unsigned int pkts_compl = 0, bytes_compl = 0;
         struct octeon_soft_command *sc;
         struct octeon_instr_irh *irh;
 
@@ -394,7 +424,7 @@ lio_process_iq_request_list(struct octeon_device *oct,
                 case REQTYPE_SOFT_COMMAND:
                         sc = buf;
 
-                        irh = (struct octeon_instr_irh *)&sc->cmd.irh;
+                        irh = (struct octeon_instr_irh *)&sc->cmd.cmd2.irh;
                         if (irh->rflag) {
                                 /* We're expecting a response from Octeon.
                                  * It's up to lio_process_ordered_list() to
@@ -430,6 +460,9 @@ lio_process_iq_request_list(struct octeon_device *oct,
  skip_this:
                 inst_count++;
                 INCR_INDEX_BY1(old, iq->max_count);
+
+                if ((napi_budget) && (inst_count >= napi_budget))
+                        break;
         }
         if (bytes_compl)
                 octeon_report_tx_completion_to_bql(iq->app_ctx, pkts_compl,
@@ -439,38 +472,63 @@ lio_process_iq_request_list(struct octeon_device *oct,
         return inst_count;
 }
 
-static inline void
-update_iq_indices(struct octeon_device *oct, struct octeon_instr_queue *iq)
+/* Can only be called from process context */
+int
+octeon_flush_iq(struct octeon_device *oct, struct octeon_instr_queue *iq,
+                u32 pending_thresh, u32 napi_budget)
 {
         u32 inst_processed = 0;
+        u32 tot_inst_processed = 0;
+        int tx_done = 1;
 
-        /* Calculate how many commands Octeon has read and move the read index
-         * accordingly.
-         */
-        iq->octeon_read_index = oct->fn_list.update_iq_read_idx(oct, iq);
+        if (!spin_trylock(&iq->iq_flush_running_lock))
+                return tx_done;
 
-        /* Move the NORESPONSE requests to the per-device completion list. */
-        if (iq->flush_index != iq->octeon_read_index)
-                inst_processed = lio_process_iq_request_list(oct, iq);
+        spin_lock_bh(&iq->lock);
 
-        if (inst_processed) {
-                atomic_sub(inst_processed, &iq->instr_pending);
-                iq->stats.instr_processed += inst_processed;
-        }
-}
+        iq->octeon_read_index = oct->fn_list.update_iq_read_idx(iq);
 
-static void
-octeon_flush_iq(struct octeon_device *oct, struct octeon_instr_queue *iq,
-                u32 pending_thresh)
-{
         if (atomic_read(&iq->instr_pending) >= (s32)pending_thresh) {
-                spin_lock_bh(&iq->lock);
-                update_iq_indices(oct, iq);
-                spin_unlock_bh(&iq->lock);
+                do {
+                        /* Process any outstanding IQ packets. */
+                        if (iq->flush_index == iq->octeon_read_index)
+                                break;
+
+                        if (napi_budget)
+                                inst_processed = lio_process_iq_request_list
+                                        (oct, iq,
+                                         napi_budget - tot_inst_processed);
+                        else
+                                inst_processed =
+                                        lio_process_iq_request_list(oct, iq, 0);
+
+                        if (inst_processed) {
+                                atomic_sub(inst_processed, &iq->instr_pending);
+                                iq->stats.instr_processed += inst_processed;
+                        }
+
+                        tot_inst_processed += inst_processed;
+                        inst_processed = 0;
+
+                } while (tot_inst_processed < napi_budget);
+
+                if (napi_budget && (tot_inst_processed >= napi_budget))
+                        tx_done = 0;
         }
+
+        iq->last_db_time = jiffies;
+
+        spin_unlock_bh(&iq->lock);
+
+        spin_unlock(&iq->iq_flush_running_lock);
+
+        return tx_done;
 }
 
-static void __check_db_timeout(struct octeon_device *oct, unsigned long iq_no)
+/* Process instruction queue after timeout.
+ * This routine gets called from a workqueue or when removing the module.
+ */
+static void __check_db_timeout(struct octeon_device *oct, u64 iq_no)
 {
         struct octeon_instr_queue *iq;
         u64 next_time;
@@ -481,24 +539,17 @@ static void __check_db_timeout(struct octeon_device *oct, unsigned long iq_no)
         if (!iq)
                 return;
 
+        /* return immediately, if no work pending */
+        if (!atomic_read(&iq->instr_pending))
+                return;
         /* If jiffies - last_db_time < db_timeout do nothing  */
         next_time = iq->last_db_time + iq->db_timeout;
         if (!time_after(jiffies, (unsigned long)next_time))
                 return;
         iq->last_db_time = jiffies;
 
-        /* Get the lock and prevent tasklets. This routine gets called from
-         * the poll thread. Instructions can now be posted in tasklet context
-         */
-        spin_lock_bh(&iq->lock);
-        if (iq->fill_cnt != 0)
-                ring_doorbell(oct, iq);
-
-        spin_unlock_bh(&iq->lock);
-
         /* Flush the instruction queue */
-        if (iq->do_auto_flush)
-                octeon_flush_iq(oct, iq, 1);
+        octeon_flush_iq(oct, iq, 1, 0);
 }
 
 /* Called by the Poll thread at regular intervals to check the instruction
@@ -523,7 +574,10 @@ octeon_send_command(struct octeon_device *oct, u32 iq_no,
         struct iq_post_status st;
         struct octeon_instr_queue *iq = oct->instr_queue[iq_no];
 
-        spin_lock_bh(&iq->lock);
+        /* Get the lock and prevent other tasks and tx interrupt handler from
+         * running.
+         */
+        spin_lock_bh(&iq->post_lock);
 
         st = __post_command2(oct, iq, force_db, cmd);
 
@@ -539,10 +593,13 @@ octeon_send_command(struct octeon_device *oct, u32 iq_no,
                 INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_dropped, 1);
         }
 
-        spin_unlock_bh(&iq->lock);
+        spin_unlock_bh(&iq->post_lock);
 
-        if (iq->do_auto_flush)
-                octeon_flush_iq(oct, iq, 2);
+        /* This is only done here to expedite packets being flushed
+         * for cases where there are no IQ completion interrupts.
+         */
+        /*if (iq->do_auto_flush)*/
+        /*      octeon_flush_iq(oct, iq, 2, 0);*/
 
         return st.status;
 }
@@ -557,7 +614,7 @@ octeon_prepare_soft_command(struct octeon_device *oct,
                             u64 ossp1)
 {
         struct octeon_config *oct_cfg;
-        struct octeon_instr_ih *ih;
+        struct octeon_instr_ih2 *ih2;
         struct octeon_instr_irh *irh;
         struct octeon_instr_rdp *rdp;
 
@@ -566,73 +623,69 @@ octeon_prepare_soft_command(struct octeon_device *oct,
 
         oct_cfg = octeon_get_conf(oct);
 
-        ih          = (struct octeon_instr_ih *)&sc->cmd.ih;
-        ih->tagtype = ATOMIC_TAG;
-        ih->tag     = LIO_CONTROL;
-        ih->raw     = 1;
-        ih->grp     = CFG_GET_CTRL_Q_GRP(oct_cfg);
+        ih2          = (struct octeon_instr_ih2 *)&sc->cmd.cmd2.ih2;
+        ih2->tagtype = ATOMIC_TAG;
+        ih2->tag     = LIO_CONTROL;
+        ih2->raw     = 1;
+        ih2->grp     = CFG_GET_CTRL_Q_GRP(oct_cfg);
 
         if (sc->datasize) {
-                ih->dlengsz = sc->datasize;
-                ih->rs = 1;
+                ih2->dlengsz = sc->datasize;
+                ih2->rs = 1;
         }
 
-        irh            = (struct octeon_instr_irh *)&sc->cmd.irh;
+        irh            = (struct octeon_instr_irh *)&sc->cmd.cmd2.irh;
         irh->opcode    = opcode;
         irh->subcode   = subcode;
 
         /* opcode/subcode specific parameters (ossp) */
         irh->ossp       = irh_ossp;
-        sc->cmd.ossp[0] = ossp0;
-        sc->cmd.ossp[1] = ossp1;
+        sc->cmd.cmd2.ossp[0] = ossp0;
+        sc->cmd.cmd2.ossp[1] = ossp1;
 
         if (sc->rdatasize) {
-                rdp            = (struct octeon_instr_rdp *)&sc->cmd.rdp;
+                rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd2.rdp;
                 rdp->pcie_port = oct->pcie_port;
                 rdp->rlen      = sc->rdatasize;
 
                 irh->rflag =  1;
-                irh->len   =  4;
-                ih->fsz    = 40; /* irh+ossp[0]+ossp[1]+rdp+rptr = 40 bytes */
+                ih2->fsz   = 40; /* irh+ossp[0]+ossp[1]+rdp+rptr = 40 bytes */
         } else {
                 irh->rflag =  0;
-                irh->len   =  2;
-                ih->fsz    = 24; /* irh + ossp[0] + ossp[1] = 24 bytes */
+                ih2->fsz   = 24; /* irh + ossp[0] + ossp[1] = 24 bytes */
         }
-
-        while (!(oct->io_qmask.iq & (1 << sc->iq_no)))
-                sc->iq_no++;
 }
 
 int octeon_send_soft_command(struct octeon_device *oct,
                              struct octeon_soft_command *sc)
 {
-        struct octeon_instr_ih *ih;
+        struct octeon_instr_ih2 *ih2;
         struct octeon_instr_irh *irh;
         struct octeon_instr_rdp *rdp;
+        u32 len;
 
-        ih = (struct octeon_instr_ih *)&sc->cmd.ih;
-        if (ih->dlengsz) {
-                BUG_ON(!sc->dmadptr);
-                sc->cmd.dptr = sc->dmadptr;
+        ih2 = (struct octeon_instr_ih2 *)&sc->cmd.cmd2.ih2;
+        if (ih2->dlengsz) {
+                WARN_ON(!sc->dmadptr);
+                sc->cmd.cmd2.dptr = sc->dmadptr;
         }
-
-        irh = (struct octeon_instr_irh *)&sc->cmd.irh;
+        irh = (struct octeon_instr_irh *)&sc->cmd.cmd2.irh;
         if (irh->rflag) {
                 BUG_ON(!sc->dmarptr);
                 BUG_ON(!sc->status_word);
                 *sc->status_word = COMPLETION_WORD_INIT;
 
-                rdp = (struct octeon_instr_rdp *)&sc->cmd.rdp;
+                rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd2.rdp;
 
-                sc->cmd.rptr = sc->dmarptr;
+                sc->cmd.cmd2.rptr = sc->dmarptr;
         }
+        len = (u32)ih2->dlengsz;
 
         if (sc->wait_time)
                 sc->timeout = jiffies + sc->wait_time;
 
-        return octeon_send_command(oct, sc->iq_no, 1, &sc->cmd, sc,
-                                   (u32)ih->dlengsz, REQTYPE_SOFT_COMMAND);
+        return (octeon_send_command(oct, sc->iq_no, 1, &sc->cmd, sc,
+                                    len, REQTYPE_SOFT_COMMAND));
 }
 
 int octeon_setup_sc_buffer_pool(struct octeon_device *oct)
diff --git a/drivers/net/ethernet/cavium/liquidio/response_manager.c b/drivers/net/ethernet/cavium/liquidio/response_manager.c
index 091f537a946e..c93210f99dda 100644
--- a/drivers/net/ethernet/cavium/liquidio/response_manager.c
+++ b/drivers/net/ethernet/cavium/liquidio/response_manager.c
@@ -54,8 +54,9 @@ int octeon_setup_response_list(struct octeon_device *oct)
                 spin_lock_init(&oct->response_list[i].lock);
                 atomic_set(&oct->response_list[i].pending_req_count, 0);
         }
+        spin_lock_init(&oct->cmd_resp_wqlock);
 
-        oct->dma_comp_wq.wq = create_workqueue("dma-comp");
+        oct->dma_comp_wq.wq = alloc_workqueue("dma-comp", WQ_MEM_RECLAIM, 0);
         if (!oct->dma_comp_wq.wq) {
                 dev_err(&oct->pci_dev->dev, "failed to create wq thread\n");
                 return -ENOMEM;
@@ -64,6 +65,7 @@ int octeon_setup_response_list(struct octeon_device *oct)
         cwq = &oct->dma_comp_wq;
         INIT_DELAYED_WORK(&cwq->wk.work, oct_poll_req_completion);
         cwq->wk.ctxptr = oct;
+        oct->cmd_resp_state = OCT_DRV_ONLINE;
         queue_delayed_work(cwq->wq, &cwq->wk.work, msecs_to_jiffies(100));
 
         return ret;
@@ -72,7 +74,6 @@ int octeon_setup_response_list(struct octeon_device *oct)
 void octeon_delete_response_list(struct octeon_device *oct)
 {
         cancel_delayed_work_sync(&oct->dma_comp_wq.wk.work);
-        flush_workqueue(oct->dma_comp_wq.wq);
         destroy_workqueue(oct->dma_comp_wq.wq);
 }
 
@@ -86,6 +87,7 @@ int lio_process_ordered_list(struct octeon_device *octeon_dev,
         u32 status;
         u64 status64;
         struct octeon_instr_rdp *rdp;
+        u64 rptr;
 
         ordered_sc_list = &octeon_dev->response_list[OCTEON_ORDERED_SC_LIST];
 
@@ -103,7 +105,8 @@ int lio_process_ordered_list(struct octeon_device *octeon_dev,
 
                 sc = (struct octeon_soft_command *)ordered_sc_list->
                     head.next;
-                rdp = (struct octeon_instr_rdp *)&sc->cmd.rdp;
+                rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd2.rdp;
+                rptr = sc->cmd.cmd2.rptr;
 
                 status = OCTEON_REQUEST_PENDING;
 
@@ -111,7 +114,7 @@ int lio_process_ordered_list(struct octeon_device *octeon_dev,
                  * to where rptr is pointing to
                  */
                 dma_sync_single_for_cpu(&octeon_dev->pci_dev->dev,
-                                        sc->cmd.rptr, rdp->rlen,
+                                        rptr, rdp->rlen,
                                         DMA_FROM_DEVICE);
                 status64 = *sc->status_word;
 
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index 477db477b133..c45de49dc963 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -64,6 +64,7 @@
 #include <net/bonding.h>
 #include <net/addrconf.h>
 #include <asm/uaccess.h>
+#include <linux/crash_dump.h>
 
 #include "cxgb4.h"
 #include "t4_regs.h"
@@ -206,7 +207,7 @@ static int rx_dma_offset = 2;
 static unsigned int num_vf[NUM_OF_PF_WITH_SRIOV];
 
 module_param_array(num_vf, uint, NULL, 0644);
-MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
+MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3, deprecated parameter - please use the pci sysfs interface.");
 #endif
 
 /* TX Queue select used to determine what algorithm to use for selecting TX
@@ -460,11 +461,8 @@ static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
         struct port_info *pi = netdev_priv(dev);
         struct adapter *adapter = pi->adapter;
 
-        if (!(dev->flags & IFF_PROMISC)) {
-                __dev_uc_sync(dev, cxgb4_mac_sync, cxgb4_mac_unsync);
-                if (!(dev->flags & IFF_ALLMULTI))
-                        __dev_mc_sync(dev, cxgb4_mac_sync, cxgb4_mac_unsync);
-        }
+        __dev_uc_sync(dev, cxgb4_mac_sync, cxgb4_mac_unsync);
+        __dev_mc_sync(dev, cxgb4_mac_sync, cxgb4_mac_unsync);
 
         return t4_set_rxmode(adapter, adapter->mbox, pi->viid, mtu,
                              (dev->flags & IFF_PROMISC) ? 1 : 0,
@@ -3735,7 +3733,8 @@ static int adap_init0(struct adapter *adap)
                 return ret;
 
         /* Contact FW, advertising Master capability */
-        ret = t4_fw_hello(adap, adap->mbox, adap->mbox, MASTER_MAY, &state);
+        ret = t4_fw_hello(adap, adap->mbox, adap->mbox,
+                          is_kdump_kernel() ? MASTER_MUST : MASTER_MAY, &state);
         if (ret < 0) {
                 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
                         ret);
@@ -4366,6 +4365,11 @@ static void cfg_queues(struct adapter *adap)
         if (q10g > netif_get_num_default_rss_queues())
                 q10g = netif_get_num_default_rss_queues();
 
+        /* Reduce memory usage in kdump environment, disable all offload.
+         */
+        if (is_kdump_kernel())
+                adap->params.offload = 0;
+
         for_each_port(adap, i) {
                 struct port_info *pi = adap2pinfo(adap, i);
 
@@ -4829,6 +4833,60 @@ static int get_chip_type(struct pci_dev *pdev, u32 pl_rev)
         return -EINVAL;
 }
 
+#ifdef CONFIG_PCI_IOV
+static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs)
+{
+        int err = 0;
+        int current_vfs = pci_num_vf(pdev);
+        u32 pcie_fw;
+        void __iomem *regs;
+
+        regs = pci_ioremap_bar(pdev, 0);
+        if (!regs) {
+                dev_err(&pdev->dev, "cannot map device registers\n");
+                return -ENOMEM;
+        }
+
+        pcie_fw = readl(regs + PCIE_FW_A);
+        iounmap(regs);
+        /* Check if cxgb4 is the MASTER and fw is initialized */
+        if (!(pcie_fw & PCIE_FW_INIT_F) ||
+            !(pcie_fw & PCIE_FW_MASTER_VLD_F) ||
+            PCIE_FW_MASTER_G(pcie_fw) != 4) {
+                dev_warn(&pdev->dev,
+                         "cxgb4 driver needs to be MASTER to support SRIOV\n");
+                return -EOPNOTSUPP;
+        }
+
+        /* If any of the VF's is already assigned to Guest OS, then
+         * SRIOV for the same cannot be modified
+         */
+        if (current_vfs && pci_vfs_assigned(pdev)) {
+                dev_err(&pdev->dev,
+                        "Cannot modify SR-IOV while VFs are assigned\n");
+                num_vfs = current_vfs;
+                return num_vfs;
+        }
+
+        /* Disable SRIOV when zero is passed.
+         * One needs to disable SRIOV before modifying it, else
+         * stack throws the below warning:
+         * " 'n' VFs already enabled. Disable before enabling 'm' VFs."
+         */
+        if (!num_vfs) {
+                pci_disable_sriov(pdev);
+                return num_vfs;
+        }
+
+        if (num_vfs != current_vfs) {
+                err = pci_enable_sriov(pdev, num_vfs);
+                if (err)
+                        return err;
+        }
+        return num_vfs;
+}
+#endif
+
 static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
         int func, i, err, s_qpp, qpp, num_seg;
@@ -5162,11 +5220,16 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 sriov:
 #ifdef CONFIG_PCI_IOV
-        if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
+        if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0) {
+                dev_warn(&pdev->dev,
+                         "Enabling SR-IOV VFs using the num_vf module "
+                         "parameter is deprecated - please use the pci sysfs "
+                         "interface instead.\n");
                 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
                         dev_info(&pdev->dev,
                                  "instantiated %u virtual functions\n",
                                  num_vf[func]);
+        }
 #endif
         return 0;
 
@@ -5259,6 +5322,9 @@ static struct pci_driver cxgb4_driver = {
         .probe    = init_one,
         .remove   = remove_one,
         .shutdown = remove_one,
+#ifdef CONFIG_PCI_IOV
+        .sriov_configure = cxgb4_iov_configure,
+#endif
         .err_handler = &cxgb4_eeh,
 };
 
diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/adapter.h b/drivers/net/ethernet/chelsio/cxgb4vf/adapter.h
index 734dd776c22f..109bc630408b 100644
--- a/drivers/net/ethernet/chelsio/cxgb4vf/adapter.h
+++ b/drivers/net/ethernet/chelsio/cxgb4vf/adapter.h
@@ -353,6 +353,10 @@ struct hash_mac_addr {
         u8 addr[ETH_ALEN];
 };
 
+struct mbox_list {
+        struct list_head list;
+};
+
 /*
  * Per-"adapter" (Virtual Function) information.
  */
@@ -387,6 +391,10 @@ struct adapter {
         /* various locks */
         spinlock_t stats_lock;
 
+        /* lock for mailbox cmd list */
+        spinlock_t mbox_lock;
+        struct mbox_list mlist;
+
         /* support for mailbox command/reply logging */
 #define T4VF_OS_LOG_MBOX_CMDS 256
         struct mbox_cmd_log *mbox_log;
diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c
index 04fc6f6d1e25..9f5526478d2f 100644
--- a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c
@@ -937,12 +937,8 @@ static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
 {
         struct port_info *pi = netdev_priv(dev);
 
-        if (!(dev->flags & IFF_PROMISC)) {
-                __dev_uc_sync(dev, cxgb4vf_mac_sync, cxgb4vf_mac_unsync);
-                if (!(dev->flags & IFF_ALLMULTI))
-                        __dev_mc_sync(dev, cxgb4vf_mac_sync,
-                                      cxgb4vf_mac_unsync);
-        }
+        __dev_uc_sync(dev, cxgb4vf_mac_sync, cxgb4vf_mac_unsync);
+        __dev_mc_sync(dev, cxgb4vf_mac_sync, cxgb4vf_mac_unsync);
         return t4vf_set_rxmode(pi->adapter, pi->viid, -1,
                                (dev->flags & IFF_PROMISC) != 0,
                                (dev->flags & IFF_ALLMULTI) != 0,
@@ -2778,6 +2774,8 @@ static int cxgb4vf_pci_probe(struct pci_dev *pdev,
          * Initialize SMP data synchronization resources.
          */
         spin_lock_init(&adapter->stats_lock);
+        spin_lock_init(&adapter->mbox_lock);
+        INIT_LIST_HEAD(&adapter->mlist.list);
 
         /*
          * Map our I/O registers in BAR0.
diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c
index 955ff7c61f1b..61bfe86da86d 100644
--- a/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c
+++ b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c
@@ -139,6 +139,7 @@ int t4vf_wr_mbox_core(struct adapter *adapter, const void *cmd, int size,
         u32 mbox_ctl = T4VF_CIM_BASE_ADDR + CIM_VF_EXT_MAILBOX_CTRL;
         u32 cmd_op = FW_CMD_OP_G(be32_to_cpu(((struct fw_cmd_hdr *)cmd)->hi));
         __be64 cmd_rpl[MBOX_LEN / 8];
+        struct mbox_list entry;
 
         /* In T6, mailbox size is changed to 128 bytes to avoid
          * invalidating the entire prefetch buffer.
@@ -156,6 +157,51 @@ int t4vf_wr_mbox_core(struct adapter *adapter, const void *cmd, int size,
             size > NUM_CIM_VF_MAILBOX_DATA_INSTANCES * 4)
                 return -EINVAL;
 
+        /* Queue ourselves onto the mailbox access list.  When our entry is at
+         * the front of the list, we have rights to access the mailbox.  So we
+         * wait [for a while] till we're at the front [or bail out with an
+         * EBUSY] ...
+         */
+        spin_lock(&adapter->mbox_lock);
+        list_add_tail(&entry.list, &adapter->mlist.list);
+        spin_unlock(&adapter->mbox_lock);
+
+        delay_idx = 0;
+        ms = delay[0];
+
+        for (i = 0; ; i += ms) {
+                /* If we've waited too long, return a busy indication.  This
+                 * really ought to be based on our initial position in the
+                 * mailbox access list but this is a start.  We very rearely
+                 * contend on access to the mailbox ...
+                 */
+                if (i > FW_CMD_MAX_TIMEOUT) {
+                        spin_lock(&adapter->mbox_lock);
+                        list_del(&entry.list);
+                        spin_unlock(&adapter->mbox_lock);
+                        ret = -EBUSY;
+                        t4vf_record_mbox(adapter, cmd, size, access, ret);
+                        return ret;
+                }
+
+                /* If we're at the head, break out and start the mailbox
+                 * protocol.
+                 */
+                if (list_first_entry(&adapter->mlist.list, struct mbox_list,
+                                     list) == &entry)
+                        break;
+
+                /* Delay for a bit before checking again ... */
+                if (sleep_ok) {
+                        ms = delay[delay_idx];  /* last element may repeat */
+                        if (delay_idx < ARRAY_SIZE(delay) - 1)
+                                delay_idx++;
+                        msleep(ms);
+                } else {
+                        mdelay(ms);
+                }
+        }
+
         /*
          * Loop trying to get ownership of the mailbox.  Return an error
          * if we can't gain ownership.
@@ -164,6 +210,9 @@ int t4vf_wr_mbox_core(struct adapter *adapter, const void *cmd, int size,
         for (i = 0; v == MBOX_OWNER_NONE && i < 3; i++)
                 v = MBOWNER_G(t4_read_reg(adapter, mbox_ctl));
         if (v != MBOX_OWNER_DRV) {
+                spin_lock(&adapter->mbox_lock);
+                list_del(&entry.list);
+                spin_unlock(&adapter->mbox_lock);
                 ret = (v == MBOX_OWNER_FW) ? -EBUSY : -ETIMEDOUT;
                 t4vf_record_mbox(adapter, cmd, size, access, ret);
                 return ret;
@@ -248,6 +297,9 @@ int t4vf_wr_mbox_core(struct adapter *adapter, const void *cmd, int size,
                         if (cmd_op != FW_VI_STATS_CMD)
                                 t4vf_record_mbox(adapter, cmd_rpl, size, access,
                                                  execute);
+                        spin_lock(&adapter->mbox_lock);
+                        list_del(&entry.list);
+                        spin_unlock(&adapter->mbox_lock);
                         return -FW_CMD_RETVAL_G(v);
                 }
         }
@@ -255,6 +307,9 @@ int t4vf_wr_mbox_core(struct adapter *adapter, const void *cmd, int size,
         /* We timed out.  Return the error ... */
         ret = -ETIMEDOUT;
         t4vf_record_mbox(adapter, cmd, size, access, ret);
+        spin_lock(&adapter->mbox_lock);
+        list_del(&entry.list);
+        spin_unlock(&adapter->mbox_lock);
         return ret;
 }
 
diff --git a/drivers/net/ethernet/cirrus/cs89x0.c b/drivers/net/ethernet/cirrus/cs89x0.c
index 60383040d6c6..c363b58552e9 100644
--- a/drivers/net/ethernet/cirrus/cs89x0.c
+++ b/drivers/net/ethernet/cirrus/cs89x0.c
@@ -53,6 +53,8 @@
 #include <linux/errno.h>
 #include <linux/netdevice.h>
 #include <linux/etherdevice.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
 #include <linux/platform_device.h>
 #include <linux/kernel.h>
 #include <linux/types.h>
@@ -1895,9 +1897,17 @@ static int cs89x0_platform_remove(struct platform_device *pdev)
         return 0;
 }
 
+static const struct __maybe_unused of_device_id cs89x0_match[] = {
+        { .compatible = "cirrus,cs8900", },
+        { .compatible = "cirrus,cs8920", },
+        { },
+};
+MODULE_DEVICE_TABLE(of, cs89x0_match);
+
 static struct platform_driver cs89x0_driver = {
         .driver = {
-                .name   = DRV_NAME,
+                .name           = DRV_NAME,
+                .of_match_table = of_match_ptr(cs89x0_match),
         },
         .remove = cs89x0_platform_remove,
 };
diff --git a/drivers/net/ethernet/cisco/enic/enic_ethtool.c b/drivers/net/ethernet/cisco/enic/enic_ethtool.c
index f44a39c40642..fd3980cc1e34 100644
--- a/drivers/net/ethernet/cisco/enic/enic_ethtool.c
+++ b/drivers/net/ethernet/cisco/enic/enic_ethtool.c
@@ -103,25 +103,29 @@ static void enic_intr_coal_set_rx(struct enic *enic, u32 timer)
         }
 }
 
-static int enic_get_settings(struct net_device *netdev,
-        struct ethtool_cmd *ecmd)
+static int enic_get_ksettings(struct net_device *netdev,
+                              struct ethtool_link_ksettings *ecmd)
 {
         struct enic *enic = netdev_priv(netdev);
+        struct ethtool_link_settings *base = &ecmd->base;
 
-        ecmd->supported = (SUPPORTED_10000baseT_Full | SUPPORTED_FIBRE);
-        ecmd->advertising = (ADVERTISED_10000baseT_Full | ADVERTISED_FIBRE);
-        ecmd->port = PORT_FIBRE;
-        ecmd->transceiver = XCVR_EXTERNAL;
+        ethtool_link_ksettings_add_link_mode(ecmd, supported,
+                                             10000baseT_Full);
+        ethtool_link_ksettings_add_link_mode(ecmd, supported, FIBRE);
+        ethtool_link_ksettings_add_link_mode(ecmd, advertising,
+                                             10000baseT_Full);
+        ethtool_link_ksettings_add_link_mode(ecmd, advertising, FIBRE);
+        base->port = PORT_FIBRE;
 
         if (netif_carrier_ok(netdev)) {
-                ethtool_cmd_speed_set(ecmd, vnic_dev_port_speed(enic->vdev));
-                ecmd->duplex = DUPLEX_FULL;
+                base->speed = vnic_dev_port_speed(enic->vdev);
+                base->duplex = DUPLEX_FULL;
         } else {
-                ethtool_cmd_speed_set(ecmd, SPEED_UNKNOWN);
-                ecmd->duplex = DUPLEX_UNKNOWN;
+                base->speed = SPEED_UNKNOWN;
+                base->duplex = DUPLEX_UNKNOWN;
         }
 
-        ecmd->autoneg = AUTONEG_DISABLE;
+        base->autoneg = AUTONEG_DISABLE;
 
         return 0;
 }
@@ -500,7 +504,6 @@ static int enic_set_rxfh(struct net_device *netdev, const u32 *indir,
 }
 
 static const struct ethtool_ops enic_ethtool_ops = {
-        .get_settings = enic_get_settings,
         .get_drvinfo = enic_get_drvinfo,
         .get_msglevel = enic_get_msglevel,
         .set_msglevel = enic_set_msglevel,
@@ -516,6 +519,7 @@ static const struct ethtool_ops enic_ethtool_ops = {
         .get_rxfh_key_size = enic_get_rxfh_key_size,
         .get_rxfh = enic_get_rxfh,
         .set_rxfh = enic_set_rxfh,
+        .get_link_ksettings = enic_get_ksettings,
 };
 
 void enic_set_ethtool_ops(struct net_device *netdev)
diff --git a/drivers/net/ethernet/dnet.c b/drivers/net/ethernet/dnet.c
index b69a9eacc531..c3b64cdd0dec 100644
--- a/drivers/net/ethernet/dnet.c
+++ b/drivers/net/ethernet/dnet.c
@@ -173,7 +173,7 @@ static int dnet_mdio_write(struct mii_bus *bus, int mii_id, int regnum,
 static void dnet_handle_link_change(struct net_device *dev)
 {
         struct dnet *bp = netdev_priv(dev);
-        struct phy_device *phydev = bp->phy_dev;
+        struct phy_device *phydev = dev->phydev;
         unsigned long flags;
         u32 mode_reg, ctl_reg;
 
@@ -295,7 +295,6 @@ static int dnet_mii_probe(struct net_device *dev)
         bp->link = 0;
         bp->speed = 0;
         bp->duplex = -1;
-        bp->phy_dev = phydev;
 
         return 0;
 }
@@ -629,16 +628,16 @@ static int dnet_open(struct net_device *dev)
         struct dnet *bp = netdev_priv(dev);
 
         /* if the phy is not yet register, retry later */
-        if (!bp->phy_dev)
+        if (!dev->phydev)
                 return -EAGAIN;
 
         napi_enable(&bp->napi);
         dnet_init_hw(bp);
 
-        phy_start_aneg(bp->phy_dev);
+        phy_start_aneg(dev->phydev);
 
         /* schedule a link state check */
-        phy_start(bp->phy_dev);
+        phy_start(dev->phydev);
 
         netif_start_queue(dev);
 
@@ -652,8 +651,8 @@ static int dnet_close(struct net_device *dev)
         netif_stop_queue(dev);
         napi_disable(&bp->napi);
 
-        if (bp->phy_dev)
-                phy_stop(bp->phy_dev);
+        if (dev->phydev)
+                phy_stop(dev->phydev);
 
         dnet_reset_hw(bp);
         netif_carrier_off(dev);
@@ -731,32 +730,9 @@ static struct net_device_stats *dnet_get_stats(struct net_device *dev)
         return nstat;
 }
 
-static int dnet_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
-{
-        struct dnet *bp = netdev_priv(dev);
-        struct phy_device *phydev = bp->phy_dev;
-
-        if (!phydev)
-                return -ENODEV;
-
-        return phy_ethtool_gset(phydev, cmd);
-}
-
-static int dnet_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
-{
-        struct dnet *bp = netdev_priv(dev);
-        struct phy_device *phydev = bp->phy_dev;
-
-        if (!phydev)
-                return -ENODEV;
-
-        return phy_ethtool_sset(phydev, cmd);
-}
-
 static int dnet_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
 {
-        struct dnet *bp = netdev_priv(dev);
-        struct phy_device *phydev = bp->phy_dev;
+        struct phy_device *phydev = dev->phydev;
 
         if (!netif_running(dev))
                 return -EINVAL;
@@ -776,11 +752,11 @@ static void dnet_get_drvinfo(struct net_device *dev,
 }
 
 static const struct ethtool_ops dnet_ethtool_ops = {
-        .get_settings           = dnet_get_settings,
-        .set_settings           = dnet_set_settings,
         .get_drvinfo            = dnet_get_drvinfo,
         .get_link               = ethtool_op_get_link,
         .get_ts_info            = ethtool_op_get_ts_info,
+        .get_link_ksettings     = phy_ethtool_get_link_ksettings,
+        .set_link_ksettings     = phy_ethtool_set_link_ksettings,
 };
 
 static const struct net_device_ops dnet_netdev_ops = {
@@ -875,7 +851,7 @@ static int dnet_probe(struct platform_device *pdev)
                (bp->capabilities & DNET_HAS_IRQ) ? "" : "no ",
                (bp->capabilities & DNET_HAS_GIGABIT) ? "" : "no ",
                (bp->capabilities & DNET_HAS_DMA) ? "" : "no ");
-        phydev = bp->phy_dev;
+        phydev = dev->phydev;
         phy_attached_info(phydev);
 
         return 0;
@@ -899,8 +875,8 @@ static int dnet_remove(struct platform_device *pdev)
 
         if (dev) {
                 bp = netdev_priv(dev);
-                if (bp->phy_dev)
-                        phy_disconnect(bp->phy_dev);
+                if (dev->phydev)
+                        phy_disconnect(dev->phydev);
                 mdiobus_unregister(bp->mii_bus);
                 mdiobus_free(bp->mii_bus);
                 unregister_netdev(dev);
diff --git a/drivers/net/ethernet/dnet.h b/drivers/net/ethernet/dnet.h
index 37f5b30fa78b..d985080bbd5d 100644
--- a/drivers/net/ethernet/dnet.h
+++ b/drivers/net/ethernet/dnet.h
@@ -216,7 +216,6 @@ struct dnet {
 
         /* PHY stuff */
         struct mii_bus                  *mii_bus;
-        struct phy_device               *phy_dev;
         unsigned int                    link;
         unsigned int                    speed;
         unsigned int                    duplex;
diff --git a/drivers/net/ethernet/emulex/benet/Kconfig b/drivers/net/ethernet/emulex/benet/Kconfig
index 7108563260ae..b4853ec9de8d 100644
--- a/drivers/net/ethernet/emulex/benet/Kconfig
+++ b/drivers/net/ethernet/emulex/benet/Kconfig
@@ -13,11 +13,3 @@ config BE2NET_HWMON
         ---help---
           Say Y here if you want to expose thermal sensor data on
           be2net network adapter.
-
-config BE2NET_VXLAN
-        bool "VXLAN offload support on be2net driver"
-        default y
-        depends on BE2NET && VXLAN && !(BE2NET=y && VXLAN=m)
-        ---help---
-          Say Y here if you want to enable VXLAN offload support on
-          be2net driver.
diff --git a/drivers/net/ethernet/emulex/benet/be.h b/drivers/net/ethernet/emulex/benet/be.h
index fe3763df3f13..4555e041ef69 100644
--- a/drivers/net/ethernet/emulex/benet/be.h
+++ b/drivers/net/ethernet/emulex/benet/be.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2005 - 2015 Emulex
+ * Copyright (C) 2005 - 2016 Broadcom
  * All rights reserved.
  *
  * This program is free software; you can redistribute it and/or
@@ -97,7 +97,8 @@
                                          * SURF/DPDK
                                          */
 
-#define MAX_RSS_IFACES          15
+#define MAX_PORT_RSS_TABLES     15
+#define MAX_NIC_FUNCS           16
 #define MAX_RX_QS               32
 #define MAX_EVT_QS              32
 #define MAX_TX_QS               32
@@ -442,8 +443,20 @@ struct be_resources {
         u16 max_iface_count;
         u16 max_mcc_count;
         u16 max_evt_qs;
+        u16 max_nic_evt_qs;     /* NIC's share of evt qs */
         u32 if_cap_flags;
         u32 vf_if_cap_flags;    /* VF if capability flags */
+        u32 flags;
+        /* Calculated PF Pool's share of RSS Tables. This is not enforced by
+         * the FW, but is a self-imposed driver limitation.
+         */
+        u16 max_rss_tables;
+};
+
+/* These are port-wide values */
+struct be_port_resources {
+        u16 max_vfs;
+        u16 nic_pfs;
 };
 
 #define be_is_os2bmc_enabled(adapter) (adapter->flags & BE_FLAGS_OS2BMC)
@@ -513,7 +526,8 @@ struct be_adapter {
         spinlock_t mcc_lock;    /* For serializing mcc cmds to BE card */
         spinlock_t mcc_cq_lock;
 
-        u16 cfg_num_qs;         /* configured via set-channels */
+        u16 cfg_num_rx_irqs;            /* configured via set-channels */
+        u16 cfg_num_tx_irqs;            /* configured via set-channels */
         u16 num_evt_qs;
         u16 num_msix_vec;
         struct be_eq_obj eq_obj[MAX_EVT_QS];
@@ -632,16 +646,42 @@ struct be_adapter {
 #define be_max_txqs(adapter)            (adapter->res.max_tx_qs)
 #define be_max_prio_txqs(adapter)       (adapter->res.max_prio_tx_qs)
 #define be_max_rxqs(adapter)            (adapter->res.max_rx_qs)
-#define be_max_eqs(adapter)             (adapter->res.max_evt_qs)
+/* Max number of EQs available for the function (NIC + RoCE (if enabled)) */
+#define be_max_func_eqs(adapter)        (adapter->res.max_evt_qs)
+/* Max number of EQs available avaialble only for NIC */
+#define be_max_nic_eqs(adapter)         (adapter->res.max_nic_evt_qs)
 #define be_if_cap_flags(adapter)        (adapter->res.if_cap_flags)
-
-static inline u16 be_max_qs(struct be_adapter *adapter)
+#define be_max_pf_pool_rss_tables(adapter)      \
+                                (adapter->pool_res.max_rss_tables)
+/* Max irqs avaialble for NIC */
+#define be_max_irqs(adapter)            \
+                        (min_t(u16, be_max_nic_eqs(adapter), num_online_cpus()))
+
+/* Max irqs *needed* for RX queues */
+static inline u16 be_max_rx_irqs(struct be_adapter *adapter)
 {
-        /* If no RSS, need atleast the one def RXQ */
+        /* If no RSS, need atleast one irq for def-RXQ */
         u16 num = max_t(u16, be_max_rss(adapter), 1);
 
-        num = min(num, be_max_eqs(adapter));
-        return min_t(u16, num, num_online_cpus());
+        return min_t(u16, num, be_max_irqs(adapter));
+}
+
+/* Max irqs *needed* for TX queues */
+static inline u16 be_max_tx_irqs(struct be_adapter *adapter)
+{
+        return min_t(u16, be_max_txqs(adapter), be_max_irqs(adapter));
+}
+
+/* Max irqs *needed* for combined queues */
+static inline u16 be_max_qp_irqs(struct be_adapter *adapter)
+{
+        return min(be_max_tx_irqs(adapter), be_max_rx_irqs(adapter));
+}
+
+/* Max irqs *needed* for RX and TX queues together */
+static inline u16 be_max_any_irqs(struct be_adapter *adapter)
+{
+        return max(be_max_tx_irqs(adapter), be_max_rx_irqs(adapter));
 }
 
 /* Is BE in pvid_tagging mode */
diff --git a/drivers/net/ethernet/emulex/benet/be_cmds.c b/drivers/net/ethernet/emulex/benet/be_cmds.c
index 22402db275f2..2cc11756859f 100644
--- a/drivers/net/ethernet/emulex/benet/be_cmds.c
+++ b/drivers/net/ethernet/emulex/benet/be_cmds.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2005 - 2015 Emulex
+ * Copyright (C) 2005 - 2016 Broadcom
  * All rights reserved.
  *
  * This program is free software; you can redistribute it and/or
@@ -87,6 +87,11 @@ static struct be_cmd_priv_map cmd_priv_map[] = {
                 CMD_SUBSYSTEM_LOWLEVEL,
                 BE_PRIV_DEVCFG | BE_PRIV_DEVSEC
         },
+        {
+                OPCODE_COMMON_SET_HSW_CONFIG,
+                CMD_SUBSYSTEM_COMMON,
+                BE_PRIV_DEVCFG | BE_PRIV_VHADM
+        },
 };
 
 static bool be_cmd_allowed(struct be_adapter *adapter, u8 opcode, u8 subsystem)
@@ -3850,6 +3855,10 @@ int be_cmd_set_hsw_config(struct be_adapter *adapter, u16 pvid,
         void *ctxt;
         int status;
 
+        if (!be_cmd_allowed(adapter, OPCODE_COMMON_SET_HSW_CONFIG,
+                            CMD_SUBSYSTEM_COMMON))
+                return -EPERM;
+
         spin_lock_bh(&adapter->mcc_lock);
 
         wrb = wrb_from_mccq(adapter);
@@ -3871,7 +3880,7 @@ int be_cmd_set_hsw_config(struct be_adapter *adapter, u16 pvid,
                 AMAP_SET_BITS(struct amap_set_hsw_context, pvid_valid, ctxt, 1);
                 AMAP_SET_BITS(struct amap_set_hsw_context, pvid, ctxt, pvid);
         }
-        if (!BEx_chip(adapter) && hsw_mode) {
+        if (hsw_mode) {
                 AMAP_SET_BITS(struct amap_set_hsw_context, interface_id,
                               ctxt, adapter->hba_port_num);
                 AMAP_SET_BITS(struct amap_set_hsw_context, pport, ctxt, 1);
@@ -4023,7 +4032,10 @@ int be_cmd_get_acpi_wol_cap(struct be_adapter *adapter)
                 resp = (struct be_cmd_resp_acpi_wol_magic_config_v1 *)cmd.va;
 
                 adapter->wol_cap = resp->wol_settings;
-                if (adapter->wol_cap & BE_WOL_CAP)
+
+                /* Non-zero macaddr indicates WOL is enabled */
+                if (adapter->wol_cap & BE_WOL_CAP &&
+                    !is_zero_ether_addr(resp->magic_mac))
                         adapter->wol_en = true;
         }
 err:
@@ -4360,9 +4372,35 @@ err:
         return status;
 }
 
+/* This routine returns a list of all the NIC PF_nums in the adapter */
+u16 be_get_nic_pf_num_list(u8 *buf, u32 desc_count, u16 *nic_pf_nums)
+{
+        struct be_res_desc_hdr *hdr = (struct be_res_desc_hdr *)buf;
+        struct be_pcie_res_desc *pcie = NULL;
+        int i;
+        u16 nic_pf_count = 0;
+
+        for (i = 0; i < desc_count; i++) {
+                if (hdr->desc_type == PCIE_RESOURCE_DESC_TYPE_V0 ||
+                    hdr->desc_type == PCIE_RESOURCE_DESC_TYPE_V1) {
+                        pcie = (struct be_pcie_res_desc *)hdr;
+                        if (pcie->pf_state && (pcie->pf_type == MISSION_NIC ||
+                                               pcie->pf_type == MISSION_RDMA)) {
+                                nic_pf_nums[nic_pf_count++] = pcie->pf_num;
+                        }
+                }
+
+                hdr->desc_len = hdr->desc_len ? : RESOURCE_DESC_SIZE_V0;
+                hdr = (void *)hdr + hdr->desc_len;
+        }
+        return nic_pf_count;
+}
+
 /* Will use MBOX only if MCCQ has not been created */
 int be_cmd_get_profile_config(struct be_adapter *adapter,
-                              struct be_resources *res, u8 query, u8 domain)
+                              struct be_resources *res,
+                              struct be_port_resources *port_res,
+                              u8 profile_type, u8 query, u8 domain)
 {
         struct be_cmd_resp_get_profile_config *resp;
         struct be_cmd_req_get_profile_config *req;
@@ -4389,7 +4427,7 @@ int be_cmd_get_profile_config(struct be_adapter *adapter,
 
         if (!lancer_chip(adapter))
                 req->hdr.version = 1;
-        req->type = ACTIVE_PROFILE_TYPE;
+        req->type = profile_type;
         req->hdr.domain = domain;
 
         /* When QUERY_MODIFIABLE_FIELDS_TYPE bit is set, cmd returns the
@@ -4406,6 +4444,28 @@ int be_cmd_get_profile_config(struct be_adapter *adapter,
         resp = cmd.va;
         desc_count = le16_to_cpu(resp->desc_count);
 
+        if (port_res) {
+                u16 nic_pf_cnt = 0, i;
+                u16 nic_pf_num_list[MAX_NIC_FUNCS];
+
+                nic_pf_cnt = be_get_nic_pf_num_list(resp->func_param,
+                                                    desc_count,
+                                                    nic_pf_num_list);
+
+                for (i = 0; i < nic_pf_cnt; i++) {
+                        nic = be_get_func_nic_desc(resp->func_param, desc_count,
+                                                   nic_pf_num_list[i]);
+                        if (nic->link_param == adapter->port_num) {
+                                port_res->nic_pfs++;
+                                pcie = be_get_pcie_desc(resp->func_param,
+                                                        desc_count,
+                                                        nic_pf_num_list[i]);
+                                port_res->max_vfs += le16_to_cpu(pcie->num_vfs);
+                        }
+                }
+                return status;
+        }
+
         pcie = be_get_pcie_desc(resp->func_param, desc_count,
                                 adapter->pf_num);
         if (pcie)
@@ -4465,7 +4525,7 @@ static int be_cmd_set_profile_config(struct be_adapter *adapter, void *desc,
 }
 
 /* Mark all fields invalid */
-static void be_reset_nic_desc(struct be_nic_res_desc *nic)
+void be_reset_nic_desc(struct be_nic_res_desc *nic)
 {
         memset(nic, 0, sizeof(*nic));
         nic->unicast_mac_count = 0xFFFF;
@@ -4534,73 +4594,9 @@ int be_cmd_config_qos(struct be_adapter *adapter, u32 max_rate, u16 link_speed,
                                          1, version, domain);
 }
 
-static void be_fill_vf_res_template(struct be_adapter *adapter,
-                                    struct be_resources pool_res,
-                                    u16 num_vfs, u16 num_vf_qs,
-                                    struct be_nic_res_desc *nic_vft)
-{
-        u32 vf_if_cap_flags = pool_res.vf_if_cap_flags;
-        struct be_resources res_mod = {0};
-
-        /* Resource with fields set to all '1's by GET_PROFILE_CONFIG cmd,
-         * which are modifiable using SET_PROFILE_CONFIG cmd.
-         */
-        be_cmd_get_profile_config(adapter, &res_mod, RESOURCE_MODIFIABLE, 0);
-
-        /* If RSS IFACE capability flags are modifiable for a VF, set the
-         * capability flag as valid and set RSS and DEFQ_RSS IFACE flags if
-         * more than 1 RSSQ is available for a VF.
-         * Otherwise, provision only 1 queue pair for VF.
-         */
-        if (res_mod.vf_if_cap_flags & BE_IF_FLAGS_RSS) {
-                nic_vft->flags |= BIT(IF_CAPS_FLAGS_VALID_SHIFT);
-                if (num_vf_qs > 1) {
-                        vf_if_cap_flags |= BE_IF_FLAGS_RSS;
-                        if (pool_res.if_cap_flags & BE_IF_FLAGS_DEFQ_RSS)
-                                vf_if_cap_flags |= BE_IF_FLAGS_DEFQ_RSS;
-                } else {
-                        vf_if_cap_flags &= ~(BE_IF_FLAGS_RSS |
-                                             BE_IF_FLAGS_DEFQ_RSS);
-                }
-        } else {
-                num_vf_qs = 1;
-        }
-
-        if (res_mod.vf_if_cap_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) {
-                nic_vft->flags |= BIT(IF_CAPS_FLAGS_VALID_SHIFT);
-                vf_if_cap_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS;
-        }
-
-        nic_vft->cap_flags = cpu_to_le32(vf_if_cap_flags);
-        nic_vft->rq_count = cpu_to_le16(num_vf_qs);
-        nic_vft->txq_count = cpu_to_le16(num_vf_qs);
-        nic_vft->rssq_count = cpu_to_le16(num_vf_qs);
-        nic_vft->cq_count = cpu_to_le16(pool_res.max_cq_count /
-                                        (num_vfs + 1));
-
-        /* Distribute unicast MACs, VLANs, IFACE count and MCCQ count equally
-         * among the PF and it's VFs, if the fields are changeable
-         */
-        if (res_mod.max_uc_mac == FIELD_MODIFIABLE)
-                nic_vft->unicast_mac_count = cpu_to_le16(pool_res.max_uc_mac /
-                                                         (num_vfs + 1));
-
-        if (res_mod.max_vlans == FIELD_MODIFIABLE)
-                nic_vft->vlan_count = cpu_to_le16(pool_res.max_vlans /
-                                                  (num_vfs + 1));
-
-        if (res_mod.max_iface_count == FIELD_MODIFIABLE)
-                nic_vft->iface_count = cpu_to_le16(pool_res.max_iface_count /
-                                                   (num_vfs + 1));
-
-        if (res_mod.max_mcc_count == FIELD_MODIFIABLE)
-                nic_vft->mcc_count = cpu_to_le16(pool_res.max_mcc_count /
-                                                 (num_vfs + 1));
-}
-
 int be_cmd_set_sriov_config(struct be_adapter *adapter,
                             struct be_resources pool_res, u16 num_vfs,
-                            u16 num_vf_qs)
+                            struct be_resources *vft_res)
 {
         struct {
                 struct be_pcie_res_desc pcie;
@@ -4620,12 +4616,26 @@ int be_cmd_set_sriov_config(struct be_adapter *adapter,
         be_reset_nic_desc(&desc.nic_vft);
         desc.nic_vft.hdr.desc_type = NIC_RESOURCE_DESC_TYPE_V1;
         desc.nic_vft.hdr.desc_len = RESOURCE_DESC_SIZE_V1;
-        desc.nic_vft.flags = BIT(VFT_SHIFT) | BIT(IMM_SHIFT) | BIT(NOSV_SHIFT);
+        desc.nic_vft.flags = vft_res->flags | BIT(VFT_SHIFT) |
+                             BIT(IMM_SHIFT) | BIT(NOSV_SHIFT);
         desc.nic_vft.pf_num = adapter->pdev->devfn;
         desc.nic_vft.vf_num = 0;
-
-        be_fill_vf_res_template(adapter, pool_res, num_vfs, num_vf_qs,
-                                &desc.nic_vft);
+        desc.nic_vft.cap_flags = cpu_to_le32(vft_res->vf_if_cap_flags);
+        desc.nic_vft.rq_count = cpu_to_le16(vft_res->max_rx_qs);
+        desc.nic_vft.txq_count = cpu_to_le16(vft_res->max_tx_qs);
+        desc.nic_vft.rssq_count = cpu_to_le16(vft_res->max_rss_qs);
+        desc.nic_vft.cq_count = cpu_to_le16(vft_res->max_cq_count);
+
+        if (vft_res->max_uc_mac)
+                desc.nic_vft.unicast_mac_count =
+                                        cpu_to_le16(vft_res->max_uc_mac);
+        if (vft_res->max_vlans)
+                desc.nic_vft.vlan_count = cpu_to_le16(vft_res->max_vlans);
+        if (vft_res->max_iface_count)
+                desc.nic_vft.iface_count =
+                                cpu_to_le16(vft_res->max_iface_count);
+        if (vft_res->max_mcc_count)
+                desc.nic_vft.mcc_count = cpu_to_le16(vft_res->max_mcc_count);
 
         return be_cmd_set_profile_config(adapter, &desc,
                                          2 * RESOURCE_DESC_SIZE_V1, 2, 1, 0);
diff --git a/drivers/net/ethernet/emulex/benet/be_cmds.h b/drivers/net/ethernet/emulex/benet/be_cmds.h
index d8540ae95e5a..0d6be224a787 100644
--- a/drivers/net/ethernet/emulex/benet/be_cmds.h
+++ b/drivers/net/ethernet/emulex/benet/be_cmds.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2005 - 2015 Emulex
+ * Copyright (C) 2005 - 2016 Broadcom
  * All rights reserved.
  *
  * This program is free software; you can redistribute it and/or
@@ -1556,7 +1556,9 @@ struct be_cmd_resp_acpi_wol_magic_config_v1 {
         u8 rsvd0[2];
         u8 wol_settings;
         u8 rsvd1[5];
-        u32 rsvd2[295];
+        u32 rsvd2[288];
+        u8 magic_mac[6];
+        u8 rsvd3[22];
 } __packed;
 
 #define BE_GET_WOL_CAP                  2
@@ -2128,6 +2130,9 @@ struct be_cmd_req_set_ext_fat_caps {
 #define IMM_SHIFT                               6       /* Immediate */
 #define NOSV_SHIFT                              7       /* No save */
 
+#define MISSION_NIC                             1
+#define MISSION_RDMA                            8
+
 struct be_res_desc_hdr {
         u8 desc_type;
         u8 desc_len;
@@ -2244,6 +2249,7 @@ struct be_cmd_req_get_profile_config {
         struct be_cmd_req_hdr hdr;
         u8 rsvd;
 #define ACTIVE_PROFILE_TYPE                     0x2
+#define SAVED_PROFILE_TYPE                      0x0
 #define QUERY_MODIFIABLE_FIELDS_TYPE            BIT(3)
         u8 type;
         u16 rsvd1;
@@ -2449,7 +2455,9 @@ int be_cmd_query_port_name(struct be_adapter *adapter);
 int be_cmd_get_func_config(struct be_adapter *adapter,
                            struct be_resources *res);
 int be_cmd_get_profile_config(struct be_adapter *adapter,
-                              struct be_resources *res, u8 query, u8 domain);
+                              struct be_resources *res,
+                              struct be_port_resources *port_res,
+                              u8 profile_type, u8 query, u8 domain);
 int be_cmd_get_active_profile(struct be_adapter *adapter, u16 *profile);
 int be_cmd_get_if_id(struct be_adapter *adapter, struct be_vf_cfg *vf_cfg,
                      int vf_num);
@@ -2461,4 +2469,4 @@ int be_cmd_set_vxlan_port(struct be_adapter *adapter, __be16 port);
 int be_cmd_manage_iface(struct be_adapter *adapter, u32 iface, u8 op);
 int be_cmd_set_sriov_config(struct be_adapter *adapter,
                             struct be_resources res, u16 num_vfs,
-                            u16 num_vf_qs);
+                            struct be_resources *vft_res);
diff --git a/drivers/net/ethernet/emulex/benet/be_ethtool.c b/drivers/net/ethernet/emulex/benet/be_ethtool.c
index 2ff691636dac..50e7be5da50c 100644
--- a/drivers/net/ethernet/emulex/benet/be_ethtool.c
+++ b/drivers/net/ethernet/emulex/benet/be_ethtool.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2005 - 2015 Emulex
+ * Copyright (C) 2005 - 2016 Broadcom
  * All rights reserved.
  *
  * This program is free software; you can redistribute it and/or
@@ -793,6 +793,11 @@ static void be_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol)
 static int be_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol)
 {
         struct be_adapter *adapter = netdev_priv(netdev);
+        struct device *dev = &adapter->pdev->dev;
+        struct be_dma_mem cmd;
+        u8 mac[ETH_ALEN];
+        bool enable;
+        int status;
 
         if (wol->wolopts & ~WAKE_MAGIC)
                 return -EOPNOTSUPP;
@@ -802,12 +807,32 @@ static int be_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol)
                 return -EOPNOTSUPP;
         }
 
-        if (wol->wolopts & WAKE_MAGIC)
-                adapter->wol_en = true;
-        else
-                adapter->wol_en = false;
+        cmd.size = sizeof(struct be_cmd_req_acpi_wol_magic_config);
+        cmd.va = dma_zalloc_coherent(dev, cmd.size, &cmd.dma, GFP_KERNEL);
+        if (!cmd.va)
+                return -ENOMEM;
 
-        return 0;
+        eth_zero_addr(mac);
+
+        enable = wol->wolopts & WAKE_MAGIC;
+        if (enable)
+                ether_addr_copy(mac, adapter->netdev->dev_addr);
+
+        status = be_cmd_enable_magic_wol(adapter, mac, &cmd);
+        if (status) {
+                dev_err(dev, "Could not set Wake-on-lan mac address\n");
+                status = be_cmd_status(status);
+                goto err;
+        }
+
+        pci_enable_wake(adapter->pdev, PCI_D3hot, enable);
+        pci_enable_wake(adapter->pdev, PCI_D3cold, enable);
+
+        adapter->wol_en = enable ? true : false;
+
+err:
+        dma_free_coherent(dev, cmd.size, cmd.va, cmd.dma);
+        return status;
 }
 
 static int be_test_ddr_dma(struct be_adapter *adapter)
@@ -1171,9 +1196,17 @@ static void be_get_channels(struct net_device *netdev,
                             struct ethtool_channels *ch)
 {
         struct be_adapter *adapter = netdev_priv(netdev);
+        u16 num_rx_irqs = max_t(u16, adapter->num_rss_qs, 1);
 
-        ch->combined_count = adapter->num_evt_qs;
-        ch->max_combined = be_max_qs(adapter);
+        /* num_tx_qs is always same as the number of irqs used for TX */
+        ch->combined_count = min(adapter->num_tx_qs, num_rx_irqs);
+        ch->rx_count = num_rx_irqs - ch->combined_count;
+        ch->tx_count = adapter->num_tx_qs - ch->combined_count;
+
+        ch->max_combined = be_max_qp_irqs(adapter);
+        /* The user must create atleast one combined channel */
+        ch->max_rx = be_max_rx_irqs(adapter) - 1;
+        ch->max_tx = be_max_tx_irqs(adapter) - 1;
 }
 
 static int be_set_channels(struct net_device  *netdev,
@@ -1182,11 +1215,22 @@ static int be_set_channels(struct net_device  *netdev,
         struct be_adapter *adapter = netdev_priv(netdev);
         int status;
 
-        if (ch->rx_count || ch->tx_count || ch->other_count ||
-            !ch->combined_count || ch->combined_count > be_max_qs(adapter))
+        /* we support either only combined channels or a combination of
+         * combined and either RX-only or TX-only channels.
+         */
+        if (ch->other_count || !ch->combined_count ||
+            (ch->rx_count && ch->tx_count))
+                return -EINVAL;
+
+        if (ch->combined_count > be_max_qp_irqs(adapter) ||
+            (ch->rx_count &&
+             (ch->rx_count + ch->combined_count) > be_max_rx_irqs(adapter)) ||
+            (ch->tx_count &&
+             (ch->tx_count + ch->combined_count) > be_max_tx_irqs(adapter)))
                 return -EINVAL;
 
-        adapter->cfg_num_qs = ch->combined_count;
+        adapter->cfg_num_rx_irqs = ch->combined_count + ch->rx_count;
+        adapter->cfg_num_tx_irqs = ch->combined_count + ch->tx_count;
 
         status = be_update_queues(adapter);
         return be_cmd_status(status);
diff --git a/drivers/net/ethernet/emulex/benet/be_main.c b/drivers/net/ethernet/emulex/benet/be_main.c
index ed98ef1ecac3..1873c74638cd 100644
--- a/drivers/net/ethernet/emulex/benet/be_main.c
+++ b/drivers/net/ethernet/emulex/benet/be_main.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2005 - 2015 Emulex
+ * Copyright (C) 2005 - 2016 Broadcom
  * All rights reserved.
  *
  * This program is free software; you can redistribute it and/or
@@ -2620,8 +2620,10 @@ static int be_evt_queues_create(struct be_adapter *adapter)
         struct be_aic_obj *aic;
         int i, rc;
 
+        /* need enough EQs to service both RX and TX queues */
         adapter->num_evt_qs = min_t(u16, num_irqs(adapter),
-                                    adapter->cfg_num_qs);
+                                    max(adapter->cfg_num_rx_irqs,
+                                        adapter->cfg_num_tx_irqs));
 
         for_all_evt_queues(adapter, eqo, i) {
                 int numa_node = dev_to_node(&adapter->pdev->dev);
@@ -2726,7 +2728,7 @@ static int be_tx_qs_create(struct be_adapter *adapter)
         struct be_eq_obj *eqo;
         int status, i;
 
-        adapter->num_tx_qs = min(adapter->num_evt_qs, be_max_txqs(adapter));
+        adapter->num_tx_qs = min(adapter->num_evt_qs, adapter->cfg_num_tx_irqs);
 
         for_all_tx_queues(adapter, txo, i) {
                 cq = &txo->cq;
@@ -2784,11 +2786,11 @@ static int be_rx_cqs_create(struct be_adapter *adapter)
         struct be_rx_obj *rxo;
         int rc, i;
 
-        /* We can create as many RSS rings as there are EQs. */
-        adapter->num_rss_qs = adapter->num_evt_qs;
+        adapter->num_rss_qs =
+                        min(adapter->num_evt_qs, adapter->cfg_num_rx_irqs);
 
         /* We'll use RSS only if atleast 2 RSS rings are supported. */
-        if (adapter->num_rss_qs <= 1)
+        if (adapter->num_rss_qs < 2)
                 adapter->num_rss_qs = 0;
 
         adapter->num_rx_qs = adapter->num_rss_qs + adapter->need_def_rxq;
@@ -3249,18 +3251,22 @@ static void be_msix_disable(struct be_adapter *adapter)
 
 static int be_msix_enable(struct be_adapter *adapter)
 {
-        int i, num_vec;
+        unsigned int i, num_vec, max_roce_eqs;
         struct device *dev = &adapter->pdev->dev;
 
-        /* If RoCE is supported, program the max number of NIC vectors that
-         * may be configured via set-channels, along with vectors needed for
-         * RoCe. Else, just program the number we'll use initially.
+        /* If RoCE is supported, program the max number of vectors that
+         * could be used for NIC and RoCE, else, just program the number
+         * we'll use initially.
          */
-        if (be_roce_supported(adapter))
-                num_vec = min_t(int, 2 * be_max_eqs(adapter),
-                                2 * num_online_cpus());
-        else
-                num_vec = adapter->cfg_num_qs;
+        if (be_roce_supported(adapter)) {
+                max_roce_eqs =
+                        be_max_func_eqs(adapter) - be_max_nic_eqs(adapter);
+                max_roce_eqs = min(max_roce_eqs, num_online_cpus());
+                num_vec = be_max_any_irqs(adapter) + max_roce_eqs;
+        } else {
+                num_vec = max(adapter->cfg_num_rx_irqs,
+                              adapter->cfg_num_tx_irqs);
+        }
 
         for (i = 0; i < num_vec; i++)
                 adapter->msix_entries[i].entry = i;
@@ -3625,10 +3631,8 @@ static int be_open(struct net_device *netdev)
                 be_link_status_update(adapter, link_status);
 
         netif_tx_start_all_queues(netdev);
-#ifdef CONFIG_BE2NET_VXLAN
         if (skyhawk_chip(adapter))
-                vxlan_get_rx_port(netdev);
-#endif
+                udp_tunnel_get_rx_info(netdev);
 
         return 0;
 err:
@@ -3636,40 +3640,6 @@ err:
         return -EIO;
 }
 
-static int be_setup_wol(struct be_adapter *adapter, bool enable)
-{
-        struct device *dev = &adapter->pdev->dev;
-        struct be_dma_mem cmd;
-        u8 mac[ETH_ALEN];
-        int status;
-
-        eth_zero_addr(mac);
-
-        cmd.size = sizeof(struct be_cmd_req_acpi_wol_magic_config);
-        cmd.va = dma_zalloc_coherent(dev, cmd.size, &cmd.dma, GFP_KERNEL);
-        if (!cmd.va)
-                return -ENOMEM;
-
-        if (enable) {
-                status = pci_write_config_dword(adapter->pdev,
-                                                PCICFG_PM_CONTROL_OFFSET,
-                                                PCICFG_PM_CONTROL_MASK);
-                if (status) {
-                        dev_err(dev, "Could not enable Wake-on-lan\n");
-                        goto err;
-                }
-        } else {
-                ether_addr_copy(mac, adapter->netdev->dev_addr);
-        }
-
-        status = be_cmd_enable_magic_wol(adapter, mac, &cmd);
-        pci_enable_wake(adapter->pdev, PCI_D3hot, enable);
-        pci_enable_wake(adapter->pdev, PCI_D3cold, enable);
-err:
-        dma_free_coherent(dev, cmd.size, cmd.va, cmd.dma);
-        return status;
-}
-
 static void be_vf_eth_addr_generate(struct be_adapter *adapter, u8 *mac)
 {
         u32 addr;
@@ -3759,6 +3729,11 @@ static void be_vf_clear(struct be_adapter *adapter)
 
                 be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1);
         }
+
+        if (BE3_chip(adapter))
+                be_cmd_set_hsw_config(adapter, 0, 0,
+                                      adapter->if_handle,
+                                      PORT_FWD_TYPE_PASSTHRU, 0);
 done:
         kfree(adapter->vf_cfg);
         adapter->num_vfs = 0;
@@ -3789,7 +3764,6 @@ static void be_cancel_err_detection(struct be_adapter *adapter)
         }
 }
 
-#ifdef CONFIG_BE2NET_VXLAN
 static void be_disable_vxlan_offloads(struct be_adapter *adapter)
 {
         struct net_device *netdev = adapter->netdev;
@@ -3808,37 +3782,87 @@ static void be_disable_vxlan_offloads(struct be_adapter *adapter)
         netdev->hw_features &= ~(NETIF_F_GSO_UDP_TUNNEL);
         netdev->features &= ~(NETIF_F_GSO_UDP_TUNNEL);
 }
-#endif
 
-static u16 be_calculate_vf_qs(struct be_adapter *adapter, u16 num_vfs)
+static void be_calculate_vf_res(struct be_adapter *adapter, u16 num_vfs,
+                                struct be_resources *vft_res)
 {
         struct be_resources res = adapter->pool_res;
+        u32 vf_if_cap_flags = res.vf_if_cap_flags;
+        struct be_resources res_mod = {0};
         u16 num_vf_qs = 1;
 
-        /* Distribute the queue resources among the PF and it's VFs
-         * Do not distribute queue resources in multi-channel configuration.
-         */
-        if (num_vfs && !be_is_mc(adapter)) {
-                 /* Divide the qpairs evenly among the VFs and the PF, capped
-                  * at VF-EQ-count. Any remainder qpairs belong to the PF.
-                  */
+        /* Distribute the queue resources among the PF and it's VFs */
+        if (num_vfs) {
+                /* Divide the rx queues evenly among the VFs and the PF, capped
+                 * at VF-EQ-count. Any remainder queues belong to the PF.
+                 */
                 num_vf_qs = min(SH_VF_MAX_NIC_EQS,
                                 res.max_rss_qs / (num_vfs + 1));
 
-                /* Skyhawk-R chip supports only MAX_RSS_IFACES RSS capable
-                 * interfaces per port. Provide RSS on VFs, only if number
-                 * of VFs requested is less than MAX_RSS_IFACES limit.
+                /* Skyhawk-R chip supports only MAX_PORT_RSS_TABLES
+                 * RSS Tables per port. Provide RSS on VFs, only if number of
+                 * VFs requested is less than it's PF Pool's RSS Tables limit.
                  */
-                if (num_vfs >= MAX_RSS_IFACES)
+                if (num_vfs >= be_max_pf_pool_rss_tables(adapter))
                         num_vf_qs = 1;
         }
-        return num_vf_qs;
+
+        /* Resource with fields set to all '1's by GET_PROFILE_CONFIG cmd,
+         * which are modifiable using SET_PROFILE_CONFIG cmd.
+         */
+        be_cmd_get_profile_config(adapter, &res_mod, NULL, ACTIVE_PROFILE_TYPE,
+                                  RESOURCE_MODIFIABLE, 0);
+
+        /* If RSS IFACE capability flags are modifiable for a VF, set the
+         * capability flag as valid and set RSS and DEFQ_RSS IFACE flags if
+         * more than 1 RSSQ is available for a VF.
+         * Otherwise, provision only 1 queue pair for VF.
+         */
+        if (res_mod.vf_if_cap_flags & BE_IF_FLAGS_RSS) {
+                vft_res->flags |= BIT(IF_CAPS_FLAGS_VALID_SHIFT);
+                if (num_vf_qs > 1) {
+                        vf_if_cap_flags |= BE_IF_FLAGS_RSS;
+                        if (res.if_cap_flags & BE_IF_FLAGS_DEFQ_RSS)
+                                vf_if_cap_flags |= BE_IF_FLAGS_DEFQ_RSS;
+                } else {
+                        vf_if_cap_flags &= ~(BE_IF_FLAGS_RSS |
+                                             BE_IF_FLAGS_DEFQ_RSS);
+                }
+        } else {
+                num_vf_qs = 1;
+        }
+
+        if (res_mod.vf_if_cap_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) {
+                vft_res->flags |= BIT(IF_CAPS_FLAGS_VALID_SHIFT);
+                vf_if_cap_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS;
+        }
+
+        vft_res->vf_if_cap_flags = vf_if_cap_flags;
+        vft_res->max_rx_qs = num_vf_qs;
+        vft_res->max_rss_qs = num_vf_qs;
+        vft_res->max_tx_qs = res.max_tx_qs / (num_vfs + 1);
+        vft_res->max_cq_count = res.max_cq_count / (num_vfs + 1);
+
+        /* Distribute unicast MACs, VLANs, IFACE count and MCCQ count equally
+         * among the PF and it's VFs, if the fields are changeable
+         */
+        if (res_mod.max_uc_mac == FIELD_MODIFIABLE)
+                vft_res->max_uc_mac = res.max_uc_mac / (num_vfs + 1);
+
+        if (res_mod.max_vlans == FIELD_MODIFIABLE)
+                vft_res->max_vlans = res.max_vlans / (num_vfs + 1);
+
+        if (res_mod.max_iface_count == FIELD_MODIFIABLE)
+                vft_res->max_iface_count = res.max_iface_count / (num_vfs + 1);
+
+        if (res_mod.max_mcc_count == FIELD_MODIFIABLE)
+                vft_res->max_mcc_count = res.max_mcc_count / (num_vfs + 1);
 }
 
 static int be_clear(struct be_adapter *adapter)
 {
         struct pci_dev *pdev = adapter->pdev;
-        u16 num_vf_qs;
+        struct  be_resources vft_res = {0};
 
         be_cancel_worker(adapter);
 
@@ -3850,16 +3874,15 @@ static int be_clear(struct be_adapter *adapter)
          */
         if (skyhawk_chip(adapter) && be_physfn(adapter) &&
             !pci_vfs_assigned(pdev)) {
-                num_vf_qs = be_calculate_vf_qs(adapter,
-                                               pci_sriov_get_totalvfs(pdev));
+                be_calculate_vf_res(adapter,
+                                    pci_sriov_get_totalvfs(pdev),
+                                    &vft_res);
                 be_cmd_set_sriov_config(adapter, adapter->pool_res,
                                         pci_sriov_get_totalvfs(pdev),
-                                        num_vf_qs);
+                                        &vft_res);
         }
 
-#ifdef CONFIG_BE2NET_VXLAN
         be_disable_vxlan_offloads(adapter);
-#endif
         kfree(adapter->pmac_id);
         adapter->pmac_id = NULL;
 
@@ -3884,7 +3907,8 @@ static int be_vfs_if_create(struct be_adapter *adapter)
 
         for_all_vfs(adapter, vf_cfg, vf) {
                 if (!BE3_chip(adapter)) {
-                        status = be_cmd_get_profile_config(adapter, &res,
+                        status = be_cmd_get_profile_config(adapter, &res, NULL,
+                                                           ACTIVE_PROFILE_TYPE,
                                                            RESOURCE_LIMITS,
                                                            vf + 1);
                         if (!status) {
@@ -4000,6 +4024,15 @@ static int be_vf_setup(struct be_adapter *adapter)
                 }
         }
 
+        if (BE3_chip(adapter)) {
+                /* On BE3, enable VEB only when SRIOV is enabled */
+                status = be_cmd_set_hsw_config(adapter, 0, 0,
+                                               adapter->if_handle,
+                                               PORT_FWD_TYPE_VEB, 0);
+                if (status)
+                        goto err;
+        }
+
         adapter->flags |= BE_FLAGS_SRIOV_ENABLED;
         return 0;
 err:
@@ -4069,8 +4102,9 @@ static void BEx_get_resources(struct be_adapter *adapter,
                 /* On a SuperNIC profile, the driver needs to use the
                  * GET_PROFILE_CONFIG cmd to query the per-function TXQ limits
                  */
-                be_cmd_get_profile_config(adapter, &super_nic_res,
-                                          RESOURCE_LIMITS, 0);
+                be_cmd_get_profile_config(adapter, &super_nic_res, NULL,
+                                          ACTIVE_PROFILE_TYPE, RESOURCE_LIMITS,
+                                          0);
                 /* Some old versions of BE3 FW don't report max_tx_qs value */
                 res->max_tx_qs = super_nic_res.max_tx_qs ? : BE3_MAX_TX_QS;
         } else {
@@ -4109,12 +4143,38 @@ static void be_setup_init(struct be_adapter *adapter)
                 adapter->cmd_privileges = MIN_PRIVILEGES;
 }
 
+/* HW supports only MAX_PORT_RSS_TABLES RSS Policy Tables per port.
+ * However, this HW limitation is not exposed to the host via any SLI cmd.
+ * As a result, in the case of SRIOV and in particular multi-partition configs
+ * the driver needs to calcuate a proportional share of RSS Tables per PF-pool
+ * for distribution between the VFs. This self-imposed limit will determine the
+ * no: of VFs for which RSS can be enabled.
+ */
+void be_calculate_pf_pool_rss_tables(struct be_adapter *adapter)
+{
+        struct be_port_resources port_res = {0};
+        u8 rss_tables_on_port;
+        u16 max_vfs = be_max_vfs(adapter);
+
+        be_cmd_get_profile_config(adapter, NULL, &port_res, SAVED_PROFILE_TYPE,
+                                  RESOURCE_LIMITS, 0);
+
+        rss_tables_on_port = MAX_PORT_RSS_TABLES - port_res.nic_pfs;
+
+        /* Each PF Pool's RSS Tables limit =
+         * PF's Max VFs / Total_Max_VFs on Port * RSS Tables on Port
+         */
+        adapter->pool_res.max_rss_tables =
+                max_vfs * rss_tables_on_port / port_res.max_vfs;
+}
+
 static int be_get_sriov_config(struct be_adapter *adapter)
 {
         struct be_resources res = {0};
         int max_vfs, old_vfs;
 
-        be_cmd_get_profile_config(adapter, &res, RESOURCE_LIMITS, 0);
+        be_cmd_get_profile_config(adapter, &res, NULL, ACTIVE_PROFILE_TYPE,
+                                  RESOURCE_LIMITS, 0);
 
         /* Some old versions of BE3 FW don't report max_vfs value */
         if (BE3_chip(adapter) && !res.max_vfs) {
@@ -4138,13 +4198,19 @@ static int be_get_sriov_config(struct be_adapter *adapter)
                 adapter->num_vfs = old_vfs;
         }
 
+        if (skyhawk_chip(adapter) && be_max_vfs(adapter) && !old_vfs) {
+                be_calculate_pf_pool_rss_tables(adapter);
+                dev_info(&adapter->pdev->dev,
+                         "RSS can be enabled for all VFs if num_vfs <= %d\n",
+                         be_max_pf_pool_rss_tables(adapter));
+        }
         return 0;
 }
 
 static void be_alloc_sriov_res(struct be_adapter *adapter)
 {
         int old_vfs = pci_num_vf(adapter->pdev);
-        u16 num_vf_qs;
+        struct  be_resources vft_res = {0};
         int status;
 
         be_get_sriov_config(adapter);
@@ -4158,9 +4224,9 @@ static void be_alloc_sriov_res(struct be_adapter *adapter)
          * Also, this is done by FW in Lancer chip.
          */
         if (skyhawk_chip(adapter) && be_max_vfs(adapter) && !old_vfs) {
-                num_vf_qs = be_calculate_vf_qs(adapter, 0);
+                be_calculate_vf_res(adapter, 0, &vft_res);
                 status = be_cmd_set_sriov_config(adapter, adapter->pool_res, 0,
-                                                 num_vf_qs);
+                                                 &vft_res);
                 if (status)
                         dev_err(&adapter->pdev->dev,
                                 "Failed to optimize SRIOV resources\n");
@@ -4173,16 +4239,13 @@ static int be_get_resources(struct be_adapter *adapter)
         struct be_resources res = {0};
         int status;
 
-        if (BEx_chip(adapter)) {
-                BEx_get_resources(adapter, &res);
-                adapter->res = res;
-        }
-
         /* For Lancer, SH etc read per-function resource limits from FW.
          * GET_FUNC_CONFIG returns per function guaranteed limits.
          * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits
          */
-        if (!BEx_chip(adapter)) {
+        if (BEx_chip(adapter)) {
+                BEx_get_resources(adapter, &res);
+        } else {
                 status = be_cmd_get_func_config(adapter, &res);
                 if (status)
                         return status;
@@ -4191,13 +4254,13 @@ static int be_get_resources(struct be_adapter *adapter)
                 if (res.max_rss_qs && res.max_rss_qs == res.max_rx_qs &&
                     !(res.if_cap_flags & BE_IF_FLAGS_DEFQ_RSS))
                         res.max_rss_qs -= 1;
-
-                /* If RoCE may be enabled stash away half the EQs for RoCE */
-                if (be_roce_supported(adapter))
-                        res.max_evt_qs /= 2;
-                adapter->res = res;
         }
 
+        /* If RoCE is supported stash away half the EQs for RoCE */
+        res.max_nic_evt_qs = be_roce_supported(adapter) ?
+                                res.max_evt_qs / 2 : res.max_evt_qs;
+        adapter->res = res;
+
         /* If FW supports RSS default queue, then skip creating non-RSS
          * queue for non-IP traffic.
          */
@@ -4206,15 +4269,17 @@ static int be_get_resources(struct be_adapter *adapter)
 
         dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n",
                  be_max_txqs(adapter), be_max_rxqs(adapter),
-                 be_max_rss(adapter), be_max_eqs(adapter),
+                 be_max_rss(adapter), be_max_nic_eqs(adapter),
                  be_max_vfs(adapter));
         dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n",
                  be_max_uc(adapter), be_max_mc(adapter),
                  be_max_vlans(adapter));
 
-        /* Sanitize cfg_num_qs based on HW and platform limits */
-        adapter->cfg_num_qs = min_t(u16, netif_get_num_default_rss_queues(),
-                                    be_max_qs(adapter));
+        /* Ensure RX and TX queues are created in pairs at init time */
+        adapter->cfg_num_rx_irqs =
+                                min_t(u16, netif_get_num_default_rss_queues(),
+                                      be_max_qp_irqs(adapter));
+        adapter->cfg_num_tx_irqs = adapter->cfg_num_rx_irqs;
         return 0;
 }
 
@@ -4241,6 +4306,8 @@ static int be_get_config(struct be_adapter *adapter)
         }
 
         be_cmd_get_acpi_wol_cap(adapter);
+        pci_enable_wake(adapter->pdev, PCI_D3hot, adapter->wol_en);
+        pci_enable_wake(adapter->pdev, PCI_D3cold, adapter->wol_en);
 
         be_cmd_query_port_name(adapter);
 
@@ -4251,15 +4318,6 @@ static int be_get_config(struct be_adapter *adapter)
                                  "Using profile 0x%x\n", profile_id);
         }
 
-        status = be_get_resources(adapter);
-        if (status)
-                return status;
-
-        adapter->pmac_id = kcalloc(be_max_uc(adapter),
-                                   sizeof(*adapter->pmac_id), GFP_KERNEL);
-        if (!adapter->pmac_id)
-                return -ENOMEM;
-
         return 0;
 }
 
@@ -4334,7 +4392,7 @@ static int be_if_create(struct be_adapter *adapter)
         u32 cap_flags = be_if_cap_flags(adapter);
         int status;
 
-        if (adapter->cfg_num_qs == 1)
+        if (adapter->cfg_num_rx_irqs == 1)
                 cap_flags &= ~(BE_IF_FLAGS_DEFQ_RSS | BE_IF_FLAGS_RSS);
 
         en_flags &= cap_flags;
@@ -4460,13 +4518,22 @@ static int be_setup(struct be_adapter *adapter)
                         return status;
         }
 
+        status = be_get_config(adapter);
+        if (status)
+                goto err;
+
         if (!BE2_chip(adapter) && be_physfn(adapter))
                 be_alloc_sriov_res(adapter);
 
-        status = be_get_config(adapter);
+        status = be_get_resources(adapter);
         if (status)
                 goto err;
 
+        adapter->pmac_id = kcalloc(be_max_uc(adapter),
+                                   sizeof(*adapter->pmac_id), GFP_KERNEL);
+        if (!adapter->pmac_id)
+                return -ENOMEM;
+
         status = be_msix_enable(adapter);
         if (status)
                 goto err;
@@ -4511,6 +4578,15 @@ static int be_setup(struct be_adapter *adapter)
                 be_cmd_set_logical_link_config(adapter,
                                                IFLA_VF_LINK_STATE_AUTO, 0);
 
+        /* BE3 EVB echoes broadcast/multicast packets back to PF's vport
+         * confusing a linux bridge or OVS that it might be connected to.
+         * Set the EVB to PASSTHRU mode which effectively disables the EVB
+         * when SRIOV is not enabled.
+         */
+        if (BE3_chip(adapter))
+                be_cmd_set_hsw_config(adapter, 0, 0, adapter->if_handle,
+                                      PORT_FWD_TYPE_PASSTHRU, 0);
+
         if (adapter->num_vfs)
                 be_vf_setup(adapter);
 
@@ -4651,7 +4727,6 @@ static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
                                        0, 0, nlflags, filter_mask, NULL);
 }
 
-#ifdef CONFIG_BE2NET_VXLAN
 /* VxLAN offload Notes:
  *
  * The stack defines tunnel offload flags (hw_enc_features) for IP and doesn't
@@ -4666,13 +4741,17 @@ static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
  * adds more than one port, disable offloads and don't re-enable them again
  * until after all the tunnels are removed.
  */
-static void be_add_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
-                              __be16 port)
+static void be_add_vxlan_port(struct net_device *netdev,
+                              struct udp_tunnel_info *ti)
 {
         struct be_adapter *adapter = netdev_priv(netdev);
         struct device *dev = &adapter->pdev->dev;
+        __be16 port = ti->port;
         int status;
 
+        if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
+                return;
+
         if (lancer_chip(adapter) || BEx_chip(adapter) || be_is_mc(adapter))
                 return;
 
@@ -4720,10 +4799,14 @@ err:
         be_disable_vxlan_offloads(adapter);
 }
 
-static void be_del_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
-                              __be16 port)
+static void be_del_vxlan_port(struct net_device *netdev,
+                              struct udp_tunnel_info *ti)
 {
         struct be_adapter *adapter = netdev_priv(netdev);
+        __be16 port = ti->port;
+
+        if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
+                return;
 
         if (lancer_chip(adapter) || BEx_chip(adapter) || be_is_mc(adapter))
                 return;
@@ -4785,7 +4868,6 @@ static netdev_features_t be_features_check(struct sk_buff *skb,
 
         return features;
 }
-#endif
 
 static int be_get_phys_port_id(struct net_device *dev,
                                struct netdev_phys_item_id *ppid)
@@ -4833,11 +4915,9 @@ static const struct net_device_ops be_netdev_ops = {
 #ifdef CONFIG_NET_RX_BUSY_POLL
         .ndo_busy_poll          = be_busy_poll,
 #endif
-#ifdef CONFIG_BE2NET_VXLAN
-        .ndo_add_vxlan_port     = be_add_vxlan_port,
-        .ndo_del_vxlan_port     = be_del_vxlan_port,
+        .ndo_udp_tunnel_add     = be_add_vxlan_port,
+        .ndo_udp_tunnel_del     = be_del_vxlan_port,
         .ndo_features_check     = be_features_check,
-#endif
         .ndo_get_phys_port_id   = be_get_phys_port_id,
 };
 
@@ -5410,9 +5490,6 @@ static int be_suspend(struct pci_dev *pdev, pm_message_t state)
 {
         struct be_adapter *adapter = pci_get_drvdata(pdev);
 
-        if (adapter->wol_en)
-                be_setup_wol(adapter, true);
-
         be_intr_set(adapter, false);
         be_cancel_err_detection(adapter);
 
@@ -5441,9 +5518,6 @@ static int be_pci_resume(struct pci_dev *pdev)
 
         be_schedule_err_detection(adapter, ERR_DETECTION_DELAY);
 
-        if (adapter->wol_en)
-                be_setup_wol(adapter, false);
-
         return 0;
 }
 
@@ -5552,7 +5626,7 @@ err:
 static int be_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
 {
         struct be_adapter *adapter = pci_get_drvdata(pdev);
-        u16 num_vf_qs;
+        struct be_resources vft_res = {0};
         int status;
 
         if (!num_vfs)
@@ -5575,9 +5649,10 @@ static int be_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
          * Also, this is done by FW in Lancer chip.
          */
         if (skyhawk_chip(adapter) && !pci_num_vf(pdev)) {
-                num_vf_qs = be_calculate_vf_qs(adapter, adapter->num_vfs);
+                be_calculate_vf_res(adapter, adapter->num_vfs,
+                                    &vft_res);
                 status = be_cmd_set_sriov_config(adapter, adapter->pool_res,
-                                                 adapter->num_vfs, num_vf_qs);
+                                                 adapter->num_vfs, &vft_res);
                 if (status)
                         dev_err(&pdev->dev,
                                 "Failed to optimize SR-IOV resources\n");
diff --git a/drivers/net/ethernet/emulex/benet/be_roce.c b/drivers/net/ethernet/emulex/benet/be_roce.c
index 4089156a7f5e..2b62841c4c63 100644
--- a/drivers/net/ethernet/emulex/benet/be_roce.c
+++ b/drivers/net/ethernet/emulex/benet/be_roce.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2005 - 2015 Emulex
+ * Copyright (C) 2005 - 2016 Broadcom
  * All rights reserved.
  *
  * This program is free software; you can redistribute it and/or
diff --git a/drivers/net/ethernet/emulex/benet/be_roce.h b/drivers/net/ethernet/emulex/benet/be_roce.h
index fde609789483..e51719a7307f 100644
--- a/drivers/net/ethernet/emulex/benet/be_roce.h
+++ b/drivers/net/ethernet/emulex/benet/be_roce.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2005 - 2015 Emulex
+ * Copyright (C) 2005 - 2016 Broadcom
  * All rights reserved.
  *
  * This program is free software; you can redistribute it and/or
diff --git a/drivers/net/ethernet/freescale/fec.h b/drivers/net/ethernet/freescale/fec.h
index f58f9ea51639..92fd5c0bf4df 100644
--- a/drivers/net/ethernet/freescale/fec.h
+++ b/drivers/net/ethernet/freescale/fec.h
@@ -442,6 +442,8 @@ struct bufdesc_ex {
 #define FEC_QUIRK_SINGLE_MDIO           (1 << 11)
 /* Controller supports RACC register */
 #define FEC_QUIRK_HAS_RACC              (1 << 12)
+/* Controller supports interrupt coalesc */
+#define FEC_QUIRK_HAS_COALESCE          (1 << 13)
 
 struct bufdesc_prop {
         int qid;
diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c
index fea0f330ddbd..4040003a74f9 100644
--- a/drivers/net/ethernet/freescale/fec_main.c
+++ b/drivers/net/ethernet/freescale/fec_main.c
@@ -111,7 +111,13 @@ static struct platform_device_id fec_devtype[] = {
                                 FEC_QUIRK_HAS_BUFDESC_EX | FEC_QUIRK_HAS_CSUM |
                                 FEC_QUIRK_HAS_VLAN | FEC_QUIRK_HAS_AVB |
                                 FEC_QUIRK_ERR007885 | FEC_QUIRK_BUG_CAPTURE |
-                                FEC_QUIRK_HAS_RACC,
+                                FEC_QUIRK_HAS_RACC | FEC_QUIRK_HAS_COALESCE,
+        }, {
+                .name = "imx6ul-fec",
+                .driver_data = FEC_QUIRK_ENET_MAC | FEC_QUIRK_HAS_GBIT |
+                                FEC_QUIRK_HAS_BUFDESC_EX | FEC_QUIRK_HAS_CSUM |
+                                FEC_QUIRK_HAS_VLAN | FEC_QUIRK_BUG_CAPTURE |
+                                FEC_QUIRK_HAS_RACC | FEC_QUIRK_HAS_COALESCE,
         }, {
                 /* sentinel */
         }
@@ -125,6 +131,7 @@ enum imx_fec_type {
         IMX6Q_FEC,
         MVF600_FEC,
         IMX6SX_FEC,
+        IMX6UL_FEC,
 };
 
 static const struct of_device_id fec_dt_ids[] = {
@@ -134,6 +141,7 @@ static const struct of_device_id fec_dt_ids[] = {
         { .compatible = "fsl,imx6q-fec", .data = &fec_devtype[IMX6Q_FEC], },
         { .compatible = "fsl,mvf600-fec", .data = &fec_devtype[MVF600_FEC], },
         { .compatible = "fsl,imx6sx-fec", .data = &fec_devtype[IMX6SX_FEC], },
+        { .compatible = "fsl,imx6ul-fec", .data = &fec_devtype[IMX6UL_FEC], },
         { /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, fec_dt_ids);
@@ -2358,9 +2366,6 @@ static void fec_enet_itr_coal_set(struct net_device *ndev)
         struct fec_enet_private *fep = netdev_priv(ndev);
         int rx_itr, tx_itr;
 
-        if (!(fep->quirks & FEC_QUIRK_HAS_AVB))
-                return;
-
         /* Must be greater than zero to avoid unpredictable behavior */
         if (!fep->rx_time_itr || !fep->rx_pkts_itr ||
             !fep->tx_time_itr || !fep->tx_pkts_itr)
@@ -2383,10 +2388,12 @@ static void fec_enet_itr_coal_set(struct net_device *ndev)
 
         writel(tx_itr, fep->hwp + FEC_TXIC0);
         writel(rx_itr, fep->hwp + FEC_RXIC0);
-        writel(tx_itr, fep->hwp + FEC_TXIC1);
-        writel(rx_itr, fep->hwp + FEC_RXIC1);
-        writel(tx_itr, fep->hwp + FEC_TXIC2);
-        writel(rx_itr, fep->hwp + FEC_RXIC2);
+        if (fep->quirks & FEC_QUIRK_HAS_AVB) {
+                writel(tx_itr, fep->hwp + FEC_TXIC1);
+                writel(rx_itr, fep->hwp + FEC_RXIC1);
+                writel(tx_itr, fep->hwp + FEC_TXIC2);
+                writel(rx_itr, fep->hwp + FEC_RXIC2);
+        }
 }
 
 static int
@@ -2394,7 +2401,7 @@ fec_enet_get_coalesce(struct net_device *ndev, struct ethtool_coalesce *ec)
 {
         struct fec_enet_private *fep = netdev_priv(ndev);
 
-        if (!(fep->quirks & FEC_QUIRK_HAS_AVB))
+        if (!(fep->quirks & FEC_QUIRK_HAS_COALESCE))
                 return -EOPNOTSUPP;
 
         ec->rx_coalesce_usecs = fep->rx_time_itr;
@@ -2412,7 +2419,7 @@ fec_enet_set_coalesce(struct net_device *ndev, struct ethtool_coalesce *ec)
         struct fec_enet_private *fep = netdev_priv(ndev);
         unsigned int cycle;
 
-        if (!(fep->quirks & FEC_QUIRK_HAS_AVB))
+        if (!(fep->quirks & FEC_QUIRK_HAS_COALESCE))
                 return -EOPNOTSUPP;
 
         if (ec->rx_max_coalesced_frames > 255) {
@@ -3191,7 +3198,12 @@ static void fec_reset_phy(struct platform_device *pdev)
                 dev_err(&pdev->dev, "failed to get phy-reset-gpios: %d\n", err);
                 return;
         }
-        msleep(msec);
+
+        if (msec > 20)
+                msleep(msec);
+        else
+                usleep_range(msec * 1000, msec * 1000 + 1000);
+
         gpio_set_value_cansleep(phy_reset, !active_high);
 }
 #else /* CONFIG_OF */
diff --git a/drivers/net/ethernet/freescale/gianfar.c b/drivers/net/ethernet/freescale/gianfar.c
index 2e6785b6e8be..d20935dc8399 100644
--- a/drivers/net/ethernet/freescale/gianfar.c
+++ b/drivers/net/ethernet/freescale/gianfar.c
@@ -2275,7 +2275,7 @@ static inline void gfar_tx_checksum(struct sk_buff *skb, struct txfcb *fcb,
         fcb->flags = flags;
 }
 
-void inline gfar_tx_vlan(struct sk_buff *skb, struct txfcb *fcb)
+static inline void gfar_tx_vlan(struct sk_buff *skb, struct txfcb *fcb)
 {
         fcb->flags |= TXFCB_VLN;
         fcb->vlctl = cpu_to_be16(skb_vlan_tag_get(skb));
diff --git a/drivers/net/ethernet/hisilicon/Kconfig b/drivers/net/ethernet/hisilicon/Kconfig
index 4ccc032633c4..2e2566230e27 100644
--- a/drivers/net/ethernet/hisilicon/Kconfig
+++ b/drivers/net/ethernet/hisilicon/Kconfig
@@ -5,7 +5,7 @@
 config NET_VENDOR_HISILICON
         bool "Hisilicon devices"
         default y
-        depends on OF && HAS_DMA
+        depends on (OF || ACPI) && HAS_DMA
         depends on ARM || ARM64 || COMPILE_TEST
         ---help---
           If you have a network (Ethernet) card belonging to this class, say Y.
diff --git a/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c b/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c
index b9f2ea59308a..275618bb4646 100644
--- a/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c
+++ b/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c
@@ -218,7 +218,6 @@ struct hix5hd2_priv {
         struct device *dev;
         struct net_device *netdev;
 
-        struct phy_device *phy;
         struct device_node *phy_node;
         phy_interface_t phy_mode;
 
@@ -402,7 +401,7 @@ static int hix5hd2_net_set_mac_address(struct net_device *dev, void *p)
 static void hix5hd2_adjust_link(struct net_device *dev)
 {
         struct hix5hd2_priv *priv = netdev_priv(dev);
-        struct phy_device *phy = priv->phy;
+        struct phy_device *phy = dev->phydev;
 
         if ((priv->speed != phy->speed) || (priv->duplex != phy->duplex)) {
                 hix5hd2_config_port(dev, phy->speed, phy->duplex);
@@ -679,6 +678,7 @@ static void hix5hd2_free_dma_desc_rings(struct hix5hd2_priv *priv)
 static int hix5hd2_net_open(struct net_device *dev)
 {
         struct hix5hd2_priv *priv = netdev_priv(dev);
+        struct phy_device *phy;
         int ret;
 
         ret = clk_prepare_enable(priv->clk);
@@ -687,12 +687,12 @@ static int hix5hd2_net_open(struct net_device *dev)
                 return ret;
         }
 
-        priv->phy = of_phy_connect(dev, priv->phy_node,
-                                   &hix5hd2_adjust_link, 0, priv->phy_mode);
-        if (!priv->phy)
+        phy = of_phy_connect(dev, priv->phy_node,
+                             &hix5hd2_adjust_link, 0, priv->phy_mode);
+        if (!phy)
                 return -ENODEV;
 
-        phy_start(priv->phy);
+        phy_start(phy);
         hix5hd2_hw_init(priv);
         hix5hd2_rx_refill(priv);
 
@@ -716,9 +716,9 @@ static int hix5hd2_net_close(struct net_device *dev)
         netif_stop_queue(dev);
         hix5hd2_free_dma_desc_rings(priv);
 
-        if (priv->phy) {
-                phy_stop(priv->phy);
-                phy_disconnect(priv->phy);
+        if (dev->phydev) {
+                phy_stop(dev->phydev);
+                phy_disconnect(dev->phydev);
         }
 
         clk_disable_unprepare(priv->clk);
@@ -750,32 +750,10 @@ static const struct net_device_ops hix5hd2_netdev_ops = {
         .ndo_set_mac_address    = hix5hd2_net_set_mac_address,
 };
 
-static int hix5hd2_get_settings(struct net_device *net_dev,
-                                struct ethtool_cmd *cmd)
-{
-        struct hix5hd2_priv *priv = netdev_priv(net_dev);
-
-        if (!priv->phy)
-                return -ENODEV;
-
-        return phy_ethtool_gset(priv->phy, cmd);
-}
-
-static int hix5hd2_set_settings(struct net_device *net_dev,
-                                struct ethtool_cmd *cmd)
-{
-        struct hix5hd2_priv *priv = netdev_priv(net_dev);
-
-        if (!priv->phy)
-                return -ENODEV;
-
-        return phy_ethtool_sset(priv->phy, cmd);
-}
-
 static struct ethtool_ops hix5hd2_ethtools_ops = {
         .get_link               = ethtool_op_get_link,
-        .get_settings           = hix5hd2_get_settings,
-        .set_settings           = hix5hd2_set_settings,
+        .get_link_ksettings     = phy_ethtool_get_link_ksettings,
+        .set_link_ksettings     = phy_ethtool_set_link_ksettings,
 };
 
 static int hix5hd2_mdio_wait_ready(struct mii_bus *bus)
diff --git a/drivers/net/ethernet/hisilicon/hns/hnae.c b/drivers/net/ethernet/hisilicon/hns/hnae.c
index 3bfe36f9405b..c54c6fac0d1d 100644
--- a/drivers/net/ethernet/hisilicon/hns/hnae.c
+++ b/drivers/net/ethernet/hisilicon/hns/hnae.c
@@ -96,16 +96,22 @@ static int __ae_match(struct device *dev, const void *data)
 {
         struct hnae_ae_dev *hdev = cls_to_ae_dev(dev);
 
-        return hdev->dev->of_node == data;
+        if (dev_of_node(hdev->dev))
+                return (data == &hdev->dev->of_node->fwnode);
+        else if (is_acpi_node(hdev->dev->fwnode))
+                return (data == hdev->dev->fwnode);
+
+        dev_err(dev, "__ae_match cannot read cfg data from OF or acpi\n");
+        return 0;
 }
 
-static struct hnae_ae_dev *find_ae(const struct device_node *ae_node)
+static struct hnae_ae_dev *find_ae(const struct fwnode_handle *fwnode)
 {
         struct device *dev;
 
-        WARN_ON(!ae_node);
+        WARN_ON(!fwnode);
 
-        dev = class_find_device(hnae_class, NULL, ae_node, __ae_match);
+        dev = class_find_device(hnae_class, NULL, fwnode, __ae_match);
 
         return dev ? cls_to_ae_dev(dev) : NULL;
 }
@@ -312,7 +318,7 @@ EXPORT_SYMBOL(hnae_reinit_handle);
  * return handle ptr or ERR_PTR
  */
 struct hnae_handle *hnae_get_handle(struct device *owner_dev,
-                                    const struct device_node *ae_node,
+                                    const struct fwnode_handle  *fwnode,
                                     u32 port_id,
                                     struct hnae_buf_ops *bops)
 {
@@ -321,7 +327,7 @@ struct hnae_handle *hnae_get_handle(struct device *owner_dev,
         int i, j;
         int ret;
 
-        dev = find_ae(ae_node);
+        dev = find_ae(fwnode);
         if (!dev)
                 return ERR_PTR(-ENODEV);
 
@@ -394,7 +400,6 @@ int hnae_ae_register(struct hnae_ae_dev *hdev, struct module *owner)
 
         if (!hdev->ops || !hdev->ops->get_handle ||
             !hdev->ops->toggle_ring_irq ||
-            !hdev->ops->toggle_queue_status ||
             !hdev->ops->get_status || !hdev->ops->adjust_link)
                 return -EINVAL;
 
diff --git a/drivers/net/ethernet/hisilicon/hns/hnae.h b/drivers/net/ethernet/hisilicon/hns/hnae.h
index e8d36aaea223..3869322690ac 100644
--- a/drivers/net/ethernet/hisilicon/hns/hnae.h
+++ b/drivers/net/ethernet/hisilicon/hns/hnae.h
@@ -27,6 +27,7 @@
  * "cb" means control block
  */
 
+#include <linux/acpi.h>
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/module.h>
@@ -453,7 +454,6 @@ struct hnae_ae_ops {
         int (*get_info)(struct hnae_handle *handle,
                         u8 *auto_neg, u16 *speed, u8 *duplex);
         void (*toggle_ring_irq)(struct hnae_ring *ring, u32 val);
-        void (*toggle_queue_status)(struct hnae_queue *queue, u32 val);
         void (*adjust_link)(struct hnae_handle *handle, int speed, int duplex);
         int (*set_loopback)(struct hnae_handle *handle,
                             enum hnae_loop loop_mode, int en);
@@ -472,6 +472,11 @@ struct hnae_ae_ops {
         int (*set_coalesce_usecs)(struct hnae_handle *handle, u32 timeout);
         int (*set_coalesce_frames)(struct hnae_handle *handle,
                                    u32 coalesce_frames);
+        void (*get_coalesce_range)(struct hnae_handle *handle,
+                                   u32 *tx_frames_low, u32 *rx_frames_low,
+                                   u32 *tx_frames_high, u32 *rx_frames_high,
+                                   u32 *tx_usecs_low, u32 *rx_usecs_low,
+                                   u32 *tx_usecs_high, u32 *rx_usecs_high);
         void (*set_promisc_mode)(struct hnae_handle *handle, u32 en);
         int (*get_mac_addr)(struct hnae_handle *handle, void **p);
         int (*set_mac_addr)(struct hnae_handle *handle, void *p);
@@ -512,7 +517,7 @@ struct hnae_ae_dev {
 struct hnae_handle {
         struct device *owner_dev; /* the device which make use of this handle */
         struct hnae_ae_dev *dev;  /* the device who provides this handle */
-        struct device_node *phy_node;
+        struct phy_device *phy_dev;
         phy_interface_t phy_if;
         u32 if_support;
         int q_num;
@@ -528,7 +533,7 @@ struct hnae_handle {
 #define ring_to_dev(ring) ((ring)->q->dev->dev)
 
 struct hnae_handle *hnae_get_handle(struct device *owner_dev,
-                                    const struct device_node *ae_node,
+                                    const struct fwnode_handle  *fwnode,
                                     u32 port_id,
                                     struct hnae_buf_ops *bops);
 
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_ae_adapt.c b/drivers/net/ethernet/hisilicon/hns/hns_ae_adapt.c
index 7a757e88c89a..835521bf1bbc 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_ae_adapt.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_ae_adapt.c
@@ -131,7 +131,7 @@ struct hnae_handle *hns_ae_get_handle(struct hnae_ae_dev *dev,
         vf_cb->mac_cb = dsaf_dev->mac_cb[port_id];
 
         ae_handle->phy_if = vf_cb->mac_cb->phy_if;
-        ae_handle->phy_node = vf_cb->mac_cb->phy_node;
+        ae_handle->phy_dev = vf_cb->mac_cb->phy_dev;
         ae_handle->if_support = vf_cb->mac_cb->if_support;
         ae_handle->port_type = vf_cb->mac_cb->mac_type;
         ae_handle->dport_id = port_id;
@@ -247,12 +247,21 @@ static void hns_ae_set_tso_stats(struct hnae_handle *handle, int enable)
 static int hns_ae_start(struct hnae_handle *handle)
 {
         int ret;
+        int k;
         struct hns_mac_cb *mac_cb = hns_get_mac_cb(handle);
 
         ret = hns_mac_vm_config_bc_en(mac_cb, 0, true);
         if (ret)
                 return ret;
 
+        for (k = 0; k < handle->q_num; k++) {
+                if (AE_IS_VER1(mac_cb->dsaf_dev->dsaf_ver))
+                        hns_rcb_int_clr_hw(handle->qs[k],
+                                           RCB_INT_FLAG_TX | RCB_INT_FLAG_RX);
+                else
+                        hns_rcbv2_int_clr_hw(handle->qs[k],
+                                             RCB_INT_FLAG_TX | RCB_INT_FLAG_RX);
+        }
         hns_ae_ring_enable_all(handle, 1);
         msleep(100);
 
@@ -313,18 +322,6 @@ static void hns_aev2_toggle_ring_irq(struct hnae_ring *ring, u32 mask)
         hns_rcbv2_int_ctrl_hw(ring->q, flag, mask);
 }
 
-static void hns_ae_toggle_queue_status(struct hnae_queue *queue, u32 val)
-{
-        struct dsaf_device *dsaf_dev = hns_ae_get_dsaf_dev(queue->dev);
-
-        if (AE_IS_VER1(dsaf_dev->dsaf_ver))
-                hns_rcb_int_clr_hw(queue, RCB_INT_FLAG_TX | RCB_INT_FLAG_RX);
-        else
-                hns_rcbv2_int_clr_hw(queue, RCB_INT_FLAG_TX | RCB_INT_FLAG_RX);
-
-        hns_rcb_start(queue, val);
-}
-
 static int hns_ae_get_link_status(struct hnae_handle *handle)
 {
         u32 link_status;
@@ -465,6 +462,30 @@ static int  hns_ae_set_coalesce_frames(struct hnae_handle *handle,
                 ring_pair->port_id_in_comm, coalesce_frames);
 }
 
+static void hns_ae_get_coalesce_range(struct hnae_handle *handle,
+                                      u32 *tx_frames_low, u32 *rx_frames_low,
+                                      u32 *tx_frames_high, u32 *rx_frames_high,
+                                      u32 *tx_usecs_low, u32 *rx_usecs_low,
+                                      u32 *tx_usecs_high, u32 *rx_usecs_high)
+{
+        struct dsaf_device *dsaf_dev;
+
+        dsaf_dev = hns_ae_get_dsaf_dev(handle->dev);
+
+        *tx_frames_low  = HNS_RCB_MIN_COALESCED_FRAMES;
+        *rx_frames_low  = HNS_RCB_MIN_COALESCED_FRAMES;
+        *tx_frames_high =
+                (dsaf_dev->desc_num - 1 > HNS_RCB_MAX_COALESCED_FRAMES) ?
+                HNS_RCB_MAX_COALESCED_FRAMES : dsaf_dev->desc_num - 1;
+        *rx_frames_high =
+                (dsaf_dev->desc_num - 1 > HNS_RCB_MAX_COALESCED_FRAMES) ?
+                 HNS_RCB_MAX_COALESCED_FRAMES : dsaf_dev->desc_num - 1;
+        *tx_usecs_low   = 0;
+        *rx_usecs_low   = 0;
+        *tx_usecs_high  = HNS_RCB_MAX_COALESCED_USECS;
+        *rx_usecs_high  = HNS_RCB_MAX_COALESCED_USECS;
+}
+
 void hns_ae_update_stats(struct hnae_handle *handle,
                          struct net_device_stats *net_stats)
 {
@@ -587,6 +608,7 @@ void hns_ae_get_strings(struct hnae_handle *handle,
         int idx;
         struct hns_mac_cb *mac_cb;
         struct hns_ppe_cb *ppe_cb;
+        struct dsaf_device *dsaf_dev = hns_ae_get_dsaf_dev(handle->dev);
         u8 *p = data;
         struct  hnae_vf_cb *vf_cb;
 
@@ -609,13 +631,14 @@ void hns_ae_get_strings(struct hnae_handle *handle,
         p += ETH_GSTRING_LEN * hns_mac_get_sset_count(mac_cb, stringset);
 
         if (mac_cb->mac_type == HNAE_PORT_SERVICE)
-                hns_dsaf_get_strings(stringset, p, port);
+                hns_dsaf_get_strings(stringset, p, port, dsaf_dev);
 }
 
 int hns_ae_get_sset_count(struct hnae_handle *handle, int stringset)
 {
         u32 sset_count = 0;
         struct hns_mac_cb *mac_cb;
+        struct dsaf_device *dsaf_dev = hns_ae_get_dsaf_dev(handle->dev);
 
         assert(handle);
 
@@ -626,7 +649,7 @@ int hns_ae_get_sset_count(struct hnae_handle *handle, int stringset)
         sset_count += hns_mac_get_sset_count(mac_cb, stringset);
 
         if (mac_cb->mac_type == HNAE_PORT_SERVICE)
-                sset_count += hns_dsaf_get_sset_count(stringset);
+                sset_count += hns_dsaf_get_sset_count(dsaf_dev, stringset);
 
         return sset_count;
 }
@@ -637,13 +660,15 @@ static int hns_ae_config_loopback(struct hnae_handle *handle,
         int ret;
         struct hnae_vf_cb *vf_cb = hns_ae_get_vf_cb(handle);
         struct hns_mac_cb *mac_cb = hns_get_mac_cb(handle);
+        struct dsaf_device *dsaf_dev = mac_cb->dsaf_dev;
 
         switch (loop) {
         case MAC_INTERNALLOOP_PHY:
                 ret = 0;
                 break;
         case MAC_INTERNALLOOP_SERDES:
-                ret = hns_mac_config_sds_loopback(vf_cb->mac_cb, en);
+                ret = dsaf_dev->misc_op->cfg_serdes_loopback(vf_cb->mac_cb,
+                                                             !!en);
                 break;
         case MAC_INTERNALLOOP_MAC:
                 ret = hns_mac_config_mac_loopback(vf_cb->mac_cb, loop, en);
@@ -780,7 +805,6 @@ static struct hnae_ae_ops hns_dsaf_ops = {
         .stop = hns_ae_stop,
         .reset = hns_ae_reset,
         .toggle_ring_irq = hns_ae_toggle_ring_irq,
-        .toggle_queue_status = hns_ae_toggle_queue_status,
         .get_status = hns_ae_get_link_status,
         .get_info = hns_ae_get_mac_info,
         .adjust_link = hns_ae_adjust_link,
@@ -794,6 +818,7 @@ static struct hnae_ae_ops hns_dsaf_ops = {
         .get_rx_max_coalesced_frames = hns_ae_get_rx_max_coalesced_frames,
         .set_coalesce_usecs = hns_ae_set_coalesce_usecs,
         .set_coalesce_frames = hns_ae_set_coalesce_frames,
+        .get_coalesce_range = hns_ae_get_coalesce_range,
         .set_promisc_mode = hns_ae_set_promisc_mode,
         .set_mac_addr = hns_ae_set_mac_address,
         .set_mc_addr = hns_ae_set_multicast_one,
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_gmac.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_gmac.c
index 44abb08de155..1235c7f2564b 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_gmac.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_gmac.c
@@ -110,7 +110,7 @@ static void hns_gmac_free(void *mac_drv)
 
         u32 mac_id = drv->mac_id;
 
-        hns_dsaf_ge_srst_by_port(dsaf_dev, mac_id, 0);
+        dsaf_dev->misc_op->ge_srst(dsaf_dev, mac_id, 0);
 }
 
 static void hns_gmac_set_tx_auto_pause_frames(void *mac_drv, u16 newval)
@@ -317,9 +317,9 @@ static void hns_gmac_init(void *mac_drv)
 
         port = drv->mac_id;
 
-        hns_dsaf_ge_srst_by_port(dsaf_dev, port, 0);
+        dsaf_dev->misc_op->ge_srst(dsaf_dev, port, 0);
         mdelay(10);
-        hns_dsaf_ge_srst_by_port(dsaf_dev, port, 1);
+        dsaf_dev->misc_op->ge_srst(dsaf_dev, port, 1);
         mdelay(10);
         hns_gmac_disable(mac_drv, MAC_COMM_MODE_RX_AND_TX);
         hns_gmac_tx_loop_pkt_dis(mac_drv);
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c
index 611581fccf2a..c526558e6367 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c
@@ -7,6 +7,7 @@
  * (at your option) any later version.
  */
 
+#include <linux/acpi.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
@@ -15,7 +16,8 @@
 #include <linux/netdevice.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
-#include <linux/phy_fixed.h>
+#include <linux/of_mdio.h>
+#include <linux/phy.h>
 #include <linux/platform_device.h>
 
 #include "hns_dsaf_main.h"
@@ -94,7 +96,7 @@ void hns_mac_get_link_status(struct hns_mac_cb *mac_cb, u32 *link_status)
         else
                 *link_status = 0;
 
-        ret = hns_mac_get_sfp_prsnt(mac_cb, &sfp_prsnt);
+        ret = mac_cb->dsaf_dev->misc_op->get_sfp_prsnt(mac_cb, &sfp_prsnt);
         if (!ret)
                 *link_status = *link_status && sfp_prsnt;
 
@@ -511,7 +513,7 @@ void hns_mac_stop(struct hns_mac_cb *mac_cb)
 
         mac_ctrl_drv->mac_en_flg = 0;
         mac_cb->link = 0;
-        cpld_led_reset(mac_cb);
+        mac_cb->dsaf_dev->misc_op->cpld_reset_led(mac_cb);
 }
 
 /**
@@ -637,6 +639,115 @@ free_mac_drv:
         return ret;
 }
 
+static int
+hns_mac_phy_parse_addr(struct device *dev, struct fwnode_handle *fwnode)
+{
+        u32 addr;
+        int ret;
+
+        ret = fwnode_property_read_u32(fwnode, "phy-addr", &addr);
+        if (ret) {
+                dev_err(dev, "has invalid PHY address ret:%d\n", ret);
+                return ret;
+        }
+
+        if (addr >= PHY_MAX_ADDR) {
+                dev_err(dev, "PHY address %i is too large\n", addr);
+                return -EINVAL;
+        }
+
+        return addr;
+}
+
+static int hns_mac_phydev_match(struct device *dev, void *fwnode)
+{
+        return dev->fwnode == fwnode;
+}
+
+static struct
+platform_device *hns_mac_find_platform_device(struct fwnode_handle *fwnode)
+{
+        struct device *dev;
+
+        dev = bus_find_device(&platform_bus_type, NULL,
+                              fwnode, hns_mac_phydev_match);
+        return dev ? to_platform_device(dev) : NULL;
+}
+
+static int
+hns_mac_register_phydev(struct mii_bus *mdio, struct hns_mac_cb *mac_cb,
+                        u32 addr)
+{
+        struct phy_device *phy;
+        const char *phy_type;
+        bool is_c45;
+        int rc;
+
+        rc = fwnode_property_read_string(mac_cb->fw_port,
+                                         "phy-mode", &phy_type);
+        if (rc < 0)
+                return rc;
+
+        if (!strcmp(phy_type, phy_modes(PHY_INTERFACE_MODE_XGMII)))
+                is_c45 = 1;
+        else if (!strcmp(phy_type, phy_modes(PHY_INTERFACE_MODE_SGMII)))
+                is_c45 = 0;
+        else
+                return -ENODATA;
+
+        phy = get_phy_device(mdio, addr, is_c45);
+        if (!phy || IS_ERR(phy))
+                return -EIO;
+
+        if (mdio->irq)
+                phy->irq = mdio->irq[addr];
+
+        /* All data is now stored in the phy struct;
+         * register it
+         */
+        rc = phy_device_register(phy);
+        if (rc) {
+                phy_device_free(phy);
+                return -ENODEV;
+        }
+
+        mac_cb->phy_dev = phy;
+
+        dev_dbg(&mdio->dev, "registered phy at address %i\n", addr);
+
+        return 0;
+}
+
+static void hns_mac_register_phy(struct hns_mac_cb *mac_cb)
+{
+        struct acpi_reference_args args;
+        struct platform_device *pdev;
+        struct mii_bus *mii_bus;
+        int rc;
+        int addr;
+
+        /* Loop over the child nodes and register a phy_device for each one */
+        if (!to_acpi_device_node(mac_cb->fw_port))
+                return;
+
+        rc = acpi_node_get_property_reference(
+                        mac_cb->fw_port, "mdio-node", 0, &args);
+        if (rc)
+                return;
+
+        addr = hns_mac_phy_parse_addr(mac_cb->dev, mac_cb->fw_port);
+        if (addr < 0)
+                return;
+
+        /* dev address in adev */
+        pdev = hns_mac_find_platform_device(acpi_fwnode_handle(args.adev));
+        mii_bus = platform_get_drvdata(pdev);
+        rc = hns_mac_register_phydev(mii_bus, mac_cb, addr);
+        if (!rc)
+                dev_dbg(mac_cb->dev, "mac%d register phy addr:%d\n",
+                        mac_cb->mac_id, addr);
+}
+
 /**
  *hns_mac_get_info  - get mac information from device node
  *@mac_cb: mac device
@@ -645,7 +756,7 @@ free_mac_drv:
  */
 static int  hns_mac_get_info(struct hns_mac_cb *mac_cb)
 {
-        struct device_node *np = mac_cb->dev->of_node;
+        struct device_node *np;
         struct regmap *syscon;
         struct of_phandle_args cpld_args;
         u32 ret;
@@ -672,63 +783,85 @@ static int  hns_mac_get_info(struct hns_mac_cb *mac_cb)
          * from dsaf node
          */
         if (!mac_cb->fw_port) {
-                mac_cb->phy_node = of_parse_phandle(np, "phy-handle",
-                                                    mac_cb->mac_id);
-                if (mac_cb->phy_node)
+                np = of_parse_phandle(mac_cb->dev->of_node, "phy-handle",
+                                      mac_cb->mac_id);
+                mac_cb->phy_dev = of_phy_find_device(np);
+                if (mac_cb->phy_dev) {
+                        /* refcount is held by of_phy_find_device()
+                         * if the phy_dev is found
+                         */
+                        put_device(&mac_cb->phy_dev->mdio.dev);
+
                         dev_dbg(mac_cb->dev, "mac%d phy_node: %s\n",
-                                mac_cb->mac_id, mac_cb->phy_node->name);
-                return 0;
-        }
-        if (!is_of_node(mac_cb->fw_port))
-                return -EINVAL;
-        /* parse property from port subnode in dsaf */
-        mac_cb->phy_node = of_parse_phandle(to_of_node(mac_cb->fw_port),
-                                            "phy-handle", 0);
-        if (mac_cb->phy_node)
-                dev_dbg(mac_cb->dev, "mac%d phy_node: %s\n",
-                        mac_cb->mac_id, mac_cb->phy_node->name);
-        syscon = syscon_node_to_regmap(
-                        of_parse_phandle(to_of_node(mac_cb->fw_port),
-                                         "serdes-syscon", 0));
-        if (IS_ERR_OR_NULL(syscon)) {
-                dev_err(mac_cb->dev, "serdes-syscon is needed!\n");
-                return -EINVAL;
-        }
-        mac_cb->serdes_ctrl = syscon;
+                                mac_cb->mac_id, np->name);
+                }
 
-        ret = fwnode_property_read_u32(mac_cb->fw_port,
-                                       "port-rst-offset",
-                                       &mac_cb->port_rst_off);
-        if (ret) {
-                dev_dbg(mac_cb->dev,
-                        "mac%d port-rst-offset not found, use default value.\n",
-                        mac_cb->mac_id);
+                return 0;
         }
 
-        ret = fwnode_property_read_u32(mac_cb->fw_port,
-                                       "port-mode-offset",
-                                       &mac_cb->port_mode_off);
-        if (ret) {
-                dev_dbg(mac_cb->dev,
-                        "mac%d port-mode-offset not found, use default value.\n",
-                        mac_cb->mac_id);
-        }
+        if (is_of_node(mac_cb->fw_port)) {
+                /* parse property from port subnode in dsaf */
+                np = of_parse_phandle(to_of_node(mac_cb->fw_port),
+                                      "phy-handle", 0);
+                mac_cb->phy_dev = of_phy_find_device(np);
+                if (mac_cb->phy_dev) {
+                        /* refcount is held by of_phy_find_device()
+                         * if the phy_dev is found
+                         */
+                        put_device(&mac_cb->phy_dev->mdio.dev);
+                        dev_dbg(mac_cb->dev, "mac%d phy_node: %s\n",
+                                mac_cb->mac_id, np->name);
+                }
 
-        ret = of_parse_phandle_with_fixed_args(to_of_node(mac_cb->fw_port),
-                                               "cpld-syscon", 1, 0, &cpld_args);
-        if (ret) {
-                dev_dbg(mac_cb->dev, "mac%d no cpld-syscon found.\n",
-                        mac_cb->mac_id);
-                mac_cb->cpld_ctrl = NULL;
-        } else {
-                syscon = syscon_node_to_regmap(cpld_args.np);
+                syscon = syscon_node_to_regmap(
+                                of_parse_phandle(to_of_node(mac_cb->fw_port),
+                                                 "serdes-syscon", 0));
                 if (IS_ERR_OR_NULL(syscon)) {
-                        dev_dbg(mac_cb->dev, "no cpld-syscon found!\n");
+                        dev_err(mac_cb->dev, "serdes-syscon is needed!\n");
+                        return -EINVAL;
+                }
+                mac_cb->serdes_ctrl = syscon;
+
+                ret = fwnode_property_read_u32(mac_cb->fw_port,
+                                               "port-rst-offset",
+                                               &mac_cb->port_rst_off);
+                if (ret) {
+                        dev_dbg(mac_cb->dev,
+                                "mac%d port-rst-offset not found, use default value.\n",
+                                mac_cb->mac_id);
+                }
+
+                ret = fwnode_property_read_u32(mac_cb->fw_port,
+                                               "port-mode-offset",
+                                               &mac_cb->port_mode_off);
+                if (ret) {
+                        dev_dbg(mac_cb->dev,
+                                "mac%d port-mode-offset not found, use default value.\n",
+                                mac_cb->mac_id);
+                }
+
+                ret = of_parse_phandle_with_fixed_args(
+                        to_of_node(mac_cb->fw_port), "cpld-syscon", 1, 0,
+                        &cpld_args);
+                if (ret) {
+                        dev_dbg(mac_cb->dev, "mac%d no cpld-syscon found.\n",
+                                mac_cb->mac_id);
                         mac_cb->cpld_ctrl = NULL;
                 } else {
-                        mac_cb->cpld_ctrl = syscon;
-                        mac_cb->cpld_ctrl_reg = cpld_args.args[0];
+                        syscon = syscon_node_to_regmap(cpld_args.np);
+                        if (IS_ERR_OR_NULL(syscon)) {
+                                dev_dbg(mac_cb->dev, "no cpld-syscon found!\n");
+                                mac_cb->cpld_ctrl = NULL;
+                        } else {
+                                mac_cb->cpld_ctrl = syscon;
+                                mac_cb->cpld_ctrl_reg = cpld_args.args[0];
+                        }
                 }
+        } else if (is_acpi_node(mac_cb->fw_port)) {
+                hns_mac_register_phy(mac_cb);
+        } else {
+                dev_err(mac_cb->dev, "mac%d cannot find phy node\n",
+                        mac_cb->mac_id);
         }
 
         return 0;
@@ -790,7 +923,7 @@ int hns_mac_get_cfg(struct dsaf_device *dsaf_dev, struct hns_mac_cb *mac_cb)
         else
                 mac_cb->mac_type = HNAE_PORT_DEBUG;
 
-        mac_cb->phy_if = hns_mac_get_phy_if(mac_cb);
+        mac_cb->phy_if = dsaf_dev->misc_op->get_phy_if(mac_cb);
 
         ret = hns_mac_get_mode(mac_cb->phy_if);
         if (ret < 0) {
@@ -805,7 +938,7 @@ int hns_mac_get_cfg(struct dsaf_device *dsaf_dev, struct hns_mac_cb *mac_cb)
         if (ret)
                 return ret;
 
-        cpld_led_reset(mac_cb);
+        mac_cb->dsaf_dev->misc_op->cpld_reset_led(mac_cb);
         mac_cb->vaddr = hns_mac_get_vaddr(dsaf_dev, mac_cb, mac_mode_idx);
 
         return 0;
@@ -892,7 +1025,7 @@ void hns_mac_uninit(struct dsaf_device *dsaf_dev)
         int max_port_num = hns_mac_get_max_port_num(dsaf_dev);
 
         for (i = 0; i < max_port_num; i++) {
-                cpld_led_reset(dsaf_dev->mac_cb[i]);
+                dsaf_dev->misc_op->cpld_reset_led(dsaf_dev->mac_cb[i]);
                 dsaf_dev->mac_cb[i] = NULL;
         }
 }
@@ -975,7 +1108,7 @@ void hns_set_led_opt(struct hns_mac_cb *mac_cb)
                 nic_data = 0;
         mac_cb->txpkt_for_led = mac_cb->hw_stats.tx_good_pkts;
         mac_cb->rxpkt_for_led = mac_cb->hw_stats.rx_good_pkts;
-        hns_cpld_set_led(mac_cb, (int)mac_cb->link,
+        mac_cb->dsaf_dev->misc_op->cpld_set_led(mac_cb, (int)mac_cb->link,
                          mac_cb->speed, nic_data);
 }
 
@@ -985,5 +1118,5 @@ int hns_cpld_led_set_id(struct hns_mac_cb *mac_cb,
         if (!mac_cb || !mac_cb->cpld_ctrl)
                 return 0;
 
-        return cpld_set_led_id(mac_cb, status);
+        return mac_cb->dsaf_dev->misc_op->cpld_set_led_id(mac_cb, status);
 }
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.h b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.h
index 97ce9a750aaf..05a6e8f7a419 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.h
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.h
@@ -338,7 +338,7 @@ struct hns_mac_cb {
         phy_interface_t phy_if;
         enum hnae_loop loop_mode;
 
-        struct device_node *phy_node;
+        struct phy_device *phy_dev;
 
         struct mac_hw_stats hw_stats;
 };
@@ -448,8 +448,6 @@ int hns_mac_set_pauseparam(struct hns_mac_cb *mac_cb, u32 rx_en, u32 tx_en);
 int hns_mac_set_mtu(struct hns_mac_cb *mac_cb, u32 new_mtu);
 int hns_mac_get_port_info(struct hns_mac_cb *mac_cb,
                           u8 *auto_neg, u16 *speed, u8 *duplex);
-phy_interface_t hns_mac_get_phy_if(struct hns_mac_cb *mac_cb);
-int hns_mac_config_sds_loopback(struct hns_mac_cb *mac_cb, u8 en);
 int hns_mac_config_mac_loopback(struct hns_mac_cb *mac_cb,
                                 enum hnae_loop loop, int en);
 void hns_mac_update_stats(struct hns_mac_cb *mac_cb);
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.c
index 1c2ddb25e776..67e8e1323205 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.c
@@ -7,6 +7,7 @@
  * (at your option) any later version.
  */
 
+#include <linux/acpi.h>
 #include <linux/device.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
@@ -24,6 +25,7 @@
 #include "hns_dsaf_main.h"
 #include "hns_dsaf_ppe.h"
 #include "hns_dsaf_rcb.h"
+#include "hns_dsaf_misc.h"
 
 const char *g_dsaf_mode_match[DSAF_MODE_MAX] = {
         [DSAF_MODE_DISABLE_2PORT_64VM] = "2port-64vf",
@@ -32,6 +34,13 @@ const char *g_dsaf_mode_match[DSAF_MODE_MAX] = {
         [DSAF_MODE_DISABLE_SP] = "single-port",
 };
 
+static const struct acpi_device_id hns_dsaf_acpi_match[] = {
+        { "HISI00B1", 0 },
+        { "HISI00B2", 0 },
+        { },
+};
+MODULE_DEVICE_TABLE(acpi, hns_dsaf_acpi_match);
+
 int hns_dsaf_get_cfg(struct dsaf_device *dsaf_dev)
 {
         int ret, i;
@@ -45,12 +54,24 @@ int hns_dsaf_get_cfg(struct dsaf_device *dsaf_dev)
         struct device_node *np = dsaf_dev->dev->of_node;
         struct platform_device *pdev = to_platform_device(dsaf_dev->dev);
 
-        if (of_device_is_compatible(np, "hisilicon,hns-dsaf-v1"))
-                dsaf_dev->dsaf_ver = AE_VERSION_1;
-        else
-                dsaf_dev->dsaf_ver = AE_VERSION_2;
+        if (dev_of_node(dsaf_dev->dev)) {
+                if (of_device_is_compatible(np, "hisilicon,hns-dsaf-v1"))
+                        dsaf_dev->dsaf_ver = AE_VERSION_1;
+                else
+                        dsaf_dev->dsaf_ver = AE_VERSION_2;
+        } else if (is_acpi_node(dsaf_dev->dev->fwnode)) {
+                if (acpi_dev_found(hns_dsaf_acpi_match[0].id))
+                        dsaf_dev->dsaf_ver = AE_VERSION_1;
+                else if (acpi_dev_found(hns_dsaf_acpi_match[1].id))
+                        dsaf_dev->dsaf_ver = AE_VERSION_2;
+                else
+                        return -ENXIO;
+        } else {
+                dev_err(dsaf_dev->dev, "cannot get cfg data from of or acpi\n");
+                return -ENXIO;
+        }
 
-        ret = of_property_read_string(np, "mode", &mode_str);
+        ret = device_property_read_string(dsaf_dev->dev, "mode", &mode_str);
         if (ret) {
                 dev_err(dsaf_dev->dev, "get dsaf mode fail, ret=%d!\n", ret);
                 return ret;
@@ -80,32 +101,40 @@ int hns_dsaf_get_cfg(struct dsaf_device *dsaf_dev)
         else
                 dsaf_dev->dsaf_tc_mode = HRD_DSAF_4TC_MODE;
 
-        syscon = syscon_node_to_regmap(
-                        of_parse_phandle(np, "subctrl-syscon", 0));
-        if (IS_ERR_OR_NULL(syscon)) {
-                res = platform_get_resource(pdev, IORESOURCE_MEM, res_idx++);
-                if (!res) {
-                        dev_err(dsaf_dev->dev, "subctrl info is needed!\n");
-                        return -ENOMEM;
-                }
-                dsaf_dev->sc_base = devm_ioremap_resource(&pdev->dev, res);
-                if (!dsaf_dev->sc_base) {
-                        dev_err(dsaf_dev->dev, "subctrl can not map!\n");
-                        return -ENOMEM;
-                }
+        if (dev_of_node(dsaf_dev->dev)) {
+                syscon = syscon_node_to_regmap(
+                                of_parse_phandle(np, "subctrl-syscon", 0));
+                if (IS_ERR_OR_NULL(syscon)) {
+                        res = platform_get_resource(pdev, IORESOURCE_MEM,
+                                                    res_idx++);
+                        if (!res) {
+                                dev_err(dsaf_dev->dev, "subctrl info is needed!\n");
+                                return -ENOMEM;
+                        }
 
-                res = platform_get_resource(pdev, IORESOURCE_MEM, res_idx++);
-                if (!res) {
-                        dev_err(dsaf_dev->dev, "serdes-ctrl info is needed!\n");
-                        return -ENOMEM;
-                }
-                dsaf_dev->sds_base = devm_ioremap_resource(&pdev->dev, res);
-                if (!dsaf_dev->sds_base) {
-                        dev_err(dsaf_dev->dev, "serdes-ctrl can not map!\n");
-                        return -ENOMEM;
+                        dsaf_dev->sc_base = devm_ioremap_resource(&pdev->dev,
+                                                                  res);
+                        if (!dsaf_dev->sc_base) {
+                                dev_err(dsaf_dev->dev, "subctrl can not map!\n");
+                                return -ENOMEM;
+                        }
+
+                        res = platform_get_resource(pdev, IORESOURCE_MEM,
+                                                    res_idx++);
+                        if (!res) {
+                                dev_err(dsaf_dev->dev, "serdes-ctrl info is needed!\n");
+                                return -ENOMEM;
+                        }
+
+                        dsaf_dev->sds_base = devm_ioremap_resource(&pdev->dev,
+                                                                   res);
+                        if (!dsaf_dev->sds_base) {
+                                dev_err(dsaf_dev->dev, "serdes-ctrl can not map!\n");
+                                return -ENOMEM;
+                        }
+                } else {
+                        dsaf_dev->sub_ctrl = syscon;
                 }
-        } else {
-                dsaf_dev->sub_ctrl = syscon;
         }
 
         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ppe-base");
@@ -142,27 +171,28 @@ int hns_dsaf_get_cfg(struct dsaf_device *dsaf_dev)
                 }
         }
 
-        ret = of_property_read_u32(np, "desc-num", &desc_num);
+        ret = device_property_read_u32(dsaf_dev->dev, "desc-num", &desc_num);
         if (ret < 0 || desc_num < HNS_DSAF_MIN_DESC_CNT ||
             desc_num > HNS_DSAF_MAX_DESC_CNT) {
                 dev_err(dsaf_dev->dev, "get desc-num(%d) fail, ret=%d!\n",
                         desc_num, ret);
-                goto unmap_base_addr;
+                return -EINVAL;
         }
         dsaf_dev->desc_num = desc_num;
 
-        ret = of_property_read_u32(np, "reset-field-offset", &reset_offset);
+        ret = device_property_read_u32(dsaf_dev->dev, "reset-field-offset",
+                                       &reset_offset);
         if (ret < 0) {
                 dev_dbg(dsaf_dev->dev,
                         "get reset-field-offset fail, ret=%d!\r\n", ret);
         }
         dsaf_dev->reset_offset = reset_offset;
 
-        ret = of_property_read_u32(np, "buf-size", &buf_size);
+        ret = device_property_read_u32(dsaf_dev->dev, "buf-size", &buf_size);
         if (ret < 0) {
                 dev_err(dsaf_dev->dev,
                         "get buf-size fail, ret=%d!\r\n", ret);
-                goto unmap_base_addr;
+                return ret;
         }
         dsaf_dev->buf_size = buf_size;
 
@@ -170,41 +200,19 @@ int hns_dsaf_get_cfg(struct dsaf_device *dsaf_dev)
         if (dsaf_dev->buf_size_type < 0) {
                 dev_err(dsaf_dev->dev,
                         "buf_size(%d) is wrong!\n", buf_size);
-                goto unmap_base_addr;
+                return -EINVAL;
         }
 
+        dsaf_dev->misc_op = hns_misc_op_get(dsaf_dev);
+        if (!dsaf_dev->misc_op)
+                return -ENOMEM;
+
         if (!dma_set_mask_and_coherent(dsaf_dev->dev, DMA_BIT_MASK(64ULL)))
                 dev_dbg(dsaf_dev->dev, "set mask to 64bit\n");
         else
                 dev_err(dsaf_dev->dev, "set mask to 64bit fail!\n");
 
         return 0;
-
-unmap_base_addr:
-        if (dsaf_dev->io_base)
-                iounmap(dsaf_dev->io_base);
-        if (dsaf_dev->ppe_base)
-                iounmap(dsaf_dev->ppe_base);
-        if (dsaf_dev->sds_base)
-                iounmap(dsaf_dev->sds_base);
-        if (dsaf_dev->sc_base)
-                iounmap(dsaf_dev->sc_base);
-        return ret;
-}
-
-static void hns_dsaf_free_cfg(struct dsaf_device *dsaf_dev)
-{
-        if (dsaf_dev->io_base)
-                iounmap(dsaf_dev->io_base);
-
-        if (dsaf_dev->ppe_base)
-                iounmap(dsaf_dev->ppe_base);
-
-        if (dsaf_dev->sds_base)
-                iounmap(dsaf_dev->sds_base);
-
-        if (dsaf_dev->sc_base)
-                iounmap(dsaf_dev->sc_base);
 }
 
 /**
@@ -508,10 +516,10 @@ static void hns_dsafv2_sbm_bp_wl_cfg(struct dsaf_device *dsaf_dev)
                 o_sbm_bp_cfg = dsaf_read_dev(dsaf_dev, reg);
                 dsaf_set_field(o_sbm_bp_cfg,
                                DSAFV2_SBM_CFG3_SET_BUF_NUM_NO_PFC_M,
-                               DSAFV2_SBM_CFG3_SET_BUF_NUM_NO_PFC_S, 110);
+                               DSAFV2_SBM_CFG3_SET_BUF_NUM_NO_PFC_S, 48);
                 dsaf_set_field(o_sbm_bp_cfg,
                                DSAFV2_SBM_CFG3_RESET_BUF_NUM_NO_PFC_M,
-                               DSAFV2_SBM_CFG3_RESET_BUF_NUM_NO_PFC_S, 160);
+                               DSAFV2_SBM_CFG3_RESET_BUF_NUM_NO_PFC_S, 80);
                 dsaf_write_dev(dsaf_dev, reg, o_sbm_bp_cfg);
 
                 /* for no enable pfc mode */
@@ -519,29 +527,39 @@ static void hns_dsafv2_sbm_bp_wl_cfg(struct dsaf_device *dsaf_dev)
                 o_sbm_bp_cfg = dsaf_read_dev(dsaf_dev, reg);
                 dsaf_set_field(o_sbm_bp_cfg,
                                DSAFV2_SBM_CFG4_SET_BUF_NUM_NO_PFC_M,
-                               DSAFV2_SBM_CFG4_SET_BUF_NUM_NO_PFC_S, 128);
+                               DSAFV2_SBM_CFG4_SET_BUF_NUM_NO_PFC_S, 192);
                 dsaf_set_field(o_sbm_bp_cfg,
                                DSAFV2_SBM_CFG4_RESET_BUF_NUM_NO_PFC_M,
-                               DSAFV2_SBM_CFG4_RESET_BUF_NUM_NO_PFC_S, 192);
+                               DSAFV2_SBM_CFG4_RESET_BUF_NUM_NO_PFC_S, 240);
                 dsaf_write_dev(dsaf_dev, reg, o_sbm_bp_cfg);
         }
 
         /* PPE */
-        reg = DSAF_SBM_BP_CFG_2_PPE_REG_0_REG + 0x80 * i;
-        o_sbm_bp_cfg = dsaf_read_dev(dsaf_dev, reg);
-        dsaf_set_field(o_sbm_bp_cfg, DSAFV2_SBM_CFG2_SET_BUF_NUM_M,
-                       DSAFV2_SBM_CFG2_SET_BUF_NUM_S, 10);
-        dsaf_set_field(o_sbm_bp_cfg, DSAFV2_SBM_CFG2_RESET_BUF_NUM_M,
-                       DSAFV2_SBM_CFG2_RESET_BUF_NUM_S, 12);
-        dsaf_write_dev(dsaf_dev, reg, o_sbm_bp_cfg);
+        for (i = 0; i < DSAFV2_SBM_PPE_CHN; i++) {
+                reg = DSAF_SBM_BP_CFG_2_PPE_REG_0_REG + 0x80 * i;
+                o_sbm_bp_cfg = dsaf_read_dev(dsaf_dev, reg);
+                dsaf_set_field(o_sbm_bp_cfg,
+                               DSAFV2_SBM_CFG2_PPE_SET_BUF_NUM_M,
+                               DSAFV2_SBM_CFG2_PPE_SET_BUF_NUM_S, 2);
+                dsaf_set_field(o_sbm_bp_cfg,
+                               DSAFV2_SBM_CFG2_PPE_RESET_BUF_NUM_M,
+                               DSAFV2_SBM_CFG2_PPE_RESET_BUF_NUM_S, 3);
+                dsaf_set_field(o_sbm_bp_cfg,
+                               DSAFV2_SBM_CFG2_PPE_CFG_USEFUL_NUM_M,
+                               DSAFV2_SBM_CFG2_PPE_CFG_USEFUL_NUM_S, 52);
+                dsaf_write_dev(dsaf_dev, reg, o_sbm_bp_cfg);
+        }
+
         /* RoCEE */
         for (i = 0; i < DASFV2_ROCEE_CRD_NUM; i++) {
                 reg = DSAFV2_SBM_BP_CFG_2_ROCEE_REG_0_REG + 0x80 * i;
                 o_sbm_bp_cfg = dsaf_read_dev(dsaf_dev, reg);
-                dsaf_set_field(o_sbm_bp_cfg, DSAFV2_SBM_CFG2_SET_BUF_NUM_M,
-                               DSAFV2_SBM_CFG2_SET_BUF_NUM_S, 2);
-                dsaf_set_field(o_sbm_bp_cfg, DSAFV2_SBM_CFG2_RESET_BUF_NUM_M,
-                               DSAFV2_SBM_CFG2_RESET_BUF_NUM_S, 4);
+                dsaf_set_field(o_sbm_bp_cfg,
+                               DSAFV2_SBM_CFG2_ROCEE_SET_BUF_NUM_M,
+                               DSAFV2_SBM_CFG2_ROCEE_SET_BUF_NUM_S, 2);
+                dsaf_set_field(o_sbm_bp_cfg,
+                               DSAFV2_SBM_CFG2_ROCEE_RESET_BUF_NUM_M,
+                               DSAFV2_SBM_CFG2_ROCEE_RESET_BUF_NUM_S, 4);
                 dsaf_write_dev(dsaf_dev, reg, o_sbm_bp_cfg);
         }
 }
@@ -852,6 +870,8 @@ static void hns_dsaf_single_line_tbl_cfg(
         struct dsaf_device *dsaf_dev,
         u32 address, struct dsaf_tbl_line_cfg *ptbl_line)
 {
+        spin_lock_bh(&dsaf_dev->tcam_lock);
+
         /*Write Addr*/
         hns_dsaf_tbl_line_addr_cfg(dsaf_dev, address);
 
@@ -860,6 +880,8 @@ static void hns_dsaf_single_line_tbl_cfg(
 
         /*Write Plus*/
         hns_dsaf_tbl_line_pul(dsaf_dev);
+
+        spin_unlock_bh(&dsaf_dev->tcam_lock);
 }
 
 /**
@@ -873,6 +895,8 @@ static void hns_dsaf_tcam_uc_cfg(
         struct dsaf_tbl_tcam_data *ptbl_tcam_data,
         struct dsaf_tbl_tcam_ucast_cfg *ptbl_tcam_ucast)
 {
+        spin_lock_bh(&dsaf_dev->tcam_lock);
+
         /*Write Addr*/
         hns_dsaf_tbl_tcam_addr_cfg(dsaf_dev, address);
         /*Write Tcam Data*/
@@ -881,6 +905,8 @@ static void hns_dsaf_tcam_uc_cfg(
         hns_dsaf_tbl_tcam_ucast_cfg(dsaf_dev, ptbl_tcam_ucast);
         /*Write Plus*/
         hns_dsaf_tbl_tcam_data_ucast_pul(dsaf_dev);
+
+        spin_unlock_bh(&dsaf_dev->tcam_lock);
 }
 
 /**
@@ -895,6 +921,8 @@ static void hns_dsaf_tcam_mc_cfg(
         struct dsaf_tbl_tcam_data *ptbl_tcam_data,
         struct dsaf_tbl_tcam_mcast_cfg *ptbl_tcam_mcast)
 {
+        spin_lock_bh(&dsaf_dev->tcam_lock);
+
         /*Write Addr*/
         hns_dsaf_tbl_tcam_addr_cfg(dsaf_dev, address);
         /*Write Tcam Data*/
@@ -903,6 +931,8 @@ static void hns_dsaf_tcam_mc_cfg(
         hns_dsaf_tbl_tcam_mcast_cfg(dsaf_dev, ptbl_tcam_mcast);
         /*Write Plus*/
         hns_dsaf_tbl_tcam_data_mcast_pul(dsaf_dev);
+
+        spin_unlock_bh(&dsaf_dev->tcam_lock);
 }
 
 /**
@@ -912,6 +942,8 @@ static void hns_dsaf_tcam_mc_cfg(
  */
 static void hns_dsaf_tcam_mc_invld(struct dsaf_device *dsaf_dev, u32 address)
 {
+        spin_lock_bh(&dsaf_dev->tcam_lock);
+
         /*Write Addr*/
         hns_dsaf_tbl_tcam_addr_cfg(dsaf_dev, address);
 
@@ -924,6 +956,8 @@ static void hns_dsaf_tcam_mc_invld(struct dsaf_device *dsaf_dev, u32 address)
 
         /*Write Plus*/
         hns_dsaf_tbl_tcam_mcast_pul(dsaf_dev);
+
+        spin_unlock_bh(&dsaf_dev->tcam_lock);
 }
 
 /**
@@ -941,6 +975,8 @@ static void hns_dsaf_tcam_uc_get(
         u32 tcam_read_data0;
         u32 tcam_read_data4;
 
+        spin_lock_bh(&dsaf_dev->tcam_lock);
+
         /*Write Addr*/
         hns_dsaf_tbl_tcam_addr_cfg(dsaf_dev, address);
 
@@ -949,9 +985,9 @@ static void hns_dsaf_tcam_uc_get(
 
         /*read tcam data*/
         ptbl_tcam_data->tbl_tcam_data_high
-                = dsaf_read_dev(dsaf_dev, DSAF_TBL_TCAM_RDATA_LOW_0_REG);
-        ptbl_tcam_data->tbl_tcam_data_low
                 = dsaf_read_dev(dsaf_dev, DSAF_TBL_TCAM_RDATA_HIGH_0_REG);
+        ptbl_tcam_data->tbl_tcam_data_low
+                = dsaf_read_dev(dsaf_dev, DSAF_TBL_TCAM_RDATA_LOW_0_REG);
 
         /*read tcam mcast*/
         tcam_read_data0 = dsaf_read_dev(dsaf_dev,
@@ -973,6 +1009,8 @@ static void hns_dsaf_tcam_uc_get(
                                  DSAF_TBL_UCAST_CFG1_OUT_PORT_S);
         ptbl_tcam_ucast->tbl_ucast_dvc
                 = dsaf_get_bit(tcam_read_data0, DSAF_TBL_UCAST_CFG1_DVC_S);
+
+        spin_unlock_bh(&dsaf_dev->tcam_lock);
 }
 
 /**
@@ -989,6 +1027,8 @@ static void hns_dsaf_tcam_mc_get(
 {
         u32 data_tmp;
 
+        spin_lock_bh(&dsaf_dev->tcam_lock);
+
         /*Write Addr*/
         hns_dsaf_tbl_tcam_addr_cfg(dsaf_dev, address);
 
@@ -997,9 +1037,9 @@ static void hns_dsaf_tcam_mc_get(
 
         /*read tcam data*/
         ptbl_tcam_data->tbl_tcam_data_high =
-                dsaf_read_dev(dsaf_dev, DSAF_TBL_TCAM_RDATA_LOW_0_REG);
-        ptbl_tcam_data->tbl_tcam_data_low =
                 dsaf_read_dev(dsaf_dev, DSAF_TBL_TCAM_RDATA_HIGH_0_REG);
+        ptbl_tcam_data->tbl_tcam_data_low =
+                dsaf_read_dev(dsaf_dev, DSAF_TBL_TCAM_RDATA_LOW_0_REG);
 
         /*read tcam mcast*/
         ptbl_tcam_mcast->tbl_mcast_port_msk[0] =
@@ -1019,6 +1059,8 @@ static void hns_dsaf_tcam_mc_get(
         ptbl_tcam_mcast->tbl_mcast_port_msk[4] =
                 dsaf_get_field(data_tmp, DSAF_TBL_MCAST_CFG4_VM128_112_M,
                                DSAF_TBL_MCAST_CFG4_VM128_112_S);
+
+        spin_unlock_bh(&dsaf_dev->tcam_lock);
 }
 
 /**
@@ -1080,10 +1122,10 @@ int hns_dsaf_set_rx_mac_pause_en(struct dsaf_device *dsaf_dev, int mac_id,
                                  u32 en)
 {
         if (AE_IS_VER1(dsaf_dev->dsaf_ver)) {
-                if (!en)
+                if (!en) {
                         dev_err(dsaf_dev->dev, "dsafv1 can't close rx_pause!\n");
-
-                return -EINVAL;
+                        return -EINVAL;
+                }
         }
 
         dsaf_set_dev_bit(dsaf_dev, DSAF_PAUSE_CFG_REG + mac_id * 4,
@@ -1295,9 +1337,9 @@ static int hns_dsaf_init_hw(struct dsaf_device *dsaf_dev)
         dev_dbg(dsaf_dev->dev,
                 "hns_dsaf_init_hw begin %s !\n", dsaf_dev->ae_dev.name);
 
-        hns_dsaf_rst(dsaf_dev, 0);
+        dsaf_dev->misc_op->dsaf_reset(dsaf_dev, 0);
         mdelay(10);
-        hns_dsaf_rst(dsaf_dev, 1);
+        dsaf_dev->misc_op->dsaf_reset(dsaf_dev, 1);
 
         hns_dsaf_comm_init(dsaf_dev);
 
@@ -1325,7 +1367,7 @@ static int hns_dsaf_init_hw(struct dsaf_device *dsaf_dev)
 static void hns_dsaf_remove_hw(struct dsaf_device *dsaf_dev)
 {
         /*reset*/
-        hns_dsaf_rst(dsaf_dev, 0);
+        dsaf_dev->misc_op->dsaf_reset(dsaf_dev, 0);
 }
 
 /**
@@ -1343,6 +1385,7 @@ static int hns_dsaf_init(struct dsaf_device *dsaf_dev)
         if (HNS_DSAF_IS_DEBUG(dsaf_dev))
                 return 0;
 
+        spin_lock_init(&dsaf_dev->tcam_lock);
         ret = hns_dsaf_init_hw(dsaf_dev);
         if (ret)
                 return ret;
@@ -2088,11 +2131,24 @@ void hns_dsaf_fix_mac_mode(struct hns_mac_cb *mac_cb)
         hns_dsaf_port_work_rate_cfg(dsaf_dev, mac_id, mode);
 }
 
+static u32 hns_dsaf_get_inode_prio_reg(int index)
+{
+        int base_index, offset;
+        u32 base_addr = DSAF_INODE_IN_PRIO_PAUSE_BASE_REG;
+
+        base_index = (index + 1) / DSAF_REG_PER_ZONE;
+        offset = (index + 1) % DSAF_REG_PER_ZONE;
+
+        return base_addr + DSAF_INODE_IN_PRIO_PAUSE_BASE_OFFSET * base_index +
+                DSAF_INODE_IN_PRIO_PAUSE_OFFSET * offset;
+}
+
 void hns_dsaf_update_stats(struct dsaf_device *dsaf_dev, u32 node_num)
 {
         struct dsaf_hw_stats *hw_stats
                 = &dsaf_dev->hw_stats[node_num];
         bool is_ver1 = AE_IS_VER1(dsaf_dev->dsaf_ver);
+        int i;
         u32 reg_tmp;
 
         hw_stats->pad_drop += dsaf_read_dev(dsaf_dev,
@@ -2127,6 +2183,18 @@ void hns_dsaf_update_stats(struct dsaf_device *dsaf_dev, u32 node_num)
         hw_stats->stp_drop += dsaf_read_dev(dsaf_dev,
                 DSAF_INODE_IN_DATA_STP_DISC_0_REG + 0x80 * (u64)node_num);
 
+        /* pfc pause frame statistics stored in dsaf inode*/
+        if ((node_num < DSAF_SERVICE_NW_NUM) && !is_ver1) {
+                for (i = 0; i < DSAF_PRIO_NR; i++) {
+                        reg_tmp = hns_dsaf_get_inode_prio_reg(i);
+                        hw_stats->rx_pfc[i] += dsaf_read_dev(dsaf_dev,
+                                reg_tmp + 0x4 * (u64)node_num);
+                        hw_stats->tx_pfc[i] += dsaf_read_dev(dsaf_dev,
+                                DSAF_XOD_XGE_PFC_PRIO_CNT_BASE_REG +
+                                DSAF_XOD_XGE_PFC_PRIO_CNT_OFFSET * i +
+                                0xF0 * (u64)node_num);
+                }
+        }
         hw_stats->tx_pkts += dsaf_read_dev(dsaf_dev,
                 DSAF_XOD_RCVPKT_CNT_0_REG + 0x90 * (u64)node_num);
 }
@@ -2464,9 +2532,12 @@ void hns_dsaf_get_regs(struct dsaf_device *ddev, u32 port, void *data)
                 p[i] = 0xdddddddd;
 }
 
-static char *hns_dsaf_get_node_stats_strings(char *data, int node)
+static char *hns_dsaf_get_node_stats_strings(char *data, int node,
+                                             struct dsaf_device *dsaf_dev)
 {
         char *buff = data;
+        int i;
+        bool is_ver1 = AE_IS_VER1(dsaf_dev->dsaf_ver);
 
         snprintf(buff, ETH_GSTRING_LEN, "innod%d_pad_drop_pkts", node);
         buff = buff + ETH_GSTRING_LEN;
@@ -2494,6 +2565,18 @@ static char *hns_dsaf_get_node_stats_strings(char *data, int node)
         buff = buff + ETH_GSTRING_LEN;
         snprintf(buff, ETH_GSTRING_LEN, "innod%d_stp_drop_pkts", node);
         buff = buff + ETH_GSTRING_LEN;
+        if ((node < DSAF_SERVICE_NW_NUM) && (!is_ver1)) {
+                for (i = 0; i < DSAF_PRIO_NR; i++) {
+                        snprintf(buff, ETH_GSTRING_LEN,
+                                 "inod%d_pfc_prio%d_pkts", node, i);
+                        buff = buff + ETH_GSTRING_LEN;
+                }
+                for (i = 0; i < DSAF_PRIO_NR; i++) {
+                        snprintf(buff, ETH_GSTRING_LEN,
+                                 "onod%d_pfc_prio%d_pkts", node, i);
+                        buff = buff + ETH_GSTRING_LEN;
+                }
+        }
         snprintf(buff, ETH_GSTRING_LEN, "onnod%d_tx_pkts", node);
         buff = buff + ETH_GSTRING_LEN;
 
@@ -2504,7 +2587,9 @@ static u64 *hns_dsaf_get_node_stats(struct dsaf_device *ddev, u64 *data,
                                     int node_num)
 {
         u64 *p = data;
+        int i;
         struct dsaf_hw_stats *hw_stats = &ddev->hw_stats[node_num];
+        bool is_ver1 = AE_IS_VER1(ddev->dsaf_ver);
 
         p[0] = hw_stats->pad_drop;
         p[1] = hw_stats->man_pkts;
@@ -2519,8 +2604,16 @@ static u64 *hns_dsaf_get_node_stats(struct dsaf_device *ddev, u64 *data,
         p[10] = hw_stats->local_addr_false;
         p[11] = hw_stats->vlan_drop;
         p[12] = hw_stats->stp_drop;
-        p[13] = hw_stats->tx_pkts;
+        if ((node_num < DSAF_SERVICE_NW_NUM) && (!is_ver1)) {
+                for (i = 0; i < DSAF_PRIO_NR; i++) {
+                        p[13 + i] = hw_stats->rx_pfc[i];
+                        p[13 + i + DSAF_PRIO_NR] = hw_stats->tx_pfc[i];
+                }
+                p[29] = hw_stats->tx_pkts;
+                return &p[30];
+        }
 
+        p[13] = hw_stats->tx_pkts;
         return &p[14];
 }
 
@@ -2548,11 +2641,16 @@ void hns_dsaf_get_stats(struct dsaf_device *ddev, u64 *data, int port)
  *@stringset: type of values in data
  *return dsaf string name count
  */
-int hns_dsaf_get_sset_count(int stringset)
+int hns_dsaf_get_sset_count(struct dsaf_device *dsaf_dev, int stringset)
 {
-        if (stringset == ETH_SS_STATS)
-                return DSAF_STATIC_NUM;
+        bool is_ver1 = AE_IS_VER1(dsaf_dev->dsaf_ver);
 
+        if (stringset == ETH_SS_STATS) {
+                if (is_ver1)
+                        return DSAF_STATIC_NUM;
+                else
+                        return DSAF_V2_STATIC_NUM;
+        }
         return 0;
 }
 
@@ -2562,7 +2660,8 @@ int hns_dsaf_get_sset_count(int stringset)
  *@data:strings name value
  *@port:port index
  */
-void hns_dsaf_get_strings(int stringset, u8 *data, int port)
+void hns_dsaf_get_strings(int stringset, u8 *data, int port,
+                          struct dsaf_device *dsaf_dev)
 {
         char *buff = (char *)data;
         int node = port;
@@ -2571,11 +2670,11 @@ void hns_dsaf_get_strings(int stringset, u8 *data, int port)
                 return;
 
         /* for ge/xge node info */
-        buff = hns_dsaf_get_node_stats_strings(buff, node);
+        buff = hns_dsaf_get_node_stats_strings(buff, node, dsaf_dev);
 
         /* for ppe node info */
         node = port + DSAF_PPE_INODE_BASE;
-        (void)hns_dsaf_get_node_stats_strings(buff, node);
+        (void)hns_dsaf_get_node_stats_strings(buff, node, dsaf_dev);
 }
 
 /**
@@ -2611,7 +2710,7 @@ static int hns_dsaf_probe(struct platform_device *pdev)
 
         ret = hns_dsaf_init(dsaf_dev);
         if (ret)
-                goto free_cfg;
+                goto free_dev;
 
         ret = hns_mac_init(dsaf_dev);
         if (ret)
@@ -2636,9 +2735,6 @@ uninit_mac:
 uninit_dsaf:
         hns_dsaf_free(dsaf_dev);
 
-free_cfg:
-        hns_dsaf_free_cfg(dsaf_dev);
-
 free_dev:
         hns_dsaf_free_dev(dsaf_dev);
 
@@ -2661,8 +2757,6 @@ static int hns_dsaf_remove(struct platform_device *pdev)
 
         hns_dsaf_free(dsaf_dev);
 
-        hns_dsaf_free_cfg(dsaf_dev);
-
         hns_dsaf_free_dev(dsaf_dev);
 
         return 0;
@@ -2680,6 +2774,7 @@ static struct platform_driver g_dsaf_driver = {
         .driver = {
                 .name = DSAF_DRV_NAME,
                 .of_match_table = g_dsaf_match,
+                .acpi_match_table = hns_dsaf_acpi_match,
         },
 };
 
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.h b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.h
index f0502ba0a677..1daf018d9071 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.h
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.h
@@ -39,6 +39,9 @@ struct hns_mac_cb;
 
 #define DSAF_DUMP_REGS_NUM 504
 #define DSAF_STATIC_NUM 28
+#define DSAF_V2_STATIC_NUM      44
+#define DSAF_PRIO_NR    8
+#define DSAF_REG_PER_ZONE       3
 
 #define DSAF_STATS_READ(p, offset) (*((u64 *)((u8 *)(p) + (offset))))
 #define HNS_DSAF_IS_DEBUG(dev) (dev->dsaf_mode == DSAF_MODE_DISABLE_SP)
@@ -176,6 +179,8 @@ struct dsaf_hw_stats {
         u64 local_addr_false;
         u64 vlan_drop;
         u64 stp_drop;
+        u64 rx_pfc[DSAF_PRIO_NR];
+        u64 tx_pfc[DSAF_PRIO_NR];
         u64 tx_pkts;
 };
 
@@ -268,6 +273,27 @@ struct dsaf_int_stat {
 
 };
 
+struct dsaf_misc_op {
+        void (*cpld_set_led)(struct hns_mac_cb *mac_cb, int link_status,
+                             u16 speed, int data);
+        void (*cpld_reset_led)(struct hns_mac_cb *mac_cb);
+        int (*cpld_set_led_id)(struct hns_mac_cb *mac_cb,
+                               enum hnae_led_state status);
+        /* reset seris function, it will be reset if the dereseet is 0 */
+        void (*dsaf_reset)(struct dsaf_device *dsaf_dev, bool dereset);
+        void (*xge_srst)(struct dsaf_device *dsaf_dev, u32 port, bool dereset);
+        void (*xge_core_srst)(struct dsaf_device *dsaf_dev, u32 port,
+                              bool dereset);
+        void (*ge_srst)(struct dsaf_device *dsaf_dev, u32 port, bool dereset);
+        void (*ppe_srst)(struct dsaf_device *dsaf_dev, u32 port, bool dereset);
+        void (*ppe_comm_srst)(struct dsaf_device *dsaf_dev, bool dereset);
+
+        phy_interface_t (*get_phy_if)(struct hns_mac_cb *mac_cb);
+        int (*get_sfp_prsnt)(struct hns_mac_cb *mac_cb, int *sfp_prsnt);
+
+        int (*cfg_serdes_loopback)(struct hns_mac_cb *mac_cb, bool en);
+};
+
 /* Dsaf device struct define ,and mac ->  dsaf */
 struct dsaf_device {
         struct device *dev;
@@ -292,9 +318,12 @@ struct dsaf_device {
         struct ppe_common_cb *ppe_common[DSAF_COMM_DEV_NUM];
         struct rcb_common_cb *rcb_common[DSAF_COMM_DEV_NUM];
         struct hns_mac_cb *mac_cb[DSAF_MAX_PORT_NUM];
+        struct dsaf_misc_op *misc_op;
 
         struct dsaf_hw_stats hw_stats[DSAF_NODE_NUM];
         struct dsaf_int_stat int_stat;
+        /* make sure tcam table config spinlock */
+        spinlock_t tcam_lock;
 };
 
 static inline void *hns_dsaf_dev_priv(const struct dsaf_device *dsaf_dev)
@@ -388,27 +417,17 @@ int hns_dsaf_get_mac_entry_by_index(
         u16 entry_index,
         struct dsaf_drv_mac_multi_dest_entry *mac_entry);
 
-void hns_dsaf_rst(struct dsaf_device *dsaf_dev, u32 val);
-
-void hns_ppe_srst_by_port(struct dsaf_device *dsaf_dev, u32 port, u32 val);
-
-void hns_ppe_com_srst(struct ppe_common_cb *ppe_common, u32 val);
-
 void hns_dsaf_fix_mac_mode(struct hns_mac_cb *mac_cb);
 
 int hns_dsaf_ae_init(struct dsaf_device *dsaf_dev);
 void hns_dsaf_ae_uninit(struct dsaf_device *dsaf_dev);
 
-void hns_dsaf_xge_srst_by_port(struct dsaf_device *dsaf_dev, u32 port, u32 val);
-void hns_dsaf_ge_srst_by_port(struct dsaf_device *dsaf_dev, u32 port, u32 val);
-void hns_dsaf_xge_core_srst_by_port(struct dsaf_device *dsaf_dev,
-                                    u32 port, u32 val);
-
 void hns_dsaf_update_stats(struct dsaf_device *dsaf_dev, u32 inode_num);
 
-int hns_dsaf_get_sset_count(int stringset);
+int hns_dsaf_get_sset_count(struct dsaf_device *dsaf_dev, int stringset);
 void hns_dsaf_get_stats(struct dsaf_device *ddev, u64 *data, int port);
-void hns_dsaf_get_strings(int stringset, u8 *data, int port);
+void hns_dsaf_get_strings(int stringset, u8 *data, int port,
+                          struct dsaf_device *dsaf_dev);
 
 void hns_dsaf_get_regs(struct dsaf_device *ddev, u32 port, void *data);
 int hns_dsaf_get_regs_count(void);
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c
index a837bb9e3839..8473287d4c8b 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c
@@ -12,6 +12,27 @@
 #include "hns_dsaf_ppe.h"
 #include "hns_dsaf_reg.h"
 
+enum _dsm_op_index {
+        HNS_OP_RESET_FUNC               = 0x1,
+        HNS_OP_SERDES_LP_FUNC           = 0x2,
+        HNS_OP_LED_SET_FUNC             = 0x3,
+        HNS_OP_GET_PORT_TYPE_FUNC       = 0x4,
+        HNS_OP_GET_SFP_STAT_FUNC        = 0x5,
+};
+
+enum _dsm_rst_type {
+        HNS_DSAF_RESET_FUNC     = 0x1,
+        HNS_PPE_RESET_FUNC      = 0x2,
+        HNS_XGE_CORE_RESET_FUNC = 0x3,
+        HNS_XGE_RESET_FUNC      = 0x4,
+        HNS_GE_RESET_FUNC       = 0x5,
+};
+
+const u8 hns_dsaf_acpi_dsm_uuid[] = {
+        0x1A, 0xAA, 0x85, 0x1A, 0x93, 0xE2, 0x5E, 0x41,
+        0x8E, 0x28, 0x8D, 0x69, 0x0A, 0x0F, 0x82, 0x0A
+};
+
 static void dsaf_write_sub(struct dsaf_device *dsaf_dev, u32 reg, u32 val)
 {
         if (dsaf_dev->sub_ctrl)
@@ -32,8 +53,8 @@ static u32 dsaf_read_sub(struct dsaf_device *dsaf_dev, u32 reg)
         return ret;
 }
 
-void hns_cpld_set_led(struct hns_mac_cb *mac_cb, int link_status,
-                      u16 speed, int data)
+static void hns_cpld_set_led(struct hns_mac_cb *mac_cb, int link_status,
+                             u16 speed, int data)
 {
         int speed_reg = 0;
         u8 value;
@@ -65,13 +86,14 @@ void hns_cpld_set_led(struct hns_mac_cb *mac_cb, int link_status,
                         mac_cb->cpld_led_value = value;
                 }
         } else {
-                dsaf_write_syscon(mac_cb->cpld_ctrl, mac_cb->cpld_ctrl_reg,
-                                  CPLD_LED_DEFAULT_VALUE);
-                mac_cb->cpld_led_value = CPLD_LED_DEFAULT_VALUE;
+                value = (mac_cb->cpld_led_value) & (0x1 << DSAF_LED_ANCHOR_B);
+                dsaf_write_syscon(mac_cb->cpld_ctrl,
+                                  mac_cb->cpld_ctrl_reg, value);
+                mac_cb->cpld_led_value = value;
         }
 }
 
-void cpld_led_reset(struct hns_mac_cb *mac_cb)
+static void cpld_led_reset(struct hns_mac_cb *mac_cb)
 {
         if (!mac_cb || !mac_cb->cpld_ctrl)
                 return;
@@ -81,8 +103,8 @@ void cpld_led_reset(struct hns_mac_cb *mac_cb)
         mac_cb->cpld_led_value = CPLD_LED_DEFAULT_VALUE;
 }
 
-int cpld_set_led_id(struct hns_mac_cb *mac_cb,
-                    enum hnae_led_state status)
+static int cpld_set_led_id(struct hns_mac_cb *mac_cb,
+                           enum hnae_led_state status)
 {
         switch (status) {
         case HNAE_LED_ACTIVE:
@@ -93,7 +115,7 @@ int cpld_set_led_id(struct hns_mac_cb *mac_cb,
                              CPLD_LED_ON_VALUE);
                 dsaf_write_syscon(mac_cb->cpld_ctrl, mac_cb->cpld_ctrl_reg,
                                   mac_cb->cpld_led_value);
-                return 2;
+                break;
         case HNAE_LED_INACTIVE:
                 dsaf_set_bit(mac_cb->cpld_led_value, DSAF_LED_ANCHOR_B,
                              CPLD_LED_DEFAULT_VALUE);
@@ -101,7 +123,8 @@ int cpld_set_led_id(struct hns_mac_cb *mac_cb,
                                   mac_cb->cpld_led_value);
                 break;
         default:
-                break;
+                dev_err(mac_cb->dev, "invalid led state: %d!", status);
+                return -EINVAL;
         }
 
         return 0;
@@ -109,12 +132,40 @@ int cpld_set_led_id(struct hns_mac_cb *mac_cb,
 
 #define RESET_REQ_OR_DREQ 1
 
-void hns_dsaf_rst(struct dsaf_device *dsaf_dev, u32 val)
+static void hns_dsaf_acpi_srst_by_port(struct dsaf_device *dsaf_dev, u8 op_type,
+                                       u32 port_type, u32 port, u32 val)
+{
+        union acpi_object *obj;
+        union acpi_object obj_args[3], argv4;
+
+        obj_args[0].integer.type = ACPI_TYPE_INTEGER;
+        obj_args[0].integer.value = port_type;
+        obj_args[1].integer.type = ACPI_TYPE_INTEGER;
+        obj_args[1].integer.value = port;
+        obj_args[2].integer.type = ACPI_TYPE_INTEGER;
+        obj_args[2].integer.value = val;
+
+        argv4.type = ACPI_TYPE_PACKAGE;
+        argv4.package.count = 3;
+        argv4.package.elements = obj_args;
+
+        obj = acpi_evaluate_dsm(ACPI_HANDLE(dsaf_dev->dev),
+                                hns_dsaf_acpi_dsm_uuid, 0, op_type, &argv4);
+        if (!obj) {
+                dev_warn(dsaf_dev->dev, "reset port_type%d port%d fail!",
+                         port_type, port);
+                return;
+        }
+
+        ACPI_FREE(obj);
+}
+
+static void hns_dsaf_rst(struct dsaf_device *dsaf_dev, bool dereset)
 {
         u32 xbar_reg_addr;
         u32 nt_reg_addr;
 
-        if (!val) {
+        if (!dereset) {
                 xbar_reg_addr = DSAF_SUB_SC_XBAR_RESET_REQ_REG;
                 nt_reg_addr = DSAF_SUB_SC_NT_RESET_REQ_REG;
         } else {
@@ -126,7 +177,15 @@ void hns_dsaf_rst(struct dsaf_device *dsaf_dev, u32 val)
         dsaf_write_sub(dsaf_dev, nt_reg_addr, RESET_REQ_OR_DREQ);
 }
 
-void hns_dsaf_xge_srst_by_port(struct dsaf_device *dsaf_dev, u32 port, u32 val)
+static void hns_dsaf_rst_acpi(struct dsaf_device *dsaf_dev, bool dereset)
+{
+        hns_dsaf_acpi_srst_by_port(dsaf_dev, HNS_OP_RESET_FUNC,
+                                   HNS_DSAF_RESET_FUNC,
+                                   0, dereset);
+}
+
+static void hns_dsaf_xge_srst_by_port(struct dsaf_device *dsaf_dev, u32 port,
+                                      bool dereset)
 {
         u32 reg_val = 0;
         u32 reg_addr;
@@ -137,7 +196,7 @@ void hns_dsaf_xge_srst_by_port(struct dsaf_device *dsaf_dev, u32 port, u32 val)
         reg_val |= RESET_REQ_OR_DREQ;
         reg_val |= 0x2082082 << dsaf_dev->mac_cb[port]->port_rst_off;
 
-        if (val == 0)
+        if (!dereset)
                 reg_addr = DSAF_SUB_SC_XGE_RESET_REQ_REG;
         else
                 reg_addr = DSAF_SUB_SC_XGE_RESET_DREQ_REG;
@@ -145,8 +204,15 @@ void hns_dsaf_xge_srst_by_port(struct dsaf_device *dsaf_dev, u32 port, u32 val)
         dsaf_write_sub(dsaf_dev, reg_addr, reg_val);
 }
 
-void hns_dsaf_xge_core_srst_by_port(struct dsaf_device *dsaf_dev,
-                                    u32 port, u32 val)
+static void hns_dsaf_xge_srst_by_port_acpi(struct dsaf_device *dsaf_dev,
+                                           u32 port, bool dereset)
+{
+        hns_dsaf_acpi_srst_by_port(dsaf_dev, HNS_OP_RESET_FUNC,
+                                   HNS_XGE_RESET_FUNC, port, dereset);
+}
+
+static void hns_dsaf_xge_core_srst_by_port(struct dsaf_device *dsaf_dev,
+                                           u32 port, bool dereset)
 {
         u32 reg_val = 0;
         u32 reg_addr;
@@ -157,7 +223,7 @@ void hns_dsaf_xge_core_srst_by_port(struct dsaf_device *dsaf_dev,
         reg_val |= XGMAC_TRX_CORE_SRST_M
                 << dsaf_dev->mac_cb[port]->port_rst_off;
 
-        if (val == 0)
+        if (!dereset)
                 reg_addr = DSAF_SUB_SC_XGE_RESET_REQ_REG;
         else
                 reg_addr = DSAF_SUB_SC_XGE_RESET_DREQ_REG;
@@ -165,7 +231,16 @@ void hns_dsaf_xge_core_srst_by_port(struct dsaf_device *dsaf_dev,
         dsaf_write_sub(dsaf_dev, reg_addr, reg_val);
 }
 
-void hns_dsaf_ge_srst_by_port(struct dsaf_device *dsaf_dev, u32 port, u32 val)
+static void
+hns_dsaf_xge_core_srst_by_port_acpi(struct dsaf_device *dsaf_dev,
+                                    u32 port, bool dereset)
+{
+        hns_dsaf_acpi_srst_by_port(dsaf_dev, HNS_OP_RESET_FUNC,
+                                   HNS_XGE_CORE_RESET_FUNC, port, dereset);
+}
+
+static void hns_dsaf_ge_srst_by_port(struct dsaf_device *dsaf_dev, u32 port,
+                                     bool dereset)
 {
         u32 reg_val_1;
         u32 reg_val_2;
@@ -183,7 +258,7 @@ void hns_dsaf_ge_srst_by_port(struct dsaf_device *dsaf_dev, u32 port, u32 val)
                 else
                         reg_val_2  = 0x2082082 << port_rst_off;
 
-                if (val == 0) {
+                if (!dereset) {
                         dsaf_write_sub(dsaf_dev, DSAF_SUB_SC_GE_RESET_REQ1_REG,
                                        reg_val_1);
 
@@ -198,9 +273,13 @@ void hns_dsaf_ge_srst_by_port(struct dsaf_device *dsaf_dev, u32 port, u32 val)
                 }
         } else {
                 reg_val_1 = 0x15540 << dsaf_dev->reset_offset;
-                reg_val_2 = 0x100 << dsaf_dev->reset_offset;
 
-                if (val == 0) {
+                if (AE_IS_VER1(dsaf_dev->dsaf_ver))
+                        reg_val_2 = 0x100 << dsaf_dev->reset_offset;
+                else
+                        reg_val_2 = 0x40 << dsaf_dev->reset_offset;
+
+                if (!dereset) {
                         dsaf_write_sub(dsaf_dev, DSAF_SUB_SC_GE_RESET_REQ1_REG,
                                        reg_val_1);
 
@@ -216,14 +295,22 @@ void hns_dsaf_ge_srst_by_port(struct dsaf_device *dsaf_dev, u32 port, u32 val)
         }
 }
 
-void hns_ppe_srst_by_port(struct dsaf_device *dsaf_dev, u32 port, u32 val)
+static void hns_dsaf_ge_srst_by_port_acpi(struct dsaf_device *dsaf_dev,
+                                          u32 port, bool dereset)
+{
+        hns_dsaf_acpi_srst_by_port(dsaf_dev, HNS_OP_RESET_FUNC,
+                                   HNS_GE_RESET_FUNC, port, dereset);
+}
+
+static void hns_ppe_srst_by_port(struct dsaf_device *dsaf_dev, u32 port,
+                                 bool dereset)
 {
         u32 reg_val = 0;
         u32 reg_addr;
 
         reg_val |= RESET_REQ_OR_DREQ << dsaf_dev->mac_cb[port]->port_rst_off;
 
-        if (val == 0)
+        if (!dereset)
                 reg_addr = DSAF_SUB_SC_PPE_RESET_REQ_REG;
         else
                 reg_addr = DSAF_SUB_SC_PPE_RESET_DREQ_REG;
@@ -231,15 +318,24 @@ void hns_ppe_srst_by_port(struct dsaf_device *dsaf_dev, u32 port, u32 val)
         dsaf_write_sub(dsaf_dev, reg_addr, reg_val);
 }
 
-void hns_ppe_com_srst(struct ppe_common_cb *ppe_common, u32 val)
+static void
+hns_ppe_srst_by_port_acpi(struct dsaf_device *dsaf_dev, u32 port, bool dereset)
+{
+        hns_dsaf_acpi_srst_by_port(dsaf_dev, HNS_OP_RESET_FUNC,
+                                   HNS_PPE_RESET_FUNC, port, dereset);
+}
+
+static void hns_ppe_com_srst(struct dsaf_device *dsaf_dev, bool dereset)
 {
-        struct dsaf_device *dsaf_dev = ppe_common->dsaf_dev;
         u32 reg_val;
         u32 reg_addr;
 
+        if (!(dev_of_node(dsaf_dev->dev)))
+                return;
+
         if (!HNS_DSAF_IS_DEBUG(dsaf_dev)) {
                 reg_val = RESET_REQ_OR_DREQ;
-                if (val == 0)
+                if (!dereset)
                         reg_addr = DSAF_SUB_SC_RCB_PPE_COM_RESET_REQ_REG;
                 else
                         reg_addr = DSAF_SUB_SC_RCB_PPE_COM_RESET_DREQ_REG;
@@ -247,7 +343,7 @@ void hns_ppe_com_srst(struct ppe_common_cb *ppe_common, u32 val)
         } else {
                 reg_val = 0x100 << dsaf_dev->reset_offset;
 
-                if (val == 0)
+                if (!dereset)
                         reg_addr = DSAF_SUB_SC_PPE_RESET_REQ_REG;
                 else
                         reg_addr = DSAF_SUB_SC_PPE_RESET_DREQ_REG;
@@ -261,7 +357,7 @@ void hns_ppe_com_srst(struct ppe_common_cb *ppe_common, u32 val)
  * @mac_cb: mac control block
  * retuen phy interface
  */
-phy_interface_t hns_mac_get_phy_if(struct hns_mac_cb *mac_cb)
+static phy_interface_t hns_mac_get_phy_if(struct hns_mac_cb *mac_cb)
 {
         u32 mode;
         u32 reg;
@@ -293,6 +389,36 @@ phy_interface_t hns_mac_get_phy_if(struct hns_mac_cb *mac_cb)
         return phy_if;
 }
 
+static phy_interface_t hns_mac_get_phy_if_acpi(struct hns_mac_cb *mac_cb)
+{
+        phy_interface_t phy_if = PHY_INTERFACE_MODE_NA;
+        union acpi_object *obj;
+        union acpi_object obj_args, argv4;
+
+        obj_args.integer.type = ACPI_TYPE_INTEGER;
+        obj_args.integer.value = mac_cb->mac_id;
+
+        argv4.type = ACPI_TYPE_PACKAGE,
+        argv4.package.count = 1,
+        argv4.package.elements = &obj_args,
+
+        obj = acpi_evaluate_dsm(ACPI_HANDLE(mac_cb->dev),
+                                hns_dsaf_acpi_dsm_uuid, 0,
+                                HNS_OP_GET_PORT_TYPE_FUNC, &argv4);
+
+        if (!obj || obj->type != ACPI_TYPE_INTEGER)
+                return phy_if;
+
+        phy_if = obj->integer.value ?
+                PHY_INTERFACE_MODE_XGMII : PHY_INTERFACE_MODE_SGMII;
+
+        dev_dbg(mac_cb->dev, "mac_id=%d, phy_if=%d\n", mac_cb->mac_id, phy_if);
+
+        ACPI_FREE(obj);
+
+        return phy_if;
+}
+
 int hns_mac_get_sfp_prsnt(struct hns_mac_cb *mac_cb, int *sfp_prsnt)
 {
         if (!mac_cb->cpld_ctrl)
@@ -309,13 +435,8 @@ int hns_mac_get_sfp_prsnt(struct hns_mac_cb *mac_cb, int *sfp_prsnt)
  * @mac_cb: mac control block
  * retuen 0 == success
  */
-int hns_mac_config_sds_loopback(struct hns_mac_cb *mac_cb, u8 en)
+static int hns_mac_config_sds_loopback(struct hns_mac_cb *mac_cb, bool en)
 {
-        /* port 0-3 hilink4 base is serdes_vaddr + 0x00280000
-         * port 4-7 hilink3 base is serdes_vaddr + 0x00200000
-         */
-        u8 *base_addr = (u8 *)mac_cb->serdes_vaddr +
-                       (mac_cb->mac_id <= 3 ? 0x00280000 : 0x00200000);
         const u8 lane_id[] = {
                 0,      /* mac 0 -> lane 0 */
                 1,      /* mac 1 -> lane 1 */
@@ -332,7 +453,7 @@ int hns_mac_config_sds_loopback(struct hns_mac_cb *mac_cb, u8 en)
         int sfp_prsnt;
         int ret = hns_mac_get_sfp_prsnt(mac_cb, &sfp_prsnt);
 
-        if (!mac_cb->phy_node) {
+        if (!mac_cb->phy_dev) {
                 if (ret)
                         pr_info("please confirm sfp is present or not\n");
                 else
@@ -341,13 +462,110 @@ int hns_mac_config_sds_loopback(struct hns_mac_cb *mac_cb, u8 en)
         }
 
         if (mac_cb->serdes_ctrl) {
-                u32 origin = dsaf_read_syscon(mac_cb->serdes_ctrl, reg_offset);
+                u32 origin;
+
+                if (!AE_IS_VER1(mac_cb->dsaf_dev->dsaf_ver)) {
+#define HILINK_ACCESS_SEL_CFG           0x40008
+                        /* hilink4 & hilink3 use the same xge training and
+                         * xge u adaptor. There is a hilink access sel cfg
+                         * register to select which one to be configed
+                         */
+                        if ((!HNS_DSAF_IS_DEBUG(mac_cb->dsaf_dev)) &&
+                            (mac_cb->mac_id <= 3))
+                                dsaf_write_syscon(mac_cb->serdes_ctrl,
+                                                  HILINK_ACCESS_SEL_CFG, 0);
+                        else
+                                dsaf_write_syscon(mac_cb->serdes_ctrl,
+                                                  HILINK_ACCESS_SEL_CFG, 3);
+                }
 
-                dsaf_set_field(origin, 1ull << 10, 10, !!en);
+                origin = dsaf_read_syscon(mac_cb->serdes_ctrl, reg_offset);
+
+                dsaf_set_field(origin, 1ull << 10, 10, en);
                 dsaf_write_syscon(mac_cb->serdes_ctrl, reg_offset, origin);
         } else {
-                dsaf_set_reg_field(base_addr, reg_offset, 1ull << 10, 10, !!en);
+                u8 *base_addr = (u8 *)mac_cb->serdes_vaddr +
+                                (mac_cb->mac_id <= 3 ? 0x00280000 : 0x00200000);
+                dsaf_set_reg_field(base_addr, reg_offset, 1ull << 10, 10, en);
+        }
+
+        return 0;
+}
+
+static int
+hns_mac_config_sds_loopback_acpi(struct hns_mac_cb *mac_cb, bool en)
+{
+        union acpi_object *obj;
+        union acpi_object obj_args[3], argv4;
+
+        obj_args[0].integer.type = ACPI_TYPE_INTEGER;
+        obj_args[0].integer.value = mac_cb->mac_id;
+        obj_args[1].integer.type = ACPI_TYPE_INTEGER;
+        obj_args[1].integer.value = !!en;
+
+        argv4.type = ACPI_TYPE_PACKAGE;
+        argv4.package.count = 2;
+        argv4.package.elements = obj_args;
+
+        obj = acpi_evaluate_dsm(ACPI_HANDLE(mac_cb->dsaf_dev->dev),
+                                hns_dsaf_acpi_dsm_uuid, 0,
+                                HNS_OP_SERDES_LP_FUNC, &argv4);
+        if (!obj) {
+                dev_warn(mac_cb->dsaf_dev->dev, "set port%d serdes lp fail!",
+                         mac_cb->mac_id);
+
+                return -ENOTSUPP;
         }
 
+        ACPI_FREE(obj);
+
         return 0;
 }
+
+struct dsaf_misc_op *hns_misc_op_get(struct dsaf_device *dsaf_dev)
+{
+        struct dsaf_misc_op *misc_op;
+
+        misc_op = devm_kzalloc(dsaf_dev->dev, sizeof(*misc_op), GFP_KERNEL);
+        if (!misc_op)
+                return NULL;
+
+        if (dev_of_node(dsaf_dev->dev)) {
+                misc_op->cpld_set_led = hns_cpld_set_led;
+                misc_op->cpld_reset_led = cpld_led_reset;
+                misc_op->cpld_set_led_id = cpld_set_led_id;
+
+                misc_op->dsaf_reset = hns_dsaf_rst;
+                misc_op->xge_srst = hns_dsaf_xge_srst_by_port;
+                misc_op->xge_core_srst = hns_dsaf_xge_core_srst_by_port;
+                misc_op->ge_srst = hns_dsaf_ge_srst_by_port;
+                misc_op->ppe_srst = hns_ppe_srst_by_port;
+                misc_op->ppe_comm_srst = hns_ppe_com_srst;
+
+                misc_op->get_phy_if = hns_mac_get_phy_if;
+                misc_op->get_sfp_prsnt = hns_mac_get_sfp_prsnt;
+
+                misc_op->cfg_serdes_loopback = hns_mac_config_sds_loopback;
+        } else if (is_acpi_node(dsaf_dev->dev->fwnode)) {
+                misc_op->cpld_set_led = hns_cpld_set_led;
+                misc_op->cpld_reset_led = cpld_led_reset;
+                misc_op->cpld_set_led_id = cpld_set_led_id;
+
+                misc_op->dsaf_reset = hns_dsaf_rst_acpi;
+                misc_op->xge_srst = hns_dsaf_xge_srst_by_port_acpi;
+                misc_op->xge_core_srst = hns_dsaf_xge_core_srst_by_port_acpi;
+                misc_op->ge_srst = hns_dsaf_ge_srst_by_port_acpi;
+                misc_op->ppe_srst = hns_ppe_srst_by_port_acpi;
+                misc_op->ppe_comm_srst = hns_ppe_com_srst;
+
+                misc_op->get_phy_if = hns_mac_get_phy_if_acpi;
+                misc_op->get_sfp_prsnt = hns_mac_get_sfp_prsnt;
+
+                misc_op->cfg_serdes_loopback = hns_mac_config_sds_loopback_acpi;
+        } else {
+                devm_kfree(dsaf_dev->dev, (void *)misc_op);
+                misc_op = NULL;
+        }
+
+        return (void *)misc_op;
+}
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.h b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.h
index 419f07aa9734..f06bb03d47a6 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.h
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.h
@@ -33,11 +33,6 @@
 #define DSAF_LED_DATA_B 4
 #define DSAF_LED_ANCHOR_B 5
 
-void hns_cpld_set_led(struct hns_mac_cb *mac_cb, int link_status,
-                      u16 speed, int data);
-void cpld_led_reset(struct hns_mac_cb *mac_cb);
-int cpld_set_led_id(struct hns_mac_cb *mac_cb,
-                    enum hnae_led_state status);
-int hns_mac_get_sfp_prsnt(struct hns_mac_cb *mac_cb, int *sfp_prsnt);
+struct dsaf_misc_op *hns_misc_op_get(struct dsaf_device *dsaf_dev);
 
 #endif
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c
index 8cd151a5245e..ff8b6a468b24 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c
@@ -112,7 +112,6 @@ void hns_ppe_common_free_cfg(struct dsaf_device *dsaf_dev, u32 comm_index)
 static void __iomem *hns_ppe_get_iobase(struct ppe_common_cb *ppe_common,
                                         int ppe_idx)
 {
-
         return ppe_common->dsaf_dev->ppe_base + ppe_idx * PPE_REG_OFFSET;
 }
 
@@ -200,11 +199,12 @@ static void hns_ppe_set_port_mode(struct hns_ppe_cb *ppe_cb,
 static int hns_ppe_common_init_hw(struct ppe_common_cb *ppe_common)
 {
         enum ppe_qid_mode qid_mode;
-        enum dsaf_mode dsaf_mode = ppe_common->dsaf_dev->dsaf_mode;
+        struct dsaf_device *dsaf_dev = ppe_common->dsaf_dev;
+        enum dsaf_mode dsaf_mode = dsaf_dev->dsaf_mode;
 
-        hns_ppe_com_srst(ppe_common, 0);
+        dsaf_dev->misc_op->ppe_comm_srst(dsaf_dev, 0);
         mdelay(100);
-        hns_ppe_com_srst(ppe_common, 1);
+        dsaf_dev->misc_op->ppe_comm_srst(dsaf_dev, 1);
         mdelay(100);
 
         if (ppe_common->ppe_mode == PPE_COMMON_MODE_SERVICE) {
@@ -288,9 +288,9 @@ static void hns_ppe_init_hw(struct hns_ppe_cb *ppe_cb)
         /* get default RSS key */
         netdev_rss_key_fill(ppe_cb->rss_key, HNS_PPEV2_RSS_KEY_SIZE);
 
-        hns_ppe_srst_by_port(dsaf_dev, port, 0);
+        dsaf_dev->misc_op->ppe_srst(dsaf_dev, port, 0);
         mdelay(10);
-        hns_ppe_srst_by_port(dsaf_dev, port, 1);
+        dsaf_dev->misc_op->ppe_srst(dsaf_dev, port, 1);
 
         /* clr and msk except irq*/
         hns_ppe_exc_irq_en(ppe_cb, 0);
@@ -328,10 +328,11 @@ static void hns_ppe_init_hw(struct hns_ppe_cb *ppe_cb)
 static void hns_ppe_uninit_hw(struct hns_ppe_cb *ppe_cb)
 {
         u32 port;
+        struct dsaf_device *dsaf_dev = ppe_cb->ppe_common_cb->dsaf_dev;
 
         if (ppe_cb->ppe_common_cb) {
                 port = ppe_cb->index;
-                hns_ppe_srst_by_port(ppe_cb->ppe_common_cb->dsaf_dev, port, 0);
+                dsaf_dev->misc_op->ppe_srst(dsaf_dev, port, 0);
         }
 }
 
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.c
index 4ef6d23d998e..ef1107777c08 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.c
@@ -458,7 +458,6 @@ void hns_rcb_get_cfg(struct rcb_common_cb *rcb_common)
         u32 i;
         u32 ring_num = rcb_common->ring_num;
         int base_irq_idx = hns_rcb_get_base_irq_idx(rcb_common);
-        struct device_node *np = rcb_common->dsaf_dev->dev->of_node;
         struct platform_device *pdev =
                 to_platform_device(rcb_common->dsaf_dev->dev);
         bool is_ver1 = AE_IS_VER1(rcb_common->dsaf_dev->dsaf_ver);
@@ -473,10 +472,10 @@ void hns_rcb_get_cfg(struct rcb_common_cb *rcb_common)
                 ring_pair_cb->port_id_in_comm =
                         hns_rcb_get_port_in_comm(rcb_common, i);
                 ring_pair_cb->virq[HNS_RCB_IRQ_IDX_TX] =
-                is_ver1 ? irq_of_parse_and_map(np, base_irq_idx + i * 2) :
+                is_ver1 ? platform_get_irq(pdev, base_irq_idx + i * 2) :
                           platform_get_irq(pdev, base_irq_idx + i * 3 + 1);
                 ring_pair_cb->virq[HNS_RCB_IRQ_IDX_RX] =
-                is_ver1 ? irq_of_parse_and_map(np, base_irq_idx + i * 2 + 1) :
+                is_ver1 ? platform_get_irq(pdev, base_irq_idx + i * 2 + 1) :
                           platform_get_irq(pdev, base_irq_idx + i * 3);
                 ring_pair_cb->q.phy_base =
                         RCB_COMM_BASE_TO_RING_BASE(rcb_common->phy_base, i);
@@ -541,7 +540,7 @@ int hns_rcb_set_coalesce_usecs(
         }
         if (timeout > HNS_RCB_MAX_COALESCED_USECS) {
                 dev_err(rcb_common->dsaf_dev->dev,
-                        "error: not support coalesce %dus!\n", timeout);
+                        "error: coalesce_usecs setting supports 0~1023us\n");
                 return -EINVAL;
         }
         hns_rcb_set_port_timeout(rcb_common, port_idx, timeout);
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.h b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.h
index bd54dac82ee0..99b4e1ba0a94 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.h
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.h
@@ -40,7 +40,7 @@ struct rcb_common_cb;
 #define HNS_RCB_DEF_COALESCED_FRAMES            50
 #define HNS_RCB_CLK_FREQ_MHZ                    350
 #define HNS_RCB_MAX_COALESCED_USECS             0x3ff
-#define HNS_RCB_DEF_COALESCED_USECS             3
+#define HNS_RCB_DEF_COALESCED_USECS             50
 
 #define HNS_RCB_COMMON_ENDIAN                   1
 
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_reg.h b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_reg.h
index 7c3b5103d151..235f74444b1d 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_reg.h
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_reg.h
@@ -32,7 +32,7 @@
 #define DSAFV2_SBM_NUM          8
 #define DSAFV2_SBM_XGE_CHN    6
 #define DSAFV2_SBM_PPE_CHN    1
-#define DASFV2_ROCEE_CRD_NUM  8
+#define DASFV2_ROCEE_CRD_NUM  1
 
 #define DSAF_VOQ_NUM            DSAF_NODE_NUM
 #define DSAF_INODE_NUM          DSAF_NODE_NUM
@@ -166,6 +166,9 @@
 #define DSAF_INODE_GE_FC_EN_0_REG               0x1B00
 #define DSAF_INODE_VC0_IN_PKT_NUM_0_REG         0x1B50
 #define DSAF_INODE_VC1_IN_PKT_NUM_0_REG         0x1C00
+#define DSAF_INODE_IN_PRIO_PAUSE_BASE_REG       0x1C00
+#define DSAF_INODE_IN_PRIO_PAUSE_BASE_OFFSET    0x100
+#define DSAF_INODE_IN_PRIO_PAUSE_OFFSET         0x50
 
 #define DSAF_SBM_CFG_REG_0_REG                  0x2000
 #define DSAF_SBM_BP_CFG_0_XGE_REG_0_REG         0x2004
@@ -175,7 +178,7 @@
 #define DSAF_SBM_BP_CFG_2_XGE_REG_0_REG         0x200C
 #define DSAF_SBM_BP_CFG_2_PPE_REG_0_REG         0x230C
 #define DSAF_SBM_BP_CFG_2_ROCEE_REG_0_REG       0x260C
-#define DSAFV2_SBM_BP_CFG_2_ROCEE_REG_0_REG              0x238C
+#define DSAFV2_SBM_BP_CFG_2_ROCEE_REG_0_REG     0x238C
 #define DSAF_SBM_FREE_CNT_0_0_REG               0x2010
 #define DSAF_SBM_FREE_CNT_1_0_REG               0x2014
 #define DSAF_SBM_BP_CNT_0_0_REG                 0x2018
@@ -232,6 +235,8 @@
 #define DSAF_XOD_ROCEE_RCVIN0_CNT_0_REG         0x3074
 #define DSAF_XOD_ROCEE_RCVIN1_CNT_0_REG         0x3078
 #define DSAF_XOD_FIFO_STATUS_0_REG              0x307C
+#define DSAF_XOD_XGE_PFC_PRIO_CNT_BASE_REG      0x3A00
+#define DSAF_XOD_XGE_PFC_PRIO_CNT_OFFSET        0x4
 
 #define DSAF_VOQ_ECC_INVERT_EN_0_REG            0x4004
 #define DSAF_VOQ_SRAM_PKT_NUM_0_REG             0x4008
@@ -791,6 +796,18 @@
 #define DSAFV2_SBM_CFG4_RESET_BUF_NUM_NO_PFC_S 9
 #define DSAFV2_SBM_CFG4_RESET_BUF_NUM_NO_PFC_M (((1ULL << 9) - 1) << 9)
 
+#define DSAFV2_SBM_CFG2_ROCEE_SET_BUF_NUM_S 0
+#define DSAFV2_SBM_CFG2_ROCEE_SET_BUF_NUM_M (((1ULL << 8) - 1) << 0)
+#define DSAFV2_SBM_CFG2_ROCEE_RESET_BUF_NUM_S 8
+#define DSAFV2_SBM_CFG2_ROCEE_RESET_BUF_NUM_M (((1ULL << 8) - 1) << 8)
+
+#define DSAFV2_SBM_CFG2_PPE_SET_BUF_NUM_S (0)
+#define DSAFV2_SBM_CFG2_PPE_SET_BUF_NUM_M (((1ULL << 6) - 1) << 0)
+#define DSAFV2_SBM_CFG2_PPE_RESET_BUF_NUM_S (6)
+#define DSAFV2_SBM_CFG2_PPE_RESET_BUF_NUM_M (((1ULL << 6) - 1) << 6)
+#define DSAFV2_SBM_CFG2_PPE_CFG_USEFUL_NUM_S (12)
+#define DSAFV2_SBM_CFG2_PPE_CFG_USEFUL_NUM_M (((1ULL << 6) - 1) << 12)
+
 #define DSAF_TBL_TCAM_ADDR_S 0
 #define DSAF_TBL_TCAM_ADDR_M ((1ULL << 9) - 1)
 
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_xgmac.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_xgmac.c
index fd90f3737963..8f4f0e8da984 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_xgmac.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_xgmac.c
@@ -119,7 +119,7 @@ static void hns_xgmac_enable(void *mac_drv, enum mac_commom_mode mode)
                 = (struct dsaf_device *)dev_get_drvdata(drv->dev);
         u32 port = drv->mac_id;
 
-        hns_dsaf_xge_core_srst_by_port(dsaf_dev, port, 1);
+        dsaf_dev->misc_op->xge_core_srst(dsaf_dev, port, 1);
         mdelay(10);
 
         /*enable XGE rX/tX */
@@ -157,7 +157,7 @@ static void hns_xgmac_disable(void *mac_drv, enum mac_commom_mode mode)
         }
 
         mdelay(10);
-        hns_dsaf_xge_core_srst_by_port(dsaf_dev, port, 0);
+        dsaf_dev->misc_op->xge_core_srst(dsaf_dev, port, 0);
 }
 
 /**
@@ -198,9 +198,9 @@ static void hns_xgmac_init(void *mac_drv)
                 = (struct dsaf_device *)dev_get_drvdata(drv->dev);
         u32 port = drv->mac_id;
 
-        hns_dsaf_xge_srst_by_port(dsaf_dev, port, 0);
+        dsaf_dev->misc_op->xge_srst(dsaf_dev, port, 0);
         mdelay(100);
-        hns_dsaf_xge_srst_by_port(dsaf_dev, port, 1);
+        dsaf_dev->misc_op->xge_srst(dsaf_dev, port, 1);
 
         mdelay(100);
         hns_xgmac_exc_irq_en(drv, 0);
@@ -425,7 +425,7 @@ static void hns_xgmac_free(void *mac_drv)
 
         u32 mac_id = drv->mac_id;
 
-        hns_dsaf_xge_srst_by_port(dsaf_dev, mac_id, 0);
+        dsaf_dev->misc_op->xge_srst(dsaf_dev, mac_id, 0);
 }
 
 /**
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_enet.c b/drivers/net/ethernet/hisilicon/hns/hns_enet.c
index e621636e69b9..d5297ecfe4a5 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_enet.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_enet.c
@@ -132,6 +132,13 @@ static void fill_v2_desc(struct hnae_ring *ring, void *priv,
         ring_ptr_move_fw(ring, next_to_use);
 }
 
+static const struct acpi_device_id hns_enet_acpi_match[] = {
+        { "HISI00C1", 0 },
+        { "HISI00C2", 0 },
+        { },
+};
+MODULE_DEVICE_TABLE(acpi, hns_enet_acpi_match);
+
 static void fill_desc(struct hnae_ring *ring, void *priv,
                       int size, dma_addr_t dma, int frag_end,
                       int buf_num, enum hns_desc_type type, int mtu)
@@ -593,6 +600,7 @@ static int hns_nic_poll_rx_skb(struct hns_nic_ring_data *ring_data,
                 ring->stats.sw_err_cnt++;
                 return -ENOMEM;
         }
+        skb_reset_mac_header(skb);
 
         prefetchw(skb->data);
         length = le16_to_cpu(desc->rx.pkt_len);
@@ -760,10 +768,10 @@ recv:
                         clean_count = 0;
                 }
 
-                /* poll one pkg*/
+                /* poll one pkt*/
                 err = hns_nic_poll_rx_skb(ring_data, &skb, &bnum);
                 if (unlikely(!skb)) /* this fault cannot be repaired */
-                        break;
+                        goto out;
 
                 recv_bds += bnum;
                 clean_count += bnum;
@@ -789,6 +797,7 @@ recv:
                 }
         }
 
+out:
         /* make all data has been write before submit */
         if (clean_count > 0)
                 hns_nic_alloc_rx_buffers(ring_data, clean_count);
@@ -983,8 +992,26 @@ static void hns_nic_adjust_link(struct net_device *ndev)
 {
         struct hns_nic_priv *priv = netdev_priv(ndev);
         struct hnae_handle *h = priv->ae_handle;
+        int state = 1;
+
+        if (priv->phy) {
+                h->dev->ops->adjust_link(h, ndev->phydev->speed,
+                                         ndev->phydev->duplex);
+                state = priv->phy->link;
+        }
+        state = state && h->dev->ops->get_status(h);
 
-        h->dev->ops->adjust_link(h, ndev->phydev->speed, ndev->phydev->duplex);
+        if (state != priv->link) {
+                if (state) {
+                        netif_carrier_on(ndev);
+                        netif_tx_wake_all_queues(ndev);
+                        netdev_info(ndev, "link up\n");
+                } else {
+                        netif_carrier_off(ndev);
+                        netdev_info(ndev, "link down\n");
+                }
+                priv->link = state;
+        }
 }
 
 /**
@@ -996,19 +1023,22 @@ static void hns_nic_adjust_link(struct net_device *ndev)
 int hns_nic_init_phy(struct net_device *ndev, struct hnae_handle *h)
 {
         struct hns_nic_priv *priv = netdev_priv(ndev);
-        struct phy_device *phy_dev = NULL;
+        struct phy_device *phy_dev = h->phy_dev;
+        int ret;
 
-        if (!h->phy_node)
+        if (!h->phy_dev)
                 return 0;
 
-        if (h->phy_if != PHY_INTERFACE_MODE_XGMII)
-                phy_dev = of_phy_connect(ndev, h->phy_node,
-                                         hns_nic_adjust_link, 0, h->phy_if);
-        else
-                phy_dev = of_phy_attach(ndev, h->phy_node, 0, h->phy_if);
+        if (h->phy_if != PHY_INTERFACE_MODE_XGMII) {
+                phy_dev->dev_flags = 0;
 
-        if (unlikely(!phy_dev) || IS_ERR(phy_dev))
-                return !phy_dev ? -ENODEV : PTR_ERR(phy_dev);
+                ret = phy_connect_direct(ndev, phy_dev, hns_nic_adjust_link,
+                                         h->phy_if);
+        } else {
+                ret = phy_attach_direct(ndev, phy_dev, 0, h->phy_if);
+        }
+        if (unlikely(ret))
+                return -ENODEV;
 
         phy_dev->supported &= h->if_support;
         phy_dev->advertising = phy_dev->supported;
@@ -1067,13 +1097,8 @@ void hns_nic_update_stats(struct net_device *netdev)
 static void hns_init_mac_addr(struct net_device *ndev)
 {
         struct hns_nic_priv *priv = netdev_priv(ndev);
-        struct device_node *node = priv->dev->of_node;
-        const void *mac_addr_temp;
 
-        mac_addr_temp = of_get_mac_address(node);
-        if (mac_addr_temp && is_valid_ether_addr(mac_addr_temp)) {
-                memcpy(ndev->dev_addr, mac_addr_temp, ndev->addr_len);
-        } else {
+        if (!device_get_mac_address(priv->dev, ndev->dev_addr, ETH_ALEN)) {
                 eth_hw_addr_random(ndev);
                 dev_warn(priv->dev, "No valid mac, use random mac %pM",
                          ndev->dev_addr);
@@ -1176,7 +1201,7 @@ static int hns_nic_net_up(struct net_device *ndev)
 {
         struct hns_nic_priv *priv = netdev_priv(ndev);
         struct hnae_handle *h = priv->ae_handle;
-        int i, j, k;
+        int i, j;
         int ret;
 
         ret = hns_nic_init_irq(priv);
@@ -1191,9 +1216,6 @@ static int hns_nic_net_up(struct net_device *ndev)
                         goto out_has_some_queues;
         }
 
-        for (k = 0; k < h->q_num; k++)
-                h->dev->ops->toggle_queue_status(h->qs[k], 1);
-
         ret = h->dev->ops->set_mac_addr(h, ndev->dev_addr);
         if (ret)
                 goto out_set_mac_addr_err;
@@ -1213,8 +1235,6 @@ static int hns_nic_net_up(struct net_device *ndev)
 out_start_err:
         netif_stop_queue(ndev);
 out_set_mac_addr_err:
-        for (k = 0; k < h->q_num; k++)
-                h->dev->ops->toggle_queue_status(h->qs[k], 0);
 out_has_some_queues:
         for (j = i - 1; j >= 0; j--)
                 hns_nic_ring_close(ndev, j);
@@ -1421,7 +1441,6 @@ static int hns_nic_set_features(struct net_device *netdev,
                                 netdev_features_t features)
 {
         struct hns_nic_priv *priv = netdev_priv(netdev);
-        struct hnae_handle *h = priv->ae_handle;
 
         switch (priv->enet_ver) {
         case AE_VERSION_1:
@@ -1434,11 +1453,9 @@ static int hns_nic_set_features(struct net_device *netdev,
                         priv->ops.maybe_stop_tx = hns_nic_maybe_stop_tso;
                         /* The chip only support 7*4096 */
                         netif_set_gso_max_size(netdev, 7 * 4096);
-                        h->dev->ops->set_tso_stats(h, 1);
                 } else {
                         priv->ops.fill_desc = fill_v2_desc;
                         priv->ops.maybe_stop_tx = hns_nic_maybe_stop_tx;
-                        h->dev->ops->set_tso_stats(h, 0);
                 }
                 break;
         }
@@ -1571,27 +1588,14 @@ static void hns_nic_update_link_status(struct net_device *netdev)
         struct hns_nic_priv *priv = netdev_priv(netdev);
 
         struct hnae_handle *h = priv->ae_handle;
-        int state = 1;
 
-        if (priv->phy) {
-                if (!genphy_update_link(priv->phy))
-                        state = priv->phy->link;
-                else
-                        state = 0;
-        }
-        state = state && h->dev->ops->get_status(h);
+        if (h->phy_dev) {
+                if (h->phy_if != PHY_INTERFACE_MODE_XGMII)
+                        return;
 
-        if (state != priv->link) {
-                if (state) {
-                        netif_carrier_on(netdev);
-                        netif_tx_wake_all_queues(netdev);
-                        netdev_info(netdev, "link up\n");
-                } else {
-                        netif_carrier_off(netdev);
-                        netdev_info(netdev, "link down\n");
-                }
-                priv->link = state;
+                (void)genphy_read_status(h->phy_dev);
         }
+        hns_nic_adjust_link(netdev);
 }
 
 /* for dumping key regs*/
@@ -1627,7 +1631,7 @@ static void hns_nic_dump(struct hns_nic_priv *priv)
         }
 }
 
-/* for resetting suntask*/
+/* for resetting subtask */
 static void hns_nic_reset_subtask(struct hns_nic_priv *priv)
 {
         enum hnae_port_type type = priv->ae_handle->port_type;
@@ -1797,11 +1801,14 @@ static void hns_nic_set_priv_ops(struct net_device *netdev)
                         priv->ops.maybe_stop_tx = hns_nic_maybe_stop_tso;
                         /* This chip only support 7*4096 */
                         netif_set_gso_max_size(netdev, 7 * 4096);
-                        h->dev->ops->set_tso_stats(h, 1);
                 } else {
                         priv->ops.fill_desc = fill_v2_desc;
                         priv->ops.maybe_stop_tx = hns_nic_maybe_stop_tx;
                 }
+                /* enable tso when init
+                 * control tso on/off through TSE bit in bd
+                 */
+                h->dev->ops->set_tso_stats(h, 1);
         }
 }
 
@@ -1812,7 +1819,7 @@ static int hns_nic_try_get_ae(struct net_device *ndev)
         int ret;
 
         h = hnae_get_handle(&priv->netdev->dev,
-                            priv->ae_node, priv->port_id, NULL);
+                            priv->fwnode, priv->port_id, NULL);
         if (IS_ERR_OR_NULL(h)) {
                 ret = -ENODEV;
                 dev_dbg(priv->dev, "has not handle, register notifier!\n");
@@ -1872,7 +1879,6 @@ static int hns_nic_dev_probe(struct platform_device *pdev)
         struct device *dev = &pdev->dev;
         struct net_device *ndev;
         struct hns_nic_priv *priv;
-        struct device_node *node = dev->of_node;
         u32 port_id;
         int ret;
 
@@ -1886,22 +1892,49 @@ static int hns_nic_dev_probe(struct platform_device *pdev)
         priv->dev = dev;
         priv->netdev = ndev;
 
-        if (of_device_is_compatible(node, "hisilicon,hns-nic-v1"))
-                priv->enet_ver = AE_VERSION_1;
-        else
-                priv->enet_ver = AE_VERSION_2;
+        if (dev_of_node(dev)) {
+                struct device_node *ae_node;
 
-        priv->ae_node = (void *)of_parse_phandle(node, "ae-handle", 0);
-        if (IS_ERR_OR_NULL(priv->ae_node)) {
-                ret = PTR_ERR(priv->ae_node);
-                dev_err(dev, "not find ae-handle\n");
-                goto out_read_prop_fail;
+                if (of_device_is_compatible(dev->of_node,
+                                            "hisilicon,hns-nic-v1"))
+                        priv->enet_ver = AE_VERSION_1;
+                else
+                        priv->enet_ver = AE_VERSION_2;
+
+                ae_node = of_parse_phandle(dev->of_node, "ae-handle", 0);
+                if (IS_ERR_OR_NULL(ae_node)) {
+                        ret = PTR_ERR(ae_node);
+                        dev_err(dev, "not find ae-handle\n");
+                        goto out_read_prop_fail;
+                }
+                priv->fwnode = &ae_node->fwnode;
+        } else if (is_acpi_node(dev->fwnode)) {
+                struct acpi_reference_args args;
+
+                if (acpi_dev_found(hns_enet_acpi_match[0].id))
+                        priv->enet_ver = AE_VERSION_1;
+                else if (acpi_dev_found(hns_enet_acpi_match[1].id))
+                        priv->enet_ver = AE_VERSION_2;
+                else
+                        return -ENXIO;
+
+                /* try to find port-idx-in-ae first */
+                ret = acpi_node_get_property_reference(dev->fwnode,
+                                                       "ae-handle", 0, &args);
+                if (ret) {
+                        dev_err(dev, "not find ae-handle\n");
+                        goto out_read_prop_fail;
+                }
+                priv->fwnode = acpi_fwnode_handle(args.adev);
+        } else {
+                dev_err(dev, "cannot read cfg data from OF or acpi\n");
+                return -ENXIO;
         }
-        /* try to find port-idx-in-ae first */
-        ret = of_property_read_u32(node, "port-idx-in-ae", &port_id);
+
+        ret = device_property_read_u32(dev, "port-idx-in-ae", &port_id);
         if (ret) {
                 /* only for old code compatible */
-                ret = of_property_read_u32(node, "port-id", &port_id);
+                ret = device_property_read_u32(dev, "port-id", &port_id);
                 if (ret)
                         goto out_read_prop_fail;
                 /* for old dts, we need to caculate the port offset */
@@ -1940,7 +1973,7 @@ static int hns_nic_dev_probe(struct platform_device *pdev)
         if (!dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)))
                 dev_dbg(dev, "set mask to 64bit\n");
         else
-                dev_err(dev, "set mask to 32bit fail!\n");
+                dev_err(dev, "set mask to 64bit fail!\n");
 
         /* carrier off reporting is important to ethtool even BEFORE open */
         netif_carrier_off(ndev);
@@ -2014,6 +2047,7 @@ static struct platform_driver hns_nic_dev_driver = {
         .driver = {
                 .name = "hns-nic",
                 .of_match_table = hns_enet_of_match,
+                .acpi_match_table = ACPI_PTR(hns_enet_acpi_match),
         },
         .probe = hns_nic_dev_probe,
         .remove = hns_nic_dev_remove,
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_enet.h b/drivers/net/ethernet/hisilicon/hns/hns_enet.h
index 337efa582bac..44bb3015eed3 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_enet.h
+++ b/drivers/net/ethernet/hisilicon/hns/hns_enet.h
@@ -54,7 +54,7 @@ struct hns_nic_ops {
 };
 
 struct hns_nic_priv {
-        const struct device_node *ae_node;
+        const struct fwnode_handle      *fwnode;
         u32 enet_ver;
         u32 port_id;
         int phy_mode;
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c b/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c
index 67a648c7d3a9..a395ca1405c3 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c
@@ -49,7 +49,7 @@ static u32 hns_nic_get_link(struct net_device *net_dev)
         h = priv->ae_handle;
 
         if (priv->phy) {
-                if (!genphy_update_link(priv->phy))
+                if (!genphy_read_status(priv->phy))
                         link_stat = priv->phy->link;
                 else
                         link_stat = 0;
@@ -242,6 +242,7 @@ static const char hns_nic_test_strs[][ETH_GSTRING_LEN] = {
 static int hns_nic_config_phy_loopback(struct phy_device *phy_dev, u8 en)
 {
 #define COPPER_CONTROL_REG 0
+#define PHY_POWER_DOWN BIT(11)
 #define PHY_LOOP_BACK BIT(14)
         u16 val = 0;
 
@@ -252,33 +253,40 @@ static int hns_nic_config_phy_loopback(struct phy_device *phy_dev, u8 en)
                 /* speed : 1000M */
                 phy_write(phy_dev, HNS_PHY_PAGE_REG, 2);
                 phy_write(phy_dev, 21, 0x1046);
+
+                phy_write(phy_dev, HNS_PHY_PAGE_REG, 0);
                 /* Force Master */
                 phy_write(phy_dev, 9, 0x1F00);
+
                 /* Soft-reset */
                 phy_write(phy_dev, 0, 0x9140);
                 /* If autoneg disabled,two soft-reset operations */
                 phy_write(phy_dev, 0, 0x9140);
-                phy_write(phy_dev, 22, 0xFA);
+
+                phy_write(phy_dev, HNS_PHY_PAGE_REG, 0xFA);
 
                 /* Default is 0x0400 */
                 phy_write(phy_dev, 1, 0x418);
 
                 /* Force 1000M Link, Default is 0x0200 */
                 phy_write(phy_dev, 7, 0x20C);
-                phy_write(phy_dev, 22, 0);
+                phy_write(phy_dev, HNS_PHY_PAGE_REG, 0);
 
-                /* Enable MAC loop-back */
+                /* Enable PHY loop-back */
                 val = phy_read(phy_dev, COPPER_CONTROL_REG);
                 val |= PHY_LOOP_BACK;
+                val &= ~PHY_POWER_DOWN;
                 phy_write(phy_dev, COPPER_CONTROL_REG, val);
         } else {
-                phy_write(phy_dev, 22, 0xFA);
+                phy_write(phy_dev, HNS_PHY_PAGE_REG, 0xFA);
                 phy_write(phy_dev, 1, 0x400);
                 phy_write(phy_dev, 7, 0x200);
-                phy_write(phy_dev, 22, 0);
+                phy_write(phy_dev, HNS_PHY_PAGE_REG, 0);
+                phy_write(phy_dev, 9, 0xF00);
 
                 val = phy_read(phy_dev, COPPER_CONTROL_REG);
                 val &= ~PHY_LOOP_BACK;
+                val |= PHY_POWER_DOWN;
                 phy_write(phy_dev, COPPER_CONTROL_REG, val);
         }
         return 0;
@@ -339,28 +347,16 @@ static int __lb_up(struct net_device *ndev,
 
         hns_nic_net_reset(ndev);
 
-        if (priv->phy) {
-                phy_disconnect(priv->phy);
-                msleep(100);
-
-                ret = hns_nic_init_phy(ndev, h);
-                if (ret)
-                        return ret;
-        }
-
         ret = __lb_setup(ndev, loop_mode);
         if (ret)
                 return ret;
 
-        msleep(100);
+        msleep(200);
 
         ret = h->dev->ops->start ? h->dev->ops->start(h) : 0;
         if (ret)
                 return ret;
 
-        if (priv->phy)
-                phy_start(priv->phy);
-
         /* link adjust duplex*/
         if (priv->ae_handle->phy_if != PHY_INTERFACE_MODE_XGMII)
                 speed = 1000;
@@ -561,9 +557,6 @@ static int __lb_down(struct net_device *ndev)
                            __func__,
                            ret);
 
-        if (priv->phy)
-                phy_stop(priv->phy);
-
         if (h->dev->ops->stop)
                 h->dev->ops->stop(h);
 
@@ -596,7 +589,7 @@ static void hns_nic_self_test(struct net_device *ndev,
         st_param[1][0] = MAC_INTERNALLOOP_SERDES;
         st_param[1][1] = 1; /*serdes must exist*/
         st_param[2][0] = MAC_INTERNALLOOP_PHY; /* only supporte phy node*/
-        st_param[2][1] = ((!!(priv->ae_handle->phy_node)) &&
+        st_param[2][1] = ((!!(priv->ae_handle->phy_dev)) &&
                 (priv->ae_handle->phy_if != PHY_INTERFACE_MODE_XGMII));
 
         if (eth_test->flags == ETH_TEST_FL_OFFLINE) {
@@ -758,6 +751,16 @@ static int hns_get_coalesce(struct net_device *net_dev,
                 &ec->tx_max_coalesced_frames,
                 &ec->rx_max_coalesced_frames);
 
+        ops->get_coalesce_range(priv->ae_handle,
+                                &ec->tx_max_coalesced_frames_low,
+                                &ec->rx_max_coalesced_frames_low,
+                                &ec->tx_max_coalesced_frames_high,
+                                &ec->rx_max_coalesced_frames_high,
+                                &ec->tx_coalesce_usecs_low,
+                                &ec->rx_coalesce_usecs_low,
+                                &ec->tx_coalesce_usecs_high,
+                                &ec->rx_coalesce_usecs_high);
+
         return 0;
 }
 
diff --git a/drivers/net/ethernet/hisilicon/hns_mdio.c b/drivers/net/ethernet/hisilicon/hns_mdio.c
index 765ddb3dcd1a..761a32fceceb 100644
--- a/drivers/net/ethernet/hisilicon/hns_mdio.c
+++ b/drivers/net/ethernet/hisilicon/hns_mdio.c
@@ -7,6 +7,7 @@
  * (at your option) any later version.
  */
 
+#include <linux/acpi.h>
 #include <linux/errno.h>
 #include <linux/etherdevice.h>
 #include <linux/init.h>
@@ -354,67 +355,64 @@ static int hns_mdio_reset(struct mii_bus *bus)
         struct hns_mdio_device *mdio_dev = (struct hns_mdio_device *)bus->priv;
         int ret;
 
-        if (!mdio_dev->subctrl_vbase) {
-                dev_err(&bus->dev, "mdio sys ctl reg has not maped\n");
-                return -ENODEV;
-        }
-
-        /*1. reset req, and read reset st check*/
-        ret = mdio_sc_cfg_reg_write(mdio_dev, MDIO_SC_RESET_REQ, 0x1,
-                                    MDIO_SC_RESET_ST, 0x1,
-                                    MDIO_CHECK_SET_ST);
-        if (ret) {
-                dev_err(&bus->dev, "MDIO reset fail\n");
-                return ret;
-        }
+        if (dev_of_node(bus->parent)) {
+                if (!mdio_dev->subctrl_vbase) {
+                        dev_err(&bus->dev, "mdio sys ctl reg has not maped\n");
+                        return -ENODEV;
+                }
 
-        /*2. dis clk, and read clk st check*/
-        ret = mdio_sc_cfg_reg_write(mdio_dev, MDIO_SC_CLK_DIS,
-                                    0x1, MDIO_SC_CLK_ST, 0x1,
-                                    MDIO_CHECK_CLR_ST);
-        if (ret) {
-                dev_err(&bus->dev, "MDIO dis clk fail\n");
-                return ret;
-        }
+                /* 1. reset req, and read reset st check */
+                ret = mdio_sc_cfg_reg_write(mdio_dev, MDIO_SC_RESET_REQ, 0x1,
+                                            MDIO_SC_RESET_ST, 0x1,
+                                            MDIO_CHECK_SET_ST);
+                if (ret) {
+                        dev_err(&bus->dev, "MDIO reset fail\n");
+                        return ret;
+                }
 
-        /*3. reset dreq, and read reset st check*/
-        ret = mdio_sc_cfg_reg_write(mdio_dev, MDIO_SC_RESET_DREQ, 0x1,
-                                    MDIO_SC_RESET_ST, 0x1,
-                                    MDIO_CHECK_CLR_ST);
-        if (ret) {
-                dev_err(&bus->dev, "MDIO dis clk fail\n");
-                return ret;
-        }
+                /* 2. dis clk, and read clk st check */
+                ret = mdio_sc_cfg_reg_write(mdio_dev, MDIO_SC_CLK_DIS,
+                                            0x1, MDIO_SC_CLK_ST, 0x1,
+                                            MDIO_CHECK_CLR_ST);
+                if (ret) {
+                        dev_err(&bus->dev, "MDIO dis clk fail\n");
+                        return ret;
+                }
 
-        /*4. en clk, and read clk st check*/
-        ret = mdio_sc_cfg_reg_write(mdio_dev, MDIO_SC_CLK_EN,
-                                    0x1, MDIO_SC_CLK_ST, 0x1,
-                                    MDIO_CHECK_SET_ST);
-        if (ret)
-                dev_err(&bus->dev, "MDIO en clk fail\n");
+                /* 3. reset dreq, and read reset st check */
+                ret = mdio_sc_cfg_reg_write(mdio_dev, MDIO_SC_RESET_DREQ, 0x1,
+                                            MDIO_SC_RESET_ST, 0x1,
+                                            MDIO_CHECK_CLR_ST);
+                if (ret) {
+                        dev_err(&bus->dev, "MDIO dis clk fail\n");
+                        return ret;
+                }
 
+                /* 4. en clk, and read clk st check */
+                ret = mdio_sc_cfg_reg_write(mdio_dev, MDIO_SC_CLK_EN,
+                                            0x1, MDIO_SC_CLK_ST, 0x1,
+                                            MDIO_CHECK_SET_ST);
+                if (ret)
+                        dev_err(&bus->dev, "MDIO en clk fail\n");
+        } else if (is_acpi_node(bus->parent->fwnode)) {
+                acpi_status s;
+
+                s = acpi_evaluate_object(ACPI_HANDLE(bus->parent),
+                                         "_RST", NULL, NULL);
+                if (ACPI_FAILURE(s)) {
+                        dev_err(&bus->dev, "Reset failed, return:%#x\n", s);
+                        ret = -EBUSY;
+                } else {
+                        ret = 0;
+                }
+        } else {
+                dev_err(&bus->dev, "Can not get cfg data from DT or ACPI\n");
+                ret = -ENXIO;
+        }
         return ret;
 }
 
 /**
- * hns_mdio_bus_name - get mdio bus name
- * @name: mdio bus name
- * @np: mdio device node pointer
- */
-static void hns_mdio_bus_name(char *name, struct device_node *np)
-{
-        const u32 *addr;
-        u64 taddr = OF_BAD_ADDR;
-
-        addr = of_get_address(np, 0, NULL, NULL);
-        if (addr)
-                taddr = of_translate_address(np, addr);
-
-        snprintf(name, MII_BUS_ID_SIZE, "%s@%llx", np->name,
-                 (unsigned long long)taddr);
-}
-
-/**
  * hns_mdio_probe - probe mdio device
  * @pdev: mdio platform device
  *
@@ -422,17 +420,16 @@ static void hns_mdio_bus_name(char *name, struct device_node *np)
  */
 static int hns_mdio_probe(struct platform_device *pdev)
 {
-        struct device_node *np;
         struct hns_mdio_device *mdio_dev;
         struct mii_bus *new_bus;
         struct resource *res;
-        int ret;
+        int ret = -ENODEV;
 
         if (!pdev) {
                 dev_err(NULL, "pdev is NULL!\r\n");
                 return -ENODEV;
         }
-        np = pdev->dev.of_node;
+
         mdio_dev = devm_kzalloc(&pdev->dev, sizeof(*mdio_dev), GFP_KERNEL);
         if (!mdio_dev)
                 return -ENOMEM;
@@ -448,7 +445,7 @@ static int hns_mdio_probe(struct platform_device *pdev)
         new_bus->write = hns_mdio_write;
         new_bus->reset = hns_mdio_reset;
         new_bus->priv = mdio_dev;
-        hns_mdio_bus_name(new_bus->id, np);
+        new_bus->parent = &pdev->dev;
 
         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
         mdio_dev->vbase = devm_ioremap_resource(&pdev->dev, res);
@@ -457,16 +454,32 @@ static int hns_mdio_probe(struct platform_device *pdev)
                 return ret;
         }
 
-        mdio_dev->subctrl_vbase =
-                syscon_node_to_regmap(of_parse_phandle(np, "subctrl-vbase", 0));
-        if (IS_ERR(mdio_dev->subctrl_vbase)) {
-                dev_warn(&pdev->dev, "no syscon hisilicon,peri-c-subctrl\n");
-                mdio_dev->subctrl_vbase = NULL;
-        }
-        new_bus->parent = &pdev->dev;
         platform_set_drvdata(pdev, new_bus);
+        snprintf(new_bus->id, MII_BUS_ID_SIZE, "%s-%s", "Mii",
+                 dev_name(&pdev->dev));
+        if (dev_of_node(&pdev->dev)) {
+                mdio_dev->subctrl_vbase = syscon_node_to_regmap(
+                        of_parse_phandle(pdev->dev.of_node,
+                                         "subctrl-vbase", 0));
+                if (IS_ERR(mdio_dev->subctrl_vbase)) {
+                        dev_warn(&pdev->dev, "no syscon hisilicon,peri-c-subctrl\n");
+                        mdio_dev->subctrl_vbase = NULL;
+                }
+                ret = of_mdiobus_register(new_bus, pdev->dev.of_node);
+        } else if (is_acpi_node(pdev->dev.fwnode)) {
+                /* Clear all the IRQ properties */
+                memset(new_bus->irq, PHY_POLL, 4 * PHY_MAX_ADDR);
+
+                /* Mask out all PHYs from auto probing. */
+                new_bus->phy_mask = ~0;
+
+                /* Register the MDIO bus */
+                ret = mdiobus_register(new_bus);
+        } else {
+                dev_err(&pdev->dev, "Can not get cfg data from DT or ACPI\n");
+                ret = -ENXIO;
+        }
 
-        ret = of_mdiobus_register(new_bus, np);
         if (ret) {
                 dev_err(&pdev->dev, "Cannot register as MDIO bus!\n");
                 platform_set_drvdata(pdev, NULL);
@@ -499,12 +512,19 @@ static const struct of_device_id hns_mdio_match[] = {
         {}
 };
 
+static const struct acpi_device_id hns_mdio_acpi_match[] = {
+        { "HISI0141", 0 },
+        { },
+};
+MODULE_DEVICE_TABLE(acpi, hns_mdio_acpi_match);
+
 static struct platform_driver hns_mdio_driver = {
         .probe = hns_mdio_probe,
         .remove = hns_mdio_remove,
         .driver = {
                    .name = MDIO_DRV_NAME,
                    .of_match_table = hns_mdio_match,
+                   .acpi_match_table = ACPI_PTR(hns_mdio_acpi_match),
                    },
 };
 
diff --git a/drivers/net/ethernet/intel/Kconfig b/drivers/net/ethernet/intel/Kconfig
index 714bd1014ddb..c0e17433f623 100644
--- a/drivers/net/ethernet/intel/Kconfig
+++ b/drivers/net/ethernet/intel/Kconfig
@@ -167,17 +167,6 @@ config IXGBE
           To compile this driver as a module, choose M here. The module
           will be called ixgbe.
 
-config IXGBE_VXLAN
-        bool "Virtual eXtensible Local Area Network Support"
-        default n
-        depends on IXGBE && VXLAN && !(IXGBE=y && VXLAN=m)
-        ---help---
-          This allows one to create VXLAN virtual interfaces that provide
-          Layer 2 Networks over Layer 3 Networks. VXLAN is often used
-          to tunnel virtual network infrastructure in virtualized environments.
-          Say Y here if you want to use Virtual eXtensible Local Area Network
-          (VXLAN) in the driver.
-
 config IXGBE_HWMON
         bool "Intel(R) 10GbE PCI Express adapters HWMON support"
         default y
@@ -236,27 +225,6 @@ config I40E
           To compile this driver as a module, choose M here. The module
           will be called i40e.
 
-config I40E_VXLAN
-        bool "Virtual eXtensible Local Area Network Support"
-        default n
-        depends on I40E && VXLAN && !(I40E=y && VXLAN=m)
-        ---help---
-          This allows one to create VXLAN virtual interfaces that provide
-          Layer 2 Networks over Layer 3 Networks. VXLAN is often used
-          to tunnel virtual network infrastructure in virtualized environments.
-          Say Y here if you want to use Virtual eXtensible Local Area Network
-          (VXLAN) in the driver.
-
-config I40E_GENEVE
-        bool "Generic Network Virtualization Encapsulation (GENEVE) Support"
-        depends on I40E && GENEVE && !(I40E=y && GENEVE=m)
-        default n
-        ---help---
-          This allows one to create GENEVE virtual interfaces that provide
-          Layer 2 Networks over Layer 3 Networks. GENEVE is often used
-          to tunnel virtual network infrastructure in virtualized environments.
-          Say Y here if you want to use GENEVE in the driver.
-
 config I40E_DCB
         bool "Data Center Bridging (DCB) Support"
         default n
@@ -307,15 +275,4 @@ config FM10K
           To compile this driver as a module, choose M here. The module
           will be called fm10k.  MSI-X interrupt support is required
 
-config FM10K_VXLAN
-        bool "Virtual eXtensible Local Area Network Support"
-        default n
-        depends on FM10K && VXLAN && !(FM10K=y && VXLAN=m)
-        ---help---
-          This allows one to create VXLAN virtual interfaces that provide
-          Layer 2 Networks over Layer 3 Networks. VXLAN is often used
-          to tunnel virtual network infrastructure in virtualized environments.
-          Say Y here if you want to use Virtual eXtensible Local Area Network
-          (VXLAN) in the driver.
-
 endif # NET_VENDOR_INTEL
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c b/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c
index 2a08d3f5b6df..d00cb193da9a 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c
@@ -20,9 +20,7 @@
 
 #include "fm10k.h"
 #include <linux/vmalloc.h>
-#ifdef CONFIG_FM10K_VXLAN
-#include <net/vxlan.h>
-#endif /* CONFIG_FM10K_VXLAN */
+#include <net/udp_tunnel.h>
 
 /**
  * fm10k_setup_tx_resources - allocate Tx resources (Descriptors)
@@ -436,6 +434,7 @@ static void fm10k_restore_vxlan_port(struct fm10k_intfc *interface)
  * @netdev: network interface device structure
  * @sa_family: Address family of new port
  * @port: port number used for VXLAN
+ * @type: Enumerated value specifying udp encapsulation type
  *
  * This function is called when a new VXLAN interface has added a new port
  * number to the range that is currently in use for VXLAN.  The new port
@@ -444,18 +443,21 @@ static void fm10k_restore_vxlan_port(struct fm10k_intfc *interface)
  * is always used as the VXLAN port number for offloads.
  **/
 static void fm10k_add_vxlan_port(struct net_device *dev,
-                                 sa_family_t sa_family, __be16 port) {
+                                 struct udp_tunnel_info *ti)
+{
         struct fm10k_intfc *interface = netdev_priv(dev);
         struct fm10k_vxlan_port *vxlan_port;
 
+        if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
+                return;
         /* only the PF supports configuring tunnels */
         if (interface->hw.mac.type != fm10k_mac_pf)
                 return;
 
         /* existing ports are pulled out so our new entry is always last */
         fm10k_vxlan_port_for_each(vxlan_port, interface) {
-                if ((vxlan_port->port == port) &&
-                    (vxlan_port->sa_family == sa_family)) {
+                if ((vxlan_port->port == ti->port) &&
+                    (vxlan_port->sa_family == ti->sa_family)) {
                         list_del(&vxlan_port->list);
                         goto insert_tail;
                 }
@@ -465,8 +467,8 @@ static void fm10k_add_vxlan_port(struct net_device *dev,
         vxlan_port = kmalloc(sizeof(*vxlan_port), GFP_ATOMIC);
         if (!vxlan_port)
                 return;
-        vxlan_port->port = port;
-        vxlan_port->sa_family = sa_family;
+        vxlan_port->port = ti->port;
+        vxlan_port->sa_family = ti->sa_family;
 
 insert_tail:
         /* add new port value to list */
@@ -480,6 +482,7 @@ insert_tail:
  * @netdev: network interface device structure
  * @sa_family: Address family of freed port
  * @port: port number used for VXLAN
+ * @type: Enumerated value specifying udp encapsulation type
  *
  * This function is called when a new VXLAN interface has freed a port
  * number from the range that is currently in use for VXLAN.  The freed
@@ -487,17 +490,20 @@ insert_tail:
  * the port number for offloads.
  **/
 static void fm10k_del_vxlan_port(struct net_device *dev,
-                                 sa_family_t sa_family, __be16 port) {
+                                 struct udp_tunnel_info *ti)
+{
         struct fm10k_intfc *interface = netdev_priv(dev);
         struct fm10k_vxlan_port *vxlan_port;
 
+        if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
+                return;
         if (interface->hw.mac.type != fm10k_mac_pf)
                 return;
 
         /* find the port in the list and free it */
         fm10k_vxlan_port_for_each(vxlan_port, interface) {
-                if ((vxlan_port->port == port) &&
-                    (vxlan_port->sa_family == sa_family)) {
+                if ((vxlan_port->port == ti->port) &&
+                    (vxlan_port->sa_family == ti->sa_family)) {
                         list_del(&vxlan_port->list);
                         kfree(vxlan_port);
                         break;
@@ -553,10 +559,8 @@ int fm10k_open(struct net_device *netdev)
         if (err)
                 goto err_set_queues;
 
-#ifdef CONFIG_FM10K_VXLAN
         /* update VXLAN port configuration */
-        vxlan_get_rx_port(netdev);
-#endif
+        udp_tunnel_get_rx_info(netdev);
 
         fm10k_up(interface);
 
@@ -1375,8 +1379,8 @@ static const struct net_device_ops fm10k_netdev_ops = {
         .ndo_set_vf_vlan        = fm10k_ndo_set_vf_vlan,
         .ndo_set_vf_rate        = fm10k_ndo_set_vf_bw,
         .ndo_get_vf_config      = fm10k_ndo_get_vf_config,
-        .ndo_add_vxlan_port     = fm10k_add_vxlan_port,
-        .ndo_del_vxlan_port     = fm10k_del_vxlan_port,
+        .ndo_udp_tunnel_add     = fm10k_add_vxlan_port,
+        .ndo_udp_tunnel_del     = fm10k_del_vxlan_port,
         .ndo_dfwd_add_station   = fm10k_dfwd_add_station,
         .ndo_dfwd_del_station   = fm10k_dfwd_del_station,
 #ifdef CONFIG_NET_POLL_CONTROLLER
diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h
index 9c44739da5e2..e83fc8afb30f 100644
--- a/drivers/net/ethernet/intel/i40e/i40e.h
+++ b/drivers/net/ethernet/intel/i40e/i40e.h
@@ -283,6 +283,7 @@ struct i40e_pf {
 #endif /* I40E_FCOE */
         u16 num_lan_qps;           /* num lan queues this PF has set up */
         u16 num_lan_msix;          /* num queue vectors for the base PF vsi */
+        u16 num_fdsb_msix;         /* num queue vectors for sideband Fdir */
         u16 num_iwarp_msix;        /* num of iwarp vectors for this PF */
         int iwarp_base_vector;
         int queues_left;           /* queues left unclaimed */
diff --git a/drivers/net/ethernet/intel/i40e/i40e_common.c b/drivers/net/ethernet/intel/i40e/i40e_common.c
index 422b41d61c9a..e447dc435464 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_common.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_common.c
@@ -1967,6 +1967,62 @@ aq_add_vsi_exit:
 }
 
 /**
+ * i40e_aq_set_default_vsi
+ * @hw: pointer to the hw struct
+ * @seid: vsi number
+ * @cmd_details: pointer to command details structure or NULL
+ **/
+i40e_status i40e_aq_set_default_vsi(struct i40e_hw *hw,
+                                    u16 seid,
+                                    struct i40e_asq_cmd_details *cmd_details)
+{
+        struct i40e_aq_desc desc;
+        struct i40e_aqc_set_vsi_promiscuous_modes *cmd =
+                (struct i40e_aqc_set_vsi_promiscuous_modes *)
+                &desc.params.raw;
+        i40e_status status;
+
+        i40e_fill_default_direct_cmd_desc(&desc,
+                                          i40e_aqc_opc_set_vsi_promiscuous_modes);
+
+        cmd->promiscuous_flags = cpu_to_le16(I40E_AQC_SET_VSI_DEFAULT);
+        cmd->valid_flags = cpu_to_le16(I40E_AQC_SET_VSI_DEFAULT);
+        cmd->seid = cpu_to_le16(seid);
+
+        status = i40e_asq_send_command(hw, &desc, NULL, 0, cmd_details);
+
+        return status;
+}
+
+/**
+ * i40e_aq_clear_default_vsi
+ * @hw: pointer to the hw struct
+ * @seid: vsi number
+ * @cmd_details: pointer to command details structure or NULL
+ **/
+i40e_status i40e_aq_clear_default_vsi(struct i40e_hw *hw,
+                                      u16 seid,
+                                      struct i40e_asq_cmd_details *cmd_details)
+{
+        struct i40e_aq_desc desc;
+        struct i40e_aqc_set_vsi_promiscuous_modes *cmd =
+                (struct i40e_aqc_set_vsi_promiscuous_modes *)
+                &desc.params.raw;
+        i40e_status status;
+
+        i40e_fill_default_direct_cmd_desc(&desc,
+                                          i40e_aqc_opc_set_vsi_promiscuous_modes);
+
+        cmd->promiscuous_flags = cpu_to_le16(0);
+        cmd->valid_flags = cpu_to_le16(I40E_AQC_SET_VSI_DEFAULT);
+        cmd->seid = cpu_to_le16(seid);
+
+        status = i40e_asq_send_command(hw, &desc, NULL, 0, cmd_details);
+
+        return status;
+}
+
+/**
  * i40e_aq_set_vsi_unicast_promiscuous
  * @hw: pointer to the hw struct
  * @seid: vsi number
diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
index 5e8d84ff7d5f..4962e855fbd3 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
@@ -313,8 +313,7 @@ static void i40e_phy_type_to_ethtool(struct i40e_pf *pf, u32 *supported,
                 *advertising |= ADVERTISED_Autoneg |
                                 ADVERTISED_40000baseCR4_Full;
         }
-        if ((phy_types & I40E_CAP_PHY_TYPE_100BASE_TX) &&
-            !(phy_types & I40E_CAP_PHY_TYPE_1000BASE_T)) {
+        if (phy_types & I40E_CAP_PHY_TYPE_100BASE_TX) {
                 *supported |= SUPPORTED_Autoneg |
                               SUPPORTED_100baseT_Full;
                 *advertising |= ADVERTISED_Autoneg |
@@ -663,6 +662,7 @@ static int i40e_set_settings(struct net_device *netdev,
         if (hw->phy.media_type != I40E_MEDIA_TYPE_BASET &&
             hw->phy.media_type != I40E_MEDIA_TYPE_FIBER &&
             hw->phy.media_type != I40E_MEDIA_TYPE_BACKPLANE &&
+            hw->phy.media_type != I40E_MEDIA_TYPE_DA &&
             hw->phy.link_info.link_info & I40E_AQ_LINK_UP)
                 return -EOPNOTSUPP;
 
diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c
index 5ea22008d721..2b1140563a64 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_main.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_main.c
@@ -31,12 +31,7 @@
 /* Local includes */
 #include "i40e.h"
 #include "i40e_diag.h"
-#if IS_ENABLED(CONFIG_VXLAN)
-#include <net/vxlan.h>
-#endif
-#if IS_ENABLED(CONFIG_GENEVE)
-#include <net/geneve.h>
-#endif
+#include <net/udp_tunnel.h>
 
 const char i40e_driver_name[] = "i40e";
 static const char i40e_driver_string[] =
@@ -45,8 +40,8 @@ static const char i40e_driver_string[] =
 #define DRV_KERN "-k"
 
 #define DRV_VERSION_MAJOR 1
-#define DRV_VERSION_MINOR 5
-#define DRV_VERSION_BUILD 16
+#define DRV_VERSION_MINOR 6
+#define DRV_VERSION_BUILD 4
 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
              __stringify(DRV_VERSION_MINOR) "." \
              __stringify(DRV_VERSION_BUILD)    DRV_KERN
@@ -1584,14 +1579,8 @@ static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
         vsi->tc_config.numtc = numtc;
         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
         /* Number of queues per enabled TC */
-        /* In MFP case we can have a much lower count of MSIx
-         * vectors available and so we need to lower the used
-         * q count.
-         */
-        if (pf->flags & I40E_FLAG_MSIX_ENABLED)
-                qcount = min_t(int, vsi->alloc_queue_pairs, pf->num_lan_msix);
-        else
-                qcount = vsi->alloc_queue_pairs;
+        qcount = vsi->alloc_queue_pairs;
+
         num_tc_qps = qcount / numtc;
         num_tc_qps = min_t(int, num_tc_qps, i40e_pf_get_max_q_per_tc(pf));
 
@@ -1845,8 +1834,10 @@ int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
 {
         struct list_head tmp_del_list, tmp_add_list;
         struct i40e_mac_filter *f, *ftmp, *fclone;
+        struct i40e_hw *hw = &vsi->back->hw;
         bool promisc_forced_on = false;
         bool add_happened = false;
+        char vsi_name[16] = "PF";
         int filter_list_len = 0;
         u32 changed_flags = 0;
         i40e_status aq_ret = 0;
@@ -1874,6 +1865,11 @@ int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
         INIT_LIST_HEAD(&tmp_del_list);
         INIT_LIST_HEAD(&tmp_add_list);
 
+        if (vsi->type == I40E_VSI_SRIOV)
+                snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id);
+        else if (vsi->type != I40E_VSI_MAIN)
+                snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid);
+
         if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
                 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
 
@@ -1925,7 +1921,7 @@ int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
         if (!list_empty(&tmp_del_list)) {
                 int del_list_size;
 
-                filter_list_len = pf->hw.aq.asq_buf_size /
+                filter_list_len = hw->aq.asq_buf_size /
                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
                 del_list_size = filter_list_len *
                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
@@ -1957,21 +1953,21 @@ int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
 
                         /* flush a full buffer */
                         if (num_del == filter_list_len) {
-                                aq_ret = i40e_aq_remove_macvlan(&pf->hw,
-                                                                vsi->seid,
-                                                                del_list,
-                                                                num_del,
-                                                                NULL);
-                                aq_err = pf->hw.aq.asq_last_status;
+                                aq_ret =
+                                        i40e_aq_remove_macvlan(hw, vsi->seid,
+                                                               del_list,
+                                                               num_del, NULL);
+                                aq_err = hw->aq.asq_last_status;
                                 num_del = 0;
                                 memset(del_list, 0, del_list_size);
 
                                 if (aq_ret && aq_err != I40E_AQ_RC_ENOENT) {
                                         retval = -EIO;
                                         dev_err(&pf->pdev->dev,
-                                                "ignoring delete macvlan error, err %s, aq_err %s while flushing a full buffer\n",
-                                                i40e_stat_str(&pf->hw, aq_ret),
-                                                i40e_aq_str(&pf->hw, aq_err));
+                                                 "ignoring delete macvlan error on %s, err %s, aq_err %s while flushing a full buffer\n",
+                                                 vsi_name,
+                                                 i40e_stat_str(hw, aq_ret),
+                                                 i40e_aq_str(hw, aq_err));
                                 }
                         }
                         /* Release memory for MAC filter entries which were
@@ -1982,17 +1978,17 @@ int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
                 }
 
                 if (num_del) {
-                        aq_ret = i40e_aq_remove_macvlan(&pf->hw, vsi->seid,
-                                                        del_list, num_del,
-                                                        NULL);
-                        aq_err = pf->hw.aq.asq_last_status;
+                        aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, del_list,
+                                                        num_del, NULL);
+                        aq_err = hw->aq.asq_last_status;
                         num_del = 0;
 
                         if (aq_ret && aq_err != I40E_AQ_RC_ENOENT)
                                 dev_info(&pf->pdev->dev,
-                                         "ignoring delete macvlan error, err %s aq_err %s\n",
-                                         i40e_stat_str(&pf->hw, aq_ret),
-                                         i40e_aq_str(&pf->hw, aq_err));
+                                         "ignoring delete macvlan error on %s, err %s aq_err %s\n",
+                                         vsi_name,
+                                         i40e_stat_str(hw, aq_ret),
+                                         i40e_aq_str(hw, aq_err));
                 }
 
                 kfree(del_list);
@@ -2003,7 +1999,7 @@ int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
                 int add_list_size;
 
                 /* do all the adds now */
-                filter_list_len = pf->hw.aq.asq_buf_size /
+                filter_list_len = hw->aq.asq_buf_size /
                                sizeof(struct i40e_aqc_add_macvlan_element_data),
                 add_list_size = filter_list_len *
                                sizeof(struct i40e_aqc_add_macvlan_element_data);
@@ -2038,10 +2034,10 @@ int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
 
                         /* flush a full buffer */
                         if (num_add == filter_list_len) {
-                                aq_ret = i40e_aq_add_macvlan(&pf->hw, vsi->seid,
+                                aq_ret = i40e_aq_add_macvlan(hw, vsi->seid,
                                                              add_list, num_add,
                                                              NULL);
-                                aq_err = pf->hw.aq.asq_last_status;
+                                aq_err = hw->aq.asq_last_status;
                                 num_add = 0;
 
                                 if (aq_ret)
@@ -2056,9 +2052,9 @@ int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
                 }
 
                 if (num_add) {
-                        aq_ret = i40e_aq_add_macvlan(&pf->hw, vsi->seid,
+                        aq_ret = i40e_aq_add_macvlan(hw, vsi->seid,
                                                      add_list, num_add, NULL);
-                        aq_err = pf->hw.aq.asq_last_status;
+                        aq_err = hw->aq.asq_last_status;
                         num_add = 0;
                 }
                 kfree(add_list);
@@ -2067,16 +2063,18 @@ int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
                 if (add_happened && aq_ret && aq_err != I40E_AQ_RC_EINVAL) {
                         retval = i40e_aq_rc_to_posix(aq_ret, aq_err);
                         dev_info(&pf->pdev->dev,
-                                 "add filter failed, err %s aq_err %s\n",
-                                 i40e_stat_str(&pf->hw, aq_ret),
-                                 i40e_aq_str(&pf->hw, aq_err));
-                        if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOSPC) &&
+                                 "add filter failed on %s, err %s aq_err %s\n",
+                                 vsi_name,
+                                 i40e_stat_str(hw, aq_ret),
+                                 i40e_aq_str(hw, aq_err));
+                        if ((hw->aq.asq_last_status == I40E_AQ_RC_ENOSPC) &&
                             !test_bit(__I40E_FILTER_OVERFLOW_PROMISC,
                                       &vsi->state)) {
                                 promisc_forced_on = true;
                                 set_bit(__I40E_FILTER_OVERFLOW_PROMISC,
                                         &vsi->state);
-                                dev_info(&pf->pdev->dev, "promiscuous mode forced on\n");
+                                dev_info(&pf->pdev->dev, "promiscuous mode forced on %s\n",
+                                         vsi_name);
                         }
                 }
         }
@@ -2098,12 +2096,12 @@ int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
                                                                NULL);
                 if (aq_ret) {
                         retval = i40e_aq_rc_to_posix(aq_ret,
-                                                     pf->hw.aq.asq_last_status);
+                                                     hw->aq.asq_last_status);
                         dev_info(&pf->pdev->dev,
-                                 "set multi promisc failed, err %s aq_err %s\n",
-                                 i40e_stat_str(&pf->hw, aq_ret),
-                                 i40e_aq_str(&pf->hw,
-                                             pf->hw.aq.asq_last_status));
+                                 "set multi promisc failed on %s, err %s aq_err %s\n",
+                                 vsi_name,
+                                 i40e_stat_str(hw, aq_ret),
+                                 i40e_aq_str(hw, hw->aq.asq_last_status));
                 }
         }
         if ((changed_flags & IFF_PROMISC) || promisc_forced_on) {
@@ -2122,33 +2120,58 @@ int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
                          */
                         if (pf->cur_promisc != cur_promisc) {
                                 pf->cur_promisc = cur_promisc;
-                                set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
+                                if (cur_promisc)
+                                        aq_ret =
+                                              i40e_aq_set_default_vsi(hw,
+                                                                      vsi->seid,
+                                                                      NULL);
+                                else
+                                        aq_ret =
+                                            i40e_aq_clear_default_vsi(hw,
+                                                                      vsi->seid,
+                                                                      NULL);
+                                if (aq_ret) {
+                                        retval = i40e_aq_rc_to_posix(aq_ret,
+                                                        hw->aq.asq_last_status);
+                                        dev_info(&pf->pdev->dev,
+                                                 "Set default VSI failed on %s, err %s, aq_err %s\n",
+                                                 vsi_name,
+                                                 i40e_stat_str(hw, aq_ret),
+                                                 i40e_aq_str(hw,
+                                                     hw->aq.asq_last_status));
+                                }
                         }
                 } else {
                         aq_ret = i40e_aq_set_vsi_unicast_promiscuous(
-                                                          &vsi->back->hw,
+                                                          hw,
                                                           vsi->seid,
                                                           cur_promisc, NULL,
                                                           true);
                         if (aq_ret) {
                                 retval =
                                 i40e_aq_rc_to_posix(aq_ret,
-                                                    pf->hw.aq.asq_last_status);
+                                                    hw->aq.asq_last_status);
                                 dev_info(&pf->pdev->dev,
-                                         "set unicast promisc failed, err %d, aq_err %d\n",
-                                         aq_ret, pf->hw.aq.asq_last_status);
+                                         "set unicast promisc failed on %s, err %s, aq_err %s\n",
+                                         vsi_name,
+                                         i40e_stat_str(hw, aq_ret),
+                                         i40e_aq_str(hw,
+                                                     hw->aq.asq_last_status));
                         }
                         aq_ret = i40e_aq_set_vsi_multicast_promiscuous(
-                                                          &vsi->back->hw,
+                                                          hw,
                                                           vsi->seid,
                                                           cur_promisc, NULL);
                         if (aq_ret) {
                                 retval =
                                 i40e_aq_rc_to_posix(aq_ret,
-                                                    pf->hw.aq.asq_last_status);
+                                                    hw->aq.asq_last_status);
                                 dev_info(&pf->pdev->dev,
-                                         "set multicast promisc failed, err %d, aq_err %d\n",
-                                         aq_ret, pf->hw.aq.asq_last_status);
+                                         "set multicast promisc failed on %s, err %s, aq_err %s\n",
+                                         vsi_name,
+                                         i40e_stat_str(hw, aq_ret),
+                                         i40e_aq_str(hw,
+                                                     hw->aq.asq_last_status));
                         }
                 }
                 aq_ret = i40e_aq_set_vsi_broadcast(&vsi->back->hw,
@@ -2159,9 +2182,9 @@ int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
                                                      pf->hw.aq.asq_last_status);
                         dev_info(&pf->pdev->dev,
                                  "set brdcast promisc failed, err %s, aq_err %s\n",
-                                 i40e_stat_str(&pf->hw, aq_ret),
-                                 i40e_aq_str(&pf->hw,
-                                             pf->hw.aq.asq_last_status));
+                                         i40e_stat_str(hw, aq_ret),
+                                         i40e_aq_str(hw,
+                                                     hw->aq.asq_last_status));
                 }
         }
 out:
@@ -3952,6 +3975,7 @@ static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
                         /* clear the affinity_mask in the IRQ descriptor */
                         irq_set_affinity_hint(pf->msix_entries[vector].vector,
                                               NULL);
+                        synchronize_irq(pf->msix_entries[vector].vector);
                         free_irq(pf->msix_entries[vector].vector,
                                  vsi->q_vectors[i]);
 
@@ -4958,7 +4982,6 @@ static void i40e_dcb_reconfigure(struct i40e_pf *pf)
                         if (pf->vsi[v]->netdev)
                                 i40e_dcbnl_set_all(pf->vsi[v]);
                 }
-                i40e_notify_client_of_l2_param_changes(pf->vsi[v]);
         }
 }
 
@@ -5183,12 +5206,6 @@ static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
                 usleep_range(1000, 2000);
         i40e_down(vsi);
 
-        /* Give a VF some time to respond to the reset.  The
-         * two second wait is based upon the watchdog cycle in
-         * the VF driver.
-         */
-        if (vsi->type == I40E_VSI_SRIOV)
-                msleep(2000);
         i40e_up(vsi);
         clear_bit(__I40E_CONFIG_BUSY, &pf->state);
 }
@@ -5231,6 +5248,9 @@ void i40e_down(struct i40e_vsi *vsi)
                 i40e_clean_tx_ring(vsi->tx_rings[i]);
                 i40e_clean_rx_ring(vsi->rx_rings[i]);
         }
+
+        i40e_notify_client_of_netdev_close(vsi, false);
+
 }
 
 /**
@@ -5342,14 +5362,7 @@ int i40e_open(struct net_device *netdev)
                                                        TCP_FLAG_CWR) >> 16);
         wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
 
-#ifdef CONFIG_I40E_VXLAN
-        vxlan_get_rx_port(netdev);
-#endif
-#ifdef CONFIG_I40E_GENEVE
-        if (pf->flags & I40E_FLAG_GENEVE_OFFLOAD_CAPABLE)
-                geneve_get_rx_port(netdev);
-#endif
-
+        udp_tunnel_get_rx_info(netdev);
         i40e_notify_client_of_netdev_open(vsi);
 
         return 0;
@@ -5716,6 +5729,8 @@ static int i40e_handle_lldp_event(struct i40e_pf *pf,
                 i40e_service_event_schedule(pf);
         } else {
                 i40e_pf_unquiesce_all_vsi(pf);
+                /* Notify the client for the DCB changes */
+                i40e_notify_client_of_l2_param_changes(pf->vsi[pf->lan_vsi]);
         }
 
 exit:
@@ -5940,7 +5955,6 @@ static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
                         dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
         }
-
 }
 
 /**
@@ -7057,7 +7071,6 @@ static void i40e_handle_mdd_event(struct i40e_pf *pf)
  **/
 static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf)
 {
-#if IS_ENABLED(CONFIG_VXLAN) || IS_ENABLED(CONFIG_GENEVE)
         struct i40e_hw *hw = &pf->hw;
         i40e_status ret;
         __be16 port;
@@ -7092,7 +7105,6 @@ static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf)
                         }
                 }
         }
-#endif
 }
 
 /**
@@ -7174,7 +7186,7 @@ static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
                 vsi->alloc_queue_pairs = 1;
                 vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT,
                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
-                vsi->num_q_vectors = 1;
+                vsi->num_q_vectors = pf->num_fdsb_msix;
                 break;
 
         case I40E_VSI_VMDQ2:
@@ -7558,9 +7570,11 @@ static int i40e_init_msix(struct i40e_pf *pf)
         /* reserve one vector for sideband flow director */
         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
                 if (vectors_left) {
+                        pf->num_fdsb_msix = 1;
                         v_budget++;
                         vectors_left--;
                 } else {
+                        pf->num_fdsb_msix = 0;
                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
                 }
         }
@@ -8579,7 +8593,9 @@ bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
                 /* Enable filters and mark for reset */
                 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
                         need_reset = true;
-                pf->flags |= I40E_FLAG_FD_SB_ENABLED;
+                /* enable FD_SB only if there is MSI-X vector */
+                if (pf->num_fdsb_msix > 0)
+                        pf->flags |= I40E_FLAG_FD_SB_ENABLED;
         } else {
                 /* turn off filters, mark for reset and clear SW filter list */
                 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
@@ -8628,7 +8644,6 @@ static int i40e_set_features(struct net_device *netdev,
         return 0;
 }
 
-#if IS_ENABLED(CONFIG_VXLAN) || IS_ENABLED(CONFIG_GENEVE)
 /**
  * i40e_get_udp_port_idx - Lookup a possibly offloaded for Rx UDP port
  * @pf: board private structure
@@ -8648,21 +8663,18 @@ static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, __be16 port)
         return i;
 }
 
-#endif
-
-#if IS_ENABLED(CONFIG_VXLAN)
 /**
- * i40e_add_vxlan_port - Get notifications about VXLAN ports that come up
+ * i40e_udp_tunnel_add - Get notifications about UDP tunnel ports that come up
  * @netdev: This physical port's netdev
- * @sa_family: Socket Family that VXLAN is notifying us about
- * @port: New UDP port number that VXLAN started listening to
+ * @ti: Tunnel endpoint information
  **/
-static void i40e_add_vxlan_port(struct net_device *netdev,
-                                sa_family_t sa_family, __be16 port)
+static void i40e_udp_tunnel_add(struct net_device *netdev,
+                                struct udp_tunnel_info *ti)
 {
         struct i40e_netdev_priv *np = netdev_priv(netdev);
         struct i40e_vsi *vsi = np->vsi;
         struct i40e_pf *pf = vsi->back;
+        __be16 port = ti->port;
         u8 next_idx;
         u8 idx;
 
@@ -8670,7 +8682,7 @@ static void i40e_add_vxlan_port(struct net_device *netdev,
 
         /* Check if port already exists */
         if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
-                netdev_info(netdev, "vxlan port %d already offloaded\n",
+                netdev_info(netdev, "port %d already offloaded\n",
                             ntohs(port));
                 return;
         }
@@ -8679,131 +8691,75 @@ static void i40e_add_vxlan_port(struct net_device *netdev,
         next_idx = i40e_get_udp_port_idx(pf, 0);
 
         if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
-                netdev_info(netdev, "maximum number of vxlan UDP ports reached, not adding port %d\n",
-                            ntohs(port));
-                return;
-        }
-
-        /* New port: add it and mark its index in the bitmap */
-        pf->udp_ports[next_idx].index = port;
-        pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN;
-        pf->pending_udp_bitmap |= BIT_ULL(next_idx);
-        pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
-}
-
-/**
- * i40e_del_vxlan_port - Get notifications about VXLAN ports that go away
- * @netdev: This physical port's netdev
- * @sa_family: Socket Family that VXLAN is notifying us about
- * @port: UDP port number that VXLAN stopped listening to
- **/
-static void i40e_del_vxlan_port(struct net_device *netdev,
-                                sa_family_t sa_family, __be16 port)
-{
-        struct i40e_netdev_priv *np = netdev_priv(netdev);
-        struct i40e_vsi *vsi = np->vsi;
-        struct i40e_pf *pf = vsi->back;
-        u8 idx;
-
-        idx = i40e_get_udp_port_idx(pf, port);
-
-        /* Check if port already exists */
-        if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
-                /* if port exists, set it to 0 (mark for deletion)
-                 * and make it pending
-                 */
-                pf->udp_ports[idx].index = 0;
-                pf->pending_udp_bitmap |= BIT_ULL(idx);
-                pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
-        } else {
-                netdev_warn(netdev, "vxlan port %d was not found, not deleting\n",
-                            ntohs(port));
-        }
-}
-#endif
-
-#if IS_ENABLED(CONFIG_GENEVE)
-/**
- * i40e_add_geneve_port - Get notifications about GENEVE ports that come up
- * @netdev: This physical port's netdev
- * @sa_family: Socket Family that GENEVE is notifying us about
- * @port: New UDP port number that GENEVE started listening to
- **/
-static void i40e_add_geneve_port(struct net_device *netdev,
-                                 sa_family_t sa_family, __be16 port)
-{
-        struct i40e_netdev_priv *np = netdev_priv(netdev);
-        struct i40e_vsi *vsi = np->vsi;
-        struct i40e_pf *pf = vsi->back;
-        u8 next_idx;
-        u8 idx;
-
-        if (!(pf->flags & I40E_FLAG_GENEVE_OFFLOAD_CAPABLE))
-                return;
-
-        idx = i40e_get_udp_port_idx(pf, port);
-
-        /* Check if port already exists */
-        if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
-                netdev_info(netdev, "udp port %d already offloaded\n",
+                netdev_info(netdev, "maximum number of offloaded UDP ports reached, not adding port %d\n",
                             ntohs(port));
                 return;
         }
 
-        /* Now check if there is space to add the new port */
-        next_idx = i40e_get_udp_port_idx(pf, 0);
-
-        if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
-                netdev_info(netdev, "maximum number of UDP ports reached, not adding port %d\n",
-                            ntohs(port));
+        switch (ti->type) {
+        case UDP_TUNNEL_TYPE_VXLAN:
+                pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN;
+                break;
+        case UDP_TUNNEL_TYPE_GENEVE:
+                if (!(pf->flags & I40E_FLAG_GENEVE_OFFLOAD_CAPABLE))
+                        return;
+                pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE;
+                break;
+        default:
                 return;
         }
 
         /* New port: add it and mark its index in the bitmap */
         pf->udp_ports[next_idx].index = port;
-        pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE;
         pf->pending_udp_bitmap |= BIT_ULL(next_idx);
         pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
-
-        dev_info(&pf->pdev->dev, "adding geneve port %d\n", ntohs(port));
 }
 
 /**
- * i40e_del_geneve_port - Get notifications about GENEVE ports that go away
+ * i40e_udp_tunnel_del - Get notifications about UDP tunnel ports that go away
  * @netdev: This physical port's netdev
- * @sa_family: Socket Family that GENEVE is notifying us about
- * @port: UDP port number that GENEVE stopped listening to
+ * @ti: Tunnel endpoint information
  **/
-static void i40e_del_geneve_port(struct net_device *netdev,
-                                 sa_family_t sa_family, __be16 port)
+static void i40e_udp_tunnel_del(struct net_device *netdev,
+                                struct udp_tunnel_info *ti)
 {
         struct i40e_netdev_priv *np = netdev_priv(netdev);
         struct i40e_vsi *vsi = np->vsi;
         struct i40e_pf *pf = vsi->back;
+        __be16 port = ti->port;
         u8 idx;
 
-        if (!(pf->flags & I40E_FLAG_GENEVE_OFFLOAD_CAPABLE))
-                return;
-
         idx = i40e_get_udp_port_idx(pf, port);
 
         /* Check if port already exists */
-        if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
-                /* if port exists, set it to 0 (mark for deletion)
-                 * and make it pending
-                 */
-                pf->udp_ports[idx].index = 0;
-                pf->pending_udp_bitmap |= BIT_ULL(idx);
-                pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
+        if (idx >= I40E_MAX_PF_UDP_OFFLOAD_PORTS)
+                goto not_found;
 
-                dev_info(&pf->pdev->dev, "deleting geneve port %d\n",
-                         ntohs(port));
-        } else {
-                netdev_warn(netdev, "geneve port %d was not found, not deleting\n",
-                            ntohs(port));
+        switch (ti->type) {
+        case UDP_TUNNEL_TYPE_VXLAN:
+                if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_VXLAN)
+                        goto not_found;
+                break;
+        case UDP_TUNNEL_TYPE_GENEVE:
+                if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_NGE)
+                        goto not_found;
+                break;
+        default:
+                goto not_found;
         }
+
+        /* if port exists, set it to 0 (mark for deletion)
+         * and make it pending
+         */
+        pf->udp_ports[idx].index = 0;
+        pf->pending_udp_bitmap |= BIT_ULL(idx);
+        pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
+
+        return;
+not_found:
+        netdev_warn(netdev, "UDP port %d was not found, not deleting\n",
+                    ntohs(port));
 }
-#endif
 
 static int i40e_get_phys_port_id(struct net_device *netdev,
                                  struct netdev_phys_item_id *ppid)
@@ -9033,14 +8989,8 @@ static const struct net_device_ops i40e_netdev_ops = {
         .ndo_set_vf_link_state  = i40e_ndo_set_vf_link_state,
         .ndo_set_vf_spoofchk    = i40e_ndo_set_vf_spoofchk,
         .ndo_set_vf_trust       = i40e_ndo_set_vf_trust,
-#if IS_ENABLED(CONFIG_VXLAN)
-        .ndo_add_vxlan_port     = i40e_add_vxlan_port,
-        .ndo_del_vxlan_port     = i40e_del_vxlan_port,
-#endif
-#if IS_ENABLED(CONFIG_GENEVE)
-        .ndo_add_geneve_port    = i40e_add_geneve_port,
-        .ndo_del_geneve_port    = i40e_del_geneve_port,
-#endif
+        .ndo_udp_tunnel_add     = i40e_udp_tunnel_add,
+        .ndo_udp_tunnel_del     = i40e_udp_tunnel_del,
         .ndo_get_phys_port_id   = i40e_get_phys_port_id,
         .ndo_fdb_add            = i40e_ndo_fdb_add,
         .ndo_features_check     = i40e_features_check,
@@ -10133,14 +10083,14 @@ void i40e_veb_release(struct i40e_veb *veb)
 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
 {
         struct i40e_pf *pf = veb->pf;
-        bool is_default = veb->pf->cur_promisc;
         bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED);
         int ret;
 
-        /* get a VEB from the hardware */
         ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid,
-                              veb->enabled_tc, is_default,
+                              veb->enabled_tc, false,
                               &veb->seid, enable_stats, NULL);
+
+        /* get a VEB from the hardware */
         if (ret) {
                 dev_info(&pf->pdev->dev,
                          "couldn't add VEB, err %s aq_err %s\n",
@@ -10689,12 +10639,8 @@ static void i40e_print_features(struct i40e_pf *pf)
         }
         if (pf->flags & I40E_FLAG_DCB_CAPABLE)
                 i += snprintf(&buf[i], REMAIN(i), " DCB");
-#if IS_ENABLED(CONFIG_VXLAN)
         i += snprintf(&buf[i], REMAIN(i), " VxLAN");
-#endif
-#if IS_ENABLED(CONFIG_GENEVE)
         i += snprintf(&buf[i], REMAIN(i), " Geneve");
-#endif
         if (pf->flags & I40E_FLAG_PTP)
                 i += snprintf(&buf[i], REMAIN(i), " PTP");
 #ifdef I40E_FCOE
@@ -11525,6 +11471,7 @@ static int i40e_suspend(struct pci_dev *pdev, pm_message_t state)
 {
         struct i40e_pf *pf = pci_get_drvdata(pdev);
         struct i40e_hw *hw = &pf->hw;
+        int retval = 0;
 
         set_bit(__I40E_SUSPENDED, &pf->state);
         set_bit(__I40E_DOWN, &pf->state);
@@ -11536,10 +11483,16 @@ static int i40e_suspend(struct pci_dev *pdev, pm_message_t state)
         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
 
+        i40e_stop_misc_vector(pf);
+
+        retval = pci_save_state(pdev);
+        if (retval)
+                return retval;
+
         pci_wake_from_d3(pdev, pf->wol_en);
         pci_set_power_state(pdev, PCI_D3hot);
 
-        return 0;
+        return retval;
 }
 
 /**
diff --git a/drivers/net/ethernet/intel/i40e/i40e_prototype.h b/drivers/net/ethernet/intel/i40e/i40e_prototype.h
index 80403c6ee7f0..4660c5abc855 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_prototype.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_prototype.h
@@ -98,6 +98,8 @@ i40e_status i40e_aq_set_phy_debug(struct i40e_hw *hw, u8 cmd_flags,
                                 struct i40e_asq_cmd_details *cmd_details);
 i40e_status i40e_aq_set_default_vsi(struct i40e_hw *hw, u16 vsi_id,
                                 struct i40e_asq_cmd_details *cmd_details);
+i40e_status i40e_aq_clear_default_vsi(struct i40e_hw *hw, u16 vsi_id,
+                                      struct i40e_asq_cmd_details *cmd_details);
 enum i40e_status_code i40e_aq_get_phy_capabilities(struct i40e_hw *hw,
                         bool qualified_modules, bool report_init,
                         struct i40e_aq_get_phy_abilities_resp *abilities,
diff --git a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
index 1fcafcfa8f14..6fcbf764f32b 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
@@ -665,6 +665,8 @@ static int i40e_alloc_vsi_res(struct i40e_vf *vf, enum i40e_vsi_type type)
                 goto error_alloc_vsi_res;
         }
         if (type == I40E_VSI_SRIOV) {
+                u64 hena = i40e_pf_get_default_rss_hena(pf);
+
                 vf->lan_vsi_idx = vsi->idx;
                 vf->lan_vsi_id = vsi->id;
                 /* If the port VLAN has been configured and then the
@@ -687,6 +689,10 @@ static int i40e_alloc_vsi_res(struct i40e_vf *vf, enum i40e_vsi_type type)
                                         vf->default_lan_addr.addr, vf->vf_id);
                 }
                 spin_unlock_bh(&vsi->mac_filter_list_lock);
+                i40e_write_rx_ctl(&pf->hw, I40E_VFQF_HENA1(0, vf->vf_id),
+                                  (u32)hena);
+                i40e_write_rx_ctl(&pf->hw, I40E_VFQF_HENA1(1, vf->vf_id),
+                                  (u32)(hena >> 32));
         }
 
         /* program mac filter */
diff --git a/drivers/net/ethernet/intel/i40evf/i40evf_main.c b/drivers/net/ethernet/intel/i40evf/i40evf_main.c
index 16c552952860..eac057b88055 100644
--- a/drivers/net/ethernet/intel/i40evf/i40evf_main.c
+++ b/drivers/net/ethernet/intel/i40evf/i40evf_main.c
@@ -37,8 +37,8 @@ static const char i40evf_driver_string[] =
 #define DRV_KERN "-k"
 
 #define DRV_VERSION_MAJOR 1
-#define DRV_VERSION_MINOR 5
-#define DRV_VERSION_BUILD 10
+#define DRV_VERSION_MINOR 6
+#define DRV_VERSION_BUILD 4
 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
              __stringify(DRV_VERSION_MINOR) "." \
              __stringify(DRV_VERSION_BUILD) \
@@ -825,7 +825,7 @@ i40evf_mac_filter *i40evf_add_filter(struct i40evf_adapter *adapter,
 
                 ether_addr_copy(f->macaddr, macaddr);
 
-                list_add(&f->list, &adapter->mac_filter_list);
+                list_add_tail(&f->list, &adapter->mac_filter_list);
                 f->add = true;
                 adapter->aq_required |= I40EVF_FLAG_AQ_ADD_MAC_FILTER;
         }
diff --git a/drivers/net/ethernet/intel/i40evf/i40evf_virtchnl.c b/drivers/net/ethernet/intel/i40evf/i40evf_virtchnl.c
index f13445691507..d76c221d4c8a 100644
--- a/drivers/net/ethernet/intel/i40evf/i40evf_virtchnl.c
+++ b/drivers/net/ethernet/intel/i40evf/i40evf_virtchnl.c
@@ -434,6 +434,8 @@ void i40evf_add_ether_addrs(struct i40evf_adapter *adapter)
                         ether_addr_copy(veal->list[i].addr, f->macaddr);
                         i++;
                         f->add = false;
+                        if (i == count)
+                                break;
                 }
         }
         if (!more)
@@ -497,6 +499,8 @@ void i40evf_del_ether_addrs(struct i40evf_adapter *adapter)
                         i++;
                         list_del(&f->list);
                         kfree(f);
+                        if (i == count)
+                                break;
                 }
         }
         if (!more)
@@ -560,6 +564,8 @@ void i40evf_add_vlans(struct i40evf_adapter *adapter)
                         vvfl->vlan_id[i] = f->vlan;
                         i++;
                         f->add = false;
+                        if (i == count)
+                                break;
                 }
         }
         if (!more)
@@ -623,6 +629,8 @@ void i40evf_del_vlans(struct i40evf_adapter *adapter)
                         i++;
                         list_del(&f->list);
                         kfree(f);
+                        if (i == count)
+                                break;
                 }
         }
         if (!more)
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
index 088c47cf27d9..468fa9ddfa06 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
@@ -50,7 +50,7 @@
 #include <linux/if_bridge.h>
 #include <linux/prefetch.h>
 #include <scsi/fc/fc_fcoe.h>
-#include <net/vxlan.h>
+#include <net/udp_tunnel.h>
 #include <net/pkt_cls.h>
 #include <net/tc_act/tc_gact.h>
 #include <net/tc_act/tc_mirred.h>
@@ -5722,9 +5722,7 @@ static int ixgbe_sw_init(struct ixgbe_adapter *adapter)
 #ifdef CONFIG_IXGBE_DCA
                 adapter->flags &= ~IXGBE_FLAG_DCA_CAPABLE;
 #endif
-#ifdef CONFIG_IXGBE_VXLAN
                 adapter->flags |= IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE;
-#endif
                 break;
         default:
                 break;
@@ -6158,9 +6156,7 @@ int ixgbe_open(struct net_device *netdev)
         ixgbe_up_complete(adapter);
 
         ixgbe_clear_vxlan_port(adapter);
-#ifdef CONFIG_IXGBE_VXLAN
-        vxlan_get_rx_port(netdev);
-#endif
+        udp_tunnel_get_rx_info(netdev);
 
         return 0;
 
@@ -7262,14 +7258,12 @@ static void ixgbe_service_task(struct work_struct *work)
                 ixgbe_service_event_complete(adapter);
                 return;
         }
-#ifdef CONFIG_IXGBE_VXLAN
-        rtnl_lock();
         if (adapter->flags2 & IXGBE_FLAG2_VXLAN_REREG_NEEDED) {
+                rtnl_lock();
                 adapter->flags2 &= ~IXGBE_FLAG2_VXLAN_REREG_NEEDED;
-                vxlan_get_rx_port(adapter->netdev);
+                udp_tunnel_get_rx_info(adapter->netdev);
+                rtnl_unlock();
         }
-        rtnl_unlock();
-#endif /* CONFIG_IXGBE_VXLAN */
         ixgbe_reset_subtask(adapter);
         ixgbe_phy_interrupt_subtask(adapter);
         ixgbe_sfp_detection_subtask(adapter);
@@ -7697,7 +7691,6 @@ static void ixgbe_atr(struct ixgbe_ring *ring,
         /* snag network header to get L4 type and address */
         skb = first->skb;
         hdr.network = skb_network_header(skb);
-#ifdef CONFIG_IXGBE_VXLAN
         if (skb->encapsulation &&
             first->protocol == htons(ETH_P_IP) &&
             hdr.ipv4->protocol != IPPROTO_UDP) {
@@ -7708,7 +7701,6 @@ static void ixgbe_atr(struct ixgbe_ring *ring,
                     udp_hdr(skb)->dest == adapter->vxlan_port)
                         hdr.network = skb_inner_network_header(skb);
         }
-#endif /* CONFIG_IXGBE_VXLAN */
 
         /* Currently only IPv4/IPv6 with TCP is supported */
         switch (hdr.ipv4->version) {
@@ -8770,14 +8762,12 @@ static int ixgbe_set_features(struct net_device *netdev,
 
         netdev->features = features;
 
-#ifdef CONFIG_IXGBE_VXLAN
         if ((adapter->flags & IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE)) {
                 if (features & NETIF_F_RXCSUM)
                         adapter->flags2 |= IXGBE_FLAG2_VXLAN_REREG_NEEDED;
                 else
                         ixgbe_clear_vxlan_port(adapter);
         }
-#endif /* CONFIG_IXGBE_VXLAN */
 
         if (need_reset)
                 ixgbe_do_reset(netdev);
@@ -8788,23 +8778,27 @@ static int ixgbe_set_features(struct net_device *netdev,
         return 0;
 }
 
-#ifdef CONFIG_IXGBE_VXLAN
 /**
  * ixgbe_add_vxlan_port - Get notifications about VXLAN ports that come up
  * @dev: The port's netdev
  * @sa_family: Socket Family that VXLAN is notifiying us about
  * @port: New UDP port number that VXLAN started listening to
+ * @type: Enumerated type specifying UDP tunnel type
  **/
-static void ixgbe_add_vxlan_port(struct net_device *dev, sa_family_t sa_family,
-                                 __be16 port)
+static void ixgbe_add_vxlan_port(struct net_device *dev,
+                                 struct udp_tunnel_info *ti)
 {
         struct ixgbe_adapter *adapter = netdev_priv(dev);
         struct ixgbe_hw *hw = &adapter->hw;
+        __be16 port = ti->port;
 
-        if (!(adapter->flags & IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE))
+        if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
                 return;
 
-        if (sa_family == AF_INET6)
+        if (ti->sa_family != AF_INET)
+                return;
+
+        if (!(adapter->flags & IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE))
                 return;
 
         if (adapter->vxlan_port == port)
@@ -8826,28 +8820,31 @@ static void ixgbe_add_vxlan_port(struct net_device *dev, sa_family_t sa_family,
  * @dev: The port's netdev
  * @sa_family: Socket Family that VXLAN is notifying us about
  * @port: UDP port number that VXLAN stopped listening to
+ * @type: Enumerated type specifying UDP tunnel type
  **/
-static void ixgbe_del_vxlan_port(struct net_device *dev, sa_family_t sa_family,
-                                 __be16 port)
+static void ixgbe_del_vxlan_port(struct net_device *dev,
+                                 struct udp_tunnel_info *ti)
 {
         struct ixgbe_adapter *adapter = netdev_priv(dev);
 
-        if (!(adapter->flags & IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE))
+        if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
                 return;
 
-        if (sa_family == AF_INET6)
+        if (ti->sa_family != AF_INET)
                 return;
 
-        if (adapter->vxlan_port != port) {
+        if (!(adapter->flags & IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE))
+                return;
+
+        if (adapter->vxlan_port != ti->port) {
                 netdev_info(dev, "Port %d was not found, not deleting\n",
-                            ntohs(port));
+                            ntohs(ti->port));
                 return;
         }
 
         ixgbe_clear_vxlan_port(adapter);
         adapter->flags2 |= IXGBE_FLAG2_VXLAN_REREG_NEEDED;
 }
-#endif /* CONFIG_IXGBE_VXLAN */
 
 static int ixgbe_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
                              struct net_device *dev,
@@ -9160,10 +9157,8 @@ static const struct net_device_ops ixgbe_netdev_ops = {
         .ndo_bridge_getlink     = ixgbe_ndo_bridge_getlink,
         .ndo_dfwd_add_station   = ixgbe_fwd_add,
         .ndo_dfwd_del_station   = ixgbe_fwd_del,
-#ifdef CONFIG_IXGBE_VXLAN
-        .ndo_add_vxlan_port     = ixgbe_add_vxlan_port,
-        .ndo_del_vxlan_port     = ixgbe_del_vxlan_port,
-#endif /* CONFIG_IXGBE_VXLAN */
+        .ndo_udp_tunnel_add     = ixgbe_add_vxlan_port,
+        .ndo_udp_tunnel_del     = ixgbe_del_vxlan_port,
         .ndo_features_check     = ixgbe_features_check,
 };
 
diff --git a/drivers/net/ethernet/lantiq_etop.c b/drivers/net/ethernet/lantiq_etop.c
index dc82b1b19574..0d2f8e934c59 100644
--- a/drivers/net/ethernet/lantiq_etop.c
+++ b/drivers/net/ethernet/lantiq_etop.c
@@ -411,7 +411,6 @@ static int
 ltq_etop_mdio_init(struct net_device *dev)
 {
         struct ltq_etop_priv *priv = netdev_priv(dev);
-        int i;
         int err;
 
         priv->mii_bus = mdiobus_alloc();
diff --git a/drivers/net/ethernet/marvell/mvpp2.c b/drivers/net/ethernet/marvell/mvpp2.c
index 868a957f24bb..0b047178cda1 100644
--- a/drivers/net/ethernet/marvell/mvpp2.c
+++ b/drivers/net/ethernet/marvell/mvpp2.c
@@ -699,7 +699,6 @@ struct mvpp2_port {
         u16 rx_ring_size;
         struct mvpp2_pcpu_stats __percpu *stats;
 
-        struct phy_device *phy_dev;
         phy_interface_t phy_interface;
         struct device_node *phy_node;
         unsigned int link;
@@ -4850,7 +4849,7 @@ static irqreturn_t mvpp2_isr(int irq, void *dev_id)
 static void mvpp2_link_event(struct net_device *dev)
 {
         struct mvpp2_port *port = netdev_priv(dev);
-        struct phy_device *phydev = port->phy_dev;
+        struct phy_device *phydev = dev->phydev;
         int status_change = 0;
         u32 val;
 
@@ -5416,6 +5415,8 @@ static int mvpp2_poll(struct napi_struct *napi, int budget)
 /* Set hw internals when starting port */
 static void mvpp2_start_dev(struct mvpp2_port *port)
 {
+        struct net_device *ndev = port->dev;
+
         mvpp2_gmac_max_rx_size_set(port);
         mvpp2_txp_max_tx_size_set(port);
 
@@ -5425,13 +5426,15 @@ static void mvpp2_start_dev(struct mvpp2_port *port)
         mvpp2_interrupts_enable(port);
 
         mvpp2_port_enable(port);
-        phy_start(port->phy_dev);
+        phy_start(ndev->phydev);
         netif_tx_start_all_queues(port->dev);
 }
 
 /* Set hw internals when stopping port */
 static void mvpp2_stop_dev(struct mvpp2_port *port)
 {
+        struct net_device *ndev = port->dev;
+
         /* Stop new packets from arriving to RXQs */
         mvpp2_ingress_disable(port);
 
@@ -5447,7 +5450,7 @@ static void mvpp2_stop_dev(struct mvpp2_port *port)
 
         mvpp2_egress_disable(port);
         mvpp2_port_disable(port);
-        phy_stop(port->phy_dev);
+        phy_stop(ndev->phydev);
 }
 
 /* Return positive if MTU is valid */
@@ -5535,7 +5538,6 @@ static int mvpp2_phy_connect(struct mvpp2_port *port)
         phy_dev->supported &= PHY_GBIT_FEATURES;
         phy_dev->advertising = phy_dev->supported;
 
-        port->phy_dev = phy_dev;
         port->link    = 0;
         port->duplex  = 0;
         port->speed   = 0;
@@ -5545,8 +5547,9 @@ static int mvpp2_phy_connect(struct mvpp2_port *port)
 
 static void mvpp2_phy_disconnect(struct mvpp2_port *port)
 {
-        phy_disconnect(port->phy_dev);
-        port->phy_dev = NULL;
+        struct net_device *ndev = port->dev;
+
+        phy_disconnect(ndev->phydev);
 }
 
 static int mvpp2_open(struct net_device *dev)
@@ -5796,13 +5799,12 @@ mvpp2_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
 
 static int mvpp2_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
 {
-        struct mvpp2_port *port = netdev_priv(dev);
         int ret;
 
-        if (!port->phy_dev)
+        if (!dev->phydev)
                 return -ENOTSUPP;
 
-        ret = phy_mii_ioctl(port->phy_dev, ifr, cmd);
+        ret = phy_mii_ioctl(dev->phydev, ifr, cmd);
         if (!ret)
                 mvpp2_link_event(dev);
 
@@ -5811,28 +5813,6 @@ static int mvpp2_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
 
 /* Ethtool methods */
 
-/* Get settings (phy address, speed) for ethtools */
-static int mvpp2_ethtool_get_settings(struct net_device *dev,
-                                      struct ethtool_cmd *cmd)
-{
-        struct mvpp2_port *port = netdev_priv(dev);
-
-        if (!port->phy_dev)
-                return -ENODEV;
-        return phy_ethtool_gset(port->phy_dev, cmd);
-}
-
-/* Set settings (phy address, speed) for ethtools */
-static int mvpp2_ethtool_set_settings(struct net_device *dev,
-                                      struct ethtool_cmd *cmd)
-{
-        struct mvpp2_port *port = netdev_priv(dev);
-
-        if (!port->phy_dev)
-                return -ENODEV;
-        return phy_ethtool_sset(port->phy_dev, cmd);
-}
-
 /* Set interrupt coalescing for ethtools */
 static int mvpp2_ethtool_set_coalesce(struct net_device *dev,
                                       struct ethtool_coalesce *c)
@@ -5967,13 +5947,13 @@ static const struct net_device_ops mvpp2_netdev_ops = {
 
 static const struct ethtool_ops mvpp2_eth_tool_ops = {
         .get_link       = ethtool_op_get_link,
-        .get_settings   = mvpp2_ethtool_get_settings,
-        .set_settings   = mvpp2_ethtool_set_settings,
         .set_coalesce   = mvpp2_ethtool_set_coalesce,
         .get_coalesce   = mvpp2_ethtool_get_coalesce,
         .get_drvinfo    = mvpp2_ethtool_get_drvinfo,
         .get_ringparam  = mvpp2_ethtool_get_ringparam,
         .set_ringparam  = mvpp2_ethtool_set_ringparam,
+        .get_link_ksettings = phy_ethtool_get_link_ksettings,
+        .set_link_ksettings = phy_ethtool_set_link_ksettings,
 };
 
 /* Driver initialization */
diff --git a/drivers/net/ethernet/mellanox/mlx4/Kconfig b/drivers/net/ethernet/mellanox/mlx4/Kconfig
index 9ca3734ebb6b..5098e7f21987 100644
--- a/drivers/net/ethernet/mellanox/mlx4/Kconfig
+++ b/drivers/net/ethernet/mellanox/mlx4/Kconfig
@@ -24,13 +24,6 @@ config MLX4_EN_DCB
 
           If unsure, set to Y
 
-config MLX4_EN_VXLAN
-        bool "VXLAN offloads Support"
-        default y
-        depends on MLX4_EN && VXLAN && !(MLX4_EN=y && VXLAN=m)
-        ---help---
-          Say Y here if you want to use VXLAN offloads in the driver.
-
 config MLX4_CORE
         tristate
         depends on PCI
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_dcb_nl.c b/drivers/net/ethernet/mellanox/mlx4/en_dcb_nl.c
index f01918c63f28..99c6bbdff501 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_dcb_nl.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_dcb_nl.c
@@ -37,6 +37,11 @@
 #include "mlx4_en.h"
 #include "fw_qos.h"
 
+enum {
+        MLX4_CEE_STATE_DOWN   = 0,
+        MLX4_CEE_STATE_UP     = 1,
+};
+
 /* Definitions for QCN
  */
 
@@ -80,13 +85,202 @@ struct mlx4_congestion_control_mb_prio_802_1_qau_statistics {
         __be32 reserved3[4];
 };
 
+static u8 mlx4_en_dcbnl_getcap(struct net_device *dev, int capid, u8 *cap)
+{
+        struct mlx4_en_priv *priv = netdev_priv(dev);
+
+        switch (capid) {
+        case DCB_CAP_ATTR_PFC:
+                *cap = true;
+                break;
+        case DCB_CAP_ATTR_DCBX:
+                *cap = priv->cee_params.dcbx_cap;
+                break;
+        case DCB_CAP_ATTR_PFC_TCS:
+                *cap = 1 <<  mlx4_max_tc(priv->mdev->dev);
+                break;
+        default:
+                *cap = false;
+                break;
+        }
+
+        return 0;
+}
+
+static u8 mlx4_en_dcbnl_getpfcstate(struct net_device *netdev)
+{
+        struct mlx4_en_priv *priv = netdev_priv(netdev);
+
+        return priv->cee_params.dcb_cfg.pfc_state;
+}
+
+static void mlx4_en_dcbnl_setpfcstate(struct net_device *netdev, u8 state)
+{
+        struct mlx4_en_priv *priv = netdev_priv(netdev);
+
+        priv->cee_params.dcb_cfg.pfc_state = state;
+}
+
+static void mlx4_en_dcbnl_get_pfc_cfg(struct net_device *netdev, int priority,
+                                      u8 *setting)
+{
+        struct mlx4_en_priv *priv = netdev_priv(netdev);
+
+        *setting = priv->cee_params.dcb_cfg.tc_config[priority].dcb_pfc;
+}
+
+static void mlx4_en_dcbnl_set_pfc_cfg(struct net_device *netdev, int priority,
+                                      u8 setting)
+{
+        struct mlx4_en_priv *priv = netdev_priv(netdev);
+
+        priv->cee_params.dcb_cfg.tc_config[priority].dcb_pfc = setting;
+        priv->cee_params.dcb_cfg.pfc_state = true;
+}
+
+static int mlx4_en_dcbnl_getnumtcs(struct net_device *netdev, int tcid, u8 *num)
+{
+        struct mlx4_en_priv *priv = netdev_priv(netdev);
+
+        if (!(priv->flags & MLX4_EN_FLAG_DCB_ENABLED))
+                return -EINVAL;
+
+        if (tcid == DCB_NUMTCS_ATTR_PFC)
+                *num = mlx4_max_tc(priv->mdev->dev);
+        else
+                *num = 0;
+
+        return 0;
+}
+
+static u8 mlx4_en_dcbnl_set_all(struct net_device *netdev)
+{
+        struct mlx4_en_priv *priv = netdev_priv(netdev);
+        struct mlx4_en_dev *mdev = priv->mdev;
+        struct mlx4_en_cee_config *dcb_cfg = &priv->cee_params.dcb_cfg;
+        int err = 0;
+
+        if (!(priv->cee_params.dcbx_cap & DCB_CAP_DCBX_VER_CEE))
+                return -EINVAL;
+
+        if (dcb_cfg->pfc_state) {
+                int tc;
+
+                priv->prof->rx_pause = 0;
+                priv->prof->tx_pause = 0;
+                for (tc = 0; tc < CEE_DCBX_MAX_PRIO; tc++) {
+                        u8 tc_mask = 1 << tc;
+
+                        switch (dcb_cfg->tc_config[tc].dcb_pfc) {
+                        case pfc_disabled:
+                                priv->prof->tx_ppp &= ~tc_mask;
+                                priv->prof->rx_ppp &= ~tc_mask;
+                                break;
+                        case pfc_enabled_full:
+                                priv->prof->tx_ppp |= tc_mask;
+                                priv->prof->rx_ppp |= tc_mask;
+                                break;
+                        case pfc_enabled_tx:
+                                priv->prof->tx_ppp |= tc_mask;
+                                priv->prof->rx_ppp &= ~tc_mask;
+                                break;
+                        case pfc_enabled_rx:
+                                priv->prof->tx_ppp &= ~tc_mask;
+                                priv->prof->rx_ppp |= tc_mask;
+                                break;
+                        default:
+                                break;
+                        }
+                }
+                en_dbg(DRV, priv, "Set pfc on\n");
+        } else {
+                priv->prof->rx_pause = 1;
+                priv->prof->tx_pause = 1;
+                en_dbg(DRV, priv, "Set pfc off\n");
+        }
+
+        err = mlx4_SET_PORT_general(mdev->dev, priv->port,
+                                    priv->rx_skb_size + ETH_FCS_LEN,
+                                    priv->prof->tx_pause,
+                                    priv->prof->tx_ppp,
+                                    priv->prof->rx_pause,
+                                    priv->prof->rx_ppp);
+        if (err)
+                en_err(priv, "Failed setting pause params\n");
+        return err;
+}
+
+static u8 mlx4_en_dcbnl_get_state(struct net_device *dev)
+{
+        struct mlx4_en_priv *priv = netdev_priv(dev);
+
+        if (priv->flags & MLX4_EN_FLAG_DCB_ENABLED)
+                return MLX4_CEE_STATE_UP;
+
+        return MLX4_CEE_STATE_DOWN;
+}
+
+static u8 mlx4_en_dcbnl_set_state(struct net_device *dev, u8 state)
+{
+        struct mlx4_en_priv *priv = netdev_priv(dev);
+        int num_tcs = 0;
+
+        if (!(priv->cee_params.dcbx_cap & DCB_CAP_DCBX_VER_CEE))
+                return 1;
+
+        if (!!(state) == !!(priv->flags & MLX4_EN_FLAG_DCB_ENABLED))
+                return 0;
+
+        if (state) {
+                priv->flags |= MLX4_EN_FLAG_DCB_ENABLED;
+                num_tcs = IEEE_8021QAZ_MAX_TCS;
+        } else {
+                priv->flags &= ~MLX4_EN_FLAG_DCB_ENABLED;
+        }
+
+        return mlx4_en_setup_tc(dev, num_tcs);
+}
+
+/* On success returns a non-zero 802.1p user priority bitmap
+ * otherwise returns 0 as the invalid user priority bitmap to
+ * indicate an error.
+ */
+static int mlx4_en_dcbnl_getapp(struct net_device *netdev, u8 idtype, u16 id)
+{
+        struct mlx4_en_priv *priv = netdev_priv(netdev);
+        struct dcb_app app = {
+                                .selector = idtype,
+                                .protocol = id,
+                             };
+        if (!(priv->cee_params.dcbx_cap & DCB_CAP_DCBX_VER_CEE))
+                return 0;
+
+        return dcb_getapp(netdev, &app);
+}
+
+static int mlx4_en_dcbnl_setapp(struct net_device *netdev, u8 idtype,
+                                u16 id, u8 up)
+{
+        struct mlx4_en_priv *priv = netdev_priv(netdev);
+        struct dcb_app app;
+
+        if (!(priv->cee_params.dcbx_cap & DCB_CAP_DCBX_VER_CEE))
+                return -EINVAL;
+
+        memset(&app, 0, sizeof(struct dcb_app));
+        app.selector = idtype;
+        app.protocol = id;
+        app.priority = up;
+
+        return dcb_setapp(netdev, &app);
+}
+
 static int mlx4_en_dcbnl_ieee_getets(struct net_device *dev,
                                    struct ieee_ets *ets)
 {
         struct mlx4_en_priv *priv = netdev_priv(dev);
         struct ieee_ets *my_ets = &priv->ets;
 
-        /* No IEEE PFC settings available */
         if (!my_ets)
                 return -EINVAL;
 
@@ -237,18 +431,51 @@ static int mlx4_en_dcbnl_ieee_setpfc(struct net_device *dev,
 
 static u8 mlx4_en_dcbnl_getdcbx(struct net_device *dev)
 {
-        return DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_IEEE;
+        struct mlx4_en_priv *priv = netdev_priv(dev);
+
+        return priv->cee_params.dcbx_cap;
 }
 
 static u8 mlx4_en_dcbnl_setdcbx(struct net_device *dev, u8 mode)
 {
+        struct mlx4_en_priv *priv = netdev_priv(dev);
+        struct ieee_ets ets = {0};
+        struct ieee_pfc pfc = {0};
+
+        if (mode == priv->cee_params.dcbx_cap)
+                return 0;
+
         if ((mode & DCB_CAP_DCBX_LLD_MANAGED) ||
-            (mode & DCB_CAP_DCBX_VER_CEE) ||
-            !(mode & DCB_CAP_DCBX_VER_IEEE) ||
+            ((mode & DCB_CAP_DCBX_VER_IEEE) &&
+             (mode & DCB_CAP_DCBX_VER_CEE)) ||
             !(mode & DCB_CAP_DCBX_HOST))
-                return 1;
+                goto err;
+
+        priv->cee_params.dcbx_cap = mode;
+
+        ets.ets_cap = IEEE_8021QAZ_MAX_TCS;
+        pfc.pfc_cap = IEEE_8021QAZ_MAX_TCS;
+
+        if (mode & DCB_CAP_DCBX_VER_IEEE) {
+                if (mlx4_en_dcbnl_ieee_setets(dev, &ets))
+                        goto err;
+                if (mlx4_en_dcbnl_ieee_setpfc(dev, &pfc))
+                        goto err;
+        } else if (mode & DCB_CAP_DCBX_VER_CEE) {
+                if (mlx4_en_dcbnl_set_all(dev))
+                        goto err;
+        } else {
+                if (mlx4_en_dcbnl_ieee_setets(dev, &ets))
+                        goto err;
+                if (mlx4_en_dcbnl_ieee_setpfc(dev, &pfc))
+                        goto err;
+                if (mlx4_en_setup_tc(dev, 0))
+                        goto err;
+        }
 
         return 0;
+err:
+        return 1;
 }
 
 #define MLX4_RATELIMIT_UNITS_IN_KB 100000 /* rate-limit HW unit in Kbps */
@@ -463,24 +690,46 @@ static int mlx4_en_dcbnl_ieee_getqcnstats(struct net_device *dev,
 }
 
 const struct dcbnl_rtnl_ops mlx4_en_dcbnl_ops = {
-        .ieee_getets    = mlx4_en_dcbnl_ieee_getets,
-        .ieee_setets    = mlx4_en_dcbnl_ieee_setets,
-        .ieee_getmaxrate = mlx4_en_dcbnl_ieee_getmaxrate,
-        .ieee_setmaxrate = mlx4_en_dcbnl_ieee_setmaxrate,
-        .ieee_getpfc    = mlx4_en_dcbnl_ieee_getpfc,
-        .ieee_setpfc    = mlx4_en_dcbnl_ieee_setpfc,
+        .ieee_getets            = mlx4_en_dcbnl_ieee_getets,
+        .ieee_setets            = mlx4_en_dcbnl_ieee_setets,
+        .ieee_getmaxrate        = mlx4_en_dcbnl_ieee_getmaxrate,
+        .ieee_setmaxrate        = mlx4_en_dcbnl_ieee_setmaxrate,
+        .ieee_getqcn            = mlx4_en_dcbnl_ieee_getqcn,
+        .ieee_setqcn            = mlx4_en_dcbnl_ieee_setqcn,
+        .ieee_getqcnstats       = mlx4_en_dcbnl_ieee_getqcnstats,
+        .ieee_getpfc            = mlx4_en_dcbnl_ieee_getpfc,
+        .ieee_setpfc            = mlx4_en_dcbnl_ieee_setpfc,
+
+        .getstate       = mlx4_en_dcbnl_get_state,
+        .setstate       = mlx4_en_dcbnl_set_state,
+        .getpfccfg      = mlx4_en_dcbnl_get_pfc_cfg,
+        .setpfccfg      = mlx4_en_dcbnl_set_pfc_cfg,
+        .setall         = mlx4_en_dcbnl_set_all,
+        .getcap         = mlx4_en_dcbnl_getcap,
+        .getnumtcs      = mlx4_en_dcbnl_getnumtcs,
+        .getpfcstate    = mlx4_en_dcbnl_getpfcstate,
+        .setpfcstate    = mlx4_en_dcbnl_setpfcstate,
+        .getapp         = mlx4_en_dcbnl_getapp,
+        .setapp         = mlx4_en_dcbnl_setapp,
 
         .getdcbx        = mlx4_en_dcbnl_getdcbx,
         .setdcbx        = mlx4_en_dcbnl_setdcbx,
-        .ieee_getqcn    = mlx4_en_dcbnl_ieee_getqcn,
-        .ieee_setqcn    = mlx4_en_dcbnl_ieee_setqcn,
-        .ieee_getqcnstats = mlx4_en_dcbnl_ieee_getqcnstats,
 };
 
 const struct dcbnl_rtnl_ops mlx4_en_dcbnl_pfc_ops = {
         .ieee_getpfc    = mlx4_en_dcbnl_ieee_getpfc,
         .ieee_setpfc    = mlx4_en_dcbnl_ieee_setpfc,
 
+        .setstate       = mlx4_en_dcbnl_set_state,
+        .getpfccfg      = mlx4_en_dcbnl_get_pfc_cfg,
+        .setpfccfg      = mlx4_en_dcbnl_set_pfc_cfg,
+        .setall         = mlx4_en_dcbnl_set_all,
+        .getnumtcs      = mlx4_en_dcbnl_getnumtcs,
+        .getpfcstate    = mlx4_en_dcbnl_getpfcstate,
+        .setpfcstate    = mlx4_en_dcbnl_setpfcstate,
+        .getapp         = mlx4_en_dcbnl_getapp,
+        .setapp         = mlx4_en_dcbnl_setapp,
+
         .getdcbx        = mlx4_en_dcbnl_getdcbx,
         .setdcbx        = mlx4_en_dcbnl_setdcbx,
 };
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c
index fc95affaf76b..51a2e8252b82 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c
@@ -1107,7 +1107,7 @@ static u32 mlx4_en_get_rxfh_indir_size(struct net_device *dev)
 {
         struct mlx4_en_priv *priv = netdev_priv(dev);
 
-        return priv->rx_ring_num;
+        return rounddown_pow_of_two(priv->rx_ring_num);
 }
 
 static u32 mlx4_en_get_rxfh_key_size(struct net_device *netdev)
@@ -1141,19 +1141,17 @@ static int mlx4_en_get_rxfh(struct net_device *dev, u32 *ring_index, u8 *key,
                             u8 *hfunc)
 {
         struct mlx4_en_priv *priv = netdev_priv(dev);
-        struct mlx4_en_rss_map *rss_map = &priv->rss_map;
-        int rss_rings;
-        size_t n = priv->rx_ring_num;
+        u32 n = mlx4_en_get_rxfh_indir_size(dev);
+        u32 i, rss_rings;
         int err = 0;
 
-        rss_rings = priv->prof->rss_rings ?: priv->rx_ring_num;
-        rss_rings = 1 << ilog2(rss_rings);
+        rss_rings = priv->prof->rss_rings ?: n;
+        rss_rings = rounddown_pow_of_two(rss_rings);
 
-        while (n--) {
+        for (i = 0; i < n; i++) {
                 if (!ring_index)
                         break;
-                ring_index[n] = rss_map->qps[n % rss_rings].qpn -
-                        rss_map->base_qpn;
+                ring_index[i] = i % rss_rings;
         }
         if (key)
                 memcpy(key, priv->rss_key, MLX4_EN_RSS_KEY_SIZE);
@@ -1166,6 +1164,7 @@ static int mlx4_en_set_rxfh(struct net_device *dev, const u32 *ring_index,
                             const u8 *key, const u8 hfunc)
 {
         struct mlx4_en_priv *priv = netdev_priv(dev);
+        u32 n = mlx4_en_get_rxfh_indir_size(dev);
         struct mlx4_en_dev *mdev = priv->mdev;
         int port_up = 0;
         int err = 0;
@@ -1175,18 +1174,18 @@ static int mlx4_en_set_rxfh(struct net_device *dev, const u32 *ring_index,
         /* Calculate RSS table size and make sure flows are spread evenly
          * between rings
          */
-        for (i = 0; i < priv->rx_ring_num; i++) {
+        for (i = 0; i < n; i++) {
                 if (!ring_index)
-                        continue;
+                        break;
                 if (i > 0 && !ring_index[i] && !rss_rings)
                         rss_rings = i;
 
-                if (ring_index[i] != (i % (rss_rings ?: priv->rx_ring_num)))
+                if (ring_index[i] != (i % (rss_rings ?: n)))
                         return -EINVAL;
         }
 
         if (!rss_rings)
-                rss_rings = priv->rx_ring_num;
+                rss_rings = n;
 
         /* RSS table size must be an order of 2 */
         if (!is_power_of_2(rss_rings))
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
index 0c0dfd6cdca6..6083775dae16 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
@@ -67,6 +67,17 @@ int mlx4_en_setup_tc(struct net_device *dev, u8 up)
                 offset += priv->num_tx_rings_p_up;
         }
 
+#ifdef CONFIG_MLX4_EN_DCB
+        if (!mlx4_is_slave(priv->mdev->dev)) {
+                if (up) {
+                        priv->flags |= MLX4_EN_FLAG_DCB_ENABLED;
+                } else {
+                        priv->flags &= ~MLX4_EN_FLAG_DCB_ENABLED;
+                        priv->cee_params.dcb_cfg.pfc_state = false;
+                }
+        }
+#endif /* CONFIG_MLX4_EN_DCB */
+
         return 0;
 }
 
@@ -1201,8 +1212,8 @@ static void mlx4_en_netpoll(struct net_device *dev)
         struct mlx4_en_cq *cq;
         int i;
 
-        for (i = 0; i < priv->rx_ring_num; i++) {
-                cq = priv->rx_cq[i];
+        for (i = 0; i < priv->tx_ring_num; i++) {
+                cq = priv->tx_cq[i];
                 napi_schedule(&cq->napi);
         }
 }
@@ -1696,10 +1707,9 @@ int mlx4_en_start_port(struct net_device *dev)
         /* Schedule multicast task to populate multicast list */
         queue_work(mdev->workqueue, &priv->rx_mode_task);
 
-#ifdef CONFIG_MLX4_EN_VXLAN
         if (priv->mdev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN)
-                vxlan_get_rx_port(dev);
-#endif
+                udp_tunnel_get_rx_info(dev);
+
         priv->port_up = true;
         netif_tx_start_all_queues(dev);
         netif_device_attach(dev);
@@ -2359,7 +2369,6 @@ static int mlx4_en_get_phys_port_id(struct net_device *dev,
         return 0;
 }
 
-#ifdef CONFIG_MLX4_EN_VXLAN
 static void mlx4_en_add_vxlan_offloads(struct work_struct *work)
 {
         int ret;
@@ -2409,15 +2418,19 @@ static void mlx4_en_del_vxlan_offloads(struct work_struct *work)
 }
 
 static void mlx4_en_add_vxlan_port(struct  net_device *dev,
-                                   sa_family_t sa_family, __be16 port)
+                                   struct udp_tunnel_info *ti)
 {
         struct mlx4_en_priv *priv = netdev_priv(dev);
+        __be16 port = ti->port;
         __be16 current_port;
 
-        if (priv->mdev->dev->caps.tunnel_offload_mode != MLX4_TUNNEL_OFFLOAD_MODE_VXLAN)
+        if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
+                return;
+
+        if (ti->sa_family != AF_INET)
                 return;
 
-        if (sa_family == AF_INET6)
+        if (priv->mdev->dev->caps.tunnel_offload_mode != MLX4_TUNNEL_OFFLOAD_MODE_VXLAN)
                 return;
 
         current_port = priv->vxlan_port;
@@ -2432,15 +2445,19 @@ static void mlx4_en_add_vxlan_port(struct  net_device *dev,
 }
 
 static void mlx4_en_del_vxlan_port(struct  net_device *dev,
-                                   sa_family_t sa_family, __be16 port)
+                                   struct udp_tunnel_info *ti)
 {
         struct mlx4_en_priv *priv = netdev_priv(dev);
+        __be16 port = ti->port;
         __be16 current_port;
 
-        if (priv->mdev->dev->caps.tunnel_offload_mode != MLX4_TUNNEL_OFFLOAD_MODE_VXLAN)
+        if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
                 return;
 
-        if (sa_family == AF_INET6)
+        if (ti->sa_family != AF_INET)
+                return;
+
+        if (priv->mdev->dev->caps.tunnel_offload_mode != MLX4_TUNNEL_OFFLOAD_MODE_VXLAN)
                 return;
 
         current_port = priv->vxlan_port;
@@ -2475,7 +2492,6 @@ static netdev_features_t mlx4_en_features_check(struct sk_buff *skb,
 
         return features;
 }
-#endif
 
 static int mlx4_en_set_tx_maxrate(struct net_device *dev, int queue_index, u32 maxrate)
 {
@@ -2528,11 +2544,9 @@ static const struct net_device_ops mlx4_netdev_ops = {
         .ndo_rx_flow_steer      = mlx4_en_filter_rfs,
 #endif
         .ndo_get_phys_port_id   = mlx4_en_get_phys_port_id,
-#ifdef CONFIG_MLX4_EN_VXLAN
-        .ndo_add_vxlan_port     = mlx4_en_add_vxlan_port,
-        .ndo_del_vxlan_port     = mlx4_en_del_vxlan_port,
+        .ndo_udp_tunnel_add     = mlx4_en_add_vxlan_port,
+        .ndo_udp_tunnel_del     = mlx4_en_del_vxlan_port,
         .ndo_features_check     = mlx4_en_features_check,
-#endif
         .ndo_set_tx_maxrate     = mlx4_en_set_tx_maxrate,
 };
 
@@ -2566,11 +2580,9 @@ static const struct net_device_ops mlx4_netdev_ops_master = {
         .ndo_rx_flow_steer      = mlx4_en_filter_rfs,
 #endif
         .ndo_get_phys_port_id   = mlx4_en_get_phys_port_id,
-#ifdef CONFIG_MLX4_EN_VXLAN
-        .ndo_add_vxlan_port     = mlx4_en_add_vxlan_port,
-        .ndo_del_vxlan_port     = mlx4_en_del_vxlan_port,
+        .ndo_udp_tunnel_add     = mlx4_en_add_vxlan_port,
+        .ndo_udp_tunnel_del     = mlx4_en_del_vxlan_port,
         .ndo_features_check     = mlx4_en_features_check,
-#endif
         .ndo_set_tx_maxrate     = mlx4_en_set_tx_maxrate,
 };
 
@@ -2836,6 +2848,9 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port,
         struct mlx4_en_priv *priv;
         int i;
         int err;
+#ifdef CONFIG_MLX4_EN_DCB
+        struct tc_configuration *tc;
+#endif
 
         dev = alloc_etherdev_mqs(sizeof(struct mlx4_en_priv),
                                  MAX_TX_RINGS, MAX_RX_RINGS);
@@ -2861,10 +2876,8 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port,
         INIT_WORK(&priv->linkstate_task, mlx4_en_linkstate);
         INIT_DELAYED_WORK(&priv->stats_task, mlx4_en_do_get_stats);
         INIT_DELAYED_WORK(&priv->service_task, mlx4_en_service_task);
-#ifdef CONFIG_MLX4_EN_VXLAN
         INIT_WORK(&priv->vxlan_add_task, mlx4_en_add_vxlan_offloads);
         INIT_WORK(&priv->vxlan_del_task, mlx4_en_del_vxlan_offloads);
-#endif
 #ifdef CONFIG_RFS_ACCEL
         INIT_LIST_HEAD(&priv->filters);
         spin_lock_init(&priv->filters_lock);
@@ -2904,6 +2917,17 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port,
         priv->msg_enable = MLX4_EN_MSG_LEVEL;
 #ifdef CONFIG_MLX4_EN_DCB
         if (!mlx4_is_slave(priv->mdev->dev)) {
+                priv->cee_params.dcbx_cap = DCB_CAP_DCBX_VER_CEE |
+                                            DCB_CAP_DCBX_HOST |
+                                            DCB_CAP_DCBX_VER_IEEE;
+                priv->flags |= MLX4_EN_DCB_ENABLED;
+                priv->cee_params.dcb_cfg.pfc_state = false;
+
+                for (i = 0; i < MLX4_EN_NUM_UP; i++) {
+                        tc = &priv->cee_params.dcb_cfg.tc_config[i];
+                        tc->dcb_pfc = pfc_disabled;
+                }
+
                 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ETS_CFG) {
                         dev->dcbnl_ops = &mlx4_en_dcbnl_ops;
                 } else {
diff --git a/drivers/net/ethernet/mellanox/mlx4/fw.c b/drivers/net/ethernet/mellanox/mlx4/fw.c
index e97094598b2d..f4497cf4d06d 100644
--- a/drivers/net/ethernet/mellanox/mlx4/fw.c
+++ b/drivers/net/ethernet/mellanox/mlx4/fw.c
@@ -1128,6 +1128,7 @@ int mlx4_QUERY_PORT(struct mlx4_dev *dev, int port, struct mlx4_port_cap *port_c
                 port_cap->max_pkeys        = 1 << (field & 0xf);
                 MLX4_GET(field, outbox, QUERY_PORT_MAX_VL_OFFSET);
                 port_cap->max_vl           = field & 0xf;
+                port_cap->max_tc_eth       = field >> 4;
                 MLX4_GET(field, outbox, QUERY_PORT_MAX_MACVLAN_OFFSET);
                 port_cap->log_max_macs  = field & 0xf;
                 port_cap->log_max_vlans = field >> 4;
diff --git a/drivers/net/ethernet/mellanox/mlx4/fw.h b/drivers/net/ethernet/mellanox/mlx4/fw.h
index 7ea258af636a..cdbd76f10ced 100644
--- a/drivers/net/ethernet/mellanox/mlx4/fw.h
+++ b/drivers/net/ethernet/mellanox/mlx4/fw.h
@@ -53,6 +53,7 @@ struct mlx4_port_cap {
         int ib_mtu;
         int max_port_width;
         int max_vl;
+        int max_tc_eth;
         int max_gids;
         int max_pkeys;
         u64 def_mac;
diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c
index 546fab0ecc3b..b673a5fc6b6c 100644
--- a/drivers/net/ethernet/mellanox/mlx4/main.c
+++ b/drivers/net/ethernet/mellanox/mlx4/main.c
@@ -292,6 +292,7 @@ static int _mlx4_dev_port(struct mlx4_dev *dev, int port,
         dev->caps.pkey_table_len[port] = port_cap->max_pkeys;
         dev->caps.port_width_cap[port] = port_cap->max_port_width;
         dev->caps.eth_mtu_cap[port]    = port_cap->eth_mtu;
+        dev->caps.max_tc_eth           = port_cap->max_tc_eth;
         dev->caps.def_mac[port]        = port_cap->def_mac;
         dev->caps.supported_type[port] = port_cap->supported_port_types;
         dev->caps.suggested_type[port] = port_cap->suggested_type;
diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
index 467d47ed2c39..d39bf594abe4 100644
--- a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
+++ b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
@@ -448,6 +448,27 @@ struct mlx4_en_frag_info {
 
 #define MLX4_EN_TC_ETS 7
 
+enum dcb_pfc_type {
+        pfc_disabled = 0,
+        pfc_enabled_full,
+        pfc_enabled_tx,
+        pfc_enabled_rx
+};
+
+struct tc_configuration {
+        enum dcb_pfc_type  dcb_pfc;
+};
+
+struct mlx4_en_cee_config {
+        bool    pfc_state;
+        struct  tc_configuration tc_config[MLX4_EN_NUM_UP];
+};
+
+struct mlx4_en_cee_params {
+        u8 dcbx_cap;
+        struct mlx4_en_cee_config dcb_cfg;
+};
+
 #endif
 
 struct ethtool_flow_id {
@@ -467,6 +488,9 @@ enum {
         MLX4_EN_FLAG_RX_FILTER_NEEDED   = (1 << 3),
         MLX4_EN_FLAG_FORCE_PROMISC      = (1 << 4),
         MLX4_EN_FLAG_RX_CSUM_NON_TCP_UDP        = (1 << 5),
+#ifdef CONFIG_MLX4_EN_DCB
+        MLX4_EN_FLAG_DCB_ENABLED        = (1 << 6),
+#endif
 };
 
 #define PORT_BEACON_MAX_LIMIT (65535)
@@ -545,10 +569,8 @@ struct mlx4_en_priv {
         struct work_struct linkstate_task;
         struct delayed_work stats_task;
         struct delayed_work service_task;
-#ifdef CONFIG_MLX4_EN_VXLAN
         struct work_struct vxlan_add_task;
         struct work_struct vxlan_del_task;
-#endif
         struct mlx4_en_perf_stats pstats;
         struct mlx4_en_pkt_stats pkstats;
         struct mlx4_en_counter_stats pf_stats;
@@ -570,9 +592,11 @@ struct mlx4_en_priv {
         u32 counter_index;
 
 #ifdef CONFIG_MLX4_EN_DCB
+#define MLX4_EN_DCB_ENABLED     0x3
         struct ieee_ets ets;
         u16 maxrate[IEEE_8021QAZ_MAX_TCS];
         enum dcbnl_cndd_states cndd_state[IEEE_8021QAZ_MAX_TCS];
+        struct mlx4_en_cee_params cee_params;
 #endif
 #ifdef CONFIG_RFS_ACCEL
         spinlock_t filters_lock;
diff --git a/drivers/net/ethernet/mellanox/mlx4/port.c b/drivers/net/ethernet/mellanox/mlx4/port.c
index 087b23b320cb..3d2095e5c61c 100644
--- a/drivers/net/ethernet/mellanox/mlx4/port.c
+++ b/drivers/net/ethernet/mellanox/mlx4/port.c
@@ -52,6 +52,7 @@
 
 #define MLX4_FLAG_V_IGNORE_FCS_MASK             0x2
 #define MLX4_IGNORE_FCS_MASK                    0x1
+#define MLNX4_TX_MAX_NUMBER                     8
 
 void mlx4_init_mac_table(struct mlx4_dev *dev, struct mlx4_mac_table *table)
 {
@@ -2015,3 +2016,14 @@ out:
         return ret;
 }
 EXPORT_SYMBOL(mlx4_get_module_info);
+
+int mlx4_max_tc(struct mlx4_dev *dev)
+{
+        u8 num_tc = dev->caps.max_tc_eth;
+
+        if (!num_tc)
+                num_tc = MLNX4_TX_MAX_NUMBER;
+
+        return num_tc;
+}
+EXPORT_SYMBOL(mlx4_max_tc);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
index 9ea7b583096a..c4f450f1c658 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
@@ -1,11 +1,13 @@
 obj-$(CONFIG_MLX5_CORE)         += mlx5_core.o
 
 mlx5_core-y :=  main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \
-                health.o mcg.o cq.o srq.o alloc.o qp.o port.o mr.o pd.o   \
-                mad.o transobj.o vport.o sriov.o fs_cmd.o fs_core.o fs_counters.o
+                health.o mcg.o cq.o srq.o alloc.o qp.o port.o mr.o pd.o \
+                mad.o transobj.o vport.o sriov.o fs_cmd.o fs_core.o \
+                fs_counters.o rl.o
 
 mlx5_core-$(CONFIG_MLX5_CORE_EN) += wq.o eswitch.o \
                 en_main.o en_fs.o en_ethtool.o en_tx.o en_rx.o \
-                en_txrx.o en_clock.o vxlan.o en_tc.o en_arfs.o
+                en_rx_am.o en_txrx.o en_clock.o vxlan.o en_tc.o \
+                en_arfs.o
 
 mlx5_core-$(CONFIG_MLX5_CORE_EN_DCB) +=  en_dcbnl.o
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
index baa991a23475..b97511bf4c7b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -79,6 +79,7 @@
 
 #define MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ                 (64 * 1024)
 #define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC      0x10
+#define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE 0x3
 #define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS      0x20
 #define MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC      0x10
 #define MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS      0x20
@@ -88,6 +89,7 @@
 #define MLX5E_LOG_INDIR_RQT_SIZE       0x7
 #define MLX5E_INDIR_RQT_SIZE           BIT(MLX5E_LOG_INDIR_RQT_SIZE)
 #define MLX5E_MAX_NUM_CHANNELS         (MLX5E_INDIR_RQT_SIZE >> 1)
+#define MLX5E_MAX_NUM_SQS              (MLX5E_MAX_NUM_CHANNELS * MLX5E_MAX_NUM_TC)
 #define MLX5E_TX_CQ_POLL_BUDGET        128
 #define MLX5E_UPDATE_STATS_INTERVAL    200 /* msecs */
 #define MLX5E_SQ_BF_BUDGET             16
@@ -143,11 +145,32 @@ struct mlx5e_umr_wqe {
         struct mlx5_wqe_data_seg       data;
 };
 
+static const char mlx5e_priv_flags[][ETH_GSTRING_LEN] = {
+        "rx_cqe_moder",
+};
+
+enum mlx5e_priv_flag {
+        MLX5E_PFLAG_RX_CQE_BASED_MODER = (1 << 0),
+};
+
+#define MLX5E_SET_PRIV_FLAG(priv, pflag, enable)    \
+        do {                                        \
+                if (enable)                         \
+                        priv->pflags |= pflag;      \
+                else                                \
+                        priv->pflags &= ~pflag;     \
+        } while (0)
+
 #ifdef CONFIG_MLX5_CORE_EN_DCB
 #define MLX5E_MAX_BW_ALLOC 100 /* Max percentage of BW allocation */
 #define MLX5E_MIN_BW_ALLOC 1   /* Min percentage of BW allocation */
 #endif
 
+struct mlx5e_cq_moder {
+        u16 usec;
+        u16 pkts;
+};
+
 struct mlx5e_params {
         u8  log_sq_size;
         u8  rq_wq_type;
@@ -156,12 +179,11 @@ struct mlx5e_params {
         u8  log_rq_size;
         u16 num_channels;
         u8  num_tc;
+        u8  rx_cq_period_mode;
         bool rx_cqe_compress_admin;
         bool rx_cqe_compress;
-        u16 rx_cq_moderation_usec;
-        u16 rx_cq_moderation_pkts;
-        u16 tx_cq_moderation_usec;
-        u16 tx_cq_moderation_pkts;
+        struct mlx5e_cq_moder rx_cq_moderation;
+        struct mlx5e_cq_moder tx_cq_moderation;
         u16 min_rx_wqes;
         bool lro_en;
         u32 lro_wqe_sz;
@@ -173,6 +195,7 @@ struct mlx5e_params {
 #ifdef CONFIG_MLX5_CORE_EN_DCB
         struct ieee_ets ets;
 #endif
+        bool rx_am_enabled;
 };
 
 struct mlx5e_tstamp {
@@ -191,6 +214,7 @@ struct mlx5e_tstamp {
 enum {
         MLX5E_RQ_STATE_POST_WQES_ENABLE,
         MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS,
+        MLX5E_RQ_STATE_AM,
 };
 
 struct mlx5e_cq {
@@ -198,6 +222,7 @@ struct mlx5e_cq {
         struct mlx5_cqwq           wq;
 
         /* data path - accessed per napi poll */
+        u16                        event_ctr;
         struct napi_struct        *napi;
         struct mlx5_core_cq        mcq;
         struct mlx5e_channel      *channel;
@@ -225,6 +250,30 @@ struct mlx5e_dma_info {
         dma_addr_t      addr;
 };
 
+struct mlx5e_rx_am_stats {
+        int ppms; /* packets per msec */
+        int epms; /* events per msec */
+};
+
+struct mlx5e_rx_am_sample {
+        ktime_t         time;
+        unsigned int    pkt_ctr;
+        u16             event_ctr;
+};
+
+struct mlx5e_rx_am { /* Adaptive Moderation */
+        u8                                      state;
+        struct mlx5e_rx_am_stats                prev_stats;
+        struct mlx5e_rx_am_sample               start_sample;
+        struct work_struct                      work;
+        u8                                      profile_ix;
+        u8                                      mode;
+        u8                                      tune_state;
+        u8                                      steps_right;
+        u8                                      steps_left;
+        u8                                      tired;
+};
+
 struct mlx5e_rq {
         /* data path */
         struct mlx5_wq_ll      wq;
@@ -245,6 +294,8 @@ struct mlx5e_rq {
         unsigned long          state;
         int                    ix;
 
+        struct mlx5e_rx_am     am; /* Adaptive Moderation */
+
         /* control */
         struct mlx5_wq_ctrl    wq_ctrl;
         u8                     wq_type;
@@ -354,6 +405,7 @@ struct mlx5e_sq {
         struct mlx5e_channel      *channel;
         int                        tc;
         struct mlx5e_ico_wqe_info *ico_wqe_info;
+        u32                        rate_limit;
 } ____cacheline_aligned_in_smp;
 
 static inline bool mlx5e_sq_has_room_for(struct mlx5e_sq *sq, u16 n)
@@ -530,6 +582,7 @@ struct mlx5e_priv {
         u32                        indir_rqtn;
         u32                        indir_tirn[MLX5E_NUM_INDIR_TIRS];
         struct mlx5e_direct_tir    direct_tir[MLX5E_MAX_NUM_CHANNELS];
+        u32                        tx_rates[MLX5E_MAX_NUM_SQS];
 
         struct mlx5e_flow_steering fs;
         struct mlx5e_vxlan_db      vxlan;
@@ -540,6 +593,7 @@ struct mlx5e_priv {
         struct work_struct         set_rx_mode_work;
         struct delayed_work        update_stats_work;
 
+        u32                        pflags;
         struct mlx5_core_dev      *mdev;
         struct net_device         *netdev;
         struct mlx5e_stats         stats;
@@ -562,6 +616,7 @@ enum mlx5e_link_mode {
         MLX5E_10GBASE_ER         = 14,
         MLX5E_40GBASE_SR4        = 15,
         MLX5E_40GBASE_LR4        = 16,
+        MLX5E_50GBASE_SR2        = 18,
         MLX5E_100GBASE_CR4       = 20,
         MLX5E_100GBASE_SR4       = 21,
         MLX5E_100GBASE_KR4       = 22,
@@ -579,6 +634,9 @@ enum mlx5e_link_mode {
 
 #define MLX5E_PROT_MASK(link_mode) (1 << link_mode)
 
+
+void mlx5e_build_ptys2ethtool_map(void);
+
 void mlx5e_send_nop(struct mlx5e_sq *sq, bool notify_hw);
 u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb,
                        void *accel_priv, select_queue_fallback_t fallback);
@@ -612,6 +670,10 @@ void mlx5e_free_rx_fragmented_mpwqe(struct mlx5e_rq *rq,
                                     struct mlx5e_mpw_info *wi);
 struct mlx5_cqe64 *mlx5e_get_cqe(struct mlx5e_cq *cq);
 
+void mlx5e_rx_am(struct mlx5e_rq *rq);
+void mlx5e_rx_am_work(struct work_struct *work);
+struct mlx5e_cq_moder mlx5e_am_get_def_profile(u8 rx_cq_period_mode);
+
 void mlx5e_update_stats(struct mlx5e_priv *priv);
 
 int mlx5e_create_flow_steering(struct mlx5e_priv *priv);
@@ -647,6 +709,9 @@ void mlx5e_build_default_indir_rqt(struct mlx5_core_dev *mdev,
                                    int num_channels);
 int mlx5e_get_max_linkspeed(struct mlx5_core_dev *mdev, u32 *speed);
 
+void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params,
+                                 u8 cq_period_mode);
+
 static inline void mlx5e_tx_notify_hw(struct mlx5e_sq *sq,
                                       struct mlx5_wqe_ctrl_seg *ctrl, int bf_sz)
 {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c b/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c
index b2db180ae2a5..e6883132b555 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c
@@ -191,7 +191,6 @@ static int mlx5e_dcbnl_ieee_setpfc(struct net_device *dev,
 {
         struct mlx5e_priv *priv = netdev_priv(dev);
         struct mlx5_core_dev *mdev = priv->mdev;
-        enum mlx5_port_status ps;
         u8 curr_pfc_en;
         int ret;
 
@@ -200,14 +199,8 @@ static int mlx5e_dcbnl_ieee_setpfc(struct net_device *dev,
         if (pfc->pfc_en == curr_pfc_en)
                 return 0;
 
-        mlx5_query_port_admin_status(mdev, &ps);
-        if (ps == MLX5_PORT_UP)
-                mlx5_set_port_admin_status(mdev, MLX5_PORT_DOWN);
-
         ret = mlx5_set_port_pfc(mdev, pfc->pfc_en, pfc->pfc_en);
-
-        if (ps == MLX5_PORT_UP)
-                mlx5_set_port_admin_status(mdev, MLX5_PORT_UP);
+        mlx5_toggle_port_link(mdev);
 
         return ret;
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
index e667a870e0c2..b29684d9fcd6 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
@@ -48,123 +48,85 @@ static void mlx5e_get_drvinfo(struct net_device *dev,
                 sizeof(drvinfo->bus_info));
 }
 
-static const struct {
-        u32 supported;
-        u32 advertised;
+struct ptys2ethtool_config {
+        __ETHTOOL_DECLARE_LINK_MODE_MASK(supported);
+        __ETHTOOL_DECLARE_LINK_MODE_MASK(advertised);
         u32 speed;
-} ptys2ethtool_table[MLX5E_LINK_MODES_NUMBER] = {
-        [MLX5E_1000BASE_CX_SGMII] = {
-                .supported  = SUPPORTED_1000baseKX_Full,
-                .advertised = ADVERTISED_1000baseKX_Full,
-                .speed      = 1000,
-        },
-        [MLX5E_1000BASE_KX] = {
-                .supported  = SUPPORTED_1000baseKX_Full,
-                .advertised = ADVERTISED_1000baseKX_Full,
-                .speed      = 1000,
-        },
-        [MLX5E_10GBASE_CX4] = {
-                .supported  = SUPPORTED_10000baseKX4_Full,
-                .advertised = ADVERTISED_10000baseKX4_Full,
-                .speed      = 10000,
-        },
-        [MLX5E_10GBASE_KX4] = {
-                .supported  = SUPPORTED_10000baseKX4_Full,
-                .advertised = ADVERTISED_10000baseKX4_Full,
-                .speed      = 10000,
-        },
-        [MLX5E_10GBASE_KR] = {
-                .supported  = SUPPORTED_10000baseKR_Full,
-                .advertised = ADVERTISED_10000baseKR_Full,
-                .speed      = 10000,
-        },
-        [MLX5E_20GBASE_KR2] = {
-                .supported  = SUPPORTED_20000baseKR2_Full,
-                .advertised = ADVERTISED_20000baseKR2_Full,
-                .speed      = 20000,
-        },
-        [MLX5E_40GBASE_CR4] = {
-                .supported  = SUPPORTED_40000baseCR4_Full,
-                .advertised = ADVERTISED_40000baseCR4_Full,
-                .speed      = 40000,
-        },
-        [MLX5E_40GBASE_KR4] = {
-                .supported  = SUPPORTED_40000baseKR4_Full,
-                .advertised = ADVERTISED_40000baseKR4_Full,
-                .speed      = 40000,
-        },
-        [MLX5E_56GBASE_R4] = {
-                .supported  = SUPPORTED_56000baseKR4_Full,
-                .advertised = ADVERTISED_56000baseKR4_Full,
-                .speed      = 56000,
-        },
-        [MLX5E_10GBASE_CR] = {
-                .supported  = SUPPORTED_10000baseKR_Full,
-                .advertised = ADVERTISED_10000baseKR_Full,
-                .speed      = 10000,
-        },
-        [MLX5E_10GBASE_SR] = {
-                .supported  = SUPPORTED_10000baseKR_Full,
-                .advertised = ADVERTISED_10000baseKR_Full,
-                .speed      = 10000,
-        },
-        [MLX5E_10GBASE_ER] = {
-                .supported  = SUPPORTED_10000baseKR_Full,
-                .advertised = ADVERTISED_10000baseKR_Full,
-                .speed      = 10000,
-        },
-        [MLX5E_40GBASE_SR4] = {
-                .supported  = SUPPORTED_40000baseSR4_Full,
-                .advertised = ADVERTISED_40000baseSR4_Full,
-                .speed      = 40000,
-        },
-        [MLX5E_40GBASE_LR4] = {
-                .supported  = SUPPORTED_40000baseLR4_Full,
-                .advertised = ADVERTISED_40000baseLR4_Full,
-                .speed      = 40000,
-        },
-        [MLX5E_100GBASE_CR4] = {
-                .speed      = 100000,
-        },
-        [MLX5E_100GBASE_SR4] = {
-                .speed      = 100000,
-        },
-        [MLX5E_100GBASE_KR4] = {
-                .speed      = 100000,
-        },
-        [MLX5E_100GBASE_LR4] = {
-                .speed      = 100000,
-        },
-        [MLX5E_100BASE_TX]   = {
-                .speed      = 100,
-        },
-        [MLX5E_1000BASE_T]    = {
-                .supported  = SUPPORTED_1000baseT_Full,
-                .advertised = ADVERTISED_1000baseT_Full,
-                .speed      = 1000,
-        },
-        [MLX5E_10GBASE_T]    = {
-                .supported  = SUPPORTED_10000baseT_Full,
-                .advertised = ADVERTISED_10000baseT_Full,
-                .speed      = 1000,
-        },
-        [MLX5E_25GBASE_CR]   = {
-                .speed      = 25000,
-        },
-        [MLX5E_25GBASE_KR]   = {
-                .speed      = 25000,
-        },
-        [MLX5E_25GBASE_SR]   = {
-                .speed      = 25000,
-        },
-        [MLX5E_50GBASE_CR2]  = {
-                .speed      = 50000,
-        },
-        [MLX5E_50GBASE_KR2]  = {
-                .speed      = 50000,
-        },
 };
 
+static struct ptys2ethtool_config ptys2ethtool_table[MLX5E_LINK_MODES_NUMBER];
+
+#define MLX5_BUILD_PTYS2ETHTOOL_CONFIG(reg_, speed_, ...)               \
+        ({                                                              \
+                struct ptys2ethtool_config *cfg;                        \
+                const unsigned int modes[] = { __VA_ARGS__ };           \
+                unsigned int i;                                         \
+                cfg = &ptys2ethtool_table[reg_];                        \
+                cfg->speed = speed_;                                    \
+                bitmap_zero(cfg->supported,                             \
+                            __ETHTOOL_LINK_MODE_MASK_NBITS);            \
+                bitmap_zero(cfg->advertised,                            \
+                            __ETHTOOL_LINK_MODE_MASK_NBITS);            \
+                for (i = 0 ; i < ARRAY_SIZE(modes) ; ++i) {             \
+                        __set_bit(modes[i], cfg->supported);            \
+                        __set_bit(modes[i], cfg->advertised);           \
+                }                                                       \
+        })
+
+void mlx5e_build_ptys2ethtool_map(void)
+{
+        MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_1000BASE_CX_SGMII, SPEED_1000,
+                                       ETHTOOL_LINK_MODE_1000baseKX_Full_BIT);
+        MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_1000BASE_KX, SPEED_1000,
+                                       ETHTOOL_LINK_MODE_1000baseKX_Full_BIT);
+        MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_10GBASE_CX4, SPEED_10000,
+                                       ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT);
+        MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_10GBASE_KX4, SPEED_10000,
+                                       ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT);
+        MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_10GBASE_KR, SPEED_10000,
+                                       ETHTOOL_LINK_MODE_10000baseKR_Full_BIT);
+        MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_20GBASE_KR2, SPEED_20000,
+                                       ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT);
+        MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_40GBASE_CR4, SPEED_40000,
+                                       ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT);
+        MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_40GBASE_KR4, SPEED_40000,
+                                       ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT);
+        MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_56GBASE_R4, SPEED_56000,
+                                       ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT);
+        MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_10GBASE_CR, SPEED_10000,
+                                       ETHTOOL_LINK_MODE_10000baseKR_Full_BIT);
+        MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_10GBASE_SR, SPEED_10000,
+                                       ETHTOOL_LINK_MODE_10000baseKR_Full_BIT);
+        MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_10GBASE_ER, SPEED_10000,
+                                       ETHTOOL_LINK_MODE_10000baseKR_Full_BIT);
+        MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_40GBASE_SR4, SPEED_40000,
+                                       ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT);
+        MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_40GBASE_LR4, SPEED_40000,
+                                       ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT);
+        MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_50GBASE_SR2, SPEED_50000,
+                                       ETHTOOL_LINK_MODE_50000baseSR2_Full_BIT);
+        MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_100GBASE_CR4, SPEED_100000,
+                                       ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT);
+        MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_100GBASE_SR4, SPEED_100000,
+                                       ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT);
+        MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_100GBASE_KR4, SPEED_100000,
+                                       ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT);
+        MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_100GBASE_LR4, SPEED_100000,
+                                       ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT);
+        MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_10GBASE_T, SPEED_10000,
+                                       ETHTOOL_LINK_MODE_10000baseT_Full_BIT);
+        MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_25GBASE_CR, SPEED_25000,
+                                       ETHTOOL_LINK_MODE_25000baseCR_Full_BIT);
+        MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_25GBASE_KR, SPEED_25000,
+                                       ETHTOOL_LINK_MODE_25000baseKR_Full_BIT);
+        MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_25GBASE_SR, SPEED_25000,
+                                       ETHTOOL_LINK_MODE_25000baseSR_Full_BIT);
+        MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_50GBASE_CR2, SPEED_50000,
+                                       ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT);
+        MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_50GBASE_KR2, SPEED_50000,
+                                       ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT);
+}
+
 static unsigned long mlx5e_query_pfc_combined(struct mlx5e_priv *priv)
 {
         struct mlx5_core_dev *mdev = priv->mdev;
@@ -200,6 +162,8 @@ static int mlx5e_get_sset_count(struct net_device *dev, int sset)
                        MLX5E_NUM_RQ_STATS(priv) +
                        MLX5E_NUM_SQ_STATS(priv) +
                        MLX5E_NUM_PFC_COUNTERS(priv);
+        case ETH_SS_PRIV_FLAGS:
+                return ARRAY_SIZE(mlx5e_priv_flags);
         /* fallthrough */
         default:
                 return -EOPNOTSUPP;
@@ -272,9 +236,12 @@ static void mlx5e_get_strings(struct net_device *dev,
                               uint32_t stringset, uint8_t *data)
 {
         struct mlx5e_priv *priv = netdev_priv(dev);
+        int i;
 
         switch (stringset) {
         case ETH_SS_PRIV_FLAGS:
+                for (i = 0; i < ARRAY_SIZE(mlx5e_priv_flags); i++)
+                        strcpy(data + i * ETH_GSTRING_LEN, mlx5e_priv_flags[i]);
                 break;
 
         case ETH_SS_TEST:
@@ -519,10 +486,11 @@ static int mlx5e_get_coalesce(struct net_device *netdev,
         if (!MLX5_CAP_GEN(priv->mdev, cq_moderation))
                 return -ENOTSUPP;
 
-        coal->rx_coalesce_usecs       = priv->params.rx_cq_moderation_usec;
-        coal->rx_max_coalesced_frames = priv->params.rx_cq_moderation_pkts;
-        coal->tx_coalesce_usecs       = priv->params.tx_cq_moderation_usec;
-        coal->tx_max_coalesced_frames = priv->params.tx_cq_moderation_pkts;
+        coal->rx_coalesce_usecs       = priv->params.rx_cq_moderation.usec;
+        coal->rx_max_coalesced_frames = priv->params.rx_cq_moderation.pkts;
+        coal->tx_coalesce_usecs       = priv->params.tx_cq_moderation.usec;
+        coal->tx_max_coalesced_frames = priv->params.tx_cq_moderation.pkts;
+        coal->use_adaptive_rx_coalesce = priv->params.rx_am_enabled;
 
         return 0;
 }
@@ -533,6 +501,10 @@ static int mlx5e_set_coalesce(struct net_device *netdev,
         struct mlx5e_priv *priv    = netdev_priv(netdev);
         struct mlx5_core_dev *mdev = priv->mdev;
         struct mlx5e_channel *c;
+        bool restart =
+                !!coal->use_adaptive_rx_coalesce != priv->params.rx_am_enabled;
+        bool was_opened;
+        int err = 0;
         int tc;
         int i;
 
@@ -540,12 +512,19 @@ static int mlx5e_set_coalesce(struct net_device *netdev,
                 return -ENOTSUPP;
 
         mutex_lock(&priv->state_lock);
-        priv->params.tx_cq_moderation_usec = coal->tx_coalesce_usecs;
-        priv->params.tx_cq_moderation_pkts = coal->tx_max_coalesced_frames;
-        priv->params.rx_cq_moderation_usec = coal->rx_coalesce_usecs;
-        priv->params.rx_cq_moderation_pkts = coal->rx_max_coalesced_frames;
 
-        if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
+        was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
+        if (was_opened && restart) {
+                mlx5e_close_locked(netdev);
+                priv->params.rx_am_enabled = !!coal->use_adaptive_rx_coalesce;
+        }
+
+        priv->params.tx_cq_moderation.usec = coal->tx_coalesce_usecs;
+        priv->params.tx_cq_moderation.pkts = coal->tx_max_coalesced_frames;
+        priv->params.rx_cq_moderation.usec = coal->rx_coalesce_usecs;
+        priv->params.rx_cq_moderation.pkts = coal->rx_max_coalesced_frames;
+
+        if (!was_opened || restart)
                 goto out;
 
         for (i = 0; i < priv->params.num_channels; ++i) {
@@ -564,35 +543,37 @@ static int mlx5e_set_coalesce(struct net_device *netdev,
         }
 
 out:
+        if (was_opened && restart)
+                err = mlx5e_open_locked(netdev);
+
         mutex_unlock(&priv->state_lock);
-        return 0;
+        return err;
 }
 
-static u32 ptys2ethtool_supported_link(u32 eth_proto_cap)
+static void ptys2ethtool_supported_link(unsigned long *supported_modes,
+                                        u32 eth_proto_cap)
 {
-        int i;
-        u32 supported_modes = 0;
+        int proto;
 
-        for (i = 0; i < MLX5E_LINK_MODES_NUMBER; ++i) {
-                if (eth_proto_cap & MLX5E_PROT_MASK(i))
-                        supported_modes |= ptys2ethtool_table[i].supported;
-        }
-        return supported_modes;
+        for_each_set_bit(proto, (unsigned long *)&eth_proto_cap, MLX5E_LINK_MODES_NUMBER)
+                bitmap_or(supported_modes, supported_modes,
+                          ptys2ethtool_table[proto].supported,
+                          __ETHTOOL_LINK_MODE_MASK_NBITS);
 }
 
-static u32 ptys2ethtool_adver_link(u32 eth_proto_cap)
+static void ptys2ethtool_adver_link(unsigned long *advertising_modes,
+                                    u32 eth_proto_cap)
 {
-        int i;
-        u32 advertising_modes = 0;
+        int proto;
 
-        for (i = 0; i < MLX5E_LINK_MODES_NUMBER; ++i) {
-                if (eth_proto_cap & MLX5E_PROT_MASK(i))
-                        advertising_modes |= ptys2ethtool_table[i].advertised;
-        }
-        return advertising_modes;
+        for_each_set_bit(proto, (unsigned long *)&eth_proto_cap, MLX5E_LINK_MODES_NUMBER)
+                bitmap_or(advertising_modes, advertising_modes,
+                          ptys2ethtool_table[proto].advertised,
+                          __ETHTOOL_LINK_MODE_MASK_NBITS);
 }
 
-static u32 ptys2ethtool_supported_port(u32 eth_proto_cap)
+static void ptys2ethtool_supported_port(struct ethtool_link_ksettings *link_ksettings,
+                                        u32 eth_proto_cap)
 {
         if (eth_proto_cap & (MLX5E_PROT_MASK(MLX5E_10GBASE_CR)
                            | MLX5E_PROT_MASK(MLX5E_10GBASE_SR)
@@ -600,7 +581,7 @@ static u32 ptys2ethtool_supported_port(u32 eth_proto_cap)
                            | MLX5E_PROT_MASK(MLX5E_40GBASE_SR4)
                            | MLX5E_PROT_MASK(MLX5E_100GBASE_SR4)
                            | MLX5E_PROT_MASK(MLX5E_1000BASE_CX_SGMII))) {
-                return SUPPORTED_FIBRE;
+                ethtool_link_ksettings_add_link_mode(link_ksettings, supported, FIBRE);
         }
 
         if (eth_proto_cap & (MLX5E_PROT_MASK(MLX5E_100GBASE_KR4)
@@ -608,9 +589,8 @@ static u32 ptys2ethtool_supported_port(u32 eth_proto_cap)
                            | MLX5E_PROT_MASK(MLX5E_10GBASE_KR)
                            | MLX5E_PROT_MASK(MLX5E_10GBASE_KX4)
                            | MLX5E_PROT_MASK(MLX5E_1000BASE_KX))) {
-                return SUPPORTED_Backplane;
+                ethtool_link_ksettings_add_link_mode(link_ksettings, supported, Backplane);
         }
-        return 0;
 }
 
 int mlx5e_get_max_linkspeed(struct mlx5_core_dev *mdev, u32 *speed)
@@ -634,7 +614,7 @@ int mlx5e_get_max_linkspeed(struct mlx5_core_dev *mdev, u32 *speed)
 
 static void get_speed_duplex(struct net_device *netdev,
                              u32 eth_proto_oper,
-                             struct ethtool_cmd *cmd)
+                             struct ethtool_link_ksettings *link_ksettings)
 {
         int i;
         u32 speed = SPEED_UNKNOWN;
@@ -651,23 +631,32 @@ static void get_speed_duplex(struct net_device *netdev,
                 }
         }
 out:
-        ethtool_cmd_speed_set(cmd, speed);
-        cmd->duplex = duplex;
+        link_ksettings->base.speed = speed;
+        link_ksettings->base.duplex = duplex;
 }
 
-static void get_supported(u32 eth_proto_cap, u32 *supported)
+static void get_supported(u32 eth_proto_cap,
+                          struct ethtool_link_ksettings *link_ksettings)
 {
-        *supported |= ptys2ethtool_supported_port(eth_proto_cap);
-        *supported |= ptys2ethtool_supported_link(eth_proto_cap);
-        *supported |= SUPPORTED_Pause | SUPPORTED_Asym_Pause;
+        unsigned long *supported = link_ksettings->link_modes.supported;
+
+        ptys2ethtool_supported_port(link_ksettings, eth_proto_cap);
+        ptys2ethtool_supported_link(supported, eth_proto_cap);
+        ethtool_link_ksettings_add_link_mode(link_ksettings, supported, Pause);
+        ethtool_link_ksettings_add_link_mode(link_ksettings, supported, Asym_Pause);
 }
 
 static void get_advertising(u32 eth_proto_cap, u8 tx_pause,
-                            u8 rx_pause, u32 *advertising)
+                            u8 rx_pause,
+                            struct ethtool_link_ksettings *link_ksettings)
 {
-        *advertising |= ptys2ethtool_adver_link(eth_proto_cap);
-        *advertising |= tx_pause ? ADVERTISED_Pause : 0;
-        *advertising |= (tx_pause ^ rx_pause) ? ADVERTISED_Asym_Pause : 0;
+        unsigned long *advertising = link_ksettings->link_modes.advertising;
+
+        ptys2ethtool_adver_link(advertising, eth_proto_cap);
+        if (tx_pause)
+                ethtool_link_ksettings_add_link_mode(link_ksettings, advertising, Pause);
+        if (tx_pause ^ rx_pause)
+                ethtool_link_ksettings_add_link_mode(link_ksettings, advertising, Asym_Pause);
 }
 
 static u8 get_connector_port(u32 eth_proto)
@@ -695,13 +684,16 @@ static u8 get_connector_port(u32 eth_proto)
         return PORT_OTHER;
 }
 
-static void get_lp_advertising(u32 eth_proto_lp, u32 *lp_advertising)
+static void get_lp_advertising(u32 eth_proto_lp,
+                               struct ethtool_link_ksettings *link_ksettings)
 {
-        *lp_advertising = ptys2ethtool_adver_link(eth_proto_lp);
+        unsigned long *lp_advertising = link_ksettings->link_modes.lp_advertising;
+
+        ptys2ethtool_adver_link(lp_advertising, eth_proto_lp);
 }
 
-static int mlx5e_get_settings(struct net_device *netdev,
-                              struct ethtool_cmd *cmd)
+static int mlx5e_get_link_ksettings(struct net_device *netdev,
+                                    struct ethtool_link_ksettings *link_ksettings)
 {
         struct mlx5e_priv *priv    = netdev_priv(netdev);
         struct mlx5_core_dev *mdev = priv->mdev;
@@ -710,6 +702,8 @@ static int mlx5e_get_settings(struct net_device *netdev,
         u32 eth_proto_admin;
         u32 eth_proto_lp;
         u32 eth_proto_oper;
+        u8 an_disable_admin;
+        u8 an_status;
         int err;
 
         err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN, 1);
@@ -720,35 +714,49 @@ static int mlx5e_get_settings(struct net_device *netdev,
                 goto err_query_ptys;
         }
 
-        eth_proto_cap   = MLX5_GET(ptys_reg, out, eth_proto_capability);
-        eth_proto_admin = MLX5_GET(ptys_reg, out, eth_proto_admin);
-        eth_proto_oper  = MLX5_GET(ptys_reg, out, eth_proto_oper);
-        eth_proto_lp    = MLX5_GET(ptys_reg, out, eth_proto_lp_advertise);
+        eth_proto_cap    = MLX5_GET(ptys_reg, out, eth_proto_capability);
+        eth_proto_admin  = MLX5_GET(ptys_reg, out, eth_proto_admin);
+        eth_proto_oper   = MLX5_GET(ptys_reg, out, eth_proto_oper);
+        eth_proto_lp     = MLX5_GET(ptys_reg, out, eth_proto_lp_advertise);
+        an_disable_admin = MLX5_GET(ptys_reg, out, an_disable_admin);
+        an_status        = MLX5_GET(ptys_reg, out, an_status);
 
-        cmd->supported   = 0;
-        cmd->advertising = 0;
+        ethtool_link_ksettings_zero_link_mode(link_ksettings, supported);
+        ethtool_link_ksettings_zero_link_mode(link_ksettings, advertising);
 
-        get_supported(eth_proto_cap, &cmd->supported);
-        get_advertising(eth_proto_admin, 0, 0, &cmd->advertising);
-        get_speed_duplex(netdev, eth_proto_oper, cmd);
+        get_supported(eth_proto_cap, link_ksettings);
+        get_advertising(eth_proto_admin, 0, 0, link_ksettings);
+        get_speed_duplex(netdev, eth_proto_oper, link_ksettings);
 
         eth_proto_oper = eth_proto_oper ? eth_proto_oper : eth_proto_cap;
 
-        cmd->port = get_connector_port(eth_proto_oper);
-        get_lp_advertising(eth_proto_lp, &cmd->lp_advertising);
+        link_ksettings->base.port = get_connector_port(eth_proto_oper);
+        get_lp_advertising(eth_proto_lp, link_ksettings);
+
+        if (an_status == MLX5_AN_COMPLETE)
+                ethtool_link_ksettings_add_link_mode(link_ksettings,
+                                                     lp_advertising, Autoneg);
 
-        cmd->transceiver = XCVR_INTERNAL;
+        link_ksettings->base.autoneg = an_disable_admin ? AUTONEG_DISABLE :
+                                                          AUTONEG_ENABLE;
+        ethtool_link_ksettings_add_link_mode(link_ksettings, supported,
+                                             Autoneg);
+        if (!an_disable_admin)
+                ethtool_link_ksettings_add_link_mode(link_ksettings,
+                                                     advertising, Autoneg);
 
 err_query_ptys:
         return err;
 }
 
-static u32 mlx5e_ethtool2ptys_adver_link(u32 link_modes)
+static u32 mlx5e_ethtool2ptys_adver_link(const unsigned long *link_modes)
 {
         u32 i, ptys_modes = 0;
 
         for (i = 0; i < MLX5E_LINK_MODES_NUMBER; ++i) {
-                if (ptys2ethtool_table[i].advertised & link_modes)
+                if (bitmap_intersects(ptys2ethtool_table[i].advertised,
+                                      link_modes,
+                                      __ETHTOOL_LINK_MODE_MASK_NBITS))
                         ptys_modes |= MLX5E_PROT_MASK(i);
         }
 
@@ -767,21 +775,25 @@ static u32 mlx5e_ethtool2ptys_speed_link(u32 speed)
         return speed_links;
 }
 
-static int mlx5e_set_settings(struct net_device *netdev,
-                              struct ethtool_cmd *cmd)
+static int mlx5e_set_link_ksettings(struct net_device *netdev,
+                                    const struct ethtool_link_ksettings *link_ksettings)
 {
         struct mlx5e_priv *priv    = netdev_priv(netdev);
         struct mlx5_core_dev *mdev = priv->mdev;
+        u32 eth_proto_cap, eth_proto_admin;
+        bool an_changes = false;
+        u8 an_disable_admin;
+        u8 an_disable_cap;
+        bool an_disable;
         u32 link_modes;
+        u8 an_status;
         u32 speed;
-        u32 eth_proto_cap, eth_proto_admin;
-        enum mlx5_port_status ps;
         int err;
 
-        speed = ethtool_cmd_speed(cmd);
+        speed = link_ksettings->base.speed;
 
-        link_modes = cmd->autoneg == AUTONEG_ENABLE ?
-                mlx5e_ethtool2ptys_adver_link(cmd->advertising) :
+        link_modes = link_ksettings->base.autoneg == AUTONEG_ENABLE ?
+                mlx5e_ethtool2ptys_adver_link(link_ksettings->link_modes.advertising) :
                 mlx5e_ethtool2ptys_speed_link(speed);
 
         err = mlx5_query_port_proto_cap(mdev, &eth_proto_cap, MLX5_PTYS_EN);
@@ -806,15 +818,18 @@ static int mlx5e_set_settings(struct net_device *netdev,
                 goto out;
         }
 
-        if (link_modes == eth_proto_admin)
+        mlx5_query_port_autoneg(mdev, MLX5_PTYS_EN, &an_status,
+                                &an_disable_cap, &an_disable_admin);
+
+        an_disable = link_ksettings->base.autoneg == AUTONEG_DISABLE;
+        an_changes = ((!an_disable && an_disable_admin) ||
+                      (an_disable && !an_disable_admin));
+
+        if (!an_changes && link_modes == eth_proto_admin)
                 goto out;
 
-        mlx5_query_port_admin_status(mdev, &ps);
-        if (ps == MLX5_PORT_UP)
-                mlx5_set_port_admin_status(mdev, MLX5_PORT_DOWN);
-        mlx5_set_port_proto(mdev, link_modes, MLX5_PTYS_EN);
-        if (ps == MLX5_PORT_UP)
-                mlx5_set_port_admin_status(mdev, MLX5_PORT_UP);
+        mlx5_set_port_ptys(mdev, an_disable, link_modes, MLX5_PTYS_EN);
+        mlx5_toggle_port_link(mdev);
 
 out:
         return err;
@@ -1272,6 +1287,87 @@ static int mlx5e_get_module_eeprom(struct net_device *netdev,
         return 0;
 }
 
+typedef int (*mlx5e_pflag_handler)(struct net_device *netdev, bool enable);
+
+static int set_pflag_rx_cqe_based_moder(struct net_device *netdev, bool enable)
+{
+        struct mlx5e_priv *priv = netdev_priv(netdev);
+        struct mlx5_core_dev *mdev = priv->mdev;
+        bool rx_mode_changed;
+        u8 rx_cq_period_mode;
+        int err = 0;
+        bool reset;
+
+        rx_cq_period_mode = enable ?
+                MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
+                MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
+        rx_mode_changed = rx_cq_period_mode != priv->params.rx_cq_period_mode;
+
+        if (rx_cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE &&
+            !MLX5_CAP_GEN(mdev, cq_period_start_from_cqe))
+                return -ENOTSUPP;
+
+        if (!rx_mode_changed)
+                return 0;
+
+        reset = test_bit(MLX5E_STATE_OPENED, &priv->state);
+        if (reset)
+                mlx5e_close_locked(netdev);
+
+        mlx5e_set_rx_cq_mode_params(&priv->params, rx_cq_period_mode);
+
+        if (reset)
+                err = mlx5e_open_locked(netdev);
+
+        return err;
+}
+
+static int mlx5e_handle_pflag(struct net_device *netdev,
+                              u32 wanted_flags,
+                              enum mlx5e_priv_flag flag,
+                              mlx5e_pflag_handler pflag_handler)
+{
+        struct mlx5e_priv *priv = netdev_priv(netdev);
+        bool enable = !!(wanted_flags & flag);
+        u32 changes = wanted_flags ^ priv->pflags;
+        int err;
+
+        if (!(changes & flag))
+                return 0;
+
+        err = pflag_handler(netdev, enable);
+        if (err) {
+                netdev_err(netdev, "%s private flag 0x%x failed err %d\n",
+                           enable ? "Enable" : "Disable", flag, err);
+                return err;
+        }
+
+        MLX5E_SET_PRIV_FLAG(priv, flag, enable);
+        return 0;
+}
+
+static int mlx5e_set_priv_flags(struct net_device *netdev, u32 pflags)
+{
+        struct mlx5e_priv *priv = netdev_priv(netdev);
+        int err;
+
+        mutex_lock(&priv->state_lock);
+
+        err = mlx5e_handle_pflag(netdev, pflags,
+                                 MLX5E_PFLAG_RX_CQE_BASED_MODER,
+                                 set_pflag_rx_cqe_based_moder);
+
+        mutex_unlock(&priv->state_lock);
+        return err ? -EINVAL : 0;
+}
+
+static u32 mlx5e_get_priv_flags(struct net_device *netdev)
+{
+        struct mlx5e_priv *priv = netdev_priv(netdev);
+
+        return priv->pflags;
+}
+
 const struct ethtool_ops mlx5e_ethtool_ops = {
         .get_drvinfo       = mlx5e_get_drvinfo,
         .get_link          = ethtool_op_get_link,
@@ -1284,8 +1380,8 @@ const struct ethtool_ops mlx5e_ethtool_ops = {
         .set_channels      = mlx5e_set_channels,
         .get_coalesce      = mlx5e_get_coalesce,
         .set_coalesce      = mlx5e_set_coalesce,
-        .get_settings      = mlx5e_get_settings,
-        .set_settings      = mlx5e_set_settings,
+        .get_link_ksettings  = mlx5e_get_link_ksettings,
+        .set_link_ksettings  = mlx5e_set_link_ksettings,
         .get_rxfh_key_size   = mlx5e_get_rxfh_key_size,
         .get_rxfh_indir_size = mlx5e_get_rxfh_indir_size,
         .get_rxfh          = mlx5e_get_rxfh,
@@ -1301,4 +1397,6 @@ const struct ethtool_ops mlx5e_ethtool_ops = {
         .set_wol           = mlx5e_set_wol,
         .get_module_info   = mlx5e_get_module_info,
         .get_module_eeprom = mlx5e_get_module_eeprom,
+        .get_priv_flags    = mlx5e_get_priv_flags,
+        .set_priv_flags    = mlx5e_set_priv_flags
 };
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index cb6defd71fc1..a64ce5df5810 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -40,8 +40,9 @@
 #include "vxlan.h"
 
 struct mlx5e_rq_param {
-        u32                        rqc[MLX5_ST_SZ_DW(rqc)];
-        struct mlx5_wq_param       wq;
+        u32                     rqc[MLX5_ST_SZ_DW(rqc)];
+        struct mlx5_wq_param    wq;
+        bool                    am_enabled;
 };
 
 struct mlx5e_sq_param {
@@ -55,6 +56,7 @@ struct mlx5e_cq_param {
         u32                        cqc[MLX5_ST_SZ_DW(cqc)];
         struct mlx5_wq_param       wq;
         u16                        eq_ix;
+        u8                         cq_period_mode;
 };
 
 struct mlx5e_channel_param {
@@ -335,6 +337,9 @@ static int mlx5e_create_rq(struct mlx5e_channel *c,
                 wqe->data.byte_count = cpu_to_be32(byte_count);
         }
 
+        INIT_WORK(&rq->am.work, mlx5e_rx_am_work);
+        rq->am.mode = priv->params.rx_cq_period_mode;
+
         rq->wq_type = priv->params.rq_wq_type;
         rq->pdev    = c->pdev;
         rq->netdev  = c->netdev;
@@ -507,6 +512,9 @@ static int mlx5e_open_rq(struct mlx5e_channel *c,
         if (err)
                 goto err_disable_rq;
 
+        if (param->am_enabled)
+                set_bit(MLX5E_RQ_STATE_AM, &c->rq.state);
+
         set_bit(MLX5E_RQ_STATE_POST_WQES_ENABLE, &rq->state);
 
         sq->ico_wqe_info[pi].opcode     = MLX5_OPCODE_NOP;
@@ -535,6 +543,8 @@ static void mlx5e_close_rq(struct mlx5e_rq *rq)
         /* avoid destroying rq before mlx5e_poll_rx_cq() is done with it */
         napi_synchronize(&rq->channel->napi);
 
+        cancel_work_sync(&rq->am.work);
+
         mlx5e_disable_rq(rq);
         mlx5e_destroy_rq(rq);
 }
@@ -701,7 +711,8 @@ static int mlx5e_enable_sq(struct mlx5e_sq *sq, struct mlx5e_sq_param *param)
         return err;
 }
 
-static int mlx5e_modify_sq(struct mlx5e_sq *sq, int curr_state, int next_state)
+static int mlx5e_modify_sq(struct mlx5e_sq *sq, int curr_state,
+                           int next_state, bool update_rl, int rl_index)
 {
         struct mlx5e_channel *c = sq->channel;
         struct mlx5e_priv *priv = c->priv;
@@ -721,6 +732,10 @@ static int mlx5e_modify_sq(struct mlx5e_sq *sq, int curr_state, int next_state)
 
         MLX5_SET(modify_sq_in, in, sq_state, curr_state);
         MLX5_SET(sqc, sqc, state, next_state);
+        if (update_rl && next_state == MLX5_SQC_STATE_RDY) {
+                MLX5_SET64(modify_sq_in, in, modify_bitmask, 1);
+                MLX5_SET(sqc,  sqc, packet_pacing_rate_limit_index, rl_index);
+        }
 
         err = mlx5_core_modify_sq(mdev, sq->sqn, in, inlen);
 
@@ -736,6 +751,8 @@ static void mlx5e_disable_sq(struct mlx5e_sq *sq)
         struct mlx5_core_dev *mdev = priv->mdev;
 
         mlx5_core_destroy_sq(mdev, sq->sqn);
+        if (sq->rate_limit)
+                mlx5_rl_remove_rate(mdev, sq->rate_limit);
 }
 
 static int mlx5e_open_sq(struct mlx5e_channel *c,
@@ -753,7 +770,8 @@ static int mlx5e_open_sq(struct mlx5e_channel *c,
         if (err)
                 goto err_destroy_sq;
 
-        err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY);
+        err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY,
+                              false, 0);
         if (err)
                 goto err_disable_sq;
 
@@ -792,7 +810,8 @@ static void mlx5e_close_sq(struct mlx5e_sq *sq)
                 if (mlx5e_sq_has_room_for(sq, 1))
                         mlx5e_send_nop(sq, true);
 
-                mlx5e_modify_sq(sq, MLX5_SQC_STATE_RDY, MLX5_SQC_STATE_ERR);
+                mlx5e_modify_sq(sq, MLX5_SQC_STATE_RDY, MLX5_SQC_STATE_ERR,
+                                false, 0);
         }
 
         while (sq->cc != sq->pc) /* wait till sq is empty */
@@ -886,6 +905,7 @@ static int mlx5e_enable_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
 
         mlx5_vector2eqn(mdev, param->eq_ix, &eqn, &irqn_not_used);
 
+        MLX5_SET(cqc,   cqc, cq_period_mode, param->cq_period_mode);
         MLX5_SET(cqc,   cqc, c_eqn,         eqn);
         MLX5_SET(cqc,   cqc, uar_page,      mcq->uar->index);
         MLX5_SET(cqc,   cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
@@ -915,8 +935,7 @@ static void mlx5e_disable_cq(struct mlx5e_cq *cq)
 static int mlx5e_open_cq(struct mlx5e_channel *c,
                          struct mlx5e_cq_param *param,
                          struct mlx5e_cq *cq,
-                         u16 moderation_usecs,
-                         u16 moderation_frames)
+                         struct mlx5e_cq_moder moderation)
 {
         int err;
         struct mlx5e_priv *priv = c->priv;
@@ -932,8 +951,8 @@ static int mlx5e_open_cq(struct mlx5e_channel *c,
 
         if (MLX5_CAP_GEN(mdev, cq_moderation))
                 mlx5_core_modify_cq_moderation(mdev, &cq->mcq,
-                                               moderation_usecs,
-                                               moderation_frames);
+                                               moderation.usec,
+                                               moderation.pkts);
         return 0;
 
 err_destroy_cq:
@@ -962,8 +981,7 @@ static int mlx5e_open_tx_cqs(struct mlx5e_channel *c,
 
         for (tc = 0; tc < c->num_tc; tc++) {
                 err = mlx5e_open_cq(c, &cparam->tx_cq, &c->sq[tc].cq,
-                                    priv->params.tx_cq_moderation_usec,
-                                    priv->params.tx_cq_moderation_pkts);
+                                    priv->params.tx_cq_moderation);
                 if (err)
                         goto err_close_tx_cqs;
         }
@@ -1023,14 +1041,91 @@ static void mlx5e_build_channeltc_to_txq_map(struct mlx5e_priv *priv, int ix)
                         ix + i * priv->params.num_channels;
 }
 
+static int mlx5e_set_sq_maxrate(struct net_device *dev,
+                                struct mlx5e_sq *sq, u32 rate)
+{
+        struct mlx5e_priv *priv = netdev_priv(dev);
+        struct mlx5_core_dev *mdev = priv->mdev;
+        u16 rl_index = 0;
+        int err;
+
+        if (rate == sq->rate_limit)
+                /* nothing to do */
+                return 0;
+
+        if (sq->rate_limit)
+                /* remove current rl index to free space to next ones */
+                mlx5_rl_remove_rate(mdev, sq->rate_limit);
+
+        sq->rate_limit = 0;
+
+        if (rate) {
+                err = mlx5_rl_add_rate(mdev, rate, &rl_index);
+                if (err) {
+                        netdev_err(dev, "Failed configuring rate %u: %d\n",
+                                   rate, err);
+                        return err;
+                }
+        }
+
+        err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RDY,
+                              MLX5_SQC_STATE_RDY, true, rl_index);
+        if (err) {
+                netdev_err(dev, "Failed configuring rate %u: %d\n",
+                           rate, err);
+                /* remove the rate from the table */
+                if (rate)
+                        mlx5_rl_remove_rate(mdev, rate);
+                return err;
+        }
+
+        sq->rate_limit = rate;
+        return 0;
+}
+
+static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
+{
+        struct mlx5e_priv *priv = netdev_priv(dev);
+        struct mlx5_core_dev *mdev = priv->mdev;
+        struct mlx5e_sq *sq = priv->txq_to_sq_map[index];
+        int err = 0;
+
+        if (!mlx5_rl_is_supported(mdev)) {
+                netdev_err(dev, "Rate limiting is not supported on this device\n");
+                return -EINVAL;
+        }
+
+        /* rate is given in Mb/sec, HW config is in Kb/sec */
+        rate = rate << 10;
+
+        /* Check whether rate in valid range, 0 is always valid */
+        if (rate && !mlx5_rl_is_in_range(mdev, rate)) {
+                netdev_err(dev, "TX rate %u, is not in range\n", rate);
+                return -ERANGE;
+        }
+
+        mutex_lock(&priv->state_lock);
+        if (test_bit(MLX5E_STATE_OPENED, &priv->state))
+                err = mlx5e_set_sq_maxrate(dev, sq, rate);
+        if (!err)
+                priv->tx_rates[index] = rate;
+        mutex_unlock(&priv->state_lock);
+
+        return err;
+}
+
 static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
                               struct mlx5e_channel_param *cparam,
                               struct mlx5e_channel **cp)
 {
+        struct mlx5e_cq_moder icosq_cq_moder = {0, 0};
         struct net_device *netdev = priv->netdev;
+        struct mlx5e_cq_moder rx_cq_profile;
         int cpu = mlx5e_get_cpu(priv, ix);
         struct mlx5e_channel *c;
+        struct mlx5e_sq *sq;
         int err;
+        int i;
 
         c = kzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu));
         if (!c)
@@ -1044,11 +1139,16 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
         c->mkey_be  = cpu_to_be32(priv->mkey.key);
         c->num_tc   = priv->params.num_tc;
 
+        if (priv->params.rx_am_enabled)
+                rx_cq_profile = mlx5e_am_get_def_profile(priv->params.rx_cq_period_mode);
+        else
+                rx_cq_profile = priv->params.rx_cq_moderation;
+
         mlx5e_build_channeltc_to_txq_map(priv, ix);
 
         netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);
 
-        err = mlx5e_open_cq(c, &cparam->icosq_cq, &c->icosq.cq, 0, 0);
+        err = mlx5e_open_cq(c, &cparam->icosq_cq, &c->icosq.cq, icosq_cq_moder);
         if (err)
                 goto err_napi_del;
 
@@ -1057,8 +1157,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
                 goto err_close_icosq_cq;
 
         err = mlx5e_open_cq(c, &cparam->rx_cq, &c->rq.cq,
-                            priv->params.rx_cq_moderation_usec,
-                            priv->params.rx_cq_moderation_pkts);
+                            rx_cq_profile);
         if (err)
                 goto err_close_tx_cqs;
 
@@ -1072,6 +1171,16 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
         if (err)
                 goto err_close_icosq;
 
+        for (i = 0; i < priv->params.num_tc; i++) {
+                u32 txq_ix = priv->channeltc_to_txq_map[ix][i];
+
+                if (priv->tx_rates[txq_ix]) {
+                        sq = priv->txq_to_sq_map[txq_ix];
+                        mlx5e_set_sq_maxrate(priv->netdev, sq,
+                                             priv->tx_rates[txq_ix]);
+                }
+        }
+
         err = mlx5e_open_rq(c, &cparam->rq, &c->rq);
         if (err)
                 goto err_close_sqs;
@@ -1148,6 +1257,8 @@ static void mlx5e_build_rq_param(struct mlx5e_priv *priv,
 
         param->wq.buf_numa_node = dev_to_node(&priv->mdev->pdev->dev);
         param->wq.linear = 1;
+
+        param->am_enabled = priv->params.rx_am_enabled;
 }
 
 static void mlx5e_build_drop_rq_param(struct mlx5e_rq_param *param)
@@ -1213,6 +1324,8 @@ static void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
         }
 
         mlx5e_build_common_cq_param(priv, param);
+
+        param->cq_period_mode = priv->params.rx_cq_period_mode;
 }
 
 static void mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
@@ -1223,6 +1336,8 @@ static void mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
         MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_sq_size);
 
         mlx5e_build_common_cq_param(priv, param);
+
+        param->cq_period_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
 }
 
 static void mlx5e_build_ico_cq_param(struct mlx5e_priv *priv,
@@ -1234,6 +1349,8 @@ static void mlx5e_build_ico_cq_param(struct mlx5e_priv *priv,
         MLX5_SET(cqc, cqc, log_cq_size, log_wq_size);
 
         mlx5e_build_common_cq_param(priv, param);
+
+        param->cq_period_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
 }
 
 static void mlx5e_build_icosq_param(struct mlx5e_priv *priv,
@@ -2519,25 +2636,31 @@ static int mlx5e_get_vf_stats(struct net_device *dev,
 }
 
 static void mlx5e_add_vxlan_port(struct net_device *netdev,
-                                 sa_family_t sa_family, __be16 port)
+                                 struct udp_tunnel_info *ti)
 {
         struct mlx5e_priv *priv = netdev_priv(netdev);
 
+        if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
+                return;
+
         if (!mlx5e_vxlan_allowed(priv->mdev))
                 return;
 
-        mlx5e_vxlan_queue_work(priv, sa_family, be16_to_cpu(port), 1);
+        mlx5e_vxlan_queue_work(priv, ti->sa_family, be16_to_cpu(ti->port), 1);
 }
 
 static void mlx5e_del_vxlan_port(struct net_device *netdev,
-                                 sa_family_t sa_family, __be16 port)
+                                 struct udp_tunnel_info *ti)
 {
         struct mlx5e_priv *priv = netdev_priv(netdev);
 
+        if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
+                return;
+
         if (!mlx5e_vxlan_allowed(priv->mdev))
                 return;
 
-        mlx5e_vxlan_queue_work(priv, sa_family, be16_to_cpu(port), 0);
+        mlx5e_vxlan_queue_work(priv, ti->sa_family, be16_to_cpu(ti->port), 0);
 }
 
 static netdev_features_t mlx5e_vxlan_features_check(struct mlx5e_priv *priv,
@@ -2604,6 +2727,7 @@ static const struct net_device_ops mlx5e_netdev_ops_basic = {
         .ndo_set_features        = mlx5e_set_features,
         .ndo_change_mtu          = mlx5e_change_mtu,
         .ndo_do_ioctl            = mlx5e_ioctl,
+        .ndo_set_tx_maxrate      = mlx5e_set_tx_maxrate,
 #ifdef CONFIG_RFS_ACCEL
         .ndo_rx_flow_steer       = mlx5e_rx_flow_steer,
 #endif
@@ -2623,8 +2747,9 @@ static const struct net_device_ops mlx5e_netdev_ops_sriov = {
         .ndo_set_features        = mlx5e_set_features,
         .ndo_change_mtu          = mlx5e_change_mtu,
         .ndo_do_ioctl            = mlx5e_ioctl,
-        .ndo_add_vxlan_port      = mlx5e_add_vxlan_port,
-        .ndo_del_vxlan_port      = mlx5e_del_vxlan_port,
+        .ndo_udp_tunnel_add      = mlx5e_add_vxlan_port,
+        .ndo_udp_tunnel_del      = mlx5e_del_vxlan_port,
+        .ndo_set_tx_maxrate      = mlx5e_set_tx_maxrate,
         .ndo_features_check      = mlx5e_features_check,
 #ifdef CONFIG_RFS_ACCEL
         .ndo_rx_flow_steer       = mlx5e_rx_flow_steer,
@@ -2753,6 +2878,20 @@ static bool cqe_compress_heuristic(u32 link_speed, u32 pci_bw)
                 (pci_bw < 40000) && (pci_bw < link_speed));
 }
 
+void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
+{
+        params->rx_cq_period_mode = cq_period_mode;
+
+        params->rx_cq_moderation.pkts =
+                MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
+        params->rx_cq_moderation.usec =
+                        MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
+
+        if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
+                params->rx_cq_moderation.usec =
+                        MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE;
+}
+
 static void mlx5e_build_netdev_priv(struct mlx5_core_dev *mdev,
                                     struct net_device *netdev,
                                     int num_channels)
@@ -2760,6 +2899,9 @@ static void mlx5e_build_netdev_priv(struct mlx5_core_dev *mdev,
         struct mlx5e_priv *priv = netdev_priv(netdev);
         u32 link_speed = 0;
         u32 pci_bw = 0;
+        u8 cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
+                                         MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
+                                         MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
 
         priv->params.log_sq_size           =
                 MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
@@ -2805,13 +2947,13 @@ static void mlx5e_build_netdev_priv(struct mlx5_core_dev *mdev,
 
         priv->params.min_rx_wqes = mlx5_min_rx_wqes(priv->params.rq_wq_type,
                                             BIT(priv->params.log_rq_size));
-        priv->params.rx_cq_moderation_usec =
-                MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
-        priv->params.rx_cq_moderation_pkts =
-                MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
-        priv->params.tx_cq_moderation_usec =
+
+        priv->params.rx_am_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
+        mlx5e_set_rx_cq_mode_params(&priv->params, cq_period_mode);
+
+        priv->params.tx_cq_moderation.usec =
                 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
-        priv->params.tx_cq_moderation_pkts =
+        priv->params.tx_cq_moderation.pkts =
                 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
         priv->params.tx_max_inline         = mlx5e_get_max_inline_cap(mdev);
         priv->params.num_tc                = 1;
@@ -2826,6 +2968,10 @@ static void mlx5e_build_netdev_priv(struct mlx5_core_dev *mdev,
         priv->params.lro_wqe_sz            =
                 MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ;
 
+        /* Initialize pflags */
+        MLX5E_SET_PRIV_FLAG(priv, MLX5E_PFLAG_RX_CQE_BASED_MODER,
+                            priv->params.rx_cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
+
         priv->mdev                         = mdev;
         priv->netdev                       = netdev;
         priv->params.num_channels          = num_channels;
@@ -3127,7 +3273,7 @@ static void *mlx5e_create_netdev(struct mlx5_core_dev *mdev)
 
         if (mlx5e_vxlan_allowed(mdev)) {
                 rtnl_lock();
-                vxlan_get_rx_port(netdev);
+                udp_tunnel_get_rx_info(netdev);
                 rtnl_unlock();
         }
 
@@ -3233,6 +3379,7 @@ static struct mlx5_interface mlx5e_interface = {
 
 void mlx5e_init(void)
 {
+        mlx5e_build_ptys2ethtool_map();
         mlx5_register_interface(&mlx5e_interface);
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c
new file mode 100644
index 000000000000..1fffe48a93cc
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c
@@ -0,0 +1,335 @@
+/*
+ * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "en.h"
+
+/* Adaptive moderation profiles */
+#define MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256
+#define MLX5E_RX_AM_DEF_PROFILE_CQE 1
+#define MLX5E_RX_AM_DEF_PROFILE_EQE 1
+#define MLX5E_PARAMS_AM_NUM_PROFILES 5
+
+/* All profiles sizes must be MLX5E_PARAMS_AM_NUM_PROFILES */
+#define MLX5_AM_EQE_PROFILES { \
+        {1,   MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
+        {8,   MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
+        {64,  MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
+        {128, MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
+        {256, MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
+}
+
+#define MLX5_AM_CQE_PROFILES { \
+        {2,  256},             \
+        {8,  128},             \
+        {16, 64},              \
+        {32, 64},              \
+        {64, 64}               \
+}
+
+static const struct mlx5e_cq_moder
+profile[MLX5_CQ_PERIOD_NUM_MODES][MLX5E_PARAMS_AM_NUM_PROFILES] = {
+        MLX5_AM_EQE_PROFILES,
+        MLX5_AM_CQE_PROFILES,
+};
+
+static inline struct mlx5e_cq_moder mlx5e_am_get_profile(u8 cq_period_mode, int ix)
+{
+        return profile[cq_period_mode][ix];
+}
+
+struct mlx5e_cq_moder mlx5e_am_get_def_profile(u8 rx_cq_period_mode)
+{
+        int default_profile_ix;
+
+        if (rx_cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
+                default_profile_ix = MLX5E_RX_AM_DEF_PROFILE_CQE;
+        else /* MLX5_CQ_PERIOD_MODE_START_FROM_EQE */
+                default_profile_ix = MLX5E_RX_AM_DEF_PROFILE_EQE;
+
+        return profile[rx_cq_period_mode][default_profile_ix];
+}
+
+/* Adaptive moderation logic */
+enum {
+        MLX5E_AM_START_MEASURE,
+        MLX5E_AM_MEASURE_IN_PROGRESS,
+        MLX5E_AM_APPLY_NEW_PROFILE,
+};
+
+enum {
+        MLX5E_AM_PARKING_ON_TOP,
+        MLX5E_AM_PARKING_TIRED,
+        MLX5E_AM_GOING_RIGHT,
+        MLX5E_AM_GOING_LEFT,
+};
+
+enum {
+        MLX5E_AM_STATS_WORSE,
+        MLX5E_AM_STATS_SAME,
+        MLX5E_AM_STATS_BETTER,
+};
+
+enum {
+        MLX5E_AM_STEPPED,
+        MLX5E_AM_TOO_TIRED,
+        MLX5E_AM_ON_EDGE,
+};
+
+static bool mlx5e_am_on_top(struct mlx5e_rx_am *am)
+{
+        switch (am->tune_state) {
+        case MLX5E_AM_PARKING_ON_TOP:
+        case MLX5E_AM_PARKING_TIRED:
+                WARN_ONCE(true, "mlx5e_am_on_top: PARKING\n");
+                return true;
+        case MLX5E_AM_GOING_RIGHT:
+                return (am->steps_left > 1) && (am->steps_right == 1);
+        default: /* MLX5E_AM_GOING_LEFT */
+                return (am->steps_right > 1) && (am->steps_left == 1);
+        }
+}
+
+static void mlx5e_am_turn(struct mlx5e_rx_am *am)
+{
+        switch (am->tune_state) {
+        case MLX5E_AM_PARKING_ON_TOP:
+        case MLX5E_AM_PARKING_TIRED:
+                WARN_ONCE(true, "mlx5e_am_turn: PARKING\n");
+                break;
+        case MLX5E_AM_GOING_RIGHT:
+                am->tune_state = MLX5E_AM_GOING_LEFT;
+                am->steps_left = 0;
+                break;
+        case MLX5E_AM_GOING_LEFT:
+                am->tune_state = MLX5E_AM_GOING_RIGHT;
+                am->steps_right = 0;
+                break;
+        }
+}
+
+static int mlx5e_am_step(struct mlx5e_rx_am *am)
+{
+        if (am->tired == (MLX5E_PARAMS_AM_NUM_PROFILES * 2))
+                return MLX5E_AM_TOO_TIRED;
+
+        switch (am->tune_state) {
+        case MLX5E_AM_PARKING_ON_TOP:
+        case MLX5E_AM_PARKING_TIRED:
+                WARN_ONCE(true, "mlx5e_am_step: PARKING\n");
+                break;
+        case MLX5E_AM_GOING_RIGHT:
+                if (am->profile_ix == (MLX5E_PARAMS_AM_NUM_PROFILES - 1))
+                        return MLX5E_AM_ON_EDGE;
+                am->profile_ix++;
+                am->steps_right++;
+                break;
+        case MLX5E_AM_GOING_LEFT:
+                if (am->profile_ix == 0)
+                        return MLX5E_AM_ON_EDGE;
+                am->profile_ix--;
+                am->steps_left++;
+                break;
+        }
+
+        am->tired++;
+        return MLX5E_AM_STEPPED;
+}
+
+static void mlx5e_am_park_on_top(struct mlx5e_rx_am *am)
+{
+        am->steps_right  = 0;
+        am->steps_left   = 0;
+        am->tired        = 0;
+        am->tune_state   = MLX5E_AM_PARKING_ON_TOP;
+}
+
+static void mlx5e_am_park_tired(struct mlx5e_rx_am *am)
+{
+        am->steps_right  = 0;
+        am->steps_left   = 0;
+        am->tune_state   = MLX5E_AM_PARKING_TIRED;
+}
+
+static void mlx5e_am_exit_parking(struct mlx5e_rx_am *am)
+{
+        am->tune_state = am->profile_ix ? MLX5E_AM_GOING_LEFT :
+                                          MLX5E_AM_GOING_RIGHT;
+        mlx5e_am_step(am);
+}
+
+static int mlx5e_am_stats_compare(struct mlx5e_rx_am_stats *curr,
+                                  struct mlx5e_rx_am_stats *prev)
+{
+        int diff;
+
+        if (!prev->ppms)
+                return curr->ppms ? MLX5E_AM_STATS_BETTER :
+                                    MLX5E_AM_STATS_SAME;
+
+        diff = curr->ppms - prev->ppms;
+        if (((100 * abs(diff)) / prev->ppms) > 10) /* more than 10% diff */
+                return (diff > 0) ? MLX5E_AM_STATS_BETTER :
+                                    MLX5E_AM_STATS_WORSE;
+
+        if (!prev->epms)
+                return curr->epms ? MLX5E_AM_STATS_WORSE :
+                                    MLX5E_AM_STATS_SAME;
+
+        diff = curr->epms - prev->epms;
+        if (((100 * abs(diff)) / prev->epms) > 10) /* more than 10% diff */
+                return (diff < 0) ? MLX5E_AM_STATS_BETTER :
+                                    MLX5E_AM_STATS_WORSE;
+
+        return MLX5E_AM_STATS_SAME;
+}
+
+static bool mlx5e_am_decision(struct mlx5e_rx_am_stats *curr_stats,
+                              struct mlx5e_rx_am *am)
+{
+        int prev_state = am->tune_state;
+        int prev_ix = am->profile_ix;
+        int stats_res;
+        int step_res;
+
+        switch (am->tune_state) {
+        case MLX5E_AM_PARKING_ON_TOP:
+                stats_res = mlx5e_am_stats_compare(curr_stats, &am->prev_stats);
+                if (stats_res != MLX5E_AM_STATS_SAME)
+                        mlx5e_am_exit_parking(am);
+                break;
+
+        case MLX5E_AM_PARKING_TIRED:
+                am->tired--;
+                if (!am->tired)
+                        mlx5e_am_exit_parking(am);
+                break;
+
+        case MLX5E_AM_GOING_RIGHT:
+        case MLX5E_AM_GOING_LEFT:
+                stats_res = mlx5e_am_stats_compare(curr_stats, &am->prev_stats);
+                if (stats_res != MLX5E_AM_STATS_BETTER)
+                        mlx5e_am_turn(am);
+
+                if (mlx5e_am_on_top(am)) {
+                        mlx5e_am_park_on_top(am);
+                        break;
+                }
+
+                step_res = mlx5e_am_step(am);
+                switch (step_res) {
+                case MLX5E_AM_ON_EDGE:
+                        mlx5e_am_park_on_top(am);
+                        break;
+                case MLX5E_AM_TOO_TIRED:
+                        mlx5e_am_park_tired(am);
+                        break;
+                }
+
+                break;
+        }
+
+        if ((prev_state     != MLX5E_AM_PARKING_ON_TOP) ||
+            (am->tune_state != MLX5E_AM_PARKING_ON_TOP))
+                am->prev_stats = *curr_stats;
+
+        return am->profile_ix != prev_ix;
+}
+
+static void mlx5e_am_sample(struct mlx5e_rq *rq,
+                            struct mlx5e_rx_am_sample *s)
+{
+        s->time      = ktime_get();
+        s->pkt_ctr   = rq->stats.packets;
+        s->event_ctr = rq->cq.event_ctr;
+}
+
+#define MLX5E_AM_NEVENTS 64
+
+static void mlx5e_am_calc_stats(struct mlx5e_rx_am_sample *start,
+                                struct mlx5e_rx_am_sample *end,
+                                struct mlx5e_rx_am_stats *curr_stats)
+{
+        /* u32 holds up to 71 minutes, should be enough */
+        u32 delta_us = ktime_us_delta(end->time, start->time);
+        unsigned int npkts = end->pkt_ctr - start->pkt_ctr;
+
+        if (!delta_us) {
+                WARN_ONCE(true, "mlx5e_am_calc_stats: delta_us=0\n");
+                return;
+        }
+
+        curr_stats->ppms =            (npkts * USEC_PER_MSEC) / delta_us;
+        curr_stats->epms = (MLX5E_AM_NEVENTS * USEC_PER_MSEC) / delta_us;
+}
+
+void mlx5e_rx_am_work(struct work_struct *work)
+{
+        struct mlx5e_rx_am *am = container_of(work, struct mlx5e_rx_am,
+                                              work);
+        struct mlx5e_rq *rq = container_of(am, struct mlx5e_rq, am);
+        struct mlx5e_cq_moder cur_profile = profile[am->mode][am->profile_ix];
+
+        mlx5_core_modify_cq_moderation(rq->priv->mdev, &rq->cq.mcq,
+                                       cur_profile.usec, cur_profile.pkts);
+
+        am->state = MLX5E_AM_START_MEASURE;
+}
+
+void mlx5e_rx_am(struct mlx5e_rq *rq)
+{
+        struct mlx5e_rx_am *am = &rq->am;
+        struct mlx5e_rx_am_sample end_sample;
+        struct mlx5e_rx_am_stats curr_stats;
+        u16 nevents;
+
+        switch (am->state) {
+        case MLX5E_AM_MEASURE_IN_PROGRESS:
+                nevents = rq->cq.event_ctr - am->start_sample.event_ctr;
+                if (nevents < MLX5E_AM_NEVENTS)
+                        break;
+                mlx5e_am_sample(rq, &end_sample);
+                mlx5e_am_calc_stats(&am->start_sample, &end_sample,
+                                    &curr_stats);
+                if (mlx5e_am_decision(&curr_stats, am)) {
+                        am->state = MLX5E_AM_APPLY_NEW_PROFILE;
+                        schedule_work(&am->work);
+                        break;
+                }
+                /* fall through */
+        case MLX5E_AM_START_MEASURE:
+                mlx5e_am_sample(rq, &am->start_sample);
+                am->state = MLX5E_AM_MEASURE_IN_PROGRESS;
+                break;
+        case MLX5E_AM_APPLY_NEW_PROFILE:
+                break;
+        }
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
index c38781fa567d..64ae2e800daa 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
@@ -136,6 +136,10 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget)
 
         for (i = 0; i < c->num_tc; i++)
                 mlx5e_cq_arm(&c->sq[i].cq);
+
+        if (test_bit(MLX5E_RQ_STATE_AM, &c->rq.state))
+                mlx5e_rx_am(&c->rq);
+
         mlx5e_cq_arm(&c->rq.cq);
         mlx5e_cq_arm(&c->icosq.cq);
 
@@ -146,6 +150,7 @@ void mlx5e_completion_event(struct mlx5_core_cq *mcq)
 {
         struct mlx5e_cq *cq = container_of(mcq, struct mlx5e_cq, mcq);
 
+        cq->event_ctr++;
         set_bit(MLX5E_CHANNEL_NAPI_SCHED, &cq->channel->flags);
         napi_schedule(cq->napi);
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c
index 75c7ae6a5cc4..77fc1aa26114 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c
@@ -151,6 +151,12 @@ int mlx5_query_hca_caps(struct mlx5_core_dev *dev)
                         return err;
         }
 
+        if (MLX5_CAP_GEN(dev, qos)) {
+                err = mlx5_core_get_caps(dev, MLX5_CAP_QOS);
+                if (err)
+                        return err;
+        }
+
         return 0;
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index c65f4a13e17e..1f3b6d6a852e 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -1144,6 +1144,13 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
                 dev_err(&pdev->dev, "Failed to init flow steering\n");
                 goto err_fs;
         }
+
+        err = mlx5_init_rl_table(dev);
+        if (err) {
+                dev_err(&pdev->dev, "Failed to init rate limiting\n");
+                goto err_rl;
+        }
+
 #ifdef CONFIG_MLX5_CORE_EN
         err = mlx5_eswitch_init(dev);
         if (err) {
@@ -1183,6 +1190,8 @@ err_sriov:
         mlx5_eswitch_cleanup(dev->priv.eswitch);
 #endif
 err_reg_dev:
+        mlx5_cleanup_rl_table(dev);
+err_rl:
         mlx5_cleanup_fs(dev);
 err_fs:
         mlx5_cleanup_mkey_table(dev);
@@ -1253,6 +1262,7 @@ static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
         mlx5_eswitch_cleanup(dev->priv.eswitch);
 #endif
 
+        mlx5_cleanup_rl_table(dev);
         mlx5_cleanup_fs(dev);
         mlx5_cleanup_mkey_table(dev);
         mlx5_cleanup_srq_table(dev);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/port.c b/drivers/net/ethernet/mellanox/mlx5/core/port.c
index 3e35611b19c3..752c08127138 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/port.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/port.c
@@ -202,15 +202,24 @@ int mlx5_query_port_proto_oper(struct mlx5_core_dev *dev,
 }
 EXPORT_SYMBOL_GPL(mlx5_query_port_proto_oper);
 
-int mlx5_set_port_proto(struct mlx5_core_dev *dev, u32 proto_admin,
-                        int proto_mask)
+int mlx5_set_port_ptys(struct mlx5_core_dev *dev, bool an_disable,
+                       u32 proto_admin, int proto_mask)
 {
-        u32 in[MLX5_ST_SZ_DW(ptys_reg)];
         u32 out[MLX5_ST_SZ_DW(ptys_reg)];
+        u32 in[MLX5_ST_SZ_DW(ptys_reg)];
+        u8 an_disable_admin;
+        u8 an_disable_cap;
+        u8 an_status;
+
+        mlx5_query_port_autoneg(dev, proto_mask, &an_status,
+                                &an_disable_cap, &an_disable_admin);
+        if (!an_disable_cap && an_disable)
+                return -EPERM;
 
         memset(in, 0, sizeof(in));
 
         MLX5_SET(ptys_reg, in, local_port, 1);
+        MLX5_SET(ptys_reg, in, an_disable_admin, an_disable);
         MLX5_SET(ptys_reg, in, proto_mask, proto_mask);
         if (proto_mask == MLX5_PTYS_EN)
                 MLX5_SET(ptys_reg, in, eth_proto_admin, proto_admin);
@@ -220,7 +229,19 @@ int mlx5_set_port_proto(struct mlx5_core_dev *dev, u32 proto_admin,
         return mlx5_core_access_reg(dev, in, sizeof(in), out,
                                     sizeof(out), MLX5_REG_PTYS, 0, 1);
 }
-EXPORT_SYMBOL_GPL(mlx5_set_port_proto);
+EXPORT_SYMBOL_GPL(mlx5_set_port_ptys);
+
+/* This function should be used after setting a port register only */
+void mlx5_toggle_port_link(struct mlx5_core_dev *dev)
+{
+        enum mlx5_port_status ps;
+
+        mlx5_query_port_admin_status(dev, &ps);
+        mlx5_set_port_admin_status(dev, MLX5_PORT_DOWN);
+        if (ps == MLX5_PORT_UP)
+                mlx5_set_port_admin_status(dev, MLX5_PORT_UP);
+}
+EXPORT_SYMBOL_GPL(mlx5_toggle_port_link);
 
 int mlx5_set_port_admin_status(struct mlx5_core_dev *dev,
                                enum mlx5_port_status status)
@@ -518,6 +539,25 @@ int mlx5_query_port_pfc(struct mlx5_core_dev *dev, u8 *pfc_en_tx, u8 *pfc_en_rx)
 }
 EXPORT_SYMBOL_GPL(mlx5_query_port_pfc);
 
+void mlx5_query_port_autoneg(struct mlx5_core_dev *dev, int proto_mask,
+                             u8 *an_status,
+                             u8 *an_disable_cap, u8 *an_disable_admin)
+{
+        u32 out[MLX5_ST_SZ_DW(ptys_reg)];
+
+        *an_status = 0;
+        *an_disable_cap = 0;
+        *an_disable_admin = 0;
+
+        if (mlx5_query_port_ptys(dev, out, sizeof(out), proto_mask, 1))
+                return;
+
+        *an_status = MLX5_GET(ptys_reg, out, an_status);
+        *an_disable_cap = MLX5_GET(ptys_reg, out, an_disable_cap);
+        *an_disable_admin = MLX5_GET(ptys_reg, out, an_disable_admin);
+}
+EXPORT_SYMBOL_GPL(mlx5_query_port_autoneg);
+
 int mlx5_max_tc(struct mlx5_core_dev *mdev)
 {
         u8 num_tc = MLX5_CAP_GEN(mdev, max_tc) ? : 8;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/rl.c b/drivers/net/ethernet/mellanox/mlx5/core/rl.c
new file mode 100644
index 000000000000..c07c28bd3d55
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/rl.c
@@ -0,0 +1,209 @@
+/*
+ * Copyright (c) 2013-2016, Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mlx5/driver.h>
+#include <linux/mlx5/cmd.h>
+#include "mlx5_core.h"
+
+/* Finds an entry where we can register the given rate
+ * If the rate already exists, return the entry where it is registered,
+ * otherwise return the first available entry.
+ * If the table is full, return NULL
+ */
+static struct mlx5_rl_entry *find_rl_entry(struct mlx5_rl_table *table,
+                                           u32 rate)
+{
+        struct mlx5_rl_entry *ret_entry = NULL;
+        bool empty_found = false;
+        int i;
+
+        for (i = 0; i < table->max_size; i++) {
+                if (table->rl_entry[i].rate == rate)
+                        return &table->rl_entry[i];
+                if (!empty_found && !table->rl_entry[i].rate) {
+                        empty_found = true;
+                        ret_entry = &table->rl_entry[i];
+                }
+        }
+
+        return ret_entry;
+}
+
+static int mlx5_set_rate_limit_cmd(struct mlx5_core_dev *dev,
+                                   u32 rate, u16 index)
+{
+        u32 in[MLX5_ST_SZ_DW(set_rate_limit_in)];
+        u32 out[MLX5_ST_SZ_DW(set_rate_limit_out)];
+
+        memset(in, 0, sizeof(in));
+        memset(out, 0, sizeof(out));
+
+        MLX5_SET(set_rate_limit_in, in, opcode,
+                 MLX5_CMD_OP_SET_RATE_LIMIT);
+        MLX5_SET(set_rate_limit_in, in, rate_limit_index, index);
+        MLX5_SET(set_rate_limit_in, in, rate_limit, rate);
+
+        return mlx5_cmd_exec_check_status(dev, in, sizeof(in),
+                                          out, sizeof(out));
+}
+
+bool mlx5_rl_is_in_range(struct mlx5_core_dev *dev, u32 rate)
+{
+        struct mlx5_rl_table *table = &dev->priv.rl_table;
+
+        return (rate <= table->max_rate && rate >= table->min_rate);
+}
+EXPORT_SYMBOL(mlx5_rl_is_in_range);
+
+int mlx5_rl_add_rate(struct mlx5_core_dev *dev, u32 rate, u16 *index)
+{
+        struct mlx5_rl_table *table = &dev->priv.rl_table;
+        struct mlx5_rl_entry *entry;
+        int err = 0;
+
+        mutex_lock(&table->rl_lock);
+
+        if (!rate || !mlx5_rl_is_in_range(dev, rate)) {
+                mlx5_core_err(dev, "Invalid rate: %u, should be %u to %u\n",
+                              rate, table->min_rate, table->max_rate);
+                err = -EINVAL;
+                goto out;
+        }
+
+        entry = find_rl_entry(table, rate);
+        if (!entry) {
+                mlx5_core_err(dev, "Max number of %u rates reached\n",
+                              table->max_size);
+                err = -ENOSPC;
+                goto out;
+        }
+        if (entry->refcount) {
+                /* rate already configured */
+                entry->refcount++;
+        } else {
+                /* new rate limit */
+                err = mlx5_set_rate_limit_cmd(dev, rate, entry->index);
+                if (err) {
+                        mlx5_core_err(dev, "Failed configuring rate: %u (%d)\n",
+                                      rate, err);
+                        goto out;
+                }
+                entry->rate = rate;
+                entry->refcount = 1;
+        }
+        *index = entry->index;
+
+out:
+        mutex_unlock(&table->rl_lock);
+        return err;
+}
+EXPORT_SYMBOL(mlx5_rl_add_rate);
+
+void mlx5_rl_remove_rate(struct mlx5_core_dev *dev, u32 rate)
+{
+        struct mlx5_rl_table *table = &dev->priv.rl_table;
+        struct mlx5_rl_entry *entry = NULL;
+
+        /* 0 is a reserved value for unlimited rate */
+        if (rate == 0)
+                return;
+
+        mutex_lock(&table->rl_lock);
+        entry = find_rl_entry(table, rate);
+        if (!entry || !entry->refcount) {
+                mlx5_core_warn(dev, "Rate %u is not configured\n", rate);
+                goto out;
+        }
+
+        entry->refcount--;
+        if (!entry->refcount) {
+                /* need to remove rate */
+                mlx5_set_rate_limit_cmd(dev, 0, entry->index);
+                entry->rate = 0;
+        }
+
+out:
+        mutex_unlock(&table->rl_lock);
+}
+EXPORT_SYMBOL(mlx5_rl_remove_rate);
+
+int mlx5_init_rl_table(struct mlx5_core_dev *dev)
+{
+        struct mlx5_rl_table *table = &dev->priv.rl_table;
+        int i;
+
+        mutex_init(&table->rl_lock);
+        if (!MLX5_CAP_GEN(dev, qos) || !MLX5_CAP_QOS(dev, packet_pacing)) {
+                table->max_size = 0;
+                return 0;
+        }
+
+        /* First entry is reserved for unlimited rate */
+        table->max_size = MLX5_CAP_QOS(dev, packet_pacing_rate_table_size) - 1;
+        table->max_rate = MLX5_CAP_QOS(dev, packet_pacing_max_rate);
+        table->min_rate = MLX5_CAP_QOS(dev, packet_pacing_min_rate);
+
+        table->rl_entry = kcalloc(table->max_size, sizeof(struct mlx5_rl_entry),
+                                  GFP_KERNEL);
+        if (!table->rl_entry)
+                return -ENOMEM;
+
+        /* The index represents the index in HW rate limit table
+         * Index 0 is reserved for unlimited rate
+         */
+        for (i = 0; i < table->max_size; i++)
+                table->rl_entry[i].index = i + 1;
+
+        /* Index 0 is reserved */
+        mlx5_core_info(dev, "Rate limit: %u rates are supported, range: %uMbps to %uMbps\n",
+                       table->max_size,
+                       table->min_rate >> 10,
+                       table->max_rate >> 10);
+
+        return 0;
+}
+
+void mlx5_cleanup_rl_table(struct mlx5_core_dev *dev)
+{
+        struct mlx5_rl_table *table = &dev->priv.rl_table;
+        int i;
+
+        /* Clear all configured rates */
+        for (i = 0; i < table->max_size; i++)
+                if (table->rl_entry[i].rate)
+                        mlx5_set_rate_limit_cmd(dev, 0,
+                                                table->rl_entry[i].index);
+
+        kfree(dev->priv.rl_table.rl_entry);
+}
diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.c b/drivers/net/ethernet/mellanox/mlxsw/core.c
index b0a0b01bb4ef..01ae54826d5c 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/core.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/core.c
@@ -1736,7 +1736,7 @@ static int __init mlxsw_core_module_init(void)
 {
         int err;
 
-        mlxsw_wq = create_workqueue(mlxsw_core_driver_name);
+        mlxsw_wq = alloc_workqueue(mlxsw_core_driver_name, WQ_MEM_RECLAIM, 0);
         if (!mlxsw_wq)
                 return -ENOMEM;
         mlxsw_core_dbg_root = debugfs_create_dir(mlxsw_core_driver_name, NULL);
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
index 660429ebfbe1..a453fffaa1a4 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
@@ -49,6 +49,7 @@
 #include <linux/jiffies.h>
 #include <linux/bitops.h>
 #include <linux/list.h>
+#include <linux/notifier.h>
 #include <linux/dcbnl.h>
 #include <net/switchdev.h>
 #include <generated/utsrelease.h>
@@ -636,14 +637,14 @@ static int mlxsw_sp_port_vlan_mode_trans(struct mlxsw_sp_port *mlxsw_sp_port)
         return 0;
 }
 
-static struct mlxsw_sp_vfid *
+static struct mlxsw_sp_fid *
 mlxsw_sp_vfid_find(const struct mlxsw_sp *mlxsw_sp, u16 vid)
 {
-        struct mlxsw_sp_vfid *vfid;
+        struct mlxsw_sp_fid *f;
 
-        list_for_each_entry(vfid, &mlxsw_sp->port_vfids.list, list) {
-                if (vfid->vid == vid)
-                        return vfid;
+        list_for_each_entry(f, &mlxsw_sp->port_vfids.list, list) {
+                if (f->vid == vid)
+                        return f;
         }
 
         return NULL;
@@ -655,75 +656,70 @@ static u16 mlxsw_sp_avail_vfid_get(const struct mlxsw_sp *mlxsw_sp)
                                    MLXSW_SP_VFID_PORT_MAX);
 }
 
-static int __mlxsw_sp_vfid_create(struct mlxsw_sp *mlxsw_sp, u16 vfid)
+static int mlxsw_sp_vfid_op(struct mlxsw_sp *mlxsw_sp, u16 fid, bool create)
 {
-        u16 fid = mlxsw_sp_vfid_to_fid(vfid);
         char sfmr_pl[MLXSW_REG_SFMR_LEN];
 
-        mlxsw_reg_sfmr_pack(sfmr_pl, MLXSW_REG_SFMR_OP_CREATE_FID, fid, 0);
+        mlxsw_reg_sfmr_pack(sfmr_pl, !create, fid, 0);
         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sfmr), sfmr_pl);
 }
 
-static void __mlxsw_sp_vfid_destroy(struct mlxsw_sp *mlxsw_sp, u16 vfid)
-{
-        u16 fid = mlxsw_sp_vfid_to_fid(vfid);
-        char sfmr_pl[MLXSW_REG_SFMR_LEN];
+static void mlxsw_sp_vport_vfid_leave(struct mlxsw_sp_port *mlxsw_sp_vport);
 
-        mlxsw_reg_sfmr_pack(sfmr_pl, MLXSW_REG_SFMR_OP_DESTROY_FID, fid, 0);
-        mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sfmr), sfmr_pl);
-}
-
-static struct mlxsw_sp_vfid *mlxsw_sp_vfid_create(struct mlxsw_sp *mlxsw_sp,
-                                                  u16 vid)
+static struct mlxsw_sp_fid *mlxsw_sp_vfid_create(struct mlxsw_sp *mlxsw_sp,
+                                                 u16 vid)
 {
         struct device *dev = mlxsw_sp->bus_info->dev;
-        struct mlxsw_sp_vfid *vfid;
-        u16 n_vfid;
+        struct mlxsw_sp_fid *f;
+        u16 vfid, fid;
         int err;
 
-        n_vfid = mlxsw_sp_avail_vfid_get(mlxsw_sp);
-        if (n_vfid == MLXSW_SP_VFID_PORT_MAX) {
+        vfid = mlxsw_sp_avail_vfid_get(mlxsw_sp);
+        if (vfid == MLXSW_SP_VFID_PORT_MAX) {
                 dev_err(dev, "No available vFIDs\n");
                 return ERR_PTR(-ERANGE);
         }
 
-        err = __mlxsw_sp_vfid_create(mlxsw_sp, n_vfid);
+        fid = mlxsw_sp_vfid_to_fid(vfid);
+        err = mlxsw_sp_vfid_op(mlxsw_sp, fid, true);
         if (err) {
-                dev_err(dev, "Failed to create vFID=%d\n", n_vfid);
+                dev_err(dev, "Failed to create FID=%d\n", fid);
                 return ERR_PTR(err);
         }
 
-        vfid = kzalloc(sizeof(*vfid), GFP_KERNEL);
-        if (!vfid)
+        f = kzalloc(sizeof(*f), GFP_KERNEL);
+        if (!f)
                 goto err_allocate_vfid;
 
-        vfid->vfid = n_vfid;
-        vfid->vid = vid;
+        f->leave = mlxsw_sp_vport_vfid_leave;
+        f->fid = fid;
+        f->vid = vid;
 
-        list_add(&vfid->list, &mlxsw_sp->port_vfids.list);
-        set_bit(n_vfid, mlxsw_sp->port_vfids.mapped);
+        list_add(&f->list, &mlxsw_sp->port_vfids.list);
+        set_bit(vfid, mlxsw_sp->port_vfids.mapped);
 
-        return vfid;
+        return f;
 
 err_allocate_vfid:
-        __mlxsw_sp_vfid_destroy(mlxsw_sp, n_vfid);
+        mlxsw_sp_vfid_op(mlxsw_sp, fid, false);
         return ERR_PTR(-ENOMEM);
 }
 
 static void mlxsw_sp_vfid_destroy(struct mlxsw_sp *mlxsw_sp,
-                                  struct mlxsw_sp_vfid *vfid)
+                                  struct mlxsw_sp_fid *f)
 {
-        clear_bit(vfid->vfid, mlxsw_sp->port_vfids.mapped);
-        list_del(&vfid->list);
+        u16 vfid = mlxsw_sp_fid_to_vfid(f->fid);
+
+        clear_bit(vfid, mlxsw_sp->port_vfids.mapped);
+        list_del(&f->list);
 
-        __mlxsw_sp_vfid_destroy(mlxsw_sp, vfid->vfid);
+        mlxsw_sp_vfid_op(mlxsw_sp, f->fid, false);
 
-        kfree(vfid);
+        kfree(f);
 }
 
 static struct mlxsw_sp_port *
-mlxsw_sp_port_vport_create(struct mlxsw_sp_port *mlxsw_sp_port,
-                           struct mlxsw_sp_vfid *vfid)
+mlxsw_sp_port_vport_create(struct mlxsw_sp_port *mlxsw_sp_port, u16 vid)
 {
         struct mlxsw_sp_port *mlxsw_sp_vport;
 
@@ -741,8 +737,7 @@ mlxsw_sp_port_vport_create(struct mlxsw_sp_port *mlxsw_sp_port,
         mlxsw_sp_vport->stp_state = BR_STATE_FORWARDING;
         mlxsw_sp_vport->lagged = mlxsw_sp_port->lagged;
         mlxsw_sp_vport->lag_id = mlxsw_sp_port->lag_id;
-        mlxsw_sp_vport->vport.vfid = vfid;
-        mlxsw_sp_vport->vport.vid = vfid->vid;
+        mlxsw_sp_vport->vport.vid = vid;
 
         list_add(&mlxsw_sp_vport->vport.list, &mlxsw_sp_port->vports_list);
 
@@ -755,13 +750,72 @@ static void mlxsw_sp_port_vport_destroy(struct mlxsw_sp_port *mlxsw_sp_vport)
         kfree(mlxsw_sp_vport);
 }
 
+static int mlxsw_sp_vport_fid_map(struct mlxsw_sp_port *mlxsw_sp_vport, u16 fid,
+                                  bool valid)
+{
+        enum mlxsw_reg_svfa_mt mt = MLXSW_REG_SVFA_MT_PORT_VID_TO_FID;
+        u16 vid = mlxsw_sp_vport_vid_get(mlxsw_sp_vport);
+
+        return mlxsw_sp_port_vid_to_fid_set(mlxsw_sp_vport, mt, valid, fid,
+                                            vid);
+}
+
+static int mlxsw_sp_vport_vfid_join(struct mlxsw_sp_port *mlxsw_sp_vport)
+{
+        u16 vid = mlxsw_sp_vport_vid_get(mlxsw_sp_vport);
+        struct mlxsw_sp_fid *f;
+        int err;
+
+        f = mlxsw_sp_vfid_find(mlxsw_sp_vport->mlxsw_sp, vid);
+        if (!f) {
+                f = mlxsw_sp_vfid_create(mlxsw_sp_vport->mlxsw_sp, vid);
+                if (IS_ERR(f))
+                        return PTR_ERR(f);
+        }
+
+        if (!f->ref_count) {
+                err = mlxsw_sp_vport_flood_set(mlxsw_sp_vport, f->fid, true);
+                if (err)
+                        goto err_vport_flood_set;
+        }
+
+        err = mlxsw_sp_vport_fid_map(mlxsw_sp_vport, f->fid, true);
+        if (err)
+                goto err_vport_fid_map;
+
+        mlxsw_sp_vport_fid_set(mlxsw_sp_vport, f);
+        f->ref_count++;
+
+        return 0;
+
+err_vport_fid_map:
+        if (!f->ref_count)
+                mlxsw_sp_vport_flood_set(mlxsw_sp_vport, f->fid, false);
+err_vport_flood_set:
+        if (!f->ref_count)
+                mlxsw_sp_vfid_destroy(mlxsw_sp_vport->mlxsw_sp, f);
+        return err;
+}
+
+static void mlxsw_sp_vport_vfid_leave(struct mlxsw_sp_port *mlxsw_sp_vport)
+{
+        struct mlxsw_sp_fid *f = mlxsw_sp_vport_fid_get(mlxsw_sp_vport);
+
+        mlxsw_sp_vport_fid_set(mlxsw_sp_vport, NULL);
+
+        mlxsw_sp_vport_fid_map(mlxsw_sp_vport, f->fid, false);
+
+        if (--f->ref_count == 0) {
+                mlxsw_sp_vport_flood_set(mlxsw_sp_vport, f->fid, false);
+                mlxsw_sp_vfid_destroy(mlxsw_sp_vport->mlxsw_sp, f);
+        }
+}
+
 int mlxsw_sp_port_add_vid(struct net_device *dev, __be16 __always_unused proto,
                           u16 vid)
 {
         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev);
-        struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
         struct mlxsw_sp_port *mlxsw_sp_vport;
-        struct mlxsw_sp_vfid *vfid;
         int err;
 
         /* VLAN 0 is added to HW filter when device goes up, but it is
@@ -775,31 +829,10 @@ int mlxsw_sp_port_add_vid(struct net_device *dev, __be16 __always_unused proto,
                 return 0;
         }
 
-        vfid = mlxsw_sp_vfid_find(mlxsw_sp, vid);
-        if (!vfid) {
-                vfid = mlxsw_sp_vfid_create(mlxsw_sp, vid);
-                if (IS_ERR(vfid)) {
-                        netdev_err(dev, "Failed to create vFID for VID=%d\n",
-                                   vid);
-                        return PTR_ERR(vfid);
-                }
-        }
-
-        mlxsw_sp_vport = mlxsw_sp_port_vport_create(mlxsw_sp_port, vfid);
+        mlxsw_sp_vport = mlxsw_sp_port_vport_create(mlxsw_sp_port, vid);
         if (!mlxsw_sp_vport) {
                 netdev_err(dev, "Failed to create vPort for VID=%d\n", vid);
-                err = -ENOMEM;
-                goto err_port_vport_create;
-        }
-
-        if (!vfid->nr_vports) {
-                err = mlxsw_sp_vport_flood_set(mlxsw_sp_vport, vfid->vfid,
-                                               true, false);
-                if (err) {
-                        netdev_err(dev, "Failed to setup flooding for vFID=%d\n",
-                                   vfid->vfid);
-                        goto err_vport_flood_set;
-                }
+                return -ENOMEM;
         }
 
         /* When adding the first VLAN interface on a bridged port we need to
@@ -814,15 +847,10 @@ int mlxsw_sp_port_add_vid(struct net_device *dev, __be16 __always_unused proto,
                 }
         }
 
-        err = mlxsw_sp_port_vid_to_fid_set(mlxsw_sp_vport,
-                                           MLXSW_REG_SVFA_MT_PORT_VID_TO_FID,
-                                           true,
-                                           mlxsw_sp_vfid_to_fid(vfid->vfid),
-                                           vid);
+        err = mlxsw_sp_vport_vfid_join(mlxsw_sp_vport);
         if (err) {
-                netdev_err(dev, "Failed to map {Port, VID=%d} to vFID=%d\n",
-                           vid, vfid->vfid);
-                goto err_port_vid_to_fid_set;
+                netdev_err(dev, "Failed to join vFID\n");
+                goto err_vport_vfid_join;
         }
 
         err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_vport, vid, false);
@@ -845,8 +873,6 @@ int mlxsw_sp_port_add_vid(struct net_device *dev, __be16 __always_unused proto,
                 goto err_port_stp_state_set;
         }
 
-        vfid->nr_vports++;
-
         return 0;
 
 err_port_stp_state_set:
@@ -854,21 +880,12 @@ err_port_stp_state_set:
 err_port_add_vid:
         mlxsw_sp_port_vid_learning_set(mlxsw_sp_vport, vid, true);
 err_port_vid_learning_set:
-        mlxsw_sp_port_vid_to_fid_set(mlxsw_sp_vport,
-                                     MLXSW_REG_SVFA_MT_PORT_VID_TO_FID, false,
-                                     mlxsw_sp_vfid_to_fid(vfid->vfid), vid);
-err_port_vid_to_fid_set:
+        mlxsw_sp_vport_vfid_leave(mlxsw_sp_vport);
+err_vport_vfid_join:
         if (list_is_singular(&mlxsw_sp_port->vports_list))
                 mlxsw_sp_port_vlan_mode_trans(mlxsw_sp_port);
 err_port_vp_mode_trans:
-        if (!vfid->nr_vports)
-                mlxsw_sp_vport_flood_set(mlxsw_sp_vport, vfid->vfid, false,
-                                         false);
-err_vport_flood_set:
         mlxsw_sp_port_vport_destroy(mlxsw_sp_vport);
-err_port_vport_create:
-        if (!vfid->nr_vports)
-                mlxsw_sp_vfid_destroy(mlxsw_sp, vfid);
         return err;
 }
 
@@ -877,7 +894,7 @@ int mlxsw_sp_port_kill_vid(struct net_device *dev,
 {
         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev);
         struct mlxsw_sp_port *mlxsw_sp_vport;
-        struct mlxsw_sp_vfid *vfid;
+        struct mlxsw_sp_fid *f;
         int err;
 
         /* VLAN 0 is removed from HW filter when device goes down, but
@@ -892,8 +909,6 @@ int mlxsw_sp_port_kill_vid(struct net_device *dev,
                 return 0;
         }
 
-        vfid = mlxsw_sp_vport->vport.vfid;
-
         err = mlxsw_sp_port_stp_state_set(mlxsw_sp_vport, vid,
                                           MLXSW_REG_SPMS_STATE_DISCARDING);
         if (err) {
@@ -914,16 +929,12 @@ int mlxsw_sp_port_kill_vid(struct net_device *dev,
                 return err;
         }
 
-        err = mlxsw_sp_port_vid_to_fid_set(mlxsw_sp_vport,
-                                           MLXSW_REG_SVFA_MT_PORT_VID_TO_FID,
-                                           false,
-                                           mlxsw_sp_vfid_to_fid(vfid->vfid),
-                                           vid);
-        if (err) {
-                netdev_err(dev, "Failed to invalidate {Port, VID=%d} to vFID=%d mapping\n",
-                           vid, vfid->vfid);
-                return err;
-        }
+        /* Drop FID reference. If this was the last reference the
+         * resources will be freed.
+         */
+        f = mlxsw_sp_vport_fid_get(mlxsw_sp_vport);
+        if (f && !WARN_ON(!f->leave))
+                f->leave(mlxsw_sp_vport);
 
         /* When removing the last VLAN interface on a bridged port we need to
          * transition all active 802.1Q bridge VLANs to use VID to FID
@@ -937,13 +948,8 @@ int mlxsw_sp_port_kill_vid(struct net_device *dev,
                 }
         }
 
-        vfid->nr_vports--;
         mlxsw_sp_port_vport_destroy(mlxsw_sp_vport);
 
-        /* Destroy the vFID if no vPorts are assigned to it anymore. */
-        if (!vfid->nr_vports)
-                mlxsw_sp_vfid_destroy(mlxsw_sp_port->mlxsw_sp, vfid);
-
         return 0;
 }
 
@@ -2403,6 +2409,7 @@ static int mlxsw_sp_init(struct mlxsw_core *mlxsw_core,
 
         mlxsw_sp->core = mlxsw_core;
         mlxsw_sp->bus_info = mlxsw_bus_info;
+        INIT_LIST_HEAD(&mlxsw_sp->fids);
         INIT_LIST_HEAD(&mlxsw_sp->port_vfids.list);
         INIT_LIST_HEAD(&mlxsw_sp->br_vfids.list);
         INIT_LIST_HEAD(&mlxsw_sp->br_mids.list);
@@ -2479,6 +2486,7 @@ static void mlxsw_sp_fini(struct mlxsw_core *mlxsw_core)
         mlxsw_sp_traps_fini(mlxsw_sp);
         mlxsw_sp_event_unregister(mlxsw_sp, MLXSW_TRAP_ID_PUDE);
         mlxsw_sp_ports_remove(mlxsw_sp);
+        WARN_ON(!list_empty(&mlxsw_sp->fids));
 }
 
 static struct mlxsw_config_profile mlxsw_sp_config_profile = {
@@ -2540,16 +2548,37 @@ static struct mlxsw_driver mlxsw_sp_driver = {
         .profile                        = &mlxsw_sp_config_profile,
 };
 
-static int
-mlxsw_sp_port_fdb_flush_by_port(const struct mlxsw_sp_port *mlxsw_sp_port)
+static bool mlxsw_sp_lag_port_fid_member(struct mlxsw_sp_port *lag_port,
+                                         u16 fid)
+{
+        if (mlxsw_sp_fid_is_vfid(fid))
+                return mlxsw_sp_port_vport_find_by_fid(lag_port, fid);
+        else
+                return test_bit(fid, lag_port->active_vlans);
+}
+
+static bool mlxsw_sp_port_fdb_should_flush(struct mlxsw_sp_port *mlxsw_sp_port,
+                                           u16 fid)
 {
         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
-        char sfdf_pl[MLXSW_REG_SFDF_LEN];
+        u8 local_port = mlxsw_sp_port->local_port;
+        u16 lag_id = mlxsw_sp_port->lag_id;
+        int i, count = 0;
 
-        mlxsw_reg_sfdf_pack(sfdf_pl, MLXSW_REG_SFDF_FLUSH_PER_PORT);
-        mlxsw_reg_sfdf_system_port_set(sfdf_pl, mlxsw_sp_port->local_port);
+        if (!mlxsw_sp_port->lagged)
+                return true;
 
-        return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sfdf), sfdf_pl);
+        for (i = 0; i < MLXSW_SP_PORT_PER_LAG_MAX; i++) {
+                struct mlxsw_sp_port *lag_port;
+
+                lag_port = mlxsw_sp_port_lagged_get(mlxsw_sp, lag_id, i);
+                if (!lag_port || lag_port->local_port == local_port)
+                        continue;
+                if (mlxsw_sp_lag_port_fid_member(lag_port, fid))
+                        count++;
+        }
+
+        return !count;
 }
 
 static int
@@ -2564,17 +2593,8 @@ mlxsw_sp_port_fdb_flush_by_port_fid(const struct mlxsw_sp_port *mlxsw_sp_port,
         mlxsw_reg_sfdf_port_fid_system_port_set(sfdf_pl,
                                                 mlxsw_sp_port->local_port);
 
-        return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sfdf), sfdf_pl);
-}
-
-static int
-mlxsw_sp_port_fdb_flush_by_lag_id(const struct mlxsw_sp_port *mlxsw_sp_port)
-{
-        struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
-        char sfdf_pl[MLXSW_REG_SFDF_LEN];
-
-        mlxsw_reg_sfdf_pack(sfdf_pl, MLXSW_REG_SFDF_FLUSH_PER_LAG);
-        mlxsw_reg_sfdf_lag_id_set(sfdf_pl, mlxsw_sp_port->lag_id);
+        netdev_dbg(mlxsw_sp_port->dev, "FDB flushed using Port=%d, FID=%d\n",
+                   mlxsw_sp_port->local_port, fid);
 
         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sfdf), sfdf_pl);
 }
@@ -2590,71 +2610,51 @@ mlxsw_sp_port_fdb_flush_by_lag_id_fid(const struct mlxsw_sp_port *mlxsw_sp_port,
         mlxsw_reg_sfdf_fid_set(sfdf_pl, fid);
         mlxsw_reg_sfdf_lag_fid_lag_id_set(sfdf_pl, mlxsw_sp_port->lag_id);
 
+        netdev_dbg(mlxsw_sp_port->dev, "FDB flushed using LAG ID=%d, FID=%d\n",
+                   mlxsw_sp_port->lag_id, fid);
+
         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sfdf), sfdf_pl);
 }
 
-static int
-__mlxsw_sp_port_fdb_flush(const struct mlxsw_sp_port *mlxsw_sp_port)
+int mlxsw_sp_port_fdb_flush(struct mlxsw_sp_port *mlxsw_sp_port, u16 fid)
 {
-        int err, last_err = 0;
-        u16 vid;
-
-        for (vid = 1; vid < VLAN_N_VID - 1; vid++) {
-                err = mlxsw_sp_port_fdb_flush_by_port_fid(mlxsw_sp_port, vid);
-                if (err)
-                        last_err = err;
-        }
+        if (!mlxsw_sp_port_fdb_should_flush(mlxsw_sp_port, fid))
+                return 0;
 
-        return last_err;
+        if (mlxsw_sp_port->lagged)
+                return mlxsw_sp_port_fdb_flush_by_lag_id_fid(mlxsw_sp_port,
+                                                             fid);
+        else
+                return mlxsw_sp_port_fdb_flush_by_port_fid(mlxsw_sp_port, fid);
 }
 
-static int
-__mlxsw_sp_port_fdb_flush_lagged(const struct mlxsw_sp_port *mlxsw_sp_port)
+static bool mlxsw_sp_port_dev_check(const struct net_device *dev)
 {
-        int err, last_err = 0;
-        u16 vid;
-
-        for (vid = 1; vid < VLAN_N_VID - 1; vid++) {
-                err = mlxsw_sp_port_fdb_flush_by_lag_id_fid(mlxsw_sp_port, vid);
-                if (err)
-                        last_err = err;
-        }
-
-        return last_err;
+        return dev->netdev_ops == &mlxsw_sp_port_netdev_ops;
 }
 
-static int mlxsw_sp_port_fdb_flush(struct mlxsw_sp_port *mlxsw_sp_port)
+static bool mlxsw_sp_master_bridge_check(struct mlxsw_sp *mlxsw_sp,
+                                         struct net_device *br_dev)
 {
-        if (!list_empty(&mlxsw_sp_port->vports_list))
-                if (mlxsw_sp_port->lagged)
-                        return __mlxsw_sp_port_fdb_flush_lagged(mlxsw_sp_port);
-                else
-                        return __mlxsw_sp_port_fdb_flush(mlxsw_sp_port);
-        else
-                if (mlxsw_sp_port->lagged)
-                        return mlxsw_sp_port_fdb_flush_by_lag_id(mlxsw_sp_port);
-                else
-                        return mlxsw_sp_port_fdb_flush_by_port(mlxsw_sp_port);
+        return !mlxsw_sp->master_bridge.dev ||
+               mlxsw_sp->master_bridge.dev == br_dev;
 }
 
-static int mlxsw_sp_vport_fdb_flush(struct mlxsw_sp_port *mlxsw_sp_vport)
+static void mlxsw_sp_master_bridge_inc(struct mlxsw_sp *mlxsw_sp,
+                                       struct net_device *br_dev)
 {
-        u16 vfid = mlxsw_sp_vport_vfid_get(mlxsw_sp_vport);
-        u16 fid = mlxsw_sp_vfid_to_fid(vfid);
-
-        if (mlxsw_sp_vport->lagged)
-                return mlxsw_sp_port_fdb_flush_by_lag_id_fid(mlxsw_sp_vport,
-                                                             fid);
-        else
-                return mlxsw_sp_port_fdb_flush_by_port_fid(mlxsw_sp_vport, fid);
+        mlxsw_sp->master_bridge.dev = br_dev;
+        mlxsw_sp->master_bridge.ref_count++;
 }
 
-static bool mlxsw_sp_port_dev_check(const struct net_device *dev)
+static void mlxsw_sp_master_bridge_dec(struct mlxsw_sp *mlxsw_sp)
 {
-        return dev->netdev_ops == &mlxsw_sp_port_netdev_ops;
+        if (--mlxsw_sp->master_bridge.ref_count == 0)
+                mlxsw_sp->master_bridge.dev = NULL;
 }
 
-static int mlxsw_sp_port_bridge_join(struct mlxsw_sp_port *mlxsw_sp_port)
+static int mlxsw_sp_port_bridge_join(struct mlxsw_sp_port *mlxsw_sp_port,
+                                     struct net_device *br_dev)
 {
         struct net_device *dev = mlxsw_sp_port->dev;
         int err;
@@ -2668,6 +2668,8 @@ static int mlxsw_sp_port_bridge_join(struct mlxsw_sp_port *mlxsw_sp_port)
         if (err)
                 return err;
 
+        mlxsw_sp_master_bridge_inc(mlxsw_sp_port->mlxsw_sp, br_dev);
+
         mlxsw_sp_port->learning = 1;
         mlxsw_sp_port->learning_sync = 1;
         mlxsw_sp_port->uc_flood = 1;
@@ -2676,16 +2678,14 @@ static int mlxsw_sp_port_bridge_join(struct mlxsw_sp_port *mlxsw_sp_port)
         return 0;
 }
 
-static int mlxsw_sp_port_bridge_leave(struct mlxsw_sp_port *mlxsw_sp_port,
-                                      bool flush_fdb)
+static void mlxsw_sp_port_bridge_leave(struct mlxsw_sp_port *mlxsw_sp_port)
 {
         struct net_device *dev = mlxsw_sp_port->dev;
 
-        if (flush_fdb && mlxsw_sp_port_fdb_flush(mlxsw_sp_port))
-                netdev_err(mlxsw_sp_port->dev, "Failed to flush FDB\n");
-
         mlxsw_sp_port_pvid_set(mlxsw_sp_port, 1);
 
+        mlxsw_sp_master_bridge_dec(mlxsw_sp_port->mlxsw_sp);
+
         mlxsw_sp_port->learning = 0;
         mlxsw_sp_port->learning_sync = 0;
         mlxsw_sp_port->uc_flood = 0;
@@ -2694,28 +2694,7 @@ static int mlxsw_sp_port_bridge_leave(struct mlxsw_sp_port *mlxsw_sp_port,
         /* Add implicit VLAN interface in the device, so that untagged
          * packets will be classified to the default vFID.
          */
-        return mlxsw_sp_port_add_vid(dev, 0, 1);
-}
-
-static bool mlxsw_sp_master_bridge_check(struct mlxsw_sp *mlxsw_sp,
-                                         struct net_device *br_dev)
-{
-        return !mlxsw_sp->master_bridge.dev ||
-               mlxsw_sp->master_bridge.dev == br_dev;
-}
-
-static void mlxsw_sp_master_bridge_inc(struct mlxsw_sp *mlxsw_sp,
-                                       struct net_device *br_dev)
-{
-        mlxsw_sp->master_bridge.dev = br_dev;
-        mlxsw_sp->master_bridge.ref_count++;
-}
-
-static void mlxsw_sp_master_bridge_dec(struct mlxsw_sp *mlxsw_sp,
-                                       struct net_device *br_dev)
-{
-        if (--mlxsw_sp->master_bridge.ref_count == 0)
-                mlxsw_sp->master_bridge.dev = NULL;
+        mlxsw_sp_port_add_vid(dev, 0, 1);
 }
 
 static int mlxsw_sp_lag_create(struct mlxsw_sp *mlxsw_sp, u16 lag_id)
@@ -2876,65 +2855,33 @@ err_col_port_add:
         return err;
 }
 
-static int mlxsw_sp_vport_bridge_leave(struct mlxsw_sp_port *mlxsw_sp_vport,
-                                       struct net_device *br_dev,
-                                       bool flush_fdb);
-
-static int mlxsw_sp_port_lag_leave(struct mlxsw_sp_port *mlxsw_sp_port,
-                                   struct net_device *lag_dev)
+static void mlxsw_sp_port_lag_leave(struct mlxsw_sp_port *mlxsw_sp_port,
+                                    struct net_device *lag_dev)
 {
         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
-        struct mlxsw_sp_port *mlxsw_sp_vport;
-        struct mlxsw_sp_upper *lag;
         u16 lag_id = mlxsw_sp_port->lag_id;
-        int err;
+        struct mlxsw_sp_upper *lag;
 
         if (!mlxsw_sp_port->lagged)
-                return 0;
+                return;
         lag = mlxsw_sp_lag_get(mlxsw_sp, lag_id);
         WARN_ON(lag->ref_count == 0);
 
-        err = mlxsw_sp_lag_col_port_disable(mlxsw_sp_port, lag_id);
-        if (err)
-                return err;
-        err = mlxsw_sp_lag_col_port_remove(mlxsw_sp_port, lag_id);
-        if (err)
-                return err;
-
-        /* In case we leave a LAG device that has bridges built on top,
-         * then their teardown sequence is never issued and we need to
-         * invoke the necessary cleanup routines ourselves.
-         */
-        list_for_each_entry(mlxsw_sp_vport, &mlxsw_sp_port->vports_list,
-                            vport.list) {
-                struct net_device *br_dev;
-
-                if (!mlxsw_sp_vport->bridged)
-                        continue;
-
-                br_dev = mlxsw_sp_vport_br_get(mlxsw_sp_vport);
-                mlxsw_sp_vport_bridge_leave(mlxsw_sp_vport, br_dev, false);
-        }
+        mlxsw_sp_lag_col_port_disable(mlxsw_sp_port, lag_id);
+        mlxsw_sp_lag_col_port_remove(mlxsw_sp_port, lag_id);
 
         if (mlxsw_sp_port->bridged) {
                 mlxsw_sp_port_active_vlans_del(mlxsw_sp_port);
-                mlxsw_sp_port_bridge_leave(mlxsw_sp_port, false);
-                mlxsw_sp_master_bridge_dec(mlxsw_sp, NULL);
+                mlxsw_sp_port_bridge_leave(mlxsw_sp_port);
         }
 
-        if (lag->ref_count == 1) {
-                if (mlxsw_sp_port_fdb_flush_by_lag_id(mlxsw_sp_port))
-                        netdev_err(mlxsw_sp_port->dev, "Failed to flush FDB\n");
-                err = mlxsw_sp_lag_destroy(mlxsw_sp, lag_id);
-                if (err)
-                        return err;
-        }
+        if (lag->ref_count == 1)
+                mlxsw_sp_lag_destroy(mlxsw_sp, lag_id);
 
         mlxsw_core_lag_mapping_clear(mlxsw_sp->core, lag_id,
                                      mlxsw_sp_port->local_port);
         mlxsw_sp_port->lagged = 0;
         lag->ref_count--;
-        return 0;
 }
 
 static int mlxsw_sp_lag_dist_port_add(struct mlxsw_sp_port *mlxsw_sp_port,
@@ -2983,42 +2930,25 @@ static int mlxsw_sp_port_vlan_link(struct mlxsw_sp_port *mlxsw_sp_port,
         u16 vid = vlan_dev_vlan_id(vlan_dev);
 
         mlxsw_sp_vport = mlxsw_sp_port_vport_find(mlxsw_sp_port, vid);
-        if (!mlxsw_sp_vport) {
-                WARN_ON(!mlxsw_sp_vport);
+        if (WARN_ON(!mlxsw_sp_vport))
                 return -EINVAL;
-        }
 
         mlxsw_sp_vport->dev = vlan_dev;
 
         return 0;
 }
 
-static int mlxsw_sp_port_vlan_unlink(struct mlxsw_sp_port *mlxsw_sp_port,
-                                     struct net_device *vlan_dev)
+static void mlxsw_sp_port_vlan_unlink(struct mlxsw_sp_port *mlxsw_sp_port,
+                                      struct net_device *vlan_dev)
 {
         struct mlxsw_sp_port *mlxsw_sp_vport;
         u16 vid = vlan_dev_vlan_id(vlan_dev);
 
         mlxsw_sp_vport = mlxsw_sp_port_vport_find(mlxsw_sp_port, vid);
-        if (!mlxsw_sp_vport) {
-                WARN_ON(!mlxsw_sp_vport);
-                return -EINVAL;
-        }
-
-        /* When removing a VLAN device while still bridged we should first
-         * remove it from the bridge, as we receive the bridge's notification
-         * when the vPort is already gone.
-         */
-        if (mlxsw_sp_vport->bridged) {
-                struct net_device *br_dev;
-
-                br_dev = mlxsw_sp_vport_br_get(mlxsw_sp_vport);
-                mlxsw_sp_vport_bridge_leave(mlxsw_sp_vport, br_dev, true);
-        }
+        if (WARN_ON(!mlxsw_sp_vport))
+                return;
 
         mlxsw_sp_vport->dev = mlxsw_sp_port->dev;
-
-        return 0;
 }
 
 static int mlxsw_sp_netdevice_port_upper_event(struct net_device *dev,
@@ -3028,7 +2958,7 @@ static int mlxsw_sp_netdevice_port_upper_event(struct net_device *dev,
         struct mlxsw_sp_port *mlxsw_sp_port;
         struct net_device *upper_dev;
         struct mlxsw_sp *mlxsw_sp;
-        int err;
+        int err = 0;
 
         mlxsw_sp_port = netdev_priv(dev);
         mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
@@ -3037,73 +2967,56 @@ static int mlxsw_sp_netdevice_port_upper_event(struct net_device *dev,
         switch (event) {
         case NETDEV_PRECHANGEUPPER:
                 upper_dev = info->upper_dev;
-                if (!info->master || !info->linking)
+                if (!is_vlan_dev(upper_dev) &&
+                    !netif_is_lag_master(upper_dev) &&
+                    !netif_is_bridge_master(upper_dev))
+                        return -EINVAL;
+                if (!info->linking)
                         break;
                 /* HW limitation forbids to put ports to multiple bridges. */
                 if (netif_is_bridge_master(upper_dev) &&
                     !mlxsw_sp_master_bridge_check(mlxsw_sp, upper_dev))
-                        return NOTIFY_BAD;
+                        return -EINVAL;
                 if (netif_is_lag_master(upper_dev) &&
                     !mlxsw_sp_master_lag_check(mlxsw_sp, upper_dev,
                                                info->upper_info))
-                        return NOTIFY_BAD;
+                        return -EINVAL;
+                if (netif_is_lag_master(upper_dev) && vlan_uses_dev(dev))
+                        return -EINVAL;
+                if (netif_is_lag_port(dev) && is_vlan_dev(upper_dev) &&
+                    !netif_is_lag_master(vlan_dev_real_dev(upper_dev)))
+                        return -EINVAL;
                 break;
         case NETDEV_CHANGEUPPER:
                 upper_dev = info->upper_dev;
                 if (is_vlan_dev(upper_dev)) {
-                        if (info->linking) {
+                        if (info->linking)
                                 err = mlxsw_sp_port_vlan_link(mlxsw_sp_port,
                                                               upper_dev);
-                                if (err) {
-                                        netdev_err(dev, "Failed to link VLAN device\n");
-                                        return NOTIFY_BAD;
-                                }
-                        } else {
-                                err = mlxsw_sp_port_vlan_unlink(mlxsw_sp_port,
-                                                                upper_dev);
-                                if (err) {
-                                        netdev_err(dev, "Failed to unlink VLAN device\n");
-                                        return NOTIFY_BAD;
-                                }
-                        }
+                        else
+                                 mlxsw_sp_port_vlan_unlink(mlxsw_sp_port,
+                                                           upper_dev);
                 } else if (netif_is_bridge_master(upper_dev)) {
-                        if (info->linking) {
-                                err = mlxsw_sp_port_bridge_join(mlxsw_sp_port);
-                                if (err) {
-                                        netdev_err(dev, "Failed to join bridge\n");
-                                        return NOTIFY_BAD;
-                                }
-                                mlxsw_sp_master_bridge_inc(mlxsw_sp, upper_dev);
-                        } else {
-                                err = mlxsw_sp_port_bridge_leave(mlxsw_sp_port,
-                                                                 true);
-                                mlxsw_sp_master_bridge_dec(mlxsw_sp, upper_dev);
-                                if (err) {
-                                        netdev_err(dev, "Failed to leave bridge\n");
-                                        return NOTIFY_BAD;
-                                }
-                        }
+                        if (info->linking)
+                                err = mlxsw_sp_port_bridge_join(mlxsw_sp_port,
+                                                                upper_dev);
+                        else
+                                mlxsw_sp_port_bridge_leave(mlxsw_sp_port);
                 } else if (netif_is_lag_master(upper_dev)) {
-                        if (info->linking) {
+                        if (info->linking)
                                 err = mlxsw_sp_port_lag_join(mlxsw_sp_port,
                                                              upper_dev);
-                                if (err) {
-                                        netdev_err(dev, "Failed to join link aggregation\n");
-                                        return NOTIFY_BAD;
-                                }
-                        } else {
-                                err = mlxsw_sp_port_lag_leave(mlxsw_sp_port,
-                                                              upper_dev);
-                                if (err) {
-                                        netdev_err(dev, "Failed to leave link aggregation\n");
-                                        return NOTIFY_BAD;
-                                }
-                        }
+                        else
+                                mlxsw_sp_port_lag_leave(mlxsw_sp_port,
+                                                        upper_dev);
+                } else {
+                        err = -EINVAL;
+                        WARN_ON(1);
                 }
                 break;
         }
 
-        return NOTIFY_DONE;
+        return err;
 }
 
 static int mlxsw_sp_netdevice_port_lower_event(struct net_device *dev,
@@ -3127,7 +3040,7 @@ static int mlxsw_sp_netdevice_port_lower_event(struct net_device *dev,
                 break;
         }
 
-        return NOTIFY_DONE;
+        return 0;
 }
 
 static int mlxsw_sp_netdevice_port_event(struct net_device *dev,
@@ -3141,7 +3054,7 @@ static int mlxsw_sp_netdevice_port_event(struct net_device *dev,
                 return mlxsw_sp_netdevice_port_lower_event(dev, event, ptr);
         }
 
-        return NOTIFY_DONE;
+        return 0;
 }
 
 static int mlxsw_sp_netdevice_lag_event(struct net_device *lag_dev,
@@ -3154,23 +3067,23 @@ static int mlxsw_sp_netdevice_lag_event(struct net_device *lag_dev,
         netdev_for_each_lower_dev(lag_dev, dev, iter) {
                 if (mlxsw_sp_port_dev_check(dev)) {
                         ret = mlxsw_sp_netdevice_port_event(dev, event, ptr);
-                        if (ret == NOTIFY_BAD)
+                        if (ret)
                                 return ret;
                 }
         }
 
-        return NOTIFY_DONE;
+        return 0;
 }
 
-static struct mlxsw_sp_vfid *
+static struct mlxsw_sp_fid *
 mlxsw_sp_br_vfid_find(const struct mlxsw_sp *mlxsw_sp,
                       const struct net_device *br_dev)
 {
-        struct mlxsw_sp_vfid *vfid;
+        struct mlxsw_sp_fid *f;
 
-        list_for_each_entry(vfid, &mlxsw_sp->br_vfids.list, list) {
-                if (vfid->br_dev == br_dev)
-                        return vfid;
+        list_for_each_entry(f, &mlxsw_sp->br_vfids.list, list) {
+                if (f->dev == br_dev)
+                        return f;
         }
 
         return NULL;
@@ -3192,180 +3105,127 @@ static u16 mlxsw_sp_avail_br_vfid_get(const struct mlxsw_sp *mlxsw_sp)
                                    MLXSW_SP_VFID_BR_MAX);
 }
 
-static struct mlxsw_sp_vfid *mlxsw_sp_br_vfid_create(struct mlxsw_sp *mlxsw_sp,
-                                                     struct net_device *br_dev)
+static void mlxsw_sp_vport_br_vfid_leave(struct mlxsw_sp_port *mlxsw_sp_vport);
+
+static struct mlxsw_sp_fid *mlxsw_sp_br_vfid_create(struct mlxsw_sp *mlxsw_sp,
+                                                    struct net_device *br_dev)
 {
         struct device *dev = mlxsw_sp->bus_info->dev;
-        struct mlxsw_sp_vfid *vfid;
-        u16 n_vfid;
+        struct mlxsw_sp_fid *f;
+        u16 vfid, fid;
         int err;
 
-        n_vfid = mlxsw_sp_br_vfid_to_vfid(mlxsw_sp_avail_br_vfid_get(mlxsw_sp));
-        if (n_vfid == MLXSW_SP_VFID_MAX) {
+        vfid = mlxsw_sp_br_vfid_to_vfid(mlxsw_sp_avail_br_vfid_get(mlxsw_sp));
+        if (vfid == MLXSW_SP_VFID_MAX) {
                 dev_err(dev, "No available vFIDs\n");
                 return ERR_PTR(-ERANGE);
         }
 
-        err = __mlxsw_sp_vfid_create(mlxsw_sp, n_vfid);
+        fid = mlxsw_sp_vfid_to_fid(vfid);
+        err = mlxsw_sp_vfid_op(mlxsw_sp, fid, true);
         if (err) {
-                dev_err(dev, "Failed to create vFID=%d\n", n_vfid);
+                dev_err(dev, "Failed to create FID=%d\n", fid);
                 return ERR_PTR(err);
         }
 
-        vfid = kzalloc(sizeof(*vfid), GFP_KERNEL);
-        if (!vfid)
+        f = kzalloc(sizeof(*f), GFP_KERNEL);
+        if (!f)
                 goto err_allocate_vfid;
 
-        vfid->vfid = n_vfid;
-        vfid->br_dev = br_dev;
+        f->leave = mlxsw_sp_vport_br_vfid_leave;
+        f->fid = fid;
+        f->dev = br_dev;
 
-        list_add(&vfid->list, &mlxsw_sp->br_vfids.list);
-        set_bit(mlxsw_sp_vfid_to_br_vfid(n_vfid), mlxsw_sp->br_vfids.mapped);
+        list_add(&f->list, &mlxsw_sp->br_vfids.list);
+        set_bit(mlxsw_sp_vfid_to_br_vfid(vfid), mlxsw_sp->br_vfids.mapped);
 
-        return vfid;
+        return f;
 
 err_allocate_vfid:
-        __mlxsw_sp_vfid_destroy(mlxsw_sp, n_vfid);
+        mlxsw_sp_vfid_op(mlxsw_sp, fid, false);
         return ERR_PTR(-ENOMEM);
 }
 
 static void mlxsw_sp_br_vfid_destroy(struct mlxsw_sp *mlxsw_sp,
-                                     struct mlxsw_sp_vfid *vfid)
+                                     struct mlxsw_sp_fid *f)
 {
-        u16 br_vfid = mlxsw_sp_vfid_to_br_vfid(vfid->vfid);
+        u16 vfid = mlxsw_sp_fid_to_vfid(f->fid);
+        u16 br_vfid = mlxsw_sp_vfid_to_br_vfid(vfid);
 
         clear_bit(br_vfid, mlxsw_sp->br_vfids.mapped);
-        list_del(&vfid->list);
+        list_del(&f->list);
 
-        __mlxsw_sp_vfid_destroy(mlxsw_sp, vfid->vfid);
+        mlxsw_sp_vfid_op(mlxsw_sp, f->fid, false);
 
-        kfree(vfid);
+        kfree(f);
 }
 
-static int mlxsw_sp_vport_bridge_leave(struct mlxsw_sp_port *mlxsw_sp_vport,
-                                       struct net_device *br_dev,
-                                       bool flush_fdb)
+static int mlxsw_sp_vport_br_vfid_join(struct mlxsw_sp_port *mlxsw_sp_vport,
+                                       struct net_device *br_dev)
 {
-        struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
-        u16 vid = mlxsw_sp_vport_vid_get(mlxsw_sp_vport);
-        struct net_device *dev = mlxsw_sp_vport->dev;
-        struct mlxsw_sp_vfid *vfid, *new_vfid;
+        struct mlxsw_sp_fid *f;
         int err;
 
-        vfid = mlxsw_sp_br_vfid_find(mlxsw_sp, br_dev);
-        if (!vfid) {
-                WARN_ON(!vfid);
-                return -EINVAL;
+        f = mlxsw_sp_br_vfid_find(mlxsw_sp_vport->mlxsw_sp, br_dev);
+        if (!f) {
+                f = mlxsw_sp_br_vfid_create(mlxsw_sp_vport->mlxsw_sp, br_dev);
+                if (IS_ERR(f))
+                        return PTR_ERR(f);
         }
 
-        /* We need a vFID to go back to after leaving the bridge's vFID. */
-        new_vfid = mlxsw_sp_vfid_find(mlxsw_sp, vid);
-        if (!new_vfid) {
-                new_vfid = mlxsw_sp_vfid_create(mlxsw_sp, vid);
-                if (IS_ERR(new_vfid)) {
-                        netdev_err(dev, "Failed to create vFID for VID=%d\n",
-                                   vid);
-                        return PTR_ERR(new_vfid);
-                }
-        }
+        err = mlxsw_sp_vport_flood_set(mlxsw_sp_vport, f->fid, true);
+        if (err)
+                goto err_vport_flood_set;
 
-        /* Invalidate existing {Port, VID} to vFID mapping and create a new
-         * one for the new vFID.
-         */
-        err = mlxsw_sp_port_vid_to_fid_set(mlxsw_sp_vport,
-                                           MLXSW_REG_SVFA_MT_PORT_VID_TO_FID,
-                                           false,
-                                           mlxsw_sp_vfid_to_fid(vfid->vfid),
-                                           vid);
-        if (err) {
-                netdev_err(dev, "Failed to invalidate {Port, VID} to vFID=%d mapping\n",
-                           vfid->vfid);
-                goto err_port_vid_to_fid_invalidate;
-        }
+        err = mlxsw_sp_vport_fid_map(mlxsw_sp_vport, f->fid, true);
+        if (err)
+                goto err_vport_fid_map;
 
-        err = mlxsw_sp_port_vid_to_fid_set(mlxsw_sp_vport,
-                                           MLXSW_REG_SVFA_MT_PORT_VID_TO_FID,
-                                           true,
-                                           mlxsw_sp_vfid_to_fid(new_vfid->vfid),
-                                           vid);
-        if (err) {
-                netdev_err(dev, "Failed to map {Port, VID} to vFID=%d\n",
-                           new_vfid->vfid);
-                goto err_port_vid_to_fid_validate;
-        }
+        mlxsw_sp_vport_fid_set(mlxsw_sp_vport, f);
+        f->ref_count++;
 
-        err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_vport, vid, false);
-        if (err) {
-                netdev_err(dev, "Failed to disable learning\n");
-                goto err_port_vid_learning_set;
-        }
+        netdev_dbg(mlxsw_sp_vport->dev, "Joined FID=%d\n", f->fid);
 
-        err = mlxsw_sp_vport_flood_set(mlxsw_sp_vport, vfid->vfid, false,
-                                       false);
-        if (err) {
-                netdev_err(dev, "Failed clear to clear flooding\n");
-                goto err_vport_flood_set;
-        }
+        return 0;
 
-        err = mlxsw_sp_port_stp_state_set(mlxsw_sp_vport, vid,
-                                          MLXSW_REG_SPMS_STATE_FORWARDING);
-        if (err) {
-                netdev_err(dev, "Failed to set STP state\n");
-                goto err_port_stp_state_set;
-        }
+err_vport_fid_map:
+        mlxsw_sp_vport_flood_set(mlxsw_sp_vport, f->fid, false);
+err_vport_flood_set:
+        if (!f->ref_count)
+                mlxsw_sp_br_vfid_destroy(mlxsw_sp_vport->mlxsw_sp, f);
+        return err;
+}
+
+static void mlxsw_sp_vport_br_vfid_leave(struct mlxsw_sp_port *mlxsw_sp_vport)
+{
+        struct mlxsw_sp_fid *f = mlxsw_sp_vport_fid_get(mlxsw_sp_vport);
 
-        if (flush_fdb && mlxsw_sp_vport_fdb_flush(mlxsw_sp_vport))
-                netdev_err(dev, "Failed to flush FDB\n");
+        netdev_dbg(mlxsw_sp_vport->dev, "Left FID=%d\n", f->fid);
 
-        /* Switch between the vFIDs and destroy the old one if needed. */
-        new_vfid->nr_vports++;
-        mlxsw_sp_vport->vport.vfid = new_vfid;
-        vfid->nr_vports--;
-        if (!vfid->nr_vports)
-                mlxsw_sp_br_vfid_destroy(mlxsw_sp, vfid);
+        mlxsw_sp_vport_fid_map(mlxsw_sp_vport, f->fid, false);
 
-        mlxsw_sp_vport->learning = 0;
-        mlxsw_sp_vport->learning_sync = 0;
-        mlxsw_sp_vport->uc_flood = 0;
-        mlxsw_sp_vport->bridged = 0;
+        mlxsw_sp_vport_flood_set(mlxsw_sp_vport, f->fid, false);
 
-        return 0;
+        mlxsw_sp_port_fdb_flush(mlxsw_sp_vport, f->fid);
 
-err_port_stp_state_set:
-err_vport_flood_set:
-err_port_vid_learning_set:
-err_port_vid_to_fid_validate:
-err_port_vid_to_fid_invalidate:
-        /* Rollback vFID only if new. */
-        if (!new_vfid->nr_vports)
-                mlxsw_sp_vfid_destroy(mlxsw_sp, new_vfid);
-        return err;
+        mlxsw_sp_vport_fid_set(mlxsw_sp_vport, NULL);
+        if (--f->ref_count == 0)
+                mlxsw_sp_br_vfid_destroy(mlxsw_sp_vport->mlxsw_sp, f);
 }
 
 static int mlxsw_sp_vport_bridge_join(struct mlxsw_sp_port *mlxsw_sp_vport,
                                       struct net_device *br_dev)
 {
-        struct mlxsw_sp_vfid *old_vfid = mlxsw_sp_vport->vport.vfid;
-        struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
         u16 vid = mlxsw_sp_vport_vid_get(mlxsw_sp_vport);
         struct net_device *dev = mlxsw_sp_vport->dev;
-        struct mlxsw_sp_vfid *vfid;
         int err;
 
-        vfid = mlxsw_sp_br_vfid_find(mlxsw_sp, br_dev);
-        if (!vfid) {
-                vfid = mlxsw_sp_br_vfid_create(mlxsw_sp, br_dev);
-                if (IS_ERR(vfid)) {
-                        netdev_err(dev, "Failed to create bridge vFID\n");
-                        return PTR_ERR(vfid);
-                }
-        }
+        mlxsw_sp_vport_vfid_leave(mlxsw_sp_vport);
 
-        err = mlxsw_sp_vport_flood_set(mlxsw_sp_vport, vfid->vfid, true, false);
+        err = mlxsw_sp_vport_br_vfid_join(mlxsw_sp_vport, br_dev);
         if (err) {
-                netdev_err(dev, "Failed to setup flooding for vFID=%d\n",
-                           vfid->vfid);
-                goto err_port_flood_set;
+                netdev_err(dev, "Failed to join vFID\n");
+                goto err_vport_br_vfid_join;
         }
 
         err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_vport, vid, true);
@@ -3374,38 +3234,6 @@ static int mlxsw_sp_vport_bridge_join(struct mlxsw_sp_port *mlxsw_sp_vport,
                 goto err_port_vid_learning_set;
         }
 
-        /* We need to invalidate existing {Port, VID} to vFID mapping and
-         * create a new one for the bridge's vFID.
-         */
-        err = mlxsw_sp_port_vid_to_fid_set(mlxsw_sp_vport,
-                                           MLXSW_REG_SVFA_MT_PORT_VID_TO_FID,
-                                           false,
-                                           mlxsw_sp_vfid_to_fid(old_vfid->vfid),
-                                           vid);
-        if (err) {
-                netdev_err(dev, "Failed to invalidate {Port, VID} to vFID=%d mapping\n",
-                           old_vfid->vfid);
-                goto err_port_vid_to_fid_invalidate;
-        }
-
-        err = mlxsw_sp_port_vid_to_fid_set(mlxsw_sp_vport,
-                                           MLXSW_REG_SVFA_MT_PORT_VID_TO_FID,
-                                           true,
-                                           mlxsw_sp_vfid_to_fid(vfid->vfid),
-                                           vid);
-        if (err) {
-                netdev_err(dev, "Failed to map {Port, VID} to vFID=%d\n",
-                           vfid->vfid);
-                goto err_port_vid_to_fid_validate;
-        }
-
-        /* Switch between the vFIDs and destroy the old one if needed. */
-        vfid->nr_vports++;
-        mlxsw_sp_vport->vport.vfid = vfid;
-        old_vfid->nr_vports--;
-        if (!old_vfid->nr_vports)
-                mlxsw_sp_vfid_destroy(mlxsw_sp, old_vfid);
-
         mlxsw_sp_vport->learning = 1;
         mlxsw_sp_vport->learning_sync = 1;
         mlxsw_sp_vport->uc_flood = 1;
@@ -3413,20 +3241,32 @@ static int mlxsw_sp_vport_bridge_join(struct mlxsw_sp_port *mlxsw_sp_vport,
 
         return 0;
 
-err_port_vid_to_fid_validate:
-        mlxsw_sp_port_vid_to_fid_set(mlxsw_sp_vport,
-                                     MLXSW_REG_SVFA_MT_PORT_VID_TO_FID, false,
-                                     mlxsw_sp_vfid_to_fid(old_vfid->vfid), vid);
-err_port_vid_to_fid_invalidate:
-        mlxsw_sp_port_vid_learning_set(mlxsw_sp_vport, vid, false);
 err_port_vid_learning_set:
-        mlxsw_sp_vport_flood_set(mlxsw_sp_vport, vfid->vfid, false, false);
-err_port_flood_set:
-        if (!vfid->nr_vports)
-                mlxsw_sp_br_vfid_destroy(mlxsw_sp, vfid);
+        mlxsw_sp_vport_br_vfid_leave(mlxsw_sp_vport);
+err_vport_br_vfid_join:
+        mlxsw_sp_vport_vfid_join(mlxsw_sp_vport);
         return err;
 }
 
+static void mlxsw_sp_vport_bridge_leave(struct mlxsw_sp_port *mlxsw_sp_vport)
+{
+        u16 vid = mlxsw_sp_vport_vid_get(mlxsw_sp_vport);
+
+        mlxsw_sp_port_vid_learning_set(mlxsw_sp_vport, vid, false);
+
+        mlxsw_sp_vport_br_vfid_leave(mlxsw_sp_vport);
+
+        mlxsw_sp_vport_vfid_join(mlxsw_sp_vport);
+
+        mlxsw_sp_port_stp_state_set(mlxsw_sp_vport, vid,
+                                    MLXSW_REG_SPMS_STATE_FORWARDING);
+
+        mlxsw_sp_vport->learning = 0;
+        mlxsw_sp_vport->learning_sync = 0;
+        mlxsw_sp_vport->uc_flood = 0;
+        mlxsw_sp_vport->bridged = 0;
+}
+
 static bool
 mlxsw_sp_port_master_bridge_check(const struct mlxsw_sp_port *mlxsw_sp_port,
                                   const struct net_device *br_dev)
@@ -3435,7 +3275,9 @@ mlxsw_sp_port_master_bridge_check(const struct mlxsw_sp_port *mlxsw_sp_port,
 
         list_for_each_entry(mlxsw_sp_vport, &mlxsw_sp_port->vports_list,
                             vport.list) {
-                if (mlxsw_sp_vport_br_get(mlxsw_sp_vport) == br_dev)
+                struct net_device *dev = mlxsw_sp_vport_br_get(mlxsw_sp_vport);
+
+                if (dev && dev == br_dev)
                         return false;
         }
 
@@ -3450,56 +3292,39 @@ static int mlxsw_sp_netdevice_vport_event(struct net_device *dev,
         struct netdev_notifier_changeupper_info *info = ptr;
         struct mlxsw_sp_port *mlxsw_sp_vport;
         struct net_device *upper_dev;
-        int err;
+        int err = 0;
 
         mlxsw_sp_vport = mlxsw_sp_port_vport_find(mlxsw_sp_port, vid);
 
         switch (event) {
         case NETDEV_PRECHANGEUPPER:
                 upper_dev = info->upper_dev;
-                if (!info->master || !info->linking)
-                        break;
                 if (!netif_is_bridge_master(upper_dev))
-                        return NOTIFY_BAD;
+                        return -EINVAL;
+                if (!info->linking)
+                        break;
                 /* We can't have multiple VLAN interfaces configured on
                  * the same port and being members in the same bridge.
                  */
                 if (!mlxsw_sp_port_master_bridge_check(mlxsw_sp_port,
                                                        upper_dev))
-                        return NOTIFY_BAD;
+                        return -EINVAL;
                 break;
         case NETDEV_CHANGEUPPER:
                 upper_dev = info->upper_dev;
-                if (!info->master)
-                        break;
                 if (info->linking) {
-                        if (!mlxsw_sp_vport) {
-                                WARN_ON(!mlxsw_sp_vport);
-                                return NOTIFY_BAD;
-                        }
+                        if (WARN_ON(!mlxsw_sp_vport))
+                                return -EINVAL;
                         err = mlxsw_sp_vport_bridge_join(mlxsw_sp_vport,
                                                          upper_dev);
-                        if (err) {
-                                netdev_err(dev, "Failed to join bridge\n");
-                                return NOTIFY_BAD;
-                        }
                 } else {
-                        /* We ignore bridge's unlinking notifications if vPort
-                         * is gone, since we already left the bridge when the
-                         * VLAN device was unlinked from the real device.
-                         */
                         if (!mlxsw_sp_vport)
-                                return NOTIFY_DONE;
-                        err = mlxsw_sp_vport_bridge_leave(mlxsw_sp_vport,
-                                                          upper_dev, true);
-                        if (err) {
-                                netdev_err(dev, "Failed to leave bridge\n");
-                                return NOTIFY_BAD;
-                        }
+                                return 0;
+                        mlxsw_sp_vport_bridge_leave(mlxsw_sp_vport);
                 }
         }
 
-        return NOTIFY_DONE;
+        return err;
 }
 
 static int mlxsw_sp_netdevice_lag_vport_event(struct net_device *lag_dev,
@@ -3514,12 +3339,12 @@ static int mlxsw_sp_netdevice_lag_vport_event(struct net_device *lag_dev,
                 if (mlxsw_sp_port_dev_check(dev)) {
                         ret = mlxsw_sp_netdevice_vport_event(dev, event, ptr,
                                                              vid);
-                        if (ret == NOTIFY_BAD)
+                        if (ret)
                                 return ret;
                 }
         }
 
-        return NOTIFY_DONE;
+        return 0;
 }
 
 static int mlxsw_sp_netdevice_vlan_event(struct net_device *vlan_dev,
@@ -3535,24 +3360,23 @@ static int mlxsw_sp_netdevice_vlan_event(struct net_device *vlan_dev,
                 return mlxsw_sp_netdevice_lag_vport_event(real_dev, event, ptr,
                                                           vid);
 
-        return NOTIFY_DONE;
+        return 0;
 }
 
 static int mlxsw_sp_netdevice_event(struct notifier_block *unused,
                                     unsigned long event, void *ptr)
 {
         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
+        int err = 0;
 
         if (mlxsw_sp_port_dev_check(dev))
-                return mlxsw_sp_netdevice_port_event(dev, event, ptr);
-
-        if (netif_is_lag_master(dev))
-                return mlxsw_sp_netdevice_lag_event(dev, event, ptr);
-
-        if (is_vlan_dev(dev))
-                return mlxsw_sp_netdevice_vlan_event(dev, event, ptr);
+                err = mlxsw_sp_netdevice_port_event(dev, event, ptr);
+        else if (netif_is_lag_master(dev))
+                err = mlxsw_sp_netdevice_lag_event(dev, event, ptr);
+        else if (is_vlan_dev(dev))
+                err = mlxsw_sp_netdevice_vlan_event(dev, event, ptr);
 
-        return NOTIFY_DONE;
+        return notifier_from_errno(err);
 }
 
 static struct notifier_block mlxsw_sp_netdevice_nb __read_mostly = {
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h
index 13b30eaa13d4..36c9835ea20b 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h
@@ -87,11 +87,12 @@ struct mlxsw_sp_upper {
         unsigned int ref_count;
 };
 
-struct mlxsw_sp_vfid {
+struct mlxsw_sp_fid {
+        void (*leave)(struct mlxsw_sp_port *mlxsw_sp_vport);
         struct list_head list;
-        u16 nr_vports;
-        u16 vfid;       /* Starting at 0 */
-        struct net_device *br_dev;
+        unsigned int ref_count;
+        struct net_device *dev;
+        u16 fid;
         u16 vid;
 };
 
@@ -155,17 +156,17 @@ struct mlxsw_sp_sb {
 struct mlxsw_sp {
         struct {
                 struct list_head list;
-                unsigned long mapped[BITS_TO_LONGS(MLXSW_SP_VFID_PORT_MAX)];
+                DECLARE_BITMAP(mapped, MLXSW_SP_VFID_PORT_MAX);
         } port_vfids;
         struct {
                 struct list_head list;
-                unsigned long mapped[BITS_TO_LONGS(MLXSW_SP_VFID_BR_MAX)];
+                DECLARE_BITMAP(mapped, MLXSW_SP_VFID_BR_MAX);
         } br_vfids;
         struct {
                 struct list_head list;
-                unsigned long mapped[BITS_TO_LONGS(MLXSW_SP_MID_MAX)];
+                DECLARE_BITMAP(mapped, MLXSW_SP_MID_MAX);
         } br_mids;
-        unsigned long active_fids[BITS_TO_LONGS(VLAN_N_VID)];
+        struct list_head fids;  /* VLAN-aware bridge FIDs */
         struct mlxsw_sp_port **ports;
         struct mlxsw_core *core;
         const struct mlxsw_bus_info *bus_info;
@@ -217,7 +218,7 @@ struct mlxsw_sp_port {
         u16 lag_id;
         struct {
                 struct list_head list;
-                struct mlxsw_sp_vfid *vfid;
+                struct mlxsw_sp_fid *f;
                 u16 vid;
         } vport;
         struct {
@@ -259,28 +260,38 @@ mlxsw_sp_port_lagged_get(struct mlxsw_sp *mlxsw_sp, u16 lag_id, u8 port_index)
         return mlxsw_sp_port && mlxsw_sp_port->lagged ? mlxsw_sp_port : NULL;
 }
 
+static inline u16
+mlxsw_sp_vport_vid_get(const struct mlxsw_sp_port *mlxsw_sp_vport)
+{
+        return mlxsw_sp_vport->vport.vid;
+}
+
 static inline bool
 mlxsw_sp_port_is_vport(const struct mlxsw_sp_port *mlxsw_sp_port)
 {
-        return mlxsw_sp_port->vport.vfid;
+        u16 vid = mlxsw_sp_vport_vid_get(mlxsw_sp_port);
+
+        return vid != 0;
 }
 
-static inline struct net_device *
-mlxsw_sp_vport_br_get(const struct mlxsw_sp_port *mlxsw_sp_vport)
+static inline void mlxsw_sp_vport_fid_set(struct mlxsw_sp_port *mlxsw_sp_vport,
+                                          struct mlxsw_sp_fid *f)
 {
-        return mlxsw_sp_vport->vport.vfid->br_dev;
+        mlxsw_sp_vport->vport.f = f;
 }
 
-static inline u16
-mlxsw_sp_vport_vid_get(const struct mlxsw_sp_port *mlxsw_sp_vport)
+static inline struct mlxsw_sp_fid *
+mlxsw_sp_vport_fid_get(const struct mlxsw_sp_port *mlxsw_sp_vport)
 {
-        return mlxsw_sp_vport->vport.vid;
+        return mlxsw_sp_vport->vport.f;
 }
 
-static inline u16
-mlxsw_sp_vport_vfid_get(const struct mlxsw_sp_port *mlxsw_sp_vport)
+static inline struct net_device *
+mlxsw_sp_vport_br_get(const struct mlxsw_sp_port *mlxsw_sp_vport)
 {
-        return mlxsw_sp_vport->vport.vfid->vfid;
+        struct mlxsw_sp_fid *f = mlxsw_sp_vport_fid_get(mlxsw_sp_vport);
+
+        return f ? f->dev : NULL;
 }
 
 static inline struct mlxsw_sp_port *
@@ -298,14 +309,16 @@ mlxsw_sp_port_vport_find(const struct mlxsw_sp_port *mlxsw_sp_port, u16 vid)
 }
 
 static inline struct mlxsw_sp_port *
-mlxsw_sp_port_vport_find_by_vfid(const struct mlxsw_sp_port *mlxsw_sp_port,
-                                 u16 vfid)
+mlxsw_sp_port_vport_find_by_fid(const struct mlxsw_sp_port *mlxsw_sp_port,
+                                u16 fid)
 {
         struct mlxsw_sp_port *mlxsw_sp_vport;
 
         list_for_each_entry(mlxsw_sp_vport, &mlxsw_sp_port->vports_list,
                             vport.list) {
-                if (mlxsw_sp_vport_vfid_get(mlxsw_sp_vport) == vfid)
+                struct mlxsw_sp_fid *f = mlxsw_sp_vport_fid_get(mlxsw_sp_vport);
+
+                if (f && f->fid == fid)
                         return mlxsw_sp_vport;
         }
 
@@ -366,10 +379,11 @@ int mlxsw_sp_port_add_vid(struct net_device *dev, __be16 __always_unused proto,
                           u16 vid);
 int mlxsw_sp_port_kill_vid(struct net_device *dev,
                            __be16 __always_unused proto, u16 vid);
-int mlxsw_sp_vport_flood_set(struct mlxsw_sp_port *mlxsw_sp_vport, u16 vfid,
-                             bool set, bool only_uc);
+int mlxsw_sp_vport_flood_set(struct mlxsw_sp_port *mlxsw_sp_vport, u16 fid,
+                             bool set);
 void mlxsw_sp_port_active_vlans_del(struct mlxsw_sp_port *mlxsw_sp_port);
 int mlxsw_sp_port_pvid_set(struct mlxsw_sp_port *mlxsw_sp_port, u16 vid);
+int mlxsw_sp_port_fdb_flush(struct mlxsw_sp_port *mlxsw_sp_port, u16 fid);
 int mlxsw_sp_port_ets_set(struct mlxsw_sp_port *mlxsw_sp_port,
                           enum mlxsw_reg_qeec_hr hr, u8 index, u8 next_index,
                           bool dwrr, u8 dwrr_weight);
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c
index 3710f19ed6bb..a0c7376ee517 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c
@@ -55,13 +55,10 @@
 static u16 mlxsw_sp_port_vid_to_fid_get(struct mlxsw_sp_port *mlxsw_sp_port,
                                         u16 vid)
 {
+        struct mlxsw_sp_fid *f = mlxsw_sp_vport_fid_get(mlxsw_sp_port);
         u16 fid = vid;
 
-        if (mlxsw_sp_port_is_vport(mlxsw_sp_port)) {
-                u16 vfid = mlxsw_sp_vport_vfid_get(mlxsw_sp_port);
-
-                fid = mlxsw_sp_vfid_to_fid(vfid);
-        }
+        fid = f ? f->fid : fid;
 
         if (!fid)
                 fid = mlxsw_sp_port->pvid;
@@ -236,7 +233,8 @@ static int mlxsw_sp_port_uc_flood_set(struct mlxsw_sp_port *mlxsw_sp_port,
         int err;
 
         if (mlxsw_sp_port_is_vport(mlxsw_sp_port)) {
-                u16 vfid = mlxsw_sp_vport_vfid_get(mlxsw_sp_port);
+                u16 fid = mlxsw_sp_vport_fid_get(mlxsw_sp_port)->fid;
+                u16 vfid = mlxsw_sp_fid_to_vfid(fid);
 
                 return  __mlxsw_sp_port_flood_set(mlxsw_sp_port, vfid, vfid,
                                                   set, true);
@@ -260,14 +258,17 @@ err_port_flood_set:
         return err;
 }
 
-int mlxsw_sp_vport_flood_set(struct mlxsw_sp_port *mlxsw_sp_vport, u16 vfid,
-                             bool set, bool only_uc)
+int mlxsw_sp_vport_flood_set(struct mlxsw_sp_port *mlxsw_sp_vport, u16 fid,
+                             bool set)
 {
+        u16 vfid;
+
         /* In case of vFIDs, index into the flooding table is relative to
          * the start of the vFIDs range.
          */
+        vfid = mlxsw_sp_fid_to_vfid(fid);
         return __mlxsw_sp_port_flood_set(mlxsw_sp_vport, vfid, vfid, set,
-                                         only_uc);
+                                         false);
 }
 
 static int mlxsw_sp_port_attr_br_flags_set(struct mlxsw_sp_port *mlxsw_sp_port,
@@ -383,6 +384,198 @@ static int mlxsw_sp_port_attr_set(struct net_device *dev,
         return err;
 }
 
+static struct mlxsw_sp_fid *mlxsw_sp_fid_find(struct mlxsw_sp *mlxsw_sp,
+                                              u16 fid)
+{
+        struct mlxsw_sp_fid *f;
+
+        list_for_each_entry(f, &mlxsw_sp->fids, list)
+                if (f->fid == fid)
+                        return f;
+
+        return NULL;
+}
+
+static int mlxsw_sp_fid_op(struct mlxsw_sp *mlxsw_sp, u16 fid, bool create)
+{
+        char sfmr_pl[MLXSW_REG_SFMR_LEN];
+
+        mlxsw_reg_sfmr_pack(sfmr_pl, !create, fid, fid);
+        return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sfmr), sfmr_pl);
+}
+
+static int mlxsw_sp_fid_map(struct mlxsw_sp *mlxsw_sp, u16 fid, bool valid)
+{
+        enum mlxsw_reg_svfa_mt mt = MLXSW_REG_SVFA_MT_VID_TO_FID;
+        char svfa_pl[MLXSW_REG_SVFA_LEN];
+
+        mlxsw_reg_svfa_pack(svfa_pl, 0, mt, valid, fid, fid);
+        return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(svfa), svfa_pl);
+}
+
+static struct mlxsw_sp_fid *mlxsw_sp_fid_alloc(u16 fid)
+{
+        struct mlxsw_sp_fid *f;
+
+        f = kzalloc(sizeof(*f), GFP_KERNEL);
+        if (!f)
+                return NULL;
+
+        f->fid = fid;
+
+        return f;
+}
+
+static struct mlxsw_sp_fid *mlxsw_sp_fid_create(struct mlxsw_sp *mlxsw_sp,
+                                                u16 fid)
+{
+        struct mlxsw_sp_fid *f;
+        int err;
+
+        err = mlxsw_sp_fid_op(mlxsw_sp, fid, true);
+        if (err)
+                return ERR_PTR(err);
+
+        /* Although all the ports member in the FID might be using a
+         * {Port, VID} to FID mapping, we create a global VID-to-FID
+         * mapping. This allows a port to transition to VLAN mode,
+         * knowing the global mapping exists.
+         */
+        err = mlxsw_sp_fid_map(mlxsw_sp, fid, true);
+        if (err)
+                goto err_fid_map;
+
+        f = mlxsw_sp_fid_alloc(fid);
+        if (!f) {
+                err = -ENOMEM;
+                goto err_allocate_fid;
+        }
+
+        list_add(&f->list, &mlxsw_sp->fids);
+
+        return f;
+
+err_allocate_fid:
+        mlxsw_sp_fid_map(mlxsw_sp, fid, false);
+err_fid_map:
+        mlxsw_sp_fid_op(mlxsw_sp, fid, false);
+        return ERR_PTR(err);
+}
+
+static void mlxsw_sp_fid_destroy(struct mlxsw_sp *mlxsw_sp,
+                                 struct mlxsw_sp_fid *f)
+{
+        u16 fid = f->fid;
+
+        list_del(&f->list);
+
+        kfree(f);
+
+        mlxsw_sp_fid_op(mlxsw_sp, fid, false);
+}
+
+static int __mlxsw_sp_port_fid_join(struct mlxsw_sp_port *mlxsw_sp_port,
+                                    u16 fid)
+{
+        struct mlxsw_sp_fid *f;
+
+        f = mlxsw_sp_fid_find(mlxsw_sp_port->mlxsw_sp, fid);
+        if (!f) {
+                f = mlxsw_sp_fid_create(mlxsw_sp_port->mlxsw_sp, fid);
+                if (IS_ERR(f))
+                        return PTR_ERR(f);
+        }
+
+        f->ref_count++;
+
+        netdev_dbg(mlxsw_sp_port->dev, "Joined FID=%d\n", fid);
+
+        return 0;
+}
+
+static void __mlxsw_sp_port_fid_leave(struct mlxsw_sp_port *mlxsw_sp_port,
+                                      u16 fid)
+{
+        struct mlxsw_sp_fid *f;
+
+        f = mlxsw_sp_fid_find(mlxsw_sp_port->mlxsw_sp, fid);
+        if (WARN_ON(!f))
+                return;
+
+        netdev_dbg(mlxsw_sp_port->dev, "Left FID=%d\n", fid);
+
+        mlxsw_sp_port_fdb_flush(mlxsw_sp_port, fid);
+
+        if (--f->ref_count == 0)
+                mlxsw_sp_fid_destroy(mlxsw_sp_port->mlxsw_sp, f);
+}
+
+static int mlxsw_sp_port_fid_map(struct mlxsw_sp_port *mlxsw_sp_port, u16 fid,
+                                 bool valid)
+{
+        enum mlxsw_reg_svfa_mt mt = MLXSW_REG_SVFA_MT_PORT_VID_TO_FID;
+
+        /* If port doesn't have vPorts, then it can use the global
+         * VID-to-FID mapping.
+         */
+        if (list_empty(&mlxsw_sp_port->vports_list))
+                return 0;
+
+        return mlxsw_sp_port_vid_to_fid_set(mlxsw_sp_port, mt, valid, fid, fid);
+}
+
+static int mlxsw_sp_port_fid_join(struct mlxsw_sp_port *mlxsw_sp_port,
+                                  u16 fid_begin, u16 fid_end)
+{
+        int fid, err;
+
+        for (fid = fid_begin; fid <= fid_end; fid++) {
+                err = __mlxsw_sp_port_fid_join(mlxsw_sp_port, fid);
+                if (err)
+                        goto err_port_fid_join;
+        }
+
+        err = __mlxsw_sp_port_flood_set(mlxsw_sp_port, fid_begin, fid_end,
+                                        true, false);
+        if (err)
+                goto err_port_flood_set;
+
+        for (fid = fid_begin; fid <= fid_end; fid++) {
+                err = mlxsw_sp_port_fid_map(mlxsw_sp_port, fid, true);
+                if (err)
+                        goto err_port_fid_map;
+        }
+
+        return 0;
+
+err_port_fid_map:
+        for (fid--; fid >= fid_begin; fid--)
+                mlxsw_sp_port_fid_map(mlxsw_sp_port, fid, false);
+        __mlxsw_sp_port_flood_set(mlxsw_sp_port, fid_begin, fid_end, false,
+                                  false);
+err_port_flood_set:
+        fid = fid_end;
+err_port_fid_join:
+        for (fid--; fid >= fid_begin; fid--)
+                __mlxsw_sp_port_fid_leave(mlxsw_sp_port, fid);
+        return err;
+}
+
+static void mlxsw_sp_port_fid_leave(struct mlxsw_sp_port *mlxsw_sp_port,
+                                    u16 fid_begin, u16 fid_end)
+{
+        int fid;
+
+        for (fid = fid_begin; fid <= fid_end; fid++)
+                mlxsw_sp_port_fid_map(mlxsw_sp_port, fid, false);
+
+        __mlxsw_sp_port_flood_set(mlxsw_sp_port, fid_begin, fid_end, false,
+                                  false);
+
+        for (fid = fid_begin; fid <= fid_end; fid++)
+                __mlxsw_sp_port_fid_leave(mlxsw_sp_port, fid);
+}
+
 static int __mlxsw_sp_port_pvid_set(struct mlxsw_sp_port *mlxsw_sp_port,
                                     u16 vid)
 {
@@ -440,55 +633,6 @@ err_port_allow_untagged_set:
         return err;
 }
 
-static int mlxsw_sp_fid_create(struct mlxsw_sp *mlxsw_sp, u16 fid)
-{
-        char sfmr_pl[MLXSW_REG_SFMR_LEN];
-        int err;
-
-        mlxsw_reg_sfmr_pack(sfmr_pl, MLXSW_REG_SFMR_OP_CREATE_FID, fid, fid);
-        err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sfmr), sfmr_pl);
-
-        if (err)
-                return err;
-
-        set_bit(fid, mlxsw_sp->active_fids);
-        return 0;
-}
-
-static void mlxsw_sp_fid_destroy(struct mlxsw_sp *mlxsw_sp, u16 fid)
-{
-        char sfmr_pl[MLXSW_REG_SFMR_LEN];
-
-        clear_bit(fid, mlxsw_sp->active_fids);
-
-        mlxsw_reg_sfmr_pack(sfmr_pl, MLXSW_REG_SFMR_OP_DESTROY_FID,
-                            fid, fid);
-        mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sfmr), sfmr_pl);
-}
-
-static int mlxsw_sp_port_fid_map(struct mlxsw_sp_port *mlxsw_sp_port, u16 fid)
-{
-        enum mlxsw_reg_svfa_mt mt;
-
-        if (!list_empty(&mlxsw_sp_port->vports_list))
-                mt = MLXSW_REG_SVFA_MT_PORT_VID_TO_FID;
-        else
-                mt = MLXSW_REG_SVFA_MT_VID_TO_FID;
-
-        return mlxsw_sp_port_vid_to_fid_set(mlxsw_sp_port, mt, true, fid, fid);
-}
-
-static int mlxsw_sp_port_fid_unmap(struct mlxsw_sp_port *mlxsw_sp_port, u16 fid)
-{
-        enum mlxsw_reg_svfa_mt mt;
-
-        if (list_empty(&mlxsw_sp_port->vports_list))
-                return 0;
-
-        mt = MLXSW_REG_SVFA_MT_PORT_VID_TO_FID;
-        return mlxsw_sp_port_vid_to_fid_set(mlxsw_sp_port, mt, false, fid, fid);
-}
-
 static int mlxsw_sp_port_add_vids(struct net_device *dev, u16 vid_begin,
                                   u16 vid_end)
 {
@@ -533,10 +677,8 @@ static int __mlxsw_sp_port_vlans_add(struct mlxsw_sp_port *mlxsw_sp_port,
                                      u16 vid_begin, u16 vid_end,
                                      bool flag_untagged, bool flag_pvid)
 {
-        struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
         struct net_device *dev = mlxsw_sp_port->dev;
-        u16 vid, last_visited_vid, old_pvid;
-        enum mlxsw_reg_svfa_mt mt;
+        u16 vid, old_pvid;
         int err;
 
         /* In case this is invoked with BRIDGE_FLAGS_SELF and port is
@@ -546,44 +688,10 @@ static int __mlxsw_sp_port_vlans_add(struct mlxsw_sp_port *mlxsw_sp_port,
         if (!mlxsw_sp_port->bridged)
                 return mlxsw_sp_port_add_vids(dev, vid_begin, vid_end);
 
-        for (vid = vid_begin; vid <= vid_end; vid++) {
-                if (!test_bit(vid, mlxsw_sp->active_fids)) {
-                        err = mlxsw_sp_fid_create(mlxsw_sp, vid);
-                        if (err) {
-                                netdev_err(dev, "Failed to create FID=%d\n",
-                                           vid);
-                                return err;
-                        }
-
-                        /* When creating a FID, we set a VID to FID mapping
-                         * regardless of the port's mode.
-                         */
-                        mt = MLXSW_REG_SVFA_MT_VID_TO_FID;
-                        err = mlxsw_sp_port_vid_to_fid_set(mlxsw_sp_port, mt,
-                                                           true, vid, vid);
-                        if (err) {
-                                netdev_err(dev, "Failed to create FID=VID=%d mapping\n",
-                                           vid);
-                                goto err_port_vid_to_fid_set;
-                        }
-                }
-        }
-
-        /* Set FID mapping according to port's mode */
-        for (vid = vid_begin; vid <= vid_end; vid++) {
-                err = mlxsw_sp_port_fid_map(mlxsw_sp_port, vid);
-                if (err) {
-                        netdev_err(dev, "Failed to map FID=%d", vid);
-                        last_visited_vid = --vid;
-                        goto err_port_fid_map;
-                }
-        }
-
-        err = __mlxsw_sp_port_flood_set(mlxsw_sp_port, vid_begin, vid_end,
-                                        true, false);
+        err = mlxsw_sp_port_fid_join(mlxsw_sp_port, vid_begin, vid_end);
         if (err) {
-                netdev_err(dev, "Failed to configure flooding\n");
-                goto err_port_flood_set;
+                netdev_err(dev, "Failed to join FIDs\n");
+                return err;
         }
 
         err = __mlxsw_sp_port_vlans_set(mlxsw_sp_port, vid_begin, vid_end,
@@ -628,10 +736,6 @@ static int __mlxsw_sp_port_vlans_add(struct mlxsw_sp_port *mlxsw_sp_port,
 
         return 0;
 
-err_port_vid_to_fid_set:
-        mlxsw_sp_fid_destroy(mlxsw_sp, vid);
-        return err;
-
 err_port_stp_state_set:
         for (vid = vid_begin; vid <= vid_end; vid++)
                 clear_bit(vid, mlxsw_sp_port->active_vlans);
@@ -641,13 +745,7 @@ err_port_pvid_set:
         __mlxsw_sp_port_vlans_set(mlxsw_sp_port, vid_begin, vid_end, false,
                                   false);
 err_port_vlans_set:
-        __mlxsw_sp_port_flood_set(mlxsw_sp_port, vid_begin, vid_end, false,
-                                  false);
-err_port_flood_set:
-        last_visited_vid = vid_end;
-err_port_fid_map:
-        for (vid = last_visited_vid; vid >= vid_begin; vid--)
-                mlxsw_sp_port_fid_unmap(mlxsw_sp_port, vid);
+        mlxsw_sp_port_fid_leave(mlxsw_sp_port, vid_begin, vid_end);
         return err;
 }
 
@@ -970,21 +1068,7 @@ static int __mlxsw_sp_port_vlans_del(struct mlxsw_sp_port *mlxsw_sp_port,
                 }
         }
 
-        err = __mlxsw_sp_port_flood_set(mlxsw_sp_port, vid_begin, vid_end,
-                                        false, false);
-        if (err) {
-                netdev_err(dev, "Failed to clear flooding\n");
-                return err;
-        }
-
-        for (vid = vid_begin; vid <= vid_end; vid++) {
-                /* Remove FID mapping in case of Virtual mode */
-                err = mlxsw_sp_port_fid_unmap(mlxsw_sp_port, vid);
-                if (err) {
-                        netdev_err(dev, "Failed to unmap FID=%d", vid);
-                        return err;
-                }
-        }
+        mlxsw_sp_port_fid_leave(mlxsw_sp_port, vid_begin, vid_end);
 
 out:
         /* Changing activity bits only if HW operation succeded */
@@ -1118,7 +1202,8 @@ static int mlxsw_sp_port_fdb_dump(struct mlxsw_sp_port *mlxsw_sp_port,
 {
         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
         struct mlxsw_sp_port *tmp;
-        u16 vport_fid = 0;
+        struct mlxsw_sp_fid *f;
+        u16 vport_fid;
         char *sfd_pl;
         char mac[ETH_ALEN];
         u16 fid;
@@ -1133,12 +1218,8 @@ static int mlxsw_sp_port_fdb_dump(struct mlxsw_sp_port *mlxsw_sp_port,
         if (!sfd_pl)
                 return -ENOMEM;
 
-        if (mlxsw_sp_port_is_vport(mlxsw_sp_port)) {
-                u16 tmp;
-
-                tmp = mlxsw_sp_vport_vfid_get(mlxsw_sp_port);
-                vport_fid = mlxsw_sp_vfid_to_fid(tmp);
-        }
+        f = mlxsw_sp_vport_fid_get(mlxsw_sp_port);
+        vport_fid = f ? f->fid : 0;
 
         mlxsw_reg_sfd_pack(sfd_pl, MLXSW_REG_SFD_OP_QUERY_DUMP, 0);
         do {
@@ -1310,11 +1391,10 @@ static void mlxsw_sp_fdb_notify_mac_process(struct mlxsw_sp *mlxsw_sp,
         }
 
         if (mlxsw_sp_fid_is_vfid(fid)) {
-                u16 vfid = mlxsw_sp_fid_to_vfid(fid);
                 struct mlxsw_sp_port *mlxsw_sp_vport;
 
-                mlxsw_sp_vport = mlxsw_sp_port_vport_find_by_vfid(mlxsw_sp_port,
-                                                                  vfid);
+                mlxsw_sp_vport = mlxsw_sp_port_vport_find_by_fid(mlxsw_sp_port,
+                                                                 fid);
                 if (!mlxsw_sp_vport) {
                         netdev_err(mlxsw_sp_port->dev, "Failed to find a matching vPort following FDB notification\n");
                         goto just_remove;
@@ -1370,11 +1450,10 @@ static void mlxsw_sp_fdb_notify_mac_lag_process(struct mlxsw_sp *mlxsw_sp,
         }
 
         if (mlxsw_sp_fid_is_vfid(fid)) {
-                u16 vfid = mlxsw_sp_fid_to_vfid(fid);
                 struct mlxsw_sp_port *mlxsw_sp_vport;
 
-                mlxsw_sp_vport = mlxsw_sp_port_vport_find_by_vfid(mlxsw_sp_port,
-                                                                  vfid);
+                mlxsw_sp_vport = mlxsw_sp_port_vport_find_by_fid(mlxsw_sp_port,
+                                                                 fid);
                 if (!mlxsw_sp_vport) {
                         netdev_err(mlxsw_sp_port->dev, "Failed to find a matching vPort following FDB notification\n");
                         goto just_remove;
@@ -1495,14 +1574,6 @@ static void mlxsw_sp_fdb_fini(struct mlxsw_sp *mlxsw_sp)
         cancel_delayed_work_sync(&mlxsw_sp->fdb_notify.dw);
 }
 
-static void mlxsw_sp_fids_fini(struct mlxsw_sp *mlxsw_sp)
-{
-        u16 fid;
-
-        for_each_set_bit(fid, mlxsw_sp->active_fids, VLAN_N_VID)
-                mlxsw_sp_fid_destroy(mlxsw_sp, fid);
-}
-
 int mlxsw_sp_switchdev_init(struct mlxsw_sp *mlxsw_sp)
 {
         return mlxsw_sp_fdb_init(mlxsw_sp);
@@ -1511,7 +1582,6 @@ int mlxsw_sp_switchdev_init(struct mlxsw_sp *mlxsw_sp)
 void mlxsw_sp_switchdev_fini(struct mlxsw_sp *mlxsw_sp)
 {
         mlxsw_sp_fdb_fini(mlxsw_sp);
-        mlxsw_sp_fids_fini(mlxsw_sp);
 }
 
 int mlxsw_sp_port_vlan_init(struct mlxsw_sp_port *mlxsw_sp_port)
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
index ba26bb356b8d..c25a8ba6cf9f 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
@@ -1979,7 +1979,7 @@ static int __nfp_net_set_config_and_enable(struct nfp_net *nn)
         if (nn->ctrl & NFP_NET_CFG_CTRL_VXLAN) {
                 memset(&nn->vxlan_ports, 0, sizeof(nn->vxlan_ports));
                 memset(&nn->vxlan_usecnt, 0, sizeof(nn->vxlan_usecnt));
-                vxlan_get_rx_port(nn->netdev);
+                udp_tunnel_get_rx_info(nn->netdev);
         }
 
         return err;
@@ -2551,26 +2551,32 @@ static int nfp_net_find_vxlan_idx(struct nfp_net *nn, __be16 port)
 }
 
 static void nfp_net_add_vxlan_port(struct net_device *netdev,
-                                   sa_family_t sa_family, __be16 port)
+                                   struct udp_tunnel_info *ti)
 {
         struct nfp_net *nn = netdev_priv(netdev);
         int idx;
 
-        idx = nfp_net_find_vxlan_idx(nn, port);
+        if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
+                return;
+
+        idx = nfp_net_find_vxlan_idx(nn, ti->port);
         if (idx == -ENOSPC)
                 return;
 
         if (!nn->vxlan_usecnt[idx]++)
-                nfp_net_set_vxlan_port(nn, idx, port);
+                nfp_net_set_vxlan_port(nn, idx, ti->port);
 }
 
 static void nfp_net_del_vxlan_port(struct net_device *netdev,
-                                   sa_family_t sa_family, __be16 port)
+                                   struct udp_tunnel_info *ti)
 {
         struct nfp_net *nn = netdev_priv(netdev);
         int idx;
 
-        idx = nfp_net_find_vxlan_idx(nn, port);
+        if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
+                return;
+
+        idx = nfp_net_find_vxlan_idx(nn, ti->port);
         if (!nn->vxlan_usecnt[idx] || idx == -ENOSPC)
                 return;
 
@@ -2589,8 +2595,8 @@ static const struct net_device_ops nfp_net_netdev_ops = {
         .ndo_set_mac_address    = eth_mac_addr,
         .ndo_set_features       = nfp_net_set_features,
         .ndo_features_check     = nfp_net_features_check,
-        .ndo_add_vxlan_port     = nfp_net_add_vxlan_port,
-        .ndo_del_vxlan_port     = nfp_net_del_vxlan_port,
+        .ndo_udp_tunnel_add     = nfp_net_add_vxlan_port,
+        .ndo_udp_tunnel_del     = nfp_net_del_vxlan_port,
 };
 
 /**
diff --git a/drivers/net/ethernet/nxp/lpc_eth.c b/drivers/net/ethernet/nxp/lpc_eth.c
index b1ce7aaa8f8b..01b50ff7c708 100644
--- a/drivers/net/ethernet/nxp/lpc_eth.c
+++ b/drivers/net/ethernet/nxp/lpc_eth.c
@@ -425,7 +425,6 @@ struct netdata_local {
         unsigned int            last_tx_idx;
         unsigned int            num_used_tx_buffs;
         struct mii_bus          *mii_bus;
-        struct phy_device       *phy_dev;
         struct clk              *clk;
         dma_addr_t              dma_buff_base_p;
         void                    *dma_buff_base_v;
@@ -750,7 +749,7 @@ static int lpc_mdio_reset(struct mii_bus *bus)
 static void lpc_handle_link_change(struct net_device *ndev)
 {
         struct netdata_local *pldat = netdev_priv(ndev);
-        struct phy_device *phydev = pldat->phy_dev;
+        struct phy_device *phydev = ndev->phydev;
         unsigned long flags;
 
         bool status_change = false;
@@ -814,7 +813,6 @@ static int lpc_mii_probe(struct net_device *ndev)
         pldat->link = 0;
         pldat->speed = 0;
         pldat->duplex = -1;
-        pldat->phy_dev = phydev;
 
         phy_attached_info(phydev);
 
@@ -1048,8 +1046,8 @@ static int lpc_eth_close(struct net_device *ndev)
         napi_disable(&pldat->napi);
         netif_stop_queue(ndev);
 
-        if (pldat->phy_dev)
-                phy_stop(pldat->phy_dev);
+        if (ndev->phydev)
+                phy_stop(ndev->phydev);
 
         spin_lock_irqsave(&pldat->lock, flags);
         __lpc_eth_reset(pldat);
@@ -1186,7 +1184,7 @@ static void lpc_eth_set_multicast_list(struct net_device *ndev)
 static int lpc_eth_ioctl(struct net_device *ndev, struct ifreq *req, int cmd)
 {
         struct netdata_local *pldat = netdev_priv(ndev);
-        struct phy_device *phydev = pldat->phy_dev;
+        struct phy_device *phydev = ndev->phydev;
 
         if (!netif_running(ndev))
                 return -EINVAL;
@@ -1207,14 +1205,14 @@ static int lpc_eth_open(struct net_device *ndev)
         __lpc_eth_clock_enable(pldat, true);
 
         /* Suspended PHY makes LPC ethernet core block, so resume now */
-        phy_resume(pldat->phy_dev);
+        phy_resume(ndev->phydev);
 
         /* Reset and initialize */
         __lpc_eth_reset(pldat);
         __lpc_eth_init(pldat);
 
         /* schedule a link state check */
-        phy_start(pldat->phy_dev);
+        phy_start(ndev->phydev);
         netif_start_queue(ndev);
         napi_enable(&pldat->napi);
 
@@ -1247,37 +1245,13 @@ static void lpc_eth_ethtool_setmsglevel(struct net_device *ndev, u32 level)
         pldat->msg_enable = level;
 }
 
-static int lpc_eth_ethtool_getsettings(struct net_device *ndev,
-        struct ethtool_cmd *cmd)
-{
-        struct netdata_local *pldat = netdev_priv(ndev);
-        struct phy_device *phydev = pldat->phy_dev;
-
-        if (!phydev)
-                return -EOPNOTSUPP;
-
-        return phy_ethtool_gset(phydev, cmd);
-}
-
-static int lpc_eth_ethtool_setsettings(struct net_device *ndev,
-        struct ethtool_cmd *cmd)
-{
-        struct netdata_local *pldat = netdev_priv(ndev);
-        struct phy_device *phydev = pldat->phy_dev;
-
-        if (!phydev)
-                return -EOPNOTSUPP;
-
-        return phy_ethtool_sset(phydev, cmd);
-}
-
 static const struct ethtool_ops lpc_eth_ethtool_ops = {
         .get_drvinfo    = lpc_eth_ethtool_getdrvinfo,
-        .get_settings   = lpc_eth_ethtool_getsettings,
-        .set_settings   = lpc_eth_ethtool_setsettings,
         .get_msglevel   = lpc_eth_ethtool_getmsglevel,
         .set_msglevel   = lpc_eth_ethtool_setmsglevel,
         .get_link       = ethtool_op_get_link,
+        .get_link_ksettings = phy_ethtool_get_link_ksettings,
+        .set_link_ksettings = phy_ethtool_set_link_ksettings,
 };
 
 static const struct net_device_ops lpc_netdev_ops = {
@@ -1460,7 +1434,7 @@ static int lpc_eth_drv_probe(struct platform_device *pdev)
         netdev_info(ndev, "LPC mac at 0x%08x irq %d\n",
                res->start, ndev->irq);
 
-        phydev = pldat->phy_dev;
+        phydev = ndev->phydev;
 
         device_init_wakeup(&pdev->dev, 1);
         device_set_wakeup_enable(&pdev->dev, 0);
diff --git a/drivers/net/ethernet/qlogic/Kconfig b/drivers/net/ethernet/qlogic/Kconfig
index 680d8c736d2b..6ba48406899e 100644
--- a/drivers/net/ethernet/qlogic/Kconfig
+++ b/drivers/net/ethernet/qlogic/Kconfig
@@ -54,16 +54,6 @@ config QLCNIC_DCB
           mode of DCB is supported. PG and PFC values are related only
           to Tx.
 
-config QLCNIC_VXLAN
-        bool "Virtual eXtensible Local Area Network (VXLAN) offload support"
-        default n
-        depends on QLCNIC && VXLAN && !(QLCNIC=y && VXLAN=m)
-        ---help---
-          This enables hardware offload support for VXLAN protocol over QLogic's
-          84XX series adapters.
-          Say Y here if you want to enable hardware offload support for
-          Virtual eXtensible Local Area Network (VXLAN) in the driver.
-
 config QLCNIC_HWMON
         bool "QLOGIC QLCNIC 82XX and 83XX family HWMON support"
         depends on QLCNIC && HWMON && !(QLCNIC=y && HWMON=m)
@@ -114,24 +104,4 @@ config QEDE
         ---help---
           This enables the support for ...
 
-config QEDE_VXLAN
-        bool "Virtual eXtensible Local Area Network support"
-        default n
-        depends on QEDE && VXLAN && !(QEDE=y && VXLAN=m)
-        ---help---
-          This enables hardware offload support for VXLAN protocol over
-          qede module. Say Y here if you want to enable hardware offload
-          support for Virtual eXtensible Local Area Network (VXLAN)
-          in the driver.
-
-config QEDE_GENEVE
-        bool "Generic Network Virtualization Encapsulation (GENEVE) support"
-        depends on QEDE && GENEVE && !(QEDE=y && GENEVE=m)
-        ---help---
-          This allows one to create GENEVE virtual interfaces that provide
-          Layer 2 Networks over Layer 3 Networks. GENEVE is often used
-          to tunnel virtual network infrastructure in virtualized environments.
-          Say Y here if you want to enable hardware offload support for
-          Generic Network Virtualization Encapsulation (GENEVE) in the driver.
-
 endif # NET_VENDOR_QLOGIC
diff --git a/drivers/net/ethernet/qlogic/qed/qed.h b/drivers/net/ethernet/qlogic/qed/qed.h
index 1042f2af854a..35e53771533f 100644
--- a/drivers/net/ethernet/qlogic/qed/qed.h
+++ b/drivers/net/ethernet/qlogic/qed/qed.h
@@ -127,6 +127,8 @@ struct qed_tunn_update_params {
  */
 enum qed_pci_personality {
         QED_PCI_ETH,
+        QED_PCI_ISCSI,
+        QED_PCI_ETH_ROCE,
         QED_PCI_DEFAULT /* default in shmem */
 };
 
@@ -170,6 +172,8 @@ enum QED_PORT_MODE {
 
 enum qed_dev_cap {
         QED_DEV_CAP_ETH,
+        QED_DEV_CAP_ISCSI,
+        QED_DEV_CAP_ROCE,
 };
 
 struct qed_hw_info {
@@ -183,6 +187,8 @@ struct qed_hw_info {
 
 #define RESC_START(_p_hwfn, resc) ((_p_hwfn)->hw_info.resc_start[resc])
 #define RESC_NUM(_p_hwfn, resc) ((_p_hwfn)->hw_info.resc_num[resc])
+#define RESC_END(_p_hwfn, resc) (RESC_START(_p_hwfn, resc) + \
+                                 RESC_NUM(_p_hwfn, resc))
 #define FEAT_NUM(_p_hwfn, resc) ((_p_hwfn)->hw_info.feat_num[resc])
 
         u8                              num_tc;
@@ -255,6 +261,7 @@ struct qed_qm_info {
         u8                              pure_lb_pq;
         u8                              offload_pq;
         u8                              pure_ack_pq;
+        u8 ooo_pq;
         u8                              vf_queues_offset;
         u16                             num_pqs;
         u16                             num_vf_pqs;
@@ -267,6 +274,7 @@ struct qed_qm_info {
         u8                              pf_wfq;
         u32                             pf_rl;
         struct qed_wfq_data             *wfq_data;
+        u8 num_pf_rls;
 };
 
 struct storm_stats {
@@ -312,6 +320,7 @@ struct qed_hwfn {
         bool                            hw_init_done;
 
         u8                              num_funcs_on_engine;
+        u8 enabled_func_idx;
 
         /* BAR access */
         void __iomem                    *regview;
@@ -350,6 +359,9 @@ struct qed_hwfn {
         /* Protocol related */
         struct qed_pf_params            pf_params;
 
+        bool b_rdma_enabled_in_prs;
+        u32 rdma_prs_search_reg;
+
         /* Array of sb_info of all status blocks */
         struct qed_sb_info              *sbs_info[MAX_SB_PER_PF_MIMD];
         u16                             num_sbs;
@@ -477,8 +489,8 @@ struct qed_dev {
 
         u32                             int_mode;
         enum qed_coalescing_mode        int_coalescing_mode;
-        u8                              rx_coalesce_usecs;
-        u8                              tx_coalesce_usecs;
+        u16                             rx_coalesce_usecs;
+        u16                             tx_coalesce_usecs;
 
         /* Start Bar offset of first hwfn */
         void __iomem                    *regview;
@@ -555,6 +567,7 @@ static inline u8 qed_concrete_to_sw_fid(struct qed_dev *cdev,
 }
 
 #define PURE_LB_TC 8
+#define OOO_LB_TC 9
 
 int qed_configure_vport_wfq(struct qed_dev *cdev, u16 vp_id, u32 rate);
 void qed_configure_vp_wfq_on_link_change(struct qed_dev *cdev, u32 min_pf_rate);
diff --git a/drivers/net/ethernet/qlogic/qed/qed_cxt.c b/drivers/net/ethernet/qlogic/qed/qed_cxt.c
index ac284c58d8c2..1c35f376143e 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_cxt.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_cxt.c
@@ -39,6 +39,14 @@
 #define DQ_RANGE_SHIFT          4
 #define DQ_RANGE_ALIGN          BIT(DQ_RANGE_SHIFT)
 
+/* Searcher constants */
+#define SRC_MIN_NUM_ELEMS 256
+
+/* Timers constants */
+#define TM_SHIFT        7
+#define TM_ALIGN        BIT(TM_SHIFT)
+#define TM_ELEM_SIZE    4
+
 /* ILT constants */
 #define ILT_DEFAULT_HW_P_SIZE           3
 #define ILT_PAGE_IN_BYTES(hw_p_size)    (1U << ((hw_p_size) + 12))
@@ -56,26 +64,71 @@
 union conn_context {
         struct core_conn_context core_ctx;
         struct eth_conn_context eth_ctx;
+        struct iscsi_conn_context iscsi_ctx;
+        struct roce_conn_context roce_ctx;
+};
+
+/* TYPE-0 task context - iSCSI */
+union type0_task_context {
+        struct iscsi_task_context iscsi_ctx;
 };
 
+/* TYPE-1 task context - ROCE */
+union type1_task_context {
+        struct rdma_task_context roce_ctx;
+};
+
+struct src_ent {
+        u8 opaque[56];
+        u64 next;
+};
+
+#define CDUT_SEG_ALIGNMET 3     /* in 4k chunks */
+#define CDUT_SEG_ALIGNMET_IN_BYTES (1 << (CDUT_SEG_ALIGNMET + 12))
+
 #define CONN_CXT_SIZE(p_hwfn) \
         ALIGNED_TYPE_SIZE(union conn_context, p_hwfn)
 
+#define SRQ_CXT_SIZE (sizeof(struct rdma_srq_context))
+
+#define TYPE0_TASK_CXT_SIZE(p_hwfn) \
+        ALIGNED_TYPE_SIZE(union type0_task_context, p_hwfn)
+
+/* Alignment is inherent to the type1_task_context structure */
+#define TYPE1_TASK_CXT_SIZE(p_hwfn) sizeof(union type1_task_context)
+
 /* PF per protocl configuration object */
+#define TASK_SEGMENTS   (NUM_TASK_PF_SEGMENTS + NUM_TASK_VF_SEGMENTS)
+#define TASK_SEGMENT_VF (NUM_TASK_PF_SEGMENTS)
+
+struct qed_tid_seg {
+        u32 count;
+        u8 type;
+        bool has_fl_mem;
+};
+
 struct qed_conn_type_cfg {
         u32 cid_count;
         u32 cid_start;
         u32 cids_per_vf;
+        struct qed_tid_seg tid_seg[TASK_SEGMENTS];
 };
 
 /* ILT Client configuration, Per connection type (protocol) resources. */
 #define ILT_CLI_PF_BLOCKS       (1 + NUM_TASK_PF_SEGMENTS * 2)
 #define ILT_CLI_VF_BLOCKS       (1 + NUM_TASK_VF_SEGMENTS * 2)
 #define CDUC_BLK                (0)
+#define SRQ_BLK                 (0)
+#define CDUT_SEG_BLK(n)         (1 + (u8)(n))
+#define CDUT_FL_SEG_BLK(n, X)   (1 + (n) + NUM_TASK_ ## X ## _SEGMENTS)
 
 enum ilt_clients {
         ILT_CLI_CDUC,
+        ILT_CLI_CDUT,
         ILT_CLI_QM,
+        ILT_CLI_TM,
+        ILT_CLI_SRC,
+        ILT_CLI_TSDM,
         ILT_CLI_MAX
 };
 
@@ -88,6 +141,7 @@ struct qed_ilt_cli_blk {
         u32 total_size; /* 0 means not active */
         u32 real_size_in_page;
         u32 start_line;
+        u32 dynamic_line_cnt;
 };
 
 struct qed_ilt_client_cfg {
@@ -131,18 +185,44 @@ struct qed_cxt_mngr {
         /* computed ILT structure */
         struct qed_ilt_client_cfg       clients[ILT_CLI_MAX];
 
+        /* Task type sizes */
+        u32 task_type_size[NUM_TASK_TYPES];
+
         /* total number of VFs for this hwfn -
          * ALL VFs are symmetric in terms of HW resources
          */
         u32                             vf_count;
 
+        /* total number of SRQ's for this hwfn */
+        u32 srq_count;
+
         /* Acquired CIDs */
         struct qed_cid_acquired_map     acquired[MAX_CONN_TYPES];
 
         /* ILT  shadow table */
         struct qed_dma_mem              *ilt_shadow;
         u32                             pf_start_line;
+
+        /* Mutex for a dynamic ILT allocation */
+        struct mutex mutex;
+
+        /* SRC T2 */
+        struct qed_dma_mem *t2;
+        u32 t2_num_pages;
+        u64 first_free;
+        u64 last_free;
 };
+static bool src_proto(enum protocol_type type)
+{
+        return type == PROTOCOLID_ISCSI ||
+               type == PROTOCOLID_ROCE;
+}
+
+static bool tm_cid_proto(enum protocol_type type)
+{
+        return type == PROTOCOLID_ISCSI ||
+               type == PROTOCOLID_ROCE;
+}
 
 /* counts the iids for the CDU/CDUC ILT client configuration */
 struct qed_cdu_iids {
@@ -161,21 +241,120 @@ static void qed_cxt_cdu_iids(struct qed_cxt_mngr *p_mngr,
         }
 }
 
+/* counts the iids for the Searcher block configuration */
+struct qed_src_iids {
+        u32 pf_cids;
+        u32 per_vf_cids;
+};
+
+static void qed_cxt_src_iids(struct qed_cxt_mngr *p_mngr,
+                             struct qed_src_iids *iids)
+{
+        u32 i;
+
+        for (i = 0; i < MAX_CONN_TYPES; i++) {
+                if (!src_proto(i))
+                        continue;
+
+                iids->pf_cids += p_mngr->conn_cfg[i].cid_count;
+                iids->per_vf_cids += p_mngr->conn_cfg[i].cids_per_vf;
+        }
+}
+
+/* counts the iids for the Timers block configuration */
+struct qed_tm_iids {
+        u32 pf_cids;
+        u32 pf_tids[NUM_TASK_PF_SEGMENTS];      /* per segment */
+        u32 pf_tids_total;
+        u32 per_vf_cids;
+        u32 per_vf_tids;
+};
+
+static void qed_cxt_tm_iids(struct qed_cxt_mngr *p_mngr,
+                            struct qed_tm_iids *iids)
+{
+        u32 i, j;
+
+        for (i = 0; i < MAX_CONN_TYPES; i++) {
+                struct qed_conn_type_cfg *p_cfg = &p_mngr->conn_cfg[i];
+
+                if (tm_cid_proto(i)) {
+                        iids->pf_cids += p_cfg->cid_count;
+                        iids->per_vf_cids += p_cfg->cids_per_vf;
+                }
+        }
+
+        iids->pf_cids = roundup(iids->pf_cids, TM_ALIGN);
+        iids->per_vf_cids = roundup(iids->per_vf_cids, TM_ALIGN);
+        iids->per_vf_tids = roundup(iids->per_vf_tids, TM_ALIGN);
+
+        for (iids->pf_tids_total = 0, j = 0; j < NUM_TASK_PF_SEGMENTS; j++) {
+                iids->pf_tids[j] = roundup(iids->pf_tids[j], TM_ALIGN);
+                iids->pf_tids_total += iids->pf_tids[j];
+        }
+}
+
 static void qed_cxt_qm_iids(struct qed_hwfn *p_hwfn,
                             struct qed_qm_iids *iids)
 {
         struct qed_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr;
-        u32 vf_cids = 0, type;
+        struct qed_tid_seg *segs;
+        u32 vf_cids = 0, type, j;
+        u32 vf_tids = 0;
 
         for (type = 0; type < MAX_CONN_TYPES; type++) {
                 iids->cids += p_mngr->conn_cfg[type].cid_count;
                 vf_cids += p_mngr->conn_cfg[type].cids_per_vf;
+
+                segs = p_mngr->conn_cfg[type].tid_seg;
+                /* for each segment there is at most one
+                 * protocol for which count is not 0.
+                 */
+                for (j = 0; j < NUM_TASK_PF_SEGMENTS; j++)
+                        iids->tids += segs[j].count;
+
+                /* The last array elelment is for the VFs. As for PF
+                 * segments there can be only one protocol for
+                 * which this value is not 0.
+                 */
+                vf_tids += segs[NUM_TASK_PF_SEGMENTS].count;
         }
 
         iids->vf_cids += vf_cids * p_mngr->vf_count;
+        iids->tids += vf_tids * p_mngr->vf_count;
+
         DP_VERBOSE(p_hwfn, QED_MSG_ILT,
-                   "iids: CIDS %08x vf_cids %08x\n",
-                   iids->cids, iids->vf_cids);
+                   "iids: CIDS %08x vf_cids %08x tids %08x vf_tids %08x\n",
+                   iids->cids, iids->vf_cids, iids->tids, vf_tids);
+}
+
+static struct qed_tid_seg *qed_cxt_tid_seg_info(struct qed_hwfn *p_hwfn,
+                                                u32 seg)
+{
+        struct qed_cxt_mngr *p_cfg = p_hwfn->p_cxt_mngr;
+        u32 i;
+
+        /* Find the protocol with tid count > 0 for this segment.
+         * Note: there can only be one and this is already validated.
+         */
+        for (i = 0; i < MAX_CONN_TYPES; i++)
+                if (p_cfg->conn_cfg[i].tid_seg[seg].count)
+                        return &p_cfg->conn_cfg[i].tid_seg[seg];
+        return NULL;
+}
+
+void qed_cxt_set_srq_count(struct qed_hwfn *p_hwfn, u32 num_srqs)
+{
+        struct qed_cxt_mngr *p_mgr = p_hwfn->p_cxt_mngr;
+
+        p_mgr->srq_count = num_srqs;
+}
+
+u32 qed_cxt_get_srq_count(struct qed_hwfn *p_hwfn)
+{
+        struct qed_cxt_mngr *p_mgr = p_hwfn->p_cxt_mngr;
+
+        return p_mgr->srq_count;
 }
 
 /* set the iids count per protocol */
@@ -188,6 +367,14 @@ static void qed_cxt_set_proto_cid_count(struct qed_hwfn *p_hwfn,
 
         p_conn->cid_count = roundup(cid_count, DQ_RANGE_ALIGN);
         p_conn->cids_per_vf = roundup(vf_cid_cnt, DQ_RANGE_ALIGN);
+
+        if (type == PROTOCOLID_ROCE) {
+                u32 page_sz = p_mgr->clients[ILT_CLI_CDUC].p_size.val;
+                u32 cxt_size = CONN_CXT_SIZE(p_hwfn);
+                u32 elems_per_page = ILT_PAGE_IN_BYTES(page_sz) / cxt_size;
+
+                p_conn->cid_count = roundup(p_conn->cid_count, elems_per_page);
+        }
 }
 
 u32 qed_cxt_get_proto_cid_count(struct qed_hwfn         *p_hwfn,
@@ -200,6 +387,37 @@ u32 qed_cxt_get_proto_cid_count(struct qed_hwfn		*p_hwfn,
         return p_hwfn->p_cxt_mngr->conn_cfg[type].cid_count;
 }
 
+u32 qed_cxt_get_proto_cid_start(struct qed_hwfn *p_hwfn,
+                                enum protocol_type type)
+{
+        return p_hwfn->p_cxt_mngr->acquired[type].start_cid;
+}
+
+u32 qed_cxt_get_proto_tid_count(struct qed_hwfn *p_hwfn,
+                                enum protocol_type type)
+{
+        u32 cnt = 0;
+        int i;
+
+        for (i = 0; i < TASK_SEGMENTS; i++)
+                cnt += p_hwfn->p_cxt_mngr->conn_cfg[type].tid_seg[i].count;
+
+        return cnt;
+}
+
+static void
+qed_cxt_set_proto_tid_count(struct qed_hwfn *p_hwfn,
+                            enum protocol_type proto,
+                            u8 seg, u8 seg_type, u32 count, bool has_fl)
+{
+        struct qed_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr;
+        struct qed_tid_seg *p_seg = &p_mngr->conn_cfg[proto].tid_seg[seg];
+
+        p_seg->count = count;
+        p_seg->has_fl_mem = has_fl;
+        p_seg->type = seg_type;
+}
+
 static void qed_ilt_cli_blk_fill(struct qed_ilt_client_cfg *p_cli,
                                  struct qed_ilt_cli_blk *p_blk,
                                  u32 start_line, u32 total_size,
@@ -241,17 +459,42 @@ static void qed_ilt_cli_adv_line(struct qed_hwfn *p_hwfn,
                    p_blk->real_size_in_page, p_blk->start_line);
 }
 
+static u32 qed_ilt_get_dynamic_line_cnt(struct qed_hwfn *p_hwfn,
+                                        enum ilt_clients ilt_client)
+{
+        u32 cid_count = p_hwfn->p_cxt_mngr->conn_cfg[PROTOCOLID_ROCE].cid_count;
+        struct qed_ilt_client_cfg *p_cli;
+        u32 lines_to_skip = 0;
+        u32 cxts_per_p;
+
+        if (ilt_client == ILT_CLI_CDUC) {
+                p_cli = &p_hwfn->p_cxt_mngr->clients[ILT_CLI_CDUC];
+
+                cxts_per_p = ILT_PAGE_IN_BYTES(p_cli->p_size.val) /
+                    (u32) CONN_CXT_SIZE(p_hwfn);
+
+                lines_to_skip = cid_count / cxts_per_p;
+        }
+
+        return lines_to_skip;
+}
+
 int qed_cxt_cfg_ilt_compute(struct qed_hwfn *p_hwfn)
 {
         struct qed_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr;
+        u32 curr_line, total, i, task_size, line;
         struct qed_ilt_client_cfg *p_cli;
         struct qed_ilt_cli_blk *p_blk;
         struct qed_cdu_iids cdu_iids;
+        struct qed_src_iids src_iids;
         struct qed_qm_iids qm_iids;
-        u32 curr_line, total, i;
+        struct qed_tm_iids tm_iids;
+        struct qed_tid_seg *p_seg;
 
         memset(&qm_iids, 0, sizeof(qm_iids));
         memset(&cdu_iids, 0, sizeof(cdu_iids));
+        memset(&src_iids, 0, sizeof(src_iids));
+        memset(&tm_iids, 0, sizeof(tm_iids));
 
         p_mngr->pf_start_line = RESC_START(p_hwfn, QED_ILT);
 
@@ -279,6 +522,9 @@ int qed_cxt_cfg_ilt_compute(struct qed_hwfn *p_hwfn)
         qed_ilt_cli_adv_line(p_hwfn, p_cli, p_blk, &curr_line, ILT_CLI_CDUC);
         p_cli->pf_total_lines = curr_line - p_blk->start_line;
 
+        p_blk->dynamic_line_cnt = qed_ilt_get_dynamic_line_cnt(p_hwfn,
+                                                               ILT_CLI_CDUC);
+
         /* CDUC VF */
         p_blk = &p_cli->vf_blks[CDUC_BLK];
         total = cdu_iids.per_vf_cids * CONN_CXT_SIZE(p_hwfn);
@@ -293,21 +539,128 @@ int qed_cxt_cfg_ilt_compute(struct qed_hwfn *p_hwfn)
                 qed_ilt_cli_adv_line(p_hwfn, p_cli, p_blk, &curr_line,
                                      ILT_CLI_CDUC);
 
+        /* CDUT PF */
+        p_cli = &p_mngr->clients[ILT_CLI_CDUT];
+        p_cli->first.val = curr_line;
+
+        /* first the 'working' task memory */
+        for (i = 0; i < NUM_TASK_PF_SEGMENTS; i++) {
+                p_seg = qed_cxt_tid_seg_info(p_hwfn, i);
+                if (!p_seg || p_seg->count == 0)
+                        continue;
+
+                p_blk = &p_cli->pf_blks[CDUT_SEG_BLK(i)];
+                total = p_seg->count * p_mngr->task_type_size[p_seg->type];
+                qed_ilt_cli_blk_fill(p_cli, p_blk, curr_line, total,
+                                     p_mngr->task_type_size[p_seg->type]);
+
+                qed_ilt_cli_adv_line(p_hwfn, p_cli, p_blk, &curr_line,
+                                     ILT_CLI_CDUT);
+        }
+
+        /* next the 'init' task memory (forced load memory) */
+        for (i = 0; i < NUM_TASK_PF_SEGMENTS; i++) {
+                p_seg = qed_cxt_tid_seg_info(p_hwfn, i);
+                if (!p_seg || p_seg->count == 0)
+                        continue;
+
+                p_blk = &p_cli->pf_blks[CDUT_FL_SEG_BLK(i, PF)];
+
+                if (!p_seg->has_fl_mem) {
+                        /* The segment is active (total size pf 'working'
+                         * memory is > 0) but has no FL (forced-load, Init)
+                         * memory. Thus:
+                         *
+                         * 1.   The total-size in the corrsponding FL block of
+                         *      the ILT client is set to 0 - No ILT line are
+                         *      provisioned and no ILT memory allocated.
+                         *
+                         * 2.   The start-line of said block is set to the
+                         *      start line of the matching working memory
+                         *      block in the ILT client. This is later used to
+                         *      configure the CDU segment offset registers and
+                         *      results in an FL command for TIDs of this
+                         *      segement behaves as regular load commands
+                         *      (loading TIDs from the working memory).
+                         */
+                        line = p_cli->pf_blks[CDUT_SEG_BLK(i)].start_line;
+
+                        qed_ilt_cli_blk_fill(p_cli, p_blk, line, 0, 0);
+                        continue;
+                }
+                total = p_seg->count * p_mngr->task_type_size[p_seg->type];
+
+                qed_ilt_cli_blk_fill(p_cli, p_blk,
+                                     curr_line, total,
+                                     p_mngr->task_type_size[p_seg->type]);
+
+                qed_ilt_cli_adv_line(p_hwfn, p_cli, p_blk, &curr_line,
+                                     ILT_CLI_CDUT);
+        }
+        p_cli->pf_total_lines = curr_line - p_cli->pf_blks[0].start_line;
+
+        /* CDUT VF */
+        p_seg = qed_cxt_tid_seg_info(p_hwfn, TASK_SEGMENT_VF);
+        if (p_seg && p_seg->count) {
+                /* Stricly speaking we need to iterate over all VF
+                 * task segment types, but a VF has only 1 segment
+                 */
+
+                /* 'working' memory */
+                total = p_seg->count * p_mngr->task_type_size[p_seg->type];
+
+                p_blk = &p_cli->vf_blks[CDUT_SEG_BLK(0)];
+                qed_ilt_cli_blk_fill(p_cli, p_blk,
+                                     curr_line, total,
+                                     p_mngr->task_type_size[p_seg->type]);
+
+                qed_ilt_cli_adv_line(p_hwfn, p_cli, p_blk, &curr_line,
+                                     ILT_CLI_CDUT);
+
+                /* 'init' memory */
+                p_blk = &p_cli->vf_blks[CDUT_FL_SEG_BLK(0, VF)];
+                if (!p_seg->has_fl_mem) {
+                        /* see comment above */
+                        line = p_cli->vf_blks[CDUT_SEG_BLK(0)].start_line;
+                        qed_ilt_cli_blk_fill(p_cli, p_blk, line, 0, 0);
+                } else {
+                        task_size = p_mngr->task_type_size[p_seg->type];
+                        qed_ilt_cli_blk_fill(p_cli, p_blk,
+                                             curr_line, total, task_size);
+                        qed_ilt_cli_adv_line(p_hwfn, p_cli, p_blk, &curr_line,
+                                             ILT_CLI_CDUT);
+                }
+                p_cli->vf_total_lines = curr_line -
+                    p_cli->vf_blks[0].start_line;
+
+                /* Now for the rest of the VFs */
+                for (i = 1; i < p_mngr->vf_count; i++) {
+                        p_blk = &p_cli->vf_blks[CDUT_SEG_BLK(0)];
+                        qed_ilt_cli_adv_line(p_hwfn, p_cli, p_blk, &curr_line,
+                                             ILT_CLI_CDUT);
+
+                        p_blk = &p_cli->vf_blks[CDUT_FL_SEG_BLK(0, VF)];
+                        qed_ilt_cli_adv_line(p_hwfn, p_cli, p_blk, &curr_line,
+                                             ILT_CLI_CDUT);
+                }
+        }
+
         /* QM */
         p_cli = &p_mngr->clients[ILT_CLI_QM];
         p_blk = &p_cli->pf_blks[0];
 
         qed_cxt_qm_iids(p_hwfn, &qm_iids);
         total = qed_qm_pf_mem_size(p_hwfn->rel_pf_id, qm_iids.cids,
-                                   qm_iids.vf_cids, 0,
+                                   qm_iids.vf_cids, qm_iids.tids,
                                    p_hwfn->qm_info.num_pqs,
                                    p_hwfn->qm_info.num_vf_pqs);
 
         DP_VERBOSE(p_hwfn,
                    QED_MSG_ILT,
-                   "QM ILT Info, (cids=%d, vf_cids=%d, num_pqs=%d, num_vf_pqs=%d, memory_size=%d)\n",
+                   "QM ILT Info, (cids=%d, vf_cids=%d, tids=%d, num_pqs=%d, num_vf_pqs=%d, memory_size=%d)\n",
                    qm_iids.cids,
                    qm_iids.vf_cids,
+                   qm_iids.tids,
                    p_hwfn->qm_info.num_pqs, p_hwfn->qm_info.num_vf_pqs, total);
 
         qed_ilt_cli_blk_fill(p_cli, p_blk,
@@ -317,6 +670,75 @@ int qed_cxt_cfg_ilt_compute(struct qed_hwfn *p_hwfn)
         qed_ilt_cli_adv_line(p_hwfn, p_cli, p_blk, &curr_line, ILT_CLI_QM);
         p_cli->pf_total_lines = curr_line - p_blk->start_line;
 
+        /* SRC */
+        p_cli = &p_mngr->clients[ILT_CLI_SRC];
+        qed_cxt_src_iids(p_mngr, &src_iids);
+
+        /* Both the PF and VFs searcher connections are stored in the per PF
+         * database. Thus sum the PF searcher cids and all the VFs searcher
+         * cids.
+         */
+        total = src_iids.pf_cids + src_iids.per_vf_cids * p_mngr->vf_count;
+        if (total) {
+                u32 local_max = max_t(u32, total,
+                                      SRC_MIN_NUM_ELEMS);
+
+                total = roundup_pow_of_two(local_max);
+
+                p_blk = &p_cli->pf_blks[0];
+                qed_ilt_cli_blk_fill(p_cli, p_blk, curr_line,
+                                     total * sizeof(struct src_ent),
+                                     sizeof(struct src_ent));
+
+                qed_ilt_cli_adv_line(p_hwfn, p_cli, p_blk, &curr_line,
+                                     ILT_CLI_SRC);
+                p_cli->pf_total_lines = curr_line - p_blk->start_line;
+        }
+
+        /* TM PF */
+        p_cli = &p_mngr->clients[ILT_CLI_TM];
+        qed_cxt_tm_iids(p_mngr, &tm_iids);
+        total = tm_iids.pf_cids + tm_iids.pf_tids_total;
+        if (total) {
+                p_blk = &p_cli->pf_blks[0];
+                qed_ilt_cli_blk_fill(p_cli, p_blk, curr_line,
+                                     total * TM_ELEM_SIZE, TM_ELEM_SIZE);
+
+                qed_ilt_cli_adv_line(p_hwfn, p_cli, p_blk, &curr_line,
+                                     ILT_CLI_TM);
+                p_cli->pf_total_lines = curr_line - p_blk->start_line;
+        }
+
+        /* TM VF */
+        total = tm_iids.per_vf_cids + tm_iids.per_vf_tids;
+        if (total) {
+                p_blk = &p_cli->vf_blks[0];
+                qed_ilt_cli_blk_fill(p_cli, p_blk, curr_line,
+                                     total * TM_ELEM_SIZE, TM_ELEM_SIZE);
+
+                qed_ilt_cli_adv_line(p_hwfn, p_cli, p_blk, &curr_line,
+                                     ILT_CLI_TM);
+                p_cli->pf_total_lines = curr_line - p_blk->start_line;
+
+                for (i = 1; i < p_mngr->vf_count; i++)
+                        qed_ilt_cli_adv_line(p_hwfn, p_cli, p_blk, &curr_line,
+                                             ILT_CLI_TM);
+        }
+
+        /* TSDM (SRQ CONTEXT) */
+        total = qed_cxt_get_srq_count(p_hwfn);
+
+        if (total) {
+                p_cli = &p_mngr->clients[ILT_CLI_TSDM];
+                p_blk = &p_cli->pf_blks[SRQ_BLK];
+                qed_ilt_cli_blk_fill(p_cli, p_blk, curr_line,
+                                     total * SRQ_CXT_SIZE, SRQ_CXT_SIZE);
+
+                qed_ilt_cli_adv_line(p_hwfn, p_cli, p_blk, &curr_line,
+                                     ILT_CLI_TSDM);
+                p_cli->pf_total_lines = curr_line - p_blk->start_line;
+        }
+
         if (curr_line - p_hwfn->p_cxt_mngr->pf_start_line >
             RESC_NUM(p_hwfn, QED_ILT)) {
                 DP_ERR(p_hwfn, "too many ilt lines...#lines=%d\n",
@@ -327,8 +749,122 @@ int qed_cxt_cfg_ilt_compute(struct qed_hwfn *p_hwfn)
         return 0;
 }
 
+static void qed_cxt_src_t2_free(struct qed_hwfn *p_hwfn)
+{
+        struct qed_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr;
+        u32 i;
+
+        if (!p_mngr->t2)
+                return;
+
+        for (i = 0; i < p_mngr->t2_num_pages; i++)
+                if (p_mngr->t2[i].p_virt)
+                        dma_free_coherent(&p_hwfn->cdev->pdev->dev,
+                                          p_mngr->t2[i].size,
+                                          p_mngr->t2[i].p_virt,
+                                          p_mngr->t2[i].p_phys);
+
+        kfree(p_mngr->t2);
+        p_mngr->t2 = NULL;
+}
+
+static int qed_cxt_src_t2_alloc(struct qed_hwfn *p_hwfn)
+{
+        struct qed_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr;
+        u32 conn_num, total_size, ent_per_page, psz, i;
+        struct qed_ilt_client_cfg *p_src;
+        struct qed_src_iids src_iids;
+        struct qed_dma_mem *p_t2;
+        int rc;
+
+        memset(&src_iids, 0, sizeof(src_iids));
+
+        /* if the SRC ILT client is inactive - there are no connection
+         * requiring the searcer, leave.
+         */
+        p_src = &p_hwfn->p_cxt_mngr->clients[ILT_CLI_SRC];
+        if (!p_src->active)
+                return 0;
+
+        qed_cxt_src_iids(p_mngr, &src_iids);
+        conn_num = src_iids.pf_cids + src_iids.per_vf_cids * p_mngr->vf_count;
+        total_size = conn_num * sizeof(struct src_ent);
+
+        /* use the same page size as the SRC ILT client */
+        psz = ILT_PAGE_IN_BYTES(p_src->p_size.val);
+        p_mngr->t2_num_pages = DIV_ROUND_UP(total_size, psz);
+
+        /* allocate t2 */
+        p_mngr->t2 = kzalloc(p_mngr->t2_num_pages * sizeof(struct qed_dma_mem),
+                             GFP_KERNEL);
+        if (!p_mngr->t2) {
+                DP_NOTICE(p_hwfn, "Failed to allocate t2 table\n");
+                rc = -ENOMEM;
+                goto t2_fail;
+        }
+
+        /* allocate t2 pages */
+        for (i = 0; i < p_mngr->t2_num_pages; i++) {
+                u32 size = min_t(u32, total_size, psz);
+                void **p_virt = &p_mngr->t2[i].p_virt;
+
+                *p_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
+                                             size,
+                                             &p_mngr->t2[i].p_phys, GFP_KERNEL);
+                if (!p_mngr->t2[i].p_virt) {
+                        rc = -ENOMEM;
+                        goto t2_fail;
+                }
+                memset(*p_virt, 0, size);
+                p_mngr->t2[i].size = size;
+                total_size -= size;
+        }
+
+        /* Set the t2 pointers */
+
+        /* entries per page - must be a power of two */
+        ent_per_page = psz / sizeof(struct src_ent);
+
+        p_mngr->first_free = (u64) p_mngr->t2[0].p_phys;
+
+        p_t2 = &p_mngr->t2[(conn_num - 1) / ent_per_page];
+        p_mngr->last_free = (u64) p_t2->p_phys +
+            ((conn_num - 1) & (ent_per_page - 1)) * sizeof(struct src_ent);
+
+        for (i = 0; i < p_mngr->t2_num_pages; i++) {
+                u32 ent_num = min_t(u32,
+                                    ent_per_page,
+                                    conn_num);
+                struct src_ent *entries = p_mngr->t2[i].p_virt;
+                u64 p_ent_phys = (u64) p_mngr->t2[i].p_phys, val;
+                u32 j;
+
+                for (j = 0; j < ent_num - 1; j++) {
+                        val = p_ent_phys + (j + 1) * sizeof(struct src_ent);
+                        entries[j].next = cpu_to_be64(val);
+                }
+
+                if (i < p_mngr->t2_num_pages - 1)
+                        val = (u64) p_mngr->t2[i + 1].p_phys;
+                else
+                        val = 0;
+                entries[j].next = cpu_to_be64(val);
+
+                conn_num -= ent_num;
+        }
+
+        return 0;
+
+t2_fail:
+        qed_cxt_src_t2_free(p_hwfn);
+        return rc;
+}
+
 #define for_each_ilt_valid_client(pos, clients) \
-                for (pos = 0; pos < ILT_CLI_MAX; pos++)
+        for (pos = 0; pos < ILT_CLI_MAX; pos++) \
+                if (!clients[pos].active) {     \
+                        continue;               \
+                } else                          \
 
 /* Total number of ILT lines used by this PF */
 static u32 qed_cxt_ilt_shadow_size(struct qed_ilt_client_cfg *ilt_clients)
@@ -336,12 +872,8 @@ static u32 qed_cxt_ilt_shadow_size(struct qed_ilt_client_cfg *ilt_clients)
         u32 size = 0;
         u32 i;
 
-        for_each_ilt_valid_client(i, ilt_clients) {
-                if (!ilt_clients[i].active)
-                        continue;
-                size += (ilt_clients[i].last.val -
-                         ilt_clients[i].first.val + 1);
-        }
+        for_each_ilt_valid_client(i, ilt_clients)
+            size += (ilt_clients[i].last.val - ilt_clients[i].first.val + 1);
 
         return size;
 }
@@ -372,15 +904,22 @@ static int qed_ilt_blk_alloc(struct qed_hwfn *p_hwfn,
                              u32 start_line_offset)
 {
         struct qed_dma_mem *ilt_shadow = p_hwfn->p_cxt_mngr->ilt_shadow;
-        u32 lines, line, sz_left;
+        u32 lines, line, sz_left, lines_to_skip = 0;
+
+        /* Special handling for RoCE that supports dynamic allocation */
+        if ((p_hwfn->hw_info.personality == QED_PCI_ETH_ROCE) &&
+            ((ilt_client == ILT_CLI_CDUT) || ilt_client == ILT_CLI_TSDM))
+                return 0;
+
+        lines_to_skip = p_blk->dynamic_line_cnt;
 
         if (!p_blk->total_size)
                 return 0;
 
         sz_left = p_blk->total_size;
-        lines = DIV_ROUND_UP(sz_left, p_blk->real_size_in_page);
+        lines = DIV_ROUND_UP(sz_left, p_blk->real_size_in_page) - lines_to_skip;
         line = p_blk->start_line + start_line_offset -
-               p_hwfn->p_cxt_mngr->pf_start_line;
+            p_hwfn->p_cxt_mngr->pf_start_line + lines_to_skip;
 
         for (; lines; lines--) {
                 dma_addr_t p_phys;
@@ -434,8 +973,6 @@ static int qed_ilt_shadow_alloc(struct qed_hwfn *p_hwfn)
                    (u32)(size * sizeof(struct qed_dma_mem)));
 
         for_each_ilt_valid_client(i, clients) {
-                if (!clients[i].active)
-                        continue;
                 for (j = 0; j < ILT_CLI_PF_BLOCKS; j++) {
                         p_blk = &clients[i].pf_blks[j];
                         rc = qed_ilt_blk_alloc(p_hwfn, p_blk, i, 0);
@@ -514,6 +1051,7 @@ cid_map_fail:
 
 int qed_cxt_mngr_alloc(struct qed_hwfn *p_hwfn)
 {
+        struct qed_ilt_client_cfg *clients;
         struct qed_cxt_mngr *p_mngr;
         u32 i;
 
@@ -524,20 +1062,42 @@ int qed_cxt_mngr_alloc(struct qed_hwfn *p_hwfn)
         }
 
         /* Initialize ILT client registers */
-        p_mngr->clients[ILT_CLI_CDUC].first.reg = ILT_CFG_REG(CDUC, FIRST_ILT);
-        p_mngr->clients[ILT_CLI_CDUC].last.reg = ILT_CFG_REG(CDUC, LAST_ILT);
-        p_mngr->clients[ILT_CLI_CDUC].p_size.reg = ILT_CFG_REG(CDUC, P_SIZE);
-
-        p_mngr->clients[ILT_CLI_QM].first.reg = ILT_CFG_REG(QM, FIRST_ILT);
-        p_mngr->clients[ILT_CLI_QM].last.reg = ILT_CFG_REG(QM, LAST_ILT);
-        p_mngr->clients[ILT_CLI_QM].p_size.reg = ILT_CFG_REG(QM, P_SIZE);
-
+        clients = p_mngr->clients;
+        clients[ILT_CLI_CDUC].first.reg = ILT_CFG_REG(CDUC, FIRST_ILT);
+        clients[ILT_CLI_CDUC].last.reg = ILT_CFG_REG(CDUC, LAST_ILT);
+        clients[ILT_CLI_CDUC].p_size.reg = ILT_CFG_REG(CDUC, P_SIZE);
+
+        clients[ILT_CLI_QM].first.reg = ILT_CFG_REG(QM, FIRST_ILT);
+        clients[ILT_CLI_QM].last.reg = ILT_CFG_REG(QM, LAST_ILT);
+        clients[ILT_CLI_QM].p_size.reg = ILT_CFG_REG(QM, P_SIZE);
+
+        clients[ILT_CLI_TM].first.reg = ILT_CFG_REG(TM, FIRST_ILT);
+        clients[ILT_CLI_TM].last.reg = ILT_CFG_REG(TM, LAST_ILT);
+        clients[ILT_CLI_TM].p_size.reg = ILT_CFG_REG(TM, P_SIZE);
+
+        clients[ILT_CLI_SRC].first.reg = ILT_CFG_REG(SRC, FIRST_ILT);
+        clients[ILT_CLI_SRC].last.reg = ILT_CFG_REG(SRC, LAST_ILT);
+        clients[ILT_CLI_SRC].p_size.reg = ILT_CFG_REG(SRC, P_SIZE);
+
+        clients[ILT_CLI_CDUT].first.reg = ILT_CFG_REG(CDUT, FIRST_ILT);
+        clients[ILT_CLI_CDUT].last.reg = ILT_CFG_REG(CDUT, LAST_ILT);
+        clients[ILT_CLI_CDUT].p_size.reg = ILT_CFG_REG(CDUT, P_SIZE);
+
+        clients[ILT_CLI_TSDM].first.reg = ILT_CFG_REG(TSDM, FIRST_ILT);
+        clients[ILT_CLI_TSDM].last.reg = ILT_CFG_REG(TSDM, LAST_ILT);
+        clients[ILT_CLI_TSDM].p_size.reg = ILT_CFG_REG(TSDM, P_SIZE);
         /* default ILT page size for all clients is 32K */
         for (i = 0; i < ILT_CLI_MAX; i++)
                 p_mngr->clients[i].p_size.val = ILT_DEFAULT_HW_P_SIZE;
 
+        /* Initialize task sizes */
+        p_mngr->task_type_size[0] = TYPE0_TASK_CXT_SIZE(p_hwfn);
+        p_mngr->task_type_size[1] = TYPE1_TASK_CXT_SIZE(p_hwfn);
+
         if (p_hwfn->cdev->p_iov_info)
                 p_mngr->vf_count = p_hwfn->cdev->p_iov_info->total_vfs;
+        /* Initialize the dynamic ILT allocation mutex */
+        mutex_init(&p_mngr->mutex);
 
         /* Set the cxt mangr pointer priori to further allocations */
         p_hwfn->p_cxt_mngr = p_mngr;
@@ -556,6 +1116,13 @@ int qed_cxt_tables_alloc(struct qed_hwfn *p_hwfn)
                 goto tables_alloc_fail;
         }
 
+        /* Allocate the T2  table */
+        rc = qed_cxt_src_t2_alloc(p_hwfn);
+        if (rc) {
+                DP_NOTICE(p_hwfn, "Failed to allocate T2 memory\n");
+                goto tables_alloc_fail;
+        }
+
         /* Allocate and initialize the acquired cids bitmaps */
         rc = qed_cid_map_alloc(p_hwfn);
         if (rc) {
@@ -576,6 +1143,7 @@ void qed_cxt_mngr_free(struct qed_hwfn *p_hwfn)
                 return;
 
         qed_cid_map_free(p_hwfn);
+        qed_cxt_src_t2_free(p_hwfn);
         qed_ilt_shadow_free(p_hwfn);
         kfree(p_hwfn->p_cxt_mngr);
 
@@ -620,6 +1188,48 @@ void qed_cxt_mngr_setup(struct qed_hwfn *p_hwfn)
 #define CDUC_NCIB_MASK \
         (CDU_REG_CID_ADDR_PARAMS_NCIB >> CDUC_NCIB_SHIFT)
 
+#define CDUT_TYPE0_CXT_SIZE_SHIFT \
+        CDU_REG_SEGMENT0_PARAMS_T0_TID_SIZE_SHIFT
+
+#define CDUT_TYPE0_CXT_SIZE_MASK                \
+        (CDU_REG_SEGMENT0_PARAMS_T0_TID_SIZE >> \
+         CDUT_TYPE0_CXT_SIZE_SHIFT)
+
+#define CDUT_TYPE0_BLOCK_WASTE_SHIFT \
+        CDU_REG_SEGMENT0_PARAMS_T0_TID_BLOCK_WASTE_SHIFT
+
+#define CDUT_TYPE0_BLOCK_WASTE_MASK                    \
+        (CDU_REG_SEGMENT0_PARAMS_T0_TID_BLOCK_WASTE >> \
+         CDUT_TYPE0_BLOCK_WASTE_SHIFT)
+
+#define CDUT_TYPE0_NCIB_SHIFT \
+        CDU_REG_SEGMENT0_PARAMS_T0_NUM_TIDS_IN_BLOCK_SHIFT
+
+#define CDUT_TYPE0_NCIB_MASK                             \
+        (CDU_REG_SEGMENT0_PARAMS_T0_NUM_TIDS_IN_BLOCK >> \
+         CDUT_TYPE0_NCIB_SHIFT)
+
+#define CDUT_TYPE1_CXT_SIZE_SHIFT \
+        CDU_REG_SEGMENT1_PARAMS_T1_TID_SIZE_SHIFT
+
+#define CDUT_TYPE1_CXT_SIZE_MASK                \
+        (CDU_REG_SEGMENT1_PARAMS_T1_TID_SIZE >> \
+         CDUT_TYPE1_CXT_SIZE_SHIFT)
+
+#define CDUT_TYPE1_BLOCK_WASTE_SHIFT \
+        CDU_REG_SEGMENT1_PARAMS_T1_TID_BLOCK_WASTE_SHIFT
+
+#define CDUT_TYPE1_BLOCK_WASTE_MASK                    \
+        (CDU_REG_SEGMENT1_PARAMS_T1_TID_BLOCK_WASTE >> \
+         CDUT_TYPE1_BLOCK_WASTE_SHIFT)
+
+#define CDUT_TYPE1_NCIB_SHIFT \
+        CDU_REG_SEGMENT1_PARAMS_T1_NUM_TIDS_IN_BLOCK_SHIFT
+
+#define CDUT_TYPE1_NCIB_MASK                             \
+        (CDU_REG_SEGMENT1_PARAMS_T1_NUM_TIDS_IN_BLOCK >> \
+         CDUT_TYPE1_NCIB_SHIFT)
+
 static void qed_cdu_init_common(struct qed_hwfn *p_hwfn)
 {
         u32 page_sz, elems_per_page, block_waste, cxt_size, cdu_params = 0;
@@ -634,6 +1244,92 @@ static void qed_cdu_init_common(struct qed_hwfn *p_hwfn)
         SET_FIELD(cdu_params, CDUC_BLOCK_WASTE, block_waste);
         SET_FIELD(cdu_params, CDUC_NCIB, elems_per_page);
         STORE_RT_REG(p_hwfn, CDU_REG_CID_ADDR_PARAMS_RT_OFFSET, cdu_params);
+
+        /* CDUT - type-0 tasks configuration */
+        page_sz = p_hwfn->p_cxt_mngr->clients[ILT_CLI_CDUT].p_size.val;
+        cxt_size = p_hwfn->p_cxt_mngr->task_type_size[0];
+        elems_per_page = ILT_PAGE_IN_BYTES(page_sz) / cxt_size;
+        block_waste = ILT_PAGE_IN_BYTES(page_sz) - elems_per_page * cxt_size;
+
+        /* cxt size and block-waste are multipes of 8 */
+        cdu_params = 0;
+        SET_FIELD(cdu_params, CDUT_TYPE0_CXT_SIZE, (cxt_size >> 3));
+        SET_FIELD(cdu_params, CDUT_TYPE0_BLOCK_WASTE, (block_waste >> 3));
+        SET_FIELD(cdu_params, CDUT_TYPE0_NCIB, elems_per_page);
+        STORE_RT_REG(p_hwfn, CDU_REG_SEGMENT0_PARAMS_RT_OFFSET, cdu_params);
+
+        /* CDUT - type-1 tasks configuration */
+        cxt_size = p_hwfn->p_cxt_mngr->task_type_size[1];
+        elems_per_page = ILT_PAGE_IN_BYTES(page_sz) / cxt_size;
+        block_waste = ILT_PAGE_IN_BYTES(page_sz) - elems_per_page * cxt_size;
+
+        /* cxt size and block-waste are multipes of 8 */
+        cdu_params = 0;
+        SET_FIELD(cdu_params, CDUT_TYPE1_CXT_SIZE, (cxt_size >> 3));
+        SET_FIELD(cdu_params, CDUT_TYPE1_BLOCK_WASTE, (block_waste >> 3));
+        SET_FIELD(cdu_params, CDUT_TYPE1_NCIB, elems_per_page);
+        STORE_RT_REG(p_hwfn, CDU_REG_SEGMENT1_PARAMS_RT_OFFSET, cdu_params);
+}
+
+/* CDU PF */
+#define CDU_SEG_REG_TYPE_SHIFT          CDU_SEG_TYPE_OFFSET_REG_TYPE_SHIFT
+#define CDU_SEG_REG_TYPE_MASK           0x1
+#define CDU_SEG_REG_OFFSET_SHIFT        0
+#define CDU_SEG_REG_OFFSET_MASK         CDU_SEG_TYPE_OFFSET_REG_OFFSET_MASK
+
+static void qed_cdu_init_pf(struct qed_hwfn *p_hwfn)
+{
+        struct qed_ilt_client_cfg *p_cli;
+        struct qed_tid_seg *p_seg;
+        u32 cdu_seg_params, offset;
+        int i;
+
+        static const u32 rt_type_offset_arr[] = {
+                CDU_REG_PF_SEG0_TYPE_OFFSET_RT_OFFSET,
+                CDU_REG_PF_SEG1_TYPE_OFFSET_RT_OFFSET,
+                CDU_REG_PF_SEG2_TYPE_OFFSET_RT_OFFSET,
+                CDU_REG_PF_SEG3_TYPE_OFFSET_RT_OFFSET
+        };
+
+        static const u32 rt_type_offset_fl_arr[] = {
+                CDU_REG_PF_FL_SEG0_TYPE_OFFSET_RT_OFFSET,
+                CDU_REG_PF_FL_SEG1_TYPE_OFFSET_RT_OFFSET,
+                CDU_REG_PF_FL_SEG2_TYPE_OFFSET_RT_OFFSET,
+                CDU_REG_PF_FL_SEG3_TYPE_OFFSET_RT_OFFSET
+        };
+
+        p_cli = &p_hwfn->p_cxt_mngr->clients[ILT_CLI_CDUT];
+
+        /* There are initializations only for CDUT during pf Phase */
+        for (i = 0; i < NUM_TASK_PF_SEGMENTS; i++) {
+                /* Segment 0 */
+                p_seg = qed_cxt_tid_seg_info(p_hwfn, i);
+                if (!p_seg)
+                        continue;
+
+                /* Note: start_line is already adjusted for the CDU
+                 * segment register granularity, so we just need to
+                 * divide. Adjustment is implicit as we assume ILT
+                 * Page size is larger than 32K!
+                 */
+                offset = (ILT_PAGE_IN_BYTES(p_cli->p_size.val) *
+                          (p_cli->pf_blks[CDUT_SEG_BLK(i)].start_line -
+                           p_cli->first.val)) / CDUT_SEG_ALIGNMET_IN_BYTES;
+
+                cdu_seg_params = 0;
+                SET_FIELD(cdu_seg_params, CDU_SEG_REG_TYPE, p_seg->type);
+                SET_FIELD(cdu_seg_params, CDU_SEG_REG_OFFSET, offset);
+                STORE_RT_REG(p_hwfn, rt_type_offset_arr[i], cdu_seg_params);
+
+                offset = (ILT_PAGE_IN_BYTES(p_cli->p_size.val) *
+                          (p_cli->pf_blks[CDUT_FL_SEG_BLK(i, PF)].start_line -
+                           p_cli->first.val)) / CDUT_SEG_ALIGNMET_IN_BYTES;
+
+                cdu_seg_params = 0;
+                SET_FIELD(cdu_seg_params, CDU_SEG_REG_TYPE, p_seg->type);
+                SET_FIELD(cdu_seg_params, CDU_SEG_REG_OFFSET, offset);
+                STORE_RT_REG(p_hwfn, rt_type_offset_fl_arr[i], cdu_seg_params);
+        }
 }
 
 void qed_qm_init_pf(struct qed_hwfn *p_hwfn)
@@ -742,14 +1438,11 @@ static void qed_ilt_bounds_init(struct qed_hwfn *p_hwfn)
 
         ilt_clients = p_hwfn->p_cxt_mngr->clients;
         for_each_ilt_valid_client(i, ilt_clients) {
-                if (!ilt_clients[i].active)
-                        continue;
                 STORE_RT_REG(p_hwfn,
                              ilt_clients[i].first.reg,
                              ilt_clients[i].first.val);
                 STORE_RT_REG(p_hwfn,
-                             ilt_clients[i].last.reg,
-                             ilt_clients[i].last.val);
+                             ilt_clients[i].last.reg, ilt_clients[i].last.val);
                 STORE_RT_REG(p_hwfn,
                              ilt_clients[i].p_size.reg,
                              ilt_clients[i].p_size.val);
@@ -786,6 +1479,33 @@ static void qed_ilt_vf_bounds_init(struct qed_hwfn *p_hwfn)
                              PSWRQ2_REG_CDUC_VF_BLOCKS_RT_OFFSET,
                              p_cli->vf_total_lines);
         }
+
+        p_cli = &p_hwfn->p_cxt_mngr->clients[ILT_CLI_CDUT];
+        blk_factor = ilog2(ILT_PAGE_IN_BYTES(p_cli->p_size.val) >> 10);
+        if (p_cli->active) {
+                STORE_RT_REG(p_hwfn,
+                             PSWRQ2_REG_CDUT_BLOCKS_FACTOR_RT_OFFSET,
+                             blk_factor);
+                STORE_RT_REG(p_hwfn,
+                             PSWRQ2_REG_CDUT_NUMBER_OF_PF_BLOCKS_RT_OFFSET,
+                             p_cli->pf_total_lines);
+                STORE_RT_REG(p_hwfn,
+                             PSWRQ2_REG_CDUT_VF_BLOCKS_RT_OFFSET,
+                             p_cli->vf_total_lines);
+        }
+
+        p_cli = &p_hwfn->p_cxt_mngr->clients[ILT_CLI_TM];
+        blk_factor = ilog2(ILT_PAGE_IN_BYTES(p_cli->p_size.val) >> 10);
+        if (p_cli->active) {
+                STORE_RT_REG(p_hwfn,
+                             PSWRQ2_REG_TM_BLOCKS_FACTOR_RT_OFFSET, blk_factor);
+                STORE_RT_REG(p_hwfn,
+                             PSWRQ2_REG_TM_NUMBER_OF_PF_BLOCKS_RT_OFFSET,
+                             p_cli->pf_total_lines);
+                STORE_RT_REG(p_hwfn,
+                             PSWRQ2_REG_TM_VF_BLOCKS_RT_OFFSET,
+                             p_cli->vf_total_lines);
+        }
 }
 
 /* ILT (PSWRQ2) PF */
@@ -804,9 +1524,6 @@ static void qed_ilt_init_pf(struct qed_hwfn *p_hwfn)
         clients = p_hwfn->p_cxt_mngr->clients;
 
         for_each_ilt_valid_client(i, clients) {
-                if (!clients[i].active)
-                        continue;
-
                 /** Client's 1st val and RT array are absolute, ILT shadows'
                  *  lines are relative.
                  */
@@ -837,6 +1554,137 @@ static void qed_ilt_init_pf(struct qed_hwfn *p_hwfn)
         }
 }
 
+/* SRC (Searcher) PF */
+static void qed_src_init_pf(struct qed_hwfn *p_hwfn)
+{
+        struct qed_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr;
+        u32 rounded_conn_num, conn_num, conn_max;
+        struct qed_src_iids src_iids;
+
+        memset(&src_iids, 0, sizeof(src_iids));
+        qed_cxt_src_iids(p_mngr, &src_iids);
+        conn_num = src_iids.pf_cids + src_iids.per_vf_cids * p_mngr->vf_count;
+        if (!conn_num)
+                return;
+
+        conn_max = max_t(u32, conn_num, SRC_MIN_NUM_ELEMS);
+        rounded_conn_num = roundup_pow_of_two(conn_max);
+
+        STORE_RT_REG(p_hwfn, SRC_REG_COUNTFREE_RT_OFFSET, conn_num);
+        STORE_RT_REG(p_hwfn, SRC_REG_NUMBER_HASH_BITS_RT_OFFSET,
+                     ilog2(rounded_conn_num));
+
+        STORE_RT_REG_AGG(p_hwfn, SRC_REG_FIRSTFREE_RT_OFFSET,
+                         p_hwfn->p_cxt_mngr->first_free);
+        STORE_RT_REG_AGG(p_hwfn, SRC_REG_LASTFREE_RT_OFFSET,
+                         p_hwfn->p_cxt_mngr->last_free);
+}
+
+/* Timers PF */
+#define TM_CFG_NUM_IDS_SHIFT            0
+#define TM_CFG_NUM_IDS_MASK             0xFFFFULL
+#define TM_CFG_PRE_SCAN_OFFSET_SHIFT    16
+#define TM_CFG_PRE_SCAN_OFFSET_MASK     0x1FFULL
+#define TM_CFG_PARENT_PF_SHIFT          25
+#define TM_CFG_PARENT_PF_MASK           0x7ULL
+
+#define TM_CFG_CID_PRE_SCAN_ROWS_SHIFT  30
+#define TM_CFG_CID_PRE_SCAN_ROWS_MASK   0x1FFULL
+
+#define TM_CFG_TID_OFFSET_SHIFT         30
+#define TM_CFG_TID_OFFSET_MASK          0x7FFFFULL
+#define TM_CFG_TID_PRE_SCAN_ROWS_SHIFT  49
+#define TM_CFG_TID_PRE_SCAN_ROWS_MASK   0x1FFULL
+
+static void qed_tm_init_pf(struct qed_hwfn *p_hwfn)
+{
+        struct qed_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr;
+        u32 active_seg_mask = 0, tm_offset, rt_reg;
+        struct qed_tm_iids tm_iids;
+        u64 cfg_word;
+        u8 i;
+
+        memset(&tm_iids, 0, sizeof(tm_iids));
+        qed_cxt_tm_iids(p_mngr, &tm_iids);
+
+        /* @@@TBD No pre-scan for now */
+
+        /* Note: We assume consecutive VFs for a PF */
+        for (i = 0; i < p_mngr->vf_count; i++) {
+                cfg_word = 0;
+                SET_FIELD(cfg_word, TM_CFG_NUM_IDS, tm_iids.per_vf_cids);
+                SET_FIELD(cfg_word, TM_CFG_PRE_SCAN_OFFSET, 0);
+                SET_FIELD(cfg_word, TM_CFG_PARENT_PF, p_hwfn->rel_pf_id);
+                SET_FIELD(cfg_word, TM_CFG_CID_PRE_SCAN_ROWS, 0);
+                rt_reg = TM_REG_CONFIG_CONN_MEM_RT_OFFSET +
+                    (sizeof(cfg_word) / sizeof(u32)) *
+                    (p_hwfn->cdev->p_iov_info->first_vf_in_pf + i);
+                STORE_RT_REG_AGG(p_hwfn, rt_reg, cfg_word);
+        }
+
+        cfg_word = 0;
+        SET_FIELD(cfg_word, TM_CFG_NUM_IDS, tm_iids.pf_cids);
+        SET_FIELD(cfg_word, TM_CFG_PRE_SCAN_OFFSET, 0);
+        SET_FIELD(cfg_word, TM_CFG_PARENT_PF, 0);       /* n/a for PF */
+        SET_FIELD(cfg_word, TM_CFG_CID_PRE_SCAN_ROWS, 0);       /* scan all   */
+
+        rt_reg = TM_REG_CONFIG_CONN_MEM_RT_OFFSET +
+            (sizeof(cfg_word) / sizeof(u32)) *
+            (NUM_OF_VFS(p_hwfn->cdev) + p_hwfn->rel_pf_id);
+        STORE_RT_REG_AGG(p_hwfn, rt_reg, cfg_word);
+
+        /* enale scan */
+        STORE_RT_REG(p_hwfn, TM_REG_PF_ENABLE_CONN_RT_OFFSET,
+                     tm_iids.pf_cids ? 0x1 : 0x0);
+
+        /* @@@TBD how to enable the scan for the VFs */
+
+        tm_offset = tm_iids.per_vf_cids;
+
+        /* Note: We assume consecutive VFs for a PF */
+        for (i = 0; i < p_mngr->vf_count; i++) {
+                cfg_word = 0;
+                SET_FIELD(cfg_word, TM_CFG_NUM_IDS, tm_iids.per_vf_tids);
+                SET_FIELD(cfg_word, TM_CFG_PRE_SCAN_OFFSET, 0);
+                SET_FIELD(cfg_word, TM_CFG_PARENT_PF, p_hwfn->rel_pf_id);
+                SET_FIELD(cfg_word, TM_CFG_TID_OFFSET, tm_offset);
+                SET_FIELD(cfg_word, TM_CFG_TID_PRE_SCAN_ROWS, (u64) 0);
+
+                rt_reg = TM_REG_CONFIG_TASK_MEM_RT_OFFSET +
+                    (sizeof(cfg_word) / sizeof(u32)) *
+                    (p_hwfn->cdev->p_iov_info->first_vf_in_pf + i);
+
+                STORE_RT_REG_AGG(p_hwfn, rt_reg, cfg_word);
+        }
+
+        tm_offset = tm_iids.pf_cids;
+        for (i = 0; i < NUM_TASK_PF_SEGMENTS; i++) {
+                cfg_word = 0;
+                SET_FIELD(cfg_word, TM_CFG_NUM_IDS, tm_iids.pf_tids[i]);
+                SET_FIELD(cfg_word, TM_CFG_PRE_SCAN_OFFSET, 0);
+                SET_FIELD(cfg_word, TM_CFG_PARENT_PF, 0);
+                SET_FIELD(cfg_word, TM_CFG_TID_OFFSET, tm_offset);
+                SET_FIELD(cfg_word, TM_CFG_TID_PRE_SCAN_ROWS, (u64) 0);
+
+                rt_reg = TM_REG_CONFIG_TASK_MEM_RT_OFFSET +
+                    (sizeof(cfg_word) / sizeof(u32)) *
+                    (NUM_OF_VFS(p_hwfn->cdev) +
+                     p_hwfn->rel_pf_id * NUM_TASK_PF_SEGMENTS + i);
+
+                STORE_RT_REG_AGG(p_hwfn, rt_reg, cfg_word);
+                active_seg_mask |= (tm_iids.pf_tids[i] ? (1 << i) : 0);
+
+                tm_offset += tm_iids.pf_tids[i];
+        }
+
+        if (p_hwfn->hw_info.personality == QED_PCI_ETH_ROCE)
+                active_seg_mask = 0;
+
+        STORE_RT_REG(p_hwfn, TM_REG_PF_ENABLE_TASK_RT_OFFSET, active_seg_mask);
+
+        /* @@@TBD how to enable the scan for the VFs */
+}
+
 void qed_cxt_hw_init_common(struct qed_hwfn *p_hwfn)
 {
         qed_cdu_init_common(p_hwfn);
@@ -847,7 +1695,10 @@ void qed_cxt_hw_init_pf(struct qed_hwfn *p_hwfn)
         qed_qm_init_pf(p_hwfn);
         qed_cm_init_pf(p_hwfn);
         qed_dq_init_pf(p_hwfn);
+        qed_cdu_init_pf(p_hwfn);
         qed_ilt_init_pf(p_hwfn);
+        qed_src_init_pf(p_hwfn);
+        qed_tm_init_pf(p_hwfn);
 }
 
 int qed_cxt_acquire_cid(struct qed_hwfn *p_hwfn,
@@ -968,17 +1819,439 @@ int qed_cxt_get_cid_info(struct qed_hwfn *p_hwfn,
         return 0;
 }
 
-int qed_cxt_set_pf_params(struct qed_hwfn *p_hwfn)
+void qed_rdma_set_pf_params(struct qed_hwfn *p_hwfn,
+                            struct qed_rdma_pf_params *p_params)
 {
-        struct qed_eth_pf_params *p_params = &p_hwfn->pf_params.eth_pf_params;
+        u32 num_cons, num_tasks, num_qps, num_mrs, num_srqs;
+        enum protocol_type proto;
+
+        num_mrs = min_t(u32, RDMA_MAX_TIDS, p_params->num_mrs);
+        num_tasks = num_mrs;    /* each mr uses a single task id */
+        num_srqs = min_t(u32, 32 * 1024, p_params->num_srqs);
+
+        switch (p_hwfn->hw_info.personality) {
+        case QED_PCI_ETH_ROCE:
+                num_qps = min_t(u32, ROCE_MAX_QPS, p_params->num_qps);
+                num_cons = num_qps * 2; /* each QP requires two connections */
+                proto = PROTOCOLID_ROCE;
+                break;
+        default:
+                return;
+        }
+
+        if (num_cons && num_tasks) {
+                qed_cxt_set_proto_cid_count(p_hwfn, proto, num_cons, 0);
+
+                /* Deliberatly passing ROCE for tasks id. This is because
+                 * iWARP / RoCE share the task id.
+                 */
+                qed_cxt_set_proto_tid_count(p_hwfn, PROTOCOLID_ROCE,
+                                            QED_CXT_ROCE_TID_SEG, 1,
+                                            num_tasks, false);
+                qed_cxt_set_srq_count(p_hwfn, num_srqs);
+        } else {
+                DP_INFO(p_hwfn->cdev,
+                        "RDMA personality used without setting params!\n");
+        }
+}
 
+int qed_cxt_set_pf_params(struct qed_hwfn *p_hwfn)
+{
         /* Set the number of required CORE connections */
         u32 core_cids = 1; /* SPQ */
 
         qed_cxt_set_proto_cid_count(p_hwfn, PROTOCOLID_CORE, core_cids, 0);
 
-        qed_cxt_set_proto_cid_count(p_hwfn, PROTOCOLID_ETH,
-                                    p_params->num_cons, 1);
+        switch (p_hwfn->hw_info.personality) {
+        case QED_PCI_ETH_ROCE:
+        {
+                qed_rdma_set_pf_params(p_hwfn,
+                                       &p_hwfn->
+                                       pf_params.rdma_pf_params);
+                /* no need for break since RoCE coexist with Ethernet */
+        }
+        case QED_PCI_ETH:
+        {
+                struct qed_eth_pf_params *p_params =
+                    &p_hwfn->pf_params.eth_pf_params;
+
+                qed_cxt_set_proto_cid_count(p_hwfn, PROTOCOLID_ETH,
+                                            p_params->num_cons, 1);
+                break;
+        }
+        case QED_PCI_ISCSI:
+        {
+                struct qed_iscsi_pf_params *p_params;
+
+                p_params = &p_hwfn->pf_params.iscsi_pf_params;
+
+                if (p_params->num_cons && p_params->num_tasks) {
+                        qed_cxt_set_proto_cid_count(p_hwfn,
+                                                    PROTOCOLID_ISCSI,
+                                                    p_params->num_cons,
+                                                    0);
+
+                        qed_cxt_set_proto_tid_count(p_hwfn,
+                                                    PROTOCOLID_ISCSI,
+                                                    QED_CXT_ISCSI_TID_SEG,
+                                                    0,
+                                                    p_params->num_tasks,
+                                                    true);
+                } else {
+                        DP_INFO(p_hwfn->cdev,
+                                "Iscsi personality used without setting params!\n");
+                }
+                break;
+        }
+        default:
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
+int qed_cxt_get_tid_mem_info(struct qed_hwfn *p_hwfn,
+                             struct qed_tid_mem *p_info)
+{
+        struct qed_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr;
+        u32 proto, seg, total_lines, i, shadow_line;
+        struct qed_ilt_client_cfg *p_cli;
+        struct qed_ilt_cli_blk *p_fl_seg;
+        struct qed_tid_seg *p_seg_info;
+
+        /* Verify the personality */
+        switch (p_hwfn->hw_info.personality) {
+        case QED_PCI_ISCSI:
+                proto = PROTOCOLID_ISCSI;
+                seg = QED_CXT_ISCSI_TID_SEG;
+                break;
+        default:
+                return -EINVAL;
+        }
+
+        p_cli = &p_mngr->clients[ILT_CLI_CDUT];
+        if (!p_cli->active)
+                return -EINVAL;
+
+        p_seg_info = &p_mngr->conn_cfg[proto].tid_seg[seg];
+        if (!p_seg_info->has_fl_mem)
+                return -EINVAL;
+
+        p_fl_seg = &p_cli->pf_blks[CDUT_FL_SEG_BLK(seg, PF)];
+        total_lines = DIV_ROUND_UP(p_fl_seg->total_size,
+                                   p_fl_seg->real_size_in_page);
+
+        for (i = 0; i < total_lines; i++) {
+                shadow_line = i + p_fl_seg->start_line -
+                    p_hwfn->p_cxt_mngr->pf_start_line;
+                p_info->blocks[i] = p_mngr->ilt_shadow[shadow_line].p_virt;
+        }
+        p_info->waste = ILT_PAGE_IN_BYTES(p_cli->p_size.val) -
+            p_fl_seg->real_size_in_page;
+        p_info->tid_size = p_mngr->task_type_size[p_seg_info->type];
+        p_info->num_tids_per_block = p_fl_seg->real_size_in_page /
+            p_info->tid_size;
+
+        return 0;
+}
+
+/* This function is very RoCE oriented, if another protocol in the future
+ * will want this feature we'll need to modify the function to be more generic
+ */
+int
+qed_cxt_dynamic_ilt_alloc(struct qed_hwfn *p_hwfn,
+                          enum qed_cxt_elem_type elem_type, u32 iid)
+{
+        u32 reg_offset, shadow_line, elem_size, hw_p_size, elems_per_p, line;
+        struct qed_ilt_client_cfg *p_cli;
+        struct qed_ilt_cli_blk *p_blk;
+        struct qed_ptt *p_ptt;
+        dma_addr_t p_phys;
+        u64 ilt_hw_entry;
+        void *p_virt;
+        int rc = 0;
+
+        switch (elem_type) {
+        case QED_ELEM_CXT:
+                p_cli = &p_hwfn->p_cxt_mngr->clients[ILT_CLI_CDUC];
+                elem_size = CONN_CXT_SIZE(p_hwfn);
+                p_blk = &p_cli->pf_blks[CDUC_BLK];
+                break;
+        case QED_ELEM_SRQ:
+                p_cli = &p_hwfn->p_cxt_mngr->clients[ILT_CLI_TSDM];
+                elem_size = SRQ_CXT_SIZE;
+                p_blk = &p_cli->pf_blks[SRQ_BLK];
+                break;
+        case QED_ELEM_TASK:
+                p_cli = &p_hwfn->p_cxt_mngr->clients[ILT_CLI_CDUT];
+                elem_size = TYPE1_TASK_CXT_SIZE(p_hwfn);
+                p_blk = &p_cli->pf_blks[CDUT_SEG_BLK(QED_CXT_ROCE_TID_SEG)];
+                break;
+        default:
+                DP_NOTICE(p_hwfn, "-EINVALID elem type = %d", elem_type);
+                return -EINVAL;
+        }
+
+        /* Calculate line in ilt */
+        hw_p_size = p_cli->p_size.val;
+        elems_per_p = ILT_PAGE_IN_BYTES(hw_p_size) / elem_size;
+        line = p_blk->start_line + (iid / elems_per_p);
+        shadow_line = line - p_hwfn->p_cxt_mngr->pf_start_line;
+
+        /* If line is already allocated, do nothing, otherwise allocate it and
+         * write it to the PSWRQ2 registers.
+         * This section can be run in parallel from different contexts and thus
+         * a mutex protection is needed.
+         */
+
+        mutex_lock(&p_hwfn->p_cxt_mngr->mutex);
+
+        if (p_hwfn->p_cxt_mngr->ilt_shadow[shadow_line].p_virt)
+                goto out0;
+
+        p_ptt = qed_ptt_acquire(p_hwfn);
+        if (!p_ptt) {
+                DP_NOTICE(p_hwfn,
+                          "QED_TIME_OUT on ptt acquire - dynamic allocation");
+                rc = -EBUSY;
+                goto out0;
+        }
+
+        p_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
+                                    p_blk->real_size_in_page,
+                                    &p_phys, GFP_KERNEL);
+        if (!p_virt) {
+                rc = -ENOMEM;
+                goto out1;
+        }
+        memset(p_virt, 0, p_blk->real_size_in_page);
+
+        /* configuration of refTagMask to 0xF is required for RoCE DIF MR only,
+         * to compensate for a HW bug, but it is configured even if DIF is not
+         * enabled. This is harmless and allows us to avoid a dedicated API. We
+         * configure the field for all of the contexts on the newly allocated
+         * page.
+         */
+        if (elem_type == QED_ELEM_TASK) {
+                u32 elem_i;
+                u8 *elem_start = (u8 *)p_virt;
+                union type1_task_context *elem;
+
+                for (elem_i = 0; elem_i < elems_per_p; elem_i++) {
+                        elem = (union type1_task_context *)elem_start;
+                        SET_FIELD(elem->roce_ctx.tdif_context.flags1,
+                                  TDIF_TASK_CONTEXT_REFTAGMASK, 0xf);
+                        elem_start += TYPE1_TASK_CXT_SIZE(p_hwfn);
+                }
+        }
+
+        p_hwfn->p_cxt_mngr->ilt_shadow[shadow_line].p_virt = p_virt;
+        p_hwfn->p_cxt_mngr->ilt_shadow[shadow_line].p_phys = p_phys;
+        p_hwfn->p_cxt_mngr->ilt_shadow[shadow_line].size =
+            p_blk->real_size_in_page;
+
+        /* compute absolute offset */
+        reg_offset = PSWRQ2_REG_ILT_MEMORY +
+            (line * ILT_REG_SIZE_IN_BYTES * ILT_ENTRY_IN_REGS);
+
+        ilt_hw_entry = 0;
+        SET_FIELD(ilt_hw_entry, ILT_ENTRY_VALID, 1ULL);
+        SET_FIELD(ilt_hw_entry,
+                  ILT_ENTRY_PHY_ADDR,
+                  (p_hwfn->p_cxt_mngr->ilt_shadow[shadow_line].p_phys >> 12));
+
+        /* Write via DMAE since the PSWRQ2_REG_ILT_MEMORY line is a wide-bus */
+        qed_dmae_host2grc(p_hwfn, p_ptt, (u64) (uintptr_t)&ilt_hw_entry,
+                          reg_offset, sizeof(ilt_hw_entry) / sizeof(u32), 0);
+
+        if (elem_type == QED_ELEM_CXT) {
+                u32 last_cid_allocated = (1 + (iid / elems_per_p)) *
+                    elems_per_p;
+
+                /* Update the relevant register in the parser */
+                qed_wr(p_hwfn, p_ptt, PRS_REG_ROCE_DEST_QP_MAX_PF,
+                       last_cid_allocated - 1);
+
+                if (!p_hwfn->b_rdma_enabled_in_prs) {
+                        /* Enable RoCE search */
+                        qed_wr(p_hwfn, p_ptt, p_hwfn->rdma_prs_search_reg, 1);
+                        p_hwfn->b_rdma_enabled_in_prs = true;
+                }
+        }
+
+out1:
+        qed_ptt_release(p_hwfn, p_ptt);
+out0:
+        mutex_unlock(&p_hwfn->p_cxt_mngr->mutex);
+
+        return rc;
+}
+
+/* This function is very RoCE oriented, if another protocol in the future
+ * will want this feature we'll need to modify the function to be more generic
+ */
+static int
+qed_cxt_free_ilt_range(struct qed_hwfn *p_hwfn,
+                       enum qed_cxt_elem_type elem_type,
+                       u32 start_iid, u32 count)
+{
+        u32 start_line, end_line, shadow_start_line, shadow_end_line;
+        u32 reg_offset, elem_size, hw_p_size, elems_per_p;
+        struct qed_ilt_client_cfg *p_cli;
+        struct qed_ilt_cli_blk *p_blk;
+        u32 end_iid = start_iid + count;
+        struct qed_ptt *p_ptt;
+        u64 ilt_hw_entry = 0;
+        u32 i;
+
+        switch (elem_type) {
+        case QED_ELEM_CXT:
+                p_cli = &p_hwfn->p_cxt_mngr->clients[ILT_CLI_CDUC];
+                elem_size = CONN_CXT_SIZE(p_hwfn);
+                p_blk = &p_cli->pf_blks[CDUC_BLK];
+                break;
+        case QED_ELEM_SRQ:
+                p_cli = &p_hwfn->p_cxt_mngr->clients[ILT_CLI_TSDM];
+                elem_size = SRQ_CXT_SIZE;
+                p_blk = &p_cli->pf_blks[SRQ_BLK];
+                break;
+        case QED_ELEM_TASK:
+                p_cli = &p_hwfn->p_cxt_mngr->clients[ILT_CLI_CDUT];
+                elem_size = TYPE1_TASK_CXT_SIZE(p_hwfn);
+                p_blk = &p_cli->pf_blks[CDUT_SEG_BLK(QED_CXT_ROCE_TID_SEG)];
+                break;
+        default:
+                DP_NOTICE(p_hwfn, "-EINVALID elem type = %d", elem_type);
+                return -EINVAL;
+        }
+
+        /* Calculate line in ilt */
+        hw_p_size = p_cli->p_size.val;
+        elems_per_p = ILT_PAGE_IN_BYTES(hw_p_size) / elem_size;
+        start_line = p_blk->start_line + (start_iid / elems_per_p);
+        end_line = p_blk->start_line + (end_iid / elems_per_p);
+        if (((end_iid + 1) / elems_per_p) != (end_iid / elems_per_p))
+                end_line--;
+
+        shadow_start_line = start_line - p_hwfn->p_cxt_mngr->pf_start_line;
+        shadow_end_line = end_line - p_hwfn->p_cxt_mngr->pf_start_line;
+
+        p_ptt = qed_ptt_acquire(p_hwfn);
+        if (!p_ptt) {
+                DP_NOTICE(p_hwfn,
+                          "QED_TIME_OUT on ptt acquire - dynamic allocation");
+                return -EBUSY;
+        }
+
+        for (i = shadow_start_line; i < shadow_end_line; i++) {
+                if (!p_hwfn->p_cxt_mngr->ilt_shadow[i].p_virt)
+                        continue;
+
+                dma_free_coherent(&p_hwfn->cdev->pdev->dev,
+                                  p_hwfn->p_cxt_mngr->ilt_shadow[i].size,
+                                  p_hwfn->p_cxt_mngr->ilt_shadow[i].p_virt,
+                                  p_hwfn->p_cxt_mngr->ilt_shadow[i].p_phys);
+
+                p_hwfn->p_cxt_mngr->ilt_shadow[i].p_virt = NULL;
+                p_hwfn->p_cxt_mngr->ilt_shadow[i].p_phys = 0;
+                p_hwfn->p_cxt_mngr->ilt_shadow[i].size = 0;
+
+                /* compute absolute offset */
+                reg_offset = PSWRQ2_REG_ILT_MEMORY +
+                    ((start_line++) * ILT_REG_SIZE_IN_BYTES *
+                     ILT_ENTRY_IN_REGS);
+
+                /* Write via DMAE since the PSWRQ2_REG_ILT_MEMORY line is a
+                 * wide-bus.
+                 */
+                qed_dmae_host2grc(p_hwfn, p_ptt,
+                                  (u64) (uintptr_t) &ilt_hw_entry,
+                                  reg_offset,
+                                  sizeof(ilt_hw_entry) / sizeof(u32),
+                                  0);
+        }
+
+        qed_ptt_release(p_hwfn, p_ptt);
+
+        return 0;
+}
+
+int qed_cxt_free_proto_ilt(struct qed_hwfn *p_hwfn, enum protocol_type proto)
+{
+        int rc;
+        u32 cid;
+
+        /* Free Connection CXT */
+        rc = qed_cxt_free_ilt_range(p_hwfn, QED_ELEM_CXT,
+                                    qed_cxt_get_proto_cid_start(p_hwfn,
+                                                                proto),
+                                    qed_cxt_get_proto_cid_count(p_hwfn,
+                                                                proto, &cid));
+
+        if (rc)
+                return rc;
+
+        /* Free Task CXT */
+        rc = qed_cxt_free_ilt_range(p_hwfn, QED_ELEM_TASK, 0,
+                                    qed_cxt_get_proto_tid_count(p_hwfn, proto));
+        if (rc)
+                return rc;
+
+        /* Free TSDM CXT */
+        rc = qed_cxt_free_ilt_range(p_hwfn, QED_ELEM_SRQ, 0,
+                                    qed_cxt_get_srq_count(p_hwfn));
+
+        return rc;
+}
+
+int qed_cxt_get_task_ctx(struct qed_hwfn *p_hwfn,
+                         u32 tid, u8 ctx_type, void **pp_task_ctx)
+{
+        struct qed_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr;
+        struct qed_ilt_client_cfg *p_cli;
+        struct qed_ilt_cli_blk *p_seg;
+        struct qed_tid_seg *p_seg_info;
+        u32 proto, seg;
+        u32 total_lines;
+        u32 tid_size, ilt_idx;
+        u32 num_tids_per_block;
+
+        /* Verify the personality */
+        switch (p_hwfn->hw_info.personality) {
+        case QED_PCI_ISCSI:
+                proto = PROTOCOLID_ISCSI;
+                seg = QED_CXT_ISCSI_TID_SEG;
+                break;
+        default:
+                return -EINVAL;
+        }
+
+        p_cli = &p_mngr->clients[ILT_CLI_CDUT];
+        if (!p_cli->active)
+                return -EINVAL;
+
+        p_seg_info = &p_mngr->conn_cfg[proto].tid_seg[seg];
+
+        if (ctx_type == QED_CTX_WORKING_MEM) {
+                p_seg = &p_cli->pf_blks[CDUT_SEG_BLK(seg)];
+        } else if (ctx_type == QED_CTX_FL_MEM) {
+                if (!p_seg_info->has_fl_mem)
+                        return -EINVAL;
+                p_seg = &p_cli->pf_blks[CDUT_FL_SEG_BLK(seg, PF)];
+        } else {
+                return -EINVAL;
+        }
+        total_lines = DIV_ROUND_UP(p_seg->total_size, p_seg->real_size_in_page);
+        tid_size = p_mngr->task_type_size[p_seg_info->type];
+        num_tids_per_block = p_seg->real_size_in_page / tid_size;
+
+        if (total_lines < tid / num_tids_per_block)
+                return -EINVAL;
+
+        ilt_idx = tid / num_tids_per_block + p_seg->start_line -
+                  p_mngr->pf_start_line;
+        *pp_task_ctx = (u8 *)p_mngr->ilt_shadow[ilt_idx].p_virt +
+                       (tid % num_tids_per_block) * tid_size;
 
         return 0;
 }
diff --git a/drivers/net/ethernet/qlogic/qed/qed_cxt.h b/drivers/net/ethernet/qlogic/qed/qed_cxt.h
index 234c0fa8db2a..c6f6f2e8192d 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_cxt.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_cxt.h
@@ -21,6 +21,14 @@ struct qed_cxt_info {
         enum protocol_type      type;
 };
 
+#define MAX_TID_BLOCKS                  512
+struct qed_tid_mem {
+        u32 tid_size;
+        u32 num_tids_per_block;
+        u32 waste;
+        u8 *blocks[MAX_TID_BLOCKS];     /* 4K */
+};
+
 /**
  * @brief qed_cxt_acquire - Acquire a new cid of a specific protocol type
  *
@@ -46,8 +54,22 @@ int qed_cxt_acquire_cid(struct qed_hwfn *p_hwfn,
 int qed_cxt_get_cid_info(struct qed_hwfn *p_hwfn,
                          struct qed_cxt_info *p_info);
 
+/**
+ * @brief qed_cxt_get_tid_mem_info
+ *
+ * @param p_hwfn
+ * @param p_info
+ *
+ * @return int
+ */
+int qed_cxt_get_tid_mem_info(struct qed_hwfn *p_hwfn,
+                             struct qed_tid_mem *p_info);
+
+#define QED_CXT_ISCSI_TID_SEG   PROTOCOLID_ISCSI
+#define QED_CXT_ROCE_TID_SEG    PROTOCOLID_ROCE
 enum qed_cxt_elem_type {
         QED_ELEM_CXT,
+        QED_ELEM_SRQ,
         QED_ELEM_TASK
 };
 
@@ -149,4 +171,6 @@ int qed_qm_reconf(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt);
 void qed_cxt_release_cid(struct qed_hwfn *p_hwfn,
                          u32 cid);
 
+#define QED_CTX_WORKING_MEM 0
+#define QED_CTX_FL_MEM 1
 #endif
diff --git a/drivers/net/ethernet/qlogic/qed/qed_dcbx.c b/drivers/net/ethernet/qlogic/qed/qed_dcbx.c
index 21ec1c2df2c7..d0dc28f93c0e 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_dcbx.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_dcbx.c
@@ -9,6 +9,7 @@
 #include <linux/types.h>
 #include <asm/byteorder.h>
 #include <linux/bitops.h>
+#include <linux/dcbnl.h>
 #include <linux/errno.h>
 #include <linux/kernel.h>
 #include <linux/slab.h>
@@ -18,6 +19,9 @@
 #include "qed_dcbx.h"
 #include "qed_hsi.h"
 #include "qed_sp.h"
+#ifdef CONFIG_DCB
+#include <linux/qed/qed_eth_if.h>
+#endif
 
 #define QED_DCBX_MAX_MIB_READ_TRY       (100)
 #define QED_ETH_TYPE_DEFAULT            (0)
@@ -252,7 +256,7 @@ qed_dcbx_process_tlv(struct qed_hwfn *p_hwfn,
                 if (p_data->arr[type].update)
                         continue;
 
-                enable = (type == DCBX_PROTOCOL_ETH) ? false : dcbx_enabled;
+                enable = !(type == DCBX_PROTOCOL_ETH);
                 qed_dcbx_update_app_info(p_data, p_hwfn, enable, true,
                                          priority, tc, type);
         }
@@ -351,6 +355,293 @@ qed_dcbx_copy_mib(struct qed_hwfn *p_hwfn,
         return rc;
 }
 
+#ifdef CONFIG_DCB
+static void
+qed_dcbx_get_priority_info(struct qed_hwfn *p_hwfn,
+                           struct qed_dcbx_app_prio *p_prio,
+                           struct qed_dcbx_results *p_results)
+{
+        u8 val;
+
+        p_prio->roce = QED_DCBX_INVALID_PRIORITY;
+        p_prio->roce_v2 = QED_DCBX_INVALID_PRIORITY;
+        p_prio->iscsi = QED_DCBX_INVALID_PRIORITY;
+        p_prio->fcoe = QED_DCBX_INVALID_PRIORITY;
+
+        if (p_results->arr[DCBX_PROTOCOL_ROCE].update &&
+            p_results->arr[DCBX_PROTOCOL_ROCE].enable)
+                p_prio->roce = p_results->arr[DCBX_PROTOCOL_ROCE].priority;
+
+        if (p_results->arr[DCBX_PROTOCOL_ROCE_V2].update &&
+            p_results->arr[DCBX_PROTOCOL_ROCE_V2].enable) {
+                val = p_results->arr[DCBX_PROTOCOL_ROCE_V2].priority;
+                p_prio->roce_v2 = val;
+        }
+
+        if (p_results->arr[DCBX_PROTOCOL_ISCSI].update &&
+            p_results->arr[DCBX_PROTOCOL_ISCSI].enable)
+                p_prio->iscsi = p_results->arr[DCBX_PROTOCOL_ISCSI].priority;
+
+        if (p_results->arr[DCBX_PROTOCOL_FCOE].update &&
+            p_results->arr[DCBX_PROTOCOL_FCOE].enable)
+                p_prio->fcoe = p_results->arr[DCBX_PROTOCOL_FCOE].priority;
+
+        if (p_results->arr[DCBX_PROTOCOL_ETH].update &&
+            p_results->arr[DCBX_PROTOCOL_ETH].enable)
+                p_prio->eth = p_results->arr[DCBX_PROTOCOL_ETH].priority;
+
+        DP_VERBOSE(p_hwfn, QED_MSG_DCB,
+                   "Priorities: iscsi %d, roce %d, roce v2 %d, fcoe %d, eth %d\n",
+                   p_prio->iscsi, p_prio->roce, p_prio->roce_v2, p_prio->fcoe,
+                   p_prio->eth);
+}
+
+static void
+qed_dcbx_get_app_data(struct qed_hwfn *p_hwfn,
+                      struct dcbx_app_priority_feature *p_app,
+                      struct dcbx_app_priority_entry *p_tbl,
+                      struct qed_dcbx_params *p_params)
+{
+        struct qed_app_entry *entry;
+        u8 pri_map;
+        int i;
+
+        p_params->app_willing = QED_MFW_GET_FIELD(p_app->flags,
+                                                  DCBX_APP_WILLING);
+        p_params->app_valid = QED_MFW_GET_FIELD(p_app->flags, DCBX_APP_ENABLED);
+        p_params->app_error = QED_MFW_GET_FIELD(p_app->flags, DCBX_APP_ERROR);
+        p_params->num_app_entries = QED_MFW_GET_FIELD(p_app->flags,
+                                                      DCBX_APP_NUM_ENTRIES);
+        for (i = 0; i < DCBX_MAX_APP_PROTOCOL; i++) {
+                entry = &p_params->app_entry[i];
+                entry->ethtype = !(QED_MFW_GET_FIELD(p_tbl[i].entry,
+                                                     DCBX_APP_SF));
+                pri_map = QED_MFW_GET_FIELD(p_tbl[i].entry, DCBX_APP_PRI_MAP);
+                entry->prio = ffs(pri_map) - 1;
+                entry->proto_id = QED_MFW_GET_FIELD(p_tbl[i].entry,
+                                                    DCBX_APP_PROTOCOL_ID);
+                qed_dcbx_get_app_protocol_type(p_hwfn, p_tbl[i].entry,
+                                               entry->proto_id,
+                                               &entry->proto_type);
+        }
+
+        DP_VERBOSE(p_hwfn, QED_MSG_DCB,
+                   "APP params: willing %d, valid %d error = %d\n",
+                   p_params->app_willing, p_params->app_valid,
+                   p_params->app_error);
+}
+
+static void
+qed_dcbx_get_pfc_data(struct qed_hwfn *p_hwfn,
+                      u32 pfc, struct qed_dcbx_params *p_params)
+{
+        u8 pfc_map;
+
+        p_params->pfc.willing = QED_MFW_GET_FIELD(pfc, DCBX_PFC_WILLING);
+        p_params->pfc.max_tc = QED_MFW_GET_FIELD(pfc, DCBX_PFC_CAPS);
+        p_params->pfc.enabled = QED_MFW_GET_FIELD(pfc, DCBX_PFC_ENABLED);
+        pfc_map = QED_MFW_GET_FIELD(pfc, DCBX_PFC_PRI_EN_BITMAP);
+        p_params->pfc.prio[0] = !!(pfc_map & DCBX_PFC_PRI_EN_BITMAP_PRI_0);
+        p_params->pfc.prio[1] = !!(pfc_map & DCBX_PFC_PRI_EN_BITMAP_PRI_1);
+        p_params->pfc.prio[2] = !!(pfc_map & DCBX_PFC_PRI_EN_BITMAP_PRI_2);
+        p_params->pfc.prio[3] = !!(pfc_map & DCBX_PFC_PRI_EN_BITMAP_PRI_3);
+        p_params->pfc.prio[4] = !!(pfc_map & DCBX_PFC_PRI_EN_BITMAP_PRI_4);
+        p_params->pfc.prio[5] = !!(pfc_map & DCBX_PFC_PRI_EN_BITMAP_PRI_5);
+        p_params->pfc.prio[6] = !!(pfc_map & DCBX_PFC_PRI_EN_BITMAP_PRI_6);
+        p_params->pfc.prio[7] = !!(pfc_map & DCBX_PFC_PRI_EN_BITMAP_PRI_7);
+
+        DP_VERBOSE(p_hwfn, QED_MSG_DCB,
+                   "PFC params: willing %d, pfc_bitmap %d\n",
+                   p_params->pfc.willing, pfc_map);
+}
+
+static void
+qed_dcbx_get_ets_data(struct qed_hwfn *p_hwfn,
+                      struct dcbx_ets_feature *p_ets,
+                      struct qed_dcbx_params *p_params)
+{
+        u32 bw_map[2], tsa_map[2], pri_map;
+        int i;
+
+        p_params->ets_willing = QED_MFW_GET_FIELD(p_ets->flags,
+                                                  DCBX_ETS_WILLING);
+        p_params->ets_enabled = QED_MFW_GET_FIELD(p_ets->flags,
+                                                  DCBX_ETS_ENABLED);
+        p_params->ets_cbs = QED_MFW_GET_FIELD(p_ets->flags, DCBX_ETS_CBS);
+        p_params->max_ets_tc = QED_MFW_GET_FIELD(p_ets->flags,
+                                                 DCBX_ETS_MAX_TCS);
+        DP_VERBOSE(p_hwfn, QED_MSG_DCB,
+                   "ETS params: willing %d, ets_cbs %d pri_tc_tbl_0 %x max_ets_tc %d\n",
+                   p_params->ets_willing,
+                   p_params->ets_cbs,
+                   p_ets->pri_tc_tbl[0], p_params->max_ets_tc);
+
+        /* 8 bit tsa and bw data corresponding to each of the 8 TC's are
+         * encoded in a type u32 array of size 2.
+         */
+        bw_map[0] = be32_to_cpu(p_ets->tc_bw_tbl[0]);
+        bw_map[1] = be32_to_cpu(p_ets->tc_bw_tbl[1]);
+        tsa_map[0] = be32_to_cpu(p_ets->tc_tsa_tbl[0]);
+        tsa_map[1] = be32_to_cpu(p_ets->tc_tsa_tbl[1]);
+        pri_map = be32_to_cpu(p_ets->pri_tc_tbl[0]);
+        for (i = 0; i < QED_MAX_PFC_PRIORITIES; i++) {
+                p_params->ets_tc_bw_tbl[i] = ((u8 *)bw_map)[i];
+                p_params->ets_tc_tsa_tbl[i] = ((u8 *)tsa_map)[i];
+                p_params->ets_pri_tc_tbl[i] = QED_DCBX_PRIO2TC(pri_map, i);
+                DP_VERBOSE(p_hwfn, QED_MSG_DCB,
+                           "elem %d  bw_tbl %x tsa_tbl %x\n",
+                           i, p_params->ets_tc_bw_tbl[i],
+                           p_params->ets_tc_tsa_tbl[i]);
+        }
+}
+
+static void
+qed_dcbx_get_common_params(struct qed_hwfn *p_hwfn,
+                           struct dcbx_app_priority_feature *p_app,
+                           struct dcbx_app_priority_entry *p_tbl,
+                           struct dcbx_ets_feature *p_ets,
+                           u32 pfc, struct qed_dcbx_params *p_params)
+{
+        qed_dcbx_get_app_data(p_hwfn, p_app, p_tbl, p_params);
+        qed_dcbx_get_ets_data(p_hwfn, p_ets, p_params);
+        qed_dcbx_get_pfc_data(p_hwfn, pfc, p_params);
+}
+
+static void
+qed_dcbx_get_local_params(struct qed_hwfn *p_hwfn,
+                          struct qed_ptt *p_ptt, struct qed_dcbx_get *params)
+{
+        struct dcbx_features *p_feat;
+
+        p_feat = &p_hwfn->p_dcbx_info->local_admin.features;
+        qed_dcbx_get_common_params(p_hwfn, &p_feat->app,
+                                   p_feat->app.app_pri_tbl, &p_feat->ets,
+                                   p_feat->pfc, &params->local.params);
+        params->local.valid = true;
+}
+
+static void
+qed_dcbx_get_remote_params(struct qed_hwfn *p_hwfn,
+                           struct qed_ptt *p_ptt, struct qed_dcbx_get *params)
+{
+        struct dcbx_features *p_feat;
+
+        p_feat = &p_hwfn->p_dcbx_info->remote.features;
+        qed_dcbx_get_common_params(p_hwfn, &p_feat->app,
+                                   p_feat->app.app_pri_tbl, &p_feat->ets,
+                                   p_feat->pfc, &params->remote.params);
+        params->remote.valid = true;
+}
+
+static void
+qed_dcbx_get_operational_params(struct qed_hwfn *p_hwfn,
+                                struct qed_ptt *p_ptt,
+                                struct qed_dcbx_get *params)
+{
+        struct qed_dcbx_operational_params *p_operational;
+        struct qed_dcbx_results *p_results;
+        struct dcbx_features *p_feat;
+        bool enabled, err;
+        u32 flags;
+        bool val;
+
+        flags = p_hwfn->p_dcbx_info->operational.flags;
+
+        /* If DCBx version is non zero, then negotiation
+         * was successfuly performed
+         */
+        p_operational = &params->operational;
+        enabled = !!(QED_MFW_GET_FIELD(flags, DCBX_CONFIG_VERSION) !=
+                     DCBX_CONFIG_VERSION_DISABLED);
+        if (!enabled) {
+                p_operational->enabled = enabled;
+                p_operational->valid = false;
+                return;
+        }
+
+        p_feat = &p_hwfn->p_dcbx_info->operational.features;
+        p_results = &p_hwfn->p_dcbx_info->results;
+
+        val = !!(QED_MFW_GET_FIELD(flags, DCBX_CONFIG_VERSION) ==
+                 DCBX_CONFIG_VERSION_IEEE);
+        p_operational->ieee = val;
+        val = !!(QED_MFW_GET_FIELD(flags, DCBX_CONFIG_VERSION) ==
+                 DCBX_CONFIG_VERSION_CEE);
+        p_operational->cee = val;
+
+        DP_VERBOSE(p_hwfn, QED_MSG_DCB, "Version support: ieee %d, cee %d\n",
+                   p_operational->ieee, p_operational->cee);
+
+        qed_dcbx_get_common_params(p_hwfn, &p_feat->app,
+                                   p_feat->app.app_pri_tbl, &p_feat->ets,
+                                   p_feat->pfc, &params->operational.params);
+        qed_dcbx_get_priority_info(p_hwfn, &p_operational->app_prio, p_results);
+        err = QED_MFW_GET_FIELD(p_feat->app.flags, DCBX_APP_ERROR);
+        p_operational->err = err;
+        p_operational->enabled = enabled;
+        p_operational->valid = true;
+}
+
+static void
+qed_dcbx_get_local_lldp_params(struct qed_hwfn *p_hwfn,
+                               struct qed_ptt *p_ptt,
+                               struct qed_dcbx_get *params)
+{
+        struct lldp_config_params_s *p_local;
+
+        p_local = &p_hwfn->p_dcbx_info->lldp_local[LLDP_NEAREST_BRIDGE];
+
+        memcpy(params->lldp_local.local_chassis_id, p_local->local_chassis_id,
+               ARRAY_SIZE(p_local->local_chassis_id));
+        memcpy(params->lldp_local.local_port_id, p_local->local_port_id,
+               ARRAY_SIZE(p_local->local_port_id));
+}
+
+static void
+qed_dcbx_get_remote_lldp_params(struct qed_hwfn *p_hwfn,
+                                struct qed_ptt *p_ptt,
+                                struct qed_dcbx_get *params)
+{
+        struct lldp_status_params_s *p_remote;
+
+        p_remote = &p_hwfn->p_dcbx_info->lldp_remote[LLDP_NEAREST_BRIDGE];
+
+        memcpy(params->lldp_remote.peer_chassis_id, p_remote->peer_chassis_id,
+               ARRAY_SIZE(p_remote->peer_chassis_id));
+        memcpy(params->lldp_remote.peer_port_id, p_remote->peer_port_id,
+               ARRAY_SIZE(p_remote->peer_port_id));
+}
+
+static int
+qed_dcbx_get_params(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
+                    struct qed_dcbx_get *p_params,
+                    enum qed_mib_read_type type)
+{
+        switch (type) {
+        case QED_DCBX_REMOTE_MIB:
+                qed_dcbx_get_remote_params(p_hwfn, p_ptt, p_params);
+                break;
+        case QED_DCBX_LOCAL_MIB:
+                qed_dcbx_get_local_params(p_hwfn, p_ptt, p_params);
+                break;
+        case QED_DCBX_OPERATIONAL_MIB:
+                qed_dcbx_get_operational_params(p_hwfn, p_ptt, p_params);
+                break;
+        case QED_DCBX_REMOTE_LLDP_MIB:
+                qed_dcbx_get_remote_lldp_params(p_hwfn, p_ptt, p_params);
+                break;
+        case QED_DCBX_LOCAL_LLDP_MIB:
+                qed_dcbx_get_local_lldp_params(p_hwfn, p_ptt, p_params);
+                break;
+        default:
+                DP_ERR(p_hwfn, "MIB read err, unknown mib type %d\n", type);
+                return -EINVAL;
+        }
+
+        return 0;
+}
+#endif
+
 static int
 qed_dcbx_read_local_lldp_mib(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 {
@@ -561,3 +852,1333 @@ void qed_dcbx_set_pf_update_params(struct qed_dcbx_results *p_src,
         p_dcb_data = &p_dest->eth_dcb_data;
         qed_dcbx_update_protocol_data(p_dcb_data, p_src, DCBX_PROTOCOL_ETH);
 }
+
+#ifdef CONFIG_DCB
+static int qed_dcbx_query_params(struct qed_hwfn *p_hwfn,
+                                 struct qed_dcbx_get *p_get,
+                                 enum qed_mib_read_type type)
+{
+        struct qed_ptt *p_ptt;
+        int rc;
+
+        p_ptt = qed_ptt_acquire(p_hwfn);
+        if (!p_ptt)
+                return -EBUSY;
+
+        rc = qed_dcbx_read_mib(p_hwfn, p_ptt, type);
+        if (rc)
+                goto out;
+
+        rc = qed_dcbx_get_params(p_hwfn, p_ptt, p_get, type);
+
+out:
+        qed_ptt_release(p_hwfn, p_ptt);
+        return rc;
+}
+
+static void
+qed_dcbx_set_pfc_data(struct qed_hwfn *p_hwfn,
+                      u32 *pfc, struct qed_dcbx_params *p_params)
+{
+        u8 pfc_map = 0;
+        int i;
+
+        if (p_params->pfc.willing)
+                *pfc |= DCBX_PFC_WILLING_MASK;
+        else
+                *pfc &= ~DCBX_PFC_WILLING_MASK;
+
+        if (p_params->pfc.enabled)
+                *pfc |= DCBX_PFC_ENABLED_MASK;
+        else
+                *pfc &= ~DCBX_PFC_ENABLED_MASK;
+
+        *pfc &= ~DCBX_PFC_CAPS_MASK;
+        *pfc |= (u32)p_params->pfc.max_tc << DCBX_PFC_CAPS_SHIFT;
+
+        for (i = 0; i < QED_MAX_PFC_PRIORITIES; i++)
+                if (p_params->pfc.prio[i])
+                        pfc_map |= BIT(i);
+
+        *pfc |= (pfc_map << DCBX_PFC_PRI_EN_BITMAP_SHIFT);
+
+        DP_VERBOSE(p_hwfn, QED_MSG_DCB, "pfc = 0x%x\n", *pfc);
+}
+
+static void
+qed_dcbx_set_ets_data(struct qed_hwfn *p_hwfn,
+                      struct dcbx_ets_feature *p_ets,
+                      struct qed_dcbx_params *p_params)
+{
+        u8 *bw_map, *tsa_map;
+        u32 val;
+        int i;
+
+        if (p_params->ets_willing)
+                p_ets->flags |= DCBX_ETS_WILLING_MASK;
+        else
+                p_ets->flags &= ~DCBX_ETS_WILLING_MASK;
+
+        if (p_params->ets_cbs)
+                p_ets->flags |= DCBX_ETS_CBS_MASK;
+        else
+                p_ets->flags &= ~DCBX_ETS_CBS_MASK;
+
+        if (p_params->ets_enabled)
+                p_ets->flags |= DCBX_ETS_ENABLED_MASK;
+        else
+                p_ets->flags &= ~DCBX_ETS_ENABLED_MASK;
+
+        p_ets->flags &= ~DCBX_ETS_MAX_TCS_MASK;
+        p_ets->flags |= (u32)p_params->max_ets_tc << DCBX_ETS_MAX_TCS_SHIFT;
+
+        bw_map = (u8 *)&p_ets->tc_bw_tbl[0];
+        tsa_map = (u8 *)&p_ets->tc_tsa_tbl[0];
+        p_ets->pri_tc_tbl[0] = 0;
+        for (i = 0; i < QED_MAX_PFC_PRIORITIES; i++) {
+                bw_map[i] = p_params->ets_tc_bw_tbl[i];
+                tsa_map[i] = p_params->ets_tc_tsa_tbl[i];
+                /* Copy the priority value to the corresponding 4 bits in the
+                 * traffic class table.
+                 */
+                val = (((u32)p_params->ets_pri_tc_tbl[i]) << ((7 - i) * 4));
+                p_ets->pri_tc_tbl[0] |= val;
+        }
+        p_ets->pri_tc_tbl[0] = cpu_to_be32(p_ets->pri_tc_tbl[0]);
+        for (i = 0; i < 2; i++) {
+                p_ets->tc_bw_tbl[i] = cpu_to_be32(p_ets->tc_bw_tbl[i]);
+                p_ets->tc_tsa_tbl[i] = cpu_to_be32(p_ets->tc_tsa_tbl[i]);
+        }
+}
+
+static void
+qed_dcbx_set_app_data(struct qed_hwfn *p_hwfn,
+                      struct dcbx_app_priority_feature *p_app,
+                      struct qed_dcbx_params *p_params)
+{
+        u32 *entry;
+        int i;
+
+        if (p_params->app_willing)
+                p_app->flags |= DCBX_APP_WILLING_MASK;
+        else
+                p_app->flags &= ~DCBX_APP_WILLING_MASK;
+
+        if (p_params->app_valid)
+                p_app->flags |= DCBX_APP_ENABLED_MASK;
+        else
+                p_app->flags &= ~DCBX_APP_ENABLED_MASK;
+
+        p_app->flags &= ~DCBX_APP_NUM_ENTRIES_MASK;
+        p_app->flags |= (u32)p_params->num_app_entries <<
+            DCBX_APP_NUM_ENTRIES_SHIFT;
+
+        for (i = 0; i < DCBX_MAX_APP_PROTOCOL; i++) {
+                entry = &p_app->app_pri_tbl[i].entry;
+                *entry &= ~DCBX_APP_SF_MASK;
+                if (p_params->app_entry[i].ethtype)
+                        *entry |= ((u32)DCBX_APP_SF_ETHTYPE <<
+                                   DCBX_APP_SF_SHIFT);
+                else
+                        *entry |= ((u32)DCBX_APP_SF_PORT << DCBX_APP_SF_SHIFT);
+                *entry &= ~DCBX_APP_PROTOCOL_ID_MASK;
+                *entry |= ((u32)p_params->app_entry[i].proto_id <<
+                           DCBX_APP_PROTOCOL_ID_SHIFT);
+                *entry &= ~DCBX_APP_PRI_MAP_MASK;
+                *entry |= ((u32)(p_params->app_entry[i].prio) <<
+                           DCBX_APP_PRI_MAP_SHIFT);
+        }
+}
+
+static void
+qed_dcbx_set_local_params(struct qed_hwfn *p_hwfn,
+                          struct dcbx_local_params *local_admin,
+                          struct qed_dcbx_set *params)
+{
+        local_admin->flags = 0;
+        memcpy(&local_admin->features,
+               &p_hwfn->p_dcbx_info->operational.features,
+               sizeof(local_admin->features));
+
+        if (params->enabled)
+                local_admin->config = params->ver_num;
+        else
+                local_admin->config = DCBX_CONFIG_VERSION_DISABLED;
+
+        if (params->override_flags & QED_DCBX_OVERRIDE_PFC_CFG)
+                qed_dcbx_set_pfc_data(p_hwfn, &local_admin->features.pfc,
+                                      &params->config.params);
+
+        if (params->override_flags & QED_DCBX_OVERRIDE_ETS_CFG)
+                qed_dcbx_set_ets_data(p_hwfn, &local_admin->features.ets,
+                                      &params->config.params);
+
+        if (params->override_flags & QED_DCBX_OVERRIDE_APP_CFG)
+                qed_dcbx_set_app_data(p_hwfn, &local_admin->features.app,
+                                      &params->config.params);
+}
+
+int qed_dcbx_config_params(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
+                           struct qed_dcbx_set *params, bool hw_commit)
+{
+        struct dcbx_local_params local_admin;
+        struct qed_dcbx_mib_meta_data data;
+        u32 resp = 0, param = 0;
+        int rc = 0;
+
+        if (!hw_commit) {
+                memcpy(&p_hwfn->p_dcbx_info->set, params,
+                       sizeof(struct qed_dcbx_set));
+                return 0;
+        }
+
+        /* clear set-parmas cache */
+        memset(&p_hwfn->p_dcbx_info->set, 0, sizeof(p_hwfn->p_dcbx_info->set));
+
+        memset(&local_admin, 0, sizeof(local_admin));
+        qed_dcbx_set_local_params(p_hwfn, &local_admin, params);
+
+        data.addr = p_hwfn->mcp_info->port_addr +
+            offsetof(struct public_port, local_admin_dcbx_mib);
+        data.local_admin = &local_admin;
+        data.size = sizeof(struct dcbx_local_params);
+        qed_memcpy_to(p_hwfn, p_ptt, data.addr, data.local_admin, data.size);
+
+        rc = qed_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_SET_DCBX,
+                         1 << DRV_MB_PARAM_LLDP_SEND_SHIFT, &resp, &param);
+        if (rc)
+                DP_NOTICE(p_hwfn, "Failed to send DCBX update request\n");
+
+        return rc;
+}
+
+int qed_dcbx_get_config_params(struct qed_hwfn *p_hwfn,
+                               struct qed_dcbx_set *params)
+{
+        struct qed_dcbx_get *dcbx_info;
+        int rc;
+
+        if (p_hwfn->p_dcbx_info->set.config.valid) {
+                memcpy(params, &p_hwfn->p_dcbx_info->set,
+                       sizeof(struct qed_dcbx_set));
+                return 0;
+        }
+
+        dcbx_info = kmalloc(sizeof(*dcbx_info), GFP_KERNEL);
+        if (!dcbx_info) {
+                DP_ERR(p_hwfn, "Failed to allocate struct qed_dcbx_info\n");
+                return -ENOMEM;
+        }
+
+        rc = qed_dcbx_query_params(p_hwfn, dcbx_info, QED_DCBX_OPERATIONAL_MIB);
+        if (rc) {
+                kfree(dcbx_info);
+                return rc;
+        }
+
+        p_hwfn->p_dcbx_info->set.override_flags = 0;
+        p_hwfn->p_dcbx_info->set.ver_num = DCBX_CONFIG_VERSION_DISABLED;
+        if (dcbx_info->operational.cee)
+                p_hwfn->p_dcbx_info->set.ver_num |= DCBX_CONFIG_VERSION_CEE;
+        if (dcbx_info->operational.ieee)
+                p_hwfn->p_dcbx_info->set.ver_num |= DCBX_CONFIG_VERSION_IEEE;
+
+        p_hwfn->p_dcbx_info->set.enabled = dcbx_info->operational.enabled;
+        memcpy(&p_hwfn->p_dcbx_info->set.config.params,
+               &dcbx_info->operational.params,
+               sizeof(struct qed_dcbx_admin_params));
+        p_hwfn->p_dcbx_info->set.config.valid = true;
+
+        memcpy(params, &p_hwfn->p_dcbx_info->set, sizeof(struct qed_dcbx_set));
+
+        kfree(dcbx_info);
+
+        return 0;
+}
+
+static struct qed_dcbx_get *qed_dcbnl_get_dcbx(struct qed_hwfn *hwfn,
+                                               enum qed_mib_read_type type)
+{
+        struct qed_dcbx_get *dcbx_info;
+
+        dcbx_info = kmalloc(sizeof(*dcbx_info), GFP_KERNEL);
+        if (!dcbx_info) {
+                DP_ERR(hwfn->cdev, "Failed to allocate memory for dcbx_info\n");
+                return NULL;
+        }
+
+        if (qed_dcbx_query_params(hwfn, dcbx_info, type)) {
+                kfree(dcbx_info);
+                return NULL;
+        }
+
+        if ((type == QED_DCBX_OPERATIONAL_MIB) &&
+            !dcbx_info->operational.enabled) {
+                DP_INFO(hwfn, "DCBX is not enabled/operational\n");
+                kfree(dcbx_info);
+                return NULL;
+        }
+
+        return dcbx_info;
+}
+
+static u8 qed_dcbnl_getstate(struct qed_dev *cdev)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_get *dcbx_info;
+        bool enabled;
+
+        dcbx_info = qed_dcbnl_get_dcbx(hwfn, QED_DCBX_OPERATIONAL_MIB);
+        if (!dcbx_info)
+                return 0;
+
+        enabled = dcbx_info->operational.enabled;
+        DP_VERBOSE(hwfn, QED_MSG_DCB, "DCB state = %d\n", enabled);
+        kfree(dcbx_info);
+
+        return enabled;
+}
+
+static u8 qed_dcbnl_setstate(struct qed_dev *cdev, u8 state)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_set dcbx_set;
+        struct qed_ptt *ptt;
+        int rc;
+
+        DP_VERBOSE(hwfn, QED_MSG_DCB, "DCB state = %d\n", state);
+
+        memset(&dcbx_set, 0, sizeof(dcbx_set));
+        rc = qed_dcbx_get_config_params(hwfn, &dcbx_set);
+        if (rc)
+                return 1;
+
+        dcbx_set.enabled = !!state;
+
+        ptt = qed_ptt_acquire(hwfn);
+        if (!ptt)
+                return 1;
+
+        rc = qed_dcbx_config_params(hwfn, ptt, &dcbx_set, 0);
+
+        qed_ptt_release(hwfn, ptt);
+
+        return rc ? 1 : 0;
+}
+
+static void qed_dcbnl_getpgtccfgtx(struct qed_dev *cdev, int tc, u8 *prio_type,
+                                   u8 *pgid, u8 *bw_pct, u8 *up_map)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_get *dcbx_info;
+
+        DP_VERBOSE(hwfn, QED_MSG_DCB, "tc = %d\n", tc);
+        *prio_type = *pgid = *bw_pct = *up_map = 0;
+        if (tc < 0 || tc >= QED_MAX_PFC_PRIORITIES) {
+                DP_INFO(hwfn, "Invalid tc %d\n", tc);
+                return;
+        }
+
+        dcbx_info = qed_dcbnl_get_dcbx(hwfn, QED_DCBX_OPERATIONAL_MIB);
+        if (!dcbx_info)
+                return;
+
+        *pgid = dcbx_info->operational.params.ets_pri_tc_tbl[tc];
+        kfree(dcbx_info);
+}
+
+static void qed_dcbnl_getpgbwgcfgtx(struct qed_dev *cdev, int pgid, u8 *bw_pct)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_get *dcbx_info;
+
+        *bw_pct = 0;
+        DP_VERBOSE(hwfn, QED_MSG_DCB, "pgid = %d\n", pgid);
+        if (pgid < 0 || pgid >= QED_MAX_PFC_PRIORITIES) {
+                DP_INFO(hwfn, "Invalid pgid %d\n", pgid);
+                return;
+        }
+
+        dcbx_info = qed_dcbnl_get_dcbx(hwfn, QED_DCBX_OPERATIONAL_MIB);
+        if (!dcbx_info)
+                return;
+
+        *bw_pct = dcbx_info->operational.params.ets_tc_bw_tbl[pgid];
+        DP_VERBOSE(hwfn, QED_MSG_DCB, "bw_pct = %d\n", *bw_pct);
+        kfree(dcbx_info);
+}
+
+static void qed_dcbnl_getpgtccfgrx(struct qed_dev *cdev, int tc, u8 *prio,
+                                   u8 *bwg_id, u8 *bw_pct, u8 *up_map)
+{
+        DP_INFO(QED_LEADING_HWFN(cdev), "Rx ETS is not supported\n");
+        *prio = *bwg_id = *bw_pct = *up_map = 0;
+}
+
+static void qed_dcbnl_getpgbwgcfgrx(struct qed_dev *cdev,
+                                    int bwg_id, u8 *bw_pct)
+{
+        DP_INFO(QED_LEADING_HWFN(cdev), "Rx ETS is not supported\n");
+        *bw_pct = 0;
+}
+
+static void qed_dcbnl_getpfccfg(struct qed_dev *cdev,
+                                int priority, u8 *setting)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_get *dcbx_info;
+
+        DP_VERBOSE(hwfn, QED_MSG_DCB, "priority = %d\n", priority);
+        if (priority < 0 || priority >= QED_MAX_PFC_PRIORITIES) {
+                DP_INFO(hwfn, "Invalid priority %d\n", priority);
+                return;
+        }
+
+        dcbx_info = qed_dcbnl_get_dcbx(hwfn, QED_DCBX_OPERATIONAL_MIB);
+        if (!dcbx_info)
+                return;
+
+        *setting = dcbx_info->operational.params.pfc.prio[priority];
+        DP_VERBOSE(hwfn, QED_MSG_DCB, "setting = %d\n", *setting);
+        kfree(dcbx_info);
+}
+
+static void qed_dcbnl_setpfccfg(struct qed_dev *cdev, int priority, u8 setting)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_set dcbx_set;
+        struct qed_ptt *ptt;
+        int rc;
+
+        DP_VERBOSE(hwfn, QED_MSG_DCB, "priority = %d setting = %d\n",
+                   priority, setting);
+        if (priority < 0 || priority >= QED_MAX_PFC_PRIORITIES) {
+                DP_INFO(hwfn, "Invalid priority %d\n", priority);
+                return;
+        }
+
+        memset(&dcbx_set, 0, sizeof(dcbx_set));
+        rc = qed_dcbx_get_config_params(hwfn, &dcbx_set);
+        if (rc)
+                return;
+
+        dcbx_set.override_flags |= QED_DCBX_OVERRIDE_PFC_CFG;
+        dcbx_set.config.params.pfc.prio[priority] = !!setting;
+
+        ptt = qed_ptt_acquire(hwfn);
+        if (!ptt)
+                return;
+
+        rc = qed_dcbx_config_params(hwfn, ptt, &dcbx_set, 0);
+
+        qed_ptt_release(hwfn, ptt);
+}
+
+static u8 qed_dcbnl_getcap(struct qed_dev *cdev, int capid, u8 *cap)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_get *dcbx_info;
+        int rc = 0;
+
+        DP_VERBOSE(hwfn, QED_MSG_DCB, "capid = %d\n", capid);
+        dcbx_info = qed_dcbnl_get_dcbx(hwfn, QED_DCBX_OPERATIONAL_MIB);
+        if (!dcbx_info)
+                return 1;
+
+        switch (capid) {
+        case DCB_CAP_ATTR_PG:
+        case DCB_CAP_ATTR_PFC:
+        case DCB_CAP_ATTR_UP2TC:
+        case DCB_CAP_ATTR_GSP:
+                *cap = true;
+                break;
+        case DCB_CAP_ATTR_PG_TCS:
+        case DCB_CAP_ATTR_PFC_TCS:
+                *cap = 0x80;
+                break;
+        case DCB_CAP_ATTR_DCBX:
+                *cap = (DCB_CAP_DCBX_LLD_MANAGED | DCB_CAP_DCBX_VER_CEE |
+                        DCB_CAP_DCBX_VER_IEEE);
+                break;
+        default:
+                *cap = false;
+                rc = 1;
+        }
+
+        DP_VERBOSE(hwfn, QED_MSG_DCB, "id = %d caps = %d\n", capid, *cap);
+        kfree(dcbx_info);
+
+        return rc;
+}
+
+static int qed_dcbnl_getnumtcs(struct qed_dev *cdev, int tcid, u8 *num)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_get *dcbx_info;
+        int rc = 0;
+
+        DP_VERBOSE(hwfn, QED_MSG_DCB, "tcid = %d\n", tcid);
+        dcbx_info = qed_dcbnl_get_dcbx(hwfn, QED_DCBX_OPERATIONAL_MIB);
+        if (!dcbx_info)
+                return -EINVAL;
+
+        switch (tcid) {
+        case DCB_NUMTCS_ATTR_PG:
+                *num = dcbx_info->operational.params.max_ets_tc;
+                break;
+        case DCB_NUMTCS_ATTR_PFC:
+                *num = dcbx_info->operational.params.pfc.max_tc;
+                break;
+        default:
+                rc = -EINVAL;
+        }
+
+        kfree(dcbx_info);
+        DP_VERBOSE(hwfn, QED_MSG_DCB, "numtcs = %d\n", *num);
+
+        return rc;
+}
+
+static u8 qed_dcbnl_getpfcstate(struct qed_dev *cdev)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_get *dcbx_info;
+        bool enabled;
+
+        dcbx_info = qed_dcbnl_get_dcbx(hwfn, QED_DCBX_OPERATIONAL_MIB);
+        if (!dcbx_info)
+                return 0;
+
+        enabled = dcbx_info->operational.params.pfc.enabled;
+        DP_VERBOSE(hwfn, QED_MSG_DCB, "pfc state = %d\n", enabled);
+        kfree(dcbx_info);
+
+        return enabled;
+}
+
+static u8 qed_dcbnl_getdcbx(struct qed_dev *cdev)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_get *dcbx_info;
+        u8 mode = 0;
+
+        dcbx_info = qed_dcbnl_get_dcbx(hwfn, QED_DCBX_OPERATIONAL_MIB);
+        if (!dcbx_info)
+                return 0;
+
+        if (dcbx_info->operational.enabled)
+                mode |= DCB_CAP_DCBX_LLD_MANAGED;
+        if (dcbx_info->operational.ieee)
+                mode |= DCB_CAP_DCBX_VER_IEEE;
+        if (dcbx_info->operational.cee)
+                mode |= DCB_CAP_DCBX_VER_CEE;
+
+        DP_VERBOSE(hwfn, QED_MSG_DCB, "dcb mode = %d\n", mode);
+        kfree(dcbx_info);
+
+        return mode;
+}
+
+static void qed_dcbnl_setpgtccfgtx(struct qed_dev *cdev,
+                                   int tc,
+                                   u8 pri_type, u8 pgid, u8 bw_pct, u8 up_map)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_set dcbx_set;
+        struct qed_ptt *ptt;
+        int rc;
+
+        DP_VERBOSE(hwfn, QED_MSG_DCB,
+                   "tc = %d pri_type = %d pgid = %d bw_pct = %d up_map = %d\n",
+                   tc, pri_type, pgid, bw_pct, up_map);
+
+        if (tc < 0 || tc >= QED_MAX_PFC_PRIORITIES) {
+                DP_INFO(hwfn, "Invalid tc %d\n", tc);
+                return;
+        }
+
+        memset(&dcbx_set, 0, sizeof(dcbx_set));
+        rc = qed_dcbx_get_config_params(hwfn, &dcbx_set);
+        if (rc)
+                return;
+
+        dcbx_set.override_flags |= QED_DCBX_OVERRIDE_ETS_CFG;
+        dcbx_set.config.params.ets_pri_tc_tbl[tc] = pgid;
+
+        ptt = qed_ptt_acquire(hwfn);
+        if (!ptt)
+                return;
+
+        rc = qed_dcbx_config_params(hwfn, ptt, &dcbx_set, 0);
+
+        qed_ptt_release(hwfn, ptt);
+}
+
+static void qed_dcbnl_setpgtccfgrx(struct qed_dev *cdev, int prio,
+                                   u8 pri_type, u8 pgid, u8 bw_pct, u8 up_map)
+{
+        DP_INFO(QED_LEADING_HWFN(cdev), "Rx ETS is not supported\n");
+}
+
+static void qed_dcbnl_setpgbwgcfgtx(struct qed_dev *cdev, int pgid, u8 bw_pct)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_set dcbx_set;
+        struct qed_ptt *ptt;
+        int rc;
+
+        DP_VERBOSE(hwfn, QED_MSG_DCB, "pgid = %d bw_pct = %d\n", pgid, bw_pct);
+        if (pgid < 0 || pgid >= QED_MAX_PFC_PRIORITIES) {
+                DP_INFO(hwfn, "Invalid pgid %d\n", pgid);
+                return;
+        }
+
+        memset(&dcbx_set, 0, sizeof(dcbx_set));
+        rc = qed_dcbx_get_config_params(hwfn, &dcbx_set);
+        if (rc)
+                return;
+
+        dcbx_set.override_flags |= QED_DCBX_OVERRIDE_ETS_CFG;
+        dcbx_set.config.params.ets_tc_bw_tbl[pgid] = bw_pct;
+
+        ptt = qed_ptt_acquire(hwfn);
+        if (!ptt)
+                return;
+
+        rc = qed_dcbx_config_params(hwfn, ptt, &dcbx_set, 0);
+
+        qed_ptt_release(hwfn, ptt);
+}
+
+static void qed_dcbnl_setpgbwgcfgrx(struct qed_dev *cdev, int pgid, u8 bw_pct)
+{
+        DP_INFO(QED_LEADING_HWFN(cdev), "Rx ETS is not supported\n");
+}
+
+static u8 qed_dcbnl_setall(struct qed_dev *cdev)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_set dcbx_set;
+        struct qed_ptt *ptt;
+        int rc;
+
+        memset(&dcbx_set, 0, sizeof(dcbx_set));
+        rc = qed_dcbx_get_config_params(hwfn, &dcbx_set);
+        if (rc)
+                return 1;
+
+        ptt = qed_ptt_acquire(hwfn);
+        if (!ptt)
+                return 1;
+
+        rc = qed_dcbx_config_params(hwfn, ptt, &dcbx_set, 1);
+
+        qed_ptt_release(hwfn, ptt);
+
+        return rc;
+}
+
+static int qed_dcbnl_setnumtcs(struct qed_dev *cdev, int tcid, u8 num)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_set dcbx_set;
+        struct qed_ptt *ptt;
+        int rc;
+
+        DP_VERBOSE(hwfn, QED_MSG_DCB, "tcid = %d num = %d\n", tcid, num);
+        memset(&dcbx_set, 0, sizeof(dcbx_set));
+        rc = qed_dcbx_get_config_params(hwfn, &dcbx_set);
+        if (rc)
+                return 1;
+
+        switch (tcid) {
+        case DCB_NUMTCS_ATTR_PG:
+                dcbx_set.override_flags |= QED_DCBX_OVERRIDE_ETS_CFG;
+                dcbx_set.config.params.max_ets_tc = num;
+                break;
+        case DCB_NUMTCS_ATTR_PFC:
+                dcbx_set.override_flags |= QED_DCBX_OVERRIDE_PFC_CFG;
+                dcbx_set.config.params.pfc.max_tc = num;
+                break;
+        default:
+                DP_INFO(hwfn, "Invalid tcid %d\n", tcid);
+                return -EINVAL;
+        }
+
+        ptt = qed_ptt_acquire(hwfn);
+        if (!ptt)
+                return -EINVAL;
+
+        rc = qed_dcbx_config_params(hwfn, ptt, &dcbx_set, 0);
+
+        qed_ptt_release(hwfn, ptt);
+
+        return 0;
+}
+
+static void qed_dcbnl_setpfcstate(struct qed_dev *cdev, u8 state)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_set dcbx_set;
+        struct qed_ptt *ptt;
+        int rc;
+
+        DP_VERBOSE(hwfn, QED_MSG_DCB, "new state = %d\n", state);
+
+        memset(&dcbx_set, 0, sizeof(dcbx_set));
+        rc = qed_dcbx_get_config_params(hwfn, &dcbx_set);
+        if (rc)
+                return;
+
+        dcbx_set.override_flags |= QED_DCBX_OVERRIDE_PFC_CFG;
+        dcbx_set.config.params.pfc.enabled = !!state;
+
+        ptt = qed_ptt_acquire(hwfn);
+        if (!ptt)
+                return;
+
+        rc = qed_dcbx_config_params(hwfn, ptt, &dcbx_set, 0);
+
+        qed_ptt_release(hwfn, ptt);
+}
+
+static int qed_dcbnl_getapp(struct qed_dev *cdev, u8 idtype, u16 idval)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_get *dcbx_info;
+        struct qed_app_entry *entry;
+        bool ethtype;
+        u8 prio = 0;
+        int i;
+
+        dcbx_info = qed_dcbnl_get_dcbx(hwfn, QED_DCBX_OPERATIONAL_MIB);
+        if (!dcbx_info)
+                return -EINVAL;
+
+        ethtype = !!(idtype == DCB_APP_IDTYPE_ETHTYPE);
+        for (i = 0; i < QED_DCBX_MAX_APP_PROTOCOL; i++) {
+                entry = &dcbx_info->operational.params.app_entry[i];
+                if ((entry->ethtype == ethtype) && (entry->proto_id == idval)) {
+                        prio = entry->prio;
+                        break;
+                }
+        }
+
+        if (i == QED_DCBX_MAX_APP_PROTOCOL) {
+                DP_ERR(cdev, "App entry (%d, %d) not found\n", idtype, idval);
+                kfree(dcbx_info);
+                return -EINVAL;
+        }
+
+        kfree(dcbx_info);
+
+        return prio;
+}
+
+static int qed_dcbnl_setapp(struct qed_dev *cdev,
+                            u8 idtype, u16 idval, u8 pri_map)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_set dcbx_set;
+        struct qed_app_entry *entry;
+        struct qed_ptt *ptt;
+        bool ethtype;
+        int rc, i;
+
+        memset(&dcbx_set, 0, sizeof(dcbx_set));
+        rc = qed_dcbx_get_config_params(hwfn, &dcbx_set);
+        if (rc)
+                return -EINVAL;
+
+        ethtype = !!(idtype == DCB_APP_IDTYPE_ETHTYPE);
+        for (i = 0; i < QED_DCBX_MAX_APP_PROTOCOL; i++) {
+                entry = &dcbx_set.config.params.app_entry[i];
+                if ((entry->ethtype == ethtype) && (entry->proto_id == idval))
+                        break;
+                /* First empty slot */
+                if (!entry->proto_id)
+                        break;
+        }
+
+        if (i == QED_DCBX_MAX_APP_PROTOCOL) {
+                DP_ERR(cdev, "App table is full\n");
+                return -EBUSY;
+        }
+
+        dcbx_set.override_flags |= QED_DCBX_OVERRIDE_APP_CFG;
+        dcbx_set.config.params.app_entry[i].ethtype = ethtype;
+        dcbx_set.config.params.app_entry[i].proto_id = idval;
+        dcbx_set.config.params.app_entry[i].prio = pri_map;
+
+        ptt = qed_ptt_acquire(hwfn);
+        if (!ptt)
+                return -EBUSY;
+
+        rc = qed_dcbx_config_params(hwfn, ptt, &dcbx_set, 0);
+
+        qed_ptt_release(hwfn, ptt);
+
+        return rc;
+}
+
+static u8 qed_dcbnl_setdcbx(struct qed_dev *cdev, u8 mode)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_set dcbx_set;
+        struct qed_ptt *ptt;
+        int rc;
+
+        DP_VERBOSE(hwfn, QED_MSG_DCB, "new mode = %x\n", mode);
+
+        if (!(mode & DCB_CAP_DCBX_VER_IEEE) && !(mode & DCB_CAP_DCBX_VER_CEE)) {
+                DP_INFO(hwfn, "Allowed mode is cee, ieee or both\n");
+                return 1;
+        }
+
+        memset(&dcbx_set, 0, sizeof(dcbx_set));
+        rc = qed_dcbx_get_config_params(hwfn, &dcbx_set);
+        if (rc)
+                return 1;
+
+        dcbx_set.ver_num = 0;
+        if (mode & DCB_CAP_DCBX_VER_CEE) {
+                dcbx_set.ver_num |= DCBX_CONFIG_VERSION_CEE;
+                dcbx_set.enabled = true;
+        }
+
+        if (mode & DCB_CAP_DCBX_VER_IEEE) {
+                dcbx_set.ver_num |= DCBX_CONFIG_VERSION_IEEE;
+                dcbx_set.enabled = true;
+        }
+
+        ptt = qed_ptt_acquire(hwfn);
+        if (!ptt)
+                return 1;
+
+        rc = qed_dcbx_config_params(hwfn, ptt, &dcbx_set, 0);
+
+        qed_ptt_release(hwfn, ptt);
+
+        return 0;
+}
+
+static u8 qed_dcbnl_getfeatcfg(struct qed_dev *cdev, int featid, u8 *flags)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_get *dcbx_info;
+
+        DP_VERBOSE(hwfn, QED_MSG_DCB, "Feature id  = %d\n", featid);
+        dcbx_info = qed_dcbnl_get_dcbx(hwfn, QED_DCBX_OPERATIONAL_MIB);
+        if (!dcbx_info)
+                return 1;
+
+        *flags = 0;
+        switch (featid) {
+        case DCB_FEATCFG_ATTR_PG:
+                if (dcbx_info->operational.params.ets_enabled)
+                        *flags = DCB_FEATCFG_ENABLE;
+                else
+                        *flags = DCB_FEATCFG_ERROR;
+                break;
+        case DCB_FEATCFG_ATTR_PFC:
+                if (dcbx_info->operational.params.pfc.enabled)
+                        *flags = DCB_FEATCFG_ENABLE;
+                else
+                        *flags = DCB_FEATCFG_ERROR;
+                break;
+        case DCB_FEATCFG_ATTR_APP:
+                if (dcbx_info->operational.params.app_valid)
+                        *flags = DCB_FEATCFG_ENABLE;
+                else
+                        *flags = DCB_FEATCFG_ERROR;
+                break;
+        default:
+                DP_INFO(hwfn, "Invalid feature-ID %d\n", featid);
+                kfree(dcbx_info);
+                return 1;
+        }
+
+        DP_VERBOSE(hwfn, QED_MSG_DCB, "flags = %d\n", *flags);
+        kfree(dcbx_info);
+
+        return 0;
+}
+
+static u8 qed_dcbnl_setfeatcfg(struct qed_dev *cdev, int featid, u8 flags)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_set dcbx_set;
+        bool enabled, willing;
+        struct qed_ptt *ptt;
+        int rc;
+
+        DP_VERBOSE(hwfn, QED_MSG_DCB, "featid = %d flags = %d\n",
+                   featid, flags);
+        memset(&dcbx_set, 0, sizeof(dcbx_set));
+        rc = qed_dcbx_get_config_params(hwfn, &dcbx_set);
+        if (rc)
+                return 1;
+
+        enabled = !!(flags & DCB_FEATCFG_ENABLE);
+        willing = !!(flags & DCB_FEATCFG_WILLING);
+        switch (featid) {
+        case DCB_FEATCFG_ATTR_PG:
+                dcbx_set.override_flags |= QED_DCBX_OVERRIDE_ETS_CFG;
+                dcbx_set.config.params.ets_enabled = enabled;
+                dcbx_set.config.params.ets_willing = willing;
+                break;
+        case DCB_FEATCFG_ATTR_PFC:
+                dcbx_set.override_flags |= QED_DCBX_OVERRIDE_PFC_CFG;
+                dcbx_set.config.params.pfc.enabled = enabled;
+                dcbx_set.config.params.pfc.willing = willing;
+                break;
+        case DCB_FEATCFG_ATTR_APP:
+                dcbx_set.override_flags |= QED_DCBX_OVERRIDE_APP_CFG;
+                dcbx_set.config.params.app_willing = willing;
+                break;
+        default:
+                DP_INFO(hwfn, "Invalid feature-ID %d\n", featid);
+                return 1;
+        }
+
+        ptt = qed_ptt_acquire(hwfn);
+        if (!ptt)
+                return 1;
+
+        rc = qed_dcbx_config_params(hwfn, ptt, &dcbx_set, 0);
+
+        qed_ptt_release(hwfn, ptt);
+
+        return 0;
+}
+
+static int qed_dcbnl_peer_getappinfo(struct qed_dev *cdev,
+                                     struct dcb_peer_app_info *info,
+                                     u16 *app_count)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_get *dcbx_info;
+
+        dcbx_info = qed_dcbnl_get_dcbx(hwfn, QED_DCBX_REMOTE_MIB);
+        if (!dcbx_info)
+                return -EINVAL;
+
+        info->willing = dcbx_info->remote.params.app_willing;
+        info->error = dcbx_info->remote.params.app_error;
+        *app_count = dcbx_info->remote.params.num_app_entries;
+        kfree(dcbx_info);
+
+        return 0;
+}
+
+static int qed_dcbnl_peer_getapptable(struct qed_dev *cdev,
+                                      struct dcb_app *table)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_get *dcbx_info;
+        int i;
+
+        dcbx_info = qed_dcbnl_get_dcbx(hwfn, QED_DCBX_REMOTE_MIB);
+        if (!dcbx_info)
+                return -EINVAL;
+
+        for (i = 0; i < dcbx_info->remote.params.num_app_entries; i++) {
+                if (dcbx_info->remote.params.app_entry[i].ethtype)
+                        table[i].selector = DCB_APP_IDTYPE_ETHTYPE;
+                else
+                        table[i].selector = DCB_APP_IDTYPE_PORTNUM;
+                table[i].priority = dcbx_info->remote.params.app_entry[i].prio;
+                table[i].protocol =
+                    dcbx_info->remote.params.app_entry[i].proto_id;
+        }
+
+        kfree(dcbx_info);
+
+        return 0;
+}
+
+static int qed_dcbnl_cee_peer_getpfc(struct qed_dev *cdev, struct cee_pfc *pfc)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_get *dcbx_info;
+        int i;
+
+        dcbx_info = qed_dcbnl_get_dcbx(hwfn, QED_DCBX_REMOTE_MIB);
+        if (!dcbx_info)
+                return -EINVAL;
+
+        for (i = 0; i < QED_MAX_PFC_PRIORITIES; i++)
+                if (dcbx_info->remote.params.pfc.prio[i])
+                        pfc->pfc_en |= BIT(i);
+
+        pfc->tcs_supported = dcbx_info->remote.params.pfc.max_tc;
+        DP_VERBOSE(hwfn, QED_MSG_DCB, "pfc state = %d tcs_supported = %d\n",
+                   pfc->pfc_en, pfc->tcs_supported);
+        kfree(dcbx_info);
+
+        return 0;
+}
+
+static int qed_dcbnl_cee_peer_getpg(struct qed_dev *cdev, struct cee_pg *pg)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_get *dcbx_info;
+        int i;
+
+        dcbx_info = qed_dcbnl_get_dcbx(hwfn, QED_DCBX_REMOTE_MIB);
+        if (!dcbx_info)
+                return -EINVAL;
+
+        pg->willing = dcbx_info->remote.params.ets_willing;
+        for (i = 0; i < QED_MAX_PFC_PRIORITIES; i++) {
+                pg->pg_bw[i] = dcbx_info->remote.params.ets_tc_bw_tbl[i];
+                pg->prio_pg[i] = dcbx_info->remote.params.ets_pri_tc_tbl[i];
+        }
+
+        DP_VERBOSE(hwfn, QED_MSG_DCB, "willing = %d", pg->willing);
+        kfree(dcbx_info);
+
+        return 0;
+}
+
+static int qed_dcbnl_get_ieee_pfc(struct qed_dev *cdev,
+                                  struct ieee_pfc *pfc, bool remote)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_params *params;
+        struct qed_dcbx_get *dcbx_info;
+        int rc, i;
+
+        dcbx_info = qed_dcbnl_get_dcbx(hwfn, QED_DCBX_OPERATIONAL_MIB);
+        if (!dcbx_info)
+                return -EINVAL;
+
+        if (!dcbx_info->operational.ieee) {
+                DP_INFO(hwfn, "DCBX is not enabled/operational in IEEE mode\n");
+                return -EINVAL;
+        }
+
+        if (remote) {
+                memset(dcbx_info, 0, sizeof(*dcbx_info));
+                rc = qed_dcbx_query_params(hwfn, dcbx_info,
+                                           QED_DCBX_REMOTE_MIB);
+                if (rc) {
+                        kfree(dcbx_info);
+                        return -EINVAL;
+                }
+
+                params = &dcbx_info->remote.params;
+        } else {
+                params = &dcbx_info->operational.params;
+        }
+
+        pfc->pfc_cap = params->pfc.max_tc;
+        pfc->pfc_en = 0;
+        for (i = 0; i < QED_MAX_PFC_PRIORITIES; i++)
+                if (params->pfc.prio[i])
+                        pfc->pfc_en |= BIT(i);
+
+        kfree(dcbx_info);
+
+        return 0;
+}
+
+static int qed_dcbnl_ieee_getpfc(struct qed_dev *cdev, struct ieee_pfc *pfc)
+{
+        return qed_dcbnl_get_ieee_pfc(cdev, pfc, false);
+}
+
+static int qed_dcbnl_ieee_setpfc(struct qed_dev *cdev, struct ieee_pfc *pfc)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_get *dcbx_info;
+        struct qed_dcbx_set dcbx_set;
+        struct qed_ptt *ptt;
+        int rc, i;
+
+        dcbx_info = qed_dcbnl_get_dcbx(hwfn, QED_DCBX_OPERATIONAL_MIB);
+        if (!dcbx_info)
+                return -EINVAL;
+
+        if (!dcbx_info->operational.ieee) {
+                DP_INFO(hwfn, "DCBX is not enabled/operational in IEEE mode\n");
+                kfree(dcbx_info);
+                return -EINVAL;
+        }
+
+        kfree(dcbx_info);
+
+        memset(&dcbx_set, 0, sizeof(dcbx_set));
+        rc = qed_dcbx_get_config_params(hwfn, &dcbx_set);
+        if (rc)
+                return -EINVAL;
+
+        dcbx_set.override_flags |= QED_DCBX_OVERRIDE_PFC_CFG;
+        for (i = 0; i < QED_MAX_PFC_PRIORITIES; i++)
+                dcbx_set.config.params.pfc.prio[i] = !!(pfc->pfc_en & BIT(i));
+
+        ptt = qed_ptt_acquire(hwfn);
+        if (!ptt)
+                return -EINVAL;
+
+        rc = qed_dcbx_config_params(hwfn, ptt, &dcbx_set, 0);
+
+        qed_ptt_release(hwfn, ptt);
+
+        return rc;
+}
+
+static int qed_dcbnl_get_ieee_ets(struct qed_dev *cdev,
+                                  struct ieee_ets *ets, bool remote)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_get *dcbx_info;
+        struct qed_dcbx_params *params;
+        int rc;
+
+        dcbx_info = qed_dcbnl_get_dcbx(hwfn, QED_DCBX_OPERATIONAL_MIB);
+        if (!dcbx_info)
+                return -EINVAL;
+
+        if (!dcbx_info->operational.ieee) {
+                DP_INFO(hwfn, "DCBX is not enabled/operational in IEEE mode\n");
+                kfree(dcbx_info);
+                return -EINVAL;
+        }
+
+        if (remote) {
+                memset(dcbx_info, 0, sizeof(*dcbx_info));
+                rc = qed_dcbx_query_params(hwfn, dcbx_info,
+                                           QED_DCBX_REMOTE_MIB);
+                if (rc) {
+                        kfree(dcbx_info);
+                        return -EINVAL;
+                }
+
+                params = &dcbx_info->remote.params;
+        } else {
+                params = &dcbx_info->operational.params;
+        }
+
+        ets->ets_cap = params->max_ets_tc;
+        ets->willing = params->ets_willing;
+        ets->cbs = params->ets_cbs;
+        memcpy(ets->tc_tx_bw, params->ets_tc_bw_tbl, sizeof(ets->tc_tx_bw));
+        memcpy(ets->tc_tsa, params->ets_tc_tsa_tbl, sizeof(ets->tc_tsa));
+        memcpy(ets->prio_tc, params->ets_pri_tc_tbl, sizeof(ets->prio_tc));
+        kfree(dcbx_info);
+
+        return 0;
+}
+
+static int qed_dcbnl_ieee_getets(struct qed_dev *cdev, struct ieee_ets *ets)
+{
+        return qed_dcbnl_get_ieee_ets(cdev, ets, false);
+}
+
+static int qed_dcbnl_ieee_setets(struct qed_dev *cdev, struct ieee_ets *ets)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_get *dcbx_info;
+        struct qed_dcbx_set dcbx_set;
+        struct qed_ptt *ptt;
+        int rc;
+
+        dcbx_info = qed_dcbnl_get_dcbx(hwfn, QED_DCBX_OPERATIONAL_MIB);
+        if (!dcbx_info)
+                return -EINVAL;
+
+        if (!dcbx_info->operational.ieee) {
+                DP_INFO(hwfn, "DCBX is not enabled/operational in IEEE mode\n");
+                kfree(dcbx_info);
+                return -EINVAL;
+        }
+
+        kfree(dcbx_info);
+
+        memset(&dcbx_set, 0, sizeof(dcbx_set));
+        rc = qed_dcbx_get_config_params(hwfn, &dcbx_set);
+        if (rc)
+                return -EINVAL;
+
+        dcbx_set.override_flags |= QED_DCBX_OVERRIDE_ETS_CFG;
+        dcbx_set.config.params.max_ets_tc = ets->ets_cap;
+        dcbx_set.config.params.ets_willing = ets->willing;
+        dcbx_set.config.params.ets_cbs = ets->cbs;
+        memcpy(dcbx_set.config.params.ets_tc_bw_tbl, ets->tc_tx_bw,
+               sizeof(ets->tc_tx_bw));
+        memcpy(dcbx_set.config.params.ets_tc_tsa_tbl, ets->tc_tsa,
+               sizeof(ets->tc_tsa));
+        memcpy(dcbx_set.config.params.ets_pri_tc_tbl, ets->prio_tc,
+               sizeof(ets->prio_tc));
+
+        ptt = qed_ptt_acquire(hwfn);
+        if (!ptt)
+                return -EINVAL;
+
+        rc = qed_dcbx_config_params(hwfn, ptt, &dcbx_set, 0);
+
+        qed_ptt_release(hwfn, ptt);
+
+        return rc;
+}
+
+int qed_dcbnl_ieee_peer_getets(struct qed_dev *cdev, struct ieee_ets *ets)
+{
+        return qed_dcbnl_get_ieee_ets(cdev, ets, true);
+}
+
+int qed_dcbnl_ieee_peer_getpfc(struct qed_dev *cdev, struct ieee_pfc *pfc)
+{
+        return qed_dcbnl_get_ieee_pfc(cdev, pfc, true);
+}
+
+int qed_dcbnl_ieee_getapp(struct qed_dev *cdev, struct dcb_app *app)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_get *dcbx_info;
+        struct qed_app_entry *entry;
+        bool ethtype;
+        u8 prio = 0;
+        int i;
+
+        dcbx_info = qed_dcbnl_get_dcbx(hwfn, QED_DCBX_OPERATIONAL_MIB);
+        if (!dcbx_info)
+                return -EINVAL;
+
+        if (!dcbx_info->operational.ieee) {
+                DP_INFO(hwfn, "DCBX is not enabled/operational in IEEE mode\n");
+                kfree(dcbx_info);
+                return -EINVAL;
+        }
+
+        /* ieee defines the selector field value for ethertype to be 1 */
+        ethtype = !!((app->selector - 1) == DCB_APP_IDTYPE_ETHTYPE);
+        for (i = 0; i < QED_DCBX_MAX_APP_PROTOCOL; i++) {
+                entry = &dcbx_info->operational.params.app_entry[i];
+                if ((entry->ethtype == ethtype) &&
+                    (entry->proto_id == app->protocol)) {
+                        prio = entry->prio;
+                        break;
+                }
+        }
+
+        if (i == QED_DCBX_MAX_APP_PROTOCOL) {
+                DP_ERR(cdev, "App entry (%d, %d) not found\n", app->selector,
+                       app->protocol);
+                kfree(dcbx_info);
+                return -EINVAL;
+        }
+
+        app->priority = ffs(prio) - 1;
+
+        kfree(dcbx_info);
+
+        return 0;
+}
+
+int qed_dcbnl_ieee_setapp(struct qed_dev *cdev, struct dcb_app *app)
+{
+        struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+        struct qed_dcbx_get *dcbx_info;
+        struct qed_dcbx_set dcbx_set;
+        struct qed_app_entry *entry;
+        struct qed_ptt *ptt;
+        bool ethtype;
+        int rc, i;
+
+        if (app->priority < 0 || app->priority >= QED_MAX_PFC_PRIORITIES) {
+                DP_INFO(hwfn, "Invalid priority %d\n", app->priority);
+                return -EINVAL;
+        }
+
+        dcbx_info = qed_dcbnl_get_dcbx(hwfn, QED_DCBX_OPERATIONAL_MIB);
+        if (!dcbx_info)
+                return -EINVAL;
+
+        if (!dcbx_info->operational.ieee) {
+                DP_INFO(hwfn, "DCBX is not enabled/operational in IEEE mode\n");
+                kfree(dcbx_info);
+                return -EINVAL;
+        }
+
+        kfree(dcbx_info);
+
+        memset(&dcbx_set, 0, sizeof(dcbx_set));
+        rc = qed_dcbx_get_config_params(hwfn, &dcbx_set);
+        if (rc)
+                return -EINVAL;
+
+        /* ieee defines the selector field value for ethertype to be 1 */
+        ethtype = !!((app->selector - 1) == DCB_APP_IDTYPE_ETHTYPE);
+        for (i = 0; i < QED_DCBX_MAX_APP_PROTOCOL; i++) {
+                entry = &dcbx_set.config.params.app_entry[i];
+                if ((entry->ethtype == ethtype) &&
+                    (entry->proto_id == app->protocol))
+                        break;
+                /* First empty slot */
+                if (!entry->proto_id)
+                        break;
+        }
+
+        if (i == QED_DCBX_MAX_APP_PROTOCOL) {
+                DP_ERR(cdev, "App table is full\n");
+                return -EBUSY;
+        }
+
+        dcbx_set.override_flags |= QED_DCBX_OVERRIDE_APP_CFG;
+        dcbx_set.config.params.app_entry[i].ethtype = ethtype;
+        dcbx_set.config.params.app_entry[i].proto_id = app->protocol;
+        dcbx_set.config.params.app_entry[i].prio = BIT(app->priority);
+
+        ptt = qed_ptt_acquire(hwfn);
+        if (!ptt)
+                return -EBUSY;
+
+        rc = qed_dcbx_config_params(hwfn, ptt, &dcbx_set, 0);
+
+        qed_ptt_release(hwfn, ptt);
+
+        return rc;
+}
+
+const struct qed_eth_dcbnl_ops qed_dcbnl_ops_pass = {
+        .getstate = qed_dcbnl_getstate,
+        .setstate = qed_dcbnl_setstate,
+        .getpgtccfgtx = qed_dcbnl_getpgtccfgtx,
+        .getpgbwgcfgtx = qed_dcbnl_getpgbwgcfgtx,
+        .getpgtccfgrx = qed_dcbnl_getpgtccfgrx,
+        .getpgbwgcfgrx = qed_dcbnl_getpgbwgcfgrx,
+        .getpfccfg = qed_dcbnl_getpfccfg,
+        .setpfccfg = qed_dcbnl_setpfccfg,
+        .getcap = qed_dcbnl_getcap,
+        .getnumtcs = qed_dcbnl_getnumtcs,
+        .getpfcstate = qed_dcbnl_getpfcstate,
+        .getdcbx = qed_dcbnl_getdcbx,
+        .setpgtccfgtx = qed_dcbnl_setpgtccfgtx,
+        .setpgtccfgrx = qed_dcbnl_setpgtccfgrx,
+        .setpgbwgcfgtx = qed_dcbnl_setpgbwgcfgtx,
+        .setpgbwgcfgrx = qed_dcbnl_setpgbwgcfgrx,
+        .setall = qed_dcbnl_setall,
+        .setnumtcs = qed_dcbnl_setnumtcs,
+        .setpfcstate = qed_dcbnl_setpfcstate,
+        .setapp = qed_dcbnl_setapp,
+        .setdcbx = qed_dcbnl_setdcbx,
+        .setfeatcfg = qed_dcbnl_setfeatcfg,
+        .getfeatcfg = qed_dcbnl_getfeatcfg,
+        .getapp = qed_dcbnl_getapp,
+        .peer_getappinfo = qed_dcbnl_peer_getappinfo,
+        .peer_getapptable = qed_dcbnl_peer_getapptable,
+        .cee_peer_getpfc = qed_dcbnl_cee_peer_getpfc,
+        .cee_peer_getpg = qed_dcbnl_cee_peer_getpg,
+        .ieee_getpfc = qed_dcbnl_ieee_getpfc,
+        .ieee_setpfc = qed_dcbnl_ieee_setpfc,
+        .ieee_getets = qed_dcbnl_ieee_getets,
+        .ieee_setets = qed_dcbnl_ieee_setets,
+        .ieee_peer_getpfc = qed_dcbnl_ieee_peer_getpfc,
+        .ieee_peer_getets = qed_dcbnl_ieee_peer_getets,
+        .ieee_getapp = qed_dcbnl_ieee_getapp,
+        .ieee_setapp = qed_dcbnl_ieee_setapp,
+};
+
+#endif
diff --git a/drivers/net/ethernet/qlogic/qed/qed_dcbx.h b/drivers/net/ethernet/qlogic/qed/qed_dcbx.h
index e7f834dbda2d..9ba681643d05 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_dcbx.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_dcbx.h
@@ -33,6 +33,24 @@ struct qed_dcbx_app_data {
         u8 tc;                  /* Traffic Class */
 };
 
+#ifdef CONFIG_DCB
+#define QED_DCBX_VERSION_DISABLED       0
+#define QED_DCBX_VERSION_IEEE           1
+#define QED_DCBX_VERSION_CEE            2
+
+struct qed_dcbx_set {
+#define QED_DCBX_OVERRIDE_STATE         BIT(0)
+#define QED_DCBX_OVERRIDE_PFC_CFG       BIT(1)
+#define QED_DCBX_OVERRIDE_ETS_CFG       BIT(2)
+#define QED_DCBX_OVERRIDE_APP_CFG       BIT(3)
+#define QED_DCBX_OVERRIDE_DSCP_CFG      BIT(4)
+        u32 override_flags;
+        bool enabled;
+        struct qed_dcbx_admin_params config;
+        u32 ver_num;
+};
+#endif
+
 struct qed_dcbx_results {
         bool dcbx_enabled;
         u8 pf_id;
@@ -55,6 +73,9 @@ struct qed_dcbx_info {
         struct qed_dcbx_results results;
         struct dcbx_mib operational;
         struct dcbx_mib remote;
+#ifdef CONFIG_DCB
+        struct qed_dcbx_set set;
+#endif
         u8 dcbx_cap;
 };
 
@@ -67,6 +88,13 @@ struct qed_dcbx_mib_meta_data {
         u32 addr;
 };
 
+#ifdef CONFIG_DCB
+int qed_dcbx_get_config_params(struct qed_hwfn *, struct qed_dcbx_set *);
+
+int qed_dcbx_config_params(struct qed_hwfn *,
+                           struct qed_ptt *, struct qed_dcbx_set *, bool);
+#endif
+
 /* QED local interface routines */
 int
 qed_dcbx_mib_update_event(struct qed_hwfn *,
diff --git a/drivers/net/ethernet/qlogic/qed/qed_dev.c b/drivers/net/ethernet/qlogic/qed/qed_dev.c
index 2d89e8c16b32..b26fe267a150 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_dev.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_dev.c
@@ -17,6 +17,7 @@
 #include <linux/pci.h>
 #include <linux/slab.h>
 #include <linux/string.h>
+#include <linux/vmalloc.h>
 #include <linux/etherdevice.h>
 #include <linux/qed/qed_chain.h>
 #include <linux/qed/qed_if.h>
@@ -160,9 +161,13 @@ static int qed_init_qm_info(struct qed_hwfn *p_hwfn, bool b_sleepable)
         u8 num_vports, vf_offset = 0, i, vport_id, num_ports, curr_queue = 0;
         struct qed_qm_info *qm_info = &p_hwfn->qm_info;
         struct init_qm_port_params *p_qm_port;
+        bool init_rdma_offload_pq = false;
+        bool init_pure_ack_pq = false;
+        bool init_ooo_pq = false;
         u16 num_pqs, multi_cos_tcs = 1;
         u8 pf_wfq = qm_info->pf_wfq;
         u32 pf_rl = qm_info->pf_rl;
+        u16 num_pf_rls = 0;
         u16 num_vfs = 0;
 
 #ifdef CONFIG_QED_SRIOV
@@ -174,6 +179,25 @@ static int qed_init_qm_info(struct qed_hwfn *p_hwfn, bool b_sleepable)
         num_pqs = multi_cos_tcs + num_vfs + 1;  /* The '1' is for pure-LB */
         num_vports = (u8)RESC_NUM(p_hwfn, QED_VPORT);
 
+        if (p_hwfn->hw_info.personality == QED_PCI_ETH_ROCE) {
+                num_pqs++;      /* for RoCE queue */
+                init_rdma_offload_pq = true;
+                /* we subtract num_vfs because each require a rate limiter,
+                 * and one default rate limiter
+                 */
+                if (p_hwfn->pf_params.rdma_pf_params.enable_dcqcn)
+                        num_pf_rls = RESC_NUM(p_hwfn, QED_RL) - num_vfs - 1;
+
+                num_pqs += num_pf_rls;
+                qm_info->num_pf_rls = (u8) num_pf_rls;
+        }
+
+        if (p_hwfn->hw_info.personality == QED_PCI_ISCSI) {
+                num_pqs += 2;   /* for iSCSI pure-ACK / OOO queue */
+                init_pure_ack_pq = true;
+                init_ooo_pq = true;
+        }
+
         /* Sanity checking that setup requires legal number of resources */
         if (num_pqs > RESC_NUM(p_hwfn, QED_PQ)) {
                 DP_ERR(p_hwfn,
@@ -211,12 +235,22 @@ static int qed_init_qm_info(struct qed_hwfn *p_hwfn, bool b_sleepable)
 
         vport_id = (u8)RESC_START(p_hwfn, QED_VPORT);
 
+        /* First init rate limited queues */
+        for (curr_queue = 0; curr_queue < num_pf_rls; curr_queue++) {
+                qm_info->qm_pq_params[curr_queue].vport_id = vport_id++;
+                qm_info->qm_pq_params[curr_queue].tc_id =
+                    p_hwfn->hw_info.non_offload_tc;
+                qm_info->qm_pq_params[curr_queue].wrr_group = 1;
+                qm_info->qm_pq_params[curr_queue].rl_valid = 1;
+        }
+
         /* First init per-TC PQs */
         for (i = 0; i < multi_cos_tcs; i++) {
                 struct init_qm_pq_params *params =
                     &qm_info->qm_pq_params[curr_queue++];
 
-                if (p_hwfn->hw_info.personality == QED_PCI_ETH) {
+                if (p_hwfn->hw_info.personality == QED_PCI_ETH_ROCE ||
+                    p_hwfn->hw_info.personality == QED_PCI_ETH) {
                         params->vport_id = vport_id;
                         params->tc_id = p_hwfn->hw_info.non_offload_tc;
                         params->wrr_group = 1;
@@ -236,6 +270,32 @@ static int qed_init_qm_info(struct qed_hwfn *p_hwfn, bool b_sleepable)
         curr_queue++;
 
         qm_info->offload_pq = 0;
+        if (init_rdma_offload_pq) {
+                qm_info->offload_pq = curr_queue;
+                qm_info->qm_pq_params[curr_queue].vport_id = vport_id;
+                qm_info->qm_pq_params[curr_queue].tc_id =
+                    p_hwfn->hw_info.offload_tc;
+                qm_info->qm_pq_params[curr_queue].wrr_group = 1;
+                curr_queue++;
+        }
+
+        if (init_pure_ack_pq) {
+                qm_info->pure_ack_pq = curr_queue;
+                qm_info->qm_pq_params[curr_queue].vport_id = vport_id;
+                qm_info->qm_pq_params[curr_queue].tc_id =
+                    p_hwfn->hw_info.offload_tc;
+                qm_info->qm_pq_params[curr_queue].wrr_group = 1;
+                curr_queue++;
+        }
+
+        if (init_ooo_pq) {
+                qm_info->ooo_pq = curr_queue;
+                qm_info->qm_pq_params[curr_queue].vport_id = vport_id;
+                qm_info->qm_pq_params[curr_queue].tc_id = DCBX_ISCSI_OOO_TC;
+                qm_info->qm_pq_params[curr_queue].wrr_group = 1;
+                curr_queue++;
+        }
+
         /* Then init per-VF PQs */
         vf_offset = curr_queue;
         for (i = 0; i < num_vfs; i++) {
@@ -244,6 +304,7 @@ static int qed_init_qm_info(struct qed_hwfn *p_hwfn, bool b_sleepable)
                 qm_info->qm_pq_params[curr_queue].tc_id =
                     p_hwfn->hw_info.non_offload_tc;
                 qm_info->qm_pq_params[curr_queue].wrr_group = 1;
+                qm_info->qm_pq_params[curr_queue].rl_valid = 1;
                 curr_queue++;
         }
 
@@ -256,7 +317,10 @@ static int qed_init_qm_info(struct qed_hwfn *p_hwfn, bool b_sleepable)
         for (i = 0; i < num_ports; i++) {
                 p_qm_port = &qm_info->qm_port_params[i];
                 p_qm_port->active = 1;
-                p_qm_port->num_active_phys_tcs = 4;
+                if (num_ports == 4)
+                        p_qm_port->active_phys_tcs = 0x7;
+                else
+                        p_qm_port->active_phys_tcs = 0x9f;
                 p_qm_port->num_pbf_cmd_lines = PBF_MAX_CMD_LINES / num_ports;
                 p_qm_port->num_btb_blocks = BTB_MAX_BLOCKS / num_ports;
         }
@@ -366,21 +430,20 @@ int qed_resc_alloc(struct qed_dev *cdev)
                 if (!p_hwfn->p_tx_cids) {
                         DP_NOTICE(p_hwfn,
                                   "Failed to allocate memory for Tx Cids\n");
-                        rc = -ENOMEM;
-                        goto alloc_err;
+                        goto alloc_no_mem;
                 }
 
                 p_hwfn->p_rx_cids = kzalloc(rx_size, GFP_KERNEL);
                 if (!p_hwfn->p_rx_cids) {
                         DP_NOTICE(p_hwfn,
                                   "Failed to allocate memory for Rx Cids\n");
-                        rc = -ENOMEM;
-                        goto alloc_err;
+                        goto alloc_no_mem;
                 }
         }
 
         for_each_hwfn(cdev, i) {
                 struct qed_hwfn *p_hwfn = &cdev->hwfns[i];
+                u32 n_eqes, num_cons;
 
                 /* First allocate the context manager structure */
                 rc = qed_cxt_mngr_alloc(p_hwfn);
@@ -429,18 +492,34 @@ int qed_resc_alloc(struct qed_dev *cdev)
                         goto alloc_err;
 
                 /* EQ */
-                p_eq = qed_eq_alloc(p_hwfn, 256);
-                if (!p_eq) {
-                        rc = -ENOMEM;
+                n_eqes = qed_chain_get_capacity(&p_hwfn->p_spq->chain);
+                if (p_hwfn->hw_info.personality == QED_PCI_ETH_ROCE) {
+                        num_cons = qed_cxt_get_proto_cid_count(p_hwfn,
+                                                               PROTOCOLID_ROCE,
+                                                               0) * 2;
+                        n_eqes += num_cons + 2 * MAX_NUM_VFS_BB;
+                } else if (p_hwfn->hw_info.personality == QED_PCI_ISCSI) {
+                        num_cons =
+                            qed_cxt_get_proto_cid_count(p_hwfn,
+                                                        PROTOCOLID_ISCSI, 0);
+                        n_eqes += 2 * num_cons;
+                }
+
+                if (n_eqes > 0xFFFF) {
+                        DP_ERR(p_hwfn,
+                               "Cannot allocate 0x%x EQ elements. The maximum of a u16 chain is 0x%x\n",
+                               n_eqes, 0xFFFF);
                         goto alloc_err;
                 }
+
+                p_eq = qed_eq_alloc(p_hwfn, (u16) n_eqes);
+                if (!p_eq)
+                        goto alloc_no_mem;
                 p_hwfn->p_eq = p_eq;
 
                 p_consq = qed_consq_alloc(p_hwfn);
-                if (!p_consq) {
-                        rc = -ENOMEM;
-                        goto alloc_err;
-                }
+                if (!p_consq)
+                        goto alloc_no_mem;
                 p_hwfn->p_consq = p_consq;
 
                 /* DMA info initialization */
@@ -469,6 +548,8 @@ int qed_resc_alloc(struct qed_dev *cdev)
 
         return 0;
 
+alloc_no_mem:
+        rc = -ENOMEM;
 alloc_err:
         qed_resc_free(cdev);
         return rc;
@@ -634,6 +715,7 @@ static int qed_hw_init_common(struct qed_hwfn *p_hwfn,
         struct qed_qm_info *qm_info = &p_hwfn->qm_info;
         struct qed_qm_common_rt_init_params params;
         struct qed_dev *cdev = p_hwfn->cdev;
+        u16 num_pfs, pf_id;
         u32 concrete_fid;
         int rc = 0;
         u8 vf_id;
@@ -682,9 +764,16 @@ static int qed_hw_init_common(struct qed_hwfn *p_hwfn,
         qed_wr(p_hwfn, p_ptt, PSWRQ2_REG_L2P_VALIDATE_VFID, 0);
         qed_wr(p_hwfn, p_ptt, PGLUE_B_REG_USE_CLIENTID_IN_TAG, 1);
 
-        /* Disable relaxed ordering in the PCI config space */
-        qed_wr(p_hwfn, p_ptt, 0x20b4,
-               qed_rd(p_hwfn, p_ptt, 0x20b4) & ~0x10);
+        if (QED_IS_BB(p_hwfn->cdev)) {
+                num_pfs = NUM_OF_ENG_PFS(p_hwfn->cdev);
+                for (pf_id = 0; pf_id < num_pfs; pf_id++) {
+                        qed_fid_pretend(p_hwfn, p_ptt, pf_id);
+                        qed_wr(p_hwfn, p_ptt, PRS_REG_SEARCH_ROCE, 0x0);
+                        qed_wr(p_hwfn, p_ptt, PRS_REG_SEARCH_TCP, 0x0);
+                }
+                /* pretend to original PF */
+                qed_fid_pretend(p_hwfn, p_ptt, p_hwfn->rel_pf_id);
+        }
 
         for (vf_id = 0; vf_id < MAX_NUM_VFS_BB; vf_id++) {
                 concrete_fid = qed_vfid_to_concrete(p_hwfn, vf_id);
@@ -703,8 +792,31 @@ static int qed_hw_init_port(struct qed_hwfn *p_hwfn,
 {
         int rc = 0;
 
-        rc = qed_init_run(p_hwfn, p_ptt, PHASE_PORT, p_hwfn->port_id,
-                          hw_mode);
+        rc = qed_init_run(p_hwfn, p_ptt, PHASE_PORT, p_hwfn->port_id, hw_mode);
+        if (rc != 0)
+                return rc;
+
+        if (hw_mode & (1 << MODE_MF_SI)) {
+                u8 pf_id = 0;
+
+                if (!qed_hw_init_first_eth(p_hwfn, p_ptt, &pf_id)) {
+                        DP_VERBOSE(p_hwfn, NETIF_MSG_IFUP,
+                                   "PF[%08x] is first eth on engine\n", pf_id);
+
+                        /* We should have configured BIT for ppfid, i.e., the
+                         * relative function number in the port. But there's a
+                         * bug in LLH in BB where the ppfid is actually engine
+                         * based, so we need to take this into account.
+                         */
+                        qed_wr(p_hwfn, p_ptt,
+                               NIG_REG_LLH_TAGMAC_DEF_PF_VECTOR, 1 << pf_id);
+                }
+
+                /* Take the protocol-based hit vector if there is a hit,
+                 * otherwise take the other vector.
+                 */
+                qed_wr(p_hwfn, p_ptt, NIG_REG_LLH_CLS_TYPE_DUALMODE, 0x2);
+        }
         return rc;
 }
 
@@ -751,7 +863,8 @@ static int qed_hw_init_pf(struct qed_hwfn *p_hwfn,
         }
 
         /* Protocl Configuration  */
-        STORE_RT_REG(p_hwfn, PRS_REG_SEARCH_TCP_RT_OFFSET, 0);
+        STORE_RT_REG(p_hwfn, PRS_REG_SEARCH_TCP_RT_OFFSET,
+                     (p_hwfn->hw_info.personality == QED_PCI_ISCSI) ? 1 : 0);
         STORE_RT_REG(p_hwfn, PRS_REG_SEARCH_FCOE_RT_OFFSET, 0);
         STORE_RT_REG(p_hwfn, PRS_REG_SEARCH_ROCE_RT_OFFSET, 0);
 
@@ -773,6 +886,21 @@ static int qed_hw_init_pf(struct qed_hwfn *p_hwfn,
         /* Pure runtime initializations - directly to the HW  */
         qed_int_igu_init_pure_rt(p_hwfn, p_ptt, true, true);
 
+        if (hw_mode & (1 << MODE_MF_SI)) {
+                u8 pf_id = 0;
+                u32 val;
+
+                if (!qed_hw_init_first_eth(p_hwfn, p_ptt, &pf_id)) {
+                        if (p_hwfn->rel_pf_id == pf_id) {
+                                DP_VERBOSE(p_hwfn, NETIF_MSG_IFUP,
+                                           "PF[%d] is first ETH on engine\n",
+                                           pf_id);
+                                val = 1;
+                        }
+                        qed_wr(p_hwfn, p_ptt, PRS_REG_MSG_INFO, val);
+                }
+        }
+
         if (b_hw_start) {
                 /* enable interrupts */
                 qed_int_igu_enable(p_hwfn, p_ptt, int_mode);
@@ -1213,8 +1341,9 @@ static void qed_hw_set_feat(struct qed_hwfn *p_hwfn)
                    num_features);
 }
 
-static void qed_hw_get_resc(struct qed_hwfn *p_hwfn)
+static int qed_hw_get_resc(struct qed_hwfn *p_hwfn)
 {
+        u8 enabled_func_idx = p_hwfn->enabled_func_idx;
         u32 *resc_start = p_hwfn->hw_info.resc_start;
         u8 num_funcs = p_hwfn->num_funcs_on_engine;
         u32 *resc_num = p_hwfn->hw_info.resc_num;
@@ -1238,14 +1367,22 @@ static void qed_hw_get_resc(struct qed_hwfn *p_hwfn)
         resc_num[QED_VPORT] = MAX_NUM_VPORTS_BB / num_funcs;
         resc_num[QED_RSS_ENG] = ETH_RSS_ENGINE_NUM_BB / num_funcs;
         resc_num[QED_PQ] = MAX_QM_TX_QUEUES_BB / num_funcs;
-        resc_num[QED_RL] = 8;
+        resc_num[QED_RL] = min_t(u32, 64, resc_num[QED_VPORT]);
         resc_num[QED_MAC] = ETH_NUM_MAC_FILTERS / num_funcs;
         resc_num[QED_VLAN] = (ETH_NUM_VLAN_FILTERS - 1 /*For vlan0*/) /
                              num_funcs;
-        resc_num[QED_ILT] = 950;
+        resc_num[QED_ILT] = PXP_NUM_ILT_RECORDS_BB / num_funcs;
 
         for (i = 0; i < QED_MAX_RESC; i++)
-                resc_start[i] = resc_num[i] * p_hwfn->rel_pf_id;
+                resc_start[i] = resc_num[i] * enabled_func_idx;
+
+        /* Sanity for ILT */
+        if (RESC_END(p_hwfn, QED_ILT) > PXP_NUM_ILT_RECORDS_BB) {
+                DP_NOTICE(p_hwfn, "Can't assign ILT pages [%08x,...,%08x]\n",
+                          RESC_START(p_hwfn, QED_ILT),
+                          RESC_END(p_hwfn, QED_ILT) - 1);
+                return -EINVAL;
+        }
 
         qed_hw_set_feat(p_hwfn);
 
@@ -1275,6 +1412,8 @@ static void qed_hw_get_resc(struct qed_hwfn *p_hwfn)
                    p_hwfn->hw_info.resc_start[QED_VLAN],
                    p_hwfn->hw_info.resc_num[QED_ILT],
                    p_hwfn->hw_info.resc_start[QED_ILT]);
+
+        return 0;
 }
 
 static int qed_hw_get_nvm_info(struct qed_hwfn *p_hwfn,
@@ -1304,31 +1443,31 @@ static int qed_hw_get_nvm_info(struct qed_hwfn *p_hwfn,
 
         switch ((core_cfg & NVM_CFG1_GLOB_NETWORK_PORT_MODE_MASK) >>
                 NVM_CFG1_GLOB_NETWORK_PORT_MODE_OFFSET) {
-        case NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_2X40G:
+        case NVM_CFG1_GLOB_NETWORK_PORT_MODE_BB_2X40G:
                 p_hwfn->hw_info.port_mode = QED_PORT_MODE_DE_2X40G;
                 break;
-        case NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_2X50G:
+        case NVM_CFG1_GLOB_NETWORK_PORT_MODE_2X50G:
                 p_hwfn->hw_info.port_mode = QED_PORT_MODE_DE_2X50G;
                 break;
-        case NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_1X100G:
+        case NVM_CFG1_GLOB_NETWORK_PORT_MODE_BB_1X100G:
                 p_hwfn->hw_info.port_mode = QED_PORT_MODE_DE_1X100G;
                 break;
-        case NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_4X10G_F:
+        case NVM_CFG1_GLOB_NETWORK_PORT_MODE_4X10G_F:
                 p_hwfn->hw_info.port_mode = QED_PORT_MODE_DE_4X10G_F;
                 break;
-        case NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_4X10G_E:
+        case NVM_CFG1_GLOB_NETWORK_PORT_MODE_BB_4X10G_E:
                 p_hwfn->hw_info.port_mode = QED_PORT_MODE_DE_4X10G_E;
                 break;
-        case NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_4X20G:
+        case NVM_CFG1_GLOB_NETWORK_PORT_MODE_BB_4X20G:
                 p_hwfn->hw_info.port_mode = QED_PORT_MODE_DE_4X20G;
                 break;
-        case NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_1X40G:
+        case NVM_CFG1_GLOB_NETWORK_PORT_MODE_1X40G:
                 p_hwfn->hw_info.port_mode = QED_PORT_MODE_DE_1X40G;
                 break;
-        case NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_2X25G:
+        case NVM_CFG1_GLOB_NETWORK_PORT_MODE_2X25G:
                 p_hwfn->hw_info.port_mode = QED_PORT_MODE_DE_2X25G;
                 break;
-        case NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_1X25G:
+        case NVM_CFG1_GLOB_NETWORK_PORT_MODE_1X25G:
                 p_hwfn->hw_info.port_mode = QED_PORT_MODE_DE_1X25G;
                 break;
         default:
@@ -1373,7 +1512,7 @@ static int qed_hw_get_nvm_info(struct qed_hwfn *p_hwfn,
         case NVM_CFG1_PORT_DRV_LINK_SPEED_50G:
                 link->speed.forced_speed = 50000;
                 break;
-        case NVM_CFG1_PORT_DRV_LINK_SPEED_100G:
+        case NVM_CFG1_PORT_DRV_LINK_SPEED_BB_100G:
                 link->speed.forced_speed = 100000;
                 break;
         default:
@@ -1429,14 +1568,20 @@ static int qed_hw_get_nvm_info(struct qed_hwfn *p_hwfn,
         if (device_capabilities & NVM_CFG1_GLOB_DEVICE_CAPABILITIES_ETHERNET)
                 __set_bit(QED_DEV_CAP_ETH,
                           &p_hwfn->hw_info.device_capabilities);
+        if (device_capabilities & NVM_CFG1_GLOB_DEVICE_CAPABILITIES_ISCSI)
+                __set_bit(QED_DEV_CAP_ISCSI,
+                          &p_hwfn->hw_info.device_capabilities);
+        if (device_capabilities & NVM_CFG1_GLOB_DEVICE_CAPABILITIES_ROCE)
+                __set_bit(QED_DEV_CAP_ROCE,
+                          &p_hwfn->hw_info.device_capabilities);
 
         return qed_mcp_fill_shmem_func_info(p_hwfn, p_ptt);
 }
 
 static void qed_get_num_funcs(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 {
-        u32 reg_function_hide, tmp, eng_mask;
-        u8 num_funcs;
+        u8 num_funcs, enabled_func_idx = p_hwfn->rel_pf_id;
+        u32 reg_function_hide, tmp, eng_mask, low_pfs_mask;
 
         num_funcs = MAX_NUM_PFS_BB;
 
@@ -1466,9 +1611,19 @@ static void qed_get_num_funcs(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
                                 num_funcs++;
                         tmp >>= 0x1;
                 }
+
+                /* Get the PF index within the enabled functions */
+                low_pfs_mask = (0x1 << p_hwfn->abs_pf_id) - 1;
+                tmp = reg_function_hide & eng_mask & low_pfs_mask;
+                while (tmp) {
+                        if (tmp & 0x1)
+                                enabled_func_idx--;
+                        tmp >>= 0x1;
+                }
         }
 
         p_hwfn->num_funcs_on_engine = num_funcs;
+        p_hwfn->enabled_func_idx = enabled_func_idx;
 
         DP_VERBOSE(p_hwfn,
                    NETIF_MSG_PROBE,
@@ -1538,9 +1693,7 @@ qed_get_hw_info(struct qed_hwfn *p_hwfn,
 
         qed_get_num_funcs(p_hwfn, p_ptt);
 
-        qed_hw_get_resc(p_hwfn);
-
-        return rc;
+        return qed_hw_get_resc(p_hwfn);
 }
 
 static int qed_get_dev_info(struct qed_dev *cdev)
@@ -1737,92 +1890,285 @@ void qed_hw_remove(struct qed_dev *cdev)
         qed_iov_free_hw_info(cdev);
 }
 
-int qed_chain_alloc(struct qed_dev *cdev,
-                    enum qed_chain_use_mode intended_use,
-                    enum qed_chain_mode mode,
-                    u16 num_elems,
-                    size_t elem_size,
-                    struct qed_chain *p_chain)
+static void qed_chain_free_next_ptr(struct qed_dev *cdev,
+                                    struct qed_chain *p_chain)
+{
+        void *p_virt = p_chain->p_virt_addr, *p_virt_next = NULL;
+        dma_addr_t p_phys = p_chain->p_phys_addr, p_phys_next = 0;
+        struct qed_chain_next *p_next;
+        u32 size, i;
+
+        if (!p_virt)
+                return;
+
+        size = p_chain->elem_size * p_chain->usable_per_page;
+
+        for (i = 0; i < p_chain->page_cnt; i++) {
+                if (!p_virt)
+                        break;
+
+                p_next = (struct qed_chain_next *)((u8 *)p_virt + size);
+                p_virt_next = p_next->next_virt;
+                p_phys_next = HILO_DMA_REGPAIR(p_next->next_phys);
+
+                dma_free_coherent(&cdev->pdev->dev,
+                                  QED_CHAIN_PAGE_SIZE, p_virt, p_phys);
+
+                p_virt = p_virt_next;
+                p_phys = p_phys_next;
+        }
+}
+
+static void qed_chain_free_single(struct qed_dev *cdev,
+                                  struct qed_chain *p_chain)
+{
+        if (!p_chain->p_virt_addr)
+                return;
+
+        dma_free_coherent(&cdev->pdev->dev,
+                          QED_CHAIN_PAGE_SIZE,
+                          p_chain->p_virt_addr, p_chain->p_phys_addr);
+}
+
+static void qed_chain_free_pbl(struct qed_dev *cdev, struct qed_chain *p_chain)
+{
+        void **pp_virt_addr_tbl = p_chain->pbl.pp_virt_addr_tbl;
+        u32 page_cnt = p_chain->page_cnt, i, pbl_size;
+        u8 *p_pbl_virt = p_chain->pbl.p_virt_table;
+
+        if (!pp_virt_addr_tbl)
+                return;
+
+        if (!p_chain->pbl.p_virt_table)
+                goto out;
+
+        for (i = 0; i < page_cnt; i++) {
+                if (!pp_virt_addr_tbl[i])
+                        break;
+
+                dma_free_coherent(&cdev->pdev->dev,
+                                  QED_CHAIN_PAGE_SIZE,
+                                  pp_virt_addr_tbl[i],
+                                  *(dma_addr_t *)p_pbl_virt);
+
+                p_pbl_virt += QED_CHAIN_PBL_ENTRY_SIZE;
+        }
+
+        pbl_size = page_cnt * QED_CHAIN_PBL_ENTRY_SIZE;
+        dma_free_coherent(&cdev->pdev->dev,
+                          pbl_size,
+                          p_chain->pbl.p_virt_table, p_chain->pbl.p_phys_table);
+out:
+        vfree(p_chain->pbl.pp_virt_addr_tbl);
+}
+
+void qed_chain_free(struct qed_dev *cdev, struct qed_chain *p_chain)
 {
-        dma_addr_t p_pbl_phys = 0;
-        void *p_pbl_virt = NULL;
+        switch (p_chain->mode) {
+        case QED_CHAIN_MODE_NEXT_PTR:
+                qed_chain_free_next_ptr(cdev, p_chain);
+                break;
+        case QED_CHAIN_MODE_SINGLE:
+                qed_chain_free_single(cdev, p_chain);
+                break;
+        case QED_CHAIN_MODE_PBL:
+                qed_chain_free_pbl(cdev, p_chain);
+                break;
+        }
+}
+
+static int
+qed_chain_alloc_sanity_check(struct qed_dev *cdev,
+                             enum qed_chain_cnt_type cnt_type,
+                             size_t elem_size, u32 page_cnt)
+{
+        u64 chain_size = ELEMS_PER_PAGE(elem_size) * page_cnt;
+
+        /* The actual chain size can be larger than the maximal possible value
+         * after rounding up the requested elements number to pages, and after
+         * taking into acount the unusuable elements (next-ptr elements).
+         * The size of a "u16" chain can be (U16_MAX + 1) since the chain
+         * size/capacity fields are of a u32 type.
+         */
+        if ((cnt_type == QED_CHAIN_CNT_TYPE_U16 &&
+             chain_size > 0x10000) ||
+            (cnt_type == QED_CHAIN_CNT_TYPE_U32 &&
+             chain_size > 0x100000000ULL)) {
+                DP_NOTICE(cdev,
+                          "The actual chain size (0x%llx) is larger than the maximal possible value\n",
+                          chain_size);
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
+static int
+qed_chain_alloc_next_ptr(struct qed_dev *cdev, struct qed_chain *p_chain)
+{
+        void *p_virt = NULL, *p_virt_prev = NULL;
         dma_addr_t p_phys = 0;
-        void *p_virt = NULL;
-        u16 page_cnt = 0;
-        size_t size;
+        u32 i;
 
-        if (mode == QED_CHAIN_MODE_SINGLE)
-                page_cnt = 1;
-        else
-                page_cnt = QED_CHAIN_PAGE_CNT(num_elems, elem_size, mode);
+        for (i = 0; i < p_chain->page_cnt; i++) {
+                p_virt = dma_alloc_coherent(&cdev->pdev->dev,
+                                            QED_CHAIN_PAGE_SIZE,
+                                            &p_phys, GFP_KERNEL);
+                if (!p_virt) {
+                        DP_NOTICE(cdev, "Failed to allocate chain memory\n");
+                        return -ENOMEM;
+                }
+
+                if (i == 0) {
+                        qed_chain_init_mem(p_chain, p_virt, p_phys);
+                        qed_chain_reset(p_chain);
+                } else {
+                        qed_chain_init_next_ptr_elem(p_chain, p_virt_prev,
+                                                     p_virt, p_phys);
+                }
+
+                p_virt_prev = p_virt;
+        }
+        /* Last page's next element should point to the beginning of the
+         * chain.
+         */
+        qed_chain_init_next_ptr_elem(p_chain, p_virt_prev,
+                                     p_chain->p_virt_addr,
+                                     p_chain->p_phys_addr);
+
+        return 0;
+}
+
+static int
+qed_chain_alloc_single(struct qed_dev *cdev, struct qed_chain *p_chain)
+{
+        dma_addr_t p_phys = 0;
+        void *p_virt = NULL;
 
-        size = page_cnt * QED_CHAIN_PAGE_SIZE;
         p_virt = dma_alloc_coherent(&cdev->pdev->dev,
-                                    size, &p_phys, GFP_KERNEL);
+                                    QED_CHAIN_PAGE_SIZE, &p_phys, GFP_KERNEL);
         if (!p_virt) {
-                DP_NOTICE(cdev, "Failed to allocate chain mem\n");
-                goto nomem;
+                DP_NOTICE(cdev, "Failed to allocate chain memory\n");
+                return -ENOMEM;
         }
 
-        if (mode == QED_CHAIN_MODE_PBL) {
-                size = page_cnt * QED_CHAIN_PBL_ENTRY_SIZE;
-                p_pbl_virt = dma_alloc_coherent(&cdev->pdev->dev,
-                                                size, &p_pbl_phys,
-                                                GFP_KERNEL);
-                if (!p_pbl_virt) {
-                        DP_NOTICE(cdev, "Failed to allocate chain pbl mem\n");
-                        goto nomem;
-                }
+        qed_chain_init_mem(p_chain, p_virt, p_phys);
+        qed_chain_reset(p_chain);
 
-                qed_chain_pbl_init(p_chain, p_virt, p_phys, page_cnt,
-                                   (u8)elem_size, intended_use,
-                                   p_pbl_phys, p_pbl_virt);
-        } else {
-                qed_chain_init(p_chain, p_virt, p_phys, page_cnt,
-                               (u8)elem_size, intended_use, mode);
+        return 0;
+}
+
+static int qed_chain_alloc_pbl(struct qed_dev *cdev, struct qed_chain *p_chain)
+{
+        u32 page_cnt = p_chain->page_cnt, size, i;
+        dma_addr_t p_phys = 0, p_pbl_phys = 0;
+        void **pp_virt_addr_tbl = NULL;
+        u8 *p_pbl_virt = NULL;
+        void *p_virt = NULL;
+
+        size = page_cnt * sizeof(*pp_virt_addr_tbl);
+        pp_virt_addr_tbl = vmalloc(size);
+        if (!pp_virt_addr_tbl) {
+                DP_NOTICE(cdev,
+                          "Failed to allocate memory for the chain virtual addresses table\n");
+                return -ENOMEM;
         }
+        memset(pp_virt_addr_tbl, 0, size);
 
-        return 0;
+        /* The allocation of the PBL table is done with its full size, since it
+         * is expected to be successive.
+         * qed_chain_init_pbl_mem() is called even in a case of an allocation
+         * failure, since pp_virt_addr_tbl was previously allocated, and it
+         * should be saved to allow its freeing during the error flow.
+         */
+        size = page_cnt * QED_CHAIN_PBL_ENTRY_SIZE;
+        p_pbl_virt = dma_alloc_coherent(&cdev->pdev->dev,
+                                        size, &p_pbl_phys, GFP_KERNEL);
+        qed_chain_init_pbl_mem(p_chain, p_pbl_virt, p_pbl_phys,
+                               pp_virt_addr_tbl);
+        if (!p_pbl_virt) {
+                DP_NOTICE(cdev, "Failed to allocate chain pbl memory\n");
+                return -ENOMEM;
+        }
 
-nomem:
-        dma_free_coherent(&cdev->pdev->dev,
-                          page_cnt * QED_CHAIN_PAGE_SIZE,
-                          p_virt, p_phys);
-        dma_free_coherent(&cdev->pdev->dev,
-                          page_cnt * QED_CHAIN_PBL_ENTRY_SIZE,
-                          p_pbl_virt, p_pbl_phys);
+        for (i = 0; i < page_cnt; i++) {
+                p_virt = dma_alloc_coherent(&cdev->pdev->dev,
+                                            QED_CHAIN_PAGE_SIZE,
+                                            &p_phys, GFP_KERNEL);
+                if (!p_virt) {
+                        DP_NOTICE(cdev, "Failed to allocate chain memory\n");
+                        return -ENOMEM;
+                }
 
-        return -ENOMEM;
+                if (i == 0) {
+                        qed_chain_init_mem(p_chain, p_virt, p_phys);
+                        qed_chain_reset(p_chain);
+                }
+
+                /* Fill the PBL table with the physical address of the page */
+                *(dma_addr_t *)p_pbl_virt = p_phys;
+                /* Keep the virtual address of the page */
+                p_chain->pbl.pp_virt_addr_tbl[i] = p_virt;
+
+                p_pbl_virt += QED_CHAIN_PBL_ENTRY_SIZE;
+        }
+
+        return 0;
 }
 
-void qed_chain_free(struct qed_dev *cdev,
-                    struct qed_chain *p_chain)
+int qed_chain_alloc(struct qed_dev *cdev,
+                    enum qed_chain_use_mode intended_use,
+                    enum qed_chain_mode mode,
+                    enum qed_chain_cnt_type cnt_type,
+                    u32 num_elems, size_t elem_size, struct qed_chain *p_chain)
 {
-        size_t size;
+        u32 page_cnt;
+        int rc = 0;
 
-        if (!p_chain->p_virt_addr)
-                return;
+        if (mode == QED_CHAIN_MODE_SINGLE)
+                page_cnt = 1;
+        else
+                page_cnt = QED_CHAIN_PAGE_CNT(num_elems, elem_size, mode);
+
+        rc = qed_chain_alloc_sanity_check(cdev, cnt_type, elem_size, page_cnt);
+        if (rc) {
+                DP_NOTICE(cdev,
+                          "Cannot allocate a chain with the given arguments:\n"
+                          "[use_mode %d, mode %d, cnt_type %d, num_elems %d, elem_size %zu]\n",
+                          intended_use, mode, cnt_type, num_elems, elem_size);
+                return rc;
+        }
+
+        qed_chain_init_params(p_chain, page_cnt, (u8) elem_size, intended_use,
+                              mode, cnt_type);
 
-        if (p_chain->mode == QED_CHAIN_MODE_PBL) {
-                size = p_chain->page_cnt * QED_CHAIN_PBL_ENTRY_SIZE;
-                dma_free_coherent(&cdev->pdev->dev, size,
-                                  p_chain->pbl.p_virt_table,
-                                  p_chain->pbl.p_phys_table);
+        switch (mode) {
+        case QED_CHAIN_MODE_NEXT_PTR:
+                rc = qed_chain_alloc_next_ptr(cdev, p_chain);
+                break;
+        case QED_CHAIN_MODE_SINGLE:
+                rc = qed_chain_alloc_single(cdev, p_chain);
+                break;
+        case QED_CHAIN_MODE_PBL:
+                rc = qed_chain_alloc_pbl(cdev, p_chain);
+                break;
         }
+        if (rc)
+                goto nomem;
+
+        return 0;
 
-        size = p_chain->page_cnt * QED_CHAIN_PAGE_SIZE;
-        dma_free_coherent(&cdev->pdev->dev, size,
-                          p_chain->p_virt_addr,
-                          p_chain->p_phys_addr);
+nomem:
+        qed_chain_free(cdev, p_chain);
+        return rc;
 }
 
-int qed_fw_l2_queue(struct qed_hwfn *p_hwfn,
-                    u16 src_id, u16 *dst_id)
+int qed_fw_l2_queue(struct qed_hwfn *p_hwfn, u16 src_id, u16 *dst_id)
 {
         if (src_id >= RESC_NUM(p_hwfn, QED_L2_QUEUE)) {
                 u16 min, max;
 
-                min = (u16)RESC_START(p_hwfn, QED_L2_QUEUE);
+                min = (u16) RESC_START(p_hwfn, QED_L2_QUEUE);
                 max = min + RESC_NUM(p_hwfn, QED_L2_QUEUE);
                 DP_NOTICE(p_hwfn,
                           "l2_queue id [%d] is not valid, available indices [%d - %d]\n",
@@ -1876,6 +2222,110 @@ int qed_fw_rss_eng(struct qed_hwfn *p_hwfn,
         return 0;
 }
 
+static int qed_set_coalesce(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
+                            u32 hw_addr, void *p_eth_qzone,
+                            size_t eth_qzone_size, u8 timeset)
+{
+        struct coalescing_timeset *p_coal_timeset;
+
+        if (p_hwfn->cdev->int_coalescing_mode != QED_COAL_MODE_ENABLE) {
+                DP_NOTICE(p_hwfn, "Coalescing configuration not enabled\n");
+                return -EINVAL;
+        }
+
+        p_coal_timeset = p_eth_qzone;
+        memset(p_coal_timeset, 0, eth_qzone_size);
+        SET_FIELD(p_coal_timeset->value, COALESCING_TIMESET_TIMESET, timeset);
+        SET_FIELD(p_coal_timeset->value, COALESCING_TIMESET_VALID, 1);
+        qed_memcpy_to(p_hwfn, p_ptt, hw_addr, p_eth_qzone, eth_qzone_size);
+
+        return 0;
+}
+
+int qed_set_rxq_coalesce(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
+                         u16 coalesce, u8 qid, u16 sb_id)
+{
+        struct ustorm_eth_queue_zone eth_qzone;
+        u8 timeset, timer_res;
+        u16 fw_qid = 0;
+        u32 address;
+        int rc;
+
+        /* Coalesce = (timeset << timer-resolution), timeset is 7bit wide */
+        if (coalesce <= 0x7F) {
+                timer_res = 0;
+        } else if (coalesce <= 0xFF) {
+                timer_res = 1;
+        } else if (coalesce <= 0x1FF) {
+                timer_res = 2;
+        } else {
+                DP_ERR(p_hwfn, "Invalid coalesce value - %d\n", coalesce);
+                return -EINVAL;
+        }
+        timeset = (u8)(coalesce >> timer_res);
+
+        rc = qed_fw_l2_queue(p_hwfn, (u16)qid, &fw_qid);
+        if (rc)
+                return rc;
+
+        rc = qed_int_set_timer_res(p_hwfn, p_ptt, timer_res, sb_id, false);
+        if (rc)
+                goto out;
+
+        address = BAR0_MAP_REG_USDM_RAM + USTORM_ETH_QUEUE_ZONE_OFFSET(fw_qid);
+
+        rc = qed_set_coalesce(p_hwfn, p_ptt, address, &eth_qzone,
+                              sizeof(struct ustorm_eth_queue_zone), timeset);
+        if (rc)
+                goto out;
+
+        p_hwfn->cdev->rx_coalesce_usecs = coalesce;
+out:
+        return rc;
+}
+
+int qed_set_txq_coalesce(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
+                         u16 coalesce, u8 qid, u16 sb_id)
+{
+        struct xstorm_eth_queue_zone eth_qzone;
+        u8 timeset, timer_res;
+        u16 fw_qid = 0;
+        u32 address;
+        int rc;
+
+        /* Coalesce = (timeset << timer-resolution), timeset is 7bit wide */
+        if (coalesce <= 0x7F) {
+                timer_res = 0;
+        } else if (coalesce <= 0xFF) {
+                timer_res = 1;
+        } else if (coalesce <= 0x1FF) {
+                timer_res = 2;
+        } else {
+                DP_ERR(p_hwfn, "Invalid coalesce value - %d\n", coalesce);
+                return -EINVAL;
+        }
+        timeset = (u8)(coalesce >> timer_res);
+
+        rc = qed_fw_l2_queue(p_hwfn, (u16)qid, &fw_qid);
+        if (rc)
+                return rc;
+
+        rc = qed_int_set_timer_res(p_hwfn, p_ptt, timer_res, sb_id, true);
+        if (rc)
+                goto out;
+
+        address = BAR0_MAP_REG_XSDM_RAM + XSTORM_ETH_QUEUE_ZONE_OFFSET(fw_qid);
+
+        rc = qed_set_coalesce(p_hwfn, p_ptt, address, &eth_qzone,
+                              sizeof(struct xstorm_eth_queue_zone), timeset);
+        if (rc)
+                goto out;
+
+        p_hwfn->cdev->tx_coalesce_usecs = coalesce;
+out:
+        return rc;
+}
+
 /* Calculate final WFQ values for all vports and configure them.
  * After this configuration each vport will have
  * approx min rate =  min_pf_rate * (vport_wfq / QED_WFQ_UNIT)
diff --git a/drivers/net/ethernet/qlogic/qed/qed_dev_api.h b/drivers/net/ethernet/qlogic/qed/qed_dev_api.h
index dde364d6f502..343bb0344f62 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_dev_api.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_dev_api.h
@@ -212,6 +212,20 @@ qed_dmae_host2grc(struct qed_hwfn *p_hwfn,
                   u32 size_in_dwords,
                   u32 flags);
 
+ /**
+ * @brief qed_dmae_grc2host - Read data from dmae data offset
+ * to source address using the given ptt
+ *
+ * @param p_ptt
+ * @param grc_addr (dmae_data_offset)
+ * @param dest_addr
+ * @param size_in_dwords
+ * @param flags - one of the flags defined above
+ */
+int qed_dmae_grc2host(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
+                      u32 grc_addr, dma_addr_t dest_addr, u32 size_in_dwords,
+                      u32 flags);
+
 /**
  * @brief qed_dmae_host2host - copy data from to source address
  * to a destination adress (for SRIOV) using the given ptt
@@ -245,9 +259,8 @@ int
 qed_chain_alloc(struct qed_dev *cdev,
                 enum qed_chain_use_mode intended_use,
                 enum qed_chain_mode mode,
-                u16 num_elems,
-                size_t elem_size,
-                struct qed_chain *p_chain);
+                enum qed_chain_cnt_type cnt_type,
+                u32 num_elems, size_t elem_size, struct qed_chain *p_chain);
 
 /**
  * @brief qed_chain_free - Free chain DMA memory
@@ -255,8 +268,7 @@ qed_chain_alloc(struct qed_dev *cdev,
  * @param p_hwfn
  * @param p_chain
  */
-void qed_chain_free(struct qed_dev *cdev,
-                    struct qed_chain *p_chain);
+void qed_chain_free(struct qed_dev *cdev, struct qed_chain *p_chain);
 
 /**
  * @@brief qed_fw_l2_queue - Get absolute L2 queue ID
@@ -310,4 +322,37 @@ int qed_fw_rss_eng(struct qed_hwfn *p_hwfn,
 int qed_final_cleanup(struct qed_hwfn *p_hwfn,
                       struct qed_ptt *p_ptt, u16 id, bool is_vf);
 
+/**
+ * @brief qed_set_rxq_coalesce - Configure coalesce parameters for an Rx queue
+ * The fact that we can configure coalescing to up to 511, but on varying
+ * accuracy [the bigger the value the less accurate] up to a mistake of 3usec
+ * for the highest values.
+ *
+ * @param p_hwfn
+ * @param p_ptt
+ * @param coalesce - Coalesce value in micro seconds.
+ * @param qid - Queue index.
+ * @param qid - SB Id
+ *
+ * @return int
+ */
+int qed_set_rxq_coalesce(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
+                         u16 coalesce, u8 qid, u16 sb_id);
+
+/**
+ * @brief qed_set_txq_coalesce - Configure coalesce parameters for a Tx queue
+ * While the API allows setting coalescing per-qid, all tx queues sharing a
+ * SB should be in same range [i.e., either 0-0x7f, 0x80-0xff or 0x100-0x1ff]
+ * otherwise configuration would break.
+ *
+ * @param p_hwfn
+ * @param p_ptt
+ * @param coalesce - Coalesce value in micro seconds.
+ * @param qid - Queue index.
+ * @param qid - SB Id
+ *
+ * @return int
+ */
+int qed_set_txq_coalesce(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
+                         u16 coalesce, u8 qid, u16 sb_id);
 #endif
diff --git a/drivers/net/ethernet/qlogic/qed/qed_hsi.h b/drivers/net/ethernet/qlogic/qed/qed_hsi.h
index e29ed5a69566..592784019994 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_hsi.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_hsi.h
@@ -17,13 +17,15 @@
 #include <linux/list.h>
 #include <linux/slab.h>
 #include <linux/qed/common_hsi.h>
+#include <linux/qed/storage_common.h>
+#include <linux/qed/tcp_common.h>
 #include <linux/qed/eth_common.h>
+#include <linux/qed/iscsi_common.h>
+#include <linux/qed/rdma_common.h>
+#include <linux/qed/roce_common.h>
 
 struct qed_hwfn;
 struct qed_ptt;
-/********************************/
-/* Add include to common target */
-/********************************/
 
 /* opcodes for the event ring */
 enum common_event_opcode {
@@ -32,9 +34,10 @@ enum common_event_opcode {
         COMMON_EVENT_VF_START,
         COMMON_EVENT_VF_STOP,
         COMMON_EVENT_VF_PF_CHANNEL,
-        COMMON_EVENT_RESERVED4,
-        COMMON_EVENT_RESERVED5,
-        COMMON_EVENT_RESERVED6,
+        COMMON_EVENT_VF_FLR,
+        COMMON_EVENT_PF_UPDATE,
+        COMMON_EVENT_MALICIOUS_VF,
+        COMMON_EVENT_RL_UPDATE,
         COMMON_EVENT_EMPTY,
         MAX_COMMON_EVENT_OPCODE
 };
@@ -42,11 +45,12 @@ enum common_event_opcode {
 /* Common Ramrod Command IDs */
 enum common_ramrod_cmd_id {
         COMMON_RAMROD_UNUSED,
-        COMMON_RAMROD_PF_START /* PF Function Start Ramrod */,
-        COMMON_RAMROD_PF_STOP /* PF Function Stop Ramrod */,
+        COMMON_RAMROD_PF_START,
+        COMMON_RAMROD_PF_STOP,
         COMMON_RAMROD_VF_START,
         COMMON_RAMROD_VF_STOP,
         COMMON_RAMROD_PF_UPDATE,
+        COMMON_RAMROD_RL_UPDATE,
         COMMON_RAMROD_EMPTY,
         MAX_COMMON_RAMROD_CMD_ID
 };
@@ -63,448 +67,448 @@ struct pstorm_core_conn_st_ctx {
 
 /* Core Slowpath Connection storm context of Xstorm */
 struct xstorm_core_conn_st_ctx {
-        __le32          spq_base_lo /* SPQ Ring Base Address low dword */;
-        __le32          spq_base_hi /* SPQ Ring Base Address high dword */;
-        struct regpair  consolid_base_addr;
-        __le16          spq_cons /* SPQ Ring Consumer */;
-        __le16          consolid_cons /* Consolidation Ring Consumer */;
-        __le32          reserved0[55] /* Pad to 15 cycles */;
+        __le32 spq_base_lo;
+        __le32 spq_base_hi;
+        struct regpair consolid_base_addr;
+        __le16 spq_cons;
+        __le16 consolid_cons;
+        __le32 reserved0[55];
 };
 
 struct xstorm_core_conn_ag_ctx {
-        u8      reserved0 /* cdu_validation */;
-        u8      core_state /* state */;
-        u8      flags0;
-#define XSTORM_CORE_CONN_AG_CTX_EXIST_IN_QM0_MASK         0x1
-#define XSTORM_CORE_CONN_AG_CTX_EXIST_IN_QM0_SHIFT        0
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED1_MASK            0x1
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED1_SHIFT           1
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED2_MASK            0x1
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED2_SHIFT           2
-#define XSTORM_CORE_CONN_AG_CTX_EXIST_IN_QM3_MASK         0x1
-#define XSTORM_CORE_CONN_AG_CTX_EXIST_IN_QM3_SHIFT        3
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED3_MASK            0x1
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED3_SHIFT           4
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED4_MASK            0x1
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED4_SHIFT           5
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED5_MASK            0x1   /* bit6 */
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED5_SHIFT           6
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED6_MASK            0x1   /* bit7 */
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED6_SHIFT           7
+        u8 reserved0;
+        u8 core_state;
+        u8 flags0;
+#define XSTORM_CORE_CONN_AG_CTX_EXIST_IN_QM0_MASK       0x1
+#define XSTORM_CORE_CONN_AG_CTX_EXIST_IN_QM0_SHIFT      0
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED1_MASK          0x1
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED1_SHIFT         1
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED2_MASK          0x1
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED2_SHIFT         2
+#define XSTORM_CORE_CONN_AG_CTX_EXIST_IN_QM3_MASK       0x1
+#define XSTORM_CORE_CONN_AG_CTX_EXIST_IN_QM3_SHIFT      3
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED3_MASK          0x1
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED3_SHIFT         4
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED4_MASK          0x1
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED4_SHIFT         5
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED5_MASK          0x1
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED5_SHIFT         6
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED6_MASK          0x1
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED6_SHIFT         7
         u8 flags1;
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED7_MASK            0x1   /* bit8 */
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED7_SHIFT           0
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED8_MASK            0x1   /* bit9 */
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED8_SHIFT           1
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED9_MASK            0x1   /* bit10 */
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED9_SHIFT           2
-#define XSTORM_CORE_CONN_AG_CTX_BIT11_MASK                0x1   /* bit11 */
-#define XSTORM_CORE_CONN_AG_CTX_BIT11_SHIFT               3
-#define XSTORM_CORE_CONN_AG_CTX_BIT12_MASK                0x1   /* bit12 */
-#define XSTORM_CORE_CONN_AG_CTX_BIT12_SHIFT               4
-#define XSTORM_CORE_CONN_AG_CTX_BIT13_MASK                0x1   /* bit13 */
-#define XSTORM_CORE_CONN_AG_CTX_BIT13_SHIFT               5
-#define XSTORM_CORE_CONN_AG_CTX_TX_RULE_ACTIVE_MASK       0x1   /* bit14 */
-#define XSTORM_CORE_CONN_AG_CTX_TX_RULE_ACTIVE_SHIFT      6
-#define XSTORM_CORE_CONN_AG_CTX_DQ_CF_ACTIVE_MASK         0x1   /* bit15 */
-#define XSTORM_CORE_CONN_AG_CTX_DQ_CF_ACTIVE_SHIFT        7
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED7_MASK          0x1
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED7_SHIFT         0
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED8_MASK          0x1
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED8_SHIFT         1
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED9_MASK          0x1
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED9_SHIFT         2
+#define XSTORM_CORE_CONN_AG_CTX_BIT11_MASK              0x1
+#define XSTORM_CORE_CONN_AG_CTX_BIT11_SHIFT             3
+#define XSTORM_CORE_CONN_AG_CTX_BIT12_MASK              0x1
+#define XSTORM_CORE_CONN_AG_CTX_BIT12_SHIFT             4
+#define XSTORM_CORE_CONN_AG_CTX_BIT13_MASK              0x1
+#define XSTORM_CORE_CONN_AG_CTX_BIT13_SHIFT             5
+#define XSTORM_CORE_CONN_AG_CTX_TX_RULE_ACTIVE_MASK     0x1
+#define XSTORM_CORE_CONN_AG_CTX_TX_RULE_ACTIVE_SHIFT    6
+#define XSTORM_CORE_CONN_AG_CTX_DQ_CF_ACTIVE_MASK       0x1
+#define XSTORM_CORE_CONN_AG_CTX_DQ_CF_ACTIVE_SHIFT      7
         u8 flags2;
-#define XSTORM_CORE_CONN_AG_CTX_CF0_MASK                  0x3   /* timer0cf */
-#define XSTORM_CORE_CONN_AG_CTX_CF0_SHIFT                 0
-#define XSTORM_CORE_CONN_AG_CTX_CF1_MASK                  0x3   /* timer1cf */
-#define XSTORM_CORE_CONN_AG_CTX_CF1_SHIFT                 2
-#define XSTORM_CORE_CONN_AG_CTX_CF2_MASK                  0x3   /* timer2cf */
-#define XSTORM_CORE_CONN_AG_CTX_CF2_SHIFT                 4
-#define XSTORM_CORE_CONN_AG_CTX_CF3_MASK                  0x3
-#define XSTORM_CORE_CONN_AG_CTX_CF3_SHIFT                 6
+#define XSTORM_CORE_CONN_AG_CTX_CF0_MASK        0x3
+#define XSTORM_CORE_CONN_AG_CTX_CF0_SHIFT       0
+#define XSTORM_CORE_CONN_AG_CTX_CF1_MASK        0x3
+#define XSTORM_CORE_CONN_AG_CTX_CF1_SHIFT       2
+#define XSTORM_CORE_CONN_AG_CTX_CF2_MASK        0x3
+#define XSTORM_CORE_CONN_AG_CTX_CF2_SHIFT       4
+#define XSTORM_CORE_CONN_AG_CTX_CF3_MASK        0x3
+#define XSTORM_CORE_CONN_AG_CTX_CF3_SHIFT       6
         u8 flags3;
-#define XSTORM_CORE_CONN_AG_CTX_CF4_MASK                  0x3   /* cf4 */
-#define XSTORM_CORE_CONN_AG_CTX_CF4_SHIFT                 0
-#define XSTORM_CORE_CONN_AG_CTX_CF5_MASK                  0x3   /* cf5 */
-#define XSTORM_CORE_CONN_AG_CTX_CF5_SHIFT                 2
-#define XSTORM_CORE_CONN_AG_CTX_CF6_MASK                  0x3   /* cf6 */
-#define XSTORM_CORE_CONN_AG_CTX_CF6_SHIFT                 4
-#define XSTORM_CORE_CONN_AG_CTX_CF7_MASK                  0x3   /* cf7 */
-#define XSTORM_CORE_CONN_AG_CTX_CF7_SHIFT                 6
+#define XSTORM_CORE_CONN_AG_CTX_CF4_MASK        0x3
+#define XSTORM_CORE_CONN_AG_CTX_CF4_SHIFT       0
+#define XSTORM_CORE_CONN_AG_CTX_CF5_MASK        0x3
+#define XSTORM_CORE_CONN_AG_CTX_CF5_SHIFT       2
+#define XSTORM_CORE_CONN_AG_CTX_CF6_MASK        0x3
+#define XSTORM_CORE_CONN_AG_CTX_CF6_SHIFT       4
+#define XSTORM_CORE_CONN_AG_CTX_CF7_MASK        0x3
+#define XSTORM_CORE_CONN_AG_CTX_CF7_SHIFT       6
         u8 flags4;
-#define XSTORM_CORE_CONN_AG_CTX_CF8_MASK                  0x3   /* cf8 */
-#define XSTORM_CORE_CONN_AG_CTX_CF8_SHIFT                 0
-#define XSTORM_CORE_CONN_AG_CTX_CF9_MASK                  0x3   /* cf9 */
-#define XSTORM_CORE_CONN_AG_CTX_CF9_SHIFT                 2
-#define XSTORM_CORE_CONN_AG_CTX_CF10_MASK                 0x3   /* cf10 */
-#define XSTORM_CORE_CONN_AG_CTX_CF10_SHIFT                4
-#define XSTORM_CORE_CONN_AG_CTX_CF11_MASK                 0x3   /* cf11 */
-#define XSTORM_CORE_CONN_AG_CTX_CF11_SHIFT                6
+#define XSTORM_CORE_CONN_AG_CTX_CF8_MASK        0x3
+#define XSTORM_CORE_CONN_AG_CTX_CF8_SHIFT       0
+#define XSTORM_CORE_CONN_AG_CTX_CF9_MASK        0x3
+#define XSTORM_CORE_CONN_AG_CTX_CF9_SHIFT       2
+#define XSTORM_CORE_CONN_AG_CTX_CF10_MASK       0x3
+#define XSTORM_CORE_CONN_AG_CTX_CF10_SHIFT      4
+#define XSTORM_CORE_CONN_AG_CTX_CF11_MASK       0x3
+#define XSTORM_CORE_CONN_AG_CTX_CF11_SHIFT      6
         u8 flags5;
-#define XSTORM_CORE_CONN_AG_CTX_CF12_MASK                 0x3   /* cf12 */
-#define XSTORM_CORE_CONN_AG_CTX_CF12_SHIFT                0
-#define XSTORM_CORE_CONN_AG_CTX_CF13_MASK                 0x3   /* cf13 */
-#define XSTORM_CORE_CONN_AG_CTX_CF13_SHIFT                2
-#define XSTORM_CORE_CONN_AG_CTX_CF14_MASK                 0x3   /* cf14 */
-#define XSTORM_CORE_CONN_AG_CTX_CF14_SHIFT                4
-#define XSTORM_CORE_CONN_AG_CTX_CF15_MASK                 0x3   /* cf15 */
-#define XSTORM_CORE_CONN_AG_CTX_CF15_SHIFT                6
+#define XSTORM_CORE_CONN_AG_CTX_CF12_MASK       0x3
+#define XSTORM_CORE_CONN_AG_CTX_CF12_SHIFT      0
+#define XSTORM_CORE_CONN_AG_CTX_CF13_MASK       0x3
+#define XSTORM_CORE_CONN_AG_CTX_CF13_SHIFT      2
+#define XSTORM_CORE_CONN_AG_CTX_CF14_MASK       0x3
+#define XSTORM_CORE_CONN_AG_CTX_CF14_SHIFT      4
+#define XSTORM_CORE_CONN_AG_CTX_CF15_MASK       0x3
+#define XSTORM_CORE_CONN_AG_CTX_CF15_SHIFT      6
         u8 flags6;
-#define XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_MASK     0x3   /* cf16 */
-#define XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_SHIFT    0
-#define XSTORM_CORE_CONN_AG_CTX_CF17_MASK                 0x3
-#define XSTORM_CORE_CONN_AG_CTX_CF17_SHIFT                2
-#define XSTORM_CORE_CONN_AG_CTX_DQ_CF_MASK                0x3   /* cf18 */
-#define XSTORM_CORE_CONN_AG_CTX_DQ_CF_SHIFT               4
-#define XSTORM_CORE_CONN_AG_CTX_TERMINATE_CF_MASK         0x3   /* cf19 */
-#define XSTORM_CORE_CONN_AG_CTX_TERMINATE_CF_SHIFT        6
+#define XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_MASK   0x3
+#define XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_SHIFT  0
+#define XSTORM_CORE_CONN_AG_CTX_CF17_MASK               0x3
+#define XSTORM_CORE_CONN_AG_CTX_CF17_SHIFT              2
+#define XSTORM_CORE_CONN_AG_CTX_DQ_CF_MASK              0x3
+#define XSTORM_CORE_CONN_AG_CTX_DQ_CF_SHIFT             4
+#define XSTORM_CORE_CONN_AG_CTX_TERMINATE_CF_MASK       0x3
+#define XSTORM_CORE_CONN_AG_CTX_TERMINATE_CF_SHIFT      6
         u8 flags7;
-#define XSTORM_CORE_CONN_AG_CTX_FLUSH_Q0_MASK             0x3   /* cf20 */
-#define XSTORM_CORE_CONN_AG_CTX_FLUSH_Q0_SHIFT            0
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED10_MASK           0x3   /* cf21 */
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED10_SHIFT          2
-#define XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_MASK            0x3   /* cf22 */
-#define XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_SHIFT           4
-#define XSTORM_CORE_CONN_AG_CTX_CF0EN_MASK                0x1   /* cf0en */
-#define XSTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT               6
-#define XSTORM_CORE_CONN_AG_CTX_CF1EN_MASK                0x1   /* cf1en */
-#define XSTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT               7
+#define XSTORM_CORE_CONN_AG_CTX_FLUSH_Q0_MASK           0x3
+#define XSTORM_CORE_CONN_AG_CTX_FLUSH_Q0_SHIFT          0
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED10_MASK         0x3
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED10_SHIFT        2
+#define XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_MASK          0x3
+#define XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_SHIFT         4
+#define XSTORM_CORE_CONN_AG_CTX_CF0EN_MASK              0x1
+#define XSTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT             6
+#define XSTORM_CORE_CONN_AG_CTX_CF1EN_MASK              0x1
+#define XSTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT             7
         u8 flags8;
-#define XSTORM_CORE_CONN_AG_CTX_CF2EN_MASK                0x1   /* cf2en */
-#define XSTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT               0
-#define XSTORM_CORE_CONN_AG_CTX_CF3EN_MASK                0x1   /* cf3en */
-#define XSTORM_CORE_CONN_AG_CTX_CF3EN_SHIFT               1
-#define XSTORM_CORE_CONN_AG_CTX_CF4EN_MASK                0x1   /* cf4en */
-#define XSTORM_CORE_CONN_AG_CTX_CF4EN_SHIFT               2
-#define XSTORM_CORE_CONN_AG_CTX_CF5EN_MASK                0x1   /* cf5en */
-#define XSTORM_CORE_CONN_AG_CTX_CF5EN_SHIFT               3
-#define XSTORM_CORE_CONN_AG_CTX_CF6EN_MASK                0x1   /* cf6en */
-#define XSTORM_CORE_CONN_AG_CTX_CF6EN_SHIFT               4
-#define XSTORM_CORE_CONN_AG_CTX_CF7EN_MASK                0x1   /* cf7en */
-#define XSTORM_CORE_CONN_AG_CTX_CF7EN_SHIFT               5
-#define XSTORM_CORE_CONN_AG_CTX_CF8EN_MASK                0x1   /* cf8en */
-#define XSTORM_CORE_CONN_AG_CTX_CF8EN_SHIFT               6
-#define XSTORM_CORE_CONN_AG_CTX_CF9EN_MASK                0x1   /* cf9en */
-#define XSTORM_CORE_CONN_AG_CTX_CF9EN_SHIFT               7
+#define XSTORM_CORE_CONN_AG_CTX_CF2EN_MASK      0x1
+#define XSTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT     0
+#define XSTORM_CORE_CONN_AG_CTX_CF3EN_MASK      0x1
+#define XSTORM_CORE_CONN_AG_CTX_CF3EN_SHIFT     1
+#define XSTORM_CORE_CONN_AG_CTX_CF4EN_MASK      0x1
+#define XSTORM_CORE_CONN_AG_CTX_CF4EN_SHIFT     2
+#define XSTORM_CORE_CONN_AG_CTX_CF5EN_MASK      0x1
+#define XSTORM_CORE_CONN_AG_CTX_CF5EN_SHIFT     3
+#define XSTORM_CORE_CONN_AG_CTX_CF6EN_MASK      0x1
+#define XSTORM_CORE_CONN_AG_CTX_CF6EN_SHIFT     4
+#define XSTORM_CORE_CONN_AG_CTX_CF7EN_MASK      0x1
+#define XSTORM_CORE_CONN_AG_CTX_CF7EN_SHIFT     5
+#define XSTORM_CORE_CONN_AG_CTX_CF8EN_MASK      0x1
+#define XSTORM_CORE_CONN_AG_CTX_CF8EN_SHIFT     6
+#define XSTORM_CORE_CONN_AG_CTX_CF9EN_MASK      0x1
+#define XSTORM_CORE_CONN_AG_CTX_CF9EN_SHIFT     7
         u8 flags9;
-#define XSTORM_CORE_CONN_AG_CTX_CF10EN_MASK               0x1   /* cf10en */
-#define XSTORM_CORE_CONN_AG_CTX_CF10EN_SHIFT              0
-#define XSTORM_CORE_CONN_AG_CTX_CF11EN_MASK               0x1   /* cf11en */
-#define XSTORM_CORE_CONN_AG_CTX_CF11EN_SHIFT              1
-#define XSTORM_CORE_CONN_AG_CTX_CF12EN_MASK               0x1   /* cf12en */
-#define XSTORM_CORE_CONN_AG_CTX_CF12EN_SHIFT              2
-#define XSTORM_CORE_CONN_AG_CTX_CF13EN_MASK               0x1   /* cf13en */
-#define XSTORM_CORE_CONN_AG_CTX_CF13EN_SHIFT              3
-#define XSTORM_CORE_CONN_AG_CTX_CF14EN_MASK               0x1   /* cf14en */
-#define XSTORM_CORE_CONN_AG_CTX_CF14EN_SHIFT              4
-#define XSTORM_CORE_CONN_AG_CTX_CF15EN_MASK               0x1   /* cf15en */
-#define XSTORM_CORE_CONN_AG_CTX_CF15EN_SHIFT              5
-#define XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_EN_MASK  0x1   /* cf16en */
-#define XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_EN_SHIFT 6
-#define XSTORM_CORE_CONN_AG_CTX_CF17EN_MASK               0x1
-#define XSTORM_CORE_CONN_AG_CTX_CF17EN_SHIFT              7
+#define XSTORM_CORE_CONN_AG_CTX_CF10EN_MASK                     0x1
+#define XSTORM_CORE_CONN_AG_CTX_CF10EN_SHIFT                    0
+#define XSTORM_CORE_CONN_AG_CTX_CF11EN_MASK                     0x1
+#define XSTORM_CORE_CONN_AG_CTX_CF11EN_SHIFT                    1
+#define XSTORM_CORE_CONN_AG_CTX_CF12EN_MASK                     0x1
+#define XSTORM_CORE_CONN_AG_CTX_CF12EN_SHIFT                    2
+#define XSTORM_CORE_CONN_AG_CTX_CF13EN_MASK                     0x1
+#define XSTORM_CORE_CONN_AG_CTX_CF13EN_SHIFT                    3
+#define XSTORM_CORE_CONN_AG_CTX_CF14EN_MASK                     0x1
+#define XSTORM_CORE_CONN_AG_CTX_CF14EN_SHIFT                    4
+#define XSTORM_CORE_CONN_AG_CTX_CF15EN_MASK                     0x1
+#define XSTORM_CORE_CONN_AG_CTX_CF15EN_SHIFT                    5
+#define XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_EN_MASK        0x1
+#define XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_EN_SHIFT       6
+#define XSTORM_CORE_CONN_AG_CTX_CF17EN_MASK                     0x1
+#define XSTORM_CORE_CONN_AG_CTX_CF17EN_SHIFT                    7
         u8 flags10;
-#define XSTORM_CORE_CONN_AG_CTX_DQ_CF_EN_MASK             0x1   /* cf18en */
-#define XSTORM_CORE_CONN_AG_CTX_DQ_CF_EN_SHIFT            0
-#define XSTORM_CORE_CONN_AG_CTX_TERMINATE_CF_EN_MASK      0x1   /* cf19en */
-#define XSTORM_CORE_CONN_AG_CTX_TERMINATE_CF_EN_SHIFT     1
-#define XSTORM_CORE_CONN_AG_CTX_FLUSH_Q0_EN_MASK          0x1   /* cf20en */
-#define XSTORM_CORE_CONN_AG_CTX_FLUSH_Q0_EN_SHIFT         2
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED11_MASK           0x1   /* cf21en */
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED11_SHIFT          3
-#define XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_EN_MASK         0x1   /* cf22en */
-#define XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_EN_SHIFT        4
-#define XSTORM_CORE_CONN_AG_CTX_CF23EN_MASK               0x1   /* cf23en */
-#define XSTORM_CORE_CONN_AG_CTX_CF23EN_SHIFT              5
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED12_MASK           0x1   /* rule0en */
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED12_SHIFT          6
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED13_MASK           0x1   /* rule1en */
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED13_SHIFT          7
+#define XSTORM_CORE_CONN_AG_CTX_DQ_CF_EN_MASK           0x1
+#define XSTORM_CORE_CONN_AG_CTX_DQ_CF_EN_SHIFT          0
+#define XSTORM_CORE_CONN_AG_CTX_TERMINATE_CF_EN_MASK    0x1
+#define XSTORM_CORE_CONN_AG_CTX_TERMINATE_CF_EN_SHIFT   1
+#define XSTORM_CORE_CONN_AG_CTX_FLUSH_Q0_EN_MASK        0x1
+#define XSTORM_CORE_CONN_AG_CTX_FLUSH_Q0_EN_SHIFT       2
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED11_MASK         0x1
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED11_SHIFT        3
+#define XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_EN_MASK       0x1
+#define XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_EN_SHIFT      4
+#define XSTORM_CORE_CONN_AG_CTX_CF23EN_MASK             0x1
+#define XSTORM_CORE_CONN_AG_CTX_CF23EN_SHIFT            5
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED12_MASK         0x1
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED12_SHIFT        6
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED13_MASK         0x1
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED13_SHIFT        7
         u8 flags11;
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED14_MASK           0x1   /* rule2en */
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED14_SHIFT          0
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED15_MASK           0x1   /* rule3en */
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED15_SHIFT          1
-#define XSTORM_CORE_CONN_AG_CTX_TX_DEC_RULE_EN_MASK       0x1   /* rule4en */
-#define XSTORM_CORE_CONN_AG_CTX_TX_DEC_RULE_EN_SHIFT      2
-#define XSTORM_CORE_CONN_AG_CTX_RULE5EN_MASK              0x1   /* rule5en */
-#define XSTORM_CORE_CONN_AG_CTX_RULE5EN_SHIFT             3
-#define XSTORM_CORE_CONN_AG_CTX_RULE6EN_MASK              0x1   /* rule6en */
-#define XSTORM_CORE_CONN_AG_CTX_RULE6EN_SHIFT             4
-#define XSTORM_CORE_CONN_AG_CTX_RULE7EN_MASK              0x1   /* rule7en */
-#define XSTORM_CORE_CONN_AG_CTX_RULE7EN_SHIFT             5
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED1_MASK         0x1   /* rule8en */
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED1_SHIFT        6
-#define XSTORM_CORE_CONN_AG_CTX_RULE9EN_MASK              0x1   /* rule9en */
-#define XSTORM_CORE_CONN_AG_CTX_RULE9EN_SHIFT             7
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED14_MASK         0x1
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED14_SHIFT        0
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED15_MASK         0x1
+#define XSTORM_CORE_CONN_AG_CTX_RESERVED15_SHIFT        1
+#define XSTORM_CORE_CONN_AG_CTX_TX_DEC_RULE_EN_MASK     0x1
+#define XSTORM_CORE_CONN_AG_CTX_TX_DEC_RULE_EN_SHIFT    2
+#define XSTORM_CORE_CONN_AG_CTX_RULE5EN_MASK            0x1
+#define XSTORM_CORE_CONN_AG_CTX_RULE5EN_SHIFT           3
+#define XSTORM_CORE_CONN_AG_CTX_RULE6EN_MASK            0x1
+#define XSTORM_CORE_CONN_AG_CTX_RULE6EN_SHIFT           4
+#define XSTORM_CORE_CONN_AG_CTX_RULE7EN_MASK            0x1
+#define XSTORM_CORE_CONN_AG_CTX_RULE7EN_SHIFT           5
+#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED1_MASK       0x1
+#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED1_SHIFT      6
+#define XSTORM_CORE_CONN_AG_CTX_RULE9EN_MASK            0x1
+#define XSTORM_CORE_CONN_AG_CTX_RULE9EN_SHIFT           7
         u8 flags12;
-#define XSTORM_CORE_CONN_AG_CTX_RULE10EN_MASK             0x1   /* rule10en */
-#define XSTORM_CORE_CONN_AG_CTX_RULE10EN_SHIFT            0
-#define XSTORM_CORE_CONN_AG_CTX_RULE11EN_MASK             0x1   /* rule11en */
-#define XSTORM_CORE_CONN_AG_CTX_RULE11EN_SHIFT            1
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED2_MASK         0x1   /* rule12en */
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED2_SHIFT        2
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED3_MASK         0x1   /* rule13en */
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED3_SHIFT        3
-#define XSTORM_CORE_CONN_AG_CTX_RULE14EN_MASK             0x1   /* rule14en */
-#define XSTORM_CORE_CONN_AG_CTX_RULE14EN_SHIFT            4
-#define XSTORM_CORE_CONN_AG_CTX_RULE15EN_MASK             0x1   /* rule15en */
-#define XSTORM_CORE_CONN_AG_CTX_RULE15EN_SHIFT            5
-#define XSTORM_CORE_CONN_AG_CTX_RULE16EN_MASK             0x1   /* rule16en */
-#define XSTORM_CORE_CONN_AG_CTX_RULE16EN_SHIFT            6
-#define XSTORM_CORE_CONN_AG_CTX_RULE17EN_MASK             0x1   /* rule17en */
-#define XSTORM_CORE_CONN_AG_CTX_RULE17EN_SHIFT            7
+#define XSTORM_CORE_CONN_AG_CTX_RULE10EN_MASK           0x1
+#define XSTORM_CORE_CONN_AG_CTX_RULE10EN_SHIFT          0
+#define XSTORM_CORE_CONN_AG_CTX_RULE11EN_MASK           0x1
+#define XSTORM_CORE_CONN_AG_CTX_RULE11EN_SHIFT          1
+#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED2_MASK       0x1
+#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED2_SHIFT      2
+#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED3_MASK       0x1
+#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED3_SHIFT      3
+#define XSTORM_CORE_CONN_AG_CTX_RULE14EN_MASK           0x1
+#define XSTORM_CORE_CONN_AG_CTX_RULE14EN_SHIFT          4
+#define XSTORM_CORE_CONN_AG_CTX_RULE15EN_MASK           0x1
+#define XSTORM_CORE_CONN_AG_CTX_RULE15EN_SHIFT          5
+#define XSTORM_CORE_CONN_AG_CTX_RULE16EN_MASK           0x1
+#define XSTORM_CORE_CONN_AG_CTX_RULE16EN_SHIFT          6
+#define XSTORM_CORE_CONN_AG_CTX_RULE17EN_MASK           0x1
+#define XSTORM_CORE_CONN_AG_CTX_RULE17EN_SHIFT          7
         u8 flags13;
-#define XSTORM_CORE_CONN_AG_CTX_RULE18EN_MASK             0x1   /* rule18en */
-#define XSTORM_CORE_CONN_AG_CTX_RULE18EN_SHIFT            0
-#define XSTORM_CORE_CONN_AG_CTX_RULE19EN_MASK             0x1   /* rule19en */
-#define XSTORM_CORE_CONN_AG_CTX_RULE19EN_SHIFT            1
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED4_MASK         0x1   /* rule20en */
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED4_SHIFT        2
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED5_MASK         0x1   /* rule21en */
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED5_SHIFT        3
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED6_MASK         0x1   /* rule22en */
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED6_SHIFT        4
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED7_MASK         0x1   /* rule23en */
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED7_SHIFT        5
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED8_MASK         0x1   /* rule24en */
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED8_SHIFT        6
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED9_MASK         0x1   /* rule25en */
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED9_SHIFT        7
+#define XSTORM_CORE_CONN_AG_CTX_RULE18EN_MASK           0x1
+#define XSTORM_CORE_CONN_AG_CTX_RULE18EN_SHIFT          0
+#define XSTORM_CORE_CONN_AG_CTX_RULE19EN_MASK           0x1
+#define XSTORM_CORE_CONN_AG_CTX_RULE19EN_SHIFT          1
+#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED4_MASK       0x1
+#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED4_SHIFT      2
+#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED5_MASK       0x1
+#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED5_SHIFT      3
+#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED6_MASK       0x1
+#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED6_SHIFT      4
+#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED7_MASK       0x1
+#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED7_SHIFT      5
+#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED8_MASK       0x1
+#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED8_SHIFT      6
+#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED9_MASK       0x1
+#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED9_SHIFT      7
         u8 flags14;
-#define XSTORM_CORE_CONN_AG_CTX_BIT16_MASK                0x1   /* bit16 */
-#define XSTORM_CORE_CONN_AG_CTX_BIT16_SHIFT               0
-#define XSTORM_CORE_CONN_AG_CTX_BIT17_MASK                0x1   /* bit17 */
-#define XSTORM_CORE_CONN_AG_CTX_BIT17_SHIFT               1
-#define XSTORM_CORE_CONN_AG_CTX_BIT18_MASK                0x1   /* bit18 */
-#define XSTORM_CORE_CONN_AG_CTX_BIT18_SHIFT               2
-#define XSTORM_CORE_CONN_AG_CTX_BIT19_MASK                0x1   /* bit19 */
-#define XSTORM_CORE_CONN_AG_CTX_BIT19_SHIFT               3
-#define XSTORM_CORE_CONN_AG_CTX_BIT20_MASK                0x1   /* bit20 */
-#define XSTORM_CORE_CONN_AG_CTX_BIT20_SHIFT               4
-#define XSTORM_CORE_CONN_AG_CTX_BIT21_MASK                0x1   /* bit21 */
-#define XSTORM_CORE_CONN_AG_CTX_BIT21_SHIFT               5
-#define XSTORM_CORE_CONN_AG_CTX_CF23_MASK                 0x3   /* cf23 */
-#define XSTORM_CORE_CONN_AG_CTX_CF23_SHIFT                6
-        u8      byte2 /* byte2 */;
-        __le16  physical_q0 /* physical_q0 */;
-        __le16  consolid_prod /* physical_q1 */;
-        __le16  reserved16 /* physical_q2 */;
-        __le16  tx_bd_cons /* word3 */;
-        __le16  tx_bd_or_spq_prod /* word4 */;
-        __le16  word5 /* word5 */;
-        __le16  conn_dpi /* conn_dpi */;
-        u8      byte3 /* byte3 */;
-        u8      byte4 /* byte4 */;
-        u8      byte5 /* byte5 */;
-        u8      byte6 /* byte6 */;
-        __le32  reg0 /* reg0 */;
-        __le32  reg1 /* reg1 */;
-        __le32  reg2 /* reg2 */;
-        __le32  reg3 /* reg3 */;
-        __le32  reg4 /* reg4 */;
-        __le32  reg5 /* cf_array0 */;
-        __le32  reg6 /* cf_array1 */;
-        __le16  word7 /* word7 */;
-        __le16  word8 /* word8 */;
-        __le16  word9 /* word9 */;
-        __le16  word10 /* word10 */;
-        __le32  reg7 /* reg7 */;
-        __le32  reg8 /* reg8 */;
-        __le32  reg9 /* reg9 */;
-        u8      byte7 /* byte7 */;
-        u8      byte8 /* byte8 */;
-        u8      byte9 /* byte9 */;
-        u8      byte10 /* byte10 */;
-        u8      byte11 /* byte11 */;
-        u8      byte12 /* byte12 */;
-        u8      byte13 /* byte13 */;
-        u8      byte14 /* byte14 */;
-        u8      byte15 /* byte15 */;
-        u8      byte16 /* byte16 */;
-        __le16  word11 /* word11 */;
-        __le32  reg10 /* reg10 */;
-        __le32  reg11 /* reg11 */;
-        __le32  reg12 /* reg12 */;
-        __le32  reg13 /* reg13 */;
-        __le32  reg14 /* reg14 */;
-        __le32  reg15 /* reg15 */;
-        __le32  reg16 /* reg16 */;
-        __le32  reg17 /* reg17 */;
-        __le32  reg18 /* reg18 */;
-        __le32  reg19 /* reg19 */;
-        __le16  word12 /* word12 */;
-        __le16  word13 /* word13 */;
-        __le16  word14 /* word14 */;
-        __le16  word15 /* word15 */;
+#define XSTORM_CORE_CONN_AG_CTX_BIT16_MASK      0x1
+#define XSTORM_CORE_CONN_AG_CTX_BIT16_SHIFT     0
+#define XSTORM_CORE_CONN_AG_CTX_BIT17_MASK      0x1
+#define XSTORM_CORE_CONN_AG_CTX_BIT17_SHIFT     1
+#define XSTORM_CORE_CONN_AG_CTX_BIT18_MASK      0x1
+#define XSTORM_CORE_CONN_AG_CTX_BIT18_SHIFT     2
+#define XSTORM_CORE_CONN_AG_CTX_BIT19_MASK      0x1
+#define XSTORM_CORE_CONN_AG_CTX_BIT19_SHIFT     3
+#define XSTORM_CORE_CONN_AG_CTX_BIT20_MASK      0x1
+#define XSTORM_CORE_CONN_AG_CTX_BIT20_SHIFT     4
+#define XSTORM_CORE_CONN_AG_CTX_BIT21_MASK      0x1
+#define XSTORM_CORE_CONN_AG_CTX_BIT21_SHIFT     5
+#define XSTORM_CORE_CONN_AG_CTX_CF23_MASK       0x3
+#define XSTORM_CORE_CONN_AG_CTX_CF23_SHIFT      6
+        u8 byte2;
+        __le16 physical_q0;
+        __le16 consolid_prod;
+        __le16 reserved16;
+        __le16 tx_bd_cons;
+        __le16 tx_bd_or_spq_prod;
+        __le16 word5;
+        __le16 conn_dpi;
+        u8 byte3;
+        u8 byte4;
+        u8 byte5;
+        u8 byte6;
+        __le32 reg0;
+        __le32 reg1;
+        __le32 reg2;
+        __le32 reg3;
+        __le32 reg4;
+        __le32 reg5;
+        __le32 reg6;
+        __le16 word7;
+        __le16 word8;
+        __le16 word9;
+        __le16 word10;
+        __le32 reg7;
+        __le32 reg8;
+        __le32 reg9;
+        u8 byte7;
+        u8 byte8;
+        u8 byte9;
+        u8 byte10;
+        u8 byte11;
+        u8 byte12;
+        u8 byte13;
+        u8 byte14;
+        u8 byte15;
+        u8 byte16;
+        __le16 word11;
+        __le32 reg10;
+        __le32 reg11;
+        __le32 reg12;
+        __le32 reg13;
+        __le32 reg14;
+        __le32 reg15;
+        __le32 reg16;
+        __le32 reg17;
+        __le32 reg18;
+        __le32 reg19;
+        __le16 word12;
+        __le16 word13;
+        __le16 word14;
+        __le16 word15;
 };
 
 struct tstorm_core_conn_ag_ctx {
-        u8      byte0 /* cdu_validation */;
-        u8      byte1 /* state */;
-        u8      flags0;
-#define TSTORM_CORE_CONN_AG_CTX_BIT0_MASK     0x1       /* exist_in_qm0 */
-#define TSTORM_CORE_CONN_AG_CTX_BIT0_SHIFT    0
-#define TSTORM_CORE_CONN_AG_CTX_BIT1_MASK     0x1       /* exist_in_qm1 */
-#define TSTORM_CORE_CONN_AG_CTX_BIT1_SHIFT    1
-#define TSTORM_CORE_CONN_AG_CTX_BIT2_MASK     0x1       /* bit2 */
-#define TSTORM_CORE_CONN_AG_CTX_BIT2_SHIFT    2
-#define TSTORM_CORE_CONN_AG_CTX_BIT3_MASK     0x1       /* bit3 */
-#define TSTORM_CORE_CONN_AG_CTX_BIT3_SHIFT    3
-#define TSTORM_CORE_CONN_AG_CTX_BIT4_MASK     0x1       /* bit4 */
-#define TSTORM_CORE_CONN_AG_CTX_BIT4_SHIFT    4
-#define TSTORM_CORE_CONN_AG_CTX_BIT5_MASK     0x1       /* bit5 */
-#define TSTORM_CORE_CONN_AG_CTX_BIT5_SHIFT    5
-#define TSTORM_CORE_CONN_AG_CTX_CF0_MASK      0x3       /* timer0cf */
-#define TSTORM_CORE_CONN_AG_CTX_CF0_SHIFT     6
+        u8 byte0;
+        u8 byte1;
+        u8 flags0;
+#define TSTORM_CORE_CONN_AG_CTX_BIT0_MASK       0x1
+#define TSTORM_CORE_CONN_AG_CTX_BIT0_SHIFT      0
+#define TSTORM_CORE_CONN_AG_CTX_BIT1_MASK       0x1
+#define TSTORM_CORE_CONN_AG_CTX_BIT1_SHIFT      1
+#define TSTORM_CORE_CONN_AG_CTX_BIT2_MASK       0x1
+#define TSTORM_CORE_CONN_AG_CTX_BIT2_SHIFT      2
+#define TSTORM_CORE_CONN_AG_CTX_BIT3_MASK       0x1
+#define TSTORM_CORE_CONN_AG_CTX_BIT3_SHIFT      3
+#define TSTORM_CORE_CONN_AG_CTX_BIT4_MASK       0x1
+#define TSTORM_CORE_CONN_AG_CTX_BIT4_SHIFT      4
+#define TSTORM_CORE_CONN_AG_CTX_BIT5_MASK       0x1
+#define TSTORM_CORE_CONN_AG_CTX_BIT5_SHIFT      5
+#define TSTORM_CORE_CONN_AG_CTX_CF0_MASK        0x3
+#define TSTORM_CORE_CONN_AG_CTX_CF0_SHIFT       6
         u8 flags1;
-#define TSTORM_CORE_CONN_AG_CTX_CF1_MASK      0x3       /* timer1cf */
-#define TSTORM_CORE_CONN_AG_CTX_CF1_SHIFT     0
-#define TSTORM_CORE_CONN_AG_CTX_CF2_MASK      0x3       /* timer2cf */
-#define TSTORM_CORE_CONN_AG_CTX_CF2_SHIFT     2
-#define TSTORM_CORE_CONN_AG_CTX_CF3_MASK      0x3       /* timer_stop_all */
-#define TSTORM_CORE_CONN_AG_CTX_CF3_SHIFT     4
-#define TSTORM_CORE_CONN_AG_CTX_CF4_MASK      0x3       /* cf4 */
-#define TSTORM_CORE_CONN_AG_CTX_CF4_SHIFT     6
+#define TSTORM_CORE_CONN_AG_CTX_CF1_MASK        0x3
+#define TSTORM_CORE_CONN_AG_CTX_CF1_SHIFT       0
+#define TSTORM_CORE_CONN_AG_CTX_CF2_MASK        0x3
+#define TSTORM_CORE_CONN_AG_CTX_CF2_SHIFT       2
+#define TSTORM_CORE_CONN_AG_CTX_CF3_MASK        0x3
+#define TSTORM_CORE_CONN_AG_CTX_CF3_SHIFT       4
+#define TSTORM_CORE_CONN_AG_CTX_CF4_MASK        0x3
+#define TSTORM_CORE_CONN_AG_CTX_CF4_SHIFT       6
         u8 flags2;
-#define TSTORM_CORE_CONN_AG_CTX_CF5_MASK      0x3       /* cf5 */
-#define TSTORM_CORE_CONN_AG_CTX_CF5_SHIFT     0
-#define TSTORM_CORE_CONN_AG_CTX_CF6_MASK      0x3       /* cf6 */
-#define TSTORM_CORE_CONN_AG_CTX_CF6_SHIFT     2
-#define TSTORM_CORE_CONN_AG_CTX_CF7_MASK      0x3       /* cf7 */
-#define TSTORM_CORE_CONN_AG_CTX_CF7_SHIFT     4
-#define TSTORM_CORE_CONN_AG_CTX_CF8_MASK      0x3       /* cf8 */
-#define TSTORM_CORE_CONN_AG_CTX_CF8_SHIFT     6
+#define TSTORM_CORE_CONN_AG_CTX_CF5_MASK        0x3
+#define TSTORM_CORE_CONN_AG_CTX_CF5_SHIFT       0
+#define TSTORM_CORE_CONN_AG_CTX_CF6_MASK        0x3
+#define TSTORM_CORE_CONN_AG_CTX_CF6_SHIFT       2
+#define TSTORM_CORE_CONN_AG_CTX_CF7_MASK        0x3
+#define TSTORM_CORE_CONN_AG_CTX_CF7_SHIFT       4
+#define TSTORM_CORE_CONN_AG_CTX_CF8_MASK        0x3
+#define TSTORM_CORE_CONN_AG_CTX_CF8_SHIFT       6
         u8 flags3;
-#define TSTORM_CORE_CONN_AG_CTX_CF9_MASK      0x3       /* cf9 */
-#define TSTORM_CORE_CONN_AG_CTX_CF9_SHIFT     0
-#define TSTORM_CORE_CONN_AG_CTX_CF10_MASK     0x3       /* cf10 */
-#define TSTORM_CORE_CONN_AG_CTX_CF10_SHIFT    2
-#define TSTORM_CORE_CONN_AG_CTX_CF0EN_MASK    0x1       /* cf0en */
-#define TSTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT   4
-#define TSTORM_CORE_CONN_AG_CTX_CF1EN_MASK    0x1       /* cf1en */
-#define TSTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT   5
-#define TSTORM_CORE_CONN_AG_CTX_CF2EN_MASK    0x1       /* cf2en */
-#define TSTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT   6
-#define TSTORM_CORE_CONN_AG_CTX_CF3EN_MASK    0x1       /* cf3en */
-#define TSTORM_CORE_CONN_AG_CTX_CF3EN_SHIFT   7
+#define TSTORM_CORE_CONN_AG_CTX_CF9_MASK        0x3
+#define TSTORM_CORE_CONN_AG_CTX_CF9_SHIFT       0
+#define TSTORM_CORE_CONN_AG_CTX_CF10_MASK       0x3
+#define TSTORM_CORE_CONN_AG_CTX_CF10_SHIFT      2
+#define TSTORM_CORE_CONN_AG_CTX_CF0EN_MASK      0x1
+#define TSTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT     4
+#define TSTORM_CORE_CONN_AG_CTX_CF1EN_MASK      0x1
+#define TSTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT     5
+#define TSTORM_CORE_CONN_AG_CTX_CF2EN_MASK      0x1
+#define TSTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT     6
+#define TSTORM_CORE_CONN_AG_CTX_CF3EN_MASK      0x1
+#define TSTORM_CORE_CONN_AG_CTX_CF3EN_SHIFT     7
         u8 flags4;
-#define TSTORM_CORE_CONN_AG_CTX_CF4EN_MASK    0x1       /* cf4en */
-#define TSTORM_CORE_CONN_AG_CTX_CF4EN_SHIFT   0
-#define TSTORM_CORE_CONN_AG_CTX_CF5EN_MASK    0x1       /* cf5en */
-#define TSTORM_CORE_CONN_AG_CTX_CF5EN_SHIFT   1
-#define TSTORM_CORE_CONN_AG_CTX_CF6EN_MASK    0x1       /* cf6en */
-#define TSTORM_CORE_CONN_AG_CTX_CF6EN_SHIFT   2
-#define TSTORM_CORE_CONN_AG_CTX_CF7EN_MASK    0x1       /* cf7en */
-#define TSTORM_CORE_CONN_AG_CTX_CF7EN_SHIFT   3
-#define TSTORM_CORE_CONN_AG_CTX_CF8EN_MASK    0x1       /* cf8en */
-#define TSTORM_CORE_CONN_AG_CTX_CF8EN_SHIFT   4
-#define TSTORM_CORE_CONN_AG_CTX_CF9EN_MASK    0x1       /* cf9en */
-#define TSTORM_CORE_CONN_AG_CTX_CF9EN_SHIFT   5
-#define TSTORM_CORE_CONN_AG_CTX_CF10EN_MASK   0x1       /* cf10en */
-#define TSTORM_CORE_CONN_AG_CTX_CF10EN_SHIFT  6
-#define TSTORM_CORE_CONN_AG_CTX_RULE0EN_MASK  0x1       /* rule0en */
-#define TSTORM_CORE_CONN_AG_CTX_RULE0EN_SHIFT 7
+#define TSTORM_CORE_CONN_AG_CTX_CF4EN_MASK      0x1
+#define TSTORM_CORE_CONN_AG_CTX_CF4EN_SHIFT     0
+#define TSTORM_CORE_CONN_AG_CTX_CF5EN_MASK      0x1
+#define TSTORM_CORE_CONN_AG_CTX_CF5EN_SHIFT     1
+#define TSTORM_CORE_CONN_AG_CTX_CF6EN_MASK      0x1
+#define TSTORM_CORE_CONN_AG_CTX_CF6EN_SHIFT     2
+#define TSTORM_CORE_CONN_AG_CTX_CF7EN_MASK      0x1
+#define TSTORM_CORE_CONN_AG_CTX_CF7EN_SHIFT     3
+#define TSTORM_CORE_CONN_AG_CTX_CF8EN_MASK      0x1
+#define TSTORM_CORE_CONN_AG_CTX_CF8EN_SHIFT     4
+#define TSTORM_CORE_CONN_AG_CTX_CF9EN_MASK      0x1
+#define TSTORM_CORE_CONN_AG_CTX_CF9EN_SHIFT     5
+#define TSTORM_CORE_CONN_AG_CTX_CF10EN_MASK     0x1
+#define TSTORM_CORE_CONN_AG_CTX_CF10EN_SHIFT    6
+#define TSTORM_CORE_CONN_AG_CTX_RULE0EN_MASK    0x1
+#define TSTORM_CORE_CONN_AG_CTX_RULE0EN_SHIFT   7
         u8 flags5;
-#define TSTORM_CORE_CONN_AG_CTX_RULE1EN_MASK  0x1       /* rule1en */
-#define TSTORM_CORE_CONN_AG_CTX_RULE1EN_SHIFT 0
-#define TSTORM_CORE_CONN_AG_CTX_RULE2EN_MASK  0x1       /* rule2en */
-#define TSTORM_CORE_CONN_AG_CTX_RULE2EN_SHIFT 1
-#define TSTORM_CORE_CONN_AG_CTX_RULE3EN_MASK  0x1       /* rule3en */
-#define TSTORM_CORE_CONN_AG_CTX_RULE3EN_SHIFT 2
-#define TSTORM_CORE_CONN_AG_CTX_RULE4EN_MASK  0x1       /* rule4en */
-#define TSTORM_CORE_CONN_AG_CTX_RULE4EN_SHIFT 3
-#define TSTORM_CORE_CONN_AG_CTX_RULE5EN_MASK  0x1       /* rule5en */
-#define TSTORM_CORE_CONN_AG_CTX_RULE5EN_SHIFT 4
-#define TSTORM_CORE_CONN_AG_CTX_RULE6EN_MASK  0x1       /* rule6en */
-#define TSTORM_CORE_CONN_AG_CTX_RULE6EN_SHIFT 5
-#define TSTORM_CORE_CONN_AG_CTX_RULE7EN_MASK  0x1       /* rule7en */
-#define TSTORM_CORE_CONN_AG_CTX_RULE7EN_SHIFT 6
-#define TSTORM_CORE_CONN_AG_CTX_RULE8EN_MASK  0x1       /* rule8en */
-#define TSTORM_CORE_CONN_AG_CTX_RULE8EN_SHIFT 7
-        __le32  reg0 /* reg0 */;
-        __le32  reg1 /* reg1 */;
-        __le32  reg2 /* reg2 */;
-        __le32  reg3 /* reg3 */;
-        __le32  reg4 /* reg4 */;
-        __le32  reg5 /* reg5 */;
-        __le32  reg6 /* reg6 */;
-        __le32  reg7 /* reg7 */;
-        __le32  reg8 /* reg8 */;
-        u8      byte2 /* byte2 */;
-        u8      byte3 /* byte3 */;
-        __le16  word0 /* word0 */;
-        u8      byte4 /* byte4 */;
-        u8      byte5 /* byte5 */;
-        __le16  word1 /* word1 */;
-        __le16  word2 /* conn_dpi */;
-        __le16  word3 /* word3 */;
-        __le32  reg9 /* reg9 */;
-        __le32  reg10 /* reg10 */;
+#define TSTORM_CORE_CONN_AG_CTX_RULE1EN_MASK    0x1
+#define TSTORM_CORE_CONN_AG_CTX_RULE1EN_SHIFT   0
+#define TSTORM_CORE_CONN_AG_CTX_RULE2EN_MASK    0x1
+#define TSTORM_CORE_CONN_AG_CTX_RULE2EN_SHIFT   1
+#define TSTORM_CORE_CONN_AG_CTX_RULE3EN_MASK    0x1
+#define TSTORM_CORE_CONN_AG_CTX_RULE3EN_SHIFT   2
+#define TSTORM_CORE_CONN_AG_CTX_RULE4EN_MASK    0x1
+#define TSTORM_CORE_CONN_AG_CTX_RULE4EN_SHIFT   3
+#define TSTORM_CORE_CONN_AG_CTX_RULE5EN_MASK    0x1
+#define TSTORM_CORE_CONN_AG_CTX_RULE5EN_SHIFT   4
+#define TSTORM_CORE_CONN_AG_CTX_RULE6EN_MASK    0x1
+#define TSTORM_CORE_CONN_AG_CTX_RULE6EN_SHIFT   5
+#define TSTORM_CORE_CONN_AG_CTX_RULE7EN_MASK    0x1
+#define TSTORM_CORE_CONN_AG_CTX_RULE7EN_SHIFT   6
+#define TSTORM_CORE_CONN_AG_CTX_RULE8EN_MASK    0x1
+#define TSTORM_CORE_CONN_AG_CTX_RULE8EN_SHIFT   7
+        __le32 reg0;
+        __le32 reg1;
+        __le32 reg2;
+        __le32 reg3;
+        __le32 reg4;
+        __le32 reg5;
+        __le32 reg6;
+        __le32 reg7;
+        __le32 reg8;
+        u8 byte2;
+        u8 byte3;
+        __le16 word0;
+        u8 byte4;
+        u8 byte5;
+        __le16 word1;
+        __le16 word2;
+        __le16 word3;
+        __le32 reg9;
+        __le32 reg10;
 };
 
 struct ustorm_core_conn_ag_ctx {
-        u8      reserved /* cdu_validation */;
-        u8      byte1 /* state */;
-        u8      flags0;
-#define USTORM_CORE_CONN_AG_CTX_BIT0_MASK     0x1       /* exist_in_qm0 */
-#define USTORM_CORE_CONN_AG_CTX_BIT0_SHIFT    0
-#define USTORM_CORE_CONN_AG_CTX_BIT1_MASK     0x1       /* exist_in_qm1 */
-#define USTORM_CORE_CONN_AG_CTX_BIT1_SHIFT    1
-#define USTORM_CORE_CONN_AG_CTX_CF0_MASK      0x3       /* timer0cf */
-#define USTORM_CORE_CONN_AG_CTX_CF0_SHIFT     2
-#define USTORM_CORE_CONN_AG_CTX_CF1_MASK      0x3       /* timer1cf */
-#define USTORM_CORE_CONN_AG_CTX_CF1_SHIFT     4
-#define USTORM_CORE_CONN_AG_CTX_CF2_MASK      0x3       /* timer2cf */
-#define USTORM_CORE_CONN_AG_CTX_CF2_SHIFT     6
+        u8 reserved;
+        u8 byte1;
+        u8 flags0;
+#define USTORM_CORE_CONN_AG_CTX_BIT0_MASK       0x1
+#define USTORM_CORE_CONN_AG_CTX_BIT0_SHIFT      0
+#define USTORM_CORE_CONN_AG_CTX_BIT1_MASK       0x1
+#define USTORM_CORE_CONN_AG_CTX_BIT1_SHIFT      1
+#define USTORM_CORE_CONN_AG_CTX_CF0_MASK        0x3
+#define USTORM_CORE_CONN_AG_CTX_CF0_SHIFT       2
+#define USTORM_CORE_CONN_AG_CTX_CF1_MASK        0x3
+#define USTORM_CORE_CONN_AG_CTX_CF1_SHIFT       4
+#define USTORM_CORE_CONN_AG_CTX_CF2_MASK        0x3
+#define USTORM_CORE_CONN_AG_CTX_CF2_SHIFT       6
         u8 flags1;
-#define USTORM_CORE_CONN_AG_CTX_CF3_MASK      0x3       /* timer_stop_all */
-#define USTORM_CORE_CONN_AG_CTX_CF3_SHIFT     0
-#define USTORM_CORE_CONN_AG_CTX_CF4_MASK      0x3       /* cf4 */
-#define USTORM_CORE_CONN_AG_CTX_CF4_SHIFT     2
-#define USTORM_CORE_CONN_AG_CTX_CF5_MASK      0x3       /* cf5 */
-#define USTORM_CORE_CONN_AG_CTX_CF5_SHIFT     4
-#define USTORM_CORE_CONN_AG_CTX_CF6_MASK      0x3       /* cf6 */
-#define USTORM_CORE_CONN_AG_CTX_CF6_SHIFT     6
+#define USTORM_CORE_CONN_AG_CTX_CF3_MASK        0x3
+#define USTORM_CORE_CONN_AG_CTX_CF3_SHIFT       0
+#define USTORM_CORE_CONN_AG_CTX_CF4_MASK        0x3
+#define USTORM_CORE_CONN_AG_CTX_CF4_SHIFT       2
+#define USTORM_CORE_CONN_AG_CTX_CF5_MASK        0x3
+#define USTORM_CORE_CONN_AG_CTX_CF5_SHIFT       4
+#define USTORM_CORE_CONN_AG_CTX_CF6_MASK        0x3
+#define USTORM_CORE_CONN_AG_CTX_CF6_SHIFT       6
         u8 flags2;
-#define USTORM_CORE_CONN_AG_CTX_CF0EN_MASK    0x1       /* cf0en */
-#define USTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT   0
-#define USTORM_CORE_CONN_AG_CTX_CF1EN_MASK    0x1       /* cf1en */
-#define USTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT   1
-#define USTORM_CORE_CONN_AG_CTX_CF2EN_MASK    0x1       /* cf2en */
-#define USTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT   2
-#define USTORM_CORE_CONN_AG_CTX_CF3EN_MASK    0x1       /* cf3en */
-#define USTORM_CORE_CONN_AG_CTX_CF3EN_SHIFT   3
-#define USTORM_CORE_CONN_AG_CTX_CF4EN_MASK    0x1       /* cf4en */
-#define USTORM_CORE_CONN_AG_CTX_CF4EN_SHIFT   4
-#define USTORM_CORE_CONN_AG_CTX_CF5EN_MASK    0x1       /* cf5en */
-#define USTORM_CORE_CONN_AG_CTX_CF5EN_SHIFT   5
-#define USTORM_CORE_CONN_AG_CTX_CF6EN_MASK    0x1       /* cf6en */
-#define USTORM_CORE_CONN_AG_CTX_CF6EN_SHIFT   6
-#define USTORM_CORE_CONN_AG_CTX_RULE0EN_MASK  0x1       /* rule0en */
-#define USTORM_CORE_CONN_AG_CTX_RULE0EN_SHIFT 7
+#define USTORM_CORE_CONN_AG_CTX_CF0EN_MASK      0x1
+#define USTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT     0
+#define USTORM_CORE_CONN_AG_CTX_CF1EN_MASK      0x1
+#define USTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT     1
+#define USTORM_CORE_CONN_AG_CTX_CF2EN_MASK      0x1
+#define USTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT     2
+#define USTORM_CORE_CONN_AG_CTX_CF3EN_MASK      0x1
+#define USTORM_CORE_CONN_AG_CTX_CF3EN_SHIFT     3
+#define USTORM_CORE_CONN_AG_CTX_CF4EN_MASK      0x1
+#define USTORM_CORE_CONN_AG_CTX_CF4EN_SHIFT     4
+#define USTORM_CORE_CONN_AG_CTX_CF5EN_MASK      0x1
+#define USTORM_CORE_CONN_AG_CTX_CF5EN_SHIFT     5
+#define USTORM_CORE_CONN_AG_CTX_CF6EN_MASK      0x1
+#define USTORM_CORE_CONN_AG_CTX_CF6EN_SHIFT     6
+#define USTORM_CORE_CONN_AG_CTX_RULE0EN_MASK    0x1
+#define USTORM_CORE_CONN_AG_CTX_RULE0EN_SHIFT   7
         u8 flags3;
-#define USTORM_CORE_CONN_AG_CTX_RULE1EN_MASK  0x1       /* rule1en */
-#define USTORM_CORE_CONN_AG_CTX_RULE1EN_SHIFT 0
-#define USTORM_CORE_CONN_AG_CTX_RULE2EN_MASK  0x1       /* rule2en */
-#define USTORM_CORE_CONN_AG_CTX_RULE2EN_SHIFT 1
-#define USTORM_CORE_CONN_AG_CTX_RULE3EN_MASK  0x1       /* rule3en */
-#define USTORM_CORE_CONN_AG_CTX_RULE3EN_SHIFT 2
-#define USTORM_CORE_CONN_AG_CTX_RULE4EN_MASK  0x1       /* rule4en */
-#define USTORM_CORE_CONN_AG_CTX_RULE4EN_SHIFT 3
-#define USTORM_CORE_CONN_AG_CTX_RULE5EN_MASK  0x1       /* rule5en */
-#define USTORM_CORE_CONN_AG_CTX_RULE5EN_SHIFT 4
-#define USTORM_CORE_CONN_AG_CTX_RULE6EN_MASK  0x1       /* rule6en */
-#define USTORM_CORE_CONN_AG_CTX_RULE6EN_SHIFT 5
-#define USTORM_CORE_CONN_AG_CTX_RULE7EN_MASK  0x1       /* rule7en */
-#define USTORM_CORE_CONN_AG_CTX_RULE7EN_SHIFT 6
-#define USTORM_CORE_CONN_AG_CTX_RULE8EN_MASK  0x1       /* rule8en */
-#define USTORM_CORE_CONN_AG_CTX_RULE8EN_SHIFT 7
-        u8      byte2 /* byte2 */;
-        u8      byte3 /* byte3 */;
-        __le16  word0 /* conn_dpi */;
-        __le16  word1 /* word1 */;
-        __le32  rx_producers /* reg0 */;
-        __le32  reg1 /* reg1 */;
-        __le32  reg2 /* reg2 */;
-        __le32  reg3 /* reg3 */;
-        __le16  word2 /* word2 */;
-        __le16  word3 /* word3 */;
+#define USTORM_CORE_CONN_AG_CTX_RULE1EN_MASK    0x1
+#define USTORM_CORE_CONN_AG_CTX_RULE1EN_SHIFT   0
+#define USTORM_CORE_CONN_AG_CTX_RULE2EN_MASK    0x1
+#define USTORM_CORE_CONN_AG_CTX_RULE2EN_SHIFT   1
+#define USTORM_CORE_CONN_AG_CTX_RULE3EN_MASK    0x1
+#define USTORM_CORE_CONN_AG_CTX_RULE3EN_SHIFT   2
+#define USTORM_CORE_CONN_AG_CTX_RULE4EN_MASK    0x1
+#define USTORM_CORE_CONN_AG_CTX_RULE4EN_SHIFT   3
+#define USTORM_CORE_CONN_AG_CTX_RULE5EN_MASK    0x1
+#define USTORM_CORE_CONN_AG_CTX_RULE5EN_SHIFT   4
+#define USTORM_CORE_CONN_AG_CTX_RULE6EN_MASK    0x1
+#define USTORM_CORE_CONN_AG_CTX_RULE6EN_SHIFT   5
+#define USTORM_CORE_CONN_AG_CTX_RULE7EN_MASK    0x1
+#define USTORM_CORE_CONN_AG_CTX_RULE7EN_SHIFT   6
+#define USTORM_CORE_CONN_AG_CTX_RULE8EN_MASK    0x1
+#define USTORM_CORE_CONN_AG_CTX_RULE8EN_SHIFT   7
+        u8 byte2;
+        u8 byte3;
+        __le16 word0;
+        __le16 word1;
+        __le32 rx_producers;
+        __le32 reg1;
+        __le32 reg2;
+        __le32 reg3;
+        __le16 word2;
+        __le16 word3;
 };
 
 /* The core storm context for the Mstorm */
@@ -519,122 +523,186 @@ struct ustorm_core_conn_st_ctx {
 
 /* core connection context */
 struct core_conn_context {
-        struct ystorm_core_conn_st_ctx  ystorm_st_context;
-        struct regpair                  ystorm_st_padding[2] /* padding */;
-        struct pstorm_core_conn_st_ctx  pstorm_st_context;
-        struct regpair                  pstorm_st_padding[2];
-        struct xstorm_core_conn_st_ctx  xstorm_st_context;
-        struct xstorm_core_conn_ag_ctx  xstorm_ag_context;
-        struct tstorm_core_conn_ag_ctx  tstorm_ag_context;
-        struct ustorm_core_conn_ag_ctx  ustorm_ag_context;
-        struct mstorm_core_conn_st_ctx  mstorm_st_context;
-        struct ustorm_core_conn_st_ctx  ustorm_st_context;
-        struct regpair                  ustorm_st_padding[2] /* padding */;
+        struct ystorm_core_conn_st_ctx ystorm_st_context;
+        struct regpair ystorm_st_padding[2];
+        struct pstorm_core_conn_st_ctx pstorm_st_context;
+        struct regpair pstorm_st_padding[2];
+        struct xstorm_core_conn_st_ctx xstorm_st_context;
+        struct xstorm_core_conn_ag_ctx xstorm_ag_context;
+        struct tstorm_core_conn_ag_ctx tstorm_ag_context;
+        struct ustorm_core_conn_ag_ctx ustorm_ag_context;
+        struct mstorm_core_conn_st_ctx mstorm_st_context;
+        struct ustorm_core_conn_st_ctx ustorm_st_context;
+        struct regpair ustorm_st_padding[2];
+};
+
+struct eth_mstorm_per_pf_stat {
+        struct regpair gre_discard_pkts;
+        struct regpair vxlan_discard_pkts;
+        struct regpair geneve_discard_pkts;
+        struct regpair lb_discard_pkts;
 };
 
 struct eth_mstorm_per_queue_stat {
-        struct regpair  ttl0_discard;
-        struct regpair  packet_too_big_discard;
-        struct regpair  no_buff_discard;
-        struct regpair  not_active_discard;
-        struct regpair  tpa_coalesced_pkts;
-        struct regpair  tpa_coalesced_events;
-        struct regpair  tpa_aborts_num;
-        struct regpair  tpa_coalesced_bytes;
+        struct regpair ttl0_discard;
+        struct regpair packet_too_big_discard;
+        struct regpair no_buff_discard;
+        struct regpair not_active_discard;
+        struct regpair tpa_coalesced_pkts;
+        struct regpair tpa_coalesced_events;
+        struct regpair tpa_aborts_num;
+        struct regpair tpa_coalesced_bytes;
+};
+
+/* Ethernet TX Per PF */
+struct eth_pstorm_per_pf_stat {
+        struct regpair sent_lb_ucast_bytes;
+        struct regpair sent_lb_mcast_bytes;
+        struct regpair sent_lb_bcast_bytes;
+        struct regpair sent_lb_ucast_pkts;
+        struct regpair sent_lb_mcast_pkts;
+        struct regpair sent_lb_bcast_pkts;
+        struct regpair sent_gre_bytes;
+        struct regpair sent_vxlan_bytes;
+        struct regpair sent_geneve_bytes;
+        struct regpair sent_gre_pkts;
+        struct regpair sent_vxlan_pkts;
+        struct regpair sent_geneve_pkts;
+        struct regpair gre_drop_pkts;
+        struct regpair vxlan_drop_pkts;
+        struct regpair geneve_drop_pkts;
+};
+
+/* Ethernet TX Per Queue Stats */
+struct eth_pstorm_per_queue_stat {
+        struct regpair sent_ucast_bytes;
+        struct regpair sent_mcast_bytes;
+        struct regpair sent_bcast_bytes;
+        struct regpair sent_ucast_pkts;
+        struct regpair sent_mcast_pkts;
+        struct regpair sent_bcast_pkts;
+        struct regpair error_drop_pkts;
+};
+
+/* ETH Rx producers data */
+struct eth_rx_rate_limit {
+        __le16 mult;
+        __le16 cnst;
+        u8 add_sub_cnst;
+        u8 reserved0;
+        __le16 reserved1;
 };
 
-struct eth_pstorm_per_queue_stat {
-        struct regpair  sent_ucast_bytes;
-        struct regpair  sent_mcast_bytes;
-        struct regpair  sent_bcast_bytes;
-        struct regpair  sent_ucast_pkts;
-        struct regpair  sent_mcast_pkts;
-        struct regpair  sent_bcast_pkts;
-        struct regpair  error_drop_pkts;
+struct eth_ustorm_per_pf_stat {
+        struct regpair rcv_lb_ucast_bytes;
+        struct regpair rcv_lb_mcast_bytes;
+        struct regpair rcv_lb_bcast_bytes;
+        struct regpair rcv_lb_ucast_pkts;
+        struct regpair rcv_lb_mcast_pkts;
+        struct regpair rcv_lb_bcast_pkts;
+        struct regpair rcv_gre_bytes;
+        struct regpair rcv_vxlan_bytes;
+        struct regpair rcv_geneve_bytes;
+        struct regpair rcv_gre_pkts;
+        struct regpair rcv_vxlan_pkts;
+        struct regpair rcv_geneve_pkts;
 };
 
 struct eth_ustorm_per_queue_stat {
-        struct regpair  rcv_ucast_bytes;
-        struct regpair  rcv_mcast_bytes;
-        struct regpair  rcv_bcast_bytes;
-        struct regpair  rcv_ucast_pkts;
-        struct regpair  rcv_mcast_pkts;
-        struct regpair  rcv_bcast_pkts;
+        struct regpair rcv_ucast_bytes;
+        struct regpair rcv_mcast_bytes;
+        struct regpair rcv_bcast_bytes;
+        struct regpair rcv_ucast_pkts;
+        struct regpair rcv_mcast_pkts;
+        struct regpair rcv_bcast_pkts;
 };
 
 /* Event Ring Next Page Address */
 struct event_ring_next_addr {
-        struct regpair  addr /* Next Page Address */;
-        __le32          reserved[2] /* Reserved */;
+        struct regpair addr;
+        __le32 reserved[2];
 };
 
+/* Event Ring Element */
 union event_ring_element {
-        struct event_ring_entry         entry /* Event Ring Entry */;
-        struct event_ring_next_addr     next_addr;
+        struct event_ring_entry entry;
+        struct event_ring_next_addr next_addr;
+};
+
+/* Major and Minor hsi Versions */
+struct hsi_fp_ver_struct {
+        u8 minor_ver_arr[2];
+        u8 major_ver_arr[2];
 };
 
+/* Mstorm non-triggering VF zone */
 struct mstorm_non_trigger_vf_zone {
         struct eth_mstorm_per_queue_stat eth_queue_stat;
+        struct eth_rx_prod_data eth_rx_queue_producers[ETH_MAX_NUM_RX_QUEUES_PER_VF];
 };
 
+/* Mstorm VF zone */
 struct mstorm_vf_zone {
         struct mstorm_non_trigger_vf_zone non_trigger;
+
 };
 
+/* personality per PF */
 enum personality_type {
         BAD_PERSONALITY_TYP,
-        PERSONALITY_RESERVED,
+        PERSONALITY_ISCSI,
         PERSONALITY_RESERVED2,
-        PERSONALITY_RDMA_AND_ETH /* Roce or Iwarp */,
+        PERSONALITY_RDMA_AND_ETH,
         PERSONALITY_RESERVED3,
         PERSONALITY_CORE,
-        PERSONALITY_ETH /* Ethernet */,
+        PERSONALITY_ETH,
         PERSONALITY_RESERVED4,
         MAX_PERSONALITY_TYPE
 };
 
+/* tunnel configuration */
 struct pf_start_tunnel_config {
-        u8      set_vxlan_udp_port_flg;
-        u8      set_geneve_udp_port_flg;
-        u8      tx_enable_vxlan /* If set, enable VXLAN tunnel in TX path. */;
-        u8      tx_enable_l2geneve;
-        u8      tx_enable_ipgeneve;
-        u8      tx_enable_l2gre /* If set, enable l2 GRE tunnel in TX path. */;
-        u8      tx_enable_ipgre /* If set, enable IP GRE tunnel in TX path. */;
-        u8      tunnel_clss_vxlan /* Classification scheme for VXLAN tunnel. */;
-        u8      tunnel_clss_l2geneve;
-        u8      tunnel_clss_ipgeneve;
-        u8      tunnel_clss_l2gre;
-        u8      tunnel_clss_ipgre;
-        __le16  vxlan_udp_port /* VXLAN tunnel UDP destination port. */;
-        __le16  geneve_udp_port /* GENEVE tunnel UDP destination port. */;
+        u8 set_vxlan_udp_port_flg;
+        u8 set_geneve_udp_port_flg;
+        u8 tx_enable_vxlan;
+        u8 tx_enable_l2geneve;
+        u8 tx_enable_ipgeneve;
+        u8 tx_enable_l2gre;
+        u8 tx_enable_ipgre;
+        u8 tunnel_clss_vxlan;
+        u8 tunnel_clss_l2geneve;
+        u8 tunnel_clss_ipgeneve;
+        u8 tunnel_clss_l2gre;
+        u8 tunnel_clss_ipgre;
+        __le16 vxlan_udp_port;
+        __le16 geneve_udp_port;
 };
 
 /* Ramrod data for PF start ramrod */
 struct pf_start_ramrod_data {
-        struct regpair                  event_ring_pbl_addr;
-        struct regpair                  consolid_q_pbl_addr;
-        struct pf_start_tunnel_config   tunnel_config;
-        __le16                          event_ring_sb_id;
-        u8                              base_vf_id;
-        u8                              num_vfs;
-        u8                              event_ring_num_pages;
-        u8                              event_ring_sb_index;
-        u8                              path_id;
-        u8                              warning_as_error;
-        u8                              dont_log_ramrods;
-        u8                              personality;
-        __le16                          log_type_mask;
-        u8                              mf_mode /* Multi function mode */;
-        u8                              integ_phase /* Integration phase */;
-        u8                              allow_npar_tx_switching;
-        u8                              inner_to_outer_pri_map[8];
-        u8                              pri_map_valid;
-        u32                             outer_tag;
-        u8                              reserved0[4];
-};
-
-/* Data for port update ramrod */
+        struct regpair event_ring_pbl_addr;
+        struct regpair consolid_q_pbl_addr;
+        struct pf_start_tunnel_config tunnel_config;
+        __le16 event_ring_sb_id;
+        u8 base_vf_id;
+        u8 num_vfs;
+        u8 event_ring_num_pages;
+        u8 event_ring_sb_index;
+        u8 path_id;
+        u8 warning_as_error;
+        u8 dont_log_ramrods;
+        u8 personality;
+        __le16 log_type_mask;
+        u8 mf_mode;
+        u8 integ_phase;
+        u8 allow_npar_tx_switching;
+        u8 inner_to_outer_pri_map[8];
+        u8 pri_map_valid;
+        __le32 outer_tag;
+        struct hsi_fp_ver_struct hsi_fp_ver;
+
+};
+
 struct protocol_dcb_data {
         u8 dcb_enable_flag;
         u8 dcb_priority;
@@ -642,25 +710,24 @@ struct protocol_dcb_data {
         u8 reserved;
 };
 
-/* tunnel configuration */
 struct pf_update_tunnel_config {
-        u8      update_rx_pf_clss;
-        u8      update_tx_pf_clss;
-        u8      set_vxlan_udp_port_flg;
-        u8      set_geneve_udp_port_flg;
-        u8      tx_enable_vxlan;
-        u8      tx_enable_l2geneve;
-        u8      tx_enable_ipgeneve;
-        u8      tx_enable_l2gre;
-        u8      tx_enable_ipgre;
-        u8      tunnel_clss_vxlan;
-        u8      tunnel_clss_l2geneve;
-        u8      tunnel_clss_ipgeneve;
-        u8      tunnel_clss_l2gre;
-        u8      tunnel_clss_ipgre;
-        __le16  vxlan_udp_port;
-        __le16  geneve_udp_port;
-        __le16  reserved[3];
+        u8 update_rx_pf_clss;
+        u8 update_tx_pf_clss;
+        u8 set_vxlan_udp_port_flg;
+        u8 set_geneve_udp_port_flg;
+        u8 tx_enable_vxlan;
+        u8 tx_enable_l2geneve;
+        u8 tx_enable_ipgeneve;
+        u8 tx_enable_l2gre;
+        u8 tx_enable_ipgre;
+        u8 tunnel_clss_vxlan;
+        u8 tunnel_clss_l2geneve;
+        u8 tunnel_clss_ipgeneve;
+        u8 tunnel_clss_l2gre;
+        u8 tunnel_clss_ipgre;
+        __le16 vxlan_udp_port;
+        __le16 geneve_udp_port;
+        __le16 reserved[3];
 };
 
 struct pf_update_ramrod_data {
@@ -669,38 +736,43 @@ struct pf_update_ramrod_data {
         u8 update_fcoe_dcb_data_flag;
         u8 update_iscsi_dcb_data_flag;
         u8 update_roce_dcb_data_flag;
+        u8 update_iwarp_dcb_data_flag;
         u8 update_mf_vlan_flag;
-        __le16 mf_vlan;
+        u8 reserved;
         struct protocol_dcb_data eth_dcb_data;
         struct protocol_dcb_data fcoe_dcb_data;
         struct protocol_dcb_data iscsi_dcb_data;
         struct protocol_dcb_data roce_dcb_data;
-        struct pf_update_tunnel_config  tunnel_config;
-};
-
-/* Tunnel classification scheme */
-enum tunnel_clss {
-        TUNNEL_CLSS_MAC_VLAN = 0,
-        TUNNEL_CLSS_MAC_VNI,
-        TUNNEL_CLSS_INNER_MAC_VLAN,
-        TUNNEL_CLSS_INNER_MAC_VNI,
-        MAX_TUNNEL_CLSS
+        struct protocol_dcb_data iwarp_dcb_data;
+        __le16 mf_vlan;
+        __le16 reserved2;
+        struct pf_update_tunnel_config tunnel_config;
 };
 
+/* Ports mode */
 enum ports_mode {
-        ENGX2_PORTX1 /* 2 engines x 1 port */,
-        ENGX2_PORTX2 /* 2 engines x 2 ports */,
-        ENGX1_PORTX1 /* 1 engine  x 1 port */,
-        ENGX1_PORTX2 /* 1 engine  x 2 ports */,
-        ENGX1_PORTX4 /* 1 engine  x 4 ports */,
+        ENGX2_PORTX1,
+        ENGX2_PORTX2,
+        ENGX1_PORTX1,
+        ENGX1_PORTX2,
+        ENGX1_PORTX4,
         MAX_PORTS_MODE
 };
 
+/* use to index in hsi_fp_[major|minor]_ver_arr per protocol */
+enum protocol_version_array_key {
+        ETH_VER_KEY = 0,
+        ROCE_VER_KEY,
+        MAX_PROTOCOL_VERSION_ARRAY_KEY
+};
+
+/* Pstorm non-triggering VF zone */
 struct pstorm_non_trigger_vf_zone {
         struct eth_pstorm_per_queue_stat eth_queue_stat;
         struct regpair reserved[2];
 };
 
+/* Pstorm VF zone */
 struct pstorm_vf_zone {
         struct pstorm_non_trigger_vf_zone non_trigger;
         struct regpair reserved[7];
@@ -708,56 +780,89 @@ struct pstorm_vf_zone {
 
 /* Ramrod Header of SPQE */
 struct ramrod_header {
-        __le32  cid /* Slowpath Connection CID */;
-        u8      cmd_id /* Ramrod Cmd (Per Protocol Type) */;
-        u8      protocol_id /* Ramrod Protocol ID */;
-        __le16  echo /* Ramrod echo */;
+        __le32 cid;
+        u8 cmd_id;
+        u8 protocol_id;
+        __le16 echo;
 };
 
 /* Slowpath Element (SPQE) */
 struct slow_path_element {
-        struct ramrod_header    hdr /* Ramrod Header */;
-        struct regpair          data_ptr;
+        struct ramrod_header hdr;
+        struct regpair data_ptr;
+};
+
+/* Tstorm non-triggering VF zone */
+struct tstorm_non_trigger_vf_zone {
+        struct regpair reserved[2];
 };
 
 struct tstorm_per_port_stat {
-        struct regpair  trunc_error_discard;
-        struct regpair  mac_error_discard;
-        struct regpair  mftag_filter_discard;
-        struct regpair  eth_mac_filter_discard;
-        struct regpair  ll2_mac_filter_discard;
-        struct regpair  ll2_conn_disabled_discard;
-        struct regpair  iscsi_irregular_pkt;
-        struct regpair  fcoe_irregular_pkt;
-        struct regpair  roce_irregular_pkt;
-        struct regpair  eth_irregular_pkt;
-        struct regpair  toe_irregular_pkt;
-        struct regpair  preroce_irregular_pkt;
+        struct regpair trunc_error_discard;
+        struct regpair mac_error_discard;
+        struct regpair mftag_filter_discard;
+        struct regpair eth_mac_filter_discard;
+        struct regpair reserved[5];
+        struct regpair eth_irregular_pkt;
+        struct regpair reserved1[2];
+        struct regpair eth_gre_tunn_filter_discard;
+        struct regpair eth_vxlan_tunn_filter_discard;
+        struct regpair eth_geneve_tunn_filter_discard;
+};
+
+/* Tstorm VF zone */
+struct tstorm_vf_zone {
+        struct tstorm_non_trigger_vf_zone non_trigger;
+};
+
+/* Tunnel classification scheme */
+enum tunnel_clss {
+        TUNNEL_CLSS_MAC_VLAN = 0,
+        TUNNEL_CLSS_MAC_VNI,
+        TUNNEL_CLSS_INNER_MAC_VLAN,
+        TUNNEL_CLSS_INNER_MAC_VNI,
+        TUNNEL_CLSS_MAC_VLAN_DUAL_STAGE,
+        MAX_TUNNEL_CLSS
 };
 
+/* Ustorm non-triggering VF zone */
 struct ustorm_non_trigger_vf_zone {
         struct eth_ustorm_per_queue_stat eth_queue_stat;
         struct regpair vf_pf_msg_addr;
 };
 
+/* Ustorm triggering VF zone */
 struct ustorm_trigger_vf_zone {
         u8 vf_pf_msg_valid;
         u8 reserved[7];
 };
 
+/* Ustorm VF zone */
 struct ustorm_vf_zone {
         struct ustorm_non_trigger_vf_zone non_trigger;
         struct ustorm_trigger_vf_zone trigger;
 };
 
+/* VF-PF channel data */
+struct vf_pf_channel_data {
+        __le32 ready;
+        u8 valid;
+        u8 reserved0;
+        __le16 reserved1;
+};
+
+/* Ramrod data for VF start ramrod */
 struct vf_start_ramrod_data {
         u8 vf_id;
         u8 enable_flr_ack;
         __le16 opaque_fid;
         u8 personality;
-        u8 reserved[3];
+        u8 reserved[7];
+        struct hsi_fp_ver_struct hsi_fp_ver;
+
 };
 
+/* Ramrod data for VF start ramrod */
 struct vf_stop_ramrod_data {
         u8 vf_id;
         u8 reserved0;
@@ -765,94 +870,474 @@ struct vf_stop_ramrod_data {
         __le32 reserved2;
 };
 
+/* Attentions status block */
 struct atten_status_block {
-        __le32  atten_bits;
-        __le32  atten_ack;
-        __le16  reserved0;
-        __le16  sb_index /* status block running index */;
-        __le32  reserved1;
+        __le32 atten_bits;
+        __le32 atten_ack;
+        __le16 reserved0;
+        __le16 sb_index;
+        __le32 reserved1;
+};
+
+enum command_type_bit {
+        IGU_COMMAND_TYPE_NOP = 0,
+        IGU_COMMAND_TYPE_SET = 1,
+        MAX_COMMAND_TYPE_BIT
+};
+
+/* DMAE command */
+struct dmae_cmd {
+        __le32 opcode;
+#define DMAE_CMD_SRC_MASK               0x1
+#define DMAE_CMD_SRC_SHIFT              0
+#define DMAE_CMD_DST_MASK               0x3
+#define DMAE_CMD_DST_SHIFT              1
+#define DMAE_CMD_C_DST_MASK             0x1
+#define DMAE_CMD_C_DST_SHIFT            3
+#define DMAE_CMD_CRC_RESET_MASK         0x1
+#define DMAE_CMD_CRC_RESET_SHIFT        4
+#define DMAE_CMD_SRC_ADDR_RESET_MASK    0x1
+#define DMAE_CMD_SRC_ADDR_RESET_SHIFT   5
+#define DMAE_CMD_DST_ADDR_RESET_MASK    0x1
+#define DMAE_CMD_DST_ADDR_RESET_SHIFT   6
+#define DMAE_CMD_COMP_FUNC_MASK         0x1
+#define DMAE_CMD_COMP_FUNC_SHIFT        7
+#define DMAE_CMD_COMP_WORD_EN_MASK      0x1
+#define DMAE_CMD_COMP_WORD_EN_SHIFT     8
+#define DMAE_CMD_COMP_CRC_EN_MASK       0x1
+#define DMAE_CMD_COMP_CRC_EN_SHIFT      9
+#define DMAE_CMD_COMP_CRC_OFFSET_MASK   0x7
+#define DMAE_CMD_COMP_CRC_OFFSET_SHIFT 10
+#define DMAE_CMD_RESERVED1_MASK         0x1
+#define DMAE_CMD_RESERVED1_SHIFT        13
+#define DMAE_CMD_ENDIANITY_MODE_MASK    0x3
+#define DMAE_CMD_ENDIANITY_MODE_SHIFT   14
+#define DMAE_CMD_ERR_HANDLING_MASK      0x3
+#define DMAE_CMD_ERR_HANDLING_SHIFT     16
+#define DMAE_CMD_PORT_ID_MASK           0x3
+#define DMAE_CMD_PORT_ID_SHIFT          18
+#define DMAE_CMD_SRC_PF_ID_MASK         0xF
+#define DMAE_CMD_SRC_PF_ID_SHIFT        20
+#define DMAE_CMD_DST_PF_ID_MASK         0xF
+#define DMAE_CMD_DST_PF_ID_SHIFT        24
+#define DMAE_CMD_SRC_VF_ID_VALID_MASK   0x1
+#define DMAE_CMD_SRC_VF_ID_VALID_SHIFT 28
+#define DMAE_CMD_DST_VF_ID_VALID_MASK   0x1
+#define DMAE_CMD_DST_VF_ID_VALID_SHIFT 29
+#define DMAE_CMD_RESERVED2_MASK         0x3
+#define DMAE_CMD_RESERVED2_SHIFT        30
+        __le32 src_addr_lo;
+        __le32 src_addr_hi;
+        __le32 dst_addr_lo;
+        __le32 dst_addr_hi;
+        __le16 length_dw;
+        __le16 opcode_b;
+#define DMAE_CMD_SRC_VF_ID_MASK         0xFF
+#define DMAE_CMD_SRC_VF_ID_SHIFT        0
+#define DMAE_CMD_DST_VF_ID_MASK         0xFF
+#define DMAE_CMD_DST_VF_ID_SHIFT        8
+        __le32 comp_addr_lo;
+        __le32 comp_addr_hi;
+        __le32 comp_val;
+        __le32 crc32;
+        __le32 crc_32_c;
+        __le16 crc16;
+        __le16 crc16_c;
+        __le16 crc10;
+        __le16 reserved;
+        __le16 xsum16;
+        __le16 xsum8;
+};
+
+enum dmae_cmd_comp_crc_en_enum {
+        dmae_cmd_comp_crc_disabled,
+        dmae_cmd_comp_crc_enabled,
+        MAX_DMAE_CMD_COMP_CRC_EN_ENUM
+};
+
+enum dmae_cmd_comp_func_enum {
+        dmae_cmd_comp_func_to_src,
+        dmae_cmd_comp_func_to_dst,
+        MAX_DMAE_CMD_COMP_FUNC_ENUM
+};
+
+enum dmae_cmd_comp_word_en_enum {
+        dmae_cmd_comp_word_disabled,
+        dmae_cmd_comp_word_enabled,
+        MAX_DMAE_CMD_COMP_WORD_EN_ENUM
+};
+
+enum dmae_cmd_c_dst_enum {
+        dmae_cmd_c_dst_pcie,
+        dmae_cmd_c_dst_grc,
+        MAX_DMAE_CMD_C_DST_ENUM
+};
+
+enum dmae_cmd_dst_enum {
+        dmae_cmd_dst_none_0,
+        dmae_cmd_dst_pcie,
+        dmae_cmd_dst_grc,
+        dmae_cmd_dst_none_3,
+        MAX_DMAE_CMD_DST_ENUM
+};
+
+enum dmae_cmd_error_handling_enum {
+        dmae_cmd_error_handling_send_regular_comp,
+        dmae_cmd_error_handling_send_comp_with_err,
+        dmae_cmd_error_handling_dont_send_comp,
+        MAX_DMAE_CMD_ERROR_HANDLING_ENUM
+};
+
+enum dmae_cmd_src_enum {
+        dmae_cmd_src_pcie,
+        dmae_cmd_src_grc,
+        MAX_DMAE_CMD_SRC_ENUM
+};
+
+/* IGU cleanup command */
+struct igu_cleanup {
+        __le32 sb_id_and_flags;
+#define IGU_CLEANUP_RESERVED0_MASK      0x7FFFFFF
+#define IGU_CLEANUP_RESERVED0_SHIFT     0
+#define IGU_CLEANUP_CLEANUP_SET_MASK    0x1
+#define IGU_CLEANUP_CLEANUP_SET_SHIFT   27
+#define IGU_CLEANUP_CLEANUP_TYPE_MASK   0x7
+#define IGU_CLEANUP_CLEANUP_TYPE_SHIFT  28
+#define IGU_CLEANUP_COMMAND_TYPE_MASK   0x1
+#define IGU_CLEANUP_COMMAND_TYPE_SHIFT  31
+        __le32 reserved1;
+};
+
+/* IGU firmware driver command */
+union igu_command {
+        struct igu_prod_cons_update prod_cons_update;
+        struct igu_cleanup cleanup;
+};
+
+/* IGU firmware driver command */
+struct igu_command_reg_ctrl {
+        __le16 opaque_fid;
+        __le16 igu_command_reg_ctrl_fields;
+#define IGU_COMMAND_REG_CTRL_PXP_BAR_ADDR_MASK  0xFFF
+#define IGU_COMMAND_REG_CTRL_PXP_BAR_ADDR_SHIFT 0
+#define IGU_COMMAND_REG_CTRL_RESERVED_MASK      0x7
+#define IGU_COMMAND_REG_CTRL_RESERVED_SHIFT     12
+#define IGU_COMMAND_REG_CTRL_COMMAND_TYPE_MASK  0x1
+#define IGU_COMMAND_REG_CTRL_COMMAND_TYPE_SHIFT 15
 };
 
+/* IGU mapping line structure */
+struct igu_mapping_line {
+        __le32 igu_mapping_line_fields;
+#define IGU_MAPPING_LINE_VALID_MASK             0x1
+#define IGU_MAPPING_LINE_VALID_SHIFT            0
+#define IGU_MAPPING_LINE_VECTOR_NUMBER_MASK     0xFF
+#define IGU_MAPPING_LINE_VECTOR_NUMBER_SHIFT    1
+#define IGU_MAPPING_LINE_FUNCTION_NUMBER_MASK   0xFF
+#define IGU_MAPPING_LINE_FUNCTION_NUMBER_SHIFT  9
+#define IGU_MAPPING_LINE_PF_VALID_MASK          0x1
+#define IGU_MAPPING_LINE_PF_VALID_SHIFT         17
+#define IGU_MAPPING_LINE_IPS_GROUP_MASK         0x3F
+#define IGU_MAPPING_LINE_IPS_GROUP_SHIFT        18
+#define IGU_MAPPING_LINE_RESERVED_MASK          0xFF
+#define IGU_MAPPING_LINE_RESERVED_SHIFT         24
+};
+
+/* IGU MSIX line structure */
+struct igu_msix_vector {
+        struct regpair address;
+        __le32 data;
+        __le32 msix_vector_fields;
+#define IGU_MSIX_VECTOR_MASK_BIT_MASK           0x1
+#define IGU_MSIX_VECTOR_MASK_BIT_SHIFT          0
+#define IGU_MSIX_VECTOR_RESERVED0_MASK          0x7FFF
+#define IGU_MSIX_VECTOR_RESERVED0_SHIFT         1
+#define IGU_MSIX_VECTOR_STEERING_TAG_MASK       0xFF
+#define IGU_MSIX_VECTOR_STEERING_TAG_SHIFT      16
+#define IGU_MSIX_VECTOR_RESERVED1_MASK          0xFF
+#define IGU_MSIX_VECTOR_RESERVED1_SHIFT         24
+};
+
+struct mstorm_core_conn_ag_ctx {
+        u8 byte0;
+        u8 byte1;
+        u8 flags0;
+#define MSTORM_CORE_CONN_AG_CTX_BIT0_MASK       0x1
+#define MSTORM_CORE_CONN_AG_CTX_BIT0_SHIFT      0
+#define MSTORM_CORE_CONN_AG_CTX_BIT1_MASK       0x1
+#define MSTORM_CORE_CONN_AG_CTX_BIT1_SHIFT      1
+#define MSTORM_CORE_CONN_AG_CTX_CF0_MASK        0x3
+#define MSTORM_CORE_CONN_AG_CTX_CF0_SHIFT       2
+#define MSTORM_CORE_CONN_AG_CTX_CF1_MASK        0x3
+#define MSTORM_CORE_CONN_AG_CTX_CF1_SHIFT       4
+#define MSTORM_CORE_CONN_AG_CTX_CF2_MASK        0x3
+#define MSTORM_CORE_CONN_AG_CTX_CF2_SHIFT       6
+        u8 flags1;
+#define MSTORM_CORE_CONN_AG_CTX_CF0EN_MASK      0x1
+#define MSTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT     0
+#define MSTORM_CORE_CONN_AG_CTX_CF1EN_MASK      0x1
+#define MSTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT     1
+#define MSTORM_CORE_CONN_AG_CTX_CF2EN_MASK      0x1
+#define MSTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT     2
+#define MSTORM_CORE_CONN_AG_CTX_RULE0EN_MASK    0x1
+#define MSTORM_CORE_CONN_AG_CTX_RULE0EN_SHIFT   3
+#define MSTORM_CORE_CONN_AG_CTX_RULE1EN_MASK    0x1
+#define MSTORM_CORE_CONN_AG_CTX_RULE1EN_SHIFT   4
+#define MSTORM_CORE_CONN_AG_CTX_RULE2EN_MASK    0x1
+#define MSTORM_CORE_CONN_AG_CTX_RULE2EN_SHIFT   5
+#define MSTORM_CORE_CONN_AG_CTX_RULE3EN_MASK    0x1
+#define MSTORM_CORE_CONN_AG_CTX_RULE3EN_SHIFT   6
+#define MSTORM_CORE_CONN_AG_CTX_RULE4EN_MASK    0x1
+#define MSTORM_CORE_CONN_AG_CTX_RULE4EN_SHIFT   7
+        __le16 word0;
+        __le16 word1;
+        __le32 reg0;
+        __le32 reg1;
+};
+
+/* per encapsulation type enabling flags */
+struct prs_reg_encapsulation_type_en {
+        u8 flags;
+#define PRS_REG_ENCAPSULATION_TYPE_EN_ETH_OVER_GRE_ENABLE_MASK          0x1
+#define PRS_REG_ENCAPSULATION_TYPE_EN_ETH_OVER_GRE_ENABLE_SHIFT         0
+#define PRS_REG_ENCAPSULATION_TYPE_EN_IP_OVER_GRE_ENABLE_MASK           0x1
+#define PRS_REG_ENCAPSULATION_TYPE_EN_IP_OVER_GRE_ENABLE_SHIFT          1
+#define PRS_REG_ENCAPSULATION_TYPE_EN_VXLAN_ENABLE_MASK                 0x1
+#define PRS_REG_ENCAPSULATION_TYPE_EN_VXLAN_ENABLE_SHIFT                2
+#define PRS_REG_ENCAPSULATION_TYPE_EN_T_TAG_ENABLE_MASK                 0x1
+#define PRS_REG_ENCAPSULATION_TYPE_EN_T_TAG_ENABLE_SHIFT                3
+#define PRS_REG_ENCAPSULATION_TYPE_EN_ETH_OVER_GENEVE_ENABLE_MASK       0x1
+#define PRS_REG_ENCAPSULATION_TYPE_EN_ETH_OVER_GENEVE_ENABLE_SHIFT      4
+#define PRS_REG_ENCAPSULATION_TYPE_EN_IP_OVER_GENEVE_ENABLE_MASK        0x1
+#define PRS_REG_ENCAPSULATION_TYPE_EN_IP_OVER_GENEVE_ENABLE_SHIFT       5
+#define PRS_REG_ENCAPSULATION_TYPE_EN_RESERVED_MASK                     0x3
+#define PRS_REG_ENCAPSULATION_TYPE_EN_RESERVED_SHIFT                    6
+};
+
+enum pxp_tph_st_hint {
+        TPH_ST_HINT_BIDIR,
+        TPH_ST_HINT_REQUESTER,
+        TPH_ST_HINT_TARGET,
+        TPH_ST_HINT_TARGET_PRIO,
+        MAX_PXP_TPH_ST_HINT
+};
+
+/* QM hardware structure of enable bypass credit mask */
+struct qm_rf_bypass_mask {
+        u8 flags;
+#define QM_RF_BYPASS_MASK_LINEVOQ_MASK          0x1
+#define QM_RF_BYPASS_MASK_LINEVOQ_SHIFT         0
+#define QM_RF_BYPASS_MASK_RESERVED0_MASK        0x1
+#define QM_RF_BYPASS_MASK_RESERVED0_SHIFT       1
+#define QM_RF_BYPASS_MASK_PFWFQ_MASK            0x1
+#define QM_RF_BYPASS_MASK_PFWFQ_SHIFT           2
+#define QM_RF_BYPASS_MASK_VPWFQ_MASK            0x1
+#define QM_RF_BYPASS_MASK_VPWFQ_SHIFT           3
+#define QM_RF_BYPASS_MASK_PFRL_MASK             0x1
+#define QM_RF_BYPASS_MASK_PFRL_SHIFT            4
+#define QM_RF_BYPASS_MASK_VPQCNRL_MASK          0x1
+#define QM_RF_BYPASS_MASK_VPQCNRL_SHIFT         5
+#define QM_RF_BYPASS_MASK_FWPAUSE_MASK          0x1
+#define QM_RF_BYPASS_MASK_FWPAUSE_SHIFT         6
+#define QM_RF_BYPASS_MASK_RESERVED1_MASK        0x1
+#define QM_RF_BYPASS_MASK_RESERVED1_SHIFT       7
+};
+
+/* QM hardware structure of opportunistic credit mask */
+struct qm_rf_opportunistic_mask {
+        __le16 flags;
+#define QM_RF_OPPORTUNISTIC_MASK_LINEVOQ_MASK           0x1
+#define QM_RF_OPPORTUNISTIC_MASK_LINEVOQ_SHIFT          0
+#define QM_RF_OPPORTUNISTIC_MASK_BYTEVOQ_MASK           0x1
+#define QM_RF_OPPORTUNISTIC_MASK_BYTEVOQ_SHIFT          1
+#define QM_RF_OPPORTUNISTIC_MASK_PFWFQ_MASK             0x1
+#define QM_RF_OPPORTUNISTIC_MASK_PFWFQ_SHIFT            2
+#define QM_RF_OPPORTUNISTIC_MASK_VPWFQ_MASK             0x1
+#define QM_RF_OPPORTUNISTIC_MASK_VPWFQ_SHIFT            3
+#define QM_RF_OPPORTUNISTIC_MASK_PFRL_MASK              0x1
+#define QM_RF_OPPORTUNISTIC_MASK_PFRL_SHIFT             4
+#define QM_RF_OPPORTUNISTIC_MASK_VPQCNRL_MASK           0x1
+#define QM_RF_OPPORTUNISTIC_MASK_VPQCNRL_SHIFT          5
+#define QM_RF_OPPORTUNISTIC_MASK_FWPAUSE_MASK           0x1
+#define QM_RF_OPPORTUNISTIC_MASK_FWPAUSE_SHIFT          6
+#define QM_RF_OPPORTUNISTIC_MASK_RESERVED0_MASK         0x1
+#define QM_RF_OPPORTUNISTIC_MASK_RESERVED0_SHIFT        7
+#define QM_RF_OPPORTUNISTIC_MASK_QUEUEEMPTY_MASK        0x1
+#define QM_RF_OPPORTUNISTIC_MASK_QUEUEEMPTY_SHIFT       8
+#define QM_RF_OPPORTUNISTIC_MASK_RESERVED1_MASK         0x7F
+#define QM_RF_OPPORTUNISTIC_MASK_RESERVED1_SHIFT        9
+};
+
+/* QM hardware structure of QM map memory */
+struct qm_rf_pq_map {
+        __le32 reg;
+#define QM_RF_PQ_MAP_PQ_VALID_MASK              0x1
+#define QM_RF_PQ_MAP_PQ_VALID_SHIFT             0
+#define QM_RF_PQ_MAP_RL_ID_MASK                 0xFF
+#define QM_RF_PQ_MAP_RL_ID_SHIFT                1
+#define QM_RF_PQ_MAP_VP_PQ_ID_MASK              0x1FF
+#define QM_RF_PQ_MAP_VP_PQ_ID_SHIFT             9
+#define QM_RF_PQ_MAP_VOQ_MASK                   0x1F
+#define QM_RF_PQ_MAP_VOQ_SHIFT                  18
+#define QM_RF_PQ_MAP_WRR_WEIGHT_GROUP_MASK      0x3
+#define QM_RF_PQ_MAP_WRR_WEIGHT_GROUP_SHIFT     23
+#define QM_RF_PQ_MAP_RL_VALID_MASK              0x1
+#define QM_RF_PQ_MAP_RL_VALID_SHIFT             25
+#define QM_RF_PQ_MAP_RESERVED_MASK              0x3F
+#define QM_RF_PQ_MAP_RESERVED_SHIFT             26
+};
+
+/* Completion params for aggregated interrupt completion */
+struct sdm_agg_int_comp_params {
+        __le16 params;
+#define SDM_AGG_INT_COMP_PARAMS_AGG_INT_INDEX_MASK      0x3F
+#define SDM_AGG_INT_COMP_PARAMS_AGG_INT_INDEX_SHIFT     0
+#define SDM_AGG_INT_COMP_PARAMS_AGG_VECTOR_ENABLE_MASK  0x1
+#define SDM_AGG_INT_COMP_PARAMS_AGG_VECTOR_ENABLE_SHIFT 6
+#define SDM_AGG_INT_COMP_PARAMS_AGG_VECTOR_BIT_MASK     0x1FF
+#define SDM_AGG_INT_COMP_PARAMS_AGG_VECTOR_BIT_SHIFT    7
+};
+
+/* SDM operation gen command (generate aggregative interrupt) */
+struct sdm_op_gen {
+        __le32 command;
+#define SDM_OP_GEN_COMP_PARAM_MASK      0xFFFF
+#define SDM_OP_GEN_COMP_PARAM_SHIFT     0
+#define SDM_OP_GEN_COMP_TYPE_MASK       0xF
+#define SDM_OP_GEN_COMP_TYPE_SHIFT      16
+#define SDM_OP_GEN_RESERVED_MASK        0xFFF
+#define SDM_OP_GEN_RESERVED_SHIFT       20
+};
+
+struct ystorm_core_conn_ag_ctx {
+        u8 byte0;
+        u8 byte1;
+        u8 flags0;
+#define YSTORM_CORE_CONN_AG_CTX_BIT0_MASK       0x1
+#define YSTORM_CORE_CONN_AG_CTX_BIT0_SHIFT      0
+#define YSTORM_CORE_CONN_AG_CTX_BIT1_MASK       0x1
+#define YSTORM_CORE_CONN_AG_CTX_BIT1_SHIFT      1
+#define YSTORM_CORE_CONN_AG_CTX_CF0_MASK        0x3
+#define YSTORM_CORE_CONN_AG_CTX_CF0_SHIFT       2
+#define YSTORM_CORE_CONN_AG_CTX_CF1_MASK        0x3
+#define YSTORM_CORE_CONN_AG_CTX_CF1_SHIFT       4
+#define YSTORM_CORE_CONN_AG_CTX_CF2_MASK        0x3
+#define YSTORM_CORE_CONN_AG_CTX_CF2_SHIFT       6
+        u8 flags1;
+#define YSTORM_CORE_CONN_AG_CTX_CF0EN_MASK      0x1
+#define YSTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT     0
+#define YSTORM_CORE_CONN_AG_CTX_CF1EN_MASK      0x1
+#define YSTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT     1
+#define YSTORM_CORE_CONN_AG_CTX_CF2EN_MASK      0x1
+#define YSTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT     2
+#define YSTORM_CORE_CONN_AG_CTX_RULE0EN_MASK    0x1
+#define YSTORM_CORE_CONN_AG_CTX_RULE0EN_SHIFT   3
+#define YSTORM_CORE_CONN_AG_CTX_RULE1EN_MASK    0x1
+#define YSTORM_CORE_CONN_AG_CTX_RULE1EN_SHIFT   4
+#define YSTORM_CORE_CONN_AG_CTX_RULE2EN_MASK    0x1
+#define YSTORM_CORE_CONN_AG_CTX_RULE2EN_SHIFT   5
+#define YSTORM_CORE_CONN_AG_CTX_RULE3EN_MASK    0x1
+#define YSTORM_CORE_CONN_AG_CTX_RULE3EN_SHIFT   6
+#define YSTORM_CORE_CONN_AG_CTX_RULE4EN_MASK    0x1
+#define YSTORM_CORE_CONN_AG_CTX_RULE4EN_SHIFT   7
+        u8 byte2;
+        u8 byte3;
+        __le16 word0;
+        __le32 reg0;
+        __le32 reg1;
+        __le16 word1;
+        __le16 word2;
+        __le16 word3;
+        __le16 word4;
+        __le32 reg2;
+        __le32 reg3;
+};
+
+/****************************************/
+/* Debug Tools HSI constants and macros */
+/****************************************/
+
 enum block_addr {
-        GRCBASE_GRC             = 0x50000,
-        GRCBASE_MISCS           = 0x9000,
-        GRCBASE_MISC            = 0x8000,
-        GRCBASE_DBU             = 0xa000,
-        GRCBASE_PGLUE_B         = 0x2a8000,
-        GRCBASE_CNIG            = 0x218000,
-        GRCBASE_CPMU            = 0x30000,
-        GRCBASE_NCSI            = 0x40000,
-        GRCBASE_OPTE            = 0x53000,
-        GRCBASE_BMB             = 0x540000,
-        GRCBASE_PCIE            = 0x54000,
-        GRCBASE_MCP             = 0xe00000,
-        GRCBASE_MCP2            = 0x52000,
-        GRCBASE_PSWHST          = 0x2a0000,
-        GRCBASE_PSWHST2         = 0x29e000,
-        GRCBASE_PSWRD           = 0x29c000,
-        GRCBASE_PSWRD2          = 0x29d000,
-        GRCBASE_PSWWR           = 0x29a000,
-        GRCBASE_PSWWR2          = 0x29b000,
-        GRCBASE_PSWRQ           = 0x280000,
-        GRCBASE_PSWRQ2          = 0x240000,
-        GRCBASE_PGLCS           = 0x0,
-        GRCBASE_PTU             = 0x560000,
-        GRCBASE_DMAE            = 0xc000,
-        GRCBASE_TCM             = 0x1180000,
-        GRCBASE_MCM             = 0x1200000,
-        GRCBASE_UCM             = 0x1280000,
-        GRCBASE_XCM             = 0x1000000,
-        GRCBASE_YCM             = 0x1080000,
-        GRCBASE_PCM             = 0x1100000,
-        GRCBASE_QM              = 0x2f0000,
-        GRCBASE_TM              = 0x2c0000,
-        GRCBASE_DORQ            = 0x100000,
-        GRCBASE_BRB             = 0x340000,
-        GRCBASE_SRC             = 0x238000,
-        GRCBASE_PRS             = 0x1f0000,
-        GRCBASE_TSDM            = 0xfb0000,
-        GRCBASE_MSDM            = 0xfc0000,
-        GRCBASE_USDM            = 0xfd0000,
-        GRCBASE_XSDM            = 0xf80000,
-        GRCBASE_YSDM            = 0xf90000,
-        GRCBASE_PSDM            = 0xfa0000,
-        GRCBASE_TSEM            = 0x1700000,
-        GRCBASE_MSEM            = 0x1800000,
-        GRCBASE_USEM            = 0x1900000,
-        GRCBASE_XSEM            = 0x1400000,
-        GRCBASE_YSEM            = 0x1500000,
-        GRCBASE_PSEM            = 0x1600000,
-        GRCBASE_RSS             = 0x238800,
-        GRCBASE_TMLD            = 0x4d0000,
-        GRCBASE_MULD            = 0x4e0000,
-        GRCBASE_YULD            = 0x4c8000,
-        GRCBASE_XYLD            = 0x4c0000,
-        GRCBASE_PRM             = 0x230000,
-        GRCBASE_PBF_PB1         = 0xda0000,
-        GRCBASE_PBF_PB2         = 0xda4000,
-        GRCBASE_RPB             = 0x23c000,
-        GRCBASE_BTB             = 0xdb0000,
-        GRCBASE_PBF             = 0xd80000,
-        GRCBASE_RDIF            = 0x300000,
-        GRCBASE_TDIF            = 0x310000,
-        GRCBASE_CDU             = 0x580000,
-        GRCBASE_CCFC            = 0x2e0000,
-        GRCBASE_TCFC            = 0x2d0000,
-        GRCBASE_IGU             = 0x180000,
-        GRCBASE_CAU             = 0x1c0000,
-        GRCBASE_UMAC            = 0x51000,
-        GRCBASE_XMAC            = 0x210000,
-        GRCBASE_DBG             = 0x10000,
-        GRCBASE_NIG             = 0x500000,
-        GRCBASE_WOL             = 0x600000,
-        GRCBASE_BMBN            = 0x610000,
-        GRCBASE_IPC             = 0x20000,
-        GRCBASE_NWM             = 0x800000,
-        GRCBASE_NWS             = 0x700000,
-        GRCBASE_MS              = 0x6a0000,
-        GRCBASE_PHY_PCIE        = 0x620000,
-        GRCBASE_MISC_AEU        = 0x8000,
-        GRCBASE_BAR0_MAP        = 0x1c00000,
+        GRCBASE_GRC = 0x50000,
+        GRCBASE_MISCS = 0x9000,
+        GRCBASE_MISC = 0x8000,
+        GRCBASE_DBU = 0xa000,
+        GRCBASE_PGLUE_B = 0x2a8000,
+        GRCBASE_CNIG = 0x218000,
+        GRCBASE_CPMU = 0x30000,
+        GRCBASE_NCSI = 0x40000,
+        GRCBASE_OPTE = 0x53000,
+        GRCBASE_BMB = 0x540000,
+        GRCBASE_PCIE = 0x54000,
+        GRCBASE_MCP = 0xe00000,
+        GRCBASE_MCP2 = 0x52000,
+        GRCBASE_PSWHST = 0x2a0000,
+        GRCBASE_PSWHST2 = 0x29e000,
+        GRCBASE_PSWRD = 0x29c000,
+        GRCBASE_PSWRD2 = 0x29d000,
+        GRCBASE_PSWWR = 0x29a000,
+        GRCBASE_PSWWR2 = 0x29b000,
+        GRCBASE_PSWRQ = 0x280000,
+        GRCBASE_PSWRQ2 = 0x240000,
+        GRCBASE_PGLCS = 0x0,
+        GRCBASE_DMAE = 0xc000,
+        GRCBASE_PTU = 0x560000,
+        GRCBASE_TCM = 0x1180000,
+        GRCBASE_MCM = 0x1200000,
+        GRCBASE_UCM = 0x1280000,
+        GRCBASE_XCM = 0x1000000,
+        GRCBASE_YCM = 0x1080000,
+        GRCBASE_PCM = 0x1100000,
+        GRCBASE_QM = 0x2f0000,
+        GRCBASE_TM = 0x2c0000,
+        GRCBASE_DORQ = 0x100000,
+        GRCBASE_BRB = 0x340000,
+        GRCBASE_SRC = 0x238000,
+        GRCBASE_PRS = 0x1f0000,
+        GRCBASE_TSDM = 0xfb0000,
+        GRCBASE_MSDM = 0xfc0000,
+        GRCBASE_USDM = 0xfd0000,
+        GRCBASE_XSDM = 0xf80000,
+        GRCBASE_YSDM = 0xf90000,
+        GRCBASE_PSDM = 0xfa0000,
+        GRCBASE_TSEM = 0x1700000,
+        GRCBASE_MSEM = 0x1800000,
+        GRCBASE_USEM = 0x1900000,
+        GRCBASE_XSEM = 0x1400000,
+        GRCBASE_YSEM = 0x1500000,
+        GRCBASE_PSEM = 0x1600000,
+        GRCBASE_RSS = 0x238800,
+        GRCBASE_TMLD = 0x4d0000,
+        GRCBASE_MULD = 0x4e0000,
+        GRCBASE_YULD = 0x4c8000,
+        GRCBASE_XYLD = 0x4c0000,
+        GRCBASE_PRM = 0x230000,
+        GRCBASE_PBF_PB1 = 0xda0000,
+        GRCBASE_PBF_PB2 = 0xda4000,
+        GRCBASE_RPB = 0x23c000,
+        GRCBASE_BTB = 0xdb0000,
+        GRCBASE_PBF = 0xd80000,
+        GRCBASE_RDIF = 0x300000,
+        GRCBASE_TDIF = 0x310000,
+        GRCBASE_CDU = 0x580000,
+        GRCBASE_CCFC = 0x2e0000,
+        GRCBASE_TCFC = 0x2d0000,
+        GRCBASE_IGU = 0x180000,
+        GRCBASE_CAU = 0x1c0000,
+        GRCBASE_UMAC = 0x51000,
+        GRCBASE_XMAC = 0x210000,
+        GRCBASE_DBG = 0x10000,
+        GRCBASE_NIG = 0x500000,
+        GRCBASE_WOL = 0x600000,
+        GRCBASE_BMBN = 0x610000,
+        GRCBASE_IPC = 0x20000,
+        GRCBASE_NWM = 0x800000,
+        GRCBASE_NWS = 0x700000,
+        GRCBASE_MS = 0x6a0000,
+        GRCBASE_PHY_PCIE = 0x620000,
+        GRCBASE_LED = 0x6b8000,
+        GRCBASE_MISC_AEU = 0x8000,
+        GRCBASE_BAR0_MAP = 0x1c00000,
         MAX_BLOCK_ADDR
 };
 
@@ -879,8 +1364,8 @@ enum block_id {
         BLOCK_PSWRQ,
         BLOCK_PSWRQ2,
         BLOCK_PGLCS,
-        BLOCK_PTU,
         BLOCK_DMAE,
+        BLOCK_PTU,
         BLOCK_TCM,
         BLOCK_MCM,
         BLOCK_UCM,
@@ -934,141 +1419,216 @@ enum block_id {
         BLOCK_NWS,
         BLOCK_MS,
         BLOCK_PHY_PCIE,
+        BLOCK_LED,
         BLOCK_MISC_AEU,
         BLOCK_BAR0_MAP,
         MAX_BLOCK_ID
 };
 
-enum command_type_bit {
-        IGU_COMMAND_TYPE_NOP    = 0,
-        IGU_COMMAND_TYPE_SET    = 1,
-        MAX_COMMAND_TYPE_BIT
+/* binary debug buffer types */
+enum bin_dbg_buffer_type {
+        BIN_BUF_DBG_MODE_TREE,
+        BIN_BUF_DBG_DUMP_REG,
+        BIN_BUF_DBG_DUMP_MEM,
+        BIN_BUF_DBG_IDLE_CHK_REGS,
+        BIN_BUF_DBG_IDLE_CHK_IMMS,
+        BIN_BUF_DBG_IDLE_CHK_RULES,
+        BIN_BUF_DBG_IDLE_CHK_PARSING_DATA,
+        BIN_BUF_DBG_ATTN_BLOCKS,
+        BIN_BUF_DBG_ATTN_REGS,
+        BIN_BUF_DBG_ATTN_INDEXES,
+        BIN_BUF_DBG_ATTN_NAME_OFFSETS,
+        BIN_BUF_DBG_PARSING_STRINGS,
+        MAX_BIN_DBG_BUFFER_TYPE
 };
 
-struct dmae_cmd {
-        __le32 opcode;
-#define DMAE_CMD_SRC_MASK              0x1
-#define DMAE_CMD_SRC_SHIFT             0
-#define DMAE_CMD_DST_MASK              0x3
-#define DMAE_CMD_DST_SHIFT             1
-#define DMAE_CMD_C_DST_MASK            0x1
-#define DMAE_CMD_C_DST_SHIFT           3
-#define DMAE_CMD_CRC_RESET_MASK        0x1
-#define DMAE_CMD_CRC_RESET_SHIFT       4
-#define DMAE_CMD_SRC_ADDR_RESET_MASK   0x1
-#define DMAE_CMD_SRC_ADDR_RESET_SHIFT  5
-#define DMAE_CMD_DST_ADDR_RESET_MASK   0x1
-#define DMAE_CMD_DST_ADDR_RESET_SHIFT  6
-#define DMAE_CMD_COMP_FUNC_MASK        0x1
-#define DMAE_CMD_COMP_FUNC_SHIFT       7
-#define DMAE_CMD_COMP_WORD_EN_MASK     0x1
-#define DMAE_CMD_COMP_WORD_EN_SHIFT    8
-#define DMAE_CMD_COMP_CRC_EN_MASK      0x1
-#define DMAE_CMD_COMP_CRC_EN_SHIFT     9
-#define DMAE_CMD_COMP_CRC_OFFSET_MASK  0x7
-#define DMAE_CMD_COMP_CRC_OFFSET_SHIFT 10
-#define DMAE_CMD_RESERVED1_MASK        0x1
-#define DMAE_CMD_RESERVED1_SHIFT       13
-#define DMAE_CMD_ENDIANITY_MODE_MASK   0x3
-#define DMAE_CMD_ENDIANITY_MODE_SHIFT  14
-#define DMAE_CMD_ERR_HANDLING_MASK     0x3
-#define DMAE_CMD_ERR_HANDLING_SHIFT    16
-#define DMAE_CMD_PORT_ID_MASK          0x3
-#define DMAE_CMD_PORT_ID_SHIFT         18
-#define DMAE_CMD_SRC_PF_ID_MASK        0xF
-#define DMAE_CMD_SRC_PF_ID_SHIFT       20
-#define DMAE_CMD_DST_PF_ID_MASK        0xF
-#define DMAE_CMD_DST_PF_ID_SHIFT       24
-#define DMAE_CMD_SRC_VF_ID_VALID_MASK  0x1
-#define DMAE_CMD_SRC_VF_ID_VALID_SHIFT 28
-#define DMAE_CMD_DST_VF_ID_VALID_MASK  0x1
-#define DMAE_CMD_DST_VF_ID_VALID_SHIFT 29
-#define DMAE_CMD_RESERVED2_MASK        0x3
-#define DMAE_CMD_RESERVED2_SHIFT       30
-        __le32  src_addr_lo;
-        __le32  src_addr_hi;
-        __le32  dst_addr_lo;
-        __le32  dst_addr_hi;
-        __le16  length /* Length in DW */;
-        __le16  opcode_b;
-#define DMAE_CMD_SRC_VF_ID_MASK        0xFF     /* Source VF id */
-#define DMAE_CMD_SRC_VF_ID_SHIFT       0
-#define DMAE_CMD_DST_VF_ID_MASK        0xFF     /* Destination VF id */
-#define DMAE_CMD_DST_VF_ID_SHIFT       8
-        __le32  comp_addr_lo /* PCIe completion address low or grc address */;
-        __le32  comp_addr_hi;
-        __le32  comp_val /* Value to write to copmletion address */;
-        __le32  crc32 /* crc16 result */;
-        __le32  crc_32_c /* crc32_c result */;
-        __le16  crc16 /* crc16 result */;
-        __le16  crc16_c /* crc16_c result */;
-        __le16  crc10 /* crc_t10 result */;
-        __le16  reserved;
-        __le16  xsum16 /* checksum16 result  */;
-        __le16  xsum8 /* checksum8 result  */;
+/* Chip IDs */
+enum chip_ids {
+        CHIP_RESERVED,
+        CHIP_BB_B0,
+        CHIP_RESERVED2,
+        MAX_CHIP_IDS
 };
 
-struct igu_cleanup {
-        __le32 sb_id_and_flags;
-#define IGU_CLEANUP_RESERVED0_MASK     0x7FFFFFF
-#define IGU_CLEANUP_RESERVED0_SHIFT    0
-#define IGU_CLEANUP_CLEANUP_SET_MASK   0x1 /* cleanup clear - 0, set - 1 */
-#define IGU_CLEANUP_CLEANUP_SET_SHIFT  27
-#define IGU_CLEANUP_CLEANUP_TYPE_MASK  0x7
-#define IGU_CLEANUP_CLEANUP_TYPE_SHIFT 28
-#define IGU_CLEANUP_COMMAND_TYPE_MASK  0x1
-#define IGU_CLEANUP_COMMAND_TYPE_SHIFT 31
-        __le32 reserved1;
+/* Attention bit mapping */
+struct dbg_attn_bit_mapping {
+        __le16 data;
+#define DBG_ATTN_BIT_MAPPING_VAL_MASK                   0x7FFF
+#define DBG_ATTN_BIT_MAPPING_VAL_SHIFT                  0
+#define DBG_ATTN_BIT_MAPPING_IS_UNUSED_BIT_CNT_MASK     0x1
+#define DBG_ATTN_BIT_MAPPING_IS_UNUSED_BIT_CNT_SHIFT    15
 };
 
-union igu_command {
-        struct igu_prod_cons_update     prod_cons_update;
-        struct igu_cleanup              cleanup;
+/* Attention block per-type data */
+struct dbg_attn_block_type_data {
+        __le16 names_offset;
+        __le16 reserved1;
+        u8 num_regs;
+        u8 reserved2;
+        __le16 regs_offset;
 };
 
-struct igu_command_reg_ctrl {
-        __le16  opaque_fid;
-        __le16  igu_command_reg_ctrl_fields;
-#define IGU_COMMAND_REG_CTRL_PXP_BAR_ADDR_MASK  0xFFF
-#define IGU_COMMAND_REG_CTRL_PXP_BAR_ADDR_SHIFT 0
-#define IGU_COMMAND_REG_CTRL_RESERVED_MASK      0x7
-#define IGU_COMMAND_REG_CTRL_RESERVED_SHIFT     12
-#define IGU_COMMAND_REG_CTRL_COMMAND_TYPE_MASK  0x1
-#define IGU_COMMAND_REG_CTRL_COMMAND_TYPE_SHIFT 15
+/* Block attentions */
+struct dbg_attn_block {
+        struct dbg_attn_block_type_data per_type_data[2];
 };
 
-struct igu_mapping_line {
-        __le32 igu_mapping_line_fields;
-#define IGU_MAPPING_LINE_VALID_MASK            0x1
-#define IGU_MAPPING_LINE_VALID_SHIFT           0
-#define IGU_MAPPING_LINE_VECTOR_NUMBER_MASK    0xFF
-#define IGU_MAPPING_LINE_VECTOR_NUMBER_SHIFT   1
-#define IGU_MAPPING_LINE_FUNCTION_NUMBER_MASK  0xFF
-#define IGU_MAPPING_LINE_FUNCTION_NUMBER_SHIFT 9
-#define IGU_MAPPING_LINE_PF_VALID_MASK         0x1      /* PF-1, VF-0 */
-#define IGU_MAPPING_LINE_PF_VALID_SHIFT        17
-#define IGU_MAPPING_LINE_IPS_GROUP_MASK        0x3F
-#define IGU_MAPPING_LINE_IPS_GROUP_SHIFT       18
-#define IGU_MAPPING_LINE_RESERVED_MASK         0xFF
-#define IGU_MAPPING_LINE_RESERVED_SHIFT        24
+/* Attention register result */
+struct dbg_attn_reg_result {
+        __le32 data;
+#define DBG_ATTN_REG_RESULT_STS_ADDRESS_MASK    0xFFFFFF
+#define DBG_ATTN_REG_RESULT_STS_ADDRESS_SHIFT   0
+#define DBG_ATTN_REG_RESULT_NUM_ATTN_IDX_MASK   0xFF
+#define DBG_ATTN_REG_RESULT_NUM_ATTN_IDX_SHIFT  24
+        __le16 attn_idx_offset;
+        __le16 reserved;
+        __le32 sts_val;
+        __le32 mask_val;
+};
+
+/* Attention block result */
+struct dbg_attn_block_result {
+        u8 block_id;
+        u8 data;
+#define DBG_ATTN_BLOCK_RESULT_ATTN_TYPE_MASK    0x3
+#define DBG_ATTN_BLOCK_RESULT_ATTN_TYPE_SHIFT   0
+#define DBG_ATTN_BLOCK_RESULT_NUM_REGS_MASK     0x3F
+#define DBG_ATTN_BLOCK_RESULT_NUM_REGS_SHIFT    2
+        __le16 names_offset;
+        struct dbg_attn_reg_result reg_results[15];
+};
+
+/* mode header */
+struct dbg_mode_hdr {
+        __le16 data;
+#define DBG_MODE_HDR_EVAL_MODE_MASK             0x1
+#define DBG_MODE_HDR_EVAL_MODE_SHIFT            0
+#define DBG_MODE_HDR_MODES_BUF_OFFSET_MASK      0x7FFF
+#define DBG_MODE_HDR_MODES_BUF_OFFSET_SHIFT     1
+};
+
+/* Attention register */
+struct dbg_attn_reg {
+        struct dbg_mode_hdr mode;
+        __le16 attn_idx_offset;
+        __le32 data;
+#define DBG_ATTN_REG_STS_ADDRESS_MASK   0xFFFFFF
+#define DBG_ATTN_REG_STS_ADDRESS_SHIFT  0
+#define DBG_ATTN_REG_NUM_ATTN_IDX_MASK  0xFF
+#define DBG_ATTN_REG_NUM_ATTN_IDX_SHIFT 24
+        __le32 sts_clr_address;
+        __le32 mask_address;
+};
+
+/* attention types */
+enum dbg_attn_type {
+        ATTN_TYPE_INTERRUPT,
+        ATTN_TYPE_PARITY,
+        MAX_DBG_ATTN_TYPE
+};
+
+/* Debug status codes */
+enum dbg_status {
+        DBG_STATUS_OK,
+        DBG_STATUS_APP_VERSION_NOT_SET,
+        DBG_STATUS_UNSUPPORTED_APP_VERSION,
+        DBG_STATUS_DBG_BLOCK_NOT_RESET,
+        DBG_STATUS_INVALID_ARGS,
+        DBG_STATUS_OUTPUT_ALREADY_SET,
+        DBG_STATUS_INVALID_PCI_BUF_SIZE,
+        DBG_STATUS_PCI_BUF_ALLOC_FAILED,
+        DBG_STATUS_PCI_BUF_NOT_ALLOCATED,
+        DBG_STATUS_TOO_MANY_INPUTS,
+        DBG_STATUS_INPUT_OVERLAP,
+        DBG_STATUS_HW_ONLY_RECORDING,
+        DBG_STATUS_STORM_ALREADY_ENABLED,
+        DBG_STATUS_STORM_NOT_ENABLED,
+        DBG_STATUS_BLOCK_ALREADY_ENABLED,
+        DBG_STATUS_BLOCK_NOT_ENABLED,
+        DBG_STATUS_NO_INPUT_ENABLED,
+        DBG_STATUS_NO_FILTER_TRIGGER_64B,
+        DBG_STATUS_FILTER_ALREADY_ENABLED,
+        DBG_STATUS_TRIGGER_ALREADY_ENABLED,
+        DBG_STATUS_TRIGGER_NOT_ENABLED,
+        DBG_STATUS_CANT_ADD_CONSTRAINT,
+        DBG_STATUS_TOO_MANY_TRIGGER_STATES,
+        DBG_STATUS_TOO_MANY_CONSTRAINTS,
+        DBG_STATUS_RECORDING_NOT_STARTED,
+        DBG_STATUS_DATA_DIDNT_TRIGGER,
+        DBG_STATUS_NO_DATA_RECORDED,
+        DBG_STATUS_DUMP_BUF_TOO_SMALL,
+        DBG_STATUS_DUMP_NOT_CHUNK_ALIGNED,
+        DBG_STATUS_UNKNOWN_CHIP,
+        DBG_STATUS_VIRT_MEM_ALLOC_FAILED,
+        DBG_STATUS_BLOCK_IN_RESET,
+        DBG_STATUS_INVALID_TRACE_SIGNATURE,
+        DBG_STATUS_INVALID_NVRAM_BUNDLE,
+        DBG_STATUS_NVRAM_GET_IMAGE_FAILED,
+        DBG_STATUS_NON_ALIGNED_NVRAM_IMAGE,
+        DBG_STATUS_NVRAM_READ_FAILED,
+        DBG_STATUS_IDLE_CHK_PARSE_FAILED,
+        DBG_STATUS_MCP_TRACE_BAD_DATA,
+        DBG_STATUS_MCP_TRACE_NO_META,
+        DBG_STATUS_MCP_COULD_NOT_HALT,
+        DBG_STATUS_MCP_COULD_NOT_RESUME,
+        DBG_STATUS_DMAE_FAILED,
+        DBG_STATUS_SEMI_FIFO_NOT_EMPTY,
+        DBG_STATUS_IGU_FIFO_BAD_DATA,
+        DBG_STATUS_MCP_COULD_NOT_MASK_PRTY,
+        DBG_STATUS_FW_ASSERTS_PARSE_FAILED,
+        DBG_STATUS_REG_FIFO_BAD_DATA,
+        DBG_STATUS_PROTECTION_OVERRIDE_BAD_DATA,
+        DBG_STATUS_DBG_ARRAY_NOT_SET,
+        MAX_DBG_STATUS
 };
 
-struct igu_msix_vector {
-        struct regpair  address;
-        __le32          data;
-        __le32          msix_vector_fields;
-#define IGU_MSIX_VECTOR_MASK_BIT_MASK      0x1
-#define IGU_MSIX_VECTOR_MASK_BIT_SHIFT     0
-#define IGU_MSIX_VECTOR_RESERVED0_MASK     0x7FFF
-#define IGU_MSIX_VECTOR_RESERVED0_SHIFT    1
-#define IGU_MSIX_VECTOR_STEERING_TAG_MASK  0xFF
-#define IGU_MSIX_VECTOR_STEERING_TAG_SHIFT 16
-#define IGU_MSIX_VECTOR_RESERVED1_MASK     0xFF
-#define IGU_MSIX_VECTOR_RESERVED1_SHIFT    24
+/********************************/
+/* HSI Init Functions constants */
+/********************************/
+
+/* Number of VLAN priorities */
+#define NUM_OF_VLAN_PRIORITIES  8
+
+/* QM per-port init parameters */
+struct init_qm_port_params {
+        u8 active;
+        u8 active_phys_tcs;
+        __le16 num_pbf_cmd_lines;
+        __le16 num_btb_blocks;
+        __le16 reserved;
 };
 
+/* QM per-PQ init parameters */
+struct init_qm_pq_params {
+        u8 vport_id;
+        u8 tc_id;
+        u8 wrr_group;
+        u8 rl_valid;
+};
+
+/* QM per-vport init parameters */
+struct init_qm_vport_params {
+        __le32 vport_rl;
+        __le16 vport_wfq;
+        __le16 first_tx_pq_id[NUM_OF_TCS];
+};
+
+/**************************************/
+/* Init Tool HSI constants and macros */
+/**************************************/
+
+/* Width of GRC address in bits (addresses are specified in dwords) */
+#define GRC_ADDR_BITS   23
+#define MAX_GRC_ADDR    ((1 << GRC_ADDR_BITS) - 1)
+
+/* indicates an init that should be applied to any phase ID */
+#define ANY_PHASE_ID    0xffff
+
+/* Max size in dwords of a zipped array */
+#define MAX_ZIPPED_SIZE 8192
+
 enum init_modes {
-        MODE_BB_A0,
+        MODE_RESERVED,
         MODE_BB_B0,
         MODE_RESERVED2,
         MODE_ASIC,
@@ -1083,7 +1643,8 @@ enum init_modes {
         MODE_PORTS_PER_ENG_2,
         MODE_PORTS_PER_ENG_4,
         MODE_100G,
-        MODE_EAGLE_ENG1_WORKAROUND,
+        MODE_40G,
+        MODE_RESERVED7,
         MAX_INIT_MODES
 };
 
@@ -1096,484 +1657,302 @@ enum init_phases {
         MAX_INIT_PHASES
 };
 
-/* per encapsulation type enabling flags */
-struct prs_reg_encapsulation_type_en {
-        u8 flags;
-#define PRS_REG_ENCAPSULATION_TYPE_EN_ETH_OVER_GRE_ENABLE_MASK     0x1
-#define PRS_REG_ENCAPSULATION_TYPE_EN_ETH_OVER_GRE_ENABLE_SHIFT    0
-#define PRS_REG_ENCAPSULATION_TYPE_EN_IP_OVER_GRE_ENABLE_MASK      0x1
-#define PRS_REG_ENCAPSULATION_TYPE_EN_IP_OVER_GRE_ENABLE_SHIFT     1
-#define PRS_REG_ENCAPSULATION_TYPE_EN_VXLAN_ENABLE_MASK            0x1
-#define PRS_REG_ENCAPSULATION_TYPE_EN_VXLAN_ENABLE_SHIFT           2
-#define PRS_REG_ENCAPSULATION_TYPE_EN_T_TAG_ENABLE_MASK            0x1
-#define PRS_REG_ENCAPSULATION_TYPE_EN_T_TAG_ENABLE_SHIFT           3
-#define PRS_REG_ENCAPSULATION_TYPE_EN_ETH_OVER_GENEVE_ENABLE_MASK  0x1
-#define PRS_REG_ENCAPSULATION_TYPE_EN_ETH_OVER_GENEVE_ENABLE_SHIFT 4
-#define PRS_REG_ENCAPSULATION_TYPE_EN_IP_OVER_GENEVE_ENABLE_MASK   0x1
-#define PRS_REG_ENCAPSULATION_TYPE_EN_IP_OVER_GENEVE_ENABLE_SHIFT  5
-#define PRS_REG_ENCAPSULATION_TYPE_EN_RESERVED_MASK                0x3
-#define PRS_REG_ENCAPSULATION_TYPE_EN_RESERVED_SHIFT               6
-};
-
-enum pxp_tph_st_hint {
-        TPH_ST_HINT_BIDIR /* Read/Write access by Host and Device */,
-        TPH_ST_HINT_REQUESTER /* Read/Write access by Device */,
-        TPH_ST_HINT_TARGET,
-        TPH_ST_HINT_TARGET_PRIO,
-        MAX_PXP_TPH_ST_HINT
-};
-
-/* QM hardware structure of enable bypass credit mask */
-struct qm_rf_bypass_mask {
-        u8 flags;
-#define QM_RF_BYPASS_MASK_LINEVOQ_MASK    0x1
-#define QM_RF_BYPASS_MASK_LINEVOQ_SHIFT   0
-#define QM_RF_BYPASS_MASK_RESERVED0_MASK  0x1
-#define QM_RF_BYPASS_MASK_RESERVED0_SHIFT 1
-#define QM_RF_BYPASS_MASK_PFWFQ_MASK      0x1
-#define QM_RF_BYPASS_MASK_PFWFQ_SHIFT     2
-#define QM_RF_BYPASS_MASK_VPWFQ_MASK      0x1
-#define QM_RF_BYPASS_MASK_VPWFQ_SHIFT     3
-#define QM_RF_BYPASS_MASK_PFRL_MASK       0x1
-#define QM_RF_BYPASS_MASK_PFRL_SHIFT      4
-#define QM_RF_BYPASS_MASK_VPQCNRL_MASK    0x1
-#define QM_RF_BYPASS_MASK_VPQCNRL_SHIFT   5
-#define QM_RF_BYPASS_MASK_FWPAUSE_MASK    0x1
-#define QM_RF_BYPASS_MASK_FWPAUSE_SHIFT   6
-#define QM_RF_BYPASS_MASK_RESERVED1_MASK  0x1
-#define QM_RF_BYPASS_MASK_RESERVED1_SHIFT 7
-};
-
-/* QM hardware structure of opportunistic credit mask */
-struct qm_rf_opportunistic_mask {
-        __le16 flags;
-#define QM_RF_OPPORTUNISTIC_MASK_LINEVOQ_MASK     0x1
-#define QM_RF_OPPORTUNISTIC_MASK_LINEVOQ_SHIFT    0
-#define QM_RF_OPPORTUNISTIC_MASK_BYTEVOQ_MASK     0x1
-#define QM_RF_OPPORTUNISTIC_MASK_BYTEVOQ_SHIFT    1
-#define QM_RF_OPPORTUNISTIC_MASK_PFWFQ_MASK       0x1
-#define QM_RF_OPPORTUNISTIC_MASK_PFWFQ_SHIFT      2
-#define QM_RF_OPPORTUNISTIC_MASK_VPWFQ_MASK       0x1
-#define QM_RF_OPPORTUNISTIC_MASK_VPWFQ_SHIFT      3
-#define QM_RF_OPPORTUNISTIC_MASK_PFRL_MASK        0x1
-#define QM_RF_OPPORTUNISTIC_MASK_PFRL_SHIFT       4
-#define QM_RF_OPPORTUNISTIC_MASK_VPQCNRL_MASK     0x1
-#define QM_RF_OPPORTUNISTIC_MASK_VPQCNRL_SHIFT    5
-#define QM_RF_OPPORTUNISTIC_MASK_FWPAUSE_MASK     0x1
-#define QM_RF_OPPORTUNISTIC_MASK_FWPAUSE_SHIFT    6
-#define QM_RF_OPPORTUNISTIC_MASK_RESERVED0_MASK   0x1
-#define QM_RF_OPPORTUNISTIC_MASK_RESERVED0_SHIFT  7
-#define QM_RF_OPPORTUNISTIC_MASK_QUEUEEMPTY_MASK  0x1
-#define QM_RF_OPPORTUNISTIC_MASK_QUEUEEMPTY_SHIFT 8
-#define QM_RF_OPPORTUNISTIC_MASK_RESERVED1_MASK   0x7F
-#define QM_RF_OPPORTUNISTIC_MASK_RESERVED1_SHIFT  9
-};
-
-/* QM hardware structure of QM map memory */
-struct qm_rf_pq_map {
-        u32 reg;
-#define QM_RF_PQ_MAP_PQ_VALID_MASK          0x1         /* PQ active */
-#define QM_RF_PQ_MAP_PQ_VALID_SHIFT         0
-#define QM_RF_PQ_MAP_RL_ID_MASK             0xFF        /* RL ID */
-#define QM_RF_PQ_MAP_RL_ID_SHIFT            1
-#define QM_RF_PQ_MAP_VP_PQ_ID_MASK          0x1FF
-#define QM_RF_PQ_MAP_VP_PQ_ID_SHIFT         9
-#define QM_RF_PQ_MAP_VOQ_MASK               0x1F        /* VOQ */
-#define QM_RF_PQ_MAP_VOQ_SHIFT              18
-#define QM_RF_PQ_MAP_WRR_WEIGHT_GROUP_MASK  0x3         /* WRR weight */
-#define QM_RF_PQ_MAP_WRR_WEIGHT_GROUP_SHIFT 23
-#define QM_RF_PQ_MAP_RL_VALID_MASK          0x1         /* RL active */
-#define QM_RF_PQ_MAP_RL_VALID_SHIFT         25
-#define QM_RF_PQ_MAP_RESERVED_MASK          0x3F
-#define QM_RF_PQ_MAP_RESERVED_SHIFT         26
-};
-
-/* Completion params for aggregated interrupt completion */
-struct sdm_agg_int_comp_params {
-        __le16 params;
-#define SDM_AGG_INT_COMP_PARAMS_AGG_INT_INDEX_MASK      0x3F
-#define SDM_AGG_INT_COMP_PARAMS_AGG_INT_INDEX_SHIFT     0
-#define SDM_AGG_INT_COMP_PARAMS_AGG_VECTOR_ENABLE_MASK  0x1
-#define SDM_AGG_INT_COMP_PARAMS_AGG_VECTOR_ENABLE_SHIFT 6
-#define SDM_AGG_INT_COMP_PARAMS_AGG_VECTOR_BIT_MASK     0x1FF
-#define SDM_AGG_INT_COMP_PARAMS_AGG_VECTOR_BIT_SHIFT    7
+enum init_split_types {
+        SPLIT_TYPE_NONE,
+        SPLIT_TYPE_PORT,
+        SPLIT_TYPE_PF,
+        SPLIT_TYPE_PORT_PF,
+        SPLIT_TYPE_VF,
+        MAX_INIT_SPLIT_TYPES
 };
 
-/* SDM operation gen command (generate aggregative interrupt) */
-struct sdm_op_gen {
-        __le32 command;
-#define SDM_OP_GEN_COMP_PARAM_MASK  0xFFFF      /* completion parameters 0-15 */
-#define SDM_OP_GEN_COMP_PARAM_SHIFT 0
-#define SDM_OP_GEN_COMP_TYPE_MASK   0xF         /* completion type 16-19 */
-#define SDM_OP_GEN_COMP_TYPE_SHIFT  16
-#define SDM_OP_GEN_RESERVED_MASK    0xFFF       /* reserved 20-31 */
-#define SDM_OP_GEN_RESERVED_SHIFT   20
-};
-
-/*********************************** Init ************************************/
-
-/* Width of GRC address in bits (addresses are specified in dwords) */
-#define GRC_ADDR_BITS                   23
-#define MAX_GRC_ADDR                    ((1 << GRC_ADDR_BITS) - 1)
-
-/* indicates an init that should be applied to any phase ID */
-#define ANY_PHASE_ID                    0xffff
-
-/* init pattern size in bytes */
-#define INIT_PATTERN_SIZE_BITS  4
-#define MAX_INIT_PATTERN_SIZE   BIT(INIT_PATTERN_SIZE_BITS)
-
-/* Max size in dwords of a zipped array */
-#define MAX_ZIPPED_SIZE                 8192
-
-/* Global PXP window */
-#define NUM_OF_PXP_WIN                  19
-#define PXP_WIN_DWORD_SIZE_BITS 10
-#define PXP_WIN_DWORD_SIZE              BIT(PXP_WIN_DWORD_SIZE_BITS)
-#define PXP_WIN_BYTE_SIZE_BITS  (PXP_WIN_DWORD_SIZE_BITS + 2)
-#define PXP_WIN_BYTE_SIZE               (PXP_WIN_DWORD_SIZE * 4)
-
-/********************************* GRC Dump **********************************/
-
-/* width of GRC dump register sequence length in bits */
-#define DUMP_SEQ_LEN_BITS                       8
-#define DUMP_SEQ_LEN_MAX_VAL            ((1 << DUMP_SEQ_LEN_BITS) - 1)
-
-/* width of GRC dump memory length in bits */
-#define DUMP_MEM_LEN_BITS                       18
-#define DUMP_MEM_LEN_MAX_VAL            ((1 << DUMP_MEM_LEN_BITS) - 1)
-
-/* width of register type ID in bits */
-#define REG_TYPE_ID_BITS                        6
-#define REG_TYPE_ID_MAX_VAL                     ((1 << REG_TYPE_ID_BITS) - 1)
-
-/* width of block ID in bits */
-#define BLOCK_ID_BITS                           8
-#define BLOCK_ID_MAX_VAL                        ((1 << BLOCK_ID_BITS) - 1)
-
-/******************************** Idle Check *********************************/
-
-/* max number of idle check predicate immediates */
-#define MAX_IDLE_CHK_PRED_IMM           3
-
-/* max number of idle check argument registers */
-#define MAX_IDLE_CHK_READ_REGS          3
-
-/* max number of idle check loops */
-#define MAX_IDLE_CHK_LOOPS                      0x10000
-
-/* max idle check address increment */
-#define MAX_IDLE_CHK_INCREMENT          0x10000
-
-/* inicates an undefined idle check line index */
-#define IDLE_CHK_UNDEFINED_LINE_IDX     0xffffff
-
-/* max number of register values following the idle check header */
-#define IDLE_CHK_MAX_DUMP_REGS          2
-
-/* arguments for IDLE_CHK_MACRO_TYPE_QM_RD_WR */
-#define IDLE_CHK_QM_RD_WR_PTR           0
-#define IDLE_CHK_QM_RD_WR_BANK          1
-
-/**************************************/
-/* HSI Functions constants and macros */
-/**************************************/
-
-/* Number of VLAN priorities */
-#define NUM_OF_VLAN_PRIORITIES                  8
-
-/* the MCP Trace meta data signautre is duplicated in the perl script that
- * generats the NVRAM images.
- */
-#define MCP_TRACE_META_IMAGE_SIGNATURE  0x669955aa
-
 /* Binary buffer header */
 struct bin_buffer_hdr {
-        u32     offset;
-        u32     length /* buffer length in bytes */;
-};
-
-/* binary buffer types */
-enum bin_buffer_type {
-        BIN_BUF_FW_VER_INFO /* fw_ver_info struct */,
-        BIN_BUF_INIT_CMD /* init commands */,
-        BIN_BUF_INIT_VAL /* init data */,
-        BIN_BUF_INIT_MODE_TREE /* init modes tree */,
-        BIN_BUF_IRO /* internal RAM offsets array */,
-        MAX_BIN_BUFFER_TYPE
+        __le32 offset;
+        __le32 length;
 };
 
-/* Chip IDs */
-enum chip_ids {
-        CHIP_BB_A0 /* BB A0 chip ID */,
-        CHIP_BB_B0 /* BB B0 chip ID */,
-        CHIP_K2 /* AH chip ID */,
-        MAX_CHIP_IDS
+/* binary init buffer types */
+enum bin_init_buffer_type {
+        BIN_BUF_FW_VER_INFO,
+        BIN_BUF_INIT_CMD,
+        BIN_BUF_INIT_VAL,
+        BIN_BUF_INIT_MODE_TREE,
+        BIN_BUF_IRO,
+        MAX_BIN_INIT_BUFFER_TYPE
 };
 
+/* init array header: raw */
 struct init_array_raw_hdr {
         __le32 data;
-#define INIT_ARRAY_RAW_HDR_TYPE_MASK    0xF
-#define INIT_ARRAY_RAW_HDR_TYPE_SHIFT   0
-#define INIT_ARRAY_RAW_HDR_PARAMS_MASK  0xFFFFFFF       /* init array params */
-#define INIT_ARRAY_RAW_HDR_PARAMS_SHIFT 4
+#define INIT_ARRAY_RAW_HDR_TYPE_MASK    0xF
+#define INIT_ARRAY_RAW_HDR_TYPE_SHIFT   0
+#define INIT_ARRAY_RAW_HDR_PARAMS_MASK  0xFFFFFFF
+#define INIT_ARRAY_RAW_HDR_PARAMS_SHIFT 4
 };
 
+/* init array header: standard */
 struct init_array_standard_hdr {
         __le32 data;
-#define INIT_ARRAY_STANDARD_HDR_TYPE_MASK  0xF
-#define INIT_ARRAY_STANDARD_HDR_TYPE_SHIFT 0
-#define INIT_ARRAY_STANDARD_HDR_SIZE_MASK  0xFFFFFFF
-#define INIT_ARRAY_STANDARD_HDR_SIZE_SHIFT 4
+#define INIT_ARRAY_STANDARD_HDR_TYPE_MASK       0xF
+#define INIT_ARRAY_STANDARD_HDR_TYPE_SHIFT      0
+#define INIT_ARRAY_STANDARD_HDR_SIZE_MASK       0xFFFFFFF
+#define INIT_ARRAY_STANDARD_HDR_SIZE_SHIFT      4
 };
 
+/* init array header: zipped */
 struct init_array_zipped_hdr {
         __le32 data;
-#define INIT_ARRAY_ZIPPED_HDR_TYPE_MASK         0xF
-#define INIT_ARRAY_ZIPPED_HDR_TYPE_SHIFT        0
-#define INIT_ARRAY_ZIPPED_HDR_ZIPPED_SIZE_MASK  0xFFFFFFF
-#define INIT_ARRAY_ZIPPED_HDR_ZIPPED_SIZE_SHIFT 4
+#define INIT_ARRAY_ZIPPED_HDR_TYPE_MASK         0xF
+#define INIT_ARRAY_ZIPPED_HDR_TYPE_SHIFT        0
+#define INIT_ARRAY_ZIPPED_HDR_ZIPPED_SIZE_MASK  0xFFFFFFF
+#define INIT_ARRAY_ZIPPED_HDR_ZIPPED_SIZE_SHIFT 4
 };
 
+/* init array header: pattern */
 struct init_array_pattern_hdr {
         __le32 data;
-#define INIT_ARRAY_PATTERN_HDR_TYPE_MASK          0xF
-#define INIT_ARRAY_PATTERN_HDR_TYPE_SHIFT         0
-#define INIT_ARRAY_PATTERN_HDR_PATTERN_SIZE_MASK  0xF
-#define INIT_ARRAY_PATTERN_HDR_PATTERN_SIZE_SHIFT 4
-#define INIT_ARRAY_PATTERN_HDR_REPETITIONS_MASK   0xFFFFFF
-#define INIT_ARRAY_PATTERN_HDR_REPETITIONS_SHIFT  8
+#define INIT_ARRAY_PATTERN_HDR_TYPE_MASK                0xF
+#define INIT_ARRAY_PATTERN_HDR_TYPE_SHIFT               0
+#define INIT_ARRAY_PATTERN_HDR_PATTERN_SIZE_MASK        0xF
+#define INIT_ARRAY_PATTERN_HDR_PATTERN_SIZE_SHIFT       4
+#define INIT_ARRAY_PATTERN_HDR_REPETITIONS_MASK         0xFFFFFF
+#define INIT_ARRAY_PATTERN_HDR_REPETITIONS_SHIFT        8
 };
 
+/* init array header union */
 union init_array_hdr {
-        struct init_array_raw_hdr       raw /* raw init array header */;
-        struct init_array_standard_hdr  standard;
-        struct init_array_zipped_hdr    zipped /* zipped init array header */;
-        struct init_array_pattern_hdr   pattern /* pattern init array header */;
+        struct init_array_raw_hdr raw;
+        struct init_array_standard_hdr standard;
+        struct init_array_zipped_hdr zipped;
+        struct init_array_pattern_hdr pattern;
 };
 
+/* init array types */
 enum init_array_types {
-        INIT_ARR_STANDARD /* standard init array */,
-        INIT_ARR_ZIPPED /* zipped init array */,
-        INIT_ARR_PATTERN /* a repeated pattern */,
+        INIT_ARR_STANDARD,
+        INIT_ARR_ZIPPED,
+        INIT_ARR_PATTERN,
         MAX_INIT_ARRAY_TYPES
 };
 
 /* init operation: callback */
 struct init_callback_op {
-        __le32  op_data;
-#define INIT_CALLBACK_OP_OP_MASK        0xF
-#define INIT_CALLBACK_OP_OP_SHIFT       0
-#define INIT_CALLBACK_OP_RESERVED_MASK  0xFFFFFFF
-#define INIT_CALLBACK_OP_RESERVED_SHIFT 4
-        __le16  callback_id /* Callback ID */;
-        __le16  block_id /* Blocks ID */;
+        __le32 op_data;
+#define INIT_CALLBACK_OP_OP_MASK        0xF
+#define INIT_CALLBACK_OP_OP_SHIFT       0
+#define INIT_CALLBACK_OP_RESERVED_MASK  0xFFFFFFF
+#define INIT_CALLBACK_OP_RESERVED_SHIFT 4
+        __le16 callback_id;
+        __le16 block_id;
 };
 
 /* init operation: delay */
 struct init_delay_op {
-        __le32  op_data;
-#define INIT_DELAY_OP_OP_MASK        0xF
-#define INIT_DELAY_OP_OP_SHIFT       0
-#define INIT_DELAY_OP_RESERVED_MASK  0xFFFFFFF
-#define INIT_DELAY_OP_RESERVED_SHIFT 4
-        __le32  delay /* delay in us */;
+        __le32 op_data;
+#define INIT_DELAY_OP_OP_MASK           0xF
+#define INIT_DELAY_OP_OP_SHIFT          0
+#define INIT_DELAY_OP_RESERVED_MASK     0xFFFFFFF
+#define INIT_DELAY_OP_RESERVED_SHIFT    4
+        __le32 delay;
 };
 
 /* init operation: if_mode */
 struct init_if_mode_op {
         __le32 op_data;
-#define INIT_IF_MODE_OP_OP_MASK          0xF
-#define INIT_IF_MODE_OP_OP_SHIFT         0
-#define INIT_IF_MODE_OP_RESERVED1_MASK   0xFFF
-#define INIT_IF_MODE_OP_RESERVED1_SHIFT  4
-#define INIT_IF_MODE_OP_CMD_OFFSET_MASK  0xFFFF
-#define INIT_IF_MODE_OP_CMD_OFFSET_SHIFT 16
-        __le16  reserved2;
-        __le16  modes_buf_offset;
+#define INIT_IF_MODE_OP_OP_MASK                 0xF
+#define INIT_IF_MODE_OP_OP_SHIFT                0
+#define INIT_IF_MODE_OP_RESERVED1_MASK          0xFFF
+#define INIT_IF_MODE_OP_RESERVED1_SHIFT         4
+#define INIT_IF_MODE_OP_CMD_OFFSET_MASK         0xFFFF
+#define INIT_IF_MODE_OP_CMD_OFFSET_SHIFT        16
+        __le16 reserved2;
+        __le16 modes_buf_offset;
 };
 
-/*  init operation: if_phase */
+/* init operation: if_phase */
 struct init_if_phase_op {
         __le32 op_data;
-#define INIT_IF_PHASE_OP_OP_MASK           0xF
-#define INIT_IF_PHASE_OP_OP_SHIFT          0
-#define INIT_IF_PHASE_OP_DMAE_ENABLE_MASK  0x1
-#define INIT_IF_PHASE_OP_DMAE_ENABLE_SHIFT 4
-#define INIT_IF_PHASE_OP_RESERVED1_MASK    0x7FF
-#define INIT_IF_PHASE_OP_RESERVED1_SHIFT   5
-#define INIT_IF_PHASE_OP_CMD_OFFSET_MASK   0xFFFF
-#define INIT_IF_PHASE_OP_CMD_OFFSET_SHIFT  16
+#define INIT_IF_PHASE_OP_OP_MASK                0xF
+#define INIT_IF_PHASE_OP_OP_SHIFT               0
+#define INIT_IF_PHASE_OP_DMAE_ENABLE_MASK       0x1
+#define INIT_IF_PHASE_OP_DMAE_ENABLE_SHIFT      4
+#define INIT_IF_PHASE_OP_RESERVED1_MASK         0x7FF
+#define INIT_IF_PHASE_OP_RESERVED1_SHIFT        5
+#define INIT_IF_PHASE_OP_CMD_OFFSET_MASK        0xFFFF
+#define INIT_IF_PHASE_OP_CMD_OFFSET_SHIFT       16
         __le32 phase_data;
-#define INIT_IF_PHASE_OP_PHASE_MASK        0xFF /* Init phase */
-#define INIT_IF_PHASE_OP_PHASE_SHIFT       0
-#define INIT_IF_PHASE_OP_RESERVED2_MASK    0xFF
-#define INIT_IF_PHASE_OP_RESERVED2_SHIFT   8
-#define INIT_IF_PHASE_OP_PHASE_ID_MASK     0xFFFF /* Init phase ID */
-#define INIT_IF_PHASE_OP_PHASE_ID_SHIFT    16
+#define INIT_IF_PHASE_OP_PHASE_MASK             0xFF
+#define INIT_IF_PHASE_OP_PHASE_SHIFT            0
+#define INIT_IF_PHASE_OP_RESERVED2_MASK         0xFF
+#define INIT_IF_PHASE_OP_RESERVED2_SHIFT        8
+#define INIT_IF_PHASE_OP_PHASE_ID_MASK          0xFFFF
+#define INIT_IF_PHASE_OP_PHASE_ID_SHIFT         16
 };
 
 /* init mode operators */
 enum init_mode_ops {
-        INIT_MODE_OP_NOT /* init mode not operator */,
-        INIT_MODE_OP_OR /* init mode or operator */,
-        INIT_MODE_OP_AND /* init mode and operator */,
+        INIT_MODE_OP_NOT,
+        INIT_MODE_OP_OR,
+        INIT_MODE_OP_AND,
         MAX_INIT_MODE_OPS
 };
 
 /* init operation: raw */
 struct init_raw_op {
-        __le32  op_data;
-#define INIT_RAW_OP_OP_MASK      0xF
-#define INIT_RAW_OP_OP_SHIFT     0
-#define INIT_RAW_OP_PARAM1_MASK  0xFFFFFFF      /* init param 1 */
-#define INIT_RAW_OP_PARAM1_SHIFT 4
-        __le32  param2 /* Init param 2 */;
+        __le32 op_data;
+#define INIT_RAW_OP_OP_MASK             0xF
+#define INIT_RAW_OP_OP_SHIFT            0
+#define INIT_RAW_OP_PARAM1_MASK         0xFFFFFFF
+#define INIT_RAW_OP_PARAM1_SHIFT        4
+        __le32 param2;
 };
 
 /* init array params */
 struct init_op_array_params {
-        __le16  size /* array size in dwords */;
-        __le16  offset /* array start offset in dwords */;
+        __le16 size;
+        __le16 offset;
 };
 
 /* Write init operation arguments */
 union init_write_args {
-        __le32                          inline_val;
-        __le32                          zeros_count;
-        __le32                          array_offset;
-        struct init_op_array_params     runtime;
+        __le32 inline_val;
+        __le32 zeros_count;
+        __le32 array_offset;
+        struct init_op_array_params runtime;
 };
 
 /* init operation: write */
 struct init_write_op {
         __le32 data;
-#define INIT_WRITE_OP_OP_MASK        0xF
-#define INIT_WRITE_OP_OP_SHIFT       0
-#define INIT_WRITE_OP_SOURCE_MASK    0x7
-#define INIT_WRITE_OP_SOURCE_SHIFT   4
-#define INIT_WRITE_OP_RESERVED_MASK  0x1
-#define INIT_WRITE_OP_RESERVED_SHIFT 7
-#define INIT_WRITE_OP_WIDE_BUS_MASK  0x1
-#define INIT_WRITE_OP_WIDE_BUS_SHIFT 8
-#define INIT_WRITE_OP_ADDRESS_MASK   0x7FFFFF
-#define INIT_WRITE_OP_ADDRESS_SHIFT  9
-        union init_write_args args /* Write init operation arguments */;
+#define INIT_WRITE_OP_OP_MASK           0xF
+#define INIT_WRITE_OP_OP_SHIFT          0
+#define INIT_WRITE_OP_SOURCE_MASK       0x7
+#define INIT_WRITE_OP_SOURCE_SHIFT      4
+#define INIT_WRITE_OP_RESERVED_MASK     0x1
+#define INIT_WRITE_OP_RESERVED_SHIFT    7
+#define INIT_WRITE_OP_WIDE_BUS_MASK     0x1
+#define INIT_WRITE_OP_WIDE_BUS_SHIFT    8
+#define INIT_WRITE_OP_ADDRESS_MASK      0x7FFFFF
+#define INIT_WRITE_OP_ADDRESS_SHIFT     9
+        union init_write_args args;
 };
 
 /* init operation: read */
 struct init_read_op {
         __le32 op_data;
-#define INIT_READ_OP_OP_MASK         0xF
-#define INIT_READ_OP_OP_SHIFT        0
-#define INIT_READ_OP_POLL_TYPE_MASK  0xF
-#define INIT_READ_OP_POLL_TYPE_SHIFT 4
-#define INIT_READ_OP_RESERVED_MASK   0x1
-#define INIT_READ_OP_RESERVED_SHIFT  8
-#define INIT_READ_OP_ADDRESS_MASK    0x7FFFFF
-#define INIT_READ_OP_ADDRESS_SHIFT   9
+#define INIT_READ_OP_OP_MASK            0xF
+#define INIT_READ_OP_OP_SHIFT           0
+#define INIT_READ_OP_POLL_TYPE_MASK     0xF
+#define INIT_READ_OP_POLL_TYPE_SHIFT    4
+#define INIT_READ_OP_RESERVED_MASK      0x1
+#define INIT_READ_OP_RESERVED_SHIFT     8
+#define INIT_READ_OP_ADDRESS_MASK       0x7FFFFF
+#define INIT_READ_OP_ADDRESS_SHIFT      9
         __le32 expected_val;
+
 };
 
 /* Init operations union */
 union init_op {
-        struct init_raw_op      raw /* raw init operation */;
-        struct init_write_op    write /* write init operation */;
-        struct init_read_op     read /* read init operation */;
-        struct init_if_mode_op  if_mode /* if_mode init operation */;
-        struct init_if_phase_op if_phase /* if_phase init operation */;
-        struct init_callback_op callback /* callback init operation */;
-        struct init_delay_op    delay /* delay init operation */;
+        struct init_raw_op raw;
+        struct init_write_op write;
+        struct init_read_op read;
+        struct init_if_mode_op if_mode;
+        struct init_if_phase_op if_phase;
+        struct init_callback_op callback;
+        struct init_delay_op delay;
 };
 
 /* Init command operation types */
 enum init_op_types {
-        INIT_OP_READ /* GRC read init command */,
-        INIT_OP_WRITE /* GRC write init command */,
+        INIT_OP_READ,
+        INIT_OP_WRITE,
         INIT_OP_IF_MODE,
         INIT_OP_IF_PHASE,
-        INIT_OP_DELAY /* delay init command */,
-        INIT_OP_CALLBACK /* callback init command */,
+        INIT_OP_DELAY,
+        INIT_OP_CALLBACK,
         MAX_INIT_OP_TYPES
 };
 
+/* init polling types */
 enum init_poll_types {
-        INIT_POLL_NONE /* No polling */,
-        INIT_POLL_EQ /* init value is included in the init command */,
-        INIT_POLL_OR /* init value is all zeros */,
-        INIT_POLL_AND /* init value is an array of values */,
+        INIT_POLL_NONE,
+        INIT_POLL_EQ,
+        INIT_POLL_OR,
+        INIT_POLL_AND,
         MAX_INIT_POLL_TYPES
 };
 
 /* init source types */
 enum init_source_types {
-        INIT_SRC_INLINE /* init value is included in the init command */,
-        INIT_SRC_ZEROS /* init value is all zeros */,
-        INIT_SRC_ARRAY /* init value is an array of values */,
-        INIT_SRC_RUNTIME /* init value is provided during runtime */,
+        INIT_SRC_INLINE,
+        INIT_SRC_ZEROS,
+        INIT_SRC_ARRAY,
+        INIT_SRC_RUNTIME,
         MAX_INIT_SOURCE_TYPES
 };
 
 /* Internal RAM Offsets macro data */
 struct iro {
-        u32     base /* RAM field offset */;
-        u16     m1 /* multiplier 1 */;
-        u16     m2 /* multiplier 2 */;
-        u16     m3 /* multiplier 3 */;
-        u16     size /* RAM field size */;
+        __le32 base;
+        __le16 m1;
+        __le16 m2;
+        __le16 m3;
+        __le16 size;
 };
 
-/* QM per-port init parameters */
-struct init_qm_port_params {
-        u8      active /* Indicates if this port is active */;
-        u8      num_active_phys_tcs;
-        u16     num_pbf_cmd_lines;
-        u16     num_btb_blocks;
-        __le16  reserved;
-};
-
-/* QM per-PQ init parameters */
-struct init_qm_pq_params {
-        u8      vport_id /* VPORT ID */;
-        u8      tc_id /* TC ID */;
-        u8      wrr_group /* WRR group */;
-        u8      reserved;
-};
+/**
+ * @brief qed_dbg_print_attn - Prints attention registers values in the specified results struct.
+ *
+ * @param p_hwfn
+ * @param results - Pointer to the attention read results
+ *
+ * @return error if one of the following holds:
+ *      - the version wasn't set
+ * Otherwise, returns ok.
+ */
+enum dbg_status qed_dbg_print_attn(struct qed_hwfn *p_hwfn,
+                                   struct dbg_attn_block_result *results);
 
-/* QM per-vport init parameters */
-struct init_qm_vport_params {
-        u32     vport_rl;
-        u16     vport_wfq;
-        u16     first_tx_pq_id[NUM_OF_TCS];
-};
+#define MAX_NAME_LEN    16
 
 /* Win 2 */
 #define GTT_BAR0_MAP_REG_IGU_CMD \
         0x00f000UL
+
 /* Win 3 */
 #define GTT_BAR0_MAP_REG_TSDM_RAM \
         0x010000UL
+
 /* Win 4 */
 #define GTT_BAR0_MAP_REG_MSDM_RAM \
         0x011000UL
+
 /* Win 5 */
 #define GTT_BAR0_MAP_REG_MSDM_RAM_1024 \
         0x012000UL
+
 /* Win 6 */
 #define GTT_BAR0_MAP_REG_USDM_RAM \
         0x013000UL
+
 /* Win 7 */
 #define GTT_BAR0_MAP_REG_USDM_RAM_1024 \
         0x014000UL
+
 /* Win 8 */
 #define GTT_BAR0_MAP_REG_USDM_RAM_2048 \
         0x015000UL
+
 /* Win 9 */
 #define GTT_BAR0_MAP_REG_XSDM_RAM \
         0x016000UL
+
 /* Win 10 */
 #define GTT_BAR0_MAP_REG_YSDM_RAM \
         0x017000UL
+
 /* Win 11 */
 #define GTT_BAR0_MAP_REG_PSDM_RAM \
         0x018000UL
@@ -1584,785 +1963,718 @@ struct init_qm_vport_params {
  * Returns the required host memory size in 4KB units.
  * Must be called before all QM init HSI functions.
  *
- * @param pf_id                 - physical function ID
- * @param num_pf_cids   - number of connections used by this PF
- * @param num_vf_cids   - number of connections used by VFs of this PF
- * @param num_tids              - number of tasks used by this PF
- * @param num_pf_pqs    - number of PQs used by this PF
- * @param num_vf_pqs    - number of PQs used by VFs of this PF
+ * @param pf_id - physical function ID
+ * @param num_pf_cids - number of connections used by this PF
+ * @param num_vf_cids - number of connections used by VFs of this PF
+ * @param num_tids - number of tasks used by this PF
+ * @param num_pf_pqs - number of PQs used by this PF
+ * @param num_vf_pqs - number of PQs used by VFs of this PF
  *
  * @return The required host memory size in 4KB units.
  */
-u32 qed_qm_pf_mem_size(u8       pf_id,
-                       u32      num_pf_cids,
-                       u32      num_vf_cids,
-                       u32      num_tids,
-                       u16      num_pf_pqs,
-                       u16      num_vf_pqs);
+u32 qed_qm_pf_mem_size(u8 pf_id,
+                       u32 num_pf_cids,
+                       u32 num_vf_cids,
+                       u32 num_tids, u16 num_pf_pqs, u16 num_vf_pqs);
 
 struct qed_qm_common_rt_init_params {
-        u8                              max_ports_per_engine;
-        u8                              max_phys_tcs_per_port;
-        bool                            pf_rl_en;
-        bool                            pf_wfq_en;
-        bool                            vport_rl_en;
-        bool                            vport_wfq_en;
-        struct init_qm_port_params      *port_params;
+        u8 max_ports_per_engine;
+        u8 max_phys_tcs_per_port;
+        bool pf_rl_en;
+        bool pf_wfq_en;
+        bool vport_rl_en;
+        bool vport_wfq_en;
+        struct init_qm_port_params *port_params;
 };
 
+int qed_qm_common_rt_init(struct qed_hwfn *p_hwfn,
+                          struct qed_qm_common_rt_init_params *p_params);
+
+struct qed_qm_pf_rt_init_params {
+        u8 port_id;
+        u8 pf_id;
+        u8 max_phys_tcs_per_port;
+        bool is_first_pf;
+        u32 num_pf_cids;
+        u32 num_vf_cids;
+        u32 num_tids;
+        u16 start_pq;
+        u16 num_pf_pqs;
+        u16 num_vf_pqs;
+        u8 start_vport;
+        u8 num_vports;
+        u8 pf_wfq;
+        u32 pf_rl;
+        struct init_qm_pq_params *pq_params;
+        struct init_qm_vport_params *vport_params;
+};
+
+int qed_qm_pf_rt_init(struct qed_hwfn *p_hwfn,
+        struct qed_ptt *p_ptt,
+        struct qed_qm_pf_rt_init_params *p_params);
+
 /**
- * @brief qed_qm_common_rt_init - Prepare QM runtime init values for the
- * engine phase.
+ * @brief qed_init_pf_wfq - Initializes the WFQ weight of the specified PF
  *
  * @param p_hwfn
- * @param max_ports_per_engine  - max number of ports per engine in HW
- * @param max_phys_tcs_per_port - max number of physical TCs per port in HW
- * @param pf_rl_en                              - enable per-PF rate limiters
- * @param pf_wfq_en                             - enable per-PF WFQ
- * @param vport_rl_en                   - enable per-VPORT rate limiters
- * @param vport_wfq_en                  - enable per-VPORT WFQ
- * @param port_params                   - array of size MAX_NUM_PORTS with
- *                                              arameters for each port
+ * @param p_ptt - ptt window used for writing the registers
+ * @param pf_id - PF ID
+ * @param pf_wfq - WFQ weight. Must be non-zero.
  *
  * @return 0 on success, -1 on error.
  */
-int qed_qm_common_rt_init(
-        struct qed_hwfn                         *p_hwfn,
-        struct qed_qm_common_rt_init_params     *p_params);
-
-struct qed_qm_pf_rt_init_params {
-        u8                              port_id;
-        u8                              pf_id;
-        u8                              max_phys_tcs_per_port;
-        bool                            is_first_pf;
-        u32                             num_pf_cids;
-        u32                             num_vf_cids;
-        u32                             num_tids;
-        u16                             start_pq;
-        u16                             num_pf_pqs;
-        u16                             num_vf_pqs;
-        u8                              start_vport;
-        u8                              num_vports;
-        u8                              pf_wfq;
-        u32                             pf_rl;
-        struct init_qm_pq_params        *pq_params;
-        struct init_qm_vport_params     *vport_params;
-};
-
-int qed_qm_pf_rt_init(struct qed_hwfn                   *p_hwfn,
-                      struct qed_ptt                    *p_ptt,
-                      struct qed_qm_pf_rt_init_params   *p_params);
+int qed_init_pf_wfq(struct qed_hwfn *p_hwfn,
+                    struct qed_ptt *p_ptt, u8 pf_id, u16 pf_wfq);
 
 /**
- * @brief qed_init_pf_rl  Initializes the rate limit of the specified PF
+ * @brief qed_init_pf_rl - Initializes the rate limit of the specified PF
  *
  * @param p_hwfn
- * @param p_ptt - ptt window used for writing the registers
- * @param pf_id - PF ID
- * @param pf_rl - rate limit in Mb/sec units
+ * @param p_ptt - ptt window used for writing the registers
+ * @param pf_id - PF ID
+ * @param pf_rl - rate limit in Mb/sec units
  *
  * @return 0 on success, -1 on error.
  */
-int qed_init_pf_rl(struct qed_hwfn      *p_hwfn,
-                   struct qed_ptt       *p_ptt,
-                   u8                   pf_id,
-                   u32                  pf_rl);
+int qed_init_pf_rl(struct qed_hwfn *p_hwfn,
+                   struct qed_ptt *p_ptt, u8 pf_id, u32 pf_rl);
 
 /**
- * @brief qed_init_vport_rl  Initializes the rate limit of the specified VPORT
+ * @brief qed_init_vport_wfq Initializes the WFQ weight of the specified VPORT
  *
  * @param p_hwfn
- * @param p_ptt         - ptt window used for writing the registers
- * @param vport_id      - VPORT ID
- * @param vport_rl      - rate limit in Mb/sec units
+ * @param p_ptt - ptt window used for writing the registers
+ * @param first_tx_pq_id- An array containing the first Tx PQ ID associated
+ *        with the VPORT for each TC. This array is filled by
+ *        qed_qm_pf_rt_init
+ * @param vport_wfq - WFQ weight. Must be non-zero.
  *
  * @return 0 on success, -1 on error.
  */
+int qed_init_vport_wfq(struct qed_hwfn *p_hwfn,
+                       struct qed_ptt *p_ptt,
+                       u16 first_tx_pq_id[NUM_OF_TCS], u16 vport_wfq);
 
-int qed_init_vport_rl(struct qed_hwfn   *p_hwfn,
-                      struct qed_ptt    *p_ptt,
-                      u8                vport_id,
-                      u32               vport_rl);
+/**
+ * @brief qed_init_vport_rl - Initializes the rate limit of the specified VPORT
+ *
+ * @param p_hwfn
+ * @param p_ptt - ptt window used for writing the registers
+ * @param vport_id - VPORT ID
+ * @param vport_rl - rate limit in Mb/sec units
+ *
+ * @return 0 on success, -1 on error.
+ */
+int qed_init_vport_rl(struct qed_hwfn *p_hwfn,
+                      struct qed_ptt *p_ptt, u8 vport_id, u32 vport_rl);
 /**
  * @brief qed_send_qm_stop_cmd  Sends a stop command to the QM
  *
  * @param p_hwfn
- * @param p_ptt          - ptt window used for writing the registers
+ * @param p_ptt
  * @param is_release_cmd - true for release, false for stop.
- * @param is_tx_pq       - true for Tx PQs, false for Other PQs.
- * @param start_pq       - first PQ ID to stop
- * @param num_pqs        - Number of PQs to stop, starting from start_pq.
+ * @param is_tx_pq - true for Tx PQs, false for Other PQs.
+ * @param start_pq - first PQ ID to stop
+ * @param num_pqs - Number of PQs to stop, starting from start_pq.
  *
- * @return bool, true if successful, false if timeout occurred while waiting
- *                                      for QM command done.
+ * @return bool, true if successful, false if timeout occured while waiting for QM command done.
  */
+bool qed_send_qm_stop_cmd(struct qed_hwfn *p_hwfn,
+                          struct qed_ptt *p_ptt,
+                          bool is_release_cmd,
+                          bool is_tx_pq, u16 start_pq, u16 num_pqs);
 
-bool qed_send_qm_stop_cmd(struct qed_hwfn       *p_hwfn,
-                          struct qed_ptt        *p_ptt,
-                          bool                  is_release_cmd,
-                          bool                  is_tx_pq,
-                          u16                   start_pq,
-                          u16                   num_pqs);
-
+/**
+ * @brief qed_set_vxlan_dest_port - initializes vxlan tunnel destination udp port
+ *
+ * @param p_ptt - ptt window used for writing the registers.
+ * @param dest_port - vxlan destination udp port.
+ */
 void qed_set_vxlan_dest_port(struct qed_hwfn *p_hwfn,
-                             struct qed_ptt  *p_ptt, u16 dest_port);
+                             struct qed_ptt *p_ptt, u16 dest_port);
+
+/**
+ * @brief qed_set_vxlan_enable - enable or disable VXLAN tunnel in HW
+ *
+ * @param p_ptt - ptt window used for writing the registers.
+ * @param vxlan_enable - vxlan enable flag.
+ */
 void qed_set_vxlan_enable(struct qed_hwfn *p_hwfn,
                           struct qed_ptt *p_ptt, bool vxlan_enable);
+
+/**
+ * @brief qed_set_gre_enable - enable or disable GRE tunnel in HW
+ *
+ * @param p_ptt - ptt window used for writing the registers.
+ * @param eth_gre_enable - eth GRE enable enable flag.
+ * @param ip_gre_enable - IP GRE enable enable flag.
+ */
 void qed_set_gre_enable(struct qed_hwfn *p_hwfn,
-                        struct qed_ptt  *p_ptt, bool eth_gre_enable,
-                        bool ip_gre_enable);
+                        struct qed_ptt *p_ptt,
+                        bool eth_gre_enable, bool ip_gre_enable);
+
+/**
+ * @brief qed_set_geneve_dest_port - initializes geneve tunnel destination udp port
+ *
+ * @param p_ptt - ptt window used for writing the registers.
+ * @param dest_port - geneve destination udp port.
+ */
 void qed_set_geneve_dest_port(struct qed_hwfn *p_hwfn,
                               struct qed_ptt *p_ptt, u16 dest_port);
+
+/**
+ * @brief qed_set_gre_enable - enable or disable GRE tunnel in HW
+ *
+ * @param p_ptt - ptt window used for writing the registers.
+ * @param eth_geneve_enable - eth GENEVE enable enable flag.
+ * @param ip_geneve_enable - IP GENEVE enable enable flag.
+ */
 void qed_set_geneve_enable(struct qed_hwfn *p_hwfn,
-                           struct qed_ptt *p_ptt, bool eth_geneve_enable,
-                           bool ip_geneve_enable);
-
-/* Ystorm flow control mode. Use enum fw_flow_ctrl_mode */
-#define YSTORM_FLOW_CONTROL_MODE_OFFSET  (IRO[0].base)
-#define YSTORM_FLOW_CONTROL_MODE_SIZE    (IRO[0].size)
-/* Tstorm port statistics */
-#define TSTORM_PORT_STAT_OFFSET(port_id) (IRO[1].base + ((port_id) * IRO[1].m1))
-#define TSTORM_PORT_STAT_SIZE            (IRO[1].size)
-/* Tstorm ll2 port statistics */
-#define TSTORM_LL2_PORT_STAT_OFFSET(port_id) \
-                                (IRO[2].base + ((port_id) * IRO[2].m1))
-#define TSTORM_LL2_PORT_STAT_SIZE            (IRO[2].size)
-/* Ustorm VF-PF Channel ready flag */
-#define USTORM_VF_PF_CHANNEL_READY_OFFSET(vf_id) \
-                                (IRO[3].base +  ((vf_id) * IRO[3].m1))
-#define USTORM_VF_PF_CHANNEL_READY_SIZE          (IRO[3].size)
-/* Ustorm Final flr cleanup ack */
-#define USTORM_FLR_FINAL_ACK_OFFSET(pf_id) (IRO[4].base + ((pf_id) * IRO[4].m1))
-#define USTORM_FLR_FINAL_ACK_SIZE          (IRO[4].size)
-/* Ustorm Event ring consumer */
-#define USTORM_EQE_CONS_OFFSET(pf_id)    (IRO[5].base + ((pf_id) * IRO[5].m1))
-#define USTORM_EQE_CONS_SIZE             (IRO[5].size)
-/* Ustorm Common Queue ring consumer */
-#define USTORM_COMMON_QUEUE_CONS_OFFSET(global_queue_id) \
-                        (IRO[6].base + ((global_queue_id) * IRO[6].m1))
-#define USTORM_COMMON_QUEUE_CONS_SIZE    (IRO[6].size)
-/* Xstorm Integration Test Data */
-#define XSTORM_INTEG_TEST_DATA_OFFSET    (IRO[7].base)
-#define XSTORM_INTEG_TEST_DATA_SIZE      (IRO[7].size)
-/* Ystorm Integration Test Data */
-#define YSTORM_INTEG_TEST_DATA_OFFSET    (IRO[8].base)
-#define YSTORM_INTEG_TEST_DATA_SIZE      (IRO[8].size)
-/* Pstorm Integration Test Data */
-#define PSTORM_INTEG_TEST_DATA_OFFSET    (IRO[9].base)
-#define PSTORM_INTEG_TEST_DATA_SIZE      (IRO[9].size)
-/* Tstorm Integration Test Data */
-#define TSTORM_INTEG_TEST_DATA_OFFSET    (IRO[10].base)
-#define TSTORM_INTEG_TEST_DATA_SIZE      (IRO[10].size)
-/* Mstorm Integration Test Data */
-#define MSTORM_INTEG_TEST_DATA_OFFSET    (IRO[11].base)
-#define MSTORM_INTEG_TEST_DATA_SIZE      (IRO[11].size)
-/* Ustorm Integration Test Data */
-#define USTORM_INTEG_TEST_DATA_OFFSET    (IRO[12].base)
-#define USTORM_INTEG_TEST_DATA_SIZE      (IRO[12].size)
-/* Tstorm producers */
-#define TSTORM_LL2_RX_PRODS_OFFSET(core_rx_queue_id) \
-                        (IRO[13].base + ((core_rx_queue_id) * IRO[13].m1))
-#define TSTORM_LL2_RX_PRODS_SIZE         (IRO[13].size)
-/* Tstorm LightL2 queue statistics */
-#define CORE_LL2_TSTORM_PER_QUEUE_STAT_OFFSET(core_rx_queue_id) \
-                        (IRO[14].base + ((core_rx_queue_id) * IRO[14].m1))
-#define CORE_LL2_TSTORM_PER_QUEUE_STAT_SIZE    (IRO[14].size)
-/* Ustorm LiteL2 queue statistics */
-#define CORE_LL2_USTORM_PER_QUEUE_STAT_OFFSET(core_rx_queue_id) \
-                        (IRO[15].base + ((core_rx_queue_id) * IRO[15].m1))
-#define CORE_LL2_USTORM_PER_QUEUE_STAT_SIZE    (IRO[15].size)
-/* Pstorm LiteL2 queue statistics */
-#define CORE_LL2_PSTORM_PER_QUEUE_STAT_OFFSET(core_tx_stats_id) \
-                        (IRO[16].base + ((core_tx_stats_id) * IRO[16].m1))
-#define CORE_LL2_PSTORM_PER_QUEUE_STAT_SIZE    (IRO[16].size)
-/* Mstorm queue statistics */
-#define MSTORM_QUEUE_STAT_OFFSET(stat_counter_id) \
-                        (IRO[17].base + ((stat_counter_id) * IRO[17].m1))
-#define MSTORM_QUEUE_STAT_SIZE                 (IRO[17].size)
-/* Mstorm producers */
-#define MSTORM_PRODS_OFFSET(queue_id) (IRO[18].base + ((queue_id) * IRO[18].m1))
-#define MSTORM_PRODS_SIZE             (IRO[18].size)
-/* TPA agregation timeout in us resolution (on ASIC) */
-#define MSTORM_TPA_TIMEOUT_US_OFFSET  (IRO[19].base)
-#define MSTORM_TPA_TIMEOUT_US_SIZE    (IRO[19].size)
-/* Ustorm queue statistics */
-#define USTORM_QUEUE_STAT_OFFSET(stat_counter_id) \
-                        (IRO[20].base + ((stat_counter_id) * IRO[20].m1))
-#define USTORM_QUEUE_STAT_SIZE        (IRO[20].size)
-/* Ustorm queue zone */
-#define USTORM_ETH_QUEUE_ZONE_OFFSET(queue_id) \
-                        (IRO[21].base + ((queue_id) * IRO[21].m1))
-#define USTORM_ETH_QUEUE_ZONE_SIZE    (IRO[21].size)
-/* Pstorm queue statistics */
-#define PSTORM_QUEUE_STAT_OFFSET(stat_counter_id) \
-                (IRO[22].base + ((stat_counter_id) * IRO[22].m1))
-#define PSTORM_QUEUE_STAT_SIZE        (IRO[22].size)
-/* Tstorm last parser message */
-#define TSTORM_ETH_PRS_INPUT_OFFSET  (IRO[23].base)
-#define TSTORM_ETH_PRS_INPUT_SIZE    (IRO[23].size)
-/* Tstorm Eth limit Rx rate */
-#define ETH_RX_RATE_LIMIT_OFFSET(pf_id) (IRO[24].base + ((pf_id) * IRO[24].m1))
-#define ETH_RX_RATE_LIMIT_SIZE       (IRO[24].size)
-/* Ystorm queue zone */
-#define YSTORM_ETH_QUEUE_ZONE_OFFSET(queue_id) \
-                        (IRO[25].base + ((queue_id) * IRO[25].m1))
-#define YSTORM_ETH_QUEUE_ZONE_SIZE   (IRO[25].size)
-/* Ystorm cqe producer */
-#define YSTORM_TOE_CQ_PROD_OFFSET(rss_id) \
-                        (IRO[26].base + ((rss_id) * IRO[26].m1))
-#define YSTORM_TOE_CQ_PROD_SIZE      (IRO[26].size)
-/* Ustorm cqe producer */
-#define USTORM_TOE_CQ_PROD_OFFSET(rss_id) \
-                        (IRO[27].base + ((rss_id) * IRO[27].m1))
-#define USTORM_TOE_CQ_PROD_SIZE      (IRO[27].size)
-/* Ustorm grq producer */
-#define USTORM_TOE_GRQ_PROD_OFFSET(pf_id) \
-                        (IRO[28].base + ((pf_id) * IRO[28].m1))
-#define USTORM_TOE_GRQ_PROD_SIZE     (IRO[28].size)
-/* Tstorm cmdq-cons of given command queue-id */
-#define TSTORM_SCSI_CMDQ_CONS_OFFSET(cmdq_queue_id) \
-                        (IRO[29].base + ((cmdq_queue_id) * IRO[29].m1))
-#define TSTORM_SCSI_CMDQ_CONS_SIZE   (IRO[29].size)
-/* Mstorm rq-cons of given queue-id */
-#define MSTORM_SCSI_RQ_CONS_OFFSET(rq_queue_id) \
-                (IRO[30].base + ((rq_queue_id) * IRO[30].m1))
-#define MSTORM_SCSI_RQ_CONS_SIZE     (IRO[30].size)
-/* Mstorm bdq-external-producer of given BDQ function ID, BDqueue-id */
-#define MSTORM_SCSI_BDQ_EXT_PROD_OFFSET(func_id, bdq_id) \
-        (IRO[31].base + ((func_id) * IRO[31].m1) + ((bdq_id) * IRO[31].m2))
-#define MSTORM_SCSI_BDQ_EXT_PROD_SIZE (IRO[31].size)
-/* Tstorm (reflects M-Storm) bdq-external-producer of given fn ID, BDqueue-id */
-#define TSTORM_SCSI_BDQ_EXT_PROD_OFFSET(func_id, bdq_id) \
-        (IRO[32].base + ((func_id) * IRO[32].m1) + ((bdq_id) * IRO[32].m2))
-#define TSTORM_SCSI_BDQ_EXT_PROD_SIZE (IRO[32].size)
-/* Tstorm iSCSI RX stats */
-#define TSTORM_ISCSI_RX_STATS_OFFSET(pf_id) \
-                                (IRO[33].base + ((pf_id) * IRO[33].m1))
-#define TSTORM_ISCSI_RX_STATS_SIZE    (IRO[33].size)
-/* Mstorm iSCSI RX stats */
-#define MSTORM_ISCSI_RX_STATS_OFFSET(pf_id) \
-                                (IRO[34].base + ((pf_id) * IRO[34].m1))
-#define MSTORM_ISCSI_RX_STATS_SIZE    (IRO[34].size)
-/* Ustorm iSCSI RX stats */
-#define USTORM_ISCSI_RX_STATS_OFFSET(pf_id) \
-                                (IRO[35].base + ((pf_id) * IRO[35].m1))
-#define USTORM_ISCSI_RX_STATS_SIZE    (IRO[35].size)
-/* Xstorm iSCSI TX stats */
-#define XSTORM_ISCSI_TX_STATS_OFFSET(pf_id) \
-                                (IRO[36].base + ((pf_id) * IRO[36].m1))
-#define XSTORM_ISCSI_TX_STATS_SIZE    (IRO[36].size)
-/* Ystorm iSCSI TX stats */
-#define YSTORM_ISCSI_TX_STATS_OFFSET(pf_id) \
-                                (IRO[37].base + ((pf_id) * IRO[37].m1))
-#define YSTORM_ISCSI_TX_STATS_SIZE    (IRO[37].size)
-/* Pstorm iSCSI TX stats */
-#define PSTORM_ISCSI_TX_STATS_OFFSET(pf_id) \
-                                (IRO[38].base + ((pf_id) * IRO[38].m1))
-#define PSTORM_ISCSI_TX_STATS_SIZE    (IRO[38].size)
-/* Tstorm FCoE RX stats */
-#define TSTORM_FCOE_RX_STATS_OFFSET(pf_id) \
-                                (IRO[39].base + ((pf_id) * IRO[39].m1))
-#define TSTORM_FCOE_RX_STATS_SIZE      (IRO[39].size)
-/* Mstorm FCoE RX stats */
-#define MSTORM_FCOE_RX_STATS_OFFSET(pf_id) \
-                                (IRO[40].base + ((pf_id) * IRO[40].m1))
-#define MSTORM_FCOE_RX_STATS_SIZE      (IRO[40].size)
-/* Pstorm FCoE TX stats */
-#define PSTORM_FCOE_TX_STATS_OFFSET(pf_id) \
-                                (IRO[41].base + ((pf_id) * IRO[41].m1))
-#define PSTORM_FCOE_TX_STATS_SIZE      (IRO[41].size)
-/* Pstorm RoCE statistics */
-#define PSTORM_ROCE_STAT_OFFSET(stat_counter_id) \
-                        (IRO[42].base + ((stat_counter_id) * IRO[42].m1))
-#define PSTORM_ROCE_STAT_SIZE          (IRO[42].size)
-/* Tstorm RoCE statistics */
-#define TSTORM_ROCE_STAT_OFFSET(stat_counter_id) \
-                        (IRO[43].base + ((stat_counter_id) * IRO[43].m1))
-#define TSTORM_ROCE_STAT_SIZE          (IRO[43].size)
-
-static const struct iro iro_arr[44] = {
-        { 0x10,    0x0,    0x0,    0x0,    0x8      },
-        { 0x47c8,  0x60,   0x0,    0x0,    0x60     },
-        { 0x5e30,  0x20,   0x0,    0x0,    0x20     },
-        { 0x510,   0x8,    0x0,    0x0,    0x4      },
-        { 0x490,   0x8,    0x0,    0x0,    0x4      },
-        { 0x10,    0x8,    0x0,    0x0,    0x2      },
-        { 0x90,    0x8,    0x0,    0x0,    0x2      },
-        { 0x4940,  0x0,    0x0,    0x0,    0x78     },
-        { 0x3de0,  0x0,    0x0,    0x0,    0x78     },
-        { 0x2998,  0x0,    0x0,    0x0,    0x78     },
-        { 0x4750,  0x0,    0x0,    0x0,    0x78     },
-        { 0x56d0,  0x0,    0x0,    0x0,    0x78     },
-        { 0x7e50,  0x0,    0x0,    0x0,    0x78     },
-        { 0x100,   0x8,    0x0,    0x0,    0x8      },
-        { 0x5c10,  0x10,   0x0,    0x0,    0x10     },
-        { 0xb508,  0x30,   0x0,    0x0,    0x30     },
-        { 0x95c0,  0x30,   0x0,    0x0,    0x30     },
-        { 0x58a0,  0x40,   0x0,    0x0,    0x40     },
-        { 0x200,   0x10,   0x0,    0x0,    0x8      },
-        { 0xa230,  0x0,    0x0,    0x0,    0x4      },
-        { 0x8058,  0x40,   0x0,    0x0,    0x30     },
-        { 0xd00,   0x8,    0x0,    0x0,    0x8      },
-        { 0x2b30,  0x80,   0x0,    0x0,    0x38     },
-        { 0xa808,  0x0,    0x0,    0x0,    0xf0     },
-        { 0xa8f8,  0x8,    0x0,    0x0,    0x8      },
-        { 0x80,    0x8,    0x0,    0x0,    0x8      },
-        { 0xac0,   0x8,    0x0,    0x0,    0x8      },
-        { 0x2580,  0x8,    0x0,    0x0,    0x8      },
-        { 0x2500,  0x8,    0x0,    0x0,    0x8      },
-        { 0x440,   0x8,    0x0,    0x0,    0x2      },
-        { 0x1800,  0x8,    0x0,    0x0,    0x2      },
-        { 0x1a00,  0x10,   0x8,    0x0,    0x2      },
-        { 0x640,   0x10,   0x8,    0x0,    0x2      },
-        { 0xd9b8,  0x38,   0x0,    0x0,    0x24     },
-        { 0x11048, 0x10,   0x0,    0x0,    0x8      },
-        { 0x11678, 0x38,   0x0,    0x0,    0x18     },
-        { 0xaec0,  0x30,   0x0,    0x0,    0x10     },
-        { 0x8700,  0x28,   0x0,    0x0,    0x18     },
-        { 0xec00,  0x10,   0x0,    0x0,    0x10     },
-        { 0xde38,  0x40,   0x0,    0x0,    0x30     },
-        { 0x121a8, 0x38,   0x0,    0x0,    0x8      },
-        { 0xf068,  0x20,   0x0,    0x0,    0x20     },
-        { 0x2b68,  0x80,   0x0,    0x0,    0x10     },
-        { 0x4ab8,  0x10,   0x0,    0x0,    0x10     },
+                           struct qed_ptt *p_ptt,
+                           bool eth_geneve_enable, bool ip_geneve_enable);
+
+#define YSTORM_FLOW_CONTROL_MODE_OFFSET                 (IRO[0].base)
+#define YSTORM_FLOW_CONTROL_MODE_SIZE                   (IRO[0].size)
+#define TSTORM_PORT_STAT_OFFSET(port_id) \
+        (IRO[1].base + ((port_id) * IRO[1].m1))
+#define TSTORM_PORT_STAT_SIZE                           (IRO[1].size)
+#define USTORM_VF_PF_CHANNEL_READY_OFFSET(vf_id) \
+        (IRO[3].base + ((vf_id) * IRO[3].m1))
+#define USTORM_VF_PF_CHANNEL_READY_SIZE                 (IRO[3].size)
+#define USTORM_FLR_FINAL_ACK_OFFSET(pf_id) \
+        (IRO[4].base + (pf_id) * IRO[4].m1)
+#define USTORM_FLR_FINAL_ACK_SIZE                       (IRO[4].size)
+#define USTORM_EQE_CONS_OFFSET(pf_id) \
+        (IRO[5].base + ((pf_id) * IRO[5].m1))
+#define USTORM_EQE_CONS_SIZE                            (IRO[5].size)
+#define USTORM_ETH_QUEUE_ZONE_OFFSET(queue_zone_id) \
+        (IRO[6].base + ((queue_zone_id) * IRO[6].m1))
+#define USTORM_ETH_QUEUE_ZONE_SIZE                      (IRO[6].size)
+#define USTORM_COMMON_QUEUE_CONS_OFFSET(queue_zone_id) \
+        (IRO[7].base + ((queue_zone_id) * IRO[7].m1))
+#define USTORM_COMMON_QUEUE_CONS_SIZE                   (IRO[7].size)
+#define MSTORM_QUEUE_STAT_OFFSET(stat_counter_id) \
+        (IRO[18].base + ((stat_counter_id) * IRO[18].m1))
+#define MSTORM_QUEUE_STAT_SIZE                          (IRO[18].size)
+#define MSTORM_ETH_PF_PRODS_OFFSET(queue_id) \
+        (IRO[19].base + ((queue_id) * IRO[19].m1))
+#define MSTORM_ETH_PF_PRODS_SIZE                        (IRO[19].size)
+#define MSTORM_TPA_TIMEOUT_US_OFFSET                    (IRO[20].base)
+#define MSTORM_TPA_TIMEOUT_US_SIZE                      (IRO[20].size)
+#define MSTORM_ETH_PF_STAT_OFFSET(pf_id) \
+        (IRO[21].base + ((pf_id) * IRO[21].m1))
+#define MSTORM_ETH_PF_STAT_SIZE                         (IRO[21].size)
+#define USTORM_QUEUE_STAT_OFFSET(stat_counter_id) \
+        (IRO[22].base + ((stat_counter_id) * IRO[22].m1))
+#define USTORM_QUEUE_STAT_SIZE                          (IRO[22].size)
+#define USTORM_ETH_PF_STAT_OFFSET(pf_id) \
+        (IRO[23].base + ((pf_id) * IRO[23].m1))
+#define USTORM_ETH_PF_STAT_SIZE                         (IRO[23].size)
+#define PSTORM_QUEUE_STAT_OFFSET(stat_counter_id) \
+        (IRO[24].base + ((stat_counter_id) * IRO[24].m1))
+#define PSTORM_QUEUE_STAT_SIZE                          (IRO[24].size)
+#define PSTORM_ETH_PF_STAT_OFFSET(pf_id) \
+        (IRO[25].base + ((pf_id) * IRO[25].m1))
+#define PSTORM_ETH_PF_STAT_SIZE                         (IRO[25].size)
+#define PSTORM_CTL_FRAME_ETHTYPE_OFFSET(ethtype) \
+        (IRO[26].base + ((ethtype) * IRO[26].m1))
+#define PSTORM_CTL_FRAME_ETHTYPE_SIZE                   (IRO[26].size)
+#define TSTORM_ETH_PRS_INPUT_OFFSET                     (IRO[27].base)
+#define TSTORM_ETH_PRS_INPUT_SIZE                       (IRO[27].size)
+#define ETH_RX_RATE_LIMIT_OFFSET(pf_id) \
+        (IRO[28].base + ((pf_id) * IRO[28].m1))
+#define ETH_RX_RATE_LIMIT_SIZE                          (IRO[28].size)
+#define XSTORM_ETH_QUEUE_ZONE_OFFSET(queue_id) \
+        (IRO[29].base + ((queue_id) * IRO[29].m1))
+#define XSTORM_ETH_QUEUE_ZONE_SIZE                      (IRO[29].size)
+
+static const struct iro iro_arr[46] = {
+        {0x0, 0x0, 0x0, 0x0, 0x8},
+        {0x4cb0, 0x78, 0x0, 0x0, 0x78},
+        {0x6318, 0x20, 0x0, 0x0, 0x20},
+        {0xb00, 0x8, 0x0, 0x0, 0x4},
+        {0xa80, 0x8, 0x0, 0x0, 0x4},
+        {0x0, 0x8, 0x0, 0x0, 0x2},
+        {0x80, 0x8, 0x0, 0x0, 0x4},
+        {0x84, 0x8, 0x0, 0x0, 0x2},
+        {0x4bc0, 0x0, 0x0, 0x0, 0x78},
+        {0x3df0, 0x0, 0x0, 0x0, 0x78},
+        {0x29b0, 0x0, 0x0, 0x0, 0x78},
+        {0x4c38, 0x0, 0x0, 0x0, 0x78},
+        {0x4a48, 0x0, 0x0, 0x0, 0x78},
+        {0x7e48, 0x0, 0x0, 0x0, 0x78},
+        {0xa28, 0x8, 0x0, 0x0, 0x8},
+        {0x60f8, 0x10, 0x0, 0x0, 0x10},
+        {0xb820, 0x30, 0x0, 0x0, 0x30},
+        {0x95b8, 0x30, 0x0, 0x0, 0x30},
+        {0x4c18, 0x80, 0x0, 0x0, 0x40},
+        {0x1f8, 0x4, 0x0, 0x0, 0x4},
+        {0xc9a8, 0x0, 0x0, 0x0, 0x4},
+        {0x4c58, 0x80, 0x0, 0x0, 0x20},
+        {0x8050, 0x40, 0x0, 0x0, 0x30},
+        {0xe770, 0x60, 0x0, 0x0, 0x60},
+        {0x2b48, 0x80, 0x0, 0x0, 0x38},
+        {0xdf88, 0x78, 0x0, 0x0, 0x78},
+        {0x1f8, 0x4, 0x0, 0x0, 0x4},
+        {0xacf0, 0x0, 0x0, 0x0, 0xf0},
+        {0xade0, 0x8, 0x0, 0x0, 0x8},
+        {0x1f8, 0x8, 0x0, 0x0, 0x8},
+        {0xac0, 0x8, 0x0, 0x0, 0x8},
+        {0x2578, 0x8, 0x0, 0x0, 0x8},
+        {0x24f8, 0x8, 0x0, 0x0, 0x8},
+        {0x0, 0x8, 0x0, 0x0, 0x8},
+        {0x200, 0x10, 0x8, 0x0, 0x8},
+        {0xb78, 0x10, 0x8, 0x0, 0x2},
+        {0xd888, 0x38, 0x0, 0x0, 0x24},
+        {0x12120, 0x10, 0x0, 0x0, 0x8},
+        {0x11b20, 0x38, 0x0, 0x0, 0x18},
+        {0xa8c0, 0x30, 0x0, 0x0, 0x10},
+        {0x86f8, 0x28, 0x0, 0x0, 0x18},
+        {0xeff8, 0x10, 0x0, 0x0, 0x10},
+        {0xdd08, 0x48, 0x0, 0x0, 0x38},
+        {0xf460, 0x20, 0x0, 0x0, 0x20},
+        {0x2b80, 0x80, 0x0, 0x0, 0x10},
+        {0x5000, 0x10, 0x0, 0x0, 0x10},
 };
 
 /* Runtime array offsets */
-#define DORQ_REG_PF_MAX_ICID_0_RT_OFFSET                                0
-#define DORQ_REG_PF_MAX_ICID_1_RT_OFFSET                                1
-#define DORQ_REG_PF_MAX_ICID_2_RT_OFFSET                                2
-#define DORQ_REG_PF_MAX_ICID_3_RT_OFFSET                                3
-#define DORQ_REG_PF_MAX_ICID_4_RT_OFFSET                                4
-#define DORQ_REG_PF_MAX_ICID_5_RT_OFFSET                                5
-#define DORQ_REG_PF_MAX_ICID_6_RT_OFFSET                                6
-#define DORQ_REG_PF_MAX_ICID_7_RT_OFFSET                                7
-#define DORQ_REG_VF_MAX_ICID_0_RT_OFFSET                                8
-#define DORQ_REG_VF_MAX_ICID_1_RT_OFFSET                                9
-#define DORQ_REG_VF_MAX_ICID_2_RT_OFFSET                                10
-#define DORQ_REG_VF_MAX_ICID_3_RT_OFFSET                                11
-#define DORQ_REG_VF_MAX_ICID_4_RT_OFFSET                                12
-#define DORQ_REG_VF_MAX_ICID_5_RT_OFFSET                                13
-#define DORQ_REG_VF_MAX_ICID_6_RT_OFFSET                                14
-#define DORQ_REG_VF_MAX_ICID_7_RT_OFFSET                                15
-#define DORQ_REG_PF_WAKE_ALL_RT_OFFSET                                  16
-#define DORQ_REG_TAG1_ETHERTYPE_RT_OFFSET                               17
-#define IGU_REG_PF_CONFIGURATION_RT_OFFSET                              18
-#define IGU_REG_VF_CONFIGURATION_RT_OFFSET                              19
-#define IGU_REG_ATTN_MSG_ADDR_L_RT_OFFSET                               20
-#define IGU_REG_ATTN_MSG_ADDR_H_RT_OFFSET                               21
-#define IGU_REG_LEADING_EDGE_LATCH_RT_OFFSET                            22
-#define IGU_REG_TRAILING_EDGE_LATCH_RT_OFFSET                           23
-#define CAU_REG_CQE_AGG_UNIT_SIZE_RT_OFFSET                             24
-#define CAU_REG_SB_VAR_MEMORY_RT_OFFSET                                 761
-#define CAU_REG_SB_VAR_MEMORY_RT_SIZE                                   736
-#define CAU_REG_SB_VAR_MEMORY_RT_OFFSET                                 761
-#define CAU_REG_SB_VAR_MEMORY_RT_SIZE                                   736
-#define CAU_REG_SB_ADDR_MEMORY_RT_OFFSET                                1497
-#define CAU_REG_SB_ADDR_MEMORY_RT_SIZE                                  736
-#define CAU_REG_PI_MEMORY_RT_OFFSET                                     2233
-#define CAU_REG_PI_MEMORY_RT_SIZE                                       4416
-#define PRS_REG_SEARCH_RESP_INITIATOR_TYPE_RT_OFFSET                    6649
-#define PRS_REG_TASK_ID_MAX_INITIATOR_PF_RT_OFFSET                      6650
-#define PRS_REG_TASK_ID_MAX_INITIATOR_VF_RT_OFFSET                      6651
-#define PRS_REG_TASK_ID_MAX_TARGET_PF_RT_OFFSET                         6652
-#define PRS_REG_TASK_ID_MAX_TARGET_VF_RT_OFFSET                         6653
-#define PRS_REG_SEARCH_TCP_RT_OFFSET                                    6654
-#define PRS_REG_SEARCH_FCOE_RT_OFFSET                                   6655
-#define PRS_REG_SEARCH_ROCE_RT_OFFSET                                   6656
-#define PRS_REG_ROCE_DEST_QP_MAX_VF_RT_OFFSET                           6657
-#define PRS_REG_ROCE_DEST_QP_MAX_PF_RT_OFFSET                           6658
-#define PRS_REG_SEARCH_OPENFLOW_RT_OFFSET                               6659
-#define PRS_REG_SEARCH_NON_IP_AS_OPENFLOW_RT_OFFSET                     6660
-#define PRS_REG_OPENFLOW_SUPPORT_ONLY_KNOWN_OVER_IP_RT_OFFSET           6661
-#define PRS_REG_OPENFLOW_SEARCH_KEY_MASK_RT_OFFSET                      6662
-#define PRS_REG_TAG_ETHERTYPE_0_RT_OFFSET                               6663
-#define PRS_REG_LIGHT_L2_ETHERTYPE_EN_RT_OFFSET                         6664
-#define SRC_REG_FIRSTFREE_RT_OFFSET                                     6665
-#define SRC_REG_FIRSTFREE_RT_SIZE                                       2
-#define SRC_REG_LASTFREE_RT_OFFSET                                      6667
-#define SRC_REG_LASTFREE_RT_SIZE                                        2
-#define SRC_REG_COUNTFREE_RT_OFFSET                                     6669
-#define SRC_REG_NUMBER_HASH_BITS_RT_OFFSET                              6670
-#define PSWRQ2_REG_CDUT_P_SIZE_RT_OFFSET                                6671
-#define PSWRQ2_REG_CDUC_P_SIZE_RT_OFFSET                                6672
-#define PSWRQ2_REG_TM_P_SIZE_RT_OFFSET                                  6673
-#define PSWRQ2_REG_QM_P_SIZE_RT_OFFSET                                  6674
-#define PSWRQ2_REG_SRC_P_SIZE_RT_OFFSET                                 6675
-#define PSWRQ2_REG_TM_FIRST_ILT_RT_OFFSET                               6676
-#define PSWRQ2_REG_TM_LAST_ILT_RT_OFFSET                                6677
-#define PSWRQ2_REG_QM_FIRST_ILT_RT_OFFSET                               6678
-#define PSWRQ2_REG_QM_LAST_ILT_RT_OFFSET                                6679
-#define PSWRQ2_REG_SRC_FIRST_ILT_RT_OFFSET                              6680
-#define PSWRQ2_REG_SRC_LAST_ILT_RT_OFFSET                               6681
-#define PSWRQ2_REG_CDUC_FIRST_ILT_RT_OFFSET                             6682
-#define PSWRQ2_REG_CDUC_LAST_ILT_RT_OFFSET                              6683
-#define PSWRQ2_REG_CDUT_FIRST_ILT_RT_OFFSET                             6684
-#define PSWRQ2_REG_CDUT_LAST_ILT_RT_OFFSET                              6685
-#define PSWRQ2_REG_TSDM_FIRST_ILT_RT_OFFSET                             6686
-#define PSWRQ2_REG_TSDM_LAST_ILT_RT_OFFSET                              6687
-#define PSWRQ2_REG_TM_NUMBER_OF_PF_BLOCKS_RT_OFFSET                     6688
-#define PSWRQ2_REG_CDUT_NUMBER_OF_PF_BLOCKS_RT_OFFSET                   6689
-#define PSWRQ2_REG_CDUC_NUMBER_OF_PF_BLOCKS_RT_OFFSET                   6690
-#define PSWRQ2_REG_TM_VF_BLOCKS_RT_OFFSET                               6691
-#define PSWRQ2_REG_CDUT_VF_BLOCKS_RT_OFFSET                             6692
-#define PSWRQ2_REG_CDUC_VF_BLOCKS_RT_OFFSET                             6693
-#define PSWRQ2_REG_TM_BLOCKS_FACTOR_RT_OFFSET                           6694
-#define PSWRQ2_REG_CDUT_BLOCKS_FACTOR_RT_OFFSET                         6695
-#define PSWRQ2_REG_CDUC_BLOCKS_FACTOR_RT_OFFSET                         6696
-#define PSWRQ2_REG_VF_BASE_RT_OFFSET                                    6697
-#define PSWRQ2_REG_VF_LAST_ILT_RT_OFFSET                                6698
-#define PSWRQ2_REG_WR_MBS0_RT_OFFSET                                    6699
-#define PSWRQ2_REG_RD_MBS0_RT_OFFSET                                    6700
-#define PSWRQ2_REG_DRAM_ALIGN_WR_RT_OFFSET                              6701
-#define PSWRQ2_REG_DRAM_ALIGN_RD_RT_OFFSET                              6702
-#define PSWRQ2_REG_ILT_MEMORY_RT_OFFSET                                 6703
-#define PSWRQ2_REG_ILT_MEMORY_RT_SIZE                                   22000
-#define PGLUE_REG_B_VF_BASE_RT_OFFSET                                   28703
-#define PGLUE_REG_B_CACHE_LINE_SIZE_RT_OFFSET                           28704
-#define PGLUE_REG_B_PF_BAR0_SIZE_RT_OFFSET                              28705
-#define PGLUE_REG_B_PF_BAR1_SIZE_RT_OFFSET                              28706
-#define PGLUE_REG_B_VF_BAR1_SIZE_RT_OFFSET                              28707
-#define TM_REG_VF_ENABLE_CONN_RT_OFFSET                                 28708
-#define TM_REG_PF_ENABLE_CONN_RT_OFFSET                                 28709
-#define TM_REG_PF_ENABLE_TASK_RT_OFFSET                                 28710
-#define TM_REG_GROUP_SIZE_RESOLUTION_CONN_RT_OFFSET                     28711
-#define TM_REG_GROUP_SIZE_RESOLUTION_TASK_RT_OFFSET                     28712
-#define TM_REG_CONFIG_CONN_MEM_RT_OFFSET                                28713
-#define TM_REG_CONFIG_CONN_MEM_RT_SIZE                                  416
-#define TM_REG_CONFIG_TASK_MEM_RT_OFFSET                                29129
-#define TM_REG_CONFIG_TASK_MEM_RT_SIZE                                  512
-#define QM_REG_MAXPQSIZE_0_RT_OFFSET                                    29641
-#define QM_REG_MAXPQSIZE_1_RT_OFFSET                                    29642
-#define QM_REG_MAXPQSIZE_2_RT_OFFSET                                    29643
-#define QM_REG_MAXPQSIZETXSEL_0_RT_OFFSET                               29644
-#define QM_REG_MAXPQSIZETXSEL_1_RT_OFFSET                               29645
-#define QM_REG_MAXPQSIZETXSEL_2_RT_OFFSET                               29646
-#define QM_REG_MAXPQSIZETXSEL_3_RT_OFFSET                               29647
-#define QM_REG_MAXPQSIZETXSEL_4_RT_OFFSET                               29648
-#define QM_REG_MAXPQSIZETXSEL_5_RT_OFFSET                               29649
-#define QM_REG_MAXPQSIZETXSEL_6_RT_OFFSET                               29650
-#define QM_REG_MAXPQSIZETXSEL_7_RT_OFFSET                               29651
-#define QM_REG_MAXPQSIZETXSEL_8_RT_OFFSET                               29652
-#define QM_REG_MAXPQSIZETXSEL_9_RT_OFFSET                               29653
-#define QM_REG_MAXPQSIZETXSEL_10_RT_OFFSET                              29654
-#define QM_REG_MAXPQSIZETXSEL_11_RT_OFFSET                              29655
-#define QM_REG_MAXPQSIZETXSEL_12_RT_OFFSET                              29656
-#define QM_REG_MAXPQSIZETXSEL_13_RT_OFFSET                              29657
-#define QM_REG_MAXPQSIZETXSEL_14_RT_OFFSET                              29658
-#define QM_REG_MAXPQSIZETXSEL_15_RT_OFFSET                              29659
-#define QM_REG_MAXPQSIZETXSEL_16_RT_OFFSET                              29660
-#define QM_REG_MAXPQSIZETXSEL_17_RT_OFFSET                              29661
-#define QM_REG_MAXPQSIZETXSEL_18_RT_OFFSET                              29662
-#define QM_REG_MAXPQSIZETXSEL_19_RT_OFFSET                              29663
-#define QM_REG_MAXPQSIZETXSEL_20_RT_OFFSET                              29664
-#define QM_REG_MAXPQSIZETXSEL_21_RT_OFFSET                              29665
-#define QM_REG_MAXPQSIZETXSEL_22_RT_OFFSET                              29666
-#define QM_REG_MAXPQSIZETXSEL_23_RT_OFFSET                              29667
-#define QM_REG_MAXPQSIZETXSEL_24_RT_OFFSET                              29668
-#define QM_REG_MAXPQSIZETXSEL_25_RT_OFFSET                              29669
-#define QM_REG_MAXPQSIZETXSEL_26_RT_OFFSET                              29670
-#define QM_REG_MAXPQSIZETXSEL_27_RT_OFFSET                              29671
-#define QM_REG_MAXPQSIZETXSEL_28_RT_OFFSET                              29672
-#define QM_REG_MAXPQSIZETXSEL_29_RT_OFFSET                              29673
-#define QM_REG_MAXPQSIZETXSEL_30_RT_OFFSET                              29674
-#define QM_REG_MAXPQSIZETXSEL_31_RT_OFFSET                              29675
-#define QM_REG_MAXPQSIZETXSEL_32_RT_OFFSET                              29676
-#define QM_REG_MAXPQSIZETXSEL_33_RT_OFFSET                              29677
-#define QM_REG_MAXPQSIZETXSEL_34_RT_OFFSET                              29678
-#define QM_REG_MAXPQSIZETXSEL_35_RT_OFFSET                              29679
-#define QM_REG_MAXPQSIZETXSEL_36_RT_OFFSET                              29680
-#define QM_REG_MAXPQSIZETXSEL_37_RT_OFFSET                              29681
-#define QM_REG_MAXPQSIZETXSEL_38_RT_OFFSET                              29682
-#define QM_REG_MAXPQSIZETXSEL_39_RT_OFFSET                              29683
-#define QM_REG_MAXPQSIZETXSEL_40_RT_OFFSET                              29684
-#define QM_REG_MAXPQSIZETXSEL_41_RT_OFFSET                              29685
-#define QM_REG_MAXPQSIZETXSEL_42_RT_OFFSET                              29686
-#define QM_REG_MAXPQSIZETXSEL_43_RT_OFFSET                              29687
-#define QM_REG_MAXPQSIZETXSEL_44_RT_OFFSET                              29688
-#define QM_REG_MAXPQSIZETXSEL_45_RT_OFFSET                              29689
-#define QM_REG_MAXPQSIZETXSEL_46_RT_OFFSET                              29690
-#define QM_REG_MAXPQSIZETXSEL_47_RT_OFFSET                              29691
-#define QM_REG_MAXPQSIZETXSEL_48_RT_OFFSET                              29692
-#define QM_REG_MAXPQSIZETXSEL_49_RT_OFFSET                              29693
-#define QM_REG_MAXPQSIZETXSEL_50_RT_OFFSET                              29694
-#define QM_REG_MAXPQSIZETXSEL_51_RT_OFFSET                              29695
-#define QM_REG_MAXPQSIZETXSEL_52_RT_OFFSET                              29696
-#define QM_REG_MAXPQSIZETXSEL_53_RT_OFFSET                              29697
-#define QM_REG_MAXPQSIZETXSEL_54_RT_OFFSET                              29698
-#define QM_REG_MAXPQSIZETXSEL_55_RT_OFFSET                              29699
-#define QM_REG_MAXPQSIZETXSEL_56_RT_OFFSET                              29700
-#define QM_REG_MAXPQSIZETXSEL_57_RT_OFFSET                              29701
-#define QM_REG_MAXPQSIZETXSEL_58_RT_OFFSET                              29702
-#define QM_REG_MAXPQSIZETXSEL_59_RT_OFFSET                              29703
-#define QM_REG_MAXPQSIZETXSEL_60_RT_OFFSET                              29704
-#define QM_REG_MAXPQSIZETXSEL_61_RT_OFFSET                              29705
-#define QM_REG_MAXPQSIZETXSEL_62_RT_OFFSET                              29706
-#define QM_REG_MAXPQSIZETXSEL_63_RT_OFFSET                              29707
-#define QM_REG_BASEADDROTHERPQ_RT_OFFSET                                29708
-#define QM_REG_BASEADDROTHERPQ_RT_SIZE                                  128
-#define QM_REG_VOQCRDLINE_RT_OFFSET                                     29836
-#define QM_REG_VOQCRDLINE_RT_SIZE                                       20
-#define QM_REG_VOQINITCRDLINE_RT_OFFSET                                 29856
-#define QM_REG_VOQINITCRDLINE_RT_SIZE                                   20
-#define QM_REG_AFULLQMBYPTHRPFWFQ_RT_OFFSET                             29876
-#define QM_REG_AFULLQMBYPTHRVPWFQ_RT_OFFSET                             29877
-#define QM_REG_AFULLQMBYPTHRPFRL_RT_OFFSET                              29878
-#define QM_REG_AFULLQMBYPTHRGLBLRL_RT_OFFSET                            29879
-#define QM_REG_AFULLOPRTNSTCCRDMASK_RT_OFFSET                           29880
-#define QM_REG_WRROTHERPQGRP_0_RT_OFFSET                                29881
-#define QM_REG_WRROTHERPQGRP_1_RT_OFFSET                                29882
-#define QM_REG_WRROTHERPQGRP_2_RT_OFFSET                                29883
-#define QM_REG_WRROTHERPQGRP_3_RT_OFFSET                                29884
-#define QM_REG_WRROTHERPQGRP_4_RT_OFFSET                                29885
-#define QM_REG_WRROTHERPQGRP_5_RT_OFFSET                                29886
-#define QM_REG_WRROTHERPQGRP_6_RT_OFFSET                                29887
-#define QM_REG_WRROTHERPQGRP_7_RT_OFFSET                                29888
-#define QM_REG_WRROTHERPQGRP_8_RT_OFFSET                                29889
-#define QM_REG_WRROTHERPQGRP_9_RT_OFFSET                                29890
-#define QM_REG_WRROTHERPQGRP_10_RT_OFFSET                               29891
-#define QM_REG_WRROTHERPQGRP_11_RT_OFFSET                               29892
-#define QM_REG_WRROTHERPQGRP_12_RT_OFFSET                               29893
-#define QM_REG_WRROTHERPQGRP_13_RT_OFFSET                               29894
-#define QM_REG_WRROTHERPQGRP_14_RT_OFFSET                               29895
-#define QM_REG_WRROTHERPQGRP_15_RT_OFFSET                               29896
-#define QM_REG_WRROTHERGRPWEIGHT_0_RT_OFFSET                            29897
-#define QM_REG_WRROTHERGRPWEIGHT_1_RT_OFFSET                            29898
-#define QM_REG_WRROTHERGRPWEIGHT_2_RT_OFFSET                            29899
-#define QM_REG_WRROTHERGRPWEIGHT_3_RT_OFFSET                            29900
-#define QM_REG_WRRTXGRPWEIGHT_0_RT_OFFSET                               29901
-#define QM_REG_WRRTXGRPWEIGHT_1_RT_OFFSET                               29902
-#define QM_REG_PQTX2PF_0_RT_OFFSET                                      29903
-#define QM_REG_PQTX2PF_1_RT_OFFSET                                      29904
-#define QM_REG_PQTX2PF_2_RT_OFFSET                                      29905
-#define QM_REG_PQTX2PF_3_RT_OFFSET                                      29906
-#define QM_REG_PQTX2PF_4_RT_OFFSET                                      29907
-#define QM_REG_PQTX2PF_5_RT_OFFSET                                      29908
-#define QM_REG_PQTX2PF_6_RT_OFFSET                                      29909
-#define QM_REG_PQTX2PF_7_RT_OFFSET                                      29910
-#define QM_REG_PQTX2PF_8_RT_OFFSET                                      29911
-#define QM_REG_PQTX2PF_9_RT_OFFSET                                      29912
-#define QM_REG_PQTX2PF_10_RT_OFFSET                                     29913
-#define QM_REG_PQTX2PF_11_RT_OFFSET                                     29914
-#define QM_REG_PQTX2PF_12_RT_OFFSET                                     29915
-#define QM_REG_PQTX2PF_13_RT_OFFSET                                     29916
-#define QM_REG_PQTX2PF_14_RT_OFFSET                                     29917
-#define QM_REG_PQTX2PF_15_RT_OFFSET                                     29918
-#define QM_REG_PQTX2PF_16_RT_OFFSET                                     29919
-#define QM_REG_PQTX2PF_17_RT_OFFSET                                     29920
-#define QM_REG_PQTX2PF_18_RT_OFFSET                                     29921
-#define QM_REG_PQTX2PF_19_RT_OFFSET                                     29922
-#define QM_REG_PQTX2PF_20_RT_OFFSET                                     29923
-#define QM_REG_PQTX2PF_21_RT_OFFSET                                     29924
-#define QM_REG_PQTX2PF_22_RT_OFFSET                                     29925
-#define QM_REG_PQTX2PF_23_RT_OFFSET                                     29926
-#define QM_REG_PQTX2PF_24_RT_OFFSET                                     29927
-#define QM_REG_PQTX2PF_25_RT_OFFSET                                     29928
-#define QM_REG_PQTX2PF_26_RT_OFFSET                                     29929
-#define QM_REG_PQTX2PF_27_RT_OFFSET                                     29930
-#define QM_REG_PQTX2PF_28_RT_OFFSET                                     29931
-#define QM_REG_PQTX2PF_29_RT_OFFSET                                     29932
-#define QM_REG_PQTX2PF_30_RT_OFFSET                                     29933
-#define QM_REG_PQTX2PF_31_RT_OFFSET                                     29934
-#define QM_REG_PQTX2PF_32_RT_OFFSET                                     29935
-#define QM_REG_PQTX2PF_33_RT_OFFSET                                     29936
-#define QM_REG_PQTX2PF_34_RT_OFFSET                                     29937
-#define QM_REG_PQTX2PF_35_RT_OFFSET                                     29938
-#define QM_REG_PQTX2PF_36_RT_OFFSET                                     29939
-#define QM_REG_PQTX2PF_37_RT_OFFSET                                     29940
-#define QM_REG_PQTX2PF_38_RT_OFFSET                                     29941
-#define QM_REG_PQTX2PF_39_RT_OFFSET                                     29942
-#define QM_REG_PQTX2PF_40_RT_OFFSET                                     29943
-#define QM_REG_PQTX2PF_41_RT_OFFSET                                     29944
-#define QM_REG_PQTX2PF_42_RT_OFFSET                                     29945
-#define QM_REG_PQTX2PF_43_RT_OFFSET                                     29946
-#define QM_REG_PQTX2PF_44_RT_OFFSET                                     29947
-#define QM_REG_PQTX2PF_45_RT_OFFSET                                     29948
-#define QM_REG_PQTX2PF_46_RT_OFFSET                                     29949
-#define QM_REG_PQTX2PF_47_RT_OFFSET                                     29950
-#define QM_REG_PQTX2PF_48_RT_OFFSET                                     29951
-#define QM_REG_PQTX2PF_49_RT_OFFSET                                     29952
-#define QM_REG_PQTX2PF_50_RT_OFFSET                                     29953
-#define QM_REG_PQTX2PF_51_RT_OFFSET                                     29954
-#define QM_REG_PQTX2PF_52_RT_OFFSET                                     29955
-#define QM_REG_PQTX2PF_53_RT_OFFSET                                     29956
-#define QM_REG_PQTX2PF_54_RT_OFFSET                                     29957
-#define QM_REG_PQTX2PF_55_RT_OFFSET                                     29958
-#define QM_REG_PQTX2PF_56_RT_OFFSET                                     29959
-#define QM_REG_PQTX2PF_57_RT_OFFSET                                     29960
-#define QM_REG_PQTX2PF_58_RT_OFFSET                                     29961
-#define QM_REG_PQTX2PF_59_RT_OFFSET                                     29962
-#define QM_REG_PQTX2PF_60_RT_OFFSET                                     29963
-#define QM_REG_PQTX2PF_61_RT_OFFSET                                     29964
-#define QM_REG_PQTX2PF_62_RT_OFFSET                                     29965
-#define QM_REG_PQTX2PF_63_RT_OFFSET                                     29966
-#define QM_REG_PQOTHER2PF_0_RT_OFFSET                                   29967
-#define QM_REG_PQOTHER2PF_1_RT_OFFSET                                   29968
-#define QM_REG_PQOTHER2PF_2_RT_OFFSET                                   29969
-#define QM_REG_PQOTHER2PF_3_RT_OFFSET                                   29970
-#define QM_REG_PQOTHER2PF_4_RT_OFFSET                                   29971
-#define QM_REG_PQOTHER2PF_5_RT_OFFSET                                   29972
-#define QM_REG_PQOTHER2PF_6_RT_OFFSET                                   29973
-#define QM_REG_PQOTHER2PF_7_RT_OFFSET                                   29974
-#define QM_REG_PQOTHER2PF_8_RT_OFFSET                                   29975
-#define QM_REG_PQOTHER2PF_9_RT_OFFSET                                   29976
-#define QM_REG_PQOTHER2PF_10_RT_OFFSET                                  29977
-#define QM_REG_PQOTHER2PF_11_RT_OFFSET                                  29978
-#define QM_REG_PQOTHER2PF_12_RT_OFFSET                                  29979
-#define QM_REG_PQOTHER2PF_13_RT_OFFSET                                  29980
-#define QM_REG_PQOTHER2PF_14_RT_OFFSET                                  29981
-#define QM_REG_PQOTHER2PF_15_RT_OFFSET                                  29982
-#define QM_REG_RLGLBLPERIOD_0_RT_OFFSET                                 29983
-#define QM_REG_RLGLBLPERIOD_1_RT_OFFSET                                 29984
-#define QM_REG_RLGLBLPERIODTIMER_0_RT_OFFSET                            29985
-#define QM_REG_RLGLBLPERIODTIMER_1_RT_OFFSET                            29986
-#define QM_REG_RLGLBLPERIODSEL_0_RT_OFFSET                              29987
-#define QM_REG_RLGLBLPERIODSEL_1_RT_OFFSET                              29988
-#define QM_REG_RLGLBLPERIODSEL_2_RT_OFFSET                              29989
-#define QM_REG_RLGLBLPERIODSEL_3_RT_OFFSET                              29990
-#define QM_REG_RLGLBLPERIODSEL_4_RT_OFFSET                              29991
-#define QM_REG_RLGLBLPERIODSEL_5_RT_OFFSET                              29992
-#define QM_REG_RLGLBLPERIODSEL_6_RT_OFFSET                              29993
-#define QM_REG_RLGLBLPERIODSEL_7_RT_OFFSET                              29994
-#define QM_REG_RLGLBLINCVAL_RT_OFFSET                                   29995
-#define QM_REG_RLGLBLINCVAL_RT_SIZE                                     256
-#define QM_REG_RLGLBLUPPERBOUND_RT_OFFSET                               30251
-#define QM_REG_RLGLBLUPPERBOUND_RT_SIZE                                 256
-#define QM_REG_RLGLBLCRD_RT_OFFSET                                      30507
-#define QM_REG_RLGLBLCRD_RT_SIZE                                        256
-#define QM_REG_RLGLBLENABLE_RT_OFFSET                                   30763
-#define QM_REG_RLPFPERIOD_RT_OFFSET                                     30764
-#define QM_REG_RLPFPERIODTIMER_RT_OFFSET                                30765
-#define QM_REG_RLPFINCVAL_RT_OFFSET                                     30766
-#define QM_REG_RLPFINCVAL_RT_SIZE                                       16
-#define QM_REG_RLPFUPPERBOUND_RT_OFFSET                                 30782
-#define QM_REG_RLPFUPPERBOUND_RT_SIZE                                   16
-#define QM_REG_RLPFCRD_RT_OFFSET                                        30798
-#define QM_REG_RLPFCRD_RT_SIZE                                          16
-#define QM_REG_RLPFENABLE_RT_OFFSET                                     30814
-#define QM_REG_RLPFVOQENABLE_RT_OFFSET                                  30815
-#define QM_REG_WFQPFWEIGHT_RT_OFFSET                                    30816
-#define QM_REG_WFQPFWEIGHT_RT_SIZE                                      16
-#define QM_REG_WFQPFUPPERBOUND_RT_OFFSET                                30832
-#define QM_REG_WFQPFUPPERBOUND_RT_SIZE                                  16
-#define QM_REG_WFQPFCRD_RT_OFFSET                                       30848
-#define QM_REG_WFQPFCRD_RT_SIZE                                         160
-#define QM_REG_WFQPFENABLE_RT_OFFSET                                    31008
-#define QM_REG_WFQVPENABLE_RT_OFFSET                                    31009
-#define QM_REG_BASEADDRTXPQ_RT_OFFSET                                   31010
-#define QM_REG_BASEADDRTXPQ_RT_SIZE                                     512
-#define QM_REG_TXPQMAP_RT_OFFSET                                        31522
-#define QM_REG_TXPQMAP_RT_SIZE                                          512
-#define QM_REG_WFQVPWEIGHT_RT_OFFSET                                    32034
-#define QM_REG_WFQVPWEIGHT_RT_SIZE                                      512
-#define QM_REG_WFQVPCRD_RT_OFFSET                                       32546
-#define QM_REG_WFQVPCRD_RT_SIZE                                         512
-#define QM_REG_WFQVPMAP_RT_OFFSET                                       33058
-#define QM_REG_WFQVPMAP_RT_SIZE                                         512
-#define QM_REG_WFQPFCRD_MSB_RT_OFFSET                                   33570
-#define QM_REG_WFQPFCRD_MSB_RT_SIZE                                     160
-#define NIG_REG_TAG_ETHERTYPE_0_RT_OFFSET                               33730
-#define NIG_REG_OUTER_TAG_VALUE_LIST0_RT_OFFSET                         33731
-#define NIG_REG_OUTER_TAG_VALUE_LIST1_RT_OFFSET                         33732
-#define NIG_REG_OUTER_TAG_VALUE_LIST2_RT_OFFSET                         33733
-#define NIG_REG_OUTER_TAG_VALUE_LIST3_RT_OFFSET                         33734
-#define NIG_REG_OUTER_TAG_VALUE_MASK_RT_OFFSET                          33735
-#define NIG_REG_LLH_FUNC_TAGMAC_CLS_TYPE_RT_OFFSET                      33736
-#define NIG_REG_LLH_FUNC_TAG_EN_RT_OFFSET                               33737
-#define NIG_REG_LLH_FUNC_TAG_EN_RT_SIZE                                 4
-#define NIG_REG_LLH_FUNC_TAG_HDR_SEL_RT_OFFSET                          33741
-#define NIG_REG_LLH_FUNC_TAG_HDR_SEL_RT_SIZE                            4
-#define NIG_REG_LLH_FUNC_TAG_VALUE_RT_OFFSET                            33745
-#define NIG_REG_LLH_FUNC_TAG_VALUE_RT_SIZE                              4
-#define NIG_REG_LLH_FUNC_NO_TAG_RT_OFFSET                               33749
-#define NIG_REG_LLH_FUNC_FILTER_VALUE_RT_OFFSET                         33750
-#define NIG_REG_LLH_FUNC_FILTER_VALUE_RT_SIZE                           32
-#define NIG_REG_LLH_FUNC_FILTER_EN_RT_OFFSET                            33782
-#define NIG_REG_LLH_FUNC_FILTER_EN_RT_SIZE                              16
-#define NIG_REG_LLH_FUNC_FILTER_MODE_RT_OFFSET                          33798
-#define NIG_REG_LLH_FUNC_FILTER_MODE_RT_SIZE                            16
-#define NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_RT_OFFSET                 33814
-#define NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_RT_SIZE                   16
-#define NIG_REG_LLH_FUNC_FILTER_HDR_SEL_RT_OFFSET                       33830
-#define NIG_REG_LLH_FUNC_FILTER_HDR_SEL_RT_SIZE                         16
-#define NIG_REG_TX_EDPM_CTRL_RT_OFFSET                                  33846
-#define CDU_REG_CID_ADDR_PARAMS_RT_OFFSET                               33847
-#define CDU_REG_SEGMENT0_PARAMS_RT_OFFSET                               33848
-#define CDU_REG_SEGMENT1_PARAMS_RT_OFFSET                               33849
-#define CDU_REG_PF_SEG0_TYPE_OFFSET_RT_OFFSET                           33850
-#define CDU_REG_PF_SEG1_TYPE_OFFSET_RT_OFFSET                           33851
-#define CDU_REG_PF_SEG2_TYPE_OFFSET_RT_OFFSET                           33852
-#define CDU_REG_PF_SEG3_TYPE_OFFSET_RT_OFFSET                           33853
-#define CDU_REG_PF_FL_SEG0_TYPE_OFFSET_RT_OFFSET                        33854
-#define CDU_REG_PF_FL_SEG1_TYPE_OFFSET_RT_OFFSET                        33855
-#define CDU_REG_PF_FL_SEG2_TYPE_OFFSET_RT_OFFSET                        33856
-#define CDU_REG_PF_FL_SEG3_TYPE_OFFSET_RT_OFFSET                        33857
-#define CDU_REG_VF_SEG_TYPE_OFFSET_RT_OFFSET                            33858
-#define CDU_REG_VF_FL_SEG_TYPE_OFFSET_RT_OFFSET                         33859
-#define PBF_REG_TAG_ETHERTYPE_0_RT_OFFSET                               33860
-#define PBF_REG_BTB_SHARED_AREA_SIZE_RT_OFFSET                          33861
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ0_RT_OFFSET                        33862
-#define PBF_REG_BTB_GUARANTEED_VOQ0_RT_OFFSET                           33863
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ0_RT_OFFSET                    33864
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ1_RT_OFFSET                        33865
-#define PBF_REG_BTB_GUARANTEED_VOQ1_RT_OFFSET                           33866
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ1_RT_OFFSET                    33867
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ2_RT_OFFSET                        33868
-#define PBF_REG_BTB_GUARANTEED_VOQ2_RT_OFFSET                           33869
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ2_RT_OFFSET                    33870
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ3_RT_OFFSET                        33871
-#define PBF_REG_BTB_GUARANTEED_VOQ3_RT_OFFSET                           33872
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ3_RT_OFFSET                    33873
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ4_RT_OFFSET                        33874
-#define PBF_REG_BTB_GUARANTEED_VOQ4_RT_OFFSET                           33875
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ4_RT_OFFSET                    33876
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ5_RT_OFFSET                        33877
-#define PBF_REG_BTB_GUARANTEED_VOQ5_RT_OFFSET                           33878
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ5_RT_OFFSET                    33879
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ6_RT_OFFSET                        33880
-#define PBF_REG_BTB_GUARANTEED_VOQ6_RT_OFFSET                           33881
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ6_RT_OFFSET                    33882
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ7_RT_OFFSET                        33883
-#define PBF_REG_BTB_GUARANTEED_VOQ7_RT_OFFSET                           33884
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ7_RT_OFFSET                    33885
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ8_RT_OFFSET                        33886
-#define PBF_REG_BTB_GUARANTEED_VOQ8_RT_OFFSET                           33887
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ8_RT_OFFSET                    33888
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ9_RT_OFFSET                        33889
-#define PBF_REG_BTB_GUARANTEED_VOQ9_RT_OFFSET                           33890
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ9_RT_OFFSET                    33891
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ10_RT_OFFSET                       33892
-#define PBF_REG_BTB_GUARANTEED_VOQ10_RT_OFFSET                          33893
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ10_RT_OFFSET                   33894
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ11_RT_OFFSET                       33895
-#define PBF_REG_BTB_GUARANTEED_VOQ11_RT_OFFSET                          33896
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ11_RT_OFFSET                   33897
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ12_RT_OFFSET                       33898
-#define PBF_REG_BTB_GUARANTEED_VOQ12_RT_OFFSET                          33899
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ12_RT_OFFSET                   33900
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ13_RT_OFFSET                       33901
-#define PBF_REG_BTB_GUARANTEED_VOQ13_RT_OFFSET                          33902
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ13_RT_OFFSET                   33903
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ14_RT_OFFSET                       33904
-#define PBF_REG_BTB_GUARANTEED_VOQ14_RT_OFFSET                          33905
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ14_RT_OFFSET                   33906
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ15_RT_OFFSET                       33907
-#define PBF_REG_BTB_GUARANTEED_VOQ15_RT_OFFSET                          33908
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ15_RT_OFFSET                   33909
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ16_RT_OFFSET                       33910
-#define PBF_REG_BTB_GUARANTEED_VOQ16_RT_OFFSET                          33911
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ16_RT_OFFSET                   33912
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ17_RT_OFFSET                       33913
-#define PBF_REG_BTB_GUARANTEED_VOQ17_RT_OFFSET                          33914
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ17_RT_OFFSET                   33915
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ18_RT_OFFSET                       33916
-#define PBF_REG_BTB_GUARANTEED_VOQ18_RT_OFFSET                          33917
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ18_RT_OFFSET                   33918
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ19_RT_OFFSET                       33919
-#define PBF_REG_BTB_GUARANTEED_VOQ19_RT_OFFSET                          33920
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ19_RT_OFFSET                   33921
-#define XCM_REG_CON_PHY_Q3_RT_OFFSET                                    33922
-
-#define RUNTIME_ARRAY_SIZE 33923
+#define DORQ_REG_PF_MAX_ICID_0_RT_OFFSET 0
+#define DORQ_REG_PF_MAX_ICID_1_RT_OFFSET 1
+#define DORQ_REG_PF_MAX_ICID_2_RT_OFFSET 2
+#define DORQ_REG_PF_MAX_ICID_3_RT_OFFSET 3
+#define DORQ_REG_PF_MAX_ICID_4_RT_OFFSET 4
+#define DORQ_REG_PF_MAX_ICID_5_RT_OFFSET 5
+#define DORQ_REG_PF_MAX_ICID_6_RT_OFFSET 6
+#define DORQ_REG_PF_MAX_ICID_7_RT_OFFSET 7
+#define DORQ_REG_VF_MAX_ICID_0_RT_OFFSET 8
+#define DORQ_REG_VF_MAX_ICID_1_RT_OFFSET 9
+#define DORQ_REG_VF_MAX_ICID_2_RT_OFFSET 10
+#define DORQ_REG_VF_MAX_ICID_3_RT_OFFSET 11
+#define DORQ_REG_VF_MAX_ICID_4_RT_OFFSET 12
+#define DORQ_REG_VF_MAX_ICID_5_RT_OFFSET 13
+#define DORQ_REG_VF_MAX_ICID_6_RT_OFFSET 14
+#define DORQ_REG_VF_MAX_ICID_7_RT_OFFSET 15
+#define DORQ_REG_PF_WAKE_ALL_RT_OFFSET 16
+#define DORQ_REG_TAG1_ETHERTYPE_RT_OFFSET 17
+#define IGU_REG_PF_CONFIGURATION_RT_OFFSET 18
+#define IGU_REG_VF_CONFIGURATION_RT_OFFSET 19
+#define IGU_REG_ATTN_MSG_ADDR_L_RT_OFFSET 20
+#define IGU_REG_ATTN_MSG_ADDR_H_RT_OFFSET 21
+#define IGU_REG_LEADING_EDGE_LATCH_RT_OFFSET 22
+#define IGU_REG_TRAILING_EDGE_LATCH_RT_OFFSET 23
+#define CAU_REG_CQE_AGG_UNIT_SIZE_RT_OFFSET 24
+#define CAU_REG_SB_VAR_MEMORY_RT_OFFSET 761
+#define CAU_REG_SB_VAR_MEMORY_RT_SIZE 736
+#define CAU_REG_SB_VAR_MEMORY_RT_OFFSET 761
+#define CAU_REG_SB_VAR_MEMORY_RT_SIZE 736
+#define CAU_REG_SB_ADDR_MEMORY_RT_OFFSET 1497
+#define CAU_REG_SB_ADDR_MEMORY_RT_SIZE 736
+#define CAU_REG_PI_MEMORY_RT_OFFSET 2233
+#define CAU_REG_PI_MEMORY_RT_SIZE 4416
+#define PRS_REG_SEARCH_RESP_INITIATOR_TYPE_RT_OFFSET 6649
+#define PRS_REG_TASK_ID_MAX_INITIATOR_PF_RT_OFFSET 6650
+#define PRS_REG_TASK_ID_MAX_INITIATOR_VF_RT_OFFSET 6651
+#define PRS_REG_TASK_ID_MAX_TARGET_PF_RT_OFFSET 6652
+#define PRS_REG_TASK_ID_MAX_TARGET_VF_RT_OFFSET 6653
+#define PRS_REG_SEARCH_TCP_RT_OFFSET 6654
+#define PRS_REG_SEARCH_FCOE_RT_OFFSET 6655
+#define PRS_REG_SEARCH_ROCE_RT_OFFSET 6656
+#define PRS_REG_ROCE_DEST_QP_MAX_VF_RT_OFFSET 6657
+#define PRS_REG_ROCE_DEST_QP_MAX_PF_RT_OFFSET 6658
+#define PRS_REG_SEARCH_OPENFLOW_RT_OFFSET 6659
+#define PRS_REG_SEARCH_NON_IP_AS_OPENFLOW_RT_OFFSET 6660
+#define PRS_REG_OPENFLOW_SUPPORT_ONLY_KNOWN_OVER_IP_RT_OFFSET 6661
+#define PRS_REG_OPENFLOW_SEARCH_KEY_MASK_RT_OFFSET 6662
+#define PRS_REG_TAG_ETHERTYPE_0_RT_OFFSET 6663
+#define PRS_REG_LIGHT_L2_ETHERTYPE_EN_RT_OFFSET 6664
+#define SRC_REG_FIRSTFREE_RT_OFFSET 6665
+#define SRC_REG_FIRSTFREE_RT_SIZE 2
+#define SRC_REG_LASTFREE_RT_OFFSET 6667
+#define SRC_REG_LASTFREE_RT_SIZE 2
+#define SRC_REG_COUNTFREE_RT_OFFSET 6669
+#define SRC_REG_NUMBER_HASH_BITS_RT_OFFSET 6670
+#define PSWRQ2_REG_CDUT_P_SIZE_RT_OFFSET 6671
+#define PSWRQ2_REG_CDUC_P_SIZE_RT_OFFSET 6672
+#define PSWRQ2_REG_TM_P_SIZE_RT_OFFSET 6673
+#define PSWRQ2_REG_QM_P_SIZE_RT_OFFSET 6674
+#define PSWRQ2_REG_SRC_P_SIZE_RT_OFFSET 6675
+#define PSWRQ2_REG_TSDM_P_SIZE_RT_OFFSET 6676
+#define PSWRQ2_REG_TM_FIRST_ILT_RT_OFFSET 6677
+#define PSWRQ2_REG_TM_LAST_ILT_RT_OFFSET 6678
+#define PSWRQ2_REG_QM_FIRST_ILT_RT_OFFSET 6679
+#define PSWRQ2_REG_QM_LAST_ILT_RT_OFFSET 6680
+#define PSWRQ2_REG_SRC_FIRST_ILT_RT_OFFSET 6681
+#define PSWRQ2_REG_SRC_LAST_ILT_RT_OFFSET 6682
+#define PSWRQ2_REG_CDUC_FIRST_ILT_RT_OFFSET 6683
+#define PSWRQ2_REG_CDUC_LAST_ILT_RT_OFFSET 6684
+#define PSWRQ2_REG_CDUT_FIRST_ILT_RT_OFFSET 6685
+#define PSWRQ2_REG_CDUT_LAST_ILT_RT_OFFSET 6686
+#define PSWRQ2_REG_TSDM_FIRST_ILT_RT_OFFSET 6687
+#define PSWRQ2_REG_TSDM_LAST_ILT_RT_OFFSET 6688
+#define PSWRQ2_REG_TM_NUMBER_OF_PF_BLOCKS_RT_OFFSET 6689
+#define PSWRQ2_REG_CDUT_NUMBER_OF_PF_BLOCKS_RT_OFFSET 6690
+#define PSWRQ2_REG_CDUC_NUMBER_OF_PF_BLOCKS_RT_OFFSET 6691
+#define PSWRQ2_REG_TM_VF_BLOCKS_RT_OFFSET 6692
+#define PSWRQ2_REG_CDUT_VF_BLOCKS_RT_OFFSET 6693
+#define PSWRQ2_REG_CDUC_VF_BLOCKS_RT_OFFSET 6694
+#define PSWRQ2_REG_TM_BLOCKS_FACTOR_RT_OFFSET 6695
+#define PSWRQ2_REG_CDUT_BLOCKS_FACTOR_RT_OFFSET 6696
+#define PSWRQ2_REG_CDUC_BLOCKS_FACTOR_RT_OFFSET 6697
+#define PSWRQ2_REG_VF_BASE_RT_OFFSET 6698
+#define PSWRQ2_REG_VF_LAST_ILT_RT_OFFSET 6699
+#define PSWRQ2_REG_WR_MBS0_RT_OFFSET 6700
+#define PSWRQ2_REG_RD_MBS0_RT_OFFSET 6701
+#define PSWRQ2_REG_DRAM_ALIGN_WR_RT_OFFSET 6702
+#define PSWRQ2_REG_DRAM_ALIGN_RD_RT_OFFSET 6703
+#define PSWRQ2_REG_ILT_MEMORY_RT_OFFSET 6704
+#define PSWRQ2_REG_ILT_MEMORY_RT_SIZE 22000
+#define PGLUE_REG_B_VF_BASE_RT_OFFSET 28704
+#define PGLUE_REG_B_CACHE_LINE_SIZE_RT_OFFSET 28705
+#define PGLUE_REG_B_PF_BAR0_SIZE_RT_OFFSET 28706
+#define PGLUE_REG_B_PF_BAR1_SIZE_RT_OFFSET 28707
+#define PGLUE_REG_B_VF_BAR1_SIZE_RT_OFFSET 28708
+#define TM_REG_VF_ENABLE_CONN_RT_OFFSET 28709
+#define TM_REG_PF_ENABLE_CONN_RT_OFFSET 28710
+#define TM_REG_PF_ENABLE_TASK_RT_OFFSET 28711
+#define TM_REG_GROUP_SIZE_RESOLUTION_CONN_RT_OFFSET 28712
+#define TM_REG_GROUP_SIZE_RESOLUTION_TASK_RT_OFFSET 28713
+#define TM_REG_CONFIG_CONN_MEM_RT_OFFSET 28714
+#define TM_REG_CONFIG_CONN_MEM_RT_SIZE 416
+#define TM_REG_CONFIG_TASK_MEM_RT_OFFSET 29130
+#define TM_REG_CONFIG_TASK_MEM_RT_SIZE 512
+#define QM_REG_MAXPQSIZE_0_RT_OFFSET 29642
+#define QM_REG_MAXPQSIZE_1_RT_OFFSET 29643
+#define QM_REG_MAXPQSIZE_2_RT_OFFSET 29644
+#define QM_REG_MAXPQSIZETXSEL_0_RT_OFFSET 29645
+#define QM_REG_MAXPQSIZETXSEL_1_RT_OFFSET 29646
+#define QM_REG_MAXPQSIZETXSEL_2_RT_OFFSET 29647
+#define QM_REG_MAXPQSIZETXSEL_3_RT_OFFSET 29648
+#define QM_REG_MAXPQSIZETXSEL_4_RT_OFFSET 29649
+#define QM_REG_MAXPQSIZETXSEL_5_RT_OFFSET 29650
+#define QM_REG_MAXPQSIZETXSEL_6_RT_OFFSET 29651
+#define QM_REG_MAXPQSIZETXSEL_7_RT_OFFSET 29652
+#define QM_REG_MAXPQSIZETXSEL_8_RT_OFFSET 29653
+#define QM_REG_MAXPQSIZETXSEL_9_RT_OFFSET 29654
+#define QM_REG_MAXPQSIZETXSEL_10_RT_OFFSET 29655
+#define QM_REG_MAXPQSIZETXSEL_11_RT_OFFSET 29656
+#define QM_REG_MAXPQSIZETXSEL_12_RT_OFFSET 29657
+#define QM_REG_MAXPQSIZETXSEL_13_RT_OFFSET 29658
+#define QM_REG_MAXPQSIZETXSEL_14_RT_OFFSET 29659
+#define QM_REG_MAXPQSIZETXSEL_15_RT_OFFSET 29660
+#define QM_REG_MAXPQSIZETXSEL_16_RT_OFFSET 29661
+#define QM_REG_MAXPQSIZETXSEL_17_RT_OFFSET 29662
+#define QM_REG_MAXPQSIZETXSEL_18_RT_OFFSET 29663
+#define QM_REG_MAXPQSIZETXSEL_19_RT_OFFSET 29664
+#define QM_REG_MAXPQSIZETXSEL_20_RT_OFFSET 29665
+#define QM_REG_MAXPQSIZETXSEL_21_RT_OFFSET 29666
+#define QM_REG_MAXPQSIZETXSEL_22_RT_OFFSET 29667
+#define QM_REG_MAXPQSIZETXSEL_23_RT_OFFSET 29668
+#define QM_REG_MAXPQSIZETXSEL_24_RT_OFFSET 29669
+#define QM_REG_MAXPQSIZETXSEL_25_RT_OFFSET 29670
+#define QM_REG_MAXPQSIZETXSEL_26_RT_OFFSET 29671
+#define QM_REG_MAXPQSIZETXSEL_27_RT_OFFSET 29672
+#define QM_REG_MAXPQSIZETXSEL_28_RT_OFFSET 29673
+#define QM_REG_MAXPQSIZETXSEL_29_RT_OFFSET 29674
+#define QM_REG_MAXPQSIZETXSEL_30_RT_OFFSET 29675
+#define QM_REG_MAXPQSIZETXSEL_31_RT_OFFSET 29676
+#define QM_REG_MAXPQSIZETXSEL_32_RT_OFFSET 29677
+#define QM_REG_MAXPQSIZETXSEL_33_RT_OFFSET 29678
+#define QM_REG_MAXPQSIZETXSEL_34_RT_OFFSET 29679
+#define QM_REG_MAXPQSIZETXSEL_35_RT_OFFSET 29680
+#define QM_REG_MAXPQSIZETXSEL_36_RT_OFFSET 29681
+#define QM_REG_MAXPQSIZETXSEL_37_RT_OFFSET 29682
+#define QM_REG_MAXPQSIZETXSEL_38_RT_OFFSET 29683
+#define QM_REG_MAXPQSIZETXSEL_39_RT_OFFSET 29684
+#define QM_REG_MAXPQSIZETXSEL_40_RT_OFFSET 29685
+#define QM_REG_MAXPQSIZETXSEL_41_RT_OFFSET 29686
+#define QM_REG_MAXPQSIZETXSEL_42_RT_OFFSET 29687
+#define QM_REG_MAXPQSIZETXSEL_43_RT_OFFSET 29688
+#define QM_REG_MAXPQSIZETXSEL_44_RT_OFFSET 29689
+#define QM_REG_MAXPQSIZETXSEL_45_RT_OFFSET 29690
+#define QM_REG_MAXPQSIZETXSEL_46_RT_OFFSET 29691
+#define QM_REG_MAXPQSIZETXSEL_47_RT_OFFSET 29692
+#define QM_REG_MAXPQSIZETXSEL_48_RT_OFFSET 29693
+#define QM_REG_MAXPQSIZETXSEL_49_RT_OFFSET 29694
+#define QM_REG_MAXPQSIZETXSEL_50_RT_OFFSET 29695
+#define QM_REG_MAXPQSIZETXSEL_51_RT_OFFSET 29696
+#define QM_REG_MAXPQSIZETXSEL_52_RT_OFFSET 29697
+#define QM_REG_MAXPQSIZETXSEL_53_RT_OFFSET 29698
+#define QM_REG_MAXPQSIZETXSEL_54_RT_OFFSET 29699
+#define QM_REG_MAXPQSIZETXSEL_55_RT_OFFSET 29700
+#define QM_REG_MAXPQSIZETXSEL_56_RT_OFFSET 29701
+#define QM_REG_MAXPQSIZETXSEL_57_RT_OFFSET 29702
+#define QM_REG_MAXPQSIZETXSEL_58_RT_OFFSET 29703
+#define QM_REG_MAXPQSIZETXSEL_59_RT_OFFSET 29704
+#define QM_REG_MAXPQSIZETXSEL_60_RT_OFFSET 29705
+#define QM_REG_MAXPQSIZETXSEL_61_RT_OFFSET 29706
+#define QM_REG_MAXPQSIZETXSEL_62_RT_OFFSET 29707
+#define QM_REG_MAXPQSIZETXSEL_63_RT_OFFSET 29708
+#define QM_REG_BASEADDROTHERPQ_RT_OFFSET 29709
+#define QM_REG_BASEADDROTHERPQ_RT_SIZE 128
+#define QM_REG_VOQCRDLINE_RT_OFFSET 29837
+#define QM_REG_VOQCRDLINE_RT_SIZE 20
+#define QM_REG_VOQINITCRDLINE_RT_OFFSET 29857
+#define QM_REG_VOQINITCRDLINE_RT_SIZE 20
+#define QM_REG_AFULLQMBYPTHRPFWFQ_RT_OFFSET 29877
+#define QM_REG_AFULLQMBYPTHRVPWFQ_RT_OFFSET 29878
+#define QM_REG_AFULLQMBYPTHRPFRL_RT_OFFSET 29879
+#define QM_REG_AFULLQMBYPTHRGLBLRL_RT_OFFSET 29880
+#define QM_REG_AFULLOPRTNSTCCRDMASK_RT_OFFSET 29881
+#define QM_REG_WRROTHERPQGRP_0_RT_OFFSET 29882
+#define QM_REG_WRROTHERPQGRP_1_RT_OFFSET 29883
+#define QM_REG_WRROTHERPQGRP_2_RT_OFFSET 29884
+#define QM_REG_WRROTHERPQGRP_3_RT_OFFSET 29885
+#define QM_REG_WRROTHERPQGRP_4_RT_OFFSET 29886
+#define QM_REG_WRROTHERPQGRP_5_RT_OFFSET 29887
+#define QM_REG_WRROTHERPQGRP_6_RT_OFFSET 29888
+#define QM_REG_WRROTHERPQGRP_7_RT_OFFSET 29889
+#define QM_REG_WRROTHERPQGRP_8_RT_OFFSET 29890
+#define QM_REG_WRROTHERPQGRP_9_RT_OFFSET 29891
+#define QM_REG_WRROTHERPQGRP_10_RT_OFFSET 29892
+#define QM_REG_WRROTHERPQGRP_11_RT_OFFSET 29893
+#define QM_REG_WRROTHERPQGRP_12_RT_OFFSET 29894
+#define QM_REG_WRROTHERPQGRP_13_RT_OFFSET 29895
+#define QM_REG_WRROTHERPQGRP_14_RT_OFFSET 29896
+#define QM_REG_WRROTHERPQGRP_15_RT_OFFSET 29897
+#define QM_REG_WRROTHERGRPWEIGHT_0_RT_OFFSET 29898
+#define QM_REG_WRROTHERGRPWEIGHT_1_RT_OFFSET 29899
+#define QM_REG_WRROTHERGRPWEIGHT_2_RT_OFFSET 29900
+#define QM_REG_WRROTHERGRPWEIGHT_3_RT_OFFSET 29901
+#define QM_REG_WRRTXGRPWEIGHT_0_RT_OFFSET 29902
+#define QM_REG_WRRTXGRPWEIGHT_1_RT_OFFSET 29903
+#define QM_REG_PQTX2PF_0_RT_OFFSET 29904
+#define QM_REG_PQTX2PF_1_RT_OFFSET 29905
+#define QM_REG_PQTX2PF_2_RT_OFFSET 29906
+#define QM_REG_PQTX2PF_3_RT_OFFSET 29907
+#define QM_REG_PQTX2PF_4_RT_OFFSET 29908
+#define QM_REG_PQTX2PF_5_RT_OFFSET 29909
+#define QM_REG_PQTX2PF_6_RT_OFFSET 29910
+#define QM_REG_PQTX2PF_7_RT_OFFSET 29911
+#define QM_REG_PQTX2PF_8_RT_OFFSET 29912
+#define QM_REG_PQTX2PF_9_RT_OFFSET 29913
+#define QM_REG_PQTX2PF_10_RT_OFFSET 29914
+#define QM_REG_PQTX2PF_11_RT_OFFSET 29915
+#define QM_REG_PQTX2PF_12_RT_OFFSET 29916
+#define QM_REG_PQTX2PF_13_RT_OFFSET 29917
+#define QM_REG_PQTX2PF_14_RT_OFFSET 29918
+#define QM_REG_PQTX2PF_15_RT_OFFSET 29919
+#define QM_REG_PQTX2PF_16_RT_OFFSET 29920
+#define QM_REG_PQTX2PF_17_RT_OFFSET 29921
+#define QM_REG_PQTX2PF_18_RT_OFFSET 29922
+#define QM_REG_PQTX2PF_19_RT_OFFSET 29923
+#define QM_REG_PQTX2PF_20_RT_OFFSET 29924
+#define QM_REG_PQTX2PF_21_RT_OFFSET 29925
+#define QM_REG_PQTX2PF_22_RT_OFFSET 29926
+#define QM_REG_PQTX2PF_23_RT_OFFSET 29927
+#define QM_REG_PQTX2PF_24_RT_OFFSET 29928
+#define QM_REG_PQTX2PF_25_RT_OFFSET 29929
+#define QM_REG_PQTX2PF_26_RT_OFFSET 29930
+#define QM_REG_PQTX2PF_27_RT_OFFSET 29931
+#define QM_REG_PQTX2PF_28_RT_OFFSET 29932
+#define QM_REG_PQTX2PF_29_RT_OFFSET 29933
+#define QM_REG_PQTX2PF_30_RT_OFFSET 29934
+#define QM_REG_PQTX2PF_31_RT_OFFSET 29935
+#define QM_REG_PQTX2PF_32_RT_OFFSET 29936
+#define QM_REG_PQTX2PF_33_RT_OFFSET 29937
+#define QM_REG_PQTX2PF_34_RT_OFFSET 29938
+#define QM_REG_PQTX2PF_35_RT_OFFSET 29939
+#define QM_REG_PQTX2PF_36_RT_OFFSET 29940
+#define QM_REG_PQTX2PF_37_RT_OFFSET 29941
+#define QM_REG_PQTX2PF_38_RT_OFFSET 29942
+#define QM_REG_PQTX2PF_39_RT_OFFSET 29943
+#define QM_REG_PQTX2PF_40_RT_OFFSET 29944
+#define QM_REG_PQTX2PF_41_RT_OFFSET 29945
+#define QM_REG_PQTX2PF_42_RT_OFFSET 29946
+#define QM_REG_PQTX2PF_43_RT_OFFSET 29947
+#define QM_REG_PQTX2PF_44_RT_OFFSET 29948
+#define QM_REG_PQTX2PF_45_RT_OFFSET 29949
+#define QM_REG_PQTX2PF_46_RT_OFFSET 29950
+#define QM_REG_PQTX2PF_47_RT_OFFSET 29951
+#define QM_REG_PQTX2PF_48_RT_OFFSET 29952
+#define QM_REG_PQTX2PF_49_RT_OFFSET 29953
+#define QM_REG_PQTX2PF_50_RT_OFFSET 29954
+#define QM_REG_PQTX2PF_51_RT_OFFSET 29955
+#define QM_REG_PQTX2PF_52_RT_OFFSET 29956
+#define QM_REG_PQTX2PF_53_RT_OFFSET 29957
+#define QM_REG_PQTX2PF_54_RT_OFFSET 29958
+#define QM_REG_PQTX2PF_55_RT_OFFSET 29959
+#define QM_REG_PQTX2PF_56_RT_OFFSET 29960
+#define QM_REG_PQTX2PF_57_RT_OFFSET 29961
+#define QM_REG_PQTX2PF_58_RT_OFFSET 29962
+#define QM_REG_PQTX2PF_59_RT_OFFSET 29963
+#define QM_REG_PQTX2PF_60_RT_OFFSET 29964
+#define QM_REG_PQTX2PF_61_RT_OFFSET 29965
+#define QM_REG_PQTX2PF_62_RT_OFFSET 29966
+#define QM_REG_PQTX2PF_63_RT_OFFSET 29967
+#define QM_REG_PQOTHER2PF_0_RT_OFFSET 29968
+#define QM_REG_PQOTHER2PF_1_RT_OFFSET 29969
+#define QM_REG_PQOTHER2PF_2_RT_OFFSET 29970
+#define QM_REG_PQOTHER2PF_3_RT_OFFSET 29971
+#define QM_REG_PQOTHER2PF_4_RT_OFFSET 29972
+#define QM_REG_PQOTHER2PF_5_RT_OFFSET 29973
+#define QM_REG_PQOTHER2PF_6_RT_OFFSET 29974
+#define QM_REG_PQOTHER2PF_7_RT_OFFSET 29975
+#define QM_REG_PQOTHER2PF_8_RT_OFFSET 29976
+#define QM_REG_PQOTHER2PF_9_RT_OFFSET 29977
+#define QM_REG_PQOTHER2PF_10_RT_OFFSET 29978
+#define QM_REG_PQOTHER2PF_11_RT_OFFSET 29979
+#define QM_REG_PQOTHER2PF_12_RT_OFFSET 29980
+#define QM_REG_PQOTHER2PF_13_RT_OFFSET 29981
+#define QM_REG_PQOTHER2PF_14_RT_OFFSET 29982
+#define QM_REG_PQOTHER2PF_15_RT_OFFSET 29983
+#define QM_REG_RLGLBLPERIOD_0_RT_OFFSET 29984
+#define QM_REG_RLGLBLPERIOD_1_RT_OFFSET 29985
+#define QM_REG_RLGLBLPERIODTIMER_0_RT_OFFSET 29986
+#define QM_REG_RLGLBLPERIODTIMER_1_RT_OFFSET 29987
+#define QM_REG_RLGLBLPERIODSEL_0_RT_OFFSET 29988
+#define QM_REG_RLGLBLPERIODSEL_1_RT_OFFSET 29989
+#define QM_REG_RLGLBLPERIODSEL_2_RT_OFFSET 29990
+#define QM_REG_RLGLBLPERIODSEL_3_RT_OFFSET 29991
+#define QM_REG_RLGLBLPERIODSEL_4_RT_OFFSET 29992
+#define QM_REG_RLGLBLPERIODSEL_5_RT_OFFSET 29993
+#define QM_REG_RLGLBLPERIODSEL_6_RT_OFFSET 29994
+#define QM_REG_RLGLBLPERIODSEL_7_RT_OFFSET 29995
+#define QM_REG_RLGLBLINCVAL_RT_OFFSET 29996
+#define QM_REG_RLGLBLINCVAL_RT_SIZE 256
+#define QM_REG_RLGLBLUPPERBOUND_RT_OFFSET 30252
+#define QM_REG_RLGLBLUPPERBOUND_RT_SIZE 256
+#define QM_REG_RLGLBLCRD_RT_OFFSET 30508
+#define QM_REG_RLGLBLCRD_RT_SIZE 256
+#define QM_REG_RLGLBLENABLE_RT_OFFSET 30764
+#define QM_REG_RLPFPERIOD_RT_OFFSET 30765
+#define QM_REG_RLPFPERIODTIMER_RT_OFFSET 30766
+#define QM_REG_RLPFINCVAL_RT_OFFSET 30767
+#define QM_REG_RLPFINCVAL_RT_SIZE 16
+#define QM_REG_RLPFUPPERBOUND_RT_OFFSET 30783
+#define QM_REG_RLPFUPPERBOUND_RT_SIZE 16
+#define QM_REG_RLPFCRD_RT_OFFSET 30799
+#define QM_REG_RLPFCRD_RT_SIZE 16
+#define QM_REG_RLPFENABLE_RT_OFFSET 30815
+#define QM_REG_RLPFVOQENABLE_RT_OFFSET 30816
+#define QM_REG_WFQPFWEIGHT_RT_OFFSET 30817
+#define QM_REG_WFQPFWEIGHT_RT_SIZE 16
+#define QM_REG_WFQPFUPPERBOUND_RT_OFFSET 30833
+#define QM_REG_WFQPFUPPERBOUND_RT_SIZE 16
+#define QM_REG_WFQPFCRD_RT_OFFSET 30849
+#define QM_REG_WFQPFCRD_RT_SIZE 160
+#define QM_REG_WFQPFENABLE_RT_OFFSET 31009
+#define QM_REG_WFQVPENABLE_RT_OFFSET 31010
+#define QM_REG_BASEADDRTXPQ_RT_OFFSET 31011
+#define QM_REG_BASEADDRTXPQ_RT_SIZE 512
+#define QM_REG_TXPQMAP_RT_OFFSET 31523
+#define QM_REG_TXPQMAP_RT_SIZE 512
+#define QM_REG_WFQVPWEIGHT_RT_OFFSET 32035
+#define QM_REG_WFQVPWEIGHT_RT_SIZE 512
+#define QM_REG_WFQVPCRD_RT_OFFSET 32547
+#define QM_REG_WFQVPCRD_RT_SIZE 512
+#define QM_REG_WFQVPMAP_RT_OFFSET 33059
+#define QM_REG_WFQVPMAP_RT_SIZE 512
+#define QM_REG_WFQPFCRD_MSB_RT_OFFSET 33571
+#define QM_REG_WFQPFCRD_MSB_RT_SIZE 160
+#define NIG_REG_TAG_ETHERTYPE_0_RT_OFFSET 33731
+#define NIG_REG_OUTER_TAG_VALUE_LIST0_RT_OFFSET 33732
+#define NIG_REG_OUTER_TAG_VALUE_LIST1_RT_OFFSET 33733
+#define NIG_REG_OUTER_TAG_VALUE_LIST2_RT_OFFSET 33734
+#define NIG_REG_OUTER_TAG_VALUE_LIST3_RT_OFFSET 33735
+#define NIG_REG_OUTER_TAG_VALUE_MASK_RT_OFFSET 33736
+#define NIG_REG_LLH_FUNC_TAGMAC_CLS_TYPE_RT_OFFSET 33737
+#define NIG_REG_LLH_FUNC_TAG_EN_RT_OFFSET 33738
+#define NIG_REG_LLH_FUNC_TAG_EN_RT_SIZE 4
+#define NIG_REG_LLH_FUNC_TAG_HDR_SEL_RT_OFFSET 33742
+#define NIG_REG_LLH_FUNC_TAG_HDR_SEL_RT_SIZE 4
+#define NIG_REG_LLH_FUNC_TAG_VALUE_RT_OFFSET 33746
+#define NIG_REG_LLH_FUNC_TAG_VALUE_RT_SIZE 4
+#define NIG_REG_LLH_FUNC_NO_TAG_RT_OFFSET 33750
+#define NIG_REG_LLH_FUNC_FILTER_VALUE_RT_OFFSET 33751
+#define NIG_REG_LLH_FUNC_FILTER_VALUE_RT_SIZE 32
+#define NIG_REG_LLH_FUNC_FILTER_EN_RT_OFFSET 33783
+#define NIG_REG_LLH_FUNC_FILTER_EN_RT_SIZE 16
+#define NIG_REG_LLH_FUNC_FILTER_MODE_RT_OFFSET 33799
+#define NIG_REG_LLH_FUNC_FILTER_MODE_RT_SIZE 16
+#define NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_RT_OFFSET 33815
+#define NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_RT_SIZE 16
+#define NIG_REG_LLH_FUNC_FILTER_HDR_SEL_RT_OFFSET 33831
+#define NIG_REG_LLH_FUNC_FILTER_HDR_SEL_RT_SIZE 16
+#define NIG_REG_TX_EDPM_CTRL_RT_OFFSET 33847
+#define NIG_REG_ROCE_DUPLICATE_TO_HOST_RT_OFFSET 33848
+#define CDU_REG_CID_ADDR_PARAMS_RT_OFFSET 33849
+#define CDU_REG_SEGMENT0_PARAMS_RT_OFFSET 33850
+#define CDU_REG_SEGMENT1_PARAMS_RT_OFFSET 33851
+#define CDU_REG_PF_SEG0_TYPE_OFFSET_RT_OFFSET 33852
+#define CDU_REG_PF_SEG1_TYPE_OFFSET_RT_OFFSET 33853
+#define CDU_REG_PF_SEG2_TYPE_OFFSET_RT_OFFSET 33854
+#define CDU_REG_PF_SEG3_TYPE_OFFSET_RT_OFFSET 33855
+#define CDU_REG_PF_FL_SEG0_TYPE_OFFSET_RT_OFFSET 33856
+#define CDU_REG_PF_FL_SEG1_TYPE_OFFSET_RT_OFFSET 33857
+#define CDU_REG_PF_FL_SEG2_TYPE_OFFSET_RT_OFFSET 33858
+#define CDU_REG_PF_FL_SEG3_TYPE_OFFSET_RT_OFFSET 33859
+#define CDU_REG_VF_SEG_TYPE_OFFSET_RT_OFFSET 33860
+#define CDU_REG_VF_FL_SEG_TYPE_OFFSET_RT_OFFSET 33861
+#define PBF_REG_TAG_ETHERTYPE_0_RT_OFFSET 33862
+#define PBF_REG_BTB_SHARED_AREA_SIZE_RT_OFFSET 33863
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ0_RT_OFFSET 33864
+#define PBF_REG_BTB_GUARANTEED_VOQ0_RT_OFFSET 33865
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ0_RT_OFFSET 33866
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ1_RT_OFFSET 33867
+#define PBF_REG_BTB_GUARANTEED_VOQ1_RT_OFFSET 33868
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ1_RT_OFFSET 33869
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ2_RT_OFFSET 33870
+#define PBF_REG_BTB_GUARANTEED_VOQ2_RT_OFFSET 33871
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ2_RT_OFFSET 33872
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ3_RT_OFFSET 33873
+#define PBF_REG_BTB_GUARANTEED_VOQ3_RT_OFFSET 33874
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ3_RT_OFFSET 33875
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ4_RT_OFFSET 33876
+#define PBF_REG_BTB_GUARANTEED_VOQ4_RT_OFFSET 33877
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ4_RT_OFFSET 33878
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ5_RT_OFFSET 33879
+#define PBF_REG_BTB_GUARANTEED_VOQ5_RT_OFFSET 33880
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ5_RT_OFFSET 33881
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ6_RT_OFFSET 33882
+#define PBF_REG_BTB_GUARANTEED_VOQ6_RT_OFFSET 33883
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ6_RT_OFFSET 33884
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ7_RT_OFFSET 33885
+#define PBF_REG_BTB_GUARANTEED_VOQ7_RT_OFFSET 33886
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ7_RT_OFFSET 33887
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ8_RT_OFFSET 33888
+#define PBF_REG_BTB_GUARANTEED_VOQ8_RT_OFFSET 33889
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ8_RT_OFFSET 33890
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ9_RT_OFFSET 33891
+#define PBF_REG_BTB_GUARANTEED_VOQ9_RT_OFFSET 33892
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ9_RT_OFFSET 33893
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ10_RT_OFFSET 33894
+#define PBF_REG_BTB_GUARANTEED_VOQ10_RT_OFFSET 33895
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ10_RT_OFFSET 33896
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ11_RT_OFFSET 33897
+#define PBF_REG_BTB_GUARANTEED_VOQ11_RT_OFFSET 33898
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ11_RT_OFFSET 33899
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ12_RT_OFFSET 33900
+#define PBF_REG_BTB_GUARANTEED_VOQ12_RT_OFFSET 33901
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ12_RT_OFFSET 33902
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ13_RT_OFFSET 33903
+#define PBF_REG_BTB_GUARANTEED_VOQ13_RT_OFFSET 33904
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ13_RT_OFFSET 33905
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ14_RT_OFFSET 33906
+#define PBF_REG_BTB_GUARANTEED_VOQ14_RT_OFFSET 33907
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ14_RT_OFFSET 33908
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ15_RT_OFFSET 33909
+#define PBF_REG_BTB_GUARANTEED_VOQ15_RT_OFFSET 33910
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ15_RT_OFFSET 33911
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ16_RT_OFFSET 33912
+#define PBF_REG_BTB_GUARANTEED_VOQ16_RT_OFFSET 33913
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ16_RT_OFFSET 33914
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ17_RT_OFFSET 33915
+#define PBF_REG_BTB_GUARANTEED_VOQ17_RT_OFFSET 33916
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ17_RT_OFFSET 33917
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ18_RT_OFFSET 33918
+#define PBF_REG_BTB_GUARANTEED_VOQ18_RT_OFFSET 33919
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ18_RT_OFFSET 33920
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ19_RT_OFFSET 33921
+#define PBF_REG_BTB_GUARANTEED_VOQ19_RT_OFFSET 33922
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ19_RT_OFFSET 33923
+#define XCM_REG_CON_PHY_Q3_RT_OFFSET 33924
+
+#define RUNTIME_ARRAY_SIZE 33925
 
 /* The eth storm context for the Tstorm */
 struct tstorm_eth_conn_st_ctx {
@@ -2380,266 +2692,266 @@ struct xstorm_eth_conn_st_ctx {
 };
 
 struct xstorm_eth_conn_ag_ctx {
-        u8      reserved0 /* cdu_validation */;
-        u8      eth_state /* state */;
-        u8      flags0;
-#define XSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM0_MASK            0x1
-#define XSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM0_SHIFT           0
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED1_MASK               0x1
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED1_SHIFT              1
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED2_MASK               0x1
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED2_SHIFT              2
-#define XSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM3_MASK            0x1
-#define XSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM3_SHIFT           3
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED3_MASK               0x1 /* bit4 */
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED3_SHIFT              4
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED4_MASK               0x1
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED4_SHIFT              5
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED5_MASK               0x1 /* bit6 */
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED5_SHIFT              6
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED6_MASK               0x1 /* bit7 */
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED6_SHIFT              7
-        u8 flags1;
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED7_MASK               0x1 /* bit8 */
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED7_SHIFT              0
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED8_MASK               0x1 /* bit9 */
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED8_SHIFT              1
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED9_MASK               0x1 /* bit10 */
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED9_SHIFT              2
-#define XSTORM_ETH_CONN_AG_CTX_BIT11_MASK                   0x1 /* bit11 */
-#define XSTORM_ETH_CONN_AG_CTX_BIT11_SHIFT                  3
-#define XSTORM_ETH_CONN_AG_CTX_BIT12_MASK                   0x1 /* bit12 */
-#define XSTORM_ETH_CONN_AG_CTX_BIT12_SHIFT                  4
-#define XSTORM_ETH_CONN_AG_CTX_BIT13_MASK                   0x1 /* bit13 */
-#define XSTORM_ETH_CONN_AG_CTX_BIT13_SHIFT                  5
-#define XSTORM_ETH_CONN_AG_CTX_TX_RULE_ACTIVE_MASK          0x1 /* bit14 */
-#define XSTORM_ETH_CONN_AG_CTX_TX_RULE_ACTIVE_SHIFT         6
-#define XSTORM_ETH_CONN_AG_CTX_DQ_CF_ACTIVE_MASK            0x1 /* bit15 */
-#define XSTORM_ETH_CONN_AG_CTX_DQ_CF_ACTIVE_SHIFT           7
+        u8 reserved0;
+        u8 eth_state;
+        u8 flags0;
+#define XSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM0_MASK        0x1
+#define XSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM0_SHIFT       0
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED1_MASK           0x1
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED1_SHIFT          1
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED2_MASK           0x1
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED2_SHIFT          2
+#define XSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM3_MASK        0x1
+#define XSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM3_SHIFT       3
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED3_MASK           0x1
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED3_SHIFT          4
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED4_MASK           0x1
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED4_SHIFT          5
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED5_MASK           0x1
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED5_SHIFT          6
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED6_MASK           0x1
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED6_SHIFT          7
+                u8 flags1;
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED7_MASK           0x1
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED7_SHIFT          0
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED8_MASK           0x1
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED8_SHIFT          1
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED9_MASK           0x1
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED9_SHIFT          2
+#define XSTORM_ETH_CONN_AG_CTX_BIT11_MASK               0x1
+#define XSTORM_ETH_CONN_AG_CTX_BIT11_SHIFT              3
+#define XSTORM_ETH_CONN_AG_CTX_BIT12_MASK               0x1
+#define XSTORM_ETH_CONN_AG_CTX_BIT12_SHIFT              4
+#define XSTORM_ETH_CONN_AG_CTX_BIT13_MASK               0x1
+#define XSTORM_ETH_CONN_AG_CTX_BIT13_SHIFT              5
+#define XSTORM_ETH_CONN_AG_CTX_TX_RULE_ACTIVE_MASK      0x1
+#define XSTORM_ETH_CONN_AG_CTX_TX_RULE_ACTIVE_SHIFT     6
+#define XSTORM_ETH_CONN_AG_CTX_DQ_CF_ACTIVE_MASK        0x1
+#define XSTORM_ETH_CONN_AG_CTX_DQ_CF_ACTIVE_SHIFT       7
         u8 flags2;
-#define XSTORM_ETH_CONN_AG_CTX_CF0_MASK                     0x3 /* timer0cf */
-#define XSTORM_ETH_CONN_AG_CTX_CF0_SHIFT                    0
-#define XSTORM_ETH_CONN_AG_CTX_CF1_MASK                     0x3 /* timer1cf */
-#define XSTORM_ETH_CONN_AG_CTX_CF1_SHIFT                    2
-#define XSTORM_ETH_CONN_AG_CTX_CF2_MASK                     0x3 /* timer2cf */
-#define XSTORM_ETH_CONN_AG_CTX_CF2_SHIFT                    4
-#define XSTORM_ETH_CONN_AG_CTX_CF3_MASK                     0x3
-#define XSTORM_ETH_CONN_AG_CTX_CF3_SHIFT                    6
+#define XSTORM_ETH_CONN_AG_CTX_CF0_MASK         0x3
+#define XSTORM_ETH_CONN_AG_CTX_CF0_SHIFT                0
+#define XSTORM_ETH_CONN_AG_CTX_CF1_MASK         0x3
+#define XSTORM_ETH_CONN_AG_CTX_CF1_SHIFT                2
+#define XSTORM_ETH_CONN_AG_CTX_CF2_MASK         0x3
+#define XSTORM_ETH_CONN_AG_CTX_CF2_SHIFT                4
+#define XSTORM_ETH_CONN_AG_CTX_CF3_MASK         0x3
+#define XSTORM_ETH_CONN_AG_CTX_CF3_SHIFT                6
         u8 flags3;
-#define XSTORM_ETH_CONN_AG_CTX_CF4_MASK                     0x3 /* cf4 */
-#define XSTORM_ETH_CONN_AG_CTX_CF4_SHIFT                    0
-#define XSTORM_ETH_CONN_AG_CTX_CF5_MASK                     0x3 /* cf5 */
-#define XSTORM_ETH_CONN_AG_CTX_CF5_SHIFT                    2
-#define XSTORM_ETH_CONN_AG_CTX_CF6_MASK                     0x3 /* cf6 */
-#define XSTORM_ETH_CONN_AG_CTX_CF6_SHIFT                    4
-#define XSTORM_ETH_CONN_AG_CTX_CF7_MASK                     0x3 /* cf7 */
-#define XSTORM_ETH_CONN_AG_CTX_CF7_SHIFT                    6
-        u8 flags4;
-#define XSTORM_ETH_CONN_AG_CTX_CF8_MASK                     0x3 /* cf8 */
-#define XSTORM_ETH_CONN_AG_CTX_CF8_SHIFT                    0
-#define XSTORM_ETH_CONN_AG_CTX_CF9_MASK                     0x3 /* cf9 */
-#define XSTORM_ETH_CONN_AG_CTX_CF9_SHIFT                    2
-#define XSTORM_ETH_CONN_AG_CTX_CF10_MASK                    0x3 /* cf10 */
-#define XSTORM_ETH_CONN_AG_CTX_CF10_SHIFT                   4
-#define XSTORM_ETH_CONN_AG_CTX_CF11_MASK                    0x3 /* cf11 */
-#define XSTORM_ETH_CONN_AG_CTX_CF11_SHIFT                   6
+#define XSTORM_ETH_CONN_AG_CTX_CF4_MASK         0x3
+#define XSTORM_ETH_CONN_AG_CTX_CF4_SHIFT                0
+#define XSTORM_ETH_CONN_AG_CTX_CF5_MASK         0x3
+#define XSTORM_ETH_CONN_AG_CTX_CF5_SHIFT                2
+#define XSTORM_ETH_CONN_AG_CTX_CF6_MASK         0x3
+#define XSTORM_ETH_CONN_AG_CTX_CF6_SHIFT                4
+#define XSTORM_ETH_CONN_AG_CTX_CF7_MASK         0x3
+#define XSTORM_ETH_CONN_AG_CTX_CF7_SHIFT                6
+                u8 flags4;
+#define XSTORM_ETH_CONN_AG_CTX_CF8_MASK         0x3
+#define XSTORM_ETH_CONN_AG_CTX_CF8_SHIFT                0
+#define XSTORM_ETH_CONN_AG_CTX_CF9_MASK         0x3
+#define XSTORM_ETH_CONN_AG_CTX_CF9_SHIFT                2
+#define XSTORM_ETH_CONN_AG_CTX_CF10_MASK                0x3
+#define XSTORM_ETH_CONN_AG_CTX_CF10_SHIFT               4
+#define XSTORM_ETH_CONN_AG_CTX_CF11_MASK                0x3
+#define XSTORM_ETH_CONN_AG_CTX_CF11_SHIFT               6
         u8 flags5;
-#define XSTORM_ETH_CONN_AG_CTX_CF12_MASK                    0x3 /* cf12 */
-#define XSTORM_ETH_CONN_AG_CTX_CF12_SHIFT                   0
-#define XSTORM_ETH_CONN_AG_CTX_CF13_MASK                    0x3 /* cf13 */
-#define XSTORM_ETH_CONN_AG_CTX_CF13_SHIFT                   2
-#define XSTORM_ETH_CONN_AG_CTX_CF14_MASK                    0x3 /* cf14 */
-#define XSTORM_ETH_CONN_AG_CTX_CF14_SHIFT                   4
-#define XSTORM_ETH_CONN_AG_CTX_CF15_MASK                    0x3 /* cf15 */
-#define XSTORM_ETH_CONN_AG_CTX_CF15_SHIFT                   6
+#define XSTORM_ETH_CONN_AG_CTX_CF12_MASK                0x3
+#define XSTORM_ETH_CONN_AG_CTX_CF12_SHIFT               0
+#define XSTORM_ETH_CONN_AG_CTX_CF13_MASK                0x3
+#define XSTORM_ETH_CONN_AG_CTX_CF13_SHIFT               2
+#define XSTORM_ETH_CONN_AG_CTX_CF14_MASK                0x3
+#define XSTORM_ETH_CONN_AG_CTX_CF14_SHIFT               4
+#define XSTORM_ETH_CONN_AG_CTX_CF15_MASK                0x3
+#define XSTORM_ETH_CONN_AG_CTX_CF15_SHIFT               6
         u8 flags6;
-#define XSTORM_ETH_CONN_AG_CTX_GO_TO_BD_CONS_CF_MASK        0x3 /* cf16 */
-#define XSTORM_ETH_CONN_AG_CTX_GO_TO_BD_CONS_CF_SHIFT       0
-#define XSTORM_ETH_CONN_AG_CTX_MULTI_UNICAST_CF_MASK        0x3
-#define XSTORM_ETH_CONN_AG_CTX_MULTI_UNICAST_CF_SHIFT       2
-#define XSTORM_ETH_CONN_AG_CTX_DQ_CF_MASK                   0x3 /* cf18 */
-#define XSTORM_ETH_CONN_AG_CTX_DQ_CF_SHIFT                  4
-#define XSTORM_ETH_CONN_AG_CTX_TERMINATE_CF_MASK            0x3 /* cf19 */
-#define XSTORM_ETH_CONN_AG_CTX_TERMINATE_CF_SHIFT           6
+#define XSTORM_ETH_CONN_AG_CTX_GO_TO_BD_CONS_CF_MASK    0x3
+#define XSTORM_ETH_CONN_AG_CTX_GO_TO_BD_CONS_CF_SHIFT   0
+#define XSTORM_ETH_CONN_AG_CTX_MULTI_UNICAST_CF_MASK    0x3
+#define XSTORM_ETH_CONN_AG_CTX_MULTI_UNICAST_CF_SHIFT   2
+#define XSTORM_ETH_CONN_AG_CTX_DQ_CF_MASK               0x3
+#define XSTORM_ETH_CONN_AG_CTX_DQ_CF_SHIFT              4
+#define XSTORM_ETH_CONN_AG_CTX_TERMINATE_CF_MASK        0x3
+#define XSTORM_ETH_CONN_AG_CTX_TERMINATE_CF_SHIFT       6
         u8 flags7;
-#define XSTORM_ETH_CONN_AG_CTX_FLUSH_Q0_MASK                0x3 /* cf20 */
-#define XSTORM_ETH_CONN_AG_CTX_FLUSH_Q0_SHIFT               0
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED10_MASK              0x3 /* cf21 */
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED10_SHIFT             2
-#define XSTORM_ETH_CONN_AG_CTX_SLOW_PATH_MASK               0x3 /* cf22 */
-#define XSTORM_ETH_CONN_AG_CTX_SLOW_PATH_SHIFT              4
-#define XSTORM_ETH_CONN_AG_CTX_CF0EN_MASK                   0x1 /* cf0en */
-#define XSTORM_ETH_CONN_AG_CTX_CF0EN_SHIFT                  6
-#define XSTORM_ETH_CONN_AG_CTX_CF1EN_MASK                   0x1 /* cf1en */
-#define XSTORM_ETH_CONN_AG_CTX_CF1EN_SHIFT                  7
+#define XSTORM_ETH_CONN_AG_CTX_FLUSH_Q0_MASK            0x3
+#define XSTORM_ETH_CONN_AG_CTX_FLUSH_Q0_SHIFT           0
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED10_MASK          0x3
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED10_SHIFT         2
+#define XSTORM_ETH_CONN_AG_CTX_SLOW_PATH_MASK           0x3
+#define XSTORM_ETH_CONN_AG_CTX_SLOW_PATH_SHIFT          4
+#define XSTORM_ETH_CONN_AG_CTX_CF0EN_MASK               0x1
+#define XSTORM_ETH_CONN_AG_CTX_CF0EN_SHIFT              6
+#define XSTORM_ETH_CONN_AG_CTX_CF1EN_MASK               0x1
+#define XSTORM_ETH_CONN_AG_CTX_CF1EN_SHIFT              7
         u8 flags8;
-#define XSTORM_ETH_CONN_AG_CTX_CF2EN_MASK                   0x1 /* cf2en */
-#define XSTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT                  0
-#define XSTORM_ETH_CONN_AG_CTX_CF3EN_MASK                   0x1 /* cf3en */
-#define XSTORM_ETH_CONN_AG_CTX_CF3EN_SHIFT                  1
-#define XSTORM_ETH_CONN_AG_CTX_CF4EN_MASK                   0x1 /* cf4en */
-#define XSTORM_ETH_CONN_AG_CTX_CF4EN_SHIFT                  2
-#define XSTORM_ETH_CONN_AG_CTX_CF5EN_MASK                   0x1 /* cf5en */
-#define XSTORM_ETH_CONN_AG_CTX_CF5EN_SHIFT                  3
-#define XSTORM_ETH_CONN_AG_CTX_CF6EN_MASK                   0x1 /* cf6en */
-#define XSTORM_ETH_CONN_AG_CTX_CF6EN_SHIFT                  4
-#define XSTORM_ETH_CONN_AG_CTX_CF7EN_MASK                   0x1 /* cf7en */
-#define XSTORM_ETH_CONN_AG_CTX_CF7EN_SHIFT                  5
-#define XSTORM_ETH_CONN_AG_CTX_CF8EN_MASK                   0x1 /* cf8en */
-#define XSTORM_ETH_CONN_AG_CTX_CF8EN_SHIFT                  6
-#define XSTORM_ETH_CONN_AG_CTX_CF9EN_MASK                   0x1 /* cf9en */
-#define XSTORM_ETH_CONN_AG_CTX_CF9EN_SHIFT                  7
+#define XSTORM_ETH_CONN_AG_CTX_CF2EN_MASK               0x1
+#define XSTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT              0
+#define XSTORM_ETH_CONN_AG_CTX_CF3EN_MASK               0x1
+#define XSTORM_ETH_CONN_AG_CTX_CF3EN_SHIFT              1
+#define XSTORM_ETH_CONN_AG_CTX_CF4EN_MASK               0x1
+#define XSTORM_ETH_CONN_AG_CTX_CF4EN_SHIFT              2
+#define XSTORM_ETH_CONN_AG_CTX_CF5EN_MASK               0x1
+#define XSTORM_ETH_CONN_AG_CTX_CF5EN_SHIFT              3
+#define XSTORM_ETH_CONN_AG_CTX_CF6EN_MASK               0x1
+#define XSTORM_ETH_CONN_AG_CTX_CF6EN_SHIFT              4
+#define XSTORM_ETH_CONN_AG_CTX_CF7EN_MASK               0x1
+#define XSTORM_ETH_CONN_AG_CTX_CF7EN_SHIFT              5
+#define XSTORM_ETH_CONN_AG_CTX_CF8EN_MASK               0x1
+#define XSTORM_ETH_CONN_AG_CTX_CF8EN_SHIFT              6
+#define XSTORM_ETH_CONN_AG_CTX_CF9EN_MASK               0x1
+#define XSTORM_ETH_CONN_AG_CTX_CF9EN_SHIFT              7
         u8 flags9;
-#define XSTORM_ETH_CONN_AG_CTX_CF10EN_MASK                  0x1 /* cf10en */
-#define XSTORM_ETH_CONN_AG_CTX_CF10EN_SHIFT                 0
-#define XSTORM_ETH_CONN_AG_CTX_CF11EN_MASK                  0x1 /* cf11en */
-#define XSTORM_ETH_CONN_AG_CTX_CF11EN_SHIFT                 1
-#define XSTORM_ETH_CONN_AG_CTX_CF12EN_MASK                  0x1 /* cf12en */
-#define XSTORM_ETH_CONN_AG_CTX_CF12EN_SHIFT                 2
-#define XSTORM_ETH_CONN_AG_CTX_CF13EN_MASK                  0x1 /* cf13en */
-#define XSTORM_ETH_CONN_AG_CTX_CF13EN_SHIFT                 3
-#define XSTORM_ETH_CONN_AG_CTX_CF14EN_MASK                  0x1 /* cf14en */
-#define XSTORM_ETH_CONN_AG_CTX_CF14EN_SHIFT                 4
-#define XSTORM_ETH_CONN_AG_CTX_CF15EN_MASK                  0x1 /* cf15en */
-#define XSTORM_ETH_CONN_AG_CTX_CF15EN_SHIFT                 5
-#define XSTORM_ETH_CONN_AG_CTX_GO_TO_BD_CONS_CF_EN_MASK     0x1 /* cf16en */
-#define XSTORM_ETH_CONN_AG_CTX_GO_TO_BD_CONS_CF_EN_SHIFT    6
-#define XSTORM_ETH_CONN_AG_CTX_MULTI_UNICAST_CF_EN_MASK     0x1
-#define XSTORM_ETH_CONN_AG_CTX_MULTI_UNICAST_CF_EN_SHIFT    7
+#define XSTORM_ETH_CONN_AG_CTX_CF10EN_MASK              0x1
+#define XSTORM_ETH_CONN_AG_CTX_CF10EN_SHIFT             0
+#define XSTORM_ETH_CONN_AG_CTX_CF11EN_MASK              0x1
+#define XSTORM_ETH_CONN_AG_CTX_CF11EN_SHIFT             1
+#define XSTORM_ETH_CONN_AG_CTX_CF12EN_MASK              0x1
+#define XSTORM_ETH_CONN_AG_CTX_CF12EN_SHIFT             2
+#define XSTORM_ETH_CONN_AG_CTX_CF13EN_MASK              0x1
+#define XSTORM_ETH_CONN_AG_CTX_CF13EN_SHIFT             3
+#define XSTORM_ETH_CONN_AG_CTX_CF14EN_MASK              0x1
+#define XSTORM_ETH_CONN_AG_CTX_CF14EN_SHIFT             4
+#define XSTORM_ETH_CONN_AG_CTX_CF15EN_MASK              0x1
+#define XSTORM_ETH_CONN_AG_CTX_CF15EN_SHIFT             5
+#define XSTORM_ETH_CONN_AG_CTX_GO_TO_BD_CONS_CF_EN_MASK 0x1
+#define XSTORM_ETH_CONN_AG_CTX_GO_TO_BD_CONS_CF_EN_SHIFT 6
+#define XSTORM_ETH_CONN_AG_CTX_MULTI_UNICAST_CF_EN_MASK 0x1
+#define XSTORM_ETH_CONN_AG_CTX_MULTI_UNICAST_CF_EN_SHIFT 7
         u8 flags10;
-#define XSTORM_ETH_CONN_AG_CTX_DQ_CF_EN_MASK                0x1 /* cf18en */
-#define XSTORM_ETH_CONN_AG_CTX_DQ_CF_EN_SHIFT               0
-#define XSTORM_ETH_CONN_AG_CTX_TERMINATE_CF_EN_MASK         0x1 /* cf19en */
-#define XSTORM_ETH_CONN_AG_CTX_TERMINATE_CF_EN_SHIFT        1
-#define XSTORM_ETH_CONN_AG_CTX_FLUSH_Q0_EN_MASK             0x1 /* cf20en */
-#define XSTORM_ETH_CONN_AG_CTX_FLUSH_Q0_EN_SHIFT            2
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED11_MASK              0x1 /* cf21en */
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED11_SHIFT             3
-#define XSTORM_ETH_CONN_AG_CTX_SLOW_PATH_EN_MASK            0x1 /* cf22en */
-#define XSTORM_ETH_CONN_AG_CTX_SLOW_PATH_EN_SHIFT           4
-#define XSTORM_ETH_CONN_AG_CTX_TPH_ENABLE_EN_RESERVED_MASK  0x1 /* cf23en */
+#define XSTORM_ETH_CONN_AG_CTX_DQ_CF_EN_MASK            0x1
+#define XSTORM_ETH_CONN_AG_CTX_DQ_CF_EN_SHIFT           0
+#define XSTORM_ETH_CONN_AG_CTX_TERMINATE_CF_EN_MASK     0x1
+#define XSTORM_ETH_CONN_AG_CTX_TERMINATE_CF_EN_SHIFT    1
+#define XSTORM_ETH_CONN_AG_CTX_FLUSH_Q0_EN_MASK         0x1
+#define XSTORM_ETH_CONN_AG_CTX_FLUSH_Q0_EN_SHIFT        2
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED11_MASK          0x1
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED11_SHIFT         3
+#define XSTORM_ETH_CONN_AG_CTX_SLOW_PATH_EN_MASK        0x1
+#define XSTORM_ETH_CONN_AG_CTX_SLOW_PATH_EN_SHIFT       4
+#define XSTORM_ETH_CONN_AG_CTX_TPH_ENABLE_EN_RESERVED_MASK 0x1
 #define XSTORM_ETH_CONN_AG_CTX_TPH_ENABLE_EN_RESERVED_SHIFT 5
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED12_MASK              0x1 /* rule0en */
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED12_SHIFT             6
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED13_MASK              0x1 /* rule1en */
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED13_SHIFT             7
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED12_MASK          0x1
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED12_SHIFT         6
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED13_MASK          0x1
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED13_SHIFT         7
         u8 flags11;
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED14_MASK              0x1 /* rule2en */
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED14_SHIFT             0
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED15_MASK              0x1 /* rule3en */
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED15_SHIFT             1
-#define XSTORM_ETH_CONN_AG_CTX_TX_DEC_RULE_EN_MASK          0x1 /* rule4en */
-#define XSTORM_ETH_CONN_AG_CTX_TX_DEC_RULE_EN_SHIFT         2
-#define XSTORM_ETH_CONN_AG_CTX_RULE5EN_MASK                 0x1 /* rule5en */
-#define XSTORM_ETH_CONN_AG_CTX_RULE5EN_SHIFT                3
-#define XSTORM_ETH_CONN_AG_CTX_RULE6EN_MASK                 0x1 /* rule6en */
-#define XSTORM_ETH_CONN_AG_CTX_RULE6EN_SHIFT                4
-#define XSTORM_ETH_CONN_AG_CTX_RULE7EN_MASK                 0x1 /* rule7en */
-#define XSTORM_ETH_CONN_AG_CTX_RULE7EN_SHIFT                5
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED1_MASK            0x1 /* rule8en */
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED1_SHIFT           6
-#define XSTORM_ETH_CONN_AG_CTX_RULE9EN_MASK                 0x1 /* rule9en */
-#define XSTORM_ETH_CONN_AG_CTX_RULE9EN_SHIFT                7
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED14_MASK          0x1
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED14_SHIFT         0
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED15_MASK          0x1
+#define XSTORM_ETH_CONN_AG_CTX_RESERVED15_SHIFT         1
+#define XSTORM_ETH_CONN_AG_CTX_TX_DEC_RULE_EN_MASK      0x1
+#define XSTORM_ETH_CONN_AG_CTX_TX_DEC_RULE_EN_SHIFT     2
+#define XSTORM_ETH_CONN_AG_CTX_RULE5EN_MASK             0x1
+#define XSTORM_ETH_CONN_AG_CTX_RULE5EN_SHIFT            3
+#define XSTORM_ETH_CONN_AG_CTX_RULE6EN_MASK             0x1
+#define XSTORM_ETH_CONN_AG_CTX_RULE6EN_SHIFT            4
+#define XSTORM_ETH_CONN_AG_CTX_RULE7EN_MASK             0x1
+#define XSTORM_ETH_CONN_AG_CTX_RULE7EN_SHIFT            5
+#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED1_MASK        0x1
+#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED1_SHIFT       6
+#define XSTORM_ETH_CONN_AG_CTX_RULE9EN_MASK             0x1
+#define XSTORM_ETH_CONN_AG_CTX_RULE9EN_SHIFT            7
         u8 flags12;
-#define XSTORM_ETH_CONN_AG_CTX_RULE10EN_MASK                0x1 /* rule10en */
-#define XSTORM_ETH_CONN_AG_CTX_RULE10EN_SHIFT               0
-#define XSTORM_ETH_CONN_AG_CTX_RULE11EN_MASK                0x1 /* rule11en */
-#define XSTORM_ETH_CONN_AG_CTX_RULE11EN_SHIFT               1
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED2_MASK            0x1 /* rule12en */
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED2_SHIFT           2
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED3_MASK            0x1 /* rule13en */
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED3_SHIFT           3
-#define XSTORM_ETH_CONN_AG_CTX_RULE14EN_MASK                0x1 /* rule14en */
-#define XSTORM_ETH_CONN_AG_CTX_RULE14EN_SHIFT               4
-#define XSTORM_ETH_CONN_AG_CTX_RULE15EN_MASK                0x1 /* rule15en */
-#define XSTORM_ETH_CONN_AG_CTX_RULE15EN_SHIFT               5
-#define XSTORM_ETH_CONN_AG_CTX_RULE16EN_MASK                0x1 /* rule16en */
-#define XSTORM_ETH_CONN_AG_CTX_RULE16EN_SHIFT               6
-#define XSTORM_ETH_CONN_AG_CTX_RULE17EN_MASK                0x1 /* rule17en */
-#define XSTORM_ETH_CONN_AG_CTX_RULE17EN_SHIFT               7
+#define XSTORM_ETH_CONN_AG_CTX_RULE10EN_MASK            0x1
+#define XSTORM_ETH_CONN_AG_CTX_RULE10EN_SHIFT           0
+#define XSTORM_ETH_CONN_AG_CTX_RULE11EN_MASK            0x1
+#define XSTORM_ETH_CONN_AG_CTX_RULE11EN_SHIFT           1
+#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED2_MASK        0x1
+#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED2_SHIFT       2
+#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED3_MASK        0x1
+#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED3_SHIFT       3
+#define XSTORM_ETH_CONN_AG_CTX_RULE14EN_MASK            0x1
+#define XSTORM_ETH_CONN_AG_CTX_RULE14EN_SHIFT           4
+#define XSTORM_ETH_CONN_AG_CTX_RULE15EN_MASK            0x1
+#define XSTORM_ETH_CONN_AG_CTX_RULE15EN_SHIFT           5
+#define XSTORM_ETH_CONN_AG_CTX_RULE16EN_MASK            0x1
+#define XSTORM_ETH_CONN_AG_CTX_RULE16EN_SHIFT           6
+#define XSTORM_ETH_CONN_AG_CTX_RULE17EN_MASK            0x1
+#define XSTORM_ETH_CONN_AG_CTX_RULE17EN_SHIFT           7
         u8 flags13;
-#define XSTORM_ETH_CONN_AG_CTX_RULE18EN_MASK                0x1 /* rule18en */
-#define XSTORM_ETH_CONN_AG_CTX_RULE18EN_SHIFT               0
-#define XSTORM_ETH_CONN_AG_CTX_RULE19EN_MASK                0x1 /* rule19en */
-#define XSTORM_ETH_CONN_AG_CTX_RULE19EN_SHIFT               1
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED4_MASK            0x1 /* rule20en */
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED4_SHIFT           2
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED5_MASK            0x1 /* rule21en */
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED5_SHIFT           3
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED6_MASK            0x1 /* rule22en */
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED6_SHIFT           4
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED7_MASK            0x1 /* rule23en */
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED7_SHIFT           5
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED8_MASK            0x1 /* rule24en */
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED8_SHIFT           6
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED9_MASK            0x1 /* rule25en */
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED9_SHIFT           7
+#define XSTORM_ETH_CONN_AG_CTX_RULE18EN_MASK            0x1
+#define XSTORM_ETH_CONN_AG_CTX_RULE18EN_SHIFT           0
+#define XSTORM_ETH_CONN_AG_CTX_RULE19EN_MASK            0x1
+#define XSTORM_ETH_CONN_AG_CTX_RULE19EN_SHIFT           1
+#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED4_MASK        0x1
+#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED4_SHIFT       2
+#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED5_MASK        0x1
+#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED5_SHIFT       3
+#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED6_MASK        0x1
+#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED6_SHIFT       4
+#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED7_MASK        0x1
+#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED7_SHIFT       5
+#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED8_MASK        0x1
+#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED8_SHIFT       6
+#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED9_MASK        0x1
+#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED9_SHIFT       7
         u8 flags14;
-#define XSTORM_ETH_CONN_AG_CTX_EDPM_USE_EXT_HDR_MASK        0x1 /* bit16 */
-#define XSTORM_ETH_CONN_AG_CTX_EDPM_USE_EXT_HDR_SHIFT       0
-#define XSTORM_ETH_CONN_AG_CTX_EDPM_SEND_RAW_L3L4_MASK      0x1 /* bit17 */
-#define XSTORM_ETH_CONN_AG_CTX_EDPM_SEND_RAW_L3L4_SHIFT     1
-#define XSTORM_ETH_CONN_AG_CTX_EDPM_INBAND_PROP_HDR_MASK    0x1 /* bit18 */
-#define XSTORM_ETH_CONN_AG_CTX_EDPM_INBAND_PROP_HDR_SHIFT   2
-#define XSTORM_ETH_CONN_AG_CTX_EDPM_SEND_EXT_TUNNEL_MASK    0x1 /* bit19 */
-#define XSTORM_ETH_CONN_AG_CTX_EDPM_SEND_EXT_TUNNEL_SHIFT   3
-#define XSTORM_ETH_CONN_AG_CTX_L2_EDPM_ENABLE_MASK          0x1 /* bit20 */
-#define XSTORM_ETH_CONN_AG_CTX_L2_EDPM_ENABLE_SHIFT         4
-#define XSTORM_ETH_CONN_AG_CTX_ROCE_EDPM_ENABLE_MASK        0x1 /* bit21 */
-#define XSTORM_ETH_CONN_AG_CTX_ROCE_EDPM_ENABLE_SHIFT       5
-#define XSTORM_ETH_CONN_AG_CTX_TPH_ENABLE_MASK              0x3 /* cf23 */
-#define XSTORM_ETH_CONN_AG_CTX_TPH_ENABLE_SHIFT             6
-        u8      edpm_event_id /* byte2 */;
-        __le16  physical_q0 /* physical_q0 */;
-        __le16  word1 /* physical_q1 */;
-        __le16  edpm_num_bds /* physical_q2 */;
-        __le16  tx_bd_cons /* word3 */;
-        __le16  tx_bd_prod /* word4 */;
-        __le16  go_to_bd_cons /* word5 */;
-        __le16  conn_dpi /* conn_dpi */;
-        u8      byte3 /* byte3 */;
-        u8      byte4 /* byte4 */;
-        u8      byte5 /* byte5 */;
-        u8      byte6 /* byte6 */;
-        __le32  reg0 /* reg0 */;
-        __le32  reg1 /* reg1 */;
-        __le32  reg2 /* reg2 */;
-        __le32  reg3 /* reg3 */;
-        __le32  reg4 /* reg4 */;
-        __le32  reg5 /* cf_array0 */;
-        __le32  reg6 /* cf_array1 */;
-        __le16  word7 /* word7 */;
-        __le16  word8 /* word8 */;
-        __le16  word9 /* word9 */;
-        __le16  word10 /* word10 */;
-        __le32  reg7 /* reg7 */;
-        __le32  reg8 /* reg8 */;
-        __le32  reg9 /* reg9 */;
-        u8      byte7 /* byte7 */;
-        u8      byte8 /* byte8 */;
-        u8      byte9 /* byte9 */;
-        u8      byte10 /* byte10 */;
-        u8      byte11 /* byte11 */;
-        u8      byte12 /* byte12 */;
-        u8      byte13 /* byte13 */;
-        u8      byte14 /* byte14 */;
-        u8      byte15 /* byte15 */;
-        u8      byte16 /* byte16 */;
-        __le16  word11 /* word11 */;
-        __le32  reg10 /* reg10 */;
-        __le32  reg11 /* reg11 */;
-        __le32  reg12 /* reg12 */;
-        __le32  reg13 /* reg13 */;
-        __le32  reg14 /* reg14 */;
-        __le32  reg15 /* reg15 */;
-        __le32  reg16 /* reg16 */;
-        __le32  reg17 /* reg17 */;
-        __le32  reg18 /* reg18 */;
-        __le32  reg19 /* reg19 */;
-        __le16  word12 /* word12 */;
-        __le16  word13 /* word13 */;
-        __le16  word14 /* word14 */;
-        __le16  word15 /* word15 */;
+#define XSTORM_ETH_CONN_AG_CTX_EDPM_USE_EXT_HDR_MASK    0x1
+#define XSTORM_ETH_CONN_AG_CTX_EDPM_USE_EXT_HDR_SHIFT   0
+#define XSTORM_ETH_CONN_AG_CTX_EDPM_SEND_RAW_L3L4_MASK  0x1
+#define XSTORM_ETH_CONN_AG_CTX_EDPM_SEND_RAW_L3L4_SHIFT 1
+#define XSTORM_ETH_CONN_AG_CTX_EDPM_INBAND_PROP_HDR_MASK 0x1
+#define XSTORM_ETH_CONN_AG_CTX_EDPM_INBAND_PROP_HDR_SHIFT 2
+#define XSTORM_ETH_CONN_AG_CTX_EDPM_SEND_EXT_TUNNEL_MASK 0x1
+#define XSTORM_ETH_CONN_AG_CTX_EDPM_SEND_EXT_TUNNEL_SHIFT 3
+#define XSTORM_ETH_CONN_AG_CTX_L2_EDPM_ENABLE_MASK      0x1
+#define XSTORM_ETH_CONN_AG_CTX_L2_EDPM_ENABLE_SHIFT     4
+#define XSTORM_ETH_CONN_AG_CTX_ROCE_EDPM_ENABLE_MASK    0x1
+#define XSTORM_ETH_CONN_AG_CTX_ROCE_EDPM_ENABLE_SHIFT   5
+#define XSTORM_ETH_CONN_AG_CTX_TPH_ENABLE_MASK          0x3
+#define XSTORM_ETH_CONN_AG_CTX_TPH_ENABLE_SHIFT         6
+        u8 edpm_event_id;
+        __le16 physical_q0;
+        __le16 quota;
+        __le16 edpm_num_bds;
+        __le16 tx_bd_cons;
+        __le16 tx_bd_prod;
+        __le16 tx_class;
+        __le16 conn_dpi;
+        u8 byte3;
+        u8 byte4;
+        u8 byte5;
+        u8 byte6;
+        __le32 reg0;
+        __le32 reg1;
+        __le32 reg2;
+        __le32 reg3;
+        __le32 reg4;
+        __le32 reg5;
+        __le32 reg6;
+        __le16 word7;
+        __le16 word8;
+        __le16 word9;
+        __le16 word10;
+        __le32 reg7;
+        __le32 reg8;
+        __le32 reg9;
+        u8 byte7;
+        u8 byte8;
+        u8 byte9;
+        u8 byte10;
+        u8 byte11;
+        u8 byte12;
+        u8 byte13;
+        u8 byte14;
+        u8 byte15;
+        u8 byte16;
+        __le16 word11;
+        __le32 reg10;
+        __le32 reg11;
+        __le32 reg12;
+        __le32 reg13;
+        __le32 reg14;
+        __le32 reg15;
+        __le32 reg16;
+        __le32 reg17;
+        __le32 reg18;
+        __le32 reg19;
+        __le16 word12;
+        __le16 word13;
+        __le16 word14;
+        __le16 word15;
 };
 
 /* The eth storm context for the Ystorm */
@@ -2648,220 +2960,220 @@ struct ystorm_eth_conn_st_ctx {
 };
 
 struct ystorm_eth_conn_ag_ctx {
-        u8      byte0 /* cdu_validation */;
-        u8      byte1 /* state */;
-        u8      flags0;
-#define YSTORM_ETH_CONN_AG_CTX_BIT0_MASK                  0x1 /* exist_in_qm0 */
-#define YSTORM_ETH_CONN_AG_CTX_BIT0_SHIFT                 0
-#define YSTORM_ETH_CONN_AG_CTX_BIT1_MASK                  0x1 /* exist_in_qm1 */
-#define YSTORM_ETH_CONN_AG_CTX_BIT1_SHIFT                 1
-#define YSTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_MASK     0x3   /* cf0 */
-#define YSTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_SHIFT    2
-#define YSTORM_ETH_CONN_AG_CTX_PMD_TERMINATE_CF_MASK      0x3   /* cf1 */
-#define YSTORM_ETH_CONN_AG_CTX_PMD_TERMINATE_CF_SHIFT     4
-#define YSTORM_ETH_CONN_AG_CTX_CF2_MASK                   0x3   /* cf2 */
-#define YSTORM_ETH_CONN_AG_CTX_CF2_SHIFT                  6
+        u8 byte0;
+        u8 state;
+        u8 flags0;
+#define YSTORM_ETH_CONN_AG_CTX_BIT0_MASK                0x1
+#define YSTORM_ETH_CONN_AG_CTX_BIT0_SHIFT               0
+#define YSTORM_ETH_CONN_AG_CTX_BIT1_MASK                0x1
+#define YSTORM_ETH_CONN_AG_CTX_BIT1_SHIFT               1
+#define YSTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_MASK   0x3
+#define YSTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_SHIFT  2
+#define YSTORM_ETH_CONN_AG_CTX_PMD_TERMINATE_CF_MASK    0x3
+#define YSTORM_ETH_CONN_AG_CTX_PMD_TERMINATE_CF_SHIFT   4
+#define YSTORM_ETH_CONN_AG_CTX_CF2_MASK                 0x3
+#define YSTORM_ETH_CONN_AG_CTX_CF2_SHIFT                6
         u8 flags1;
-#define YSTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_EN_MASK  0x1   /* cf0en */
-#define YSTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_EN_SHIFT 0
-#define YSTORM_ETH_CONN_AG_CTX_PMD_TERMINATE_CF_EN_MASK   0x1   /* cf1en */
-#define YSTORM_ETH_CONN_AG_CTX_PMD_TERMINATE_CF_EN_SHIFT  1
-#define YSTORM_ETH_CONN_AG_CTX_CF2EN_MASK                 0x1   /* cf2en */
-#define YSTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT                2
-#define YSTORM_ETH_CONN_AG_CTX_RULE0EN_MASK               0x1   /* rule0en */
-#define YSTORM_ETH_CONN_AG_CTX_RULE0EN_SHIFT              3
-#define YSTORM_ETH_CONN_AG_CTX_RULE1EN_MASK               0x1   /* rule1en */
-#define YSTORM_ETH_CONN_AG_CTX_RULE1EN_SHIFT              4
-#define YSTORM_ETH_CONN_AG_CTX_RULE2EN_MASK               0x1   /* rule2en */
-#define YSTORM_ETH_CONN_AG_CTX_RULE2EN_SHIFT              5
-#define YSTORM_ETH_CONN_AG_CTX_RULE3EN_MASK               0x1   /* rule3en */
-#define YSTORM_ETH_CONN_AG_CTX_RULE3EN_SHIFT              6
-#define YSTORM_ETH_CONN_AG_CTX_RULE4EN_MASK               0x1   /* rule4en */
-#define YSTORM_ETH_CONN_AG_CTX_RULE4EN_SHIFT              7
-        u8      byte2 /* byte2 */;
-        u8      byte3 /* byte3 */;
-        __le16  word0 /* word0 */;
-        __le32  terminate_spqe /* reg0 */;
-        __le32  reg1 /* reg1 */;
-        __le16  tx_bd_cons_upd /* word1 */;
-        __le16  word2 /* word2 */;
-        __le16  word3 /* word3 */;
-        __le16  word4 /* word4 */;
-        __le32  reg2 /* reg2 */;
-        __le32  reg3 /* reg3 */;
+#define YSTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_EN_MASK        0x1
+#define YSTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_EN_SHIFT       0
+#define YSTORM_ETH_CONN_AG_CTX_PMD_TERMINATE_CF_EN_MASK         0x1
+#define YSTORM_ETH_CONN_AG_CTX_PMD_TERMINATE_CF_EN_SHIFT        1
+#define YSTORM_ETH_CONN_AG_CTX_CF2EN_MASK                       0x1
+#define YSTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT                      2
+#define YSTORM_ETH_CONN_AG_CTX_RULE0EN_MASK                     0x1
+#define YSTORM_ETH_CONN_AG_CTX_RULE0EN_SHIFT                    3
+#define YSTORM_ETH_CONN_AG_CTX_RULE1EN_MASK                     0x1
+#define YSTORM_ETH_CONN_AG_CTX_RULE1EN_SHIFT                    4
+#define YSTORM_ETH_CONN_AG_CTX_RULE2EN_MASK                     0x1
+#define YSTORM_ETH_CONN_AG_CTX_RULE2EN_SHIFT                    5
+#define YSTORM_ETH_CONN_AG_CTX_RULE3EN_MASK                     0x1
+#define YSTORM_ETH_CONN_AG_CTX_RULE3EN_SHIFT                    6
+#define YSTORM_ETH_CONN_AG_CTX_RULE4EN_MASK                     0x1
+#define YSTORM_ETH_CONN_AG_CTX_RULE4EN_SHIFT                    7
+        u8 tx_q0_int_coallecing_timeset;
+        u8 byte3;
+        __le16 word0;
+        __le32 terminate_spqe;
+        __le32 reg1;
+        __le16 tx_bd_cons_upd;
+        __le16 word2;
+        __le16 word3;
+        __le16 word4;
+        __le32 reg2;
+        __le32 reg3;
 };
 
 struct tstorm_eth_conn_ag_ctx {
-        u8      byte0 /* cdu_validation */;
-        u8      byte1 /* state */;
-        u8      flags0;
-#define TSTORM_ETH_CONN_AG_CTX_BIT0_MASK      0x1       /* exist_in_qm0 */
-#define TSTORM_ETH_CONN_AG_CTX_BIT0_SHIFT     0
-#define TSTORM_ETH_CONN_AG_CTX_BIT1_MASK      0x1       /* exist_in_qm1 */
-#define TSTORM_ETH_CONN_AG_CTX_BIT1_SHIFT     1
-#define TSTORM_ETH_CONN_AG_CTX_BIT2_MASK      0x1       /* bit2 */
-#define TSTORM_ETH_CONN_AG_CTX_BIT2_SHIFT     2
-#define TSTORM_ETH_CONN_AG_CTX_BIT3_MASK      0x1       /* bit3 */
-#define TSTORM_ETH_CONN_AG_CTX_BIT3_SHIFT     3
-#define TSTORM_ETH_CONN_AG_CTX_BIT4_MASK      0x1       /* bit4 */
-#define TSTORM_ETH_CONN_AG_CTX_BIT4_SHIFT     4
-#define TSTORM_ETH_CONN_AG_CTX_BIT5_MASK      0x1       /* bit5 */
-#define TSTORM_ETH_CONN_AG_CTX_BIT5_SHIFT     5
-#define TSTORM_ETH_CONN_AG_CTX_CF0_MASK       0x3       /* timer0cf */
-#define TSTORM_ETH_CONN_AG_CTX_CF0_SHIFT      6
+        u8 byte0;
+        u8 byte1;
+        u8 flags0;
+#define TSTORM_ETH_CONN_AG_CTX_BIT0_MASK                0x1
+#define TSTORM_ETH_CONN_AG_CTX_BIT0_SHIFT               0
+#define TSTORM_ETH_CONN_AG_CTX_BIT1_MASK                0x1
+#define TSTORM_ETH_CONN_AG_CTX_BIT1_SHIFT               1
+#define TSTORM_ETH_CONN_AG_CTX_BIT2_MASK                0x1
+#define TSTORM_ETH_CONN_AG_CTX_BIT2_SHIFT               2
+#define TSTORM_ETH_CONN_AG_CTX_BIT3_MASK                0x1
+#define TSTORM_ETH_CONN_AG_CTX_BIT3_SHIFT               3
+#define TSTORM_ETH_CONN_AG_CTX_BIT4_MASK                0x1
+#define TSTORM_ETH_CONN_AG_CTX_BIT4_SHIFT               4
+#define TSTORM_ETH_CONN_AG_CTX_BIT5_MASK                0x1
+#define TSTORM_ETH_CONN_AG_CTX_BIT5_SHIFT               5
+#define TSTORM_ETH_CONN_AG_CTX_CF0_MASK                 0x3
+#define TSTORM_ETH_CONN_AG_CTX_CF0_SHIFT                6
         u8 flags1;
-#define TSTORM_ETH_CONN_AG_CTX_CF1_MASK       0x3       /* timer1cf */
-#define TSTORM_ETH_CONN_AG_CTX_CF1_SHIFT      0
-#define TSTORM_ETH_CONN_AG_CTX_CF2_MASK       0x3       /* timer2cf */
-#define TSTORM_ETH_CONN_AG_CTX_CF2_SHIFT      2
-#define TSTORM_ETH_CONN_AG_CTX_CF3_MASK       0x3       /* timer_stop_all */
-#define TSTORM_ETH_CONN_AG_CTX_CF3_SHIFT      4
-#define TSTORM_ETH_CONN_AG_CTX_CF4_MASK       0x3       /* cf4 */
-#define TSTORM_ETH_CONN_AG_CTX_CF4_SHIFT      6
+#define TSTORM_ETH_CONN_AG_CTX_CF1_MASK                 0x3
+#define TSTORM_ETH_CONN_AG_CTX_CF1_SHIFT                0
+#define TSTORM_ETH_CONN_AG_CTX_CF2_MASK                 0x3
+#define TSTORM_ETH_CONN_AG_CTX_CF2_SHIFT                2
+#define TSTORM_ETH_CONN_AG_CTX_CF3_MASK                 0x3
+#define TSTORM_ETH_CONN_AG_CTX_CF3_SHIFT                4
+#define TSTORM_ETH_CONN_AG_CTX_CF4_MASK                 0x3
+#define TSTORM_ETH_CONN_AG_CTX_CF4_SHIFT                6
         u8 flags2;
-#define TSTORM_ETH_CONN_AG_CTX_CF5_MASK       0x3       /* cf5 */
-#define TSTORM_ETH_CONN_AG_CTX_CF5_SHIFT      0
-#define TSTORM_ETH_CONN_AG_CTX_CF6_MASK       0x3       /* cf6 */
-#define TSTORM_ETH_CONN_AG_CTX_CF6_SHIFT      2
-#define TSTORM_ETH_CONN_AG_CTX_CF7_MASK       0x3       /* cf7 */
-#define TSTORM_ETH_CONN_AG_CTX_CF7_SHIFT      4
-#define TSTORM_ETH_CONN_AG_CTX_CF8_MASK       0x3       /* cf8 */
-#define TSTORM_ETH_CONN_AG_CTX_CF8_SHIFT      6
+#define TSTORM_ETH_CONN_AG_CTX_CF5_MASK                 0x3
+#define TSTORM_ETH_CONN_AG_CTX_CF5_SHIFT                0
+#define TSTORM_ETH_CONN_AG_CTX_CF6_MASK                 0x3
+#define TSTORM_ETH_CONN_AG_CTX_CF6_SHIFT                2
+#define TSTORM_ETH_CONN_AG_CTX_CF7_MASK                 0x3
+#define TSTORM_ETH_CONN_AG_CTX_CF7_SHIFT                4
+#define TSTORM_ETH_CONN_AG_CTX_CF8_MASK                 0x3
+#define TSTORM_ETH_CONN_AG_CTX_CF8_SHIFT                6
         u8 flags3;
-#define TSTORM_ETH_CONN_AG_CTX_CF9_MASK       0x3       /* cf9 */
-#define TSTORM_ETH_CONN_AG_CTX_CF9_SHIFT      0
-#define TSTORM_ETH_CONN_AG_CTX_CF10_MASK      0x3       /* cf10 */
-#define TSTORM_ETH_CONN_AG_CTX_CF10_SHIFT     2
-#define TSTORM_ETH_CONN_AG_CTX_CF0EN_MASK     0x1       /* cf0en */
-#define TSTORM_ETH_CONN_AG_CTX_CF0EN_SHIFT    4
-#define TSTORM_ETH_CONN_AG_CTX_CF1EN_MASK     0x1       /* cf1en */
-#define TSTORM_ETH_CONN_AG_CTX_CF1EN_SHIFT    5
-#define TSTORM_ETH_CONN_AG_CTX_CF2EN_MASK     0x1       /* cf2en */
-#define TSTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT    6
-#define TSTORM_ETH_CONN_AG_CTX_CF3EN_MASK     0x1       /* cf3en */
-#define TSTORM_ETH_CONN_AG_CTX_CF3EN_SHIFT    7
+#define TSTORM_ETH_CONN_AG_CTX_CF9_MASK                 0x3
+#define TSTORM_ETH_CONN_AG_CTX_CF9_SHIFT                0
+#define TSTORM_ETH_CONN_AG_CTX_CF10_MASK                0x3
+#define TSTORM_ETH_CONN_AG_CTX_CF10_SHIFT               2
+#define TSTORM_ETH_CONN_AG_CTX_CF0EN_MASK               0x1
+#define TSTORM_ETH_CONN_AG_CTX_CF0EN_SHIFT              4
+#define TSTORM_ETH_CONN_AG_CTX_CF1EN_MASK               0x1
+#define TSTORM_ETH_CONN_AG_CTX_CF1EN_SHIFT              5
+#define TSTORM_ETH_CONN_AG_CTX_CF2EN_MASK               0x1
+#define TSTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT              6
+#define TSTORM_ETH_CONN_AG_CTX_CF3EN_MASK               0x1
+#define TSTORM_ETH_CONN_AG_CTX_CF3EN_SHIFT              7
         u8 flags4;
-#define TSTORM_ETH_CONN_AG_CTX_CF4EN_MASK     0x1       /* cf4en */
-#define TSTORM_ETH_CONN_AG_CTX_CF4EN_SHIFT    0
-#define TSTORM_ETH_CONN_AG_CTX_CF5EN_MASK     0x1       /* cf5en */
-#define TSTORM_ETH_CONN_AG_CTX_CF5EN_SHIFT    1
-#define TSTORM_ETH_CONN_AG_CTX_CF6EN_MASK     0x1       /* cf6en */
-#define TSTORM_ETH_CONN_AG_CTX_CF6EN_SHIFT    2
-#define TSTORM_ETH_CONN_AG_CTX_CF7EN_MASK     0x1       /* cf7en */
-#define TSTORM_ETH_CONN_AG_CTX_CF7EN_SHIFT    3
-#define TSTORM_ETH_CONN_AG_CTX_CF8EN_MASK     0x1       /* cf8en */
-#define TSTORM_ETH_CONN_AG_CTX_CF8EN_SHIFT    4
-#define TSTORM_ETH_CONN_AG_CTX_CF9EN_MASK     0x1       /* cf9en */
-#define TSTORM_ETH_CONN_AG_CTX_CF9EN_SHIFT    5
-#define TSTORM_ETH_CONN_AG_CTX_CF10EN_MASK    0x1       /* cf10en */
-#define TSTORM_ETH_CONN_AG_CTX_CF10EN_SHIFT   6
-#define TSTORM_ETH_CONN_AG_CTX_RULE0EN_MASK   0x1       /* rule0en */
-#define TSTORM_ETH_CONN_AG_CTX_RULE0EN_SHIFT  7
+#define TSTORM_ETH_CONN_AG_CTX_CF4EN_MASK               0x1
+#define TSTORM_ETH_CONN_AG_CTX_CF4EN_SHIFT              0
+#define TSTORM_ETH_CONN_AG_CTX_CF5EN_MASK               0x1
+#define TSTORM_ETH_CONN_AG_CTX_CF5EN_SHIFT              1
+#define TSTORM_ETH_CONN_AG_CTX_CF6EN_MASK               0x1
+#define TSTORM_ETH_CONN_AG_CTX_CF6EN_SHIFT              2
+#define TSTORM_ETH_CONN_AG_CTX_CF7EN_MASK               0x1
+#define TSTORM_ETH_CONN_AG_CTX_CF7EN_SHIFT              3
+#define TSTORM_ETH_CONN_AG_CTX_CF8EN_MASK               0x1
+#define TSTORM_ETH_CONN_AG_CTX_CF8EN_SHIFT              4
+#define TSTORM_ETH_CONN_AG_CTX_CF9EN_MASK               0x1
+#define TSTORM_ETH_CONN_AG_CTX_CF9EN_SHIFT              5
+#define TSTORM_ETH_CONN_AG_CTX_CF10EN_MASK              0x1
+#define TSTORM_ETH_CONN_AG_CTX_CF10EN_SHIFT             6
+#define TSTORM_ETH_CONN_AG_CTX_RULE0EN_MASK             0x1
+#define TSTORM_ETH_CONN_AG_CTX_RULE0EN_SHIFT            7
         u8 flags5;
-#define TSTORM_ETH_CONN_AG_CTX_RULE1EN_MASK   0x1       /* rule1en */
-#define TSTORM_ETH_CONN_AG_CTX_RULE1EN_SHIFT  0
-#define TSTORM_ETH_CONN_AG_CTX_RULE2EN_MASK   0x1       /* rule2en */
-#define TSTORM_ETH_CONN_AG_CTX_RULE2EN_SHIFT  1
-#define TSTORM_ETH_CONN_AG_CTX_RULE3EN_MASK   0x1       /* rule3en */
-#define TSTORM_ETH_CONN_AG_CTX_RULE3EN_SHIFT  2
-#define TSTORM_ETH_CONN_AG_CTX_RULE4EN_MASK   0x1       /* rule4en */
-#define TSTORM_ETH_CONN_AG_CTX_RULE4EN_SHIFT  3
-#define TSTORM_ETH_CONN_AG_CTX_RULE5EN_MASK   0x1       /* rule5en */
-#define TSTORM_ETH_CONN_AG_CTX_RULE5EN_SHIFT  4
-#define TSTORM_ETH_CONN_AG_CTX_RX_BD_EN_MASK  0x1       /* rule6en */
-#define TSTORM_ETH_CONN_AG_CTX_RX_BD_EN_SHIFT 5
-#define TSTORM_ETH_CONN_AG_CTX_RULE7EN_MASK   0x1       /* rule7en */
-#define TSTORM_ETH_CONN_AG_CTX_RULE7EN_SHIFT  6
-#define TSTORM_ETH_CONN_AG_CTX_RULE8EN_MASK   0x1       /* rule8en */
-#define TSTORM_ETH_CONN_AG_CTX_RULE8EN_SHIFT  7
-        __le32  reg0 /* reg0 */;
-        __le32  reg1 /* reg1 */;
-        __le32  reg2 /* reg2 */;
-        __le32  reg3 /* reg3 */;
-        __le32  reg4 /* reg4 */;
-        __le32  reg5 /* reg5 */;
-        __le32  reg6 /* reg6 */;
-        __le32  reg7 /* reg7 */;
-        __le32  reg8 /* reg8 */;
-        u8      byte2 /* byte2 */;
-        u8      byte3 /* byte3 */;
-        __le16  rx_bd_cons /* word0 */;
-        u8      byte4 /* byte4 */;
-        u8      byte5 /* byte5 */;
-        __le16  rx_bd_prod /* word1 */;
-        __le16  word2 /* conn_dpi */;
-        __le16  word3 /* word3 */;
-        __le32  reg9 /* reg9 */;
-        __le32  reg10 /* reg10 */;
+#define TSTORM_ETH_CONN_AG_CTX_RULE1EN_MASK             0x1
+#define TSTORM_ETH_CONN_AG_CTX_RULE1EN_SHIFT            0
+#define TSTORM_ETH_CONN_AG_CTX_RULE2EN_MASK             0x1
+#define TSTORM_ETH_CONN_AG_CTX_RULE2EN_SHIFT            1
+#define TSTORM_ETH_CONN_AG_CTX_RULE3EN_MASK             0x1
+#define TSTORM_ETH_CONN_AG_CTX_RULE3EN_SHIFT            2
+#define TSTORM_ETH_CONN_AG_CTX_RULE4EN_MASK             0x1
+#define TSTORM_ETH_CONN_AG_CTX_RULE4EN_SHIFT            3
+#define TSTORM_ETH_CONN_AG_CTX_RULE5EN_MASK             0x1
+#define TSTORM_ETH_CONN_AG_CTX_RULE5EN_SHIFT            4
+#define TSTORM_ETH_CONN_AG_CTX_RX_BD_EN_MASK            0x1
+#define TSTORM_ETH_CONN_AG_CTX_RX_BD_EN_SHIFT           5
+#define TSTORM_ETH_CONN_AG_CTX_RULE7EN_MASK             0x1
+#define TSTORM_ETH_CONN_AG_CTX_RULE7EN_SHIFT            6
+#define TSTORM_ETH_CONN_AG_CTX_RULE8EN_MASK             0x1
+#define TSTORM_ETH_CONN_AG_CTX_RULE8EN_SHIFT            7
+        __le32 reg0;
+        __le32 reg1;
+        __le32 reg2;
+        __le32 reg3;
+        __le32 reg4;
+        __le32 reg5;
+        __le32 reg6;
+        __le32 reg7;
+        __le32 reg8;
+        u8 byte2;
+        u8 byte3;
+        __le16 rx_bd_cons;
+        u8 byte4;
+        u8 byte5;
+        __le16 rx_bd_prod;
+        __le16 word2;
+        __le16 word3;
+        __le32 reg9;
+        __le32 reg10;
 };
 
 struct ustorm_eth_conn_ag_ctx {
-        u8      byte0 /* cdu_validation */;
-        u8      byte1 /* state */;
-        u8      flags0;
-#define USTORM_ETH_CONN_AG_CTX_BIT0_MASK                  0x1 /* exist_in_qm0 */
-#define USTORM_ETH_CONN_AG_CTX_BIT0_SHIFT                 0
-#define USTORM_ETH_CONN_AG_CTX_BIT1_MASK                  0x1 /* exist_in_qm1 */
-#define USTORM_ETH_CONN_AG_CTX_BIT1_SHIFT                 1
-#define USTORM_ETH_CONN_AG_CTX_TX_PMD_TERMINATE_CF_MASK   0x3 /* timer0cf */
-#define USTORM_ETH_CONN_AG_CTX_TX_PMD_TERMINATE_CF_SHIFT  2
-#define USTORM_ETH_CONN_AG_CTX_RX_PMD_TERMINATE_CF_MASK   0x3 /* timer1cf */
-#define USTORM_ETH_CONN_AG_CTX_RX_PMD_TERMINATE_CF_SHIFT  4
-#define USTORM_ETH_CONN_AG_CTX_CF2_MASK                   0x3 /* timer2cf */
-#define USTORM_ETH_CONN_AG_CTX_CF2_SHIFT                  6
+        u8 byte0;
+        u8 byte1;
+        u8 flags0;
+#define USTORM_ETH_CONN_AG_CTX_BIT0_MASK                        0x1
+#define USTORM_ETH_CONN_AG_CTX_BIT0_SHIFT                       0
+#define USTORM_ETH_CONN_AG_CTX_BIT1_MASK                        0x1
+#define USTORM_ETH_CONN_AG_CTX_BIT1_SHIFT                       1
+#define USTORM_ETH_CONN_AG_CTX_TX_PMD_TERMINATE_CF_MASK         0x3
+#define USTORM_ETH_CONN_AG_CTX_TX_PMD_TERMINATE_CF_SHIFT        2
+#define USTORM_ETH_CONN_AG_CTX_RX_PMD_TERMINATE_CF_MASK         0x3
+#define USTORM_ETH_CONN_AG_CTX_RX_PMD_TERMINATE_CF_SHIFT        4
+#define USTORM_ETH_CONN_AG_CTX_CF2_MASK                         0x3
+#define USTORM_ETH_CONN_AG_CTX_CF2_SHIFT                        6
         u8 flags1;
-#define USTORM_ETH_CONN_AG_CTX_CF3_MASK                 0x3 /* timer_stop_all */
-#define USTORM_ETH_CONN_AG_CTX_CF3_SHIFT                0
-#define USTORM_ETH_CONN_AG_CTX_TX_ARM_CF_MASK           0x3 /* cf4 */
-#define USTORM_ETH_CONN_AG_CTX_TX_ARM_CF_SHIFT          2
-#define USTORM_ETH_CONN_AG_CTX_RX_ARM_CF_MASK           0x3 /* cf5 */
-#define USTORM_ETH_CONN_AG_CTX_RX_ARM_CF_SHIFT          4
-#define USTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_MASK   0x3 /* cf6 */
-#define USTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_SHIFT  6
+#define USTORM_ETH_CONN_AG_CTX_CF3_MASK                         0x3
+#define USTORM_ETH_CONN_AG_CTX_CF3_SHIFT                        0
+#define USTORM_ETH_CONN_AG_CTX_TX_ARM_CF_MASK                   0x3
+#define USTORM_ETH_CONN_AG_CTX_TX_ARM_CF_SHIFT                  2
+#define USTORM_ETH_CONN_AG_CTX_RX_ARM_CF_MASK                   0x3
+#define USTORM_ETH_CONN_AG_CTX_RX_ARM_CF_SHIFT                  4
+#define USTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_MASK           0x3
+#define USTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_SHIFT          6
         u8 flags2;
-#define USTORM_ETH_CONN_AG_CTX_TX_PMD_TERMINATE_CF_EN_MASK  0x1 /* cf0en */
-#define USTORM_ETH_CONN_AG_CTX_TX_PMD_TERMINATE_CF_EN_SHIFT 0
-#define USTORM_ETH_CONN_AG_CTX_RX_PMD_TERMINATE_CF_EN_MASK  0x1 /* cf1en */
-#define USTORM_ETH_CONN_AG_CTX_RX_PMD_TERMINATE_CF_EN_SHIFT 1
-#define USTORM_ETH_CONN_AG_CTX_CF2EN_MASK                   0x1 /* cf2en */
-#define USTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT                  2
-#define USTORM_ETH_CONN_AG_CTX_CF3EN_MASK                   0x1 /* cf3en */
-#define USTORM_ETH_CONN_AG_CTX_CF3EN_SHIFT                  3
-#define USTORM_ETH_CONN_AG_CTX_TX_ARM_CF_EN_MASK            0x1 /* cf4en */
-#define USTORM_ETH_CONN_AG_CTX_TX_ARM_CF_EN_SHIFT           4
-#define USTORM_ETH_CONN_AG_CTX_RX_ARM_CF_EN_MASK            0x1 /* cf5en */
-#define USTORM_ETH_CONN_AG_CTX_RX_ARM_CF_EN_SHIFT           5
-#define USTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_EN_MASK    0x1 /* cf6en */
-#define USTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_EN_SHIFT   6
-#define USTORM_ETH_CONN_AG_CTX_RULE0EN_MASK                 0x1 /* rule0en */
-#define USTORM_ETH_CONN_AG_CTX_RULE0EN_SHIFT                7
+#define USTORM_ETH_CONN_AG_CTX_TX_PMD_TERMINATE_CF_EN_MASK      0x1
+#define USTORM_ETH_CONN_AG_CTX_TX_PMD_TERMINATE_CF_EN_SHIFT     0
+#define USTORM_ETH_CONN_AG_CTX_RX_PMD_TERMINATE_CF_EN_MASK      0x1
+#define USTORM_ETH_CONN_AG_CTX_RX_PMD_TERMINATE_CF_EN_SHIFT     1
+#define USTORM_ETH_CONN_AG_CTX_CF2EN_MASK                       0x1
+#define USTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT                      2
+#define USTORM_ETH_CONN_AG_CTX_CF3EN_MASK                       0x1
+#define USTORM_ETH_CONN_AG_CTX_CF3EN_SHIFT                      3
+#define USTORM_ETH_CONN_AG_CTX_TX_ARM_CF_EN_MASK                0x1
+#define USTORM_ETH_CONN_AG_CTX_TX_ARM_CF_EN_SHIFT               4
+#define USTORM_ETH_CONN_AG_CTX_RX_ARM_CF_EN_MASK                0x1
+#define USTORM_ETH_CONN_AG_CTX_RX_ARM_CF_EN_SHIFT               5
+#define USTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_EN_MASK        0x1
+#define USTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_EN_SHIFT       6
+#define USTORM_ETH_CONN_AG_CTX_RULE0EN_MASK                     0x1
+#define USTORM_ETH_CONN_AG_CTX_RULE0EN_SHIFT                    7
         u8 flags3;
-#define USTORM_ETH_CONN_AG_CTX_RULE1EN_MASK                 0x1 /* rule1en */
-#define USTORM_ETH_CONN_AG_CTX_RULE1EN_SHIFT                0
-#define USTORM_ETH_CONN_AG_CTX_RULE2EN_MASK                 0x1 /* rule2en */
-#define USTORM_ETH_CONN_AG_CTX_RULE2EN_SHIFT                1
-#define USTORM_ETH_CONN_AG_CTX_RULE3EN_MASK                 0x1 /* rule3en */
-#define USTORM_ETH_CONN_AG_CTX_RULE3EN_SHIFT                2
-#define USTORM_ETH_CONN_AG_CTX_RULE4EN_MASK                 0x1 /* rule4en */
-#define USTORM_ETH_CONN_AG_CTX_RULE4EN_SHIFT                3
-#define USTORM_ETH_CONN_AG_CTX_RULE5EN_MASK                 0x1 /* rule5en */
-#define USTORM_ETH_CONN_AG_CTX_RULE5EN_SHIFT                4
-#define USTORM_ETH_CONN_AG_CTX_RULE6EN_MASK                 0x1 /* rule6en */
-#define USTORM_ETH_CONN_AG_CTX_RULE6EN_SHIFT                5
-#define USTORM_ETH_CONN_AG_CTX_RULE7EN_MASK                 0x1 /* rule7en */
-#define USTORM_ETH_CONN_AG_CTX_RULE7EN_SHIFT                6
-#define USTORM_ETH_CONN_AG_CTX_RULE8EN_MASK                 0x1 /* rule8en */
-#define USTORM_ETH_CONN_AG_CTX_RULE8EN_SHIFT                7
-        u8      byte2 /* byte2 */;
-        u8      byte3 /* byte3 */;
-        __le16  word0 /* conn_dpi */;
-        __le16  tx_bd_cons /* word1 */;
-        __le32  reg0 /* reg0 */;
-        __le32  reg1 /* reg1 */;
-        __le32  reg2 /* reg2 */;
-        __le32  tx_int_coallecing_timeset /* reg3 */;
-        __le16  tx_drv_bd_cons /* word2 */;
-        __le16  rx_drv_cqe_cons /* word3 */;
+#define USTORM_ETH_CONN_AG_CTX_RULE1EN_MASK                     0x1
+#define USTORM_ETH_CONN_AG_CTX_RULE1EN_SHIFT                    0
+#define USTORM_ETH_CONN_AG_CTX_RULE2EN_MASK                     0x1
+#define USTORM_ETH_CONN_AG_CTX_RULE2EN_SHIFT                    1
+#define USTORM_ETH_CONN_AG_CTX_RULE3EN_MASK                     0x1
+#define USTORM_ETH_CONN_AG_CTX_RULE3EN_SHIFT                    2
+#define USTORM_ETH_CONN_AG_CTX_RULE4EN_MASK                     0x1
+#define USTORM_ETH_CONN_AG_CTX_RULE4EN_SHIFT                    3
+#define USTORM_ETH_CONN_AG_CTX_RULE5EN_MASK                     0x1
+#define USTORM_ETH_CONN_AG_CTX_RULE5EN_SHIFT                    4
+#define USTORM_ETH_CONN_AG_CTX_RULE6EN_MASK                     0x1
+#define USTORM_ETH_CONN_AG_CTX_RULE6EN_SHIFT                    5
+#define USTORM_ETH_CONN_AG_CTX_RULE7EN_MASK                     0x1
+#define USTORM_ETH_CONN_AG_CTX_RULE7EN_SHIFT                    6
+#define USTORM_ETH_CONN_AG_CTX_RULE8EN_MASK                     0x1
+#define USTORM_ETH_CONN_AG_CTX_RULE8EN_SHIFT                    7
+        u8 byte2;
+        u8 byte3;
+        __le16 word0;
+        __le16 tx_bd_cons;
+        __le32 reg0;
+        __le32 reg1;
+        __le32 reg2;
+        __le32 tx_int_coallecing_timeset;
+        __le16 tx_drv_bd_cons;
+        __le16 rx_drv_cqe_cons;
 };
 
 /* The eth storm context for the Ustorm */
@@ -2876,47 +3188,75 @@ struct mstorm_eth_conn_st_ctx {
 
 /* eth connection context */
 struct eth_conn_context {
-        struct tstorm_eth_conn_st_ctx   tstorm_st_context;
-        struct regpair                  tstorm_st_padding[2];
-        struct pstorm_eth_conn_st_ctx   pstorm_st_context;
-        struct xstorm_eth_conn_st_ctx   xstorm_st_context;
-        struct xstorm_eth_conn_ag_ctx   xstorm_ag_context;
-        struct ystorm_eth_conn_st_ctx   ystorm_st_context;
-        struct ystorm_eth_conn_ag_ctx   ystorm_ag_context;
-        struct tstorm_eth_conn_ag_ctx   tstorm_ag_context;
-        struct ustorm_eth_conn_ag_ctx   ustorm_ag_context;
-        struct ustorm_eth_conn_st_ctx   ustorm_st_context;
-        struct mstorm_eth_conn_st_ctx   mstorm_st_context;
+        struct tstorm_eth_conn_st_ctx tstorm_st_context;
+        struct regpair tstorm_st_padding[2];
+        struct pstorm_eth_conn_st_ctx pstorm_st_context;
+        struct xstorm_eth_conn_st_ctx xstorm_st_context;
+        struct xstorm_eth_conn_ag_ctx xstorm_ag_context;
+        struct ystorm_eth_conn_st_ctx ystorm_st_context;
+        struct ystorm_eth_conn_ag_ctx ystorm_ag_context;
+        struct tstorm_eth_conn_ag_ctx tstorm_ag_context;
+        struct ustorm_eth_conn_ag_ctx ustorm_ag_context;
+        struct ustorm_eth_conn_st_ctx ustorm_st_context;
+        struct mstorm_eth_conn_st_ctx mstorm_st_context;
 };
 
+/* opcodes for the event ring */
+enum eth_event_opcode {
+        ETH_EVENT_UNUSED,
+        ETH_EVENT_VPORT_START,
+        ETH_EVENT_VPORT_UPDATE,
+        ETH_EVENT_VPORT_STOP,
+        ETH_EVENT_TX_QUEUE_START,
+        ETH_EVENT_TX_QUEUE_STOP,
+        ETH_EVENT_RX_QUEUE_START,
+        ETH_EVENT_RX_QUEUE_UPDATE,
+        ETH_EVENT_RX_QUEUE_STOP,
+        ETH_EVENT_FILTERS_UPDATE,
+        ETH_EVENT_RESERVED,
+        ETH_EVENT_RESERVED2,
+        ETH_EVENT_RESERVED3,
+        ETH_EVENT_RX_ADD_UDP_FILTER,
+        ETH_EVENT_RX_DELETE_UDP_FILTER,
+        ETH_EVENT_RESERVED4,
+        ETH_EVENT_RESERVED5,
+        MAX_ETH_EVENT_OPCODE
+};
+
+/* Classify rule types in E2/E3 */
 enum eth_filter_action {
+        ETH_FILTER_ACTION_UNUSED,
         ETH_FILTER_ACTION_REMOVE,
         ETH_FILTER_ACTION_ADD,
         ETH_FILTER_ACTION_REMOVE_ALL,
         MAX_ETH_FILTER_ACTION
 };
 
+/* Command for adding/removing a classification rule $$KEEP_ENDIANNESS$$ */
 struct eth_filter_cmd {
-        u8      type /* Filter Type (MAC/VLAN/Pair/VNI) */;
-        u8      vport_id /* the vport id */;
-        u8      action /* filter command action: add/remove/replace */;
-        u8      reserved0;
-        __le32  vni;
-        __le16  mac_lsb;
-        __le16  mac_mid;
-        __le16  mac_msb;
-        __le16  vlan_id;
+        u8 type;
+        u8 vport_id;
+        u8 action;
+        u8 reserved0;
+        __le32 vni;
+        __le16 mac_lsb;
+        __le16 mac_mid;
+        __le16 mac_msb;
+        __le16 vlan_id;
 };
 
+/*      $$KEEP_ENDIANNESS$$ */
 struct eth_filter_cmd_header {
-        u8      rx;
-        u8      tx;
-        u8      cmd_cnt;
-        u8      assert_on_error;
-        u8      reserved1[4];
+        u8 rx;
+        u8 tx;
+        u8 cmd_cnt;
+        u8 assert_on_error;
+        u8 reserved1[4];
 };
 
+/* Ethernet filter types: mac/vlan/pair */
 enum eth_filter_type {
+        ETH_FILTER_TYPE_UNUSED,
         ETH_FILTER_TYPE_MAC,
         ETH_FILTER_TYPE_VLAN,
         ETH_FILTER_TYPE_PAIR,
@@ -2929,463 +3269,3515 @@ enum eth_filter_type {
         MAX_ETH_FILTER_TYPE
 };
 
+/* Ethernet Ramrod Command IDs */
 enum eth_ramrod_cmd_id {
         ETH_RAMROD_UNUSED,
-        ETH_RAMROD_VPORT_START /* VPort Start Ramrod */,
-        ETH_RAMROD_VPORT_UPDATE /* VPort Update Ramrod */,
-        ETH_RAMROD_VPORT_STOP /* VPort Stop Ramrod */,
-        ETH_RAMROD_RX_QUEUE_START /* RX Queue Start Ramrod */,
-        ETH_RAMROD_RX_QUEUE_STOP /* RX Queue Stop Ramrod */,
-        ETH_RAMROD_TX_QUEUE_START /* TX Queue Start Ramrod */,
-        ETH_RAMROD_TX_QUEUE_STOP /* TX Queue Stop Ramrod */,
-        ETH_RAMROD_FILTERS_UPDATE /* Add or Remove Mac/Vlan/Pair filters */,
-        ETH_RAMROD_RX_QUEUE_UPDATE /* RX Queue Update Ramrod */,
-        ETH_RAMROD_RESERVED,
-        ETH_RAMROD_RESERVED2,
-        ETH_RAMROD_RESERVED3,
-        ETH_RAMROD_RESERVED4,
-        ETH_RAMROD_RESERVED5,
-        ETH_RAMROD_RESERVED6,
-        ETH_RAMROD_RESERVED7,
-        ETH_RAMROD_RESERVED8,
+        ETH_RAMROD_VPORT_START,
+        ETH_RAMROD_VPORT_UPDATE,
+        ETH_RAMROD_VPORT_STOP,
+        ETH_RAMROD_RX_QUEUE_START,
+        ETH_RAMROD_RX_QUEUE_STOP,
+        ETH_RAMROD_TX_QUEUE_START,
+        ETH_RAMROD_TX_QUEUE_STOP,
+        ETH_RAMROD_FILTERS_UPDATE,
+        ETH_RAMROD_RX_QUEUE_UPDATE,
+        ETH_RAMROD_RX_CREATE_OPENFLOW_ACTION,
+        ETH_RAMROD_RX_ADD_OPENFLOW_FILTER,
+        ETH_RAMROD_RX_DELETE_OPENFLOW_FILTER,
+        ETH_RAMROD_RX_ADD_UDP_FILTER,
+        ETH_RAMROD_RX_DELETE_UDP_FILTER,
+        ETH_RAMROD_RX_CREATE_GFT_ACTION,
+        ETH_RAMROD_GFT_UPDATE_FILTER,
         MAX_ETH_RAMROD_CMD_ID
 };
 
+/* return code from eth sp ramrods */
+struct eth_return_code {
+        u8 value;
+#define ETH_RETURN_CODE_ERR_CODE_MASK   0x1F
+#define ETH_RETURN_CODE_ERR_CODE_SHIFT  0
+#define ETH_RETURN_CODE_RESERVED_MASK   0x3
+#define ETH_RETURN_CODE_RESERVED_SHIFT  5
+#define ETH_RETURN_CODE_RX_TX_MASK      0x1
+#define ETH_RETURN_CODE_RX_TX_SHIFT     7
+};
+
+/* What to do in case an error occurs */
 enum eth_tx_err {
-        ETH_TX_ERR_DROP /* Drop erronous packet. */,
+        ETH_TX_ERR_DROP,
         ETH_TX_ERR_ASSERT_MALICIOUS,
         MAX_ETH_TX_ERR
 };
 
+/* Array of the different error type behaviors */
 struct eth_tx_err_vals {
         __le16 values;
-#define ETH_TX_ERR_VALS_ILLEGAL_VLAN_MODE_MASK            0x1
-#define ETH_TX_ERR_VALS_ILLEGAL_VLAN_MODE_SHIFT           0
-#define ETH_TX_ERR_VALS_PACKET_TOO_SMALL_MASK             0x1
-#define ETH_TX_ERR_VALS_PACKET_TOO_SMALL_SHIFT            1
-#define ETH_TX_ERR_VALS_ANTI_SPOOFING_ERR_MASK            0x1
-#define ETH_TX_ERR_VALS_ANTI_SPOOFING_ERR_SHIFT           2
-#define ETH_TX_ERR_VALS_ILLEGAL_INBAND_TAGS_MASK          0x1
-#define ETH_TX_ERR_VALS_ILLEGAL_INBAND_TAGS_SHIFT         3
-#define ETH_TX_ERR_VALS_VLAN_INSERTION_W_INBAND_TAG_MASK  0x1
-#define ETH_TX_ERR_VALS_VLAN_INSERTION_W_INBAND_TAG_SHIFT 4
-#define ETH_TX_ERR_VALS_MTU_VIOLATION_MASK                0x1
-#define ETH_TX_ERR_VALS_MTU_VIOLATION_SHIFT               5
-#define ETH_TX_ERR_VALS_ILLEGAL_CONTROL_FRAME_MASK        0x1
-#define ETH_TX_ERR_VALS_ILLEGAL_CONTROL_FRAME_SHIFT       6
-#define ETH_TX_ERR_VALS_RESERVED_MASK                     0x1FF
-#define ETH_TX_ERR_VALS_RESERVED_SHIFT                    7
-};
-
+#define ETH_TX_ERR_VALS_ILLEGAL_VLAN_MODE_MASK                  0x1
+#define ETH_TX_ERR_VALS_ILLEGAL_VLAN_MODE_SHIFT                 0
+#define ETH_TX_ERR_VALS_PACKET_TOO_SMALL_MASK                   0x1
+#define ETH_TX_ERR_VALS_PACKET_TOO_SMALL_SHIFT                  1
+#define ETH_TX_ERR_VALS_ANTI_SPOOFING_ERR_MASK                  0x1
+#define ETH_TX_ERR_VALS_ANTI_SPOOFING_ERR_SHIFT                 2
+#define ETH_TX_ERR_VALS_ILLEGAL_INBAND_TAGS_MASK                0x1
+#define ETH_TX_ERR_VALS_ILLEGAL_INBAND_TAGS_SHIFT               3
+#define ETH_TX_ERR_VALS_VLAN_INSERTION_W_INBAND_TAG_MASK        0x1
+#define ETH_TX_ERR_VALS_VLAN_INSERTION_W_INBAND_TAG_SHIFT       4
+#define ETH_TX_ERR_VALS_MTU_VIOLATION_MASK                      0x1
+#define ETH_TX_ERR_VALS_MTU_VIOLATION_SHIFT                     5
+#define ETH_TX_ERR_VALS_ILLEGAL_CONTROL_FRAME_MASK              0x1
+#define ETH_TX_ERR_VALS_ILLEGAL_CONTROL_FRAME_SHIFT             6
+#define ETH_TX_ERR_VALS_RESERVED_MASK                           0x1FF
+#define ETH_TX_ERR_VALS_RESERVED_SHIFT                          7
+};
+
+/* vport rss configuration data */
 struct eth_vport_rss_config {
         __le16 capabilities;
-#define ETH_VPORT_RSS_CONFIG_IPV4_CAPABILITY_MASK       0x1
-#define ETH_VPORT_RSS_CONFIG_IPV4_CAPABILITY_SHIFT       0
-#define ETH_VPORT_RSS_CONFIG_IPV6_CAPABILITY_MASK       0x1
-#define ETH_VPORT_RSS_CONFIG_IPV6_CAPABILITY_SHIFT       1
-#define ETH_VPORT_RSS_CONFIG_IPV4_TCP_CAPABILITY_MASK    0x1
-#define ETH_VPORT_RSS_CONFIG_IPV4_TCP_CAPABILITY_SHIFT   2
-#define ETH_VPORT_RSS_CONFIG_IPV6_TCP_CAPABILITY_MASK    0x1
-#define ETH_VPORT_RSS_CONFIG_IPV6_TCP_CAPABILITY_SHIFT   3
-#define ETH_VPORT_RSS_CONFIG_IPV4_UDP_CAPABILITY_MASK    0x1
-#define ETH_VPORT_RSS_CONFIG_IPV4_UDP_CAPABILITY_SHIFT   4
-#define ETH_VPORT_RSS_CONFIG_IPV6_UDP_CAPABILITY_MASK    0x1
-#define ETH_VPORT_RSS_CONFIG_IPV6_UDP_CAPABILITY_SHIFT   5
-#define ETH_VPORT_RSS_CONFIG_EN_5_TUPLE_CAPABILITY_MASK  0x1
-#define ETH_VPORT_RSS_CONFIG_EN_5_TUPLE_CAPABILITY_SHIFT 6
-#define ETH_VPORT_RSS_CONFIG_RESERVED0_MASK           0x1FF
-#define ETH_VPORT_RSS_CONFIG_RESERVED0_SHIFT         7
-        u8      rss_id;
-        u8      rss_mode;
-        u8      update_rss_key;
-        u8      update_rss_ind_table;
-        u8      update_rss_capabilities;
-        u8      tbl_size;
-        __le32  reserved2[2];
-        __le16  indirection_table[ETH_RSS_IND_TABLE_ENTRIES_NUM];
-        __le32  rss_key[ETH_RSS_KEY_SIZE_REGS];
-        __le32  reserved3[2];
-};
-
+#define ETH_VPORT_RSS_CONFIG_IPV4_CAPABILITY_MASK               0x1
+#define ETH_VPORT_RSS_CONFIG_IPV4_CAPABILITY_SHIFT              0
+#define ETH_VPORT_RSS_CONFIG_IPV6_CAPABILITY_MASK               0x1
+#define ETH_VPORT_RSS_CONFIG_IPV6_CAPABILITY_SHIFT              1
+#define ETH_VPORT_RSS_CONFIG_IPV4_TCP_CAPABILITY_MASK           0x1
+#define ETH_VPORT_RSS_CONFIG_IPV4_TCP_CAPABILITY_SHIFT          2
+#define ETH_VPORT_RSS_CONFIG_IPV6_TCP_CAPABILITY_MASK           0x1
+#define ETH_VPORT_RSS_CONFIG_IPV6_TCP_CAPABILITY_SHIFT          3
+#define ETH_VPORT_RSS_CONFIG_IPV4_UDP_CAPABILITY_MASK           0x1
+#define ETH_VPORT_RSS_CONFIG_IPV4_UDP_CAPABILITY_SHIFT          4
+#define ETH_VPORT_RSS_CONFIG_IPV6_UDP_CAPABILITY_MASK           0x1
+#define ETH_VPORT_RSS_CONFIG_IPV6_UDP_CAPABILITY_SHIFT          5
+#define ETH_VPORT_RSS_CONFIG_EN_5_TUPLE_CAPABILITY_MASK         0x1
+#define ETH_VPORT_RSS_CONFIG_EN_5_TUPLE_CAPABILITY_SHIFT        6
+#define ETH_VPORT_RSS_CONFIG_RESERVED0_MASK                     0x1FF
+#define ETH_VPORT_RSS_CONFIG_RESERVED0_SHIFT                    7
+        u8 rss_id;
+        u8 rss_mode;
+        u8 update_rss_key;
+        u8 update_rss_ind_table;
+        u8 update_rss_capabilities;
+        u8 tbl_size;
+        __le32 reserved2[2];
+        __le16 indirection_table[ETH_RSS_IND_TABLE_ENTRIES_NUM];
+
+        __le32 rss_key[ETH_RSS_KEY_SIZE_REGS];
+        __le32 reserved3[2];
+};
+
+/* eth vport RSS mode */
 enum eth_vport_rss_mode {
         ETH_VPORT_RSS_MODE_DISABLED,
         ETH_VPORT_RSS_MODE_REGULAR,
         MAX_ETH_VPORT_RSS_MODE
 };
 
+/* Command for setting classification flags for a vport $$KEEP_ENDIANNESS$$ */
 struct eth_vport_rx_mode {
         __le16 state;
-#define ETH_VPORT_RX_MODE_UCAST_DROP_ALL_MASK     0x1
-#define ETH_VPORT_RX_MODE_UCAST_DROP_ALL_SHIFT   0
-#define ETH_VPORT_RX_MODE_UCAST_ACCEPT_ALL_MASK 0x1
-#define ETH_VPORT_RX_MODE_UCAST_ACCEPT_ALL_SHIFT       1
-#define ETH_VPORT_RX_MODE_UCAST_ACCEPT_UNMATCHED_MASK  0x1
-#define ETH_VPORT_RX_MODE_UCAST_ACCEPT_UNMATCHED_SHIFT 2
-#define ETH_VPORT_RX_MODE_MCAST_DROP_ALL_MASK     0x1
-#define ETH_VPORT_RX_MODE_MCAST_DROP_ALL_SHIFT   3
-#define ETH_VPORT_RX_MODE_MCAST_ACCEPT_ALL_MASK 0x1
-#define ETH_VPORT_RX_MODE_MCAST_ACCEPT_ALL_SHIFT       4
-#define ETH_VPORT_RX_MODE_BCAST_ACCEPT_ALL_MASK 0x1
-#define ETH_VPORT_RX_MODE_BCAST_ACCEPT_ALL_SHIFT       5
-#define ETH_VPORT_RX_MODE_RESERVED1_MASK               0x3FF
-#define ETH_VPORT_RX_MODE_RESERVED1_SHIFT             6
+#define ETH_VPORT_RX_MODE_UCAST_DROP_ALL_MASK           0x1
+#define ETH_VPORT_RX_MODE_UCAST_DROP_ALL_SHIFT          0
+#define ETH_VPORT_RX_MODE_UCAST_ACCEPT_ALL_MASK         0x1
+#define ETH_VPORT_RX_MODE_UCAST_ACCEPT_ALL_SHIFT        1
+#define ETH_VPORT_RX_MODE_UCAST_ACCEPT_UNMATCHED_MASK   0x1
+#define ETH_VPORT_RX_MODE_UCAST_ACCEPT_UNMATCHED_SHIFT  2
+#define ETH_VPORT_RX_MODE_MCAST_DROP_ALL_MASK           0x1
+#define ETH_VPORT_RX_MODE_MCAST_DROP_ALL_SHIFT          3
+#define ETH_VPORT_RX_MODE_MCAST_ACCEPT_ALL_MASK         0x1
+#define ETH_VPORT_RX_MODE_MCAST_ACCEPT_ALL_SHIFT        4
+#define ETH_VPORT_RX_MODE_BCAST_ACCEPT_ALL_MASK         0x1
+#define ETH_VPORT_RX_MODE_BCAST_ACCEPT_ALL_SHIFT        5
+#define ETH_VPORT_RX_MODE_RESERVED1_MASK                0x3FF
+#define ETH_VPORT_RX_MODE_RESERVED1_SHIFT               6
         __le16 reserved2[3];
 };
 
+/* Command for setting tpa parameters */
 struct eth_vport_tpa_param {
-        u8      tpa_ipv4_en_flg;
-        u8      tpa_ipv6_en_flg;
-        u8      tpa_ipv4_tunn_en_flg;
-        u8      tpa_ipv6_tunn_en_flg;
-        u8      tpa_pkt_split_flg;
-        u8      tpa_hdr_data_split_flg;
-        u8      tpa_gro_consistent_flg;
-        u8      tpa_max_aggs_num;
-        u16     tpa_max_size;
-        u16     tpa_min_size_to_start;
-        u16     tpa_min_size_to_cont;
-        u8      max_buff_num;
-        u8      reserved;
+        u8 tpa_ipv4_en_flg;
+        u8 tpa_ipv6_en_flg;
+        u8 tpa_ipv4_tunn_en_flg;
+        u8 tpa_ipv6_tunn_en_flg;
+        u8 tpa_pkt_split_flg;
+        u8 tpa_hdr_data_split_flg;
+        u8 tpa_gro_consistent_flg;
+
+        u8 tpa_max_aggs_num;
+
+        __le16 tpa_max_size;
+        __le16 tpa_min_size_to_start;
+
+        __le16 tpa_min_size_to_cont;
+        u8 max_buff_num;
+        u8 reserved;
 };
 
+/* Command for setting classification flags for a vport $$KEEP_ENDIANNESS$$ */
 struct eth_vport_tx_mode {
         __le16 state;
-#define ETH_VPORT_TX_MODE_UCAST_DROP_ALL_MASK    0x1
-#define ETH_VPORT_TX_MODE_UCAST_DROP_ALL_SHIFT   0
-#define ETH_VPORT_TX_MODE_UCAST_ACCEPT_ALL_MASK  0x1
-#define ETH_VPORT_TX_MODE_UCAST_ACCEPT_ALL_SHIFT 1
-#define ETH_VPORT_TX_MODE_MCAST_DROP_ALL_MASK    0x1
-#define ETH_VPORT_TX_MODE_MCAST_DROP_ALL_SHIFT   2
-#define ETH_VPORT_TX_MODE_MCAST_ACCEPT_ALL_MASK  0x1
-#define ETH_VPORT_TX_MODE_MCAST_ACCEPT_ALL_SHIFT 3
-#define ETH_VPORT_TX_MODE_BCAST_ACCEPT_ALL_MASK  0x1
-#define ETH_VPORT_TX_MODE_BCAST_ACCEPT_ALL_SHIFT 4
-#define ETH_VPORT_TX_MODE_RESERVED1_MASK         0x7FF
-#define ETH_VPORT_TX_MODE_RESERVED1_SHIFT       5
+#define ETH_VPORT_TX_MODE_UCAST_DROP_ALL_MASK           0x1
+#define ETH_VPORT_TX_MODE_UCAST_DROP_ALL_SHIFT          0
+#define ETH_VPORT_TX_MODE_UCAST_ACCEPT_ALL_MASK         0x1
+#define ETH_VPORT_TX_MODE_UCAST_ACCEPT_ALL_SHIFT        1
+#define ETH_VPORT_TX_MODE_MCAST_DROP_ALL_MASK           0x1
+#define ETH_VPORT_TX_MODE_MCAST_DROP_ALL_SHIFT          2
+#define ETH_VPORT_TX_MODE_MCAST_ACCEPT_ALL_MASK         0x1
+#define ETH_VPORT_TX_MODE_MCAST_ACCEPT_ALL_SHIFT        3
+#define ETH_VPORT_TX_MODE_BCAST_ACCEPT_ALL_MASK         0x1
+#define ETH_VPORT_TX_MODE_BCAST_ACCEPT_ALL_SHIFT        4
+#define ETH_VPORT_TX_MODE_RESERVED1_MASK                0x7FF
+#define ETH_VPORT_TX_MODE_RESERVED1_SHIFT               5
         __le16 reserved2[3];
 };
 
+/* Ramrod data for rx queue start ramrod */
 struct rx_queue_start_ramrod_data {
-        __le16    rx_queue_id;
-        __le16    num_of_pbl_pages;
-        __le16    bd_max_bytes;
-        __le16    sb_id;
-        u8            sb_index;
-        u8            vport_id;
-        u8            default_rss_queue_flg;
-        u8            complete_cqe_flg;
-        u8            complete_event_flg;
-        u8            stats_counter_id;
-        u8            pin_context;
-        u8            pxp_tph_valid_bd;
-        u8            pxp_tph_valid_pkt;
-        u8            pxp_st_hint;
-        __le16    pxp_st_index;
-        u8              pmd_mode;
-        u8              notify_en;
-        u8              toggle_val;
-        u8              reserved[7];
-        __le16          reserved1;
-        struct regpair  cqe_pbl_addr;
-        struct regpair  bd_base;
-        struct regpair  reserved2;
+        __le16 rx_queue_id;
+        __le16 num_of_pbl_pages;
+        __le16 bd_max_bytes;
+        __le16 sb_id;
+        u8 sb_index;
+        u8 vport_id;
+        u8 default_rss_queue_flg;
+        u8 complete_cqe_flg;
+        u8 complete_event_flg;
+        u8 stats_counter_id;
+        u8 pin_context;
+        u8 pxp_tph_valid_bd;
+        u8 pxp_tph_valid_pkt;
+        u8 pxp_st_hint;
+
+        __le16 pxp_st_index;
+        u8 pmd_mode;
+
+        u8 notify_en;
+        u8 toggle_val;
+
+        u8 vf_rx_prod_index;
+
+        u8 reserved[6];
+        __le16 reserved1;
+        struct regpair cqe_pbl_addr;
+        struct regpair bd_base;
+        struct regpair reserved2;
 };
 
+/* Ramrod data for rx queue start ramrod */
 struct rx_queue_stop_ramrod_data {
-        __le16  rx_queue_id;
-        u8      complete_cqe_flg;
-        u8      complete_event_flg;
-        u8      vport_id;
-        u8      reserved[3];
+        __le16 rx_queue_id;
+        u8 complete_cqe_flg;
+        u8 complete_event_flg;
+        u8 vport_id;
+        u8 reserved[3];
 };
 
+/* Ramrod data for rx queue update ramrod */
 struct rx_queue_update_ramrod_data {
-        __le16  rx_queue_id;
-        u8      complete_cqe_flg;
-        u8      complete_event_flg;
-        u8      vport_id;
-        u8      reserved[4];
-        u8      reserved1;
-        u8      reserved2;
-        u8      reserved3;
-        __le16  reserved4;
-        __le16  reserved5;
+        __le16 rx_queue_id;
+        u8 complete_cqe_flg;
+        u8 complete_event_flg;
+        u8 vport_id;
+        u8 reserved[4];
+        u8 reserved1;
+        u8 reserved2;
+        u8 reserved3;
+        __le16 reserved4;
+        __le16 reserved5;
         struct regpair reserved6;
 };
 
-struct tx_queue_start_ramrod_data {
-        __le16  sb_id;
-        u8      sb_index;
-        u8      vport_id;
-        u8      reserved0;
-        u8      stats_counter_id;
-        __le16  qm_pq_id;
-        u8      flags;
-#define TX_QUEUE_START_RAMROD_DATA_DISABLE_OPPORTUNISTIC_MASK  0x1
-#define TX_QUEUE_START_RAMROD_DATA_DISABLE_OPPORTUNISTIC_SHIFT 0
-#define TX_QUEUE_START_RAMROD_DATA_TEST_MODE_PKT_DUP_MASK      0x1
-#define TX_QUEUE_START_RAMROD_DATA_TEST_MODE_PKT_DUP_SHIFT     1
-#define TX_QUEUE_START_RAMROD_DATA_TEST_MODE_TX_DEST_MASK      0x1
-#define TX_QUEUE_START_RAMROD_DATA_TEST_MODE_TX_DEST_SHIFT     2
-#define TX_QUEUE_START_RAMROD_DATA_PMD_MODE_MASK               0x1
-#define TX_QUEUE_START_RAMROD_DATA_PMD_MODE_SHIFT              3
-#define TX_QUEUE_START_RAMROD_DATA_NOTIFY_EN_MASK              0x1
-#define TX_QUEUE_START_RAMROD_DATA_NOTIFY_EN_SHIFT             4
-#define TX_QUEUE_START_RAMROD_DATA_PIN_CONTEXT_MASK            0x1
-#define TX_QUEUE_START_RAMROD_DATA_PIN_CONTEXT_SHIFT           5
-#define TX_QUEUE_START_RAMROD_DATA_RESERVED1_MASK              0x3
-#define TX_QUEUE_START_RAMROD_DATA_RESERVED1_SHIFT             6
-        u8      pxp_st_hint;
-        u8      pxp_tph_valid_bd;
-        u8      pxp_tph_valid_pkt;
-        __le16  pxp_st_index;
-        __le16  comp_agg_size;
-        __le16  queue_zone_id;
-        __le16  test_dup_count;
-        __le16  pbl_size;
-        __le16  tx_queue_id;
-        struct regpair  pbl_base_addr;
-        struct regpair  bd_cons_address;
+/* Ramrod data for rx Add UDP Filter */
+struct rx_udp_filter_data {
+        __le16 action_icid;
+        __le16 vlan_id;
+        u8 ip_type;
+        u8 tenant_id_exists;
+        __le16 reserved1;
+        __le32 ip_dst_addr[4];
+        __le32 ip_src_addr[4];
+        __le16 udp_dst_port;
+        __le16 udp_src_port;
+        __le32 tenant_id;
 };
 
+/* Ramrod data for rx queue start ramrod */
+struct tx_queue_start_ramrod_data {
+        __le16 sb_id;
+        u8 sb_index;
+        u8 vport_id;
+        u8 reserved0;
+        u8 stats_counter_id;
+        __le16 qm_pq_id;
+        u8 flags;
+#define TX_QUEUE_START_RAMROD_DATA_DISABLE_OPPORTUNISTIC_MASK   0x1
+#define TX_QUEUE_START_RAMROD_DATA_DISABLE_OPPORTUNISTIC_SHIFT  0
+#define TX_QUEUE_START_RAMROD_DATA_TEST_MODE_PKT_DUP_MASK       0x1
+#define TX_QUEUE_START_RAMROD_DATA_TEST_MODE_PKT_DUP_SHIFT      1
+#define TX_QUEUE_START_RAMROD_DATA_TEST_MODE_TX_DEST_MASK       0x1
+#define TX_QUEUE_START_RAMROD_DATA_TEST_MODE_TX_DEST_SHIFT      2
+#define TX_QUEUE_START_RAMROD_DATA_PMD_MODE_MASK                0x1
+#define TX_QUEUE_START_RAMROD_DATA_PMD_MODE_SHIFT               3
+#define TX_QUEUE_START_RAMROD_DATA_NOTIFY_EN_MASK               0x1
+#define TX_QUEUE_START_RAMROD_DATA_NOTIFY_EN_SHIFT              4
+#define TX_QUEUE_START_RAMROD_DATA_PIN_CONTEXT_MASK             0x1
+#define TX_QUEUE_START_RAMROD_DATA_PIN_CONTEXT_SHIFT            5
+#define TX_QUEUE_START_RAMROD_DATA_RESERVED1_MASK               0x3
+#define TX_QUEUE_START_RAMROD_DATA_RESERVED1_SHIFT              6
+        u8 pxp_st_hint;
+        u8 pxp_tph_valid_bd;
+        u8 pxp_tph_valid_pkt;
+        __le16 pxp_st_index;
+        __le16 comp_agg_size;
+        __le16 queue_zone_id;
+        __le16 test_dup_count;
+        __le16 pbl_size;
+        __le16 tx_queue_id;
+
+        struct regpair pbl_base_addr;
+        struct regpair bd_cons_address;
+};
+
+/* Ramrod data for tx queue stop ramrod */
 struct tx_queue_stop_ramrod_data {
         __le16 reserved[4];
 };
 
+/* Ramrod data for vport update ramrod */
 struct vport_filter_update_ramrod_data {
-        struct eth_filter_cmd_header    filter_cmd_hdr;
-        struct eth_filter_cmd      filter_cmds[ETH_FILTER_RULES_COUNT];
+        struct eth_filter_cmd_header filter_cmd_hdr;
+        struct eth_filter_cmd filter_cmds[ETH_FILTER_RULES_COUNT];
 };
 
+/* Ramrod data for vport start ramrod */
 struct vport_start_ramrod_data {
-        u8                            vport_id;
-        u8                            sw_fid;
-        __le16                    mtu;
-        u8                            drop_ttl0_en;
-        u8                            inner_vlan_removal_en;
-        struct eth_vport_rx_mode        rx_mode;
-        struct eth_vport_tx_mode        tx_mode;
-        struct eth_vport_tpa_param      tpa_param;
-        __le16                          default_vlan;
-        u8                              tx_switching_en;
-        u8                              anti_spoofing_en;
-        u8                              default_vlan_en;
-        u8                              handle_ptp_pkts;
-        u8                              silent_vlan_removal_en;
-        u8                              untagged;
-        struct eth_tx_err_vals          tx_err_behav;
-        u8                              zero_placement_offset;
-        u8                              reserved[7];
-};
-
+        u8 vport_id;
+        u8 sw_fid;
+        __le16 mtu;
+        u8 drop_ttl0_en;
+        u8 inner_vlan_removal_en;
+        struct eth_vport_rx_mode rx_mode;
+        struct eth_vport_tx_mode tx_mode;
+        struct eth_vport_tpa_param tpa_param;
+        __le16 default_vlan;
+        u8 tx_switching_en;
+        u8 anti_spoofing_en;
+
+        u8 default_vlan_en;
+
+        u8 handle_ptp_pkts;
+        u8 silent_vlan_removal_en;
+        u8 untagged;
+        struct eth_tx_err_vals tx_err_behav;
+
+        u8 zero_placement_offset;
+        u8 ctl_frame_mac_check_en;
+        u8 ctl_frame_ethtype_check_en;
+        u8 reserved[5];
+};
+
+/* Ramrod data for vport stop ramrod */
 struct vport_stop_ramrod_data {
-        u8      vport_id;
-        u8      reserved[7];
+        u8 vport_id;
+        u8 reserved[7];
 };
 
+/* Ramrod data for vport update ramrod */
 struct vport_update_ramrod_data_cmn {
-        u8      vport_id;
-        u8      update_rx_active_flg;
-        u8      rx_active_flg;
-        u8      update_tx_active_flg;
-        u8      tx_active_flg;
-        u8      update_rx_mode_flg;
-        u8      update_tx_mode_flg;
-        u8      update_approx_mcast_flg;
-        u8      update_rss_flg;
-        u8      update_inner_vlan_removal_en_flg;
-        u8      inner_vlan_removal_en;
-        u8      update_tpa_param_flg;
-        u8      update_tpa_en_flg;
-        u8      update_tx_switching_en_flg;
-        u8      tx_switching_en;
-        u8      update_anti_spoofing_en_flg;
-        u8      anti_spoofing_en;
-        u8      update_handle_ptp_pkts;
-        u8      handle_ptp_pkts;
-        u8      update_default_vlan_en_flg;
-        u8      default_vlan_en;
-        u8      update_default_vlan_flg;
-        __le16  default_vlan;
-        u8      update_accept_any_vlan_flg;
-        u8      accept_any_vlan;
-        u8      silent_vlan_removal_en;
-        u8      update_mtu_flg;
-        __le16  mtu;
-        u8      reserved[2];
+        u8 vport_id;
+        u8 update_rx_active_flg;
+        u8 rx_active_flg;
+        u8 update_tx_active_flg;
+        u8 tx_active_flg;
+        u8 update_rx_mode_flg;
+        u8 update_tx_mode_flg;
+        u8 update_approx_mcast_flg;
+
+        u8 update_rss_flg;
+        u8 update_inner_vlan_removal_en_flg;
+
+        u8 inner_vlan_removal_en;
+        u8 update_tpa_param_flg;
+        u8 update_tpa_en_flg;
+        u8 update_tx_switching_en_flg;
+
+        u8 tx_switching_en;
+        u8 update_anti_spoofing_en_flg;
+
+        u8 anti_spoofing_en;
+        u8 update_handle_ptp_pkts;
+
+        u8 handle_ptp_pkts;
+        u8 update_default_vlan_en_flg;
+
+        u8 default_vlan_en;
+
+        u8 update_default_vlan_flg;
+
+        __le16 default_vlan;
+        u8 update_accept_any_vlan_flg;
+
+        u8 accept_any_vlan;
+        u8 silent_vlan_removal_en;
+        u8 update_mtu_flg;
+
+        __le16 mtu;
+        u8 reserved[2];
 };
 
 struct vport_update_ramrod_mcast {
         __le32 bins[ETH_MULTICAST_MAC_BINS_IN_REGS];
 };
 
+/* Ramrod data for vport update ramrod */
 struct vport_update_ramrod_data {
-        struct vport_update_ramrod_data_cmn     common;
-        struct eth_vport_rx_mode                rx_mode;
-        struct eth_vport_tx_mode                tx_mode;
-        struct eth_vport_tpa_param            tpa_param;
-        struct vport_update_ramrod_mcast        approx_mcast;
-        struct eth_vport_rss_config          rss_config;
+        struct vport_update_ramrod_data_cmn common;
+
+        struct eth_vport_rx_mode rx_mode;
+        struct eth_vport_tx_mode tx_mode;
+        struct eth_vport_tpa_param tpa_param;
+        struct vport_update_ramrod_mcast approx_mcast;
+        struct eth_vport_rss_config rss_config;
+};
+
+struct mstorm_rdma_task_st_ctx {
+        struct regpair temp[4];
+};
+
+struct rdma_close_func_ramrod_data {
+        u8 cnq_start_offset;
+        u8 num_cnqs;
+        u8 vf_id;
+        u8 vf_valid;
+        u8 reserved[4];
+};
+
+struct rdma_cnq_params {
+        __le16 sb_num;
+        u8 sb_index;
+        u8 num_pbl_pages;
+        __le32 reserved;
+        struct regpair pbl_base_addr;
+        __le16 queue_zone_num;
+        u8 reserved1[6];
+};
+
+struct rdma_create_cq_ramrod_data {
+        struct regpair cq_handle;
+        struct regpair pbl_addr;
+        __le32 max_cqes;
+        __le16 pbl_num_pages;
+        __le16 dpi;
+        u8 is_two_level_pbl;
+        u8 cnq_id;
+        u8 pbl_log_page_size;
+        u8 toggle_bit;
+        __le16 int_timeout;
+        __le16 reserved1;
 };
 
-#define VF_MAX_STATIC 192       /* In case of K2 */
+struct rdma_deregister_tid_ramrod_data {
+        __le32 itid;
+        __le32 reserved;
+};
 
-#define MCP_GLOB_PATH_MAX       2
-#define MCP_PORT_MAX            2       /* Global */
-#define MCP_GLOB_PORT_MAX       4       /* Global */
-#define MCP_GLOB_FUNC_MAX       16      /* Global */
+struct rdma_destroy_cq_output_params {
+        __le16 cnq_num;
+        __le16 reserved0;
+        __le32 reserved1;
+};
+
+struct rdma_destroy_cq_ramrod_data {
+        struct regpair output_params_addr;
+};
+
+enum rdma_event_opcode {
+        RDMA_EVENT_UNUSED,
+        RDMA_EVENT_FUNC_INIT,
+        RDMA_EVENT_FUNC_CLOSE,
+        RDMA_EVENT_REGISTER_MR,
+        RDMA_EVENT_DEREGISTER_MR,
+        RDMA_EVENT_CREATE_CQ,
+        RDMA_EVENT_RESIZE_CQ,
+        RDMA_EVENT_DESTROY_CQ,
+        RDMA_EVENT_CREATE_SRQ,
+        RDMA_EVENT_MODIFY_SRQ,
+        RDMA_EVENT_DESTROY_SRQ,
+        MAX_RDMA_EVENT_OPCODE
+};
+
+enum rdma_fw_return_code {
+        RDMA_RETURN_OK = 0,
+        RDMA_RETURN_REGISTER_MR_BAD_STATE_ERR,
+        RDMA_RETURN_DEREGISTER_MR_BAD_STATE_ERR,
+        RDMA_RETURN_RESIZE_CQ_ERR,
+        RDMA_RETURN_NIG_DRAIN_REQ,
+        MAX_RDMA_FW_RETURN_CODE
+};
+
+struct rdma_init_func_hdr {
+        u8 cnq_start_offset;
+        u8 num_cnqs;
+        u8 cq_ring_mode;
+        u8 cnp_vlan_priority;
+        __le32 cnp_send_timeout;
+        u8 cnp_dscp;
+        u8 vf_id;
+        u8 vf_valid;
+        u8 reserved[5];
+};
+
+struct rdma_init_func_ramrod_data {
+        struct rdma_init_func_hdr params_header;
+        struct rdma_cnq_params cnq_params[NUM_OF_GLOBAL_QUEUES];
+};
+
+enum rdma_ramrod_cmd_id {
+        RDMA_RAMROD_UNUSED,
+        RDMA_RAMROD_FUNC_INIT,
+        RDMA_RAMROD_FUNC_CLOSE,
+        RDMA_RAMROD_REGISTER_MR,
+        RDMA_RAMROD_DEREGISTER_MR,
+        RDMA_RAMROD_CREATE_CQ,
+        RDMA_RAMROD_RESIZE_CQ,
+        RDMA_RAMROD_DESTROY_CQ,
+        RDMA_RAMROD_CREATE_SRQ,
+        RDMA_RAMROD_MODIFY_SRQ,
+        RDMA_RAMROD_DESTROY_SRQ,
+        MAX_RDMA_RAMROD_CMD_ID
+};
+
+struct rdma_register_tid_ramrod_data {
+        __le32 flags;
+#define RDMA_REGISTER_TID_RAMROD_DATA_MAX_ID_MASK             0x3FFFF
+#define RDMA_REGISTER_TID_RAMROD_DATA_MAX_ID_SHIFT            0
+#define RDMA_REGISTER_TID_RAMROD_DATA_PAGE_SIZE_LOG_MASK      0x1F
+#define RDMA_REGISTER_TID_RAMROD_DATA_PAGE_SIZE_LOG_SHIFT     18
+#define RDMA_REGISTER_TID_RAMROD_DATA_TWO_LEVEL_PBL_MASK      0x1
+#define RDMA_REGISTER_TID_RAMROD_DATA_TWO_LEVEL_PBL_SHIFT     23
+#define RDMA_REGISTER_TID_RAMROD_DATA_ZERO_BASED_MASK         0x1
+#define RDMA_REGISTER_TID_RAMROD_DATA_ZERO_BASED_SHIFT        24
+#define RDMA_REGISTER_TID_RAMROD_DATA_PHY_MR_MASK             0x1
+#define RDMA_REGISTER_TID_RAMROD_DATA_PHY_MR_SHIFT            25
+#define RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_READ_MASK        0x1
+#define RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_READ_SHIFT       26
+#define RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_WRITE_MASK       0x1
+#define RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_WRITE_SHIFT      27
+#define RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_ATOMIC_MASK      0x1
+#define RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_ATOMIC_SHIFT     28
+#define RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_WRITE_MASK        0x1
+#define RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_WRITE_SHIFT       29
+#define RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_READ_MASK         0x1
+#define RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_READ_SHIFT        30
+#define RDMA_REGISTER_TID_RAMROD_DATA_ENABLE_MW_BIND_MASK     0x1
+#define RDMA_REGISTER_TID_RAMROD_DATA_ENABLE_MW_BIND_SHIFT    31
+        u8 flags1;
+#define RDMA_REGISTER_TID_RAMROD_DATA_PBL_PAGE_SIZE_LOG_MASK  0x1F
+#define RDMA_REGISTER_TID_RAMROD_DATA_PBL_PAGE_SIZE_LOG_SHIFT 0
+#define RDMA_REGISTER_TID_RAMROD_DATA_TID_TYPE_MASK           0x7
+#define RDMA_REGISTER_TID_RAMROD_DATA_TID_TYPE_SHIFT          5
+        u8 flags2;
+#define RDMA_REGISTER_TID_RAMROD_DATA_DMA_MR_MASK             0x1
+#define RDMA_REGISTER_TID_RAMROD_DATA_DMA_MR_SHIFT            0
+#define RDMA_REGISTER_TID_RAMROD_DATA_DIF_ON_HOST_FLG_MASK    0x1
+#define RDMA_REGISTER_TID_RAMROD_DATA_DIF_ON_HOST_FLG_SHIFT   1
+#define RDMA_REGISTER_TID_RAMROD_DATA_RESERVED1_MASK          0x3F
+#define RDMA_REGISTER_TID_RAMROD_DATA_RESERVED1_SHIFT         2
+        u8 key;
+        u8 length_hi;
+        u8 vf_id;
+        u8 vf_valid;
+        __le16 pd;
+        __le32 length_lo;
+        __le32 itid;
+        __le32 reserved2;
+        struct regpair va;
+        struct regpair pbl_base;
+        struct regpair dif_error_addr;
+        struct regpair dif_runt_addr;
+        __le32 reserved3[2];
+};
+
+struct rdma_resize_cq_output_params {
+        __le32 old_cq_cons;
+        __le32 old_cq_prod;
+};
+
+struct rdma_resize_cq_ramrod_data {
+        u8 flags;
+#define RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK        0x1
+#define RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_SHIFT       0
+#define RDMA_RESIZE_CQ_RAMROD_DATA_IS_TWO_LEVEL_PBL_MASK  0x1
+#define RDMA_RESIZE_CQ_RAMROD_DATA_IS_TWO_LEVEL_PBL_SHIFT 1
+#define RDMA_RESIZE_CQ_RAMROD_DATA_RESERVED_MASK          0x3F
+#define RDMA_RESIZE_CQ_RAMROD_DATA_RESERVED_SHIFT         2
+        u8 pbl_log_page_size;
+        __le16 pbl_num_pages;
+        __le32 max_cqes;
+        struct regpair pbl_addr;
+        struct regpair output_params_addr;
+};
+
+struct rdma_srq_context {
+        struct regpair temp[8];
+};
+
+struct rdma_srq_create_ramrod_data {
+        struct regpair pbl_base_addr;
+        __le16 pages_in_srq_pbl;
+        __le16 pd_id;
+        struct rdma_srq_id srq_id;
+        __le16 page_size;
+        __le16 reserved1;
+        __le32 reserved2;
+        struct regpair producers_addr;
+};
+
+struct rdma_srq_destroy_ramrod_data {
+        struct rdma_srq_id srq_id;
+        __le32 reserved;
+};
+
+struct rdma_srq_modify_ramrod_data {
+        struct rdma_srq_id srq_id;
+        __le32 wqe_limit;
+};
+
+struct ystorm_rdma_task_st_ctx {
+        struct regpair temp[4];
+};
+
+struct ystorm_rdma_task_ag_ctx {
+        u8 reserved;
+        u8 byte1;
+        __le16 msem_ctx_upd_seq;
+        u8 flags0;
+#define YSTORM_RDMA_TASK_AG_CTX_CONNECTION_TYPE_MASK  0xF
+#define YSTORM_RDMA_TASK_AG_CTX_CONNECTION_TYPE_SHIFT 0
+#define YSTORM_RDMA_TASK_AG_CTX_EXIST_IN_QM0_MASK     0x1
+#define YSTORM_RDMA_TASK_AG_CTX_EXIST_IN_QM0_SHIFT    4
+#define YSTORM_RDMA_TASK_AG_CTX_BIT1_MASK             0x1
+#define YSTORM_RDMA_TASK_AG_CTX_BIT1_SHIFT            5
+#define YSTORM_RDMA_TASK_AG_CTX_VALID_MASK            0x1
+#define YSTORM_RDMA_TASK_AG_CTX_VALID_SHIFT           6
+#define YSTORM_RDMA_TASK_AG_CTX_BIT3_MASK             0x1
+#define YSTORM_RDMA_TASK_AG_CTX_BIT3_SHIFT            7
+        u8 flags1;
+#define YSTORM_RDMA_TASK_AG_CTX_CF0_MASK              0x3
+#define YSTORM_RDMA_TASK_AG_CTX_CF0_SHIFT             0
+#define YSTORM_RDMA_TASK_AG_CTX_CF1_MASK              0x3
+#define YSTORM_RDMA_TASK_AG_CTX_CF1_SHIFT             2
+#define YSTORM_RDMA_TASK_AG_CTX_CF2SPECIAL_MASK       0x3
+#define YSTORM_RDMA_TASK_AG_CTX_CF2SPECIAL_SHIFT      4
+#define YSTORM_RDMA_TASK_AG_CTX_CF0EN_MASK            0x1
+#define YSTORM_RDMA_TASK_AG_CTX_CF0EN_SHIFT           6
+#define YSTORM_RDMA_TASK_AG_CTX_CF1EN_MASK            0x1
+#define YSTORM_RDMA_TASK_AG_CTX_CF1EN_SHIFT           7
+        u8 flags2;
+#define YSTORM_RDMA_TASK_AG_CTX_BIT4_MASK             0x1
+#define YSTORM_RDMA_TASK_AG_CTX_BIT4_SHIFT            0
+#define YSTORM_RDMA_TASK_AG_CTX_RULE0EN_MASK          0x1
+#define YSTORM_RDMA_TASK_AG_CTX_RULE0EN_SHIFT         1
+#define YSTORM_RDMA_TASK_AG_CTX_RULE1EN_MASK          0x1
+#define YSTORM_RDMA_TASK_AG_CTX_RULE1EN_SHIFT         2
+#define YSTORM_RDMA_TASK_AG_CTX_RULE2EN_MASK          0x1
+#define YSTORM_RDMA_TASK_AG_CTX_RULE2EN_SHIFT         3
+#define YSTORM_RDMA_TASK_AG_CTX_RULE3EN_MASK          0x1
+#define YSTORM_RDMA_TASK_AG_CTX_RULE3EN_SHIFT         4
+#define YSTORM_RDMA_TASK_AG_CTX_RULE4EN_MASK          0x1
+#define YSTORM_RDMA_TASK_AG_CTX_RULE4EN_SHIFT         5
+#define YSTORM_RDMA_TASK_AG_CTX_RULE5EN_MASK          0x1
+#define YSTORM_RDMA_TASK_AG_CTX_RULE5EN_SHIFT         6
+#define YSTORM_RDMA_TASK_AG_CTX_RULE6EN_MASK          0x1
+#define YSTORM_RDMA_TASK_AG_CTX_RULE6EN_SHIFT         7
+        u8 key;
+        __le32 mw_cnt;
+        u8 ref_cnt_seq;
+        u8 ctx_upd_seq;
+        __le16 dif_flags;
+        __le16 tx_ref_count;
+        __le16 last_used_ltid;
+        __le16 parent_mr_lo;
+        __le16 parent_mr_hi;
+        __le32 fbo_lo;
+        __le32 fbo_hi;
+};
+
+struct mstorm_rdma_task_ag_ctx {
+        u8 reserved;
+        u8 byte1;
+        __le16 icid;
+        u8 flags0;
+#define MSTORM_RDMA_TASK_AG_CTX_CONNECTION_TYPE_MASK  0xF
+#define MSTORM_RDMA_TASK_AG_CTX_CONNECTION_TYPE_SHIFT 0
+#define MSTORM_RDMA_TASK_AG_CTX_EXIST_IN_QM0_MASK     0x1
+#define MSTORM_RDMA_TASK_AG_CTX_EXIST_IN_QM0_SHIFT    4
+#define MSTORM_RDMA_TASK_AG_CTX_BIT1_MASK             0x1
+#define MSTORM_RDMA_TASK_AG_CTX_BIT1_SHIFT            5
+#define MSTORM_RDMA_TASK_AG_CTX_BIT2_MASK             0x1
+#define MSTORM_RDMA_TASK_AG_CTX_BIT2_SHIFT            6
+#define MSTORM_RDMA_TASK_AG_CTX_BIT3_MASK             0x1
+#define MSTORM_RDMA_TASK_AG_CTX_BIT3_SHIFT            7
+        u8 flags1;
+#define MSTORM_RDMA_TASK_AG_CTX_CF0_MASK              0x3
+#define MSTORM_RDMA_TASK_AG_CTX_CF0_SHIFT             0
+#define MSTORM_RDMA_TASK_AG_CTX_CF1_MASK              0x3
+#define MSTORM_RDMA_TASK_AG_CTX_CF1_SHIFT             2
+#define MSTORM_RDMA_TASK_AG_CTX_CF2_MASK              0x3
+#define MSTORM_RDMA_TASK_AG_CTX_CF2_SHIFT             4
+#define MSTORM_RDMA_TASK_AG_CTX_CF0EN_MASK            0x1
+#define MSTORM_RDMA_TASK_AG_CTX_CF0EN_SHIFT           6
+#define MSTORM_RDMA_TASK_AG_CTX_CF1EN_MASK            0x1
+#define MSTORM_RDMA_TASK_AG_CTX_CF1EN_SHIFT           7
+        u8 flags2;
+#define MSTORM_RDMA_TASK_AG_CTX_CF2EN_MASK            0x1
+#define MSTORM_RDMA_TASK_AG_CTX_CF2EN_SHIFT           0
+#define MSTORM_RDMA_TASK_AG_CTX_RULE0EN_MASK          0x1
+#define MSTORM_RDMA_TASK_AG_CTX_RULE0EN_SHIFT         1
+#define MSTORM_RDMA_TASK_AG_CTX_RULE1EN_MASK          0x1
+#define MSTORM_RDMA_TASK_AG_CTX_RULE1EN_SHIFT         2
+#define MSTORM_RDMA_TASK_AG_CTX_RULE2EN_MASK          0x1
+#define MSTORM_RDMA_TASK_AG_CTX_RULE2EN_SHIFT         3
+#define MSTORM_RDMA_TASK_AG_CTX_RULE3EN_MASK          0x1
+#define MSTORM_RDMA_TASK_AG_CTX_RULE3EN_SHIFT         4
+#define MSTORM_RDMA_TASK_AG_CTX_RULE4EN_MASK          0x1
+#define MSTORM_RDMA_TASK_AG_CTX_RULE4EN_SHIFT         5
+#define MSTORM_RDMA_TASK_AG_CTX_RULE5EN_MASK          0x1
+#define MSTORM_RDMA_TASK_AG_CTX_RULE5EN_SHIFT         6
+#define MSTORM_RDMA_TASK_AG_CTX_RULE6EN_MASK          0x1
+#define MSTORM_RDMA_TASK_AG_CTX_RULE6EN_SHIFT         7
+        u8 key;
+        __le32 mw_cnt;
+        u8 ref_cnt_seq;
+        u8 ctx_upd_seq;
+        __le16 dif_flags;
+        __le16 tx_ref_count;
+        __le16 last_used_ltid;
+        __le16 parent_mr_lo;
+        __le16 parent_mr_hi;
+        __le32 fbo_lo;
+        __le32 fbo_hi;
+};
+
+struct ustorm_rdma_task_st_ctx {
+        struct regpair temp[2];
+};
+
+struct ustorm_rdma_task_ag_ctx {
+        u8 reserved;
+        u8 byte1;
+        __le16 icid;
+        u8 flags0;
+#define USTORM_RDMA_TASK_AG_CTX_CONNECTION_TYPE_MASK         0xF
+#define USTORM_RDMA_TASK_AG_CTX_CONNECTION_TYPE_SHIFT        0
+#define USTORM_RDMA_TASK_AG_CTX_EXIST_IN_QM0_MASK            0x1
+#define USTORM_RDMA_TASK_AG_CTX_EXIST_IN_QM0_SHIFT           4
+#define USTORM_RDMA_TASK_AG_CTX_DIF_RUNT_VALID_MASK          0x1
+#define USTORM_RDMA_TASK_AG_CTX_DIF_RUNT_VALID_SHIFT         5
+#define USTORM_RDMA_TASK_AG_CTX_DIF_WRITE_RESULT_CF_MASK     0x3
+#define USTORM_RDMA_TASK_AG_CTX_DIF_WRITE_RESULT_CF_SHIFT    6
+        u8 flags1;
+#define USTORM_RDMA_TASK_AG_CTX_DIF_RESULT_TOGGLE_BIT_MASK   0x3
+#define USTORM_RDMA_TASK_AG_CTX_DIF_RESULT_TOGGLE_BIT_SHIFT  0
+#define USTORM_RDMA_TASK_AG_CTX_DIF_TX_IO_FLG_MASK           0x3
+#define USTORM_RDMA_TASK_AG_CTX_DIF_TX_IO_FLG_SHIFT          2
+#define USTORM_RDMA_TASK_AG_CTX_CF3_MASK                     0x3
+#define USTORM_RDMA_TASK_AG_CTX_CF3_SHIFT                    4
+#define USTORM_RDMA_TASK_AG_CTX_DIF_ERROR_CF_MASK            0x3
+#define USTORM_RDMA_TASK_AG_CTX_DIF_ERROR_CF_SHIFT           6
+        u8 flags2;
+#define USTORM_RDMA_TASK_AG_CTX_DIF_WRITE_RESULT_CF_EN_MASK  0x1
+#define USTORM_RDMA_TASK_AG_CTX_DIF_WRITE_RESULT_CF_EN_SHIFT 0
+#define USTORM_RDMA_TASK_AG_CTX_RESERVED2_MASK               0x1
+#define USTORM_RDMA_TASK_AG_CTX_RESERVED2_SHIFT              1
+#define USTORM_RDMA_TASK_AG_CTX_RESERVED3_MASK               0x1
+#define USTORM_RDMA_TASK_AG_CTX_RESERVED3_SHIFT              2
+#define USTORM_RDMA_TASK_AG_CTX_CF3EN_MASK                   0x1
+#define USTORM_RDMA_TASK_AG_CTX_CF3EN_SHIFT                  3
+#define USTORM_RDMA_TASK_AG_CTX_DIF_ERROR_CF_EN_MASK         0x1
+#define USTORM_RDMA_TASK_AG_CTX_DIF_ERROR_CF_EN_SHIFT        4
+#define USTORM_RDMA_TASK_AG_CTX_RULE0EN_MASK                 0x1
+#define USTORM_RDMA_TASK_AG_CTX_RULE0EN_SHIFT                5
+#define USTORM_RDMA_TASK_AG_CTX_RULE1EN_MASK                 0x1
+#define USTORM_RDMA_TASK_AG_CTX_RULE1EN_SHIFT                6
+#define USTORM_RDMA_TASK_AG_CTX_RULE2EN_MASK                 0x1
+#define USTORM_RDMA_TASK_AG_CTX_RULE2EN_SHIFT                7
+        u8 flags3;
+#define USTORM_RDMA_TASK_AG_CTX_RULE3EN_MASK                 0x1
+#define USTORM_RDMA_TASK_AG_CTX_RULE3EN_SHIFT                0
+#define USTORM_RDMA_TASK_AG_CTX_RULE4EN_MASK                 0x1
+#define USTORM_RDMA_TASK_AG_CTX_RULE4EN_SHIFT                1
+#define USTORM_RDMA_TASK_AG_CTX_RULE5EN_MASK                 0x1
+#define USTORM_RDMA_TASK_AG_CTX_RULE5EN_SHIFT                2
+#define USTORM_RDMA_TASK_AG_CTX_RULE6EN_MASK                 0x1
+#define USTORM_RDMA_TASK_AG_CTX_RULE6EN_SHIFT                3
+#define USTORM_RDMA_TASK_AG_CTX_DIF_ERROR_TYPE_MASK          0xF
+#define USTORM_RDMA_TASK_AG_CTX_DIF_ERROR_TYPE_SHIFT         4
+        __le32 dif_err_intervals;
+        __le32 dif_error_1st_interval;
+        __le32 reg2;
+        __le32 dif_runt_value;
+        __le32 reg4;
+        __le32 reg5;
+};
+
+struct rdma_task_context {
+        struct ystorm_rdma_task_st_ctx ystorm_st_context;
+        struct ystorm_rdma_task_ag_ctx ystorm_ag_context;
+        struct tdif_task_context tdif_context;
+        struct mstorm_rdma_task_ag_ctx mstorm_ag_context;
+        struct mstorm_rdma_task_st_ctx mstorm_st_context;
+        struct rdif_task_context rdif_context;
+        struct ustorm_rdma_task_st_ctx ustorm_st_context;
+        struct regpair ustorm_st_padding[2];
+        struct ustorm_rdma_task_ag_ctx ustorm_ag_context;
+};
+
+enum rdma_tid_type {
+        RDMA_TID_REGISTERED_MR,
+        RDMA_TID_FMR,
+        RDMA_TID_MW_TYPE1,
+        RDMA_TID_MW_TYPE2A,
+        MAX_RDMA_TID_TYPE
+};
+
+struct mstorm_rdma_conn_ag_ctx {
+        u8 byte0;
+        u8 byte1;
+        u8 flags0;
+#define MSTORM_RDMA_CONN_AG_CTX_BIT0_MASK     0x1
+#define MSTORM_RDMA_CONN_AG_CTX_BIT0_SHIFT    0
+#define MSTORM_RDMA_CONN_AG_CTX_BIT1_MASK     0x1
+#define MSTORM_RDMA_CONN_AG_CTX_BIT1_SHIFT    1
+#define MSTORM_RDMA_CONN_AG_CTX_CF0_MASK      0x3
+#define MSTORM_RDMA_CONN_AG_CTX_CF0_SHIFT     2
+#define MSTORM_RDMA_CONN_AG_CTX_CF1_MASK      0x3
+#define MSTORM_RDMA_CONN_AG_CTX_CF1_SHIFT     4
+#define MSTORM_RDMA_CONN_AG_CTX_CF2_MASK      0x3
+#define MSTORM_RDMA_CONN_AG_CTX_CF2_SHIFT     6
+        u8 flags1;
+#define MSTORM_RDMA_CONN_AG_CTX_CF0EN_MASK    0x1
+#define MSTORM_RDMA_CONN_AG_CTX_CF0EN_SHIFT   0
+#define MSTORM_RDMA_CONN_AG_CTX_CF1EN_MASK    0x1
+#define MSTORM_RDMA_CONN_AG_CTX_CF1EN_SHIFT   1
+#define MSTORM_RDMA_CONN_AG_CTX_CF2EN_MASK    0x1
+#define MSTORM_RDMA_CONN_AG_CTX_CF2EN_SHIFT   2
+#define MSTORM_RDMA_CONN_AG_CTX_RULE0EN_MASK  0x1
+#define MSTORM_RDMA_CONN_AG_CTX_RULE0EN_SHIFT 3
+#define MSTORM_RDMA_CONN_AG_CTX_RULE1EN_MASK  0x1
+#define MSTORM_RDMA_CONN_AG_CTX_RULE1EN_SHIFT 4
+#define MSTORM_RDMA_CONN_AG_CTX_RULE2EN_MASK  0x1
+#define MSTORM_RDMA_CONN_AG_CTX_RULE2EN_SHIFT 5
+#define MSTORM_RDMA_CONN_AG_CTX_RULE3EN_MASK  0x1
+#define MSTORM_RDMA_CONN_AG_CTX_RULE3EN_SHIFT 6
+#define MSTORM_RDMA_CONN_AG_CTX_RULE4EN_MASK  0x1
+#define MSTORM_RDMA_CONN_AG_CTX_RULE4EN_SHIFT 7
+        __le16 word0;
+        __le16 word1;
+        __le32 reg0;
+        __le32 reg1;
+};
+
+struct tstorm_rdma_conn_ag_ctx {
+        u8 reserved0;
+        u8 byte1;
+        u8 flags0;
+#define TSTORM_RDMA_CONN_AG_CTX_EXIST_IN_QM0_MASK          0x1
+#define TSTORM_RDMA_CONN_AG_CTX_EXIST_IN_QM0_SHIFT         0
+#define TSTORM_RDMA_CONN_AG_CTX_BIT1_MASK                  0x1
+#define TSTORM_RDMA_CONN_AG_CTX_BIT1_SHIFT                 1
+#define TSTORM_RDMA_CONN_AG_CTX_BIT2_MASK                  0x1
+#define TSTORM_RDMA_CONN_AG_CTX_BIT2_SHIFT                 2
+#define TSTORM_RDMA_CONN_AG_CTX_BIT3_MASK                  0x1
+#define TSTORM_RDMA_CONN_AG_CTX_BIT3_SHIFT                 3
+#define TSTORM_RDMA_CONN_AG_CTX_BIT4_MASK                  0x1
+#define TSTORM_RDMA_CONN_AG_CTX_BIT4_SHIFT                 4
+#define TSTORM_RDMA_CONN_AG_CTX_BIT5_MASK                  0x1
+#define TSTORM_RDMA_CONN_AG_CTX_BIT5_SHIFT                 5
+#define TSTORM_RDMA_CONN_AG_CTX_CF0_MASK                   0x3
+#define TSTORM_RDMA_CONN_AG_CTX_CF0_SHIFT                  6
+        u8 flags1;
+#define TSTORM_RDMA_CONN_AG_CTX_CF1_MASK                   0x3
+#define TSTORM_RDMA_CONN_AG_CTX_CF1_SHIFT                  0
+#define TSTORM_RDMA_CONN_AG_CTX_CF2_MASK                   0x3
+#define TSTORM_RDMA_CONN_AG_CTX_CF2_SHIFT                  2
+#define TSTORM_RDMA_CONN_AG_CTX_TIMER_STOP_ALL_CF_MASK     0x3
+#define TSTORM_RDMA_CONN_AG_CTX_TIMER_STOP_ALL_CF_SHIFT    4
+#define TSTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_MASK           0x3
+#define TSTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_SHIFT          6
+        u8 flags2;
+#define TSTORM_RDMA_CONN_AG_CTX_MSTORM_FLUSH_CF_MASK       0x3
+#define TSTORM_RDMA_CONN_AG_CTX_MSTORM_FLUSH_CF_SHIFT      0
+#define TSTORM_RDMA_CONN_AG_CTX_CF6_MASK                   0x3
+#define TSTORM_RDMA_CONN_AG_CTX_CF6_SHIFT                  2
+#define TSTORM_RDMA_CONN_AG_CTX_CF7_MASK                   0x3
+#define TSTORM_RDMA_CONN_AG_CTX_CF7_SHIFT                  4
+#define TSTORM_RDMA_CONN_AG_CTX_CF8_MASK                   0x3
+#define TSTORM_RDMA_CONN_AG_CTX_CF8_SHIFT                  6
+        u8 flags3;
+#define TSTORM_RDMA_CONN_AG_CTX_CF9_MASK                   0x3
+#define TSTORM_RDMA_CONN_AG_CTX_CF9_SHIFT                  0
+#define TSTORM_RDMA_CONN_AG_CTX_CF10_MASK                  0x3
+#define TSTORM_RDMA_CONN_AG_CTX_CF10_SHIFT                 2
+#define TSTORM_RDMA_CONN_AG_CTX_CF0EN_MASK                 0x1
+#define TSTORM_RDMA_CONN_AG_CTX_CF0EN_SHIFT                4
+#define TSTORM_RDMA_CONN_AG_CTX_CF1EN_MASK                 0x1
+#define TSTORM_RDMA_CONN_AG_CTX_CF1EN_SHIFT                5
+#define TSTORM_RDMA_CONN_AG_CTX_CF2EN_MASK                 0x1
+#define TSTORM_RDMA_CONN_AG_CTX_CF2EN_SHIFT                6
+#define TSTORM_RDMA_CONN_AG_CTX_TIMER_STOP_ALL_CF_EN_MASK  0x1
+#define TSTORM_RDMA_CONN_AG_CTX_TIMER_STOP_ALL_CF_EN_SHIFT 7
+        u8 flags4;
+#define TSTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_EN_MASK        0x1
+#define TSTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_EN_SHIFT       0
+#define TSTORM_RDMA_CONN_AG_CTX_MSTORM_FLUSH_CF_EN_MASK    0x1
+#define TSTORM_RDMA_CONN_AG_CTX_MSTORM_FLUSH_CF_EN_SHIFT   1
+#define TSTORM_RDMA_CONN_AG_CTX_CF6EN_MASK                 0x1
+#define TSTORM_RDMA_CONN_AG_CTX_CF6EN_SHIFT                2
+#define TSTORM_RDMA_CONN_AG_CTX_CF7EN_MASK                 0x1
+#define TSTORM_RDMA_CONN_AG_CTX_CF7EN_SHIFT                3
+#define TSTORM_RDMA_CONN_AG_CTX_CF8EN_MASK                 0x1
+#define TSTORM_RDMA_CONN_AG_CTX_CF8EN_SHIFT                4
+#define TSTORM_RDMA_CONN_AG_CTX_CF9EN_MASK                 0x1
+#define TSTORM_RDMA_CONN_AG_CTX_CF9EN_SHIFT                5
+#define TSTORM_RDMA_CONN_AG_CTX_CF10EN_MASK                0x1
+#define TSTORM_RDMA_CONN_AG_CTX_CF10EN_SHIFT               6
+#define TSTORM_RDMA_CONN_AG_CTX_RULE0EN_MASK               0x1
+#define TSTORM_RDMA_CONN_AG_CTX_RULE0EN_SHIFT              7
+        u8 flags5;
+#define TSTORM_RDMA_CONN_AG_CTX_RULE1EN_MASK               0x1
+#define TSTORM_RDMA_CONN_AG_CTX_RULE1EN_SHIFT              0
+#define TSTORM_RDMA_CONN_AG_CTX_RULE2EN_MASK               0x1
+#define TSTORM_RDMA_CONN_AG_CTX_RULE2EN_SHIFT              1
+#define TSTORM_RDMA_CONN_AG_CTX_RULE3EN_MASK               0x1
+#define TSTORM_RDMA_CONN_AG_CTX_RULE3EN_SHIFT              2
+#define TSTORM_RDMA_CONN_AG_CTX_RULE4EN_MASK               0x1
+#define TSTORM_RDMA_CONN_AG_CTX_RULE4EN_SHIFT              3
+#define TSTORM_RDMA_CONN_AG_CTX_RULE5EN_MASK               0x1
+#define TSTORM_RDMA_CONN_AG_CTX_RULE5EN_SHIFT              4
+#define TSTORM_RDMA_CONN_AG_CTX_RULE6EN_MASK               0x1
+#define TSTORM_RDMA_CONN_AG_CTX_RULE6EN_SHIFT              5
+#define TSTORM_RDMA_CONN_AG_CTX_RULE7EN_MASK               0x1
+#define TSTORM_RDMA_CONN_AG_CTX_RULE7EN_SHIFT              6
+#define TSTORM_RDMA_CONN_AG_CTX_RULE8EN_MASK               0x1
+#define TSTORM_RDMA_CONN_AG_CTX_RULE8EN_SHIFT              7
+        __le32 reg0;
+        __le32 reg1;
+        __le32 reg2;
+        __le32 reg3;
+        __le32 reg4;
+        __le32 reg5;
+        __le32 reg6;
+        __le32 reg7;
+        __le32 reg8;
+        u8 byte2;
+        u8 byte3;
+        __le16 word0;
+        u8 byte4;
+        u8 byte5;
+        __le16 word1;
+        __le16 word2;
+        __le16 word3;
+        __le32 reg9;
+        __le32 reg10;
+};
+
+struct tstorm_rdma_task_ag_ctx {
+        u8 byte0;
+        u8 byte1;
+        __le16 word0;
+        u8 flags0;
+#define TSTORM_RDMA_TASK_AG_CTX_NIBBLE0_MASK  0xF
+#define TSTORM_RDMA_TASK_AG_CTX_NIBBLE0_SHIFT 0
+#define TSTORM_RDMA_TASK_AG_CTX_BIT0_MASK     0x1
+#define TSTORM_RDMA_TASK_AG_CTX_BIT0_SHIFT    4
+#define TSTORM_RDMA_TASK_AG_CTX_BIT1_MASK     0x1
+#define TSTORM_RDMA_TASK_AG_CTX_BIT1_SHIFT    5
+#define TSTORM_RDMA_TASK_AG_CTX_BIT2_MASK     0x1
+#define TSTORM_RDMA_TASK_AG_CTX_BIT2_SHIFT    6
+#define TSTORM_RDMA_TASK_AG_CTX_BIT3_MASK     0x1
+#define TSTORM_RDMA_TASK_AG_CTX_BIT3_SHIFT    7
+        u8 flags1;
+#define TSTORM_RDMA_TASK_AG_CTX_BIT4_MASK     0x1
+#define TSTORM_RDMA_TASK_AG_CTX_BIT4_SHIFT    0
+#define TSTORM_RDMA_TASK_AG_CTX_BIT5_MASK     0x1
+#define TSTORM_RDMA_TASK_AG_CTX_BIT5_SHIFT    1
+#define TSTORM_RDMA_TASK_AG_CTX_CF0_MASK      0x3
+#define TSTORM_RDMA_TASK_AG_CTX_CF0_SHIFT     2
+#define TSTORM_RDMA_TASK_AG_CTX_CF1_MASK      0x3
+#define TSTORM_RDMA_TASK_AG_CTX_CF1_SHIFT     4
+#define TSTORM_RDMA_TASK_AG_CTX_CF2_MASK      0x3
+#define TSTORM_RDMA_TASK_AG_CTX_CF2_SHIFT     6
+        u8 flags2;
+#define TSTORM_RDMA_TASK_AG_CTX_CF3_MASK      0x3
+#define TSTORM_RDMA_TASK_AG_CTX_CF3_SHIFT     0
+#define TSTORM_RDMA_TASK_AG_CTX_CF4_MASK      0x3
+#define TSTORM_RDMA_TASK_AG_CTX_CF4_SHIFT     2
+#define TSTORM_RDMA_TASK_AG_CTX_CF5_MASK      0x3
+#define TSTORM_RDMA_TASK_AG_CTX_CF5_SHIFT     4
+#define TSTORM_RDMA_TASK_AG_CTX_CF6_MASK      0x3
+#define TSTORM_RDMA_TASK_AG_CTX_CF6_SHIFT     6
+        u8 flags3;
+#define TSTORM_RDMA_TASK_AG_CTX_CF7_MASK      0x3
+#define TSTORM_RDMA_TASK_AG_CTX_CF7_SHIFT     0
+#define TSTORM_RDMA_TASK_AG_CTX_CF0EN_MASK    0x1
+#define TSTORM_RDMA_TASK_AG_CTX_CF0EN_SHIFT   2
+#define TSTORM_RDMA_TASK_AG_CTX_CF1EN_MASK    0x1
+#define TSTORM_RDMA_TASK_AG_CTX_CF1EN_SHIFT   3
+#define TSTORM_RDMA_TASK_AG_CTX_CF2EN_MASK    0x1
+#define TSTORM_RDMA_TASK_AG_CTX_CF2EN_SHIFT   4
+#define TSTORM_RDMA_TASK_AG_CTX_CF3EN_MASK    0x1
+#define TSTORM_RDMA_TASK_AG_CTX_CF3EN_SHIFT   5
+#define TSTORM_RDMA_TASK_AG_CTX_CF4EN_MASK    0x1
+#define TSTORM_RDMA_TASK_AG_CTX_CF4EN_SHIFT   6
+#define TSTORM_RDMA_TASK_AG_CTX_CF5EN_MASK    0x1
+#define TSTORM_RDMA_TASK_AG_CTX_CF5EN_SHIFT   7
+        u8 flags4;
+#define TSTORM_RDMA_TASK_AG_CTX_CF6EN_MASK    0x1
+#define TSTORM_RDMA_TASK_AG_CTX_CF6EN_SHIFT   0
+#define TSTORM_RDMA_TASK_AG_CTX_CF7EN_MASK    0x1
+#define TSTORM_RDMA_TASK_AG_CTX_CF7EN_SHIFT   1
+#define TSTORM_RDMA_TASK_AG_CTX_RULE0EN_MASK  0x1
+#define TSTORM_RDMA_TASK_AG_CTX_RULE0EN_SHIFT 2
+#define TSTORM_RDMA_TASK_AG_CTX_RULE1EN_MASK  0x1
+#define TSTORM_RDMA_TASK_AG_CTX_RULE1EN_SHIFT 3
+#define TSTORM_RDMA_TASK_AG_CTX_RULE2EN_MASK  0x1
+#define TSTORM_RDMA_TASK_AG_CTX_RULE2EN_SHIFT 4
+#define TSTORM_RDMA_TASK_AG_CTX_RULE3EN_MASK  0x1
+#define TSTORM_RDMA_TASK_AG_CTX_RULE3EN_SHIFT 5
+#define TSTORM_RDMA_TASK_AG_CTX_RULE4EN_MASK  0x1
+#define TSTORM_RDMA_TASK_AG_CTX_RULE4EN_SHIFT 6
+#define TSTORM_RDMA_TASK_AG_CTX_RULE5EN_MASK  0x1
+#define TSTORM_RDMA_TASK_AG_CTX_RULE5EN_SHIFT 7
+        u8 byte2;
+        __le16 word1;
+        __le32 reg0;
+        u8 byte3;
+        u8 byte4;
+        __le16 word2;
+        __le16 word3;
+        __le16 word4;
+        __le32 reg1;
+        __le32 reg2;
+};
+
+struct ustorm_rdma_conn_ag_ctx {
+        u8 reserved;
+        u8 byte1;
+        u8 flags0;
+#define USTORM_RDMA_CONN_AG_CTX_EXIST_IN_QM0_MASK     0x1
+#define USTORM_RDMA_CONN_AG_CTX_EXIST_IN_QM0_SHIFT    0
+#define USTORM_RDMA_CONN_AG_CTX_BIT1_MASK             0x1
+#define USTORM_RDMA_CONN_AG_CTX_BIT1_SHIFT            1
+#define USTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_MASK      0x3
+#define USTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_SHIFT     2
+#define USTORM_RDMA_CONN_AG_CTX_CF1_MASK              0x3
+#define USTORM_RDMA_CONN_AG_CTX_CF1_SHIFT             4
+#define USTORM_RDMA_CONN_AG_CTX_CF2_MASK              0x3
+#define USTORM_RDMA_CONN_AG_CTX_CF2_SHIFT             6
+        u8 flags1;
+#define USTORM_RDMA_CONN_AG_CTX_CF3_MASK              0x3
+#define USTORM_RDMA_CONN_AG_CTX_CF3_SHIFT             0
+#define USTORM_RDMA_CONN_AG_CTX_CQ_ARM_SE_CF_MASK     0x3
+#define USTORM_RDMA_CONN_AG_CTX_CQ_ARM_SE_CF_SHIFT    2
+#define USTORM_RDMA_CONN_AG_CTX_CQ_ARM_CF_MASK        0x3
+#define USTORM_RDMA_CONN_AG_CTX_CQ_ARM_CF_SHIFT       4
+#define USTORM_RDMA_CONN_AG_CTX_CF6_MASK              0x3
+#define USTORM_RDMA_CONN_AG_CTX_CF6_SHIFT             6
+        u8 flags2;
+#define USTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_EN_MASK   0x1
+#define USTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_EN_SHIFT  0
+#define USTORM_RDMA_CONN_AG_CTX_CF1EN_MASK            0x1
+#define USTORM_RDMA_CONN_AG_CTX_CF1EN_SHIFT           1
+#define USTORM_RDMA_CONN_AG_CTX_CF2EN_MASK            0x1
+#define USTORM_RDMA_CONN_AG_CTX_CF2EN_SHIFT           2
+#define USTORM_RDMA_CONN_AG_CTX_CF3EN_MASK            0x1
+#define USTORM_RDMA_CONN_AG_CTX_CF3EN_SHIFT           3
+#define USTORM_RDMA_CONN_AG_CTX_CQ_ARM_SE_CF_EN_MASK  0x1
+#define USTORM_RDMA_CONN_AG_CTX_CQ_ARM_SE_CF_EN_SHIFT 4
+#define USTORM_RDMA_CONN_AG_CTX_CQ_ARM_CF_EN_MASK     0x1
+#define USTORM_RDMA_CONN_AG_CTX_CQ_ARM_CF_EN_SHIFT    5
+#define USTORM_RDMA_CONN_AG_CTX_CF6EN_MASK            0x1
+#define USTORM_RDMA_CONN_AG_CTX_CF6EN_SHIFT           6
+#define USTORM_RDMA_CONN_AG_CTX_CQ_SE_EN_MASK         0x1
+#define USTORM_RDMA_CONN_AG_CTX_CQ_SE_EN_SHIFT        7
+        u8 flags3;
+#define USTORM_RDMA_CONN_AG_CTX_CQ_EN_MASK            0x1
+#define USTORM_RDMA_CONN_AG_CTX_CQ_EN_SHIFT           0
+#define USTORM_RDMA_CONN_AG_CTX_RULE2EN_MASK          0x1
+#define USTORM_RDMA_CONN_AG_CTX_RULE2EN_SHIFT         1
+#define USTORM_RDMA_CONN_AG_CTX_RULE3EN_MASK          0x1
+#define USTORM_RDMA_CONN_AG_CTX_RULE3EN_SHIFT         2
+#define USTORM_RDMA_CONN_AG_CTX_RULE4EN_MASK          0x1
+#define USTORM_RDMA_CONN_AG_CTX_RULE4EN_SHIFT         3
+#define USTORM_RDMA_CONN_AG_CTX_RULE5EN_MASK          0x1
+#define USTORM_RDMA_CONN_AG_CTX_RULE5EN_SHIFT         4
+#define USTORM_RDMA_CONN_AG_CTX_RULE6EN_MASK          0x1
+#define USTORM_RDMA_CONN_AG_CTX_RULE6EN_SHIFT         5
+#define USTORM_RDMA_CONN_AG_CTX_RULE7EN_MASK          0x1
+#define USTORM_RDMA_CONN_AG_CTX_RULE7EN_SHIFT         6
+#define USTORM_RDMA_CONN_AG_CTX_RULE8EN_MASK          0x1
+#define USTORM_RDMA_CONN_AG_CTX_RULE8EN_SHIFT         7
+        u8 byte2;
+        u8 byte3;
+        __le16 conn_dpi;
+        __le16 word1;
+        __le32 cq_cons;
+        __le32 cq_se_prod;
+        __le32 cq_prod;
+        __le32 reg3;
+        __le16 int_timeout;
+        __le16 word3;
+};
+
+struct xstorm_roce_conn_ag_ctx_dq_ext_ld_part {
+        u8 reserved0;
+        u8 state;
+        u8 flags0;
+#define XSTORMROCECONNAGCTXDQEXTLDPART_EXIST_IN_QM0_MASK      0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_EXIST_IN_QM0_SHIFT     0
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT1_MASK              0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT1_SHIFT             1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT2_MASK              0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT2_SHIFT             2
+#define XSTORMROCECONNAGCTXDQEXTLDPART_EXIST_IN_QM3_MASK      0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_EXIST_IN_QM3_SHIFT     3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT4_MASK              0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT4_SHIFT             4
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT5_MASK              0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT5_SHIFT             5
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT6_MASK              0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT6_SHIFT             6
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT7_MASK              0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT7_SHIFT             7
+        u8 flags1;
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT8_MASK              0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT8_SHIFT             0
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT9_MASK              0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT9_SHIFT             1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT10_MASK             0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT10_SHIFT            2
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT11_MASK             0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT11_SHIFT            3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT12_MASK             0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT12_SHIFT            4
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT13_MASK             0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT13_SHIFT            5
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT14_MASK             0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT14_SHIFT            6
+#define XSTORMROCECONNAGCTXDQEXTLDPART_YSTORM_FLUSH_MASK      0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_YSTORM_FLUSH_SHIFT     7
+        u8 flags2;
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF0_MASK               0x3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF0_SHIFT              0
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF1_MASK               0x3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF1_SHIFT              2
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF2_MASK               0x3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF2_SHIFT              4
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF3_MASK               0x3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF3_SHIFT              6
+        u8 flags3;
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF4_MASK               0x3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF4_SHIFT              0
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF5_MASK               0x3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF5_SHIFT              2
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF6_MASK               0x3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF6_SHIFT              4
+#define XSTORMROCECONNAGCTXDQEXTLDPART_FLUSH_Q0_CF_MASK       0x3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_FLUSH_Q0_CF_SHIFT      6
+        u8 flags4;
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF8_MASK               0x3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF8_SHIFT              0
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF9_MASK               0x3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF9_SHIFT              2
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF10_MASK              0x3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF10_SHIFT             4
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF11_MASK              0x3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF11_SHIFT             6
+        u8 flags5;
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF12_MASK              0x3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF12_SHIFT             0
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF13_MASK              0x3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF13_SHIFT             2
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF14_MASK              0x3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF14_SHIFT             4
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF15_MASK              0x3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF15_SHIFT             6
+        u8 flags6;
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF16_MASK              0x3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF16_SHIFT             0
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF17_MASK              0x3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF17_SHIFT             2
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF18_MASK              0x3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF18_SHIFT             4
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF19_MASK              0x3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF19_SHIFT             6
+        u8 flags7;
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF20_MASK              0x3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF20_SHIFT             0
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF21_MASK              0x3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF21_SHIFT             2
+#define XSTORMROCECONNAGCTXDQEXTLDPART_SLOW_PATH_MASK         0x3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_SLOW_PATH_SHIFT        4
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF0EN_MASK             0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF0EN_SHIFT            6
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF1EN_MASK             0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF1EN_SHIFT            7
+        u8 flags8;
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF2EN_MASK             0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF2EN_SHIFT            0
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF3EN_MASK             0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF3EN_SHIFT            1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF4EN_MASK             0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF4EN_SHIFT            2
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF5EN_MASK             0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF5EN_SHIFT            3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF6EN_MASK             0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF6EN_SHIFT            4
+#define XSTORMROCECONNAGCTXDQEXTLDPART_FLUSH_Q0_CF_EN_MASK    0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_FLUSH_Q0_CF_EN_SHIFT   5
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF8EN_MASK             0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF8EN_SHIFT            6
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF9EN_MASK             0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF9EN_SHIFT            7
+        u8 flags9;
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF10EN_MASK            0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF10EN_SHIFT           0
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF11EN_MASK            0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF11EN_SHIFT           1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF12EN_MASK            0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF12EN_SHIFT           2
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF13EN_MASK            0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF13EN_SHIFT           3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF14EN_MASK            0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF14EN_SHIFT           4
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF15EN_MASK            0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF15EN_SHIFT           5
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF16EN_MASK            0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF16EN_SHIFT           6
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF17EN_MASK            0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF17EN_SHIFT           7
+        u8 flags10;
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF18EN_MASK            0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF18EN_SHIFT           0
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF19EN_MASK            0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF19EN_SHIFT           1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF20EN_MASK            0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF20EN_SHIFT           2
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF21EN_MASK            0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF21EN_SHIFT           3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_SLOW_PATH_EN_MASK      0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_SLOW_PATH_EN_SHIFT     4
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF23EN_MASK            0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF23EN_SHIFT           5
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE0EN_MASK           0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE0EN_SHIFT          6
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE1EN_MASK           0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE1EN_SHIFT          7
+        u8 flags11;
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE2EN_MASK           0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE2EN_SHIFT          0
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE3EN_MASK           0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE3EN_SHIFT          1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE4EN_MASK           0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE4EN_SHIFT          2
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE5EN_MASK           0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE5EN_SHIFT          3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE6EN_MASK           0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE6EN_SHIFT          4
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE7EN_MASK           0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE7EN_SHIFT          5
+#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED1_MASK      0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED1_SHIFT     6
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE9EN_MASK           0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE9EN_SHIFT          7
+        u8 flags12;
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE10EN_MASK          0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE10EN_SHIFT         0
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE11EN_MASK          0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE11EN_SHIFT         1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED2_MASK      0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED2_SHIFT     2
+#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED3_MASK      0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED3_SHIFT     3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE14EN_MASK          0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE14EN_SHIFT         4
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE15EN_MASK          0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE15EN_SHIFT         5
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE16EN_MASK          0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE16EN_SHIFT         6
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE17EN_MASK          0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE17EN_SHIFT         7
+        u8 flags13;
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE18EN_MASK          0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE18EN_SHIFT         0
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE19EN_MASK          0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE19EN_SHIFT         1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED4_MASK      0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED4_SHIFT     2
+#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED5_MASK      0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED5_SHIFT     3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED6_MASK      0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED6_SHIFT     4
+#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED7_MASK      0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED7_SHIFT     5
+#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED8_MASK      0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED8_SHIFT     6
+#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED9_MASK      0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED9_SHIFT     7
+        u8 flags14;
+#define XSTORMROCECONNAGCTXDQEXTLDPART_MIGRATION_MASK         0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_MIGRATION_SHIFT        0
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT17_MASK             0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT17_SHIFT            1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_DPM_PORT_NUM_MASK      0x3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_DPM_PORT_NUM_SHIFT     2
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RESERVED_MASK          0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_RESERVED_SHIFT         4
+#define XSTORMROCECONNAGCTXDQEXTLDPART_ROCE_EDPM_ENABLE_MASK  0x1
+#define XSTORMROCECONNAGCTXDQEXTLDPART_ROCE_EDPM_ENABLE_SHIFT 5
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF23_MASK              0x3
+#define XSTORMROCECONNAGCTXDQEXTLDPART_CF23_SHIFT             6
+        u8 byte2;
+        __le16 physical_q0;
+        __le16 word1;
+        __le16 word2;
+        __le16 word3;
+        __le16 word4;
+        __le16 word5;
+        __le16 conn_dpi;
+        u8 byte3;
+        u8 byte4;
+        u8 byte5;
+        u8 byte6;
+        __le32 reg0;
+        __le32 reg1;
+        __le32 reg2;
+        __le32 snd_nxt_psn;
+        __le32 reg4;
+};
+
+struct xstorm_rdma_conn_ag_ctx {
+        u8 reserved0;
+        u8 state;
+        u8 flags0;
+#define XSTORM_RDMA_CONN_AG_CTX_EXIST_IN_QM0_MASK      0x1
+#define XSTORM_RDMA_CONN_AG_CTX_EXIST_IN_QM0_SHIFT     0
+#define XSTORM_RDMA_CONN_AG_CTX_BIT1_MASK              0x1
+#define XSTORM_RDMA_CONN_AG_CTX_BIT1_SHIFT             1
+#define XSTORM_RDMA_CONN_AG_CTX_BIT2_MASK              0x1
+#define XSTORM_RDMA_CONN_AG_CTX_BIT2_SHIFT             2
+#define XSTORM_RDMA_CONN_AG_CTX_EXIST_IN_QM3_MASK      0x1
+#define XSTORM_RDMA_CONN_AG_CTX_EXIST_IN_QM3_SHIFT     3
+#define XSTORM_RDMA_CONN_AG_CTX_BIT4_MASK              0x1
+#define XSTORM_RDMA_CONN_AG_CTX_BIT4_SHIFT             4
+#define XSTORM_RDMA_CONN_AG_CTX_BIT5_MASK              0x1
+#define XSTORM_RDMA_CONN_AG_CTX_BIT5_SHIFT             5
+#define XSTORM_RDMA_CONN_AG_CTX_BIT6_MASK              0x1
+#define XSTORM_RDMA_CONN_AG_CTX_BIT6_SHIFT             6
+#define XSTORM_RDMA_CONN_AG_CTX_BIT7_MASK              0x1
+#define XSTORM_RDMA_CONN_AG_CTX_BIT7_SHIFT             7
+        u8 flags1;
+#define XSTORM_RDMA_CONN_AG_CTX_BIT8_MASK              0x1
+#define XSTORM_RDMA_CONN_AG_CTX_BIT8_SHIFT             0
+#define XSTORM_RDMA_CONN_AG_CTX_BIT9_MASK              0x1
+#define XSTORM_RDMA_CONN_AG_CTX_BIT9_SHIFT             1
+#define XSTORM_RDMA_CONN_AG_CTX_BIT10_MASK             0x1
+#define XSTORM_RDMA_CONN_AG_CTX_BIT10_SHIFT            2
+#define XSTORM_RDMA_CONN_AG_CTX_BIT11_MASK             0x1
+#define XSTORM_RDMA_CONN_AG_CTX_BIT11_SHIFT            3
+#define XSTORM_RDMA_CONN_AG_CTX_BIT12_MASK             0x1
+#define XSTORM_RDMA_CONN_AG_CTX_BIT12_SHIFT            4
+#define XSTORM_RDMA_CONN_AG_CTX_BIT13_MASK             0x1
+#define XSTORM_RDMA_CONN_AG_CTX_BIT13_SHIFT            5
+#define XSTORM_RDMA_CONN_AG_CTX_BIT14_MASK             0x1
+#define XSTORM_RDMA_CONN_AG_CTX_BIT14_SHIFT            6
+#define XSTORM_RDMA_CONN_AG_CTX_YSTORM_FLUSH_MASK      0x1
+#define XSTORM_RDMA_CONN_AG_CTX_YSTORM_FLUSH_SHIFT     7
+        u8 flags2;
+#define XSTORM_RDMA_CONN_AG_CTX_CF0_MASK               0x3
+#define XSTORM_RDMA_CONN_AG_CTX_CF0_SHIFT              0
+#define XSTORM_RDMA_CONN_AG_CTX_CF1_MASK               0x3
+#define XSTORM_RDMA_CONN_AG_CTX_CF1_SHIFT              2
+#define XSTORM_RDMA_CONN_AG_CTX_CF2_MASK               0x3
+#define XSTORM_RDMA_CONN_AG_CTX_CF2_SHIFT              4
+#define XSTORM_RDMA_CONN_AG_CTX_CF3_MASK               0x3
+#define XSTORM_RDMA_CONN_AG_CTX_CF3_SHIFT              6
+        u8 flags3;
+#define XSTORM_RDMA_CONN_AG_CTX_CF4_MASK               0x3
+#define XSTORM_RDMA_CONN_AG_CTX_CF4_SHIFT              0
+#define XSTORM_RDMA_CONN_AG_CTX_CF5_MASK               0x3
+#define XSTORM_RDMA_CONN_AG_CTX_CF5_SHIFT              2
+#define XSTORM_RDMA_CONN_AG_CTX_CF6_MASK               0x3
+#define XSTORM_RDMA_CONN_AG_CTX_CF6_SHIFT              4
+#define XSTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_MASK       0x3
+#define XSTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_SHIFT      6
+        u8 flags4;
+#define XSTORM_RDMA_CONN_AG_CTX_CF8_MASK               0x3
+#define XSTORM_RDMA_CONN_AG_CTX_CF8_SHIFT              0
+#define XSTORM_RDMA_CONN_AG_CTX_CF9_MASK               0x3
+#define XSTORM_RDMA_CONN_AG_CTX_CF9_SHIFT              2
+#define XSTORM_RDMA_CONN_AG_CTX_CF10_MASK              0x3
+#define XSTORM_RDMA_CONN_AG_CTX_CF10_SHIFT             4
+#define XSTORM_RDMA_CONN_AG_CTX_CF11_MASK              0x3
+#define XSTORM_RDMA_CONN_AG_CTX_CF11_SHIFT             6
+        u8 flags5;
+#define XSTORM_RDMA_CONN_AG_CTX_CF12_MASK              0x3
+#define XSTORM_RDMA_CONN_AG_CTX_CF12_SHIFT             0
+#define XSTORM_RDMA_CONN_AG_CTX_CF13_MASK              0x3
+#define XSTORM_RDMA_CONN_AG_CTX_CF13_SHIFT             2
+#define XSTORM_RDMA_CONN_AG_CTX_CF14_MASK              0x3
+#define XSTORM_RDMA_CONN_AG_CTX_CF14_SHIFT             4
+#define XSTORM_RDMA_CONN_AG_CTX_CF15_MASK              0x3
+#define XSTORM_RDMA_CONN_AG_CTX_CF15_SHIFT             6
+        u8 flags6;
+#define XSTORM_RDMA_CONN_AG_CTX_CF16_MASK              0x3
+#define XSTORM_RDMA_CONN_AG_CTX_CF16_SHIFT             0
+#define XSTORM_RDMA_CONN_AG_CTX_CF17_MASK              0x3
+#define XSTORM_RDMA_CONN_AG_CTX_CF17_SHIFT             2
+#define XSTORM_RDMA_CONN_AG_CTX_CF18_MASK              0x3
+#define XSTORM_RDMA_CONN_AG_CTX_CF18_SHIFT             4
+#define XSTORM_RDMA_CONN_AG_CTX_CF19_MASK              0x3
+#define XSTORM_RDMA_CONN_AG_CTX_CF19_SHIFT             6
+        u8 flags7;
+#define XSTORM_RDMA_CONN_AG_CTX_CF20_MASK              0x3
+#define XSTORM_RDMA_CONN_AG_CTX_CF20_SHIFT             0
+#define XSTORM_RDMA_CONN_AG_CTX_CF21_MASK              0x3
+#define XSTORM_RDMA_CONN_AG_CTX_CF21_SHIFT             2
+#define XSTORM_RDMA_CONN_AG_CTX_SLOW_PATH_MASK         0x3
+#define XSTORM_RDMA_CONN_AG_CTX_SLOW_PATH_SHIFT        4
+#define XSTORM_RDMA_CONN_AG_CTX_CF0EN_MASK             0x1
+#define XSTORM_RDMA_CONN_AG_CTX_CF0EN_SHIFT            6
+#define XSTORM_RDMA_CONN_AG_CTX_CF1EN_MASK             0x1
+#define XSTORM_RDMA_CONN_AG_CTX_CF1EN_SHIFT            7
+        u8 flags8;
+#define XSTORM_RDMA_CONN_AG_CTX_CF2EN_MASK             0x1
+#define XSTORM_RDMA_CONN_AG_CTX_CF2EN_SHIFT            0
+#define XSTORM_RDMA_CONN_AG_CTX_CF3EN_MASK             0x1
+#define XSTORM_RDMA_CONN_AG_CTX_CF3EN_SHIFT            1
+#define XSTORM_RDMA_CONN_AG_CTX_CF4EN_MASK             0x1
+#define XSTORM_RDMA_CONN_AG_CTX_CF4EN_SHIFT            2
+#define XSTORM_RDMA_CONN_AG_CTX_CF5EN_MASK             0x1
+#define XSTORM_RDMA_CONN_AG_CTX_CF5EN_SHIFT            3
+#define XSTORM_RDMA_CONN_AG_CTX_CF6EN_MASK             0x1
+#define XSTORM_RDMA_CONN_AG_CTX_CF6EN_SHIFT            4
+#define XSTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_EN_MASK    0x1
+#define XSTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_EN_SHIFT   5
+#define XSTORM_RDMA_CONN_AG_CTX_CF8EN_MASK             0x1
+#define XSTORM_RDMA_CONN_AG_CTX_CF8EN_SHIFT            6
+#define XSTORM_RDMA_CONN_AG_CTX_CF9EN_MASK             0x1
+#define XSTORM_RDMA_CONN_AG_CTX_CF9EN_SHIFT            7
+        u8 flags9;
+#define XSTORM_RDMA_CONN_AG_CTX_CF10EN_MASK            0x1
+#define XSTORM_RDMA_CONN_AG_CTX_CF10EN_SHIFT           0
+#define XSTORM_RDMA_CONN_AG_CTX_CF11EN_MASK            0x1
+#define XSTORM_RDMA_CONN_AG_CTX_CF11EN_SHIFT           1
+#define XSTORM_RDMA_CONN_AG_CTX_CF12EN_MASK            0x1
+#define XSTORM_RDMA_CONN_AG_CTX_CF12EN_SHIFT           2
+#define XSTORM_RDMA_CONN_AG_CTX_CF13EN_MASK            0x1
+#define XSTORM_RDMA_CONN_AG_CTX_CF13EN_SHIFT           3
+#define XSTORM_RDMA_CONN_AG_CTX_CF14EN_MASK            0x1
+#define XSTORM_RDMA_CONN_AG_CTX_CF14EN_SHIFT           4
+#define XSTORM_RDMA_CONN_AG_CTX_CF15EN_MASK            0x1
+#define XSTORM_RDMA_CONN_AG_CTX_CF15EN_SHIFT           5
+#define XSTORM_RDMA_CONN_AG_CTX_CF16EN_MASK            0x1
+#define XSTORM_RDMA_CONN_AG_CTX_CF16EN_SHIFT           6
+#define XSTORM_RDMA_CONN_AG_CTX_CF17EN_MASK            0x1
+#define XSTORM_RDMA_CONN_AG_CTX_CF17EN_SHIFT           7
+        u8 flags10;
+#define XSTORM_RDMA_CONN_AG_CTX_CF18EN_MASK            0x1
+#define XSTORM_RDMA_CONN_AG_CTX_CF18EN_SHIFT           0
+#define XSTORM_RDMA_CONN_AG_CTX_CF19EN_MASK            0x1
+#define XSTORM_RDMA_CONN_AG_CTX_CF19EN_SHIFT           1
+#define XSTORM_RDMA_CONN_AG_CTX_CF20EN_MASK            0x1
+#define XSTORM_RDMA_CONN_AG_CTX_CF20EN_SHIFT           2
+#define XSTORM_RDMA_CONN_AG_CTX_CF21EN_MASK            0x1
+#define XSTORM_RDMA_CONN_AG_CTX_CF21EN_SHIFT           3
+#define XSTORM_RDMA_CONN_AG_CTX_SLOW_PATH_EN_MASK      0x1
+#define XSTORM_RDMA_CONN_AG_CTX_SLOW_PATH_EN_SHIFT     4
+#define XSTORM_RDMA_CONN_AG_CTX_CF23EN_MASK            0x1
+#define XSTORM_RDMA_CONN_AG_CTX_CF23EN_SHIFT           5
+#define XSTORM_RDMA_CONN_AG_CTX_RULE0EN_MASK           0x1
+#define XSTORM_RDMA_CONN_AG_CTX_RULE0EN_SHIFT          6
+#define XSTORM_RDMA_CONN_AG_CTX_RULE1EN_MASK           0x1
+#define XSTORM_RDMA_CONN_AG_CTX_RULE1EN_SHIFT          7
+        u8 flags11;
+#define XSTORM_RDMA_CONN_AG_CTX_RULE2EN_MASK           0x1
+#define XSTORM_RDMA_CONN_AG_CTX_RULE2EN_SHIFT          0
+#define XSTORM_RDMA_CONN_AG_CTX_RULE3EN_MASK           0x1
+#define XSTORM_RDMA_CONN_AG_CTX_RULE3EN_SHIFT          1
+#define XSTORM_RDMA_CONN_AG_CTX_RULE4EN_MASK           0x1
+#define XSTORM_RDMA_CONN_AG_CTX_RULE4EN_SHIFT          2
+#define XSTORM_RDMA_CONN_AG_CTX_RULE5EN_MASK           0x1
+#define XSTORM_RDMA_CONN_AG_CTX_RULE5EN_SHIFT          3
+#define XSTORM_RDMA_CONN_AG_CTX_RULE6EN_MASK           0x1
+#define XSTORM_RDMA_CONN_AG_CTX_RULE6EN_SHIFT          4
+#define XSTORM_RDMA_CONN_AG_CTX_RULE7EN_MASK           0x1
+#define XSTORM_RDMA_CONN_AG_CTX_RULE7EN_SHIFT          5
+#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED1_MASK      0x1
+#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED1_SHIFT     6
+#define XSTORM_RDMA_CONN_AG_CTX_RULE9EN_MASK           0x1
+#define XSTORM_RDMA_CONN_AG_CTX_RULE9EN_SHIFT          7
+        u8 flags12;
+#define XSTORM_RDMA_CONN_AG_CTX_RULE10EN_MASK          0x1
+#define XSTORM_RDMA_CONN_AG_CTX_RULE10EN_SHIFT         0
+#define XSTORM_RDMA_CONN_AG_CTX_RULE11EN_MASK          0x1
+#define XSTORM_RDMA_CONN_AG_CTX_RULE11EN_SHIFT         1
+#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED2_MASK      0x1
+#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED2_SHIFT     2
+#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED3_MASK      0x1
+#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED3_SHIFT     3
+#define XSTORM_RDMA_CONN_AG_CTX_RULE14EN_MASK          0x1
+#define XSTORM_RDMA_CONN_AG_CTX_RULE14EN_SHIFT         4
+#define XSTORM_RDMA_CONN_AG_CTX_RULE15EN_MASK          0x1
+#define XSTORM_RDMA_CONN_AG_CTX_RULE15EN_SHIFT         5
+#define XSTORM_RDMA_CONN_AG_CTX_RULE16EN_MASK          0x1
+#define XSTORM_RDMA_CONN_AG_CTX_RULE16EN_SHIFT         6
+#define XSTORM_RDMA_CONN_AG_CTX_RULE17EN_MASK          0x1
+#define XSTORM_RDMA_CONN_AG_CTX_RULE17EN_SHIFT         7
+        u8 flags13;
+#define XSTORM_RDMA_CONN_AG_CTX_RULE18EN_MASK          0x1
+#define XSTORM_RDMA_CONN_AG_CTX_RULE18EN_SHIFT         0
+#define XSTORM_RDMA_CONN_AG_CTX_RULE19EN_MASK          0x1
+#define XSTORM_RDMA_CONN_AG_CTX_RULE19EN_SHIFT         1
+#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED4_MASK      0x1
+#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED4_SHIFT     2
+#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED5_MASK      0x1
+#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED5_SHIFT     3
+#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED6_MASK      0x1
+#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED6_SHIFT     4
+#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED7_MASK      0x1
+#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED7_SHIFT     5
+#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED8_MASK      0x1
+#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED8_SHIFT     6
+#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED9_MASK      0x1
+#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED9_SHIFT     7
+        u8 flags14;
+#define XSTORM_RDMA_CONN_AG_CTX_MIGRATION_MASK         0x1
+#define XSTORM_RDMA_CONN_AG_CTX_MIGRATION_SHIFT        0
+#define XSTORM_RDMA_CONN_AG_CTX_BIT17_MASK             0x1
+#define XSTORM_RDMA_CONN_AG_CTX_BIT17_SHIFT            1
+#define XSTORM_RDMA_CONN_AG_CTX_DPM_PORT_NUM_MASK      0x3
+#define XSTORM_RDMA_CONN_AG_CTX_DPM_PORT_NUM_SHIFT     2
+#define XSTORM_RDMA_CONN_AG_CTX_RESERVED_MASK          0x1
+#define XSTORM_RDMA_CONN_AG_CTX_RESERVED_SHIFT         4
+#define XSTORM_RDMA_CONN_AG_CTX_ROCE_EDPM_ENABLE_MASK  0x1
+#define XSTORM_RDMA_CONN_AG_CTX_ROCE_EDPM_ENABLE_SHIFT 5
+#define XSTORM_RDMA_CONN_AG_CTX_CF23_MASK              0x3
+#define XSTORM_RDMA_CONN_AG_CTX_CF23_SHIFT             6
+        u8 byte2;
+        __le16 physical_q0;
+        __le16 word1;
+        __le16 word2;
+        __le16 word3;
+        __le16 word4;
+        __le16 word5;
+        __le16 conn_dpi;
+        u8 byte3;
+        u8 byte4;
+        u8 byte5;
+        u8 byte6;
+        __le32 reg0;
+        __le32 reg1;
+        __le32 reg2;
+        __le32 snd_nxt_psn;
+        __le32 reg4;
+        __le32 reg5;
+        __le32 reg6;
+};
+
+struct ystorm_rdma_conn_ag_ctx {
+        u8 byte0;
+        u8 byte1;
+        u8 flags0;
+#define YSTORM_RDMA_CONN_AG_CTX_BIT0_MASK     0x1
+#define YSTORM_RDMA_CONN_AG_CTX_BIT0_SHIFT    0
+#define YSTORM_RDMA_CONN_AG_CTX_BIT1_MASK     0x1
+#define YSTORM_RDMA_CONN_AG_CTX_BIT1_SHIFT    1
+#define YSTORM_RDMA_CONN_AG_CTX_CF0_MASK      0x3
+#define YSTORM_RDMA_CONN_AG_CTX_CF0_SHIFT     2
+#define YSTORM_RDMA_CONN_AG_CTX_CF1_MASK      0x3
+#define YSTORM_RDMA_CONN_AG_CTX_CF1_SHIFT     4
+#define YSTORM_RDMA_CONN_AG_CTX_CF2_MASK      0x3
+#define YSTORM_RDMA_CONN_AG_CTX_CF2_SHIFT     6
+        u8 flags1;
+#define YSTORM_RDMA_CONN_AG_CTX_CF0EN_MASK    0x1
+#define YSTORM_RDMA_CONN_AG_CTX_CF0EN_SHIFT   0
+#define YSTORM_RDMA_CONN_AG_CTX_CF1EN_MASK    0x1
+#define YSTORM_RDMA_CONN_AG_CTX_CF1EN_SHIFT   1
+#define YSTORM_RDMA_CONN_AG_CTX_CF2EN_MASK    0x1
+#define YSTORM_RDMA_CONN_AG_CTX_CF2EN_SHIFT   2
+#define YSTORM_RDMA_CONN_AG_CTX_RULE0EN_MASK  0x1
+#define YSTORM_RDMA_CONN_AG_CTX_RULE0EN_SHIFT 3
+#define YSTORM_RDMA_CONN_AG_CTX_RULE1EN_MASK  0x1
+#define YSTORM_RDMA_CONN_AG_CTX_RULE1EN_SHIFT 4
+#define YSTORM_RDMA_CONN_AG_CTX_RULE2EN_MASK  0x1
+#define YSTORM_RDMA_CONN_AG_CTX_RULE2EN_SHIFT 5
+#define YSTORM_RDMA_CONN_AG_CTX_RULE3EN_MASK  0x1
+#define YSTORM_RDMA_CONN_AG_CTX_RULE3EN_SHIFT 6
+#define YSTORM_RDMA_CONN_AG_CTX_RULE4EN_MASK  0x1
+#define YSTORM_RDMA_CONN_AG_CTX_RULE4EN_SHIFT 7
+        u8 byte2;
+        u8 byte3;
+        __le16 word0;
+        __le32 reg0;
+        __le32 reg1;
+        __le16 word1;
+        __le16 word2;
+        __le16 word3;
+        __le16 word4;
+        __le32 reg2;
+        __le32 reg3;
+};
+
+struct mstorm_roce_conn_st_ctx {
+        struct regpair temp[6];
+};
+
+struct pstorm_roce_conn_st_ctx {
+        struct regpair temp[16];
+};
+
+struct ystorm_roce_conn_st_ctx {
+        struct regpair temp[2];
+};
+
+struct xstorm_roce_conn_st_ctx {
+        struct regpair temp[22];
+};
+
+struct tstorm_roce_conn_st_ctx {
+        struct regpair temp[30];
+};
+
+struct ustorm_roce_conn_st_ctx {
+        struct regpair temp[12];
+};
+
+struct roce_conn_context {
+        struct ystorm_roce_conn_st_ctx ystorm_st_context;
+        struct regpair ystorm_st_padding[2];
+        struct pstorm_roce_conn_st_ctx pstorm_st_context;
+        struct xstorm_roce_conn_st_ctx xstorm_st_context;
+        struct regpair xstorm_st_padding[2];
+        struct xstorm_rdma_conn_ag_ctx xstorm_ag_context;
+        struct tstorm_rdma_conn_ag_ctx tstorm_ag_context;
+        struct timers_context timer_context;
+        struct ustorm_rdma_conn_ag_ctx ustorm_ag_context;
+        struct tstorm_roce_conn_st_ctx tstorm_st_context;
+        struct mstorm_roce_conn_st_ctx mstorm_st_context;
+        struct ustorm_roce_conn_st_ctx ustorm_st_context;
+        struct regpair ustorm_st_padding[2];
+};
+
+struct roce_create_qp_req_ramrod_data {
+        __le16 flags;
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_ROCE_FLAVOR_MASK          0x3
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_ROCE_FLAVOR_SHIFT         0
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_FMR_AND_RESERVED_EN_MASK  0x1
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_FMR_AND_RESERVED_EN_SHIFT 2
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_SIGNALED_COMP_MASK        0x1
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_SIGNALED_COMP_SHIFT       3
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_PRI_MASK                  0x7
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_PRI_SHIFT                 4
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_RESERVED_MASK             0x1
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_RESERVED_SHIFT            7
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_MASK        0xF
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_SHIFT       8
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_MASK          0xF
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_SHIFT         12
+        u8 max_ord;
+        u8 traffic_class;
+        u8 hop_limit;
+        u8 orq_num_pages;
+        __le16 p_key;
+        __le32 flow_label;
+        __le32 dst_qp_id;
+        __le32 ack_timeout_val;
+        __le32 initial_psn;
+        __le16 mtu;
+        __le16 pd;
+        __le16 sq_num_pages;
+        __le16 reseved2;
+        struct regpair sq_pbl_addr;
+        struct regpair orq_pbl_addr;
+        __le16 local_mac_addr[3];
+        __le16 remote_mac_addr[3];
+        __le16 vlan_id;
+        __le16 udp_src_port;
+        __le32 src_gid[4];
+        __le32 dst_gid[4];
+        struct regpair qp_handle_for_cqe;
+        struct regpair qp_handle_for_async;
+        u8 stats_counter_id;
+        u8 reserved3[7];
+        __le32 cq_cid;
+        __le16 physical_queue0;
+        __le16 dpi;
+};
+
+struct roce_create_qp_resp_ramrod_data {
+        __le16 flags;
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_ROCE_FLAVOR_MASK          0x3
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_ROCE_FLAVOR_SHIFT         0
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_RD_EN_MASK           0x1
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_RD_EN_SHIFT          2
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_WR_EN_MASK           0x1
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_WR_EN_SHIFT          3
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_ATOMIC_EN_MASK            0x1
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_ATOMIC_EN_SHIFT           4
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_SRQ_FLG_MASK              0x1
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_SRQ_FLG_SHIFT             5
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_E2E_FLOW_CONTROL_EN_MASK  0x1
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_E2E_FLOW_CONTROL_EN_SHIFT 6
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_RESERVED0_MASK            0x1
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_RESERVED0_SHIFT           7
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_PRI_MASK                  0x7
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_PRI_SHIFT                 8
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_MASK    0x1F
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_SHIFT   11
+        u8 max_ird;
+        u8 traffic_class;
+        u8 hop_limit;
+        u8 irq_num_pages;
+        __le16 p_key;
+        __le32 flow_label;
+        __le32 dst_qp_id;
+        u8 stats_counter_id;
+        u8 reserved1;
+        __le16 mtu;
+        __le32 initial_psn;
+        __le16 pd;
+        __le16 rq_num_pages;
+        struct rdma_srq_id srq_id;
+        struct regpair rq_pbl_addr;
+        struct regpair irq_pbl_addr;
+        __le16 local_mac_addr[3];
+        __le16 remote_mac_addr[3];
+        __le16 vlan_id;
+        __le16 udp_src_port;
+        __le32 src_gid[4];
+        __le32 dst_gid[4];
+        struct regpair qp_handle_for_cqe;
+        struct regpair qp_handle_for_async;
+        __le32 reserved2[2];
+        __le32 cq_cid;
+        __le16 physical_queue0;
+        __le16 dpi;
+};
+
+struct roce_destroy_qp_req_output_params {
+        __le32 num_bound_mw;
+        __le32 reserved;
+};
+
+struct roce_destroy_qp_req_ramrod_data {
+        struct regpair output_params_addr;
+};
+
+struct roce_destroy_qp_resp_output_params {
+        __le32 num_invalidated_mw;
+        __le32 reserved;
+};
+
+struct roce_destroy_qp_resp_ramrod_data {
+        struct regpair output_params_addr;
+};
+
+enum roce_event_opcode {
+        ROCE_EVENT_CREATE_QP = 11,
+        ROCE_EVENT_MODIFY_QP,
+        ROCE_EVENT_QUERY_QP,
+        ROCE_EVENT_DESTROY_QP,
+        MAX_ROCE_EVENT_OPCODE
+};
+
+struct roce_modify_qp_req_ramrod_data {
+        __le16 flags;
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_ERR_FLG_MASK      0x1
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_ERR_FLG_SHIFT     0
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_SQD_FLG_MASK      0x1
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_SQD_FLG_SHIFT     1
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_EN_SQD_ASYNC_NOTIFY_MASK  0x1
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_EN_SQD_ASYNC_NOTIFY_SHIFT 2
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_P_KEY_FLG_MASK            0x1
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_P_KEY_FLG_SHIFT           3
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_ADDRESS_VECTOR_FLG_MASK   0x1
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_ADDRESS_VECTOR_FLG_SHIFT  4
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_MAX_ORD_FLG_MASK          0x1
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_MAX_ORD_FLG_SHIFT         5
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_FLG_MASK      0x1
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_FLG_SHIFT     6
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_FLG_MASK    0x1
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_FLG_SHIFT   7
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_ACK_TIMEOUT_FLG_MASK      0x1
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_ACK_TIMEOUT_FLG_SHIFT     8
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_PRI_FLG_MASK              0x1
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_PRI_FLG_SHIFT             9
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_PRI_MASK                  0x7
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_PRI_SHIFT                 10
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_RESERVED1_MASK            0x7
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_RESERVED1_SHIFT           13
+        u8 fields;
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_MASK        0xF
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_SHIFT       0
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_MASK          0xF
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_SHIFT         4
+        u8 max_ord;
+        u8 traffic_class;
+        u8 hop_limit;
+        __le16 p_key;
+        __le32 flow_label;
+        __le32 ack_timeout_val;
+        __le16 mtu;
+        __le16 reserved2;
+        __le32 reserved3[3];
+        __le32 src_gid[4];
+        __le32 dst_gid[4];
+};
+
+struct roce_modify_qp_resp_ramrod_data {
+        __le16 flags;
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_MOVE_TO_ERR_FLG_MASK        0x1
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_MOVE_TO_ERR_FLG_SHIFT       0
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_RD_EN_MASK             0x1
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_RD_EN_SHIFT            1
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_WR_EN_MASK             0x1
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_WR_EN_SHIFT            2
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_ATOMIC_EN_MASK              0x1
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_ATOMIC_EN_SHIFT             3
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_P_KEY_FLG_MASK              0x1
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_P_KEY_FLG_SHIFT             4
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_ADDRESS_VECTOR_FLG_MASK     0x1
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_ADDRESS_VECTOR_FLG_SHIFT    5
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_MAX_IRD_FLG_MASK            0x1
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_MAX_IRD_FLG_SHIFT           6
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_PRI_FLG_MASK                0x1
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_PRI_FLG_SHIFT               7
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_FLG_MASK  0x1
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_FLG_SHIFT 8
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_OPS_EN_FLG_MASK        0x1
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_OPS_EN_FLG_SHIFT       9
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_RESERVED1_MASK              0x3F
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_RESERVED1_SHIFT             10
+        u8 fields;
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_PRI_MASK                    0x7
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_PRI_SHIFT                   0
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_MASK      0x1F
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_SHIFT     3
+        u8 max_ird;
+        u8 traffic_class;
+        u8 hop_limit;
+        __le16 p_key;
+        __le32 flow_label;
+        __le16 mtu;
+        __le16 reserved2;
+        __le32 src_gid[4];
+        __le32 dst_gid[4];
+};
+
+struct roce_query_qp_req_output_params {
+        __le32 psn;
+        __le32 flags;
+#define ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_ERR_FLG_MASK          0x1
+#define ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_ERR_FLG_SHIFT         0
+#define ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_SQ_DRAINING_FLG_MASK  0x1
+#define ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_SQ_DRAINING_FLG_SHIFT 1
+#define ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_RESERVED0_MASK        0x3FFFFFFF
+#define ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_RESERVED0_SHIFT       2
+};
+
+struct roce_query_qp_req_ramrod_data {
+        struct regpair output_params_addr;
+};
+
+struct roce_query_qp_resp_output_params {
+        __le32 psn;
+        __le32 err_flag;
+#define ROCE_QUERY_QP_RESP_OUTPUT_PARAMS_ERROR_FLG_MASK  0x1
+#define ROCE_QUERY_QP_RESP_OUTPUT_PARAMS_ERROR_FLG_SHIFT 0
+#define ROCE_QUERY_QP_RESP_OUTPUT_PARAMS_RESERVED0_MASK  0x7FFFFFFF
+#define ROCE_QUERY_QP_RESP_OUTPUT_PARAMS_RESERVED0_SHIFT 1
+};
+
+struct roce_query_qp_resp_ramrod_data {
+        struct regpair output_params_addr;
+};
+
+enum roce_ramrod_cmd_id {
+        ROCE_RAMROD_CREATE_QP = 11,
+        ROCE_RAMROD_MODIFY_QP,
+        ROCE_RAMROD_QUERY_QP,
+        ROCE_RAMROD_DESTROY_QP,
+        MAX_ROCE_RAMROD_CMD_ID
+};
+
+struct mstorm_roce_req_conn_ag_ctx {
+        u8 byte0;
+        u8 byte1;
+        u8 flags0;
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_BIT0_MASK     0x1
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_BIT0_SHIFT    0
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_BIT1_MASK     0x1
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_BIT1_SHIFT    1
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_CF0_MASK      0x3
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_CF0_SHIFT     2
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_CF1_MASK      0x3
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_CF1_SHIFT     4
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_CF2_MASK      0x3
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_CF2_SHIFT     6
+        u8 flags1;
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_CF0EN_MASK    0x1
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_CF0EN_SHIFT   0
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_MASK    0x1
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_SHIFT   1
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_CF2EN_MASK    0x1
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_CF2EN_SHIFT   2
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_MASK  0x1
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_SHIFT 3
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_MASK  0x1
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_SHIFT 4
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_MASK  0x1
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_SHIFT 5
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_MASK  0x1
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_SHIFT 6
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_MASK  0x1
+#define MSTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_SHIFT 7
+        __le16 word0;
+        __le16 word1;
+        __le32 reg0;
+        __le32 reg1;
+};
+
+struct mstorm_roce_resp_conn_ag_ctx {
+        u8 byte0;
+        u8 byte1;
+        u8 flags0;
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_BIT0_MASK     0x1
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_BIT0_SHIFT    0
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_BIT1_MASK     0x1
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_BIT1_SHIFT    1
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_CF0_MASK      0x3
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_CF0_SHIFT     2
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_CF1_MASK      0x3
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_CF1_SHIFT     4
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_CF2_MASK      0x3
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_CF2_SHIFT     6
+        u8 flags1;
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_MASK    0x1
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_SHIFT   0
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_CF1EN_MASK    0x1
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_CF1EN_SHIFT   1
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_CF2EN_MASK    0x1
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_CF2EN_SHIFT   2
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_MASK  0x1
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_SHIFT 3
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_MASK  0x1
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_SHIFT 4
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_MASK  0x1
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_SHIFT 5
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_MASK  0x1
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_SHIFT 6
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_MASK  0x1
+#define MSTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_SHIFT 7
+        __le16 word0;
+        __le16 word1;
+        __le32 reg0;
+        __le32 reg1;
+};
+
+enum roce_flavor {
+        PLAIN_ROCE /* RoCE v1 */ ,
+        RROCE_IPV4 /* RoCE v2 (Routable RoCE) over ipv4 */ ,
+        RROCE_IPV6 /* RoCE v2 (Routable RoCE) over ipv6 */ ,
+        MAX_ROCE_FLAVOR
+};
+
+struct tstorm_roce_req_conn_ag_ctx {
+        u8 reserved0;
+        u8 state;
+        u8 flags0;
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_EXIST_IN_QM0_MASK                0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_EXIST_IN_QM0_SHIFT               0
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_RX_ERROR_OCCURED_MASK            0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_RX_ERROR_OCCURED_SHIFT           1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_TX_CQE_ERROR_OCCURED_MASK        0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_TX_CQE_ERROR_OCCURED_SHIFT       2
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_BIT3_MASK                        0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_BIT3_SHIFT                       3
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_MSTORM_FLUSH_MASK                0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_MSTORM_FLUSH_SHIFT               4
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_CACHED_ORQ_MASK                  0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_CACHED_ORQ_SHIFT                 5
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_TIMER_CF_MASK                    0x3
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_TIMER_CF_SHIFT                   6
+        u8 flags1;
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_CF1_MASK                         0x3
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_CF1_SHIFT                        0
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_SQ_CF_MASK                 0x3
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_SQ_CF_SHIFT                2
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_TIMER_STOP_ALL_CF_MASK           0x3
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_TIMER_STOP_ALL_CF_SHIFT          4
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_Q0_CF_MASK                 0x3
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_Q0_CF_SHIFT                6
+        u8 flags2;
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_MSTORM_FLUSH_CF_MASK             0x3
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_MSTORM_FLUSH_CF_SHIFT            0
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_SET_TIMER_CF_MASK                0x3
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_SET_TIMER_CF_SHIFT               2
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_TX_ASYNC_ERROR_CF_MASK           0x3
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_TX_ASYNC_ERROR_CF_SHIFT          4
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_RXMIT_DONE_CF_MASK               0x3
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_RXMIT_DONE_CF_SHIFT              6
+        u8 flags3;
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_ERROR_SCAN_COMPLETED_CF_MASK     0x3
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_ERROR_SCAN_COMPLETED_CF_SHIFT    0
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_SQ_DRAIN_COMPLETED_CF_MASK       0x3
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_SQ_DRAIN_COMPLETED_CF_SHIFT      2
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_TIMER_CF_EN_MASK                 0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_TIMER_CF_EN_SHIFT                4
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_MASK                       0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_SHIFT                      5
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_SQ_CF_EN_MASK              0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_SQ_CF_EN_SHIFT             6
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_TIMER_STOP_ALL_CF_EN_MASK        0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_TIMER_STOP_ALL_CF_EN_SHIFT       7
+        u8 flags4;
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_Q0_CF_EN_MASK              0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_Q0_CF_EN_SHIFT             0
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_MSTORM_FLUSH_CF_EN_MASK          0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_MSTORM_FLUSH_CF_EN_SHIFT         1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_SET_TIMER_CF_EN_MASK             0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_SET_TIMER_CF_EN_SHIFT            2
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_TX_ASYNC_ERROR_CF_EN_MASK        0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_TX_ASYNC_ERROR_CF_EN_SHIFT       3
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_RXMIT_DONE_CF_EN_MASK            0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_RXMIT_DONE_CF_EN_SHIFT           4
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_ERROR_SCAN_COMPLETED_CF_EN_MASK  0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_ERROR_SCAN_COMPLETED_CF_EN_SHIFT 5
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_SQ_DRAIN_COMPLETED_CF_EN_MASK    0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_SQ_DRAIN_COMPLETED_CF_EN_SHIFT   6
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_MASK                     0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_SHIFT                    7
+        u8 flags5;
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_MASK                     0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_SHIFT                    0
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_MASK                     0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_SHIFT                    1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_MASK                     0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_SHIFT                    2
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_MASK                     0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_SHIFT                    3
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE5EN_MASK                     0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE5EN_SHIFT                    4
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_SND_SQ_CONS_EN_MASK              0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_SND_SQ_CONS_EN_SHIFT             5
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE7EN_MASK                     0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE7EN_SHIFT                    6
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE8EN_MASK                     0x1
+#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE8EN_SHIFT                    7
+        __le32 reg0;
+        __le32 snd_nxt_psn;
+        __le32 snd_max_psn;
+        __le32 orq_prod;
+        __le32 reg4;
+        __le32 reg5;
+        __le32 reg6;
+        __le32 reg7;
+        __le32 reg8;
+        u8 tx_cqe_error_type;
+        u8 orq_cache_idx;
+        __le16 snd_sq_cons_th;
+        u8 byte4;
+        u8 byte5;
+        __le16 snd_sq_cons;
+        __le16 word2;
+        __le16 word3;
+        __le32 reg9;
+        __le32 reg10;
+};
+
+struct tstorm_roce_resp_conn_ag_ctx {
+        u8 byte0;
+        u8 state;
+        u8 flags0;
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_EXIST_IN_QM0_MASK        0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_EXIST_IN_QM0_SHIFT       0
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_BIT1_MASK                0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_BIT1_SHIFT               1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_BIT2_MASK                0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_BIT2_SHIFT               2
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_BIT3_MASK                0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_BIT3_SHIFT               3
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_MSTORM_FLUSH_MASK        0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_MSTORM_FLUSH_SHIFT       4
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_BIT5_MASK                0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_BIT5_SHIFT               5
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF0_MASK                 0x3
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF0_SHIFT                6
+        u8 flags1;
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_CF_MASK         0x3
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_CF_SHIFT        0
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_TX_ERROR_CF_MASK         0x3
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_TX_ERROR_CF_SHIFT        2
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF3_MASK                 0x3
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF3_SHIFT                4
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_FLUSH_Q0_CF_MASK         0x3
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_FLUSH_Q0_CF_SHIFT        6
+        u8 flags2;
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_MSTORM_FLUSH_CF_MASK     0x3
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_MSTORM_FLUSH_CF_SHIFT    0
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF6_MASK                 0x3
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF6_SHIFT                2
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF7_MASK                 0x3
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF7_SHIFT                4
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF8_MASK                 0x3
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF8_SHIFT                6
+        u8 flags3;
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF9_MASK                 0x3
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF9_SHIFT                0
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF10_MASK                0x3
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF10_SHIFT               2
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_MASK               0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_SHIFT              4
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_CF_EN_MASK      0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_CF_EN_SHIFT     5
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_TX_ERROR_CF_EN_MASK      0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_TX_ERROR_CF_EN_SHIFT     6
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF3EN_MASK               0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF3EN_SHIFT              7
+        u8 flags4;
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_FLUSH_Q0_CF_EN_MASK      0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_FLUSH_Q0_CF_EN_SHIFT     0
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_MSTORM_FLUSH_CF_EN_MASK  0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_MSTORM_FLUSH_CF_EN_SHIFT 1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF6EN_MASK               0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF6EN_SHIFT              2
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF7EN_MASK               0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF7EN_SHIFT              3
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF8EN_MASK               0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF8EN_SHIFT              4
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF9EN_MASK               0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF9EN_SHIFT              5
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF10EN_MASK              0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF10EN_SHIFT             6
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_MASK             0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_SHIFT            7
+        u8 flags5;
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_MASK             0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_SHIFT            0
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_MASK             0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_SHIFT            1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_MASK             0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_SHIFT            2
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_MASK             0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_SHIFT            3
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE5EN_MASK             0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE5EN_SHIFT            4
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_RQ_RULE_EN_MASK          0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_RQ_RULE_EN_SHIFT         5
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE7EN_MASK             0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE7EN_SHIFT            6
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE8EN_MASK             0x1
+#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE8EN_SHIFT            7
+        __le32 psn_and_rxmit_id_echo;
+        __le32 reg1;
+        __le32 reg2;
+        __le32 reg3;
+        __le32 reg4;
+        __le32 reg5;
+        __le32 reg6;
+        __le32 reg7;
+        __le32 reg8;
+        u8 tx_async_error_type;
+        u8 byte3;
+        __le16 rq_cons;
+        u8 byte4;
+        u8 byte5;
+        __le16 rq_prod;
+        __le16 conn_dpi;
+        __le16 irq_cons;
+        __le32 num_invlidated_mw;
+        __le32 reg10;
+};
+
+struct ustorm_roce_req_conn_ag_ctx {
+        u8 byte0;
+        u8 byte1;
+        u8 flags0;
+#define USTORM_ROCE_REQ_CONN_AG_CTX_BIT0_MASK     0x1
+#define USTORM_ROCE_REQ_CONN_AG_CTX_BIT0_SHIFT    0
+#define USTORM_ROCE_REQ_CONN_AG_CTX_BIT1_MASK     0x1
+#define USTORM_ROCE_REQ_CONN_AG_CTX_BIT1_SHIFT    1
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF0_MASK      0x3
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF0_SHIFT     2
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF1_MASK      0x3
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF1_SHIFT     4
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF2_MASK      0x3
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF2_SHIFT     6
+        u8 flags1;
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF3_MASK      0x3
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF3_SHIFT     0
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF4_MASK      0x3
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF4_SHIFT     2
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF5_MASK      0x3
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF5_SHIFT     4
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF6_MASK      0x3
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF6_SHIFT     6
+        u8 flags2;
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF0EN_MASK    0x1
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF0EN_SHIFT   0
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_MASK    0x1
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_SHIFT   1
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF2EN_MASK    0x1
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF2EN_SHIFT   2
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF3EN_MASK    0x1
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF3EN_SHIFT   3
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF4EN_MASK    0x1
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF4EN_SHIFT   4
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF5EN_MASK    0x1
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF5EN_SHIFT   5
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF6EN_MASK    0x1
+#define USTORM_ROCE_REQ_CONN_AG_CTX_CF6EN_SHIFT   6
+#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_MASK  0x1
+#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_SHIFT 7
+        u8 flags3;
+#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_MASK  0x1
+#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_SHIFT 0
+#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_MASK  0x1
+#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_SHIFT 1
+#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_MASK  0x1
+#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_SHIFT 2
+#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_MASK  0x1
+#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_SHIFT 3
+#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE5EN_MASK  0x1
+#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE5EN_SHIFT 4
+#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE6EN_MASK  0x1
+#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE6EN_SHIFT 5
+#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE7EN_MASK  0x1
+#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE7EN_SHIFT 6
+#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE8EN_MASK  0x1
+#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE8EN_SHIFT 7
+        u8 byte2;
+        u8 byte3;
+        __le16 word0;
+        __le16 word1;
+        __le32 reg0;
+        __le32 reg1;
+        __le32 reg2;
+        __le32 reg3;
+        __le16 word2;
+        __le16 word3;
+};
+
+struct ustorm_roce_resp_conn_ag_ctx {
+        u8 byte0;
+        u8 byte1;
+        u8 flags0;
+#define USTORM_ROCE_RESP_CONN_AG_CTX_BIT0_MASK     0x1
+#define USTORM_ROCE_RESP_CONN_AG_CTX_BIT0_SHIFT    0
+#define USTORM_ROCE_RESP_CONN_AG_CTX_BIT1_MASK     0x1
+#define USTORM_ROCE_RESP_CONN_AG_CTX_BIT1_SHIFT    1
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF0_MASK      0x3
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF0_SHIFT     2
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF1_MASK      0x3
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF1_SHIFT     4
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF2_MASK      0x3
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF2_SHIFT     6
+        u8 flags1;
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF3_MASK      0x3
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF3_SHIFT     0
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF4_MASK      0x3
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF4_SHIFT     2
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF5_MASK      0x3
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF5_SHIFT     4
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF6_MASK      0x3
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF6_SHIFT     6
+        u8 flags2;
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_MASK    0x1
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_SHIFT   0
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF1EN_MASK    0x1
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF1EN_SHIFT   1
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF2EN_MASK    0x1
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF2EN_SHIFT   2
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF3EN_MASK    0x1
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF3EN_SHIFT   3
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF4EN_MASK    0x1
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF4EN_SHIFT   4
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF5EN_MASK    0x1
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF5EN_SHIFT   5
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF6EN_MASK    0x1
+#define USTORM_ROCE_RESP_CONN_AG_CTX_CF6EN_SHIFT   6
+#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_MASK  0x1
+#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_SHIFT 7
+        u8 flags3;
+#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_MASK  0x1
+#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_SHIFT 0
+#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_MASK  0x1
+#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_SHIFT 1
+#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_MASK  0x1
+#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_SHIFT 2
+#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_MASK  0x1
+#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_SHIFT 3
+#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE5EN_MASK  0x1
+#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE5EN_SHIFT 4
+#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE6EN_MASK  0x1
+#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE6EN_SHIFT 5
+#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE7EN_MASK  0x1
+#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE7EN_SHIFT 6
+#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE8EN_MASK  0x1
+#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE8EN_SHIFT 7
+        u8 byte2;
+        u8 byte3;
+        __le16 word0;
+        __le16 word1;
+        __le32 reg0;
+        __le32 reg1;
+        __le32 reg2;
+        __le32 reg3;
+        __le16 word2;
+        __le16 word3;
+};
+
+struct xstorm_roce_req_conn_ag_ctx {
+        u8 reserved0;
+        u8 state;
+        u8 flags0;
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_EXIST_IN_QM0_MASK        0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_EXIST_IN_QM0_SHIFT       0
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED1_MASK           0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED1_SHIFT          1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED2_MASK           0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED2_SHIFT          2
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_EXIST_IN_QM3_MASK        0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_EXIST_IN_QM3_SHIFT       3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED3_MASK           0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED3_SHIFT          4
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED4_MASK           0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED4_SHIFT          5
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED5_MASK           0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED5_SHIFT          6
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED6_MASK           0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED6_SHIFT          7
+        u8 flags1;
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED7_MASK           0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED7_SHIFT          0
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED8_MASK           0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED8_SHIFT          1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_BIT10_MASK               0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_BIT10_SHIFT              2
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_BIT11_MASK               0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_BIT11_SHIFT              3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_BIT12_MASK               0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_BIT12_SHIFT              4
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_BIT13_MASK               0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_BIT13_SHIFT              5
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_ERROR_STATE_MASK         0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_ERROR_STATE_SHIFT        6
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_YSTORM_FLUSH_MASK        0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_YSTORM_FLUSH_SHIFT       7
+        u8 flags2;
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF0_MASK                 0x3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF0_SHIFT                0
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF1_MASK                 0x3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF1_SHIFT                2
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF2_MASK                 0x3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF2_SHIFT                4
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF3_MASK                 0x3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF3_SHIFT                6
+        u8 flags3;
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_SQ_FLUSH_CF_MASK         0x3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_SQ_FLUSH_CF_SHIFT        0
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RX_ERROR_CF_MASK         0x3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RX_ERROR_CF_SHIFT        2
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_SND_RXMIT_CF_MASK        0x3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_SND_RXMIT_CF_SHIFT       4
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_Q0_CF_MASK         0x3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_Q0_CF_SHIFT        6
+        u8 flags4;
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF8_MASK                 0x3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF8_SHIFT                0
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF9_MASK                 0x3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF9_SHIFT                2
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF10_MASK                0x3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF10_SHIFT               4
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF11_MASK                0x3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF11_SHIFT               6
+        u8 flags5;
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF12_MASK                0x3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF12_SHIFT               0
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF13_MASK                0x3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF13_SHIFT               2
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_FMR_ENDED_CF_MASK        0x3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_FMR_ENDED_CF_SHIFT       4
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF15_MASK                0x3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF15_SHIFT               6
+        u8 flags6;
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF16_MASK                0x3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF16_SHIFT               0
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF17_MASK                0x3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF17_SHIFT               2
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF18_MASK                0x3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF18_SHIFT               4
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF19_MASK                0x3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF19_SHIFT               6
+        u8 flags7;
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF20_MASK                0x3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF20_SHIFT               0
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF21_MASK                0x3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF21_SHIFT               2
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_SLOW_PATH_MASK           0x3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_SLOW_PATH_SHIFT          4
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF0EN_MASK               0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF0EN_SHIFT              6
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_MASK               0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_SHIFT              7
+        u8 flags8;
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF2EN_MASK               0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF2EN_SHIFT              0
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF3EN_MASK               0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF3EN_SHIFT              1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_SQ_FLUSH_CF_EN_MASK      0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_SQ_FLUSH_CF_EN_SHIFT     2
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RX_ERROR_CF_EN_MASK      0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RX_ERROR_CF_EN_SHIFT     3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_SND_RXMIT_CF_EN_MASK     0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_SND_RXMIT_CF_EN_SHIFT    4
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_Q0_CF_EN_MASK      0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_Q0_CF_EN_SHIFT     5
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF8EN_MASK               0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF8EN_SHIFT              6
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF9EN_MASK               0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF9EN_SHIFT              7
+        u8 flags9;
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF10EN_MASK              0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF10EN_SHIFT             0
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF11EN_MASK              0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF11EN_SHIFT             1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF12EN_MASK              0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF12EN_SHIFT             2
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF13EN_MASK              0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF13EN_SHIFT             3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_FME_ENDED_CF_EN_MASK     0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_FME_ENDED_CF_EN_SHIFT    4
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF15EN_MASK              0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF15EN_SHIFT             5
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF16EN_MASK              0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF16EN_SHIFT             6
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF17EN_MASK              0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF17EN_SHIFT             7
+        u8 flags10;
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF18EN_MASK              0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF18EN_SHIFT             0
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF19EN_MASK              0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF19EN_SHIFT             1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF20EN_MASK              0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF20EN_SHIFT             2
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF21EN_MASK              0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF21EN_SHIFT             3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_SLOW_PATH_EN_MASK        0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_SLOW_PATH_EN_SHIFT       4
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF23EN_MASK              0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF23EN_SHIFT             5
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_MASK             0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_SHIFT            6
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_MASK             0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_SHIFT            7
+        u8 flags11;
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_MASK             0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_SHIFT            0
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_MASK             0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_SHIFT            1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_MASK             0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_SHIFT            2
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE5EN_MASK             0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE5EN_SHIFT            3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE6EN_MASK             0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE6EN_SHIFT            4
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_E2E_CREDIT_RULE_EN_MASK  0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_E2E_CREDIT_RULE_EN_SHIFT 5
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED1_MASK        0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED1_SHIFT       6
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE9EN_MASK             0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE9EN_SHIFT            7
+        u8 flags12;
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_SQ_PROD_EN_MASK          0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_SQ_PROD_EN_SHIFT         0
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE11EN_MASK            0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE11EN_SHIFT           1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED2_MASK        0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED2_SHIFT       2
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED3_MASK        0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED3_SHIFT       3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_INV_FENCE_RULE_EN_MASK   0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_INV_FENCE_RULE_EN_SHIFT  4
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE15EN_MASK            0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE15EN_SHIFT           5
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_ORQ_FENCE_RULE_EN_MASK   0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_ORQ_FENCE_RULE_EN_SHIFT  6
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_MAX_ORD_RULE_EN_MASK     0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_MAX_ORD_RULE_EN_SHIFT    7
+        u8 flags13;
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE18EN_MASK            0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE18EN_SHIFT           0
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE19EN_MASK            0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE19EN_SHIFT           1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED4_MASK        0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED4_SHIFT       2
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED5_MASK        0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED5_SHIFT       3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED6_MASK        0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED6_SHIFT       4
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED7_MASK        0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED7_SHIFT       5
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED8_MASK        0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED8_SHIFT       6
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED9_MASK        0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED9_SHIFT       7
+        u8 flags14;
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_MIGRATION_FLAG_MASK      0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_MIGRATION_FLAG_SHIFT     0
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_BIT17_MASK               0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_BIT17_SHIFT              1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_DPM_PORT_NUM_MASK        0x3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_DPM_PORT_NUM_SHIFT       2
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED_MASK            0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED_SHIFT           4
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_ROCE_EDPM_ENABLE_MASK    0x1
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_ROCE_EDPM_ENABLE_SHIFT   5
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF23_MASK                0x3
+#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF23_SHIFT               6
+        u8 byte2;
+        __le16 physical_q0;
+        __le16 word1;
+        __le16 sq_cmp_cons;
+        __le16 sq_cons;
+        __le16 sq_prod;
+        __le16 word5;
+        __le16 conn_dpi;
+        u8 byte3;
+        u8 byte4;
+        u8 byte5;
+        u8 byte6;
+        __le32 lsn;
+        __le32 ssn;
+        __le32 snd_una_psn;
+        __le32 snd_nxt_psn;
+        __le32 reg4;
+        __le32 orq_cons_th;
+        __le32 orq_cons;
+};
+
+struct xstorm_roce_resp_conn_ag_ctx {
+        u8 reserved0;
+        u8 state;
+        u8 flags0;
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_EXIST_IN_QM0_MASK      0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_EXIST_IN_QM0_SHIFT     0
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED1_MASK         0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED1_SHIFT        1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED2_MASK         0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED2_SHIFT        2
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_EXIST_IN_QM3_MASK      0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_EXIST_IN_QM3_SHIFT     3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED3_MASK         0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED3_SHIFT        4
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED4_MASK         0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED4_SHIFT        5
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED5_MASK         0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED5_SHIFT        6
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED6_MASK         0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED6_SHIFT        7
+        u8 flags1;
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED7_MASK         0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED7_SHIFT        0
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED8_MASK         0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED8_SHIFT        1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT10_MASK             0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT10_SHIFT            2
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT11_MASK             0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT11_SHIFT            3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT12_MASK             0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT12_SHIFT            4
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT13_MASK             0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT13_SHIFT            5
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_ERROR_STATE_MASK       0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_ERROR_STATE_SHIFT      6
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_YSTORM_FLUSH_MASK      0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_YSTORM_FLUSH_SHIFT     7
+        u8 flags2;
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF0_MASK               0x3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF0_SHIFT              0
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF1_MASK               0x3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF1_SHIFT              2
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF2_MASK               0x3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF2_SHIFT              4
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF3_MASK               0x3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF3_SHIFT              6
+        u8 flags3;
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RXMIT_CF_MASK          0x3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RXMIT_CF_SHIFT         0
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_CF_MASK       0x3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_CF_SHIFT      2
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_FORCE_ACK_CF_MASK      0x3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_FORCE_ACK_CF_SHIFT     4
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_FLUSH_Q0_CF_MASK       0x3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_FLUSH_Q0_CF_SHIFT      6
+        u8 flags4;
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF8_MASK               0x3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF8_SHIFT              0
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF9_MASK               0x3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF9_SHIFT              2
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF10_MASK              0x3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF10_SHIFT             4
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF11_MASK              0x3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF11_SHIFT             6
+        u8 flags5;
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF12_MASK              0x3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF12_SHIFT             0
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF13_MASK              0x3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF13_SHIFT             2
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF14_MASK              0x3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF14_SHIFT             4
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF15_MASK              0x3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF15_SHIFT             6
+        u8 flags6;
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF16_MASK              0x3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF16_SHIFT             0
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF17_MASK              0x3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF17_SHIFT             2
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF18_MASK              0x3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF18_SHIFT             4
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF19_MASK              0x3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF19_SHIFT             6
+        u8 flags7;
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF20_MASK              0x3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF20_SHIFT             0
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF21_MASK              0x3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF21_SHIFT             2
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_SLOW_PATH_MASK         0x3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_SLOW_PATH_SHIFT        4
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_MASK             0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_SHIFT            6
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF1EN_MASK             0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF1EN_SHIFT            7
+        u8 flags8;
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF2EN_MASK             0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF2EN_SHIFT            0
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF3EN_MASK             0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF3EN_SHIFT            1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RXMIT_CF_EN_MASK       0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RXMIT_CF_EN_SHIFT      2
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_CF_EN_MASK    0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_CF_EN_SHIFT   3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_FORCE_ACK_CF_EN_MASK   0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_FORCE_ACK_CF_EN_SHIFT  4
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_FLUSH_Q0_CF_EN_MASK    0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_FLUSH_Q0_CF_EN_SHIFT   5
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF8EN_MASK             0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF8EN_SHIFT            6
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF9EN_MASK             0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF9EN_SHIFT            7
+        u8 flags9;
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF10EN_MASK            0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF10EN_SHIFT           0
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF11EN_MASK            0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF11EN_SHIFT           1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF12EN_MASK            0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF12EN_SHIFT           2
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF13EN_MASK            0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF13EN_SHIFT           3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF14EN_MASK            0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF14EN_SHIFT           4
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF15EN_MASK            0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF15EN_SHIFT           5
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF16EN_MASK            0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF16EN_SHIFT           6
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF17EN_MASK            0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF17EN_SHIFT           7
+        u8 flags10;
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF18EN_MASK            0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF18EN_SHIFT           0
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF19EN_MASK            0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF19EN_SHIFT           1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF20EN_MASK            0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF20EN_SHIFT           2
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF21EN_MASK            0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF21EN_SHIFT           3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_SLOW_PATH_EN_MASK      0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_SLOW_PATH_EN_SHIFT     4
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF23EN_MASK            0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF23EN_SHIFT           5
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_MASK           0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_SHIFT          6
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_MASK           0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_SHIFT          7
+        u8 flags11;
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_MASK           0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_SHIFT          0
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_MASK           0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_SHIFT          1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_MASK           0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_SHIFT          2
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE5EN_MASK           0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE5EN_SHIFT          3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE6EN_MASK           0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE6EN_SHIFT          4
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE7EN_MASK           0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE7EN_SHIFT          5
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED1_MASK      0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED1_SHIFT     6
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE9EN_MASK           0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE9EN_SHIFT          7
+        u8 flags12;
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE10EN_MASK          0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE10EN_SHIFT         0
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_IRQ_PROD_RULE_EN_MASK  0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_IRQ_PROD_RULE_EN_SHIFT 1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED2_MASK      0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED2_SHIFT     2
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED3_MASK      0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED3_SHIFT     3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE14EN_MASK          0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE14EN_SHIFT         4
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE15EN_MASK          0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE15EN_SHIFT         5
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE16EN_MASK          0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE16EN_SHIFT         6
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE17EN_MASK          0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE17EN_SHIFT         7
+        u8 flags13;
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE18EN_MASK          0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE18EN_SHIFT         0
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE19EN_MASK          0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE19EN_SHIFT         1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED4_MASK      0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED4_SHIFT     2
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED5_MASK      0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED5_SHIFT     3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED6_MASK      0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED6_SHIFT     4
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED7_MASK      0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED7_SHIFT     5
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED8_MASK      0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED8_SHIFT     6
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED9_MASK      0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED9_SHIFT     7
+        u8 flags14;
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT16_MASK             0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT16_SHIFT            0
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT17_MASK             0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT17_SHIFT            1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT18_MASK             0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT18_SHIFT            2
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT19_MASK             0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT19_SHIFT            3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT20_MASK             0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT20_SHIFT            4
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT21_MASK             0x1
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT21_SHIFT            5
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF23_MASK              0x3
+#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF23_SHIFT             6
+        u8 byte2;
+        __le16 physical_q0;
+        __le16 word1;
+        __le16 irq_prod;
+        __le16 word3;
+        __le16 word4;
+        __le16 word5;
+        __le16 irq_cons;
+        u8 rxmit_opcode;
+        u8 byte4;
+        u8 byte5;
+        u8 byte6;
+        __le32 rxmit_psn_and_id;
+        __le32 rxmit_bytes_length;
+        __le32 psn;
+        __le32 reg3;
+        __le32 reg4;
+        __le32 reg5;
+        __le32 msn_and_syndrome;
+};
+
+struct ystorm_roce_req_conn_ag_ctx {
+        u8 byte0;
+        u8 byte1;
+        u8 flags0;
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_BIT0_MASK     0x1
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_BIT0_SHIFT    0
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_BIT1_MASK     0x1
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_BIT1_SHIFT    1
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_CF0_MASK      0x3
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_CF0_SHIFT     2
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_CF1_MASK      0x3
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_CF1_SHIFT     4
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_CF2_MASK      0x3
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_CF2_SHIFT     6
+        u8 flags1;
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_CF0EN_MASK    0x1
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_CF0EN_SHIFT   0
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_MASK    0x1
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_SHIFT   1
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_CF2EN_MASK    0x1
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_CF2EN_SHIFT   2
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_MASK  0x1
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_SHIFT 3
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_MASK  0x1
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_SHIFT 4
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_MASK  0x1
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_SHIFT 5
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_MASK  0x1
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_SHIFT 6
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_MASK  0x1
+#define YSTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_SHIFT 7
+        u8 byte2;
+        u8 byte3;
+        __le16 word0;
+        __le32 reg0;
+        __le32 reg1;
+        __le16 word1;
+        __le16 word2;
+        __le16 word3;
+        __le16 word4;
+        __le32 reg2;
+        __le32 reg3;
+};
+
+struct ystorm_roce_resp_conn_ag_ctx {
+        u8 byte0;
+        u8 byte1;
+        u8 flags0;
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_BIT0_MASK     0x1
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_BIT0_SHIFT    0
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_BIT1_MASK     0x1
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_BIT1_SHIFT    1
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_CF0_MASK      0x3
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_CF0_SHIFT     2
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_CF1_MASK      0x3
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_CF1_SHIFT     4
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_CF2_MASK      0x3
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_CF2_SHIFT     6
+        u8 flags1;
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_MASK    0x1
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_SHIFT   0
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_CF1EN_MASK    0x1
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_CF1EN_SHIFT   1
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_CF2EN_MASK    0x1
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_CF2EN_SHIFT   2
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_MASK  0x1
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_SHIFT 3
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_MASK  0x1
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_SHIFT 4
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_MASK  0x1
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_SHIFT 5
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_MASK  0x1
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_SHIFT 6
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_MASK  0x1
+#define YSTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_SHIFT 7
+        u8 byte2;
+        u8 byte3;
+        __le16 word0;
+        __le32 reg0;
+        __le32 reg1;
+        __le16 word1;
+        __le16 word2;
+        __le16 word3;
+        __le16 word4;
+        __le32 reg2;
+        __le32 reg3;
+};
+
+struct ystorm_iscsi_conn_st_ctx {
+        __le32 reserved[4];
+};
+
+struct pstorm_iscsi_tcp_conn_st_ctx {
+        __le32 tcp[32];
+        __le32 iscsi[4];
+};
+
+struct xstorm_iscsi_tcp_conn_st_ctx {
+        __le32 reserved_iscsi[40];
+        __le32 reserved_tcp[4];
+};
+
+struct xstorm_iscsi_conn_ag_ctx {
+        u8 cdu_validation;
+        u8 state;
+        u8 flags0;
+#define XSTORM_ISCSI_CONN_AG_CTX_EXIST_IN_QM0_MASK                0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_EXIST_IN_QM0_SHIFT               0
+#define XSTORM_ISCSI_CONN_AG_CTX_EXIST_IN_QM1_MASK                0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_EXIST_IN_QM1_SHIFT               1
+#define XSTORM_ISCSI_CONN_AG_CTX_RESERVED1_MASK                   0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_RESERVED1_SHIFT                  2
+#define XSTORM_ISCSI_CONN_AG_CTX_EXIST_IN_QM3_MASK                0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_EXIST_IN_QM3_SHIFT               3
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT4_MASK                        0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT4_SHIFT                       4
+#define XSTORM_ISCSI_CONN_AG_CTX_RESERVED2_MASK                   0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_RESERVED2_SHIFT                  5
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT6_MASK                        0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT6_SHIFT                       6
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT7_MASK                        0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT7_SHIFT                       7
+        u8 flags1;
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT8_MASK                        0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT8_SHIFT                       0
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT9_MASK                        0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT9_SHIFT                       1
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT10_MASK                       0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT10_SHIFT                      2
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT11_MASK                       0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT11_SHIFT                      3
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT12_MASK                       0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT12_SHIFT                      4
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT13_MASK                       0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT13_SHIFT                      5
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT14_MASK                       0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT14_SHIFT                      6
+#define XSTORM_ISCSI_CONN_AG_CTX_TX_TRUNCATE_MASK                 0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_TX_TRUNCATE_SHIFT                7
+        u8 flags2;
+#define XSTORM_ISCSI_CONN_AG_CTX_CF0_MASK                         0x3
+#define XSTORM_ISCSI_CONN_AG_CTX_CF0_SHIFT                        0
+#define XSTORM_ISCSI_CONN_AG_CTX_CF1_MASK                         0x3
+#define XSTORM_ISCSI_CONN_AG_CTX_CF1_SHIFT                        2
+#define XSTORM_ISCSI_CONN_AG_CTX_CF2_MASK                         0x3
+#define XSTORM_ISCSI_CONN_AG_CTX_CF2_SHIFT                        4
+#define XSTORM_ISCSI_CONN_AG_CTX_TIMER_STOP_ALL_MASK              0x3
+#define XSTORM_ISCSI_CONN_AG_CTX_TIMER_STOP_ALL_SHIFT             6
+        u8 flags3;
+#define XSTORM_ISCSI_CONN_AG_CTX_CF4_MASK                         0x3
+#define XSTORM_ISCSI_CONN_AG_CTX_CF4_SHIFT                        0
+#define XSTORM_ISCSI_CONN_AG_CTX_CF5_MASK                         0x3
+#define XSTORM_ISCSI_CONN_AG_CTX_CF5_SHIFT                        2
+#define XSTORM_ISCSI_CONN_AG_CTX_CF6_MASK                         0x3
+#define XSTORM_ISCSI_CONN_AG_CTX_CF6_SHIFT                        4
+#define XSTORM_ISCSI_CONN_AG_CTX_CF7_MASK                         0x3
+#define XSTORM_ISCSI_CONN_AG_CTX_CF7_SHIFT                        6
+        u8 flags4;
+#define XSTORM_ISCSI_CONN_AG_CTX_CF8_MASK                         0x3
+#define XSTORM_ISCSI_CONN_AG_CTX_CF8_SHIFT                        0
+#define XSTORM_ISCSI_CONN_AG_CTX_CF9_MASK                         0x3
+#define XSTORM_ISCSI_CONN_AG_CTX_CF9_SHIFT                        2
+#define XSTORM_ISCSI_CONN_AG_CTX_CF10_MASK                        0x3
+#define XSTORM_ISCSI_CONN_AG_CTX_CF10_SHIFT                       4
+#define XSTORM_ISCSI_CONN_AG_CTX_CF11_MASK                        0x3
+#define XSTORM_ISCSI_CONN_AG_CTX_CF11_SHIFT                       6
+        u8 flags5;
+#define XSTORM_ISCSI_CONN_AG_CTX_CF12_MASK                        0x3
+#define XSTORM_ISCSI_CONN_AG_CTX_CF12_SHIFT                       0
+#define XSTORM_ISCSI_CONN_AG_CTX_CF13_MASK                        0x3
+#define XSTORM_ISCSI_CONN_AG_CTX_CF13_SHIFT                       2
+#define XSTORM_ISCSI_CONN_AG_CTX_CF14_MASK                        0x3
+#define XSTORM_ISCSI_CONN_AG_CTX_CF14_SHIFT                       4
+#define XSTORM_ISCSI_CONN_AG_CTX_UPDATE_STATE_TO_BASE_CF_MASK     0x3
+#define XSTORM_ISCSI_CONN_AG_CTX_UPDATE_STATE_TO_BASE_CF_SHIFT    6
+        u8 flags6;
+#define XSTORM_ISCSI_CONN_AG_CTX_CF16_MASK                        0x3
+#define XSTORM_ISCSI_CONN_AG_CTX_CF16_SHIFT                       0
+#define XSTORM_ISCSI_CONN_AG_CTX_CF17_MASK                        0x3
+#define XSTORM_ISCSI_CONN_AG_CTX_CF17_SHIFT                       2
+#define XSTORM_ISCSI_CONN_AG_CTX_CF18_MASK                        0x3
+#define XSTORM_ISCSI_CONN_AG_CTX_CF18_SHIFT                       4
+#define XSTORM_ISCSI_CONN_AG_CTX_DQ_FLUSH_MASK                    0x3
+#define XSTORM_ISCSI_CONN_AG_CTX_DQ_FLUSH_SHIFT                   6
+        u8 flags7;
+#define XSTORM_ISCSI_CONN_AG_CTX_FLUSH_Q0_MASK                    0x3
+#define XSTORM_ISCSI_CONN_AG_CTX_FLUSH_Q0_SHIFT                   0
+#define XSTORM_ISCSI_CONN_AG_CTX_FLUSH_Q1_MASK                    0x3
+#define XSTORM_ISCSI_CONN_AG_CTX_FLUSH_Q1_SHIFT                   2
+#define XSTORM_ISCSI_CONN_AG_CTX_SLOW_PATH_MASK                   0x3
+#define XSTORM_ISCSI_CONN_AG_CTX_SLOW_PATH_SHIFT                  4
+#define XSTORM_ISCSI_CONN_AG_CTX_CF0EN_MASK                       0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_CF0EN_SHIFT                      6
+#define XSTORM_ISCSI_CONN_AG_CTX_CF1EN_MASK                       0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_CF1EN_SHIFT                      7
+        u8 flags8;
+#define XSTORM_ISCSI_CONN_AG_CTX_CF2EN_MASK                       0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_CF2EN_SHIFT                      0
+#define XSTORM_ISCSI_CONN_AG_CTX_TIMER_STOP_ALL_EN_MASK           0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_TIMER_STOP_ALL_EN_SHIFT          1
+#define XSTORM_ISCSI_CONN_AG_CTX_CF4EN_MASK                       0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_CF4EN_SHIFT                      2
+#define XSTORM_ISCSI_CONN_AG_CTX_CF5EN_MASK                       0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_CF5EN_SHIFT                      3
+#define XSTORM_ISCSI_CONN_AG_CTX_CF6EN_MASK                       0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_CF6EN_SHIFT                      4
+#define XSTORM_ISCSI_CONN_AG_CTX_CF7EN_MASK                       0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_CF7EN_SHIFT                      5
+#define XSTORM_ISCSI_CONN_AG_CTX_CF8EN_MASK                       0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_CF8EN_SHIFT                      6
+#define XSTORM_ISCSI_CONN_AG_CTX_CF9EN_MASK                       0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_CF9EN_SHIFT                      7
+        u8 flags9;
+#define XSTORM_ISCSI_CONN_AG_CTX_CF10EN_MASK                      0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_CF10EN_SHIFT                     0
+#define XSTORM_ISCSI_CONN_AG_CTX_CF11EN_MASK                      0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_CF11EN_SHIFT                     1
+#define XSTORM_ISCSI_CONN_AG_CTX_CF12EN_MASK                      0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_CF12EN_SHIFT                     2
+#define XSTORM_ISCSI_CONN_AG_CTX_CF13EN_MASK                      0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_CF13EN_SHIFT                     3
+#define XSTORM_ISCSI_CONN_AG_CTX_CF14EN_MASK                      0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_CF14EN_SHIFT                     4
+#define XSTORM_ISCSI_CONN_AG_CTX_UPDATE_STATE_TO_BASE_CF_EN_MASK  0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_UPDATE_STATE_TO_BASE_CF_EN_SHIFT 5
+#define XSTORM_ISCSI_CONN_AG_CTX_CF16EN_MASK                      0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_CF16EN_SHIFT                     6
+#define XSTORM_ISCSI_CONN_AG_CTX_CF17EN_MASK                      0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_CF17EN_SHIFT                     7
+        u8 flags10;
+#define XSTORM_ISCSI_CONN_AG_CTX_CF18EN_MASK                      0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_CF18EN_SHIFT                     0
+#define XSTORM_ISCSI_CONN_AG_CTX_DQ_FLUSH_EN_MASK                 0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_DQ_FLUSH_EN_SHIFT                1
+#define XSTORM_ISCSI_CONN_AG_CTX_FLUSH_Q0_EN_MASK                 0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_FLUSH_Q0_EN_SHIFT                2
+#define XSTORM_ISCSI_CONN_AG_CTX_FLUSH_Q1_EN_MASK                 0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_FLUSH_Q1_EN_SHIFT                3
+#define XSTORM_ISCSI_CONN_AG_CTX_SLOW_PATH_EN_MASK                0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_SLOW_PATH_EN_SHIFT               4
+#define XSTORM_ISCSI_CONN_AG_CTX_PROC_ONLY_CLEANUP_EN_MASK        0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_PROC_ONLY_CLEANUP_EN_SHIFT       5
+#define XSTORM_ISCSI_CONN_AG_CTX_RULE0EN_MASK                     0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_RULE0EN_SHIFT                    6
+#define XSTORM_ISCSI_CONN_AG_CTX_MORE_TO_SEND_DEC_RULE_EN_MASK    0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_MORE_TO_SEND_DEC_RULE_EN_SHIFT   7
+        u8 flags11;
+#define XSTORM_ISCSI_CONN_AG_CTX_RULE2EN_MASK                     0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_RULE2EN_SHIFT                    0
+#define XSTORM_ISCSI_CONN_AG_CTX_RULE3EN_MASK                     0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_RULE3EN_SHIFT                    1
+#define XSTORM_ISCSI_CONN_AG_CTX_RESERVED3_MASK                   0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_RESERVED3_SHIFT                  2
+#define XSTORM_ISCSI_CONN_AG_CTX_RULE5EN_MASK                     0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_RULE5EN_SHIFT                    3
+#define XSTORM_ISCSI_CONN_AG_CTX_RULE6EN_MASK                     0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_RULE6EN_SHIFT                    4
+#define XSTORM_ISCSI_CONN_AG_CTX_RULE7EN_MASK                     0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_RULE7EN_SHIFT                    5
+#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED1_MASK                0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED1_SHIFT               6
+#define XSTORM_ISCSI_CONN_AG_CTX_RULE9EN_MASK                     0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_RULE9EN_SHIFT                    7
+        u8 flags12;
+#define XSTORM_ISCSI_CONN_AG_CTX_SQ_DEC_RULE_EN_MASK              0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_SQ_DEC_RULE_EN_SHIFT             0
+#define XSTORM_ISCSI_CONN_AG_CTX_RULE11EN_MASK                    0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_RULE11EN_SHIFT                   1
+#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED2_MASK                0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED2_SHIFT               2
+#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED3_MASK                0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED3_SHIFT               3
+#define XSTORM_ISCSI_CONN_AG_CTX_RULE14EN_MASK                    0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_RULE14EN_SHIFT                   4
+#define XSTORM_ISCSI_CONN_AG_CTX_RULE15EN_MASK                    0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_RULE15EN_SHIFT                   5
+#define XSTORM_ISCSI_CONN_AG_CTX_RULE16EN_MASK                    0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_RULE16EN_SHIFT                   6
+#define XSTORM_ISCSI_CONN_AG_CTX_RULE17EN_MASK                    0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_RULE17EN_SHIFT                   7
+        u8 flags13;
+#define XSTORM_ISCSI_CONN_AG_CTX_R2TQ_DEC_RULE_EN_MASK            0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_R2TQ_DEC_RULE_EN_SHIFT           0
+#define XSTORM_ISCSI_CONN_AG_CTX_HQ_DEC_RULE_EN_MASK              0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_HQ_DEC_RULE_EN_SHIFT             1
+#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED4_MASK                0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED4_SHIFT               2
+#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED5_MASK                0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED5_SHIFT               3
+#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED6_MASK                0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED6_SHIFT               4
+#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED7_MASK                0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED7_SHIFT               5
+#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED8_MASK                0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED8_SHIFT               6
+#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED9_MASK                0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED9_SHIFT               7
+        u8 flags14;
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT16_MASK                       0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT16_SHIFT                      0
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT17_MASK                       0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT17_SHIFT                      1
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT18_MASK                       0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT18_SHIFT                      2
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT19_MASK                       0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT19_SHIFT                      3
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT20_MASK                       0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_BIT20_SHIFT                      4
+#define XSTORM_ISCSI_CONN_AG_CTX_DUMMY_READ_DONE_MASK             0x1
+#define XSTORM_ISCSI_CONN_AG_CTX_DUMMY_READ_DONE_SHIFT            5
+#define XSTORM_ISCSI_CONN_AG_CTX_PROC_ONLY_CLEANUP_MASK           0x3
+#define XSTORM_ISCSI_CONN_AG_CTX_PROC_ONLY_CLEANUP_SHIFT          6
+        u8 byte2;
+        __le16 physical_q0;
+        __le16 physical_q1;
+        __le16 dummy_dorq_var;
+        __le16 sq_cons;
+        __le16 sq_prod;
+        __le16 word5;
+        __le16 slow_io_total_data_tx_update;
+        u8 byte3;
+        u8 byte4;
+        u8 byte5;
+        u8 byte6;
+        __le32 reg0;
+        __le32 reg1;
+        __le32 reg2;
+        __le32 more_to_send_seq;
+        __le32 reg4;
+        __le32 reg5;
+        __le32 hq_scan_next_relevant_ack;
+        __le16 r2tq_prod;
+        __le16 r2tq_cons;
+        __le16 hq_prod;
+        __le16 hq_cons;
+        __le32 remain_seq;
+        __le32 bytes_to_next_pdu;
+        __le32 hq_tcp_seq;
+        u8 byte7;
+        u8 byte8;
+        u8 byte9;
+        u8 byte10;
+        u8 byte11;
+        u8 byte12;
+        u8 byte13;
+        u8 byte14;
+        u8 byte15;
+        u8 byte16;
+        __le16 word11;
+        __le32 reg10;
+        __le32 reg11;
+        __le32 exp_stat_sn;
+        __le32 reg13;
+        __le32 reg14;
+        __le32 reg15;
+        __le32 reg16;
+        __le32 reg17;
+};
+
+struct tstorm_iscsi_conn_ag_ctx {
+        u8 reserved0;
+        u8 state;
+        u8 flags0;
+#define TSTORM_ISCSI_CONN_AG_CTX_EXIST_IN_QM0_MASK       0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_EXIST_IN_QM0_SHIFT      0
+#define TSTORM_ISCSI_CONN_AG_CTX_BIT1_MASK               0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_BIT1_SHIFT              1
+#define TSTORM_ISCSI_CONN_AG_CTX_BIT2_MASK               0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_BIT2_SHIFT              2
+#define TSTORM_ISCSI_CONN_AG_CTX_BIT3_MASK               0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_BIT3_SHIFT              3
+#define TSTORM_ISCSI_CONN_AG_CTX_BIT4_MASK               0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_BIT4_SHIFT              4
+#define TSTORM_ISCSI_CONN_AG_CTX_BIT5_MASK               0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_BIT5_SHIFT              5
+#define TSTORM_ISCSI_CONN_AG_CTX_CF0_MASK                0x3
+#define TSTORM_ISCSI_CONN_AG_CTX_CF0_SHIFT               6
+        u8 flags1;
+#define TSTORM_ISCSI_CONN_AG_CTX_CF1_MASK                0x3
+#define TSTORM_ISCSI_CONN_AG_CTX_CF1_SHIFT               0
+#define TSTORM_ISCSI_CONN_AG_CTX_CF2_MASK                0x3
+#define TSTORM_ISCSI_CONN_AG_CTX_CF2_SHIFT               2
+#define TSTORM_ISCSI_CONN_AG_CTX_TIMER_STOP_ALL_MASK     0x3
+#define TSTORM_ISCSI_CONN_AG_CTX_TIMER_STOP_ALL_SHIFT    4
+#define TSTORM_ISCSI_CONN_AG_CTX_CF4_MASK                0x3
+#define TSTORM_ISCSI_CONN_AG_CTX_CF4_SHIFT               6
+        u8 flags2;
+#define TSTORM_ISCSI_CONN_AG_CTX_CF5_MASK                0x3
+#define TSTORM_ISCSI_CONN_AG_CTX_CF5_SHIFT               0
+#define TSTORM_ISCSI_CONN_AG_CTX_CF6_MASK                0x3
+#define TSTORM_ISCSI_CONN_AG_CTX_CF6_SHIFT               2
+#define TSTORM_ISCSI_CONN_AG_CTX_CF7_MASK                0x3
+#define TSTORM_ISCSI_CONN_AG_CTX_CF7_SHIFT               4
+#define TSTORM_ISCSI_CONN_AG_CTX_CF8_MASK                0x3
+#define TSTORM_ISCSI_CONN_AG_CTX_CF8_SHIFT               6
+        u8 flags3;
+#define TSTORM_ISCSI_CONN_AG_CTX_FLUSH_Q0_MASK           0x3
+#define TSTORM_ISCSI_CONN_AG_CTX_FLUSH_Q0_SHIFT          0
+#define TSTORM_ISCSI_CONN_AG_CTX_CF10_MASK               0x3
+#define TSTORM_ISCSI_CONN_AG_CTX_CF10_SHIFT              2
+#define TSTORM_ISCSI_CONN_AG_CTX_CF0EN_MASK              0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_CF0EN_SHIFT             4
+#define TSTORM_ISCSI_CONN_AG_CTX_CF1EN_MASK              0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_CF1EN_SHIFT             5
+#define TSTORM_ISCSI_CONN_AG_CTX_CF2EN_MASK              0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_CF2EN_SHIFT             6
+#define TSTORM_ISCSI_CONN_AG_CTX_TIMER_STOP_ALL_EN_MASK  0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_TIMER_STOP_ALL_EN_SHIFT 7
+        u8 flags4;
+#define TSTORM_ISCSI_CONN_AG_CTX_CF4EN_MASK              0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_CF4EN_SHIFT             0
+#define TSTORM_ISCSI_CONN_AG_CTX_CF5EN_MASK              0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_CF5EN_SHIFT             1
+#define TSTORM_ISCSI_CONN_AG_CTX_CF6EN_MASK              0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_CF6EN_SHIFT             2
+#define TSTORM_ISCSI_CONN_AG_CTX_CF7EN_MASK              0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_CF7EN_SHIFT             3
+#define TSTORM_ISCSI_CONN_AG_CTX_CF8EN_MASK              0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_CF8EN_SHIFT             4
+#define TSTORM_ISCSI_CONN_AG_CTX_FLUSH_Q0_EN_MASK        0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_FLUSH_Q0_EN_SHIFT       5
+#define TSTORM_ISCSI_CONN_AG_CTX_CF10EN_MASK             0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_CF10EN_SHIFT            6
+#define TSTORM_ISCSI_CONN_AG_CTX_RULE0EN_MASK            0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_RULE0EN_SHIFT           7
+        u8 flags5;
+#define TSTORM_ISCSI_CONN_AG_CTX_RULE1EN_MASK            0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_RULE1EN_SHIFT           0
+#define TSTORM_ISCSI_CONN_AG_CTX_RULE2EN_MASK            0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_RULE2EN_SHIFT           1
+#define TSTORM_ISCSI_CONN_AG_CTX_RULE3EN_MASK            0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_RULE3EN_SHIFT           2
+#define TSTORM_ISCSI_CONN_AG_CTX_RULE4EN_MASK            0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_RULE4EN_SHIFT           3
+#define TSTORM_ISCSI_CONN_AG_CTX_RULE5EN_MASK            0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_RULE5EN_SHIFT           4
+#define TSTORM_ISCSI_CONN_AG_CTX_RULE6EN_MASK            0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_RULE6EN_SHIFT           5
+#define TSTORM_ISCSI_CONN_AG_CTX_RULE7EN_MASK            0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_RULE7EN_SHIFT           6
+#define TSTORM_ISCSI_CONN_AG_CTX_RULE8EN_MASK            0x1
+#define TSTORM_ISCSI_CONN_AG_CTX_RULE8EN_SHIFT           7
+        __le32 reg0;
+        __le32 reg1;
+        __le32 reg2;
+        __le32 reg3;
+        __le32 reg4;
+        __le32 reg5;
+        __le32 reg6;
+        __le32 reg7;
+        __le32 reg8;
+        u8 byte2;
+        u8 byte3;
+        __le16 word0;
+};
+
+struct ustorm_iscsi_conn_ag_ctx {
+        u8 byte0;
+        u8 byte1;
+        u8 flags0;
+#define USTORM_ISCSI_CONN_AG_CTX_BIT0_MASK     0x1
+#define USTORM_ISCSI_CONN_AG_CTX_BIT0_SHIFT    0
+#define USTORM_ISCSI_CONN_AG_CTX_BIT1_MASK     0x1
+#define USTORM_ISCSI_CONN_AG_CTX_BIT1_SHIFT    1
+#define USTORM_ISCSI_CONN_AG_CTX_CF0_MASK      0x3
+#define USTORM_ISCSI_CONN_AG_CTX_CF0_SHIFT     2
+#define USTORM_ISCSI_CONN_AG_CTX_CF1_MASK      0x3
+#define USTORM_ISCSI_CONN_AG_CTX_CF1_SHIFT     4
+#define USTORM_ISCSI_CONN_AG_CTX_CF2_MASK      0x3
+#define USTORM_ISCSI_CONN_AG_CTX_CF2_SHIFT     6
+        u8 flags1;
+#define USTORM_ISCSI_CONN_AG_CTX_CF3_MASK      0x3
+#define USTORM_ISCSI_CONN_AG_CTX_CF3_SHIFT     0
+#define USTORM_ISCSI_CONN_AG_CTX_CF4_MASK      0x3
+#define USTORM_ISCSI_CONN_AG_CTX_CF4_SHIFT     2
+#define USTORM_ISCSI_CONN_AG_CTX_CF5_MASK      0x3
+#define USTORM_ISCSI_CONN_AG_CTX_CF5_SHIFT     4
+#define USTORM_ISCSI_CONN_AG_CTX_CF6_MASK      0x3
+#define USTORM_ISCSI_CONN_AG_CTX_CF6_SHIFT     6
+        u8 flags2;
+#define USTORM_ISCSI_CONN_AG_CTX_CF0EN_MASK    0x1
+#define USTORM_ISCSI_CONN_AG_CTX_CF0EN_SHIFT   0
+#define USTORM_ISCSI_CONN_AG_CTX_CF1EN_MASK    0x1
+#define USTORM_ISCSI_CONN_AG_CTX_CF1EN_SHIFT   1
+#define USTORM_ISCSI_CONN_AG_CTX_CF2EN_MASK    0x1
+#define USTORM_ISCSI_CONN_AG_CTX_CF2EN_SHIFT   2
+#define USTORM_ISCSI_CONN_AG_CTX_CF3EN_MASK    0x1
+#define USTORM_ISCSI_CONN_AG_CTX_CF3EN_SHIFT   3
+#define USTORM_ISCSI_CONN_AG_CTX_CF4EN_MASK    0x1
+#define USTORM_ISCSI_CONN_AG_CTX_CF4EN_SHIFT   4
+#define USTORM_ISCSI_CONN_AG_CTX_CF5EN_MASK    0x1
+#define USTORM_ISCSI_CONN_AG_CTX_CF5EN_SHIFT   5
+#define USTORM_ISCSI_CONN_AG_CTX_CF6EN_MASK    0x1
+#define USTORM_ISCSI_CONN_AG_CTX_CF6EN_SHIFT   6
+#define USTORM_ISCSI_CONN_AG_CTX_RULE0EN_MASK  0x1
+#define USTORM_ISCSI_CONN_AG_CTX_RULE0EN_SHIFT 7
+        u8 flags3;
+#define USTORM_ISCSI_CONN_AG_CTX_RULE1EN_MASK  0x1
+#define USTORM_ISCSI_CONN_AG_CTX_RULE1EN_SHIFT 0
+#define USTORM_ISCSI_CONN_AG_CTX_RULE2EN_MASK  0x1
+#define USTORM_ISCSI_CONN_AG_CTX_RULE2EN_SHIFT 1
+#define USTORM_ISCSI_CONN_AG_CTX_RULE3EN_MASK  0x1
+#define USTORM_ISCSI_CONN_AG_CTX_RULE3EN_SHIFT 2
+#define USTORM_ISCSI_CONN_AG_CTX_RULE4EN_MASK  0x1
+#define USTORM_ISCSI_CONN_AG_CTX_RULE4EN_SHIFT 3
+#define USTORM_ISCSI_CONN_AG_CTX_RULE5EN_MASK  0x1
+#define USTORM_ISCSI_CONN_AG_CTX_RULE5EN_SHIFT 4
+#define USTORM_ISCSI_CONN_AG_CTX_RULE6EN_MASK  0x1
+#define USTORM_ISCSI_CONN_AG_CTX_RULE6EN_SHIFT 5
+#define USTORM_ISCSI_CONN_AG_CTX_RULE7EN_MASK  0x1
+#define USTORM_ISCSI_CONN_AG_CTX_RULE7EN_SHIFT 6
+#define USTORM_ISCSI_CONN_AG_CTX_RULE8EN_MASK  0x1
+#define USTORM_ISCSI_CONN_AG_CTX_RULE8EN_SHIFT 7
+        u8 byte2;
+        u8 byte3;
+        __le16 word0;
+        __le16 word1;
+        __le32 reg0;
+        __le32 reg1;
+        __le32 reg2;
+        __le32 reg3;
+        __le16 word2;
+        __le16 word3;
+};
+
+struct tstorm_iscsi_conn_st_ctx {
+        __le32 reserved[40];
+};
+
+struct mstorm_iscsi_conn_ag_ctx {
+        u8 reserved;
+        u8 state;
+        u8 flags0;
+#define MSTORM_ISCSI_CONN_AG_CTX_BIT0_MASK     0x1
+#define MSTORM_ISCSI_CONN_AG_CTX_BIT0_SHIFT    0
+#define MSTORM_ISCSI_CONN_AG_CTX_BIT1_MASK     0x1
+#define MSTORM_ISCSI_CONN_AG_CTX_BIT1_SHIFT    1
+#define MSTORM_ISCSI_CONN_AG_CTX_CF0_MASK      0x3
+#define MSTORM_ISCSI_CONN_AG_CTX_CF0_SHIFT     2
+#define MSTORM_ISCSI_CONN_AG_CTX_CF1_MASK      0x3
+#define MSTORM_ISCSI_CONN_AG_CTX_CF1_SHIFT     4
+#define MSTORM_ISCSI_CONN_AG_CTX_CF2_MASK      0x3
+#define MSTORM_ISCSI_CONN_AG_CTX_CF2_SHIFT     6
+        u8 flags1;
+#define MSTORM_ISCSI_CONN_AG_CTX_CF0EN_MASK    0x1
+#define MSTORM_ISCSI_CONN_AG_CTX_CF0EN_SHIFT   0
+#define MSTORM_ISCSI_CONN_AG_CTX_CF1EN_MASK    0x1
+#define MSTORM_ISCSI_CONN_AG_CTX_CF1EN_SHIFT   1
+#define MSTORM_ISCSI_CONN_AG_CTX_CF2EN_MASK    0x1
+#define MSTORM_ISCSI_CONN_AG_CTX_CF2EN_SHIFT   2
+#define MSTORM_ISCSI_CONN_AG_CTX_RULE0EN_MASK  0x1
+#define MSTORM_ISCSI_CONN_AG_CTX_RULE0EN_SHIFT 3
+#define MSTORM_ISCSI_CONN_AG_CTX_RULE1EN_MASK  0x1
+#define MSTORM_ISCSI_CONN_AG_CTX_RULE1EN_SHIFT 4
+#define MSTORM_ISCSI_CONN_AG_CTX_RULE2EN_MASK  0x1
+#define MSTORM_ISCSI_CONN_AG_CTX_RULE2EN_SHIFT 5
+#define MSTORM_ISCSI_CONN_AG_CTX_RULE3EN_MASK  0x1
+#define MSTORM_ISCSI_CONN_AG_CTX_RULE3EN_SHIFT 6
+#define MSTORM_ISCSI_CONN_AG_CTX_RULE4EN_MASK  0x1
+#define MSTORM_ISCSI_CONN_AG_CTX_RULE4EN_SHIFT 7
+        __le16 word0;
+        __le16 word1;
+        __le32 reg0;
+        __le32 reg1;
+};
+
+struct mstorm_iscsi_tcp_conn_st_ctx {
+        __le32 reserved_tcp[20];
+        __le32 reserved_iscsi[8];
+};
+
+struct ustorm_iscsi_conn_st_ctx {
+        __le32 reserved[52];
+};
+
+struct iscsi_conn_context {
+        struct ystorm_iscsi_conn_st_ctx ystorm_st_context;
+        struct regpair ystorm_st_padding[2];
+        struct pstorm_iscsi_tcp_conn_st_ctx pstorm_st_context;
+        struct regpair pstorm_st_padding[2];
+        struct pb_context xpb2_context;
+        struct xstorm_iscsi_tcp_conn_st_ctx xstorm_st_context;
+        struct regpair xstorm_st_padding[2];
+        struct xstorm_iscsi_conn_ag_ctx xstorm_ag_context;
+        struct tstorm_iscsi_conn_ag_ctx tstorm_ag_context;
+        struct regpair tstorm_ag_padding[2];
+        struct timers_context timer_context;
+        struct ustorm_iscsi_conn_ag_ctx ustorm_ag_context;
+        struct pb_context upb_context;
+        struct tstorm_iscsi_conn_st_ctx tstorm_st_context;
+        struct regpair tstorm_st_padding[2];
+        struct mstorm_iscsi_conn_ag_ctx mstorm_ag_context;
+        struct mstorm_iscsi_tcp_conn_st_ctx mstorm_st_context;
+        struct ustorm_iscsi_conn_st_ctx ustorm_st_context;
+};
+
+struct iscsi_init_ramrod_params {
+        struct iscsi_spe_func_init iscsi_init_spe;
+        struct tcp_init_params tcp_init;
+};
+
+struct ystorm_iscsi_conn_ag_ctx {
+        u8 byte0;
+        u8 byte1;
+        u8 flags0;
+#define YSTORM_ISCSI_CONN_AG_CTX_BIT0_MASK     0x1
+#define YSTORM_ISCSI_CONN_AG_CTX_BIT0_SHIFT    0
+#define YSTORM_ISCSI_CONN_AG_CTX_BIT1_MASK     0x1
+#define YSTORM_ISCSI_CONN_AG_CTX_BIT1_SHIFT    1
+#define YSTORM_ISCSI_CONN_AG_CTX_CF0_MASK      0x3
+#define YSTORM_ISCSI_CONN_AG_CTX_CF0_SHIFT     2
+#define YSTORM_ISCSI_CONN_AG_CTX_CF1_MASK      0x3
+#define YSTORM_ISCSI_CONN_AG_CTX_CF1_SHIFT     4
+#define YSTORM_ISCSI_CONN_AG_CTX_CF2_MASK      0x3
+#define YSTORM_ISCSI_CONN_AG_CTX_CF2_SHIFT     6
+        u8 flags1;
+#define YSTORM_ISCSI_CONN_AG_CTX_CF0EN_MASK    0x1
+#define YSTORM_ISCSI_CONN_AG_CTX_CF0EN_SHIFT   0
+#define YSTORM_ISCSI_CONN_AG_CTX_CF1EN_MASK    0x1
+#define YSTORM_ISCSI_CONN_AG_CTX_CF1EN_SHIFT   1
+#define YSTORM_ISCSI_CONN_AG_CTX_CF2EN_MASK    0x1
+#define YSTORM_ISCSI_CONN_AG_CTX_CF2EN_SHIFT   2
+#define YSTORM_ISCSI_CONN_AG_CTX_RULE0EN_MASK  0x1
+#define YSTORM_ISCSI_CONN_AG_CTX_RULE0EN_SHIFT 3
+#define YSTORM_ISCSI_CONN_AG_CTX_RULE1EN_MASK  0x1
+#define YSTORM_ISCSI_CONN_AG_CTX_RULE1EN_SHIFT 4
+#define YSTORM_ISCSI_CONN_AG_CTX_RULE2EN_MASK  0x1
+#define YSTORM_ISCSI_CONN_AG_CTX_RULE2EN_SHIFT 5
+#define YSTORM_ISCSI_CONN_AG_CTX_RULE3EN_MASK  0x1
+#define YSTORM_ISCSI_CONN_AG_CTX_RULE3EN_SHIFT 6
+#define YSTORM_ISCSI_CONN_AG_CTX_RULE4EN_MASK  0x1
+#define YSTORM_ISCSI_CONN_AG_CTX_RULE4EN_SHIFT 7
+        u8 byte2;
+        u8 byte3;
+        __le16 word0;
+        __le32 reg0;
+        __le32 reg1;
+        __le16 word1;
+        __le16 word2;
+        __le16 word3;
+        __le16 word4;
+        __le32 reg2;
+        __le32 reg3;
+};
+#define VF_MAX_STATIC 192
+
+#define MCP_GLOB_PATH_MAX       2
+#define MCP_PORT_MAX            2
+#define MCP_GLOB_PORT_MAX       4
+#define MCP_GLOB_FUNC_MAX       16
 
-typedef u32 offsize_t;                  /* In DWORDS !!! */
 /* Offset from the beginning of the MCP scratchpad */
-#define OFFSIZE_OFFSET_SHIFT    0
-#define OFFSIZE_OFFSET_MASK     0x0000ffff
+#define OFFSIZE_OFFSET_SHIFT    0
+#define OFFSIZE_OFFSET_MASK     0x0000ffff
 /* Size of specific element (not the whole array if any) */
-#define OFFSIZE_SIZE_SHIFT      16
-#define OFFSIZE_SIZE_MASK       0xffff0000
+#define OFFSIZE_SIZE_SHIFT      16
+#define OFFSIZE_SIZE_MASK       0xffff0000
 
-/* SECTION_OFFSET is calculating the offset in bytes out of offsize */
-#define SECTION_OFFSET(_offsize)        ((((_offsize &              \
-                                            OFFSIZE_OFFSET_MASK) >> \
-                                           OFFSIZE_OFFSET_SHIFT) << 2))
+#define SECTION_OFFSET(_offsize) ((((_offsize &                 \
+                                     OFFSIZE_OFFSET_MASK) >>    \
+                                    OFFSIZE_OFFSET_SHIFT) << 2))
 
-/* QED_SECTION_SIZE is calculating the size in bytes out of offsize */
-#define QED_SECTION_SIZE(_offsize)              (((_offsize &            \
-                                                   OFFSIZE_SIZE_MASK) >> \
-                                                  OFFSIZE_SIZE_SHIFT) << 2)
+#define QED_SECTION_SIZE(_offsize) (((_offsize &                \
+                                      OFFSIZE_SIZE_MASK) >>     \
+                                     OFFSIZE_SIZE_SHIFT) << 2)
 
-/* SECTION_ADDR returns the GRC addr of a section, given offsize and index
- * within section.
- */
-#define SECTION_ADDR(_offsize, idx)     (MCP_REG_SCRATCH +          \
-                                         SECTION_OFFSET(_offsize) + \
-                                         (QED_SECTION_SIZE(_offsize) * idx))
+#define SECTION_ADDR(_offsize, idx) (MCP_REG_SCRATCH +                  \
+                                     SECTION_OFFSET(_offsize) +         \
+                                     (QED_SECTION_SIZE(_offsize) * idx))
+
+#define SECTION_OFFSIZE_ADDR(_pub_base, _section)       \
+        (_pub_base + offsetof(struct mcp_public_data, sections[_section]))
 
-/* SECTION_OFFSIZE_ADDR returns the GRC addr to the offsize address.
- * Use offsetof, since the OFFSETUP collide with the firmware definition
- */
-#define SECTION_OFFSIZE_ADDR(_pub_base, _section) (_pub_base +               \
-                                                   offsetof(struct           \
-                                                            mcp_public_data, \
-                                                            sections[_section]))
 /* PHY configuration */
-struct pmm_phy_cfg {
-        u32     speed;
-#define PMM_SPEED_AUTONEG   0
-
-        u32     pause;  /* bitmask */
-#define PMM_PAUSE_NONE          0x0
-#define PMM_PAUSE_AUTONEG       0x1
-#define PMM_PAUSE_RX            0x2
-#define PMM_PAUSE_TX            0x4
-
-        u32     adv_speed;  /* Default should be the speed_cap_mask */
-        u32     loopback_mode;
-#define PMM_LOOPBACK_NONE               0
-#define PMM_LOOPBACK_INT_PHY    1
-#define PMM_LOOPBACK_EXT_PHY    2
-#define PMM_LOOPBACK_EXT                3
-#define PMM_LOOPBACK_MAC                4
-
-        /* features */
+struct eth_phy_cfg {
+        u32 speed;
+#define ETH_SPEED_AUTONEG       0
+#define ETH_SPEED_SMARTLINQ     0x8
+
+        u32 pause;
+#define ETH_PAUSE_NONE          0x0
+#define ETH_PAUSE_AUTONEG       0x1
+#define ETH_PAUSE_RX            0x2
+#define ETH_PAUSE_TX            0x4
+
+        u32 adv_speed;
+        u32 loopback_mode;
+#define ETH_LOOPBACK_NONE               (0)
+#define ETH_LOOPBACK_INT_PHY            (1)
+#define ETH_LOOPBACK_EXT_PHY            (2)
+#define ETH_LOOPBACK_EXT                (3)
+#define ETH_LOOPBACK_MAC                (4)
+
         u32 feature_config_flags;
+#define ETH_EEE_MODE_ADV_LPI            (1 << 0)
 };
 
 struct port_mf_cfg {
-        u32     dynamic_cfg; /* device control channel */
-#define PORT_MF_CFG_OV_TAG_MASK              0x0000ffff
-#define PORT_MF_CFG_OV_TAG_SHIFT             0
-#define PORT_MF_CFG_OV_TAG_DEFAULT         PORT_MF_CFG_OV_TAG_MASK
-
-        u32     reserved[1];
-};
-
-/* DO NOT add new fields in the middle
- * MUST be synced with struct pmm_stats_map
- */
-struct pmm_stats {
-        u64     r64;    /* 0x00 (Offset 0x00 ) RX 64-byte frame counter*/
-        u64     r127;   /* 0x01 (Offset 0x08 ) RX 65 to 127 byte frame counter*/
-        u64     r255;
-        u64     r511;
-        u64     r1023;
-        u64     r1518;
-        u64     r1522;
-        u64     r2047;
-        u64     r4095;
-        u64     r9216;
-        u64     r16383;
-        u64     rfcs;   /* 0x0F (Offset 0x58 ) RX FCS error frame counter*/
-        u64     rxcf;   /* 0x10 (Offset 0x60 ) RX control frame counter*/
-        u64     rxpf;   /* 0x11 (Offset 0x68 ) RX pause frame counter*/
-        u64     rxpp;   /* 0x12 (Offset 0x70 ) RX PFC frame counter*/
-        u64     raln;   /* 0x16 (Offset 0x78 ) RX alignment error counter*/
-        u64     rfcr;   /* 0x19 (Offset 0x80 ) RX false carrier counter */
-        u64     rovr;   /* 0x1A (Offset 0x88 ) RX oversized frame counter*/
-        u64     rjbr;   /* 0x1B (Offset 0x90 ) RX jabber frame counter */
-        u64     rund;   /* 0x34 (Offset 0x98 ) RX undersized frame counter */
-        u64     rfrg;   /* 0x35 (Offset 0xa0 ) RX fragment counter */
-        u64     t64;    /* 0x40 (Offset 0xa8 ) TX 64-byte frame counter */
-        u64     t127;
-        u64     t255;
-        u64     t511;
-        u64     t1023;
-        u64     t1518;
-        u64     t2047;
-        u64     t4095;
-        u64     t9216;
-        u64     t16383;
-        u64     txpf;   /* 0x50 (Offset 0xf8 ) TX pause frame counter */
-        u64     txpp;   /* 0x51 (Offset 0x100) TX PFC frame counter */
-        u64     tlpiec;
-        u64     tncl;
-        u64     rbyte;  /* 0x3d (Offset 0x118) RX byte counter */
-        u64     rxuca;  /* 0x0c (Offset 0x120) RX UC frame counter */
-        u64     rxmca;  /* 0x0d (Offset 0x128) RX MC frame counter */
-        u64     rxbca;  /* 0x0e (Offset 0x130) RX BC frame counter */
-        u64     rxpok;
-        u64     tbyte;  /* 0x6f (Offset 0x140) TX byte counter */
-        u64     txuca;  /* 0x4d (Offset 0x148) TX UC frame counter */
-        u64     txmca;  /* 0x4e (Offset 0x150) TX MC frame counter */
-        u64     txbca;  /* 0x4f (Offset 0x158) TX BC frame counter */
-        u64     txcf;   /* 0x54 (Offset 0x160) TX control frame counter */
+        u32 dynamic_cfg;
+#define PORT_MF_CFG_OV_TAG_MASK         0x0000ffff
+#define PORT_MF_CFG_OV_TAG_SHIFT        0
+#define PORT_MF_CFG_OV_TAG_DEFAULT      PORT_MF_CFG_OV_TAG_MASK
+
+        u32 reserved[1];
+};
+
+struct eth_stats {
+        u64 r64;
+        u64 r127;
+        u64 r255;
+        u64 r511;
+        u64 r1023;
+        u64 r1518;
+        u64 r1522;
+        u64 r2047;
+        u64 r4095;
+        u64 r9216;
+        u64 r16383;
+        u64 rfcs;
+        u64 rxcf;
+        u64 rxpf;
+        u64 rxpp;
+        u64 raln;
+        u64 rfcr;
+        u64 rovr;
+        u64 rjbr;
+        u64 rund;
+        u64 rfrg;
+        u64 t64;
+        u64 t127;
+        u64 t255;
+        u64 t511;
+        u64 t1023;
+        u64 t1518;
+        u64 t2047;
+        u64 t4095;
+        u64 t9216;
+        u64 t16383;
+        u64 txpf;
+        u64 txpp;
+        u64 tlpiec;
+        u64 tncl;
+        u64 rbyte;
+        u64 rxuca;
+        u64 rxmca;
+        u64 rxbca;
+        u64 rxpok;
+        u64 tbyte;
+        u64 txuca;
+        u64 txmca;
+        u64 txbca;
+        u64 txcf;
 };
 
 struct brb_stats {
-        u64     brb_truncate[8];
-        u64     brb_discard[8];
+        u64 brb_truncate[8];
+        u64 brb_discard[8];
 };
 
 struct port_stats {
-        struct brb_stats        brb;
-        struct pmm_stats        pmm;
+        struct brb_stats brb;
+        struct eth_stats eth;
 };
 
-#define CMT_TEAM0 0
-#define CMT_TEAM1 1
-#define CMT_TEAM_MAX 2
-
 struct couple_mode_teaming {
         u8 port_cmt[MCP_GLOB_PORT_MAX];
-#define PORT_CMT_IN_TEAM                BIT(0)
+#define PORT_CMT_IN_TEAM        (1 << 0)
 
-#define PORT_CMT_PORT_ROLE              BIT(1)
-#define PORT_CMT_PORT_INACTIVE      (0 << 1)
-#define PORT_CMT_PORT_ACTIVE            BIT(1)
+#define PORT_CMT_PORT_ROLE      (1 << 1)
+#define PORT_CMT_PORT_INACTIVE  (0 << 1)
+#define PORT_CMT_PORT_ACTIVE    (1 << 1)
 
-#define PORT_CMT_TEAM_MASK              BIT(2)
-#define PORT_CMT_TEAM0              (0 << 2)
-#define PORT_CMT_TEAM1                  BIT(2)
+#define PORT_CMT_TEAM_MASK      (1 << 2)
+#define PORT_CMT_TEAM0          (0 << 2)
+#define PORT_CMT_TEAM1          (1 << 2)
 };
 
-/**************************************
-*     LLDP and DCBX HSI structures
-**************************************/
-#define LLDP_CHASSIS_ID_STAT_LEN 4
-#define LLDP_PORT_ID_STAT_LEN 4
-#define DCBX_MAX_APP_PROTOCOL           32
-#define MAX_SYSTEM_LLDP_TLV_DATA    32
+#define LLDP_CHASSIS_ID_STAT_LEN        4
+#define LLDP_PORT_ID_STAT_LEN           4
+#define DCBX_MAX_APP_PROTOCOL           32
+#define MAX_SYSTEM_LLDP_TLV_DATA        32
 
-enum lldp_agent_e {
+enum _lldp_agent {
         LLDP_NEAREST_BRIDGE = 0,
         LLDP_NEAREST_NON_TPMR_BRIDGE,
         LLDP_NEAREST_CUSTOMER_BRIDGE,
@@ -3394,690 +6786,517 @@ enum lldp_agent_e {
 
 struct lldp_config_params_s {
         u32 config;
-#define LLDP_CONFIG_TX_INTERVAL_MASK        0x000000ff
-#define LLDP_CONFIG_TX_INTERVAL_SHIFT       0
-#define LLDP_CONFIG_HOLD_MASK               0x00000f00
-#define LLDP_CONFIG_HOLD_SHIFT              8
-#define LLDP_CONFIG_MAX_CREDIT_MASK         0x0000f000
-#define LLDP_CONFIG_MAX_CREDIT_SHIFT        12
-#define LLDP_CONFIG_ENABLE_RX_MASK          0x40000000
-#define LLDP_CONFIG_ENABLE_RX_SHIFT         30
-#define LLDP_CONFIG_ENABLE_TX_MASK          0x80000000
-#define LLDP_CONFIG_ENABLE_TX_SHIFT         31
-        u32     local_chassis_id[LLDP_CHASSIS_ID_STAT_LEN];
-        u32     local_port_id[LLDP_PORT_ID_STAT_LEN];
+#define LLDP_CONFIG_TX_INTERVAL_MASK    0x000000ff
+#define LLDP_CONFIG_TX_INTERVAL_SHIFT   0
+#define LLDP_CONFIG_HOLD_MASK           0x00000f00
+#define LLDP_CONFIG_HOLD_SHIFT          8
+#define LLDP_CONFIG_MAX_CREDIT_MASK     0x0000f000
+#define LLDP_CONFIG_MAX_CREDIT_SHIFT    12
+#define LLDP_CONFIG_ENABLE_RX_MASK      0x40000000
+#define LLDP_CONFIG_ENABLE_RX_SHIFT     30
+#define LLDP_CONFIG_ENABLE_TX_MASK      0x80000000
+#define LLDP_CONFIG_ENABLE_TX_SHIFT     31
+        u32 local_chassis_id[LLDP_CHASSIS_ID_STAT_LEN];
+        u32 local_port_id[LLDP_PORT_ID_STAT_LEN];
 };
 
 struct lldp_status_params_s {
-        u32     prefix_seq_num;
-        u32     status; /* TBD */
-
-        /* Holds remote Chassis ID TLV header, subtype and 9B of payload. */
-        u32     peer_chassis_id[LLDP_CHASSIS_ID_STAT_LEN];
-
-        /* Holds remote Port ID TLV header, subtype and 9B of payload. */
-        u32     peer_port_id[LLDP_PORT_ID_STAT_LEN];
-        u32     suffix_seq_num;
+        u32 prefix_seq_num;
+        u32 status;
+        u32 peer_chassis_id[LLDP_CHASSIS_ID_STAT_LEN];
+        u32 peer_port_id[LLDP_PORT_ID_STAT_LEN];
+        u32 suffix_seq_num;
 };
 
 struct dcbx_ets_feature {
         u32 flags;
-#define DCBX_ETS_ENABLED_MASK                   0x00000001
-#define DCBX_ETS_ENABLED_SHIFT                  0
-#define DCBX_ETS_WILLING_MASK                   0x00000002
-#define DCBX_ETS_WILLING_SHIFT                  1
-#define DCBX_ETS_ERROR_MASK                     0x00000004
-#define DCBX_ETS_ERROR_SHIFT                    2
-#define DCBX_ETS_CBS_MASK                       0x00000008
-#define DCBX_ETS_CBS_SHIFT                      3
-#define DCBX_ETS_MAX_TCS_MASK                   0x000000f0
-#define DCBX_ETS_MAX_TCS_SHIFT                  4
-        u32     pri_tc_tbl[1];
-#define DCBX_ISCSI_OOO_TC                       4
-#define NIG_ETS_ISCSI_OOO_CLIENT_OFFSET         (DCBX_ISCSI_OOO_TC + 1)
-        u32     tc_bw_tbl[2];
-        u32     tc_tsa_tbl[2];
-#define DCBX_ETS_TSA_STRICT                     0
-#define DCBX_ETS_TSA_CBS                        1
-#define DCBX_ETS_TSA_ETS                        2
+#define DCBX_ETS_ENABLED_MASK   0x00000001
+#define DCBX_ETS_ENABLED_SHIFT  0
+#define DCBX_ETS_WILLING_MASK   0x00000002
+#define DCBX_ETS_WILLING_SHIFT  1
+#define DCBX_ETS_ERROR_MASK     0x00000004
+#define DCBX_ETS_ERROR_SHIFT    2
+#define DCBX_ETS_CBS_MASK       0x00000008
+#define DCBX_ETS_CBS_SHIFT      3
+#define DCBX_ETS_MAX_TCS_MASK   0x000000f0
+#define DCBX_ETS_MAX_TCS_SHIFT  4
+#define DCBX_ISCSI_OOO_TC_MASK  0x00000f00
+#define DCBX_ISCSI_OOO_TC_SHIFT 8
+        u32 pri_tc_tbl[1];
+#define DCBX_ISCSI_OOO_TC       (4)
+
+#define NIG_ETS_ISCSI_OOO_CLIENT_OFFSET (DCBX_ISCSI_OOO_TC + 1)
+#define DCBX_CEE_STRICT_PRIORITY        0xf
+        u32 tc_bw_tbl[2];
+        u32 tc_tsa_tbl[2];
+#define DCBX_ETS_TSA_STRICT     0
+#define DCBX_ETS_TSA_CBS        1
+#define DCBX_ETS_TSA_ETS        2
 };
 
 struct dcbx_app_priority_entry {
         u32 entry;
-#define DCBX_APP_PRI_MAP_MASK       0x000000ff
-#define DCBX_APP_PRI_MAP_SHIFT      0
-#define DCBX_APP_PRI_0              0x01
-#define DCBX_APP_PRI_1              0x02
-#define DCBX_APP_PRI_2              0x04
-#define DCBX_APP_PRI_3              0x08
-#define DCBX_APP_PRI_4              0x10
-#define DCBX_APP_PRI_5              0x20
-#define DCBX_APP_PRI_6              0x40
-#define DCBX_APP_PRI_7              0x80
-#define DCBX_APP_SF_MASK            0x00000300
-#define DCBX_APP_SF_SHIFT           8
-#define DCBX_APP_SF_ETHTYPE         0
-#define DCBX_APP_SF_PORT            1
-#define DCBX_APP_PROTOCOL_ID_MASK   0xffff0000
-#define DCBX_APP_PROTOCOL_ID_SHIFT  16
-};
-
-/* FW structure in BE */
+#define DCBX_APP_PRI_MAP_MASK           0x000000ff
+#define DCBX_APP_PRI_MAP_SHIFT          0
+#define DCBX_APP_PRI_0                  0x01
+#define DCBX_APP_PRI_1                  0x02
+#define DCBX_APP_PRI_2                  0x04
+#define DCBX_APP_PRI_3                  0x08
+#define DCBX_APP_PRI_4                  0x10
+#define DCBX_APP_PRI_5                  0x20
+#define DCBX_APP_PRI_6                  0x40
+#define DCBX_APP_PRI_7                  0x80
+#define DCBX_APP_SF_MASK                0x00000300
+#define DCBX_APP_SF_SHIFT               8
+#define DCBX_APP_SF_ETHTYPE             0
+#define DCBX_APP_SF_PORT                1
+#define DCBX_APP_PROTOCOL_ID_MASK       0xffff0000
+#define DCBX_APP_PROTOCOL_ID_SHIFT      16
+};
+
 struct dcbx_app_priority_feature {
         u32 flags;
-#define DCBX_APP_ENABLED_MASK           0x00000001
-#define DCBX_APP_ENABLED_SHIFT          0
-#define DCBX_APP_WILLING_MASK           0x00000002
-#define DCBX_APP_WILLING_SHIFT          1
-#define DCBX_APP_ERROR_MASK             0x00000004
-#define DCBX_APP_ERROR_SHIFT            2
-/* Not in use
- * #define DCBX_APP_DEFAULT_PRI_MASK       0x00000f00
- * #define DCBX_APP_DEFAULT_PRI_SHIFT      8
- */
-#define DCBX_APP_MAX_TCS_MASK           0x0000f000
-#define DCBX_APP_MAX_TCS_SHIFT          12
-#define DCBX_APP_NUM_ENTRIES_MASK       0x00ff0000
-#define DCBX_APP_NUM_ENTRIES_SHIFT      16
+#define DCBX_APP_ENABLED_MASK           0x00000001
+#define DCBX_APP_ENABLED_SHIFT          0
+#define DCBX_APP_WILLING_MASK           0x00000002
+#define DCBX_APP_WILLING_SHIFT          1
+#define DCBX_APP_ERROR_MASK             0x00000004
+#define DCBX_APP_ERROR_SHIFT            2
+#define DCBX_APP_MAX_TCS_MASK           0x0000f000
+#define DCBX_APP_MAX_TCS_SHIFT          12
+#define DCBX_APP_NUM_ENTRIES_MASK       0x00ff0000
+#define DCBX_APP_NUM_ENTRIES_SHIFT      16
         struct dcbx_app_priority_entry app_pri_tbl[DCBX_MAX_APP_PROTOCOL];
 };
 
-/* FW structure in BE */
 struct dcbx_features {
-        /* PG feature */
         struct dcbx_ets_feature ets;
+        u32 pfc;
+#define DCBX_PFC_PRI_EN_BITMAP_MASK     0x000000ff
+#define DCBX_PFC_PRI_EN_BITMAP_SHIFT    0
+#define DCBX_PFC_PRI_EN_BITMAP_PRI_0    0x01
+#define DCBX_PFC_PRI_EN_BITMAP_PRI_1    0x02
+#define DCBX_PFC_PRI_EN_BITMAP_PRI_2    0x04
+#define DCBX_PFC_PRI_EN_BITMAP_PRI_3    0x08
+#define DCBX_PFC_PRI_EN_BITMAP_PRI_4    0x10
+#define DCBX_PFC_PRI_EN_BITMAP_PRI_5    0x20
+#define DCBX_PFC_PRI_EN_BITMAP_PRI_6    0x40
+#define DCBX_PFC_PRI_EN_BITMAP_PRI_7    0x80
+
+#define DCBX_PFC_FLAGS_MASK             0x0000ff00
+#define DCBX_PFC_FLAGS_SHIFT            8
+#define DCBX_PFC_CAPS_MASK              0x00000f00
+#define DCBX_PFC_CAPS_SHIFT             8
+#define DCBX_PFC_MBC_MASK               0x00004000
+#define DCBX_PFC_MBC_SHIFT              14
+#define DCBX_PFC_WILLING_MASK           0x00008000
+#define DCBX_PFC_WILLING_SHIFT          15
+#define DCBX_PFC_ENABLED_MASK           0x00010000
+#define DCBX_PFC_ENABLED_SHIFT          16
+#define DCBX_PFC_ERROR_MASK             0x00020000
+#define DCBX_PFC_ERROR_SHIFT            17
 
-        /* PFC feature */
-        u32                     pfc;
-#define DCBX_PFC_PRI_EN_BITMAP_MASK             0x000000ff
-#define DCBX_PFC_PRI_EN_BITMAP_SHIFT            0
-#define DCBX_PFC_PRI_EN_BITMAP_PRI_0            0x01
-#define DCBX_PFC_PRI_EN_BITMAP_PRI_1            0x02
-#define DCBX_PFC_PRI_EN_BITMAP_PRI_2            0x04
-#define DCBX_PFC_PRI_EN_BITMAP_PRI_3            0x08
-#define DCBX_PFC_PRI_EN_BITMAP_PRI_4            0x10
-#define DCBX_PFC_PRI_EN_BITMAP_PRI_5            0x20
-#define DCBX_PFC_PRI_EN_BITMAP_PRI_6            0x40
-#define DCBX_PFC_PRI_EN_BITMAP_PRI_7            0x80
-
-#define DCBX_PFC_FLAGS_MASK                     0x0000ff00
-#define DCBX_PFC_FLAGS_SHIFT                    8
-#define DCBX_PFC_CAPS_MASK                      0x00000f00
-#define DCBX_PFC_CAPS_SHIFT                     8
-#define DCBX_PFC_MBC_MASK                       0x00004000
-#define DCBX_PFC_MBC_SHIFT                      14
-#define DCBX_PFC_WILLING_MASK                   0x00008000
-#define DCBX_PFC_WILLING_SHIFT                  15
-#define DCBX_PFC_ENABLED_MASK                   0x00010000
-#define DCBX_PFC_ENABLED_SHIFT                  16
-#define DCBX_PFC_ERROR_MASK                     0x00020000
-#define DCBX_PFC_ERROR_SHIFT                    17
-
-        /* APP feature */
         struct dcbx_app_priority_feature app;
 };
 
 struct dcbx_local_params {
         u32 config;
-#define DCBX_CONFIG_VERSION_MASK            0x00000003
-#define DCBX_CONFIG_VERSION_SHIFT           0
-#define DCBX_CONFIG_VERSION_DISABLED        0
-#define DCBX_CONFIG_VERSION_IEEE            1
-#define DCBX_CONFIG_VERSION_CEE             2
+#define DCBX_CONFIG_VERSION_MASK        0x00000007
+#define DCBX_CONFIG_VERSION_SHIFT       0
+#define DCBX_CONFIG_VERSION_DISABLED    0
+#define DCBX_CONFIG_VERSION_IEEE        1
+#define DCBX_CONFIG_VERSION_CEE         2
+#define DCBX_CONFIG_VERSION_STATIC      4
 
-        u32                     flags;
-        struct dcbx_features    features;
+        u32 flags;
+        struct dcbx_features features;
 };
 
 struct dcbx_mib {
-        u32     prefix_seq_num;
-        u32     flags;
-        struct dcbx_features    features;
-        u32                     suffix_seq_num;
+        u32 prefix_seq_num;
+        u32 flags;
+        struct dcbx_features features;
+        u32 suffix_seq_num;
 };
 
 struct lldp_system_tlvs_buffer_s {
-        u16     valid;
-        u16     length;
-        u32     data[MAX_SYSTEM_LLDP_TLV_DATA];
+        u16 valid;
+        u16 length;
+        u32 data[MAX_SYSTEM_LLDP_TLV_DATA];
 };
 
-/**************************************/
-/*                                    */
-/*     P U B L I C      G L O B A L   */
-/*                                    */
-/**************************************/
-struct public_global {
-        u32                             max_path;
-#define MAX_PATH_BIG_BEAR       2
-#define MAX_PATH_K2             1
-        u32                             max_ports;
-#define MODE_1P 1
-#define MODE_2P 2
-#define MODE_3P 3
-#define MODE_4P 4
-        u32                             debug_mb_offset;
-        u32                             phymod_dbg_mb_offset;
-        struct couple_mode_teaming      cmt;
-        s32                             internal_temperature;
-        u32                             mfw_ver;
-        u32                             running_bundle_id;
+struct dcb_dscp_map {
+        u32 flags;
+#define DCB_DSCP_ENABLE_MASK    0x1
+#define DCB_DSCP_ENABLE_SHIFT   0
+#define DCB_DSCP_ENABLE 1
+        u32 dscp_pri_map[8];
 };
 
-/**************************************/
-/*                                    */
-/*     P U B L I C      P A T H       */
-/*                                    */
-/**************************************/
+struct public_global {
+        u32 max_path;
+        u32 max_ports;
+        u32 debug_mb_offset;
+        u32 phymod_dbg_mb_offset;
+        struct couple_mode_teaming cmt;
+        s32 internal_temperature;
+        u32 mfw_ver;
+        u32 running_bundle_id;
+        s32 external_temperature;
+        u32 mdump_reason;
+};
 
-/****************************************************************************
-* Shared Memory 2 Region                                                   *
-****************************************************************************/
-/* The fw_flr_ack is actually built in the following way:                   */
-/* 8 bit:  PF ack                                                           */
-/* 128 bit: VF ack                                                           */
-/* 8 bit:  ios_dis_ack                                                      */
-/* In order to maintain endianity in the mailbox hsi, we want to keep using */
-/* u32. The fw must have the VF right after the PF since this is how it     */
-/* access arrays(it expects always the VF to reside after the PF, and that  */
-/* makes the calculation much easier for it. )                              */
-/* In order to answer both limitations, and keep the struct small, the code */
-/* will abuse the structure defined here to achieve the actual partition    */
-/* above                                                                    */
-/****************************************************************************/
 struct fw_flr_mb {
-        u32     aggint;
-        u32     opgen_addr;
-        u32     accum_ack;  /* 0..15:PF, 16..207:VF, 256..271:IOV_DIS */
-#define ACCUM_ACK_PF_BASE       0
-#define ACCUM_ACK_PF_SHIFT      0
-
-#define ACCUM_ACK_VF_BASE       8
-#define ACCUM_ACK_VF_SHIFT      3
-
-#define ACCUM_ACK_IOV_DIS_BASE  256
-#define ACCUM_ACK_IOV_DIS_SHIFT 8
+        u32 aggint;
+        u32 opgen_addr;
+        u32 accum_ack;
 };
 
 struct public_path {
-        struct fw_flr_mb        flr_mb;
-        u32                     mcp_vf_disabled[VF_MAX_STATIC / 32];
-
-        u32                     process_kill;
-#define PROCESS_KILL_COUNTER_MASK               0x0000ffff
-#define PROCESS_KILL_COUNTER_SHIFT              0
-#define PROCESS_KILL_GLOB_AEU_BIT_MASK          0xffff0000
-#define PROCESS_KILL_GLOB_AEU_BIT_SHIFT         16
+        struct fw_flr_mb flr_mb;
+        u32 mcp_vf_disabled[VF_MAX_STATIC / 32];
+
+        u32 process_kill;
+#define PROCESS_KILL_COUNTER_MASK       0x0000ffff
+#define PROCESS_KILL_COUNTER_SHIFT      0
+#define PROCESS_KILL_GLOB_AEU_BIT_MASK  0xffff0000
+#define PROCESS_KILL_GLOB_AEU_BIT_SHIFT 16
 #define GLOBAL_AEU_BIT(aeu_reg_id, aeu_bit) (aeu_reg_id * 32 + aeu_bit)
 };
 
-/**************************************/
-/*                                    */
-/*     P U B L I C      P O R T       */
-/*                                    */
-/**************************************/
-
-/****************************************************************************
-* Driver <-> FW Mailbox                                                    *
-****************************************************************************/
-
 struct public_port {
-        u32 validity_map;   /* 0x0 (4*2 = 0x8) */
-
-        /* validity bits */
-#define MCP_VALIDITY_PCI_CFG                    0x00100000
-#define MCP_VALIDITY_MB                         0x00200000
-#define MCP_VALIDITY_DEV_INFO                   0x00400000
-#define MCP_VALIDITY_RESERVED                   0x00000007
-
-        /* One licensing bit should be set */
-#define MCP_VALIDITY_LIC_KEY_IN_EFFECT_MASK     0x00000038
-#define MCP_VALIDITY_LIC_MANUF_KEY_IN_EFFECT    0x00000008
-#define MCP_VALIDITY_LIC_UPGRADE_KEY_IN_EFFECT  0x00000010
-#define MCP_VALIDITY_LIC_NO_KEY_IN_EFFECT       0x00000020
-
-        /* Active MFW */
-#define MCP_VALIDITY_ACTIVE_MFW_UNKNOWN         0x00000000
-#define MCP_VALIDITY_ACTIVE_MFW_MASK            0x000001c0
-#define MCP_VALIDITY_ACTIVE_MFW_NCSI            0x00000040
-#define MCP_VALIDITY_ACTIVE_MFW_NONE            0x000001c0
+        u32 validity_map;
 
         u32 link_status;
-#define LINK_STATUS_LINK_UP \
-        0x00000001
-#define LINK_STATUS_SPEED_AND_DUPLEX_MASK                       0x0000001e
-#define LINK_STATUS_SPEED_AND_DUPLEX_1000THD            BIT(1)
-#define LINK_STATUS_SPEED_AND_DUPLEX_1000TFD            (2 << 1)
-#define LINK_STATUS_SPEED_AND_DUPLEX_10G                        (3 << 1)
-#define LINK_STATUS_SPEED_AND_DUPLEX_20G                        (4 << 1)
-#define LINK_STATUS_SPEED_AND_DUPLEX_40G                        (5 << 1)
-#define LINK_STATUS_SPEED_AND_DUPLEX_50G                        (6 << 1)
-#define LINK_STATUS_SPEED_AND_DUPLEX_100G                       (7 << 1)
-#define LINK_STATUS_SPEED_AND_DUPLEX_25G                        (8 << 1)
-
-#define LINK_STATUS_AUTO_NEGOTIATE_ENABLED                      0x00000020
-
-#define LINK_STATUS_AUTO_NEGOTIATE_COMPLETE                     0x00000040
-#define LINK_STATUS_PARALLEL_DETECTION_USED                     0x00000080
-
-#define LINK_STATUS_PFC_ENABLED \
-        0x00000100
-#define LINK_STATUS_LINK_PARTNER_1000TFD_CAPABLE        0x00000200
-#define LINK_STATUS_LINK_PARTNER_1000THD_CAPABLE        0x00000400
-#define LINK_STATUS_LINK_PARTNER_10G_CAPABLE            0x00000800
-#define LINK_STATUS_LINK_PARTNER_20G_CAPABLE            0x00001000
-#define LINK_STATUS_LINK_PARTNER_40G_CAPABLE            0x00002000
-#define LINK_STATUS_LINK_PARTNER_50G_CAPABLE            0x00004000
-#define LINK_STATUS_LINK_PARTNER_100G_CAPABLE           0x00008000
-#define LINK_STATUS_LINK_PARTNER_25G_CAPABLE            0x00010000
-
-#define LINK_STATUS_LINK_PARTNER_FLOW_CONTROL_MASK      0x000C0000
-#define LINK_STATUS_LINK_PARTNER_NOT_PAUSE_CAPABLE      (0 << 18)
-#define LINK_STATUS_LINK_PARTNER_SYMMETRIC_PAUSE        BIT(18)
-#define LINK_STATUS_LINK_PARTNER_ASYMMETRIC_PAUSE       (2 << 18)
-#define LINK_STATUS_LINK_PARTNER_BOTH_PAUSE                     (3 << 18)
-
-#define LINK_STATUS_SFP_TX_FAULT \
-        0x00100000
-#define LINK_STATUS_TX_FLOW_CONTROL_ENABLED                     0x00200000
-#define LINK_STATUS_RX_FLOW_CONTROL_ENABLED                     0x00400000
-
-        u32                     link_status1;
-        u32                     ext_phy_fw_version;
-        u32                     drv_phy_cfg_addr;
-
-        u32                     port_stx;
-
-        u32                     stat_nig_timer;
-
-        struct port_mf_cfg      port_mf_config;
-        struct port_stats       stats;
-
-        u32                     media_type;
-#define MEDIA_UNSPECIFIED       0x0
-#define MEDIA_SFPP_10G_FIBER    0x1
-#define MEDIA_XFP_FIBER         0x2
-#define MEDIA_DA_TWINAX         0x3
-#define MEDIA_BASE_T            0x4
-#define MEDIA_SFP_1G_FIBER      0x5
-#define MEDIA_MODULE_FIBER      0x6
-#define MEDIA_KR                0xf0
-#define MEDIA_NOT_PRESENT       0xff
+#define LINK_STATUS_LINK_UP                     0x00000001
+#define LINK_STATUS_SPEED_AND_DUPLEX_MASK       0x0000001e
+#define LINK_STATUS_SPEED_AND_DUPLEX_1000THD    (1 << 1)
+#define LINK_STATUS_SPEED_AND_DUPLEX_1000TFD    (2 << 1)
+#define LINK_STATUS_SPEED_AND_DUPLEX_10G        (3 << 1)
+#define LINK_STATUS_SPEED_AND_DUPLEX_20G        (4 << 1)
+#define LINK_STATUS_SPEED_AND_DUPLEX_40G        (5 << 1)
+#define LINK_STATUS_SPEED_AND_DUPLEX_50G        (6 << 1)
+#define LINK_STATUS_SPEED_AND_DUPLEX_100G       (7 << 1)
+#define LINK_STATUS_SPEED_AND_DUPLEX_25G        (8 << 1)
+
+#define LINK_STATUS_AUTO_NEGOTIATE_ENABLED      0x00000020
+
+#define LINK_STATUS_AUTO_NEGOTIATE_COMPLETE     0x00000040
+#define LINK_STATUS_PARALLEL_DETECTION_USED     0x00000080
+
+#define LINK_STATUS_PFC_ENABLED                         0x00000100
+#define LINK_STATUS_LINK_PARTNER_1000TFD_CAPABLE 0x00000200
+#define LINK_STATUS_LINK_PARTNER_1000THD_CAPABLE 0x00000400
+#define LINK_STATUS_LINK_PARTNER_10G_CAPABLE            0x00000800
+#define LINK_STATUS_LINK_PARTNER_20G_CAPABLE            0x00001000
+#define LINK_STATUS_LINK_PARTNER_40G_CAPABLE            0x00002000
+#define LINK_STATUS_LINK_PARTNER_50G_CAPABLE            0x00004000
+#define LINK_STATUS_LINK_PARTNER_100G_CAPABLE           0x00008000
+#define LINK_STATUS_LINK_PARTNER_25G_CAPABLE            0x00010000
+
+#define LINK_STATUS_LINK_PARTNER_FLOW_CONTROL_MASK      0x000C0000
+#define LINK_STATUS_LINK_PARTNER_NOT_PAUSE_CAPABLE      (0 << 18)
+#define LINK_STATUS_LINK_PARTNER_SYMMETRIC_PAUSE        (1 << 18)
+#define LINK_STATUS_LINK_PARTNER_ASYMMETRIC_PAUSE       (2 << 18)
+#define LINK_STATUS_LINK_PARTNER_BOTH_PAUSE             (3 << 18)
+
+#define LINK_STATUS_SFP_TX_FAULT                        0x00100000
+#define LINK_STATUS_TX_FLOW_CONTROL_ENABLED             0x00200000
+#define LINK_STATUS_RX_FLOW_CONTROL_ENABLED             0x00400000
+#define LINK_STATUS_RX_SIGNAL_PRESENT                   0x00800000
+#define LINK_STATUS_MAC_LOCAL_FAULT                     0x01000000
+#define LINK_STATUS_MAC_REMOTE_FAULT                    0x02000000
+#define LINK_STATUS_UNSUPPORTED_SPD_REQ                 0x04000000
+
+        u32 link_status1;
+        u32 ext_phy_fw_version;
+        u32 drv_phy_cfg_addr;
+
+        u32 port_stx;
+
+        u32 stat_nig_timer;
+
+        struct port_mf_cfg port_mf_config;
+        struct port_stats stats;
+
+        u32 media_type;
+#define MEDIA_UNSPECIFIED       0x0
+#define MEDIA_SFPP_10G_FIBER    0x1
+#define MEDIA_XFP_FIBER         0x2
+#define MEDIA_DA_TWINAX         0x3
+#define MEDIA_BASE_T            0x4
+#define MEDIA_SFP_1G_FIBER      0x5
+#define MEDIA_MODULE_FIBER      0x6
+#define MEDIA_KR                0xf0
+#define MEDIA_NOT_PRESENT       0xff
 
         u32 lfa_status;
-#define LFA_LINK_FLAP_REASON_OFFSET             0
-#define LFA_LINK_FLAP_REASON_MASK               0x000000ff
-#define LFA_NO_REASON                                   (0 << 0)
-#define LFA_LINK_DOWN                                   BIT(0)
-#define LFA_FORCE_INIT                                  BIT(1)
-#define LFA_LOOPBACK_MISMATCH                           BIT(2)
-#define LFA_SPEED_MISMATCH                              BIT(3)
-#define LFA_FLOW_CTRL_MISMATCH                          BIT(4)
-#define LFA_ADV_SPEED_MISMATCH                          BIT(5)
-#define LINK_FLAP_AVOIDANCE_COUNT_OFFSET        8
-#define LINK_FLAP_AVOIDANCE_COUNT_MASK          0x0000ff00
-#define LINK_FLAP_COUNT_OFFSET                  16
-#define LINK_FLAP_COUNT_MASK                    0x00ff0000
-
-        u32                                     link_change_count;
-
-        /* LLDP params */
-        struct lldp_config_params_s             lldp_config_params[
-                LLDP_MAX_LLDP_AGENTS];
-        struct lldp_status_params_s             lldp_status_params[
-                LLDP_MAX_LLDP_AGENTS];
-        struct lldp_system_tlvs_buffer_s        system_lldp_tlvs_buf;
+        u32 link_change_count;
+
+        struct lldp_config_params_s lldp_config_params[LLDP_MAX_LLDP_AGENTS];
+        struct lldp_status_params_s lldp_status_params[LLDP_MAX_LLDP_AGENTS];
+        struct lldp_system_tlvs_buffer_s system_lldp_tlvs_buf;
 
         /* DCBX related MIB */
-        struct dcbx_local_params                local_admin_dcbx_mib;
-        struct dcbx_mib                         remote_dcbx_mib;
-        struct dcbx_mib                         operational_dcbx_mib;
+        struct dcbx_local_params local_admin_dcbx_mib;
+        struct dcbx_mib remote_dcbx_mib;
+        struct dcbx_mib operational_dcbx_mib;
 
-        u32                                     fc_npiv_nvram_tbl_addr;
-        u32                                     fc_npiv_nvram_tbl_size;
-        u32                                     transceiver_data;
-#define PMM_TRANSCEIVER_STATE_MASK              0x000000FF
-#define PMM_TRANSCEIVER_STATE_SHIFT             0x00000000
-#define PMM_TRANSCEIVER_STATE_PRESENT           0x00000001
-};
+        u32 reserved[2];
+        u32 transceiver_data;
+#define ETH_TRANSCEIVER_STATE_MASK      0x000000FF
+#define ETH_TRANSCEIVER_STATE_SHIFT     0x00000000
+#define ETH_TRANSCEIVER_STATE_UNPLUGGED 0x00000000
+#define ETH_TRANSCEIVER_STATE_PRESENT   0x00000001
+#define ETH_TRANSCEIVER_STATE_VALID     0x00000003
+#define ETH_TRANSCEIVER_STATE_UPDATING  0x00000008
 
-/**************************************/
-/*                                    */
-/*     P U B L I C      F U N C       */
-/*                                    */
-/**************************************/
+        u32 wol_info;
+        u32 wol_pkt_len;
+        u32 wol_pkt_details;
+        struct dcb_dscp_map dcb_dscp_map;
+};
 
 struct public_func {
-        u32     iscsi_boot_signature;
-        u32     iscsi_boot_block_offset;
-
-        u32     mtu_size;
-        u32     c2s_pcp_map_lower;
-        u32     c2s_pcp_map_upper;
-        u32     c2s_pcp_map_default;
-        u32     reserved[4];
-
-        u32     config;
-
-        /* E/R/I/D */
-        /* function 0 of each port cannot be hidden */
-#define FUNC_MF_CFG_FUNC_HIDE                   0x00000001
-#define FUNC_MF_CFG_PAUSE_ON_HOST_RING          0x00000002
-#define FUNC_MF_CFG_PAUSE_ON_HOST_RING_SHIFT    0x00000001
-
-#define FUNC_MF_CFG_PROTOCOL_MASK               0x000000f0
-#define FUNC_MF_CFG_PROTOCOL_SHIFT              4
-#define FUNC_MF_CFG_PROTOCOL_ETHERNET           0x00000000
+        u32 reserved0[2];
+
+        u32 mtu_size;
+
+        u32 reserved[7];
+
+        u32 config;
+#define FUNC_MF_CFG_FUNC_HIDE                   0x00000001
+#define FUNC_MF_CFG_PAUSE_ON_HOST_RING          0x00000002
+#define FUNC_MF_CFG_PAUSE_ON_HOST_RING_SHIFT    0x00000001
+
+#define FUNC_MF_CFG_PROTOCOL_MASK       0x000000f0
+#define FUNC_MF_CFG_PROTOCOL_SHIFT      4
+#define FUNC_MF_CFG_PROTOCOL_ETHERNET   0x00000000
 #define FUNC_MF_CFG_PROTOCOL_ISCSI              0x00000010
-#define FUNC_MF_CFG_PROTOCOL_FCOE               0x00000020
 #define FUNC_MF_CFG_PROTOCOL_ROCE               0x00000030
-#define FUNC_MF_CFG_PROTOCOL_MAX                0x00000030
+#define FUNC_MF_CFG_PROTOCOL_MAX        0x00000030
 
-        /* MINBW, MAXBW */
-        /* value range - 0..100, increments in 1 %  */
-#define FUNC_MF_CFG_MIN_BW_MASK                 0x0000ff00
-#define FUNC_MF_CFG_MIN_BW_SHIFT                8
-#define FUNC_MF_CFG_MIN_BW_DEFAULT              0x00000000
-#define FUNC_MF_CFG_MAX_BW_MASK                 0x00ff0000
-#define FUNC_MF_CFG_MAX_BW_SHIFT                16
-#define FUNC_MF_CFG_MAX_BW_DEFAULT              0x00640000
+#define FUNC_MF_CFG_MIN_BW_MASK         0x0000ff00
+#define FUNC_MF_CFG_MIN_BW_SHIFT        8
+#define FUNC_MF_CFG_MIN_BW_DEFAULT      0x00000000
+#define FUNC_MF_CFG_MAX_BW_MASK         0x00ff0000
+#define FUNC_MF_CFG_MAX_BW_SHIFT        16
+#define FUNC_MF_CFG_MAX_BW_DEFAULT      0x00640000
 
-        u32     status;
-#define FUNC_STATUS_VLINK_DOWN                  0x00000001
+        u32 status;
+#define FUNC_STATUS_VLINK_DOWN          0x00000001
 
-        u32     mac_upper;  /* MAC */
-#define FUNC_MF_CFG_UPPERMAC_MASK               0x0000ffff
-#define FUNC_MF_CFG_UPPERMAC_SHIFT              0
-#define FUNC_MF_CFG_UPPERMAC_DEFAULT            FUNC_MF_CFG_UPPERMAC_MASK
-        u32     mac_lower;
-#define FUNC_MF_CFG_LOWERMAC_DEFAULT            0xffffffff
+        u32 mac_upper;
+#define FUNC_MF_CFG_UPPERMAC_MASK       0x0000ffff
+#define FUNC_MF_CFG_UPPERMAC_SHIFT      0
+#define FUNC_MF_CFG_UPPERMAC_DEFAULT    FUNC_MF_CFG_UPPERMAC_MASK
+        u32 mac_lower;
+#define FUNC_MF_CFG_LOWERMAC_DEFAULT    0xffffffff
 
-        u32     fcoe_wwn_port_name_upper;
-        u32     fcoe_wwn_port_name_lower;
+        u32 fcoe_wwn_port_name_upper;
+        u32 fcoe_wwn_port_name_lower;
 
-        u32     fcoe_wwn_node_name_upper;
-        u32     fcoe_wwn_node_name_lower;
+        u32 fcoe_wwn_node_name_upper;
+        u32 fcoe_wwn_node_name_lower;
 
-        u32     ovlan_stag; /* tags */
-#define FUNC_MF_CFG_OV_STAG_MASK              0x0000ffff
-#define FUNC_MF_CFG_OV_STAG_SHIFT             0
-#define FUNC_MF_CFG_OV_STAG_DEFAULT           FUNC_MF_CFG_OV_STAG_MASK
+        u32 ovlan_stag;
+#define FUNC_MF_CFG_OV_STAG_MASK        0x0000ffff
+#define FUNC_MF_CFG_OV_STAG_SHIFT       0
+#define FUNC_MF_CFG_OV_STAG_DEFAULT     FUNC_MF_CFG_OV_STAG_MASK
 
-        u32     pf_allocation;  /* vf per pf */
+        u32 pf_allocation;
 
-        u32     preserve_data;  /* Will be used bt CCM */
+        u32 preserve_data;
 
-        u32     driver_last_activity_ts;
+        u32 driver_last_activity_ts;
 
-        u32     drv_ack_vf_disabled[VF_MAX_STATIC / 32]; /* 0x0044 */
+        u32 drv_ack_vf_disabled[VF_MAX_STATIC / 32];
 
-        u32     drv_id;
-#define DRV_ID_PDA_COMP_VER_MASK        0x0000ffff
-#define DRV_ID_PDA_COMP_VER_SHIFT       0
+        u32 drv_id;
+#define DRV_ID_PDA_COMP_VER_MASK        0x0000ffff
+#define DRV_ID_PDA_COMP_VER_SHIFT       0
 
-#define DRV_ID_MCP_HSI_VER_MASK         0x00ff0000
-#define DRV_ID_MCP_HSI_VER_SHIFT        16
-#define DRV_ID_MCP_HSI_VER_CURRENT      BIT(DRV_ID_MCP_HSI_VER_SHIFT)
+#define DRV_ID_MCP_HSI_VER_MASK         0x00ff0000
+#define DRV_ID_MCP_HSI_VER_SHIFT        16
+#define DRV_ID_MCP_HSI_VER_CURRENT      (1 << DRV_ID_MCP_HSI_VER_SHIFT)
 
-#define DRV_ID_DRV_TYPE_MASK            0x7f000000
-#define DRV_ID_DRV_TYPE_SHIFT           24
-#define DRV_ID_DRV_TYPE_UNKNOWN         (0 << DRV_ID_DRV_TYPE_SHIFT)
-#define DRV_ID_DRV_TYPE_LINUX           (1 << DRV_ID_DRV_TYPE_SHIFT)
-#define DRV_ID_DRV_TYPE_WINDOWS         (2 << DRV_ID_DRV_TYPE_SHIFT)
-#define DRV_ID_DRV_TYPE_DIAG            (3 << DRV_ID_DRV_TYPE_SHIFT)
-#define DRV_ID_DRV_TYPE_PREBOOT         (4 << DRV_ID_DRV_TYPE_SHIFT)
-#define DRV_ID_DRV_TYPE_SOLARIS         (5 << DRV_ID_DRV_TYPE_SHIFT)
-#define DRV_ID_DRV_TYPE_VMWARE          (6 << DRV_ID_DRV_TYPE_SHIFT)
-#define DRV_ID_DRV_TYPE_FREEBSD         (7 << DRV_ID_DRV_TYPE_SHIFT)
-#define DRV_ID_DRV_TYPE_AIX             (8 << DRV_ID_DRV_TYPE_SHIFT)
+#define DRV_ID_DRV_TYPE_MASK            0x7f000000
+#define DRV_ID_DRV_TYPE_SHIFT           24
+#define DRV_ID_DRV_TYPE_UNKNOWN         (0 << DRV_ID_DRV_TYPE_SHIFT)
+#define DRV_ID_DRV_TYPE_LINUX           (1 << DRV_ID_DRV_TYPE_SHIFT)
 
-#define DRV_ID_DRV_INIT_HW_MASK         0x80000000
-#define DRV_ID_DRV_INIT_HW_SHIFT        31
-#define DRV_ID_DRV_INIT_HW_FLAG         BIT(DRV_ID_DRV_INIT_HW_SHIFT)
+#define DRV_ID_DRV_INIT_HW_MASK         0x80000000
+#define DRV_ID_DRV_INIT_HW_SHIFT        31
+#define DRV_ID_DRV_INIT_HW_FLAG         (1 << DRV_ID_DRV_INIT_HW_SHIFT)
 };
 
-/**************************************/
-/*                                    */
-/*     P U B L I C       M B          */
-/*                                    */
-/**************************************/
-/* This is the only section that the driver can write to, and each */
-/* Basically each driver request to set feature parameters,
- * will be done using a different command, which will be linked
- * to a specific data structure from the union below.
- * For huge strucuture, the common blank structure should be used.
- */
-
 struct mcp_mac {
-        u32     mac_upper;  /* Upper 16 bits are always zeroes */
-        u32     mac_lower;
+        u32 mac_upper;
+        u32 mac_lower;
 };
 
 struct mcp_val64 {
-        u32     lo;
-        u32     hi;
+        u32 lo;
+        u32 hi;
 };
 
 struct mcp_file_att {
-        u32     nvm_start_addr;
-        u32     len;
+        u32 nvm_start_addr;
+        u32 len;
+};
+
+struct bist_nvm_image_att {
+        u32 return_code;
+        u32 image_type;
+        u32 nvm_start_addr;
+        u32 len;
 };
 
 #define MCP_DRV_VER_STR_SIZE 16
 #define MCP_DRV_VER_STR_SIZE_DWORD (MCP_DRV_VER_STR_SIZE / sizeof(u32))
 #define MCP_DRV_NVM_BUF_LEN 32
 struct drv_version_stc {
-        u32     version;
-        u8      name[MCP_DRV_VER_STR_SIZE - 4];
+        u32 version;
+        u8 name[MCP_DRV_VER_STR_SIZE - 4];
+};
+
+struct lan_stats_stc {
+        u64 ucast_rx_pkts;
+        u64 ucast_tx_pkts;
+        u32 fcs_err;
+        u32 rserved;
+};
+
+struct ocbb_data_stc {
+        u32 ocbb_host_addr;
+        u32 ocsd_host_addr;
+        u32 ocsd_req_update_interval;
+};
+
+#define MAX_NUM_OF_SENSORS 7
+struct temperature_status_stc {
+        u32 num_of_sensors;
+        u32 sensor[MAX_NUM_OF_SENSORS];
+};
+
+/* crash dump configuration header */
+struct mdump_config_stc {
+        u32 version;
+        u32 config;
+        u32 epoc;
+        u32 num_of_logs;
+        u32 valid_logs;
 };
 
 union drv_union_data {
-        u32                     ver_str[MCP_DRV_VER_STR_SIZE_DWORD];
-        struct mcp_mac          wol_mac;
+        u32 ver_str[MCP_DRV_VER_STR_SIZE_DWORD];
+        struct mcp_mac wol_mac;
+
+        struct eth_phy_cfg drv_phy_cfg;
 
-        struct pmm_phy_cfg      drv_phy_cfg;
+        struct mcp_val64 val64;
 
-        struct mcp_val64        val64; /* For PHY / AVS commands */
+        u8 raw_data[MCP_DRV_NVM_BUF_LEN];
 
-        u8                      raw_data[MCP_DRV_NVM_BUF_LEN];
+        struct mcp_file_att file_att;
 
-        struct mcp_file_att     file_att;
+        u32 ack_vf_disabled[VF_MAX_STATIC / 32];
 
-        u32                     ack_vf_disabled[VF_MAX_STATIC / 32];
+        struct drv_version_stc drv_version;
 
-        struct drv_version_stc  drv_version;
+        struct lan_stats_stc lan_stats;
+        u64 reserved_stats[11];
+        struct ocbb_data_stc ocbb_info;
+        struct temperature_status_stc temp_info;
+        struct bist_nvm_image_att nvm_image_att;
+        struct mdump_config_stc mdump_config;
 };
 
 struct public_drv_mb {
         u32 drv_mb_header;
-#define DRV_MSG_CODE_MASK                       0xffff0000
-#define DRV_MSG_CODE_LOAD_REQ                   0x10000000
-#define DRV_MSG_CODE_LOAD_DONE                  0x11000000
-#define DRV_MSG_CODE_INIT_HW                    0x12000000
-#define DRV_MSG_CODE_UNLOAD_REQ                 0x20000000
-#define DRV_MSG_CODE_UNLOAD_DONE                0x21000000
-#define DRV_MSG_CODE_INIT_PHY                   0x22000000
-        /* Params - FORCE - Reinitialize the link regardless of LFA */
-        /*        - DONT_CARE - Don't flap the link if up */
-#define DRV_MSG_CODE_LINK_RESET                 0x23000000
-
-#define DRV_MSG_CODE_SET_LLDP                   0x24000000
-#define DRV_MSG_CODE_SET_DCBX                   0x25000000
+#define DRV_MSG_CODE_MASK                       0xffff0000
+#define DRV_MSG_CODE_LOAD_REQ                   0x10000000
+#define DRV_MSG_CODE_LOAD_DONE                  0x11000000
+#define DRV_MSG_CODE_INIT_HW                    0x12000000
+#define DRV_MSG_CODE_UNLOAD_REQ                 0x20000000
+#define DRV_MSG_CODE_UNLOAD_DONE                0x21000000
+#define DRV_MSG_CODE_INIT_PHY                   0x22000000
+#define DRV_MSG_CODE_LINK_RESET                 0x23000000
+#define DRV_MSG_CODE_SET_DCBX                   0x25000000
+
 #define DRV_MSG_CODE_BW_UPDATE_ACK              0x32000000
-#define DRV_MSG_CODE_NIG_DRAIN                  0x30000000
-
-#define DRV_MSG_CODE_INITIATE_FLR               0x02000000
-#define DRV_MSG_CODE_VF_DISABLED_DONE           0xc0000000
-#define DRV_MSG_CODE_CFG_VF_MSIX                0xc0010000
-#define DRV_MSG_CODE_NVM_PUT_FILE_BEGIN         0x00010000
-#define DRV_MSG_CODE_NVM_PUT_FILE_DATA          0x00020000
-#define DRV_MSG_CODE_NVM_GET_FILE_ATT           0x00030000
-#define DRV_MSG_CODE_NVM_READ_NVRAM             0x00050000
-#define DRV_MSG_CODE_NVM_WRITE_NVRAM            0x00060000
-#define DRV_MSG_CODE_NVM_DEL_FILE               0x00080000
-#define DRV_MSG_CODE_MCP_RESET                  0x00090000
-#define DRV_MSG_CODE_SET_SECURE_MODE            0x000a0000
-#define DRV_MSG_CODE_PHY_RAW_READ               0x000b0000
-#define DRV_MSG_CODE_PHY_RAW_WRITE              0x000c0000
-#define DRV_MSG_CODE_PHY_CORE_READ              0x000d0000
-#define DRV_MSG_CODE_PHY_CORE_WRITE             0x000e0000
-#define DRV_MSG_CODE_SET_VERSION                0x000f0000
-
-#define DRV_MSG_CODE_BIST_TEST                  0x001e0000
-#define DRV_MSG_CODE_SET_LED_MODE               0x00200000
-
-#define DRV_MSG_SEQ_NUMBER_MASK                 0x0000ffff
+#define DRV_MSG_CODE_NIG_DRAIN                  0x30000000
+#define DRV_MSG_CODE_VF_DISABLED_DONE           0xc0000000
+#define DRV_MSG_CODE_CFG_VF_MSIX                0xc0010000
+#define DRV_MSG_CODE_MCP_RESET                  0x00090000
+#define DRV_MSG_CODE_SET_VERSION                0x000f0000
+
+#define DRV_MSG_CODE_BIST_TEST                  0x001e0000
+#define DRV_MSG_CODE_SET_LED_MODE               0x00200000
+
+#define DRV_MSG_SEQ_NUMBER_MASK                 0x0000ffff
 
         u32 drv_mb_param;
+#define DRV_MB_PARAM_UNLOAD_WOL_MCP             0x00000001
+#define DRV_MB_PARAM_DCBX_NOTIFY_MASK           0x000000FF
+#define DRV_MB_PARAM_DCBX_NOTIFY_SHIFT          3
+#define DRV_MB_PARAM_CFG_VF_MSIX_VF_ID_SHIFT    0
+#define DRV_MB_PARAM_CFG_VF_MSIX_VF_ID_MASK     0x000000FF
+#define DRV_MB_PARAM_CFG_VF_MSIX_SB_NUM_SHIFT   8
+#define DRV_MB_PARAM_CFG_VF_MSIX_SB_NUM_MASK    0x0000FF00
+#define DRV_MB_PARAM_LLDP_SEND_MASK             0x00000001
+#define DRV_MB_PARAM_LLDP_SEND_SHIFT            0
+
+
+#define DRV_MB_PARAM_SET_LED_MODE_OPER          0x0
+#define DRV_MB_PARAM_SET_LED_MODE_ON            0x1
+#define DRV_MB_PARAM_SET_LED_MODE_OFF           0x2
+
+#define DRV_MB_PARAM_BIST_REGISTER_TEST         1
+#define DRV_MB_PARAM_BIST_CLOCK_TEST            2
+
+#define DRV_MB_PARAM_BIST_RC_UNKNOWN            0
+#define DRV_MB_PARAM_BIST_RC_PASSED             1
+#define DRV_MB_PARAM_BIST_RC_FAILED             2
+#define DRV_MB_PARAM_BIST_RC_INVALID_PARAMETER  3
 
-        /* UNLOAD_REQ params */
-#define DRV_MB_PARAM_UNLOAD_WOL_UNKNOWN         0x00000000
-#define DRV_MB_PARAM_UNLOAD_WOL_MCP             0x00000001
-#define DRV_MB_PARAM_UNLOAD_WOL_DISABLED        0x00000002
-#define DRV_MB_PARAM_UNLOAD_WOL_ENABLED         0x00000003
-
-        /* UNLOAD_DONE_params */
-#define DRV_MB_PARAM_UNLOAD_NON_D3_POWER        0x00000001
-
-        /* INIT_PHY params */
-#define DRV_MB_PARAM_INIT_PHY_FORCE             0x00000001
-#define DRV_MB_PARAM_INIT_PHY_DONT_CARE         0x00000002
-
-        /* LLDP / DCBX params*/
-#define DRV_MB_PARAM_LLDP_SEND_MASK             0x00000001
-#define DRV_MB_PARAM_LLDP_SEND_SHIFT            0
-#define DRV_MB_PARAM_LLDP_AGENT_MASK            0x00000006
-#define DRV_MB_PARAM_LLDP_AGENT_SHIFT           1
-#define DRV_MB_PARAM_DCBX_NOTIFY_MASK           0x00000008
-#define DRV_MB_PARAM_DCBX_NOTIFY_SHIFT          3
-
-#define DRV_MB_PARAM_NIG_DRAIN_PERIOD_MS_MASK   0x000000FF
-#define DRV_MB_PARAM_NIG_DRAIN_PERIOD_MS_SHIFT  0
-
-#define DRV_MB_PARAM_NVM_PUT_FILE_BEGIN_MFW     0x1
-#define DRV_MB_PARAM_NVM_PUT_FILE_BEGIN_IMAGE   0x2
-
-#define DRV_MB_PARAM_NVM_OFFSET_SHIFT           0
-#define DRV_MB_PARAM_NVM_OFFSET_MASK            0x00FFFFFF
-#define DRV_MB_PARAM_NVM_LEN_SHIFT              24
-#define DRV_MB_PARAM_NVM_LEN_MASK               0xFF000000
-
-#define DRV_MB_PARAM_PHY_ADDR_SHIFT             0
-#define DRV_MB_PARAM_PHY_ADDR_MASK              0x1FF0FFFF
-#define DRV_MB_PARAM_PHY_LANE_SHIFT             16
-#define DRV_MB_PARAM_PHY_LANE_MASK              0x000F0000
-#define DRV_MB_PARAM_PHY_SELECT_PORT_SHIFT      29
-#define DRV_MB_PARAM_PHY_SELECT_PORT_MASK       0x20000000
-#define DRV_MB_PARAM_PHY_PORT_SHIFT             30
-#define DRV_MB_PARAM_PHY_PORT_MASK              0xc0000000
-
-/* configure vf MSIX params*/
-#define DRV_MB_PARAM_CFG_VF_MSIX_VF_ID_SHIFT    0
-#define DRV_MB_PARAM_CFG_VF_MSIX_VF_ID_MASK     0x000000FF
-#define DRV_MB_PARAM_CFG_VF_MSIX_SB_NUM_SHIFT   8
-#define DRV_MB_PARAM_CFG_VF_MSIX_SB_NUM_MASK    0x0000FF00
-
-#define DRV_MB_PARAM_SET_LED_MODE_OPER          0x0
-#define DRV_MB_PARAM_SET_LED_MODE_ON            0x1
-#define DRV_MB_PARAM_SET_LED_MODE_OFF           0x2
-
-#define DRV_MB_PARAM_BIST_UNKNOWN_TEST          0
-#define DRV_MB_PARAM_BIST_REGISTER_TEST         1
-#define DRV_MB_PARAM_BIST_CLOCK_TEST            2
-
-#define DRV_MB_PARAM_BIST_RC_UNKNOWN            0
-#define DRV_MB_PARAM_BIST_RC_PASSED             1
-#define DRV_MB_PARAM_BIST_RC_FAILED             2
-#define DRV_MB_PARAM_BIST_RC_INVALID_PARAMETER          3
-
-#define DRV_MB_PARAM_BIST_TEST_INDEX_SHIFT      0
-#define DRV_MB_PARAM_BIST_TEST_INDEX_MASK       0x000000FF
+#define DRV_MB_PARAM_BIST_TEST_INDEX_SHIFT      0
+#define DRV_MB_PARAM_BIST_TEST_INDEX_MASK       0x000000FF
 
         u32 fw_mb_header;
-#define FW_MSG_CODE_MASK                        0xffff0000
-#define FW_MSG_CODE_DRV_LOAD_ENGINE             0x10100000
-#define FW_MSG_CODE_DRV_LOAD_PORT               0x10110000
-#define FW_MSG_CODE_DRV_LOAD_FUNCTION           0x10120000
-#define FW_MSG_CODE_DRV_LOAD_REFUSED_PDA        0x10200000
-#define FW_MSG_CODE_DRV_LOAD_REFUSED_HSI        0x10210000
-#define FW_MSG_CODE_DRV_LOAD_REFUSED_DIAG       0x10220000
-#define FW_MSG_CODE_DRV_LOAD_DONE               0x11100000
-#define FW_MSG_CODE_DRV_UNLOAD_ENGINE           0x20110000
-#define FW_MSG_CODE_DRV_UNLOAD_PORT             0x20120000
-#define FW_MSG_CODE_DRV_UNLOAD_FUNCTION         0x20130000
-#define FW_MSG_CODE_DRV_UNLOAD_DONE             0x21100000
-#define FW_MSG_CODE_INIT_PHY_DONE               0x21200000
-#define FW_MSG_CODE_INIT_PHY_ERR_INVALID_ARGS   0x21300000
-#define FW_MSG_CODE_LINK_RESET_DONE             0x23000000
-#define FW_MSG_CODE_SET_LLDP_DONE               0x24000000
-#define FW_MSG_CODE_SET_LLDP_UNSUPPORTED_AGENT  0x24010000
-#define FW_MSG_CODE_SET_DCBX_DONE               0x25000000
-#define FW_MSG_CODE_NIG_DRAIN_DONE              0x30000000
-#define FW_MSG_CODE_VF_DISABLED_DONE            0xb0000000
-#define FW_MSG_CODE_DRV_CFG_VF_MSIX_DONE        0xb0010000
-#define FW_MSG_CODE_FLR_ACK                     0x02000000
-#define FW_MSG_CODE_FLR_NACK                    0x02100000
-
-#define FW_MSG_CODE_NVM_OK                      0x00010000
-#define FW_MSG_CODE_NVM_INVALID_MODE            0x00020000
-#define FW_MSG_CODE_NVM_PREV_CMD_WAS_NOT_FINISHED       0x00030000
-#define FW_MSG_CODE_NVM_FAILED_TO_ALLOCATE_PAGE 0x00040000
-#define FW_MSG_CODE_NVM_INVALID_DIR_FOUND       0x00050000
-#define FW_MSG_CODE_NVM_PAGE_NOT_FOUND          0x00060000
-#define FW_MSG_CODE_NVM_FAILED_PARSING_BNDLE_HEADER 0x00070000
-#define FW_MSG_CODE_NVM_FAILED_PARSING_IMAGE_HEADER 0x00080000
-#define FW_MSG_CODE_NVM_PARSING_OUT_OF_SYNC     0x00090000
-#define FW_MSG_CODE_NVM_FAILED_UPDATING_DIR     0x000a0000
-#define FW_MSG_CODE_NVM_FAILED_TO_FREE_PAGE     0x000b0000
-#define FW_MSG_CODE_NVM_FILE_NOT_FOUND          0x000c0000
-#define FW_MSG_CODE_NVM_OPERATION_FAILED        0x000d0000
-#define FW_MSG_CODE_NVM_FAILED_UNALIGNED        0x000e0000
-#define FW_MSG_CODE_NVM_BAD_OFFSET              0x000f0000
-#define FW_MSG_CODE_NVM_BAD_SIGNATURE           0x00100000
-#define FW_MSG_CODE_NVM_FILE_READ_ONLY          0x00200000
-#define FW_MSG_CODE_NVM_UNKNOWN_FILE            0x00300000
-#define FW_MSG_CODE_NVM_PUT_FILE_FINISH_OK      0x00400000
-#define FW_MSG_CODE_MCP_RESET_REJECT            0x00600000
-#define FW_MSG_CODE_PHY_OK                      0x00110000
-#define FW_MSG_CODE_PHY_ERROR                   0x00120000
-#define FW_MSG_CODE_SET_SECURE_MODE_ERROR       0x00130000
-#define FW_MSG_CODE_SET_SECURE_MODE_OK          0x00140000
-#define FW_MSG_MODE_PHY_PRIVILEGE_ERROR         0x00150000
-#define FW_MSG_CODE_OK                          0x00160000
-
-#define FW_MSG_SEQ_NUMBER_MASK                  0x0000ffff
-
-        u32     fw_mb_param;
-
-        u32     drv_pulse_mb;
-#define DRV_PULSE_SEQ_MASK                      0x00007fff
-#define DRV_PULSE_SYSTEM_TIME_MASK              0xffff0000
-#define DRV_PULSE_ALWAYS_ALIVE                  0x00008000
+#define FW_MSG_CODE_MASK                        0xffff0000
+#define FW_MSG_CODE_DRV_LOAD_ENGINE             0x10100000
+#define FW_MSG_CODE_DRV_LOAD_PORT               0x10110000
+#define FW_MSG_CODE_DRV_LOAD_FUNCTION           0x10120000
+#define FW_MSG_CODE_DRV_LOAD_REFUSED_PDA        0x10200000
+#define FW_MSG_CODE_DRV_LOAD_REFUSED_HSI        0x10210000
+#define FW_MSG_CODE_DRV_LOAD_REFUSED_DIAG       0x10220000
+#define FW_MSG_CODE_DRV_LOAD_DONE               0x11100000
+#define FW_MSG_CODE_DRV_UNLOAD_ENGINE           0x20110000
+#define FW_MSG_CODE_DRV_UNLOAD_PORT             0x20120000
+#define FW_MSG_CODE_DRV_UNLOAD_FUNCTION         0x20130000
+#define FW_MSG_CODE_DRV_UNLOAD_DONE             0x21100000
+#define FW_MSG_CODE_DRV_CFG_VF_MSIX_DONE        0xb0010000
+#define FW_MSG_CODE_OK                          0x00160000
+
+#define FW_MSG_SEQ_NUMBER_MASK                  0x0000ffff
+
+        u32 fw_mb_param;
+
+        u32 drv_pulse_mb;
+#define DRV_PULSE_SEQ_MASK                      0x00007fff
+#define DRV_PULSE_SYSTEM_TIME_MASK              0xffff0000
+#define DRV_PULSE_ALWAYS_ALIVE                  0x00008000
+
         u32 mcp_pulse_mb;
-#define MCP_PULSE_SEQ_MASK                      0x00007fff
-#define MCP_PULSE_ALWAYS_ALIVE                  0x00008000
-#define MCP_EVENT_MASK                          0xffff0000
-#define MCP_EVENT_OTHER_DRIVER_RESET_REQ        0x00010000
+#define MCP_PULSE_SEQ_MASK                      0x00007fff
+#define MCP_PULSE_ALWAYS_ALIVE                  0x00008000
+#define MCP_EVENT_MASK                          0xffff0000
+#define MCP_EVENT_OTHER_DRIVER_RESET_REQ        0x00010000
 
         union drv_union_data union_data;
 };
 
-/* MFW - DRV MB */
-/**********************************************************************
-* Description
-*   Incremental Aggregative
-*   8-bit MFW counter per message
-*   8-bit ack-counter per message
-* Capabilities
-*   Provides up to 256 aggregative message per type
-*   Provides 4 message types in dword
-*   Message type pointers to byte offset
-*   Backward Compatibility by using sizeof for the counters.
-*   No lock requires for 32bit messages
-* Limitations:
-* In case of messages greater than 32bit, a dedicated mechanism(e.g lock)
-* is required to prevent data corruption.
-**********************************************************************/
 enum MFW_DRV_MSG_TYPE {
         MFW_DRV_MSG_LINK_CHANGE,
         MFW_DRV_MSG_FLR_FW_ACK_FAILED,
@@ -4085,37 +7304,33 @@ enum MFW_DRV_MSG_TYPE {
         MFW_DRV_MSG_LLDP_DATA_UPDATED,
         MFW_DRV_MSG_DCBX_REMOTE_MIB_UPDATED,
         MFW_DRV_MSG_DCBX_OPERATIONAL_MIB_UPDATED,
-        MFW_DRV_MSG_ERROR_RECOVERY,
+        MFW_DRV_MSG_RESERVED4,
         MFW_DRV_MSG_BW_UPDATE,
-        MFW_DRV_MSG_S_TAG_UPDATE,
-        MFW_DRV_MSG_GET_LAN_STATS,
-        MFW_DRV_MSG_GET_FCOE_STATS,
-        MFW_DRV_MSG_GET_ISCSI_STATS,
-        MFW_DRV_MSG_GET_RDMA_STATS,
-        MFW_DRV_MSG_FAILURE_DETECTED,
+        MFW_DRV_MSG_BW_UPDATE5,
+        MFW_DRV_MSG_BW_UPDATE6,
+        MFW_DRV_MSG_BW_UPDATE7,
+        MFW_DRV_MSG_BW_UPDATE8,
+        MFW_DRV_MSG_BW_UPDATE9,
+        MFW_DRV_MSG_BW_UPDATE10,
         MFW_DRV_MSG_TRANSCEIVER_STATE_CHANGE,
+        MFW_DRV_MSG_BW_UPDATE11,
         MFW_DRV_MSG_MAX
 };
 
-#define MFW_DRV_MSG_MAX_DWORDS(msgs)    (((msgs - 1) >> 2) + 1)
-#define MFW_DRV_MSG_DWORD(msg_id)       (msg_id >> 2)
-#define MFW_DRV_MSG_OFFSET(msg_id)      ((msg_id & 0x3) << 3)
-#define MFW_DRV_MSG_MASK(msg_id)        (0xff << MFW_DRV_MSG_OFFSET(msg_id))
+#define MFW_DRV_MSG_MAX_DWORDS(msgs)    (((msgs - 1) >> 2) + 1)
+#define MFW_DRV_MSG_DWORD(msg_id)       (msg_id >> 2)
+#define MFW_DRV_MSG_OFFSET(msg_id)      ((msg_id & 0x3) << 3)
+#define MFW_DRV_MSG_MASK(msg_id)        (0xff << MFW_DRV_MSG_OFFSET(msg_id))
 
 struct public_mfw_mb {
-        u32     sup_msgs;
-        u32     msg[MFW_DRV_MSG_MAX_DWORDS(MFW_DRV_MSG_MAX)];
-        u32     ack[MFW_DRV_MSG_MAX_DWORDS(MFW_DRV_MSG_MAX)];
+        u32 sup_msgs;
+        u32 msg[MFW_DRV_MSG_MAX_DWORDS(MFW_DRV_MSG_MAX)];
+        u32 ack[MFW_DRV_MSG_MAX_DWORDS(MFW_DRV_MSG_MAX)];
 };
 
-/**************************************/
-/*                                    */
-/*     P U B L I C       D A T A      */
-/*                                    */
-/**************************************/
 enum public_sections {
-        PUBLIC_DRV_MB,          /* Points to the first drv_mb of path0 */
-        PUBLIC_MFW_MB,          /* Points to the first mfw_mb of path0 */
+        PUBLIC_DRV_MB,
+        PUBLIC_MFW_MB,
         PUBLIC_GLOBAL,
         PUBLIC_PATH,
         PUBLIC_PORT,
@@ -4123,1080 +7338,179 @@ enum public_sections {
         PUBLIC_MAX_SECTIONS
 };
 
-struct drv_ver_info_stc {
-        u32     ver;
-        u8      name[32];
-};
-
 struct mcp_public_data {
-        /* The sections fields is an array */
-        u32                     num_sections;
-        offsize_t               sections[PUBLIC_MAX_SECTIONS];
-        struct public_drv_mb    drv_mb[MCP_GLOB_FUNC_MAX];
-        struct public_mfw_mb    mfw_mb[MCP_GLOB_FUNC_MAX];
-        struct public_global    global;
-        struct public_path      path[MCP_GLOB_PATH_MAX];
-        struct public_port      port[MCP_GLOB_PORT_MAX];
-        struct public_func      func[MCP_GLOB_FUNC_MAX];
-        struct drv_ver_info_stc drv_info;
+        u32 num_sections;
+        u32 sections[PUBLIC_MAX_SECTIONS];
+        struct public_drv_mb drv_mb[MCP_GLOB_FUNC_MAX];
+        struct public_mfw_mb mfw_mb[MCP_GLOB_FUNC_MAX];
+        struct public_global global;
+        struct public_path path[MCP_GLOB_PATH_MAX];
+        struct public_port port[MCP_GLOB_PORT_MAX];
+        struct public_func func[MCP_GLOB_FUNC_MAX];
 };
 
 struct nvm_cfg_mac_address {
-        u32     mac_addr_hi;
-#define NVM_CFG_MAC_ADDRESS_HI_MASK                             0x0000FFFF
-#define NVM_CFG_MAC_ADDRESS_HI_OFFSET                           0
-
-        u32     mac_addr_lo;
+        u32 mac_addr_hi;
+#define NVM_CFG_MAC_ADDRESS_HI_MASK     0x0000FFFF
+#define NVM_CFG_MAC_ADDRESS_HI_OFFSET   0
+        u32 mac_addr_lo;
 };
 
-/******************************************
-* nvm_cfg1 structs
-******************************************/
-
 struct nvm_cfg1_glob {
-        u32 generic_cont0;                                      /* 0x0 */
-#define NVM_CFG1_GLOB_BOARD_SWAP_MASK                           0x0000000F
-#define NVM_CFG1_GLOB_BOARD_SWAP_OFFSET                         0
-#define NVM_CFG1_GLOB_BOARD_SWAP_NONE                           0x0
-#define NVM_CFG1_GLOB_BOARD_SWAP_PATH                           0x1
-#define NVM_CFG1_GLOB_BOARD_SWAP_PORT                           0x2
-#define NVM_CFG1_GLOB_BOARD_SWAP_BOTH                           0x3
-#define NVM_CFG1_GLOB_MF_MODE_MASK                              0x00000FF0
-#define NVM_CFG1_GLOB_MF_MODE_OFFSET                            4
-#define NVM_CFG1_GLOB_MF_MODE_MF_ALLOWED                        0x0
-#define NVM_CFG1_GLOB_MF_MODE_DEFAULT                           0x1
-#define NVM_CFG1_GLOB_MF_MODE_SPIO4                             0x2
-#define NVM_CFG1_GLOB_MF_MODE_NPAR1_0                           0x3
-#define NVM_CFG1_GLOB_MF_MODE_NPAR1_5                           0x4
-#define NVM_CFG1_GLOB_MF_MODE_NPAR2_0                           0x5
-#define NVM_CFG1_GLOB_MF_MODE_BD                                0x6
-#define NVM_CFG1_GLOB_MF_MODE_UFP                               0x7
-#define NVM_CFG1_GLOB_FAN_FAILURE_ENFORCEMENT_MASK              0x00001000
-#define NVM_CFG1_GLOB_FAN_FAILURE_ENFORCEMENT_OFFSET            12
-#define NVM_CFG1_GLOB_FAN_FAILURE_ENFORCEMENT_DISABLED          0x0
-#define NVM_CFG1_GLOB_FAN_FAILURE_ENFORCEMENT_ENABLED           0x1
-#define NVM_CFG1_GLOB_AVS_MARGIN_LOW_MASK                       0x001FE000
-#define NVM_CFG1_GLOB_AVS_MARGIN_LOW_OFFSET                     13
-#define NVM_CFG1_GLOB_AVS_MARGIN_HIGH_MASK                      0x1FE00000
-#define NVM_CFG1_GLOB_AVS_MARGIN_HIGH_OFFSET                    21
-#define NVM_CFG1_GLOB_ENABLE_SRIOV_MASK                         0x20000000
-#define NVM_CFG1_GLOB_ENABLE_SRIOV_OFFSET                       29
-#define NVM_CFG1_GLOB_ENABLE_SRIOV_DISABLED                     0x0
-#define NVM_CFG1_GLOB_ENABLE_SRIOV_ENABLED                      0x1
-#define NVM_CFG1_GLOB_ENABLE_ATC_MASK                           0x40000000
-#define NVM_CFG1_GLOB_ENABLE_ATC_OFFSET                         30
-#define NVM_CFG1_GLOB_ENABLE_ATC_DISABLED                       0x0
-#define NVM_CFG1_GLOB_ENABLE_ATC_ENABLED                        0x1
-#define NVM_CFG1_GLOB_CLOCK_SLOWDOWN_MASK                       0x80000000
-#define NVM_CFG1_GLOB_CLOCK_SLOWDOWN_OFFSET                     31
-#define NVM_CFG1_GLOB_CLOCK_SLOWDOWN_DISABLED                   0x0
-#define NVM_CFG1_GLOB_CLOCK_SLOWDOWN_ENABLED                    0x1
-
-        u32     engineering_change[3];                          /* 0x4 */
-
-        u32     manufacturing_id;                               /* 0x10 */
-
-        u32     serial_number[4];                               /* 0x14 */
-
-        u32     pcie_cfg;                                       /* 0x24 */
-#define NVM_CFG1_GLOB_PCI_GEN_MASK                              0x00000003
-#define NVM_CFG1_GLOB_PCI_GEN_OFFSET                            0
-#define NVM_CFG1_GLOB_PCI_GEN_PCI_GEN1                          0x0
-#define NVM_CFG1_GLOB_PCI_GEN_PCI_GEN2                          0x1
-#define NVM_CFG1_GLOB_PCI_GEN_PCI_GEN3                          0x2
-#define NVM_CFG1_GLOB_BEACON_WOL_ENABLED_MASK                   0x00000004
-#define NVM_CFG1_GLOB_BEACON_WOL_ENABLED_OFFSET                 2
-#define NVM_CFG1_GLOB_BEACON_WOL_ENABLED_DISABLED               0x0
-#define NVM_CFG1_GLOB_BEACON_WOL_ENABLED_ENABLED                0x1
-#define NVM_CFG1_GLOB_ASPM_SUPPORT_MASK                         0x00000018
-#define NVM_CFG1_GLOB_ASPM_SUPPORT_OFFSET                       3
-#define NVM_CFG1_GLOB_ASPM_SUPPORT_L0S_L1_ENABLED               0x0
-#define NVM_CFG1_GLOB_ASPM_SUPPORT_L0S_DISABLED                 0x1
-#define NVM_CFG1_GLOB_ASPM_SUPPORT_L1_DISABLED                  0x2
-#define NVM_CFG1_GLOB_ASPM_SUPPORT_L0S_L1_DISABLED              0x3
-#define NVM_CFG1_GLOB_PREVENT_PCIE_L1_MENTRY_MASK               0x00000020
-#define NVM_CFG1_GLOB_PREVENT_PCIE_L1_MENTRY_OFFSET             5
-#define NVM_CFG1_GLOB_PREVENT_PCIE_L1_MENTRY_DISABLED           0x0
-#define NVM_CFG1_GLOB_PREVENT_PCIE_L1_MENTRY_ENABLED            0x1
-#define NVM_CFG1_GLOB_PCIE_G2_TX_AMPLITUDE_MASK                 0x000003C0
-#define NVM_CFG1_GLOB_PCIE_G2_TX_AMPLITUDE_OFFSET               6
-#define NVM_CFG1_GLOB_PCIE_PREEMPHASIS_MASK                     0x00001C00
-#define NVM_CFG1_GLOB_PCIE_PREEMPHASIS_OFFSET                   10
-#define NVM_CFG1_GLOB_PCIE_PREEMPHASIS_HW                       0x0
-#define NVM_CFG1_GLOB_PCIE_PREEMPHASIS_0DB                      0x1
-#define NVM_CFG1_GLOB_PCIE_PREEMPHASIS_3_5DB                    0x2
-#define NVM_CFG1_GLOB_PCIE_PREEMPHASIS_6_0DB                    0x3
-#define NVM_CFG1_GLOB_WWN_NODE_PREFIX0_MASK                     0x001FE000
-#define NVM_CFG1_GLOB_WWN_NODE_PREFIX0_OFFSET                   13
-#define NVM_CFG1_GLOB_WWN_NODE_PREFIX1_MASK                     0x1FE00000
-#define NVM_CFG1_GLOB_WWN_NODE_PREFIX1_OFFSET                   21
-#define NVM_CFG1_GLOB_NCSI_PACKAGE_ID_MASK                      0x60000000
-#define NVM_CFG1_GLOB_NCSI_PACKAGE_ID_OFFSET                    29
-
-        u32 mgmt_traffic;                                       /* 0x28 */
-#define NVM_CFG1_GLOB_RESERVED60_MASK                           0x00000001
-#define NVM_CFG1_GLOB_RESERVED60_OFFSET                         0
-#define NVM_CFG1_GLOB_RESERVED60_100KHZ                         0x0
-#define NVM_CFG1_GLOB_RESERVED60_400KHZ                         0x1
-#define NVM_CFG1_GLOB_WWN_PORT_PREFIX0_MASK                     0x000001FE
-#define NVM_CFG1_GLOB_WWN_PORT_PREFIX0_OFFSET                   1
-#define NVM_CFG1_GLOB_WWN_PORT_PREFIX1_MASK                     0x0001FE00
-#define NVM_CFG1_GLOB_WWN_PORT_PREFIX1_OFFSET                   9
-#define NVM_CFG1_GLOB_SMBUS_ADDRESS_MASK                        0x01FE0000
-#define NVM_CFG1_GLOB_SMBUS_ADDRESS_OFFSET                      17
-#define NVM_CFG1_GLOB_SIDEBAND_MODE_MASK                        0x06000000
-#define NVM_CFG1_GLOB_SIDEBAND_MODE_OFFSET                      25
-#define NVM_CFG1_GLOB_SIDEBAND_MODE_DISABLED                    0x0
-#define NVM_CFG1_GLOB_SIDEBAND_MODE_RMII                        0x1
-#define NVM_CFG1_GLOB_SIDEBAND_MODE_SGMII                       0x2
-
-        u32 core_cfg;                                           /* 0x2C */
-#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_MASK                    0x000000FF
-#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_OFFSET                  0
-#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_2X40G                0x0
-#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_2X50G                0x1
-#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_1X100G               0x2
-#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_4X10G_F              0x3
-#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_4X10G_E              0x4
-#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_4X20G                0x5
-#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_1X40G                0xB
-#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_2X25G                0xC
-#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_1X25G                0xD
-#define NVM_CFG1_GLOB_EAGLE_ENFORCE_TX_FIR_CFG_MASK             0x00000100
-#define NVM_CFG1_GLOB_EAGLE_ENFORCE_TX_FIR_CFG_OFFSET           8
-#define NVM_CFG1_GLOB_EAGLE_ENFORCE_TX_FIR_CFG_DISABLED         0x0
-#define NVM_CFG1_GLOB_EAGLE_ENFORCE_TX_FIR_CFG_ENABLED          0x1
-#define NVM_CFG1_GLOB_FALCON_ENFORCE_TX_FIR_CFG_MASK            0x00000200
-#define NVM_CFG1_GLOB_FALCON_ENFORCE_TX_FIR_CFG_OFFSET          9
-#define NVM_CFG1_GLOB_FALCON_ENFORCE_TX_FIR_CFG_DISABLED        0x0
-#define NVM_CFG1_GLOB_FALCON_ENFORCE_TX_FIR_CFG_ENABLED         0x1
-#define NVM_CFG1_GLOB_EAGLE_CORE_ADDR_MASK                      0x0003FC00
-#define NVM_CFG1_GLOB_EAGLE_CORE_ADDR_OFFSET                    10
-#define NVM_CFG1_GLOB_FALCON_CORE_ADDR_MASK                     0x03FC0000
-#define NVM_CFG1_GLOB_FALCON_CORE_ADDR_OFFSET                   18
-#define NVM_CFG1_GLOB_AVS_MODE_MASK                             0x1C000000
-#define NVM_CFG1_GLOB_AVS_MODE_OFFSET                           26
-#define NVM_CFG1_GLOB_AVS_MODE_CLOSE_LOOP                       0x0
-#define NVM_CFG1_GLOB_AVS_MODE_OPEN_LOOP                        0x1
-#define NVM_CFG1_GLOB_AVS_MODE_DISABLED                         0x3
-#define NVM_CFG1_GLOB_OVERRIDE_SECURE_MODE_MASK                 0x60000000
-#define NVM_CFG1_GLOB_OVERRIDE_SECURE_MODE_OFFSET               29
-#define NVM_CFG1_GLOB_OVERRIDE_SECURE_MODE_DISABLED             0x0
-#define NVM_CFG1_GLOB_OVERRIDE_SECURE_MODE_ENABLED              0x1
-
-        u32 e_lane_cfg1;                                        /* 0x30 */
-#define NVM_CFG1_GLOB_RX_LANE0_SWAP_MASK                        0x0000000F
-#define NVM_CFG1_GLOB_RX_LANE0_SWAP_OFFSET                      0
-#define NVM_CFG1_GLOB_RX_LANE1_SWAP_MASK                        0x000000F0
-#define NVM_CFG1_GLOB_RX_LANE1_SWAP_OFFSET                      4
-#define NVM_CFG1_GLOB_RX_LANE2_SWAP_MASK                        0x00000F00
-#define NVM_CFG1_GLOB_RX_LANE2_SWAP_OFFSET                      8
-#define NVM_CFG1_GLOB_RX_LANE3_SWAP_MASK                        0x0000F000
-#define NVM_CFG1_GLOB_RX_LANE3_SWAP_OFFSET                      12
-#define NVM_CFG1_GLOB_TX_LANE0_SWAP_MASK                        0x000F0000
-#define NVM_CFG1_GLOB_TX_LANE0_SWAP_OFFSET                      16
-#define NVM_CFG1_GLOB_TX_LANE1_SWAP_MASK                        0x00F00000
-#define NVM_CFG1_GLOB_TX_LANE1_SWAP_OFFSET                      20
-#define NVM_CFG1_GLOB_TX_LANE2_SWAP_MASK                        0x0F000000
-#define NVM_CFG1_GLOB_TX_LANE2_SWAP_OFFSET                      24
-#define NVM_CFG1_GLOB_TX_LANE3_SWAP_MASK                        0xF0000000
-#define NVM_CFG1_GLOB_TX_LANE3_SWAP_OFFSET                      28
-
-        u32 e_lane_cfg2;                                        /* 0x34 */
-#define NVM_CFG1_GLOB_RX_LANE0_POL_FLIP_MASK                    0x00000001
-#define NVM_CFG1_GLOB_RX_LANE0_POL_FLIP_OFFSET                  0
-#define NVM_CFG1_GLOB_RX_LANE1_POL_FLIP_MASK                    0x00000002
-#define NVM_CFG1_GLOB_RX_LANE1_POL_FLIP_OFFSET                  1
-#define NVM_CFG1_GLOB_RX_LANE2_POL_FLIP_MASK                    0x00000004
-#define NVM_CFG1_GLOB_RX_LANE2_POL_FLIP_OFFSET                  2
-#define NVM_CFG1_GLOB_RX_LANE3_POL_FLIP_MASK                    0x00000008
-#define NVM_CFG1_GLOB_RX_LANE3_POL_FLIP_OFFSET                  3
-#define NVM_CFG1_GLOB_TX_LANE0_POL_FLIP_MASK                    0x00000010
-#define NVM_CFG1_GLOB_TX_LANE0_POL_FLIP_OFFSET                  4
-#define NVM_CFG1_GLOB_TX_LANE1_POL_FLIP_MASK                    0x00000020
-#define NVM_CFG1_GLOB_TX_LANE1_POL_FLIP_OFFSET                  5
-#define NVM_CFG1_GLOB_TX_LANE2_POL_FLIP_MASK                    0x00000040
-#define NVM_CFG1_GLOB_TX_LANE2_POL_FLIP_OFFSET                  6
-#define NVM_CFG1_GLOB_TX_LANE3_POL_FLIP_MASK                    0x00000080
-#define NVM_CFG1_GLOB_TX_LANE3_POL_FLIP_OFFSET                  7
-#define NVM_CFG1_GLOB_SMBUS_MODE_MASK                           0x00000F00
-#define NVM_CFG1_GLOB_SMBUS_MODE_OFFSET                         8
-#define NVM_CFG1_GLOB_SMBUS_MODE_DISABLED                       0x0
-#define NVM_CFG1_GLOB_SMBUS_MODE_100KHZ                         0x1
-#define NVM_CFG1_GLOB_SMBUS_MODE_400KHZ                         0x2
-#define NVM_CFG1_GLOB_NCSI_MASK                                 0x0000F000
-#define NVM_CFG1_GLOB_NCSI_OFFSET                               12
-#define NVM_CFG1_GLOB_NCSI_DISABLED                             0x0
-#define NVM_CFG1_GLOB_NCSI_ENABLED                              0x1
-
-        u32 f_lane_cfg1;                                        /* 0x38 */
-#define NVM_CFG1_GLOB_RX_LANE0_SWAP_MASK                        0x0000000F
-#define NVM_CFG1_GLOB_RX_LANE0_SWAP_OFFSET                      0
-#define NVM_CFG1_GLOB_RX_LANE1_SWAP_MASK                        0x000000F0
-#define NVM_CFG1_GLOB_RX_LANE1_SWAP_OFFSET                      4
-#define NVM_CFG1_GLOB_RX_LANE2_SWAP_MASK                        0x00000F00
-#define NVM_CFG1_GLOB_RX_LANE2_SWAP_OFFSET                      8
-#define NVM_CFG1_GLOB_RX_LANE3_SWAP_MASK                        0x0000F000
-#define NVM_CFG1_GLOB_RX_LANE3_SWAP_OFFSET                      12
-#define NVM_CFG1_GLOB_TX_LANE0_SWAP_MASK                        0x000F0000
-#define NVM_CFG1_GLOB_TX_LANE0_SWAP_OFFSET                      16
-#define NVM_CFG1_GLOB_TX_LANE1_SWAP_MASK                        0x00F00000
-#define NVM_CFG1_GLOB_TX_LANE1_SWAP_OFFSET                      20
-#define NVM_CFG1_GLOB_TX_LANE2_SWAP_MASK                        0x0F000000
-#define NVM_CFG1_GLOB_TX_LANE2_SWAP_OFFSET                      24
-#define NVM_CFG1_GLOB_TX_LANE3_SWAP_MASK                        0xF0000000
-#define NVM_CFG1_GLOB_TX_LANE3_SWAP_OFFSET                      28
-
-        u32 f_lane_cfg2;                                        /* 0x3C */
-#define NVM_CFG1_GLOB_RX_LANE0_POL_FLIP_MASK                    0x00000001
-#define NVM_CFG1_GLOB_RX_LANE0_POL_FLIP_OFFSET                  0
-#define NVM_CFG1_GLOB_RX_LANE1_POL_FLIP_MASK                    0x00000002
-#define NVM_CFG1_GLOB_RX_LANE1_POL_FLIP_OFFSET                  1
-#define NVM_CFG1_GLOB_RX_LANE2_POL_FLIP_MASK                    0x00000004
-#define NVM_CFG1_GLOB_RX_LANE2_POL_FLIP_OFFSET                  2
-#define NVM_CFG1_GLOB_RX_LANE3_POL_FLIP_MASK                    0x00000008
-#define NVM_CFG1_GLOB_RX_LANE3_POL_FLIP_OFFSET                  3
-#define NVM_CFG1_GLOB_TX_LANE0_POL_FLIP_MASK                    0x00000010
-#define NVM_CFG1_GLOB_TX_LANE0_POL_FLIP_OFFSET                  4
-#define NVM_CFG1_GLOB_TX_LANE1_POL_FLIP_MASK                    0x00000020
-#define NVM_CFG1_GLOB_TX_LANE1_POL_FLIP_OFFSET                  5
-#define NVM_CFG1_GLOB_TX_LANE2_POL_FLIP_MASK                    0x00000040
-#define NVM_CFG1_GLOB_TX_LANE2_POL_FLIP_OFFSET                  6
-#define NVM_CFG1_GLOB_TX_LANE3_POL_FLIP_MASK                    0x00000080
-#define NVM_CFG1_GLOB_TX_LANE3_POL_FLIP_OFFSET                  7
-
-        u32 eagle_preemphasis;                                  /* 0x40 */
-#define NVM_CFG1_GLOB_LANE0_PREEMP_MASK                         0x000000FF
-#define NVM_CFG1_GLOB_LANE0_PREEMP_OFFSET                       0
-#define NVM_CFG1_GLOB_LANE1_PREEMP_MASK                         0x0000FF00
-#define NVM_CFG1_GLOB_LANE1_PREEMP_OFFSET                       8
-#define NVM_CFG1_GLOB_LANE2_PREEMP_MASK                         0x00FF0000
-#define NVM_CFG1_GLOB_LANE2_PREEMP_OFFSET                       16
-#define NVM_CFG1_GLOB_LANE3_PREEMP_MASK                         0xFF000000
-#define NVM_CFG1_GLOB_LANE3_PREEMP_OFFSET                       24
-
-        u32 eagle_driver_current;                               /* 0x44 */
-#define NVM_CFG1_GLOB_LANE0_AMP_MASK                            0x000000FF
-#define NVM_CFG1_GLOB_LANE0_AMP_OFFSET                          0
-#define NVM_CFG1_GLOB_LANE1_AMP_MASK                            0x0000FF00
-#define NVM_CFG1_GLOB_LANE1_AMP_OFFSET                          8
-#define NVM_CFG1_GLOB_LANE2_AMP_MASK                            0x00FF0000
-#define NVM_CFG1_GLOB_LANE2_AMP_OFFSET                          16
-#define NVM_CFG1_GLOB_LANE3_AMP_MASK                            0xFF000000
-#define NVM_CFG1_GLOB_LANE3_AMP_OFFSET                          24
-
-        u32 falcon_preemphasis;                                 /* 0x48 */
-#define NVM_CFG1_GLOB_LANE0_PREEMP_MASK                         0x000000FF
-#define NVM_CFG1_GLOB_LANE0_PREEMP_OFFSET                       0
-#define NVM_CFG1_GLOB_LANE1_PREEMP_MASK                         0x0000FF00
-#define NVM_CFG1_GLOB_LANE1_PREEMP_OFFSET                       8
-#define NVM_CFG1_GLOB_LANE2_PREEMP_MASK                         0x00FF0000
-#define NVM_CFG1_GLOB_LANE2_PREEMP_OFFSET                       16
-#define NVM_CFG1_GLOB_LANE3_PREEMP_MASK                         0xFF000000
-#define NVM_CFG1_GLOB_LANE3_PREEMP_OFFSET                       24
-
-        u32 falcon_driver_current;                              /* 0x4C */
-#define NVM_CFG1_GLOB_LANE0_AMP_MASK                            0x000000FF
-#define NVM_CFG1_GLOB_LANE0_AMP_OFFSET                          0
-#define NVM_CFG1_GLOB_LANE1_AMP_MASK                            0x0000FF00
-#define NVM_CFG1_GLOB_LANE1_AMP_OFFSET                          8
-#define NVM_CFG1_GLOB_LANE2_AMP_MASK                            0x00FF0000
-#define NVM_CFG1_GLOB_LANE2_AMP_OFFSET                          16
-#define NVM_CFG1_GLOB_LANE3_AMP_MASK                            0xFF000000
-#define NVM_CFG1_GLOB_LANE3_AMP_OFFSET                          24
-
-        u32     pci_id;                                         /* 0x50 */
-#define NVM_CFG1_GLOB_VENDOR_ID_MASK                            0x0000FFFF
-#define NVM_CFG1_GLOB_VENDOR_ID_OFFSET                          0
-
-        u32     pci_subsys_id;                                  /* 0x54 */
-#define NVM_CFG1_GLOB_SUBSYSTEM_VENDOR_ID_MASK                  0x0000FFFF
-#define NVM_CFG1_GLOB_SUBSYSTEM_VENDOR_ID_OFFSET                0
-#define NVM_CFG1_GLOB_SUBSYSTEM_DEVICE_ID_MASK                  0xFFFF0000
-#define NVM_CFG1_GLOB_SUBSYSTEM_DEVICE_ID_OFFSET                16
-
-        u32     bar;                                            /* 0x58 */
-#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_MASK                   0x0000000F
-#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_OFFSET                 0
-#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_DISABLED               0x0
-#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_2K                     0x1
-#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_4K                     0x2
-#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_8K                     0x3
-#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_16K                    0x4
-#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_32K                    0x5
-#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_64K                    0x6
-#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_128K                   0x7
-#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_256K                   0x8
-#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_512K                   0x9
-#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_1M                     0xA
-#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_2M                     0xB
-#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_4M                     0xC
-#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_8M                     0xD
-#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_16M                    0xE
-#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_32M                    0xF
-#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_MASK                     0x000000F0
-#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_OFFSET                   4
-#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_DISABLED                 0x0
-#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_4K                       0x1
-#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_8K                       0x2
-#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_16K                      0x3
-#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_32K                      0x4
-#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_64K                      0x5
-#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_128K                     0x6
-#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_256K                     0x7
-#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_512K                     0x8
-#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_1M                       0x9
-#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_2M                       0xA
-#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_4M                       0xB
-#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_8M                       0xC
-#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_16M                      0xD
-#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_32M                      0xE
-#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_64M                      0xF
-#define NVM_CFG1_GLOB_BAR2_SIZE_MASK                            0x00000F00
-#define NVM_CFG1_GLOB_BAR2_SIZE_OFFSET                          8
-#define NVM_CFG1_GLOB_BAR2_SIZE_DISABLED                        0x0
-#define NVM_CFG1_GLOB_BAR2_SIZE_64K                             0x1
-#define NVM_CFG1_GLOB_BAR2_SIZE_128K                            0x2
-#define NVM_CFG1_GLOB_BAR2_SIZE_256K                            0x3
-#define NVM_CFG1_GLOB_BAR2_SIZE_512K                            0x4
-#define NVM_CFG1_GLOB_BAR2_SIZE_1M                              0x5
-#define NVM_CFG1_GLOB_BAR2_SIZE_2M                              0x6
-#define NVM_CFG1_GLOB_BAR2_SIZE_4M                              0x7
-#define NVM_CFG1_GLOB_BAR2_SIZE_8M                              0x8
-#define NVM_CFG1_GLOB_BAR2_SIZE_16M                             0x9
-#define NVM_CFG1_GLOB_BAR2_SIZE_32M                             0xA
-#define NVM_CFG1_GLOB_BAR2_SIZE_64M                             0xB
-#define NVM_CFG1_GLOB_BAR2_SIZE_128M                            0xC
-#define NVM_CFG1_GLOB_BAR2_SIZE_256M                            0xD
-#define NVM_CFG1_GLOB_BAR2_SIZE_512M                            0xE
-#define NVM_CFG1_GLOB_BAR2_SIZE_1G                              0xF
-
-        u32 eagle_txfir_main;                                   /* 0x5C */
-#define NVM_CFG1_GLOB_LANE0_TXFIR_MAIN_MASK                     0x000000FF
-#define NVM_CFG1_GLOB_LANE0_TXFIR_MAIN_OFFSET                   0
-#define NVM_CFG1_GLOB_LANE1_TXFIR_MAIN_MASK                     0x0000FF00
-#define NVM_CFG1_GLOB_LANE1_TXFIR_MAIN_OFFSET                   8
-#define NVM_CFG1_GLOB_LANE2_TXFIR_MAIN_MASK                     0x00FF0000
-#define NVM_CFG1_GLOB_LANE2_TXFIR_MAIN_OFFSET                   16
-#define NVM_CFG1_GLOB_LANE3_TXFIR_MAIN_MASK                     0xFF000000
-#define NVM_CFG1_GLOB_LANE3_TXFIR_MAIN_OFFSET                   24
-
-        u32 eagle_txfir_post;                                   /* 0x60 */
-#define NVM_CFG1_GLOB_LANE0_TXFIR_POST_MASK                     0x000000FF
-#define NVM_CFG1_GLOB_LANE0_TXFIR_POST_OFFSET                   0
-#define NVM_CFG1_GLOB_LANE1_TXFIR_POST_MASK                     0x0000FF00
-#define NVM_CFG1_GLOB_LANE1_TXFIR_POST_OFFSET                   8
-#define NVM_CFG1_GLOB_LANE2_TXFIR_POST_MASK                     0x00FF0000
-#define NVM_CFG1_GLOB_LANE2_TXFIR_POST_OFFSET                   16
-#define NVM_CFG1_GLOB_LANE3_TXFIR_POST_MASK                     0xFF000000
-#define NVM_CFG1_GLOB_LANE3_TXFIR_POST_OFFSET                   24
-
-        u32 falcon_txfir_main;                                  /* 0x64 */
-#define NVM_CFG1_GLOB_LANE0_TXFIR_MAIN_MASK                     0x000000FF
-#define NVM_CFG1_GLOB_LANE0_TXFIR_MAIN_OFFSET                   0
-#define NVM_CFG1_GLOB_LANE1_TXFIR_MAIN_MASK                     0x0000FF00
-#define NVM_CFG1_GLOB_LANE1_TXFIR_MAIN_OFFSET                   8
-#define NVM_CFG1_GLOB_LANE2_TXFIR_MAIN_MASK                     0x00FF0000
-#define NVM_CFG1_GLOB_LANE2_TXFIR_MAIN_OFFSET                   16
-#define NVM_CFG1_GLOB_LANE3_TXFIR_MAIN_MASK                     0xFF000000
-#define NVM_CFG1_GLOB_LANE3_TXFIR_MAIN_OFFSET                   24
-
-        u32 falcon_txfir_post;                                  /* 0x68 */
-#define NVM_CFG1_GLOB_LANE0_TXFIR_POST_MASK                     0x000000FF
-#define NVM_CFG1_GLOB_LANE0_TXFIR_POST_OFFSET                   0
-#define NVM_CFG1_GLOB_LANE1_TXFIR_POST_MASK                     0x0000FF00
-#define NVM_CFG1_GLOB_LANE1_TXFIR_POST_OFFSET                   8
-#define NVM_CFG1_GLOB_LANE2_TXFIR_POST_MASK                     0x00FF0000
-#define NVM_CFG1_GLOB_LANE2_TXFIR_POST_OFFSET                   16
-#define NVM_CFG1_GLOB_LANE3_TXFIR_POST_MASK                     0xFF000000
-#define NVM_CFG1_GLOB_LANE3_TXFIR_POST_OFFSET                   24
-
-        u32 manufacture_ver;                                    /* 0x6C */
-#define NVM_CFG1_GLOB_MANUF0_VER_MASK                           0x0000003F
-#define NVM_CFG1_GLOB_MANUF0_VER_OFFSET                         0
-#define NVM_CFG1_GLOB_MANUF1_VER_MASK                           0x00000FC0
-#define NVM_CFG1_GLOB_MANUF1_VER_OFFSET                         6
-#define NVM_CFG1_GLOB_MANUF2_VER_MASK                           0x0003F000
-#define NVM_CFG1_GLOB_MANUF2_VER_OFFSET                         12
-#define NVM_CFG1_GLOB_MANUF3_VER_MASK                           0x00FC0000
-#define NVM_CFG1_GLOB_MANUF3_VER_OFFSET                         18
-#define NVM_CFG1_GLOB_MANUF4_VER_MASK                           0x3F000000
-#define NVM_CFG1_GLOB_MANUF4_VER_OFFSET                         24
-
-        u32 manufacture_time;                                   /* 0x70 */
-#define NVM_CFG1_GLOB_MANUF0_TIME_MASK                          0x0000003F
-#define NVM_CFG1_GLOB_MANUF0_TIME_OFFSET                        0
-#define NVM_CFG1_GLOB_MANUF1_TIME_MASK                          0x00000FC0
-#define NVM_CFG1_GLOB_MANUF1_TIME_OFFSET                        6
-#define NVM_CFG1_GLOB_MANUF2_TIME_MASK                          0x0003F000
-#define NVM_CFG1_GLOB_MANUF2_TIME_OFFSET                        12
-
-        u32 led_global_settings;                                /* 0x74 */
-#define NVM_CFG1_GLOB_LED_SWAP_0_MASK                           0x0000000F
-#define NVM_CFG1_GLOB_LED_SWAP_0_OFFSET                         0
-#define NVM_CFG1_GLOB_LED_SWAP_1_MASK                           0x000000F0
-#define NVM_CFG1_GLOB_LED_SWAP_1_OFFSET                         4
-#define NVM_CFG1_GLOB_LED_SWAP_2_MASK                           0x00000F00
-#define NVM_CFG1_GLOB_LED_SWAP_2_OFFSET                         8
-#define NVM_CFG1_GLOB_LED_SWAP_3_MASK                           0x0000F000
-#define NVM_CFG1_GLOB_LED_SWAP_3_OFFSET                         12
-
-        u32     generic_cont1;                                  /* 0x78 */
-#define NVM_CFG1_GLOB_AVS_DAC_CODE_MASK                         0x000003FF
-#define NVM_CFG1_GLOB_AVS_DAC_CODE_OFFSET                       0
-
-        u32     mbi_version;                                    /* 0x7C */
-#define NVM_CFG1_GLOB_MBI_VERSION_0_MASK                        0x000000FF
-#define NVM_CFG1_GLOB_MBI_VERSION_0_OFFSET                      0
-#define NVM_CFG1_GLOB_MBI_VERSION_1_MASK                        0x0000FF00
-#define NVM_CFG1_GLOB_MBI_VERSION_1_OFFSET                      8
-#define NVM_CFG1_GLOB_MBI_VERSION_2_MASK                        0x00FF0000
-#define NVM_CFG1_GLOB_MBI_VERSION_2_OFFSET                      16
-
-        u32     mbi_date;                                       /* 0x80 */
-
-        u32     misc_sig;                                       /* 0x84 */
-
-        /*  Define the GPIO mapping to switch i2c mux */
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO_0_MASK                   0x000000FF
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO_0_OFFSET                 0
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO_1_MASK                   0x0000FF00
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO_1_OFFSET                 8
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__NA                      0x0
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO0                   0x1
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO1                   0x2
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO2                   0x3
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO3                   0x4
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO4                   0x5
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO5                   0x6
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO6                   0x7
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO7                   0x8
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO8                   0x9
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO9                   0xA
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO10                  0xB
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO11                  0xC
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO12                  0xD
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO13                  0xE
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO14                  0xF
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO15                  0x10
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO16                  0x11
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO17                  0x12
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO18                  0x13
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO19                  0x14
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO20                  0x15
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO21                  0x16
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO22                  0x17
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO23                  0x18
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO24                  0x19
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO25                  0x1A
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO26                  0x1B
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO27                  0x1C
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO28                  0x1D
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO29                  0x1E
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO30                  0x1F
-#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO31                  0x20
-        u32     device_capabilities;                            /* 0x88 */
-#define NVM_CFG1_GLOB_DEVICE_CAPABILITIES_ETHERNET              0x1
-        u32     power_dissipated;                               /* 0x8C */
-        u32 power_consumed;                                     /* 0x90 */
-        u32     efi_version;                                    /* 0x94 */
-        u32     reserved[42];                                   /* 0x98 */
+        u32 generic_cont0;
+#define NVM_CFG1_GLOB_MF_MODE_MASK              0x00000FF0
+#define NVM_CFG1_GLOB_MF_MODE_OFFSET            4
+#define NVM_CFG1_GLOB_MF_MODE_MF_ALLOWED        0x0
+#define NVM_CFG1_GLOB_MF_MODE_DEFAULT           0x1
+#define NVM_CFG1_GLOB_MF_MODE_SPIO4             0x2
+#define NVM_CFG1_GLOB_MF_MODE_NPAR1_0           0x3
+#define NVM_CFG1_GLOB_MF_MODE_NPAR1_5           0x4
+#define NVM_CFG1_GLOB_MF_MODE_NPAR2_0           0x5
+#define NVM_CFG1_GLOB_MF_MODE_BD                0x6
+#define NVM_CFG1_GLOB_MF_MODE_UFP               0x7
+        u32 engineering_change[3];
+        u32 manufacturing_id;
+        u32 serial_number[4];
+        u32 pcie_cfg;
+        u32 mgmt_traffic;
+        u32 core_cfg;
+#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_MASK            0x000000FF
+#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_OFFSET          0
+#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_BB_2X40G        0x0
+#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_2X50G           0x1
+#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_BB_1X100G       0x2
+#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_4X10G_F         0x3
+#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_BB_4X10G_E      0x4
+#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_BB_4X20G        0x5
+#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_1X40G           0xB
+#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_2X25G           0xC
+#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_1X25G           0xD
+#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_4X25G           0xE
+        u32 e_lane_cfg1;
+        u32 e_lane_cfg2;
+        u32 f_lane_cfg1;
+        u32 f_lane_cfg2;
+        u32 mps10_preemphasis;
+        u32 mps10_driver_current;
+        u32 mps25_preemphasis;
+        u32 mps25_driver_current;
+        u32 pci_id;
+        u32 pci_subsys_id;
+        u32 bar;
+        u32 mps10_txfir_main;
+        u32 mps10_txfir_post;
+        u32 mps25_txfir_main;
+        u32 mps25_txfir_post;
+        u32 manufacture_ver;
+        u32 manufacture_time;
+        u32 led_global_settings;
+        u32 generic_cont1;
+        u32 mbi_version;
+        u32 mbi_date;
+        u32 misc_sig;
+        u32 device_capabilities;
+#define NVM_CFG1_GLOB_DEVICE_CAPABILITIES_ETHERNET      0x1
+#define NVM_CFG1_GLOB_DEVICE_CAPABILITIES_ISCSI         0x4
+#define NVM_CFG1_GLOB_DEVICE_CAPABILITIES_ROCE          0x8
+        u32 power_dissipated;
+        u32 power_consumed;
+        u32 efi_version;
+        u32 multi_network_modes_capability;
+        u32 reserved[41];
 };
 
 struct nvm_cfg1_path {
-        u32 reserved[30];                                       /* 0x0 */
+        u32 reserved[30];
 };
 
 struct nvm_cfg1_port {
-        u32     reserved__m_relocated_to_option_123;           /* 0x0 */
-        u32     reserved__m_relocated_to_option_124;           /* 0x4 */
-        u32 generic_cont0;                                      /* 0x8 */
-#define NVM_CFG1_PORT_LED_MODE_MASK                             0x000000FF
-#define NVM_CFG1_PORT_LED_MODE_OFFSET                           0
-#define NVM_CFG1_PORT_LED_MODE_MAC1                             0x0
-#define NVM_CFG1_PORT_LED_MODE_PHY1                             0x1
-#define NVM_CFG1_PORT_LED_MODE_PHY2                             0x2
-#define NVM_CFG1_PORT_LED_MODE_PHY3                             0x3
-#define NVM_CFG1_PORT_LED_MODE_MAC2                             0x4
-#define NVM_CFG1_PORT_LED_MODE_PHY4                             0x5
-#define NVM_CFG1_PORT_LED_MODE_PHY5                             0x6
-#define NVM_CFG1_PORT_LED_MODE_PHY6                             0x7
-#define NVM_CFG1_PORT_LED_MODE_MAC3                             0x8
-#define NVM_CFG1_PORT_LED_MODE_PHY7                             0x9
-#define NVM_CFG1_PORT_LED_MODE_PHY8                             0xA
-#define NVM_CFG1_PORT_LED_MODE_PHY9                             0xB
-#define NVM_CFG1_PORT_LED_MODE_MAC4                             0xC
-#define NVM_CFG1_PORT_LED_MODE_PHY10                            0xD
-#define NVM_CFG1_PORT_LED_MODE_PHY11                            0xE
-#define NVM_CFG1_PORT_LED_MODE_PHY12                            0xF
-#define NVM_CFG1_PORT_ROCE_PRIORITY_MASK                        0x0000FF00
-#define NVM_CFG1_PORT_ROCE_PRIORITY_OFFSET                      8
-#define NVM_CFG1_PORT_DCBX_MODE_MASK                            0x000F0000
-#define NVM_CFG1_PORT_DCBX_MODE_OFFSET                          16
-#define NVM_CFG1_PORT_DCBX_MODE_DISABLED                        0x0
-#define NVM_CFG1_PORT_DCBX_MODE_IEEE                            0x1
-#define NVM_CFG1_PORT_DCBX_MODE_CEE                             0x2
-#define NVM_CFG1_PORT_DCBX_MODE_DYNAMIC                         0x3
-#define NVM_CFG1_PORT_DEFAULT_ENABLED_PROTOCOLS_MASK            0x00F00000
-#define NVM_CFG1_PORT_DEFAULT_ENABLED_PROTOCOLS_OFFSET          20
-#define NVM_CFG1_PORT_DEFAULT_ENABLED_PROTOCOLS_ETHERNET        0x1
-        u32     pcie_cfg;                                       /* 0xC */
-#define NVM_CFG1_PORT_RESERVED15_MASK                           0x00000007
-#define NVM_CFG1_PORT_RESERVED15_OFFSET                         0
-
-        u32     features;                                       /* 0x10 */
-#define NVM_CFG1_PORT_ENABLE_WOL_ON_ACPI_PATTERN_MASK           0x00000001
-#define NVM_CFG1_PORT_ENABLE_WOL_ON_ACPI_PATTERN_OFFSET         0
-#define NVM_CFG1_PORT_ENABLE_WOL_ON_ACPI_PATTERN_DISABLED       0x0
-#define NVM_CFG1_PORT_ENABLE_WOL_ON_ACPI_PATTERN_ENABLED        0x1
-#define NVM_CFG1_PORT_MAGIC_PACKET_WOL_MASK                     0x00000002
-#define NVM_CFG1_PORT_MAGIC_PACKET_WOL_OFFSET                   1
-#define NVM_CFG1_PORT_MAGIC_PACKET_WOL_DISABLED                 0x0
-#define NVM_CFG1_PORT_MAGIC_PACKET_WOL_ENABLED                  0x1
-
-        u32 speed_cap_mask;                                     /* 0x14 */
-#define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_MASK            0x0000FFFF
-#define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_OFFSET          0
-#define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_1G              0x1
-#define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_10G             0x2
-#define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_25G             0x8
-#define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_40G             0x10
-#define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_50G             0x20
-#define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_100G            0x40
-#define NVM_CFG1_PORT_MFW_SPEED_CAPABILITY_MASK_MASK            0xFFFF0000
-#define NVM_CFG1_PORT_MFW_SPEED_CAPABILITY_MASK_OFFSET          16
-#define NVM_CFG1_PORT_MFW_SPEED_CAPABILITY_MASK_1G              0x1
-#define NVM_CFG1_PORT_MFW_SPEED_CAPABILITY_MASK_10G             0x2
-#define NVM_CFG1_PORT_MFW_SPEED_CAPABILITY_MASK_25G             0x8
-#define NVM_CFG1_PORT_MFW_SPEED_CAPABILITY_MASK_40G             0x10
-#define NVM_CFG1_PORT_MFW_SPEED_CAPABILITY_MASK_50G             0x20
-#define NVM_CFG1_PORT_MFW_SPEED_CAPABILITY_MASK_100G            0x40
-
-        u32 link_settings;                                      /* 0x18 */
-#define NVM_CFG1_PORT_DRV_LINK_SPEED_MASK                       0x0000000F
-#define NVM_CFG1_PORT_DRV_LINK_SPEED_OFFSET                     0
-#define NVM_CFG1_PORT_DRV_LINK_SPEED_AUTONEG                    0x0
-#define NVM_CFG1_PORT_DRV_LINK_SPEED_1G                         0x1
-#define NVM_CFG1_PORT_DRV_LINK_SPEED_10G                        0x2
-#define NVM_CFG1_PORT_DRV_LINK_SPEED_25G                        0x4
-#define NVM_CFG1_PORT_DRV_LINK_SPEED_40G                        0x5
-#define NVM_CFG1_PORT_DRV_LINK_SPEED_50G                        0x6
-#define NVM_CFG1_PORT_DRV_LINK_SPEED_100G                       0x7
-#define NVM_CFG1_PORT_DRV_FLOW_CONTROL_MASK                     0x00000070
-#define NVM_CFG1_PORT_DRV_FLOW_CONTROL_OFFSET                   4
-#define NVM_CFG1_PORT_DRV_FLOW_CONTROL_AUTONEG                  0x1
-#define NVM_CFG1_PORT_DRV_FLOW_CONTROL_RX                       0x2
-#define NVM_CFG1_PORT_DRV_FLOW_CONTROL_TX                       0x4
-#define NVM_CFG1_PORT_MFW_LINK_SPEED_MASK                       0x00000780
-#define NVM_CFG1_PORT_MFW_LINK_SPEED_OFFSET                     7
-#define NVM_CFG1_PORT_MFW_LINK_SPEED_AUTONEG                    0x0
-#define NVM_CFG1_PORT_MFW_LINK_SPEED_1G                         0x1
-#define NVM_CFG1_PORT_MFW_LINK_SPEED_10G                        0x2
-#define NVM_CFG1_PORT_MFW_LINK_SPEED_25G                        0x4
-#define NVM_CFG1_PORT_MFW_LINK_SPEED_40G                        0x5
-#define NVM_CFG1_PORT_MFW_LINK_SPEED_50G                        0x6
-#define NVM_CFG1_PORT_MFW_LINK_SPEED_100G                       0x7
-#define NVM_CFG1_PORT_MFW_FLOW_CONTROL_MASK                     0x00003800
-#define NVM_CFG1_PORT_MFW_FLOW_CONTROL_OFFSET                   11
-#define NVM_CFG1_PORT_MFW_FLOW_CONTROL_AUTONEG                  0x1
-#define NVM_CFG1_PORT_MFW_FLOW_CONTROL_RX                       0x2
-#define NVM_CFG1_PORT_MFW_FLOW_CONTROL_TX                       0x4
-#define NVM_CFG1_PORT_OPTIC_MODULE_VENDOR_ENFORCEMENT_MASK      0x00004000
-#define NVM_CFG1_PORT_OPTIC_MODULE_VENDOR_ENFORCEMENT_OFFSET    14
-#define NVM_CFG1_PORT_OPTIC_MODULE_VENDOR_ENFORCEMENT_DISABLED  0x0
-#define NVM_CFG1_PORT_OPTIC_MODULE_VENDOR_ENFORCEMENT_ENABLED   0x1
-
-        u32 phy_cfg;                                            /* 0x1C */
-#define NVM_CFG1_PORT_OPTIONAL_LINK_MODES_MASK                  0x0000FFFF
-#define NVM_CFG1_PORT_OPTIONAL_LINK_MODES_OFFSET                0
-#define NVM_CFG1_PORT_OPTIONAL_LINK_MODES_HIGIG                 0x1
-#define NVM_CFG1_PORT_OPTIONAL_LINK_MODES_SCRAMBLER             0x2
-#define NVM_CFG1_PORT_OPTIONAL_LINK_MODES_FIBER                 0x4
-#define NVM_CFG1_PORT_OPTIONAL_LINK_MODES_DISABLE_CL72_AN       0x8
-#define NVM_CFG1_PORT_OPTIONAL_LINK_MODES_DISABLE_FEC_AN        0x10
-#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_MASK                 0x00FF0000
-#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_OFFSET               16
-#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_BYPASS               0x0
-#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_KR                   0x2
-#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_KR2                  0x3
-#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_KR4                  0x4
-#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_XFI                  0x8
-#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_SFI                  0x9
-#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_1000X                0xB
-#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_SGMII                0xC
-#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_XLAUI                0x11
-#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_XLPPI                0x12
-#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_CAUI                 0x21
-#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_CPPI                 0x22
-#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_25GAUI               0x31
-#define NVM_CFG1_PORT_AN_MODE_MASK                              0xFF000000
-#define NVM_CFG1_PORT_AN_MODE_OFFSET                            24
-#define NVM_CFG1_PORT_AN_MODE_NONE                              0x0
-#define NVM_CFG1_PORT_AN_MODE_CL73                              0x1
-#define NVM_CFG1_PORT_AN_MODE_CL37                              0x2
-#define NVM_CFG1_PORT_AN_MODE_CL73_BAM                          0x3
-#define NVM_CFG1_PORT_AN_MODE_CL37_BAM                          0x4
-#define NVM_CFG1_PORT_AN_MODE_HPAM                              0x5
-#define NVM_CFG1_PORT_AN_MODE_SGMII                             0x6
-
-        u32 mgmt_traffic;                                       /* 0x20 */
-#define NVM_CFG1_PORT_RESERVED61_MASK                           0x0000000F
-#define NVM_CFG1_PORT_RESERVED61_OFFSET                         0
-
-        u32 ext_phy;                                            /* 0x24 */
-#define NVM_CFG1_PORT_EXTERNAL_PHY_TYPE_MASK                    0x000000FF
-#define NVM_CFG1_PORT_EXTERNAL_PHY_TYPE_OFFSET                  0
-#define NVM_CFG1_PORT_EXTERNAL_PHY_TYPE_NONE                    0x0
-#define NVM_CFG1_PORT_EXTERNAL_PHY_TYPE_BCM84844                0x1
-#define NVM_CFG1_PORT_EXTERNAL_PHY_ADDRESS_MASK                 0x0000FF00
-#define NVM_CFG1_PORT_EXTERNAL_PHY_ADDRESS_OFFSET               8
-
-        u32 mba_cfg1;                                           /* 0x28 */
-#define NVM_CFG1_PORT_PREBOOT_OPROM_MASK                        0x00000001
-#define NVM_CFG1_PORT_PREBOOT_OPROM_OFFSET                      0
-#define NVM_CFG1_PORT_PREBOOT_OPROM_DISABLED                    0x0
-#define NVM_CFG1_PORT_PREBOOT_OPROM_ENABLED                     0x1
-#define NVM_CFG1_PORT_RESERVED__M_MBA_BOOT_TYPE_MASK            0x00000006
-#define NVM_CFG1_PORT_RESERVED__M_MBA_BOOT_TYPE_OFFSET          1
-#define NVM_CFG1_PORT_MBA_DELAY_TIME_MASK                       0x00000078
-#define NVM_CFG1_PORT_MBA_DELAY_TIME_OFFSET                     3
-#define NVM_CFG1_PORT_MBA_SETUP_HOT_KEY_MASK                    0x00000080
-#define NVM_CFG1_PORT_MBA_SETUP_HOT_KEY_OFFSET                  7
-#define NVM_CFG1_PORT_MBA_SETUP_HOT_KEY_CTRL_S                  0x0
-#define NVM_CFG1_PORT_MBA_SETUP_HOT_KEY_CTRL_B                  0x1
-#define NVM_CFG1_PORT_MBA_HIDE_SETUP_PROMPT_MASK                0x00000100
-#define NVM_CFG1_PORT_MBA_HIDE_SETUP_PROMPT_OFFSET              8
-#define NVM_CFG1_PORT_MBA_HIDE_SETUP_PROMPT_DISABLED            0x0
-#define NVM_CFG1_PORT_MBA_HIDE_SETUP_PROMPT_ENABLED             0x1
-#define NVM_CFG1_PORT_RESERVED5_MASK                            0x0001FE00
-#define NVM_CFG1_PORT_RESERVED5_OFFSET                          9
-#define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_MASK                   0x001E0000
-#define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_OFFSET                 17
-#define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_AUTONEG                0x0
-#define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_1G                     0x1
-#define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_10G                    0x2
-#define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_25G                    0x4
-#define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_40G                    0x5
-#define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_50G                    0x6
-#define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_100G                   0x7
-#define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_SMARTLINQ              0x8
-#define NVM_CFG1_PORT_RESERVED__M_MBA_BOOT_RETRY_COUNT_MASK     0x00E00000
-#define NVM_CFG1_PORT_RESERVED__M_MBA_BOOT_RETRY_COUNT_OFFSET   21
-
-        u32     mba_cfg2;                                       /* 0x2C */
-#define NVM_CFG1_PORT_RESERVED65_MASK                           0x0000FFFF
-#define NVM_CFG1_PORT_RESERVED65_OFFSET                         0
-#define NVM_CFG1_PORT_RESERVED66_MASK                           0x00010000
-#define NVM_CFG1_PORT_RESERVED66_OFFSET                         16
-
-        u32     vf_cfg;                                         /* 0x30 */
-#define NVM_CFG1_PORT_RESERVED8_MASK                            0x0000FFFF
-#define NVM_CFG1_PORT_RESERVED8_OFFSET                          0
-#define NVM_CFG1_PORT_RESERVED6_MASK                            0x000F0000
-#define NVM_CFG1_PORT_RESERVED6_OFFSET                          16
-
-        struct nvm_cfg_mac_address      lldp_mac_address;       /* 0x34 */
-
-        u32                             led_port_settings;      /* 0x3C */
-#define NVM_CFG1_PORT_LANE_LED_SPD_0_SEL_MASK                   0x000000FF
-#define NVM_CFG1_PORT_LANE_LED_SPD_0_SEL_OFFSET                 0
-#define NVM_CFG1_PORT_LANE_LED_SPD_1_SEL_MASK                   0x0000FF00
-#define NVM_CFG1_PORT_LANE_LED_SPD_1_SEL_OFFSET                 8
-#define NVM_CFG1_PORT_LANE_LED_SPD_2_SEL_MASK                   0x00FF0000
-#define NVM_CFG1_PORT_LANE_LED_SPD_2_SEL_OFFSET                 16
-#define NVM_CFG1_PORT_LANE_LED_SPD__SEL_1G                      0x1
-#define NVM_CFG1_PORT_LANE_LED_SPD__SEL_10G                     0x2
-#define NVM_CFG1_PORT_LANE_LED_SPD__SEL_25G                     0x8
-#define NVM_CFG1_PORT_LANE_LED_SPD__SEL_40G                     0x10
-#define NVM_CFG1_PORT_LANE_LED_SPD__SEL_50G                     0x20
-#define NVM_CFG1_PORT_LANE_LED_SPD__SEL_100G                    0x40
-
-        u32 transceiver_00;                                     /* 0x40 */
-
-        /*  Define for mapping of transceiver signal module absent */
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_MASK                     0x000000FF
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_OFFSET                   0
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_NA                       0x0
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO0                    0x1
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO1                    0x2
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO2                    0x3
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO3                    0x4
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO4                    0x5
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO5                    0x6
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO6                    0x7
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO7                    0x8
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO8                    0x9
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO9                    0xA
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO10                   0xB
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO11                   0xC
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO12                   0xD
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO13                   0xE
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO14                   0xF
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO15                   0x10
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO16                   0x11
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO17                   0x12
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO18                   0x13
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO19                   0x14
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO20                   0x15
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO21                   0x16
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO22                   0x17
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO23                   0x18
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO24                   0x19
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO25                   0x1A
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO26                   0x1B
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO27                   0x1C
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO28                   0x1D
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO29                   0x1E
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO30                   0x1F
-#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO31                   0x20
-        /*  Define the GPIO mux settings  to switch i2c mux to this port */
-#define NVM_CFG1_PORT_I2C_MUX_SEL_VALUE_0_MASK                  0x00000F00
-#define NVM_CFG1_PORT_I2C_MUX_SEL_VALUE_0_OFFSET                8
-#define NVM_CFG1_PORT_I2C_MUX_SEL_VALUE_1_MASK                  0x0000F000
-#define NVM_CFG1_PORT_I2C_MUX_SEL_VALUE_1_OFFSET                12
-
-        u32 reserved[133];                                      /* 0x44 */
+        u32 reserved__m_relocated_to_option_123;
+        u32 reserved__m_relocated_to_option_124;
+        u32 generic_cont0;
+#define NVM_CFG1_PORT_DCBX_MODE_MASK                            0x000F0000
+#define NVM_CFG1_PORT_DCBX_MODE_OFFSET                          16
+#define NVM_CFG1_PORT_DCBX_MODE_DISABLED                        0x0
+#define NVM_CFG1_PORT_DCBX_MODE_IEEE                            0x1
+#define NVM_CFG1_PORT_DCBX_MODE_CEE                             0x2
+#define NVM_CFG1_PORT_DCBX_MODE_DYNAMIC                         0x3
+#define NVM_CFG1_PORT_DEFAULT_ENABLED_PROTOCOLS_MASK            0x00F00000
+#define NVM_CFG1_PORT_DEFAULT_ENABLED_PROTOCOLS_OFFSET          20
+#define NVM_CFG1_PORT_DEFAULT_ENABLED_PROTOCOLS_ETHERNET        0x1
+#define NVM_CFG1_PORT_DEFAULT_ENABLED_PROTOCOLS_FCOE            0x2
+#define NVM_CFG1_PORT_DEFAULT_ENABLED_PROTOCOLS_ISCSI           0x4
+        u32 pcie_cfg;
+        u32 features;
+        u32 speed_cap_mask;
+#define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_MASK            0x0000FFFF
+#define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_OFFSET          0
+#define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_1G              0x1
+#define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_10G             0x2
+#define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_25G             0x8
+#define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_40G             0x10
+#define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_50G             0x20
+#define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_BB_100G         0x40
+        u32 link_settings;
+#define NVM_CFG1_PORT_DRV_LINK_SPEED_MASK                       0x0000000F
+#define NVM_CFG1_PORT_DRV_LINK_SPEED_OFFSET                     0
+#define NVM_CFG1_PORT_DRV_LINK_SPEED_AUTONEG                    0x0
+#define NVM_CFG1_PORT_DRV_LINK_SPEED_1G                         0x1
+#define NVM_CFG1_PORT_DRV_LINK_SPEED_10G                        0x2
+#define NVM_CFG1_PORT_DRV_LINK_SPEED_25G                        0x4
+#define NVM_CFG1_PORT_DRV_LINK_SPEED_40G                        0x5
+#define NVM_CFG1_PORT_DRV_LINK_SPEED_50G                        0x6
+#define NVM_CFG1_PORT_DRV_LINK_SPEED_BB_100G                    0x7
+#define NVM_CFG1_PORT_DRV_LINK_SPEED_SMARTLINQ                  0x8
+#define NVM_CFG1_PORT_DRV_FLOW_CONTROL_MASK                     0x00000070
+#define NVM_CFG1_PORT_DRV_FLOW_CONTROL_OFFSET                   4
+#define NVM_CFG1_PORT_DRV_FLOW_CONTROL_AUTONEG                  0x1
+#define NVM_CFG1_PORT_DRV_FLOW_CONTROL_RX                       0x2
+#define NVM_CFG1_PORT_DRV_FLOW_CONTROL_TX                       0x4
+        u32 phy_cfg;
+        u32 mgmt_traffic;
+        u32 ext_phy;
+        u32 mba_cfg1;
+        u32 mba_cfg2;
+        u32 vf_cfg;
+        struct nvm_cfg_mac_address lldp_mac_address;
+        u32 led_port_settings;
+        u32 transceiver_00;
+        u32 device_ids;
+        u32 board_cfg;
+        u32 mnm_10g_cap;
+        u32 mnm_10g_ctrl;
+        u32 mnm_10g_misc;
+        u32 mnm_25g_cap;
+        u32 mnm_25g_ctrl;
+        u32 mnm_25g_misc;
+        u32 mnm_40g_cap;
+        u32 mnm_40g_ctrl;
+        u32 mnm_40g_misc;
+        u32 mnm_50g_cap;
+        u32 mnm_50g_ctrl;
+        u32 mnm_50g_misc;
+        u32 mnm_100g_cap;
+        u32 mnm_100g_ctrl;
+        u32 mnm_100g_misc;
+        u32 reserved[116];
 };
 
 struct nvm_cfg1_func {
-        struct nvm_cfg_mac_address      mac_address;            /* 0x0 */
-
-        u32                             rsrv1;                  /* 0x8 */
-#define NVM_CFG1_FUNC_RESERVED1_MASK                            0x0000FFFF
-#define NVM_CFG1_FUNC_RESERVED1_OFFSET                          0
-#define NVM_CFG1_FUNC_RESERVED2_MASK                            0xFFFF0000
-#define NVM_CFG1_FUNC_RESERVED2_OFFSET                          16
-
-        u32                             rsrv2;                  /* 0xC */
-#define NVM_CFG1_FUNC_RESERVED3_MASK                            0x0000FFFF
-#define NVM_CFG1_FUNC_RESERVED3_OFFSET                          0
-#define NVM_CFG1_FUNC_RESERVED4_MASK                            0xFFFF0000
-#define NVM_CFG1_FUNC_RESERVED4_OFFSET                          16
-
-        u32                             device_id;              /* 0x10 */
-#define NVM_CFG1_FUNC_MF_VENDOR_DEVICE_ID_MASK                  0x0000FFFF
-#define NVM_CFG1_FUNC_MF_VENDOR_DEVICE_ID_OFFSET                0
-#define NVM_CFG1_FUNC_RESERVED77_MASK                           0xFFFF0000
-#define NVM_CFG1_FUNC_RESERVED77_OFFSET                         16
-
-        u32                             cmn_cfg;                /* 0x14 */
-#define NVM_CFG1_FUNC_PREBOOT_BOOT_PROTOCOL_MASK                0x00000007
-#define NVM_CFG1_FUNC_PREBOOT_BOOT_PROTOCOL_OFFSET              0
-#define NVM_CFG1_FUNC_PREBOOT_BOOT_PROTOCOL_PXE                 0x0
-#define NVM_CFG1_FUNC_PREBOOT_BOOT_PROTOCOL_ISCSI_BOOT          0x3
-#define NVM_CFG1_FUNC_PREBOOT_BOOT_PROTOCOL_FCOE_BOOT           0x4
-#define NVM_CFG1_FUNC_PREBOOT_BOOT_PROTOCOL_NONE                0x7
-#define NVM_CFG1_FUNC_VF_PCI_DEVICE_ID_MASK                     0x0007FFF8
-#define NVM_CFG1_FUNC_VF_PCI_DEVICE_ID_OFFSET                   3
-#define NVM_CFG1_FUNC_PERSONALITY_MASK                          0x00780000
-#define NVM_CFG1_FUNC_PERSONALITY_OFFSET                        19
-#define NVM_CFG1_FUNC_PERSONALITY_ETHERNET                      0x0
-#define NVM_CFG1_FUNC_PERSONALITY_ISCSI                         0x1
-#define NVM_CFG1_FUNC_PERSONALITY_FCOE                          0x2
-#define NVM_CFG1_FUNC_PERSONALITY_ROCE                          0x3
-#define NVM_CFG1_FUNC_BANDWIDTH_WEIGHT_MASK                     0x7F800000
-#define NVM_CFG1_FUNC_BANDWIDTH_WEIGHT_OFFSET                   23
-#define NVM_CFG1_FUNC_PAUSE_ON_HOST_RING_MASK                   0x80000000
-#define NVM_CFG1_FUNC_PAUSE_ON_HOST_RING_OFFSET                 31
-#define NVM_CFG1_FUNC_PAUSE_ON_HOST_RING_DISABLED               0x0
-#define NVM_CFG1_FUNC_PAUSE_ON_HOST_RING_ENABLED                0x1
-
-        u32 pci_cfg;                                            /* 0x18 */
-#define NVM_CFG1_FUNC_NUMBER_OF_VFS_PER_PF_MASK                 0x0000007F
-#define NVM_CFG1_FUNC_NUMBER_OF_VFS_PER_PF_OFFSET               0
-#define NVM_CFG1_FUNC_RESERVESD12_MASK                          0x00003F80
-#define NVM_CFG1_FUNC_RESERVESD12_OFFSET                        7
-#define NVM_CFG1_FUNC_BAR1_SIZE_MASK                            0x0003C000
-#define NVM_CFG1_FUNC_BAR1_SIZE_OFFSET                          14
-#define NVM_CFG1_FUNC_BAR1_SIZE_DISABLED                        0x0
-#define NVM_CFG1_FUNC_BAR1_SIZE_64K                             0x1
-#define NVM_CFG1_FUNC_BAR1_SIZE_128K                            0x2
-#define NVM_CFG1_FUNC_BAR1_SIZE_256K                            0x3
-#define NVM_CFG1_FUNC_BAR1_SIZE_512K                            0x4
-#define NVM_CFG1_FUNC_BAR1_SIZE_1M                              0x5
-#define NVM_CFG1_FUNC_BAR1_SIZE_2M                              0x6
-#define NVM_CFG1_FUNC_BAR1_SIZE_4M                              0x7
-#define NVM_CFG1_FUNC_BAR1_SIZE_8M                              0x8
-#define NVM_CFG1_FUNC_BAR1_SIZE_16M                             0x9
-#define NVM_CFG1_FUNC_BAR1_SIZE_32M                             0xA
-#define NVM_CFG1_FUNC_BAR1_SIZE_64M                             0xB
-#define NVM_CFG1_FUNC_BAR1_SIZE_128M                            0xC
-#define NVM_CFG1_FUNC_BAR1_SIZE_256M                            0xD
-#define NVM_CFG1_FUNC_BAR1_SIZE_512M                            0xE
-#define NVM_CFG1_FUNC_BAR1_SIZE_1G                              0xF
-#define NVM_CFG1_FUNC_MAX_BANDWIDTH_MASK                        0x03FC0000
-#define NVM_CFG1_FUNC_MAX_BANDWIDTH_OFFSET                      18
-
-        struct nvm_cfg_mac_address      fcoe_node_wwn_mac_addr; /* 0x1C */
-
-        struct nvm_cfg_mac_address      fcoe_port_wwn_mac_addr; /* 0x24 */
-        u32                             preboot_generic_cfg;    /* 0x2C */
-        u32                             reserved[8];            /* 0x30 */
+        struct nvm_cfg_mac_address mac_address;
+        u32 rsrv1;
+        u32 rsrv2;
+        u32 device_id;
+        u32 cmn_cfg;
+        u32 pci_cfg;
+        struct nvm_cfg_mac_address fcoe_node_wwn_mac_addr;
+        struct nvm_cfg_mac_address fcoe_port_wwn_mac_addr;
+        u32 preboot_generic_cfg;
+        u32 reserved[8];
 };
 
 struct nvm_cfg1 {
-        struct nvm_cfg1_glob    glob;                           /* 0x0 */
-
-        struct nvm_cfg1_path    path[MCP_GLOB_PATH_MAX];        /* 0x140 */
-
-        struct nvm_cfg1_port    port[MCP_GLOB_PORT_MAX];        /* 0x230 */
-
-        struct nvm_cfg1_func    func[MCP_GLOB_FUNC_MAX];        /* 0xB90 */
-};
-
-/******************************************
-* nvm_cfg structs
-******************************************/
-
-enum nvm_cfg_sections {
-        NVM_CFG_SECTION_NVM_CFG1,
-        NVM_CFG_SECTION_MAX
-};
-
-struct nvm_cfg {
-        u32             num_sections;
-        u32             sections_offset[NVM_CFG_SECTION_MAX];
-        struct nvm_cfg1 cfg1;
-};
-
-#define PORT_0          0
-#define PORT_1          1
-#define PORT_2          2
-#define PORT_3          3
-
-extern struct spad_layout g_spad;
-
-#define MCP_SPAD_SIZE                       0x00028000  /* 160 KB */
-
-#define SPAD_OFFSET(addr) (((u32)addr - (u32)CPU_SPAD_BASE))
-
-#define TO_OFFSIZE(_offset, _size)                              \
-        (u32)((((u32)(_offset) >> 2) << OFFSIZE_OFFSET_SHIFT) | \
-              (((u32)(_size) >> 2) << OFFSIZE_SIZE_SHIFT))
-
-enum spad_sections {
-        SPAD_SECTION_TRACE,
-        SPAD_SECTION_NVM_CFG,
-        SPAD_SECTION_PUBLIC,
-        SPAD_SECTION_PRIVATE,
-        SPAD_SECTION_MAX
-};
-
-struct spad_layout {
-        struct nvm_cfg          nvm_cfg;
-        struct mcp_public_data  public_data;
+        struct nvm_cfg1_glob glob;
+        struct nvm_cfg1_path path[MCP_GLOB_PATH_MAX];
+        struct nvm_cfg1_port port[MCP_GLOB_PORT_MAX];
+        struct nvm_cfg1_func func[MCP_GLOB_FUNC_MAX];
 };
-
-#define CRC_MAGIC_VALUE                     0xDEBB20E3
-#define CRC32_POLYNOMIAL                    0xEDB88320
-#define NVM_CRC_SIZE                            (sizeof(u32))
-
-enum nvm_sw_arbitrator {
-        NVM_SW_ARB_HOST,
-        NVM_SW_ARB_MCP,
-        NVM_SW_ARB_UART,
-        NVM_SW_ARB_RESERVED
-};
-
-/****************************************************************************
-* Boot Strap Region                                                        *
-****************************************************************************/
-struct legacy_bootstrap_region {
-        u32     magic_value;
-#define NVM_MAGIC_VALUE          0x669955aa
-        u32     sram_start_addr;
-        u32     code_len;               /* boot code length (in dwords) */
-        u32     code_start_addr;
-        u32     crc;                    /* 32-bit CRC */
-};
-
-/****************************************************************************
-* Directories Region                                                       *
-****************************************************************************/
-struct nvm_code_entry {
-        u32     image_type;             /* Image type */
-        u32     nvm_start_addr;         /* NVM address of the image */
-        u32     len;                    /* Include CRC */
-        u32     sram_start_addr;
-        u32     sram_run_addr;          /* Relevant in case of MIM only */
-};
-
-enum nvm_image_type {
-        NVM_TYPE_TIM1           = 0x01,
-        NVM_TYPE_TIM2           = 0x02,
-        NVM_TYPE_MIM1           = 0x03,
-        NVM_TYPE_MIM2           = 0x04,
-        NVM_TYPE_MBA            = 0x05,
-        NVM_TYPE_MODULES_PN     = 0x06,
-        NVM_TYPE_VPD            = 0x07,
-        NVM_TYPE_MFW_TRACE1     = 0x08,
-        NVM_TYPE_MFW_TRACE2     = 0x09,
-        NVM_TYPE_NVM_CFG1       = 0x0a,
-        NVM_TYPE_L2B            = 0x0b,
-        NVM_TYPE_DIR1           = 0x0c,
-        NVM_TYPE_EAGLE_FW1      = 0x0d,
-        NVM_TYPE_FALCON_FW1     = 0x0e,
-        NVM_TYPE_PCIE_FW1       = 0x0f,
-        NVM_TYPE_HW_SET         = 0x10,
-        NVM_TYPE_LIM            = 0x11,
-        NVM_TYPE_AVS_FW1        = 0x12,
-        NVM_TYPE_DIR2           = 0x13,
-        NVM_TYPE_CCM            = 0x14,
-        NVM_TYPE_EAGLE_FW2      = 0x15,
-        NVM_TYPE_FALCON_FW2     = 0x16,
-        NVM_TYPE_PCIE_FW2       = 0x17,
-        NVM_TYPE_AVS_FW2        = 0x18,
-
-        NVM_TYPE_MAX,
-};
-
-#define MAX_NVM_DIR_ENTRIES 200
-
-struct nvm_dir {
-        s32 seq;
-#define NVM_DIR_NEXT_MFW_MASK   0x00000001
-#define NVM_DIR_SEQ_MASK        0xfffffffe
-#define NVM_DIR_NEXT_MFW(seq) ((seq) & NVM_DIR_NEXT_MFW_MASK)
-
-#define IS_DIR_SEQ_VALID(seq) ((seq & NVM_DIR_SEQ_MASK) != NVM_DIR_SEQ_MASK)
-
-        u32                     num_images;
-        u32                     rsrv;
-        struct nvm_code_entry   code[1]; /* Up to MAX_NVM_DIR_ENTRIES */
-};
-
-#define NVM_DIR_SIZE(_num_images) (sizeof(struct nvm_dir) +              \
-                                   (_num_images -                        \
-                                    1) * sizeof(struct nvm_code_entry) + \
-                                   NVM_CRC_SIZE)
-
-struct nvm_vpd_image {
-        u32     format_revision;
-#define VPD_IMAGE_VERSION        1
-
-        /* This array length depends on the number of VPD fields */
-        u8      vpd_data[1];
-};
-
-/****************************************************************************
-* NVRAM FULL MAP                                                           *
-****************************************************************************/
-#define DIR_ID_1    (0)
-#define DIR_ID_2    (1)
-#define MAX_DIR_IDS (2)
-
-#define MFW_BUNDLE_1    (0)
-#define MFW_BUNDLE_2    (1)
-#define MAX_MFW_BUNDLES (2)
-
-#define FLASH_PAGE_SIZE 0x1000
-#define NVM_DIR_MAX_SIZE    (FLASH_PAGE_SIZE)           /* 4Kb */
-#define ASIC_MIM_MAX_SIZE   (300 * FLASH_PAGE_SIZE)     /* 1.2Mb */
-#define FPGA_MIM_MAX_SIZE   (25 * FLASH_PAGE_SIZE)      /* 60Kb */
-
-#define LIM_MAX_SIZE        ((2 *                                     \
-                              FLASH_PAGE_SIZE) -                      \
-                             sizeof(struct legacy_bootstrap_region) - \
-                             NVM_RSV_SIZE)
-#define LIM_OFFSET          (NVM_OFFSET(lim_image))
-#define NVM_RSV_SIZE            (44)
-#define MIM_MAX_SIZE(is_asic) ((is_asic) ? ASIC_MIM_MAX_SIZE : \
-                               FPGA_MIM_MAX_SIZE)
-#define MIM_OFFSET(idx, is_asic) (NVM_OFFSET(dir[MAX_MFW_BUNDLES]) + \
-                                  ((idx ==                           \
-                                    NVM_TYPE_MIM2) ? MIM_MAX_SIZE(is_asic) : 0))
-#define NVM_FIXED_AREA_SIZE(is_asic) (sizeof(struct nvm_image) + \
-                                      MIM_MAX_SIZE(is_asic) * 2)
-
-union nvm_dir_union {
-        struct nvm_dir  dir;
-        u8              page[FLASH_PAGE_SIZE];
-};
-
-/*                        Address
- *  +-------------------+ 0x000000
- *  |    Bootstrap:     |
- *  | magic_number      |
- *  | sram_start_addr   |
- *  | code_len          |
- *  | code_start_addr   |
- *  | crc               |
- *  +-------------------+ 0x000014
- *  | rsrv              |
- *  +-------------------+ 0x000040
- *  | LIM               |
- *  +-------------------+ 0x002000
- *  | Dir1              |
- *  +-------------------+ 0x003000
- *  | Dir2              |
- *  +-------------------+ 0x004000
- *  | MIM1              |
- *  +-------------------+ 0x130000
- *  | MIM2              |
- *  +-------------------+ 0x25C000
- *  | Rest Images:      |
- *  | TIM1/2            |
- *  | MFW_TRACE1/2      |
- *  | Eagle/Falcon FW   |
- *  | PCIE/AVS FW       |
- *  | MBA/CCM/L2B       |
- *  | VPD               |
- *  | optic_modules     |
- *  |  ...              |
- *  +-------------------+ 0x400000
- */
-struct nvm_image {
-/*********** !!!  FIXED SECTIONS  !!! DO NOT MODIFY !!! **********************/
-        /* NVM Offset  (size) */
-        struct legacy_bootstrap_region  bootstrap;
-        u8                              rsrv[NVM_RSV_SIZE];
-        u8                              lim_image[LIM_MAX_SIZE];
-        union nvm_dir_union             dir[MAX_MFW_BUNDLES];
-
-        /* MIM1_IMAGE                              0x004000 (0x12c000) */
-        /* MIM2_IMAGE                              0x130000 (0x12c000) */
-/*********** !!!  FIXED SECTIONS  !!! DO NOT MODIFY !!! **********************/
-};                              /* 0x134 */
-
-#define NVM_OFFSET(f)   ((u32_t)((int_ptr_t)(&(((struct nvm_image *)0)->f))))
-
-struct hw_set_info {
-        u32     reg_type;
-#define GRC_REG_TYPE 1
-#define PHY_REG_TYPE 2
-#define PCI_REG_TYPE 4
-
-        u32     bank_num;
-        u32     pf_num;
-        u32     operation;
-#define READ_OP     1
-#define WRITE_OP    2
-#define RMW_SET_OP  3
-#define RMW_CLR_OP  4
-
-        u32     reg_addr;
-        u32     reg_data;
-
-        u32     reset_type;
-#define POR_RESET_TYPE  BIT(0)
-#define HARD_RESET_TYPE BIT(1)
-#define CORE_RESET_TYPE BIT(2)
-#define MCP_RESET_TYPE  BIT(3)
-#define PERSET_ASSERT   BIT(4)
-#define PERSET_DEASSERT BIT(5)
-};
-
-struct hw_set_image {
-        u32                     format_version;
-#define HW_SET_IMAGE_VERSION        1
-        u32                     no_hw_sets;
-
-        /* This array length depends on the no_hw_sets */
-        struct hw_set_info      hw_sets[1];
-};
-
-int qed_init_pf_wfq(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
-                    u8 pf_id, u16 pf_wfq);
-int qed_init_vport_wfq(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
-                       u16 first_tx_pq_id[NUM_OF_TCS], u16 vport_wfq);
 #endif
diff --git a/drivers/net/ethernet/qlogic/qed/qed_hw.c b/drivers/net/ethernet/qlogic/qed/qed_hw.c
index 0ada7fdb91bc..e17885321faf 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_hw.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_hw.c
@@ -446,7 +446,7 @@ qed_dmae_post_command(struct qed_hwfn *p_hwfn,
                            idx_cmd,
                            le32_to_cpu(command->opcode),
                            le16_to_cpu(command->opcode_b),
-                           le16_to_cpu(command->length),
+                           le16_to_cpu(command->length_dw),
                            le32_to_cpu(command->src_addr_hi),
                            le32_to_cpu(command->src_addr_lo),
                            le32_to_cpu(command->dst_addr_hi),
@@ -461,7 +461,7 @@ qed_dmae_post_command(struct qed_hwfn *p_hwfn,
                    idx_cmd,
                    le32_to_cpu(command->opcode),
                    le16_to_cpu(command->opcode_b),
-                   le16_to_cpu(command->length),
+                   le16_to_cpu(command->length_dw),
                    le32_to_cpu(command->src_addr_hi),
                    le32_to_cpu(command->src_addr_lo),
                    le32_to_cpu(command->dst_addr_hi),
@@ -645,7 +645,7 @@ static int qed_dmae_execute_sub_operation(struct qed_hwfn *p_hwfn,
                 return -EINVAL;
         }
 
-        cmd->length = cpu_to_le16((u16)length);
+        cmd->length_dw = cpu_to_le16((u16)length);
 
         qed_dmae_post_command(p_hwfn, p_ptt);
 
@@ -769,6 +769,29 @@ int qed_dmae_host2grc(struct qed_hwfn *p_hwfn,
 }
 
 int
+qed_dmae_grc2host(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u32 grc_addr,
+                  dma_addr_t dest_addr, u32 size_in_dwords, u32 flags)
+{
+        u32 grc_addr_in_dw = grc_addr / sizeof(u32);
+        struct qed_dmae_params params;
+        int rc;
+
+        memset(&params, 0, sizeof(struct qed_dmae_params));
+        params.flags = flags;
+
+        mutex_lock(&p_hwfn->dmae_info.mutex);
+
+        rc = qed_dmae_execute_command(p_hwfn, p_ptt, grc_addr_in_dw,
+                                      dest_addr, QED_DMAE_ADDRESS_GRC,
+                                      QED_DMAE_ADDRESS_HOST_VIRT,
+                                      size_in_dwords, &params);
+
+        mutex_unlock(&p_hwfn->dmae_info.mutex);
+
+        return rc;
+}
+
+int
 qed_dmae_host2host(struct qed_hwfn *p_hwfn,
                    struct qed_ptt *p_ptt,
                    dma_addr_t source_addr,
@@ -791,16 +814,16 @@ qed_dmae_host2host(struct qed_hwfn *p_hwfn,
 }
 
 u16 qed_get_qm_pq(struct qed_hwfn *p_hwfn,
-                  enum protocol_type proto,
-                  union qed_qm_pq_params *p_params)
+                  enum protocol_type proto, union qed_qm_pq_params *p_params)
 {
         u16 pq_id = 0;
 
-        if ((proto == PROTOCOLID_CORE || proto == PROTOCOLID_ETH) &&
-            !p_params) {
+        if ((proto == PROTOCOLID_CORE ||
+             proto == PROTOCOLID_ETH ||
+             proto == PROTOCOLID_ISCSI ||
+             proto == PROTOCOLID_ROCE) && !p_params) {
                 DP_NOTICE(p_hwfn,
-                          "Protocol %d received NULL PQ params\n",
-                          proto);
+                          "Protocol %d received NULL PQ params\n", proto);
                 return 0;
         }
 
@@ -808,6 +831,8 @@ u16 qed_get_qm_pq(struct qed_hwfn *p_hwfn,
         case PROTOCOLID_CORE:
                 if (p_params->core.tc == LB_TC)
                         pq_id = p_hwfn->qm_info.pure_lb_pq;
+                else if (p_params->core.tc == OOO_LB_TC)
+                        pq_id = p_hwfn->qm_info.ooo_pq;
                 else
                         pq_id = p_hwfn->qm_info.offload_pq;
                 break;
@@ -817,6 +842,18 @@ u16 qed_get_qm_pq(struct qed_hwfn *p_hwfn,
                         pq_id += p_hwfn->qm_info.vf_queues_offset +
                                  p_params->eth.vf_id;
                 break;
+        case PROTOCOLID_ISCSI:
+                if (p_params->iscsi.q_idx == 1)
+                        pq_id = p_hwfn->qm_info.pure_ack_pq;
+                break;
+        case PROTOCOLID_ROCE:
+                if (p_params->roce.dcqcn)
+                        pq_id = p_params->roce.qpid;
+                else
+                        pq_id = p_hwfn->qm_info.offload_pq;
+                if (pq_id > p_hwfn->qm_info.num_pf_rls)
+                        pq_id = p_hwfn->qm_info.offload_pq;
+                break;
         default:
                 pq_id = 0;
         }
diff --git a/drivers/net/ethernet/qlogic/qed/qed_hw.h b/drivers/net/ethernet/qlogic/qed/qed_hw.h
index 4367363ade40..d01557092868 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_hw.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_hw.h
@@ -254,6 +254,10 @@ void qed_dmae_info_free(struct qed_hwfn *p_hwfn);
 
 union qed_qm_pq_params {
         struct {
+                u8 q_idx;
+        } iscsi;
+
+        struct {
                 u8 tc;
         }       core;
 
@@ -262,11 +266,15 @@ union qed_qm_pq_params {
                 u8      vf_id;
                 u8      tc;
         }       eth;
+
+        struct {
+                u8 dcqcn;
+                u8 qpid;        /* roce relative */
+        } roce;
 };
 
 u16 qed_get_qm_pq(struct qed_hwfn *p_hwfn,
-                  enum protocol_type proto,
-                  union qed_qm_pq_params *params);
+                  enum protocol_type proto, union qed_qm_pq_params *params);
 
 int qed_init_fw_data(struct qed_dev *cdev,
                      const u8 *fw_data);
diff --git a/drivers/net/ethernet/qlogic/qed/qed_init_fw_funcs.c b/drivers/net/ethernet/qlogic/qed/qed_init_fw_funcs.c
index e8a3b9da59b5..23e455f22adc 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_init_fw_funcs.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_init_fw_funcs.c
@@ -31,7 +31,6 @@ enum cminterface {
 };
 
 /* general constants */
-#define QM_PQ_ELEMENT_SIZE                      4 /* in bytes */
 #define QM_PQ_MEM_4KB(pq_size)  (pq_size ? DIV_ROUND_UP((pq_size + 1) * \
                                                         QM_PQ_ELEMENT_SIZE, \
                                                         0x1000) : 0)
@@ -44,28 +43,28 @@ enum cminterface {
 /* other PQ constants */
 #define QM_OTHER_PQS_PER_PF                     4
 /* WFQ constants */
-#define QM_WFQ_UPPER_BOUND              6250000
+#define QM_WFQ_UPPER_BOUND              62500000
 #define QM_WFQ_VP_PQ_VOQ_SHIFT          0
 #define QM_WFQ_VP_PQ_PF_SHIFT           5
 #define QM_WFQ_INC_VAL(weight)          ((weight) * 0x9000)
-#define QM_WFQ_MAX_INC_VAL                      4375000
-#define QM_WFQ_INIT_CRD(inc_val)        (2 * (inc_val))
+#define QM_WFQ_MAX_INC_VAL                      43750000
+
 /* RL constants */
-#define QM_RL_UPPER_BOUND                       6250000
+#define QM_RL_UPPER_BOUND                       62500000
 #define QM_RL_PERIOD                            5               /* in us */
 #define QM_RL_PERIOD_CLK_25M            (25 * QM_RL_PERIOD)
+#define QM_RL_MAX_INC_VAL                       43750000
 #define QM_RL_INC_VAL(rate)             max_t(u32,      \
-                                              (((rate ? rate : 1000000) \
-                                                * QM_RL_PERIOD) / 8), 1)
-#define QM_RL_MAX_INC_VAL                       4375000
+                                              (u32)(((rate ? rate : \
+                                                      1000000) *    \
+                                                     QM_RL_PERIOD * \
+                                                     101) / (8 * 100)), 1)
 /* AFullOprtnstcCrdMask constants */
 #define QM_OPPOR_LINE_VOQ_DEF           1
 #define QM_OPPOR_FW_STOP_DEF            0
 #define QM_OPPOR_PQ_EMPTY_DEF           1
-#define EAGLE_WORKAROUND_TC                     7
 /* Command Queue constants */
 #define PBF_CMDQ_PURE_LB_LINES                          150
-#define PBF_CMDQ_EAGLE_WORKAROUND_LINES         8
 #define PBF_CMDQ_LINES_RT_OFFSET(voq)           (                \
                 PBF_REG_YCMD_QS_NUM_LINES_VOQ0_RT_OFFSET + voq * \
                 (PBF_REG_YCMD_QS_NUM_LINES_VOQ1_RT_OFFSET -      \
@@ -80,7 +79,6 @@ enum cminterface {
 /* BTB: blocks constants (block size = 256B) */
 #define BTB_JUMBO_PKT_BLOCKS            38
 #define BTB_HEADROOM_BLOCKS                     BTB_JUMBO_PKT_BLOCKS
-#define BTB_EAGLE_WORKAROUND_BLOCKS     4
 #define BTB_PURE_LB_FACTOR                      10
 #define BTB_PURE_LB_RATIO                       7
 /* QM stop command constants */
@@ -107,9 +105,9 @@ enum cminterface {
                                                  cmd ## _ ## field,       \
                                                  value)
 /* QM: VOQ macros */
-#define PHYS_VOQ(port, tc, max_phy_tcs_pr_port) ((port) *       \
-                                                 (max_phy_tcs_pr_port) \
-                                                 + (tc))
+#define PHYS_VOQ(port, tc, max_phys_tcs_per_port) ((port) *     \
+                                                   (max_phys_tcs_per_port) + \
+                                                   (tc))
 #define LB_VOQ(port)                            ( \
                 MAX_PHYS_VOQS + (port))
 #define VOQ(port, tc, max_phy_tcs_pr_port)      \
@@ -120,8 +118,7 @@ enum cminterface {
                 : LB_VOQ(port))
 /******************** INTERNAL IMPLEMENTATION *********************/
 /* Prepare PF RL enable/disable runtime init values */
-static void qed_enable_pf_rl(struct qed_hwfn *p_hwfn,
-                             bool pf_rl_en)
+static void qed_enable_pf_rl(struct qed_hwfn *p_hwfn, bool pf_rl_en)
 {
         STORE_RT_REG(p_hwfn, QM_REG_RLPFENABLE_RT_OFFSET, pf_rl_en ? 1 : 0);
         if (pf_rl_en) {
@@ -130,8 +127,7 @@ static void qed_enable_pf_rl(struct qed_hwfn *p_hwfn,
                              (1 << MAX_NUM_VOQS) - 1);
                 /* write RL period */
                 STORE_RT_REG(p_hwfn,
-                             QM_REG_RLPFPERIOD_RT_OFFSET,
-                             QM_RL_PERIOD_CLK_25M);
+                             QM_REG_RLPFPERIOD_RT_OFFSET, QM_RL_PERIOD_CLK_25M);
                 STORE_RT_REG(p_hwfn,
                              QM_REG_RLPFPERIODTIMER_RT_OFFSET,
                              QM_RL_PERIOD_CLK_25M);
@@ -144,8 +140,7 @@ static void qed_enable_pf_rl(struct qed_hwfn *p_hwfn,
 }
 
 /* Prepare PF WFQ enable/disable runtime init values */
-static void qed_enable_pf_wfq(struct qed_hwfn *p_hwfn,
-                              bool pf_wfq_en)
+static void qed_enable_pf_wfq(struct qed_hwfn *p_hwfn, bool pf_wfq_en)
 {
         STORE_RT_REG(p_hwfn, QM_REG_WFQPFENABLE_RT_OFFSET, pf_wfq_en ? 1 : 0);
         /* set credit threshold for QM bypass flow */
@@ -156,8 +151,7 @@ static void qed_enable_pf_wfq(struct qed_hwfn *p_hwfn,
 }
 
 /* Prepare VPORT RL enable/disable runtime init values */
-static void qed_enable_vport_rl(struct qed_hwfn *p_hwfn,
-                                bool vport_rl_en)
+static void qed_enable_vport_rl(struct qed_hwfn *p_hwfn, bool vport_rl_en)
 {
         STORE_RT_REG(p_hwfn, QM_REG_RLGLBLENABLE_RT_OFFSET,
                      vport_rl_en ? 1 : 0);
@@ -178,8 +172,7 @@ static void qed_enable_vport_rl(struct qed_hwfn *p_hwfn,
 }
 
 /* Prepare VPORT WFQ enable/disable runtime init values */
-static void qed_enable_vport_wfq(struct qed_hwfn *p_hwfn,
-                                 bool vport_wfq_en)
+static void qed_enable_vport_wfq(struct qed_hwfn *p_hwfn, bool vport_wfq_en)
 {
         STORE_RT_REG(p_hwfn, QM_REG_WFQVPENABLE_RT_OFFSET,
                      vport_wfq_en ? 1 : 0);
@@ -194,8 +187,7 @@ static void qed_enable_vport_wfq(struct qed_hwfn *p_hwfn,
  * the specified VOQ
  */
 static void qed_cmdq_lines_voq_rt_init(struct qed_hwfn *p_hwfn,
-                                       u8 voq,
-                                       u16 cmdq_lines)
+                                       u8 voq, u16 cmdq_lines)
 {
         u32 qm_line_crd;
 
@@ -221,7 +213,7 @@ static void qed_cmdq_lines_rt_init(
         u8 max_phys_tcs_per_port,
         struct init_qm_port_params port_params[MAX_NUM_PORTS])
 {
-        u8 tc, voq, port_id;
+        u8 tc, voq, port_id, num_tcs_in_port;
 
         /* clear PBF lines for all VOQs */
         for (voq = 0; voq < MAX_NUM_VOQS; voq++)
@@ -229,22 +221,31 @@ static void qed_cmdq_lines_rt_init(
         for (port_id = 0; port_id < max_ports_per_engine; port_id++) {
                 if (port_params[port_id].active) {
                         u16 phys_lines, phys_lines_per_tc;
-                        u8 phys_tcs = port_params[port_id].num_active_phys_tcs;
 
-                        /* find #lines to divide between the active
-                         * physical TCs.
-                         */
+                        /* find #lines to divide between active phys TCs */
                         phys_lines = port_params[port_id].num_pbf_cmd_lines -
                                      PBF_CMDQ_PURE_LB_LINES;
                         /* find #lines per active physical TC */
-                        phys_lines_per_tc = phys_lines / phys_tcs;
+                        num_tcs_in_port = 0;
+                        for (tc = 0; tc < NUM_OF_PHYS_TCS; tc++) {
+                                if (((port_params[port_id].active_phys_tcs >>
+                                      tc) & 0x1) == 1)
+                                        num_tcs_in_port++;
+                        }
+
+                        phys_lines_per_tc = phys_lines / num_tcs_in_port;
                         /* init registers per active TC */
-                        for (tc = 0; tc < phys_tcs; tc++) {
+                        for (tc = 0; tc < NUM_OF_PHYS_TCS; tc++) {
+                                if (((port_params[port_id].active_phys_tcs >>
+                                      tc) & 0x1) != 1)
+                                        continue;
+
                                 voq = PHYS_VOQ(port_id, tc,
                                                max_phys_tcs_per_port);
                                 qed_cmdq_lines_voq_rt_init(p_hwfn, voq,
                                                            phys_lines_per_tc);
                         }
+
                         /* init registers for pure LB TC */
                         qed_cmdq_lines_voq_rt_init(p_hwfn, LB_VOQ(port_id),
                                                    PBF_CMDQ_PURE_LB_LINES);
@@ -259,34 +260,42 @@ static void qed_btb_blocks_rt_init(
         struct init_qm_port_params port_params[MAX_NUM_PORTS])
 {
         u32 usable_blocks, pure_lb_blocks, phys_blocks;
-        u8 tc, voq, port_id;
+        u8 tc, voq, port_id, num_tcs_in_port;
 
         for (port_id = 0; port_id < max_ports_per_engine; port_id++) {
                 u32 temp;
-                u8 phys_tcs;
 
                 if (!port_params[port_id].active)
                         continue;
 
-                phys_tcs = port_params[port_id].num_active_phys_tcs;
-
                 /* subtract headroom blocks */
                 usable_blocks = port_params[port_id].num_btb_blocks -
                                 BTB_HEADROOM_BLOCKS;
 
-                /* find blocks per physical TC. use factor to avoid
-                 * floating arithmethic.
-                 */
+                /* find blocks per physical TC */
+                num_tcs_in_port = 0;
+                for (tc = 0; tc < NUM_OF_PHYS_TCS; tc++) {
+                        if (((port_params[port_id].active_phys_tcs >>
+                              tc) & 0x1) == 1)
+                                num_tcs_in_port++;
+                }
+
                 pure_lb_blocks = (usable_blocks * BTB_PURE_LB_FACTOR) /
-                                 (phys_tcs * BTB_PURE_LB_FACTOR +
+                                 (num_tcs_in_port * BTB_PURE_LB_FACTOR +
                                   BTB_PURE_LB_RATIO);
                 pure_lb_blocks = max_t(u32, BTB_JUMBO_PKT_BLOCKS,
                                        pure_lb_blocks / BTB_PURE_LB_FACTOR);
-                phys_blocks = (usable_blocks - pure_lb_blocks) / phys_tcs;
+                phys_blocks = (usable_blocks - pure_lb_blocks) /
+                              num_tcs_in_port;
 
                 /* init physical TCs */
-                for (tc = 0; tc < phys_tcs; tc++) {
-                        voq = PHYS_VOQ(port_id, tc, max_phys_tcs_per_port);
+                for (tc = 0; tc < NUM_OF_PHYS_TCS; tc++) {
+                        if (((port_params[port_id].active_phys_tcs >>
+                              tc) & 0x1) != 1)
+                                continue;
+
+                        voq = PHYS_VOQ(port_id, tc,
+                                       max_phys_tcs_per_port);
                         STORE_RT_REG(p_hwfn, PBF_BTB_GUARANTEED_RT_OFFSET(voq),
                                      phys_blocks);
                 }
@@ -360,10 +369,11 @@ static void qed_tx_pq_map_rt_init(
                 memset(&tx_pq_map, 0, sizeof(tx_pq_map));
                 SET_FIELD(tx_pq_map.reg, QM_RF_PQ_MAP_PQ_VALID, 1);
                 SET_FIELD(tx_pq_map.reg, QM_RF_PQ_MAP_RL_VALID,
-                          is_vf_pq ? 1 : 0);
+                          p_params->pq_params[i].rl_valid ? 1 : 0);
                 SET_FIELD(tx_pq_map.reg, QM_RF_PQ_MAP_VP_PQ_ID, first_tx_pq_id);
                 SET_FIELD(tx_pq_map.reg, QM_RF_PQ_MAP_RL_ID,
-                          is_vf_pq ? p_params->pq_params[i].vport_id : 0);
+                          p_params->pq_params[i].rl_valid ?
+                          p_params->pq_params[i].vport_id : 0);
                 SET_FIELD(tx_pq_map.reg, QM_RF_PQ_MAP_VOQ, voq);
                 SET_FIELD(tx_pq_map.reg, QM_RF_PQ_MAP_WRR_WEIGHT_GROUP,
                           p_params->pq_params[i].wrr_group);
@@ -390,25 +400,11 @@ static void qed_tx_pq_map_rt_init(
         /* store Tx PQ VF mask to size select register */
         for (i = 0; i < num_tx_pq_vf_masks; i++) {
                 if (tx_pq_vf_mask[i]) {
-                        if (is_bb_a0) {
-                                u32 curr_mask = 0, addr;
-
-                                addr = QM_REG_MAXPQSIZETXSEL_0 + (i * 4);
-                                if (!p_params->is_first_pf)
-                                        curr_mask = qed_rd(p_hwfn, p_ptt,
-                                                           addr);
-
-                                addr = QM_REG_MAXPQSIZETXSEL_0_RT_OFFSET + i;
-
-                                STORE_RT_REG(p_hwfn, addr,
-                                             curr_mask | tx_pq_vf_mask[i]);
-                        } else {
-                                u32 addr;
+                        u32 addr;
 
-                                addr = QM_REG_MAXPQSIZETXSEL_0_RT_OFFSET + i;
-                                STORE_RT_REG(p_hwfn, addr,
-                                             tx_pq_vf_mask[i]);
-                        }
+                        addr = QM_REG_MAXPQSIZETXSEL_0_RT_OFFSET + i;
+                        STORE_RT_REG(p_hwfn, addr,
+                                     tx_pq_vf_mask[i]);
                 }
         }
 }
@@ -418,8 +414,7 @@ static void qed_other_pq_map_rt_init(struct qed_hwfn *p_hwfn,
                                      u8 port_id,
                                      u8 pf_id,
                                      u32 num_pf_cids,
-                                     u32 num_tids,
-                                     u32 base_mem_addr_4kb)
+                                     u32 num_tids, u32 base_mem_addr_4kb)
 {
         u16 i, pq_id;
 
@@ -465,15 +460,10 @@ static int qed_pf_wfq_rt_init(struct qed_hwfn *p_hwfn,
                                  (p_params->pf_id % MAX_NUM_PFS_BB);
 
         inc_val = QM_WFQ_INC_VAL(p_params->pf_wfq);
-        if (inc_val > QM_WFQ_MAX_INC_VAL) {
+        if (!inc_val || inc_val > QM_WFQ_MAX_INC_VAL) {
                 DP_NOTICE(p_hwfn, "Invalid PF WFQ weight configuration");
                 return -1;
         }
-        STORE_RT_REG(p_hwfn, QM_REG_WFQPFWEIGHT_RT_OFFSET + p_params->pf_id,
-                     inc_val);
-        STORE_RT_REG(p_hwfn,
-                     QM_REG_WFQPFUPPERBOUND_RT_OFFSET + p_params->pf_id,
-                     QM_WFQ_UPPER_BOUND | QM_WFQ_CRD_REG_SIGN_BIT);
 
         for (i = 0; i < num_tx_pqs; i++) {
                 u8 voq = VOQ(p_params->port_id, p_params->pq_params[i].tc_id,
@@ -481,19 +471,21 @@ static int qed_pf_wfq_rt_init(struct qed_hwfn *p_hwfn,
 
                 OVERWRITE_RT_REG(p_hwfn,
                                  crd_reg_offset + voq * MAX_NUM_PFS_BB,
-                                 QM_WFQ_INIT_CRD(inc_val) |
                                  QM_WFQ_CRD_REG_SIGN_BIT);
         }
 
+        STORE_RT_REG(p_hwfn, QM_REG_WFQPFWEIGHT_RT_OFFSET + p_params->pf_id,
+                     inc_val);
+        STORE_RT_REG(p_hwfn,
+                     QM_REG_WFQPFUPPERBOUND_RT_OFFSET + p_params->pf_id,
+                     QM_WFQ_UPPER_BOUND | QM_WFQ_CRD_REG_SIGN_BIT);
         return 0;
 }
 
 /* Prepare PF RL runtime init values for the specified PF.
  * Return -1 on error.
  */
-static int qed_pf_rl_rt_init(struct qed_hwfn *p_hwfn,
-                             u8 pf_id,
-                             u32 pf_rl)
+static int qed_pf_rl_rt_init(struct qed_hwfn *p_hwfn, u8 pf_id, u32 pf_rl)
 {
         u32 inc_val = QM_RL_INC_VAL(pf_rl);
 
@@ -607,9 +599,7 @@ static bool qed_poll_on_qm_cmd_ready(struct qed_hwfn *p_hwfn,
 
 static bool qed_send_qm_cmd(struct qed_hwfn *p_hwfn,
                             struct qed_ptt *p_ptt,
-                            u32 cmd_addr,
-                            u32 cmd_data_lsb,
-                            u32 cmd_data_msb)
+                            u32 cmd_addr, u32 cmd_data_lsb, u32 cmd_data_msb)
 {
         if (!qed_poll_on_qm_cmd_ready(p_hwfn, p_ptt))
                 return false;
@@ -627,9 +617,7 @@ static bool qed_send_qm_cmd(struct qed_hwfn *p_hwfn,
 u32 qed_qm_pf_mem_size(u8 pf_id,
                        u32 num_pf_cids,
                        u32 num_vf_cids,
-                       u32 num_tids,
-                       u16 num_pf_pqs,
-                       u16 num_vf_pqs)
+                       u32 num_tids, u16 num_pf_pqs, u16 num_vf_pqs)
 {
         return QM_PQ_MEM_4KB(num_pf_cids) * num_pf_pqs +
                QM_PQ_MEM_4KB(num_vf_cids) * num_vf_pqs +
@@ -713,8 +701,7 @@ int qed_qm_pf_rt_init(struct qed_hwfn *p_hwfn,
 }
 
 int qed_init_pf_wfq(struct qed_hwfn *p_hwfn,
-                    struct qed_ptt *p_ptt,
-                    u8 pf_id, u16 pf_wfq)
+                    struct qed_ptt *p_ptt, u8 pf_id, u16 pf_wfq)
 {
         u32 inc_val = QM_WFQ_INC_VAL(pf_wfq);
 
@@ -728,9 +715,7 @@ int qed_init_pf_wfq(struct qed_hwfn *p_hwfn,
 }
 
 int qed_init_pf_rl(struct qed_hwfn *p_hwfn,
-                   struct qed_ptt *p_ptt,
-                   u8 pf_id,
-                   u32 pf_rl)
+                   struct qed_ptt *p_ptt, u8 pf_id, u32 pf_rl)
 {
         u32 inc_val = QM_RL_INC_VAL(pf_rl);
 
@@ -749,8 +734,7 @@ int qed_init_pf_rl(struct qed_hwfn *p_hwfn,
 
 int qed_init_vport_wfq(struct qed_hwfn *p_hwfn,
                        struct qed_ptt *p_ptt,
-                       u16 first_tx_pq_id[NUM_OF_TCS],
-                       u16 vport_wfq)
+                       u16 first_tx_pq_id[NUM_OF_TCS], u16 vport_wfq)
 {
         u32 inc_val = QM_WFQ_INC_VAL(vport_wfq);
         u8 tc;
@@ -773,9 +757,7 @@ int qed_init_vport_wfq(struct qed_hwfn *p_hwfn,
 }
 
 int qed_init_vport_rl(struct qed_hwfn *p_hwfn,
-                      struct qed_ptt *p_ptt,
-                      u8 vport_id,
-                      u32 vport_rl)
+                      struct qed_ptt *p_ptt, u8 vport_id, u32 vport_rl)
 {
         u32 inc_val = QM_RL_INC_VAL(vport_rl);
 
@@ -795,9 +777,7 @@ int qed_init_vport_rl(struct qed_hwfn *p_hwfn,
 bool qed_send_qm_stop_cmd(struct qed_hwfn *p_hwfn,
                           struct qed_ptt *p_ptt,
                           bool is_release_cmd,
-                          bool is_tx_pq,
-                          u16 start_pq,
-                          u16 num_pqs)
+                          bool is_tx_pq, u16 start_pq, u16 num_pqs)
 {
         u32 cmd_arr[QM_CMD_STRUCT_SIZE(QM_STOP_CMD)] = { 0 };
         u32 pq_mask = 0, last_pq = start_pq + num_pqs - 1, pq_id;
@@ -841,17 +821,15 @@ qed_set_tunnel_type_enable_bit(unsigned long *var, int bit, bool enable)
 #define PRS_ETH_TUNN_FIC_FORMAT -188897008
 
 void qed_set_vxlan_dest_port(struct qed_hwfn *p_hwfn,
-                             struct qed_ptt *p_ptt,
-                             u16 dest_port)
+                             struct qed_ptt *p_ptt, u16 dest_port)
 {
         qed_wr(p_hwfn, p_ptt, PRS_REG_VXLAN_PORT, dest_port);
-        qed_wr(p_hwfn, p_ptt, NIG_REG_VXLAN_PORT, dest_port);
+        qed_wr(p_hwfn, p_ptt, NIG_REG_VXLAN_CTRL, dest_port);
         qed_wr(p_hwfn, p_ptt, PBF_REG_VXLAN_PORT, dest_port);
 }
 
 void qed_set_vxlan_enable(struct qed_hwfn *p_hwfn,
-                          struct qed_ptt *p_ptt,
-                          bool vxlan_enable)
+                          struct qed_ptt *p_ptt, bool vxlan_enable)
 {
         unsigned long reg_val = 0;
         u8 shift;
@@ -908,8 +886,7 @@ void qed_set_gre_enable(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
 }
 
 void qed_set_geneve_dest_port(struct qed_hwfn *p_hwfn,
-                              struct qed_ptt *p_ptt,
-                              u16 dest_port)
+                              struct qed_ptt *p_ptt, u16 dest_port)
 {
         qed_wr(p_hwfn, p_ptt, PRS_REG_NGE_PORT, dest_port);
         qed_wr(p_hwfn, p_ptt, NIG_REG_NGE_PORT, dest_port);
@@ -918,8 +895,7 @@ void qed_set_geneve_dest_port(struct qed_hwfn *p_hwfn,
 
 void qed_set_geneve_enable(struct qed_hwfn *p_hwfn,
                            struct qed_ptt *p_ptt,
-                           bool eth_geneve_enable,
-                           bool ip_geneve_enable)
+                           bool eth_geneve_enable, bool ip_geneve_enable)
 {
         unsigned long reg_val = 0;
         u8 shift;
diff --git a/drivers/net/ethernet/qlogic/qed/qed_init_ops.c b/drivers/net/ethernet/qlogic/qed/qed_init_ops.c
index d358c3bb1308..9866a20d2128 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_init_ops.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_init_ops.c
@@ -543,8 +543,7 @@ void qed_gtt_init(struct qed_hwfn *p_hwfn)
                                pxp_global_win[i]);
 }
 
-int qed_init_fw_data(struct qed_dev *cdev,
-                     const u8 *data)
+int qed_init_fw_data(struct qed_dev *cdev, const u8 *data)
 {
         struct qed_fw_data *fw = cdev->fw_data;
         struct bin_buffer_hdr *buf_hdr;
@@ -555,7 +554,11 @@ int qed_init_fw_data(struct qed_dev *cdev,
                 return -EINVAL;
         }
 
-        buf_hdr = (struct bin_buffer_hdr *)data;
+        /* First Dword contains metadata and should be skipped */
+        buf_hdr = (struct bin_buffer_hdr *)(data + sizeof(u32));
+
+        offset = buf_hdr[BIN_BUF_FW_VER_INFO].offset;
+        fw->fw_ver_info = (struct fw_ver_info *)(data + offset);
 
         offset = buf_hdr[BIN_BUF_INIT_CMD].offset;
         fw->init_ops = (union init_op *)(data + offset);
diff --git a/drivers/net/ethernet/qlogic/qed/qed_int.c b/drivers/net/ethernet/qlogic/qed/qed_int.c
index 09a6ad3d22dd..8fa50fa23c8d 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_int.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_int.c
@@ -2418,6 +2418,7 @@ void qed_init_cau_sb_entry(struct qed_hwfn *p_hwfn,
 {
         struct qed_dev *cdev = p_hwfn->cdev;
         u32 cau_state;
+        u8 timer_res;
 
         memset(p_sb_entry, 0, sizeof(*p_sb_entry));
 
@@ -2443,6 +2444,23 @@ void qed_init_cau_sb_entry(struct qed_hwfn *p_hwfn,
                         cdev->tx_coalesce_usecs = QED_CAU_DEF_TX_USECS;
         }
 
+        /* Coalesce = (timeset << timer-res), timeset is 7bit wide */
+        if (cdev->rx_coalesce_usecs <= 0x7F)
+                timer_res = 0;
+        else if (cdev->rx_coalesce_usecs <= 0xFF)
+                timer_res = 1;
+        else
+                timer_res = 2;
+        SET_FIELD(p_sb_entry->params, CAU_SB_ENTRY_TIMER_RES0, timer_res);
+
+        if (cdev->tx_coalesce_usecs <= 0x7F)
+                timer_res = 0;
+        else if (cdev->tx_coalesce_usecs <= 0xFF)
+                timer_res = 1;
+        else
+                timer_res = 2;
+        SET_FIELD(p_sb_entry->params, CAU_SB_ENTRY_TIMER_RES1, timer_res);
+
         SET_FIELD(p_sb_entry->data, CAU_SB_ENTRY_STATE0, cau_state);
         SET_FIELD(p_sb_entry->data, CAU_SB_ENTRY_STATE1, cau_state);
 }
@@ -2484,17 +2502,28 @@ void qed_int_cau_conf_sb(struct qed_hwfn *p_hwfn,
 
         /* Configure pi coalescing if set */
         if (p_hwfn->cdev->int_coalescing_mode == QED_COAL_MODE_ENABLE) {
-                u8 timeset = p_hwfn->cdev->rx_coalesce_usecs >>
-                             (QED_CAU_DEF_RX_TIMER_RES + 1);
+                u8 timeset, timer_res;
                 u8 num_tc = 1, i;
 
+                /* timeset = (coalesce >> timer-res), timeset is 7bit wide */
+                if (p_hwfn->cdev->rx_coalesce_usecs <= 0x7F)
+                        timer_res = 0;
+                else if (p_hwfn->cdev->rx_coalesce_usecs <= 0xFF)
+                        timer_res = 1;
+                else
+                        timer_res = 2;
+                timeset = (u8)(p_hwfn->cdev->rx_coalesce_usecs >> timer_res);
                 qed_int_cau_conf_pi(p_hwfn, p_ptt, igu_sb_id, RX_PI,
                                     QED_COAL_RX_STATE_MACHINE,
                                     timeset);
 
-                timeset = p_hwfn->cdev->tx_coalesce_usecs >>
-                          (QED_CAU_DEF_TX_TIMER_RES + 1);
-
+                if (p_hwfn->cdev->tx_coalesce_usecs <= 0x7F)
+                        timer_res = 0;
+                else if (p_hwfn->cdev->tx_coalesce_usecs <= 0xFF)
+                        timer_res = 1;
+                else
+                        timer_res = 2;
+                timeset = (u8)(p_hwfn->cdev->tx_coalesce_usecs >> timer_res);
                 for (i = 0; i < num_tc; i++) {
                         qed_int_cau_conf_pi(p_hwfn, p_ptt,
                                             igu_sb_id, TX_PI(i),
@@ -3199,3 +3228,39 @@ void qed_int_disable_post_isr_release(struct qed_dev *cdev)
         for_each_hwfn(cdev, i)
                 cdev->hwfns[i].b_int_requested = false;
 }
+
+int qed_int_set_timer_res(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
+                          u8 timer_res, u16 sb_id, bool tx)
+{
+        struct cau_sb_entry sb_entry;
+        int rc;
+
+        if (!p_hwfn->hw_init_done) {
+                DP_ERR(p_hwfn, "hardware not initialized yet\n");
+                return -EINVAL;
+        }
+
+        rc = qed_dmae_grc2host(p_hwfn, p_ptt, CAU_REG_SB_VAR_MEMORY +
+                               sb_id * sizeof(u64),
+                               (u64)(uintptr_t)&sb_entry, 2, 0);
+        if (rc) {
+                DP_ERR(p_hwfn, "dmae_grc2host failed %d\n", rc);
+                return rc;
+        }
+
+        if (tx)
+                SET_FIELD(sb_entry.params, CAU_SB_ENTRY_TIMER_RES1, timer_res);
+        else
+                SET_FIELD(sb_entry.params, CAU_SB_ENTRY_TIMER_RES0, timer_res);
+
+        rc = qed_dmae_host2grc(p_hwfn, p_ptt,
+                               (u64)(uintptr_t)&sb_entry,
+                               CAU_REG_SB_VAR_MEMORY +
+                               sb_id * sizeof(u64), 2, 0);
+        if (rc) {
+                DP_ERR(p_hwfn, "dmae_host2grc failed %d\n", rc);
+                return rc;
+        }
+
+        return rc;
+}
diff --git a/drivers/net/ethernet/qlogic/qed/qed_int.h b/drivers/net/ethernet/qlogic/qed/qed_int.h
index 20b468637504..0948be64dc78 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_int.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_int.h
@@ -389,6 +389,9 @@ void qed_init_cau_sb_entry(struct qed_hwfn *p_hwfn,
                            u16 vf_number,
                            u8 vf_valid);
 
+int qed_int_set_timer_res(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
+                          u8 timer_res, u16 sb_id, bool tx);
+
 #define QED_MAPPING_MEMORY_SIZE(dev)    (NUM_OF_SBS(dev))
 
 #endif
diff --git a/drivers/net/ethernet/qlogic/qed/qed_l2.c b/drivers/net/ethernet/qlogic/qed/qed_l2.c
index aada4c7e095f..a12c6caa6c66 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_l2.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_l2.c
@@ -572,9 +572,12 @@ int qed_sp_eth_rxq_start_ramrod(struct qed_hwfn *p_hwfn,
         p_ramrod->num_of_pbl_pages      = cpu_to_le16(cqe_pbl_size);
         DMA_REGPAIR_LE(p_ramrod->cqe_pbl_addr, cqe_pbl_addr);
 
-        rc = qed_spq_post(p_hwfn, p_ent, NULL);
+        p_ramrod->vf_rx_prod_index = params->vf_qid;
+        if (params->vf_qid)
+                DP_VERBOSE(p_hwfn, QED_MSG_SP,
+                           "Queue is meant for VF rxq[%04x]\n", params->vf_qid);
 
-        return rc;
+        return qed_spq_post(p_hwfn, p_ent, NULL);
 }
 
 static int
@@ -612,7 +615,7 @@ qed_sp_eth_rx_queue_start(struct qed_hwfn *p_hwfn,
 
         *pp_prod = (u8 __iomem *)p_hwfn->regview +
                                  GTT_BAR0_MAP_REG_MSDM_RAM +
-                                 MSTORM_PRODS_OFFSET(abs_l2_queue);
+                                 MSTORM_ETH_PF_PRODS_OFFSET(abs_l2_queue);
 
         /* Init the rcq, rx bd and rx sge (if valid) producers to 0 */
         __internal_ram_wr(p_hwfn, *pp_prod, sizeof(u64),
@@ -756,9 +759,9 @@ int qed_sp_eth_txq_start_ramrod(struct qed_hwfn  *p_hwfn,
         struct qed_spq_entry *p_ent = NULL;
         struct qed_sp_init_data init_data;
         struct qed_hw_cid_data *p_tx_cid;
-        u8 abs_vport_id;
+        u16 pq_id, abs_tx_q_id = 0;
         int rc = -EINVAL;
-        u16 pq_id;
+        u8 abs_vport_id;
 
         /* Store information for the stop */
         p_tx_cid = &p_hwfn->p_tx_cids[p_params->queue_id];
@@ -769,6 +772,10 @@ int qed_sp_eth_txq_start_ramrod(struct qed_hwfn  *p_hwfn,
         if (rc)
                 return rc;
 
+        rc = qed_fw_l2_queue(p_hwfn, p_params->queue_id, &abs_tx_q_id);
+        if (rc)
+                return rc;
+
         /* Get SPQ entry */
         memset(&init_data, 0, sizeof(init_data));
         init_data.cid = cid;
@@ -788,6 +795,7 @@ int qed_sp_eth_txq_start_ramrod(struct qed_hwfn  *p_hwfn,
         p_ramrod->sb_index              = p_params->sb_idx;
         p_ramrod->stats_counter_id      = stats_id;
 
+        p_ramrod->queue_zone_id         = cpu_to_le16(abs_tx_q_id);
         p_ramrod->pbl_size              = cpu_to_le16(pbl_size);
         DMA_REGPAIR_LE(p_ramrod->pbl_base_addr, pbl_addr);
 
@@ -1482,51 +1490,51 @@ static void __qed_get_vport_port_stats(struct qed_hwfn *p_hwfn,
                         offsetof(struct public_port, stats),
                         sizeof(port_stats));
 
-        p_stats->rx_64_byte_packets             += port_stats.pmm.r64;
-        p_stats->rx_65_to_127_byte_packets      += port_stats.pmm.r127;
-        p_stats->rx_128_to_255_byte_packets     += port_stats.pmm.r255;
-        p_stats->rx_256_to_511_byte_packets     += port_stats.pmm.r511;
-        p_stats->rx_512_to_1023_byte_packets    += port_stats.pmm.r1023;
-        p_stats->rx_1024_to_1518_byte_packets   += port_stats.pmm.r1518;
-        p_stats->rx_1519_to_1522_byte_packets   += port_stats.pmm.r1522;
-        p_stats->rx_1519_to_2047_byte_packets   += port_stats.pmm.r2047;
-        p_stats->rx_2048_to_4095_byte_packets   += port_stats.pmm.r4095;
-        p_stats->rx_4096_to_9216_byte_packets   += port_stats.pmm.r9216;
-        p_stats->rx_9217_to_16383_byte_packets  += port_stats.pmm.r16383;
-        p_stats->rx_crc_errors                  += port_stats.pmm.rfcs;
-        p_stats->rx_mac_crtl_frames             += port_stats.pmm.rxcf;
-        p_stats->rx_pause_frames                += port_stats.pmm.rxpf;
-        p_stats->rx_pfc_frames                  += port_stats.pmm.rxpp;
-        p_stats->rx_align_errors                += port_stats.pmm.raln;
-        p_stats->rx_carrier_errors              += port_stats.pmm.rfcr;
-        p_stats->rx_oversize_packets            += port_stats.pmm.rovr;
-        p_stats->rx_jabbers                     += port_stats.pmm.rjbr;
-        p_stats->rx_undersize_packets           += port_stats.pmm.rund;
-        p_stats->rx_fragments                   += port_stats.pmm.rfrg;
-        p_stats->tx_64_byte_packets             += port_stats.pmm.t64;
-        p_stats->tx_65_to_127_byte_packets      += port_stats.pmm.t127;
-        p_stats->tx_128_to_255_byte_packets     += port_stats.pmm.t255;
-        p_stats->tx_256_to_511_byte_packets     += port_stats.pmm.t511;
-        p_stats->tx_512_to_1023_byte_packets    += port_stats.pmm.t1023;
-        p_stats->tx_1024_to_1518_byte_packets   += port_stats.pmm.t1518;
-        p_stats->tx_1519_to_2047_byte_packets   += port_stats.pmm.t2047;
-        p_stats->tx_2048_to_4095_byte_packets   += port_stats.pmm.t4095;
-        p_stats->tx_4096_to_9216_byte_packets   += port_stats.pmm.t9216;
-        p_stats->tx_9217_to_16383_byte_packets  += port_stats.pmm.t16383;
-        p_stats->tx_pause_frames                += port_stats.pmm.txpf;
-        p_stats->tx_pfc_frames                  += port_stats.pmm.txpp;
-        p_stats->tx_lpi_entry_count             += port_stats.pmm.tlpiec;
-        p_stats->tx_total_collisions            += port_stats.pmm.tncl;
-        p_stats->rx_mac_bytes                   += port_stats.pmm.rbyte;
-        p_stats->rx_mac_uc_packets              += port_stats.pmm.rxuca;
-        p_stats->rx_mac_mc_packets              += port_stats.pmm.rxmca;
-        p_stats->rx_mac_bc_packets              += port_stats.pmm.rxbca;
-        p_stats->rx_mac_frames_ok               += port_stats.pmm.rxpok;
-        p_stats->tx_mac_bytes                   += port_stats.pmm.tbyte;
-        p_stats->tx_mac_uc_packets              += port_stats.pmm.txuca;
-        p_stats->tx_mac_mc_packets              += port_stats.pmm.txmca;
-        p_stats->tx_mac_bc_packets              += port_stats.pmm.txbca;
-        p_stats->tx_mac_ctrl_frames             += port_stats.pmm.txcf;
+        p_stats->rx_64_byte_packets             += port_stats.eth.r64;
+        p_stats->rx_65_to_127_byte_packets      += port_stats.eth.r127;
+        p_stats->rx_128_to_255_byte_packets     += port_stats.eth.r255;
+        p_stats->rx_256_to_511_byte_packets     += port_stats.eth.r511;
+        p_stats->rx_512_to_1023_byte_packets    += port_stats.eth.r1023;
+        p_stats->rx_1024_to_1518_byte_packets   += port_stats.eth.r1518;
+        p_stats->rx_1519_to_1522_byte_packets   += port_stats.eth.r1522;
+        p_stats->rx_1519_to_2047_byte_packets   += port_stats.eth.r2047;
+        p_stats->rx_2048_to_4095_byte_packets   += port_stats.eth.r4095;
+        p_stats->rx_4096_to_9216_byte_packets   += port_stats.eth.r9216;
+        p_stats->rx_9217_to_16383_byte_packets  += port_stats.eth.r16383;
+        p_stats->rx_crc_errors                  += port_stats.eth.rfcs;
+        p_stats->rx_mac_crtl_frames             += port_stats.eth.rxcf;
+        p_stats->rx_pause_frames                += port_stats.eth.rxpf;
+        p_stats->rx_pfc_frames                  += port_stats.eth.rxpp;
+        p_stats->rx_align_errors                += port_stats.eth.raln;
+        p_stats->rx_carrier_errors              += port_stats.eth.rfcr;
+        p_stats->rx_oversize_packets            += port_stats.eth.rovr;
+        p_stats->rx_jabbers                     += port_stats.eth.rjbr;
+        p_stats->rx_undersize_packets           += port_stats.eth.rund;
+        p_stats->rx_fragments                   += port_stats.eth.rfrg;
+        p_stats->tx_64_byte_packets             += port_stats.eth.t64;
+        p_stats->tx_65_to_127_byte_packets      += port_stats.eth.t127;
+        p_stats->tx_128_to_255_byte_packets     += port_stats.eth.t255;
+        p_stats->tx_256_to_511_byte_packets     += port_stats.eth.t511;
+        p_stats->tx_512_to_1023_byte_packets    += port_stats.eth.t1023;
+        p_stats->tx_1024_to_1518_byte_packets   += port_stats.eth.t1518;
+        p_stats->tx_1519_to_2047_byte_packets   += port_stats.eth.t2047;
+        p_stats->tx_2048_to_4095_byte_packets   += port_stats.eth.t4095;
+        p_stats->tx_4096_to_9216_byte_packets   += port_stats.eth.t9216;
+        p_stats->tx_9217_to_16383_byte_packets  += port_stats.eth.t16383;
+        p_stats->tx_pause_frames                += port_stats.eth.txpf;
+        p_stats->tx_pfc_frames                  += port_stats.eth.txpp;
+        p_stats->tx_lpi_entry_count             += port_stats.eth.tlpiec;
+        p_stats->tx_total_collisions            += port_stats.eth.tncl;
+        p_stats->rx_mac_bytes                   += port_stats.eth.rbyte;
+        p_stats->rx_mac_uc_packets              += port_stats.eth.rxuca;
+        p_stats->rx_mac_mc_packets              += port_stats.eth.rxmca;
+        p_stats->rx_mac_bc_packets              += port_stats.eth.rxbca;
+        p_stats->rx_mac_frames_ok               += port_stats.eth.rxpok;
+        p_stats->tx_mac_bytes                   += port_stats.eth.tbyte;
+        p_stats->tx_mac_uc_packets              += port_stats.eth.txuca;
+        p_stats->tx_mac_mc_packets              += port_stats.eth.txmca;
+        p_stats->tx_mac_bc_packets              += port_stats.eth.txbca;
+        p_stats->tx_mac_ctrl_frames             += port_stats.eth.txcf;
         for (j = 0; j < 8; j++) {
                 p_stats->brb_truncates  += port_stats.brb.brb_truncate[j];
                 p_stats->brb_discards   += port_stats.brb.brb_discard[j];
@@ -2156,11 +2164,18 @@ static int qed_fp_cqe_completion(struct qed_dev *dev,
 extern const struct qed_iov_hv_ops qed_iov_ops_pass;
 #endif
 
+#ifdef CONFIG_DCB
+extern const struct qed_eth_dcbnl_ops qed_dcbnl_ops_pass;
+#endif
+
 static const struct qed_eth_ops qed_eth_ops_pass = {
         .common = &qed_common_ops_pass,
 #ifdef CONFIG_QED_SRIOV
         .iov = &qed_iov_ops_pass,
 #endif
+#ifdef CONFIG_DCB
+        .dcb = &qed_dcbnl_ops_pass,
+#endif
         .fill_dev_info = &qed_fill_eth_dev_info,
         .register_ops = &qed_register_eth_ops,
         .check_mac = &qed_check_mac,
diff --git a/drivers/net/ethernet/qlogic/qed/qed_main.c b/drivers/net/ethernet/qlogic/qed/qed_main.c
index c7e01b303540..1f13abb5c316 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_main.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_main.c
@@ -207,6 +207,8 @@ int qed_fill_dev_info(struct qed_dev *cdev,
         dev_info->pci_mem_start = cdev->pci_params.mem_start;
         dev_info->pci_mem_end = cdev->pci_params.mem_end;
         dev_info->pci_irq = cdev->pci_params.irq;
+        dev_info->rdma_supported =
+            (cdev->hwfns[0].hw_info.personality == QED_PCI_ETH_ROCE);
         dev_info->is_mf_default = IS_MF_DEFAULT(&cdev->hwfns[0]);
         ether_addr_copy(dev_info->hw_mac, cdev->hwfns[0].hw_info.hw_mac_addr);
 
@@ -832,7 +834,8 @@ static int qed_slowpath_start(struct qed_dev *cdev,
                         goto err2;
                 }
 
-                data = cdev->firmware->data;
+                /* First Dword used to diffrentiate between various sources */
+                data = cdev->firmware->data + sizeof(u32);
         }
 
         memset(&tunn_info, 0, sizeof(tunn_info));
@@ -900,7 +903,8 @@ static int qed_slowpath_stop(struct qed_dev *cdev)
 
         if (IS_PF(cdev)) {
                 qed_free_stream_mem(cdev);
-                qed_sriov_disable(cdev, true);
+                if (IS_QED_ETH_IF(cdev))
+                        qed_sriov_disable(cdev, true);
 
                 qed_nic_stop(cdev);
                 qed_slowpath_irq_free(cdev);
@@ -991,8 +995,7 @@ static bool qed_can_link_change(struct qed_dev *cdev)
         return true;
 }
 
-static int qed_set_link(struct qed_dev *cdev,
-                        struct qed_link_params *params)
+static int qed_set_link(struct qed_dev *cdev, struct qed_link_params *params)
 {
         struct qed_hwfn *hwfn;
         struct qed_mcp_link_params *link_params;
@@ -1032,7 +1035,7 @@ static int qed_set_link(struct qed_dev *cdev,
                                 NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_50G;
                 if (params->adv_speeds & 0)
                         link_params->speed.advertised_speeds |=
-                                NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_100G;
+                            NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_BB_100G;
         }
         if (params->override_flags & QED_LINK_OVERRIDE_SPEED_FORCED_SPEED)
                 link_params->speed.forced_speed = params->forced_speed;
@@ -1053,19 +1056,19 @@ static int qed_set_link(struct qed_dev *cdev,
         if (params->override_flags & QED_LINK_OVERRIDE_LOOPBACK_MODE) {
                 switch (params->loopback_mode) {
                 case QED_LINK_LOOPBACK_INT_PHY:
-                        link_params->loopback_mode = PMM_LOOPBACK_INT_PHY;
+                        link_params->loopback_mode = ETH_LOOPBACK_INT_PHY;
                         break;
                 case QED_LINK_LOOPBACK_EXT_PHY:
-                        link_params->loopback_mode = PMM_LOOPBACK_EXT_PHY;
+                        link_params->loopback_mode = ETH_LOOPBACK_EXT_PHY;
                         break;
                 case QED_LINK_LOOPBACK_EXT:
-                        link_params->loopback_mode = PMM_LOOPBACK_EXT;
+                        link_params->loopback_mode = ETH_LOOPBACK_EXT;
                         break;
                 case QED_LINK_LOOPBACK_MAC:
-                        link_params->loopback_mode = PMM_LOOPBACK_MAC;
+                        link_params->loopback_mode = ETH_LOOPBACK_MAC;
                         break;
                 default:
-                        link_params->loopback_mode = PMM_LOOPBACK_NONE;
+                        link_params->loopback_mode = ETH_LOOPBACK_NONE;
                         break;
                 }
         }
@@ -1185,7 +1188,7 @@ static void qed_fill_link(struct qed_hwfn *hwfn,
                 NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_50G)
                 if_link->advertised_caps |= 0;
         if (params.speed.advertised_speeds &
-                NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_100G)
+            NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_BB_100G)
                 if_link->advertised_caps |= 0;
 
         if (link_caps.speed_capabilities &
@@ -1202,7 +1205,7 @@ static void qed_fill_link(struct qed_hwfn *hwfn,
                 NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_50G)
                 if_link->supported_caps |= 0;
         if (link_caps.speed_capabilities &
-                NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_100G)
+            NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_BB_100G)
                 if_link->supported_caps |= 0;
 
         if (link.link_up)
@@ -1301,6 +1304,38 @@ static int qed_drain(struct qed_dev *cdev)
         return 0;
 }
 
+static void qed_get_coalesce(struct qed_dev *cdev, u16 *rx_coal, u16 *tx_coal)
+{
+        *rx_coal = cdev->rx_coalesce_usecs;
+        *tx_coal = cdev->tx_coalesce_usecs;
+}
+
+static int qed_set_coalesce(struct qed_dev *cdev, u16 rx_coal, u16 tx_coal,
+                            u8 qid, u16 sb_id)
+{
+        struct qed_hwfn *hwfn;
+        struct qed_ptt *ptt;
+        int hwfn_index;
+        int status = 0;
+
+        hwfn_index = qid % cdev->num_hwfns;
+        hwfn = &cdev->hwfns[hwfn_index];
+        ptt = qed_ptt_acquire(hwfn);
+        if (!ptt)
+                return -EAGAIN;
+
+        status = qed_set_rxq_coalesce(hwfn, ptt, rx_coal,
+                                      qid / cdev->num_hwfns, sb_id);
+        if (status)
+                goto out;
+        status = qed_set_txq_coalesce(hwfn, ptt, tx_coal,
+                                      qid / cdev->num_hwfns, sb_id);
+out:
+        qed_ptt_release(hwfn, ptt);
+
+        return status;
+}
+
 static int qed_set_led(struct qed_dev *cdev, enum qed_led_mode mode)
 {
         struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
@@ -1347,5 +1382,7 @@ const struct qed_common_ops qed_common_ops_pass = {
         .update_msglvl = &qed_init_dp,
         .chain_alloc = &qed_chain_alloc,
         .chain_free = &qed_chain_free,
+        .get_coalesce = &qed_get_coalesce,
+        .set_coalesce = &qed_set_coalesce,
         .set_led = &qed_set_led,
 };
diff --git a/drivers/net/ethernet/qlogic/qed/qed_mcp.c b/drivers/net/ethernet/qlogic/qed/qed_mcp.c
index 1182361798b5..a240f26344a4 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_mcp.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_mcp.c
@@ -531,9 +531,9 @@ static void qed_mcp_handle_transceiver_change(struct qed_hwfn *p_hwfn,
                                   transceiver_data)));
 
         transceiver_state = GET_FIELD(transceiver_state,
-                                      PMM_TRANSCEIVER_STATE);
+                                      ETH_TRANSCEIVER_STATE);
 
-        if (transceiver_state == PMM_TRANSCEIVER_STATE_PRESENT)
+        if (transceiver_state == ETH_TRANSCEIVER_STATE_PRESENT)
                 DP_NOTICE(p_hwfn, "Transceiver is present.\n");
         else
                 DP_NOTICE(p_hwfn, "Transceiver is unplugged.\n");
@@ -668,14 +668,12 @@ static void qed_mcp_handle_link_change(struct qed_hwfn *p_hwfn,
         qed_link_update(p_hwfn);
 }
 
-int qed_mcp_set_link(struct qed_hwfn *p_hwfn,
-                     struct qed_ptt *p_ptt,
-                     bool b_up)
+int qed_mcp_set_link(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, bool b_up)
 {
         struct qed_mcp_link_params *params = &p_hwfn->mcp_info->link_input;
         struct qed_mcp_mb_params mb_params;
         union drv_union_data union_data;
-        struct pmm_phy_cfg *phy_cfg;
+        struct eth_phy_cfg *phy_cfg;
         int rc = 0;
         u32 cmd;
 
@@ -685,9 +683,9 @@ int qed_mcp_set_link(struct qed_hwfn *p_hwfn,
         cmd = b_up ? DRV_MSG_CODE_INIT_PHY : DRV_MSG_CODE_LINK_RESET;
         if (!params->speed.autoneg)
                 phy_cfg->speed = params->speed.forced_speed;
-        phy_cfg->pause |= (params->pause.autoneg) ? PMM_PAUSE_AUTONEG : 0;
-        phy_cfg->pause |= (params->pause.forced_rx) ? PMM_PAUSE_RX : 0;
-        phy_cfg->pause |= (params->pause.forced_tx) ? PMM_PAUSE_TX : 0;
+        phy_cfg->pause |= (params->pause.autoneg) ? ETH_PAUSE_AUTONEG : 0;
+        phy_cfg->pause |= (params->pause.forced_rx) ? ETH_PAUSE_RX : 0;
+        phy_cfg->pause |= (params->pause.forced_tx) ? ETH_PAUSE_TX : 0;
         phy_cfg->adv_speed = params->speed.advertised_speeds;
         phy_cfg->loopback_mode = params->loopback_mode;
 
@@ -773,6 +771,34 @@ static u32 qed_mcp_get_shmem_func(struct qed_hwfn *p_hwfn,
         return size;
 }
 
+int qed_hw_init_first_eth(struct qed_hwfn *p_hwfn,
+                          struct qed_ptt *p_ptt, u8 *p_pf)
+{
+        struct public_func shmem_info;
+        int i;
+
+        /* Find first Ethernet interface in port */
+        for (i = 0; i < NUM_OF_ENG_PFS(p_hwfn->cdev);
+             i += p_hwfn->cdev->num_ports_in_engines) {
+                qed_mcp_get_shmem_func(p_hwfn, p_ptt, &shmem_info,
+                                       MCP_PF_ID_BY_REL(p_hwfn, i));
+
+                if (shmem_info.config & FUNC_MF_CFG_FUNC_HIDE)
+                        continue;
+
+                if ((shmem_info.config & FUNC_MF_CFG_PROTOCOL_MASK) ==
+                    FUNC_MF_CFG_PROTOCOL_ETHERNET) {
+                        *p_pf = (u8)i;
+                        return 0;
+                }
+        }
+
+        DP_NOTICE(p_hwfn,
+                  "Failed to find on port an ethernet interface in MF_SI mode\n");
+
+        return -EINVAL;
+}
+
 static void qed_mcp_update_bw(struct qed_hwfn *p_hwfn,
                               struct qed_ptt *p_ptt)
 {
@@ -951,7 +977,18 @@ qed_mcp_get_shmem_proto(struct qed_hwfn *p_hwfn,
 
         switch (p_info->config & FUNC_MF_CFG_PROTOCOL_MASK) {
         case FUNC_MF_CFG_PROTOCOL_ETHERNET:
-                *p_proto = QED_PCI_ETH;
+                if (test_bit(QED_DEV_CAP_ROCE,
+                             &p_hwfn->hw_info.device_capabilities))
+                        *p_proto = QED_PCI_ETH_ROCE;
+                else
+                        *p_proto = QED_PCI_ETH;
+                break;
+        case FUNC_MF_CFG_PROTOCOL_ISCSI:
+                *p_proto = QED_PCI_ISCSI;
+                break;
+        case FUNC_MF_CFG_PROTOCOL_ROCE:
+                DP_NOTICE(p_hwfn, "RoCE personality is not a valid value!\n");
+                rc = -EINVAL;
                 break;
         default:
                 rc = -EINVAL;
diff --git a/drivers/net/ethernet/qlogic/qed/qed_mcp.h b/drivers/net/ethernet/qlogic/qed/qed_mcp.h
index 6dd59eb7f4c6..7f319aa1b229 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_mcp.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_mcp.h
@@ -457,4 +457,7 @@ int __qed_configure_pf_min_bandwidth(struct qed_hwfn *p_hwfn,
                                      struct qed_ptt *p_ptt,
                                      struct qed_mcp_link_state *p_link,
                                      u8 min_bw);
+
+int qed_hw_init_first_eth(struct qed_hwfn *p_hwfn,
+                          struct qed_ptt *p_ptt, u8 *p_pf);
 #endif
diff --git a/drivers/net/ethernet/qlogic/qed/qed_reg_addr.h b/drivers/net/ethernet/qlogic/qed/qed_reg_addr.h
index 3a6c506f0d71..f6b86ca1ff79 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_reg_addr.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_reg_addr.h
@@ -27,6 +27,35 @@
 #define  CDU_REG_CID_ADDR_PARAMS_NCIB                   ( \
                 0xff << 24)
 
+#define CDU_REG_SEGMENT0_PARAMS \
+        0x580904UL
+#define CDU_REG_SEGMENT0_PARAMS_T0_NUM_TIDS_IN_BLOCK \
+        (0xfff << 0)
+#define CDU_REG_SEGMENT0_PARAMS_T0_NUM_TIDS_IN_BLOCK_SHIFT \
+        0
+#define CDU_REG_SEGMENT0_PARAMS_T0_TID_BLOCK_WASTE \
+        (0xff << 16)
+#define CDU_REG_SEGMENT0_PARAMS_T0_TID_BLOCK_WASTE_SHIFT \
+        16
+#define CDU_REG_SEGMENT0_PARAMS_T0_TID_SIZE \
+        (0xff << 24)
+#define CDU_REG_SEGMENT0_PARAMS_T0_TID_SIZE_SHIFT \
+        24
+#define CDU_REG_SEGMENT1_PARAMS \
+        0x580908UL
+#define CDU_REG_SEGMENT1_PARAMS_T1_NUM_TIDS_IN_BLOCK \
+        (0xfff << 0)
+#define CDU_REG_SEGMENT1_PARAMS_T1_NUM_TIDS_IN_BLOCK_SHIFT \
+        0
+#define CDU_REG_SEGMENT1_PARAMS_T1_TID_BLOCK_WASTE \
+        (0xff << 16)
+#define CDU_REG_SEGMENT1_PARAMS_T1_TID_BLOCK_WASTE_SHIFT \
+        16
+#define CDU_REG_SEGMENT1_PARAMS_T1_TID_SIZE \
+        (0xff << 24)
+#define CDU_REG_SEGMENT1_PARAMS_T1_TID_SIZE_SHIFT \
+        24
+
 #define  XSDM_REG_OPERATION_GEN \
         0xf80408UL
 #define  NIG_REG_RX_BRB_OUT_EN \
@@ -51,6 +80,8 @@
         0x1f00000UL
 #define  BAR0_MAP_REG_TSDM_RAM \
         0x1c80000UL
+#define BAR0_MAP_REG_XSDM_RAM \
+        0x1e00000UL
 #define  NIG_REG_RX_LLH_BRB_GATE_DNTFWD_PERPF \
         0x5011f4UL
 #define  PRS_REG_SEARCH_TCP \
@@ -167,6 +198,10 @@
         0x1800004UL
 #define  NIG_REG_CM_HDR \
         0x500840UL
+#define NIG_REG_LLH_TAGMAC_DEF_PF_VECTOR \
+        0x50196cUL
+#define NIG_REG_LLH_CLS_TYPE_DUALMODE \
+        0x501964UL
 #define  NCSI_REG_CONFIG        \
         0x040200UL
 #define  PBF_REG_INIT \
@@ -219,6 +254,10 @@
         0x230000UL
 #define  PRS_REG_SOFT_RST \
         0x1f0000UL
+#define PRS_REG_MSG_INFO \
+        0x1f0a1cUL
+#define PRS_REG_ROCE_DEST_QP_MAX_PF \
+        0x1f0430UL
 #define  PSDM_REG_ENABLE_IN1 \
         0xfa0004UL
 #define  PSEM_REG_ENABLE_IN \
@@ -227,6 +266,8 @@
         0x280020UL
 #define  PSWRQ2_REG_CDUT_P_SIZE \
         0x24000cUL
+#define PSWRQ2_REG_ILT_MEMORY \
+        0x260000UL
 #define  PSWHST_REG_DISCARD_INTERNAL_WRITES \
         0x2a0040UL
 #define  PSWHST2_REG_DBGSYN_ALMOST_FULL_THR \
@@ -460,7 +501,7 @@
 #define NIG_REG_ENC_TYPE_ENABLE_VXLAN_ENABLE                    (0x1 << 2)
 #define NIG_REG_ENC_TYPE_ENABLE_VXLAN_ENABLE_SHIFT              2
 
-#define NIG_REG_VXLAN_PORT              0x50105cUL
+#define NIG_REG_VXLAN_CTRL              0x50105cUL
 #define PBF_REG_VXLAN_PORT              0xd80518UL
 #define PBF_REG_NGE_PORT                0xd8051cUL
 #define PRS_REG_NGE_PORT                0x1f086cUL
diff --git a/drivers/net/ethernet/qlogic/qed/qed_sp.h b/drivers/net/ethernet/qlogic/qed/qed_sp.h
index ea4e9ce53e0a..a548504c3420 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_sp.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_sp.h
@@ -63,6 +63,32 @@ union ramrod_data {
         struct vport_update_ramrod_data vport_update;
         struct vport_filter_update_ramrod_data vport_filter_update;
 
+        struct rdma_init_func_ramrod_data rdma_init_func;
+        struct rdma_close_func_ramrod_data rdma_close_func;
+        struct rdma_register_tid_ramrod_data rdma_register_tid;
+        struct rdma_deregister_tid_ramrod_data rdma_deregister_tid;
+        struct roce_create_qp_resp_ramrod_data roce_create_qp_resp;
+        struct roce_create_qp_req_ramrod_data roce_create_qp_req;
+        struct roce_modify_qp_resp_ramrod_data roce_modify_qp_resp;
+        struct roce_modify_qp_req_ramrod_data roce_modify_qp_req;
+        struct roce_query_qp_resp_ramrod_data roce_query_qp_resp;
+        struct roce_query_qp_req_ramrod_data roce_query_qp_req;
+        struct roce_destroy_qp_resp_ramrod_data roce_destroy_qp_resp;
+        struct roce_destroy_qp_req_ramrod_data roce_destroy_qp_req;
+        struct rdma_create_cq_ramrod_data rdma_create_cq;
+        struct rdma_resize_cq_ramrod_data rdma_resize_cq;
+        struct rdma_destroy_cq_ramrod_data rdma_destroy_cq;
+        struct rdma_srq_create_ramrod_data rdma_create_srq;
+        struct rdma_srq_destroy_ramrod_data rdma_destroy_srq;
+        struct rdma_srq_modify_ramrod_data rdma_modify_srq;
+
+        struct iscsi_slow_path_hdr iscsi_empty;
+        struct iscsi_init_ramrod_params iscsi_init;
+        struct iscsi_spe_func_dstry iscsi_destroy;
+        struct iscsi_spe_conn_offload iscsi_conn_offload;
+        struct iscsi_conn_update_ramrod_params iscsi_conn_update;
+        struct iscsi_spe_conn_termination iscsi_conn_terminate;
+
         struct vf_start_ramrod_data vf_start;
         struct vf_stop_ramrod_data vf_stop;
 };
diff --git a/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c b/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c
index 67f6ce3c84c8..a52f3fc051f5 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c
@@ -308,6 +308,7 @@ int qed_sp_pf_start(struct qed_hwfn *p_hwfn,
         struct qed_spq_entry *p_ent = NULL;
         struct qed_sp_init_data init_data;
         int rc = -EINVAL;
+        u8 page_cnt;
 
         /* update initial eq producer */
         qed_eq_prod_update(p_hwfn,
@@ -332,7 +333,7 @@ int qed_sp_pf_start(struct qed_hwfn *p_hwfn,
         p_ramrod->path_id               = QED_PATH_ID(p_hwfn);
         p_ramrod->dont_log_ramrods      = 0;
         p_ramrod->log_type_mask         = cpu_to_le16(0xf);
-        p_ramrod->mf_mode = mode;
+
         switch (mode) {
         case QED_MF_DEFAULT:
         case QED_MF_NPAR:
@@ -350,24 +351,41 @@ int qed_sp_pf_start(struct qed_hwfn *p_hwfn,
         /* Place EQ address in RAMROD */
         DMA_REGPAIR_LE(p_ramrod->event_ring_pbl_addr,
                        p_hwfn->p_eq->chain.pbl.p_phys_table);
-        p_ramrod->event_ring_num_pages = (u8)p_hwfn->p_eq->chain.page_cnt;
-
+        page_cnt = (u8)qed_chain_get_page_cnt(&p_hwfn->p_eq->chain);
+        p_ramrod->event_ring_num_pages = page_cnt;
         DMA_REGPAIR_LE(p_ramrod->consolid_q_pbl_addr,
                        p_hwfn->p_consq->chain.pbl.p_phys_table);
 
         qed_tunn_set_pf_start_params(p_hwfn, p_tunn,
                                      &p_ramrod->tunnel_config);
-        p_hwfn->hw_info.personality = PERSONALITY_ETH;
 
         if (IS_MF_SI(p_hwfn))
                 p_ramrod->allow_npar_tx_switching = allow_npar_tx_switch;
 
+        switch (p_hwfn->hw_info.personality) {
+        case QED_PCI_ETH:
+                p_ramrod->personality = PERSONALITY_ETH;
+                break;
+        case QED_PCI_ISCSI:
+                p_ramrod->personality = PERSONALITY_ISCSI;
+                break;
+        case QED_PCI_ETH_ROCE:
+                p_ramrod->personality = PERSONALITY_RDMA_AND_ETH;
+                break;
+        default:
+                DP_NOTICE(p_hwfn, "Unkown personality %d\n",
+                          p_hwfn->hw_info.personality);
+                p_ramrod->personality = PERSONALITY_ETH;
+        }
+
         if (p_hwfn->cdev->p_iov_info) {
                 struct qed_hw_sriov_info *p_iov = p_hwfn->cdev->p_iov_info;
 
                 p_ramrod->base_vf_id = (u8) p_iov->first_vf_in_pf;
                 p_ramrod->num_vfs = (u8) p_iov->total_vfs;
         }
+        p_ramrod->hsi_fp_ver.major_ver_arr[ETH_VER_KEY] = ETH_HSI_VER_MAJOR;
+        p_ramrod->hsi_fp_ver.minor_ver_arr[ETH_VER_KEY] = ETH_HSI_VER_MINOR;
 
         DP_VERBOSE(p_hwfn, QED_MSG_SPQ,
                    "Setting event_ring_sb [id %04x index %02x], outer_tag [%d]\n",
diff --git a/drivers/net/ethernet/qlogic/qed/qed_spq.c b/drivers/net/ethernet/qlogic/qed/qed_spq.c
index b122f6013b6c..97ffeae262bb 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_spq.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_spq.c
@@ -339,6 +339,7 @@ struct qed_eq *qed_eq_alloc(struct qed_hwfn *p_hwfn,
         if (qed_chain_alloc(p_hwfn->cdev,
                             QED_CHAIN_USE_TO_PRODUCE,
                             QED_CHAIN_MODE_PBL,
+                            QED_CHAIN_CNT_TYPE_U16,
                             num_elem,
                             sizeof(union event_ring_element),
                             &p_eq->chain)) {
@@ -412,10 +413,10 @@ int qed_eth_cqe_completion(struct qed_hwfn *p_hwfn,
 ***************************************************************************/
 void qed_spq_setup(struct qed_hwfn *p_hwfn)
 {
-        struct qed_spq          *p_spq  = p_hwfn->p_spq;
-        struct qed_spq_entry    *p_virt = NULL;
-        dma_addr_t              p_phys  = 0;
-        unsigned int            i       = 0;
+        struct qed_spq *p_spq = p_hwfn->p_spq;
+        struct qed_spq_entry *p_virt = NULL;
+        dma_addr_t p_phys = 0;
+        u32 i, capacity;
 
         INIT_LIST_HEAD(&p_spq->pending);
         INIT_LIST_HEAD(&p_spq->completion_pending);
@@ -427,7 +428,8 @@ void qed_spq_setup(struct qed_hwfn *p_hwfn)
         p_phys  = p_spq->p_phys + offsetof(struct qed_spq_entry, ramrod);
         p_virt  = p_spq->p_virt;
 
-        for (i = 0; i < p_spq->chain.capacity; i++) {
+        capacity = qed_chain_get_capacity(&p_spq->chain);
+        for (i = 0; i < capacity; i++) {
                 DMA_REGPAIR_LE(p_virt->elem.data_ptr, p_phys);
 
                 list_add_tail(&p_virt->list, &p_spq->free_pool);
@@ -455,9 +457,10 @@ void qed_spq_setup(struct qed_hwfn *p_hwfn)
 
 int qed_spq_alloc(struct qed_hwfn *p_hwfn)
 {
-        struct qed_spq          *p_spq  = NULL;
-        dma_addr_t              p_phys  = 0;
-        struct qed_spq_entry    *p_virt = NULL;
+        struct qed_spq_entry *p_virt = NULL;
+        struct qed_spq *p_spq = NULL;
+        dma_addr_t p_phys = 0;
+        u32 capacity;
 
         /* SPQ struct */
         p_spq =
@@ -471,6 +474,7 @@ int qed_spq_alloc(struct qed_hwfn *p_hwfn)
         if (qed_chain_alloc(p_hwfn->cdev,
                             QED_CHAIN_USE_TO_PRODUCE,
                             QED_CHAIN_MODE_SINGLE,
+                            QED_CHAIN_CNT_TYPE_U16,
                             0,   /* N/A when the mode is SINGLE */
                             sizeof(struct slow_path_element),
                             &p_spq->chain)) {
@@ -479,11 +483,11 @@ int qed_spq_alloc(struct qed_hwfn *p_hwfn)
         }
 
         /* allocate and fill the SPQ elements (incl. ramrod data list) */
+        capacity = qed_chain_get_capacity(&p_spq->chain);
         p_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
-                                    p_spq->chain.capacity *
+                                    capacity *
                                     sizeof(struct qed_spq_entry),
-                                    &p_phys,
-                                    GFP_KERNEL);
+                                    &p_phys, GFP_KERNEL);
 
         if (!p_virt)
                 goto spq_allocate_fail;
@@ -503,16 +507,18 @@ spq_allocate_fail:
 void qed_spq_free(struct qed_hwfn *p_hwfn)
 {
         struct qed_spq *p_spq = p_hwfn->p_spq;
+        u32 capacity;
 
         if (!p_spq)
                 return;
 
-        if (p_spq->p_virt)
+        if (p_spq->p_virt) {
+                capacity = qed_chain_get_capacity(&p_spq->chain);
                 dma_free_coherent(&p_hwfn->cdev->pdev->dev,
-                                  p_spq->chain.capacity *
+                                  capacity *
                                   sizeof(struct qed_spq_entry),
-                                  p_spq->p_virt,
-                                  p_spq->p_phys);
+                                  p_spq->p_virt, p_spq->p_phys);
+        }
 
         qed_chain_free(p_hwfn->cdev, &p_spq->chain);
         ;
@@ -882,9 +888,9 @@ struct qed_consq *qed_consq_alloc(struct qed_hwfn *p_hwfn)
         if (qed_chain_alloc(p_hwfn->cdev,
                             QED_CHAIN_USE_TO_PRODUCE,
                             QED_CHAIN_MODE_PBL,
+                            QED_CHAIN_CNT_TYPE_U16,
                             QED_CHAIN_PAGE_SIZE / 0x80,
-                            0x80,
-                            &p_consq->chain)) {
+                            0x80, &p_consq->chain)) {
                 DP_NOTICE(p_hwfn, "Failed to allocate consq chain");
                 goto consq_allocate_fail;
         }
diff --git a/drivers/net/ethernet/qlogic/qed/qed_sriov.c b/drivers/net/ethernet/qlogic/qed/qed_sriov.c
index c325ee857ecd..4d161c751c12 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_sriov.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_sriov.c
@@ -21,18 +21,18 @@
 #include "qed_vf.h"
 
 /* IOV ramrods */
-static int qed_sp_vf_start(struct qed_hwfn *p_hwfn,
-                           u32 concrete_vfid, u16 opaque_vfid)
+static int qed_sp_vf_start(struct qed_hwfn *p_hwfn, struct qed_vf_info *p_vf)
 {
         struct vf_start_ramrod_data *p_ramrod = NULL;
         struct qed_spq_entry *p_ent = NULL;
         struct qed_sp_init_data init_data;
         int rc = -EINVAL;
+        u8 fp_minor;
 
         /* Get SPQ entry */
         memset(&init_data, 0, sizeof(init_data));
         init_data.cid = qed_spq_get_cid(p_hwfn);
-        init_data.opaque_fid = opaque_vfid;
+        init_data.opaque_fid = p_vf->opaque_fid;
         init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
 
         rc = qed_sp_init_request(p_hwfn, &p_ent,
@@ -43,10 +43,39 @@ static int qed_sp_vf_start(struct qed_hwfn *p_hwfn,
 
         p_ramrod = &p_ent->ramrod.vf_start;
 
-        p_ramrod->vf_id = GET_FIELD(concrete_vfid, PXP_CONCRETE_FID_VFID);
-        p_ramrod->opaque_fid = cpu_to_le16(opaque_vfid);
+        p_ramrod->vf_id = GET_FIELD(p_vf->concrete_fid, PXP_CONCRETE_FID_VFID);
+        p_ramrod->opaque_fid = cpu_to_le16(p_vf->opaque_fid);
 
-        p_ramrod->personality = PERSONALITY_ETH;
+        switch (p_hwfn->hw_info.personality) {
+        case QED_PCI_ETH:
+                p_ramrod->personality = PERSONALITY_ETH;
+                break;
+        case QED_PCI_ETH_ROCE:
+                p_ramrod->personality = PERSONALITY_RDMA_AND_ETH;
+                break;
+        default:
+                DP_NOTICE(p_hwfn, "Unknown VF personality %d\n",
+                          p_hwfn->hw_info.personality);
+                return -EINVAL;
+        }
+
+        fp_minor = p_vf->acquire.vfdev_info.eth_fp_hsi_minor;
+        if (fp_minor > ETH_HSI_VER_MINOR) {
+                DP_VERBOSE(p_hwfn,
+                           QED_MSG_IOV,
+                           "VF [%d] - Requested fp hsi %02x.%02x which is slightly newer than PF's %02x.%02x; Configuring PFs version\n",
+                           p_vf->abs_vf_id,
+                           ETH_HSI_VER_MAJOR,
+                           fp_minor, ETH_HSI_VER_MAJOR, ETH_HSI_VER_MINOR);
+                fp_minor = ETH_HSI_VER_MINOR;
+        }
+
+        p_ramrod->hsi_fp_ver.major_ver_arr[ETH_VER_KEY] = ETH_HSI_VER_MAJOR;
+        p_ramrod->hsi_fp_ver.minor_ver_arr[ETH_VER_KEY] = fp_minor;
+
+        DP_VERBOSE(p_hwfn, QED_MSG_IOV,
+                   "VF[%d] - Starting using HSI %02x.%02x\n",
+                   p_vf->abs_vf_id, ETH_HSI_VER_MAJOR, fp_minor);
 
         return qed_spq_post(p_hwfn, p_ent, NULL);
 }
@@ -117,6 +146,45 @@ static struct qed_vf_info *qed_iov_get_vf_info(struct qed_hwfn *p_hwfn,
         return vf;
 }
 
+static bool qed_iov_validate_rxq(struct qed_hwfn *p_hwfn,
+                                 struct qed_vf_info *p_vf, u16 rx_qid)
+{
+        if (rx_qid >= p_vf->num_rxqs)
+                DP_VERBOSE(p_hwfn,
+                           QED_MSG_IOV,
+                           "VF[0x%02x] - can't touch Rx queue[%04x]; Only 0x%04x are allocated\n",
+                           p_vf->abs_vf_id, rx_qid, p_vf->num_rxqs);
+        return rx_qid < p_vf->num_rxqs;
+}
+
+static bool qed_iov_validate_txq(struct qed_hwfn *p_hwfn,
+                                 struct qed_vf_info *p_vf, u16 tx_qid)
+{
+        if (tx_qid >= p_vf->num_txqs)
+                DP_VERBOSE(p_hwfn,
+                           QED_MSG_IOV,
+                           "VF[0x%02x] - can't touch Tx queue[%04x]; Only 0x%04x are allocated\n",
+                           p_vf->abs_vf_id, tx_qid, p_vf->num_txqs);
+        return tx_qid < p_vf->num_txqs;
+}
+
+static bool qed_iov_validate_sb(struct qed_hwfn *p_hwfn,
+                                struct qed_vf_info *p_vf, u16 sb_idx)
+{
+        int i;
+
+        for (i = 0; i < p_vf->num_sbs; i++)
+                if (p_vf->igu_sbs[i] == sb_idx)
+                        return true;
+
+        DP_VERBOSE(p_hwfn,
+                   QED_MSG_IOV,
+                   "VF[0%02x] - tried using sb_idx %04x which doesn't exist as one of its 0x%02x SBs\n",
+                   p_vf->abs_vf_id, sb_idx, p_vf->num_sbs);
+
+        return false;
+}
+
 int qed_iov_post_vf_bulletin(struct qed_hwfn *p_hwfn,
                              int vfid, struct qed_ptt *p_ptt)
 {
@@ -293,6 +361,9 @@ static void qed_iov_setup_vfdb(struct qed_hwfn *p_hwfn)
                 vf->opaque_fid = (p_hwfn->hw_info.opaque_fid & 0xff) |
                                  (vf->abs_vf_id << 8);
                 vf->vport_id = idx + 1;
+
+                vf->num_mac_filters = QED_ETH_VF_NUM_MAC_FILTERS;
+                vf->num_vlan_filters = QED_ETH_VF_NUM_VLAN_FILTERS;
         }
 }
 
@@ -598,17 +669,6 @@ static int qed_iov_enable_vf_access(struct qed_hwfn *p_hwfn,
         /* unpretend */
         qed_fid_pretend(p_hwfn, p_ptt, (u16) p_hwfn->hw_info.concrete_fid);
 
-        if (vf->state != VF_STOPPED) {
-                DP_NOTICE(p_hwfn, "VF[%02x] is already started\n",
-                          vf->abs_vf_id);
-                return -EINVAL;
-        }
-
-        /* Start VF */
-        rc = qed_sp_vf_start(p_hwfn, vf->concrete_fid, vf->opaque_fid);
-        if (rc)
-                DP_NOTICE(p_hwfn, "Failed to start VF[%02x]\n", vf->abs_vf_id);
-
         vf->state = VF_FREE;
 
         return rc;
@@ -852,7 +912,6 @@ static int qed_iov_release_hw_for_vf(struct qed_hwfn *p_hwfn,
         struct qed_mcp_link_params params;
         struct qed_mcp_link_state link;
         struct qed_vf_info *vf = NULL;
-        int rc = 0;
 
         vf = qed_iov_get_vf_info(p_hwfn, rel_vf_id, true);
         if (!vf) {
@@ -874,18 +933,8 @@ static int qed_iov_release_hw_for_vf(struct qed_hwfn *p_hwfn,
         memcpy(&caps, qed_mcp_get_link_capabilities(p_hwfn), sizeof(caps));
         qed_iov_set_link(p_hwfn, rel_vf_id, &params, &link, &caps);
 
-        if (vf->state != VF_STOPPED) {
-                /* Stopping the VF */
-                rc = qed_sp_vf_stop(p_hwfn, vf->concrete_fid, vf->opaque_fid);
-
-                if (rc != 0) {
-                        DP_ERR(p_hwfn, "qed_sp_vf_stop returned error %d\n",
-                               rc);
-                        return rc;
-                }
-
-                vf->state = VF_STOPPED;
-        }
+        /* Forget the VF's acquisition message */
+        memset(&vf->acquire, 0, sizeof(vf->acquire));
 
         /* disablng interrupts and resetting permission table was done during
          * vf-close, however, we could get here without going through vf_close
@@ -1116,8 +1165,6 @@ static void qed_iov_vf_cleanup(struct qed_hwfn *p_hwfn,
 
         p_vf->vf_bulletin = 0;
         p_vf->vport_instance = 0;
-        p_vf->num_mac_filters = 0;
-        p_vf->num_vlan_filters = 0;
         p_vf->configured_features = 0;
 
         /* If VF previously requested less resources, go back to default */
@@ -1130,9 +1177,95 @@ static void qed_iov_vf_cleanup(struct qed_hwfn *p_hwfn,
                 p_vf->vf_queues[i].rxq_active = 0;
 
         memset(&p_vf->shadow_config, 0, sizeof(p_vf->shadow_config));
+        memset(&p_vf->acquire, 0, sizeof(p_vf->acquire));
         qed_iov_clean_vf(p_hwfn, p_vf->relative_vf_id);
 }
 
+static u8 qed_iov_vf_mbx_acquire_resc(struct qed_hwfn *p_hwfn,
+                                      struct qed_ptt *p_ptt,
+                                      struct qed_vf_info *p_vf,
+                                      struct vf_pf_resc_request *p_req,
+                                      struct pf_vf_resc *p_resp)
+{
+        int i;
+
+        /* Queue related information */
+        p_resp->num_rxqs = p_vf->num_rxqs;
+        p_resp->num_txqs = p_vf->num_txqs;
+        p_resp->num_sbs = p_vf->num_sbs;
+
+        for (i = 0; i < p_resp->num_sbs; i++) {
+                p_resp->hw_sbs[i].hw_sb_id = p_vf->igu_sbs[i];
+                p_resp->hw_sbs[i].sb_qid = 0;
+        }
+
+        /* These fields are filled for backward compatibility.
+         * Unused by modern vfs.
+         */
+        for (i = 0; i < p_resp->num_rxqs; i++) {
+                qed_fw_l2_queue(p_hwfn, p_vf->vf_queues[i].fw_rx_qid,
+                                (u16 *)&p_resp->hw_qid[i]);
+                p_resp->cid[i] = p_vf->vf_queues[i].fw_cid;
+        }
+
+        /* Filter related information */
+        p_resp->num_mac_filters = min_t(u8, p_vf->num_mac_filters,
+                                        p_req->num_mac_filters);
+        p_resp->num_vlan_filters = min_t(u8, p_vf->num_vlan_filters,
+                                         p_req->num_vlan_filters);
+
+        /* This isn't really needed/enforced, but some legacy VFs might depend
+         * on the correct filling of this field.
+         */
+        p_resp->num_mc_filters = QED_MAX_MC_ADDRS;
+
+        /* Validate sufficient resources for VF */
+        if (p_resp->num_rxqs < p_req->num_rxqs ||
+            p_resp->num_txqs < p_req->num_txqs ||
+            p_resp->num_sbs < p_req->num_sbs ||
+            p_resp->num_mac_filters < p_req->num_mac_filters ||
+            p_resp->num_vlan_filters < p_req->num_vlan_filters ||
+            p_resp->num_mc_filters < p_req->num_mc_filters) {
+                DP_VERBOSE(p_hwfn,
+                           QED_MSG_IOV,
+                           "VF[%d] - Insufficient resources: rxq [%02x/%02x] txq [%02x/%02x] sbs [%02x/%02x] mac [%02x/%02x] vlan [%02x/%02x] mc [%02x/%02x]\n",
+                           p_vf->abs_vf_id,
+                           p_req->num_rxqs,
+                           p_resp->num_rxqs,
+                           p_req->num_rxqs,
+                           p_resp->num_txqs,
+                           p_req->num_sbs,
+                           p_resp->num_sbs,
+                           p_req->num_mac_filters,
+                           p_resp->num_mac_filters,
+                           p_req->num_vlan_filters,
+                           p_resp->num_vlan_filters,
+                           p_req->num_mc_filters, p_resp->num_mc_filters);
+                return PFVF_STATUS_NO_RESOURCE;
+        }
+
+        return PFVF_STATUS_SUCCESS;
+}
+
+static void qed_iov_vf_mbx_acquire_stats(struct qed_hwfn *p_hwfn,
+                                         struct pfvf_stats_info *p_stats)
+{
+        p_stats->mstats.address = PXP_VF_BAR0_START_MSDM_ZONE_B +
+                                  offsetof(struct mstorm_vf_zone,
+                                           non_trigger.eth_queue_stat);
+        p_stats->mstats.len = sizeof(struct eth_mstorm_per_queue_stat);
+        p_stats->ustats.address = PXP_VF_BAR0_START_USDM_ZONE_B +
+                                  offsetof(struct ustorm_vf_zone,
+                                           non_trigger.eth_queue_stat);
+        p_stats->ustats.len = sizeof(struct eth_ustorm_per_queue_stat);
+        p_stats->pstats.address = PXP_VF_BAR0_START_PSDM_ZONE_B +
+                                  offsetof(struct pstorm_vf_zone,
+                                           non_trigger.eth_queue_stat);
+        p_stats->pstats.len = sizeof(struct eth_pstorm_per_queue_stat);
+        p_stats->tstats.address = 0;
+        p_stats->tstats.len = 0;
+}
+
 static void qed_iov_vf_mbx_acquire(struct qed_hwfn *p_hwfn,
                                    struct qed_ptt *p_ptt,
                                    struct qed_vf_info *vf)
@@ -1141,25 +1274,27 @@ static void qed_iov_vf_mbx_acquire(struct qed_hwfn *p_hwfn,
         struct pfvf_acquire_resp_tlv *resp = &mbx->reply_virt->acquire_resp;
         struct pf_vf_pfdev_info *pfdev_info = &resp->pfdev_info;
         struct vfpf_acquire_tlv *req = &mbx->req_virt->acquire;
-        u8 i, vfpf_status = PFVF_STATUS_SUCCESS;
+        u8 vfpf_status = PFVF_STATUS_NOT_SUPPORTED;
         struct pf_vf_resc *resc = &resp->resc;
+        int rc;
+
+        memset(resp, 0, sizeof(*resp));
 
         /* Validate FW compatibility */
-        if (req->vfdev_info.fw_major != FW_MAJOR_VERSION ||
-            req->vfdev_info.fw_minor != FW_MINOR_VERSION ||
-            req->vfdev_info.fw_revision != FW_REVISION_VERSION ||
-            req->vfdev_info.fw_engineering != FW_ENGINEERING_VERSION) {
+        if (req->vfdev_info.eth_fp_hsi_major != ETH_HSI_VER_MAJOR) {
                 DP_INFO(p_hwfn,
-                        "VF[%d] is running an incompatible driver [VF needs FW %02x:%02x:%02x:%02x but Hypervisor is using %02x:%02x:%02x:%02x]\n",
+                        "VF[%d] needs fastpath HSI %02x.%02x, which is incompatible with loaded FW's faspath HSI %02x.%02x\n",
                         vf->abs_vf_id,
-                        req->vfdev_info.fw_major,
-                        req->vfdev_info.fw_minor,
-                        req->vfdev_info.fw_revision,
-                        req->vfdev_info.fw_engineering,
-                        FW_MAJOR_VERSION,
-                        FW_MINOR_VERSION,
-                        FW_REVISION_VERSION, FW_ENGINEERING_VERSION);
-                vfpf_status = PFVF_STATUS_NOT_SUPPORTED;
+                        req->vfdev_info.eth_fp_hsi_major,
+                        req->vfdev_info.eth_fp_hsi_minor,
+                        ETH_HSI_VER_MAJOR, ETH_HSI_VER_MINOR);
+
+                /* Write the PF version so that VF would know which version
+                 * is supported.
+                 */
+                pfdev_info->major_fp_hsi = ETH_HSI_VER_MAJOR;
+                pfdev_info->minor_fp_hsi = ETH_HSI_VER_MINOR;
+
                 goto out;
         }
 
@@ -1169,16 +1304,13 @@ static void qed_iov_vf_mbx_acquire(struct qed_hwfn *p_hwfn,
                 DP_INFO(p_hwfn,
                         "VF[%d] is running an old driver that doesn't support 100g\n",
                         vf->abs_vf_id);
-                vfpf_status = PFVF_STATUS_NOT_SUPPORTED;
                 goto out;
         }
 
-        memset(resp, 0, sizeof(*resp));
+        /* Store the acquire message */
+        memcpy(&vf->acquire, req, sizeof(vf->acquire));
 
-        /* Fill in vf info stuff */
         vf->opaque_fid = req->vfdev_info.opaque_fid;
-        vf->num_mac_filters = 1;
-        vf->num_vlan_filters = QED_ETH_VF_NUM_VLAN_FILTERS;
 
         vf->vf_bulletin = req->bulletin_addr;
         vf->bulletin.size = (vf->bulletin.size < req->bulletin_size) ?
@@ -1194,26 +1326,7 @@ static void qed_iov_vf_mbx_acquire(struct qed_hwfn *p_hwfn,
         if (p_hwfn->cdev->num_hwfns > 1)
                 pfdev_info->capabilities |= PFVF_ACQUIRE_CAP_100G;
 
-        pfdev_info->stats_info.mstats.address =
-            PXP_VF_BAR0_START_MSDM_ZONE_B +
-            offsetof(struct mstorm_vf_zone, non_trigger.eth_queue_stat);
-        pfdev_info->stats_info.mstats.len =
-            sizeof(struct eth_mstorm_per_queue_stat);
-
-        pfdev_info->stats_info.ustats.address =
-            PXP_VF_BAR0_START_USDM_ZONE_B +
-            offsetof(struct ustorm_vf_zone, non_trigger.eth_queue_stat);
-        pfdev_info->stats_info.ustats.len =
-            sizeof(struct eth_ustorm_per_queue_stat);
-
-        pfdev_info->stats_info.pstats.address =
-            PXP_VF_BAR0_START_PSDM_ZONE_B +
-            offsetof(struct pstorm_vf_zone, non_trigger.eth_queue_stat);
-        pfdev_info->stats_info.pstats.len =
-            sizeof(struct eth_pstorm_per_queue_stat);
-
-        pfdev_info->stats_info.tstats.address = 0;
-        pfdev_info->stats_info.tstats.len = 0;
+        qed_iov_vf_mbx_acquire_stats(p_hwfn, &pfdev_info->stats_info);
 
         memcpy(pfdev_info->port_mac, p_hwfn->hw_info.hw_mac_addr, ETH_ALEN);
 
@@ -1221,36 +1334,31 @@ static void qed_iov_vf_mbx_acquire(struct qed_hwfn *p_hwfn,
         pfdev_info->fw_minor = FW_MINOR_VERSION;
         pfdev_info->fw_rev = FW_REVISION_VERSION;
         pfdev_info->fw_eng = FW_ENGINEERING_VERSION;
+        pfdev_info->minor_fp_hsi = min_t(u8,
+                                         ETH_HSI_VER_MINOR,
+                                         req->vfdev_info.eth_fp_hsi_minor);
         pfdev_info->os_type = VFPF_ACQUIRE_OS_LINUX;
         qed_mcp_get_mfw_ver(p_hwfn, p_ptt, &pfdev_info->mfw_ver, NULL);
 
         pfdev_info->dev_type = p_hwfn->cdev->type;
         pfdev_info->chip_rev = p_hwfn->cdev->chip_rev;
 
-        resc->num_rxqs = vf->num_rxqs;
-        resc->num_txqs = vf->num_txqs;
-        resc->num_sbs = vf->num_sbs;
-        for (i = 0; i < resc->num_sbs; i++) {
-                resc->hw_sbs[i].hw_sb_id = vf->igu_sbs[i];
-                resc->hw_sbs[i].sb_qid = 0;
-        }
+        /* Fill resources available to VF; Make sure there are enough to
+         * satisfy the VF's request.
+         */
+        vfpf_status = qed_iov_vf_mbx_acquire_resc(p_hwfn, p_ptt, vf,
+                                                  &req->resc_request, resc);
+        if (vfpf_status != PFVF_STATUS_SUCCESS)
+                goto out;
 
-        for (i = 0; i < resc->num_rxqs; i++) {
-                qed_fw_l2_queue(p_hwfn, vf->vf_queues[i].fw_rx_qid,
-                                (u16 *)&resc->hw_qid[i]);
-                resc->cid[i] = vf->vf_queues[i].fw_cid;
+        /* Start the VF in FW */
+        rc = qed_sp_vf_start(p_hwfn, vf);
+        if (rc) {
+                DP_NOTICE(p_hwfn, "Failed to start VF[%02x]\n", vf->abs_vf_id);
+                vfpf_status = PFVF_STATUS_FAILURE;
+                goto out;
         }
 
-        resc->num_mac_filters = min_t(u8, vf->num_mac_filters,
-                                      req->resc_request.num_mac_filters);
-        resc->num_vlan_filters = min_t(u8, vf->num_vlan_filters,
-                                       req->resc_request.num_vlan_filters);
-
-        /* This isn't really required as VF isn't limited, but some VFs might
-         * actually test this value, so need to provide it.
-         */
-        resc->num_mc_filters = req->resc_request.num_mc_filters;
-
         /* Fill agreed size of bulletin board in response */
         resp->bulletin_size = vf->bulletin.size;
         qed_iov_post_vf_bulletin(p_hwfn, vf->relative_vf_id, p_ptt);
@@ -1585,10 +1693,6 @@ static void qed_iov_vf_mbx_stop_vport(struct qed_hwfn *p_hwfn,
                              sizeof(struct pfvf_def_resp_tlv), status);
 }
 
-#define TSTORM_QZONE_START   PXP_VF_BAR0_START_SDM_ZONE_A
-#define MSTORM_QZONE_START(dev)   (TSTORM_QZONE_START + \
-                                   (TSTORM_QZONE_SIZE * NUM_OF_L2_QUEUES(dev)))
-
 static void qed_iov_vf_mbx_start_rxq_resp(struct qed_hwfn *p_hwfn,
                                           struct qed_ptt *p_ptt,
                                           struct qed_vf_info *vf, u8 status)
@@ -1606,16 +1710,11 @@ static void qed_iov_vf_mbx_start_rxq_resp(struct qed_hwfn *p_hwfn,
 
         /* Update the TLV with the response */
         if (status == PFVF_STATUS_SUCCESS) {
-                u16 hw_qid = 0;
-
                 req = &mbx->req_virt->start_rxq;
-                qed_fw_l2_queue(p_hwfn, vf->vf_queues[req->rx_qid].fw_rx_qid,
-                                &hw_qid);
-
-                p_tlv->offset = MSTORM_QZONE_START(p_hwfn->cdev) +
-                                hw_qid * MSTORM_QZONE_SIZE +
-                                offsetof(struct mstorm_eth_queue_zone,
-                                         rx_producers);
+                p_tlv->offset = PXP_VF_BAR0_START_MSDM_ZONE_B +
+                                offsetof(struct mstorm_vf_zone,
+                                         non_trigger.eth_rx_queue_producers) +
+                                sizeof(struct eth_rx_prod_data) * req->rx_qid;
         }
 
         qed_iov_send_response(p_hwfn, p_ptt, vf, sizeof(*p_tlv), status);
@@ -1627,13 +1726,19 @@ static void qed_iov_vf_mbx_start_rxq(struct qed_hwfn *p_hwfn,
 {
         struct qed_queue_start_common_params params;
         struct qed_iov_vf_mbx *mbx = &vf->vf_mbx;
-        u8 status = PFVF_STATUS_SUCCESS;
+        u8 status = PFVF_STATUS_NO_RESOURCE;
         struct vfpf_start_rxq_tlv *req;
         int rc;
 
         memset(&params, 0, sizeof(params));
         req = &mbx->req_virt->start_rxq;
+
+        if (!qed_iov_validate_rxq(p_hwfn, vf, req->rx_qid) ||
+            !qed_iov_validate_sb(p_hwfn, vf, req->hw_sb))
+                goto out;
+
         params.queue_id =  vf->vf_queues[req->rx_qid].fw_rx_qid;
+        params.vf_qid = req->rx_qid;
         params.vport_id = vf->vport_id;
         params.sb = req->hw_sb;
         params.sb_idx = req->sb_index;
@@ -1649,22 +1754,48 @@ static void qed_iov_vf_mbx_start_rxq(struct qed_hwfn *p_hwfn,
         if (rc) {
                 status = PFVF_STATUS_FAILURE;
         } else {
+                status = PFVF_STATUS_SUCCESS;
                 vf->vf_queues[req->rx_qid].rxq_active = true;
                 vf->num_active_rxqs++;
         }
 
+out:
         qed_iov_vf_mbx_start_rxq_resp(p_hwfn, p_ptt, vf, status);
 }
 
+static void qed_iov_vf_mbx_start_txq_resp(struct qed_hwfn *p_hwfn,
+                                          struct qed_ptt *p_ptt,
+                                          struct qed_vf_info *p_vf, u8 status)
+{
+        struct qed_iov_vf_mbx *mbx = &p_vf->vf_mbx;
+        struct pfvf_start_queue_resp_tlv *p_tlv;
+
+        mbx->offset = (u8 *)mbx->reply_virt;
+
+        p_tlv = qed_add_tlv(p_hwfn, &mbx->offset, CHANNEL_TLV_START_TXQ,
+                            sizeof(*p_tlv));
+        qed_add_tlv(p_hwfn, &mbx->offset, CHANNEL_TLV_LIST_END,
+                    sizeof(struct channel_list_end_tlv));
+
+        /* Update the TLV with the response */
+        if (status == PFVF_STATUS_SUCCESS) {
+                u16 qid = mbx->req_virt->start_txq.tx_qid;
+
+                p_tlv->offset = qed_db_addr(p_vf->vf_queues[qid].fw_cid,
+                                            DQ_DEMS_LEGACY);
+        }
+
+        qed_iov_send_response(p_hwfn, p_ptt, p_vf, sizeof(*p_tlv), status);
+}
+
 static void qed_iov_vf_mbx_start_txq(struct qed_hwfn *p_hwfn,
                                      struct qed_ptt *p_ptt,
                                      struct qed_vf_info *vf)
 {
-        u16 length = sizeof(struct pfvf_def_resp_tlv);
         struct qed_queue_start_common_params params;
         struct qed_iov_vf_mbx *mbx = &vf->vf_mbx;
+        u8 status = PFVF_STATUS_NO_RESOURCE;
         union qed_qm_pq_params pq_params;
-        u8 status = PFVF_STATUS_SUCCESS;
         struct vfpf_start_txq_tlv *req;
         int rc;
 
@@ -1675,6 +1806,11 @@ static void qed_iov_vf_mbx_start_txq(struct qed_hwfn *p_hwfn,
 
         memset(&params, 0, sizeof(params));
         req = &mbx->req_virt->start_txq;
+
+        if (!qed_iov_validate_txq(p_hwfn, vf, req->tx_qid) ||
+            !qed_iov_validate_sb(p_hwfn, vf, req->hw_sb))
+                goto out;
+
         params.queue_id =  vf->vf_queues[req->tx_qid].fw_tx_qid;
         params.vport_id = vf->vport_id;
         params.sb = req->hw_sb;
@@ -1688,13 +1824,15 @@ static void qed_iov_vf_mbx_start_txq(struct qed_hwfn *p_hwfn,
                                          req->pbl_addr,
                                          req->pbl_size, &pq_params);
 
-        if (rc)
+        if (rc) {
                 status = PFVF_STATUS_FAILURE;
-        else
+        } else {
+                status = PFVF_STATUS_SUCCESS;
                 vf->vf_queues[req->tx_qid].txq_active = true;
+        }
 
-        qed_iov_prepare_resp(p_hwfn, p_ptt, vf, CHANNEL_TLV_START_TXQ,
-                             length, status);
+out:
+        qed_iov_vf_mbx_start_txq_resp(p_hwfn, p_ptt, vf, status);
 }
 
 static int qed_iov_vf_stop_rxqs(struct qed_hwfn *p_hwfn,
@@ -2119,6 +2257,16 @@ static void qed_iov_vf_mbx_vport_update(struct qed_hwfn *p_hwfn,
         u16 length;
         int rc;
 
+        /* Valiate PF can send such a request */
+        if (!vf->vport_instance) {
+                DP_VERBOSE(p_hwfn,
+                           QED_MSG_IOV,
+                           "No VPORT instance available for VF[%d], failing vport update\n",
+                           vf->abs_vf_id);
+                status = PFVF_STATUS_FAILURE;
+                goto out;
+        }
+
         memset(&params, 0, sizeof(params));
         params.opaque_fid = vf->opaque_fid;
         params.vport_id = vf->vport_id;
@@ -2161,15 +2309,12 @@ out:
         qed_iov_send_response(p_hwfn, p_ptt, vf, length, status);
 }
 
-static int qed_iov_vf_update_unicast_shadow(struct qed_hwfn *p_hwfn,
-                                            struct qed_vf_info *p_vf,
-                                            struct qed_filter_ucast *p_params)
+static int qed_iov_vf_update_vlan_shadow(struct qed_hwfn *p_hwfn,
+                                         struct qed_vf_info *p_vf,
+                                         struct qed_filter_ucast *p_params)
 {
         int i;
 
-        if (p_params->type == QED_FILTER_MAC)
-                return 0;
-
         /* First remove entries and then add new ones */
         if (p_params->opcode == QED_FILTER_REMOVE) {
                 for (i = 0; i < QED_ETH_VF_NUM_VLAN_FILTERS + 1; i++)
@@ -2222,6 +2367,80 @@ static int qed_iov_vf_update_unicast_shadow(struct qed_hwfn *p_hwfn,
         return 0;
 }
 
+static int qed_iov_vf_update_mac_shadow(struct qed_hwfn *p_hwfn,
+                                        struct qed_vf_info *p_vf,
+                                        struct qed_filter_ucast *p_params)
+{
+        int i;
+
+        /* If we're in forced-mode, we don't allow any change */
+        if (p_vf->bulletin.p_virt->valid_bitmap & (1 << MAC_ADDR_FORCED))
+                return 0;
+
+        /* First remove entries and then add new ones */
+        if (p_params->opcode == QED_FILTER_REMOVE) {
+                for (i = 0; i < QED_ETH_VF_NUM_MAC_FILTERS; i++) {
+                        if (ether_addr_equal(p_vf->shadow_config.macs[i],
+                                             p_params->mac)) {
+                                memset(p_vf->shadow_config.macs[i], 0,
+                                       ETH_ALEN);
+                                break;
+                        }
+                }
+
+                if (i == QED_ETH_VF_NUM_MAC_FILTERS) {
+                        DP_VERBOSE(p_hwfn, QED_MSG_IOV,
+                                   "MAC isn't configured\n");
+                        return -EINVAL;
+                }
+        } else if (p_params->opcode == QED_FILTER_REPLACE ||
+                   p_params->opcode == QED_FILTER_FLUSH) {
+                for (i = 0; i < QED_ETH_VF_NUM_MAC_FILTERS; i++)
+                        memset(p_vf->shadow_config.macs[i], 0, ETH_ALEN);
+        }
+
+        /* List the new MAC address */
+        if (p_params->opcode != QED_FILTER_ADD &&
+            p_params->opcode != QED_FILTER_REPLACE)
+                return 0;
+
+        for (i = 0; i < QED_ETH_VF_NUM_MAC_FILTERS; i++) {
+                if (is_zero_ether_addr(p_vf->shadow_config.macs[i])) {
+                        ether_addr_copy(p_vf->shadow_config.macs[i],
+                                        p_params->mac);
+                        DP_VERBOSE(p_hwfn, QED_MSG_IOV,
+                                   "Added MAC at %d entry in shadow\n", i);
+                        break;
+                }
+        }
+
+        if (i == QED_ETH_VF_NUM_MAC_FILTERS) {
+                DP_VERBOSE(p_hwfn, QED_MSG_IOV, "No available place for MAC\n");
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
+static int
+qed_iov_vf_update_unicast_shadow(struct qed_hwfn *p_hwfn,
+                                 struct qed_vf_info *p_vf,
+                                 struct qed_filter_ucast *p_params)
+{
+        int rc = 0;
+
+        if (p_params->type == QED_FILTER_MAC) {
+                rc = qed_iov_vf_update_mac_shadow(p_hwfn, p_vf, p_params);
+                if (rc)
+                        return rc;
+        }
+
+        if (p_params->type == QED_FILTER_VLAN)
+                rc = qed_iov_vf_update_vlan_shadow(p_hwfn, p_vf, p_params);
+
+        return rc;
+}
+
 int qed_iov_chk_ucast(struct qed_hwfn *hwfn,
                       int vfid, struct qed_filter_ucast *params)
 {
@@ -2366,11 +2585,27 @@ static void qed_iov_vf_mbx_release(struct qed_hwfn *p_hwfn,
                                    struct qed_vf_info *p_vf)
 {
         u16 length = sizeof(struct pfvf_def_resp_tlv);
+        u8 status = PFVF_STATUS_SUCCESS;
+        int rc = 0;
 
         qed_iov_vf_cleanup(p_hwfn, p_vf);
 
+        if (p_vf->state != VF_STOPPED && p_vf->state != VF_FREE) {
+                /* Stopping the VF */
+                rc = qed_sp_vf_stop(p_hwfn, p_vf->concrete_fid,
+                                    p_vf->opaque_fid);
+
+                if (rc) {
+                        DP_ERR(p_hwfn, "qed_sp_vf_stop returned error %d\n",
+                               rc);
+                        status = PFVF_STATUS_FAILURE;
+                }
+
+                p_vf->state = VF_STOPPED;
+        }
+
         qed_iov_prepare_resp(p_hwfn, p_ptt, p_vf, CHANNEL_TLV_RELEASE,
-                             length, PFVF_STATUS_SUCCESS);
+                             length, status);
 }
 
 static int
@@ -2622,7 +2857,6 @@ static void qed_iov_process_mbx_req(struct qed_hwfn *p_hwfn,
 {
         struct qed_iov_vf_mbx *mbx;
         struct qed_vf_info *p_vf;
-        int i;
 
         p_vf = qed_iov_get_vf_info(p_hwfn, (u16) vfid, true);
         if (!p_vf)
@@ -2631,9 +2865,8 @@ static void qed_iov_process_mbx_req(struct qed_hwfn *p_hwfn,
         mbx = &p_vf->vf_mbx;
 
         /* qed_iov_process_mbx_request */
-        DP_VERBOSE(p_hwfn,
-                   QED_MSG_IOV,
-                   "qed_iov_process_mbx_req vfid %d\n", p_vf->abs_vf_id);
+        DP_VERBOSE(p_hwfn, QED_MSG_IOV,
+                   "VF[%02x]: Processing mailbox message\n", p_vf->abs_vf_id);
 
         mbx->first_tlv = mbx->req_virt->first_tlv;
 
@@ -2687,15 +2920,28 @@ static void qed_iov_process_mbx_req(struct qed_hwfn *p_hwfn,
                  * support them. Or this may be because someone wrote a crappy
                  * VF driver and is sending garbage over the channel.
                  */
-                DP_ERR(p_hwfn,
-                       "unknown TLV. type %d length %d. first 20 bytes of mailbox buffer:\n",
-                       mbx->first_tlv.tl.type, mbx->first_tlv.tl.length);
-
-                for (i = 0; i < 20; i++) {
+                DP_NOTICE(p_hwfn,
+                          "VF[%02x]: unknown TLV. type %04x length %04x padding %08x reply address %llu\n",
+                          p_vf->abs_vf_id,
+                          mbx->first_tlv.tl.type,
+                          mbx->first_tlv.tl.length,
+                          mbx->first_tlv.padding, mbx->first_tlv.reply_address);
+
+                /* Try replying in case reply address matches the acquisition's
+                 * posted address.
+                 */
+                if (p_vf->acquire.first_tlv.reply_address &&
+                    (mbx->first_tlv.reply_address ==
+                     p_vf->acquire.first_tlv.reply_address)) {
+                        qed_iov_prepare_resp(p_hwfn, p_ptt, p_vf,
+                                             mbx->first_tlv.tl.type,
+                                             sizeof(struct pfvf_def_resp_tlv),
+                                             PFVF_STATUS_NOT_SUPPORTED);
+                } else {
                         DP_VERBOSE(p_hwfn,
                                    QED_MSG_IOV,
-                                   "%x ",
-                                   mbx->req_virt->tlv_buf_size.tlv_buffer[i]);
+                                   "VF[%02x]: Can't respond to TLV - no valid reply address\n",
+                                   p_vf->abs_vf_id);
                 }
         }
 }
diff --git a/drivers/net/ethernet/qlogic/qed/qed_sriov.h b/drivers/net/ethernet/qlogic/qed/qed_sriov.h
index c90b2b6ad969..0dd23e409b3f 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_sriov.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_sriov.h
@@ -10,6 +10,9 @@
 #define _QED_SRIOV_H
 #include <linux/types.h>
 #include "qed_vf.h"
+
+#define QED_ETH_VF_NUM_MAC_FILTERS 1
+#define QED_ETH_VF_NUM_VLAN_FILTERS 2
 #define QED_VF_ARRAY_LENGTH (3)
 
 #ifdef CONFIG_QED_SRIOV
@@ -24,7 +27,6 @@
 #define IS_PF_SRIOV_ALLOC(p_hwfn)       (!!((p_hwfn)->pf_iov_info))
 
 #define QED_MAX_VF_CHAINS_PER_PF 16
-#define QED_ETH_VF_NUM_VLAN_FILTERS 2
 
 #define QED_ETH_MAX_VF_NUM_VLAN_FILTERS \
         (MAX_NUM_VFS * QED_ETH_VF_NUM_VLAN_FILTERS)
@@ -120,6 +122,8 @@ struct qed_vf_shadow_config {
         /* Shadow copy of all guest vlans */
         struct qed_vf_vlan_shadow vlans[QED_ETH_VF_NUM_VLAN_FILTERS + 1];
 
+        /* Shadow copy of all configured MACs; Empty if forcing MACs */
+        u8 macs[QED_ETH_VF_NUM_MAC_FILTERS][ETH_ALEN];
         u8 inner_vlan_removal;
 };
 
@@ -133,6 +137,9 @@ struct qed_vf_info {
         struct qed_bulletin bulletin;
         dma_addr_t vf_bulletin;
 
+        /* PF saves a copy of the last VF acquire message */
+        struct vfpf_acquire_tlv acquire;
+
         u32 concrete_fid;
         u16 opaque_fid;
         u16 mtu;
diff --git a/drivers/net/ethernet/qlogic/qed/qed_vf.c b/drivers/net/ethernet/qlogic/qed/qed_vf.c
index 72e69c0ec10d..9819230947bf 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_vf.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_vf.c
@@ -117,36 +117,64 @@ exit:
 }
 
 #define VF_ACQUIRE_THRESH 3
-#define VF_ACQUIRE_MAC_FILTERS 1
+static void qed_vf_pf_acquire_reduce_resc(struct qed_hwfn *p_hwfn,
+                                          struct vf_pf_resc_request *p_req,
+                                          struct pf_vf_resc *p_resp)
+{
+        DP_VERBOSE(p_hwfn,
+                   QED_MSG_IOV,
+                   "PF unwilling to fullill resource request: rxq [%02x/%02x] txq [%02x/%02x] sbs [%02x/%02x] mac [%02x/%02x] vlan [%02x/%02x] mc [%02x/%02x]. Try PF recommended amount\n",
+                   p_req->num_rxqs,
+                   p_resp->num_rxqs,
+                   p_req->num_rxqs,
+                   p_resp->num_txqs,
+                   p_req->num_sbs,
+                   p_resp->num_sbs,
+                   p_req->num_mac_filters,
+                   p_resp->num_mac_filters,
+                   p_req->num_vlan_filters,
+                   p_resp->num_vlan_filters,
+                   p_req->num_mc_filters, p_resp->num_mc_filters);
+
+        /* humble our request */
+        p_req->num_txqs = p_resp->num_txqs;
+        p_req->num_rxqs = p_resp->num_rxqs;
+        p_req->num_sbs = p_resp->num_sbs;
+        p_req->num_mac_filters = p_resp->num_mac_filters;
+        p_req->num_vlan_filters = p_resp->num_vlan_filters;
+        p_req->num_mc_filters = p_resp->num_mc_filters;
+}
 
 static int qed_vf_pf_acquire(struct qed_hwfn *p_hwfn)
 {
         struct qed_vf_iov *p_iov = p_hwfn->vf_iov_info;
         struct pfvf_acquire_resp_tlv *resp = &p_iov->pf2vf_reply->acquire_resp;
         struct pf_vf_pfdev_info *pfdev_info = &resp->pfdev_info;
-        u8 rx_count = 1, tx_count = 1, num_sbs = 1;
-        u8 num_mac = VF_ACQUIRE_MAC_FILTERS;
+        struct vf_pf_resc_request *p_resc;
         bool resources_acquired = false;
         struct vfpf_acquire_tlv *req;
         int rc = 0, attempts = 0;
 
         /* clear mailbox and prep first tlv */
         req = qed_vf_pf_prep(p_hwfn, CHANNEL_TLV_ACQUIRE, sizeof(*req));
+        p_resc = &req->resc_request;
 
         /* starting filling the request */
         req->vfdev_info.opaque_fid = p_hwfn->hw_info.opaque_fid;
 
-        req->resc_request.num_rxqs = rx_count;
-        req->resc_request.num_txqs = tx_count;
-        req->resc_request.num_sbs = num_sbs;
-        req->resc_request.num_mac_filters = num_mac;
-        req->resc_request.num_vlan_filters = QED_ETH_VF_NUM_VLAN_FILTERS;
+        p_resc->num_rxqs = QED_MAX_VF_CHAINS_PER_PF;
+        p_resc->num_txqs = QED_MAX_VF_CHAINS_PER_PF;
+        p_resc->num_sbs = QED_MAX_VF_CHAINS_PER_PF;
+        p_resc->num_mac_filters = QED_ETH_VF_NUM_MAC_FILTERS;
+        p_resc->num_vlan_filters = QED_ETH_VF_NUM_VLAN_FILTERS;
 
         req->vfdev_info.os_type = VFPF_ACQUIRE_OS_LINUX;
         req->vfdev_info.fw_major = FW_MAJOR_VERSION;
         req->vfdev_info.fw_minor = FW_MINOR_VERSION;
         req->vfdev_info.fw_revision = FW_REVISION_VERSION;
         req->vfdev_info.fw_engineering = FW_ENGINEERING_VERSION;
+        req->vfdev_info.eth_fp_hsi_major = ETH_HSI_VER_MAJOR;
+        req->vfdev_info.eth_fp_hsi_minor = ETH_HSI_VER_MINOR;
 
         /* Fill capability field with any non-deprecated config we support */
         req->vfdev_info.capabilities |= VFPF_ACQUIRE_CAP_100G;
@@ -185,21 +213,21 @@ static int qed_vf_pf_acquire(struct qed_hwfn *p_hwfn)
                         resources_acquired = true;
                 } else if (resp->hdr.status == PFVF_STATUS_NO_RESOURCE &&
                            attempts < VF_ACQUIRE_THRESH) {
-                        DP_VERBOSE(p_hwfn,
-                                   QED_MSG_IOV,
-                                   "PF unwilling to fullfill resource request. Try PF recommended amount\n");
-
-                        /* humble our request */
-                        req->resc_request.num_txqs = resp->resc.num_txqs;
-                        req->resc_request.num_rxqs = resp->resc.num_rxqs;
-                        req->resc_request.num_sbs = resp->resc.num_sbs;
-                        req->resc_request.num_mac_filters =
-                            resp->resc.num_mac_filters;
-                        req->resc_request.num_vlan_filters =
-                            resp->resc.num_vlan_filters;
+                        qed_vf_pf_acquire_reduce_resc(p_hwfn, p_resc,
+                                                      &resp->resc);
 
                         /* Clear response buffer */
                         memset(p_iov->pf2vf_reply, 0, sizeof(union pfvf_tlvs));
+                } else if ((resp->hdr.status == PFVF_STATUS_NOT_SUPPORTED) &&
+                           pfdev_info->major_fp_hsi &&
+                           (pfdev_info->major_fp_hsi != ETH_HSI_VER_MAJOR)) {
+                        DP_NOTICE(p_hwfn,
+                                  "PF uses an incompatible fastpath HSI %02x.%02x [VF requires %02x.%02x]. Please change to a VF driver using %02x.xx.\n",
+                                  pfdev_info->major_fp_hsi,
+                                  pfdev_info->minor_fp_hsi,
+                                  ETH_HSI_VER_MAJOR,
+                                  ETH_HSI_VER_MINOR, pfdev_info->major_fp_hsi);
+                        return -EINVAL;
                 } else {
                         DP_ERR(p_hwfn,
                                "PF returned error %d to VF acquisition request\n",
@@ -225,6 +253,13 @@ static int qed_vf_pf_acquire(struct qed_hwfn *p_hwfn)
                 }
         }
 
+        if (ETH_HSI_VER_MINOR &&
+            (resp->pfdev_info.minor_fp_hsi < ETH_HSI_VER_MINOR)) {
+                DP_INFO(p_hwfn,
+                        "PF is using older fastpath HSI; %02x.%02x is configured\n",
+                        ETH_HSI_VER_MAJOR, resp->pfdev_info.minor_fp_hsi);
+        }
+
         return 0;
 }
 
@@ -405,8 +440,8 @@ int qed_vf_pf_txq_start(struct qed_hwfn *p_hwfn,
                         u16 pbl_size, void __iomem **pp_doorbell)
 {
         struct qed_vf_iov *p_iov = p_hwfn->vf_iov_info;
+        struct pfvf_start_queue_resp_tlv *resp;
         struct vfpf_start_txq_tlv *req;
-        struct pfvf_def_resp_tlv *resp;
         int rc;
 
         /* clear mailbox and prep first tlv */
@@ -424,20 +459,24 @@ int qed_vf_pf_txq_start(struct qed_hwfn *p_hwfn,
         qed_add_tlv(p_hwfn, &p_iov->offset,
                     CHANNEL_TLV_LIST_END, sizeof(struct channel_list_end_tlv));
 
-        resp = &p_iov->pf2vf_reply->default_resp;
+        resp = &p_iov->pf2vf_reply->queue_start;
         rc = qed_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp));
         if (rc)
-                return rc;
+                goto exit;
 
-        if (resp->hdr.status != PFVF_STATUS_SUCCESS)
-                return -EINVAL;
+        if (resp->hdr.status != PFVF_STATUS_SUCCESS) {
+                rc = -EINVAL;
+                goto exit;
+        }
 
         if (pp_doorbell) {
-                u8 cid = p_iov->acquire_resp.resc.cid[tx_queue_id];
+                *pp_doorbell = (u8 __iomem *)p_hwfn->doorbells + resp->offset;
 
-                *pp_doorbell = (u8 __iomem *)p_hwfn->doorbells +
-                                             qed_db_addr(cid, DQ_DEMS_LEGACY);
+                DP_VERBOSE(p_hwfn, QED_MSG_IOV,
+                           "Txq[0x%02x]: doorbell at %p [offset 0x%08x]\n",
+                           tx_queue_id, *pp_doorbell, resp->offset);
         }
+exit:
 
         return rc;
 }
diff --git a/drivers/net/ethernet/qlogic/qed/qed_vf.h b/drivers/net/ethernet/qlogic/qed/qed_vf.h
index b82fda964bbd..b23ce58e932f 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_vf.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_vf.h
@@ -96,7 +96,9 @@ struct vfpf_acquire_tlv {
                 u32 driver_version;
                 u16 opaque_fid; /* ME register value */
                 u8 os_type;     /* VFPF_ACQUIRE_OS_* value */
-                u8 padding[5];
+                u8 eth_fp_hsi_major;
+                u8 eth_fp_hsi_minor;
+                u8 padding[3];
         } vfdev_info;
 
         struct vf_pf_resc_request resc_request;
@@ -171,7 +173,14 @@ struct pfvf_acquire_resp_tlv {
                 struct pfvf_stats_info stats_info;
 
                 u8 port_mac[ETH_ALEN];
-                u8 padding2[2];
+
+                /* It's possible PF had to configure an older fastpath HSI
+                 * [in case VF is newer than PF]. This is communicated back
+                 * to the VF. It can also be used in case of error due to
+                 * non-matching versions to shed light in VF about failure.
+                 */
+                u8 major_fp_hsi;
+                u8 minor_fp_hsi;
         } pfdev_info;
 
         struct pf_vf_resc {
diff --git a/drivers/net/ethernet/qlogic/qede/Makefile b/drivers/net/ethernet/qlogic/qede/Makefile
index 06ff90d87572..74a49850d74d 100644
--- a/drivers/net/ethernet/qlogic/qede/Makefile
+++ b/drivers/net/ethernet/qlogic/qede/Makefile
@@ -1,3 +1,4 @@
 obj-$(CONFIG_QEDE) := qede.o
 
 qede-y := qede_main.o qede_ethtool.o
+qede-$(CONFIG_DCB) += qede_dcbnl.o
diff --git a/drivers/net/ethernet/qlogic/qede/qede.h b/drivers/net/ethernet/qlogic/qede/qede.h
index 47d6b22252f6..1441c8f6d414 100644
--- a/drivers/net/ethernet/qlogic/qede/qede.h
+++ b/drivers/net/ethernet/qlogic/qede/qede.h
@@ -304,6 +304,9 @@ union qede_reload_args {
         u16 mtu;
 };
 
+#ifdef CONFIG_DCB
+void qede_set_dcbnl_ops(struct net_device *ndev);
+#endif
 void qede_config_debug(uint debug, u32 *p_dp_module, u8 *p_dp_level);
 void qede_set_ethtool_ops(struct net_device *netdev);
 void qede_reload(struct qede_dev *edev,
diff --git a/drivers/net/ethernet/qlogic/qede/qede_dcbnl.c b/drivers/net/ethernet/qlogic/qede/qede_dcbnl.c
new file mode 100644
index 000000000000..03e8c0212433
--- /dev/null
+++ b/drivers/net/ethernet/qlogic/qede/qede_dcbnl.c
@@ -0,0 +1,348 @@
+/* QLogic qede NIC Driver
+* Copyright (c) 2015 QLogic Corporation
+*
+* This software is available under the terms of the GNU General Public License
+* (GPL) Version 2, available from the file COPYING in the main directory of
+* this source tree.
+*/
+
+#include <linux/types.h>
+#include <linux/netdevice.h>
+#include <linux/rtnetlink.h>
+#include <net/dcbnl.h>
+#include "qede.h"
+
+static u8 qede_dcbnl_getstate(struct net_device *netdev)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->getstate(edev->cdev);
+}
+
+static u8 qede_dcbnl_setstate(struct net_device *netdev, u8 state)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->setstate(edev->cdev, state);
+}
+
+static void qede_dcbnl_getpermhwaddr(struct net_device *netdev,
+                                     u8 *perm_addr)
+{
+        memcpy(perm_addr, netdev->dev_addr, netdev->addr_len);
+}
+
+static void qede_dcbnl_getpgtccfgtx(struct net_device *netdev, int prio,
+                                    u8 *prio_type, u8 *pgid, u8 *bw_pct,
+                                    u8 *up_map)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        edev->ops->dcb->getpgtccfgtx(edev->cdev, prio, prio_type,
+                                     pgid, bw_pct, up_map);
+}
+
+static void qede_dcbnl_getpgbwgcfgtx(struct net_device *netdev,
+                                     int pgid, u8 *bw_pct)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        edev->ops->dcb->getpgbwgcfgtx(edev->cdev, pgid, bw_pct);
+}
+
+static void qede_dcbnl_getpgtccfgrx(struct net_device *netdev, int prio,
+                                    u8 *prio_type, u8 *pgid, u8 *bw_pct,
+                                    u8 *up_map)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        edev->ops->dcb->getpgtccfgrx(edev->cdev, prio, prio_type, pgid, bw_pct,
+                                     up_map);
+}
+
+static void qede_dcbnl_getpgbwgcfgrx(struct net_device *netdev,
+                                     int pgid, u8 *bw_pct)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        edev->ops->dcb->getpgbwgcfgrx(edev->cdev, pgid, bw_pct);
+}
+
+static void qede_dcbnl_getpfccfg(struct net_device *netdev, int prio,
+                                 u8 *setting)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        edev->ops->dcb->getpfccfg(edev->cdev, prio, setting);
+}
+
+static void qede_dcbnl_setpfccfg(struct net_device *netdev, int prio,
+                                 u8 setting)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        edev->ops->dcb->setpfccfg(edev->cdev, prio, setting);
+}
+
+static u8 qede_dcbnl_getcap(struct net_device *netdev, int capid, u8 *cap)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->getcap(edev->cdev, capid, cap);
+}
+
+static int qede_dcbnl_getnumtcs(struct net_device *netdev, int tcid, u8 *num)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->getnumtcs(edev->cdev, tcid, num);
+}
+
+static u8 qede_dcbnl_getpfcstate(struct net_device *netdev)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->getpfcstate(edev->cdev);
+}
+
+static int qede_dcbnl_getapp(struct net_device *netdev, u8 idtype, u16 id)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->getapp(edev->cdev, idtype, id);
+}
+
+static u8 qede_dcbnl_getdcbx(struct net_device *netdev)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->getdcbx(edev->cdev);
+}
+
+static void qede_dcbnl_setpgtccfgtx(struct net_device *netdev, int prio,
+                                    u8 pri_type, u8 pgid, u8 bw_pct, u8 up_map)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->setpgtccfgtx(edev->cdev, prio, pri_type, pgid,
+                                            bw_pct, up_map);
+}
+
+static void qede_dcbnl_setpgtccfgrx(struct net_device *netdev, int prio,
+                                    u8 pri_type, u8 pgid, u8 bw_pct, u8 up_map)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->setpgtccfgrx(edev->cdev, prio, pri_type, pgid,
+                                            bw_pct, up_map);
+}
+
+static void qede_dcbnl_setpgbwgcfgtx(struct net_device *netdev, int pgid,
+                                     u8 bw_pct)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->setpgbwgcfgtx(edev->cdev, pgid, bw_pct);
+}
+
+static void qede_dcbnl_setpgbwgcfgrx(struct net_device *netdev, int pgid,
+                                     u8 bw_pct)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->setpgbwgcfgrx(edev->cdev, pgid, bw_pct);
+}
+
+static u8 qede_dcbnl_setall(struct net_device *netdev)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->setall(edev->cdev);
+}
+
+static int qede_dcbnl_setnumtcs(struct net_device *netdev, int tcid, u8 num)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->setnumtcs(edev->cdev, tcid, num);
+}
+
+static void qede_dcbnl_setpfcstate(struct net_device *netdev, u8 state)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->setpfcstate(edev->cdev, state);
+}
+
+static int qede_dcbnl_setapp(struct net_device *netdev, u8 idtype, u16 idval,
+                             u8 up)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->setapp(edev->cdev, idtype, idval, up);
+}
+
+static u8 qede_dcbnl_setdcbx(struct net_device *netdev, u8 state)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->setdcbx(edev->cdev, state);
+}
+
+static u8 qede_dcbnl_getfeatcfg(struct net_device *netdev, int featid,
+                                u8 *flags)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->getfeatcfg(edev->cdev, featid, flags);
+}
+
+static u8 qede_dcbnl_setfeatcfg(struct net_device *netdev, int featid, u8 flags)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->setfeatcfg(edev->cdev, featid, flags);
+}
+
+static int qede_dcbnl_peer_getappinfo(struct net_device *netdev,
+                                      struct dcb_peer_app_info *info,
+                                      u16 *count)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->peer_getappinfo(edev->cdev, info, count);
+}
+
+static int qede_dcbnl_peer_getapptable(struct net_device *netdev,
+                                       struct dcb_app *app)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->peer_getapptable(edev->cdev, app);
+}
+
+static int qede_dcbnl_cee_peer_getpfc(struct net_device *netdev,
+                                      struct cee_pfc *pfc)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->cee_peer_getpfc(edev->cdev, pfc);
+}
+
+static int qede_dcbnl_cee_peer_getpg(struct net_device *netdev,
+                                     struct cee_pg *pg)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->cee_peer_getpg(edev->cdev, pg);
+}
+
+static int qede_dcbnl_ieee_getpfc(struct net_device *netdev,
+                                  struct ieee_pfc *pfc)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->ieee_getpfc(edev->cdev, pfc);
+}
+
+static int qede_dcbnl_ieee_setpfc(struct net_device *netdev,
+                                  struct ieee_pfc *pfc)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->ieee_setpfc(edev->cdev, pfc);
+}
+
+static int qede_dcbnl_ieee_getets(struct net_device *netdev,
+                                  struct ieee_ets *ets)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->ieee_getets(edev->cdev, ets);
+}
+
+static int qede_dcbnl_ieee_setets(struct net_device *netdev,
+                                  struct ieee_ets *ets)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->ieee_setets(edev->cdev, ets);
+}
+
+static int qede_dcbnl_ieee_getapp(struct net_device *netdev,
+                                  struct dcb_app *app)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->ieee_getapp(edev->cdev, app);
+}
+
+static int qede_dcbnl_ieee_setapp(struct net_device *netdev,
+                                  struct dcb_app *app)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->ieee_setapp(edev->cdev, app);
+}
+
+static int qede_dcbnl_ieee_peer_getpfc(struct net_device *netdev,
+                                       struct ieee_pfc *pfc)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->ieee_peer_getpfc(edev->cdev, pfc);
+}
+
+static int qede_dcbnl_ieee_peer_getets(struct net_device *netdev,
+                                       struct ieee_ets *ets)
+{
+        struct qede_dev *edev = netdev_priv(netdev);
+
+        return edev->ops->dcb->ieee_peer_getets(edev->cdev, ets);
+}
+
+static const struct dcbnl_rtnl_ops qede_dcbnl_ops = {
+        .ieee_getpfc = qede_dcbnl_ieee_getpfc,
+        .ieee_setpfc = qede_dcbnl_ieee_setpfc,
+        .ieee_getets = qede_dcbnl_ieee_getets,
+        .ieee_setets = qede_dcbnl_ieee_setets,
+        .ieee_getapp = qede_dcbnl_ieee_getapp,
+        .ieee_setapp = qede_dcbnl_ieee_setapp,
+        .getdcbx = qede_dcbnl_getdcbx,
+        .ieee_peer_getpfc = qede_dcbnl_ieee_peer_getpfc,
+        .ieee_peer_getets = qede_dcbnl_ieee_peer_getets,
+        .getstate = qede_dcbnl_getstate,
+        .setstate = qede_dcbnl_setstate,
+        .getpermhwaddr = qede_dcbnl_getpermhwaddr,
+        .getpgtccfgtx = qede_dcbnl_getpgtccfgtx,
+        .getpgbwgcfgtx = qede_dcbnl_getpgbwgcfgtx,
+        .getpgtccfgrx = qede_dcbnl_getpgtccfgrx,
+        .getpgbwgcfgrx = qede_dcbnl_getpgbwgcfgrx,
+        .getpfccfg = qede_dcbnl_getpfccfg,
+        .setpfccfg = qede_dcbnl_setpfccfg,
+        .getcap = qede_dcbnl_getcap,
+        .getnumtcs = qede_dcbnl_getnumtcs,
+        .getpfcstate = qede_dcbnl_getpfcstate,
+        .getapp = qede_dcbnl_getapp,
+        .getdcbx = qede_dcbnl_getdcbx,
+        .setpgtccfgtx = qede_dcbnl_setpgtccfgtx,
+        .setpgtccfgrx = qede_dcbnl_setpgtccfgrx,
+        .setpgbwgcfgtx = qede_dcbnl_setpgbwgcfgtx,
+        .setpgbwgcfgrx = qede_dcbnl_setpgbwgcfgrx,
+        .setall = qede_dcbnl_setall,
+        .setnumtcs = qede_dcbnl_setnumtcs,
+        .setpfcstate = qede_dcbnl_setpfcstate,
+        .setapp = qede_dcbnl_setapp,
+        .setdcbx = qede_dcbnl_setdcbx,
+        .setfeatcfg = qede_dcbnl_setfeatcfg,
+        .getfeatcfg = qede_dcbnl_getfeatcfg,
+        .peer_getappinfo = qede_dcbnl_peer_getappinfo,
+        .peer_getapptable = qede_dcbnl_peer_getapptable,
+        .cee_peer_getpfc = qede_dcbnl_cee_peer_getpfc,
+        .cee_peer_getpg = qede_dcbnl_cee_peer_getpg,
+};
+
+void qede_set_dcbnl_ops(struct net_device *dev)
+{
+        dev->dcbnl_ops = &qede_dcbnl_ops;
+}
diff --git a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c
index ad3cae3b7243..c5c658ab0724 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c
@@ -426,6 +426,59 @@ static u32 qede_get_link(struct net_device *dev)
         return current_link.link_up;
 }
 
+static int qede_get_coalesce(struct net_device *dev,
+                             struct ethtool_coalesce *coal)
+{
+        struct qede_dev *edev = netdev_priv(dev);
+        u16 rxc, txc;
+
+        memset(coal, 0, sizeof(struct ethtool_coalesce));
+        edev->ops->common->get_coalesce(edev->cdev, &rxc, &txc);
+
+        coal->rx_coalesce_usecs = rxc;
+        coal->tx_coalesce_usecs = txc;
+
+        return 0;
+}
+
+static int qede_set_coalesce(struct net_device *dev,
+                             struct ethtool_coalesce *coal)
+{
+        struct qede_dev *edev = netdev_priv(dev);
+        int i, rc = 0;
+        u16 rxc, txc;
+        u8 sb_id;
+
+        if (!netif_running(dev)) {
+                DP_INFO(edev, "Interface is down\n");
+                return -EINVAL;
+        }
+
+        if (coal->rx_coalesce_usecs > QED_COALESCE_MAX ||
+            coal->tx_coalesce_usecs > QED_COALESCE_MAX) {
+                DP_INFO(edev,
+                        "Can't support requested %s coalesce value [max supported value %d]\n",
+                        coal->rx_coalesce_usecs > QED_COALESCE_MAX ? "rx"
+                                                                   : "tx",
+                        QED_COALESCE_MAX);
+                return -EINVAL;
+        }
+
+        rxc = (u16)coal->rx_coalesce_usecs;
+        txc = (u16)coal->tx_coalesce_usecs;
+        for_each_rss(i) {
+                sb_id = edev->fp_array[i].sb_info->igu_sb_id;
+                rc = edev->ops->common->set_coalesce(edev->cdev, rxc, txc,
+                                                     (u8)i, sb_id);
+                if (rc) {
+                        DP_INFO(edev, "Set coalesce error, rc = %d\n", rc);
+                        return rc;
+                }
+        }
+
+        return rc;
+}
+
 static void qede_get_ringparam(struct net_device *dev,
                                struct ethtool_ringparam *ering)
 {
@@ -910,6 +963,8 @@ static int qede_selftest_transmit_traffic(struct qede_dev *edev,
         memset(first_bd, 0, sizeof(*first_bd));
         val = 1 << ETH_TX_1ST_BD_FLAGS_START_BD_SHIFT;
         first_bd->data.bd_flags.bitfields = val;
+        val = skb->len & ETH_TX_DATA_1ST_BD_PKT_LEN_MASK;
+        first_bd->data.bitfields |= (val << ETH_TX_DATA_1ST_BD_PKT_LEN_SHIFT);
 
         /* Map skb linear data for DMA and set in the first BD */
         mapping = dma_map_single(&edev->pdev->dev, skb->data,
@@ -1137,6 +1192,8 @@ static const struct ethtool_ops qede_ethtool_ops = {
         .set_msglevel = qede_set_msglevel,
         .nway_reset = qede_nway_reset,
         .get_link = qede_get_link,
+        .get_coalesce = qede_get_coalesce,
+        .set_coalesce = qede_set_coalesce,
         .get_ringparam = qede_get_ringparam,
         .set_ringparam = qede_set_ringparam,
         .get_pauseparam = qede_get_pauseparam,
diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c
index f8e11f953acb..19bc631e1f04 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_main.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_main.c
@@ -24,12 +24,7 @@
 #include <linux/netdev_features.h>
 #include <linux/udp.h>
 #include <linux/tcp.h>
-#ifdef CONFIG_QEDE_VXLAN
-#include <net/vxlan.h>
-#endif
-#ifdef CONFIG_QEDE_GENEVE
-#include <net/geneve.h>
-#endif
+#include <net/udp_tunnel.h>
 #include <linux/ip.h>
 #include <net/ipv6.h>
 #include <net/tcp.h>
@@ -579,8 +574,6 @@ netdev_tx_t qede_start_xmit(struct sk_buff *skb,
 
         /* Fill the parsing flags & params according to the requested offload */
         if (xmit_type & XMIT_L4_CSUM) {
-                u16 temp = 1 << ETH_TX_DATA_1ST_BD_TUNN_CFG_OVERRIDE_SHIFT;
-
                 /* We don't re-calculate IP checksum as it is already done by
                  * the upper stack
                  */
@@ -590,14 +583,8 @@ netdev_tx_t qede_start_xmit(struct sk_buff *skb,
                 if (xmit_type & XMIT_ENC) {
                         first_bd->data.bd_flags.bitfields |=
                                 1 << ETH_TX_1ST_BD_FLAGS_IP_CSUM_SHIFT;
-                } else {
-                        /* In cases when OS doesn't indicate for inner offloads
-                         * when packet is tunnelled, we need to override the HW
-                         * tunnel configuration so that packets are treated as
-                         * regular non tunnelled packets and no inner offloads
-                         * are done by the hardware.
-                         */
-                        first_bd->data.bitfields |= cpu_to_le16(temp);
+                        first_bd->data.bitfields |=
+                            1 << ETH_TX_DATA_1ST_BD_TUNN_FLAG_SHIFT;
                 }
 
                 /* If the packet is IPv6 with extension header, indicate that
@@ -655,6 +642,10 @@ netdev_tx_t qede_start_xmit(struct sk_buff *skb,
                         tx_data_bd = (struct eth_tx_bd *)third_bd;
                         data_split = true;
                 }
+        } else {
+                first_bd->data.bitfields |=
+                    (skb->len & ETH_TX_DATA_1ST_BD_PKT_LEN_MASK) <<
+                    ETH_TX_DATA_1ST_BD_PKT_LEN_SHIFT;
         }
 
         /* Handle fragmented skb */
@@ -2116,75 +2107,75 @@ int qede_set_features(struct net_device *dev, netdev_features_t features)
         return 0;
 }
 
-#ifdef CONFIG_QEDE_VXLAN
-static void qede_add_vxlan_port(struct net_device *dev,
-                                sa_family_t sa_family, __be16 port)
+static void qede_udp_tunnel_add(struct net_device *dev,
+                                struct udp_tunnel_info *ti)
 {
         struct qede_dev *edev = netdev_priv(dev);
-        u16 t_port = ntohs(port);
+        u16 t_port = ntohs(ti->port);
 
-        if (edev->vxlan_dst_port)
-                return;
+        switch (ti->type) {
+        case UDP_TUNNEL_TYPE_VXLAN:
+                if (edev->vxlan_dst_port)
+                        return;
 
-        edev->vxlan_dst_port = t_port;
+                edev->vxlan_dst_port = t_port;
 
-        DP_VERBOSE(edev, QED_MSG_DEBUG, "Added vxlan port=%d", t_port);
+                DP_VERBOSE(edev, QED_MSG_DEBUG, "Added vxlan port=%d",
+                           t_port);
 
-        set_bit(QEDE_SP_VXLAN_PORT_CONFIG, &edev->sp_flags);
-        schedule_delayed_work(&edev->sp_task, 0);
-}
+                set_bit(QEDE_SP_VXLAN_PORT_CONFIG, &edev->sp_flags);
+                break;
+        case UDP_TUNNEL_TYPE_GENEVE:
+                if (edev->geneve_dst_port)
+                        return;
 
-static void qede_del_vxlan_port(struct net_device *dev,
-                                sa_family_t sa_family, __be16 port)
-{
-        struct qede_dev *edev = netdev_priv(dev);
-        u16 t_port = ntohs(port);
+                edev->geneve_dst_port = t_port;
 
-        if (t_port != edev->vxlan_dst_port)
+                DP_VERBOSE(edev, QED_MSG_DEBUG, "Added geneve port=%d",
+                           t_port);
+                set_bit(QEDE_SP_GENEVE_PORT_CONFIG, &edev->sp_flags);
+                break;
+        default:
                 return;
+        }
 
-        edev->vxlan_dst_port = 0;
-
-        DP_VERBOSE(edev, QED_MSG_DEBUG, "Deleted vxlan port=%d", t_port);
-
-        set_bit(QEDE_SP_VXLAN_PORT_CONFIG, &edev->sp_flags);
         schedule_delayed_work(&edev->sp_task, 0);
 }
-#endif
 
-#ifdef CONFIG_QEDE_GENEVE
-static void qede_add_geneve_port(struct net_device *dev,
-                                 sa_family_t sa_family, __be16 port)
+static void qede_udp_tunnel_del(struct net_device *dev,
+                                struct udp_tunnel_info *ti)
 {
         struct qede_dev *edev = netdev_priv(dev);
-        u16 t_port = ntohs(port);
+        u16 t_port = ntohs(ti->port);
 
-        if (edev->geneve_dst_port)
-                return;
+        switch (ti->type) {
+        case UDP_TUNNEL_TYPE_VXLAN:
+                if (t_port != edev->vxlan_dst_port)
+                        return;
 
-        edev->geneve_dst_port = t_port;
+                edev->vxlan_dst_port = 0;
 
-        DP_VERBOSE(edev, QED_MSG_DEBUG, "Added geneve port=%d", t_port);
-        set_bit(QEDE_SP_GENEVE_PORT_CONFIG, &edev->sp_flags);
-        schedule_delayed_work(&edev->sp_task, 0);
-}
+                DP_VERBOSE(edev, QED_MSG_DEBUG, "Deleted vxlan port=%d",
+                           t_port);
 
-static void qede_del_geneve_port(struct net_device *dev,
-                                 sa_family_t sa_family, __be16 port)
-{
-        struct qede_dev *edev = netdev_priv(dev);
-        u16 t_port = ntohs(port);
+                set_bit(QEDE_SP_VXLAN_PORT_CONFIG, &edev->sp_flags);
+                break;
+        case UDP_TUNNEL_TYPE_GENEVE:
+                if (t_port != edev->geneve_dst_port)
+                        return;
 
-        if (t_port != edev->geneve_dst_port)
-                return;
+                edev->geneve_dst_port = 0;
 
-        edev->geneve_dst_port = 0;
+                DP_VERBOSE(edev, QED_MSG_DEBUG, "Deleted geneve port=%d",
+                           t_port);
+                set_bit(QEDE_SP_GENEVE_PORT_CONFIG, &edev->sp_flags);
+                break;
+        default:
+                return;
+        }
 
-        DP_VERBOSE(edev, QED_MSG_DEBUG, "Deleted geneve port=%d", t_port);
-        set_bit(QEDE_SP_GENEVE_PORT_CONFIG, &edev->sp_flags);
         schedule_delayed_work(&edev->sp_task, 0);
 }
-#endif
 
 static const struct net_device_ops qede_netdev_ops = {
         .ndo_open = qede_open,
@@ -2208,14 +2199,8 @@ static const struct net_device_ops qede_netdev_ops = {
         .ndo_get_vf_config = qede_get_vf_config,
         .ndo_set_vf_rate = qede_set_vf_rate,
 #endif
-#ifdef CONFIG_QEDE_VXLAN
-        .ndo_add_vxlan_port = qede_add_vxlan_port,
-        .ndo_del_vxlan_port = qede_del_vxlan_port,
-#endif
-#ifdef CONFIG_QEDE_GENEVE
-        .ndo_add_geneve_port = qede_add_geneve_port,
-        .ndo_del_geneve_port = qede_del_geneve_port,
-#endif
+        .ndo_udp_tunnel_add = qede_udp_tunnel_add,
+        .ndo_udp_tunnel_del = qede_udp_tunnel_del,
 };
 
 /* -------------------------------------------------------------------------
@@ -2505,6 +2490,10 @@ static int __qede_probe(struct pci_dev *pdev, u32 dp_module, u8 dp_level,
 
         edev->ops->register_ops(cdev, &qede_ll_ops, edev);
 
+#ifdef CONFIG_DCB
+        qede_set_dcbnl_ops(edev->ndev);
+#endif
+
         INIT_DELAYED_WORK(&edev->sp_task, qede_sp_task);
         mutex_init(&edev->qede_lock);
 
@@ -2823,6 +2812,7 @@ static int qede_alloc_mem_rxq(struct qede_dev *edev,
         rc = edev->ops->common->chain_alloc(edev->cdev,
                                             QED_CHAIN_USE_TO_CONSUME_PRODUCE,
                                             QED_CHAIN_MODE_NEXT_PTR,
+                                            QED_CHAIN_CNT_TYPE_U16,
                                             RX_RING_SIZE,
                                             sizeof(struct eth_rx_bd),
                                             &rxq->rx_bd_ring);
@@ -2834,6 +2824,7 @@ static int qede_alloc_mem_rxq(struct qede_dev *edev,
         rc = edev->ops->common->chain_alloc(edev->cdev,
                                             QED_CHAIN_USE_TO_CONSUME,
                                             QED_CHAIN_MODE_PBL,
+                                            QED_CHAIN_CNT_TYPE_U16,
                                             RX_RING_SIZE,
                                             sizeof(union eth_rx_cqe),
                                             &rxq->rx_comp_ring);
@@ -2885,9 +2876,9 @@ static int qede_alloc_mem_txq(struct qede_dev *edev,
         rc = edev->ops->common->chain_alloc(edev->cdev,
                                             QED_CHAIN_USE_TO_CONSUME_PRODUCE,
                                             QED_CHAIN_MODE_PBL,
+                                            QED_CHAIN_CNT_TYPE_U16,
                                             NUM_TX_BDS_MAX,
-                                            sizeof(*p_virt),
-                                            &txq->tx_pbl);
+                                            sizeof(*p_virt), &txq->tx_pbl);
         if (rc)
                 goto err;
 
@@ -3578,12 +3569,8 @@ static int qede_open(struct net_device *ndev)
         if (rc)
                 return rc;
 
-#ifdef CONFIG_QEDE_VXLAN
-        vxlan_get_rx_port(ndev);
-#endif
-#ifdef CONFIG_QEDE_GENEVE
-        geneve_get_rx_port(ndev);
-#endif
+        udp_tunnel_get_rx_info(ndev);
+
         return 0;
 }
 
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic.h b/drivers/net/ethernet/qlogic/qlcnic/qlcnic.h
index caf6ddb7ea76..fd973f4f16c7 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic.h
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic.h
@@ -1026,10 +1026,8 @@ struct qlcnic_ipaddr {
 #define QLCNIC_HAS_PHYS_PORT_ID         0x40000
 #define QLCNIC_TSS_RSS                  0x80000
 
-#ifdef CONFIG_QLCNIC_VXLAN
 #define QLCNIC_ADD_VXLAN_PORT           0x100000
 #define QLCNIC_DEL_VXLAN_PORT           0x200000
-#endif
 
 #define QLCNIC_VLAN_FILTERING           0x800000
 
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_hw.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_hw.c
index f9640d5ce6ba..bdbcd2b088a0 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_hw.c
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_hw.c
@@ -2159,7 +2159,6 @@ int qlcnic_83xx_get_mac_address(struct qlcnic_adapter *adapter, u8 *mac,
         struct qlcnic_cmd_args cmd;
         u32 mac_low, mac_high;
 
-        function = 0;
         err = qlcnic_alloc_mbx_args(&cmd, adapter, QLCNIC_CMD_MAC_ADDRESS);
         if (err)
                 return err;
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_init.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_init.c
index bf892160dd5f..a496390b8632 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_init.c
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_init.c
@@ -1020,7 +1020,6 @@ static int qlcnic_83xx_idc_check_state_validity(struct qlcnic_adapter *adapter,
         return 0;
 }
 
-#ifdef CONFIG_QLCNIC_VXLAN
 #define QLC_83XX_ENCAP_TYPE_VXLAN       BIT_1
 #define QLC_83XX_MATCH_ENCAP_ID         BIT_2
 #define QLC_83XX_SET_VXLAN_UDP_DPORT    BIT_3
@@ -1089,14 +1088,12 @@ static int qlcnic_set_vxlan_parsing(struct qlcnic_adapter *adapter,
 
         return ret;
 }
-#endif
 
 static void qlcnic_83xx_periodic_tasks(struct qlcnic_adapter *adapter)
 {
         if (adapter->fhash.fnum)
                 qlcnic_prune_lb_filters(adapter);
 
-#ifdef CONFIG_QLCNIC_VXLAN
         if (adapter->flags & QLCNIC_ADD_VXLAN_PORT) {
                 if (qlcnic_set_vxlan_port(adapter))
                         return;
@@ -1112,7 +1109,6 @@ static void qlcnic_83xx_periodic_tasks(struct qlcnic_adapter *adapter)
                 adapter->ahw->vxlan_port = 0;
                 adapter->flags &= ~QLCNIC_DEL_VXLAN_PORT;
         }
-#endif
 }
 
 /**
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c
index 1c29105b6c36..3ebef27e0964 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c
@@ -16,9 +16,7 @@
 #include <linux/aer.h>
 #include <linux/log2.h>
 #include <linux/pci.h>
-#ifdef CONFIG_QLCNIC_VXLAN
 #include <net/vxlan.h>
-#endif
 
 #include "qlcnic.h"
 #include "qlcnic_sriov.h"
@@ -474,13 +472,15 @@ static int qlcnic_get_phys_port_id(struct net_device *netdev,
         return 0;
 }
 
-#ifdef CONFIG_QLCNIC_VXLAN
 static void qlcnic_add_vxlan_port(struct net_device *netdev,
-                                  sa_family_t sa_family, __be16 port)
+                                  struct udp_tunnel_info *ti)
 {
         struct qlcnic_adapter *adapter = netdev_priv(netdev);
         struct qlcnic_hardware_context *ahw = adapter->ahw;
 
+        if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
+                return;
+
         /* Adapter supports only one VXLAN port. Use very first port
          * for enabling offload
          */
@@ -488,23 +488,26 @@ static void qlcnic_add_vxlan_port(struct net_device *netdev,
                 return;
         if (!ahw->vxlan_port_count) {
                 ahw->vxlan_port_count = 1;
-                ahw->vxlan_port = ntohs(port);
+                ahw->vxlan_port = ntohs(ti->port);
                 adapter->flags |= QLCNIC_ADD_VXLAN_PORT;
                 return;
         }
-        if (ahw->vxlan_port == ntohs(port))
+        if (ahw->vxlan_port == ntohs(ti->port))
                 ahw->vxlan_port_count++;
 
 }
 
 static void qlcnic_del_vxlan_port(struct net_device *netdev,
-                                  sa_family_t sa_family, __be16 port)
+                                  struct udp_tunnel_info *ti)
 {
         struct qlcnic_adapter *adapter = netdev_priv(netdev);
         struct qlcnic_hardware_context *ahw = adapter->ahw;
 
+        if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
+                return;
+
         if (!qlcnic_encap_rx_offload(adapter) || !ahw->vxlan_port_count ||
-            (ahw->vxlan_port != ntohs(port)))
+            (ahw->vxlan_port != ntohs(ti->port)))
                 return;
 
         ahw->vxlan_port_count--;
@@ -519,7 +522,6 @@ static netdev_features_t qlcnic_features_check(struct sk_buff *skb,
         features = vlan_features_check(skb, features);
         return vxlan_features_check(skb, features);
 }
-#endif
 
 static const struct net_device_ops qlcnic_netdev_ops = {
         .ndo_open          = qlcnic_open,
@@ -539,11 +541,9 @@ static const struct net_device_ops qlcnic_netdev_ops = {
         .ndo_fdb_del            = qlcnic_fdb_del,
         .ndo_fdb_dump           = qlcnic_fdb_dump,
         .ndo_get_phys_port_id   = qlcnic_get_phys_port_id,
-#ifdef CONFIG_QLCNIC_VXLAN
-        .ndo_add_vxlan_port     = qlcnic_add_vxlan_port,
-        .ndo_del_vxlan_port     = qlcnic_del_vxlan_port,
+        .ndo_udp_tunnel_add     = qlcnic_add_vxlan_port,
+        .ndo_udp_tunnel_del     = qlcnic_del_vxlan_port,
         .ndo_features_check     = qlcnic_features_check,
-#endif
 #ifdef CONFIG_NET_POLL_CONTROLLER
         .ndo_poll_controller = qlcnic_poll_controller,
 #endif
@@ -2015,10 +2015,8 @@ qlcnic_attach(struct qlcnic_adapter *adapter)
 
         qlcnic_create_sysfs_entries(adapter);
 
-#ifdef CONFIG_QLCNIC_VXLAN
         if (qlcnic_encap_rx_offload(adapter))
-                vxlan_get_rx_port(netdev);
-#endif
+                udp_tunnel_get_rx_info(netdev);
 
         adapter->is_up = QLCNIC_ADAPTER_UP_MAGIC;
         return 0;
diff --git a/drivers/net/ethernet/rdc/r6040.c b/drivers/net/ethernet/rdc/r6040.c
index 6b541e57c96a..7a7a395d0512 100644
--- a/drivers/net/ethernet/rdc/r6040.c
+++ b/drivers/net/ethernet/rdc/r6040.c
@@ -200,7 +200,6 @@ struct r6040_private {
         struct mii_bus *mii_bus;
         struct napi_struct napi;
         void __iomem *base;
-        struct phy_device *phydev;
         int old_link;
         int old_duplex;
 };
@@ -474,7 +473,7 @@ static void r6040_down(struct net_device *dev)
         iowrite16(adrp[1], ioaddr + MID_0M);
         iowrite16(adrp[2], ioaddr + MID_0H);
 
-        phy_stop(lp->phydev);
+        phy_stop(dev->phydev);
 }
 
 static int r6040_close(struct net_device *dev)
@@ -515,12 +514,10 @@ static int r6040_close(struct net_device *dev)
 
 static int r6040_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
 {
-        struct r6040_private *lp = netdev_priv(dev);
-
-        if (!lp->phydev)
+        if (!dev->phydev)
                 return -EINVAL;
 
-        return phy_mii_ioctl(lp->phydev, rq, cmd);
+        return phy_mii_ioctl(dev->phydev, rq, cmd);
 }
 
 static int r6040_rx(struct net_device *dev, int limit)
@@ -732,7 +729,7 @@ static int r6040_up(struct net_device *dev)
         /* Initialize all MAC registers */
         r6040_init_mac_regs(dev);
 
-        phy_start(lp->phydev);
+        phy_start(dev->phydev);
 
         return 0;
 }
@@ -957,26 +954,12 @@ static void netdev_get_drvinfo(struct net_device *dev,
         strlcpy(info->bus_info, pci_name(rp->pdev), sizeof(info->bus_info));
 }
 
-static int netdev_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
-{
-        struct r6040_private *rp = netdev_priv(dev);
-
-        return  phy_ethtool_gset(rp->phydev, cmd);
-}
-
-static int netdev_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
-{
-        struct r6040_private *rp = netdev_priv(dev);
-
-        return phy_ethtool_sset(rp->phydev, cmd);
-}
-
 static const struct ethtool_ops netdev_ethtool_ops = {
         .get_drvinfo            = netdev_get_drvinfo,
-        .get_settings           = netdev_get_settings,
-        .set_settings           = netdev_set_settings,
         .get_link               = ethtool_op_get_link,
         .get_ts_info            = ethtool_op_get_ts_info,
+        .get_link_ksettings     = phy_ethtool_get_link_ksettings,
+        .set_link_ksettings     = phy_ethtool_set_link_ksettings,
 };
 
 static const struct net_device_ops r6040_netdev_ops = {
@@ -998,7 +981,7 @@ static const struct net_device_ops r6040_netdev_ops = {
 static void r6040_adjust_link(struct net_device *dev)
 {
         struct r6040_private *lp = netdev_priv(dev);
-        struct phy_device *phydev = lp->phydev;
+        struct phy_device *phydev = dev->phydev;
         int status_changed = 0;
         void __iomem *ioaddr = lp->base;
 
@@ -1057,7 +1040,6 @@ static int r6040_mii_probe(struct net_device *dev)
                                 | SUPPORTED_TP);
 
         phydev->advertising = phydev->supported;
-        lp->phydev = phydev;
         lp->old_link = 0;
         lp->old_duplex = -1;
 
diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c
index 867caf6e7a5a..5349284203ff 100644
--- a/drivers/net/ethernet/renesas/ravb_main.c
+++ b/drivers/net/ethernet/renesas/ravb_main.c
@@ -362,8 +362,6 @@ static void ravb_emac_init(struct net_device *ndev)
         ravb_write(ndev,
                    (ndev->dev_addr[4] << 8)  | (ndev->dev_addr[5]), MALR);
 
-        ravb_write(ndev, 1, MPR);
-
         /* E-MAC status register clear */
         ravb_write(ndev, ECSR_ICD | ECSR_MPD, ECSR);
 
@@ -402,7 +400,8 @@ static int ravb_dmac_init(struct net_device *ndev)
 #endif
 
         /* Set AVB RX */
-        ravb_write(ndev, RCR_EFFS | RCR_ENCF | RCR_ETS0 | 0x18000000, RCR);
+        ravb_write(ndev,
+                   RCR_EFFS | RCR_ENCF | RCR_ETS0 | RCR_ESF | 0x18000000, RCR);
 
         /* Set FIFO size */
         ravb_write(ndev, TGC_TQP_AVBMODE1 | 0x00222200, TGC);
@@ -2111,8 +2110,7 @@ static int ravb_runtime_nop(struct device *dev)
 }
 
 static const struct dev_pm_ops ravb_dev_pm_ops = {
-        .runtime_suspend = ravb_runtime_nop,
-        .runtime_resume = ravb_runtime_nop,
+        SET_RUNTIME_PM_OPS(ravb_runtime_nop, ravb_runtime_nop, NULL)
 };
 
 #define RAVB_PM_OPS (&ravb_dev_pm_ops)
diff --git a/drivers/net/ethernet/samsung/sxgbe/sxgbe_common.h b/drivers/net/ethernet/samsung/sxgbe/sxgbe_common.h
index 45019649bbbd..5cb51b609f02 100644
--- a/drivers/net/ethernet/samsung/sxgbe/sxgbe_common.h
+++ b/drivers/net/ethernet/samsung/sxgbe/sxgbe_common.h
@@ -475,7 +475,6 @@ struct sxgbe_priv_data {
         int rxcsum_insertion;
         spinlock_t stats_lock;  /* lock for tx/rx statatics */
 
-        struct phy_device *phydev;
         int oldlink;
         int speed;
         int oldduplex;
diff --git a/drivers/net/ethernet/samsung/sxgbe/sxgbe_ethtool.c b/drivers/net/ethernet/samsung/sxgbe/sxgbe_ethtool.c
index c0981ae45874..542b67d436df 100644
--- a/drivers/net/ethernet/samsung/sxgbe/sxgbe_ethtool.c
+++ b/drivers/net/ethernet/samsung/sxgbe/sxgbe_ethtool.c
@@ -147,7 +147,7 @@ static int sxgbe_get_eee(struct net_device *dev,
         edata->eee_active = priv->eee_active;
         edata->tx_lpi_timer = priv->tx_lpi_timer;
 
-        return phy_ethtool_get_eee(priv->phydev, edata);
+        return phy_ethtool_get_eee(dev->phydev, edata);
 }
 
 static int sxgbe_set_eee(struct net_device *dev,
@@ -172,7 +172,7 @@ static int sxgbe_set_eee(struct net_device *dev,
                 priv->tx_lpi_timer = edata->tx_lpi_timer;
         }
 
-        return phy_ethtool_set_eee(priv->phydev, edata);
+        return phy_ethtool_set_eee(dev->phydev, edata);
 }
 
 static void sxgbe_getdrvinfo(struct net_device *dev,
@@ -182,27 +182,6 @@ static void sxgbe_getdrvinfo(struct net_device *dev,
         strlcpy(info->version, DRV_VERSION, sizeof(info->version));
 }
 
-static int sxgbe_getsettings(struct net_device *dev,
-                             struct ethtool_cmd *cmd)
-{
-        struct sxgbe_priv_data *priv = netdev_priv(dev);
-
-        if (priv->phydev)
-                return phy_ethtool_gset(priv->phydev, cmd);
-
-        return -EOPNOTSUPP;
-}
-
-static int sxgbe_setsettings(struct net_device *dev, struct ethtool_cmd *cmd)
-{
-        struct sxgbe_priv_data *priv = netdev_priv(dev);
-
-        if (priv->phydev)
-                return phy_ethtool_sset(priv->phydev, cmd);
-
-        return -EOPNOTSUPP;
-}
-
 static u32 sxgbe_getmsglevel(struct net_device *dev)
 {
         struct sxgbe_priv_data *priv = netdev_priv(dev);
@@ -255,7 +234,7 @@ static void sxgbe_get_ethtool_stats(struct net_device *dev,
         char *p;
 
         if (priv->eee_enabled) {
-                int val = phy_get_eee_err(priv->phydev);
+                int val = phy_get_eee_err(dev->phydev);
 
                 if (val)
                         priv->xstats.eee_wakeup_error_n = val;
@@ -499,8 +478,6 @@ static int sxgbe_get_regs_len(struct net_device *dev)
 
 static const struct ethtool_ops sxgbe_ethtool_ops = {
         .get_drvinfo = sxgbe_getdrvinfo,
-        .get_settings = sxgbe_getsettings,
-        .set_settings = sxgbe_setsettings,
         .get_msglevel = sxgbe_getmsglevel,
         .set_msglevel = sxgbe_setmsglevel,
         .get_link = ethtool_op_get_link,
@@ -516,6 +493,8 @@ static const struct ethtool_ops sxgbe_ethtool_ops = {
         .get_regs_len = sxgbe_get_regs_len,
         .get_eee = sxgbe_get_eee,
         .set_eee = sxgbe_set_eee,
+        .get_link_ksettings = phy_ethtool_get_link_ksettings,
+        .set_link_ksettings = phy_ethtool_set_link_ksettings,
 };
 
 void sxgbe_set_ethtool_ops(struct net_device *netdev)
diff --git a/drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c b/drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c
index 413ea14ab91f..ea44a2456ce1 100644
--- a/drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c
+++ b/drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c
@@ -124,12 +124,13 @@ static void sxgbe_eee_ctrl_timer(unsigned long arg)
  */
 bool sxgbe_eee_init(struct sxgbe_priv_data * const priv)
 {
+        struct net_device *ndev = priv->dev;
         bool ret = false;
 
         /* MAC core supports the EEE feature. */
         if (priv->hw_cap.eee) {
                 /* Check if the PHY supports EEE */
-                if (phy_init_eee(priv->phydev, 1))
+                if (phy_init_eee(ndev->phydev, 1))
                         return false;
 
                 priv->eee_active = 1;
@@ -152,12 +153,14 @@ bool sxgbe_eee_init(struct sxgbe_priv_data * const priv)
 
 static void sxgbe_eee_adjust(const struct sxgbe_priv_data *priv)
 {
+        struct net_device *ndev = priv->dev;
+
         /* When the EEE has been already initialised we have to
          * modify the PLS bit in the LPI ctrl & status reg according
          * to the PHY link status. For this reason.
          */
         if (priv->eee_enabled)
-                priv->hw->mac->set_eee_pls(priv->ioaddr, priv->phydev->link);
+                priv->hw->mac->set_eee_pls(priv->ioaddr, ndev->phydev->link);
 }
 
 /**
@@ -203,7 +206,7 @@ static inline u32 sxgbe_tx_avail(struct sxgbe_tx_queue *queue, int tx_qsize)
 static void sxgbe_adjust_link(struct net_device *dev)
 {
         struct sxgbe_priv_data *priv = netdev_priv(dev);
-        struct phy_device *phydev = priv->phydev;
+        struct phy_device *phydev = dev->phydev;
         u8 new_state = 0;
         u8 speed = 0xff;
 
@@ -306,9 +309,6 @@ static int sxgbe_init_phy(struct net_device *ndev)
         netdev_dbg(ndev, "%s: attached to PHY (UID 0x%x) Link = %d\n",
                    __func__, phydev->phy_id, phydev->link);
 
-        /* save phy device in private structure */
-        priv->phydev = phydev;
-
         return 0;
 }
 
@@ -1173,8 +1173,8 @@ static int sxgbe_open(struct net_device *dev)
         priv->hw->dma->start_tx(priv->ioaddr, SXGBE_TX_QUEUES);
         priv->hw->dma->start_rx(priv->ioaddr, SXGBE_RX_QUEUES);
 
-        if (priv->phydev)
-                phy_start(priv->phydev);
+        if (dev->phydev)
+                phy_start(dev->phydev);
 
         /* initialise TX coalesce parameters */
         sxgbe_tx_init_coalesce(priv);
@@ -1194,8 +1194,8 @@ static int sxgbe_open(struct net_device *dev)
 
 init_error:
         free_dma_desc_resources(priv);
-        if (priv->phydev)
-                phy_disconnect(priv->phydev);
+        if (dev->phydev)
+                phy_disconnect(dev->phydev);
 phy_error:
         clk_disable_unprepare(priv->sxgbe_clk);
 
@@ -1216,10 +1216,9 @@ static int sxgbe_release(struct net_device *dev)
                 del_timer_sync(&priv->eee_ctrl_timer);
 
         /* Stop and disconnect the PHY */
-        if (priv->phydev) {
-                phy_stop(priv->phydev);
-                phy_disconnect(priv->phydev);
-                priv->phydev = NULL;
+        if (dev->phydev) {
+                phy_stop(dev->phydev);
+                phy_disconnect(dev->phydev);
         }
 
         netif_tx_stop_all_queues(dev);
@@ -1969,7 +1968,6 @@ static void sxgbe_poll_controller(struct net_device *dev)
  */
 static int sxgbe_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
 {
-        struct sxgbe_priv_data *priv = netdev_priv(dev);
         int ret = -EOPNOTSUPP;
 
         if (!netif_running(dev))
@@ -1979,9 +1977,9 @@ static int sxgbe_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
         case SIOCGMIIPHY:
         case SIOCGMIIREG:
         case SIOCSMIIREG:
-                if (!priv->phydev)
+                if (!dev->phydev)
                         return -EINVAL;
-                ret = phy_mii_ioctl(priv->phydev, rq, cmd);
+                ret = phy_mii_ioctl(dev->phydev, rq, cmd);
                 break;
         default:
                 break;
diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c
index 1f309127457d..f658fee74f18 100644
--- a/drivers/net/ethernet/sfc/ef10.c
+++ b/drivers/net/ethernet/sfc/ef10.c
@@ -50,14 +50,34 @@ enum {
 #define HUNT_FILTER_TBL_ROWS 8192
 
 #define EFX_EF10_FILTER_ID_INVALID 0xffff
+
+#define EFX_EF10_FILTER_DEV_UC_MAX      32
+#define EFX_EF10_FILTER_DEV_MC_MAX      256
+
+/* VLAN list entry */
+struct efx_ef10_vlan {
+        struct list_head list;
+        u16 vid;
+};
+
+/* Per-VLAN filters information */
+struct efx_ef10_filter_vlan {
+        struct list_head list;
+        u16 vid;
+        u16 uc[EFX_EF10_FILTER_DEV_UC_MAX];
+        u16 mc[EFX_EF10_FILTER_DEV_MC_MAX];
+        u16 ucdef;
+        u16 bcast;
+        u16 mcdef;
+};
+
 struct efx_ef10_dev_addr {
         u8 addr[ETH_ALEN];
-        u16 id;
 };
 
 struct efx_ef10_filter_table {
-/* The RX match field masks supported by this fw & hw, in order of priority */
-        enum efx_filter_match_flags rx_match_flags[
+/* The MCDI match masks supported by this fw & hw, in order of priority */
+        u32 rx_match_mcdi_flags[
                 MC_CMD_GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES_MAXNUM];
         unsigned int rx_match_count;
 
@@ -73,16 +93,16 @@ struct efx_ef10_filter_table {
         } *entry;
         wait_queue_head_t waitq;
 /* Shadow of net_device address lists, guarded by mac_lock */
-#define EFX_EF10_FILTER_DEV_UC_MAX      32
-#define EFX_EF10_FILTER_DEV_MC_MAX      256
         struct efx_ef10_dev_addr dev_uc_list[EFX_EF10_FILTER_DEV_UC_MAX];
         struct efx_ef10_dev_addr dev_mc_list[EFX_EF10_FILTER_DEV_MC_MAX];
         int dev_uc_count;
         int dev_mc_count;
-/* Indices (like efx_ef10_dev_addr.id) for promisc/allmulti filters */
-        u16 ucdef_id;
-        u16 bcast_id;
-        u16 mcdef_id;
+        bool uc_promisc;
+        bool mc_promisc;
+/* Whether in multicast promiscuous mode when last changed */
+        bool mc_promisc_last;
+        bool vlan_filter;
+        struct list_head vlan_list;
 };
 
 /* An arbitrary search limit for the software hash table */
@@ -90,6 +110,10 @@ struct efx_ef10_filter_table {
 
 static void efx_ef10_rx_free_indir_table(struct efx_nic *efx);
 static void efx_ef10_filter_table_remove(struct efx_nic *efx);
+static int efx_ef10_filter_add_vlan(struct efx_nic *efx, u16 vid);
+static void efx_ef10_filter_del_vlan_internal(struct efx_nic *efx,
+                                              struct efx_ef10_filter_vlan *vlan);
+static void efx_ef10_filter_del_vlan(struct efx_nic *efx, u16 vid);
 
 static int efx_ef10_get_warm_boot_count(struct efx_nic *efx)
 {
@@ -275,6 +299,131 @@ static ssize_t efx_ef10_show_primary_flag(struct device *dev,
                        ? 1 : 0);
 }
 
+static struct efx_ef10_vlan *efx_ef10_find_vlan(struct efx_nic *efx, u16 vid)
+{
+        struct efx_ef10_nic_data *nic_data = efx->nic_data;
+        struct efx_ef10_vlan *vlan;
+
+        WARN_ON(!mutex_is_locked(&nic_data->vlan_lock));
+
+        list_for_each_entry(vlan, &nic_data->vlan_list, list) {
+                if (vlan->vid == vid)
+                        return vlan;
+        }
+
+        return NULL;
+}
+
+static int efx_ef10_add_vlan(struct efx_nic *efx, u16 vid)
+{
+        struct efx_ef10_nic_data *nic_data = efx->nic_data;
+        struct efx_ef10_vlan *vlan;
+        int rc;
+
+        mutex_lock(&nic_data->vlan_lock);
+
+        vlan = efx_ef10_find_vlan(efx, vid);
+        if (vlan) {
+                /* We add VID 0 on init. 8021q adds it on module init
+                 * for all interfaces with VLAN filtring feature.
+                 */
+                if (vid == 0)
+                        goto done_unlock;
+                netif_warn(efx, drv, efx->net_dev,
+                           "VLAN %u already added\n", vid);
+                rc = -EALREADY;
+                goto fail_exist;
+        }
+
+        rc = -ENOMEM;
+        vlan = kzalloc(sizeof(*vlan), GFP_KERNEL);
+        if (!vlan)
+                goto fail_alloc;
+
+        vlan->vid = vid;
+
+        list_add_tail(&vlan->list, &nic_data->vlan_list);
+
+        if (efx->filter_state) {
+                mutex_lock(&efx->mac_lock);
+                down_write(&efx->filter_sem);
+                rc = efx_ef10_filter_add_vlan(efx, vlan->vid);
+                up_write(&efx->filter_sem);
+                mutex_unlock(&efx->mac_lock);
+                if (rc)
+                        goto fail_filter_add_vlan;
+        }
+
+done_unlock:
+        mutex_unlock(&nic_data->vlan_lock);
+        return 0;
+
+fail_filter_add_vlan:
+        list_del(&vlan->list);
+        kfree(vlan);
+fail_alloc:
+fail_exist:
+        mutex_unlock(&nic_data->vlan_lock);
+        return rc;
+}
+
+static void efx_ef10_del_vlan_internal(struct efx_nic *efx,
+                                       struct efx_ef10_vlan *vlan)
+{
+        struct efx_ef10_nic_data *nic_data = efx->nic_data;
+
+        WARN_ON(!mutex_is_locked(&nic_data->vlan_lock));
+
+        if (efx->filter_state) {
+                down_write(&efx->filter_sem);
+                efx_ef10_filter_del_vlan(efx, vlan->vid);
+                up_write(&efx->filter_sem);
+        }
+
+        list_del(&vlan->list);
+        kfree(vlan);
+}
+
+static int efx_ef10_del_vlan(struct efx_nic *efx, u16 vid)
+{
+        struct efx_ef10_nic_data *nic_data = efx->nic_data;
+        struct efx_ef10_vlan *vlan;
+        int rc = 0;
+
+        /* 8021q removes VID 0 on module unload for all interfaces
+         * with VLAN filtering feature. We need to keep it to receive
+         * untagged traffic.
+         */
+        if (vid == 0)
+                return 0;
+
+        mutex_lock(&nic_data->vlan_lock);
+
+        vlan = efx_ef10_find_vlan(efx, vid);
+        if (!vlan) {
+                netif_err(efx, drv, efx->net_dev,
+                          "VLAN %u to be deleted not found\n", vid);
+                rc = -ENOENT;
+        } else {
+                efx_ef10_del_vlan_internal(efx, vlan);
+        }
+
+        mutex_unlock(&nic_data->vlan_lock);
+
+        return rc;
+}
+
+static void efx_ef10_cleanup_vlans(struct efx_nic *efx)
+{
+        struct efx_ef10_nic_data *nic_data = efx->nic_data;
+        struct efx_ef10_vlan *vlan, *next_vlan;
+
+        mutex_lock(&nic_data->vlan_lock);
+        list_for_each_entry_safe(vlan, next_vlan, &nic_data->vlan_list, list)
+                efx_ef10_del_vlan_internal(efx, vlan);
+        mutex_unlock(&nic_data->vlan_lock);
+}
+
 static DEVICE_ATTR(link_control_flag, 0444, efx_ef10_show_link_control_flag,
                    NULL);
 static DEVICE_ATTR(primary_flag, 0444, efx_ef10_show_primary_flag, NULL);
@@ -421,8 +570,30 @@ static int efx_ef10_probe(struct efx_nic *efx)
 #endif
                 ether_addr_copy(nic_data->port_id, efx->net_dev->perm_addr);
 
+        INIT_LIST_HEAD(&nic_data->vlan_list);
+        mutex_init(&nic_data->vlan_lock);
+
+        /* Add unspecified VID to support VLAN filtering being disabled */
+        rc = efx_ef10_add_vlan(efx, EFX_FILTER_VID_UNSPEC);
+        if (rc)
+                goto fail_add_vid_unspec;
+
+        /* If VLAN filtering is enabled, we need VID 0 to get untagged
+         * traffic.  It is added automatically if 8021q module is loaded,
+         * but we can't rely on it since module may be not loaded.
+         */
+        rc = efx_ef10_add_vlan(efx, 0);
+        if (rc)
+                goto fail_add_vid_0;
+
         return 0;
 
+fail_add_vid_0:
+        efx_ef10_cleanup_vlans(efx);
+fail_add_vid_unspec:
+        mutex_destroy(&nic_data->vlan_lock);
+        efx_ptp_remove(efx);
+        efx_mcdi_mon_remove(efx);
 fail5:
         device_remove_file(&efx->pci_dev->dev, &dev_attr_primary_flag);
 fail4:
@@ -676,6 +847,9 @@ static void efx_ef10_remove(struct efx_nic *efx)
         }
 #endif
 
+        efx_ef10_cleanup_vlans(efx);
+        mutex_destroy(&nic_data->vlan_lock);
+
         efx_ptp_remove(efx);
 
         efx_mcdi_mon_remove(efx);
@@ -704,6 +878,45 @@ static int efx_ef10_probe_pf(struct efx_nic *efx)
         return efx_ef10_probe(efx);
 }
 
+int efx_ef10_vadaptor_query(struct efx_nic *efx, unsigned int port_id,
+                            u32 *port_flags, u32 *vadaptor_flags,
+                            unsigned int *vlan_tags)
+{
+        struct efx_ef10_nic_data *nic_data = efx->nic_data;
+        MCDI_DECLARE_BUF(inbuf, MC_CMD_VADAPTOR_QUERY_IN_LEN);
+        MCDI_DECLARE_BUF(outbuf, MC_CMD_VADAPTOR_QUERY_OUT_LEN);
+        size_t outlen;
+        int rc;
+
+        if (nic_data->datapath_caps &
+            (1 << MC_CMD_GET_CAPABILITIES_OUT_VADAPTOR_QUERY_LBN)) {
+                MCDI_SET_DWORD(inbuf, VADAPTOR_QUERY_IN_UPSTREAM_PORT_ID,
+                               port_id);
+
+                rc = efx_mcdi_rpc(efx, MC_CMD_VADAPTOR_QUERY, inbuf, sizeof(inbuf),
+                                  outbuf, sizeof(outbuf), &outlen);
+                if (rc)
+                        return rc;
+
+                if (outlen < sizeof(outbuf)) {
+                        rc = -EIO;
+                        return rc;
+                }
+        }
+
+        if (port_flags)
+                *port_flags = MCDI_DWORD(outbuf, VADAPTOR_QUERY_OUT_PORT_FLAGS);
+        if (vadaptor_flags)
+                *vadaptor_flags =
+                        MCDI_DWORD(outbuf, VADAPTOR_QUERY_OUT_VADAPTOR_FLAGS);
+        if (vlan_tags)
+                *vlan_tags =
+                        MCDI_DWORD(outbuf,
+                                   VADAPTOR_QUERY_OUT_NUM_AVAILABLE_VLAN_TAGS);
+
+        return 0;
+}
+
 int efx_ef10_vadaptor_alloc(struct efx_nic *efx, unsigned int port_id)
 {
         MCDI_DECLARE_BUF(inbuf, MC_CMD_VADAPTOR_ALLOC_IN_LEN);
@@ -3040,15 +3253,55 @@ static int efx_ef10_filter_push(struct efx_nic *efx,
         return rc;
 }
 
-static int efx_ef10_filter_rx_match_pri(struct efx_ef10_filter_table *table,
-                                        enum efx_filter_match_flags match_flags)
+static u32 efx_ef10_filter_mcdi_flags_from_spec(const struct efx_filter_spec *spec)
 {
+        unsigned int match_flags = spec->match_flags;
+        u32 mcdi_flags = 0;
+
+        if (match_flags & EFX_FILTER_MATCH_LOC_MAC_IG) {
+                match_flags &= ~EFX_FILTER_MATCH_LOC_MAC_IG;
+                mcdi_flags |=
+                        is_multicast_ether_addr(spec->loc_mac) ?
+                        (1 << MC_CMD_FILTER_OP_IN_MATCH_UNKNOWN_MCAST_DST_LBN) :
+                        (1 << MC_CMD_FILTER_OP_IN_MATCH_UNKNOWN_UCAST_DST_LBN);
+        }
+
+#define MAP_FILTER_TO_MCDI_FLAG(gen_flag, mcdi_field) {                 \
+                unsigned int old_match_flags = match_flags;             \
+                match_flags &= ~EFX_FILTER_MATCH_ ## gen_flag;          \
+                if (match_flags != old_match_flags)                     \
+                        mcdi_flags |=                                   \
+                                (1 << MC_CMD_FILTER_OP_IN_MATCH_ ##     \
+                                 mcdi_field ## _LBN);                   \
+        }
+        MAP_FILTER_TO_MCDI_FLAG(REM_HOST, SRC_IP);
+        MAP_FILTER_TO_MCDI_FLAG(LOC_HOST, DST_IP);
+        MAP_FILTER_TO_MCDI_FLAG(REM_MAC, SRC_MAC);
+        MAP_FILTER_TO_MCDI_FLAG(REM_PORT, SRC_PORT);
+        MAP_FILTER_TO_MCDI_FLAG(LOC_MAC, DST_MAC);
+        MAP_FILTER_TO_MCDI_FLAG(LOC_PORT, DST_PORT);
+        MAP_FILTER_TO_MCDI_FLAG(ETHER_TYPE, ETHER_TYPE);
+        MAP_FILTER_TO_MCDI_FLAG(INNER_VID, INNER_VLAN);
+        MAP_FILTER_TO_MCDI_FLAG(OUTER_VID, OUTER_VLAN);
+        MAP_FILTER_TO_MCDI_FLAG(IP_PROTO, IP_PROTO);
+#undef MAP_FILTER_TO_MCDI_FLAG
+
+        /* Did we map them all? */
+        WARN_ON_ONCE(match_flags);
+
+        return mcdi_flags;
+}
+
+static int efx_ef10_filter_pri(struct efx_ef10_filter_table *table,
+                               const struct efx_filter_spec *spec)
+{
+        u32 mcdi_flags = efx_ef10_filter_mcdi_flags_from_spec(spec);
         unsigned int match_pri;
 
         for (match_pri = 0;
              match_pri < table->rx_match_count;
              match_pri++)
-                if (table->rx_match_flags[match_pri] == match_flags)
+                if (table->rx_match_mcdi_flags[match_pri] == mcdi_flags)
                         return match_pri;
 
         return -EPROTONOSUPPORT;
@@ -3074,7 +3327,7 @@ static s32 efx_ef10_filter_insert(struct efx_nic *efx,
             EFX_FILTER_FLAG_RX)
                 return -EINVAL;
 
-        rc = efx_ef10_filter_rx_match_pri(table, spec->match_flags);
+        rc = efx_ef10_filter_pri(table, spec);
         if (rc < 0)
                 return rc;
         match_pri = rc;
@@ -3313,7 +3566,7 @@ static int efx_ef10_filter_remove_internal(struct efx_nic *efx,
         spec = efx_ef10_filter_entry_spec(table, filter_idx);
         if (!spec ||
             (!by_index &&
-             efx_ef10_filter_rx_match_pri(table, spec->match_flags) !=
+             efx_ef10_filter_pri(table, spec) !=
              filter_id / HUNT_FILTER_TBL_ROWS)) {
                 rc = -ENOENT;
                 goto out_unlock;
@@ -3394,12 +3647,13 @@ static u32 efx_ef10_filter_get_unsafe_id(struct efx_nic *efx, u32 filter_id)
         return filter_id % HUNT_FILTER_TBL_ROWS;
 }
 
-static int efx_ef10_filter_remove_unsafe(struct efx_nic *efx,
-                                         enum efx_filter_priority priority,
-                                         u32 filter_id)
+static void efx_ef10_filter_remove_unsafe(struct efx_nic *efx,
+                                          enum efx_filter_priority priority,
+                                          u32 filter_id)
 {
-        return efx_ef10_filter_remove_internal(efx, 1U << priority,
-                                               filter_id, true);
+        if (filter_id == EFX_EF10_FILTER_ID_INVALID)
+                return;
+        efx_ef10_filter_remove_internal(efx, 1U << priority, filter_id, true);
 }
 
 static int efx_ef10_filter_get_safe(struct efx_nic *efx,
@@ -3414,7 +3668,7 @@ static int efx_ef10_filter_get_safe(struct efx_nic *efx,
         spin_lock_bh(&efx->filter_lock);
         saved_spec = efx_ef10_filter_entry_spec(table, filter_idx);
         if (saved_spec && saved_spec->priority == priority &&
-            efx_ef10_filter_rx_match_pri(table, saved_spec->match_flags) ==
+            efx_ef10_filter_pri(table, saved_spec) ==
             filter_id / HUNT_FILTER_TBL_ROWS) {
                 *spec = *saved_spec;
                 rc = 0;
@@ -3487,8 +3741,7 @@ static s32 efx_ef10_filter_get_rx_ids(struct efx_nic *efx,
                                 count = -EMSGSIZE;
                                 break;
                         }
-                        buf[count++] = (efx_ef10_filter_rx_match_pri(
-                                                table, spec->match_flags) *
+                        buf[count++] = (efx_ef10_filter_pri(table, spec) *
                                         HUNT_FILTER_TBL_ROWS +
                                         filter_idx);
                 }
@@ -3724,15 +3977,58 @@ static int efx_ef10_filter_match_flags_from_mcdi(u32 mcdi_flags)
         return match_flags;
 }
 
+static void efx_ef10_filter_cleanup_vlans(struct efx_nic *efx)
+{
+        struct efx_ef10_filter_table *table = efx->filter_state;
+        struct efx_ef10_filter_vlan *vlan, *next_vlan;
+
+        /* See comment in efx_ef10_filter_table_remove() */
+        if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
+                return;
+
+        if (!table)
+                return;
+
+        list_for_each_entry_safe(vlan, next_vlan, &table->vlan_list, list)
+                efx_ef10_filter_del_vlan_internal(efx, vlan);
+}
+
+static bool efx_ef10_filter_match_supported(struct efx_ef10_filter_table *table,
+                                            enum efx_filter_match_flags match_flags)
+{
+        unsigned int match_pri;
+        int mf;
+
+        for (match_pri = 0;
+             match_pri < table->rx_match_count;
+             match_pri++) {
+                mf = efx_ef10_filter_match_flags_from_mcdi(
+                                table->rx_match_mcdi_flags[match_pri]);
+                if (mf == match_flags)
+                        return true;
+        }
+
+        return false;
+}
+
 static int efx_ef10_filter_table_probe(struct efx_nic *efx)
 {
         MCDI_DECLARE_BUF(inbuf, MC_CMD_GET_PARSER_DISP_INFO_IN_LEN);
         MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMAX);
+        struct efx_ef10_nic_data *nic_data = efx->nic_data;
+        struct net_device *net_dev = efx->net_dev;
         unsigned int pd_match_pri, pd_match_count;
         struct efx_ef10_filter_table *table;
+        struct efx_ef10_vlan *vlan;
         size_t outlen;
         int rc;
 
+        if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
+                return -EINVAL;
+
+        if (efx->filter_state) /* already probed */
+                return 0;
+
         table = kzalloc(sizeof(*table), GFP_KERNEL);
         if (!table)
                 return -ENOMEM;
@@ -3765,24 +4061,48 @@ static int efx_ef10_filter_table_probe(struct efx_nic *efx)
                                   "%s: fw flags %#x pri %u supported as driver flags %#x pri %u\n",
                                   __func__, mcdi_flags, pd_match_pri,
                                   rc, table->rx_match_count);
-                        table->rx_match_flags[table->rx_match_count++] = rc;
+                        table->rx_match_mcdi_flags[table->rx_match_count] = mcdi_flags;
+                        table->rx_match_count++;
                 }
         }
 
+        if ((efx_supported_features(efx) & NETIF_F_HW_VLAN_CTAG_FILTER) &&
+            !(efx_ef10_filter_match_supported(table,
+                (EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC)) &&
+              efx_ef10_filter_match_supported(table,
+                (EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC_IG)))) {
+                netif_info(efx, probe, net_dev,
+                           "VLAN filters are not supported in this firmware variant\n");
+                net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
+                efx->fixed_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
+                net_dev->hw_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
+        }
+
         table->entry = vzalloc(HUNT_FILTER_TBL_ROWS * sizeof(*table->entry));
         if (!table->entry) {
                 rc = -ENOMEM;
                 goto fail;
         }
 
-        table->ucdef_id = EFX_EF10_FILTER_ID_INVALID;
-        table->bcast_id = EFX_EF10_FILTER_ID_INVALID;
-        table->mcdef_id = EFX_EF10_FILTER_ID_INVALID;
+        table->mc_promisc_last = false;
+        table->vlan_filter =
+                !!(efx->net_dev->features & NETIF_F_HW_VLAN_CTAG_FILTER);
+        INIT_LIST_HEAD(&table->vlan_list);
 
         efx->filter_state = table;
         init_waitqueue_head(&table->waitq);
+
+        list_for_each_entry(vlan, &nic_data->vlan_list, list) {
+                rc = efx_ef10_filter_add_vlan(efx, vlan->vid);
+                if (rc)
+                        goto fail_add_vlan;
+        }
+
         return 0;
 
+fail_add_vlan:
+        efx_ef10_filter_cleanup_vlans(efx);
+        efx->filter_state = NULL;
 fail:
         kfree(table);
         return rc;
@@ -3843,7 +4163,6 @@ static void efx_ef10_filter_table_restore(struct efx_nic *efx)
                 nic_data->must_restore_filters = false;
 }
 
-/* Caller must hold efx->filter_sem for write */
 static void efx_ef10_filter_table_remove(struct efx_nic *efx)
 {
         struct efx_ef10_filter_table *table = efx->filter_state;
@@ -3852,7 +4171,17 @@ static void efx_ef10_filter_table_remove(struct efx_nic *efx)
         unsigned int filter_idx;
         int rc;
 
+        efx_ef10_filter_cleanup_vlans(efx);
         efx->filter_state = NULL;
+        /* If we were called without locking, then it's not safe to free
+         * the table as others might be using it.  So we just WARN, leak
+         * the memory, and potentially get an inconsistent filter table
+         * state.
+         * This should never actually happen.
+         */
+        if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
+                return;
+
         if (!table)
                 return;
 
@@ -3880,37 +4209,54 @@ static void efx_ef10_filter_table_remove(struct efx_nic *efx)
         kfree(table);
 }
 
-#define EFX_EF10_FILTER_DO_MARK_OLD(id) \
-        if (id != EFX_EF10_FILTER_ID_INVALID) { \
-                filter_idx = efx_ef10_filter_get_unsafe_id(efx, id); \
-                if (!table->entry[filter_idx].spec) \
-                        netif_dbg(efx, drv, efx->net_dev, \
-                                  "%s: marked null spec old %04x:%04x\n", \
-                                  __func__, id, filter_idx); \
-                table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_AUTO_OLD;\
-        }
-static void efx_ef10_filter_mark_old(struct efx_nic *efx)
+static void efx_ef10_filter_mark_one_old(struct efx_nic *efx, uint16_t *id)
 {
         struct efx_ef10_filter_table *table = efx->filter_state;
-        unsigned int filter_idx, i;
+        unsigned int filter_idx;
 
-        if (!table)
-                return;
+        if (*id != EFX_EF10_FILTER_ID_INVALID) {
+                filter_idx = efx_ef10_filter_get_unsafe_id(efx, *id);
+                if (!table->entry[filter_idx].spec)
+                        netif_dbg(efx, drv, efx->net_dev,
+                                  "marked null spec old %04x:%04x\n", *id,
+                                  filter_idx);
+                table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_AUTO_OLD;
+                *id = EFX_EF10_FILTER_ID_INVALID;
+        }
+}
+
+/* Mark old per-VLAN filters that may need to be removed */
+static void _efx_ef10_filter_vlan_mark_old(struct efx_nic *efx,
+                                           struct efx_ef10_filter_vlan *vlan)
+{
+        struct efx_ef10_filter_table *table = efx->filter_state;
+        unsigned int i;
 
-        /* Mark old filters that may need to be removed */
-        spin_lock_bh(&efx->filter_lock);
         for (i = 0; i < table->dev_uc_count; i++)
-                EFX_EF10_FILTER_DO_MARK_OLD(table->dev_uc_list[i].id);
+                efx_ef10_filter_mark_one_old(efx, &vlan->uc[i]);
         for (i = 0; i < table->dev_mc_count; i++)
-                EFX_EF10_FILTER_DO_MARK_OLD(table->dev_mc_list[i].id);
-        EFX_EF10_FILTER_DO_MARK_OLD(table->ucdef_id);
-        EFX_EF10_FILTER_DO_MARK_OLD(table->bcast_id);
-        EFX_EF10_FILTER_DO_MARK_OLD(table->mcdef_id);
+                efx_ef10_filter_mark_one_old(efx, &vlan->mc[i]);
+        efx_ef10_filter_mark_one_old(efx, &vlan->ucdef);
+        efx_ef10_filter_mark_one_old(efx, &vlan->bcast);
+        efx_ef10_filter_mark_one_old(efx, &vlan->mcdef);
+}
+
+/* Mark old filters that may need to be removed.
+ * Caller must hold efx->filter_sem for read if race against
+ * efx_ef10_filter_table_remove() is possible
+ */
+static void efx_ef10_filter_mark_old(struct efx_nic *efx)
+{
+        struct efx_ef10_filter_table *table = efx->filter_state;
+        struct efx_ef10_filter_vlan *vlan;
+
+        spin_lock_bh(&efx->filter_lock);
+        list_for_each_entry(vlan, &table->vlan_list, list)
+                _efx_ef10_filter_vlan_mark_old(efx, vlan);
         spin_unlock_bh(&efx->filter_lock);
 }
-#undef EFX_EF10_FILTER_DO_MARK_OLD
 
-static void efx_ef10_filter_uc_addr_list(struct efx_nic *efx, bool *promisc)
+static void efx_ef10_filter_uc_addr_list(struct efx_nic *efx)
 {
         struct efx_ef10_filter_table *table = efx->filter_state;
         struct net_device *net_dev = efx->net_dev;
@@ -3918,45 +4264,38 @@ static void efx_ef10_filter_uc_addr_list(struct efx_nic *efx, bool *promisc)
         int addr_count;
         unsigned int i;
 
-        table->ucdef_id = EFX_EF10_FILTER_ID_INVALID;
         addr_count = netdev_uc_count(net_dev);
-        if (net_dev->flags & IFF_PROMISC)
-                *promisc = true;
+        table->uc_promisc = !!(net_dev->flags & IFF_PROMISC);
         table->dev_uc_count = 1 + addr_count;
         ether_addr_copy(table->dev_uc_list[0].addr, net_dev->dev_addr);
         i = 1;
         netdev_for_each_uc_addr(uc, net_dev) {
                 if (i >= EFX_EF10_FILTER_DEV_UC_MAX) {
-                        *promisc = true;
+                        table->uc_promisc = true;
                         break;
                 }
                 ether_addr_copy(table->dev_uc_list[i].addr, uc->addr);
-                table->dev_uc_list[i].id = EFX_EF10_FILTER_ID_INVALID;
                 i++;
         }
 }
 
-static void efx_ef10_filter_mc_addr_list(struct efx_nic *efx, bool *promisc)
+static void efx_ef10_filter_mc_addr_list(struct efx_nic *efx)
 {
         struct efx_ef10_filter_table *table = efx->filter_state;
         struct net_device *net_dev = efx->net_dev;
         struct netdev_hw_addr *mc;
         unsigned int i, addr_count;
 
-        table->mcdef_id = EFX_EF10_FILTER_ID_INVALID;
-        table->bcast_id = EFX_EF10_FILTER_ID_INVALID;
-        if (net_dev->flags & (IFF_PROMISC | IFF_ALLMULTI))
-                *promisc = true;
+        table->mc_promisc = !!(net_dev->flags & (IFF_PROMISC | IFF_ALLMULTI));
 
         addr_count = netdev_mc_count(net_dev);
         i = 0;
         netdev_for_each_mc_addr(mc, net_dev) {
                 if (i >= EFX_EF10_FILTER_DEV_MC_MAX) {
-                        *promisc = true;
+                        table->mc_promisc = true;
                         break;
                 }
                 ether_addr_copy(table->dev_mc_list[i].addr, mc->addr);
-                table->dev_mc_list[i].id = EFX_EF10_FILTER_ID_INVALID;
                 i++;
         }
 
@@ -3964,7 +4303,8 @@ static void efx_ef10_filter_mc_addr_list(struct efx_nic *efx, bool *promisc)
 }
 
 static int efx_ef10_filter_insert_addr_list(struct efx_nic *efx,
-                                             bool multicast, bool rollback)
+                                            struct efx_ef10_filter_vlan *vlan,
+                                            bool multicast, bool rollback)
 {
         struct efx_ef10_filter_table *table = efx->filter_state;
         struct efx_ef10_dev_addr *addr_list;
@@ -3973,14 +4313,17 @@ static int efx_ef10_filter_insert_addr_list(struct efx_nic *efx,
         u8 baddr[ETH_ALEN];
         unsigned int i, j;
         int addr_count;
+        u16 *ids;
         int rc;
 
         if (multicast) {
                 addr_list = table->dev_mc_list;
                 addr_count = table->dev_mc_count;
+                ids = vlan->mc;
         } else {
                 addr_list = table->dev_uc_list;
                 addr_count = table->dev_uc_count;
+                ids = vlan->uc;
         }
 
         filter_flags = efx_rss_enabled(efx) ? EFX_FILTER_FLAG_RX_RSS : 0;
@@ -3988,8 +4331,7 @@ static int efx_ef10_filter_insert_addr_list(struct efx_nic *efx,
         /* Insert/renew filters */
         for (i = 0; i < addr_count; i++) {
                 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0);
-                efx_filter_set_eth_local(&spec, EFX_FILTER_VID_UNSPEC,
-                                         addr_list[i].addr);
+                efx_filter_set_eth_local(&spec, vlan->vid, addr_list[i].addr);
                 rc = efx_ef10_filter_insert(efx, &spec, true);
                 if (rc < 0) {
                         if (rollback) {
@@ -3998,12 +4340,10 @@ static int efx_ef10_filter_insert_addr_list(struct efx_nic *efx,
                                            rc);
                                 /* Fall back to promiscuous */
                                 for (j = 0; j < i; j++) {
-                                        if (addr_list[j].id == EFX_EF10_FILTER_ID_INVALID)
-                                                continue;
                                         efx_ef10_filter_remove_unsafe(
                                                 efx, EFX_FILTER_PRI_AUTO,
-                                                addr_list[j].id);
-                                        addr_list[j].id = EFX_EF10_FILTER_ID_INVALID;
+                                                ids[j]);
+                                        ids[j] = EFX_EF10_FILTER_ID_INVALID;
                                 }
                                 return rc;
                         } else {
@@ -4011,40 +4351,40 @@ static int efx_ef10_filter_insert_addr_list(struct efx_nic *efx,
                                 rc = EFX_EF10_FILTER_ID_INVALID;
                         }
                 }
-                addr_list[i].id = efx_ef10_filter_get_unsafe_id(efx, rc);
+                ids[i] = efx_ef10_filter_get_unsafe_id(efx, rc);
         }
 
         if (multicast && rollback) {
                 /* Also need an Ethernet broadcast filter */
                 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0);
                 eth_broadcast_addr(baddr);
-                efx_filter_set_eth_local(&spec, EFX_FILTER_VID_UNSPEC, baddr);
+                efx_filter_set_eth_local(&spec, vlan->vid, baddr);
                 rc = efx_ef10_filter_insert(efx, &spec, true);
                 if (rc < 0) {
                         netif_warn(efx, drv, efx->net_dev,
                                    "Broadcast filter insert failed rc=%d\n", rc);
                         /* Fall back to promiscuous */
                         for (j = 0; j < i; j++) {
-                                if (addr_list[j].id == EFX_EF10_FILTER_ID_INVALID)
-                                        continue;
                                 efx_ef10_filter_remove_unsafe(
                                         efx, EFX_FILTER_PRI_AUTO,
-                                        addr_list[j].id);
-                                addr_list[j].id = EFX_EF10_FILTER_ID_INVALID;
+                                        ids[j]);
+                                ids[j] = EFX_EF10_FILTER_ID_INVALID;
                         }
                         return rc;
                 } else {
-                        table->bcast_id = efx_ef10_filter_get_unsafe_id(efx, rc);
+                        EFX_WARN_ON_PARANOID(vlan->bcast !=
+                                             EFX_EF10_FILTER_ID_INVALID);
+                        vlan->bcast = efx_ef10_filter_get_unsafe_id(efx, rc);
                 }
         }
 
         return 0;
 }
 
-static int efx_ef10_filter_insert_def(struct efx_nic *efx, bool multicast,
-                                      bool rollback)
+static int efx_ef10_filter_insert_def(struct efx_nic *efx,
+                                      struct efx_ef10_filter_vlan *vlan,
+                                      bool multicast, bool rollback)
 {
-        struct efx_ef10_filter_table *table = efx->filter_state;
         struct efx_ef10_nic_data *nic_data = efx->nic_data;
         enum efx_filter_flags filter_flags;
         struct efx_filter_spec spec;
@@ -4060,6 +4400,9 @@ static int efx_ef10_filter_insert_def(struct efx_nic *efx, bool multicast,
         else
                 efx_filter_set_uc_def(&spec);
 
+        if (vlan->vid != EFX_FILTER_VID_UNSPEC)
+                efx_filter_set_eth_local(&spec, vlan->vid, NULL);
+
         rc = efx_ef10_filter_insert(efx, &spec, true);
         if (rc < 0) {
                 netif_printk(efx, drv, rc == -EPERM ? KERN_DEBUG : KERN_WARNING,
@@ -4067,14 +4410,14 @@ static int efx_ef10_filter_insert_def(struct efx_nic *efx, bool multicast,
                              "%scast mismatch filter insert failed rc=%d\n",
                              multicast ? "Multi" : "Uni", rc);
         } else if (multicast) {
-                table->mcdef_id = efx_ef10_filter_get_unsafe_id(efx, rc);
+                EFX_WARN_ON_PARANOID(vlan->mcdef != EFX_EF10_FILTER_ID_INVALID);
+                vlan->mcdef = efx_ef10_filter_get_unsafe_id(efx, rc);
                 if (!nic_data->workaround_26807) {
                         /* Also need an Ethernet broadcast filter */
                         efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO,
                                            filter_flags, 0);
                         eth_broadcast_addr(baddr);
-                        efx_filter_set_eth_local(&spec, EFX_FILTER_VID_UNSPEC,
-                                                 baddr);
+                        efx_filter_set_eth_local(&spec, vlan->vid, baddr);
                         rc = efx_ef10_filter_insert(efx, &spec, true);
                         if (rc < 0) {
                                 netif_warn(efx, drv, efx->net_dev,
@@ -4084,17 +4427,20 @@ static int efx_ef10_filter_insert_def(struct efx_nic *efx, bool multicast,
                                         /* Roll back the mc_def filter */
                                         efx_ef10_filter_remove_unsafe(
                                                         efx, EFX_FILTER_PRI_AUTO,
-                                                        table->mcdef_id);
-                                        table->mcdef_id = EFX_EF10_FILTER_ID_INVALID;
+                                                        vlan->mcdef);
+                                        vlan->mcdef = EFX_EF10_FILTER_ID_INVALID;
                                         return rc;
                                 }
                         } else {
-                                table->bcast_id = efx_ef10_filter_get_unsafe_id(efx, rc);
+                                EFX_WARN_ON_PARANOID(vlan->bcast !=
+                                                     EFX_EF10_FILTER_ID_INVALID);
+                                vlan->bcast = efx_ef10_filter_get_unsafe_id(efx, rc);
                         }
                 }
                 rc = 0;
         } else {
-                table->ucdef_id = rc;
+                EFX_WARN_ON_PARANOID(vlan->ucdef != EFX_EF10_FILTER_ID_INVALID);
+                vlan->ucdef = rc;
                 rc = 0;
         }
         return rc;
@@ -4203,64 +4549,55 @@ reset_nic:
 /* Caller must hold efx->filter_sem for read if race against
  * efx_ef10_filter_table_remove() is possible
  */
-static void efx_ef10_filter_sync_rx_mode(struct efx_nic *efx)
+static void efx_ef10_filter_vlan_sync_rx_mode(struct efx_nic *efx,
+                                              struct efx_ef10_filter_vlan *vlan)
 {
         struct efx_ef10_filter_table *table = efx->filter_state;
         struct efx_ef10_nic_data *nic_data = efx->nic_data;
-        struct net_device *net_dev = efx->net_dev;
-        bool uc_promisc = false, mc_promisc = false;
 
-        if (!efx_dev_registered(efx))
-                return;
-
-        if (!table)
-                return;
-
-        efx_ef10_filter_mark_old(efx);
-
-        /* Copy/convert the address lists; add the primary station
-         * address and broadcast address
+        /* Do not install unspecified VID if VLAN filtering is enabled.
+         * Do not install all specified VIDs if VLAN filtering is disabled.
          */
-        netif_addr_lock_bh(net_dev);
-        efx_ef10_filter_uc_addr_list(efx, &uc_promisc);
-        efx_ef10_filter_mc_addr_list(efx, &mc_promisc);
-        netif_addr_unlock_bh(net_dev);
+        if ((vlan->vid == EFX_FILTER_VID_UNSPEC) == table->vlan_filter)
+                return;
 
         /* Insert/renew unicast filters */
-        if (uc_promisc) {
-                efx_ef10_filter_insert_def(efx, false, false);
-                efx_ef10_filter_insert_addr_list(efx, false, false);
+        if (table->uc_promisc) {
+                efx_ef10_filter_insert_def(efx, vlan, false, false);
+                efx_ef10_filter_insert_addr_list(efx, vlan, false, false);
         } else {
                 /* If any of the filters failed to insert, fall back to
                  * promiscuous mode - add in the uc_def filter.  But keep
                  * our individual unicast filters.
                  */
-                if (efx_ef10_filter_insert_addr_list(efx, false, false))
-                        efx_ef10_filter_insert_def(efx, false, false);
+                if (efx_ef10_filter_insert_addr_list(efx, vlan, false, false))
+                        efx_ef10_filter_insert_def(efx, vlan, false, false);
         }
 
         /* Insert/renew multicast filters */
         /* If changing promiscuous state with cascaded multicast filters, remove
          * old filters first, so that packets are dropped rather than duplicated
          */
-        if (nic_data->workaround_26807 && efx->mc_promisc != mc_promisc)
+        if (nic_data->workaround_26807 &&
+            table->mc_promisc_last != table->mc_promisc)
                 efx_ef10_filter_remove_old(efx);
-        if (mc_promisc) {
+        if (table->mc_promisc) {
                 if (nic_data->workaround_26807) {
                         /* If we failed to insert promiscuous filters, rollback
                          * and fall back to individual multicast filters
                          */
-                        if (efx_ef10_filter_insert_def(efx, true, true)) {
+                        if (efx_ef10_filter_insert_def(efx, vlan, true, true)) {
                                 /* Changing promisc state, so remove old filters */
                                 efx_ef10_filter_remove_old(efx);
-                                efx_ef10_filter_insert_addr_list(efx, true, false);
+                                efx_ef10_filter_insert_addr_list(efx, vlan,
+                                                                 true, false);
                         }
                 } else {
                         /* If we failed to insert promiscuous filters, don't
                          * rollback.  Regardless, also insert the mc_list
                          */
-                        efx_ef10_filter_insert_def(efx, true, false);
-                        efx_ef10_filter_insert_addr_list(efx, true, false);
+                        efx_ef10_filter_insert_def(efx, vlan, true, false);
+                        efx_ef10_filter_insert_addr_list(efx, vlan, true, false);
                 }
         } else {
                 /* If any filters failed to insert, rollback and fall back to
@@ -4268,17 +4605,153 @@ static void efx_ef10_filter_sync_rx_mode(struct efx_nic *efx)
                  * that fails, roll back again and insert as many of our
                  * individual multicast filters as we can.
                  */
-                if (efx_ef10_filter_insert_addr_list(efx, true, true)) {
+                if (efx_ef10_filter_insert_addr_list(efx, vlan, true, true)) {
                         /* Changing promisc state, so remove old filters */
                         if (nic_data->workaround_26807)
                                 efx_ef10_filter_remove_old(efx);
-                        if (efx_ef10_filter_insert_def(efx, true, true))
-                                efx_ef10_filter_insert_addr_list(efx, true, false);
+                        if (efx_ef10_filter_insert_def(efx, vlan, true, true))
+                                efx_ef10_filter_insert_addr_list(efx, vlan,
+                                                                 true, false);
                 }
         }
+}
+
+/* Caller must hold efx->filter_sem for read if race against
+ * efx_ef10_filter_table_remove() is possible
+ */
+static void efx_ef10_filter_sync_rx_mode(struct efx_nic *efx)
+{
+        struct efx_ef10_filter_table *table = efx->filter_state;
+        struct net_device *net_dev = efx->net_dev;
+        struct efx_ef10_filter_vlan *vlan;
+        bool vlan_filter;
+
+        if (!efx_dev_registered(efx))
+                return;
+
+        if (!table)
+                return;
+
+        efx_ef10_filter_mark_old(efx);
+
+        /* Copy/convert the address lists; add the primary station
+         * address and broadcast address
+         */
+        netif_addr_lock_bh(net_dev);
+        efx_ef10_filter_uc_addr_list(efx);
+        efx_ef10_filter_mc_addr_list(efx);
+        netif_addr_unlock_bh(net_dev);
+
+        /* If VLAN filtering changes, all old filters are finally removed.
+         * Do it in advance to avoid conflicts for unicast untagged and
+         * VLAN 0 tagged filters.
+         */
+        vlan_filter = !!(net_dev->features & NETIF_F_HW_VLAN_CTAG_FILTER);
+        if (table->vlan_filter != vlan_filter) {
+                table->vlan_filter = vlan_filter;
+                efx_ef10_filter_remove_old(efx);
+        }
+
+        list_for_each_entry(vlan, &table->vlan_list, list)
+                efx_ef10_filter_vlan_sync_rx_mode(efx, vlan);
 
         efx_ef10_filter_remove_old(efx);
-        efx->mc_promisc = mc_promisc;
+        table->mc_promisc_last = table->mc_promisc;
+}
+
+static struct efx_ef10_filter_vlan *efx_ef10_filter_find_vlan(struct efx_nic *efx, u16 vid)
+{
+        struct efx_ef10_filter_table *table = efx->filter_state;
+        struct efx_ef10_filter_vlan *vlan;
+
+        WARN_ON(!rwsem_is_locked(&efx->filter_sem));
+
+        list_for_each_entry(vlan, &table->vlan_list, list) {
+                if (vlan->vid == vid)
+                        return vlan;
+        }
+
+        return NULL;
+}
+
+static int efx_ef10_filter_add_vlan(struct efx_nic *efx, u16 vid)
+{
+        struct efx_ef10_filter_table *table = efx->filter_state;
+        struct efx_ef10_filter_vlan *vlan;
+        unsigned int i;
+
+        if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
+                return -EINVAL;
+
+        vlan = efx_ef10_filter_find_vlan(efx, vid);
+        if (WARN_ON(vlan)) {
+                netif_err(efx, drv, efx->net_dev,
+                          "VLAN %u already added\n", vid);
+                return -EALREADY;
+        }
+
+        vlan = kzalloc(sizeof(*vlan), GFP_KERNEL);
+        if (!vlan)
+                return -ENOMEM;
+
+        vlan->vid = vid;
+
+        for (i = 0; i < ARRAY_SIZE(vlan->uc); i++)
+                vlan->uc[i] = EFX_EF10_FILTER_ID_INVALID;
+        for (i = 0; i < ARRAY_SIZE(vlan->mc); i++)
+                vlan->mc[i] = EFX_EF10_FILTER_ID_INVALID;
+        vlan->ucdef = EFX_EF10_FILTER_ID_INVALID;
+        vlan->bcast = EFX_EF10_FILTER_ID_INVALID;
+        vlan->mcdef = EFX_EF10_FILTER_ID_INVALID;
+
+        list_add_tail(&vlan->list, &table->vlan_list);
+
+        if (efx_dev_registered(efx))
+                efx_ef10_filter_vlan_sync_rx_mode(efx, vlan);
+
+        return 0;
+}
+
+static void efx_ef10_filter_del_vlan_internal(struct efx_nic *efx,
+                                              struct efx_ef10_filter_vlan *vlan)
+{
+        unsigned int i;
+
+        /* See comment in efx_ef10_filter_table_remove() */
+        if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
+                return;
+
+        list_del(&vlan->list);
+
+        for (i = 0; i < ARRAY_SIZE(vlan->uc); i++)
+                efx_ef10_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO,
+                                              vlan->uc[i]);
+        for (i = 0; i < ARRAY_SIZE(vlan->mc); i++)
+                efx_ef10_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO,
+                                              vlan->mc[i]);
+        efx_ef10_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO, vlan->ucdef);
+        efx_ef10_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO, vlan->bcast);
+        efx_ef10_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO, vlan->mcdef);
+
+        kfree(vlan);
+}
+
+static void efx_ef10_filter_del_vlan(struct efx_nic *efx, u16 vid)
+{
+        struct efx_ef10_filter_vlan *vlan;
+
+        /* See comment in efx_ef10_filter_table_remove() */
+        if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
+                return;
+
+        vlan = efx_ef10_filter_find_vlan(efx, vid);
+        if (!vlan) {
+                netif_err(efx, drv, efx->net_dev,
+                          "VLAN %u not found in filter state\n", vid);
+                return;
+        }
+
+        efx_ef10_filter_del_vlan_internal(efx, vlan);
 }
 
 static int efx_ef10_set_mac_address(struct efx_nic *efx)
@@ -4290,6 +4763,8 @@ static int efx_ef10_set_mac_address(struct efx_nic *efx)
 
         efx_device_detach_sync(efx);
         efx_net_stop(efx->net_dev);
+
+        mutex_lock(&efx->mac_lock);
         down_write(&efx->filter_sem);
         efx_ef10_filter_table_remove(efx);
 
@@ -4302,6 +4777,8 @@ static int efx_ef10_set_mac_address(struct efx_nic *efx)
 
         efx_ef10_filter_table_probe(efx);
         up_write(&efx->filter_sem);
+        mutex_unlock(&efx->mac_lock);
+
         if (was_enabled)
                 efx_net_open(efx->net_dev);
         netif_device_attach(efx->net_dev);
@@ -4703,6 +5180,29 @@ static int efx_ef10_ptp_set_ts_config(struct efx_nic *efx,
         }
 }
 
+static int efx_ef10_vlan_rx_add_vid(struct efx_nic *efx, __be16 proto, u16 vid)
+{
+        if (proto != htons(ETH_P_8021Q))
+                return -EINVAL;
+
+        return efx_ef10_add_vlan(efx, vid);
+}
+
+static int efx_ef10_vlan_rx_kill_vid(struct efx_nic *efx, __be16 proto, u16 vid)
+{
+        if (proto != htons(ETH_P_8021Q))
+                return -EINVAL;
+
+        return efx_ef10_del_vlan(efx, vid);
+}
+
+#define EF10_OFFLOAD_FEATURES           \
+        (NETIF_F_IP_CSUM |              \
+         NETIF_F_HW_VLAN_CTAG_FILTER |  \
+         NETIF_F_IPV6_CSUM |            \
+         NETIF_F_RXHASH |               \
+         NETIF_F_NTUPLE)
+
 const struct efx_nic_type efx_hunt_a0_vf_nic_type = {
         .is_vf = true,
         .mem_bar = EFX_MEM_VF_BAR,
@@ -4780,6 +5280,8 @@ const struct efx_nic_type efx_hunt_a0_vf_nic_type = {
 #endif
         .ptp_write_host_time = efx_ef10_ptp_write_host_time_vf,
         .ptp_set_ts_config = efx_ef10_ptp_set_ts_config_vf,
+        .vlan_rx_add_vid = efx_ef10_vlan_rx_add_vid,
+        .vlan_rx_kill_vid = efx_ef10_vlan_rx_kill_vid,
 #ifdef CONFIG_SFC_SRIOV
         .vswitching_probe = efx_ef10_vswitching_probe_vf,
         .vswitching_restore = efx_ef10_vswitching_restore_vf,
@@ -4798,8 +5300,7 @@ const struct efx_nic_type efx_hunt_a0_vf_nic_type = {
         .always_rx_scatter = true,
         .max_interrupt_mode = EFX_INT_MODE_MSIX,
         .timer_period_max = 1 << ERF_DD_EVQ_IND_TIMER_VAL_WIDTH,
-        .offload_features = (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
-                             NETIF_F_RXHASH | NETIF_F_NTUPLE),
+        .offload_features = EF10_OFFLOAD_FEATURES,
         .mcdi_max_ver = 2,
         .max_rx_ip_filters = HUNT_FILTER_TBL_ROWS,
         .hwtstamp_filters = 1 << HWTSTAMP_FILTER_NONE |
@@ -4891,6 +5392,8 @@ const struct efx_nic_type efx_hunt_a0_nic_type = {
         .ptp_write_host_time = efx_ef10_ptp_write_host_time,
         .ptp_set_ts_sync_events = efx_ef10_ptp_set_ts_sync_events,
         .ptp_set_ts_config = efx_ef10_ptp_set_ts_config,
+        .vlan_rx_add_vid = efx_ef10_vlan_rx_add_vid,
+        .vlan_rx_kill_vid = efx_ef10_vlan_rx_kill_vid,
 #ifdef CONFIG_SFC_SRIOV
         .sriov_configure = efx_ef10_sriov_configure,
         .sriov_init = efx_ef10_sriov_init,
@@ -4919,8 +5422,7 @@ const struct efx_nic_type efx_hunt_a0_nic_type = {
         .always_rx_scatter = true,
         .max_interrupt_mode = EFX_INT_MODE_MSIX,
         .timer_period_max = 1 << ERF_DD_EVQ_IND_TIMER_VAL_WIDTH,
-        .offload_features = (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
-                             NETIF_F_RXHASH | NETIF_F_NTUPLE),
+        .offload_features = EF10_OFFLOAD_FEATURES,
         .mcdi_max_ver = 2,
         .max_rx_ip_filters = HUNT_FILTER_TBL_ROWS,
         .hwtstamp_filters = 1 << HWTSTAMP_FILTER_NONE |
diff --git a/drivers/net/ethernet/sfc/ef10_sriov.c b/drivers/net/ethernet/sfc/ef10_sriov.c
index 3c17f274e802..a949b9d27329 100644
--- a/drivers/net/ethernet/sfc/ef10_sriov.c
+++ b/drivers/net/ethernet/sfc/ef10_sriov.c
@@ -232,6 +232,35 @@ fail:
         return rc;
 }
 
+static int efx_ef10_vadaptor_alloc_set_features(struct efx_nic *efx)
+{
+        struct efx_ef10_nic_data *nic_data = efx->nic_data;
+        u32 port_flags;
+        int rc;
+
+        rc = efx_ef10_vadaptor_alloc(efx, nic_data->vport_id);
+        if (rc)
+                goto fail_vadaptor_alloc;
+
+        rc = efx_ef10_vadaptor_query(efx, nic_data->vport_id,
+                                     &port_flags, NULL, NULL);
+        if (rc)
+                goto fail_vadaptor_query;
+
+        if (port_flags &
+            (1 << MC_CMD_VPORT_ALLOC_IN_FLAG_VLAN_RESTRICT_LBN))
+                efx->fixed_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
+        else
+                efx->fixed_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
+
+        return 0;
+
+fail_vadaptor_query:
+        efx_ef10_vadaptor_free(efx, EVB_PORT_ID_ASSIGNED);
+fail_vadaptor_alloc:
+        return rc;
+}
+
 /* On top of the default firmware vswitch setup, create a VEB vswitch and
  * expansion vport for use by this function.
  */
@@ -243,7 +272,7 @@ int efx_ef10_vswitching_probe_pf(struct efx_nic *efx)
 
         if (pci_sriov_get_totalvfs(efx->pci_dev) <= 0) {
                 /* vswitch not needed as we have no VFs */
-                efx_ef10_vadaptor_alloc(efx, nic_data->vport_id);
+                efx_ef10_vadaptor_alloc_set_features(efx);
                 return 0;
         }
 
@@ -263,7 +292,7 @@ int efx_ef10_vswitching_probe_pf(struct efx_nic *efx)
                 goto fail3;
         ether_addr_copy(nic_data->vport_mac, net_dev->dev_addr);
 
-        rc = efx_ef10_vadaptor_alloc(efx, nic_data->vport_id);
+        rc = efx_ef10_vadaptor_alloc_set_features(efx);
         if (rc)
                 goto fail4;
 
@@ -282,9 +311,7 @@ fail1:
 
 int efx_ef10_vswitching_probe_vf(struct efx_nic *efx)
 {
-        struct efx_ef10_nic_data *nic_data = efx->nic_data;
-
-        return efx_ef10_vadaptor_alloc(efx, nic_data->vport_id);
+        return efx_ef10_vadaptor_alloc_set_features(efx);
 }
 
 int efx_ef10_vswitching_restore_pf(struct efx_nic *efx)
@@ -554,6 +581,7 @@ int efx_ef10_sriov_set_vf_vlan(struct efx_nic *efx, int vf_i, u16 vlan,
                 efx_device_detach_sync(vf->efx);
                 efx_net_stop(vf->efx->net_dev);
 
+                mutex_lock(&vf->efx->mac_lock);
                 down_write(&vf->efx->filter_sem);
                 vf->efx->type->filter_table_remove(vf->efx);
 
@@ -630,6 +658,7 @@ restore_filters:
                         goto reset_nic_up_write;
 
                 up_write(&vf->efx->filter_sem);
+                mutex_unlock(&vf->efx->mac_lock);
 
                 up_write(&vf->efx->filter_sem);
 
@@ -642,9 +671,10 @@ restore_filters:
         return rc;
 
 reset_nic_up_write:
-        if (vf->efx)
+        if (vf->efx) {
                 up_write(&vf->efx->filter_sem);
-
+                mutex_unlock(&vf->efx->mac_lock);
+        }
 reset_nic:
         if (vf->efx) {
                 netif_err(efx, drv, efx->net_dev,
diff --git a/drivers/net/ethernet/sfc/ef10_sriov.h b/drivers/net/ethernet/sfc/ef10_sriov.h
index 6d25b92cb45e..9ceb7ef0a210 100644
--- a/drivers/net/ethernet/sfc/ef10_sriov.h
+++ b/drivers/net/ethernet/sfc/ef10_sriov.h
@@ -70,6 +70,9 @@ int efx_ef10_vport_add_mac(struct efx_nic *efx,
 int efx_ef10_vport_del_mac(struct efx_nic *efx,
                            unsigned int port_id, u8 *mac);
 int efx_ef10_vadaptor_alloc(struct efx_nic *efx, unsigned int port_id);
+int efx_ef10_vadaptor_query(struct efx_nic *efx, unsigned int port_id,
+                            u32 *port_flags, u32 *vadaptor_flags,
+                            unsigned int *vlan_tags);
 int efx_ef10_vadaptor_free(struct efx_nic *efx, unsigned int port_id);
 
 #endif /* EF10_SRIOV_H */
diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c
index 097f363f1630..14b821b1c880 100644
--- a/drivers/net/ethernet/sfc/efx.c
+++ b/drivers/net/ethernet/sfc/efx.c
@@ -600,6 +600,7 @@ fail:
  */
 static void efx_start_datapath(struct efx_nic *efx)
 {
+        netdev_features_t old_features = efx->net_dev->features;
         bool old_rx_scatter = efx->rx_scatter;
         struct efx_tx_queue *tx_queue;
         struct efx_rx_queue *rx_queue;
@@ -644,6 +645,15 @@ static void efx_start_datapath(struct efx_nic *efx)
                           efx->rx_dma_len, efx->rx_page_buf_step,
                           efx->rx_bufs_per_page, efx->rx_pages_per_batch);
 
+        /* Restore previously fixed features in hw_features and remove
+         * features which are fixed now
+         */
+        efx->net_dev->hw_features |= efx->net_dev->features;
+        efx->net_dev->hw_features &= ~efx->fixed_features;
+        efx->net_dev->features |= efx->fixed_features;
+        if (efx->net_dev->features != old_features)
+                netdev_features_change(efx->net_dev);
+
         /* RX filters may also have scatter-enabled flags */
         if (efx->rx_scatter != old_rx_scatter)
                 efx->type->filter_update_rx_scatter(efx);
@@ -1719,6 +1729,7 @@ static int efx_probe_filters(struct efx_nic *efx)
 
         spin_lock_init(&efx->filter_lock);
         init_rwsem(&efx->filter_sem);
+        mutex_lock(&efx->mac_lock);
         down_write(&efx->filter_sem);
         rc = efx->type->filter_table_probe(efx);
         if (rc)
@@ -1757,6 +1768,7 @@ static int efx_probe_filters(struct efx_nic *efx)
 #endif
 out_unlock:
         up_write(&efx->filter_sem);
+        mutex_unlock(&efx->mac_lock);
         return rc;
 }
 
@@ -2312,14 +2324,46 @@ static void efx_set_rx_mode(struct net_device *net_dev)
 static int efx_set_features(struct net_device *net_dev, netdev_features_t data)
 {
         struct efx_nic *efx = netdev_priv(net_dev);
+        int rc;
 
         /* If disabling RX n-tuple filtering, clear existing filters */
-        if (net_dev->features & ~data & NETIF_F_NTUPLE)
-                return efx->type->filter_clear_rx(efx, EFX_FILTER_PRI_MANUAL);
+        if (net_dev->features & ~data & NETIF_F_NTUPLE) {
+                rc = efx->type->filter_clear_rx(efx, EFX_FILTER_PRI_MANUAL);
+                if (rc)
+                        return rc;
+        }
+
+        /* If Rx VLAN filter is changed, update filters via mac_reconfigure */
+        if ((net_dev->features ^ data) & NETIF_F_HW_VLAN_CTAG_FILTER) {
+                /* efx_set_rx_mode() will schedule MAC work to update filters
+                 * when a new features are finally set in net_dev.
+                 */
+                efx_set_rx_mode(net_dev);
+        }
 
         return 0;
 }
 
+static int efx_vlan_rx_add_vid(struct net_device *net_dev, __be16 proto, u16 vid)
+{
+        struct efx_nic *efx = netdev_priv(net_dev);
+
+        if (efx->type->vlan_rx_add_vid)
+                return efx->type->vlan_rx_add_vid(efx, proto, vid);
+        else
+                return -EOPNOTSUPP;
+}
+
+static int efx_vlan_rx_kill_vid(struct net_device *net_dev, __be16 proto, u16 vid)
+{
+        struct efx_nic *efx = netdev_priv(net_dev);
+
+        if (efx->type->vlan_rx_kill_vid)
+                return efx->type->vlan_rx_kill_vid(efx, proto, vid);
+        else
+                return -EOPNOTSUPP;
+}
+
 static const struct net_device_ops efx_netdev_ops = {
         .ndo_open               = efx_net_open,
         .ndo_stop               = efx_net_stop,
@@ -2332,6 +2376,8 @@ static const struct net_device_ops efx_netdev_ops = {
         .ndo_set_mac_address    = efx_set_mac_address,
         .ndo_set_rx_mode        = efx_set_rx_mode,
         .ndo_set_features       = efx_set_features,
+        .ndo_vlan_rx_add_vid    = efx_vlan_rx_add_vid,
+        .ndo_vlan_rx_kill_vid   = efx_vlan_rx_kill_vid,
 #ifdef CONFIG_SFC_SRIOV
         .ndo_set_vf_mac         = efx_sriov_set_vf_mac,
         .ndo_set_vf_vlan        = efx_sriov_set_vf_vlan,
@@ -3147,17 +3193,25 @@ static int efx_pci_probe(struct pci_dev *pci_dev,
                 return -ENOMEM;
         efx = netdev_priv(net_dev);
         efx->type = (const struct efx_nic_type *) entry->driver_data;
+        efx->fixed_features |= NETIF_F_HIGHDMA;
         net_dev->features |= (efx->type->offload_features | NETIF_F_SG |
-                              NETIF_F_HIGHDMA | NETIF_F_TSO |
-                              NETIF_F_RXCSUM);
+                              NETIF_F_TSO | NETIF_F_RXCSUM);
         if (efx->type->offload_features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM))
                 net_dev->features |= NETIF_F_TSO6;
         /* Mask for features that also apply to VLAN devices */
         net_dev->vlan_features |= (NETIF_F_HW_CSUM | NETIF_F_SG |
                                    NETIF_F_HIGHDMA | NETIF_F_ALL_TSO |
                                    NETIF_F_RXCSUM);
-        /* All offloads can be toggled */
-        net_dev->hw_features = net_dev->features & ~NETIF_F_HIGHDMA;
+
+        net_dev->hw_features = net_dev->features & ~efx->fixed_features;
+
+        /* Disable VLAN filtering by default.  It may be enforced if
+         * the feature is fixed (i.e. VLAN filters are required to
+         * receive VLAN tagged packets due to vPort restrictions).
+         */
+        net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
+        net_dev->features |= efx->fixed_features;
+
         pci_set_drvdata(pci_dev, efx);
         SET_NETDEV_DEV(net_dev, &pci_dev->dev);
         rc = efx_init_struct(efx, pci_dev, net_dev);
diff --git a/drivers/net/ethernet/sfc/efx.h b/drivers/net/ethernet/sfc/efx.h
index 5e3f93f04e62..c3ae739e9c7a 100644
--- a/drivers/net/ethernet/sfc/efx.h
+++ b/drivers/net/ethernet/sfc/efx.h
@@ -274,4 +274,13 @@ static inline void efx_device_detach_sync(struct efx_nic *efx)
         netif_tx_unlock_bh(dev);
 }
 
+static inline bool efx_rwsem_assert_write_locked(struct rw_semaphore *sem)
+{
+        if (WARN_ON(down_read_trylock(sem))) {
+                up_read(sem);
+                return false;
+        }
+        return true;
+}
+
 #endif /* EFX_EFX_H */
diff --git a/drivers/net/ethernet/sfc/mcdi_pcol.h b/drivers/net/ethernet/sfc/mcdi_pcol.h
index 4cc772164a79..c9a5b003caaf 100644
--- a/drivers/net/ethernet/sfc/mcdi_pcol.h
+++ b/drivers/net/ethernet/sfc/mcdi_pcol.h
@@ -273,6 +273,9 @@
  * have already installed filters. See the comment at
  * MC_CMD_WORKAROUND_BUG26807. */
 #define MC_CMD_ERR_FILTERS_PRESENT 0x1014
+/* The clock whose frequency you've attempted to set set
+ * doesn't exist on this NIC */
+#define MC_CMD_ERR_NO_CLOCK 0x1015
 
 #define MC_CMD_ERR_CODE_OFST 0
 
@@ -292,9 +295,11 @@
 /* Point to the copycode entry point. */
 #define SIENA_MC_BOOTROM_COPYCODE_VEC (0x800 - 3 * 0x4)
 #define HUNT_MC_BOOTROM_COPYCODE_VEC (0x8000 - 3 * 0x4)
+#define MEDFORD_MC_BOOTROM_COPYCODE_VEC (0x10000 - 3 * 0x4)
 /* Points to the recovery mode entry point. */
 #define SIENA_MC_BOOTROM_NOFLASH_VEC (0x800 - 2 * 0x4)
 #define HUNT_MC_BOOTROM_NOFLASH_VEC (0x8000 - 2 * 0x4)
+#define MEDFORD_MC_BOOTROM_NOFLASH_VEC (0x10000 - 2 * 0x4)
 
 /* The command set exported by the boot ROM (MCDI v0) */
 #define MC_CMD_GET_VERSION_V0_SUPPORTED_FUNCS {         \
@@ -686,6 +691,12 @@
 #define          FCDI_EVENT_CODE_PTP_STATUS 0x9
 /* enum: Port id config to map MC-FC port idx */
 #define          FCDI_EVENT_CODE_PORT_CONFIG 0xa
+/* enum: Boot result or error code */
+#define          FCDI_EVENT_CODE_BOOT_RESULT 0xb
+#define       FCDI_EVENT_REBOOT_SRC_LBN 36
+#define       FCDI_EVENT_REBOOT_SRC_WIDTH 8
+#define          FCDI_EVENT_REBOOT_FC_FW 0x0 /* enum */
+#define          FCDI_EVENT_REBOOT_FC_BOOTLOADER 0x1 /* enum */
 #define       FCDI_EVENT_ASSERT_INSTR_ADDRESS_OFST 0
 #define       FCDI_EVENT_ASSERT_INSTR_ADDRESS_LBN 0
 #define       FCDI_EVENT_ASSERT_INSTR_ADDRESS_WIDTH 32
@@ -717,6 +728,11 @@
 #define       FCDI_EVENT_PORT_CONFIG_DATA_OFST 0
 #define       FCDI_EVENT_PORT_CONFIG_DATA_LBN 0
 #define       FCDI_EVENT_PORT_CONFIG_DATA_WIDTH 32
+#define       FCDI_EVENT_BOOT_RESULT_OFST 0
+/*            Enum values, see field(s): */
+/*               MC_CMD_AOE/MC_CMD_AOE_OUT_INFO/FC_BOOT_RESULT */
+#define       FCDI_EVENT_BOOT_RESULT_LBN 0
+#define       FCDI_EVENT_BOOT_RESULT_WIDTH 32
 
 /* FCDI_EXTENDED_EVENT_PPS structuredef: Extended FCDI event to send PPS events
  * to the MC. Note that this structure | is overlayed over a normal FCDI event
@@ -1649,15 +1665,30 @@
 
 /* MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS msgresponse */
 #define    MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_LEN 16
-/* Uncorrected error on transmit timestamps in NIC clock format */
+/* Uncorrected error on PTP transmit timestamps in NIC clock format */
 #define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_TRANSMIT_OFST 0
-/* Uncorrected error on receive timestamps in NIC clock format */
+/* Uncorrected error on PTP receive timestamps in NIC clock format */
 #define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_RECEIVE_OFST 4
 /* Uncorrected error on PPS output in NIC clock format */
 #define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_PPS_OUT_OFST 8
 /* Uncorrected error on PPS input in NIC clock format */
 #define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_PPS_IN_OFST 12
 
+/* MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2 msgresponse */
+#define    MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_LEN 24
+/* Uncorrected error on PTP transmit timestamps in NIC clock format */
+#define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_PTP_TX_OFST 0
+/* Uncorrected error on PTP receive timestamps in NIC clock format */
+#define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_PTP_RX_OFST 4
+/* Uncorrected error on PPS output in NIC clock format */
+#define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_PPS_OUT_OFST 8
+/* Uncorrected error on PPS input in NIC clock format */
+#define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_PPS_IN_OFST 12
+/* Uncorrected error on non-PTP transmit timestamps in NIC clock format */
+#define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_GENERAL_TX_OFST 16
+/* Uncorrected error on non-PTP receive timestamps in NIC clock format */
+#define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_GENERAL_RX_OFST 20
+
 /* MC_CMD_PTP_OUT_MANFTEST_PPS msgresponse */
 #define    MC_CMD_PTP_OUT_MANFTEST_PPS_LEN 4
 /* Results of testing */
@@ -2158,8 +2189,12 @@
 
 /* MC_CMD_DRV_ATTACH_IN msgrequest */
 #define    MC_CMD_DRV_ATTACH_IN_LEN 12
-/* new state (0=detached, 1=attached) to set if UPDATE=1 */
+/* new state to set if UPDATE=1 */
 #define       MC_CMD_DRV_ATTACH_IN_NEW_STATE_OFST 0
+#define        MC_CMD_DRV_ATTACH_LBN 0
+#define        MC_CMD_DRV_ATTACH_WIDTH 1
+#define        MC_CMD_DRV_PREBOOT_LBN 1
+#define        MC_CMD_DRV_PREBOOT_WIDTH 1
 /* 1 to set new state, or 0 to just report the existing state */
 #define       MC_CMD_DRV_ATTACH_IN_UPDATE_OFST 4
 /* preferred datapath firmware (for Huntington; ignored for Siena) */
@@ -2181,12 +2216,12 @@
 
 /* MC_CMD_DRV_ATTACH_OUT msgresponse */
 #define    MC_CMD_DRV_ATTACH_OUT_LEN 4
-/* previous or existing state (0=detached, 1=attached) */
+/* previous or existing state, see the bitmask at NEW_STATE */
 #define       MC_CMD_DRV_ATTACH_OUT_OLD_STATE_OFST 0
 
 /* MC_CMD_DRV_ATTACH_EXT_OUT msgresponse */
 #define    MC_CMD_DRV_ATTACH_EXT_OUT_LEN 8
-/* previous or existing state (0=detached, 1=attached) */
+/* previous or existing state, see the bitmask at NEW_STATE */
 #define       MC_CMD_DRV_ATTACH_EXT_OUT_OLD_STATE_OFST 0
 /* Flags associated with this function */
 #define       MC_CMD_DRV_ATTACH_EXT_OUT_FUNC_FLAGS_OFST 4
@@ -2198,6 +2233,10 @@
 #define          MC_CMD_DRV_ATTACH_EXT_OUT_FLAG_LINKCTRL 0x1
 /* enum: The function can perform privileged operations */
 #define          MC_CMD_DRV_ATTACH_EXT_OUT_FLAG_TRUSTED 0x2
+/* enum: The function does not have an active port associated with it. The port
+ * refers to the Sorrento external FPGA port.
+ */
+#define          MC_CMD_DRV_ATTACH_EXT_OUT_FLAG_NO_ACTIVE_PORT 0x3
 
 
 /***********************************/
@@ -2892,7 +2931,7 @@
  */
 #define MC_CMD_SET_MAC 0x2c
 
-#define MC_CMD_0x2c_PRIVILEGE_CTG SRIOV_CTG_LINK
+#define MC_CMD_0x2c_PRIVILEGE_CTG SRIOV_CTG_GENERAL
 
 /* MC_CMD_SET_MAC_IN msgrequest */
 #define    MC_CMD_SET_MAC_IN_LEN 28
@@ -2927,9 +2966,66 @@
 #define        MC_CMD_SET_MAC_IN_FLAG_INCLUDE_FCS_LBN 0
 #define        MC_CMD_SET_MAC_IN_FLAG_INCLUDE_FCS_WIDTH 1
 
+/* MC_CMD_SET_MAC_EXT_IN msgrequest */
+#define    MC_CMD_SET_MAC_EXT_IN_LEN 32
+/* The MTU is the MTU programmed directly into the XMAC/GMAC (inclusive of
+ * EtherII, VLAN, bug16011 padding).
+ */
+#define       MC_CMD_SET_MAC_EXT_IN_MTU_OFST 0
+#define       MC_CMD_SET_MAC_EXT_IN_DRAIN_OFST 4
+#define       MC_CMD_SET_MAC_EXT_IN_ADDR_OFST 8
+#define       MC_CMD_SET_MAC_EXT_IN_ADDR_LEN 8
+#define       MC_CMD_SET_MAC_EXT_IN_ADDR_LO_OFST 8
+#define       MC_CMD_SET_MAC_EXT_IN_ADDR_HI_OFST 12
+#define       MC_CMD_SET_MAC_EXT_IN_REJECT_OFST 16
+#define        MC_CMD_SET_MAC_EXT_IN_REJECT_UNCST_LBN 0
+#define        MC_CMD_SET_MAC_EXT_IN_REJECT_UNCST_WIDTH 1
+#define        MC_CMD_SET_MAC_EXT_IN_REJECT_BRDCST_LBN 1
+#define        MC_CMD_SET_MAC_EXT_IN_REJECT_BRDCST_WIDTH 1
+#define       MC_CMD_SET_MAC_EXT_IN_FCNTL_OFST 20
+/* enum: Flow control is off. */
+/*               MC_CMD_FCNTL_OFF 0x0 */
+/* enum: Respond to flow control. */
+/*               MC_CMD_FCNTL_RESPOND 0x1 */
+/* enum: Respond to and Issue flow control. */
+/*               MC_CMD_FCNTL_BIDIR 0x2 */
+/* enum: Auto neg flow control. */
+/*               MC_CMD_FCNTL_AUTO 0x3 */
+/* enum: Priority flow control (eftest builds only). */
+/*               MC_CMD_FCNTL_QBB 0x4 */
+/* enum: Issue flow control. */
+/*               MC_CMD_FCNTL_GENERATE 0x5 */
+#define       MC_CMD_SET_MAC_EXT_IN_FLAGS_OFST 24
+#define        MC_CMD_SET_MAC_EXT_IN_FLAG_INCLUDE_FCS_LBN 0
+#define        MC_CMD_SET_MAC_EXT_IN_FLAG_INCLUDE_FCS_WIDTH 1
+/* Select which parameters to configure. A parameter will only be modified if
+ * the corresponding control flag is set. If SET_MAC_ENHANCED is not set in
+ * capabilities then this field is ignored (and all flags are assumed to be
+ * set).
+ */
+#define       MC_CMD_SET_MAC_EXT_IN_CONTROL_OFST 28
+#define        MC_CMD_SET_MAC_EXT_IN_CFG_MTU_LBN 0
+#define        MC_CMD_SET_MAC_EXT_IN_CFG_MTU_WIDTH 1
+#define        MC_CMD_SET_MAC_EXT_IN_CFG_DRAIN_LBN 1
+#define        MC_CMD_SET_MAC_EXT_IN_CFG_DRAIN_WIDTH 1
+#define        MC_CMD_SET_MAC_EXT_IN_CFG_REJECT_LBN 2
+#define        MC_CMD_SET_MAC_EXT_IN_CFG_REJECT_WIDTH 1
+#define        MC_CMD_SET_MAC_EXT_IN_CFG_FCNTL_LBN 3
+#define        MC_CMD_SET_MAC_EXT_IN_CFG_FCNTL_WIDTH 1
+#define        MC_CMD_SET_MAC_EXT_IN_CFG_FCS_LBN 4
+#define        MC_CMD_SET_MAC_EXT_IN_CFG_FCS_WIDTH 1
+
 /* MC_CMD_SET_MAC_OUT msgresponse */
 #define    MC_CMD_SET_MAC_OUT_LEN 0
 
+/* MC_CMD_SET_MAC_V2_OUT msgresponse */
+#define    MC_CMD_SET_MAC_V2_OUT_LEN 4
+/* MTU as configured after processing the request. See comment at
+ * MC_CMD_SET_MAC_IN/MTU. To query MTU without doing any changes, set CONTROL
+ * to 0.
+ */
+#define       MC_CMD_SET_MAC_V2_OUT_MTU_OFST 0
+
 
 /***********************************/
 /* MC_CMD_PHY_STATS
@@ -3521,6 +3617,26 @@
 #define       MC_CMD_NVRAM_INFO_OUT_PHYSDEV_OFST 16
 #define       MC_CMD_NVRAM_INFO_OUT_PHYSADDR_OFST 20
 
+/* MC_CMD_NVRAM_INFO_V2_OUT msgresponse */
+#define    MC_CMD_NVRAM_INFO_V2_OUT_LEN 28
+#define       MC_CMD_NVRAM_INFO_V2_OUT_TYPE_OFST 0
+/*            Enum values, see field(s): */
+/*               MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */
+#define       MC_CMD_NVRAM_INFO_V2_OUT_SIZE_OFST 4
+#define       MC_CMD_NVRAM_INFO_V2_OUT_ERASESIZE_OFST 8
+#define       MC_CMD_NVRAM_INFO_V2_OUT_FLAGS_OFST 12
+#define        MC_CMD_NVRAM_INFO_V2_OUT_PROTECTED_LBN 0
+#define        MC_CMD_NVRAM_INFO_V2_OUT_PROTECTED_WIDTH 1
+#define        MC_CMD_NVRAM_INFO_V2_OUT_TLV_LBN 1
+#define        MC_CMD_NVRAM_INFO_V2_OUT_TLV_WIDTH 1
+#define        MC_CMD_NVRAM_INFO_V2_OUT_A_B_LBN 7
+#define        MC_CMD_NVRAM_INFO_V2_OUT_A_B_WIDTH 1
+#define       MC_CMD_NVRAM_INFO_V2_OUT_PHYSDEV_OFST 16
+#define       MC_CMD_NVRAM_INFO_V2_OUT_PHYSADDR_OFST 20
+/* Writes must be multiples of this size. Added to support the MUM on Sorrento.
+ */
+#define       MC_CMD_NVRAM_INFO_V2_OUT_WRITESIZE_OFST 24
+
 
 /***********************************/
 /* MC_CMD_NVRAM_UPDATE_START
@@ -3561,6 +3677,37 @@
 /* amount to read in bytes */
 #define       MC_CMD_NVRAM_READ_IN_LENGTH_OFST 8
 
+/* MC_CMD_NVRAM_READ_IN_V2 msgrequest */
+#define    MC_CMD_NVRAM_READ_IN_V2_LEN 16
+#define       MC_CMD_NVRAM_READ_IN_V2_TYPE_OFST 0
+/*            Enum values, see field(s): */
+/*               MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */
+#define       MC_CMD_NVRAM_READ_IN_V2_OFFSET_OFST 4
+/* amount to read in bytes */
+#define       MC_CMD_NVRAM_READ_IN_V2_LENGTH_OFST 8
+/* Optional control info. If a partition is stored with an A/B versioning
+ * scheme (i.e. in more than one physical partition in NVRAM) the host can set
+ * this to control which underlying physical partition is used to read data
+ * from. This allows it to perform a read-modify-write-verify with the write
+ * lock continuously held by calling NVRAM_UPDATE_START, reading the old
+ * contents using MODE=TARGET_CURRENT, overwriting the old partition and then
+ * verifying by reading with MODE=TARGET_BACKUP.
+ */
+#define       MC_CMD_NVRAM_READ_IN_V2_MODE_OFST 12
+/* enum: Same as omitting MODE: caller sees data in current partition unless it
+ * holds the write lock in which case it sees data in the partition it is
+ * updating.
+ */
+#define          MC_CMD_NVRAM_READ_IN_V2_DEFAULT 0x0
+/* enum: Read from the current partition of an A/B pair, even if holding the
+ * write lock.
+ */
+#define          MC_CMD_NVRAM_READ_IN_V2_TARGET_CURRENT 0x1
+/* enum: Read from the non-current (i.e. to be updated) partition of an A/B
+ * pair
+ */
+#define          MC_CMD_NVRAM_READ_IN_V2_TARGET_BACKUP 0x2
+
 /* MC_CMD_NVRAM_READ_OUT msgresponse */
 #define    MC_CMD_NVRAM_READ_OUT_LENMIN 1
 #define    MC_CMD_NVRAM_READ_OUT_LENMAX 252
@@ -3895,6 +4042,8 @@
 #define          MC_CMD_SENSOR_CCOM_AVREG_1V8_SUPPLY  0x39
 /* enum: CCOM AVREG 1v8 supply (external ADC): mV */
 #define          MC_CMD_SENSOR_CCOM_AVREG_1V8_SUPPLY_EXTADC  0x3a
+/* enum: CCOM RTS temperature: degC */
+#define          MC_CMD_SENSOR_CONTROLLER_RTS  0x3b
 /* enum: Not a sensor: reserved for the next page flag */
 #define          MC_CMD_SENSOR_PAGE1_NEXT  0x3f
 /* enum: controller internal temperature sensor voltage on master core
@@ -3931,6 +4080,12 @@
 #define          MC_CMD_SENSOR_PHY0_VCC  0x4c
 /* enum: Voltage supplied to the QSFP #1 from their power supply: mV */
 #define          MC_CMD_SENSOR_PHY1_VCC  0x4d
+/* enum: Controller die temperature (TDIODE): degC */
+#define          MC_CMD_SENSOR_CONTROLLER_TDIODE_TEMP  0x4e
+/* enum: Board temperature (front): degC */
+#define          MC_CMD_SENSOR_BOARD_FRONT_TEMP  0x4f
+/* enum: Board temperature (back): degC */
+#define          MC_CMD_SENSOR_BOARD_BACK_TEMP  0x50
 /* MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF */
 #define       MC_CMD_SENSOR_ENTRY_OFST 4
 #define       MC_CMD_SENSOR_ENTRY_LEN 8
@@ -4007,7 +4162,7 @@
 
 /* MC_CMD_READ_SENSORS_EXT_IN msgrequest */
 #define    MC_CMD_READ_SENSORS_EXT_IN_LEN 12
-/* DMA address of host buffer for sensor readings */
+/* DMA address of host buffer for sensor readings (must be 4Kbyte aligned). */
 #define       MC_CMD_READ_SENSORS_EXT_IN_DMA_ADDR_OFST 0
 #define       MC_CMD_READ_SENSORS_EXT_IN_DMA_ADDR_LEN 8
 #define       MC_CMD_READ_SENSORS_EXT_IN_DMA_ADDR_LO_OFST 0
@@ -4608,6 +4763,10 @@
  * operations
  */
 #define          MC_CMD_MUM_OP_QSFP 0xc
+/* enum: Request discrete and SODIMM DDR info (type, size, speed grade, voltage
+ * level) from MUM
+ */
+#define          MC_CMD_MUM_OP_READ_DDR_INFO 0xd
 
 /* MC_CMD_MUM_IN_NULL msgrequest */
 #define    MC_CMD_MUM_IN_NULL_LEN 4
@@ -4793,6 +4952,10 @@
 #define       MC_CMD_MUM_IN_PROGRAM_CLOCKS_FLAGS_OFST 8
 #define        MC_CMD_MUM_IN_PROGRAM_CLOCKS_OVERCLOCK_110_LBN 0
 #define        MC_CMD_MUM_IN_PROGRAM_CLOCKS_OVERCLOCK_110_WIDTH 1
+#define        MC_CMD_MUM_IN_PROGRAM_CLOCKS_CLOCK_NIC_FROM_FPGA_LBN 1
+#define        MC_CMD_MUM_IN_PROGRAM_CLOCKS_CLOCK_NIC_FROM_FPGA_WIDTH 1
+#define        MC_CMD_MUM_IN_PROGRAM_CLOCKS_CLOCK_REF_FROM_XO_LBN 2
+#define        MC_CMD_MUM_IN_PROGRAM_CLOCKS_CLOCK_REF_FROM_XO_WIDTH 1
 
 /* MC_CMD_MUM_IN_FPGA_LOAD msgrequest */
 #define    MC_CMD_MUM_IN_FPGA_LOAD_LEN 8
@@ -4862,6 +5025,11 @@
 #define       MC_CMD_MUM_IN_QSFP_POLL_BIST_HDR_OFST 4
 #define       MC_CMD_MUM_IN_QSFP_POLL_BIST_IDX_OFST 8
 
+/* MC_CMD_MUM_IN_READ_DDR_INFO msgrequest */
+#define    MC_CMD_MUM_IN_READ_DDR_INFO_LEN 4
+/* MUM cmd header */
+/*            MC_CMD_MUM_IN_CMD_OFST 0 */
+
 /* MC_CMD_MUM_OUT msgresponse */
 #define    MC_CMD_MUM_OUT_LEN 0
 
@@ -5004,6 +5172,69 @@
 #define    MC_CMD_MUM_OUT_QSFP_POLL_BIST_LEN 4
 #define       MC_CMD_MUM_OUT_QSFP_POLL_BIST_TEST_OFST 0
 
+/* MC_CMD_MUM_OUT_READ_DDR_INFO msgresponse */
+#define    MC_CMD_MUM_OUT_READ_DDR_INFO_LENMIN 24
+#define    MC_CMD_MUM_OUT_READ_DDR_INFO_LENMAX 248
+#define    MC_CMD_MUM_OUT_READ_DDR_INFO_LEN(num) (8+8*(num))
+/* Discrete (soldered) DDR resistor strap info */
+#define       MC_CMD_MUM_OUT_READ_DDR_INFO_DISCRETE_DDR_INFO_OFST 0
+#define        MC_CMD_MUM_OUT_READ_DDR_INFO_VRATIO_LBN 0
+#define        MC_CMD_MUM_OUT_READ_DDR_INFO_VRATIO_WIDTH 16
+#define        MC_CMD_MUM_OUT_READ_DDR_INFO_RESERVED1_LBN 16
+#define        MC_CMD_MUM_OUT_READ_DDR_INFO_RESERVED1_WIDTH 16
+/* Number of SODIMM info records */
+#define       MC_CMD_MUM_OUT_READ_DDR_INFO_NUM_RECORDS_OFST 4
+/* Array of SODIMM info records */
+#define       MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_OFST 8
+#define       MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_LEN 8
+#define       MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_LO_OFST 8
+#define       MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_HI_OFST 12
+#define       MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_MINNUM 2
+#define       MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_MAXNUM 30
+#define        MC_CMD_MUM_OUT_READ_DDR_INFO_BANK_ID_LBN 0
+#define        MC_CMD_MUM_OUT_READ_DDR_INFO_BANK_ID_WIDTH 8
+/* enum: SODIMM bank 1 (Top SODIMM for Sorrento) */
+#define          MC_CMD_MUM_OUT_READ_DDR_INFO_BANK1 0x0
+/* enum: SODIMM bank 2 (Bottom SODDIMM for Sorrento) */
+#define          MC_CMD_MUM_OUT_READ_DDR_INFO_BANK2 0x1
+/* enum: Total number of SODIMM banks */
+#define          MC_CMD_MUM_OUT_READ_DDR_INFO_NUM_BANKS 0x2
+#define        MC_CMD_MUM_OUT_READ_DDR_INFO_TYPE_LBN 8
+#define        MC_CMD_MUM_OUT_READ_DDR_INFO_TYPE_WIDTH 8
+#define        MC_CMD_MUM_OUT_READ_DDR_INFO_RANK_LBN 16
+#define        MC_CMD_MUM_OUT_READ_DDR_INFO_RANK_WIDTH 4
+#define        MC_CMD_MUM_OUT_READ_DDR_INFO_VOLTAGE_LBN 20
+#define        MC_CMD_MUM_OUT_READ_DDR_INFO_VOLTAGE_WIDTH 4
+#define          MC_CMD_MUM_OUT_READ_DDR_INFO_NOT_POWERED 0x0 /* enum */
+#define          MC_CMD_MUM_OUT_READ_DDR_INFO_1V25 0x1 /* enum */
+#define          MC_CMD_MUM_OUT_READ_DDR_INFO_1V35 0x2 /* enum */
+#define          MC_CMD_MUM_OUT_READ_DDR_INFO_1V5 0x3 /* enum */
+/* enum: Values 5-15 are reserved for future usage */
+#define          MC_CMD_MUM_OUT_READ_DDR_INFO_1V8 0x4
+#define        MC_CMD_MUM_OUT_READ_DDR_INFO_SIZE_LBN 24
+#define        MC_CMD_MUM_OUT_READ_DDR_INFO_SIZE_WIDTH 8
+#define        MC_CMD_MUM_OUT_READ_DDR_INFO_SPEED_LBN 32
+#define        MC_CMD_MUM_OUT_READ_DDR_INFO_SPEED_WIDTH 16
+#define        MC_CMD_MUM_OUT_READ_DDR_INFO_STATE_LBN 48
+#define        MC_CMD_MUM_OUT_READ_DDR_INFO_STATE_WIDTH 4
+/* enum: No module present */
+#define          MC_CMD_MUM_OUT_READ_DDR_INFO_ABSENT 0x0
+/* enum: Module present supported and powered on */
+#define          MC_CMD_MUM_OUT_READ_DDR_INFO_PRESENT_POWERED 0x1
+/* enum: Module present but bad type */
+#define          MC_CMD_MUM_OUT_READ_DDR_INFO_PRESENT_BAD_TYPE 0x2
+/* enum: Module present but incompatible voltage */
+#define          MC_CMD_MUM_OUT_READ_DDR_INFO_PRESENT_BAD_VOLTAGE 0x3
+/* enum: Module present but unknown SPD */
+#define          MC_CMD_MUM_OUT_READ_DDR_INFO_PRESENT_BAD_SPD 0x4
+/* enum: Module present but slot cannot support it */
+#define          MC_CMD_MUM_OUT_READ_DDR_INFO_PRESENT_BAD_SLOT 0x5
+/* enum: Modules may or may not be present, but cannot establish contact by I2C
+ */
+#define          MC_CMD_MUM_OUT_READ_DDR_INFO_NOT_REACHABLE 0x6
+#define        MC_CMD_MUM_OUT_READ_DDR_INFO_RESERVED2_LBN 52
+#define        MC_CMD_MUM_OUT_READ_DDR_INFO_RESERVED2_WIDTH 12
+
 /* MC_CMD_RESOURCE_SPECIFIER enum */
 /* enum: Any */
 #define          MC_CMD_RESOURCE_INSTANCE_ANY 0xffffffff
@@ -5076,6 +5307,8 @@
 #define          NVRAM_PARTITION_TYPE_DYNAMIC_CONFIG       0x500
 /* enum: Expansion ROM configuration data for port 0 */
 #define          NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT0  0x600
+/* enum: Synonym for EXPROM_CONFIG_PORT0 as used in pmap files */
+#define          NVRAM_PARTITION_TYPE_EXPROM_CONFIG        0x600
 /* enum: Expansion ROM configuration data for port 1 */
 #define          NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT1  0x601
 /* enum: Expansion ROM configuration data for port 2 */
@@ -5084,6 +5317,8 @@
 #define          NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT3  0x603
 /* enum: Non-volatile log output partition */
 #define          NVRAM_PARTITION_TYPE_LOG                  0x700
+/* enum: Non-volatile log output of second core on dual-core device */
+#define          NVRAM_PARTITION_TYPE_LOG_SLAVE            0x701
 /* enum: Device state dump output partition */
 #define          NVRAM_PARTITION_TYPE_DUMP                 0x800
 /* enum: Application license key storage partition */
@@ -5116,6 +5351,20 @@
 #define          NVRAM_PARTITION_TYPE_MUM_USER_ROM         0xc05
 /* enum: MUM fuses and lockbits partition. */
 #define          NVRAM_PARTITION_TYPE_MUM_FUSELOCK         0xc06
+/* enum: UEFI expansion ROM if separate from PXE */
+#define          NVRAM_PARTITION_TYPE_EXPANSION_UEFI       0xd00
+/* enum: Spare partition 0 */
+#define          NVRAM_PARTITION_TYPE_SPARE_0              0x1000
+/* enum: Spare partition 1 */
+#define          NVRAM_PARTITION_TYPE_SPARE_1              0x1100
+/* enum: Spare partition 2 */
+#define          NVRAM_PARTITION_TYPE_SPARE_2              0x1200
+/* enum: Spare partition 3 */
+#define          NVRAM_PARTITION_TYPE_SPARE_3              0x1300
+/* enum: Spare partition 4 */
+#define          NVRAM_PARTITION_TYPE_SPARE_4              0x1400
+/* enum: Spare partition 5 */
+#define          NVRAM_PARTITION_TYPE_SPARE_5              0x1500
 /* enum: Start of reserved value range (firmware may use for any purpose) */
 #define          NVRAM_PARTITION_TYPE_RESERVED_VALUES_MIN  0xff00
 /* enum: End of reserved value range (firmware may use for any purpose) */
@@ -5149,6 +5398,90 @@
 #define       LICENSED_APP_ID_ID_LBN 0
 #define       LICENSED_APP_ID_ID_WIDTH 32
 
+/* LICENSED_FEATURES structuredef */
+#define    LICENSED_FEATURES_LEN 8
+/* Bitmask of licensed firmware features */
+#define       LICENSED_FEATURES_MASK_OFST 0
+#define       LICENSED_FEATURES_MASK_LEN 8
+#define       LICENSED_FEATURES_MASK_LO_OFST 0
+#define       LICENSED_FEATURES_MASK_HI_OFST 4
+#define        LICENSED_FEATURES_RX_CUT_THROUGH_LBN 0
+#define        LICENSED_FEATURES_RX_CUT_THROUGH_WIDTH 1
+#define        LICENSED_FEATURES_PIO_LBN 1
+#define        LICENSED_FEATURES_PIO_WIDTH 1
+#define        LICENSED_FEATURES_EVQ_TIMER_LBN 2
+#define        LICENSED_FEATURES_EVQ_TIMER_WIDTH 1
+#define        LICENSED_FEATURES_CLOCK_LBN 3
+#define        LICENSED_FEATURES_CLOCK_WIDTH 1
+#define        LICENSED_FEATURES_RX_TIMESTAMPS_LBN 4
+#define        LICENSED_FEATURES_RX_TIMESTAMPS_WIDTH 1
+#define        LICENSED_FEATURES_TX_TIMESTAMPS_LBN 5
+#define        LICENSED_FEATURES_TX_TIMESTAMPS_WIDTH 1
+#define        LICENSED_FEATURES_RX_SNIFF_LBN 6
+#define        LICENSED_FEATURES_RX_SNIFF_WIDTH 1
+#define        LICENSED_FEATURES_TX_SNIFF_LBN 7
+#define        LICENSED_FEATURES_TX_SNIFF_WIDTH 1
+#define        LICENSED_FEATURES_PROXY_FILTER_OPS_LBN 8
+#define        LICENSED_FEATURES_PROXY_FILTER_OPS_WIDTH 1
+#define        LICENSED_FEATURES_EVENT_CUT_THROUGH_LBN 9
+#define        LICENSED_FEATURES_EVENT_CUT_THROUGH_WIDTH 1
+#define       LICENSED_FEATURES_MASK_LBN 0
+#define       LICENSED_FEATURES_MASK_WIDTH 64
+
+/* LICENSED_V3_APPS structuredef */
+#define    LICENSED_V3_APPS_LEN 8
+/* Bitmask of licensed applications */
+#define       LICENSED_V3_APPS_MASK_OFST 0
+#define       LICENSED_V3_APPS_MASK_LEN 8
+#define       LICENSED_V3_APPS_MASK_LO_OFST 0
+#define       LICENSED_V3_APPS_MASK_HI_OFST 4
+#define        LICENSED_V3_APPS_ONLOAD_LBN 0
+#define        LICENSED_V3_APPS_ONLOAD_WIDTH 1
+#define        LICENSED_V3_APPS_PTP_LBN 1
+#define        LICENSED_V3_APPS_PTP_WIDTH 1
+#define        LICENSED_V3_APPS_SOLARCAPTURE_PRO_LBN 2
+#define        LICENSED_V3_APPS_SOLARCAPTURE_PRO_WIDTH 1
+#define        LICENSED_V3_APPS_SOLARSECURE_LBN 3
+#define        LICENSED_V3_APPS_SOLARSECURE_WIDTH 1
+#define        LICENSED_V3_APPS_PERF_MONITOR_LBN 4
+#define        LICENSED_V3_APPS_PERF_MONITOR_WIDTH 1
+#define        LICENSED_V3_APPS_SOLARCAPTURE_LIVE_LBN 5
+#define        LICENSED_V3_APPS_SOLARCAPTURE_LIVE_WIDTH 1
+#define        LICENSED_V3_APPS_CAPTURE_SOLARSYSTEM_LBN 6
+#define        LICENSED_V3_APPS_CAPTURE_SOLARSYSTEM_WIDTH 1
+#define        LICENSED_V3_APPS_NETWORK_ACCESS_CONTROL_LBN 7
+#define        LICENSED_V3_APPS_NETWORK_ACCESS_CONTROL_WIDTH 1
+#define       LICENSED_V3_APPS_MASK_LBN 0
+#define       LICENSED_V3_APPS_MASK_WIDTH 64
+
+/* LICENSED_V3_FEATURES structuredef */
+#define    LICENSED_V3_FEATURES_LEN 8
+/* Bitmask of licensed firmware features */
+#define       LICENSED_V3_FEATURES_MASK_OFST 0
+#define       LICENSED_V3_FEATURES_MASK_LEN 8
+#define       LICENSED_V3_FEATURES_MASK_LO_OFST 0
+#define       LICENSED_V3_FEATURES_MASK_HI_OFST 4
+#define        LICENSED_V3_FEATURES_RX_CUT_THROUGH_LBN 0
+#define        LICENSED_V3_FEATURES_RX_CUT_THROUGH_WIDTH 1
+#define        LICENSED_V3_FEATURES_PIO_LBN 1
+#define        LICENSED_V3_FEATURES_PIO_WIDTH 1
+#define        LICENSED_V3_FEATURES_EVQ_TIMER_LBN 2
+#define        LICENSED_V3_FEATURES_EVQ_TIMER_WIDTH 1
+#define        LICENSED_V3_FEATURES_CLOCK_LBN 3
+#define        LICENSED_V3_FEATURES_CLOCK_WIDTH 1
+#define        LICENSED_V3_FEATURES_RX_TIMESTAMPS_LBN 4
+#define        LICENSED_V3_FEATURES_RX_TIMESTAMPS_WIDTH 1
+#define        LICENSED_V3_FEATURES_TX_TIMESTAMPS_LBN 5
+#define        LICENSED_V3_FEATURES_TX_TIMESTAMPS_WIDTH 1
+#define        LICENSED_V3_FEATURES_RX_SNIFF_LBN 6
+#define        LICENSED_V3_FEATURES_RX_SNIFF_WIDTH 1
+#define        LICENSED_V3_FEATURES_TX_SNIFF_LBN 7
+#define        LICENSED_V3_FEATURES_TX_SNIFF_WIDTH 1
+#define        LICENSED_V3_FEATURES_PROXY_FILTER_OPS_LBN 8
+#define        LICENSED_V3_FEATURES_PROXY_FILTER_OPS_WIDTH 1
+#define       LICENSED_V3_FEATURES_MASK_LBN 0
+#define       LICENSED_V3_FEATURES_MASK_WIDTH 64
+
 /* TX_TIMESTAMP_EVENT structuredef */
 #define    TX_TIMESTAMP_EVENT_LEN 6
 /* lower 16 bits of timestamp data */
@@ -5258,6 +5591,8 @@
 #define        MC_CMD_INIT_EVQ_IN_FLAG_RX_MERGE_WIDTH 1
 #define        MC_CMD_INIT_EVQ_IN_FLAG_TX_MERGE_LBN 5
 #define        MC_CMD_INIT_EVQ_IN_FLAG_TX_MERGE_WIDTH 1
+#define        MC_CMD_INIT_EVQ_IN_FLAG_USE_TIMER_LBN 6
+#define        MC_CMD_INIT_EVQ_IN_FLAG_USE_TIMER_WIDTH 1
 #define       MC_CMD_INIT_EVQ_IN_TMR_MODE_OFST 20
 /* enum: Disabled */
 #define          MC_CMD_INIT_EVQ_IN_TMR_MODE_DIS 0x0
@@ -5362,6 +5697,8 @@
 #define        MC_CMD_INIT_RXQ_IN_FLAG_PREFIX_WIDTH 1
 #define        MC_CMD_INIT_RXQ_IN_FLAG_DISABLE_SCATTER_LBN 9
 #define        MC_CMD_INIT_RXQ_IN_FLAG_DISABLE_SCATTER_WIDTH 1
+#define        MC_CMD_INIT_RXQ_IN_FLAG_FORCE_EV_MERGING_LBN 10
+#define        MC_CMD_INIT_RXQ_IN_FLAG_FORCE_EV_MERGING_WIDTH 1
 /* Owner ID to use if in buffer mode (zero if physical) */
 #define       MC_CMD_INIT_RXQ_IN_OWNER_ID_OFST 20
 /* The port ID associated with the v-adaptor which should contain this DMAQ. */
@@ -5422,6 +5759,8 @@
 #define          MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K  0x4 /* enum */
 #define        MC_CMD_INIT_RXQ_EXT_IN_FLAG_WANT_OUTER_CLASSES_LBN 18
 #define        MC_CMD_INIT_RXQ_EXT_IN_FLAG_WANT_OUTER_CLASSES_WIDTH 1
+#define        MC_CMD_INIT_RXQ_EXT_IN_FLAG_FORCE_EV_MERGING_LBN 19
+#define        MC_CMD_INIT_RXQ_EXT_IN_FLAG_FORCE_EV_MERGING_WIDTH 1
 /* Owner ID to use if in buffer mode (zero if physical) */
 #define       MC_CMD_INIT_RXQ_EXT_IN_OWNER_ID_OFST 20
 /* The port ID associated with the v-adaptor which should contain this DMAQ. */
@@ -5535,6 +5874,8 @@
 #define        MC_CMD_INIT_TXQ_EXT_IN_FLAG_INNER_IP_CSUM_EN_WIDTH 1
 #define        MC_CMD_INIT_TXQ_EXT_IN_FLAG_INNER_TCP_CSUM_EN_LBN 11
 #define        MC_CMD_INIT_TXQ_EXT_IN_FLAG_INNER_TCP_CSUM_EN_WIDTH 1
+#define        MC_CMD_INIT_TXQ_EXT_IN_FLAG_TSOV2_EN_LBN 12
+#define        MC_CMD_INIT_TXQ_EXT_IN_FLAG_TSOV2_EN_WIDTH 1
 /* Owner ID to use if in buffer mode (zero if physical) */
 #define       MC_CMD_INIT_TXQ_EXT_IN_OWNER_ID_OFST 20
 /* The port ID associated with the v-adaptor which should contain this DMAQ. */
@@ -5747,6 +6088,46 @@
 #define       MC_CMD_PROXY_CONFIGURE_IN_ALLOWED_MCDI_MASK_OFST 44
 #define       MC_CMD_PROXY_CONFIGURE_IN_ALLOWED_MCDI_MASK_LEN 64
 
+/* MC_CMD_PROXY_CONFIGURE_EXT_IN msgrequest */
+#define    MC_CMD_PROXY_CONFIGURE_EXT_IN_LEN 112
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_FLAGS_OFST 0
+#define        MC_CMD_PROXY_CONFIGURE_EXT_IN_ENABLE_LBN 0
+#define        MC_CMD_PROXY_CONFIGURE_EXT_IN_ENABLE_WIDTH 1
+/* Host provides a contiguous memory buffer that contains at least NUM_BLOCKS
+ * of blocks, each of the size REQUEST_BLOCK_SIZE.
+ */
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_STATUS_BUFF_ADDR_OFST 4
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_STATUS_BUFF_ADDR_LEN 8
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_STATUS_BUFF_ADDR_LO_OFST 4
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_STATUS_BUFF_ADDR_HI_OFST 8
+/* Must be a power of 2 */
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_STATUS_BLOCK_SIZE_OFST 12
+/* Host provides a contiguous memory buffer that contains at least NUM_BLOCKS
+ * of blocks, each of the size REPLY_BLOCK_SIZE.
+ */
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_REQUEST_BUFF_ADDR_OFST 16
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_REQUEST_BUFF_ADDR_LEN 8
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_REQUEST_BUFF_ADDR_LO_OFST 16
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_REQUEST_BUFF_ADDR_HI_OFST 20
+/* Must be a power of 2 */
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_REQUEST_BLOCK_SIZE_OFST 24
+/* Host provides a contiguous memory buffer that contains at least NUM_BLOCKS
+ * of blocks, each of the size STATUS_BLOCK_SIZE. This buffer is only needed if
+ * host intends to complete proxied operations by using MC_CMD_PROXY_CMD.
+ */
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_REPLY_BUFF_ADDR_OFST 28
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_REPLY_BUFF_ADDR_LEN 8
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_REPLY_BUFF_ADDR_LO_OFST 28
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_REPLY_BUFF_ADDR_HI_OFST 32
+/* Must be a power of 2, or zero if this buffer is not provided */
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_REPLY_BLOCK_SIZE_OFST 36
+/* Applies to all three buffers */
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_NUM_BLOCKS_OFST 40
+/* A bit mask defining which MCDI operations may be proxied */
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_ALLOWED_MCDI_MASK_OFST 44
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_ALLOWED_MCDI_MASK_LEN 64
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_RESERVED_OFST 108
+
 /* MC_CMD_PROXY_CONFIGURE_OUT msgresponse */
 #define    MC_CMD_PROXY_CONFIGURE_OUT_LEN 0
 
@@ -6323,6 +6704,15 @@
  * client
  */
 #define          MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_RESTRICTIONS  0x2
+/* enum: read properties relating to security rules (Medford-only; for use by
+ * SolarSecure apps, not directly by drivers. See SF-114946-SW.)
+ */
+#define          MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SECURITY_RULE_INFO  0x3
+/* enum: read the list of supported RX filter matches for VXLAN/NVGRE
+ * encapsulated frames, which follow a different match sequence to normal
+ * frames (Medford only)
+ */
+#define          MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_ENCAP_RX_MATCHES  0x4
 
 /* MC_CMD_GET_PARSER_DISP_INFO_OUT msgresponse */
 #define    MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMIN 8
@@ -6356,7 +6746,10 @@
 
 /***********************************/
 /* MC_CMD_PARSER_DISP_RW
- * Direct read/write of parser-dispatcher state (DICPUs and LUE) for debugging
+ * Direct read/write of parser-dispatcher state (DICPUs and LUE) for debugging.
+ * Please note that this interface is only of use to debug tools which have
+ * knowledge of firmware and hardware data structures; nothing here is intended
+ * for use by normal driver code.
  */
 #define MC_CMD_PARSER_DISP_RW 0xe5
 
@@ -6374,6 +6767,12 @@
 #define          MC_CMD_PARSER_DISP_RW_IN_LUE  0x2
 /* enum: Lookup engine (with requested metadata format) */
 #define          MC_CMD_PARSER_DISP_RW_IN_LUE_VERSIONED_METADATA  0x3
+/* enum: RX0 dispatcher CPU (alias for RX_DICPU; Medford has 2 RX DICPUs) */
+#define          MC_CMD_PARSER_DISP_RW_IN_RX0_DICPU  0x0
+/* enum: RX1 dispatcher CPU (only valid for Medford) */
+#define          MC_CMD_PARSER_DISP_RW_IN_RX1_DICPU  0x4
+/* enum: Miscellaneous other state (only valid for Medford) */
+#define          MC_CMD_PARSER_DISP_RW_IN_MISC_STATE  0x5
 /* identifies the type of operation requested */
 #define       MC_CMD_PARSER_DISP_RW_IN_OP_OFST 4
 /* enum: read a word of DICPU DMEM or a LUE entry */
@@ -6382,8 +6781,12 @@
 #define          MC_CMD_PARSER_DISP_RW_IN_WRITE  0x1
 /* enum: read-modify-write a word of DICPU DMEM (not valid for LUE) */
 #define          MC_CMD_PARSER_DISP_RW_IN_RMW  0x2
-/* data memory address or LUE index */
+/* data memory address (DICPU targets) or LUE index (LUE targets) */
 #define       MC_CMD_PARSER_DISP_RW_IN_ADDRESS_OFST 8
+/* selector (for MISC_STATE target) */
+#define       MC_CMD_PARSER_DISP_RW_IN_SELECTOR_OFST 8
+/* enum: Port to datapath mapping */
+#define          MC_CMD_PARSER_DISP_RW_IN_PORT_DP_MAPPING  0x1
 /* value to write (for DMEM writes) */
 #define       MC_CMD_PARSER_DISP_RW_IN_DMEM_WRITE_VALUE_OFST 12
 /* XOR value (for DMEM read-modify-writes: new = (old & mask) ^ value) */
@@ -6408,6 +6811,12 @@
  */
 #define       MC_CMD_PARSER_DISP_RW_OUT_LUE_MGR_STATE_OFST 20
 #define       MC_CMD_PARSER_DISP_RW_OUT_LUE_MGR_STATE_LEN 32
+/* datapath(s) used for each port (for MISC_STATE PORT_DP_MAPPING selector) */
+#define       MC_CMD_PARSER_DISP_RW_OUT_PORT_DP_MAPPING_OFST 0
+#define       MC_CMD_PARSER_DISP_RW_OUT_PORT_DP_MAPPING_LEN 4
+#define       MC_CMD_PARSER_DISP_RW_OUT_PORT_DP_MAPPING_NUM 4
+#define          MC_CMD_PARSER_DISP_RW_OUT_DP0  0x1 /* enum */
+#define          MC_CMD_PARSER_DISP_RW_OUT_DP1  0x2 /* enum */
 
 
 /***********************************/
@@ -7071,6 +7480,24 @@
 #define    MC_CMD_GET_CAPABILITIES_OUT_LEN 20
 /* First word of flags. */
 #define       MC_CMD_GET_CAPABILITIES_OUT_FLAGS1_OFST 0
+#define        MC_CMD_GET_CAPABILITIES_OUT_VPORT_RECONFIGURE_LBN 3
+#define        MC_CMD_GET_CAPABILITIES_OUT_VPORT_RECONFIGURE_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_OUT_TX_STRIPING_LBN 4
+#define        MC_CMD_GET_CAPABILITIES_OUT_TX_STRIPING_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_OUT_VADAPTOR_QUERY_LBN 5
+#define        MC_CMD_GET_CAPABILITIES_OUT_VADAPTOR_QUERY_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_OUT_EVB_PORT_VLAN_RESTRICT_LBN 6
+#define        MC_CMD_GET_CAPABILITIES_OUT_EVB_PORT_VLAN_RESTRICT_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_OUT_DRV_ATTACH_PREBOOT_LBN 7
+#define        MC_CMD_GET_CAPABILITIES_OUT_DRV_ATTACH_PREBOOT_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_OUT_RX_FORCE_EVENT_MERGING_LBN 8
+#define        MC_CMD_GET_CAPABILITIES_OUT_RX_FORCE_EVENT_MERGING_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_OUT_SET_MAC_ENHANCED_LBN 9
+#define        MC_CMD_GET_CAPABILITIES_OUT_SET_MAC_ENHANCED_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_OUT_UNKNOWN_UCAST_DST_FILTER_ALWAYS_MULTI_RECIPIENT_LBN 10
+#define        MC_CMD_GET_CAPABILITIES_OUT_UNKNOWN_UCAST_DST_FILTER_ALWAYS_MULTI_RECIPIENT_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_OUT_VADAPTOR_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED_LBN 11
+#define        MC_CMD_GET_CAPABILITIES_OUT_VADAPTOR_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED_WIDTH 1
 #define        MC_CMD_GET_CAPABILITIES_OUT_TX_MAC_SECURITY_FILTERING_LBN 12
 #define        MC_CMD_GET_CAPABILITIES_OUT_TX_MAC_SECURITY_FILTERING_WIDTH 1
 #define        MC_CMD_GET_CAPABILITIES_OUT_ADDITIONAL_RSS_MODES_LBN 13
@@ -7138,6 +7565,8 @@
 #define          MC_CMD_GET_CAPABILITIES_OUT_RXDP_TEST_FW_RX_HDR_SPLIT  0x107
 /* enum: RXDP Test firmware image 8 */
 #define          MC_CMD_GET_CAPABILITIES_OUT_RXDP_TEST_FW_DISABLE_DL  0x108
+/* enum: RXDP Test firmware image 9 */
+#define          MC_CMD_GET_CAPABILITIES_OUT_RXDP_TEST_FW_DOORBELL_DELAY  0x10b
 /* TxDPCPU firmware id. */
 #define       MC_CMD_GET_CAPABILITIES_OUT_TX_DPCPU_FW_ID_OFST 6
 #define       MC_CMD_GET_CAPABILITIES_OUT_TX_DPCPU_FW_ID_LEN 2
@@ -7153,6 +7582,8 @@
 #define          MC_CMD_GET_CAPABILITIES_OUT_TXDP_TEST_FW_TSO_EDIT  0x101
 /* enum: TXDP Test firmware image 2 */
 #define          MC_CMD_GET_CAPABILITIES_OUT_TXDP_TEST_FW_PACKET_EDITS  0x102
+/* enum: TXDP CSR bus test firmware */
+#define          MC_CMD_GET_CAPABILITIES_OUT_TXDP_TEST_FW_CSR  0x103
 #define       MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_VERSION_OFST 8
 #define       MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_VERSION_LEN 2
 #define        MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_VERSION_REV_LBN 0
@@ -7227,6 +7658,258 @@
 /* Licensed capabilities */
 #define       MC_CMD_GET_CAPABILITIES_OUT_LICENSE_CAPABILITIES_OFST 16
 
+/* MC_CMD_GET_CAPABILITIES_V2_IN msgrequest */
+#define    MC_CMD_GET_CAPABILITIES_V2_IN_LEN 0
+
+/* MC_CMD_GET_CAPABILITIES_V2_OUT msgresponse */
+#define    MC_CMD_GET_CAPABILITIES_V2_OUT_LEN 72
+/* First word of flags. */
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_FLAGS1_OFST 0
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_VPORT_RECONFIGURE_LBN 3
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_VPORT_RECONFIGURE_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_STRIPING_LBN 4
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_STRIPING_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_VADAPTOR_QUERY_LBN 5
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_VADAPTOR_QUERY_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_EVB_PORT_VLAN_RESTRICT_LBN 6
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_EVB_PORT_VLAN_RESTRICT_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_DRV_ATTACH_PREBOOT_LBN 7
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_DRV_ATTACH_PREBOOT_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_FORCE_EVENT_MERGING_LBN 8
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_FORCE_EVENT_MERGING_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_SET_MAC_ENHANCED_LBN 9
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_SET_MAC_ENHANCED_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_UNKNOWN_UCAST_DST_FILTER_ALWAYS_MULTI_RECIPIENT_LBN 10
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_UNKNOWN_UCAST_DST_FILTER_ALWAYS_MULTI_RECIPIENT_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_VADAPTOR_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED_LBN 11
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_VADAPTOR_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_MAC_SECURITY_FILTERING_LBN 12
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_MAC_SECURITY_FILTERING_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_ADDITIONAL_RSS_MODES_LBN 13
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_ADDITIONAL_RSS_MODES_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_QBB_LBN 14
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_QBB_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_PACKED_STREAM_VAR_BUFFERS_LBN 15
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_PACKED_STREAM_VAR_BUFFERS_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_RSS_LIMITED_LBN 16
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_RSS_LIMITED_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_PACKED_STREAM_LBN 17
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_PACKED_STREAM_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_INCLUDE_FCS_LBN 18
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_INCLUDE_FCS_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_VLAN_INSERTION_LBN 19
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_VLAN_INSERTION_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_VLAN_STRIPPING_LBN 20
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_VLAN_STRIPPING_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TSO_LBN 21
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TSO_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_PREFIX_LEN_0_LBN 22
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_PREFIX_LEN_0_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_PREFIX_LEN_14_LBN 23
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_PREFIX_LEN_14_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_TIMESTAMP_LBN 24
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_TIMESTAMP_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_BATCHING_LBN 25
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_BATCHING_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_MCAST_FILTER_CHAINING_LBN 26
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_MCAST_FILTER_CHAINING_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_PM_AND_RXDP_COUNTERS_LBN 27
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_PM_AND_RXDP_COUNTERS_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_DISABLE_SCATTER_LBN 28
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_DISABLE_SCATTER_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_MCAST_UDP_LOOPBACK_LBN 29
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_MCAST_UDP_LOOPBACK_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_EVB_LBN 30
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_EVB_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_VXLAN_NVGRE_LBN 31
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_VXLAN_NVGRE_WIDTH 1
+/* RxDPCPU firmware id. */
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_RX_DPCPU_FW_ID_OFST 4
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_RX_DPCPU_FW_ID_LEN 2
+/* enum: Standard RXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP  0x0
+/* enum: Low latency RXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_LOW_LATENCY  0x1
+/* enum: Packed stream RXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_PACKED_STREAM  0x2
+/* enum: BIST RXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_BIST  0x10a
+/* enum: RXDP Test firmware image 1 */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_TEST_FW_TO_MC_CUT_THROUGH  0x101
+/* enum: RXDP Test firmware image 2 */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_TEST_FW_TO_MC_STORE_FORWARD  0x102
+/* enum: RXDP Test firmware image 3 */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_TEST_FW_TO_MC_STORE_FORWARD_FIRST  0x103
+/* enum: RXDP Test firmware image 4 */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_TEST_EVERY_EVENT_BATCHABLE  0x104
+/* enum: RXDP Test firmware image 5 */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_TEST_BACKPRESSURE  0x105
+/* enum: RXDP Test firmware image 6 */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_TEST_FW_PACKET_EDITS  0x106
+/* enum: RXDP Test firmware image 7 */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_TEST_FW_RX_HDR_SPLIT  0x107
+/* enum: RXDP Test firmware image 8 */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_TEST_FW_DISABLE_DL  0x108
+/* enum: RXDP Test firmware image 9 */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_TEST_FW_DOORBELL_DELAY  0x10b
+/* TxDPCPU firmware id. */
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_TX_DPCPU_FW_ID_OFST 6
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_TX_DPCPU_FW_ID_LEN 2
+/* enum: Standard TXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXDP  0x0
+/* enum: Low latency TXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXDP_LOW_LATENCY  0x1
+/* enum: High packet rate TXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXDP_HIGH_PACKET_RATE  0x3
+/* enum: BIST TXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXDP_BIST  0x12d
+/* enum: TXDP Test firmware image 1 */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXDP_TEST_FW_TSO_EDIT  0x101
+/* enum: TXDP Test firmware image 2 */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXDP_TEST_FW_PACKET_EDITS  0x102
+/* enum: TXDP CSR bus test firmware */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXDP_TEST_FW_CSR  0x103
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_VERSION_OFST 8
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_VERSION_LEN 2
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_VERSION_REV_LBN 0
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_VERSION_REV_WIDTH 12
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_VERSION_TYPE_LBN 12
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_VERSION_TYPE_WIDTH 4
+/* enum: reserved value - do not use (may indicate alternative interpretation
+ * of REV field in future)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_RESERVED  0x0
+/* enum: Trivial RX PD firmware for early Huntington development (Huntington
+ * development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_FIRST_PKT  0x1
+/* enum: RX PD firmware with approximately Siena-compatible behaviour
+ * (Huntington development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_SIENA_COMPAT  0x2
+/* enum: Virtual switching (full feature) RX PD production firmware */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_VSWITCH  0x3
+/* enum: siena_compat variant RX PD firmware using PM rather than MAC
+ * (Huntington development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_SIENA_COMPAT_PM  0x4
+/* enum: Low latency RX PD production firmware */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_LOW_LATENCY  0x5
+/* enum: Packed stream RX PD production firmware */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_PACKED_STREAM  0x6
+/* enum: RX PD firmware handling layer 2 only for high packet rate performance
+ * tests (Medford development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_LAYER2_PERF  0x7
+/* enum: RX PD firmware for GUE parsing prototype (Medford development only) */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_TESTFW_GUE_PROTOTYPE  0xe
+/* enum: RX PD firmware parsing but not filtering network overlay tunnel
+ * encapsulations (Medford development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_TESTFW_ENCAP_PARSING_ONLY  0xf
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_VERSION_OFST 10
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_VERSION_LEN 2
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_VERSION_REV_LBN 0
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_VERSION_REV_WIDTH 12
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_VERSION_TYPE_LBN 12
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_VERSION_TYPE_WIDTH 4
+/* enum: reserved value - do not use (may indicate alternative interpretation
+ * of REV field in future)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_TYPE_RESERVED  0x0
+/* enum: Trivial TX PD firmware for early Huntington development (Huntington
+ * development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_TYPE_FIRST_PKT  0x1
+/* enum: TX PD firmware with approximately Siena-compatible behaviour
+ * (Huntington development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_TYPE_SIENA_COMPAT  0x2
+/* enum: Virtual switching (full feature) TX PD production firmware */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_TYPE_VSWITCH  0x3
+/* enum: siena_compat variant TX PD firmware using PM rather than MAC
+ * (Huntington development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_TYPE_SIENA_COMPAT_PM  0x4
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_TYPE_LOW_LATENCY  0x5 /* enum */
+/* enum: TX PD firmware handling layer 2 only for high packet rate performance
+ * tests (Medford development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_TYPE_LAYER2_PERF  0x7
+/* enum: RX PD firmware for GUE parsing prototype (Medford development only) */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_TYPE_TESTFW_GUE_PROTOTYPE  0xe
+/* Hardware capabilities of NIC */
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_HW_CAPABILITIES_OFST 12
+/* Licensed capabilities */
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_LICENSE_CAPABILITIES_OFST 16
+/* Second word of flags. Not present on older firmware (check the length). */
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_FLAGS2_OFST 20
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TSO_V2_LBN 0
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TSO_V2_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TSO_V2_ENCAP_LBN 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TSO_V2_ENCAP_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_EVQ_TIMER_CTRL_LBN 2
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_EVQ_TIMER_CTRL_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_EVENT_CUT_THROUGH_LBN 3
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_EVENT_CUT_THROUGH_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_CUT_THROUGH_LBN 4
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_CUT_THROUGH_WIDTH 1
+/* Number of FATSOv2 contexts per datapath supported by this NIC. Not present
+ * on older firmware (check the length).
+ */
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TSO_V2_N_CONTEXTS_OFST 24
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TSO_V2_N_CONTEXTS_LEN 2
+/* One byte per PF containing the number of the external port assigned to this
+ * PF, indexed by PF number. Special values indicate that a PF is either not
+ * present or not assigned.
+ */
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_PFS_TO_PORTS_ASSIGNMENT_OFST 26
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_PFS_TO_PORTS_ASSIGNMENT_LEN 1
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_PFS_TO_PORTS_ASSIGNMENT_NUM 16
+/* enum: The caller is not permitted to access information on this PF. */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_ACCESS_NOT_PERMITTED  0xff
+/* enum: PF does not exist. */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_PF_NOT_PRESENT  0xfe
+/* enum: PF does exist but is not assigned to any external port. */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_PF_NOT_ASSIGNED  0xfd
+/* enum: This value indicates that PF is assigned, but it cannot be expressed
+ * in this field. It is intended for a possible future situation where a more
+ * complex scheme of PFs to ports mapping is being used. The future driver
+ * should look for a new field supporting the new scheme. The current/old
+ * driver should treat this value as PF_NOT_ASSIGNED.
+ */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_INCOMPATIBLE_ASSIGNMENT  0xfc
+/* One byte per PF containing the number of its VFs, indexed by PF number. A
+ * special value indicates that a PF is not present.
+ */
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_NUM_VFS_PER_PF_OFST 42
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_NUM_VFS_PER_PF_LEN 1
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_NUM_VFS_PER_PF_NUM 16
+/* enum: The caller is not permitted to access information on this PF. */
+/*               MC_CMD_GET_CAPABILITIES_V2_OUT_ACCESS_NOT_PERMITTED  0xff */
+/* enum: PF does not exist. */
+/*               MC_CMD_GET_CAPABILITIES_V2_OUT_PF_NOT_PRESENT  0xfe */
+/* Number of VIs available for each external port */
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_NUM_VIS_PER_PORT_OFST 58
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_NUM_VIS_PER_PORT_LEN 2
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_NUM_VIS_PER_PORT_NUM 4
+/* Size of RX descriptor cache expressed as binary logarithm The actual size
+ * equals (2 ^ RX_DESC_CACHE_SIZE)
+ */
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_RX_DESC_CACHE_SIZE_OFST 66
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_RX_DESC_CACHE_SIZE_LEN 1
+/* Size of TX descriptor cache expressed as binary logarithm The actual size
+ * equals (2 ^ TX_DESC_CACHE_SIZE)
+ */
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_TX_DESC_CACHE_SIZE_OFST 67
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_TX_DESC_CACHE_SIZE_LEN 1
+/* Total number of available PIO buffers */
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_NUM_PIO_BUFFS_OFST 68
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_NUM_PIO_BUFFS_LEN 2
+/* Size of a single PIO buffer */
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_SIZE_PIO_BUFF_OFST 70
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_SIZE_PIO_BUFF_LEN 2
+
 
 /***********************************/
 /* MC_CMD_V2_EXTN
@@ -7475,6 +8158,25 @@
 
 
 /***********************************/
+/* MC_CMD_VSWITCH_QUERY
+ * read some config of v-switch. For now this command is an empty placeholder.
+ * It may be used to check if a v-switch is connected to a given EVB port (if
+ * not, then the command returns ENOENT).
+ */
+#define MC_CMD_VSWITCH_QUERY 0x63
+
+#define MC_CMD_0x63_PRIVILEGE_CTG SRIOV_CTG_GENERAL
+
+/* MC_CMD_VSWITCH_QUERY_IN msgrequest */
+#define    MC_CMD_VSWITCH_QUERY_IN_LEN 4
+/* The port to which the v-switch is connected. */
+#define       MC_CMD_VSWITCH_QUERY_IN_UPSTREAM_PORT_ID_OFST 0
+
+/* MC_CMD_VSWITCH_QUERY_OUT msgresponse */
+#define    MC_CMD_VSWITCH_QUERY_OUT_LEN 0
+
+
+/***********************************/
 /* MC_CMD_VPORT_ALLOC
  * allocate a v-port.
  */
@@ -7510,6 +8212,8 @@
 #define       MC_CMD_VPORT_ALLOC_IN_FLAGS_OFST 8
 #define        MC_CMD_VPORT_ALLOC_IN_FLAG_AUTO_PORT_LBN 0
 #define        MC_CMD_VPORT_ALLOC_IN_FLAG_AUTO_PORT_WIDTH 1
+#define        MC_CMD_VPORT_ALLOC_IN_FLAG_VLAN_RESTRICT_LBN 1
+#define        MC_CMD_VPORT_ALLOC_IN_FLAG_VLAN_RESTRICT_WIDTH 1
 /* The number of VLAN tags to insert/remove. An error will be returned if
  * incompatible with the number of VLAN tags specified for the upstream
  * v-switch.
@@ -7561,6 +8265,8 @@
 #define       MC_CMD_VADAPTOR_ALLOC_IN_FLAGS_OFST 8
 #define        MC_CMD_VADAPTOR_ALLOC_IN_FLAG_AUTO_VADAPTOR_LBN 0
 #define        MC_CMD_VADAPTOR_ALLOC_IN_FLAG_AUTO_VADAPTOR_WIDTH 1
+#define        MC_CMD_VADAPTOR_ALLOC_IN_FLAG_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED_LBN 1
+#define        MC_CMD_VADAPTOR_ALLOC_IN_FLAG_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED_WIDTH 1
 /* The number of VLAN tags to strip on receive */
 #define       MC_CMD_VADAPTOR_ALLOC_IN_NUM_VLANS_OFST 12
 /* The number of VLAN tags to transparently insert/remove. */
@@ -7639,6 +8345,29 @@
 
 
 /***********************************/
+/* MC_CMD_VADAPTOR_QUERY
+ * read some config of v-adaptor.
+ */
+#define MC_CMD_VADAPTOR_QUERY 0x61
+
+#define MC_CMD_0x61_PRIVILEGE_CTG SRIOV_CTG_GENERAL
+
+/* MC_CMD_VADAPTOR_QUERY_IN msgrequest */
+#define    MC_CMD_VADAPTOR_QUERY_IN_LEN 4
+/* The port to which the v-adaptor is connected. */
+#define       MC_CMD_VADAPTOR_QUERY_IN_UPSTREAM_PORT_ID_OFST 0
+
+/* MC_CMD_VADAPTOR_QUERY_OUT msgresponse */
+#define    MC_CMD_VADAPTOR_QUERY_OUT_LEN 12
+/* The EVB port flags as defined at MC_CMD_VPORT_ALLOC. */
+#define       MC_CMD_VADAPTOR_QUERY_OUT_PORT_FLAGS_OFST 0
+/* The v-adaptor flags as defined at MC_CMD_VADAPTOR_ALLOC. */
+#define       MC_CMD_VADAPTOR_QUERY_OUT_VADAPTOR_FLAGS_OFST 4
+/* The number of VLAN tags that may still be added */
+#define       MC_CMD_VADAPTOR_QUERY_OUT_NUM_AVAILABLE_VLAN_TAGS_OFST 8
+
+
+/***********************************/
 /* MC_CMD_EVB_PORT_ASSIGN
  * assign a port to a PCI function.
  */
@@ -7875,10 +8604,17 @@
 #define    MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN 8
 /* The handle of the RSS context */
 #define       MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID_OFST 0
-/* Hash control flags. The _EN bits are always supported. The _MODE bits only
- * work when the firmware reports ADDITIONAL_RSS_MODES in
- * MC_CMD_GET_CAPABILITIES and override the _EN bits if any of them are not 0.
- * See the RSS_MODE structure for the meaning of the mode bits.
+/* Hash control flags. The _EN bits are always supported, but new modes are
+ * available when ADDITIONAL_RSS_MODES is reported by MC_CMD_GET_CAPABILITIES:
+ * in this case, the MODE fields may be set to non-zero values, and will take
+ * effect regardless of the settings of the _EN flags. See the RSS_MODE
+ * structure for the meaning of the mode bits. Drivers must check the
+ * capability before trying to set any _MODE fields, as older firmware will
+ * reject any attempt to set the FLAGS field to a value > 0xff with EINVAL. In
+ * the case where all the _MODE flags are zero, the _EN flags take effect,
+ * providing backward compatibility for existing drivers. (Setting all _MODE
+ * *and* all _EN flags to zero is valid, to disable RSS spreading for that
+ * particular packet type.)
  */
 #define       MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_FLAGS_OFST 4
 #define        MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN_LBN 0
@@ -7923,11 +8659,18 @@
 
 /* MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT msgresponse */
 #define    MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN 8
-/* Hash control flags. If any _MODE bits are non-zero (which will only be true
- * when the firmware reports ADDITIONAL_RSS_MODES) then the _EN bits should be
- * disregarded (but are guaranteed to be consistent with the _MODE bits if
- * RSS_CONTEXT_SET_FLAGS has never been called for this context since it was
- * allocated).
+/* Hash control flags. If all _MODE bits are zero (which will always be true
+ * for older firmware which does not report the ADDITIONAL_RSS_MODES
+ * capability), the _EN bits report the state. If any _MODE bits are non-zero
+ * (which will only be true when the firmware reports ADDITIONAL_RSS_MODES)
+ * then the _EN bits should be disregarded, although the _MODE flags are
+ * guaranteed to be consistent with the _EN flags for a freshly-allocated RSS
+ * context and in the case where the _EN flags were used in the SET. This
+ * provides backward compatibility: old drivers will not be attempting to
+ * derive any meaning from the _MODE bits (and can never set them to any value
+ * not representable by the _EN bits); new drivers can always determine the
+ * mode by looking only at the _MODE bits; the value returned by a GET can
+ * always be used for a SET regardless of old/new driver vs. old/new firmware.
  */
 #define       MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_FLAGS_OFST 4
 #define        MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_IPV4_EN_LBN 0
@@ -8155,6 +8898,74 @@
 
 
 /***********************************/
+/* MC_CMD_VPORT_RECONFIGURE
+ * Replace VLAN tags and/or MAC addresses of an existing v-port. If the v-port
+ * has already been passed to another function (v-port's user), then that
+ * function will be reset before applying the changes.
+ */
+#define MC_CMD_VPORT_RECONFIGURE 0xeb
+
+#define MC_CMD_0xeb_PRIVILEGE_CTG SRIOV_CTG_GENERAL
+
+/* MC_CMD_VPORT_RECONFIGURE_IN msgrequest */
+#define    MC_CMD_VPORT_RECONFIGURE_IN_LEN 44
+/* The handle of the v-port */
+#define       MC_CMD_VPORT_RECONFIGURE_IN_VPORT_ID_OFST 0
+/* Flags requesting what should be changed. */
+#define       MC_CMD_VPORT_RECONFIGURE_IN_FLAGS_OFST 4
+#define        MC_CMD_VPORT_RECONFIGURE_IN_REPLACE_VLAN_TAGS_LBN 0
+#define        MC_CMD_VPORT_RECONFIGURE_IN_REPLACE_VLAN_TAGS_WIDTH 1
+#define        MC_CMD_VPORT_RECONFIGURE_IN_REPLACE_MACADDRS_LBN 1
+#define        MC_CMD_VPORT_RECONFIGURE_IN_REPLACE_MACADDRS_WIDTH 1
+/* The number of VLAN tags to insert/remove. An error will be returned if
+ * incompatible with the number of VLAN tags specified for the upstream
+ * v-switch.
+ */
+#define       MC_CMD_VPORT_RECONFIGURE_IN_NUM_VLAN_TAGS_OFST 8
+/* The actual VLAN tags to insert/remove */
+#define       MC_CMD_VPORT_RECONFIGURE_IN_VLAN_TAGS_OFST 12
+#define        MC_CMD_VPORT_RECONFIGURE_IN_VLAN_TAG_0_LBN 0
+#define        MC_CMD_VPORT_RECONFIGURE_IN_VLAN_TAG_0_WIDTH 16
+#define        MC_CMD_VPORT_RECONFIGURE_IN_VLAN_TAG_1_LBN 16
+#define        MC_CMD_VPORT_RECONFIGURE_IN_VLAN_TAG_1_WIDTH 16
+/* The number of MAC addresses to add */
+#define       MC_CMD_VPORT_RECONFIGURE_IN_NUM_MACADDRS_OFST 16
+/* MAC addresses to add */
+#define       MC_CMD_VPORT_RECONFIGURE_IN_MACADDRS_OFST 20
+#define       MC_CMD_VPORT_RECONFIGURE_IN_MACADDRS_LEN 6
+#define       MC_CMD_VPORT_RECONFIGURE_IN_MACADDRS_NUM 4
+
+/* MC_CMD_VPORT_RECONFIGURE_OUT msgresponse */
+#define    MC_CMD_VPORT_RECONFIGURE_OUT_LEN 4
+#define       MC_CMD_VPORT_RECONFIGURE_OUT_FLAGS_OFST 0
+#define        MC_CMD_VPORT_RECONFIGURE_OUT_RESET_DONE_LBN 0
+#define        MC_CMD_VPORT_RECONFIGURE_OUT_RESET_DONE_WIDTH 1
+
+
+/***********************************/
+/* MC_CMD_EVB_PORT_QUERY
+ * read some config of v-port.
+ */
+#define MC_CMD_EVB_PORT_QUERY 0x62
+
+#define MC_CMD_0x62_PRIVILEGE_CTG SRIOV_CTG_GENERAL
+
+/* MC_CMD_EVB_PORT_QUERY_IN msgrequest */
+#define    MC_CMD_EVB_PORT_QUERY_IN_LEN 4
+/* The handle of the v-port */
+#define       MC_CMD_EVB_PORT_QUERY_IN_PORT_ID_OFST 0
+
+/* MC_CMD_EVB_PORT_QUERY_OUT msgresponse */
+#define    MC_CMD_EVB_PORT_QUERY_OUT_LEN 8
+/* The EVB port flags as defined at MC_CMD_VPORT_ALLOC. */
+#define       MC_CMD_EVB_PORT_QUERY_OUT_PORT_FLAGS_OFST 0
+/* The number of VLAN tags that may be used on a v-adaptor connected to this
+ * EVB port.
+ */
+#define       MC_CMD_EVB_PORT_QUERY_OUT_NUM_AVAILABLE_VLAN_TAGS_OFST 4
+
+
+/***********************************/
 /* MC_CMD_DUMP_BUFTBL_ENTRIES
  * Dump buffer table entries, mainly for command client debug use. Dumps
  * absolute entries, and does not use chunk handles. All entries must be in
@@ -8196,6 +9007,14 @@
 #define       MC_CMD_SET_RXDP_CONFIG_IN_DATA_OFST 0
 #define        MC_CMD_SET_RXDP_CONFIG_IN_PAD_HOST_DMA_LBN 0
 #define        MC_CMD_SET_RXDP_CONFIG_IN_PAD_HOST_DMA_WIDTH 1
+#define        MC_CMD_SET_RXDP_CONFIG_IN_PAD_HOST_LEN_LBN 1
+#define        MC_CMD_SET_RXDP_CONFIG_IN_PAD_HOST_LEN_WIDTH 2
+/* enum: pad to 64 bytes */
+#define          MC_CMD_SET_RXDP_CONFIG_IN_PAD_HOST_64  0x0
+/* enum: pad to 128 bytes (Medford only) */
+#define          MC_CMD_SET_RXDP_CONFIG_IN_PAD_HOST_128  0x1
+/* enum: pad to 256 bytes (Medford only) */
+#define          MC_CMD_SET_RXDP_CONFIG_IN_PAD_HOST_256   0x2
 
 /* MC_CMD_SET_RXDP_CONFIG_OUT msgresponse */
 #define    MC_CMD_SET_RXDP_CONFIG_OUT_LEN 0
@@ -8217,6 +9036,10 @@
 #define       MC_CMD_GET_RXDP_CONFIG_OUT_DATA_OFST 0
 #define        MC_CMD_GET_RXDP_CONFIG_OUT_PAD_HOST_DMA_LBN 0
 #define        MC_CMD_GET_RXDP_CONFIG_OUT_PAD_HOST_DMA_WIDTH 1
+#define        MC_CMD_GET_RXDP_CONFIG_OUT_PAD_HOST_LEN_LBN 1
+#define        MC_CMD_GET_RXDP_CONFIG_OUT_PAD_HOST_LEN_WIDTH 2
+/*             Enum values, see field(s): */
+/*                MC_CMD_SET_RXDP_CONFIG/MC_CMD_SET_RXDP_CONFIG_IN/PAD_HOST_LEN */
 
 
 /***********************************/
@@ -8788,32 +9611,38 @@
 #define       MC_CMD_KR_TUNE_RXEQ_GET_OUT_PARAM_MAXNUM 63
 #define        MC_CMD_KR_TUNE_RXEQ_GET_OUT_PARAM_ID_LBN 0
 #define        MC_CMD_KR_TUNE_RXEQ_GET_OUT_PARAM_ID_WIDTH 8
-/* enum: Attenuation (0-15, TBD for Medford) */
+/* enum: Attenuation (0-15, Huntington) */
 #define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_ATT  0x0
-/* enum: CTLE Boost (0-15, TBD for Medford) */
+/* enum: CTLE Boost (0-15, Huntington) */
 #define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_BOOST  0x1
-/* enum: Edge DFE Tap1 (0 - max negative, 64 - zero, 127 - max positive, TBD
- * for Medford)
+/* enum: Edge DFE Tap1 (Huntington - 0 - max negative, 64 - zero, 127 - max
+ * positive, Medford - 0-31)
  */
 #define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_EDFE_TAP1  0x2
-/* enum: Edge DFE Tap2 (0 - max negative, 32 - zero, 63 - max positive, TBD for
- * Medford)
+/* enum: Edge DFE Tap2 (Huntington - 0 - max negative, 32 - zero, 63 - max
+ * positive, Medford - 0-31)
  */
 #define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_EDFE_TAP2  0x3
-/* enum: Edge DFE Tap3 (0 - max negative, 32 - zero, 63 - max positive, TBD for
- * Medford)
+/* enum: Edge DFE Tap3 (Huntington - 0 - max negative, 32 - zero, 63 - max
+ * positive, Medford - 0-16)
  */
 #define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_EDFE_TAP3  0x4
-/* enum: Edge DFE Tap4 (0 - max negative, 32 - zero, 63 - max positive, TBD for
- * Medford)
+/* enum: Edge DFE Tap4 (Huntington - 0 - max negative, 32 - zero, 63 - max
+ * positive, Medford - 0-16)
  */
 #define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_EDFE_TAP4  0x5
-/* enum: Edge DFE Tap5 (0 - max negative, 32 - zero, 63 - max positive, TBD for
- * Medford)
+/* enum: Edge DFE Tap5 (Huntington - 0 - max negative, 32 - zero, 63 - max
+ * positive, Medford - 0-16)
  */
 #define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_EDFE_TAP5  0x6
-/* enum: Edge DFE DLEV (TBD for Medford) */
+/* enum: Edge DFE DLEV (0-128 for Medford) */
 #define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_EDFE_DLEV  0x7
+/* enum: Variable Gain Amplifier (0-15, Medford) */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_VGA  0x8
+/* enum: CTLE EQ Capacitor (0-15, Medford) */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_CTLE_EQC  0x9
+/* enum: CTLE EQ Resistor (0-7, Medford) */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_CTLE_EQRES  0xa
 #define        MC_CMD_KR_TUNE_RXEQ_GET_OUT_PARAM_LANE_LBN 8
 #define        MC_CMD_KR_TUNE_RXEQ_GET_OUT_PARAM_LANE_WIDTH 3
 #define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_LANE_0  0x0 /* enum */
@@ -8885,26 +9714,32 @@
 #define       MC_CMD_KR_TUNE_TXEQ_GET_OUT_PARAM_MAXNUM 63
 #define        MC_CMD_KR_TUNE_TXEQ_GET_OUT_PARAM_ID_LBN 0
 #define        MC_CMD_KR_TUNE_TXEQ_GET_OUT_PARAM_ID_WIDTH 8
-/* enum: TX Amplitude */
+/* enum: TX Amplitude (Huntington, Medford) */
 #define          MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_LEV  0x0
-/* enum: De-Emphasis Tap1 Magnitude (0-7) */
+/* enum: De-Emphasis Tap1 Magnitude (0-7) (Huntington) */
 #define          MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_MODE  0x1
 /* enum: De-Emphasis Tap1 Fine */
 #define          MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_DTLEV  0x2
-/* enum: De-Emphasis Tap2 Magnitude (0-6) */
+/* enum: De-Emphasis Tap2 Magnitude (0-6) (Huntington) */
 #define          MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_D2  0x3
-/* enum: De-Emphasis Tap2 Fine */
+/* enum: De-Emphasis Tap2 Fine (Huntington) */
 #define          MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_D2TLEV  0x4
-/* enum: Pre-Emphasis Magnitude */
+/* enum: Pre-Emphasis Magnitude (Huntington) */
 #define          MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_E  0x5
-/* enum: Pre-Emphasis Fine */
+/* enum: Pre-Emphasis Fine (Huntington) */
 #define          MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_ETLEV  0x6
-/* enum: TX Slew Rate Coarse control */
+/* enum: TX Slew Rate Coarse control (Huntington) */
 #define          MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_PREDRV_DLY  0x7
-/* enum: TX Slew Rate Fine control */
+/* enum: TX Slew Rate Fine control (Huntington) */
 #define          MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_SR_SET  0x8
-/* enum: TX Termination Impedance control */
+/* enum: TX Termination Impedance control (Huntington) */
 #define          MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_RT_SET  0x9
+/* enum: TX Amplitude Fine control (Medford) */
+#define          MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_LEV_FINE  0xa
+/* enum: Pre-shoot Tap (Medford) */
+#define          MC_CMD_KR_TUNE_TXEQ_GET_OUT_TAP_ADV  0xb
+/* enum: De-emphasis Tap (Medford) */
+#define          MC_CMD_KR_TUNE_TXEQ_GET_OUT_TAP_DLY  0xc
 #define        MC_CMD_KR_TUNE_TXEQ_GET_OUT_PARAM_LANE_LBN 8
 #define        MC_CMD_KR_TUNE_TXEQ_GET_OUT_PARAM_LANE_WIDTH 3
 #define          MC_CMD_KR_TUNE_TXEQ_GET_OUT_LANE_0  0x0 /* enum */
@@ -9086,8 +9921,16 @@
 #define          MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_DFE_TAP4  0x5
 /* enum: DFE Tap5 (0 - max negative, 32 - zero, 63 - max positive) */
 #define          MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_DFE_TAP5  0x6
+/* enum: DFE DLev */
+#define          MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_DFE_DLEV  0x7
+/* enum: Figure of Merit */
+#define          MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_FOM  0x8
+/* enum: CTLE EQ Capacitor (HF Gain) */
+#define          MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_CTLE_EQC  0x9
+/* enum: CTLE EQ Resistor (DC Gain) */
+#define          MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_CTLE_EQRES  0xa
 #define        MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_PARAM_LANE_LBN 8
-#define        MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_PARAM_LANE_WIDTH 4
+#define        MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_PARAM_LANE_WIDTH 5
 #define          MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_0  0x0 /* enum */
 #define          MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_1  0x1 /* enum */
 #define          MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_2  0x2 /* enum */
@@ -9096,12 +9939,57 @@
 #define          MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_5  0x5 /* enum */
 #define          MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_6  0x6 /* enum */
 #define          MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_7  0x7 /* enum */
-#define          MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_ALL  0x8 /* enum */
-#define        MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_RESERVED_LBN 12
-#define        MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_RESERVED_WIDTH 12
+#define          MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_8  0x8 /* enum */
+#define          MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_9  0x9 /* enum */
+#define          MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_10  0xa /* enum */
+#define          MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_11  0xb /* enum */
+#define          MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_12  0xc /* enum */
+#define          MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_13  0xd /* enum */
+#define          MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_14  0xe /* enum */
+#define          MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_15  0xf /* enum */
+#define          MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_ALL  0x10 /* enum */
+#define        MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_PARAM_AUTOCAL_LBN 13
+#define        MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_PARAM_AUTOCAL_WIDTH 1
+#define        MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_RESERVED_LBN 14
+#define        MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_RESERVED_WIDTH 10
 #define        MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_PARAM_CURRENT_LBN 24
 #define        MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_PARAM_CURRENT_WIDTH 8
 
+/* MC_CMD_PCIE_TUNE_RXEQ_SET_IN msgrequest */
+#define    MC_CMD_PCIE_TUNE_RXEQ_SET_IN_LENMIN 8
+#define    MC_CMD_PCIE_TUNE_RXEQ_SET_IN_LENMAX 252
+#define    MC_CMD_PCIE_TUNE_RXEQ_SET_IN_LEN(num) (4+4*(num))
+/* Requested operation */
+#define       MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PCIE_TUNE_OP_OFST 0
+#define       MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PCIE_TUNE_OP_LEN 1
+/* Align the arguments to 32 bits */
+#define       MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PCIE_TUNE_RSVD_OFST 1
+#define       MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PCIE_TUNE_RSVD_LEN 3
+/* RXEQ Parameter */
+#define       MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PARAM_OFST 4
+#define       MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PARAM_LEN 4
+#define       MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PARAM_MINNUM 1
+#define       MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PARAM_MAXNUM 62
+#define        MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PARAM_ID_LBN 0
+#define        MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PARAM_ID_WIDTH 8
+/*             Enum values, see field(s): */
+/*                MC_CMD_PCIE_TUNE_RXEQ_GET_OUT/PARAM_ID */
+#define        MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PARAM_LANE_LBN 8
+#define        MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PARAM_LANE_WIDTH 5
+/*             Enum values, see field(s): */
+/*                MC_CMD_PCIE_TUNE_RXEQ_GET_OUT/PARAM_LANE */
+#define        MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PARAM_AUTOCAL_LBN 13
+#define        MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PARAM_AUTOCAL_WIDTH 1
+#define        MC_CMD_PCIE_TUNE_RXEQ_SET_IN_RESERVED_LBN 14
+#define        MC_CMD_PCIE_TUNE_RXEQ_SET_IN_RESERVED_WIDTH 2
+#define        MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PARAM_INITIAL_LBN 16
+#define        MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PARAM_INITIAL_WIDTH 8
+#define        MC_CMD_PCIE_TUNE_RXEQ_SET_IN_RESERVED2_LBN 24
+#define        MC_CMD_PCIE_TUNE_RXEQ_SET_IN_RESERVED2_WIDTH 8
+
+/* MC_CMD_PCIE_TUNE_RXEQ_SET_OUT msgresponse */
+#define    MC_CMD_PCIE_TUNE_RXEQ_SET_OUT_LEN 0
+
 /* MC_CMD_PCIE_TUNE_TXEQ_GET_IN msgrequest */
 #define    MC_CMD_PCIE_TUNE_TXEQ_GET_IN_LEN 4
 /* Requested operation */
@@ -9176,6 +10064,7 @@
 /***********************************/
 /* MC_CMD_LICENSING
  * Operations on the NVRAM_PARTITION_TYPE_LICENSE application license partition
+ * - not used for V3 licensing
  */
 #define MC_CMD_LICENSING 0xf3
 
@@ -9220,6 +10109,93 @@
 
 
 /***********************************/
+/* MC_CMD_LICENSING_V3
+ * Operations on the NVRAM_PARTITION_TYPE_LICENSE application license partition
+ * - V3 licensing (Medford)
+ */
+#define MC_CMD_LICENSING_V3 0xd0
+
+#define MC_CMD_0xd0_PRIVILEGE_CTG SRIOV_CTG_GENERAL
+
+/* MC_CMD_LICENSING_V3_IN msgrequest */
+#define    MC_CMD_LICENSING_V3_IN_LEN 4
+/* identifies the type of operation requested */
+#define       MC_CMD_LICENSING_V3_IN_OP_OFST 0
+/* enum: re-read and apply licenses after a license key partition update; note
+ * that this operation returns a zero-length response
+ */
+#define          MC_CMD_LICENSING_V3_IN_OP_UPDATE_LICENSE  0x0
+/* enum: report counts of installed licenses */
+#define          MC_CMD_LICENSING_V3_IN_OP_REPORT_LICENSE  0x1
+
+/* MC_CMD_LICENSING_V3_OUT msgresponse */
+#define    MC_CMD_LICENSING_V3_OUT_LEN 88
+/* count of keys which are valid */
+#define       MC_CMD_LICENSING_V3_OUT_VALID_KEYS_OFST 0
+/* sum of UNVERIFIABLE_KEYS + WRONG_NODE_KEYS (for compatibility with
+ * MC_CMD_FC_OP_LICENSE)
+ */
+#define       MC_CMD_LICENSING_V3_OUT_INVALID_KEYS_OFST 4
+/* count of keys which are invalid due to being unverifiable */
+#define       MC_CMD_LICENSING_V3_OUT_UNVERIFIABLE_KEYS_OFST 8
+/* count of keys which are invalid due to being for the wrong node */
+#define       MC_CMD_LICENSING_V3_OUT_WRONG_NODE_KEYS_OFST 12
+/* licensing state (for diagnostics; the exact meaning of the bits in this
+ * field are private to the firmware)
+ */
+#define       MC_CMD_LICENSING_V3_OUT_LICENSING_STATE_OFST 16
+/* licensing subsystem self-test report (for manftest) */
+#define       MC_CMD_LICENSING_V3_OUT_LICENSING_SELF_TEST_OFST 20
+/* enum: licensing subsystem self-test failed */
+#define          MC_CMD_LICENSING_V3_OUT_SELF_TEST_FAIL  0x0
+/* enum: licensing subsystem self-test passed */
+#define          MC_CMD_LICENSING_V3_OUT_SELF_TEST_PASS  0x1
+/* bitmask of licensed applications */
+#define       MC_CMD_LICENSING_V3_OUT_LICENSED_APPS_OFST 24
+#define       MC_CMD_LICENSING_V3_OUT_LICENSED_APPS_LEN 8
+#define       MC_CMD_LICENSING_V3_OUT_LICENSED_APPS_LO_OFST 24
+#define       MC_CMD_LICENSING_V3_OUT_LICENSED_APPS_HI_OFST 28
+/* reserved for future use */
+#define       MC_CMD_LICENSING_V3_OUT_RESERVED_0_OFST 32
+#define       MC_CMD_LICENSING_V3_OUT_RESERVED_0_LEN 24
+/* bitmask of licensed features */
+#define       MC_CMD_LICENSING_V3_OUT_LICENSED_FEATURES_OFST 56
+#define       MC_CMD_LICENSING_V3_OUT_LICENSED_FEATURES_LEN 8
+#define       MC_CMD_LICENSING_V3_OUT_LICENSED_FEATURES_LO_OFST 56
+#define       MC_CMD_LICENSING_V3_OUT_LICENSED_FEATURES_HI_OFST 60
+/* reserved for future use */
+#define       MC_CMD_LICENSING_V3_OUT_RESERVED_1_OFST 64
+#define       MC_CMD_LICENSING_V3_OUT_RESERVED_1_LEN 24
+
+
+/***********************************/
+/* MC_CMD_LICENSING_GET_ID_V3
+ * Get ID and type from the NVRAM_PARTITION_TYPE_LICENSE application license
+ * partition - V3 licensing (Medford)
+ */
+#define MC_CMD_LICENSING_GET_ID_V3 0xd1
+
+#define MC_CMD_0xd1_PRIVILEGE_CTG SRIOV_CTG_GENERAL
+
+/* MC_CMD_LICENSING_GET_ID_V3_IN msgrequest */
+#define    MC_CMD_LICENSING_GET_ID_V3_IN_LEN 0
+
+/* MC_CMD_LICENSING_GET_ID_V3_OUT msgresponse */
+#define    MC_CMD_LICENSING_GET_ID_V3_OUT_LENMIN 8
+#define    MC_CMD_LICENSING_GET_ID_V3_OUT_LENMAX 252
+#define    MC_CMD_LICENSING_GET_ID_V3_OUT_LEN(num) (8+1*(num))
+/* type of license (eg 3) */
+#define       MC_CMD_LICENSING_GET_ID_V3_OUT_LICENSE_TYPE_OFST 0
+/* length of the license ID (in bytes) */
+#define       MC_CMD_LICENSING_GET_ID_V3_OUT_LICENSE_ID_LENGTH_OFST 4
+/* the unique license ID of the adapter */
+#define       MC_CMD_LICENSING_GET_ID_V3_OUT_LICENSE_ID_OFST 8
+#define       MC_CMD_LICENSING_GET_ID_V3_OUT_LICENSE_ID_LEN 1
+#define       MC_CMD_LICENSING_GET_ID_V3_OUT_LICENSE_ID_MINNUM 0
+#define       MC_CMD_LICENSING_GET_ID_V3_OUT_LICENSE_ID_MAXNUM 244
+
+
+/***********************************/
 /* MC_CMD_MC2MC_PROXY
  * Execute an arbitrary MCDI command on the slave MC of a dual-core device.
  * This will fail on a single-core system.
@@ -9239,7 +10215,7 @@
 /* MC_CMD_GET_LICENSED_APP_STATE
  * Query the state of an individual licensed application. (Note that the actual
  * state may be invalidated by the MC_CMD_LICENSING OP_UPDATE_LICENSE operation
- * or a reboot of the MC.)
+ * or a reboot of the MC.) Not used for V3 licensing
  */
 #define MC_CMD_GET_LICENSED_APP_STATE 0xf5
 
@@ -9261,8 +10237,68 @@
 
 
 /***********************************/
+/* MC_CMD_GET_LICENSED_V3_APP_STATE
+ * Query the state of an individual licensed application. (Note that the actual
+ * state may be invalidated by the MC_CMD_LICENSING_V3 OP_UPDATE_LICENSE
+ * operation or a reboot of the MC.) Used for V3 licensing (Medford)
+ */
+#define MC_CMD_GET_LICENSED_V3_APP_STATE 0xd2
+
+#define MC_CMD_0xd2_PRIVILEGE_CTG SRIOV_CTG_GENERAL
+
+/* MC_CMD_GET_LICENSED_V3_APP_STATE_IN msgrequest */
+#define    MC_CMD_GET_LICENSED_V3_APP_STATE_IN_LEN 8
+/* application ID to query (LICENSED_V3_APPS_xxx) expressed as a single bit
+ * mask
+ */
+#define       MC_CMD_GET_LICENSED_V3_APP_STATE_IN_APP_ID_OFST 0
+#define       MC_CMD_GET_LICENSED_V3_APP_STATE_IN_APP_ID_LEN 8
+#define       MC_CMD_GET_LICENSED_V3_APP_STATE_IN_APP_ID_LO_OFST 0
+#define       MC_CMD_GET_LICENSED_V3_APP_STATE_IN_APP_ID_HI_OFST 4
+
+/* MC_CMD_GET_LICENSED_V3_APP_STATE_OUT msgresponse */
+#define    MC_CMD_GET_LICENSED_V3_APP_STATE_OUT_LEN 4
+/* state of this application */
+#define       MC_CMD_GET_LICENSED_V3_APP_STATE_OUT_STATE_OFST 0
+/* enum: no (or invalid) license is present for the application */
+#define          MC_CMD_GET_LICENSED_V3_APP_STATE_OUT_NOT_LICENSED  0x0
+/* enum: a valid license is present for the application */
+#define          MC_CMD_GET_LICENSED_V3_APP_STATE_OUT_LICENSED  0x1
+
+
+/***********************************/
+/* MC_CMD_GET_LICENSED_V3_FEATURE_STATES
+ * Query the state of an one or more licensed features. (Note that the actual
+ * state may be invalidated by the MC_CMD_LICENSING_V3 OP_UPDATE_LICENSE
+ * operation or a reboot of the MC.) Used for V3 licensing (Medford)
+ */
+#define MC_CMD_GET_LICENSED_V3_FEATURE_STATES 0xd3
+
+#define MC_CMD_0xd3_PRIVILEGE_CTG SRIOV_CTG_GENERAL
+
+/* MC_CMD_GET_LICENSED_V3_FEATURE_STATES_IN msgrequest */
+#define    MC_CMD_GET_LICENSED_V3_FEATURE_STATES_IN_LEN 8
+/* features to query (LICENSED_V3_FEATURES_xxx) expressed as a mask with one or
+ * more bits set
+ */
+#define       MC_CMD_GET_LICENSED_V3_FEATURE_STATES_IN_FEATURES_OFST 0
+#define       MC_CMD_GET_LICENSED_V3_FEATURE_STATES_IN_FEATURES_LEN 8
+#define       MC_CMD_GET_LICENSED_V3_FEATURE_STATES_IN_FEATURES_LO_OFST 0
+#define       MC_CMD_GET_LICENSED_V3_FEATURE_STATES_IN_FEATURES_HI_OFST 4
+
+/* MC_CMD_GET_LICENSED_V3_FEATURE_STATES_OUT msgresponse */
+#define    MC_CMD_GET_LICENSED_V3_FEATURE_STATES_OUT_LEN 8
+/* states of these features - bit set for licensed, clear for not licensed */
+#define       MC_CMD_GET_LICENSED_V3_FEATURE_STATES_OUT_STATES_OFST 0
+#define       MC_CMD_GET_LICENSED_V3_FEATURE_STATES_OUT_STATES_LEN 8
+#define       MC_CMD_GET_LICENSED_V3_FEATURE_STATES_OUT_STATES_LO_OFST 0
+#define       MC_CMD_GET_LICENSED_V3_FEATURE_STATES_OUT_STATES_HI_OFST 4
+
+
+/***********************************/
 /* MC_CMD_LICENSED_APP_OP
- * Perform an action for an individual licensed application.
+ * Perform an action for an individual licensed application - not used for V3
+ * licensing.
  */
 #define MC_CMD_LICENSED_APP_OP 0xf6
 
@@ -9328,6 +10364,67 @@
 
 
 /***********************************/
+/* MC_CMD_LICENSED_V3_VALIDATE_APP
+ * Perform validation for an individual licensed application - V3 licensing
+ * (Medford)
+ */
+#define MC_CMD_LICENSED_V3_VALIDATE_APP 0xd4
+
+#define MC_CMD_0xd4_PRIVILEGE_CTG SRIOV_CTG_GENERAL
+
+/* MC_CMD_LICENSED_V3_VALIDATE_APP_IN msgrequest */
+#define    MC_CMD_LICENSED_V3_VALIDATE_APP_IN_LEN 72
+/* application ID expressed as a single bit mask */
+#define       MC_CMD_LICENSED_V3_VALIDATE_APP_IN_APP_ID_OFST 0
+#define       MC_CMD_LICENSED_V3_VALIDATE_APP_IN_APP_ID_LEN 8
+#define       MC_CMD_LICENSED_V3_VALIDATE_APP_IN_APP_ID_LO_OFST 0
+#define       MC_CMD_LICENSED_V3_VALIDATE_APP_IN_APP_ID_HI_OFST 4
+/* challenge for validation */
+#define       MC_CMD_LICENSED_V3_VALIDATE_APP_IN_CHALLENGE_OFST 8
+#define       MC_CMD_LICENSED_V3_VALIDATE_APP_IN_CHALLENGE_LEN 64
+
+/* MC_CMD_LICENSED_V3_VALIDATE_APP_OUT msgresponse */
+#define    MC_CMD_LICENSED_V3_VALIDATE_APP_OUT_LEN 72
+/* application expiry time */
+#define       MC_CMD_LICENSED_V3_VALIDATE_APP_OUT_EXPIRY_TIME_OFST 0
+/* application expiry units */
+#define       MC_CMD_LICENSED_V3_VALIDATE_APP_OUT_EXPIRY_UNITS_OFST 4
+/* enum: expiry units are accounting units */
+#define          MC_CMD_LICENSED_V3_VALIDATE_APP_OUT_EXPIRY_UNIT_ACC  0x0
+/* enum: expiry units are calendar days */
+#define          MC_CMD_LICENSED_V3_VALIDATE_APP_OUT_EXPIRY_UNIT_DAYS  0x1
+/* validation response to challenge */
+#define       MC_CMD_LICENSED_V3_VALIDATE_APP_OUT_RESPONSE_OFST 8
+#define       MC_CMD_LICENSED_V3_VALIDATE_APP_OUT_RESPONSE_LEN 64
+
+
+/***********************************/
+/* MC_CMD_LICENSED_V3_MASK_FEATURES
+ * Mask features - V3 licensing (Medford)
+ */
+#define MC_CMD_LICENSED_V3_MASK_FEATURES 0xd5
+
+#define MC_CMD_0xd5_PRIVILEGE_CTG SRIOV_CTG_GENERAL
+
+/* MC_CMD_LICENSED_V3_MASK_FEATURES_IN msgrequest */
+#define    MC_CMD_LICENSED_V3_MASK_FEATURES_IN_LEN 12
+/* mask to be applied to features to be changed */
+#define       MC_CMD_LICENSED_V3_MASK_FEATURES_IN_MASK_OFST 0
+#define       MC_CMD_LICENSED_V3_MASK_FEATURES_IN_MASK_LEN 8
+#define       MC_CMD_LICENSED_V3_MASK_FEATURES_IN_MASK_LO_OFST 0
+#define       MC_CMD_LICENSED_V3_MASK_FEATURES_IN_MASK_HI_OFST 4
+/* whether to turn on or turn off the masked features */
+#define       MC_CMD_LICENSED_V3_MASK_FEATURES_IN_FLAG_OFST 8
+/* enum: turn the features off */
+#define          MC_CMD_LICENSED_V3_MASK_FEATURES_IN_OFF  0x0
+/* enum: turn the features back on */
+#define          MC_CMD_LICENSED_V3_MASK_FEATURES_IN_ON  0x1
+
+/* MC_CMD_LICENSED_V3_MASK_FEATURES_OUT msgresponse */
+#define    MC_CMD_LICENSED_V3_MASK_FEATURES_OUT_LEN 0
+
+
+/***********************************/
 /* MC_CMD_SET_PORT_SNIFF_CONFIG
  * Configure RX port sniffing for the physical port associated with the calling
  * function. Only a privileged function may change the port sniffing
@@ -9696,12 +10793,27 @@
 #define          MC_CMD_PRIVILEGE_MASK_IN_GRP_ONLOAD            0x4 /* enum */
 #define          MC_CMD_PRIVILEGE_MASK_IN_GRP_PTP               0x8 /* enum */
 #define          MC_CMD_PRIVILEGE_MASK_IN_GRP_INSECURE_FILTERS  0x10 /* enum */
-#define          MC_CMD_PRIVILEGE_MASK_IN_GRP_MAC_SPOOFING      0x20 /* enum */
+/* enum: Deprecated. Equivalent to MAC_SPOOFING_TX combined with CHANGE_MAC. */
+#define          MC_CMD_PRIVILEGE_MASK_IN_GRP_MAC_SPOOFING      0x20
 #define          MC_CMD_PRIVILEGE_MASK_IN_GRP_UNICAST           0x40 /* enum */
 #define          MC_CMD_PRIVILEGE_MASK_IN_GRP_MULTICAST         0x80 /* enum */
 #define          MC_CMD_PRIVILEGE_MASK_IN_GRP_BROADCAST         0x100 /* enum */
 #define          MC_CMD_PRIVILEGE_MASK_IN_GRP_ALL_MULTICAST     0x200 /* enum */
 #define          MC_CMD_PRIVILEGE_MASK_IN_GRP_PROMISCUOUS       0x400 /* enum */
+/* enum: Allows to set the TX packets' source MAC address to any arbitrary MAC
+ * adress.
+ */
+#define          MC_CMD_PRIVILEGE_MASK_IN_GRP_MAC_SPOOFING_TX   0x800
+/* enum: Privilege that allows a Function to change the MAC address configured
+ * in its associated vAdapter/vPort.
+ */
+#define          MC_CMD_PRIVILEGE_MASK_IN_GRP_CHANGE_MAC        0x1000
+/* enum: Privilege that allows a Function to install filters that specify VLANs
+ * that are not in the permit list for the associated vPort. This privilege is
+ * primarily to support ESX where vPorts are created that restrict traffic to
+ * only a set of permitted VLANs. See the vPort flag FLAG_VLAN_RESTRICT.
+ */
+#define          MC_CMD_PRIVILEGE_MASK_IN_GRP_UNRESTRICTED_VLAN  0x2000
 /* enum: Set this bit to indicate that a new privilege mask is to be set,
  * otherwise the command will only read the existing mask.
  */
@@ -9951,7 +11063,7 @@
 /* Sector type */
 #define       MC_CMD_XPM_WRITE_SECTOR_IN_TYPE_OFST 4
 /*            Enum values, see field(s): */
-/*               MC_CMD_XPM_READ_SECTOR_OUT/TYPE */
+/*               MC_CMD_XPM_READ_SECTOR/MC_CMD_XPM_READ_SECTOR_OUT/TYPE */
 /* Sector size */
 #define       MC_CMD_XPM_WRITE_SECTOR_IN_SIZE_OFST 8
 /* Sector data */
@@ -10067,4 +11179,123 @@
 #define    MC_CMD_XPM_WRITE_TEST_OUT_LEN 0
 
 
+/***********************************/
+/* MC_CMD_EXEC_SIGNED
+ * Check the CMAC of the contents of IMEM and DMEM against the value supplied
+ * and if correct begin execution from the start of IMEM. The caller supplies a
+ * key ID, the length of IMEM and DMEM to validate and the expected CMAC. CMAC
+ * computation runs from the start of IMEM, and from the start of DMEM + 16k,
+ * to match flash booting. The command will respond with EINVAL if the CMAC
+ * does match, otherwise it will respond with success before it jumps to IMEM.
+ */
+#define MC_CMD_EXEC_SIGNED 0x10c
+
+#define MC_CMD_0x10c_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+
+/* MC_CMD_EXEC_SIGNED_IN msgrequest */
+#define    MC_CMD_EXEC_SIGNED_IN_LEN 28
+/* the length of code to include in the CMAC */
+#define       MC_CMD_EXEC_SIGNED_IN_CODELEN_OFST 0
+/* the length of date to include in the CMAC */
+#define       MC_CMD_EXEC_SIGNED_IN_DATALEN_OFST 4
+/* the XPM sector containing the key to use */
+#define       MC_CMD_EXEC_SIGNED_IN_KEYSECTOR_OFST 8
+/* the expected CMAC value */
+#define       MC_CMD_EXEC_SIGNED_IN_CMAC_OFST 12
+#define       MC_CMD_EXEC_SIGNED_IN_CMAC_LEN 16
+
+/* MC_CMD_EXEC_SIGNED_OUT msgresponse */
+#define    MC_CMD_EXEC_SIGNED_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_PREPARE_SIGNED
+ * Prepare to upload a signed image. This will scrub the specified length of
+ * the data region, which must be at least as large as the DATALEN supplied to
+ * MC_CMD_EXEC_SIGNED.
+ */
+#define MC_CMD_PREPARE_SIGNED 0x10d
+
+#define MC_CMD_0x10d_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+
+/* MC_CMD_PREPARE_SIGNED_IN msgrequest */
+#define    MC_CMD_PREPARE_SIGNED_IN_LEN 4
+/* the length of data area to clear */
+#define       MC_CMD_PREPARE_SIGNED_IN_DATALEN_OFST 0
+
+/* MC_CMD_PREPARE_SIGNED_OUT msgresponse */
+#define    MC_CMD_PREPARE_SIGNED_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS
+ * Configure UDP ports for tunnel encapsulation hardware acceleration. The
+ * parser-dispatcher will attempt to parse traffic on these ports as tunnel
+ * encapsulation PDUs and filter them using the tunnel encapsulation filter
+ * chain rather than the standard filter chain. Note that this command can
+ * cause all functions to see a reset. (Available on Medford only.)
+ */
+#define MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS 0x117
+
+#define MC_CMD_0x117_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+
+/* MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN msgrequest */
+#define    MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_LENMIN 4
+#define    MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_LENMAX 68
+#define    MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_LEN(num) (4+4*(num))
+/* Flags */
+#define       MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_FLAGS_OFST 0
+#define       MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_FLAGS_LEN 2
+#define        MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_UNLOADING_LBN 0
+#define        MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_UNLOADING_WIDTH 1
+/* The number of entries in the ENTRIES array */
+#define       MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_NUM_ENTRIES_OFST 2
+#define       MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_NUM_ENTRIES_LEN 2
+/* Entries defining the UDP port to protocol mapping, each laid out as a
+ * TUNNEL_ENCAP_UDP_PORT_ENTRY
+ */
+#define       MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_ENTRIES_OFST 4
+#define       MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_ENTRIES_LEN 4
+#define       MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_ENTRIES_MINNUM 0
+#define       MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_ENTRIES_MAXNUM 16
+
+/* MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_OUT msgresponse */
+#define    MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_OUT_LEN 2
+/* Flags */
+#define       MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_OUT_FLAGS_OFST 0
+#define       MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_OUT_FLAGS_LEN 2
+#define        MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_OUT_RESETTING_LBN 0
+#define        MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_OUT_RESETTING_WIDTH 1
+
+
+/***********************************/
+/* MC_CMD_RX_BALANCING
+ * Configure a port upconverter to distribute the packets on both RX engines.
+ * Packets are distributed based on a table with the destination vFIFO. The
+ * index of the table is a hash of source and destination of IPV4 and VLAN
+ * priority.
+ */
+#define MC_CMD_RX_BALANCING 0x118
+
+#define MC_CMD_0x118_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+
+/* MC_CMD_RX_BALANCING_IN msgrequest */
+#define    MC_CMD_RX_BALANCING_IN_LEN 4
+/* The RX port whose upconverter table will be modified */
+#define       MC_CMD_RX_BALANCING_IN_PORT_OFST 0
+#define       MC_CMD_RX_BALANCING_IN_PORT_LEN 1
+/* The VLAN priority associated to the table index and vFIFO */
+#define       MC_CMD_RX_BALANCING_IN_PRIORITY_OFST 1
+#define       MC_CMD_RX_BALANCING_IN_PRIORITY_LEN 1
+/* The resulting bit of SRC^DST for indexing the table */
+#define       MC_CMD_RX_BALANCING_IN_SRC_DST_OFST 2
+#define       MC_CMD_RX_BALANCING_IN_SRC_DST_LEN 1
+/* The RX engine to which the vFIFO in the table entry will point to */
+#define       MC_CMD_RX_BALANCING_IN_ENG_OFST 3
+#define       MC_CMD_RX_BALANCING_IN_ENG_LEN 1
+
+/* MC_CMD_RX_BALANCING_OUT msgresponse */
+#define    MC_CMD_RX_BALANCING_OUT_LEN 0
+
+
 #endif /* MCDI_PCOL_H */
diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h
index d13ddf9703ff..9ff062a36ea8 100644
--- a/drivers/net/ethernet/sfc/net_driver.h
+++ b/drivers/net/ethernet/sfc/net_driver.h
@@ -868,6 +868,7 @@ struct vfdi_status;
  *      be held to modify it.
  * @port_initialized: Port initialized?
  * @net_dev: Operating system network device. Consider holding the rtnl lock
+ * @fixed_features: Features which cannot be turned off
  * @stats_buffer: DMA buffer for statistics
  * @phy_type: PHY type
  * @phy_op: PHY interface
@@ -916,7 +917,6 @@ struct vfdi_status;
  * @stats_lock: Statistics update lock. Must be held when calling
  *      efx_nic_type::{update,start,stop}_stats.
  * @n_rx_noskb_drops: Count of RX packets dropped due to failure to allocate an skb
- * @mc_promisc: Whether in multicast promiscuous mode when last changed
  *
  * This is stored in the private area of the &struct net_device.
  */
@@ -1008,6 +1008,8 @@ struct efx_nic {
         bool port_initialized;
         struct net_device *net_dev;
 
+        netdev_features_t fixed_features;
+
         struct efx_buffer stats_buffer;
         u64 rx_nodesc_drops_total;
         u64 rx_nodesc_drops_while_down;
@@ -1065,7 +1067,6 @@ struct efx_nic {
         int last_irq_cpu;
         spinlock_t stats_lock;
         atomic_t n_rx_noskb_drops;
-        bool mc_promisc;
 };
 
 static inline int efx_dev_registered(struct efx_nic *efx)
@@ -1333,6 +1334,8 @@ struct efx_nic_type {
         int (*ptp_set_ts_config)(struct efx_nic *efx,
                                  struct hwtstamp_config *init);
         int (*sriov_configure)(struct efx_nic *efx, int num_vfs);
+        int (*vlan_rx_add_vid)(struct efx_nic *efx, __be16 proto, u16 vid);
+        int (*vlan_rx_kill_vid)(struct efx_nic *efx, __be16 proto, u16 vid);
         int (*sriov_init)(struct efx_nic *efx);
         void (*sriov_fini)(struct efx_nic *efx);
         bool (*sriov_wanted)(struct efx_nic *efx);
@@ -1521,4 +1524,16 @@ static inline void efx_xmit_hwtstamp_pending(struct sk_buff *skb)
         skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
 }
 
+/* Get all supported features.
+ * If a feature is not fixed, it is present in hw_features.
+ * If a feature is fixed, it does not present in hw_features, but
+ * always in features.
+ */
+static inline netdev_features_t efx_supported_features(const struct efx_nic *efx)
+{
+        const struct net_device *net_dev = efx->net_dev;
+
+        return net_dev->features | net_dev->hw_features;
+}
+
 #endif /* EFX_NET_DRIVER_H */
diff --git a/drivers/net/ethernet/sfc/nic.h b/drivers/net/ethernet/sfc/nic.h
index 0b536e27d3b2..96944c3c9d14 100644
--- a/drivers/net/ethernet/sfc/nic.h
+++ b/drivers/net/ethernet/sfc/nic.h
@@ -519,6 +519,9 @@ enum {
 #ifdef CONFIG_SFC_SRIOV
  * @vf: Pointer to VF data structure
 #endif
+ * @vport_mac: The MAC address on the vport, only for PFs; VFs will be zero
+ * @vlan_list: List of VLANs added over the interface. Serialised by vlan_lock.
+ * @vlan_lock: Lock to serialize access to vlan_list.
  */
 struct efx_ef10_nic_data {
         struct efx_buffer mcdi_buf;
@@ -550,6 +553,8 @@ struct efx_ef10_nic_data {
         struct ef10_vf *vf;
 #endif
         u8 vport_mac[ETH_ALEN];
+        struct list_head vlan_list;
+        struct mutex vlan_lock;
 };
 
 int efx_init_sriov(void);
diff --git a/drivers/net/ethernet/stmicro/stmmac/Kconfig b/drivers/net/ethernet/stmicro/stmmac/Kconfig
index cec147d1d34f..8f06a6621ab1 100644
--- a/drivers/net/ethernet/stmicro/stmmac/Kconfig
+++ b/drivers/net/ethernet/stmicro/stmmac/Kconfig
@@ -40,7 +40,7 @@ config DWMAC_GENERIC
 config DWMAC_IPQ806X
         tristate "QCA IPQ806x DWMAC support"
         default ARCH_QCOM
-        depends on OF
+        depends on OF && (ARCH_QCOM || COMPILE_TEST)
         select MFD_SYSCON
         help
           Support for QCA IPQ806X DWMAC Ethernet.
@@ -53,7 +53,7 @@ config DWMAC_IPQ806X
 config DWMAC_LPC18XX
         tristate "NXP LPC18xx/43xx DWMAC support"
         default ARCH_LPC18XX
-        depends on OF
+        depends on OF && (ARCH_LPC18XX || COMPILE_TEST)
         select MFD_SYSCON
         ---help---
           Support for NXP LPC18xx/43xx DWMAC Ethernet.
@@ -61,7 +61,7 @@ config DWMAC_LPC18XX
 config DWMAC_MESON
         tristate "Amlogic Meson dwmac support"
         default ARCH_MESON
-        depends on OF
+        depends on OF && (ARCH_MESON || COMPILE_TEST)
         help
           Support for Ethernet controller on Amlogic Meson SoCs.
 
@@ -72,7 +72,7 @@ config DWMAC_MESON
 config DWMAC_ROCKCHIP
         tristate "Rockchip dwmac support"
         default ARCH_ROCKCHIP
-        depends on OF
+        depends on OF && (ARCH_ROCKCHIP || COMPILE_TEST)
         select MFD_SYSCON
         help
           Support for Ethernet controller on Rockchip RK3288 SoC.
@@ -83,7 +83,7 @@ config DWMAC_ROCKCHIP
 config DWMAC_SOCFPGA
         tristate "SOCFPGA dwmac support"
         default ARCH_SOCFPGA
-        depends on OF
+        depends on OF && (ARCH_SOCFPGA || COMPILE_TEST)
         select MFD_SYSCON
         help
           Support for ethernet controller on Altera SOCFPGA
@@ -95,7 +95,7 @@ config DWMAC_SOCFPGA
 config DWMAC_STI
         tristate "STi GMAC support"
         default ARCH_STI
-        depends on OF
+        depends on OF && (ARCH_STI || COMPILE_TEST)
         select MFD_SYSCON
         ---help---
           Support for ethernet controller on STi SOCs.
@@ -107,7 +107,7 @@ config DWMAC_STI
 config DWMAC_SUNXI
         tristate "Allwinner GMAC support"
         default ARCH_SUNXI
-        depends on OF
+        depends on OF && (ARCH_SUNXI || COMPILE_TEST)
         ---help---
           Support for Allwinner A20/A31 GMAC ethernet controllers.
 
diff --git a/drivers/net/ethernet/stmicro/stmmac/common.h b/drivers/net/ethernet/stmicro/stmmac/common.h
index fc60368df2e7..2533b91f1421 100644
--- a/drivers/net/ethernet/stmicro/stmmac/common.h
+++ b/drivers/net/ethernet/stmicro/stmmac/common.h
@@ -232,6 +232,11 @@ struct stmmac_extra_stats {
 #define DMA_HW_FEAT_ACTPHYIF    0x70000000      /* Active/selected PHY iface */
 #define DEFAULT_DMA_PBL         8
 
+/* PCS status and mask defines */
+#define PCS_ANE_IRQ             BIT(2)  /* PCS Auto-Negotiation */
+#define PCS_LINK_IRQ            BIT(1)  /* PCS Link */
+#define PCS_RGSMIIIS_IRQ        BIT(0)  /* RGMII or SMII Interrupt */
+
 /* Max/Min RI Watchdog Timer count value */
 #define MAX_DMA_RIWT            0xff
 #define MIN_DMA_RIWT            0x20
@@ -272,9 +277,6 @@ enum dma_irq_status {
 #define CORE_IRQ_RX_PATH_IN_LPI_MODE    (1 << 2)
 #define CORE_IRQ_RX_PATH_EXIT_LPI_MODE  (1 << 3)
 
-#define CORE_PCS_ANE_COMPLETE           (1 << 5)
-#define CORE_PCS_LINK_STATUS            (1 << 6)
-#define CORE_RGMII_IRQ                  (1 << 7)
 #define CORE_IRQ_MTL_RX_OVERFLOW        BIT(8)
 
 /* Physical Coding Sublayer */
@@ -469,9 +471,12 @@ struct stmmac_ops {
         void (*reset_eee_mode)(struct mac_device_info *hw);
         void (*set_eee_timer)(struct mac_device_info *hw, int ls, int tw);
         void (*set_eee_pls)(struct mac_device_info *hw, int link);
-        void (*ctrl_ane)(struct mac_device_info *hw, bool restart);
-        void (*get_adv)(struct mac_device_info *hw, struct rgmii_adv *adv);
         void (*debug)(void __iomem *ioaddr, struct stmmac_extra_stats *x);
+        /* PCS calls */
+        void (*pcs_ctrl_ane)(void __iomem *ioaddr, bool ane, bool srgmi_ral,
+                             bool loopback);
+        void (*pcs_rane)(void __iomem *ioaddr, bool restart);
+        void (*pcs_get_adv_lp)(void __iomem *ioaddr, struct rgmii_adv *adv);
 };
 
 /* PTP and HW Timer helpers */
@@ -524,6 +529,9 @@ struct mac_device_info {
         int unicast_filter_entries;
         int mcast_bits_log2;
         unsigned int rx_csum;
+        unsigned int pcs;
+        unsigned int pmt;
+        unsigned int ps;
 };
 
 struct mac_device_info *dwmac1000_setup(void __iomem *ioaddr, int mcbins,
@@ -546,6 +554,7 @@ void stmmac_dwmac4_get_mac_addr(void __iomem *ioaddr, unsigned char *addr,
 void stmmac_dwmac4_set_mac(void __iomem *ioaddr, bool enable);
 
 void dwmac_dma_flush_tx_fifo(void __iomem *ioaddr);
+
 extern const struct stmmac_mode_ops ring_mode_ops;
 extern const struct stmmac_mode_ops chain_mode_ops;
 extern const struct stmmac_desc_ops dwmac4_desc_ops;
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c
index 0cd3ecff768b..92105916ef40 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c
@@ -46,6 +46,7 @@ struct rk_priv_data {
         struct platform_device *pdev;
         int phy_iface;
         struct regulator *regulator;
+        bool suspended;
         const struct rk_gmac_ops *ops;
 
         bool clk_enabled;
@@ -72,6 +73,122 @@ struct rk_priv_data {
 #define GRF_BIT(nr)     (BIT(nr) | BIT(nr+16))
 #define GRF_CLR_BIT(nr) (BIT(nr+16))
 
+#define RK3228_GRF_MAC_CON0     0x0900
+#define RK3228_GRF_MAC_CON1     0x0904
+
+/* RK3228_GRF_MAC_CON0 */
+#define RK3228_GMAC_CLK_RX_DL_CFG(val)  HIWORD_UPDATE(val, 0x7F, 7)
+#define RK3228_GMAC_CLK_TX_DL_CFG(val)  HIWORD_UPDATE(val, 0x7F, 0)
+
+/* RK3228_GRF_MAC_CON1 */
+#define RK3228_GMAC_PHY_INTF_SEL_RGMII  \
+                (GRF_BIT(4) | GRF_CLR_BIT(5) | GRF_CLR_BIT(6))
+#define RK3228_GMAC_PHY_INTF_SEL_RMII   \
+                (GRF_CLR_BIT(4) | GRF_CLR_BIT(5) | GRF_BIT(6))
+#define RK3228_GMAC_FLOW_CTRL           GRF_BIT(3)
+#define RK3228_GMAC_FLOW_CTRL_CLR       GRF_CLR_BIT(3)
+#define RK3228_GMAC_SPEED_10M           GRF_CLR_BIT(2)
+#define RK3228_GMAC_SPEED_100M          GRF_BIT(2)
+#define RK3228_GMAC_RMII_CLK_25M        GRF_BIT(7)
+#define RK3228_GMAC_RMII_CLK_2_5M       GRF_CLR_BIT(7)
+#define RK3228_GMAC_CLK_125M            (GRF_CLR_BIT(8) | GRF_CLR_BIT(9))
+#define RK3228_GMAC_CLK_25M             (GRF_BIT(8) | GRF_BIT(9))
+#define RK3228_GMAC_CLK_2_5M            (GRF_CLR_BIT(8) | GRF_BIT(9))
+#define RK3228_GMAC_RMII_MODE           GRF_BIT(10)
+#define RK3228_GMAC_RMII_MODE_CLR       GRF_CLR_BIT(10)
+#define RK3228_GMAC_TXCLK_DLY_ENABLE    GRF_BIT(0)
+#define RK3228_GMAC_TXCLK_DLY_DISABLE   GRF_CLR_BIT(0)
+#define RK3228_GMAC_RXCLK_DLY_ENABLE    GRF_BIT(1)
+#define RK3228_GMAC_RXCLK_DLY_DISABLE   GRF_CLR_BIT(1)
+
+static void rk3228_set_to_rgmii(struct rk_priv_data *bsp_priv,
+                                int tx_delay, int rx_delay)
+{
+        struct device *dev = &bsp_priv->pdev->dev;
+
+        if (IS_ERR(bsp_priv->grf)) {
+                dev_err(dev, "Missing rockchip,grf property\n");
+                return;
+        }
+
+        regmap_write(bsp_priv->grf, RK3228_GRF_MAC_CON1,
+                     RK3228_GMAC_PHY_INTF_SEL_RGMII |
+                     RK3228_GMAC_RMII_MODE_CLR |
+                     RK3228_GMAC_RXCLK_DLY_ENABLE |
+                     RK3228_GMAC_TXCLK_DLY_ENABLE);
+
+        regmap_write(bsp_priv->grf, RK3228_GRF_MAC_CON0,
+                     RK3228_GMAC_CLK_RX_DL_CFG(rx_delay) |
+                     RK3228_GMAC_CLK_TX_DL_CFG(tx_delay));
+}
+
+static void rk3228_set_to_rmii(struct rk_priv_data *bsp_priv)
+{
+        struct device *dev = &bsp_priv->pdev->dev;
+
+        if (IS_ERR(bsp_priv->grf)) {
+                dev_err(dev, "Missing rockchip,grf property\n");
+                return;
+        }
+
+        regmap_write(bsp_priv->grf, RK3228_GRF_MAC_CON1,
+                     RK3228_GMAC_PHY_INTF_SEL_RMII |
+                     RK3228_GMAC_RMII_MODE);
+
+        /* set MAC to RMII mode */
+        regmap_write(bsp_priv->grf, RK3228_GRF_MAC_CON1, GRF_BIT(11));
+}
+
+static void rk3228_set_rgmii_speed(struct rk_priv_data *bsp_priv, int speed)
+{
+        struct device *dev = &bsp_priv->pdev->dev;
+
+        if (IS_ERR(bsp_priv->grf)) {
+                dev_err(dev, "Missing rockchip,grf property\n");
+                return;
+        }
+
+        if (speed == 10)
+                regmap_write(bsp_priv->grf, RK3228_GRF_MAC_CON1,
+                             RK3228_GMAC_CLK_2_5M);
+        else if (speed == 100)
+                regmap_write(bsp_priv->grf, RK3228_GRF_MAC_CON1,
+                             RK3228_GMAC_CLK_25M);
+        else if (speed == 1000)
+                regmap_write(bsp_priv->grf, RK3228_GRF_MAC_CON1,
+                             RK3228_GMAC_CLK_125M);
+        else
+                dev_err(dev, "unknown speed value for RGMII! speed=%d", speed);
+}
+
+static void rk3228_set_rmii_speed(struct rk_priv_data *bsp_priv, int speed)
+{
+        struct device *dev = &bsp_priv->pdev->dev;
+
+        if (IS_ERR(bsp_priv->grf)) {
+                dev_err(dev, "Missing rockchip,grf property\n");
+                return;
+        }
+
+        if (speed == 10)
+                regmap_write(bsp_priv->grf, RK3228_GRF_MAC_CON1,
+                             RK3228_GMAC_RMII_CLK_2_5M |
+                             RK3228_GMAC_SPEED_10M);
+        else if (speed == 100)
+                regmap_write(bsp_priv->grf, RK3228_GRF_MAC_CON1,
+                             RK3228_GMAC_RMII_CLK_25M |
+                             RK3228_GMAC_SPEED_100M);
+        else
+                dev_err(dev, "unknown speed value for RMII! speed=%d", speed);
+}
+
+static const struct rk_gmac_ops rk3228_ops = {
+        .set_to_rgmii = rk3228_set_to_rgmii,
+        .set_to_rmii = rk3228_set_to_rmii,
+        .set_rgmii_speed = rk3228_set_rgmii_speed,
+        .set_rmii_speed = rk3228_set_rmii_speed,
+};
+
 #define RK3288_GRF_SOC_CON1     0x0248
 #define RK3288_GRF_SOC_CON3     0x0250
 
@@ -529,9 +646,8 @@ static struct rk_priv_data *rk_gmac_setup(struct platform_device *pdev,
         return bsp_priv;
 }
 
-static int rk_gmac_init(struct platform_device *pdev, void *priv)
+static int rk_gmac_powerup(struct rk_priv_data *bsp_priv)
 {
-        struct rk_priv_data *bsp_priv = priv;
         int ret;
 
         ret = phy_power_on(bsp_priv, true);
@@ -545,14 +661,50 @@ static int rk_gmac_init(struct platform_device *pdev, void *priv)
         return 0;
 }
 
-static void rk_gmac_exit(struct platform_device *pdev, void *priv)
+static void rk_gmac_powerdown(struct rk_priv_data *gmac)
 {
-        struct rk_priv_data *gmac = priv;
-
         phy_power_on(gmac, false);
         gmac_clk_enable(gmac, false);
 }
 
+static int rk_gmac_init(struct platform_device *pdev, void *priv)
+{
+        struct rk_priv_data *bsp_priv = priv;
+
+        return rk_gmac_powerup(bsp_priv);
+}
+
+static void rk_gmac_exit(struct platform_device *pdev, void *priv)
+{
+        struct rk_priv_data *bsp_priv = priv;
+
+        rk_gmac_powerdown(bsp_priv);
+}
+
+static void rk_gmac_suspend(struct platform_device *pdev, void *priv)
+{
+        struct rk_priv_data *bsp_priv = priv;
+
+        /* Keep the PHY up if we use Wake-on-Lan. */
+        if (device_may_wakeup(&pdev->dev))
+                return;
+
+        rk_gmac_powerdown(bsp_priv);
+        bsp_priv->suspended = true;
+}
+
+static void rk_gmac_resume(struct platform_device *pdev, void *priv)
+{
+        struct rk_priv_data *bsp_priv = priv;
+
+        /* The PHY was up for Wake-on-Lan. */
+        if (!bsp_priv->suspended)
+                return;
+
+        rk_gmac_powerup(bsp_priv);
+        bsp_priv->suspended = false;
+}
+
 static void rk_fix_speed(void *priv, unsigned int speed)
 {
         struct rk_priv_data *bsp_priv = priv;
@@ -591,6 +743,8 @@ static int rk_gmac_probe(struct platform_device *pdev)
         plat_dat->init = rk_gmac_init;
         plat_dat->exit = rk_gmac_exit;
         plat_dat->fix_mac_speed = rk_fix_speed;
+        plat_dat->suspend = rk_gmac_suspend;
+        plat_dat->resume = rk_gmac_resume;
 
         plat_dat->bsp_priv = rk_gmac_setup(pdev, data);
         if (IS_ERR(plat_dat->bsp_priv))
@@ -604,6 +758,7 @@ static int rk_gmac_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id rk_gmac_dwmac_match[] = {
+        { .compatible = "rockchip,rk3228-gmac", .data = &rk3228_ops },
         { .compatible = "rockchip,rk3288-gmac", .data = &rk3288_ops },
         { .compatible = "rockchip,rk3368-gmac", .data = &rk3368_ops },
         { }
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h b/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h
index b0593a4268ee..ff3e5ab39bd0 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h
@@ -38,19 +38,26 @@
 #define GMAC_WAKEUP_FILTER      0x00000028      /* Wake-up Frame Filter */
 
 #define GMAC_INT_STATUS         0x00000038      /* interrupt status register */
-enum dwmac1000_irq_status {
-        lpiis_irq = 0x400,
-        time_stamp_irq = 0x0200,
-        mmc_rx_csum_offload_irq = 0x0080,
-        mmc_tx_irq = 0x0040,
-        mmc_rx_irq = 0x0020,
-        mmc_irq = 0x0010,
-        pmt_irq = 0x0008,
-        pcs_ane_irq = 0x0004,
-        pcs_link_irq = 0x0002,
-        rgmii_irq = 0x0001,
-};
-#define GMAC_INT_MASK           0x0000003c      /* interrupt mask register */
+#define GMAC_INT_STATUS_PMT     BIT(3)
+#define GMAC_INT_STATUS_MMCIS   BIT(4)
+#define GMAC_INT_STATUS_MMCRIS  BIT(5)
+#define GMAC_INT_STATUS_MMCTIS  BIT(6)
+#define GMAC_INT_STATUS_MMCCSUM BIT(7)
+#define GMAC_INT_STATUS_TSTAMP  BIT(9)
+#define GMAC_INT_STATUS_LPIIS   BIT(10)
+
+/* interrupt mask register */
+#define GMAC_INT_MASK           0x0000003c
+#define GMAC_INT_DISABLE_RGMII          BIT(0)
+#define GMAC_INT_DISABLE_PCSLINK        BIT(1)
+#define GMAC_INT_DISABLE_PCSAN          BIT(2)
+#define GMAC_INT_DISABLE_PMT            BIT(3)
+#define GMAC_INT_DISABLE_TIMESTAMP      BIT(9)
+#define GMAC_INT_DISABLE_PCS    (GMAC_INT_DISABLE_RGMII | \
+                                 GMAC_INT_DISABLE_PCSLINK | \
+                                 GMAC_INT_DISABLE_PCSAN)
+#define GMAC_INT_DEFAULT_MASK   (GMAC_INT_DISABLE_TIMESTAMP | \
+                                 GMAC_INT_DISABLE_PCS)
 
 /* PMT Control and Status */
 #define GMAC_PMT                0x0000002c
@@ -90,42 +97,23 @@ enum power_event {
                                 (reg * 8))
 #define GMAC_MAX_PERFECT_ADDRESSES      1
 
-/* PCS registers (AN/TBI/SGMII/RGMII) offset */
-#define GMAC_AN_CTRL    0x000000c0      /* AN control */
-#define GMAC_AN_STATUS  0x000000c4      /* AN status */
-#define GMAC_ANE_ADV    0x000000c8      /* Auto-Neg. Advertisement */
-#define GMAC_ANE_LPA    0x000000cc      /* Auto-Neg. link partener ability */
-#define GMAC_ANE_EXP    0x000000d0      /* ANE expansion */
-#define GMAC_TBI        0x000000d4      /* TBI extend status */
-#define GMAC_S_R_GMII   0x000000d8      /* SGMII RGMII status */
-
-/* AN Configuration defines */
-#define GMAC_AN_CTRL_RAN        0x00000200      /* Restart Auto-Negotiation */
-#define GMAC_AN_CTRL_ANE        0x00001000      /* Auto-Negotiation Enable */
-#define GMAC_AN_CTRL_ELE        0x00004000      /* External Loopback Enable */
-#define GMAC_AN_CTRL_ECD        0x00010000      /* Enable Comma Detect */
-#define GMAC_AN_CTRL_LR         0x00020000      /* Lock to Reference */
-#define GMAC_AN_CTRL_SGMRAL     0x00040000      /* SGMII RAL Control */
-
-/* AN Status defines */
-#define GMAC_AN_STATUS_LS       0x00000004      /* Link Status 0:down 1:up */
-#define GMAC_AN_STATUS_ANA      0x00000008      /* Auto-Negotiation Ability */
-#define GMAC_AN_STATUS_ANC      0x00000020      /* Auto-Negotiation Complete */
-#define GMAC_AN_STATUS_ES       0x00000100      /* Extended Status */
-
-/* Register 54 (SGMII/RGMII status register) */
-#define GMAC_S_R_GMII_LINK              0x8
-#define GMAC_S_R_GMII_SPEED             0x5
-#define GMAC_S_R_GMII_SPEED_SHIFT       0x1
-#define GMAC_S_R_GMII_MODE              0x1
-#define GMAC_S_R_GMII_SPEED_125         2
-#define GMAC_S_R_GMII_SPEED_25          1
-
-/* Common ADV and LPA defines */
-#define GMAC_ANE_FD             (1 << 5)
-#define GMAC_ANE_HD             (1 << 6)
-#define GMAC_ANE_PSE            (3 << 7)
-#define GMAC_ANE_PSE_SHIFT      7
+#define GMAC_PCS_BASE           0x000000c0      /* PCS register base */
+#define GMAC_RGSMIIIS           0x000000d8      /* RGMII/SMII status */
+
+/* SGMII/RGMII status register */
+#define GMAC_RGSMIIIS_LNKMODE           BIT(0)
+#define GMAC_RGSMIIIS_SPEED             GENMASK(2, 1)
+#define GMAC_RGSMIIIS_SPEED_SHIFT       1
+#define GMAC_RGSMIIIS_LNKSTS            BIT(3)
+#define GMAC_RGSMIIIS_JABTO             BIT(4)
+#define GMAC_RGSMIIIS_FALSECARDET       BIT(5)
+#define GMAC_RGSMIIIS_SMIDRXS           BIT(16)
+/* LNKMOD */
+#define GMAC_RGSMIIIS_LNKMOD_MASK       0x1
+/* LNKSPEED */
+#define GMAC_RGSMIIIS_SPEED_125         0x2
+#define GMAC_RGSMIIIS_SPEED_25          0x1
+#define GMAC_RGSMIIIS_SPEED_2_5         0x0
 
 /* GMAC Configuration defines */
 #define GMAC_CONTROL_2K 0x08000000      /* IEEE 802.3as 2K packets */
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c
index fb1eb578e34e..cbefe9e2207c 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c
@@ -30,22 +30,48 @@
 #include <linux/slab.h>
 #include <linux/ethtool.h>
 #include <asm/io.h>
+#include "stmmac_pcs.h"
 #include "dwmac1000.h"
 
 static void dwmac1000_core_init(struct mac_device_info *hw, int mtu)
 {
         void __iomem *ioaddr = hw->pcsr;
         u32 value = readl(ioaddr + GMAC_CONTROL);
+
+        /* Configure GMAC core */
         value |= GMAC_CORE_INIT;
+
         if (mtu > 1500)
                 value |= GMAC_CONTROL_2K;
         if (mtu > 2000)
                 value |= GMAC_CONTROL_JE;
 
+        if (hw->ps) {
+                value |= GMAC_CONTROL_TE;
+
+                if (hw->ps == SPEED_1000) {
+                        value &= ~GMAC_CONTROL_PS;
+                } else {
+                        value |= GMAC_CONTROL_PS;
+
+                        if (hw->ps == SPEED_10)
+                                value &= ~GMAC_CONTROL_FES;
+                        else
+                                value |= GMAC_CONTROL_FES;
+                }
+        }
+
         writel(value, ioaddr + GMAC_CONTROL);
 
         /* Mask GMAC interrupts */
-        writel(0x207, ioaddr + GMAC_INT_MASK);
+        value = GMAC_INT_DEFAULT_MASK;
+
+        if (hw->pmt)
+                value &= ~GMAC_INT_DISABLE_PMT;
+        if (hw->pcs)
+                value &= ~GMAC_INT_DISABLE_PCS;
+
+        writel(value, ioaddr + GMAC_INT_MASK);
 
 #ifdef STMMAC_VLAN_TAG_USED
         /* Tag detection without filtering */
@@ -241,6 +267,39 @@ static void dwmac1000_pmt(struct mac_device_info *hw, unsigned long mode)
         writel(pmt, ioaddr + GMAC_PMT);
 }
 
+/* RGMII or SMII interface */
+static void dwmac1000_rgsmii(void __iomem *ioaddr, struct stmmac_extra_stats *x)
+{
+        u32 status;
+
+        status = readl(ioaddr + GMAC_RGSMIIIS);
+        x->irq_rgmii_n++;
+
+        /* Check the link status */
+        if (status & GMAC_RGSMIIIS_LNKSTS) {
+                int speed_value;
+
+                x->pcs_link = 1;
+
+                speed_value = ((status & GMAC_RGSMIIIS_SPEED) >>
+                               GMAC_RGSMIIIS_SPEED_SHIFT);
+                if (speed_value == GMAC_RGSMIIIS_SPEED_125)
+                        x->pcs_speed = SPEED_1000;
+                else if (speed_value == GMAC_RGSMIIIS_SPEED_25)
+                        x->pcs_speed = SPEED_100;
+                else
+                        x->pcs_speed = SPEED_10;
+
+                x->pcs_duplex = (status & GMAC_RGSMIIIS_LNKMOD_MASK);
+
+                pr_info("Link is Up - %d/%s\n", (int)x->pcs_speed,
+                        x->pcs_duplex ? "Full" : "Half");
+        } else {
+                x->pcs_link = 0;
+                pr_info("Link is Down\n");
+        }
+}
+
 static int dwmac1000_irq_status(struct mac_device_info *hw,
                                 struct stmmac_extra_stats *x)
 {
@@ -249,19 +308,20 @@ static int dwmac1000_irq_status(struct mac_device_info *hw,
         int ret = 0;
 
         /* Not used events (e.g. MMC interrupts) are not handled. */
-        if ((intr_status & mmc_tx_irq))
+        if ((intr_status & GMAC_INT_STATUS_MMCTIS))
                 x->mmc_tx_irq_n++;
-        if (unlikely(intr_status & mmc_rx_irq))
+        if (unlikely(intr_status & GMAC_INT_STATUS_MMCRIS))
                 x->mmc_rx_irq_n++;
-        if (unlikely(intr_status & mmc_rx_csum_offload_irq))
+        if (unlikely(intr_status & GMAC_INT_STATUS_MMCCSUM))
                 x->mmc_rx_csum_offload_irq_n++;
-        if (unlikely(intr_status & pmt_irq)) {
+        if (unlikely(intr_status & GMAC_INT_DISABLE_PMT)) {
                 /* clear the PMT bits 5 and 6 by reading the PMT status reg */
                 readl(ioaddr + GMAC_PMT);
                 x->irq_receive_pmt_irq_n++;
         }
-        /* MAC trx/rx EEE LPI entry/exit interrupts */
-        if (intr_status & lpiis_irq) {
+
+        /* MAC tx/rx EEE LPI entry/exit interrupts */
+        if (intr_status & GMAC_INT_STATUS_LPIIS) {
                 /* Clean LPI interrupt by reading the Reg 12 */
                 ret = readl(ioaddr + LPI_CTRL_STATUS);
 
@@ -275,36 +335,10 @@ static int dwmac1000_irq_status(struct mac_device_info *hw,
                         x->irq_rx_path_exit_lpi_mode_n++;
         }
 
-        if ((intr_status & pcs_ane_irq) || (intr_status & pcs_link_irq)) {
-                readl(ioaddr + GMAC_AN_STATUS);
-                x->irq_pcs_ane_n++;
-        }
-        if (intr_status & rgmii_irq) {
-                u32 status = readl(ioaddr + GMAC_S_R_GMII);
-                x->irq_rgmii_n++;
-
-                /* Save and dump the link status. */
-                if (status & GMAC_S_R_GMII_LINK) {
-                        int speed_value = (status & GMAC_S_R_GMII_SPEED) >>
-                            GMAC_S_R_GMII_SPEED_SHIFT;
-                        x->pcs_duplex = (status & GMAC_S_R_GMII_MODE);
-
-                        if (speed_value == GMAC_S_R_GMII_SPEED_125)
-                                x->pcs_speed = SPEED_1000;
-                        else if (speed_value == GMAC_S_R_GMII_SPEED_25)
-                                x->pcs_speed = SPEED_100;
-                        else
-                                x->pcs_speed = SPEED_10;
+        dwmac_pcs_isr(ioaddr, GMAC_PCS_BASE, intr_status, x);
 
-                        x->pcs_link = 1;
-                        pr_debug("%s: Link is Up - %d/%s\n", __func__,
-                                 (int)x->pcs_speed,
-                                 x->pcs_duplex ? "Full" : "Half");
-                } else {
-                        x->pcs_link = 0;
-                        pr_debug("%s: Link is Down\n", __func__);
-                }
-        }
+        if (intr_status & PCS_RGSMIIIS_IRQ)
+                dwmac1000_rgsmii(ioaddr, x);
 
         return ret;
 }
@@ -363,38 +397,20 @@ static void dwmac1000_set_eee_timer(struct mac_device_info *hw, int ls, int tw)
         writel(value, ioaddr + LPI_TIMER_CTRL);
 }
 
-static void dwmac1000_ctrl_ane(struct mac_device_info *hw, bool restart)
+static void dwmac1000_ctrl_ane(void __iomem *ioaddr, bool ane, bool srgmi_ral,
+                               bool loopback)
 {
-        void __iomem *ioaddr = hw->pcsr;
-        /* auto negotiation enable and External Loopback enable */
-        u32 value = GMAC_AN_CTRL_ANE | GMAC_AN_CTRL_ELE;
-
-        if (restart)
-                value |= GMAC_AN_CTRL_RAN;
-
-        writel(value, ioaddr + GMAC_AN_CTRL);
+        dwmac_ctrl_ane(ioaddr, GMAC_PCS_BASE, ane, srgmi_ral, loopback);
 }
 
-static void dwmac1000_get_adv(struct mac_device_info *hw, struct rgmii_adv *adv)
+static void dwmac1000_rane(void __iomem *ioaddr, bool restart)
 {
-        void __iomem *ioaddr = hw->pcsr;
-        u32 value = readl(ioaddr + GMAC_ANE_ADV);
-
-        if (value & GMAC_ANE_FD)
-                adv->duplex = DUPLEX_FULL;
-        if (value & GMAC_ANE_HD)
-                adv->duplex |= DUPLEX_HALF;
-
-        adv->pause = (value & GMAC_ANE_PSE) >> GMAC_ANE_PSE_SHIFT;
-
-        value = readl(ioaddr + GMAC_ANE_LPA);
-
-        if (value & GMAC_ANE_FD)
-                adv->lp_duplex = DUPLEX_FULL;
-        if (value & GMAC_ANE_HD)
-                adv->lp_duplex = DUPLEX_HALF;
+        dwmac_rane(ioaddr, GMAC_PCS_BASE, restart);
+}
 
-        adv->lp_pause = (value & GMAC_ANE_PSE) >> GMAC_ANE_PSE_SHIFT;
+static void dwmac1000_get_adv_lp(void __iomem *ioaddr, struct rgmii_adv *adv)
+{
+        dwmac_get_adv_lp(ioaddr, GMAC_PCS_BASE, adv);
 }
 
 static void dwmac1000_debug(void __iomem *ioaddr, struct stmmac_extra_stats *x)
@@ -485,9 +501,10 @@ static const struct stmmac_ops dwmac1000_ops = {
         .reset_eee_mode = dwmac1000_reset_eee_mode,
         .set_eee_timer = dwmac1000_set_eee_timer,
         .set_eee_pls = dwmac1000_set_eee_pls,
-        .ctrl_ane = dwmac1000_ctrl_ane,
-        .get_adv = dwmac1000_get_adv,
         .debug = dwmac1000_debug,
+        .pcs_ctrl_ane = dwmac1000_ctrl_ane,
+        .pcs_rane = dwmac1000_rane,
+        .pcs_get_adv_lp = dwmac1000_get_adv_lp,
 };
 
 struct mac_device_info *dwmac1000_setup(void __iomem *ioaddr, int mcbins,
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4.h b/drivers/net/ethernet/stmicro/stmmac/dwmac4.h
index bc50952a18e7..6f4f5ce25114 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4.h
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4.h
@@ -24,10 +24,8 @@
 #define GMAC_QX_TX_FLOW_CTRL(x)         (0x70 + x * 4)
 #define GMAC_INT_STATUS                 0x000000b0
 #define GMAC_INT_EN                     0x000000b4
-#define GMAC_AN_CTRL                    0x000000e0
-#define GMAC_AN_STATUS                  0x000000e4
-#define GMAC_AN_ADV                     0x000000e8
-#define GMAC_AN_LPA                     0x000000ec
+#define GMAC_PCS_BASE                   0x000000e0
+#define GMAC_PHYIF_CONTROL_STATUS       0x000000f8
 #define GMAC_PMT                        0x000000c0
 #define GMAC_VERSION                    0x00000110
 #define GMAC_DEBUG                      0x00000114
@@ -54,9 +52,18 @@
 #define GMAC_TX_FLOW_CTRL_PT_SHIFT      16
 
 /*  MAC Interrupt bitmap*/
+#define GMAC_INT_RGSMIIS                BIT(0)
+#define GMAC_INT_PCS_LINK               BIT(1)
+#define GMAC_INT_PCS_ANE                BIT(2)
+#define GMAC_INT_PCS_PHYIS              BIT(3)
 #define GMAC_INT_PMT_EN                 BIT(4)
 #define GMAC_INT_LPI_EN                 BIT(5)
 
+#define GMAC_PCS_IRQ_DEFAULT    (GMAC_INT_RGSMIIS | GMAC_INT_PCS_LINK | \
+                                 GMAC_INT_PCS_ANE)
+
+#define GMAC_INT_DEFAULT_MASK   GMAC_INT_PMT_EN
+
 enum dwmac4_irq_status {
         time_stamp_irq = 0x00001000,
         mmc_rx_csum_offload_irq = 0x00000800,
@@ -64,19 +71,8 @@ enum dwmac4_irq_status {
         mmc_rx_irq = 0x00000200,
         mmc_irq = 0x00000100,
         pmt_irq = 0x00000010,
-        pcs_ane_irq = 0x00000004,
-        pcs_link_irq = 0x00000002,
 };
 
-/* MAC Auto-Neg bitmap*/
-#define GMAC_AN_CTRL_RAN                BIT(9)
-#define GMAC_AN_CTRL_ANE                BIT(12)
-#define GMAC_AN_CTRL_ELE                BIT(14)
-#define GMAC_AN_FD                      BIT(5)
-#define GMAC_AN_HD                      BIT(6)
-#define GMAC_AN_PSE_MASK                GENMASK(8, 7)
-#define GMAC_AN_PSE_SHIFT               7
-
 /* MAC PMT bitmap */
 enum power_event {
         pointer_reset = 0x80000000,
@@ -250,6 +246,23 @@ enum power_event {
 #define MTL_DEBUG_RRCSTS_FLUSH          3
 #define MTL_DEBUG_RWCSTS                BIT(0)
 
+/* SGMII/RGMII status register */
+#define GMAC_PHYIF_CTRLSTATUS_TC                BIT(0)
+#define GMAC_PHYIF_CTRLSTATUS_LUD               BIT(1)
+#define GMAC_PHYIF_CTRLSTATUS_SMIDRXS           BIT(4)
+#define GMAC_PHYIF_CTRLSTATUS_LNKMOD            BIT(16)
+#define GMAC_PHYIF_CTRLSTATUS_SPEED             GENMASK(18, 17)
+#define GMAC_PHYIF_CTRLSTATUS_SPEED_SHIFT       17
+#define GMAC_PHYIF_CTRLSTATUS_LNKSTS            BIT(19)
+#define GMAC_PHYIF_CTRLSTATUS_JABTO             BIT(20)
+#define GMAC_PHYIF_CTRLSTATUS_FALSECARDET       BIT(21)
+/* LNKMOD */
+#define GMAC_PHYIF_CTRLSTATUS_LNKMOD_MASK       0x1
+/* LNKSPEED */
+#define GMAC_PHYIF_CTRLSTATUS_SPEED_125         0x2
+#define GMAC_PHYIF_CTRLSTATUS_SPEED_25          0x1
+#define GMAC_PHYIF_CTRLSTATUS_SPEED_2_5         0x0
+
 extern const struct stmmac_dma_ops dwmac4_dma_ops;
 extern const struct stmmac_dma_ops dwmac410_dma_ops;
 #endif /* __DWMAC4_H__ */
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
index 44da877d2483..df5580dcdfed 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
@@ -17,6 +17,7 @@
 #include <linux/slab.h>
 #include <linux/ethtool.h>
 #include <linux/io.h>
+#include "stmmac_pcs.h"
 #include "dwmac4.h"
 
 static void dwmac4_core_init(struct mac_device_info *hw, int mtu)
@@ -31,10 +32,31 @@ static void dwmac4_core_init(struct mac_device_info *hw, int mtu)
         if (mtu > 2000)
                 value |= GMAC_CONFIG_JE;
 
+        if (hw->ps) {
+                value |= GMAC_CONFIG_TE;
+
+                if (hw->ps == SPEED_1000) {
+                        value &= ~GMAC_CONFIG_PS;
+                } else {
+                        value |= GMAC_CONFIG_PS;
+
+                        if (hw->ps == SPEED_10)
+                                value &= ~GMAC_CONFIG_FES;
+                        else
+                                value |= GMAC_CONFIG_FES;
+                }
+        }
+
         writel(value, ioaddr + GMAC_CONFIG);
 
         /* Mask GMAC interrupts */
-        writel(GMAC_INT_PMT_EN, ioaddr + GMAC_INT_EN);
+        value = GMAC_INT_DEFAULT_MASK;
+        if (hw->pmt)
+                value |= GMAC_INT_PMT_EN;
+        if (hw->pcs)
+                value |= GMAC_PCS_IRQ_DEFAULT;
+
+        writel(value, ioaddr + GMAC_INT_EN);
 }
 
 static void dwmac4_dump_regs(struct mac_device_info *hw)
@@ -190,39 +212,53 @@ static void dwmac4_flow_ctrl(struct mac_device_info *hw, unsigned int duplex,
         }
 }
 
-static void dwmac4_ctrl_ane(struct mac_device_info *hw, bool restart)
+static void dwmac4_ctrl_ane(void __iomem *ioaddr, bool ane, bool srgmi_ral,
+                            bool loopback)
 {
-        void __iomem *ioaddr = hw->pcsr;
-
-        /* auto negotiation enable and External Loopback enable */
-        u32 value = GMAC_AN_CTRL_ANE | GMAC_AN_CTRL_ELE;
+        dwmac_ctrl_ane(ioaddr, GMAC_PCS_BASE, ane, srgmi_ral, loopback);
+}
 
-        if (restart)
-                value |= GMAC_AN_CTRL_RAN;
+static void dwmac4_rane(void __iomem *ioaddr, bool restart)
+{
+        dwmac_rane(ioaddr, GMAC_PCS_BASE, restart);
+}
 
-        writel(value, ioaddr + GMAC_AN_CTRL);
+static void dwmac4_get_adv_lp(void __iomem *ioaddr, struct rgmii_adv *adv)
+{
+        dwmac_get_adv_lp(ioaddr, GMAC_PCS_BASE, adv);
 }
 
-static void dwmac4_get_adv(struct mac_device_info *hw, struct rgmii_adv *adv)
+/* RGMII or SMII interface */
+static void dwmac4_phystatus(void __iomem *ioaddr, struct stmmac_extra_stats *x)
 {
-        void __iomem *ioaddr = hw->pcsr;
-        u32 value = readl(ioaddr + GMAC_AN_ADV);
+        u32 status;
 
-        if (value & GMAC_AN_FD)
-                adv->duplex = DUPLEX_FULL;
-        if (value & GMAC_AN_HD)
-                adv->duplex |= DUPLEX_HALF;
+        status = readl(ioaddr + GMAC_PHYIF_CONTROL_STATUS);
+        x->irq_rgmii_n++;
 
-        adv->pause = (value & GMAC_AN_PSE_MASK) >> GMAC_AN_PSE_SHIFT;
+        /* Check the link status */
+        if (status & GMAC_PHYIF_CTRLSTATUS_LNKSTS) {
+                int speed_value;
 
-        value = readl(ioaddr + GMAC_AN_LPA);
+                x->pcs_link = 1;
+
+                speed_value = ((status & GMAC_PHYIF_CTRLSTATUS_SPEED) >>
+                               GMAC_PHYIF_CTRLSTATUS_SPEED_SHIFT);
+                if (speed_value == GMAC_PHYIF_CTRLSTATUS_SPEED_125)
+                        x->pcs_speed = SPEED_1000;
+                else if (speed_value == GMAC_PHYIF_CTRLSTATUS_SPEED_25)
+                        x->pcs_speed = SPEED_100;
+                else
+                        x->pcs_speed = SPEED_10;
 
-        if (value & GMAC_AN_FD)
-                adv->lp_duplex = DUPLEX_FULL;
-        if (value & GMAC_AN_HD)
-                adv->lp_duplex = DUPLEX_HALF;
+                x->pcs_duplex = (status & GMAC_PHYIF_CTRLSTATUS_LNKMOD_MASK);
 
-        adv->lp_pause = (value & GMAC_AN_PSE_MASK) >> GMAC_AN_PSE_SHIFT;
+                pr_info("Link is Up - %d/%s\n", (int)x->pcs_speed,
+                        x->pcs_duplex ? "Full" : "Half");
+        } else {
+                x->pcs_link = 0;
+                pr_info("Link is Down\n");
+        }
 }
 
 static int dwmac4_irq_status(struct mac_device_info *hw,
@@ -248,11 +284,6 @@ static int dwmac4_irq_status(struct mac_device_info *hw,
                 x->irq_receive_pmt_irq_n++;
         }
 
-        if ((intr_status & pcs_ane_irq) || (intr_status & pcs_link_irq)) {
-                readl(ioaddr + GMAC_AN_STATUS);
-                x->irq_pcs_ane_n++;
-        }
-
         mtl_int_qx_status = readl(ioaddr + MTL_INT_STATUS);
         /* Check MTL Interrupt: Currently only one queue is used: Q0. */
         if (mtl_int_qx_status & MTL_INT_Q0) {
@@ -267,6 +298,10 @@ static int dwmac4_irq_status(struct mac_device_info *hw,
                 }
         }
 
+        dwmac_pcs_isr(ioaddr, GMAC_PCS_BASE, intr_status, x);
+        if (intr_status & PCS_RGSMIIIS_IRQ)
+                dwmac4_phystatus(ioaddr, x);
+
         return ret;
 }
 
@@ -363,8 +398,9 @@ static const struct stmmac_ops dwmac4_ops = {
         .pmt = dwmac4_pmt,
         .set_umac_addr = dwmac4_set_umac_addr,
         .get_umac_addr = dwmac4_get_umac_addr,
-        .ctrl_ane = dwmac4_ctrl_ane,
-        .get_adv = dwmac4_get_adv,
+        .pcs_ctrl_ane = dwmac4_ctrl_ane,
+        .pcs_rane = dwmac4_rane,
+        .pcs_get_adv_lp = dwmac4_get_adv_lp,
         .debug = dwmac4_debug,
         .set_filter = dwmac4_set_filter,
 };
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac.h b/drivers/net/ethernet/stmicro/stmmac/stmmac.h
index 59ae6088cd22..8dc9056c1001 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac.h
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac.h
@@ -117,7 +117,6 @@ struct stmmac_priv {
         int eee_enabled;
         int eee_active;
         int tx_lpi_timer;
-        int pcs;
         unsigned int mode;
         int extend_desc;
         struct ptp_clock *ptp_clock;
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c
index e2b98b01647e..1e06173fc9d7 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c
@@ -276,7 +276,8 @@ static int stmmac_ethtool_getsettings(struct net_device *dev,
         struct phy_device *phy = priv->phydev;
         int rc;
 
-        if ((priv->pcs & STMMAC_PCS_RGMII) || (priv->pcs & STMMAC_PCS_SGMII)) {
+        if (priv->hw->pcs & STMMAC_PCS_RGMII ||
+            priv->hw->pcs & STMMAC_PCS_SGMII) {
                 struct rgmii_adv adv;
 
                 if (!priv->xstats.pcs_link) {
@@ -289,10 +290,10 @@ static int stmmac_ethtool_getsettings(struct net_device *dev,
                 ethtool_cmd_speed_set(cmd, priv->xstats.pcs_speed);
 
                 /* Get and convert ADV/LP_ADV from the HW AN registers */
-                if (!priv->hw->mac->get_adv)
+                if (!priv->hw->mac->pcs_get_adv_lp)
                         return -EOPNOTSUPP;     /* should never happen indeed */
 
-                priv->hw->mac->get_adv(priv->hw, &adv);
+                priv->hw->mac->pcs_get_adv_lp(priv->ioaddr, &adv);
 
                 /* Encoding of PSE bits is defined in 802.3z, 37.2.1.4 */
 
@@ -361,7 +362,8 @@ static int stmmac_ethtool_setsettings(struct net_device *dev,
         struct phy_device *phy = priv->phydev;
         int rc;
 
-        if ((priv->pcs & STMMAC_PCS_RGMII) || (priv->pcs & STMMAC_PCS_SGMII)) {
+        if (priv->hw->pcs & STMMAC_PCS_RGMII ||
+            priv->hw->pcs & STMMAC_PCS_SGMII) {
                 u32 mask = ADVERTISED_Autoneg | ADVERTISED_Pause;
 
                 /* Only support ANE */
@@ -376,8 +378,11 @@ static int stmmac_ethtool_setsettings(struct net_device *dev,
                         ADVERTISED_10baseT_Full);
 
                 spin_lock(&priv->lock);
-                if (priv->hw->mac->ctrl_ane)
-                        priv->hw->mac->ctrl_ane(priv->hw, 1);
+
+                if (priv->hw->mac->pcs_ctrl_ane)
+                        priv->hw->mac->pcs_ctrl_ane(priv->ioaddr, 1,
+                                                    priv->hw->ps, 0);
+
                 spin_unlock(&priv->lock);
 
                 return 0;
@@ -452,11 +457,22 @@ stmmac_get_pauseparam(struct net_device *netdev,
 {
         struct stmmac_priv *priv = netdev_priv(netdev);
 
-        if (priv->pcs)  /* FIXME */
-                return;
-
         pause->rx_pause = 0;
         pause->tx_pause = 0;
+
+        if (priv->hw->pcs && priv->hw->mac->pcs_get_adv_lp) {
+                struct rgmii_adv adv_lp;
+
+                pause->autoneg = 1;
+                priv->hw->mac->pcs_get_adv_lp(priv->ioaddr, &adv_lp);
+                if (!adv_lp.pause)
+                        return;
+        } else {
+                if (!(priv->phydev->supported & SUPPORTED_Pause) ||
+                    !(priv->phydev->supported & SUPPORTED_Asym_Pause))
+                        return;
+        }
+
         pause->autoneg = priv->phydev->autoneg;
 
         if (priv->flow_ctrl & FLOW_RX)
@@ -473,10 +489,19 @@ stmmac_set_pauseparam(struct net_device *netdev,
         struct stmmac_priv *priv = netdev_priv(netdev);
         struct phy_device *phy = priv->phydev;
         int new_pause = FLOW_OFF;
-        int ret = 0;
 
-        if (priv->pcs)  /* FIXME */
-                return -EOPNOTSUPP;
+        if (priv->hw->pcs && priv->hw->mac->pcs_get_adv_lp) {
+                struct rgmii_adv adv_lp;
+
+                pause->autoneg = 1;
+                priv->hw->mac->pcs_get_adv_lp(priv->ioaddr, &adv_lp);
+                if (!adv_lp.pause)
+                        return -EOPNOTSUPP;
+        } else {
+                if (!(phy->supported & SUPPORTED_Pause) ||
+                    !(phy->supported & SUPPORTED_Asym_Pause))
+                        return -EOPNOTSUPP;
+        }
 
         if (pause->rx_pause)
                 new_pause |= FLOW_RX;
@@ -488,11 +513,12 @@ stmmac_set_pauseparam(struct net_device *netdev,
 
         if (phy->autoneg) {
                 if (netif_running(netdev))
-                        ret = phy_start_aneg(phy);
-        } else
-                priv->hw->mac->flow_ctrl(priv->hw, phy->duplex,
-                                         priv->flow_ctrl, priv->pause);
-        return ret;
+                        return phy_start_aneg(phy);
+        }
+
+        priv->hw->mac->flow_ctrl(priv->hw, phy->duplex, priv->flow_ctrl,
+                                 priv->pause);
+        return 0;
 }
 
 static void stmmac_get_ethtool_stats(struct net_device *dev,
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
index a473c182c91d..aab777c1ba33 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
@@ -285,8 +285,9 @@ bool stmmac_eee_init(struct stmmac_priv *priv)
         /* Using PCS we cannot dial with the phy registers at this stage
          * so we do not support extra feature like EEE.
          */
-        if ((priv->pcs == STMMAC_PCS_RGMII) || (priv->pcs == STMMAC_PCS_TBI) ||
-            (priv->pcs == STMMAC_PCS_RTBI))
+        if ((priv->hw->pcs == STMMAC_PCS_RGMII) ||
+            (priv->hw->pcs == STMMAC_PCS_TBI) ||
+            (priv->hw->pcs == STMMAC_PCS_RTBI))
                 goto out;
 
         /* MAC core supports the EEE feature. */
@@ -799,10 +800,10 @@ static void stmmac_check_pcs_mode(struct stmmac_priv *priv)
                     (interface == PHY_INTERFACE_MODE_RGMII_RXID) ||
                     (interface == PHY_INTERFACE_MODE_RGMII_TXID)) {
                         pr_debug("STMMAC: PCS RGMII support enable\n");
-                        priv->pcs = STMMAC_PCS_RGMII;
+                        priv->hw->pcs = STMMAC_PCS_RGMII;
                 } else if (interface == PHY_INTERFACE_MODE_SGMII) {
                         pr_debug("STMMAC: PCS SGMII support enable\n");
-                        priv->pcs = STMMAC_PCS_SGMII;
+                        priv->hw->pcs = STMMAC_PCS_SGMII;
                 }
         }
 }
@@ -1665,6 +1666,19 @@ static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
         if (priv->plat->bus_setup)
                 priv->plat->bus_setup(priv->ioaddr);
 
+        /* PS and related bits will be programmed according to the speed */
+        if (priv->hw->pcs) {
+                int speed = priv->plat->mac_port_sel_speed;
+
+                if ((speed == SPEED_10) || (speed == SPEED_100) ||
+                    (speed == SPEED_1000)) {
+                        priv->hw->ps = speed;
+                } else {
+                        dev_warn(priv->device, "invalid port speed\n");
+                        priv->hw->ps = 0;
+                }
+        }
+
         /* Initialize the MAC Core */
         priv->hw->mac->core_init(priv->hw, dev->mtu);
 
@@ -1714,8 +1728,8 @@ static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
                 priv->hw->dma->rx_watchdog(priv->ioaddr, MAX_DMA_RIWT);
         }
 
-        if (priv->pcs && priv->hw->mac->ctrl_ane)
-                priv->hw->mac->ctrl_ane(priv->hw, 0);
+        if (priv->hw->pcs && priv->hw->mac->pcs_ctrl_ane)
+                priv->hw->mac->pcs_ctrl_ane(priv->hw, 1, priv->hw->ps, 0);
 
         /*  set TX ring length */
         if (priv->hw->dma->set_tx_ring_len)
@@ -1748,8 +1762,9 @@ static int stmmac_open(struct net_device *dev)
 
         stmmac_check_ether_addr(priv);
 
-        if (priv->pcs != STMMAC_PCS_RGMII && priv->pcs != STMMAC_PCS_TBI &&
-            priv->pcs != STMMAC_PCS_RTBI) {
+        if (priv->hw->pcs != STMMAC_PCS_RGMII &&
+            priv->hw->pcs != STMMAC_PCS_TBI &&
+            priv->hw->pcs != STMMAC_PCS_RTBI) {
                 ret = stmmac_init_phy(dev);
                 if (ret) {
                         pr_err("%s: Cannot attach to PHY (error: %d)\n",
@@ -2809,6 +2824,14 @@ static irqreturn_t stmmac_interrupt(int irq, void *dev_id)
                                                         priv->rx_tail_addr,
                                                         STMMAC_CHAN0);
                 }
+
+                /* PCS link status */
+                if (priv->hw->pcs) {
+                        if (priv->xstats.pcs_link)
+                                netif_carrier_on(dev);
+                        else
+                                netif_carrier_off(dev);
+                }
         }
 
         /* To handle DMA interrupts */
@@ -3130,6 +3153,7 @@ static int stmmac_hw_init(struct stmmac_priv *priv)
                  */
                 priv->plat->enh_desc = priv->dma_cap.enh_desc;
                 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up;
+                priv->hw->pmt = priv->plat->pmt;
 
                 /* TXCOE doesn't work in thresh DMA mode */
                 if (priv->plat->force_thresh_dma_mode)
@@ -3325,8 +3349,9 @@ int stmmac_dvr_probe(struct device *device,
 
         stmmac_check_pcs_mode(priv);
 
-        if (priv->pcs != STMMAC_PCS_RGMII && priv->pcs != STMMAC_PCS_TBI &&
-            priv->pcs != STMMAC_PCS_RTBI) {
+        if (priv->hw->pcs != STMMAC_PCS_RGMII  &&
+            priv->hw->pcs != STMMAC_PCS_TBI &&
+            priv->hw->pcs != STMMAC_PCS_RTBI) {
                 /* MDIO bus Registration */
                 ret = stmmac_mdio_register(ndev);
                 if (ret < 0) {
@@ -3376,8 +3401,9 @@ int stmmac_dvr_remove(struct device *dev)
                 reset_control_assert(priv->stmmac_rst);
         clk_disable_unprepare(priv->pclk);
         clk_disable_unprepare(priv->stmmac_clk);
-        if (priv->pcs != STMMAC_PCS_RGMII && priv->pcs != STMMAC_PCS_TBI &&
-            priv->pcs != STMMAC_PCS_RTBI)
+        if (priv->hw->pcs != STMMAC_PCS_RGMII &&
+            priv->hw->pcs != STMMAC_PCS_TBI &&
+            priv->hw->pcs != STMMAC_PCS_RTBI)
                 stmmac_mdio_unregister(ndev);
         free_netdev(ndev);
 
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_pcs.h b/drivers/net/ethernet/stmicro/stmmac/stmmac_pcs.h
new file mode 100644
index 000000000000..eba41c24b7a7
--- /dev/null
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_pcs.h
@@ -0,0 +1,159 @@
+/*
+ * stmmac_pcs.h: Physical Coding Sublayer Header File
+ *
+ * Copyright (C) 2016 STMicroelectronics (R&D) Limited
+ * Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef __STMMAC_PCS_H__
+#define __STMMAC_PCS_H__
+
+#include <linux/slab.h>
+#include <linux/io.h>
+#include "common.h"
+
+/* PCS registers (AN/TBI/SGMII/RGMII) offsets */
+#define GMAC_AN_CTRL(x)         (x)             /* AN control */
+#define GMAC_AN_STATUS(x)       (x + 0x4)       /* AN status */
+#define GMAC_ANE_ADV(x)         (x + 0x8)       /* ANE Advertisement */
+#define GMAC_ANE_LPA(x)         (x + 0xc)       /* ANE link partener ability */
+#define GMAC_ANE_EXP(x)         (x + 0x10)      /* ANE expansion */
+#define GMAC_TBI(x)             (x + 0x14)      /* TBI extend status */
+
+/* AN Configuration defines */
+#define GMAC_AN_CTRL_RAN        BIT(9)  /* Restart Auto-Negotiation */
+#define GMAC_AN_CTRL_ANE        BIT(12) /* Auto-Negotiation Enable */
+#define GMAC_AN_CTRL_ELE        BIT(14) /* External Loopback Enable */
+#define GMAC_AN_CTRL_ECD        BIT(16) /* Enable Comma Detect */
+#define GMAC_AN_CTRL_LR         BIT(17) /* Lock to Reference */
+#define GMAC_AN_CTRL_SGMRAL     BIT(18) /* SGMII RAL Control */
+
+/* AN Status defines */
+#define GMAC_AN_STATUS_LS       BIT(2)  /* Link Status 0:down 1:up */
+#define GMAC_AN_STATUS_ANA      BIT(3)  /* Auto-Negotiation Ability */
+#define GMAC_AN_STATUS_ANC      BIT(5)  /* Auto-Negotiation Complete */
+#define GMAC_AN_STATUS_ES       BIT(8)  /* Extended Status */
+
+/* ADV and LPA defines */
+#define GMAC_ANE_FD             BIT(5)
+#define GMAC_ANE_HD             BIT(6)
+#define GMAC_ANE_PSE            GENMASK(8, 7)
+#define GMAC_ANE_PSE_SHIFT      7
+#define GMAC_ANE_RFE            GENMASK(13, 12)
+#define GMAC_ANE_RFE_SHIFT      12
+#define GMAC_ANE_ACK            BIT(14)
+
+/**
+ * dwmac_pcs_isr - TBI, RTBI, or SGMII PHY ISR
+ * @ioaddr: IO registers pointer
+ * @reg: Base address of the AN Control Register.
+ * @intr_status: GMAC core interrupt status
+ * @x: pointer to log these events as stats
+ * Description: it is the ISR for PCS events: Auto-Negotiation Completed and
+ * Link status.
+ */
+static inline void dwmac_pcs_isr(void __iomem *ioaddr, u32 reg,
+                                 unsigned int intr_status,
+                                 struct stmmac_extra_stats *x)
+{
+        u32 val = readl(ioaddr + GMAC_AN_STATUS(reg));
+
+        if (intr_status & PCS_ANE_IRQ) {
+                x->irq_pcs_ane_n++;
+                if (val & GMAC_AN_STATUS_ANC)
+                        pr_info("stmmac_pcs: ANE process completed\n");
+        }
+
+        if (intr_status & PCS_LINK_IRQ) {
+                x->irq_pcs_link_n++;
+                if (val & GMAC_AN_STATUS_LS)
+                        pr_info("stmmac_pcs: Link Up\n");
+                else
+                        pr_info("stmmac_pcs: Link Down\n");
+        }
+}
+
+/**
+ * dwmac_rane - To restart ANE
+ * @ioaddr: IO registers pointer
+ * @reg: Base address of the AN Control Register.
+ * @restart: to restart ANE
+ * Description: this is to just restart the Auto-Negotiation.
+ */
+static inline void dwmac_rane(void __iomem *ioaddr, u32 reg, bool restart)
+{
+        u32 value = readl(ioaddr + GMAC_AN_CTRL(reg));
+
+        if (restart)
+                value |= GMAC_AN_CTRL_RAN;
+
+        writel(value, ioaddr + GMAC_AN_CTRL(reg));
+}
+
+/**
+ * dwmac_ctrl_ane - To program the AN Control Register.
+ * @ioaddr: IO registers pointer
+ * @reg: Base address of the AN Control Register.
+ * @ane: to enable the auto-negotiation
+ * @srgmi_ral: to manage MAC-2-MAC SGMII connections.
+ * @loopback: to cause the PHY to loopback tx data into rx path.
+ * Description: this is the main function to configure the AN control register
+ * and init the ANE, select loopback (usually for debugging purpose) and
+ * configure SGMII RAL.
+ */
+static inline void dwmac_ctrl_ane(void __iomem *ioaddr, u32 reg, bool ane,
+                                  bool srgmi_ral, bool loopback)
+{
+        u32 value = readl(ioaddr + GMAC_AN_CTRL(reg));
+
+        /* Enable and restart the Auto-Negotiation */
+        if (ane)
+                value |= GMAC_AN_CTRL_ANE | GMAC_AN_CTRL_RAN;
+
+        /* In case of MAC-2-MAC connection, block is configured to operate
+         * according to MAC conf register.
+         */
+        if (srgmi_ral)
+                value |= GMAC_AN_CTRL_SGMRAL;
+
+        if (loopback)
+                value |= GMAC_AN_CTRL_ELE;
+
+        writel(value, ioaddr + GMAC_AN_CTRL(reg));
+}
+
+/**
+ * dwmac_get_adv_lp - Get ADV and LP cap
+ * @ioaddr: IO registers pointer
+ * @reg: Base address of the AN Control Register.
+ * @adv_lp: structure to store the adv,lp status
+ * Description: this is to expose the ANE advertisement and Link partner ability
+ * status to ethtool support.
+ */
+static inline void dwmac_get_adv_lp(void __iomem *ioaddr, u32 reg,
+                                    struct rgmii_adv *adv_lp)
+{
+        u32 value = readl(ioaddr + GMAC_ANE_ADV(reg));
+
+        if (value & GMAC_ANE_FD)
+                adv_lp->duplex = DUPLEX_FULL;
+        if (value & GMAC_ANE_HD)
+                adv_lp->duplex |= DUPLEX_HALF;
+
+        adv_lp->pause = (value & GMAC_ANE_PSE) >> GMAC_ANE_PSE_SHIFT;
+
+        value = readl(ioaddr + GMAC_ANE_LPA(reg));
+
+        if (value & GMAC_ANE_FD)
+                adv_lp->lp_duplex = DUPLEX_FULL;
+        if (value & GMAC_ANE_HD)
+                adv_lp->lp_duplex = DUPLEX_HALF;
+
+        adv_lp->lp_pause = (value & GMAC_ANE_PSE) >> GMAC_ANE_PSE_SHIFT;
+}
+#endif /* __STMMAC_PCS_H__ */
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
index 409db913b117..f7dfc0ae8e9c 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
@@ -319,6 +319,8 @@ stmmac_probe_config_dt(struct platform_device *pdev, const char **mac)
                 pr_warn("force_sf_dma_mode is ignored if force_thresh_dma_mode is set.");
         }
 
+        of_property_read_u32(np, "snps,ps-speed", &plat->mac_port_sel_speed);
+
         plat->axi = stmmac_axi_setup(pdev);
 
         return plat;
@@ -411,7 +413,9 @@ static int stmmac_pltfr_suspend(struct device *dev)
         struct platform_device *pdev = to_platform_device(dev);
 
         ret = stmmac_suspend(dev);
-        if (priv->plat->exit)
+        if (priv->plat->suspend)
+                priv->plat->suspend(pdev, priv->plat->bsp_priv);
+        else if (priv->plat->exit)
                 priv->plat->exit(pdev, priv->plat->bsp_priv);
 
         return ret;
@@ -430,7 +434,9 @@ static int stmmac_pltfr_resume(struct device *dev)
         struct stmmac_priv *priv = netdev_priv(ndev);
         struct platform_device *pdev = to_platform_device(dev);
 
-        if (priv->plat->init)
+        if (priv->plat->resume)
+                priv->plat->resume(pdev, priv->plat->bsp_priv);
+        else if (priv->plat->init)
                 priv->plat->init(pdev, priv->plat->bsp_priv);
 
         return stmmac_resume(dev);
diff --git a/drivers/net/ethernet/synopsys/dwc_eth_qos.c b/drivers/net/ethernet/synopsys/dwc_eth_qos.c
index 158213cd6cdd..c14fa91c825f 100644
--- a/drivers/net/ethernet/synopsys/dwc_eth_qos.c
+++ b/drivers/net/ethernet/synopsys/dwc_eth_qos.c
@@ -598,7 +598,6 @@ struct net_local {
         struct work_struct txtimeout_reinit;
 
         phy_interface_t phy_interface;
-        struct phy_device *phy_dev;
         struct mii_bus *mii_bus;
 
         unsigned int link;
@@ -816,7 +815,7 @@ static int dwceqos_mdio_write(struct mii_bus *bus, int mii_id, int phyreg,
 static int dwceqos_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
 {
         struct net_local *lp = netdev_priv(ndev);
-        struct phy_device *phydev = lp->phy_dev;
+        struct phy_device *phydev = ndev->phydev;
 
         if (!netif_running(ndev))
                 return -EINVAL;
@@ -850,6 +849,7 @@ static void dwceqos_link_down(struct net_local *lp)
 
 static void dwceqos_link_up(struct net_local *lp)
 {
+        struct net_device *ndev = lp->ndev;
         u32 regval;
         unsigned long flags;
 
@@ -860,7 +860,7 @@ static void dwceqos_link_up(struct net_local *lp)
         dwceqos_write(lp, REG_DWCEQOS_MAC_LPI_CTRL_STATUS, regval);
         spin_unlock_irqrestore(&lp->hw_lock, flags);
 
-        lp->eee_active = !phy_init_eee(lp->phy_dev, 0);
+        lp->eee_active = !phy_init_eee(ndev->phydev, 0);
 
         /* Check for changed EEE capability */
         if (!lp->eee_active && lp->eee_enabled) {
@@ -876,7 +876,8 @@ static void dwceqos_link_up(struct net_local *lp)
 
 static void dwceqos_set_speed(struct net_local *lp)
 {
-        struct phy_device *phydev = lp->phy_dev;
+        struct net_device *ndev = lp->ndev;
+        struct phy_device *phydev = ndev->phydev;
         u32 regval;
 
         regval = dwceqos_read(lp, REG_DWCEQOS_MAC_CFG);
@@ -903,7 +904,7 @@ static void dwceqos_set_speed(struct net_local *lp)
 static void dwceqos_adjust_link(struct net_device *ndev)
 {
         struct net_local *lp = netdev_priv(ndev);
-        struct phy_device *phydev = lp->phy_dev;
+        struct phy_device *phydev = ndev->phydev;
         int status_change = 0;
 
         if (lp->phy_defer)
@@ -987,7 +988,6 @@ static int dwceqos_mii_probe(struct net_device *ndev)
         lp->link    = 0;
         lp->speed   = 0;
         lp->duplex  = DUPLEX_UNKNOWN;
-        lp->phy_dev = phydev;
 
         return 0;
 }
@@ -1531,6 +1531,7 @@ static void dwceqos_configure_bus(struct net_local *lp)
 
 static void dwceqos_init_hw(struct net_local *lp)
 {
+        struct net_device *ndev = lp->ndev;
         u32 regval;
         u32 buswidth;
         u32 dma_skip;
@@ -1645,10 +1646,10 @@ static void dwceqos_init_hw(struct net_local *lp)
                       regval | DWCEQOS_MAC_CFG_TE | DWCEQOS_MAC_CFG_RE);
 
         lp->phy_defer = false;
-        mutex_lock(&lp->phy_dev->lock);
-        phy_read_status(lp->phy_dev);
+        mutex_lock(&ndev->phydev->lock);
+        phy_read_status(ndev->phydev);
         dwceqos_adjust_link(lp->ndev);
-        mutex_unlock(&lp->phy_dev->lock);
+        mutex_unlock(&ndev->phydev->lock);
 }
 
 static void dwceqos_tx_reclaim(unsigned long data)
@@ -1898,7 +1899,7 @@ static int dwceqos_open(struct net_device *ndev)
          * hence the unusual init order with phy_start first.
          */
         lp->phy_defer = true;
-        phy_start(lp->phy_dev);
+        phy_start(ndev->phydev);
         dwceqos_init_hw(lp);
         napi_enable(&lp->napi);
 
@@ -1943,7 +1944,7 @@ static int dwceqos_stop(struct net_device *ndev)
 
         dwceqos_drain_dma(lp);
         dwceqos_reset_hw(lp);
-        phy_stop(lp->phy_dev);
+        phy_stop(ndev->phydev);
 
         dwceqos_descriptor_free(lp);
 
@@ -2523,30 +2524,6 @@ dwceqos_get_stats64(struct net_device *ndev, struct rtnl_link_stats64 *s)
         return s;
 }
 
-static int
-dwceqos_get_settings(struct net_device *ndev, struct ethtool_cmd *ecmd)
-{
-        struct net_local *lp = netdev_priv(ndev);
-        struct phy_device *phydev = lp->phy_dev;
-
-        if (!phydev)
-                return -ENODEV;
-
-        return phy_ethtool_gset(phydev, ecmd);
-}
-
-static int
-dwceqos_set_settings(struct net_device *ndev, struct ethtool_cmd *ecmd)
-{
-        struct net_local *lp = netdev_priv(ndev);
-        struct phy_device *phydev = lp->phy_dev;
-
-        if (!phydev)
-                return -ENODEV;
-
-        return phy_ethtool_sset(phydev, ecmd);
-}
-
 static void
 dwceqos_get_drvinfo(struct net_device *ndev, struct ethtool_drvinfo *ed)
 {
@@ -2574,17 +2551,17 @@ static int dwceqos_set_pauseparam(struct net_device *ndev,
 
         lp->flowcontrol.autoneg = pp->autoneg;
         if (pp->autoneg) {
-                lp->phy_dev->advertising |= ADVERTISED_Pause;
-                lp->phy_dev->advertising |= ADVERTISED_Asym_Pause;
+                ndev->phydev->advertising |= ADVERTISED_Pause;
+                ndev->phydev->advertising |= ADVERTISED_Asym_Pause;
         } else {
-                lp->phy_dev->advertising &= ~ADVERTISED_Pause;
-                lp->phy_dev->advertising &= ~ADVERTISED_Asym_Pause;
+                ndev->phydev->advertising &= ~ADVERTISED_Pause;
+                ndev->phydev->advertising &= ~ADVERTISED_Asym_Pause;
                 lp->flowcontrol.rx = pp->rx_pause;
                 lp->flowcontrol.tx = pp->tx_pause;
         }
 
         if (netif_running(ndev))
-                ret = phy_start_aneg(lp->phy_dev);
+                ret = phy_start_aneg(ndev->phydev);
 
         return ret;
 }
@@ -2705,7 +2682,7 @@ static int dwceqos_get_eee(struct net_device *ndev, struct ethtool_eee *edata)
                             dwceqos_get_tx_lpi_state(regval));
         }
 
-        return phy_ethtool_get_eee(lp->phy_dev, edata);
+        return phy_ethtool_get_eee(ndev->phydev, edata);
 }
 
 static int dwceqos_set_eee(struct net_device *ndev, struct ethtool_eee *edata)
@@ -2747,7 +2724,7 @@ static int dwceqos_set_eee(struct net_device *ndev, struct ethtool_eee *edata)
                 spin_unlock_irqrestore(&lp->hw_lock, flags);
         }
 
-        return phy_ethtool_set_eee(lp->phy_dev, edata);
+        return phy_ethtool_set_eee(ndev->phydev, edata);
 }
 
 static u32 dwceqos_get_msglevel(struct net_device *ndev)
@@ -2765,8 +2742,6 @@ static void dwceqos_set_msglevel(struct net_device *ndev, u32 msglevel)
 }
 
 static struct ethtool_ops dwceqos_ethtool_ops = {
-        .get_settings   = dwceqos_get_settings,
-        .set_settings   = dwceqos_set_settings,
         .get_drvinfo    = dwceqos_get_drvinfo,
         .get_link       = ethtool_op_get_link,
         .get_pauseparam = dwceqos_get_pauseparam,
@@ -2780,6 +2755,8 @@ static struct ethtool_ops dwceqos_ethtool_ops = {
         .set_eee        = dwceqos_set_eee,
         .get_msglevel   = dwceqos_get_msglevel,
         .set_msglevel   = dwceqos_set_msglevel,
+        .get_link_ksettings = phy_ethtool_get_link_ksettings,
+        .set_link_ksettings = phy_ethtool_set_link_ksettings,
 };
 
 static struct net_device_ops netdev_ops = {
@@ -2981,8 +2958,8 @@ static int dwceqos_remove(struct platform_device *pdev)
         if (ndev) {
                 lp = netdev_priv(ndev);
 
-                if (lp->phy_dev)
-                        phy_disconnect(lp->phy_dev);
+                if (ndev->phydev)
+                        phy_disconnect(ndev->phydev);
                 mdiobus_unregister(lp->mii_bus);
                 mdiobus_free(lp->mii_bus);
 
diff --git a/drivers/net/ethernet/ti/cpsw.c b/drivers/net/ethernet/ti/cpsw.c
index 53190894f17a..1a93a1f28433 100644
--- a/drivers/net/ethernet/ti/cpsw.c
+++ b/drivers/net/ethernet/ti/cpsw.c
@@ -364,7 +364,6 @@ static inline void slave_write(struct cpsw_slave *slave, u32 val, u32 offset)
 }
 
 struct cpsw_priv {
-        spinlock_t                      lock;
         struct platform_device          *pdev;
         struct net_device               *ndev;
         struct napi_struct              napi_rx;
@@ -1244,6 +1243,7 @@ static void cpsw_slave_stop(struct cpsw_slave *slave, struct cpsw_priv *priv)
         slave->phy = NULL;
         cpsw_ale_control_set(priv->ale, slave_port,
                              ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
+        soft_reset_slave(slave);
 }
 
 static int cpsw_ndo_open(struct net_device *ndev)
@@ -1252,7 +1252,11 @@ static int cpsw_ndo_open(struct net_device *ndev)
         int i, ret;
         u32 reg;
 
-        pm_runtime_get_sync(&priv->pdev->dev);
+        ret = pm_runtime_get_sync(&priv->pdev->dev);
+        if (ret < 0) {
+                pm_runtime_put_noidle(&priv->pdev->dev);
+                return ret;
+        }
 
         if (!cpsw_common_res_usage_state(priv))
                 cpsw_intr_disable(priv);
@@ -1278,6 +1282,7 @@ static int cpsw_ndo_open(struct net_device *ndev)
 
         if (!cpsw_common_res_usage_state(priv)) {
                 struct cpsw_priv *priv_sl0 = cpsw_get_slave_priv(priv, 0);
+                int buf_num;
 
                 /* setup tx dma to fixed prio and zero offset */
                 cpdma_control_set(priv->dma, CPDMA_TX_PRIO_FIXED, 1);
@@ -1305,10 +1310,8 @@ static int cpsw_ndo_open(struct net_device *ndev)
                         enable_irq(priv->irqs_table[0]);
                 }
 
-                if (WARN_ON(!priv->data.rx_descs))
-                        priv->data.rx_descs = 128;
-
-                for (i = 0; i < priv->data.rx_descs; i++) {
+                buf_num = cpdma_chan_get_rx_buf_num(priv->dma);
+                for (i = 0; i < buf_num; i++) {
                         struct sk_buff *skb;
 
                         ret = -ENOMEM;
@@ -1611,10 +1614,17 @@ static int cpsw_ndo_set_mac_address(struct net_device *ndev, void *p)
         struct sockaddr *addr = (struct sockaddr *)p;
         int flags = 0;
         u16 vid = 0;
+        int ret;
 
         if (!is_valid_ether_addr(addr->sa_data))
                 return -EADDRNOTAVAIL;
 
+        ret = pm_runtime_get_sync(&priv->pdev->dev);
+        if (ret < 0) {
+                pm_runtime_put_noidle(&priv->pdev->dev);
+                return ret;
+        }
+
         if (priv->data.dual_emac) {
                 vid = priv->slaves[priv->emac_port].port_vlan;
                 flags = ALE_VLAN;
@@ -1629,6 +1639,8 @@ static int cpsw_ndo_set_mac_address(struct net_device *ndev, void *p)
         memcpy(ndev->dev_addr, priv->mac_addr, ETH_ALEN);
         for_each_slave(priv, cpsw_set_slave_mac, priv);
 
+        pm_runtime_put(&priv->pdev->dev);
+
         return 0;
 }
 
@@ -1693,10 +1705,17 @@ static int cpsw_ndo_vlan_rx_add_vid(struct net_device *ndev,
                                     __be16 proto, u16 vid)
 {
         struct cpsw_priv *priv = netdev_priv(ndev);
+        int ret;
 
         if (vid == priv->data.default_vlan)
                 return 0;
 
+        ret = pm_runtime_get_sync(&priv->pdev->dev);
+        if (ret < 0) {
+                pm_runtime_put_noidle(&priv->pdev->dev);
+                return ret;
+        }
+
         if (priv->data.dual_emac) {
                 /* In dual EMAC, reserved VLAN id should not be used for
                  * creating VLAN interfaces as this can break the dual
@@ -1711,7 +1730,10 @@ static int cpsw_ndo_vlan_rx_add_vid(struct net_device *ndev,
         }
 
         dev_info(priv->dev, "Adding vlanid %d to vlan filter\n", vid);
-        return cpsw_add_vlan_ale_entry(priv, vid);
+        ret = cpsw_add_vlan_ale_entry(priv, vid);
+
+        pm_runtime_put(&priv->pdev->dev);
+        return ret;
 }
 
 static int cpsw_ndo_vlan_rx_kill_vid(struct net_device *ndev,
@@ -1723,6 +1745,12 @@ static int cpsw_ndo_vlan_rx_kill_vid(struct net_device *ndev,
         if (vid == priv->data.default_vlan)
                 return 0;
 
+        ret = pm_runtime_get_sync(&priv->pdev->dev);
+        if (ret < 0) {
+                pm_runtime_put_noidle(&priv->pdev->dev);
+                return ret;
+        }
+
         if (priv->data.dual_emac) {
                 int i;
 
@@ -1742,8 +1770,10 @@ static int cpsw_ndo_vlan_rx_kill_vid(struct net_device *ndev,
         if (ret != 0)
                 return ret;
 
-        return cpsw_ale_del_mcast(priv->ale, priv->ndev->broadcast,
-                                  0, ALE_VLAN, vid);
+        ret = cpsw_ale_del_mcast(priv->ale, priv->ndev->broadcast,
+                                 0, ALE_VLAN, vid);
+        pm_runtime_put(&priv->pdev->dev);
+        return ret;
 }
 
 static const struct net_device_ops cpsw_netdev_ops = {
@@ -1902,10 +1932,33 @@ static int cpsw_set_pauseparam(struct net_device *ndev,
         priv->tx_pause = pause->tx_pause ? true : false;
 
         for_each_slave(priv, _cpsw_adjust_link, priv, &link);
-
         return 0;
 }
 
+static int cpsw_ethtool_op_begin(struct net_device *ndev)
+{
+        struct cpsw_priv *priv = netdev_priv(ndev);
+        int ret;
+
+        ret = pm_runtime_get_sync(&priv->pdev->dev);
+        if (ret < 0) {
+                cpsw_err(priv, drv, "ethtool begin failed %d\n", ret);
+                pm_runtime_put_noidle(&priv->pdev->dev);
+        }
+
+        return ret;
+}
+
+static void cpsw_ethtool_op_complete(struct net_device *ndev)
+{
+        struct cpsw_priv *priv = netdev_priv(ndev);
+        int ret;
+
+        ret = pm_runtime_put(&priv->pdev->dev);
+        if (ret < 0)
+                cpsw_err(priv, drv, "ethtool complete failed %d\n", ret);
+}
+
 static const struct ethtool_ops cpsw_ethtool_ops = {
         .get_drvinfo    = cpsw_get_drvinfo,
         .get_msglevel   = cpsw_get_msglevel,
@@ -1925,6 +1978,8 @@ static const struct ethtool_ops cpsw_ethtool_ops = {
         .set_wol        = cpsw_set_wol,
         .get_regs_len   = cpsw_get_regs_len,
         .get_regs       = cpsw_get_regs,
+        .begin          = cpsw_ethtool_op_begin,
+        .complete       = cpsw_ethtool_op_complete,
 };
 
 static void cpsw_slave_init(struct cpsw_slave *slave, struct cpsw_priv *priv,
@@ -1999,12 +2054,6 @@ static int cpsw_probe_dt(struct cpsw_platform_data *data,
         }
         data->bd_ram_size = prop;
 
-        if (of_property_read_u32(node, "rx_descs", &prop)) {
-                dev_err(&pdev->dev, "Missing rx_descs property in the DT.\n");
-                return -EINVAL;
-        }
-        data->rx_descs = prop;
-
         if (of_property_read_u32(node, "mac_control", &prop)) {
                 dev_err(&pdev->dev, "Missing mac_control property in the DT.\n");
                 return -EINVAL;
@@ -2022,7 +2071,7 @@ static int cpsw_probe_dt(struct cpsw_platform_data *data,
         if (ret)
                 dev_warn(&pdev->dev, "Doesn't have any child node\n");
 
-        for_each_child_of_node(node, slave_node) {
+        for_each_available_child_of_node(node, slave_node) {
                 struct cpsw_slave_data *slave_data = data->slave_data + i;
                 const void *mac_addr = NULL;
                 int lenp;
@@ -2124,7 +2173,6 @@ static int cpsw_probe_dual_emac(struct platform_device *pdev,
         }
 
         priv_sl2 = netdev_priv(ndev);
-        spin_lock_init(&priv_sl2->lock);
         priv_sl2->data = *data;
         priv_sl2->pdev = pdev;
         priv_sl2->ndev = ndev;
@@ -2243,7 +2291,6 @@ static int cpsw_probe(struct platform_device *pdev)
 
         platform_set_drvdata(pdev, ndev);
         priv = netdev_priv(ndev);
-        spin_lock_init(&priv->lock);
         priv->pdev = pdev;
         priv->ndev = ndev;
         priv->dev  = &ndev->dev;
@@ -2321,7 +2368,11 @@ static int cpsw_probe(struct platform_device *pdev)
         /* Need to enable clocks with runtime PM api to access module
          * registers
          */
-        pm_runtime_get_sync(&pdev->dev);
+        ret = pm_runtime_get_sync(&pdev->dev);
+        if (ret < 0) {
+                pm_runtime_put_noidle(&pdev->dev);
+                goto clean_runtime_disable_ret;
+        }
         priv->version = readl(&priv->regs->id_ver);
         pm_runtime_put_sync(&pdev->dev);
 
@@ -2554,16 +2605,12 @@ static int cpsw_suspend(struct device *dev)
                 for (i = 0; i < priv->data.slaves; i++) {
                         if (netif_running(priv->slaves[i].ndev))
                                 cpsw_ndo_stop(priv->slaves[i].ndev);
-                        soft_reset_slave(priv->slaves + i);
                 }
         } else {
                 if (netif_running(ndev))
                         cpsw_ndo_stop(ndev);
-                for_each_slave(priv, soft_reset_slave);
         }
 
-        pm_runtime_put_sync(&pdev->dev);
-
         /* Select sleep pin state */
         pinctrl_pm_select_sleep_state(&pdev->dev);
 
@@ -2576,8 +2623,6 @@ static int cpsw_resume(struct device *dev)
         struct net_device       *ndev = platform_get_drvdata(pdev);
         struct cpsw_priv        *priv = netdev_priv(ndev);
 
-        pm_runtime_get_sync(&pdev->dev);
-
         /* Select default pin state */
         pinctrl_pm_select_default_state(&pdev->dev);
 
diff --git a/drivers/net/ethernet/ti/cpsw.h b/drivers/net/ethernet/ti/cpsw.h
index e50afd1b2eda..16b54c6f32c2 100644
--- a/drivers/net/ethernet/ti/cpsw.h
+++ b/drivers/net/ethernet/ti/cpsw.h
@@ -35,7 +35,6 @@ struct cpsw_platform_data {
         u32     cpts_clock_shift; /* convert input clock ticks to nanoseconds */
         u32     ale_entries;    /* ale table size */
         u32     bd_ram_size;  /*buffer descriptor ram size */
-        u32     rx_descs;       /* Number of Rx Descriptios */
         u32     mac_control;    /* Mac control register */
         u16     default_vlan;   /* Def VLAN for ALE lookup in VLAN aware mode*/
         bool    dual_emac;      /* Enable Dual EMAC mode */
diff --git a/drivers/net/ethernet/ti/davinci_cpdma.c b/drivers/net/ethernet/ti/davinci_cpdma.c
index 18bf3a8fdc50..1c653ca7c316 100644
--- a/drivers/net/ethernet/ti/davinci_cpdma.c
+++ b/drivers/net/ethernet/ti/davinci_cpdma.c
@@ -21,7 +21,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/io.h>
 #include <linux/delay.h>
-
+#include <linux/genalloc.h>
 #include "davinci_cpdma.h"
 
 /* DMA Registers */
@@ -87,9 +87,8 @@ struct cpdma_desc_pool {
         void                    *cpumap;        /* dma_alloc map */
         int                     desc_size, mem_size;
         int                     num_desc, used_desc;
-        unsigned long           *bitmap;
         struct device           *dev;
-        spinlock_t              lock;
+        struct gen_pool         *gen_pool;
 };
 
 enum cpdma_state {
@@ -117,6 +116,7 @@ struct cpdma_chan {
         int                             chan_num;
         spinlock_t                      lock;
         int                             count;
+        u32                             desc_num;
         u32                             mask;
         cpdma_handler_fn                handler;
         enum dma_data_direction         dir;
@@ -145,6 +145,19 @@ struct cpdma_chan {
                                  (directed << CPDMA_TO_PORT_SHIFT));    \
         } while (0)
 
+static void cpdma_desc_pool_destroy(struct cpdma_desc_pool *pool)
+{
+        if (!pool)
+                return;
+
+        WARN_ON(pool->used_desc);
+        if (pool->cpumap)
+                dma_free_coherent(pool->dev, pool->mem_size, pool->cpumap,
+                                  pool->phys);
+        else
+                iounmap(pool->iomap);
+}
+
 /*
  * Utility constructs for a cpdma descriptor pool.  Some devices (e.g. davinci
  * emac) have dedicated on-chip memory for these descriptors.  Some other
@@ -155,24 +168,25 @@ static struct cpdma_desc_pool *
 cpdma_desc_pool_create(struct device *dev, u32 phys, dma_addr_t hw_addr,
                                 int size, int align)
 {
-        int bitmap_size;
         struct cpdma_desc_pool *pool;
+        int ret;
 
         pool = devm_kzalloc(dev, sizeof(*pool), GFP_KERNEL);
         if (!pool)
-                goto fail;
-
-        spin_lock_init(&pool->lock);
+                goto gen_pool_create_fail;
 
         pool->dev       = dev;
         pool->mem_size  = size;
         pool->desc_size = ALIGN(sizeof(struct cpdma_desc), align);
         pool->num_desc  = size / pool->desc_size;
 
-        bitmap_size  = (pool->num_desc / BITS_PER_LONG) * sizeof(long);
-        pool->bitmap = devm_kzalloc(dev, bitmap_size, GFP_KERNEL);
-        if (!pool->bitmap)
-                goto fail;
+        pool->gen_pool = devm_gen_pool_create(dev, ilog2(pool->desc_size), -1,
+                                              "cpdma");
+        if (IS_ERR(pool->gen_pool)) {
+                dev_err(dev, "pool create failed %ld\n",
+                        PTR_ERR(pool->gen_pool));
+                goto gen_pool_create_fail;
+        }
 
         if (phys) {
                 pool->phys  = phys;
@@ -185,24 +199,22 @@ cpdma_desc_pool_create(struct device *dev, u32 phys, dma_addr_t hw_addr,
                 pool->phys = pool->hw_addr; /* assumes no IOMMU, don't use this value */
         }
 
-        if (pool->iomap)
-                return pool;
-fail:
-        return NULL;
-}
-
-static void cpdma_desc_pool_destroy(struct cpdma_desc_pool *pool)
-{
-        if (!pool)
-                return;
+        if (!pool->iomap)
+                goto gen_pool_create_fail;
 
-        WARN_ON(pool->used_desc);
-        if (pool->cpumap) {
-                dma_free_coherent(pool->dev, pool->mem_size, pool->cpumap,
-                                  pool->phys);
-        } else {
-                iounmap(pool->iomap);
+        ret = gen_pool_add_virt(pool->gen_pool, (unsigned long)pool->iomap,
+                                pool->phys, pool->mem_size, -1);
+        if (ret < 0) {
+                dev_err(dev, "pool add failed %d\n", ret);
+                goto gen_pool_add_virt_fail;
         }
+
+        return pool;
+
+gen_pool_add_virt_fail:
+        cpdma_desc_pool_destroy(pool);
+gen_pool_create_fail:
+        return NULL;
 }
 
 static inline dma_addr_t desc_phys(struct cpdma_desc_pool *pool,
@@ -220,47 +232,23 @@ desc_from_phys(struct cpdma_desc_pool *pool, dma_addr_t dma)
 }
 
 static struct cpdma_desc __iomem *
-cpdma_desc_alloc(struct cpdma_desc_pool *pool, int num_desc, bool is_rx)
+cpdma_desc_alloc(struct cpdma_desc_pool *pool)
 {
-        unsigned long flags;
-        int index;
-        int desc_start;
-        int desc_end;
         struct cpdma_desc __iomem *desc = NULL;
 
-        spin_lock_irqsave(&pool->lock, flags);
-
-        if (is_rx) {
-                desc_start = 0;
-                desc_end = pool->num_desc/2;
-         } else {
-                desc_start = pool->num_desc/2;
-                desc_end = pool->num_desc;
-        }
-
-        index = bitmap_find_next_zero_area(pool->bitmap,
-                                desc_end, desc_start, num_desc, 0);
-        if (index < desc_end) {
-                bitmap_set(pool->bitmap, index, num_desc);
-                desc = pool->iomap + pool->desc_size * index;
+        desc = (struct cpdma_desc __iomem *)gen_pool_alloc(pool->gen_pool,
+                                                           pool->desc_size);
+        if (desc)
                 pool->used_desc++;
-        }
 
-        spin_unlock_irqrestore(&pool->lock, flags);
         return desc;
 }
 
 static void cpdma_desc_free(struct cpdma_desc_pool *pool,
                             struct cpdma_desc __iomem *desc, int num_desc)
 {
-        unsigned long flags, index;
-
-        index = ((unsigned long)desc - (unsigned long)pool->iomap) /
-                pool->desc_size;
-        spin_lock_irqsave(&pool->lock, flags);
-        bitmap_clear(pool->bitmap, index, num_desc);
+        gen_pool_free(pool->gen_pool, (unsigned long)desc, pool->desc_size);
         pool->used_desc--;
-        spin_unlock_irqrestore(&pool->lock, flags);
 }
 
 struct cpdma_ctlr *cpdma_ctlr_create(struct cpdma_params *params)
@@ -516,6 +504,7 @@ struct cpdma_chan *cpdma_chan_create(struct cpdma_ctlr *ctlr, int chan_num,
         chan->state     = CPDMA_STATE_IDLE;
         chan->chan_num  = chan_num;
         chan->handler   = handler;
+        chan->desc_num = ctlr->pool->num_desc / 2;
 
         if (is_rx_chan(chan)) {
                 chan->hdp       = ctlr->params.rxhdp + offset;
@@ -543,6 +532,12 @@ struct cpdma_chan *cpdma_chan_create(struct cpdma_ctlr *ctlr, int chan_num,
 }
 EXPORT_SYMBOL_GPL(cpdma_chan_create);
 
+int cpdma_chan_get_rx_buf_num(struct cpdma_ctlr *ctlr)
+{
+        return ctlr->pool->num_desc / 2;
+}
+EXPORT_SYMBOL_GPL(cpdma_chan_get_rx_buf_num);
+
 int cpdma_chan_destroy(struct cpdma_chan *chan)
 {
         struct cpdma_ctlr *ctlr;
@@ -675,7 +670,13 @@ int cpdma_chan_submit(struct cpdma_chan *chan, void *token, void *data,
                 goto unlock_ret;
         }
 
-        desc = cpdma_desc_alloc(ctlr->pool, 1, is_rx_chan(chan));
+        if (chan->count >= chan->desc_num)      {
+                chan->stats.desc_alloc_fail++;
+                ret = -ENOMEM;
+                goto unlock_ret;
+        }
+
+        desc = cpdma_desc_alloc(ctlr->pool);
         if (!desc) {
                 chan->stats.desc_alloc_fail++;
                 ret = -ENOMEM;
@@ -721,24 +722,16 @@ EXPORT_SYMBOL_GPL(cpdma_chan_submit);
 
 bool cpdma_check_free_tx_desc(struct cpdma_chan *chan)
 {
-        unsigned long flags;
-        int index;
-        bool ret;
         struct cpdma_ctlr       *ctlr = chan->ctlr;
         struct cpdma_desc_pool  *pool = ctlr->pool;
+        bool                    free_tx_desc;
+        unsigned long           flags;
 
-        spin_lock_irqsave(&pool->lock, flags);
-
-        index = bitmap_find_next_zero_area(pool->bitmap,
-                                pool->num_desc, pool->num_desc/2, 1, 0);
-
-        if (index < pool->num_desc)
-                ret = true;
-        else
-                ret = false;
-
-        spin_unlock_irqrestore(&pool->lock, flags);
-        return ret;
+        spin_lock_irqsave(&chan->lock, flags);
+        free_tx_desc = (chan->count < chan->desc_num) &&
+                         gen_pool_avail(pool->gen_pool);
+        spin_unlock_irqrestore(&chan->lock, flags);
+        return free_tx_desc;
 }
 EXPORT_SYMBOL_GPL(cpdma_check_free_tx_desc);
 
diff --git a/drivers/net/ethernet/ti/davinci_cpdma.h b/drivers/net/ethernet/ti/davinci_cpdma.h
index 86dee487f2f0..80c015cbbce5 100644
--- a/drivers/net/ethernet/ti/davinci_cpdma.h
+++ b/drivers/net/ethernet/ti/davinci_cpdma.h
@@ -81,6 +81,7 @@ int cpdma_ctlr_dump(struct cpdma_ctlr *ctlr);
 
 struct cpdma_chan *cpdma_chan_create(struct cpdma_ctlr *ctlr, int chan_num,
                                      cpdma_handler_fn handler);
+int cpdma_chan_get_rx_buf_num(struct cpdma_ctlr *ctlr);
 int cpdma_chan_destroy(struct cpdma_chan *chan);
 int cpdma_chan_start(struct cpdma_chan *chan);
 int cpdma_chan_stop(struct cpdma_chan *chan);
diff --git a/drivers/net/ethernet/ti/davinci_mdio.c b/drivers/net/ethernet/ti/davinci_mdio.c
index 4e7c9b9b042a..33df340db1f1 100644
--- a/drivers/net/ethernet/ti/davinci_mdio.c
+++ b/drivers/net/ethernet/ti/davinci_mdio.c
@@ -53,6 +53,10 @@
 
 #define DEF_OUT_FREQ            2200000         /* 2.2 MHz */
 
+struct davinci_mdio_of_param {
+        int autosuspend_delay_ms;
+};
+
 struct davinci_mdio_regs {
         u32     version;
         u32     control;
@@ -90,19 +94,19 @@ static const struct mdio_platform_data default_pdata = {
 struct davinci_mdio_data {
         struct mdio_platform_data pdata;
         struct davinci_mdio_regs __iomem *regs;
-        spinlock_t      lock;
         struct clk      *clk;
         struct device   *dev;
         struct mii_bus  *bus;
-        bool            suspended;
+        bool            active_in_suspend;
         unsigned long   access_time; /* jiffies */
         /* Indicates that driver shouldn't modify phy_mask in case
          * if MDIO bus is registered from DT.
          */
         bool            skip_scan;
+        u32             clk_div;
 };
 
-static void __davinci_mdio_reset(struct davinci_mdio_data *data)
+static void davinci_mdio_init_clk(struct davinci_mdio_data *data)
 {
         u32 mdio_in, div, mdio_out_khz, access_time;
 
@@ -111,9 +115,7 @@ static void __davinci_mdio_reset(struct davinci_mdio_data *data)
         if (div > CONTROL_MAX_DIV)
                 div = CONTROL_MAX_DIV;
 
-        /* set enable and clock divider */
-        __raw_writel(div | CONTROL_ENABLE, &data->regs->control);
-
+        data->clk_div = div;
         /*
          * One mdio transaction consists of:
          *      32 bits of preamble
@@ -134,12 +136,23 @@ static void __davinci_mdio_reset(struct davinci_mdio_data *data)
                 data->access_time = 1;
 }
 
+static void davinci_mdio_enable(struct davinci_mdio_data *data)
+{
+        /* set enable and clock divider */
+        __raw_writel(data->clk_div | CONTROL_ENABLE, &data->regs->control);
+}
+
 static int davinci_mdio_reset(struct mii_bus *bus)
 {
         struct davinci_mdio_data *data = bus->priv;
         u32 phy_mask, ver;
+        int ret;
 
-        __davinci_mdio_reset(data);
+        ret = pm_runtime_get_sync(data->dev);
+        if (ret < 0) {
+                pm_runtime_put_noidle(data->dev);
+                return ret;
+        }
 
         /* wait for scan logic to settle */
         msleep(PHY_MAX_ADDR * data->access_time);
@@ -150,7 +163,7 @@ static int davinci_mdio_reset(struct mii_bus *bus)
                  (ver >> 8) & 0xff, ver & 0xff);
 
         if (data->skip_scan)
-                return 0;
+                goto done;
 
         /* get phy mask from the alive register */
         phy_mask = __raw_readl(&data->regs->alive);
@@ -165,6 +178,10 @@ static int davinci_mdio_reset(struct mii_bus *bus)
         }
         data->bus->phy_mask = phy_mask;
 
+done:
+        pm_runtime_mark_last_busy(data->dev);
+        pm_runtime_put_autosuspend(data->dev);
+
         return 0;
 }
 
@@ -190,7 +207,7 @@ static inline int wait_for_user_access(struct davinci_mdio_data *data)
                  * operation
                  */
                 dev_warn(data->dev, "resetting idled controller\n");
-                __davinci_mdio_reset(data);
+                davinci_mdio_enable(data);
                 return -EAGAIN;
         }
 
@@ -225,11 +242,10 @@ static int davinci_mdio_read(struct mii_bus *bus, int phy_id, int phy_reg)
         if (phy_reg & ~PHY_REG_MASK || phy_id & ~PHY_ID_MASK)
                 return -EINVAL;
 
-        spin_lock(&data->lock);
-
-        if (data->suspended) {
-                spin_unlock(&data->lock);
-                return -ENODEV;
+        ret = pm_runtime_get_sync(data->dev);
+        if (ret < 0) {
+                pm_runtime_put_noidle(data->dev);
+                return ret;
         }
 
         reg = (USERACCESS_GO | USERACCESS_READ | (phy_reg << 21) |
@@ -255,8 +271,8 @@ static int davinci_mdio_read(struct mii_bus *bus, int phy_id, int phy_reg)
                 break;
         }
 
-        spin_unlock(&data->lock);
-
+        pm_runtime_mark_last_busy(data->dev);
+        pm_runtime_put_autosuspend(data->dev);
         return ret;
 }
 
@@ -270,11 +286,10 @@ static int davinci_mdio_write(struct mii_bus *bus, int phy_id,
         if (phy_reg & ~PHY_REG_MASK || phy_id & ~PHY_ID_MASK)
                 return -EINVAL;
 
-        spin_lock(&data->lock);
-
-        if (data->suspended) {
-                spin_unlock(&data->lock);
-                return -ENODEV;
+        ret = pm_runtime_get_sync(data->dev);
+        if (ret < 0) {
+                pm_runtime_put_noidle(data->dev);
+                return ret;
         }
 
         reg = (USERACCESS_GO | USERACCESS_WRITE | (phy_reg << 21) |
@@ -295,9 +310,10 @@ static int davinci_mdio_write(struct mii_bus *bus, int phy_id,
                 break;
         }
 
-        spin_unlock(&data->lock);
+        pm_runtime_mark_last_busy(data->dev);
+        pm_runtime_put_autosuspend(data->dev);
 
-        return 0;
+        return ret;
 }
 
 #if IS_ENABLED(CONFIG_OF)
@@ -320,6 +336,19 @@ static int davinci_mdio_probe_dt(struct mdio_platform_data *data,
 }
 #endif
 
+#if IS_ENABLED(CONFIG_OF)
+static const struct davinci_mdio_of_param of_cpsw_mdio_data = {
+        .autosuspend_delay_ms = 100,
+};
+
+static const struct of_device_id davinci_mdio_of_mtable[] = {
+        { .compatible = "ti,davinci_mdio", },
+        { .compatible = "ti,cpsw-mdio", .data = &of_cpsw_mdio_data},
+        { /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, davinci_mdio_of_mtable);
+#endif
+
 static int davinci_mdio_probe(struct platform_device *pdev)
 {
         struct mdio_platform_data *pdata = dev_get_platdata(&pdev->dev);
@@ -328,6 +357,7 @@ static int davinci_mdio_probe(struct platform_device *pdev)
         struct resource *res;
         struct phy_device *phy;
         int ret, addr;
+        int autosuspend_delay_ms = -1;
 
         data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
         if (!data)
@@ -340,9 +370,22 @@ static int davinci_mdio_probe(struct platform_device *pdev)
         }
 
         if (dev->of_node) {
-                if (davinci_mdio_probe_dt(&data->pdata, pdev))
-                        data->pdata = default_pdata;
+                const struct of_device_id       *of_id;
+
+                ret = davinci_mdio_probe_dt(&data->pdata, pdev);
+                if (ret)
+                        return ret;
                 snprintf(data->bus->id, MII_BUS_ID_SIZE, "%s", pdev->name);
+
+                of_id = of_match_device(davinci_mdio_of_mtable, &pdev->dev);
+                if (of_id) {
+                        const struct davinci_mdio_of_param *of_mdio_data;
+
+                        of_mdio_data = of_id->data;
+                        if (of_mdio_data)
+                                autosuspend_delay_ms =
+                                        of_mdio_data->autosuspend_delay_ms;
+                }
         } else {
                 data->pdata = pdata ? (*pdata) : default_pdata;
                 snprintf(data->bus->id, MII_BUS_ID_SIZE, "%s-%x",
@@ -356,26 +399,25 @@ static int davinci_mdio_probe(struct platform_device *pdev)
         data->bus->parent       = dev;
         data->bus->priv         = data;
 
-        pm_runtime_enable(&pdev->dev);
-        pm_runtime_get_sync(&pdev->dev);
         data->clk = devm_clk_get(dev, "fck");
         if (IS_ERR(data->clk)) {
                 dev_err(dev, "failed to get device clock\n");
-                ret = PTR_ERR(data->clk);
-                data->clk = NULL;
-                goto bail_out;
+                return PTR_ERR(data->clk);
         }
 
         dev_set_drvdata(dev, data);
         data->dev = dev;
-        spin_lock_init(&data->lock);
 
         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
         data->regs = devm_ioremap_resource(dev, res);
-        if (IS_ERR(data->regs)) {
-                ret = PTR_ERR(data->regs);
-                goto bail_out;
-        }
+        if (IS_ERR(data->regs))
+                return PTR_ERR(data->regs);
+
+        davinci_mdio_init_clk(data);
+
+        pm_runtime_set_autosuspend_delay(&pdev->dev, autosuspend_delay_ms);
+        pm_runtime_use_autosuspend(&pdev->dev);
+        pm_runtime_enable(&pdev->dev);
 
         /* register the mii bus
          * Create PHYs from DT only in case if PHY child nodes are explicitly
@@ -404,9 +446,8 @@ static int davinci_mdio_probe(struct platform_device *pdev)
         return 0;
 
 bail_out:
-        pm_runtime_put_sync(&pdev->dev);
+        pm_runtime_dont_use_autosuspend(&pdev->dev);
         pm_runtime_disable(&pdev->dev);
-
         return ret;
 }
 
@@ -417,29 +458,47 @@ static int davinci_mdio_remove(struct platform_device *pdev)
         if (data->bus)
                 mdiobus_unregister(data->bus);
 
-        pm_runtime_put_sync(&pdev->dev);
+        pm_runtime_dont_use_autosuspend(&pdev->dev);
         pm_runtime_disable(&pdev->dev);
 
         return 0;
 }
 
-#ifdef CONFIG_PM_SLEEP
-static int davinci_mdio_suspend(struct device *dev)
+#ifdef CONFIG_PM
+static int davinci_mdio_runtime_suspend(struct device *dev)
 {
         struct davinci_mdio_data *data = dev_get_drvdata(dev);
         u32 ctrl;
 
-        spin_lock(&data->lock);
-
         /* shutdown the scan state machine */
         ctrl = __raw_readl(&data->regs->control);
         ctrl &= ~CONTROL_ENABLE;
         __raw_writel(ctrl, &data->regs->control);
         wait_for_idle(data);
 
-        data->suspended = true;
-        spin_unlock(&data->lock);
-        pm_runtime_put_sync(data->dev);
+        return 0;
+}
+
+static int davinci_mdio_runtime_resume(struct device *dev)
+{
+        struct davinci_mdio_data *data = dev_get_drvdata(dev);
+
+        davinci_mdio_enable(data);
+        return 0;
+}
+#endif
+
+#ifdef CONFIG_PM_SLEEP
+static int davinci_mdio_suspend(struct device *dev)
+{
+        struct davinci_mdio_data *data = dev_get_drvdata(dev);
+        int ret = 0;
+
+        data->active_in_suspend = !pm_runtime_status_suspended(dev);
+        if (data->active_in_suspend)
+                ret = pm_runtime_force_suspend(dev);
+        if (ret < 0)
+                return ret;
 
         /* Select sleep pin state */
         pinctrl_pm_select_sleep_state(dev);
@@ -454,31 +513,19 @@ static int davinci_mdio_resume(struct device *dev)
         /* Select default pin state */
         pinctrl_pm_select_default_state(dev);
 
-        pm_runtime_get_sync(data->dev);
-
-        spin_lock(&data->lock);
-        /* restart the scan state machine */
-        __davinci_mdio_reset(data);
-
-        data->suspended = false;
-        spin_unlock(&data->lock);
+        if (data->active_in_suspend)
+                pm_runtime_force_resume(dev);
 
         return 0;
 }
 #endif
 
 static const struct dev_pm_ops davinci_mdio_pm_ops = {
+        SET_RUNTIME_PM_OPS(davinci_mdio_runtime_suspend,
+                           davinci_mdio_runtime_resume, NULL)
         SET_LATE_SYSTEM_SLEEP_PM_OPS(davinci_mdio_suspend, davinci_mdio_resume)
 };
 
-#if IS_ENABLED(CONFIG_OF)
-static const struct of_device_id davinci_mdio_of_mtable[] = {
-        { .compatible = "ti,davinci_mdio", },
-        { /* sentinel */ },
-};
-MODULE_DEVICE_TABLE(of, davinci_mdio_of_mtable);
-#endif
-
 static struct platform_driver davinci_mdio_driver = {
         .driver = {
                 .name    = "davinci_mdio",
diff --git a/drivers/net/ethernet/ti/tlan.c b/drivers/net/ethernet/ti/tlan.c
index 561703317312..ece0ea0f6b38 100644
--- a/drivers/net/ethernet/ti/tlan.c
+++ b/drivers/net/ethernet/ti/tlan.c
@@ -1651,7 +1651,6 @@ static u32 tlan_handle_tx_eoc(struct net_device *dev, u16 host_int)
         dma_addr_t              head_list_phys;
         u32                     ack = 1;
 
-        host_int = 0;
         if (priv->tlan_rev < 0x30) {
                 TLAN_DBG(TLAN_DEBUG_TX,
                          "TRANSMIT:  handling TX EOC (Head=%d Tail=%d) -- IRQ\n",
diff --git a/drivers/net/ethernet/toshiba/tc35815.c b/drivers/net/ethernet/toshiba/tc35815.c
index 54874783476a..74e671906ddb 100644
--- a/drivers/net/ethernet/toshiba/tc35815.c
+++ b/drivers/net/ethernet/toshiba/tc35815.c
@@ -280,7 +280,7 @@ struct tc35815_regs {
  * Descriptors
  */
 
-/* Frame descripter */
+/* Frame descriptor */
 struct FDesc {
         volatile __u32 FDNext;
         volatile __u32 FDSystem;
@@ -288,7 +288,7 @@ struct FDesc {
         volatile __u32 FDCtl;
 };
 
-/* Buffer descripter */
+/* Buffer descriptor */
 struct BDesc {
         volatile __u32 BuffData;
         volatile __u32 BDCtl;
@@ -296,7 +296,7 @@ struct BDesc {
 
 #define FD_ALIGN        16
 
-/* Frame Descripter bit assign ---------------------------------------------- */
+/* Frame Descriptor bit assign ---------------------------------------------- */
 #define FD_FDLength_MASK       0x0000FFFF /* Length MASK                     */
 #define FD_BDCnt_MASK          0x001F0000 /* BD count MASK in FD             */
 #define FD_FrmOpt_MASK         0x7C000000 /* Frame option MASK               */
@@ -309,7 +309,7 @@ struct BDesc {
 #define FD_Next_EOL            0x00000001 /* FD EOL indicator                */
 #define FD_BDCnt_SHIFT         16
 
-/* Buffer Descripter bit assign --------------------------------------------- */
+/* Buffer Descriptor bit assign --------------------------------------------- */
 #define BD_BuffLength_MASK     0x0000FFFF /* Receive Data Size               */
 #define BD_RxBDID_MASK         0x00FF0000 /* BD ID Number MASK               */
 #define BD_RxBDSeqN_MASK       0x7F000000 /* Rx BD Sequence Number           */
@@ -1387,7 +1387,7 @@ static int tc35815_do_interrupt(struct net_device *dev, u32 status, int limit)
         if (status & Int_IntExBD) {
                 if (netif_msg_rx_err(lp))
                         dev_warn(&dev->dev,
-                                 "Excessive Buffer Descriptiors (%#x).\n",
+                                 "Excessive Buffer Descriptors (%#x).\n",
                                  status);
                 dev->stats.rx_length_errors++;
                 ret = 0;
diff --git a/drivers/net/ethernet/wiznet/w5100.c b/drivers/net/ethernet/wiznet/w5100.c
index 4f6255cf62ce..37ab46cdbec4 100644
--- a/drivers/net/ethernet/wiznet/w5100.c
+++ b/drivers/net/ethernet/wiznet/w5100.c
@@ -1154,7 +1154,7 @@ int w5100_probe(struct device *dev, const struct w5100_ops *ops,
         if (err < 0)
                 goto err_register;
 
-        priv->xfer_wq = create_workqueue(netdev_name(ndev));
+        priv->xfer_wq = alloc_workqueue(netdev_name(ndev), WQ_MEM_RECLAIM, 0);
         if (!priv->xfer_wq) {
                 err = -ENOMEM;
                 goto err_wq;
@@ -1233,7 +1233,6 @@ int w5100_remove(struct device *dev)
 
         flush_work(&priv->setrx_work);
         flush_work(&priv->restart_work);
-        flush_workqueue(priv->xfer_wq);
         destroy_workqueue(priv->xfer_wq);
 
         unregister_netdev(ndev);
diff --git a/drivers/net/ethernet/xircom/xirc2ps_cs.c b/drivers/net/ethernet/xircom/xirc2ps_cs.c
index 7b44968e02e6..ddced28e8247 100644
--- a/drivers/net/ethernet/xircom/xirc2ps_cs.c
+++ b/drivers/net/ethernet/xircom/xirc2ps_cs.c
@@ -1144,8 +1144,8 @@ xirc2ps_interrupt(int irq, void *dev_id)
             dev->stats.tx_packets += lp->last_ptr_value - n;
         netif_wake_queue(dev);
     }
-    if (tx_status & 0x0002) {   /* Execessive collissions */
-        pr_debug("tx restarted due to execssive collissions\n");
+    if (tx_status & 0x0002) {   /* Excessive collisions */
+        pr_debug("tx restarted due to excessive collisions\n");
         PutByte(XIRCREG_CR, RestartTx);  /* restart transmitter process */
     }
     if (tx_status & 0x0040)
diff --git a/drivers/net/fjes/fjes_main.c b/drivers/net/fjes/fjes_main.c
index 86c331bb5eb3..9006877c53f2 100644
--- a/drivers/net/fjes/fjes_main.c
+++ b/drivers/net/fjes/fjes_main.c
@@ -1187,8 +1187,9 @@ static int fjes_probe(struct platform_device *plat_dev)
         adapter->force_reset = false;
         adapter->open_guard = false;
 
-        adapter->txrx_wq = create_workqueue(DRV_NAME "/txrx");
-        adapter->control_wq = create_workqueue(DRV_NAME "/control");
+        adapter->txrx_wq = alloc_workqueue(DRV_NAME "/txrx", WQ_MEM_RECLAIM, 0);
+        adapter->control_wq = alloc_workqueue(DRV_NAME "/control",
+                                              WQ_MEM_RECLAIM, 0);
 
         INIT_WORK(&adapter->tx_stall_task, fjes_tx_stall_task);
         INIT_WORK(&adapter->raise_intr_rxdata_task,
diff --git a/drivers/net/geneve.c b/drivers/net/geneve.c
index cc39cefeae45..310e0b9c2657 100644
--- a/drivers/net/geneve.c
+++ b/drivers/net/geneve.c
@@ -12,7 +12,6 @@
 
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/netdevice.h>
 #include <linux/etherdevice.h>
 #include <linux/hash.h>
 #include <net/dst_metadata.h>
@@ -397,23 +396,6 @@ static struct socket *geneve_create_sock(struct net *net, bool ipv6,
         return sock;
 }
 
-static void geneve_notify_add_rx_port(struct geneve_sock *gs)
-{
-        struct net_device *dev;
-        struct sock *sk = gs->sock->sk;
-        struct net *net = sock_net(sk);
-        sa_family_t sa_family = geneve_get_sk_family(gs);
-        __be16 port = inet_sk(sk)->inet_sport;
-
-        rcu_read_lock();
-        for_each_netdev_rcu(net, dev) {
-                if (dev->netdev_ops->ndo_add_geneve_port)
-                        dev->netdev_ops->ndo_add_geneve_port(dev, sa_family,
-                                                             port);
-        }
-        rcu_read_unlock();
-}
-
 static int geneve_hlen(struct genevehdr *gh)
 {
         return sizeof(*gh) + gh->opt_len * 4;
@@ -533,7 +515,7 @@ static struct geneve_sock *geneve_socket_create(struct net *net, __be16 port,
                 INIT_HLIST_HEAD(&gs->vni_list[h]);
 
         /* Initialize the geneve udp offloads structure */
-        geneve_notify_add_rx_port(gs);
+        udp_tunnel_notify_add_rx_port(gs->sock, UDP_TUNNEL_TYPE_GENEVE);
 
         /* Mark socket as an encapsulation socket */
         memset(&tunnel_cfg, 0, sizeof(tunnel_cfg));
@@ -548,31 +530,13 @@ static struct geneve_sock *geneve_socket_create(struct net *net, __be16 port,
         return gs;
 }
 
-static void geneve_notify_del_rx_port(struct geneve_sock *gs)
-{
-        struct net_device *dev;
-        struct sock *sk = gs->sock->sk;
-        struct net *net = sock_net(sk);
-        sa_family_t sa_family = geneve_get_sk_family(gs);
-        __be16 port = inet_sk(sk)->inet_sport;
-
-        rcu_read_lock();
-        for_each_netdev_rcu(net, dev) {
-                if (dev->netdev_ops->ndo_del_geneve_port)
-                        dev->netdev_ops->ndo_del_geneve_port(dev, sa_family,
-                                                             port);
-        }
-
-        rcu_read_unlock();
-}
-
 static void __geneve_sock_release(struct geneve_sock *gs)
 {
         if (!gs || --gs->refcnt)
                 return;
 
         list_del(&gs->list);
-        geneve_notify_del_rx_port(gs);
+        udp_tunnel_notify_del_rx_port(gs->sock, UDP_TUNNEL_TYPE_GENEVE);
         udp_tunnel_sock_release(gs->sock);
         kfree_rcu(gs, rcu);
 }
@@ -1165,29 +1129,20 @@ static struct device_type geneve_type = {
         .name = "geneve",
 };
 
-/* Calls the ndo_add_geneve_port of the caller in order to
+/* Calls the ndo_add_udp_enc_port of the caller in order to
  * supply the listening GENEVE udp ports. Callers are expected
- * to implement the ndo_add_geneve_port.
+ * to implement the ndo_add_udp_enc_port.
  */
 static void geneve_push_rx_ports(struct net_device *dev)
 {
         struct net *net = dev_net(dev);
         struct geneve_net *gn = net_generic(net, geneve_net_id);
         struct geneve_sock *gs;
-        sa_family_t sa_family;
-        struct sock *sk;
-        __be16 port;
-
-        if (!dev->netdev_ops->ndo_add_geneve_port)
-                return;
 
         rcu_read_lock();
-        list_for_each_entry_rcu(gs, &gn->sock_list, list) {
-                sk = gs->sock->sk;
-                sa_family = sk->sk_family;
-                port = inet_sk(sk)->inet_sport;
-                dev->netdev_ops->ndo_add_geneve_port(dev, sa_family, port);
-        }
+        list_for_each_entry_rcu(gs, &gn->sock_list, list)
+                udp_tunnel_push_rx_port(dev, gs->sock,
+                                        UDP_TUNNEL_TYPE_GENEVE);
         rcu_read_unlock();
 }
 
@@ -1550,7 +1505,7 @@ static int geneve_netdevice_event(struct notifier_block *unused,
 {
         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
 
-        if (event == NETDEV_OFFLOAD_PUSH_GENEVE)
+        if (event == NETDEV_UDP_TUNNEL_PUSH_INFO)
                 geneve_push_rx_ports(dev);
 
         return NOTIFY_DONE;
diff --git a/drivers/net/gtp.c b/drivers/net/gtp.c
index 4e976a0d5a76..97e0cbca0a08 100644
--- a/drivers/net/gtp.c
+++ b/drivers/net/gtp.c
@@ -16,7 +16,6 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include <linux/module.h>
-#include <linux/version.h>
 #include <linux/skbuff.h>
 #include <linux/udp.h>
 #include <linux/rculist.h>
diff --git a/drivers/net/hamradio/baycom_par.c b/drivers/net/hamradio/baycom_par.c
index acb636963e90..072cddce9264 100644
--- a/drivers/net/hamradio/baycom_par.c
+++ b/drivers/net/hamradio/baycom_par.c
@@ -156,7 +156,7 @@ struct baycom_state {
 
 /* --------------------------------------------------------------------- */
 
-static void __inline__ baycom_int_freq(struct baycom_state *bc)
+static inline void baycom_int_freq(struct baycom_state *bc)
 {
 #ifdef BAYCOM_DEBUG
         unsigned long cur_jiffies = jiffies;
@@ -192,7 +192,7 @@ static void __inline__ baycom_int_freq(struct baycom_state *bc)
 
 /* --------------------------------------------------------------------- */
 
-static __inline__ void par96_tx(struct net_device *dev, struct baycom_state *bc)
+static inline void par96_tx(struct net_device *dev, struct baycom_state *bc)
 {
         int i;
         unsigned int data = hdlcdrv_getbits(&bc->hdrv);
@@ -216,7 +216,7 @@ static __inline__ void par96_tx(struct net_device *dev, struct baycom_state *bc)
 
 /* --------------------------------------------------------------------- */
 
-static __inline__ void par96_rx(struct net_device *dev, struct baycom_state *bc)
+static inline void par96_rx(struct net_device *dev, struct baycom_state *bc)
 {
         int i;
         unsigned int data, mask, mask2, descx;
diff --git a/drivers/net/hyperv/hyperv_net.h b/drivers/net/hyperv/hyperv_net.h
index c270c5a54f3a..467fb8b4d083 100644
--- a/drivers/net/hyperv/hyperv_net.h
+++ b/drivers/net/hyperv/hyperv_net.h
@@ -173,6 +173,7 @@ struct rndis_device {
 
 /* Interface */
 struct rndis_message;
+struct netvsc_device;
 int netvsc_device_add(struct hv_device *device, void *additional_info);
 int netvsc_device_remove(struct hv_device *device);
 int netvsc_send(struct hv_device *device,
@@ -189,8 +190,8 @@ int netvsc_recv_callback(struct hv_device *device_obj,
                         struct vmbus_channel *channel,
                         u16 vlan_tci);
 void netvsc_channel_cb(void *context);
-int rndis_filter_open(struct hv_device *dev);
-int rndis_filter_close(struct hv_device *dev);
+int rndis_filter_open(struct netvsc_device *nvdev);
+int rndis_filter_close(struct netvsc_device *nvdev);
 int rndis_filter_device_add(struct hv_device *dev,
                         void *additional_info);
 void rndis_filter_device_remove(struct hv_device *dev);
@@ -200,7 +201,7 @@ int rndis_filter_receive(struct hv_device *dev,
                         struct vmbus_channel *channel);
 
 int rndis_filter_set_packet_filter(struct rndis_device *dev, u32 new_filter);
-int rndis_filter_set_device_mac(struct hv_device *hdev, char *mac);
+int rndis_filter_set_device_mac(struct net_device *ndev, char *mac);
 
 void netvsc_switch_datapath(struct net_device *nv_dev, bool vf);
 
@@ -743,6 +744,18 @@ struct netvsc_device {
         atomic_t vf_use_cnt;
 };
 
+static inline struct netvsc_device *
+net_device_to_netvsc_device(struct net_device *ndev)
+{
+        return ((struct net_device_context *)netdev_priv(ndev))->nvdev;
+}
+
+static inline struct netvsc_device *
+hv_device_to_netvsc_device(struct hv_device *device)
+{
+        return net_device_to_netvsc_device(hv_get_drvdata(device));
+}
+
 /* NdisInitialize message */
 struct rndis_initialize_request {
         u32 req_id;
diff --git a/drivers/net/hyperv/netvsc.c b/drivers/net/hyperv/netvsc.c
index 719cb3578e55..6909c322de4e 100644
--- a/drivers/net/hyperv/netvsc.c
+++ b/drivers/net/hyperv/netvsc.c
@@ -95,9 +95,7 @@ static void free_netvsc_device(struct netvsc_device *nvdev)
 
 static struct netvsc_device *get_outbound_net_device(struct hv_device *device)
 {
-        struct net_device *ndev = hv_get_drvdata(device);
-        struct net_device_context *net_device_ctx = netdev_priv(ndev);
-        struct netvsc_device *net_device = net_device_ctx->nvdev;
+        struct netvsc_device *net_device = hv_device_to_netvsc_device(device);
 
         if (net_device && net_device->destroy)
                 net_device = NULL;
@@ -107,9 +105,7 @@ static struct netvsc_device *get_outbound_net_device(struct hv_device *device)
 
 static struct netvsc_device *get_inbound_net_device(struct hv_device *device)
 {
-        struct net_device *ndev = hv_get_drvdata(device);
-        struct net_device_context *net_device_ctx = netdev_priv(ndev);
-        struct netvsc_device *net_device = net_device_ctx->nvdev;
+        struct netvsc_device *net_device = hv_device_to_netvsc_device(device);
 
         if (!net_device)
                 goto get_in_err;
@@ -128,8 +124,7 @@ static int netvsc_destroy_buf(struct hv_device *device)
         struct nvsp_message *revoke_packet;
         int ret = 0;
         struct net_device *ndev = hv_get_drvdata(device);
-        struct net_device_context *net_device_ctx = netdev_priv(ndev);
-        struct netvsc_device *net_device = net_device_ctx->nvdev;
+        struct netvsc_device *net_device = net_device_to_netvsc_device(ndev);
 
         /*
          * If we got a section count, it means we received a
@@ -249,7 +244,6 @@ static int netvsc_destroy_buf(struct hv_device *device)
 static int netvsc_init_buf(struct hv_device *device)
 {
         int ret = 0;
-        unsigned long t;
         struct netvsc_device *net_device;
         struct nvsp_message *init_packet;
         struct net_device *ndev;
@@ -310,9 +304,7 @@ static int netvsc_init_buf(struct hv_device *device)
                 goto cleanup;
         }
 
-        t = wait_for_completion_timeout(&net_device->channel_init_wait, 5*HZ);
-        BUG_ON(t == 0);
-
+        wait_for_completion(&net_device->channel_init_wait);
 
         /* Check the response */
         if (init_packet->msg.v1_msg.
@@ -395,8 +387,7 @@ static int netvsc_init_buf(struct hv_device *device)
                 goto cleanup;
         }
 
-        t = wait_for_completion_timeout(&net_device->channel_init_wait, 5*HZ);
-        BUG_ON(t == 0);
+        wait_for_completion(&net_device->channel_init_wait);
 
         /* Check the response */
         if (init_packet->msg.v1_msg.
@@ -450,7 +441,6 @@ static int negotiate_nvsp_ver(struct hv_device *device,
 {
         struct net_device *ndev = hv_get_drvdata(device);
         int ret;
-        unsigned long t;
 
         memset(init_packet, 0, sizeof(struct nvsp_message));
         init_packet->hdr.msg_type = NVSP_MSG_TYPE_INIT;
@@ -467,10 +457,7 @@ static int negotiate_nvsp_ver(struct hv_device *device,
         if (ret != 0)
                 return ret;
 
-        t = wait_for_completion_timeout(&net_device->channel_init_wait, 5*HZ);
-
-        if (t == 0)
-                return -ETIMEDOUT;
+        wait_for_completion(&net_device->channel_init_wait);
 
         if (init_packet->msg.init_msg.init_complete.status !=
             NVSP_STAT_SUCCESS)
diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c
index 6a69b5cc9fe2..787a20235e5c 100644
--- a/drivers/net/hyperv/netvsc_drv.c
+++ b/drivers/net/hyperv/netvsc_drv.c
@@ -98,16 +98,14 @@ static void netvsc_set_multicast_list(struct net_device *net)
 
 static int netvsc_open(struct net_device *net)
 {
-        struct net_device_context *net_device_ctx = netdev_priv(net);
-        struct hv_device *device_obj = net_device_ctx->device_ctx;
-        struct netvsc_device *nvdev = net_device_ctx->nvdev;
+        struct netvsc_device *nvdev = net_device_to_netvsc_device(net);
         struct rndis_device *rdev;
         int ret = 0;
 
         netif_carrier_off(net);
 
         /* Open up the device */
-        ret = rndis_filter_open(device_obj);
+        ret = rndis_filter_open(nvdev);
         if (ret != 0) {
                 netdev_err(net, "unable to open device (ret %d).\n", ret);
                 return ret;
@@ -125,7 +123,6 @@ static int netvsc_open(struct net_device *net)
 static int netvsc_close(struct net_device *net)
 {
         struct net_device_context *net_device_ctx = netdev_priv(net);
-        struct hv_device *device_obj = net_device_ctx->device_ctx;
         struct netvsc_device *nvdev = net_device_ctx->nvdev;
         int ret;
         u32 aread, awrite, i, msec = 10, retry = 0, retry_max = 20;
@@ -135,7 +132,7 @@ static int netvsc_close(struct net_device *net)
 
         /* Make sure netvsc_set_multicast_list doesn't re-enable filter! */
         cancel_work_sync(&net_device_ctx->work);
-        ret = rndis_filter_close(device_obj);
+        ret = rndis_filter_close(nvdev);
         if (ret != 0) {
                 netdev_err(net, "unable to close device (ret %d).\n", ret);
                 return ret;
@@ -701,7 +698,6 @@ int netvsc_recv_callback(struct hv_device *device_obj,
         }
 
 vf_injection_done:
-        net_device_ctx = netdev_priv(net);
         rx_stats = this_cpu_ptr(net_device_ctx->rx_stats);
 
         /* Allocate a skb - TODO direct I/O to pages? */
@@ -986,8 +982,6 @@ static struct rtnl_link_stats64 *netvsc_get_stats64(struct net_device *net,
 
 static int netvsc_set_mac_addr(struct net_device *ndev, void *p)
 {
-        struct net_device_context *ndevctx = netdev_priv(ndev);
-        struct hv_device *hdev =  ndevctx->device_ctx;
         struct sockaddr *addr = p;
         char save_adr[ETH_ALEN];
         unsigned char save_aatype;
@@ -1000,7 +994,7 @@ static int netvsc_set_mac_addr(struct net_device *ndev, void *p)
         if (err != 0)
                 return err;
 
-        err = rndis_filter_set_device_mac(hdev, addr->sa_data);
+        err = rndis_filter_set_device_mac(ndev, addr->sa_data);
         if (err != 0) {
                 /* roll back to saved MAC */
                 memcpy(ndev->dev_addr, save_adr, ETH_ALEN);
@@ -1248,7 +1242,7 @@ static int netvsc_vf_up(struct net_device *vf_netdev)
         /*
          * Open the device before switching data path.
          */
-        rndis_filter_open(net_device_ctx->device_ctx);
+        rndis_filter_open(netvsc_dev);
 
         /*
          * notify the host to switch the data path.
@@ -1303,7 +1297,7 @@ static int netvsc_vf_down(struct net_device *vf_netdev)
                 udelay(50);
         netvsc_switch_datapath(ndev, false);
         netdev_info(ndev, "Data path switched from VF: %s\n", vf_netdev->name);
-        rndis_filter_close(net_device_ctx->device_ctx);
+        rndis_filter_close(netvsc_dev);
         netif_carrier_on(ndev);
         /*
          * Notify peers.
@@ -1500,6 +1494,10 @@ static int netvsc_netdev_event(struct notifier_block *this,
 {
         struct net_device *event_dev = netdev_notifier_info_to_dev(ptr);
 
+        /* Avoid Vlan dev with same MAC registering as VF */
+        if (event_dev->priv_flags & IFF_802_1Q_VLAN)
+                return NOTIFY_DONE;
+
         switch (event) {
         case NETDEV_REGISTER:
                 return netvsc_register_vf(event_dev);
diff --git a/drivers/net/hyperv/rndis_filter.c b/drivers/net/hyperv/rndis_filter.c
index 97c292b7dbea..8e830f741d47 100644
--- a/drivers/net/hyperv/rndis_filter.c
+++ b/drivers/net/hyperv/rndis_filter.c
@@ -466,7 +466,6 @@ static int rndis_filter_query_device(struct rndis_device *dev, u32 oid,
         struct rndis_query_request *query;
         struct rndis_query_complete *query_complete;
         int ret = 0;
-        unsigned long t;
 
         if (!result)
                 return -EINVAL;
@@ -503,11 +502,7 @@ static int rndis_filter_query_device(struct rndis_device *dev, u32 oid,
         if (ret != 0)
                 goto cleanup;
 
-        t = wait_for_completion_timeout(&request->wait_event, 5*HZ);
-        if (t == 0) {
-                ret = -ETIMEDOUT;
-                goto cleanup;
-        }
+        wait_for_completion(&request->wait_event);
 
         /* Copy the response back */
         query_complete = &request->response_msg.msg.query_complete;
@@ -543,11 +538,9 @@ static int rndis_filter_query_device_mac(struct rndis_device *dev)
 #define NWADR_STR "NetworkAddress"
 #define NWADR_STRLEN 14
 
-int rndis_filter_set_device_mac(struct hv_device *hdev, char *mac)
+int rndis_filter_set_device_mac(struct net_device *ndev, char *mac)
 {
-        struct net_device *ndev = hv_get_drvdata(hdev);
-        struct net_device_context *net_device_ctx = netdev_priv(ndev);
-        struct netvsc_device *nvdev = net_device_ctx->nvdev;
+        struct netvsc_device *nvdev = net_device_to_netvsc_device(ndev);
         struct rndis_device *rdev = nvdev->extension;
         struct rndis_request *request;
         struct rndis_set_request *set;
@@ -558,7 +551,6 @@ int rndis_filter_set_device_mac(struct hv_device *hdev, char *mac)
         u32 extlen = sizeof(struct rndis_config_parameter_info) +
                 2*NWADR_STRLEN + 4*ETH_ALEN;
         int ret;
-        unsigned long t;
 
         request = get_rndis_request(rdev, RNDIS_MSG_SET,
                 RNDIS_MESSAGE_SIZE(struct rndis_set_request) + extlen);
@@ -599,21 +591,13 @@ int rndis_filter_set_device_mac(struct hv_device *hdev, char *mac)
         if (ret != 0)
                 goto cleanup;
 
-        t = wait_for_completion_timeout(&request->wait_event, 5*HZ);
-        if (t == 0) {
-                netdev_err(ndev, "timeout before we got a set response...\n");
-                /*
-                 * can't put_rndis_request, since we may still receive a
-                 * send-completion.
-                 */
-                return -EBUSY;
-        } else {
-                set_complete = &request->response_msg.msg.set_complete;
-                if (set_complete->status != RNDIS_STATUS_SUCCESS) {
-                        netdev_err(ndev, "Fail to set MAC on host side:0x%x\n",
-                                   set_complete->status);
-                        ret = -EINVAL;
-                }
+        wait_for_completion(&request->wait_event);
+
+        set_complete = &request->response_msg.msg.set_complete;
+        if (set_complete->status != RNDIS_STATUS_SUCCESS) {
+                netdev_err(ndev, "Fail to set MAC on host side:0x%x\n",
+                           set_complete->status);
+                ret = -EINVAL;
         }
 
 cleanup:
@@ -622,12 +606,10 @@ cleanup:
 }
 
 static int
-rndis_filter_set_offload_params(struct hv_device *hdev,
+rndis_filter_set_offload_params(struct net_device *ndev,
                                 struct ndis_offload_params *req_offloads)
 {
-        struct net_device *ndev = hv_get_drvdata(hdev);
-        struct net_device_context *net_device_ctx = netdev_priv(ndev);
-        struct netvsc_device *nvdev = net_device_ctx->nvdev;
+        struct netvsc_device *nvdev = net_device_to_netvsc_device(ndev);
         struct rndis_device *rdev = nvdev->extension;
         struct rndis_request *request;
         struct rndis_set_request *set;
@@ -635,7 +617,6 @@ rndis_filter_set_offload_params(struct hv_device *hdev,
         struct rndis_set_complete *set_complete;
         u32 extlen = sizeof(struct ndis_offload_params);
         int ret;
-        unsigned long t;
         u32 vsp_version = nvdev->nvsp_version;
 
         if (vsp_version <= NVSP_PROTOCOL_VERSION_4) {
@@ -669,20 +650,12 @@ rndis_filter_set_offload_params(struct hv_device *hdev,
         if (ret != 0)
                 goto cleanup;
 
-        t = wait_for_completion_timeout(&request->wait_event, 5*HZ);
-        if (t == 0) {
-                netdev_err(ndev, "timeout before we got aOFFLOAD set response...\n");
-                /* can't put_rndis_request, since we may still receive a
-                 * send-completion.
-                 */
-                return -EBUSY;
-        } else {
-                set_complete = &request->response_msg.msg.set_complete;
-                if (set_complete->status != RNDIS_STATUS_SUCCESS) {
-                        netdev_err(ndev, "Fail to set offload on host side:0x%x\n",
-                                   set_complete->status);
-                        ret = -EINVAL;
-                }
+        wait_for_completion(&request->wait_event);
+        set_complete = &request->response_msg.msg.set_complete;
+        if (set_complete->status != RNDIS_STATUS_SUCCESS) {
+                netdev_err(ndev, "Fail to set offload on host side:0x%x\n",
+                           set_complete->status);
+                ret = -EINVAL;
         }
 
 cleanup:
@@ -710,7 +683,6 @@ static int rndis_filter_set_rss_param(struct rndis_device *rdev, int num_queue)
         u32 *itab;
         u8 *keyp;
         int i, ret;
-        unsigned long t;
 
         request = get_rndis_request(
                         rdev, RNDIS_MSG_SET,
@@ -753,20 +725,12 @@ static int rndis_filter_set_rss_param(struct rndis_device *rdev, int num_queue)
         if (ret != 0)
                 goto cleanup;
 
-        t = wait_for_completion_timeout(&request->wait_event, 5*HZ);
-        if (t == 0) {
-                netdev_err(ndev, "timeout before we got a set response...\n");
-                /* can't put_rndis_request, since we may still receive a
-                 * send-completion.
-                 */
-                return -ETIMEDOUT;
-        } else {
-                set_complete = &request->response_msg.msg.set_complete;
-                if (set_complete->status != RNDIS_STATUS_SUCCESS) {
-                        netdev_err(ndev, "Fail to set RSS parameters:0x%x\n",
-                                   set_complete->status);
-                        ret = -EINVAL;
-                }
+        wait_for_completion(&request->wait_event);
+        set_complete = &request->response_msg.msg.set_complete;
+        if (set_complete->status != RNDIS_STATUS_SUCCESS) {
+                netdev_err(ndev, "Fail to set RSS parameters:0x%x\n",
+                           set_complete->status);
+                ret = -EINVAL;
         }
 
 cleanup:
@@ -795,8 +759,6 @@ int rndis_filter_set_packet_filter(struct rndis_device *dev, u32 new_filter)
         struct rndis_set_complete *set_complete;
         u32 status;
         int ret;
-        unsigned long t;
-        struct net_device *ndev = dev->ndev;
 
         request = get_rndis_request(dev, RNDIS_MSG_SET,
                         RNDIS_MESSAGE_SIZE(struct rndis_set_request) +
@@ -819,26 +781,14 @@ int rndis_filter_set_packet_filter(struct rndis_device *dev, u32 new_filter)
         if (ret != 0)
                 goto cleanup;
 
-        t = wait_for_completion_timeout(&request->wait_event, 5*HZ);
+        wait_for_completion(&request->wait_event);
 
-        if (t == 0) {
-                netdev_err(ndev,
-                        "timeout before we got a set response...\n");
-                ret = -ETIMEDOUT;
-                /*
-                 * We can't deallocate the request since we may still receive a
-                 * send completion for it.
-                 */
-                goto exit;
-        } else {
-                set_complete = &request->response_msg.msg.set_complete;
-                status = set_complete->status;
-        }
+        set_complete = &request->response_msg.msg.set_complete;
+        status = set_complete->status;
 
 cleanup:
         if (request)
                 put_rndis_request(dev, request);
-exit:
         return ret;
 }
 
@@ -850,9 +800,7 @@ static int rndis_filter_init_device(struct rndis_device *dev)
         struct rndis_initialize_complete *init_complete;
         u32 status;
         int ret;
-        unsigned long t;
-        struct net_device_context *net_device_ctx = netdev_priv(dev->ndev);
-        struct netvsc_device *nvdev = net_device_ctx->nvdev;
+        struct netvsc_device *nvdev = net_device_to_netvsc_device(dev->ndev);
 
         request = get_rndis_request(dev, RNDIS_MSG_INIT,
                         RNDIS_MESSAGE_SIZE(struct rndis_initialize_request));
@@ -875,12 +823,7 @@ static int rndis_filter_init_device(struct rndis_device *dev)
                 goto cleanup;
         }
 
-        t = wait_for_completion_timeout(&request->wait_event, 5*HZ);
-
-        if (t == 0) {
-                ret = -ETIMEDOUT;
-                goto cleanup;
-        }
+        wait_for_completion(&request->wait_event);
 
         init_complete = &request->response_msg.msg.init_complete;
         status = init_complete->status;
@@ -977,8 +920,7 @@ static void netvsc_sc_open(struct vmbus_channel *new_sc)
 {
         struct net_device *ndev =
                 hv_get_drvdata(new_sc->primary_channel->device_obj);
-        struct net_device_context *net_device_ctx = netdev_priv(ndev);
-        struct netvsc_device *nvscdev = net_device_ctx->nvdev;
+        struct netvsc_device *nvscdev = net_device_to_netvsc_device(ndev);
         u16 chn_index = new_sc->offermsg.offer.sub_channel_index;
         int ret;
         unsigned long flags;
@@ -1014,7 +956,6 @@ int rndis_filter_device_add(struct hv_device *dev,
         struct netvsc_device_info *device_info = additional_info;
         struct ndis_offload_params offloads;
         struct nvsp_message *init_packet;
-        unsigned long t;
         struct ndis_recv_scale_cap rsscap;
         u32 rsscap_size = sizeof(struct ndis_recv_scale_cap);
         u32 mtu, size;
@@ -1088,7 +1029,7 @@ int rndis_filter_device_add(struct hv_device *dev,
         offloads.lso_v2_ipv4 = NDIS_OFFLOAD_PARAMETERS_LSOV2_ENABLED;
 
 
-        ret = rndis_filter_set_offload_params(dev, &offloads);
+        ret = rndis_filter_set_offload_params(net, &offloads);
         if (ret)
                 goto err_dev_remv;
 
@@ -1157,11 +1098,8 @@ int rndis_filter_device_add(struct hv_device *dev,
                                VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
         if (ret)
                 goto out;
-        t = wait_for_completion_timeout(&net_device->channel_init_wait, 5*HZ);
-        if (t == 0) {
-                ret = -ETIMEDOUT;
-                goto out;
-        }
+        wait_for_completion(&net_device->channel_init_wait);
+
         if (init_packet->msg.v5_msg.subchn_comp.status !=
             NVSP_STAT_SUCCESS) {
                 ret = -ENODEV;
@@ -1196,21 +1134,14 @@ err_dev_remv:
 
 void rndis_filter_device_remove(struct hv_device *dev)
 {
-        struct net_device *ndev = hv_get_drvdata(dev);
-        struct net_device_context *net_device_ctx = netdev_priv(ndev);
-        struct netvsc_device *net_dev = net_device_ctx->nvdev;
+        struct netvsc_device *net_dev = hv_device_to_netvsc_device(dev);
         struct rndis_device *rndis_dev = net_dev->extension;
-        unsigned long t;
 
         /* If not all subchannel offers are complete, wait for them until
          * completion to avoid race.
          */
-        while (net_dev->num_sc_offered > 0) {
-                t = wait_for_completion_timeout(&net_dev->channel_init_wait,
-                                                10 * HZ);
-                if (t == 0)
-                        WARN(1, "Netvsc: Waiting for sub-channel processing");
-        }
+        if (net_dev->num_sc_offered > 0)
+                wait_for_completion(&net_dev->channel_init_wait);
 
         /* Halt and release the rndis device */
         rndis_filter_halt_device(rndis_dev);
@@ -1222,27 +1153,19 @@ void rndis_filter_device_remove(struct hv_device *dev)
 }
 
 
-int rndis_filter_open(struct hv_device *dev)
+int rndis_filter_open(struct netvsc_device *nvdev)
 {
-        struct net_device *ndev = hv_get_drvdata(dev);
-        struct net_device_context *net_device_ctx = netdev_priv(ndev);
-        struct netvsc_device *net_device = net_device_ctx->nvdev;
-
-        if (!net_device)
+        if (!nvdev)
                 return -EINVAL;
 
-        if (atomic_inc_return(&net_device->open_cnt) != 1)
+        if (atomic_inc_return(&nvdev->open_cnt) != 1)
                 return 0;
 
-        return rndis_filter_open_device(net_device->extension);
+        return rndis_filter_open_device(nvdev->extension);
 }
 
-int rndis_filter_close(struct hv_device *dev)
+int rndis_filter_close(struct netvsc_device *nvdev)
 {
-        struct net_device *ndev = hv_get_drvdata(dev);
-        struct net_device_context *net_device_ctx = netdev_priv(ndev);
-        struct netvsc_device *nvdev = net_device_ctx->nvdev;
-
         if (!nvdev)
                 return -EINVAL;
 
diff --git a/drivers/net/ipvlan/ipvlan_main.c b/drivers/net/ipvlan/ipvlan_main.c
index 1c4d395fbd49..18b4e8c7f68a 100644
--- a/drivers/net/ipvlan/ipvlan_main.c
+++ b/drivers/net/ipvlan/ipvlan_main.c
@@ -80,13 +80,6 @@ static void ipvlan_port_destroy(struct net_device *dev)
         kfree_rcu(port, rcu);
 }
 
-/* ipvlan network devices have devices nesting below it and are a special
- * "super class" of normal network devices; split their locks off into a
- * separate class since they always nest.
- */
-static struct lock_class_key ipvlan_netdev_xmit_lock_key;
-static struct lock_class_key ipvlan_netdev_addr_lock_key;
-
 #define IPVLAN_FEATURES \
         (NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST | \
          NETIF_F_GSO | NETIF_F_TSO | NETIF_F_UFO | NETIF_F_GSO_ROBUST | \
@@ -96,19 +89,6 @@ static struct lock_class_key ipvlan_netdev_addr_lock_key;
 #define IPVLAN_STATE_MASK \
         ((1<<__LINK_STATE_NOCARRIER) | (1<<__LINK_STATE_DORMANT))
 
-static void ipvlan_set_lockdep_class_one(struct net_device *dev,
-                                         struct netdev_queue *txq,
-                                         void *_unused)
-{
-        lockdep_set_class(&txq->_xmit_lock, &ipvlan_netdev_xmit_lock_key);
-}
-
-static void ipvlan_set_lockdep_class(struct net_device *dev)
-{
-        lockdep_set_class(&dev->addr_list_lock, &ipvlan_netdev_addr_lock_key);
-        netdev_for_each_tx_queue(dev, ipvlan_set_lockdep_class_one, NULL);
-}
-
 static int ipvlan_init(struct net_device *dev)
 {
         struct ipvl_dev *ipvlan = netdev_priv(dev);
@@ -123,7 +103,7 @@ static int ipvlan_init(struct net_device *dev)
         dev->gso_max_segs = phy_dev->gso_max_segs;
         dev->hard_header_len = phy_dev->hard_header_len;
 
-        ipvlan_set_lockdep_class(dev);
+        netdev_lockdep_set_classes(dev);
 
         ipvlan->pcpu_stats = alloc_percpu(struct ipvl_pcpu_stats);
         if (!ipvlan->pcpu_stats)
diff --git a/drivers/net/loopback.c b/drivers/net/loopback.c
index a400288cb37b..6255973e3dda 100644
--- a/drivers/net/loopback.c
+++ b/drivers/net/loopback.c
@@ -169,10 +169,9 @@ static void loopback_setup(struct net_device *dev)
         dev->flags              = IFF_LOOPBACK;
         dev->priv_flags         |= IFF_LIVE_ADDR_CHANGE | IFF_NO_QUEUE;
         netif_keep_dst(dev);
-        dev->hw_features        = NETIF_F_ALL_TSO | NETIF_F_UFO;
+        dev->hw_features        = NETIF_F_GSO_SOFTWARE;
         dev->features           = NETIF_F_SG | NETIF_F_FRAGLIST
-                | NETIF_F_ALL_TSO
-                | NETIF_F_UFO
+                | NETIF_F_GSO_SOFTWARE
                 | NETIF_F_HW_CSUM
                 | NETIF_F_RXCSUM
                 | NETIF_F_SCTP_CRC
diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c
index cb01023eab41..cd9b53834bf6 100644
--- a/drivers/net/macvlan.c
+++ b/drivers/net/macvlan.c
@@ -49,6 +49,7 @@ struct macvlan_port {
         bool                    passthru;
         int                     count;
         struct hlist_head       vlan_source_hash[MACVLAN_HASH_SIZE];
+        DECLARE_BITMAP(mc_filter, MACVLAN_MC_FILTER_SZ);
 };
 
 struct macvlan_source_entry {
@@ -305,11 +306,14 @@ static void macvlan_process_broadcast(struct work_struct *w)
 
                 rcu_read_unlock();
 
+                if (src)
+                        dev_put(src->dev);
                 kfree_skb(skb);
         }
 }
 
 static void macvlan_broadcast_enqueue(struct macvlan_port *port,
+                                      const struct macvlan_dev *src,
                                       struct sk_buff *skb)
 {
         struct sk_buff *nskb;
@@ -319,8 +323,12 @@ static void macvlan_broadcast_enqueue(struct macvlan_port *port,
         if (!nskb)
                 goto err;
 
+        MACVLAN_SKB_CB(nskb)->src = src;
+
         spin_lock(&port->bc_queue.lock);
         if (skb_queue_len(&port->bc_queue) < MACVLAN_BC_QUEUE_LEN) {
+                if (src)
+                        dev_hold(src->dev);
                 __skb_queue_tail(&port->bc_queue, nskb);
                 err = 0;
         }
@@ -412,6 +420,8 @@ static rx_handler_result_t macvlan_handle_frame(struct sk_buff **pskb)
 
         port = macvlan_port_get_rcu(skb->dev);
         if (is_multicast_ether_addr(eth->h_dest)) {
+                unsigned int hash;
+
                 skb = ip_check_defrag(dev_net(skb->dev), skb, IP_DEFRAG_MACVLAN);
                 if (!skb)
                         return RX_HANDLER_CONSUMED;
@@ -429,8 +439,9 @@ static rx_handler_result_t macvlan_handle_frame(struct sk_buff **pskb)
                         goto out;
                 }
 
-                MACVLAN_SKB_CB(skb)->src = src;
-                macvlan_broadcast_enqueue(port, skb);
+                hash = mc_hash(NULL, eth->h_dest);
+                if (test_bit(hash, port->mc_filter))
+                        macvlan_broadcast_enqueue(port, src, skb);
 
                 return RX_HANDLER_PASS;
         }
@@ -716,12 +727,12 @@ static void macvlan_change_rx_flags(struct net_device *dev, int change)
         }
 }
 
-static void macvlan_set_mac_lists(struct net_device *dev)
+static void macvlan_compute_filter(unsigned long *mc_filter,
+                                   struct net_device *dev,
+                                   struct macvlan_dev *vlan)
 {
-        struct macvlan_dev *vlan = netdev_priv(dev);
-
         if (dev->flags & (IFF_PROMISC | IFF_ALLMULTI)) {
-                bitmap_fill(vlan->mc_filter, MACVLAN_MC_FILTER_SZ);
+                bitmap_fill(mc_filter, MACVLAN_MC_FILTER_SZ);
         } else {
                 struct netdev_hw_addr *ha;
                 DECLARE_BITMAP(filter, MACVLAN_MC_FILTER_SZ);
@@ -733,10 +744,33 @@ static void macvlan_set_mac_lists(struct net_device *dev)
 
                 __set_bit(mc_hash(vlan, dev->broadcast), filter);
 
-                bitmap_copy(vlan->mc_filter, filter, MACVLAN_MC_FILTER_SZ);
+                bitmap_copy(mc_filter, filter, MACVLAN_MC_FILTER_SZ);
         }
+}
+
+static void macvlan_set_mac_lists(struct net_device *dev)
+{
+        struct macvlan_dev *vlan = netdev_priv(dev);
+
+        macvlan_compute_filter(vlan->mc_filter, dev, vlan);
+
         dev_uc_sync(vlan->lowerdev, dev);
         dev_mc_sync(vlan->lowerdev, dev);
+
+        /* This is slightly inaccurate as we're including the subscription
+         * list of vlan->lowerdev too.
+         *
+         * Bug alert: This only works if everyone has the same broadcast
+         * address as lowerdev.  As soon as someone changes theirs this
+         * will break.
+         *
+         * However, this is already broken as when you change your broadcast
+         * address we don't get called.
+         *
+         * The solution is to maintain a list of broadcast addresses like
+         * we do for uc/mc, if you care.
+         */
+        macvlan_compute_filter(vlan->port->mc_filter, vlan->lowerdev, NULL);
 }
 
 static int macvlan_change_mtu(struct net_device *dev, int new_mtu)
@@ -754,7 +788,6 @@ static int macvlan_change_mtu(struct net_device *dev, int new_mtu)
  * "super class" of normal network devices; split their locks off into a
  * separate class since they always nest.
  */
-static struct lock_class_key macvlan_netdev_xmit_lock_key;
 static struct lock_class_key macvlan_netdev_addr_lock_key;
 
 #define ALWAYS_ON_FEATURES \
@@ -775,20 +808,12 @@ static int macvlan_get_nest_level(struct net_device *dev)
         return ((struct macvlan_dev *)netdev_priv(dev))->nest_level;
 }
 
-static void macvlan_set_lockdep_class_one(struct net_device *dev,
-                                          struct netdev_queue *txq,
-                                          void *_unused)
-{
-        lockdep_set_class(&txq->_xmit_lock,
-                          &macvlan_netdev_xmit_lock_key);
-}
-
 static void macvlan_set_lockdep_class(struct net_device *dev)
 {
+        netdev_lockdep_set_classes(dev);
         lockdep_set_class_and_subclass(&dev->addr_list_lock,
                                        &macvlan_netdev_addr_lock_key,
                                        macvlan_get_nest_level(dev));
-        netdev_for_each_tx_queue(dev, macvlan_set_lockdep_class_one, NULL);
 }
 
 static int macvlan_init(struct net_device *dev)
diff --git a/drivers/net/macvtap.c b/drivers/net/macvtap.c
index bd6720962b1f..95a13321f688 100644
--- a/drivers/net/macvtap.c
+++ b/drivers/net/macvtap.c
@@ -627,93 +627,6 @@ static inline struct sk_buff *macvtap_alloc_skb(struct sock *sk, size_t prepad,
         return skb;
 }
 
-/*
- * macvtap_skb_from_vnet_hdr and macvtap_skb_to_vnet_hdr should
- * be shared with the tun/tap driver.
- */
-static int macvtap_skb_from_vnet_hdr(struct macvtap_queue *q,
-                                     struct sk_buff *skb,
-                                     struct virtio_net_hdr *vnet_hdr)
-{
-        unsigned short gso_type = 0;
-        if (vnet_hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
-                switch (vnet_hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
-                case VIRTIO_NET_HDR_GSO_TCPV4:
-                        gso_type = SKB_GSO_TCPV4;
-                        break;
-                case VIRTIO_NET_HDR_GSO_TCPV6:
-                        gso_type = SKB_GSO_TCPV6;
-                        break;
-                case VIRTIO_NET_HDR_GSO_UDP:
-                        gso_type = SKB_GSO_UDP;
-                        break;
-                default:
-                        return -EINVAL;
-                }
-
-                if (vnet_hdr->gso_type & VIRTIO_NET_HDR_GSO_ECN)
-                        gso_type |= SKB_GSO_TCP_ECN;
-
-                if (vnet_hdr->gso_size == 0)
-                        return -EINVAL;
-        }
-
-        if (vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
-                if (!skb_partial_csum_set(skb, macvtap16_to_cpu(q, vnet_hdr->csum_start),
-                                          macvtap16_to_cpu(q, vnet_hdr->csum_offset)))
-                        return -EINVAL;
-        }
-
-        if (vnet_hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
-                skb_shinfo(skb)->gso_size = macvtap16_to_cpu(q, vnet_hdr->gso_size);
-                skb_shinfo(skb)->gso_type = gso_type;
-
-                /* Header must be checked, and gso_segs computed. */
-                skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
-                skb_shinfo(skb)->gso_segs = 0;
-        }
-        return 0;
-}
-
-static void macvtap_skb_to_vnet_hdr(struct macvtap_queue *q,
-                                    const struct sk_buff *skb,
-                                    struct virtio_net_hdr *vnet_hdr)
-{
-        memset(vnet_hdr, 0, sizeof(*vnet_hdr));
-
-        if (skb_is_gso(skb)) {
-                struct skb_shared_info *sinfo = skb_shinfo(skb);
-
-                /* This is a hint as to how much should be linear. */
-                vnet_hdr->hdr_len = cpu_to_macvtap16(q, skb_headlen(skb));
-                vnet_hdr->gso_size = cpu_to_macvtap16(q, sinfo->gso_size);
-                if (sinfo->gso_type & SKB_GSO_TCPV4)
-                        vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
-                else if (sinfo->gso_type & SKB_GSO_TCPV6)
-                        vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
-                else if (sinfo->gso_type & SKB_GSO_UDP)
-                        vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP;
-                else
-                        BUG();
-                if (sinfo->gso_type & SKB_GSO_TCP_ECN)
-                        vnet_hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN;
-        } else
-                vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_NONE;
-
-        if (skb->ip_summed == CHECKSUM_PARTIAL) {
-                vnet_hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
-                if (skb_vlan_tag_present(skb))
-                        vnet_hdr->csum_start = cpu_to_macvtap16(q,
-                                skb_checksum_start_offset(skb) + VLAN_HLEN);
-                else
-                        vnet_hdr->csum_start = cpu_to_macvtap16(q,
-                                skb_checksum_start_offset(skb));
-                vnet_hdr->csum_offset = cpu_to_macvtap16(q, skb->csum_offset);
-        } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
-                vnet_hdr->flags = VIRTIO_NET_HDR_F_DATA_VALID;
-        } /* else everything is zero */
-}
-
 /* Neighbour code has some assumptions on HH_DATA_MOD alignment */
 #define MACVTAP_RESERVE HH_DATA_OFF(ETH_HLEN)
 
@@ -812,7 +725,8 @@ static ssize_t macvtap_get_user(struct macvtap_queue *q, struct msghdr *m,
         skb->protocol = eth_hdr(skb)->h_proto;
 
         if (vnet_hdr_len) {
-                err = macvtap_skb_from_vnet_hdr(q, skb, &vnet_hdr);
+                err = virtio_net_hdr_to_skb(skb, &vnet_hdr,
+                                            macvtap_is_little_endian(q));
                 if (err)
                         goto err_kfree;
         }
@@ -880,7 +794,10 @@ static ssize_t macvtap_put_user(struct macvtap_queue *q,
                 if (iov_iter_count(iter) < vnet_hdr_len)
                         return -EINVAL;
 
-                macvtap_skb_to_vnet_hdr(q, skb, &vnet_hdr);
+                ret = virtio_net_hdr_from_skb(skb, &vnet_hdr,
+                                              macvtap_is_little_endian(q));
+                if (ret)
+                        BUG();
 
                 if (copy_to_iter(&vnet_hdr, sizeof(vnet_hdr), iter) !=
                     sizeof(vnet_hdr))
diff --git a/drivers/net/phy/Kconfig b/drivers/net/phy/Kconfig
index 6dad9a9c356c..f96829415ce6 100644
--- a/drivers/net/phy/Kconfig
+++ b/drivers/net/phy/Kconfig
@@ -12,6 +12,9 @@ menuconfig PHYLIB
 
 if PHYLIB
 
+config SWPHY
+        bool
+
 comment "MII PHY device drivers"
 
 config AQUANTIA_PHY
@@ -159,6 +162,7 @@ config MICROCHIP_PHY
 config FIXED_PHY
         tristate "Driver for MDIO Bus/PHY emulation with fixed speed/link PHYs"
         depends on PHYLIB
+        select SWPHY
         ---help---
           Adds the platform "fixed" MDIO Bus to cover the boards that use
           PHYs that are not connected to the real MDIO bus.
@@ -254,6 +258,17 @@ config MDIO_BUS_MUX_MMIOREG
 
           Currently, only 8-bit registers are supported.
 
+config MDIO_BUS_MUX_BCM_IPROC
+        tristate "Support for iProc based MDIO bus multiplexers"
+        depends on OF && OF_MDIO && (ARCH_BCM_IPROC || COMPILE_TEST)
+        select MDIO_BUS_MUX
+        default ARCH_BCM_IPROC
+        help
+          This module provides a driver for MDIO bus multiplexers found in
+          iProc based Broadcom SoCs. This multiplexer connects one of several
+          child MDIO bus to a parent bus. Buses could be internal as well as
+          external and selection logic lies inside the same multiplexer.
+
 config MDIO_BCM_UNIMAC
         tristate "Broadcom UniMAC MDIO bus controller"
         depends on HAS_IOMEM
@@ -271,6 +286,14 @@ config MDIO_BCM_IPROC
           This module provides a driver for the MDIO busses found in the
           Broadcom iProc SoC's.
 
+config INTEL_XWAY_PHY
+        tristate "Driver for Intel XWAY PHYs"
+        ---help---
+          Supports the Intel XWAY (former Lantiq) 11G and 22E PHYs.
+          These PHYs are marked as standalone chips under the names
+          PEF 7061, PEF 7071 and PEF 7072 or integrated into the Intel
+          SoCs xRX200, xRX300, xRX330, xRX350 and xRX550.
+
 endif # PHYLIB
 
 config MICREL_KS8995MA
diff --git a/drivers/net/phy/Makefile b/drivers/net/phy/Makefile
index fcdbb9299fab..7158274327d0 100644
--- a/drivers/net/phy/Makefile
+++ b/drivers/net/phy/Makefile
@@ -1,6 +1,7 @@
 # Makefile for Linux PHY drivers
 
-libphy-objs                     := phy.o phy_device.o mdio_bus.o mdio_device.o
+libphy-y                        := phy.o phy_device.o mdio_bus.o mdio_device.o
+libphy-$(CONFIG_SWPHY)          += swphy.o
 
 obj-$(CONFIG_PHYLIB)            += libphy.o
 obj-$(CONFIG_AQUANTIA_PHY)      += aquantia.o
@@ -39,8 +40,10 @@ obj-$(CONFIG_AMD_PHY)		+= amd.o
 obj-$(CONFIG_MDIO_BUS_MUX)      += mdio-mux.o
 obj-$(CONFIG_MDIO_BUS_MUX_GPIO) += mdio-mux-gpio.o
 obj-$(CONFIG_MDIO_BUS_MUX_MMIOREG) += mdio-mux-mmioreg.o
+obj-$(CONFIG_MDIO_BUS_MUX_BCM_IPROC)    += mdio-mux-bcm-iproc.o
 obj-$(CONFIG_MDIO_SUN4I)        += mdio-sun4i.o
 obj-$(CONFIG_MDIO_MOXART)       += mdio-moxart.o
 obj-$(CONFIG_MDIO_BCM_UNIMAC)   += mdio-bcm-unimac.o
 obj-$(CONFIG_MICROCHIP_PHY)     += microchip.o
 obj-$(CONFIG_MDIO_BCM_IPROC)    += mdio-bcm-iproc.o
+obj-$(CONFIG_INTEL_XWAY_PHY)    += intel-xway.o
diff --git a/drivers/net/phy/fixed_phy.c b/drivers/net/phy/fixed_phy.c
index 9ec7f7353434..c649c101bbab 100644
--- a/drivers/net/phy/fixed_phy.c
+++ b/drivers/net/phy/fixed_phy.c
@@ -23,9 +23,10 @@
 #include <linux/slab.h>
 #include <linux/of.h>
 #include <linux/gpio.h>
+#include <linux/seqlock.h>
 #include <linux/idr.h>
 
-#define MII_REGS_NUM 29
+#include "swphy.h"
 
 struct fixed_mdio_bus {
         struct mii_bus *mii_bus;
@@ -34,8 +35,8 @@ struct fixed_mdio_bus {
 
 struct fixed_phy {
         int addr;
-        u16 regs[MII_REGS_NUM];
         struct phy_device *phydev;
+        seqcount_t seqcount;
         struct fixed_phy_status status;
         int (*link_update)(struct net_device *, struct fixed_phy_status *);
         struct list_head node;
@@ -47,103 +48,10 @@ static struct fixed_mdio_bus platform_fmb = {
         .phys = LIST_HEAD_INIT(platform_fmb.phys),
 };
 
-static int fixed_phy_update_regs(struct fixed_phy *fp)
+static void fixed_phy_update(struct fixed_phy *fp)
 {
-        u16 bmsr = BMSR_ANEGCAPABLE;
-        u16 bmcr = 0;
-        u16 lpagb = 0;
-        u16 lpa = 0;
-
         if (gpio_is_valid(fp->link_gpio))
                 fp->status.link = !!gpio_get_value_cansleep(fp->link_gpio);
-
-        if (fp->status.duplex) {
-                switch (fp->status.speed) {
-                case 1000:
-                        bmsr |= BMSR_ESTATEN;
-                        break;
-                case 100:
-                        bmsr |= BMSR_100FULL;
-                        break;
-                case 10:
-                        bmsr |= BMSR_10FULL;
-                        break;
-                default:
-                        break;
-                }
-        } else {
-                switch (fp->status.speed) {
-                case 1000:
-                        bmsr |= BMSR_ESTATEN;
-                        break;
-                case 100:
-                        bmsr |= BMSR_100HALF;
-                        break;
-                case 10:
-                        bmsr |= BMSR_10HALF;
-                        break;
-                default:
-                        break;
-                }
-        }
-
-        if (fp->status.link) {
-                bmsr |= BMSR_LSTATUS | BMSR_ANEGCOMPLETE;
-
-                if (fp->status.duplex) {
-                        bmcr |= BMCR_FULLDPLX;
-
-                        switch (fp->status.speed) {
-                        case 1000:
-                                bmcr |= BMCR_SPEED1000;
-                                lpagb |= LPA_1000FULL;
-                                break;
-                        case 100:
-                                bmcr |= BMCR_SPEED100;
-                                lpa |= LPA_100FULL;
-                                break;
-                        case 10:
-                                lpa |= LPA_10FULL;
-                                break;
-                        default:
-                                pr_warn("fixed phy: unknown speed\n");
-                                return -EINVAL;
-                        }
-                } else {
-                        switch (fp->status.speed) {
-                        case 1000:
-                                bmcr |= BMCR_SPEED1000;
-                                lpagb |= LPA_1000HALF;
-                                break;
-                        case 100:
-                                bmcr |= BMCR_SPEED100;
-                                lpa |= LPA_100HALF;
-                                break;
-                        case 10:
-                                lpa |= LPA_10HALF;
-                                break;
-                        default:
-                                pr_warn("fixed phy: unknown speed\n");
-                        return -EINVAL;
-                        }
-                }
-
-                if (fp->status.pause)
-                        lpa |= LPA_PAUSE_CAP;
-
-                if (fp->status.asym_pause)
-                        lpa |= LPA_PAUSE_ASYM;
-        }
-
-        fp->regs[MII_PHYSID1] = 0;
-        fp->regs[MII_PHYSID2] = 0;
-
-        fp->regs[MII_BMSR] = bmsr;
-        fp->regs[MII_BMCR] = bmcr;
-        fp->regs[MII_LPA] = lpa;
-        fp->regs[MII_STAT1000] = lpagb;
-
-        return 0;
 }
 
 static int fixed_mdio_read(struct mii_bus *bus, int phy_addr, int reg_num)
@@ -151,29 +59,23 @@ static int fixed_mdio_read(struct mii_bus *bus, int phy_addr, int reg_num)
         struct fixed_mdio_bus *fmb = bus->priv;
         struct fixed_phy *fp;
 
-        if (reg_num >= MII_REGS_NUM)
-                return -1;
-
-        /* We do not support emulating Clause 45 over Clause 22 register reads
-         * return an error instead of bogus data.
-         */
-        switch (reg_num) {
-        case MII_MMD_CTRL:
-        case MII_MMD_DATA:
-                return -1;
-        default:
-                break;
-        }
-
         list_for_each_entry(fp, &fmb->phys, node) {
                 if (fp->addr == phy_addr) {
-                        /* Issue callback if user registered it. */
-                        if (fp->link_update) {
-                                fp->link_update(fp->phydev->attached_dev,
-                                                &fp->status);
-                                fixed_phy_update_regs(fp);
-                        }
-                        return fp->regs[reg_num];
+                        struct fixed_phy_status state;
+                        int s;
+
+                        do {
+                                s = read_seqcount_begin(&fp->seqcount);
+                                /* Issue callback if user registered it. */
+                                if (fp->link_update) {
+                                        fp->link_update(fp->phydev->attached_dev,
+                                                        &fp->status);
+                                        fixed_phy_update(fp);
+                                }
+                                state = fp->status;
+                        } while (read_seqcount_retry(&fp->seqcount, s));
+
+                        return swphy_read_reg(reg_num, &state);
                 }
         }
 
@@ -225,6 +127,7 @@ int fixed_phy_update_state(struct phy_device *phydev,
 
         list_for_each_entry(fp, &fmb->phys, node) {
                 if (fp->addr == phydev->mdio.addr) {
+                        write_seqcount_begin(&fp->seqcount);
 #define _UPD(x) if (changed->x) \
         fp->status.x = status->x
                         _UPD(link);
@@ -233,7 +136,8 @@ int fixed_phy_update_state(struct phy_device *phydev,
                         _UPD(pause);
                         _UPD(asym_pause);
 #undef _UPD
-                        fixed_phy_update_regs(fp);
+                        fixed_phy_update(fp);
+                        write_seqcount_end(&fp->seqcount);
                         return 0;
                 }
         }
@@ -250,11 +154,15 @@ int fixed_phy_add(unsigned int irq, int phy_addr,
         struct fixed_mdio_bus *fmb = &platform_fmb;
         struct fixed_phy *fp;
 
+        ret = swphy_validate_state(status);
+        if (ret < 0)
+                return ret;
+
         fp = kzalloc(sizeof(*fp), GFP_KERNEL);
         if (!fp)
                 return -ENOMEM;
 
-        memset(fp->regs, 0xFF,  sizeof(fp->regs[0]) * MII_REGS_NUM);
+        seqcount_init(&fp->seqcount);
 
         if (irq != PHY_POLL)
                 fmb->mii_bus->irq[phy_addr] = irq;
@@ -270,17 +178,12 @@ int fixed_phy_add(unsigned int irq, int phy_addr,
                         goto err_regs;
         }
 
-        ret = fixed_phy_update_regs(fp);
-        if (ret)
-                goto err_gpio;
+        fixed_phy_update(fp);
 
         list_add_tail(&fp->node, &fmb->phys);
 
         return 0;
 
-err_gpio:
-        if (gpio_is_valid(fp->link_gpio))
-                gpio_free(fp->link_gpio);
 err_regs:
         kfree(fp);
         return ret;
diff --git a/drivers/net/phy/intel-xway.c b/drivers/net/phy/intel-xway.c
new file mode 100644
index 000000000000..c300ab5587b8
--- /dev/null
+++ b/drivers/net/phy/intel-xway.c
@@ -0,0 +1,376 @@
+/*
+ * Copyright (C) 2012 Daniel Schwierzeck <daniel.schwierzeck@googlemail.com>
+ * Copyright (C) 2016 Hauke Mehrtens <hauke@hauke-m.de>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/mdio.h>
+#include <linux/module.h>
+#include <linux/phy.h>
+#include <linux/of.h>
+
+#define XWAY_MDIO_IMASK                 0x19    /* interrupt mask */
+#define XWAY_MDIO_ISTAT                 0x1A    /* interrupt status */
+
+#define XWAY_MDIO_INIT_WOL              BIT(15) /* Wake-On-LAN */
+#define XWAY_MDIO_INIT_MSRE             BIT(14)
+#define XWAY_MDIO_INIT_NPRX             BIT(13)
+#define XWAY_MDIO_INIT_NPTX             BIT(12)
+#define XWAY_MDIO_INIT_ANE              BIT(11) /* Auto-Neg error */
+#define XWAY_MDIO_INIT_ANC              BIT(10) /* Auto-Neg complete */
+#define XWAY_MDIO_INIT_ADSC             BIT(5)  /* Link auto-downspeed detect */
+#define XWAY_MDIO_INIT_MPIPC            BIT(4)
+#define XWAY_MDIO_INIT_MDIXC            BIT(3)
+#define XWAY_MDIO_INIT_DXMC             BIT(2)  /* Duplex mode change */
+#define XWAY_MDIO_INIT_LSPC             BIT(1)  /* Link speed change */
+#define XWAY_MDIO_INIT_LSTC             BIT(0)  /* Link state change */
+#define XWAY_MDIO_INIT_MASK             (XWAY_MDIO_INIT_LSTC | \
+                                         XWAY_MDIO_INIT_ADSC)
+
+#define ADVERTISED_MPD                  BIT(10) /* Multi-port device */
+
+/* LED Configuration */
+#define XWAY_MMD_LEDCH                  0x01E0
+/* Inverse of SCAN Function */
+#define  XWAY_MMD_LEDCH_NACS_NONE       0x0000
+#define  XWAY_MMD_LEDCH_NACS_LINK       0x0001
+#define  XWAY_MMD_LEDCH_NACS_PDOWN      0x0002
+#define  XWAY_MMD_LEDCH_NACS_EEE        0x0003
+#define  XWAY_MMD_LEDCH_NACS_ANEG       0x0004
+#define  XWAY_MMD_LEDCH_NACS_ABIST      0x0005
+#define  XWAY_MMD_LEDCH_NACS_CDIAG      0x0006
+#define  XWAY_MMD_LEDCH_NACS_TEST       0x0007
+/* Slow Blink Frequency */
+#define  XWAY_MMD_LEDCH_SBF_F02HZ       0x0000
+#define  XWAY_MMD_LEDCH_SBF_F04HZ       0x0010
+#define  XWAY_MMD_LEDCH_SBF_F08HZ       0x0020
+#define  XWAY_MMD_LEDCH_SBF_F16HZ       0x0030
+/* Fast Blink Frequency */
+#define  XWAY_MMD_LEDCH_FBF_F02HZ       0x0000
+#define  XWAY_MMD_LEDCH_FBF_F04HZ       0x0040
+#define  XWAY_MMD_LEDCH_FBF_F08HZ       0x0080
+#define  XWAY_MMD_LEDCH_FBF_F16HZ       0x00C0
+/* LED Configuration */
+#define XWAY_MMD_LEDCL                  0x01E1
+/* Complex Blinking Configuration */
+#define  XWAY_MMD_LEDCH_CBLINK_NONE     0x0000
+#define  XWAY_MMD_LEDCH_CBLINK_LINK     0x0001
+#define  XWAY_MMD_LEDCH_CBLINK_PDOWN    0x0002
+#define  XWAY_MMD_LEDCH_CBLINK_EEE      0x0003
+#define  XWAY_MMD_LEDCH_CBLINK_ANEG     0x0004
+#define  XWAY_MMD_LEDCH_CBLINK_ABIST    0x0005
+#define  XWAY_MMD_LEDCH_CBLINK_CDIAG    0x0006
+#define  XWAY_MMD_LEDCH_CBLINK_TEST     0x0007
+/* Complex SCAN Configuration */
+#define  XWAY_MMD_LEDCH_SCAN_NONE       0x0000
+#define  XWAY_MMD_LEDCH_SCAN_LINK       0x0010
+#define  XWAY_MMD_LEDCH_SCAN_PDOWN      0x0020
+#define  XWAY_MMD_LEDCH_SCAN_EEE        0x0030
+#define  XWAY_MMD_LEDCH_SCAN_ANEG       0x0040
+#define  XWAY_MMD_LEDCH_SCAN_ABIST      0x0050
+#define  XWAY_MMD_LEDCH_SCAN_CDIAG      0x0060
+#define  XWAY_MMD_LEDCH_SCAN_TEST       0x0070
+/* Configuration for LED Pin x */
+#define XWAY_MMD_LED0H                  0x01E2
+/* Fast Blinking Configuration */
+#define  XWAY_MMD_LEDxH_BLINKF_MASK     0x000F
+#define  XWAY_MMD_LEDxH_BLINKF_NONE     0x0000
+#define  XWAY_MMD_LEDxH_BLINKF_LINK10   0x0001
+#define  XWAY_MMD_LEDxH_BLINKF_LINK100  0x0002
+#define  XWAY_MMD_LEDxH_BLINKF_LINK10X  0x0003
+#define  XWAY_MMD_LEDxH_BLINKF_LINK1000 0x0004
+#define  XWAY_MMD_LEDxH_BLINKF_LINK10_0 0x0005
+#define  XWAY_MMD_LEDxH_BLINKF_LINK100X 0x0006
+#define  XWAY_MMD_LEDxH_BLINKF_LINK10XX 0x0007
+#define  XWAY_MMD_LEDxH_BLINKF_PDOWN    0x0008
+#define  XWAY_MMD_LEDxH_BLINKF_EEE      0x0009
+#define  XWAY_MMD_LEDxH_BLINKF_ANEG     0x000A
+#define  XWAY_MMD_LEDxH_BLINKF_ABIST    0x000B
+#define  XWAY_MMD_LEDxH_BLINKF_CDIAG    0x000C
+/* Constant On Configuration */
+#define  XWAY_MMD_LEDxH_CON_MASK        0x00F0
+#define  XWAY_MMD_LEDxH_CON_NONE        0x0000
+#define  XWAY_MMD_LEDxH_CON_LINK10      0x0010
+#define  XWAY_MMD_LEDxH_CON_LINK100     0x0020
+#define  XWAY_MMD_LEDxH_CON_LINK10X     0x0030
+#define  XWAY_MMD_LEDxH_CON_LINK1000    0x0040
+#define  XWAY_MMD_LEDxH_CON_LINK10_0    0x0050
+#define  XWAY_MMD_LEDxH_CON_LINK100X    0x0060
+#define  XWAY_MMD_LEDxH_CON_LINK10XX    0x0070
+#define  XWAY_MMD_LEDxH_CON_PDOWN       0x0080
+#define  XWAY_MMD_LEDxH_CON_EEE         0x0090
+#define  XWAY_MMD_LEDxH_CON_ANEG        0x00A0
+#define  XWAY_MMD_LEDxH_CON_ABIST       0x00B0
+#define  XWAY_MMD_LEDxH_CON_CDIAG       0x00C0
+#define  XWAY_MMD_LEDxH_CON_COPPER      0x00D0
+#define  XWAY_MMD_LEDxH_CON_FIBER       0x00E0
+/* Configuration for LED Pin x */
+#define XWAY_MMD_LED0L                  0x01E3
+/* Pulsing Configuration */
+#define  XWAY_MMD_LEDxL_PULSE_MASK      0x000F
+#define  XWAY_MMD_LEDxL_PULSE_NONE      0x0000
+#define  XWAY_MMD_LEDxL_PULSE_TXACT     0x0001
+#define  XWAY_MMD_LEDxL_PULSE_RXACT     0x0002
+#define  XWAY_MMD_LEDxL_PULSE_COL       0x0004
+/* Slow Blinking Configuration */
+#define  XWAY_MMD_LEDxL_BLINKS_MASK     0x00F0
+#define  XWAY_MMD_LEDxL_BLINKS_NONE     0x0000
+#define  XWAY_MMD_LEDxL_BLINKS_LINK10   0x0010
+#define  XWAY_MMD_LEDxL_BLINKS_LINK100  0x0020
+#define  XWAY_MMD_LEDxL_BLINKS_LINK10X  0x0030
+#define  XWAY_MMD_LEDxL_BLINKS_LINK1000 0x0040
+#define  XWAY_MMD_LEDxL_BLINKS_LINK10_0 0x0050
+#define  XWAY_MMD_LEDxL_BLINKS_LINK100X 0x0060
+#define  XWAY_MMD_LEDxL_BLINKS_LINK10XX 0x0070
+#define  XWAY_MMD_LEDxL_BLINKS_PDOWN    0x0080
+#define  XWAY_MMD_LEDxL_BLINKS_EEE      0x0090
+#define  XWAY_MMD_LEDxL_BLINKS_ANEG     0x00A0
+#define  XWAY_MMD_LEDxL_BLINKS_ABIST    0x00B0
+#define  XWAY_MMD_LEDxL_BLINKS_CDIAG    0x00C0
+#define XWAY_MMD_LED1H                  0x01E4
+#define XWAY_MMD_LED1L                  0x01E5
+#define XWAY_MMD_LED2H                  0x01E6
+#define XWAY_MMD_LED2L                  0x01E7
+#define XWAY_MMD_LED3H                  0x01E8
+#define XWAY_MMD_LED3L                  0x01E9
+
+#define PHY_ID_PHY11G_1_3               0x030260D1
+#define PHY_ID_PHY22F_1_3               0x030260E1
+#define PHY_ID_PHY11G_1_4               0xD565A400
+#define PHY_ID_PHY22F_1_4               0xD565A410
+#define PHY_ID_PHY11G_1_5               0xD565A401
+#define PHY_ID_PHY22F_1_5               0xD565A411
+#define PHY_ID_PHY11G_VR9               0xD565A409
+#define PHY_ID_PHY22F_VR9               0xD565A419
+
+static int xway_gphy_config_init(struct phy_device *phydev)
+{
+        int err;
+        u32 ledxh;
+        u32 ledxl;
+
+        /* Mask all interrupts */
+        err = phy_write(phydev, XWAY_MDIO_IMASK, 0);
+        if (err)
+                return err;
+
+        /* Clear all pending interrupts */
+        phy_read(phydev, XWAY_MDIO_ISTAT);
+
+        phy_write_mmd_indirect(phydev, XWAY_MMD_LEDCH, MDIO_MMD_VEND2,
+                               XWAY_MMD_LEDCH_NACS_NONE |
+                               XWAY_MMD_LEDCH_SBF_F02HZ |
+                               XWAY_MMD_LEDCH_FBF_F16HZ);
+        phy_write_mmd_indirect(phydev, XWAY_MMD_LEDCL, MDIO_MMD_VEND2,
+                               XWAY_MMD_LEDCH_CBLINK_NONE |
+                               XWAY_MMD_LEDCH_SCAN_NONE);
+
+        /**
+         * In most cases only one LED is connected to this phy, so
+         * configure them all to constant on and pulse mode. LED3 is
+         * only available in some packages, leave it in its reset
+         * configuration.
+         */
+        ledxh = XWAY_MMD_LEDxH_BLINKF_NONE | XWAY_MMD_LEDxH_CON_LINK10XX;
+        ledxl = XWAY_MMD_LEDxL_PULSE_TXACT | XWAY_MMD_LEDxL_PULSE_RXACT |
+                XWAY_MMD_LEDxL_BLINKS_NONE;
+        phy_write_mmd_indirect(phydev, XWAY_MMD_LED0H, MDIO_MMD_VEND2, ledxh);
+        phy_write_mmd_indirect(phydev, XWAY_MMD_LED0L, MDIO_MMD_VEND2, ledxl);
+        phy_write_mmd_indirect(phydev, XWAY_MMD_LED1H, MDIO_MMD_VEND2, ledxh);
+        phy_write_mmd_indirect(phydev, XWAY_MMD_LED1L, MDIO_MMD_VEND2, ledxl);
+        phy_write_mmd_indirect(phydev, XWAY_MMD_LED2H, MDIO_MMD_VEND2, ledxh);
+        phy_write_mmd_indirect(phydev, XWAY_MMD_LED2L, MDIO_MMD_VEND2, ledxl);
+
+        return 0;
+}
+
+static int xway_gphy14_config_aneg(struct phy_device *phydev)
+{
+        int reg, err;
+
+        /* Advertise as multi-port device, see IEEE802.3-2002 40.5.1.1 */
+        /* This is a workaround for an errata in rev < 1.5 devices */
+        reg = phy_read(phydev, MII_CTRL1000);
+        reg |= ADVERTISED_MPD;
+        err = phy_write(phydev, MII_CTRL1000, reg);
+        if (err)
+                return err;
+
+        return genphy_config_aneg(phydev);
+}
+
+static int xway_gphy_ack_interrupt(struct phy_device *phydev)
+{
+        int reg;
+
+        reg = phy_read(phydev, XWAY_MDIO_ISTAT);
+        return (reg < 0) ? reg : 0;
+}
+
+static int xway_gphy_did_interrupt(struct phy_device *phydev)
+{
+        int reg;
+
+        reg = phy_read(phydev, XWAY_MDIO_ISTAT);
+        return reg & XWAY_MDIO_INIT_MASK;
+}
+
+static int xway_gphy_config_intr(struct phy_device *phydev)
+{
+        u16 mask = 0;
+
+        if (phydev->interrupts == PHY_INTERRUPT_ENABLED)
+                mask = XWAY_MDIO_INIT_MASK;
+
+        return phy_write(phydev, XWAY_MDIO_IMASK, mask);
+}
+
+static struct phy_driver xway_gphy[] = {
+        {
+                .phy_id         = PHY_ID_PHY11G_1_3,
+                .phy_id_mask    = 0xffffffff,
+                .name           = "Intel XWAY PHY11G (PEF 7071/PEF 7072) v1.3",
+                .features       = (PHY_GBIT_FEATURES | SUPPORTED_Pause |
+                                   SUPPORTED_Asym_Pause),
+                .flags          = PHY_HAS_INTERRUPT,
+                .config_init    = xway_gphy_config_init,
+                .config_aneg    = xway_gphy14_config_aneg,
+                .read_status    = genphy_read_status,
+                .ack_interrupt  = xway_gphy_ack_interrupt,
+                .did_interrupt  = xway_gphy_did_interrupt,
+                .config_intr    = xway_gphy_config_intr,
+                .suspend        = genphy_suspend,
+                .resume         = genphy_resume,
+        }, {
+                .phy_id         = PHY_ID_PHY22F_1_3,
+                .phy_id_mask    = 0xffffffff,
+                .name           = "Intel XWAY PHY22F (PEF 7061) v1.3",
+                .features       = (PHY_BASIC_FEATURES | SUPPORTED_Pause |
+                                   SUPPORTED_Asym_Pause),
+                .flags          = PHY_HAS_INTERRUPT,
+                .config_init    = xway_gphy_config_init,
+                .config_aneg    = xway_gphy14_config_aneg,
+                .read_status    = genphy_read_status,
+                .ack_interrupt  = xway_gphy_ack_interrupt,
+                .did_interrupt  = xway_gphy_did_interrupt,
+                .config_intr    = xway_gphy_config_intr,
+                .suspend        = genphy_suspend,
+                .resume         = genphy_resume,
+        }, {
+                .phy_id         = PHY_ID_PHY11G_1_4,
+                .phy_id_mask    = 0xffffffff,
+                .name           = "Intel XWAY PHY11G (PEF 7071/PEF 7072) v1.4",
+                .features       = (PHY_GBIT_FEATURES | SUPPORTED_Pause |
+                                   SUPPORTED_Asym_Pause),
+                .flags          = PHY_HAS_INTERRUPT,
+                .config_init    = xway_gphy_config_init,
+                .config_aneg    = xway_gphy14_config_aneg,
+                .read_status    = genphy_read_status,
+                .ack_interrupt  = xway_gphy_ack_interrupt,
+                .did_interrupt  = xway_gphy_did_interrupt,
+                .config_intr    = xway_gphy_config_intr,
+                .suspend        = genphy_suspend,
+                .resume         = genphy_resume,
+        }, {
+                .phy_id         = PHY_ID_PHY22F_1_4,
+                .phy_id_mask    = 0xffffffff,
+                .name           = "Intel XWAY PHY22F (PEF 7061) v1.4",
+                .features       = (PHY_BASIC_FEATURES | SUPPORTED_Pause |
+                                   SUPPORTED_Asym_Pause),
+                .flags          = PHY_HAS_INTERRUPT,
+                .config_init    = xway_gphy_config_init,
+                .config_aneg    = xway_gphy14_config_aneg,
+                .read_status    = genphy_read_status,
+                .ack_interrupt  = xway_gphy_ack_interrupt,
+                .did_interrupt  = xway_gphy_did_interrupt,
+                .config_intr    = xway_gphy_config_intr,
+                .suspend        = genphy_suspend,
+                .resume         = genphy_resume,
+        }, {
+                .phy_id         = PHY_ID_PHY11G_1_5,
+                .phy_id_mask    = 0xffffffff,
+                .name           = "Intel XWAY PHY11G (PEF 7071/PEF 7072) v1.5 / v1.6",
+                .features       = (PHY_GBIT_FEATURES | SUPPORTED_Pause |
+                                   SUPPORTED_Asym_Pause),
+                .flags          = PHY_HAS_INTERRUPT,
+                .config_init    = xway_gphy_config_init,
+                .config_aneg    = genphy_config_aneg,
+                .read_status    = genphy_read_status,
+                .ack_interrupt  = xway_gphy_ack_interrupt,
+                .did_interrupt  = xway_gphy_did_interrupt,
+                .config_intr    = xway_gphy_config_intr,
+                .suspend        = genphy_suspend,
+                .resume         = genphy_resume,
+        }, {
+                .phy_id         = PHY_ID_PHY22F_1_5,
+                .phy_id_mask    = 0xffffffff,
+                .name           = "Intel XWAY PHY22F (PEF 7061) v1.5 / v1.6",
+                .features       = (PHY_BASIC_FEATURES | SUPPORTED_Pause |
+                                   SUPPORTED_Asym_Pause),
+                .flags          = PHY_HAS_INTERRUPT,
+                .config_init    = xway_gphy_config_init,
+                .config_aneg    = genphy_config_aneg,
+                .read_status    = genphy_read_status,
+                .ack_interrupt  = xway_gphy_ack_interrupt,
+                .did_interrupt  = xway_gphy_did_interrupt,
+                .config_intr    = xway_gphy_config_intr,
+                .suspend        = genphy_suspend,
+                .resume         = genphy_resume,
+        }, {
+                .phy_id         = PHY_ID_PHY11G_VR9,
+                .phy_id_mask    = 0xffffffff,
+                .name           = "Intel XWAY PHY11G (xRX integrated)",
+                .features       = (PHY_GBIT_FEATURES | SUPPORTED_Pause |
+                                   SUPPORTED_Asym_Pause),
+                .flags          = PHY_HAS_INTERRUPT,
+                .config_init    = xway_gphy_config_init,
+                .config_aneg    = genphy_config_aneg,
+                .read_status    = genphy_read_status,
+                .ack_interrupt  = xway_gphy_ack_interrupt,
+                .did_interrupt  = xway_gphy_did_interrupt,
+                .config_intr    = xway_gphy_config_intr,
+                .suspend        = genphy_suspend,
+                .resume         = genphy_resume,
+        }, {
+                .phy_id         = PHY_ID_PHY22F_VR9,
+                .phy_id_mask    = 0xffffffff,
+                .name           = "Intel XWAY PHY22F (xRX integrated)",
+                .features       = (PHY_BASIC_FEATURES | SUPPORTED_Pause |
+                                   SUPPORTED_Asym_Pause),
+                .flags          = PHY_HAS_INTERRUPT,
+                .config_init    = xway_gphy_config_init,
+                .config_aneg    = genphy_config_aneg,
+                .read_status    = genphy_read_status,
+                .ack_interrupt  = xway_gphy_ack_interrupt,
+                .did_interrupt  = xway_gphy_did_interrupt,
+                .config_intr    = xway_gphy_config_intr,
+                .suspend        = genphy_suspend,
+                .resume         = genphy_resume,
+        },
+};
+module_phy_driver(xway_gphy);
+
+static struct mdio_device_id __maybe_unused xway_gphy_tbl[] = {
+        { PHY_ID_PHY11G_1_3, 0xffffffff },
+        { PHY_ID_PHY22F_1_3, 0xffffffff },
+        { PHY_ID_PHY11G_1_4, 0xffffffff },
+        { PHY_ID_PHY22F_1_4, 0xffffffff },
+        { PHY_ID_PHY11G_1_5, 0xffffffff },
+        { PHY_ID_PHY22F_1_5, 0xffffffff },
+        { PHY_ID_PHY11G_VR9, 0xffffffff },
+        { PHY_ID_PHY22F_VR9, 0xffffffff },
+        { }
+};
+MODULE_DEVICE_TABLE(mdio, xway_gphy_tbl);
+
+MODULE_DESCRIPTION("Intel XWAY PHY driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/phy/mdio-mux-bcm-iproc.c b/drivers/net/phy/mdio-mux-bcm-iproc.c
new file mode 100644
index 000000000000..0a0412524cec
--- /dev/null
+++ b/drivers/net/phy/mdio-mux-bcm-iproc.c
@@ -0,0 +1,248 @@
+/*
+ * Copyright 2016 Broadcom
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation (the "GPL").
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 (GPLv2) for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 (GPLv2) along with this source code.
+ */
+
+#include <linux/platform_device.h>
+#include <linux/device.h>
+#include <linux/of_mdio.h>
+#include <linux/module.h>
+#include <linux/phy.h>
+#include <linux/mdio-mux.h>
+#include <linux/delay.h>
+
+#define MDIO_PARAM_OFFSET               0x00
+#define MDIO_PARAM_MIIM_CYCLE           29
+#define MDIO_PARAM_INTERNAL_SEL         25
+#define MDIO_PARAM_BUS_ID               22
+#define MDIO_PARAM_C45_SEL              21
+#define MDIO_PARAM_PHY_ID               16
+#define MDIO_PARAM_PHY_DATA             0
+
+#define MDIO_READ_OFFSET                0x04
+#define MDIO_READ_DATA_MASK             0xffff
+#define MDIO_ADDR_OFFSET                0x08
+
+#define MDIO_CTRL_OFFSET                0x0C
+#define MDIO_CTRL_WRITE_OP              0x1
+#define MDIO_CTRL_READ_OP               0x2
+
+#define MDIO_STAT_OFFSET                0x10
+#define MDIO_STAT_DONE                  1
+
+#define BUS_MAX_ADDR                    32
+#define EXT_BUS_START_ADDR              16
+
+struct iproc_mdiomux_desc {
+        void *mux_handle;
+        void __iomem *base;
+        struct device *dev;
+        struct mii_bus *mii_bus;
+};
+
+static int iproc_mdio_wait_for_idle(void __iomem *base, bool result)
+{
+        unsigned int timeout = 1000; /* loop for 1s */
+        u32 val;
+
+        do {
+                val = readl(base + MDIO_STAT_OFFSET);
+                if ((val & MDIO_STAT_DONE) == result)
+                        return 0;
+
+                usleep_range(1000, 2000);
+        } while (timeout--);
+
+        return -ETIMEDOUT;
+}
+
+/* start_miim_ops- Program and start MDIO transaction over mdio bus.
+ * @base: Base address
+ * @phyid: phyid of the selected bus.
+ * @reg: register offset to be read/written.
+ * @val :0 if read op else value to be written in @reg;
+ * @op: Operation that need to be carried out.
+ *      MDIO_CTRL_READ_OP: Read transaction.
+ *      MDIO_CTRL_WRITE_OP: Write transaction.
+ *
+ * Return value: Successful Read operation returns read reg values and write
+ *      operation returns 0. Failure operation returns negative error code.
+ */
+static int start_miim_ops(void __iomem *base,
+                          u16 phyid, u32 reg, u16 val, u32 op)
+{
+        u32 param;
+        int ret;
+
+        writel(0, base + MDIO_CTRL_OFFSET);
+        ret = iproc_mdio_wait_for_idle(base, 0);
+        if (ret)
+                goto err;
+
+        param = readl(base + MDIO_PARAM_OFFSET);
+        param |= phyid << MDIO_PARAM_PHY_ID;
+        param |= val << MDIO_PARAM_PHY_DATA;
+        if (reg & MII_ADDR_C45)
+                param |= BIT(MDIO_PARAM_C45_SEL);
+
+        writel(param, base + MDIO_PARAM_OFFSET);
+
+        writel(reg, base + MDIO_ADDR_OFFSET);
+
+        writel(op, base + MDIO_CTRL_OFFSET);
+
+        ret = iproc_mdio_wait_for_idle(base, 1);
+        if (ret)
+                goto err;
+
+        if (op == MDIO_CTRL_READ_OP)
+                ret = readl(base + MDIO_READ_OFFSET) & MDIO_READ_DATA_MASK;
+err:
+        return ret;
+}
+
+static int iproc_mdiomux_read(struct mii_bus *bus, int phyid, int reg)
+{
+        struct iproc_mdiomux_desc *md = bus->priv;
+        int ret;
+
+        ret = start_miim_ops(md->base, phyid, reg, 0, MDIO_CTRL_READ_OP);
+        if (ret < 0)
+                dev_err(&bus->dev, "mdiomux read operation failed!!!");
+
+        return ret;
+}
+
+static int iproc_mdiomux_write(struct mii_bus *bus,
+                               int phyid, int reg, u16 val)
+{
+        struct iproc_mdiomux_desc *md = bus->priv;
+        int ret;
+
+        /* Write val at reg offset */
+        ret = start_miim_ops(md->base, phyid, reg, val, MDIO_CTRL_WRITE_OP);
+        if (ret < 0)
+                dev_err(&bus->dev, "mdiomux write operation failed!!!");
+
+        return ret;
+}
+
+static int mdio_mux_iproc_switch_fn(int current_child, int desired_child,
+                                    void *data)
+{
+        struct iproc_mdiomux_desc *md = data;
+        u32 param, bus_id;
+        bool bus_dir;
+
+        /* select bus and its properties */
+        bus_dir = (desired_child < EXT_BUS_START_ADDR);
+        bus_id = bus_dir ? desired_child : (desired_child - EXT_BUS_START_ADDR);
+
+        param = (bus_dir ? 1 : 0) << MDIO_PARAM_INTERNAL_SEL;
+        param |= (bus_id << MDIO_PARAM_BUS_ID);
+
+        writel(param, md->base + MDIO_PARAM_OFFSET);
+        return 0;
+}
+
+static int mdio_mux_iproc_probe(struct platform_device *pdev)
+{
+        struct iproc_mdiomux_desc *md;
+        struct mii_bus *bus;
+        struct resource *res;
+        int rc;
+
+        md = devm_kzalloc(&pdev->dev, sizeof(*md), GFP_KERNEL);
+        if (!md)
+                return -ENOMEM;
+        md->dev = &pdev->dev;
+
+        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+        md->base = devm_ioremap_resource(&pdev->dev, res);
+        if (IS_ERR(md->base)) {
+                dev_err(&pdev->dev, "failed to ioremap register\n");
+                return PTR_ERR(md->base);
+        }
+
+        md->mii_bus = mdiobus_alloc();
+        if (!md->mii_bus) {
+                dev_err(&pdev->dev, "mdiomux bus alloc failed\n");
+                return -ENOMEM;
+        }
+
+        bus = md->mii_bus;
+        bus->priv = md;
+        bus->name = "iProc MDIO mux bus";
+        snprintf(bus->id, MII_BUS_ID_SIZE, "%s-%d", pdev->name, pdev->id);
+        bus->parent = &pdev->dev;
+        bus->read = iproc_mdiomux_read;
+        bus->write = iproc_mdiomux_write;
+
+        bus->phy_mask = ~0;
+        bus->dev.of_node = pdev->dev.of_node;
+        rc = mdiobus_register(bus);
+        if (rc) {
+                dev_err(&pdev->dev, "mdiomux registration failed\n");
+                goto out;
+        }
+
+        platform_set_drvdata(pdev, md);
+
+        rc = mdio_mux_init(md->dev, mdio_mux_iproc_switch_fn,
+                           &md->mux_handle, md, md->mii_bus);
+        if (rc) {
+                dev_info(md->dev, "mdiomux initialization failed\n");
+                goto out;
+        }
+
+        dev_info(md->dev, "iProc mdiomux registered\n");
+        return 0;
+out:
+        mdiobus_free(bus);
+        return rc;
+}
+
+static int mdio_mux_iproc_remove(struct platform_device *pdev)
+{
+        struct iproc_mdiomux_desc *md = dev_get_platdata(&pdev->dev);
+
+        mdio_mux_uninit(md->mux_handle);
+        mdiobus_unregister(md->mii_bus);
+        mdiobus_free(md->mii_bus);
+
+        return 0;
+}
+
+static const struct of_device_id mdio_mux_iproc_match[] = {
+        {
+                .compatible = "brcm,mdio-mux-iproc",
+        },
+        {},
+};
+MODULE_DEVICE_TABLE(of, mdio_mux_iproc_match);
+
+static struct platform_driver mdiomux_iproc_driver = {
+        .driver = {
+                .name           = "mdio-mux-iproc",
+                .of_match_table = mdio_mux_iproc_match,
+        },
+        .probe          = mdio_mux_iproc_probe,
+        .remove         = mdio_mux_iproc_remove,
+};
+
+module_platform_driver(mdiomux_iproc_driver);
+
+MODULE_DESCRIPTION("iProc MDIO Mux Bus Driver");
+MODULE_AUTHOR("Pramod Kumar <pramod.kumar@broadcom.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/net/phy/mdio-mux-gpio.c b/drivers/net/phy/mdio-mux-gpio.c
index 7ddb1ab70891..919949960a10 100644
--- a/drivers/net/phy/mdio-mux-gpio.c
+++ b/drivers/net/phy/mdio-mux-gpio.c
@@ -55,7 +55,7 @@ static int mdio_mux_gpio_probe(struct platform_device *pdev)
                 return PTR_ERR(s->gpios);
 
         r = mdio_mux_init(&pdev->dev,
-                          mdio_mux_gpio_switch_fn, &s->mux_handle, s);
+                          mdio_mux_gpio_switch_fn, &s->mux_handle, s, NULL);
 
         if (r != 0) {
                 gpiod_put_array(s->gpios);
diff --git a/drivers/net/phy/mdio-mux-mmioreg.c b/drivers/net/phy/mdio-mux-mmioreg.c
index 7fde454fbc4f..d0bed52c8d16 100644
--- a/drivers/net/phy/mdio-mux-mmioreg.c
+++ b/drivers/net/phy/mdio-mux-mmioreg.c
@@ -126,7 +126,7 @@ static int mdio_mux_mmioreg_probe(struct platform_device *pdev)
         }
 
         ret = mdio_mux_init(&pdev->dev, mdio_mux_mmioreg_switch_fn,
-                            &s->mux_handle, s);
+                            &s->mux_handle, s, NULL);
         if (ret) {
                 dev_err(&pdev->dev, "failed to register mdio-mux bus %s\n",
                         np->full_name);
diff --git a/drivers/net/phy/mdio-mux.c b/drivers/net/phy/mdio-mux.c
index 5c81d6faf304..963838d4fac1 100644
--- a/drivers/net/phy/mdio-mux.c
+++ b/drivers/net/phy/mdio-mux.c
@@ -89,7 +89,8 @@ static int parent_count;
 int mdio_mux_init(struct device *dev,
                   int (*switch_fn)(int cur, int desired, void *data),
                   void **mux_handle,
-                  void *data)
+                  void *data,
+                  struct mii_bus *mux_bus)
 {
         struct device_node *parent_bus_node;
         struct device_node *child_bus_node;
@@ -101,10 +102,22 @@ int mdio_mux_init(struct device *dev,
         if (!dev->of_node)
                 return -ENODEV;
 
-        parent_bus_node = of_parse_phandle(dev->of_node, "mdio-parent-bus", 0);
+        if (!mux_bus) {
+                parent_bus_node = of_parse_phandle(dev->of_node,
+                                                   "mdio-parent-bus", 0);
 
-        if (!parent_bus_node)
-                return -ENODEV;
+                if (!parent_bus_node)
+                        return -ENODEV;
+
+                parent_bus = of_mdio_find_bus(parent_bus_node);
+                if (!parent_bus) {
+                        ret_val = -EPROBE_DEFER;
+                        goto err_parent_bus;
+                }
+        } else {
+                parent_bus_node = NULL;
+                parent_bus = mux_bus;
+        }
 
         pb = devm_kzalloc(dev, sizeof(*pb), GFP_KERNEL);
         if (pb == NULL) {
@@ -112,11 +125,6 @@ int mdio_mux_init(struct device *dev,
                 goto err_parent_bus;
         }
 
-        parent_bus = of_mdio_find_bus(parent_bus_node);
-        if (parent_bus == NULL) {
-                ret_val = -EPROBE_DEFER;
-                goto err_parent_bus;
-        }
 
         pb->switch_data = data;
         pb->switch_fn = switch_fn;
diff --git a/drivers/net/phy/swphy.c b/drivers/net/phy/swphy.c
new file mode 100644
index 000000000000..34f58f2349e9
--- /dev/null
+++ b/drivers/net/phy/swphy.c
@@ -0,0 +1,179 @@
+/*
+ * Software PHY emulation
+ *
+ * Code taken from fixed_phy.c by Russell King <rmk+kernel@arm.linux.org.uk>
+ *
+ * Author: Vitaly Bordug <vbordug@ru.mvista.com>
+ *         Anton Vorontsov <avorontsov@ru.mvista.com>
+ *
+ * Copyright (c) 2006-2007 MontaVista Software, Inc.
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+#include <linux/export.h>
+#include <linux/mii.h>
+#include <linux/phy.h>
+#include <linux/phy_fixed.h>
+
+#include "swphy.h"
+
+#define MII_REGS_NUM 29
+
+struct swmii_regs {
+        u16 bmcr;
+        u16 bmsr;
+        u16 lpa;
+        u16 lpagb;
+};
+
+enum {
+        SWMII_SPEED_10 = 0,
+        SWMII_SPEED_100,
+        SWMII_SPEED_1000,
+        SWMII_DUPLEX_HALF = 0,
+        SWMII_DUPLEX_FULL,
+};
+
+/*
+ * These two tables get bitwise-anded together to produce the final result.
+ * This means the speed table must contain both duplex settings, and the
+ * duplex table must contain all speed settings.
+ */
+static const struct swmii_regs speed[] = {
+        [SWMII_SPEED_10] = {
+                .bmcr  = BMCR_FULLDPLX,
+                .lpa   = LPA_10FULL | LPA_10HALF,
+        },
+        [SWMII_SPEED_100] = {
+                .bmcr  = BMCR_FULLDPLX | BMCR_SPEED100,
+                .bmsr  = BMSR_100FULL | BMSR_100HALF,
+                .lpa   = LPA_100FULL | LPA_100HALF,
+        },
+        [SWMII_SPEED_1000] = {
+                .bmcr  = BMCR_FULLDPLX | BMCR_SPEED1000,
+                .bmsr  = BMSR_ESTATEN,
+                .lpagb = LPA_1000FULL | LPA_1000HALF,
+        },
+};
+
+static const struct swmii_regs duplex[] = {
+        [SWMII_DUPLEX_HALF] = {
+                .bmcr  = ~BMCR_FULLDPLX,
+                .bmsr  = BMSR_ESTATEN | BMSR_100HALF,
+                .lpa   = LPA_10HALF | LPA_100HALF,
+                .lpagb = LPA_1000HALF,
+        },
+        [SWMII_DUPLEX_FULL] = {
+                .bmcr  = ~0,
+                .bmsr  = BMSR_ESTATEN | BMSR_100FULL,
+                .lpa   = LPA_10FULL | LPA_100FULL,
+                .lpagb = LPA_1000FULL,
+        },
+};
+
+static int swphy_decode_speed(int speed)
+{
+        switch (speed) {
+        case 1000:
+                return SWMII_SPEED_1000;
+        case 100:
+                return SWMII_SPEED_100;
+        case 10:
+                return SWMII_SPEED_10;
+        default:
+                return -EINVAL;
+        }
+}
+
+/**
+ * swphy_validate_state - validate the software phy status
+ * @state: software phy status
+ *
+ * This checks that we can represent the state stored in @state can be
+ * represented in the emulated MII registers.  Returns 0 if it can,
+ * otherwise returns -EINVAL.
+ */
+int swphy_validate_state(const struct fixed_phy_status *state)
+{
+        int err;
+
+        if (state->link) {
+                err = swphy_decode_speed(state->speed);
+                if (err < 0) {
+                        pr_warn("swphy: unknown speed\n");
+                        return -EINVAL;
+                }
+        }
+        return 0;
+}
+EXPORT_SYMBOL_GPL(swphy_validate_state);
+
+/**
+ * swphy_read_reg - return a MII register from the fixed phy state
+ * @reg: MII register
+ * @state: fixed phy status
+ *
+ * Return the MII @reg register generated from the fixed phy state @state.
+ */
+int swphy_read_reg(int reg, const struct fixed_phy_status *state)
+{
+        int speed_index, duplex_index;
+        u16 bmsr = BMSR_ANEGCAPABLE;
+        u16 bmcr = 0;
+        u16 lpagb = 0;
+        u16 lpa = 0;
+
+        if (reg > MII_REGS_NUM)
+                return -1;
+
+        speed_index = swphy_decode_speed(state->speed);
+        if (WARN_ON(speed_index < 0))
+                return 0;
+
+        duplex_index = state->duplex ? SWMII_DUPLEX_FULL : SWMII_DUPLEX_HALF;
+
+        bmsr |= speed[speed_index].bmsr & duplex[duplex_index].bmsr;
+
+        if (state->link) {
+                bmsr |= BMSR_LSTATUS | BMSR_ANEGCOMPLETE;
+
+                bmcr  |= speed[speed_index].bmcr  & duplex[duplex_index].bmcr;
+                lpa   |= speed[speed_index].lpa   & duplex[duplex_index].lpa;
+                lpagb |= speed[speed_index].lpagb & duplex[duplex_index].lpagb;
+
+                if (state->pause)
+                        lpa |= LPA_PAUSE_CAP;
+
+                if (state->asym_pause)
+                        lpa |= LPA_PAUSE_ASYM;
+        }
+
+        switch (reg) {
+        case MII_BMCR:
+                return bmcr;
+        case MII_BMSR:
+                return bmsr;
+        case MII_PHYSID1:
+        case MII_PHYSID2:
+                return 0;
+        case MII_LPA:
+                return lpa;
+        case MII_STAT1000:
+                return lpagb;
+
+        /*
+         * We do not support emulating Clause 45 over Clause 22 register
+         * reads.  Return an error instead of bogus data.
+         */
+        case MII_MMD_CTRL:
+        case MII_MMD_DATA:
+                return -1;
+
+        default:
+                return 0xffff;
+        }
+}
+EXPORT_SYMBOL_GPL(swphy_read_reg);
diff --git a/drivers/net/phy/swphy.h b/drivers/net/phy/swphy.h
new file mode 100644
index 000000000000..2f09ac324e18
--- /dev/null
+++ b/drivers/net/phy/swphy.h
@@ -0,0 +1,9 @@
+#ifndef SWPHY_H
+#define SWPHY_H
+
+struct fixed_phy_status;
+
+int swphy_validate_state(const struct fixed_phy_status *state);
+int swphy_read_reg(int reg, const struct fixed_phy_status *state);
+
+#endif
diff --git a/drivers/net/ppp/ppp_generic.c b/drivers/net/ppp/ppp_generic.c
index 8dedafa1a95d..17953ab15000 100644
--- a/drivers/net/ppp/ppp_generic.c
+++ b/drivers/net/ppp/ppp_generic.c
@@ -1312,10 +1312,9 @@ ppp_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats64)
         return stats64;
 }
 
-static struct lock_class_key ppp_tx_busylock;
 static int ppp_dev_init(struct net_device *dev)
 {
-        dev->qdisc_tx_busylock = &ppp_tx_busylock;
+        netdev_lockdep_set_classes(dev);
         return 0;
 }
 
diff --git a/drivers/net/team/team.c b/drivers/net/team/team.c
index fdee77207323..f9eebea83516 100644
--- a/drivers/net/team/team.c
+++ b/drivers/net/team/team.c
@@ -1576,23 +1576,6 @@ static const struct team_option team_options[] = {
         },
 };
 
-static struct lock_class_key team_netdev_xmit_lock_key;
-static struct lock_class_key team_netdev_addr_lock_key;
-static struct lock_class_key team_tx_busylock_key;
-
-static void team_set_lockdep_class_one(struct net_device *dev,
-                                       struct netdev_queue *txq,
-                                       void *unused)
-{
-        lockdep_set_class(&txq->_xmit_lock, &team_netdev_xmit_lock_key);
-}
-
-static void team_set_lockdep_class(struct net_device *dev)
-{
-        lockdep_set_class(&dev->addr_list_lock, &team_netdev_addr_lock_key);
-        netdev_for_each_tx_queue(dev, team_set_lockdep_class_one, NULL);
-        dev->qdisc_tx_busylock = &team_tx_busylock_key;
-}
 
 static int team_init(struct net_device *dev)
 {
@@ -1628,7 +1611,7 @@ static int team_init(struct net_device *dev)
                 goto err_options_register;
         netif_carrier_off(dev);
 
-        team_set_lockdep_class(dev);
+        netdev_lockdep_set_classes(dev);
 
         return 0;
 
diff --git a/drivers/net/tun.c b/drivers/net/tun.c
index e16487cc6a9a..4884802e0af1 100644
--- a/drivers/net/tun.c
+++ b/drivers/net/tun.c
@@ -1254,13 +1254,11 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
                 return -EFAULT;
         }
 
-        if (gso.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
-                if (!skb_partial_csum_set(skb, tun16_to_cpu(tun, gso.csum_start),
-                                          tun16_to_cpu(tun, gso.csum_offset))) {
-                        this_cpu_inc(tun->pcpu_stats->rx_frame_errors);
-                        kfree_skb(skb);
-                        return -EINVAL;
-                }
+        err = virtio_net_hdr_to_skb(skb, &gso, tun_is_little_endian(tun));
+        if (err) {
+                this_cpu_inc(tun->pcpu_stats->rx_frame_errors);
+                kfree_skb(skb);
+                return -EINVAL;
         }
 
         switch (tun->flags & TUN_TYPE_MASK) {
@@ -1289,39 +1287,6 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
                 break;
         }
 
-        if (gso.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
-                pr_debug("GSO!\n");
-                switch (gso.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
-                case VIRTIO_NET_HDR_GSO_TCPV4:
-                        skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
-                        break;
-                case VIRTIO_NET_HDR_GSO_TCPV6:
-                        skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
-                        break;
-                case VIRTIO_NET_HDR_GSO_UDP:
-                        skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
-                        break;
-                default:
-                        this_cpu_inc(tun->pcpu_stats->rx_frame_errors);
-                        kfree_skb(skb);
-                        return -EINVAL;
-                }
-
-                if (gso.gso_type & VIRTIO_NET_HDR_GSO_ECN)
-                        skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
-
-                skb_shinfo(skb)->gso_size = tun16_to_cpu(tun, gso.gso_size);
-                if (skb_shinfo(skb)->gso_size == 0) {
-                        this_cpu_inc(tun->pcpu_stats->rx_frame_errors);
-                        kfree_skb(skb);
-                        return -EINVAL;
-                }
-
-                /* Header must be checked, and gso_segs computed. */
-                skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
-                skb_shinfo(skb)->gso_segs = 0;
-        }
-
         /* copy skb_ubuf_info for callback when skb has no error */
         if (zerocopy) {
                 skb_shinfo(skb)->destructor_arg = msg_control;
@@ -1399,46 +1364,26 @@ static ssize_t tun_put_user(struct tun_struct *tun,
 
         if (vnet_hdr_sz) {
                 struct virtio_net_hdr gso = { 0 }; /* no info leak */
+                int ret;
+
                 if (iov_iter_count(iter) < vnet_hdr_sz)
                         return -EINVAL;
 
-                if (skb_is_gso(skb)) {
+                ret = virtio_net_hdr_from_skb(skb, &gso,
+                                              tun_is_little_endian(tun));
+                if (ret) {
                         struct skb_shared_info *sinfo = skb_shinfo(skb);
-
-                        /* This is a hint as to how much should be linear. */
-                        gso.hdr_len = cpu_to_tun16(tun, skb_headlen(skb));
-                        gso.gso_size = cpu_to_tun16(tun, sinfo->gso_size);
-                        if (sinfo->gso_type & SKB_GSO_TCPV4)
-                                gso.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
-                        else if (sinfo->gso_type & SKB_GSO_TCPV6)
-                                gso.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
-                        else if (sinfo->gso_type & SKB_GSO_UDP)
-                                gso.gso_type = VIRTIO_NET_HDR_GSO_UDP;
-                        else {
-                                pr_err("unexpected GSO type: "
-                                       "0x%x, gso_size %d, hdr_len %d\n",
-                                       sinfo->gso_type, tun16_to_cpu(tun, gso.gso_size),
-                                       tun16_to_cpu(tun, gso.hdr_len));
-                                print_hex_dump(KERN_ERR, "tun: ",
-                                               DUMP_PREFIX_NONE,
-                                               16, 1, skb->head,
-                                               min((int)tun16_to_cpu(tun, gso.hdr_len), 64), true);
-                                WARN_ON_ONCE(1);
-                                return -EINVAL;
-                        }
-                        if (sinfo->gso_type & SKB_GSO_TCP_ECN)
-                                gso.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
-                } else
-                        gso.gso_type = VIRTIO_NET_HDR_GSO_NONE;
-
-                if (skb->ip_summed == CHECKSUM_PARTIAL) {
-                        gso.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
-                        gso.csum_start = cpu_to_tun16(tun, skb_checksum_start_offset(skb) +
-                                                      vlan_hlen);
-                        gso.csum_offset = cpu_to_tun16(tun, skb->csum_offset);
-                } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
-                        gso.flags = VIRTIO_NET_HDR_F_DATA_VALID;
-                } /* else everything is zero */
+                        pr_err("unexpected GSO type: "
+                               "0x%x, gso_size %d, hdr_len %d\n",
+                               sinfo->gso_type, tun16_to_cpu(tun, gso.gso_size),
+                               tun16_to_cpu(tun, gso.hdr_len));
+                        print_hex_dump(KERN_ERR, "tun: ",
+                                       DUMP_PREFIX_NONE,
+                                       16, 1, skb->head,
+                                       min((int)tun16_to_cpu(tun, gso.hdr_len), 64), true);
+                        WARN_ON_ONCE(1);
+                        return -EINVAL;
+                }
 
                 if (copy_to_iter(&gso, sizeof(gso), iter) != sizeof(gso))
                         return -EFAULT;
diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c
index 4e257b8d8f3e..24d367280ecf 100644
--- a/drivers/net/usb/r8152.c
+++ b/drivers/net/usb/r8152.c
@@ -607,7 +607,7 @@ struct r8152 {
         struct list_head rx_done, tx_free;
         struct sk_buff_head tx_queue, rx_queue;
         spinlock_t rx_lock, tx_lock;
-        struct delayed_work schedule;
+        struct delayed_work schedule, hw_phy_work;
         struct mii_if_info mii;
         struct mutex control;   /* use for hw setting */
 #ifdef CONFIG_PM_SLEEP
@@ -624,6 +624,7 @@ struct r8152 {
                 int (*eee_get)(struct r8152 *, struct ethtool_eee *);
                 int (*eee_set)(struct r8152 *, struct ethtool_eee *);
                 bool (*in_nway)(struct r8152 *);
+                void (*hw_phy_cfg)(struct r8152 *);
         } rtl_ops;
 
         int intr_interval;
@@ -632,8 +633,11 @@ struct r8152 {
         u32 tx_qlen;
         u32 coalesce;
         u16 ocp_base;
+        u16 speed;
         u8 *intr_buff;
         u8 version;
+        u8 duplex;
+        u8 autoneg;
 };
 
 enum rtl_version {
@@ -1747,7 +1751,7 @@ static int rx_bottom(struct r8152 *tp, int budget)
                         pkt_len -= CRC_SIZE;
                         rx_data += sizeof(struct rx_desc);
 
-                        skb = netdev_alloc_skb_ip_align(netdev, pkt_len);
+                        skb = napi_alloc_skb(&tp->napi, pkt_len);
                         if (!skb) {
                                 stats->rx_dropped++;
                                 goto find_next_rx;
@@ -2515,8 +2519,6 @@ static void r8152b_exit_oob(struct r8152 *tp)
 
         rxdy_gated_en(tp, true);
         r8153_teredo_off(tp);
-        r8152b_hw_phy_cfg(tp);
-
         ocp_write_byte(tp, MCU_TYPE_PLA, PLA_CRWECR, CRWECR_NORAML);
         ocp_write_byte(tp, MCU_TYPE_PLA, PLA_CR, 0x00);
 
@@ -2694,8 +2696,6 @@ static void r8153_first_init(struct r8152 *tp)
         ocp_data &= ~RCR_ACPT_ALL;
         ocp_write_dword(tp, MCU_TYPE_PLA, PLA_RCR, ocp_data);
 
-        r8153_hw_phy_cfg(tp);
-
         rtl8152_nic_reset(tp);
         rtl_reset_bmu(tp);
 
@@ -2891,7 +2891,7 @@ static int rtl8152_set_speed(struct r8152 *tp, u8 autoneg, u16 speed, u8 duplex)
                 bmcr = BMCR_ANENABLE | BMCR_ANRESTART;
         }
 
-        if (test_bit(PHY_RESET, &tp->flags))
+        if (test_and_clear_bit(PHY_RESET, &tp->flags))
                 bmcr |= BMCR_RESET;
 
         if (tp->mii.supports_gmii)
@@ -2900,7 +2900,7 @@ static int rtl8152_set_speed(struct r8152 *tp, u8 autoneg, u16 speed, u8 duplex)
         r8152_mdio_write(tp, MII_ADVERTISE, anar);
         r8152_mdio_write(tp, MII_BMCR, bmcr);
 
-        if (test_and_clear_bit(PHY_RESET, &tp->flags)) {
+        if (bmcr & BMCR_RESET) {
                 int i;
 
                 for (i = 0; i < 50; i++) {
@@ -3059,6 +3059,27 @@ out1:
         usb_autopm_put_interface(tp->intf);
 }
 
+static void rtl_hw_phy_work_func_t(struct work_struct *work)
+{
+        struct r8152 *tp = container_of(work, struct r8152, hw_phy_work.work);
+
+        if (test_bit(RTL8152_UNPLUG, &tp->flags))
+                return;
+
+        if (usb_autopm_get_interface(tp->intf) < 0)
+                return;
+
+        mutex_lock(&tp->control);
+
+        tp->rtl_ops.hw_phy_cfg(tp);
+
+        rtl8152_set_speed(tp, tp->autoneg, tp->speed, tp->duplex);
+
+        mutex_unlock(&tp->control);
+
+        usb_autopm_put_interface(tp->intf);
+}
+
 #ifdef CONFIG_PM_SLEEP
 static int rtl_notifier(struct notifier_block *nb, unsigned long action,
                         void *data)
@@ -3107,9 +3128,6 @@ static int rtl8152_open(struct net_device *netdev)
 
         tp->rtl_ops.up(tp);
 
-        rtl8152_set_speed(tp, AUTONEG_ENABLE,
-                          tp->mii.supports_gmii ? SPEED_1000 : SPEED_100,
-                          DUPLEX_FULL);
         netif_carrier_off(netdev);
         netif_start_queue(netdev);
         set_bit(WORK_ENABLE, &tp->flags);
@@ -3533,6 +3551,7 @@ static int rtl8152_resume(struct usb_interface *intf)
 
         if (!test_bit(SELECTIVE_SUSPEND, &tp->flags)) {
                 tp->rtl_ops.init(tp);
+                queue_delayed_work(system_long_wq, &tp->hw_phy_work, 0);
                 netif_device_attach(tp->netdev);
         }
 
@@ -3547,10 +3566,6 @@ static int rtl8152_resume(struct usb_interface *intf)
                         napi_enable(&tp->napi);
                 } else {
                         tp->rtl_ops.up(tp);
-                        rtl8152_set_speed(tp, AUTONEG_ENABLE,
-                                          tp->mii.supports_gmii ?
-                                          SPEED_1000 : SPEED_100,
-                                          DUPLEX_FULL);
                         netif_carrier_off(tp->netdev);
                         set_bit(WORK_ENABLE, &tp->flags);
                 }
@@ -3680,6 +3695,11 @@ static int rtl8152_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
         mutex_lock(&tp->control);
 
         ret = rtl8152_set_speed(tp, cmd->autoneg, cmd->speed, cmd->duplex);
+        if (!ret) {
+                tp->autoneg = cmd->autoneg;
+                tp->speed = cmd->speed;
+                tp->duplex = cmd->duplex;
+        }
 
         mutex_unlock(&tp->control);
 
@@ -4137,6 +4157,7 @@ static int rtl_ops_init(struct r8152 *tp)
                 ops->eee_get            = r8152_get_eee;
                 ops->eee_set            = r8152_set_eee;
                 ops->in_nway            = rtl8152_in_nway;
+                ops->hw_phy_cfg         = r8152b_hw_phy_cfg;
                 break;
 
         case RTL_VER_03:
@@ -4152,6 +4173,7 @@ static int rtl_ops_init(struct r8152 *tp)
                 ops->eee_get            = r8153_get_eee;
                 ops->eee_set            = r8153_set_eee;
                 ops->in_nway            = rtl8153_in_nway;
+                ops->hw_phy_cfg         = r8153_hw_phy_cfg;
                 break;
 
         default:
@@ -4198,6 +4220,7 @@ static int rtl8152_probe(struct usb_interface *intf,
 
         mutex_init(&tp->control);
         INIT_DELAYED_WORK(&tp->schedule, rtl_work_func_t);
+        INIT_DELAYED_WORK(&tp->hw_phy_work, rtl_hw_phy_work_func_t);
 
         netdev->netdev_ops = &rtl8152_netdev_ops;
         netdev->watchdog_timeo = RTL8152_TX_TIMEOUT;
@@ -4237,9 +4260,14 @@ static int rtl8152_probe(struct usb_interface *intf,
                 break;
         }
 
+        tp->autoneg = AUTONEG_ENABLE;
+        tp->speed = tp->mii.supports_gmii ? SPEED_1000 : SPEED_100;
+        tp->duplex = DUPLEX_FULL;
+
         intf->needs_remote_wakeup = 1;
 
         tp->rtl_ops.init(tp);
+        queue_delayed_work(system_long_wq, &tp->hw_phy_work, 0);
         set_ethernet_addr(tp);
 
         usb_set_intfdata(intf, tp);
@@ -4285,6 +4313,7 @@ static void rtl8152_disconnect(struct usb_interface *intf)
 
                 netif_napi_del(&tp->napi);
                 unregister_netdev(tp->netdev);
+                cancel_delayed_work_sync(&tp->hw_phy_work);
                 tp->rtl_ops.unload(tp);
                 free_netdev(tp->netdev);
         }
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index e0638e556fe7..1dd08d4b9c31 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -479,53 +479,21 @@ static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
         stats->rx_packets++;
         u64_stats_update_end(&stats->rx_syncp);
 
-        if (hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
-                pr_debug("Needs csum!\n");
-                if (!skb_partial_csum_set(skb,
-                          virtio16_to_cpu(vi->vdev, hdr->hdr.csum_start),
-                          virtio16_to_cpu(vi->vdev, hdr->hdr.csum_offset)))
-                        goto frame_err;
-        } else if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID) {
+        if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID)
                 skb->ip_summed = CHECKSUM_UNNECESSARY;
+
+        if (virtio_net_hdr_to_skb(skb, &hdr->hdr,
+                                  virtio_is_little_endian(vi->vdev))) {
+                net_warn_ratelimited("%s: bad gso: type: %u, size: %u\n",
+                                     dev->name, hdr->hdr.gso_type,
+                                     hdr->hdr.gso_size);
+                goto frame_err;
         }
 
         skb->protocol = eth_type_trans(skb, dev);
         pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
                  ntohs(skb->protocol), skb->len, skb->pkt_type);
 
-        if (hdr->hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
-                pr_debug("GSO!\n");
-                switch (hdr->hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
-                case VIRTIO_NET_HDR_GSO_TCPV4:
-                        skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
-                        break;
-                case VIRTIO_NET_HDR_GSO_UDP:
-                        skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
-                        break;
-                case VIRTIO_NET_HDR_GSO_TCPV6:
-                        skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
-                        break;
-                default:
-                        net_warn_ratelimited("%s: bad gso type %u.\n",
-                                             dev->name, hdr->hdr.gso_type);
-                        goto frame_err;
-                }
-
-                if (hdr->hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
-                        skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
-
-                skb_shinfo(skb)->gso_size = virtio16_to_cpu(vi->vdev,
-                                                            hdr->hdr.gso_size);
-                if (skb_shinfo(skb)->gso_size == 0) {
-                        net_warn_ratelimited("%s: zero gso size.\n", dev->name);
-                        goto frame_err;
-                }
-
-                /* Header must be checked, and gso_segs computed. */
-                skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
-                skb_shinfo(skb)->gso_segs = 0;
-        }
-
         napi_gro_receive(&rq->napi, skb);
         return;
 
@@ -868,35 +836,9 @@ static int xmit_skb(struct send_queue *sq, struct sk_buff *skb)
         else
                 hdr = skb_vnet_hdr(skb);
 
-        if (skb->ip_summed == CHECKSUM_PARTIAL) {
-                hdr->hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
-                hdr->hdr.csum_start = cpu_to_virtio16(vi->vdev,
-                                                skb_checksum_start_offset(skb));
-                hdr->hdr.csum_offset = cpu_to_virtio16(vi->vdev,
-                                                         skb->csum_offset);
-        } else {
-                hdr->hdr.flags = 0;
-                hdr->hdr.csum_offset = hdr->hdr.csum_start = 0;
-        }
-
-        if (skb_is_gso(skb)) {
-                hdr->hdr.hdr_len = cpu_to_virtio16(vi->vdev, skb_headlen(skb));
-                hdr->hdr.gso_size = cpu_to_virtio16(vi->vdev,
-                                                    skb_shinfo(skb)->gso_size);
-                if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
-                        hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
-                else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
-                        hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
-                else if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP)
-                        hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
-                else
-                        BUG();
-                if (skb_shinfo(skb)->gso_type & SKB_GSO_TCP_ECN)
-                        hdr->hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
-        } else {
-                hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
-                hdr->hdr.gso_size = hdr->hdr.hdr_len = 0;
-        }
+        if (virtio_net_hdr_from_skb(skb, &hdr->hdr,
+                                    virtio_is_little_endian(vi->vdev)))
+                BUG();
 
         if (vi->mergeable_rx_bufs)
                 hdr->num_buffers = 0;
@@ -1780,6 +1722,7 @@ static int virtnet_probe(struct virtio_device *vdev)
         struct net_device *dev;
         struct virtnet_info *vi;
         u16 max_queue_pairs;
+        int mtu;
 
         if (!vdev->config->get) {
                 dev_err(&vdev->dev, "%s failure: config access disabled\n",
@@ -1896,6 +1839,14 @@ static int virtnet_probe(struct virtio_device *vdev)
         if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
                 vi->has_cvq = true;
 
+        if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) {
+                mtu = virtio_cread16(vdev,
+                                     offsetof(struct virtio_net_config,
+                                              mtu));
+                if (virtnet_change_mtu(dev, mtu))
+                        __virtio_clear_bit(vdev, VIRTIO_NET_F_MTU);
+        }
+
         if (vi->any_header_sg)
                 dev->needed_headroom = vi->hdr_len;
 
@@ -2067,6 +2018,7 @@ static unsigned int features[] = {
         VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ,
         VIRTIO_NET_F_CTRL_MAC_ADDR,
         VIRTIO_F_ANY_LAYOUT,
+        VIRTIO_NET_F_MTU,
 };
 
 static struct virtio_driver virtio_net_driver = {
diff --git a/drivers/net/vmxnet3/Makefile b/drivers/net/vmxnet3/Makefile
index 880f5098eac9..8cdbb63d1bb0 100644
--- a/drivers/net/vmxnet3/Makefile
+++ b/drivers/net/vmxnet3/Makefile
@@ -2,7 +2,7 @@
 #
 # Linux driver for VMware's vmxnet3 ethernet NIC.
 #
-# Copyright (C) 2007-2009, VMware, Inc. All Rights Reserved.
+# Copyright (C) 2007-2016, VMware, Inc. All Rights Reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -21,7 +21,7 @@
 # The full GNU General Public License is included in this distribution in
 # the file called "COPYING".
 #
-# Maintained by: Shreyas Bhatewara <pv-drivers@vmware.com>
+# Maintained by: pv-drivers@vmware.com
 #
 #
 ################################################################################
diff --git a/drivers/net/vmxnet3/upt1_defs.h b/drivers/net/vmxnet3/upt1_defs.h
index 969c751ee404..db9f1fde3aac 100644
--- a/drivers/net/vmxnet3/upt1_defs.h
+++ b/drivers/net/vmxnet3/upt1_defs.h
@@ -1,7 +1,7 @@
 /*
  * Linux driver for VMware's vmxnet3 ethernet NIC.
  *
- * Copyright (C) 2008-2009, VMware, Inc. All Rights Reserved.
+ * Copyright (C) 2008-2016, VMware, Inc. All Rights Reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -20,7 +20,7 @@
  * The full GNU General Public License is included in this distribution in
  * the file called "COPYING".
  *
- * Maintained by: Shreyas Bhatewara <pv-drivers@vmware.com>
+ * Maintained by: pv-drivers@vmware.com
  *
  */
 
diff --git a/drivers/net/vmxnet3/vmxnet3_defs.h b/drivers/net/vmxnet3/vmxnet3_defs.h
index 72ba8ae7f09a..c3a31646189f 100644
--- a/drivers/net/vmxnet3/vmxnet3_defs.h
+++ b/drivers/net/vmxnet3/vmxnet3_defs.h
@@ -1,7 +1,7 @@
 /*
  * Linux driver for VMware's vmxnet3 ethernet NIC.
  *
- * Copyright (C) 2008-2015, VMware, Inc. All Rights Reserved.
+ * Copyright (C) 2008-2016, VMware, Inc. All Rights Reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -20,7 +20,7 @@
  * The full GNU General Public License is included in this distribution in
  * the file called "COPYING".
  *
- * Maintained by: Shreyas Bhatewara <pv-drivers@vmware.com>
+ * Maintained by: pv-drivers@vmware.com
  *
  */
 
@@ -76,7 +76,12 @@ enum {
         VMXNET3_CMD_UPDATE_IML,
         VMXNET3_CMD_UPDATE_PMCFG,
         VMXNET3_CMD_UPDATE_FEATURE,
+        VMXNET3_CMD_RESERVED1,
         VMXNET3_CMD_LOAD_PLUGIN,
+        VMXNET3_CMD_RESERVED2,
+        VMXNET3_CMD_RESERVED3,
+        VMXNET3_CMD_SET_COALESCE,
+        VMXNET3_CMD_REGISTER_MEMREGS,
 
         VMXNET3_CMD_FIRST_GET = 0xF00D0000,
         VMXNET3_CMD_GET_QUEUE_STATUS = VMXNET3_CMD_FIRST_GET,
@@ -87,7 +92,10 @@ enum {
         VMXNET3_CMD_GET_DID_LO,
         VMXNET3_CMD_GET_DID_HI,
         VMXNET3_CMD_GET_DEV_EXTRA_INFO,
-        VMXNET3_CMD_GET_CONF_INTR
+        VMXNET3_CMD_GET_CONF_INTR,
+        VMXNET3_CMD_GET_RESERVED1,
+        VMXNET3_CMD_GET_TXDATA_DESC_SIZE,
+        VMXNET3_CMD_GET_COALESCE,
 };
 
 /*
@@ -169,6 +177,8 @@ struct Vmxnet3_TxDataDesc {
         u8              data[VMXNET3_HDR_COPY_SIZE];
 };
 
+typedef u8 Vmxnet3_RxDataDesc;
+
 #define VMXNET3_TCD_GEN_SHIFT   31
 #define VMXNET3_TCD_GEN_SIZE    1
 #define VMXNET3_TCD_TXIDX_SHIFT 0
@@ -373,6 +383,14 @@ union Vmxnet3_GenericDesc {
 #define VMXNET3_RING_SIZE_ALIGN 32
 #define VMXNET3_RING_SIZE_MASK  (VMXNET3_RING_SIZE_ALIGN - 1)
 
+/* Tx Data Ring buffer size must be a multiple of 64 */
+#define VMXNET3_TXDATA_DESC_SIZE_ALIGN 64
+#define VMXNET3_TXDATA_DESC_SIZE_MASK  (VMXNET3_TXDATA_DESC_SIZE_ALIGN - 1)
+
+/* Rx Data Ring buffer size must be a multiple of 64 */
+#define VMXNET3_RXDATA_DESC_SIZE_ALIGN 64
+#define VMXNET3_RXDATA_DESC_SIZE_MASK  (VMXNET3_RXDATA_DESC_SIZE_ALIGN - 1)
+
 /* Max ring size */
 #define VMXNET3_TX_RING_MAX_SIZE   4096
 #define VMXNET3_TC_RING_MAX_SIZE   4096
@@ -380,6 +398,11 @@ union Vmxnet3_GenericDesc {
 #define VMXNET3_RX_RING2_MAX_SIZE  4096
 #define VMXNET3_RC_RING_MAX_SIZE   8192
 
+#define VMXNET3_TXDATA_DESC_MIN_SIZE 128
+#define VMXNET3_TXDATA_DESC_MAX_SIZE 2048
+
+#define VMXNET3_RXDATA_DESC_MAX_SIZE 2048
+
 /* a list of reasons for queue stop */
 
 enum {
@@ -466,7 +489,9 @@ struct Vmxnet3_TxQueueConf {
         __le32          compRingSize; /* # of comp desc */
         __le32          ddLen;        /* size of driver data */
         u8              intrIdx;
-        u8              _pad[7];
+        u8              _pad1[1];
+        __le16          txDataRingDescSize;
+        u8              _pad2[4];
 };
 
 
@@ -474,12 +499,14 @@ struct Vmxnet3_RxQueueConf {
         __le64          rxRingBasePA[2];
         __le64          compRingBasePA;
         __le64          ddPA;            /* driver data */
-        __le64          reserved;
+        __le64          rxDataRingBasePA;
         __le32          rxRingSize[2];   /* # of rx desc */
         __le32          compRingSize;    /* # of rx comp desc */
         __le32          ddLen;           /* size of driver data */
         u8              intrIdx;
-        u8              _pad[7];
+        u8              _pad1[1];
+        __le16          rxDataRingDescSize;  /* size of rx data ring buffer */
+        u8              _pad2[4];
 };
 
 
@@ -609,6 +636,63 @@ struct Vmxnet3_RxQueueDesc {
         u8                                    __pad[88]; /* 128 aligned */
 };
 
+struct Vmxnet3_SetPolling {
+        u8                                      enablePolling;
+};
+
+#define VMXNET3_COAL_STATIC_MAX_DEPTH           128
+#define VMXNET3_COAL_RBC_MIN_RATE               100
+#define VMXNET3_COAL_RBC_MAX_RATE               100000
+
+enum Vmxnet3_CoalesceMode {
+        VMXNET3_COALESCE_DISABLED   = 0,
+        VMXNET3_COALESCE_ADAPT      = 1,
+        VMXNET3_COALESCE_STATIC     = 2,
+        VMXNET3_COALESCE_RBC        = 3
+};
+
+struct Vmxnet3_CoalesceRbc {
+        u32                                     rbc_rate;
+};
+
+struct Vmxnet3_CoalesceStatic {
+        u32                                     tx_depth;
+        u32                                     tx_comp_depth;
+        u32                                     rx_depth;
+};
+
+struct Vmxnet3_CoalesceScheme {
+        enum Vmxnet3_CoalesceMode               coalMode;
+        union {
+                struct Vmxnet3_CoalesceRbc      coalRbc;
+                struct Vmxnet3_CoalesceStatic   coalStatic;
+        } coalPara;
+};
+
+struct Vmxnet3_MemoryRegion {
+        __le64                                  startPA;
+        __le32                                  length;
+        __le16                                  txQueueBits;
+        __le16                                  rxQueueBits;
+};
+
+#define MAX_MEMORY_REGION_PER_QUEUE 16
+#define MAX_MEMORY_REGION_PER_DEVICE 256
+
+struct Vmxnet3_MemRegs {
+        __le16                                  numRegs;
+        __le16                                  pad[3];
+        struct Vmxnet3_MemoryRegion             memRegs[1];
+};
+
+/* If the command data <= 16 bytes, use the shared memory directly.
+ * otherwise, use variable length configuration descriptor.
+ */
+union Vmxnet3_CmdInfo {
+        struct Vmxnet3_VariableLenConfDesc      varConf;
+        struct Vmxnet3_SetPolling               setPolling;
+        __le64                                  data[2];
+};
 
 struct Vmxnet3_DSDevRead {
         /* read-only region for device, read by dev in response to a SET cmd */
@@ -627,7 +711,14 @@ struct Vmxnet3_DriverShared {
         __le32                          pad;
         struct Vmxnet3_DSDevRead        devRead;
         __le32                          ecr;
-        __le32                          reserved[5];
+        __le32                          reserved;
+        union {
+                __le32                  reserved1[4];
+                union Vmxnet3_CmdInfo   cmdInfo; /* only valid in the context of
+                                                  * executing the relevant
+                                                  * command
+                                                  */
+        } cu;
 };
 
 
diff --git a/drivers/net/vmxnet3/vmxnet3_drv.c b/drivers/net/vmxnet3/vmxnet3_drv.c
index 08885bc8d6db..c68fe495d3f9 100644
--- a/drivers/net/vmxnet3/vmxnet3_drv.c
+++ b/drivers/net/vmxnet3/vmxnet3_drv.c
@@ -1,7 +1,7 @@
 /*
  * Linux driver for VMware's vmxnet3 ethernet NIC.
  *
- * Copyright (C) 2008-2009, VMware, Inc. All Rights Reserved.
+ * Copyright (C) 2008-2016, VMware, Inc. All Rights Reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -20,7 +20,7 @@
  * The full GNU General Public License is included in this distribution in
  * the file called "COPYING".
  *
- * Maintained by: Shreyas Bhatewara <pv-drivers@vmware.com>
+ * Maintained by: pv-drivers@vmware.com
  *
  */
 
@@ -435,8 +435,8 @@ vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq,
                 tq->tx_ring.base = NULL;
         }
         if (tq->data_ring.base) {
-                dma_free_coherent(&adapter->pdev->dev, tq->data_ring.size *
-                                  sizeof(struct Vmxnet3_TxDataDesc),
+                dma_free_coherent(&adapter->pdev->dev,
+                                  tq->data_ring.size * tq->txdata_desc_size,
                                   tq->data_ring.base, tq->data_ring.basePA);
                 tq->data_ring.base = NULL;
         }
@@ -478,8 +478,8 @@ vmxnet3_tq_init(struct vmxnet3_tx_queue *tq,
         tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
         tq->tx_ring.gen = VMXNET3_INIT_GEN;
 
-        memset(tq->data_ring.base, 0, tq->data_ring.size *
-               sizeof(struct Vmxnet3_TxDataDesc));
+        memset(tq->data_ring.base, 0,
+               tq->data_ring.size * tq->txdata_desc_size);
 
         /* reset the tx comp ring contents to 0 and reset comp ring states */
         memset(tq->comp_ring.base, 0, tq->comp_ring.size *
@@ -514,10 +514,10 @@ vmxnet3_tq_create(struct vmxnet3_tx_queue *tq,
         }
 
         tq->data_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
-                        tq->data_ring.size * sizeof(struct Vmxnet3_TxDataDesc),
+                        tq->data_ring.size * tq->txdata_desc_size,
                         &tq->data_ring.basePA, GFP_KERNEL);
         if (!tq->data_ring.base) {
-                netdev_err(adapter->netdev, "failed to allocate data ring\n");
+                netdev_err(adapter->netdev, "failed to allocate tx data ring\n");
                 goto err;
         }
 
@@ -689,7 +689,7 @@ vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx,
         if (ctx->copy_size) {
                 ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA +
                                         tq->tx_ring.next2fill *
-                                        sizeof(struct Vmxnet3_TxDataDesc));
+                                        tq->txdata_desc_size);
                 ctx->sop_txd->dword[2] = cpu_to_le32(dw2 | ctx->copy_size);
                 ctx->sop_txd->dword[3] = 0;
 
@@ -873,8 +873,9 @@ vmxnet3_parse_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
                         ctx->eth_ip_hdr_size = 0;
                         ctx->l4_hdr_size = 0;
                         /* copy as much as allowed */
-                        ctx->copy_size = min((unsigned int)VMXNET3_HDR_COPY_SIZE
-                                             , skb_headlen(skb));
+                        ctx->copy_size = min_t(unsigned int,
+                                               tq->txdata_desc_size,
+                                               skb_headlen(skb));
                 }
 
                 if (skb->len <= VMXNET3_HDR_COPY_SIZE)
@@ -885,7 +886,7 @@ vmxnet3_parse_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
                         goto err;
         }
 
-        if (unlikely(ctx->copy_size > VMXNET3_HDR_COPY_SIZE)) {
+        if (unlikely(ctx->copy_size > tq->txdata_desc_size)) {
                 tq->stats.oversized_hdr++;
                 ctx->copy_size = 0;
                 return 0;
@@ -1283,9 +1284,10 @@ vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
                          */
                         break;
                 }
-                BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2);
+                BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2 &&
+                       rcd->rqID != rq->dataRingQid);
                 idx = rcd->rxdIdx;
-                ring_idx = rcd->rqID < adapter->num_rx_queues ? 0 : 1;
+                ring_idx = VMXNET3_GET_RING_IDX(adapter, rcd->rqID);
                 ring = rq->rx_ring + ring_idx;
                 vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd,
                                   &rxCmdDesc);
@@ -1300,8 +1302,12 @@ vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
                 }
 
                 if (rcd->sop) { /* first buf of the pkt */
+                        bool rxDataRingUsed;
+                        u16 len;
+
                         BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD ||
-                               rcd->rqID != rq->qid);
+                               (rcd->rqID != rq->qid &&
+                                rcd->rqID != rq->dataRingQid));
 
                         BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB);
                         BUG_ON(ctx->skb != NULL || rbi->skb == NULL);
@@ -1317,8 +1323,12 @@ vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
 
                         skip_page_frags = false;
                         ctx->skb = rbi->skb;
+
+                        rxDataRingUsed =
+                                VMXNET3_RX_DATA_RING(adapter, rcd->rqID);
+                        len = rxDataRingUsed ? rcd->len : rbi->len;
                         new_skb = netdev_alloc_skb_ip_align(adapter->netdev,
-                                                            rbi->len);
+                                                            len);
                         if (new_skb == NULL) {
                                 /* Skb allocation failed, do not handover this
                                  * skb to stack. Reuse it. Drop the existing pkt
@@ -1329,25 +1339,48 @@ vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
                                 skip_page_frags = true;
                                 goto rcd_done;
                         }
-                        new_dma_addr = dma_map_single(&adapter->pdev->dev,
-                                                      new_skb->data, rbi->len,
-                                                      PCI_DMA_FROMDEVICE);
-                        if (dma_mapping_error(&adapter->pdev->dev,
-                                              new_dma_addr)) {
-                                dev_kfree_skb(new_skb);
-                                /* Skb allocation failed, do not handover this
-                                 * skb to stack. Reuse it. Drop the existing pkt
-                                 */
-                                rq->stats.rx_buf_alloc_failure++;
-                                ctx->skb = NULL;
-                                rq->stats.drop_total++;
-                                skip_page_frags = true;
-                                goto rcd_done;
-                        }
 
-                        dma_unmap_single(&adapter->pdev->dev, rbi->dma_addr,
-                                         rbi->len,
-                                         PCI_DMA_FROMDEVICE);
+                        if (rxDataRingUsed) {
+                                size_t sz;
+
+                                BUG_ON(rcd->len > rq->data_ring.desc_size);
+
+                                ctx->skb = new_skb;
+                                sz = rcd->rxdIdx * rq->data_ring.desc_size;
+                                memcpy(new_skb->data,
+                                       &rq->data_ring.base[sz], rcd->len);
+                        } else {
+                                ctx->skb = rbi->skb;
+
+                                new_dma_addr =
+                                        dma_map_single(&adapter->pdev->dev,
+                                                       new_skb->data, rbi->len,
+                                                       PCI_DMA_FROMDEVICE);
+                                if (dma_mapping_error(&adapter->pdev->dev,
+                                                      new_dma_addr)) {
+                                        dev_kfree_skb(new_skb);
+                                        /* Skb allocation failed, do not
+                                         * handover this skb to stack. Reuse
+                                         * it. Drop the existing pkt.
+                                         */
+                                        rq->stats.rx_buf_alloc_failure++;
+                                        ctx->skb = NULL;
+                                        rq->stats.drop_total++;
+                                        skip_page_frags = true;
+                                        goto rcd_done;
+                                }
+
+                                dma_unmap_single(&adapter->pdev->dev,
+                                                 rbi->dma_addr,
+                                                 rbi->len,
+                                                 PCI_DMA_FROMDEVICE);
+
+                                /* Immediate refill */
+                                rbi->skb = new_skb;
+                                rbi->dma_addr = new_dma_addr;
+                                rxd->addr = cpu_to_le64(rbi->dma_addr);
+                                rxd->len = rbi->len;
+                        }
 
 #ifdef VMXNET3_RSS
                         if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE &&
@@ -1358,12 +1391,7 @@ vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
 #endif
                         skb_put(ctx->skb, rcd->len);
 
-                        /* Immediate refill */
-                        rbi->skb = new_skb;
-                        rbi->dma_addr = new_dma_addr;
-                        rxd->addr = cpu_to_le64(rbi->dma_addr);
-                        rxd->len = rbi->len;
-                        if (adapter->version == 2 &&
+                        if (VMXNET3_VERSION_GE_2(adapter) &&
                             rcd->type == VMXNET3_CDTYPE_RXCOMP_LRO) {
                                 struct Vmxnet3_RxCompDescExt *rcdlro;
                                 rcdlro = (struct Vmxnet3_RxCompDescExt *)rcd;
@@ -1589,6 +1617,13 @@ static void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq,
                 rq->buf_info[i] = NULL;
         }
 
+        if (rq->data_ring.base) {
+                dma_free_coherent(&adapter->pdev->dev,
+                                  rq->rx_ring[0].size * rq->data_ring.desc_size,
+                                  rq->data_ring.base, rq->data_ring.basePA);
+                rq->data_ring.base = NULL;
+        }
+
         if (rq->comp_ring.base) {
                 dma_free_coherent(&adapter->pdev->dev, rq->comp_ring.size
                                   * sizeof(struct Vmxnet3_RxCompDesc),
@@ -1604,6 +1639,25 @@ static void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq,
         }
 }
 
+void
+vmxnet3_rq_destroy_all_rxdataring(struct vmxnet3_adapter *adapter)
+{
+        int i;
+
+        for (i = 0; i < adapter->num_rx_queues; i++) {
+                struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
+
+                if (rq->data_ring.base) {
+                        dma_free_coherent(&adapter->pdev->dev,
+                                          (rq->rx_ring[0].size *
+                                          rq->data_ring.desc_size),
+                                          rq->data_ring.base,
+                                          rq->data_ring.basePA);
+                        rq->data_ring.base = NULL;
+                        rq->data_ring.desc_size = 0;
+                }
+        }
+}
 
 static int
 vmxnet3_rq_init(struct vmxnet3_rx_queue *rq,
@@ -1697,6 +1751,22 @@ vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter)
                 }
         }
 
+        if ((adapter->rxdataring_enabled) && (rq->data_ring.desc_size != 0)) {
+                sz = rq->rx_ring[0].size * rq->data_ring.desc_size;
+                rq->data_ring.base =
+                        dma_alloc_coherent(&adapter->pdev->dev, sz,
+                                           &rq->data_ring.basePA,
+                                           GFP_KERNEL);
+                if (!rq->data_ring.base) {
+                        netdev_err(adapter->netdev,
+                                   "rx data ring will be disabled\n");
+                        adapter->rxdataring_enabled = false;
+                }
+        } else {
+                rq->data_ring.base = NULL;
+                rq->data_ring.desc_size = 0;
+        }
+
         sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc);
         rq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, sz,
                                                 &rq->comp_ring.basePA,
@@ -1729,6 +1799,8 @@ vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter)
 {
         int i, err = 0;
 
+        adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter);
+
         for (i = 0; i < adapter->num_rx_queues; i++) {
                 err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter);
                 if (unlikely(err)) {
@@ -1738,6 +1810,10 @@ vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter)
                         goto err_out;
                 }
         }
+
+        if (!adapter->rxdataring_enabled)
+                vmxnet3_rq_destroy_all_rxdataring(adapter);
+
         return err;
 err_out:
         vmxnet3_rq_destroy_all(adapter);
@@ -2045,10 +2121,9 @@ vmxnet3_request_irqs(struct vmxnet3_adapter *adapter)
                         struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
                         rq->qid = i;
                         rq->qid2 = i + adapter->num_rx_queues;
+                        rq->dataRingQid = i + 2 * adapter->num_rx_queues;
                 }
 
-
-
                 /* init our intr settings */
                 for (i = 0; i < intr->num_intrs; i++)
                         intr->mod_levels[i] = UPT1_IML_ADAPTIVE;
@@ -2336,6 +2411,7 @@ vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter)
                 tqc->ddPA           = cpu_to_le64(tq->buf_info_pa);
                 tqc->txRingSize     = cpu_to_le32(tq->tx_ring.size);
                 tqc->dataRingSize   = cpu_to_le32(tq->data_ring.size);
+                tqc->txDataRingDescSize = cpu_to_le32(tq->txdata_desc_size);
                 tqc->compRingSize   = cpu_to_le32(tq->comp_ring.size);
                 tqc->ddLen          = cpu_to_le32(
                                         sizeof(struct vmxnet3_tx_buf_info) *
@@ -2360,6 +2436,12 @@ vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter)
                                         (rqc->rxRingSize[0] +
                                          rqc->rxRingSize[1]));
                 rqc->intrIdx         = rq->comp_ring.intr_idx;
+                if (VMXNET3_VERSION_GE_3(adapter)) {
+                        rqc->rxDataRingBasePA =
+                                cpu_to_le64(rq->data_ring.basePA);
+                        rqc->rxDataRingDescSize =
+                                cpu_to_le16(rq->data_ring.desc_size);
+                }
         }
 
 #ifdef VMXNET3_RSS
@@ -2409,6 +2491,32 @@ vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter)
         /* the rest are already zeroed */
 }
 
+static void
+vmxnet3_init_coalesce(struct vmxnet3_adapter *adapter)
+{
+        struct Vmxnet3_DriverShared *shared = adapter->shared;
+        union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo;
+        unsigned long flags;
+
+        if (!VMXNET3_VERSION_GE_3(adapter))
+                return;
+
+        spin_lock_irqsave(&adapter->cmd_lock, flags);
+        cmdInfo->varConf.confVer = 1;
+        cmdInfo->varConf.confLen =
+                cpu_to_le32(sizeof(*adapter->coal_conf));
+        cmdInfo->varConf.confPA  = cpu_to_le64(adapter->coal_conf_pa);
+
+        if (adapter->default_coal_mode) {
+                VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
+                                       VMXNET3_CMD_GET_COALESCE);
+        } else {
+                VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
+                                       VMXNET3_CMD_SET_COALESCE);
+        }
+
+        spin_unlock_irqrestore(&adapter->cmd_lock, flags);
+}
 
 int
 vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)
@@ -2458,6 +2566,8 @@ vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)
                 goto activate_err;
         }
 
+        vmxnet3_init_coalesce(adapter);
+
         for (i = 0; i < adapter->num_rx_queues; i++) {
                 VMXNET3_WRITE_BAR0_REG(adapter,
                                 VMXNET3_REG_RXPROD + i * VMXNET3_REG_ALIGN,
@@ -2689,7 +2799,8 @@ vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter)
 
 int
 vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size,
-                      u32 rx_ring_size, u32 rx_ring2_size)
+                      u32 rx_ring_size, u32 rx_ring2_size,
+                      u16 txdata_desc_size, u16 rxdata_desc_size)
 {
         int err = 0, i;
 
@@ -2698,6 +2809,7 @@ vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size,
                 tq->tx_ring.size   = tx_ring_size;
                 tq->data_ring.size = tx_ring_size;
                 tq->comp_ring.size = tx_ring_size;
+                tq->txdata_desc_size = txdata_desc_size;
                 tq->shared = &adapter->tqd_start[i].ctrl;
                 tq->stopped = true;
                 tq->adapter = adapter;
@@ -2714,12 +2826,15 @@ vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size,
         adapter->rx_queue[0].rx_ring[0].size = rx_ring_size;
         adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size;
         vmxnet3_adjust_rx_ring_size(adapter);
+
+        adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter);
         for (i = 0; i < adapter->num_rx_queues; i++) {
                 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
                 /* qid and qid2 for rx queues will be assigned later when num
                  * of rx queues is finalized after allocating intrs */
                 rq->shared = &adapter->rqd_start[i].ctrl;
                 rq->adapter = adapter;
+                rq->data_ring.desc_size = rxdata_desc_size;
                 err = vmxnet3_rq_create(rq, adapter);
                 if (err) {
                         if (i == 0) {
@@ -2737,6 +2852,10 @@ vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size,
                         }
                 }
         }
+
+        if (!adapter->rxdataring_enabled)
+                vmxnet3_rq_destroy_all_rxdataring(adapter);
+
         return err;
 queue_err:
         vmxnet3_tq_destroy_all(adapter);
@@ -2754,9 +2873,35 @@ vmxnet3_open(struct net_device *netdev)
         for (i = 0; i < adapter->num_tx_queues; i++)
                 spin_lock_init(&adapter->tx_queue[i].tx_lock);
 
-        err = vmxnet3_create_queues(adapter, adapter->tx_ring_size,
+        if (VMXNET3_VERSION_GE_3(adapter)) {
+                unsigned long flags;
+                u16 txdata_desc_size;
+
+                spin_lock_irqsave(&adapter->cmd_lock, flags);
+                VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
+                                       VMXNET3_CMD_GET_TXDATA_DESC_SIZE);
+                txdata_desc_size = VMXNET3_READ_BAR1_REG(adapter,
+                                                         VMXNET3_REG_CMD);
+                spin_unlock_irqrestore(&adapter->cmd_lock, flags);
+
+                if ((txdata_desc_size < VMXNET3_TXDATA_DESC_MIN_SIZE) ||
+                    (txdata_desc_size > VMXNET3_TXDATA_DESC_MAX_SIZE) ||
+                    (txdata_desc_size & VMXNET3_TXDATA_DESC_SIZE_MASK)) {
+                        adapter->txdata_desc_size =
+                                sizeof(struct Vmxnet3_TxDataDesc);
+                } else {
+                        adapter->txdata_desc_size = txdata_desc_size;
+                }
+        } else {
+                adapter->txdata_desc_size = sizeof(struct Vmxnet3_TxDataDesc);
+        }
+
+        err = vmxnet3_create_queues(adapter,
+                                    adapter->tx_ring_size,
                                     adapter->rx_ring_size,
-                                    adapter->rx_ring2_size);
+                                    adapter->rx_ring2_size,
+                                    adapter->txdata_desc_size,
+                                    adapter->rxdata_desc_size);
         if (err)
                 goto queue_err;
 
@@ -3200,12 +3345,21 @@ vmxnet3_probe_device(struct pci_dev *pdev,
                 goto err_alloc_pci;
 
         ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS);
-        if (ver & 2) {
-                VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_VRRS, 2);
-                adapter->version = 2;
-        } else if (ver & 1) {
-                VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_VRRS, 1);
-                adapter->version = 1;
+        if (ver & (1 << VMXNET3_REV_3)) {
+                VMXNET3_WRITE_BAR1_REG(adapter,
+                                       VMXNET3_REG_VRRS,
+                                       1 << VMXNET3_REV_3);
+                adapter->version = VMXNET3_REV_3 + 1;
+        } else if (ver & (1 << VMXNET3_REV_2)) {
+                VMXNET3_WRITE_BAR1_REG(adapter,
+                                       VMXNET3_REG_VRRS,
+                                       1 << VMXNET3_REV_2);
+                adapter->version = VMXNET3_REV_2 + 1;
+        } else if (ver & (1 << VMXNET3_REV_1)) {
+                VMXNET3_WRITE_BAR1_REG(adapter,
+                                       VMXNET3_REG_VRRS,
+                                       1 << VMXNET3_REV_1);
+                adapter->version = VMXNET3_REV_1 + 1;
         } else {
                 dev_err(&pdev->dev,
                         "Incompatible h/w version (0x%x) for adapter\n", ver);
@@ -3224,9 +3378,28 @@ vmxnet3_probe_device(struct pci_dev *pdev,
                 goto err_ver;
         }
 
+        if (VMXNET3_VERSION_GE_3(adapter)) {
+                adapter->coal_conf =
+                        dma_alloc_coherent(&adapter->pdev->dev,
+                                           sizeof(struct Vmxnet3_CoalesceScheme)
+                                           ,
+                                           &adapter->coal_conf_pa,
+                                           GFP_KERNEL);
+                if (!adapter->coal_conf) {
+                        err = -ENOMEM;
+                        goto err_ver;
+                }
+                memset(adapter->coal_conf, 0, sizeof(*adapter->coal_conf));
+                adapter->coal_conf->coalMode = VMXNET3_COALESCE_DISABLED;
+                adapter->default_coal_mode = true;
+        }
+
         SET_NETDEV_DEV(netdev, &pdev->dev);
         vmxnet3_declare_features(adapter, dma64);
 
+        adapter->rxdata_desc_size = VMXNET3_VERSION_GE_3(adapter) ?
+                VMXNET3_DEF_RXDATA_DESC_SIZE : 0;
+
         if (adapter->num_tx_queues == adapter->num_rx_queues)
                 adapter->share_intr = VMXNET3_INTR_BUDDYSHARE;
         else
@@ -3283,6 +3456,11 @@ vmxnet3_probe_device(struct pci_dev *pdev,
         return 0;
 
 err_register:
+        if (VMXNET3_VERSION_GE_3(adapter)) {
+                dma_free_coherent(&adapter->pdev->dev,
+                                  sizeof(struct Vmxnet3_CoalesceScheme),
+                                  adapter->coal_conf, adapter->coal_conf_pa);
+        }
         vmxnet3_free_intr_resources(adapter);
 err_ver:
         vmxnet3_free_pci_resources(adapter);
@@ -3333,6 +3511,11 @@ vmxnet3_remove_device(struct pci_dev *pdev)
 
         vmxnet3_free_intr_resources(adapter);
         vmxnet3_free_pci_resources(adapter);
+        if (VMXNET3_VERSION_GE_3(adapter)) {
+                dma_free_coherent(&adapter->pdev->dev,
+                                  sizeof(struct Vmxnet3_CoalesceScheme),
+                                  adapter->coal_conf, adapter->coal_conf_pa);
+        }
 #ifdef VMXNET3_RSS
         dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),
                           adapter->rss_conf, adapter->rss_conf_pa);
diff --git a/drivers/net/vmxnet3/vmxnet3_ethtool.c b/drivers/net/vmxnet3/vmxnet3_ethtool.c
index 9ba11d737753..aabc6ef366b4 100644
--- a/drivers/net/vmxnet3/vmxnet3_ethtool.c
+++ b/drivers/net/vmxnet3/vmxnet3_ethtool.c
@@ -1,7 +1,7 @@
 /*
  * Linux driver for VMware's vmxnet3 ethernet NIC.
  *
- * Copyright (C) 2008-2009, VMware, Inc. All Rights Reserved.
+ * Copyright (C) 2008-2016, VMware, Inc. All Rights Reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -20,7 +20,7 @@
  * The full GNU General Public License is included in this distribution in
  * the file called "COPYING".
  *
- * Maintained by: Shreyas Bhatewara <pv-drivers@vmware.com>
+ * Maintained by: pv-drivers@vmware.com
  *
  */
 
@@ -396,8 +396,7 @@ vmxnet3_get_regs(struct net_device *netdev, struct ethtool_regs *regs, void *p)
                 buf[j++] = VMXNET3_GET_ADDR_LO(tq->data_ring.basePA);
                 buf[j++] = VMXNET3_GET_ADDR_HI(tq->data_ring.basePA);
                 buf[j++] = tq->data_ring.size;
-                /* transmit data ring buffer size */
-                buf[j++] = VMXNET3_HDR_COPY_SIZE;
+                buf[j++] = tq->txdata_desc_size;
 
                 buf[j++] = VMXNET3_GET_ADDR_LO(tq->comp_ring.basePA);
                 buf[j++] = VMXNET3_GET_ADDR_HI(tq->comp_ring.basePA);
@@ -431,11 +430,10 @@ vmxnet3_get_regs(struct net_device *netdev, struct ethtool_regs *regs, void *p)
                 buf[j++] = rq->rx_ring[1].next2comp;
                 buf[j++] = rq->rx_ring[1].gen;
 
-                /* receive data ring */
-                buf[j++] = 0;
-                buf[j++] = 0;
-                buf[j++] = 0;
-                buf[j++] = 0;
+                buf[j++] = VMXNET3_GET_ADDR_LO(rq->data_ring.basePA);
+                buf[j++] = VMXNET3_GET_ADDR_HI(rq->data_ring.basePA);
+                buf[j++] = rq->rx_ring[0].size;
+                buf[j++] = rq->data_ring.desc_size;
 
                 buf[j++] = VMXNET3_GET_ADDR_LO(rq->comp_ring.basePA);
                 buf[j++] = VMXNET3_GET_ADDR_HI(rq->comp_ring.basePA);
@@ -504,12 +502,14 @@ vmxnet3_get_ringparam(struct net_device *netdev,
 
         param->rx_max_pending = VMXNET3_RX_RING_MAX_SIZE;
         param->tx_max_pending = VMXNET3_TX_RING_MAX_SIZE;
-        param->rx_mini_max_pending = 0;
+        param->rx_mini_max_pending = VMXNET3_VERSION_GE_3(adapter) ?
+                VMXNET3_RXDATA_DESC_MAX_SIZE : 0;
         param->rx_jumbo_max_pending = VMXNET3_RX_RING2_MAX_SIZE;
 
         param->rx_pending = adapter->rx_ring_size;
         param->tx_pending = adapter->tx_ring_size;
-        param->rx_mini_pending = 0;
+        param->rx_mini_pending = VMXNET3_VERSION_GE_3(adapter) ?
+                adapter->rxdata_desc_size : 0;
         param->rx_jumbo_pending = adapter->rx_ring2_size;
 }
 
@@ -520,6 +520,7 @@ vmxnet3_set_ringparam(struct net_device *netdev,
 {
         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
         u32 new_tx_ring_size, new_rx_ring_size, new_rx_ring2_size;
+        u16 new_rxdata_desc_size;
         u32 sz;
         int err = 0;
 
@@ -542,6 +543,15 @@ vmxnet3_set_ringparam(struct net_device *netdev,
                 return -EOPNOTSUPP;
         }
 
+        if (VMXNET3_VERSION_GE_3(adapter)) {
+                if (param->rx_mini_pending < 0 ||
+                    param->rx_mini_pending > VMXNET3_RXDATA_DESC_MAX_SIZE) {
+                        return -EINVAL;
+                }
+        } else if (param->rx_mini_pending != 0) {
+                return -EINVAL;
+        }
+
         /* round it up to a multiple of VMXNET3_RING_SIZE_ALIGN */
         new_tx_ring_size = (param->tx_pending + VMXNET3_RING_SIZE_MASK) &
                                                         ~VMXNET3_RING_SIZE_MASK;
@@ -568,9 +578,19 @@ vmxnet3_set_ringparam(struct net_device *netdev,
         new_rx_ring2_size = min_t(u32, new_rx_ring2_size,
                                   VMXNET3_RX_RING2_MAX_SIZE);
 
+        /* rx data ring buffer size has to be a multiple of
+         * VMXNET3_RXDATA_DESC_SIZE_ALIGN
+         */
+        new_rxdata_desc_size =
+                (param->rx_mini_pending + VMXNET3_RXDATA_DESC_SIZE_MASK) &
+                ~VMXNET3_RXDATA_DESC_SIZE_MASK;
+        new_rxdata_desc_size = min_t(u16, new_rxdata_desc_size,
+                                     VMXNET3_RXDATA_DESC_MAX_SIZE);
+
         if (new_tx_ring_size == adapter->tx_ring_size &&
             new_rx_ring_size == adapter->rx_ring_size &&
-            new_rx_ring2_size == adapter->rx_ring2_size) {
+            new_rx_ring2_size == adapter->rx_ring2_size &&
+            new_rxdata_desc_size == adapter->rxdata_desc_size) {
                 return 0;
         }
 
@@ -591,8 +611,9 @@ vmxnet3_set_ringparam(struct net_device *netdev,
                 vmxnet3_rq_destroy_all(adapter);
 
                 err = vmxnet3_create_queues(adapter, new_tx_ring_size,
-                        new_rx_ring_size, new_rx_ring2_size);
-
+                                            new_rx_ring_size, new_rx_ring2_size,
+                                            adapter->txdata_desc_size,
+                                            new_rxdata_desc_size);
                 if (err) {
                         /* failed, most likely because of OOM, try default
                          * size */
@@ -601,10 +622,15 @@ vmxnet3_set_ringparam(struct net_device *netdev,
                         new_rx_ring_size = VMXNET3_DEF_RX_RING_SIZE;
                         new_rx_ring2_size = VMXNET3_DEF_RX_RING2_SIZE;
                         new_tx_ring_size = VMXNET3_DEF_TX_RING_SIZE;
+                        new_rxdata_desc_size = VMXNET3_VERSION_GE_3(adapter) ?
+                                VMXNET3_DEF_RXDATA_DESC_SIZE : 0;
+
                         err = vmxnet3_create_queues(adapter,
                                                     new_tx_ring_size,
                                                     new_rx_ring_size,
-                                                    new_rx_ring2_size);
+                                                    new_rx_ring2_size,
+                                                    adapter->txdata_desc_size,
+                                                    new_rxdata_desc_size);
                         if (err) {
                                 netdev_err(netdev, "failed to create queues "
                                            "with default sizes. Closing it\n");
@@ -620,6 +646,7 @@ vmxnet3_set_ringparam(struct net_device *netdev,
         adapter->tx_ring_size = new_tx_ring_size;
         adapter->rx_ring_size = new_rx_ring_size;
         adapter->rx_ring2_size = new_rx_ring2_size;
+        adapter->rxdata_desc_size = new_rxdata_desc_size;
 
 out:
         clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
@@ -698,6 +725,162 @@ vmxnet3_set_rss(struct net_device *netdev, const u32 *p, const u8 *key,
 }
 #endif
 
+static int
+vmxnet3_get_coalesce(struct net_device *netdev, struct ethtool_coalesce *ec)
+{
+        struct vmxnet3_adapter *adapter = netdev_priv(netdev);
+
+        if (!VMXNET3_VERSION_GE_3(adapter))
+                return -EOPNOTSUPP;
+
+        switch (adapter->coal_conf->coalMode) {
+        case VMXNET3_COALESCE_DISABLED:
+                /* struct ethtool_coalesce is already initialized to 0 */
+                break;
+        case VMXNET3_COALESCE_ADAPT:
+                ec->use_adaptive_rx_coalesce = true;
+                break;
+        case VMXNET3_COALESCE_STATIC:
+                ec->tx_max_coalesced_frames =
+                        adapter->coal_conf->coalPara.coalStatic.tx_comp_depth;
+                ec->rx_max_coalesced_frames =
+                        adapter->coal_conf->coalPara.coalStatic.rx_depth;
+                break;
+        case VMXNET3_COALESCE_RBC: {
+                u32 rbc_rate;
+
+                rbc_rate = adapter->coal_conf->coalPara.coalRbc.rbc_rate;
+                ec->rx_coalesce_usecs = VMXNET3_COAL_RBC_USECS(rbc_rate);
+        }
+                break;
+        default:
+                return -EOPNOTSUPP;
+        }
+
+        return 0;
+}
+
+static int
+vmxnet3_set_coalesce(struct net_device *netdev, struct ethtool_coalesce *ec)
+{
+        struct vmxnet3_adapter *adapter = netdev_priv(netdev);
+        struct Vmxnet3_DriverShared *shared = adapter->shared;
+        union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo;
+        unsigned long flags;
+
+        if (!VMXNET3_VERSION_GE_3(adapter))
+                return -EOPNOTSUPP;
+
+        if (ec->rx_coalesce_usecs_irq ||
+            ec->rx_max_coalesced_frames_irq ||
+            ec->tx_coalesce_usecs ||
+            ec->tx_coalesce_usecs_irq ||
+            ec->tx_max_coalesced_frames_irq ||
+            ec->stats_block_coalesce_usecs ||
+            ec->use_adaptive_tx_coalesce ||
+            ec->pkt_rate_low ||
+            ec->rx_coalesce_usecs_low ||
+            ec->rx_max_coalesced_frames_low ||
+            ec->tx_coalesce_usecs_low ||
+            ec->tx_max_coalesced_frames_low ||
+            ec->pkt_rate_high ||
+            ec->rx_coalesce_usecs_high ||
+            ec->rx_max_coalesced_frames_high ||
+            ec->tx_coalesce_usecs_high ||
+            ec->tx_max_coalesced_frames_high ||
+            ec->rate_sample_interval) {
+                return -EINVAL;
+        }
+
+        if ((ec->rx_coalesce_usecs == 0) &&
+            (ec->use_adaptive_rx_coalesce == 0) &&
+            (ec->tx_max_coalesced_frames == 0) &&
+            (ec->rx_max_coalesced_frames == 0)) {
+                memset(adapter->coal_conf, 0, sizeof(*adapter->coal_conf));
+                adapter->coal_conf->coalMode = VMXNET3_COALESCE_DISABLED;
+                goto done;
+        }
+
+        if (ec->rx_coalesce_usecs != 0) {
+                u32 rbc_rate;
+
+                if ((ec->use_adaptive_rx_coalesce != 0) ||
+                    (ec->tx_max_coalesced_frames != 0) ||
+                    (ec->rx_max_coalesced_frames != 0)) {
+                        return -EINVAL;
+                }
+
+                rbc_rate = VMXNET3_COAL_RBC_RATE(ec->rx_coalesce_usecs);
+                if (rbc_rate < VMXNET3_COAL_RBC_MIN_RATE ||
+                    rbc_rate > VMXNET3_COAL_RBC_MAX_RATE) {
+                        return -EINVAL;
+                }
+
+                memset(adapter->coal_conf, 0, sizeof(*adapter->coal_conf));
+                adapter->coal_conf->coalMode = VMXNET3_COALESCE_RBC;
+                adapter->coal_conf->coalPara.coalRbc.rbc_rate = rbc_rate;
+                goto done;
+        }
+
+        if (ec->use_adaptive_rx_coalesce != 0) {
+                if ((ec->rx_coalesce_usecs != 0) ||
+                    (ec->tx_max_coalesced_frames != 0) ||
+                    (ec->rx_max_coalesced_frames != 0)) {
+                        return -EINVAL;
+                }
+                memset(adapter->coal_conf, 0, sizeof(*adapter->coal_conf));
+                adapter->coal_conf->coalMode = VMXNET3_COALESCE_ADAPT;
+                goto done;
+        }
+
+        if ((ec->tx_max_coalesced_frames != 0) ||
+            (ec->rx_max_coalesced_frames != 0)) {
+                if ((ec->rx_coalesce_usecs != 0) ||
+                    (ec->use_adaptive_rx_coalesce != 0)) {
+                        return -EINVAL;
+                }
+
+                if ((ec->tx_max_coalesced_frames >
+                    VMXNET3_COAL_STATIC_MAX_DEPTH) ||
+                    (ec->rx_max_coalesced_frames >
+                     VMXNET3_COAL_STATIC_MAX_DEPTH)) {
+                        return -EINVAL;
+                }
+
+                memset(adapter->coal_conf, 0, sizeof(*adapter->coal_conf));
+                adapter->coal_conf->coalMode = VMXNET3_COALESCE_STATIC;
+
+                adapter->coal_conf->coalPara.coalStatic.tx_comp_depth =
+                        (ec->tx_max_coalesced_frames ?
+                         ec->tx_max_coalesced_frames :
+                         VMXNET3_COAL_STATIC_DEFAULT_DEPTH);
+
+                adapter->coal_conf->coalPara.coalStatic.rx_depth =
+                        (ec->rx_max_coalesced_frames ?
+                         ec->rx_max_coalesced_frames :
+                         VMXNET3_COAL_STATIC_DEFAULT_DEPTH);
+
+                adapter->coal_conf->coalPara.coalStatic.tx_depth =
+                         VMXNET3_COAL_STATIC_DEFAULT_DEPTH;
+                goto done;
+        }
+
+done:
+        adapter->default_coal_mode = false;
+        if (netif_running(netdev)) {
+                spin_lock_irqsave(&adapter->cmd_lock, flags);
+                cmdInfo->varConf.confVer = 1;
+                cmdInfo->varConf.confLen =
+                        cpu_to_le32(sizeof(*adapter->coal_conf));
+                cmdInfo->varConf.confPA  = cpu_to_le64(adapter->coal_conf_pa);
+                VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
+                                       VMXNET3_CMD_SET_COALESCE);
+                spin_unlock_irqrestore(&adapter->cmd_lock, flags);
+        }
+
+        return 0;
+}
+
 static const struct ethtool_ops vmxnet3_ethtool_ops = {
         .get_settings      = vmxnet3_get_settings,
         .get_drvinfo       = vmxnet3_get_drvinfo,
@@ -706,6 +889,8 @@ static const struct ethtool_ops vmxnet3_ethtool_ops = {
         .get_wol           = vmxnet3_get_wol,
         .set_wol           = vmxnet3_set_wol,
         .get_link          = ethtool_op_get_link,
+        .get_coalesce      = vmxnet3_get_coalesce,
+        .set_coalesce      = vmxnet3_set_coalesce,
         .get_strings       = vmxnet3_get_strings,
         .get_sset_count    = vmxnet3_get_sset_count,
         .get_ethtool_stats = vmxnet3_get_ethtool_stats,
diff --git a/drivers/net/vmxnet3/vmxnet3_int.h b/drivers/net/vmxnet3/vmxnet3_int.h
index 3d2b64e63408..74fc03072b87 100644
--- a/drivers/net/vmxnet3/vmxnet3_int.h
+++ b/drivers/net/vmxnet3/vmxnet3_int.h
@@ -1,7 +1,7 @@
 /*
  * Linux driver for VMware's vmxnet3 ethernet NIC.
  *
- * Copyright (C) 2008-2009, VMware, Inc. All Rights Reserved.
+ * Copyright (C) 2008-2016, VMware, Inc. All Rights Reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -20,7 +20,7 @@
  * The full GNU General Public License is included in this distribution in
  * the file called "COPYING".
  *
- * Maintained by: Shreyas Bhatewara <pv-drivers@vmware.com>
+ * Maintained by: pv-drivers@vmware.com
  *
  */
 
@@ -69,16 +69,20 @@
 /*
  * Version numbers
  */
-#define VMXNET3_DRIVER_VERSION_STRING   "1.4.8.0-k"
+#define VMXNET3_DRIVER_VERSION_STRING   "1.4.9.0-k"
 
 /* a 32-bit int, each byte encode a verion number in VMXNET3_DRIVER_VERSION */
-#define VMXNET3_DRIVER_VERSION_NUM      0x01040800
+#define VMXNET3_DRIVER_VERSION_NUM      0x01040900
 
 #if defined(CONFIG_PCI_MSI)
         /* RSS only makes sense if MSI-X is supported. */
         #define VMXNET3_RSS
 #endif
 
+#define VMXNET3_REV_3           2       /* Vmxnet3 Rev. 3 */
+#define VMXNET3_REV_2           1       /* Vmxnet3 Rev. 2 */
+#define VMXNET3_REV_1           0       /* Vmxnet3 Rev. 1 */
+
 /*
  * Capabilities
  */
@@ -237,6 +241,7 @@ struct vmxnet3_tx_queue {
         int                             num_stop;  /* # of times the queue is
                                                     * stopped */
         int                             qid;
+        u16                             txdata_desc_size;
 } __attribute__((__aligned__(SMP_CACHE_BYTES)));
 
 enum vmxnet3_rx_buf_type {
@@ -267,15 +272,23 @@ struct vmxnet3_rq_driver_stats {
         u64 rx_buf_alloc_failure;
 };
 
+struct vmxnet3_rx_data_ring {
+        Vmxnet3_RxDataDesc *base;
+        dma_addr_t basePA;
+        u16 desc_size;
+};
+
 struct vmxnet3_rx_queue {
         char                    name[IFNAMSIZ + 8]; /* To identify interrupt */
         struct vmxnet3_adapter    *adapter;
         struct napi_struct        napi;
         struct vmxnet3_cmd_ring   rx_ring[2];
+        struct vmxnet3_rx_data_ring data_ring;
         struct vmxnet3_comp_ring  comp_ring;
         struct vmxnet3_rx_ctx     rx_ctx;
         u32 qid;            /* rqID in RCD for buffer from 1st ring */
         u32 qid2;           /* rqID in RCD for buffer from 2nd ring */
+        u32 dataRingQid;    /* rqID in RCD for buffer from data ring */
         struct vmxnet3_rx_buf_info     *buf_info[2];
         dma_addr_t                      buf_info_pa;
         struct Vmxnet3_RxQueueCtrl            *shared;
@@ -345,6 +358,7 @@ struct vmxnet3_adapter {
         int             rx_buf_per_pkt;  /* only apply to the 1st ring */
         dma_addr_t                      shared_pa;
         dma_addr_t queue_desc_pa;
+        dma_addr_t coal_conf_pa;
 
         /* Wake-on-LAN */
         u32     wol;
@@ -359,12 +373,21 @@ struct vmxnet3_adapter {
         u32 rx_ring_size;
         u32 rx_ring2_size;
 
+        /* Size of buffer in the data ring */
+        u16 txdata_desc_size;
+        u16 rxdata_desc_size;
+
+        bool rxdataring_enabled;
+
         struct work_struct work;
 
         unsigned long  state;    /* VMXNET3_STATE_BIT_xxx */
 
         int share_intr;
 
+        struct Vmxnet3_CoalesceScheme *coal_conf;
+        bool   default_coal_mode;
+
         dma_addr_t adapter_pa;
         dma_addr_t pm_conf_pa;
         dma_addr_t rss_conf_pa;
@@ -387,14 +410,34 @@ struct vmxnet3_adapter {
 #define VMXNET3_GET_ADDR_LO(dma)   ((u32)(dma))
 #define VMXNET3_GET_ADDR_HI(dma)   ((u32)(((u64)(dma)) >> 32))
 
+#define VMXNET3_VERSION_GE_2(adapter) \
+        (adapter->version >= VMXNET3_REV_2 + 1)
+#define VMXNET3_VERSION_GE_3(adapter) \
+        (adapter->version >= VMXNET3_REV_3 + 1)
+
 /* must be a multiple of VMXNET3_RING_SIZE_ALIGN */
 #define VMXNET3_DEF_TX_RING_SIZE    512
 #define VMXNET3_DEF_RX_RING_SIZE    256
 #define VMXNET3_DEF_RX_RING2_SIZE   128
 
+#define VMXNET3_DEF_RXDATA_DESC_SIZE 128
+
 #define VMXNET3_MAX_ETH_HDR_SIZE    22
 #define VMXNET3_MAX_SKB_BUF_SIZE    (3*1024)
 
+#define VMXNET3_GET_RING_IDX(adapter, rqID)             \
+        ((rqID >= adapter->num_rx_queues &&             \
+         rqID < 2 * adapter->num_rx_queues) ? 1 : 0)    \
+
+#define VMXNET3_RX_DATA_RING(adapter, rqID)             \
+        (rqID >= 2 * adapter->num_rx_queues &&          \
+        rqID < 3 * adapter->num_rx_queues)              \
+
+#define VMXNET3_COAL_STATIC_DEFAULT_DEPTH       64
+
+#define VMXNET3_COAL_RBC_RATE(usecs) (1000000 / usecs)
+#define VMXNET3_COAL_RBC_USECS(rbc_rate) (1000000 / rbc_rate)
+
 int
 vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter);
 
@@ -418,7 +461,8 @@ vmxnet3_set_features(struct net_device *netdev, netdev_features_t features);
 
 int
 vmxnet3_create_queues(struct vmxnet3_adapter *adapter,
-                      u32 tx_ring_size, u32 rx_ring_size, u32 rx_ring2_size);
+                      u32 tx_ring_size, u32 rx_ring_size, u32 rx_ring2_size,
+                      u16 txdata_desc_size, u16 rxdata_desc_size);
 
 void vmxnet3_set_ethtool_ops(struct net_device *netdev);
 
diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c
index 8bd8c7e1ee87..b3762822b653 100644
--- a/drivers/net/vrf.c
+++ b/drivers/net/vrf.c
@@ -35,6 +35,7 @@
 #include <net/route.h>
 #include <net/addrconf.h>
 #include <net/l3mdev.h>
+#include <net/fib_rules.h>
 
 #define RT_FL_TOS(oldflp4) \
         ((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))
@@ -42,9 +43,14 @@
 #define DRV_NAME        "vrf"
 #define DRV_VERSION     "1.0"
 
+#define FIB_RULE_PREF  1000       /* default preference for FIB rules */
+static bool add_fib_rules = true;
+
 struct net_vrf {
         struct rtable __rcu     *rth;
+        struct rtable __rcu     *rth_local;
         struct rt6_info __rcu   *rt6;
+        struct rt6_info __rcu   *rt6_local;
         u32                     tb_id;
 };
 
@@ -54,9 +60,20 @@ struct pcpu_dstats {
         u64                     tx_drps;
         u64                     rx_pkts;
         u64                     rx_bytes;
+        u64                     rx_drps;
         struct u64_stats_sync   syncp;
 };
 
+static void vrf_rx_stats(struct net_device *dev, int len)
+{
+        struct pcpu_dstats *dstats = this_cpu_ptr(dev->dstats);
+
+        u64_stats_update_begin(&dstats->syncp);
+        dstats->rx_pkts++;
+        dstats->rx_bytes += len;
+        u64_stats_update_end(&dstats->syncp);
+}
+
 static void vrf_tx_error(struct net_device *vrf_dev, struct sk_buff *skb)
 {
         vrf_dev->stats.tx_errors++;
@@ -91,6 +108,34 @@ static struct rtnl_link_stats64 *vrf_get_stats64(struct net_device *dev,
         return stats;
 }
 
+/* Local traffic destined to local address. Reinsert the packet to rx
+ * path, similar to loopback handling.
+ */
+static int vrf_local_xmit(struct sk_buff *skb, struct net_device *dev,
+                          struct dst_entry *dst)
+{
+        int len = skb->len;
+
+        skb_orphan(skb);
+
+        skb_dst_set(skb, dst);
+        skb_dst_force(skb);
+
+        /* set pkt_type to avoid skb hitting packet taps twice -
+         * once on Tx and again in Rx processing
+         */
+        skb->pkt_type = PACKET_LOOPBACK;
+
+        skb->protocol = eth_type_trans(skb, dev);
+
+        if (likely(netif_rx(skb) == NET_RX_SUCCESS))
+                vrf_rx_stats(dev, len);
+        else
+                this_cpu_inc(dev->dstats->rx_drps);
+
+        return NETDEV_TX_OK;
+}
+
 #if IS_ENABLED(CONFIG_IPV6)
 static netdev_tx_t vrf_process_v6_outbound(struct sk_buff *skb,
                                            struct net_device *dev)
@@ -117,8 +162,51 @@ static netdev_tx_t vrf_process_v6_outbound(struct sk_buff *skb,
                 goto err;
 
         skb_dst_drop(skb);
+
+        /* if dst.dev is loopback or the VRF device again this is locally
+         * originated traffic destined to a local address. Short circuit
+         * to Rx path using our local dst
+         */
+        if (dst->dev == net->loopback_dev || dst->dev == dev) {
+                struct net_vrf *vrf = netdev_priv(dev);
+                struct rt6_info *rt6_local;
+
+                /* release looked up dst and use cached local dst */
+                dst_release(dst);
+
+                rcu_read_lock();
+
+                rt6_local = rcu_dereference(vrf->rt6_local);
+                if (unlikely(!rt6_local)) {
+                        rcu_read_unlock();
+                        goto err;
+                }
+
+                /* Ordering issue: cached local dst is created on newlink
+                 * before the IPv6 initialization. Using the local dst
+                 * requires rt6i_idev to be set so make sure it is.
+                 */
+                if (unlikely(!rt6_local->rt6i_idev)) {
+                        rt6_local->rt6i_idev = in6_dev_get(dev);
+                        if (!rt6_local->rt6i_idev) {
+                                rcu_read_unlock();
+                                goto err;
+                        }
+                }
+
+                dst = &rt6_local->dst;
+                dst_hold(dst);
+
+                rcu_read_unlock();
+
+                return vrf_local_xmit(skb, dev, &rt6_local->dst);
+        }
+
         skb_dst_set(skb, dst);
 
+        /* strip the ethernet header added for pass through VRF device */
+        __skb_pull(skb, skb_network_offset(skb));
+
         ret = ip6_local_out(net, skb->sk, skb);
         if (unlikely(net_xmit_eval(ret)))
                 dev->stats.tx_errors++;
@@ -139,29 +227,6 @@ static netdev_tx_t vrf_process_v6_outbound(struct sk_buff *skb,
 }
 #endif
 
-static int vrf_send_v4_prep(struct sk_buff *skb, struct flowi4 *fl4,
-                            struct net_device *vrf_dev)
-{
-        struct rtable *rt;
-        int err = 1;
-
-        rt = ip_route_output_flow(dev_net(vrf_dev), fl4, NULL);
-        if (IS_ERR(rt))
-                goto out;
-
-        /* TO-DO: what about broadcast ? */
-        if (rt->rt_type != RTN_UNICAST && rt->rt_type != RTN_LOCAL) {
-                ip_rt_put(rt);
-                goto out;
-        }
-
-        skb_dst_drop(skb);
-        skb_dst_set(skb, &rt->dst);
-        err = 0;
-out:
-        return err;
-}
-
 static netdev_tx_t vrf_process_v4_outbound(struct sk_buff *skb,
                                            struct net_device *vrf_dev)
 {
@@ -176,9 +241,51 @@ static netdev_tx_t vrf_process_v4_outbound(struct sk_buff *skb,
                                 FLOWI_FLAG_SKIP_NH_OIF,
                 .daddr = ip4h->daddr,
         };
+        struct net *net = dev_net(vrf_dev);
+        struct rtable *rt;
+
+        rt = ip_route_output_flow(net, &fl4, NULL);
+        if (IS_ERR(rt))
+                goto err;
 
-        if (vrf_send_v4_prep(skb, &fl4, vrf_dev))
+        if (rt->rt_type != RTN_UNICAST && rt->rt_type != RTN_LOCAL) {
+                ip_rt_put(rt);
                 goto err;
+        }
+
+        skb_dst_drop(skb);
+
+        /* if dst.dev is loopback or the VRF device again this is locally
+         * originated traffic destined to a local address. Short circuit
+         * to Rx path using our local dst
+         */
+        if (rt->dst.dev == net->loopback_dev || rt->dst.dev == vrf_dev) {
+                struct net_vrf *vrf = netdev_priv(vrf_dev);
+                struct rtable *rth_local;
+                struct dst_entry *dst = NULL;
+
+                ip_rt_put(rt);
+
+                rcu_read_lock();
+
+                rth_local = rcu_dereference(vrf->rth_local);
+                if (likely(rth_local)) {
+                        dst = &rth_local->dst;
+                        dst_hold(dst);
+                }
+
+                rcu_read_unlock();
+
+                if (unlikely(!dst))
+                        goto err;
+
+                return vrf_local_xmit(skb, vrf_dev, dst);
+        }
+
+        skb_dst_set(skb, &rt->dst);
+
+        /* strip the ethernet header added for pass through VRF device */
+        __skb_pull(skb, skb_network_offset(skb));
 
         if (!ip4h->saddr) {
                 ip4h->saddr = inet_select_addr(skb_dst(skb)->dev, 0,
@@ -200,9 +307,6 @@ err:
 
 static netdev_tx_t is_ip_tx_frame(struct sk_buff *skb, struct net_device *dev)
 {
-        /* strip the ethernet header added for pass through VRF device */
-        __skb_pull(skb, skb_network_offset(skb));
-
         switch (skb->protocol) {
         case htons(ETH_P_IP):
                 return vrf_process_v4_outbound(skb, dev);
@@ -274,45 +378,92 @@ static int vrf_output6(struct net *net, struct sock *sk, struct sk_buff *skb)
 }
 
 /* holding rtnl */
-static void vrf_rt6_release(struct net_vrf *vrf)
+static void vrf_rt6_release(struct net_device *dev, struct net_vrf *vrf)
 {
         struct rt6_info *rt6 = rtnl_dereference(vrf->rt6);
+        struct rt6_info *rt6_local = rtnl_dereference(vrf->rt6_local);
+        struct net *net = dev_net(dev);
+        struct dst_entry *dst;
 
-        rcu_assign_pointer(vrf->rt6, NULL);
+        RCU_INIT_POINTER(vrf->rt6, NULL);
+        RCU_INIT_POINTER(vrf->rt6_local, NULL);
+        synchronize_rcu();
+
+        /* move dev in dst's to loopback so this VRF device can be deleted
+         * - based on dst_ifdown
+         */
+        if (rt6) {
+                dst = &rt6->dst;
+                dev_put(dst->dev);
+                dst->dev = net->loopback_dev;
+                dev_hold(dst->dev);
+                dst_release(dst);
+        }
 
-        if (rt6)
-                dst_release(&rt6->dst);
+        if (rt6_local) {
+                if (rt6_local->rt6i_idev)
+                        in6_dev_put(rt6_local->rt6i_idev);
+
+                dst = &rt6_local->dst;
+                dev_put(dst->dev);
+                dst->dev = net->loopback_dev;
+                dev_hold(dst->dev);
+                dst_release(dst);
+        }
 }
 
 static int vrf_rt6_create(struct net_device *dev)
 {
+        int flags = DST_HOST | DST_NOPOLICY | DST_NOXFRM | DST_NOCACHE;
         struct net_vrf *vrf = netdev_priv(dev);
         struct net *net = dev_net(dev);
         struct fib6_table *rt6i_table;
-        struct rt6_info *rt6;
+        struct rt6_info *rt6, *rt6_local;
         int rc = -ENOMEM;
 
+        /* IPv6 can be CONFIG enabled and then disabled runtime */
+        if (!ipv6_mod_enabled())
+                return 0;
+
         rt6i_table = fib6_new_table(net, vrf->tb_id);
         if (!rt6i_table)
                 goto out;
 
-        rt6 = ip6_dst_alloc(net, dev,
-                            DST_HOST | DST_NOPOLICY | DST_NOXFRM | DST_NOCACHE);
+        /* create a dst for routing packets out a VRF device */
+        rt6 = ip6_dst_alloc(net, dev, flags);
         if (!rt6)
                 goto out;
 
         dst_hold(&rt6->dst);
 
         rt6->rt6i_table = rt6i_table;
-        rt6->dst.output = vrf_output6;
+        rt6->dst.output = vrf_output6;
+
+        /* create a dst for local routing - packets sent locally
+         * to local address via the VRF device as a loopback
+         */
+        rt6_local = ip6_dst_alloc(net, dev, flags);
+        if (!rt6_local) {
+                dst_release(&rt6->dst);
+                goto out;
+        }
+
+        dst_hold(&rt6_local->dst);
+
+        rt6_local->rt6i_idev  = in6_dev_get(dev);
+        rt6_local->rt6i_flags = RTF_UP | RTF_NONEXTHOP | RTF_LOCAL;
+        rt6_local->rt6i_table = rt6i_table;
+        rt6_local->dst.input  = ip6_input;
+
         rcu_assign_pointer(vrf->rt6, rt6);
+        rcu_assign_pointer(vrf->rt6_local, rt6_local);
 
         rc = 0;
 out:
         return rc;
 }
 #else
-static void vrf_rt6_release(struct net_vrf *vrf)
+static void vrf_rt6_release(struct net_device *dev, struct net_vrf *vrf)
 {
 }
 
@@ -381,32 +532,66 @@ static int vrf_output(struct net *net, struct sock *sk, struct sk_buff *skb)
 }
 
 /* holding rtnl */
-static void vrf_rtable_release(struct net_vrf *vrf)
+static void vrf_rtable_release(struct net_device *dev, struct net_vrf *vrf)
 {
         struct rtable *rth = rtnl_dereference(vrf->rth);
+        struct rtable *rth_local = rtnl_dereference(vrf->rth_local);
+        struct net *net = dev_net(dev);
+        struct dst_entry *dst;
 
-        rcu_assign_pointer(vrf->rth, NULL);
+        RCU_INIT_POINTER(vrf->rth, NULL);
+        RCU_INIT_POINTER(vrf->rth_local, NULL);
+        synchronize_rcu();
+
+        /* move dev in dst's to loopback so this VRF device can be deleted
+         * - based on dst_ifdown
+         */
+        if (rth) {
+                dst = &rth->dst;
+                dev_put(dst->dev);
+                dst->dev = net->loopback_dev;
+                dev_hold(dst->dev);
+                dst_release(dst);
+        }
 
-        if (rth)
-                dst_release(&rth->dst);
+        if (rth_local) {
+                dst = &rth_local->dst;
+                dev_put(dst->dev);
+                dst->dev = net->loopback_dev;
+                dev_hold(dst->dev);
+                dst_release(dst);
+        }
 }
 
 static int vrf_rtable_create(struct net_device *dev)
 {
         struct net_vrf *vrf = netdev_priv(dev);
-        struct rtable *rth;
+        struct rtable *rth, *rth_local;
 
         if (!fib_new_table(dev_net(dev), vrf->tb_id))
                 return -ENOMEM;
 
+        /* create a dst for routing packets out through a VRF device */
         rth = rt_dst_alloc(dev, 0, RTN_UNICAST, 1, 1, 0);
         if (!rth)
                 return -ENOMEM;
 
-        rth->dst.output = vrf_output;
+        /* create a dst for local ingress routing - packets sent locally
+         * to local address via the VRF device as a loopback
+         */
+        rth_local = rt_dst_alloc(dev, RTCF_LOCAL, RTN_LOCAL, 1, 1, 0);
+        if (!rth_local) {
+                dst_release(&rth->dst);
+                return -ENOMEM;
+        }
+
+        rth->dst.output = vrf_output;
         rth->rt_table_id = vrf->tb_id;
 
+        rth_local->rt_table_id = vrf->tb_id;
+
         rcu_assign_pointer(vrf->rth, rth);
+        rcu_assign_pointer(vrf->rth_local, rth_local);
 
         return 0;
 }
@@ -477,8 +662,8 @@ static void vrf_dev_uninit(struct net_device *dev)
         struct net_device *port_dev;
         struct list_head *iter;
 
-        vrf_rtable_release(vrf);
-        vrf_rt6_release(vrf);
+        vrf_rtable_release(dev, vrf);
+        vrf_rt6_release(dev, vrf);
 
         netdev_for_each_lower_dev(dev, port_dev, iter)
                 vrf_del_slave(dev, port_dev);
@@ -504,10 +689,16 @@ static int vrf_dev_init(struct net_device *dev)
 
         dev->flags = IFF_MASTER | IFF_NOARP;
 
+        /* MTU is irrelevant for VRF device; set to 64k similar to lo */
+        dev->mtu = 64 * 1024;
+
+        /* similarly, oper state is irrelevant; set to up to avoid confusion */
+        dev->operstate = IF_OPER_UP;
+        netdev_lockdep_set_classes(dev);
         return 0;
 
 out_rth:
-        vrf_rtable_release(vrf);
+        vrf_rtable_release(dev, vrf);
 out_stats:
         free_percpu(dev->dstats);
         dev->dstats = NULL;
@@ -623,11 +814,78 @@ out:
         return rc;
 }
 
+static struct rt6_info *vrf_ip6_route_lookup(struct net *net,
+                                             const struct net_device *dev,
+                                             struct flowi6 *fl6,
+                                             int ifindex,
+                                             int flags)
+{
+        struct net_vrf *vrf = netdev_priv(dev);
+        struct fib6_table *table = NULL;
+        struct rt6_info *rt6;
+
+        rcu_read_lock();
+
+        /* fib6_table does not have a refcnt and can not be freed */
+        rt6 = rcu_dereference(vrf->rt6);
+        if (likely(rt6))
+                table = rt6->rt6i_table;
+
+        rcu_read_unlock();
+
+        if (!table)
+                return NULL;
+
+        return ip6_pol_route(net, table, ifindex, fl6, flags);
+}
+
+static void vrf_ip6_input_dst(struct sk_buff *skb, struct net_device *vrf_dev,
+                              int ifindex)
+{
+        const struct ipv6hdr *iph = ipv6_hdr(skb);
+        struct flowi6 fl6 = {
+                .daddr          = iph->daddr,
+                .saddr          = iph->saddr,
+                .flowlabel      = ip6_flowinfo(iph),
+                .flowi6_mark    = skb->mark,
+                .flowi6_proto   = iph->nexthdr,
+                .flowi6_iif     = ifindex,
+        };
+        struct net *net = dev_net(vrf_dev);
+        struct rt6_info *rt6;
+
+        rt6 = vrf_ip6_route_lookup(net, vrf_dev, &fl6, ifindex,
+                                   RT6_LOOKUP_F_HAS_SADDR | RT6_LOOKUP_F_IFACE);
+        if (unlikely(!rt6))
+                return;
+
+        if (unlikely(&rt6->dst == &net->ipv6.ip6_null_entry->dst))
+                return;
+
+        skb_dst_set(skb, &rt6->dst);
+}
+
 static struct sk_buff *vrf_ip6_rcv(struct net_device *vrf_dev,
                                    struct sk_buff *skb)
 {
-        /* if packet is NDISC keep the ingress interface */
-        if (!ipv6_ndisc_frame(skb)) {
+        int orig_iif = skb->skb_iif;
+        bool need_strict;
+
+        /* loopback traffic; do not push through packet taps again.
+         * Reset pkt_type for upper layers to process skb
+         */
+        if (skb->pkt_type == PACKET_LOOPBACK) {
+                skb->dev = vrf_dev;
+                skb->skb_iif = vrf_dev->ifindex;
+                skb->pkt_type = PACKET_HOST;
+                goto out;
+        }
+
+        /* if packet is NDISC or addressed to multicast or link-local
+         * then keep the ingress interface
+         */
+        need_strict = rt6_need_strict(&ipv6_hdr(skb)->daddr);
+        if (!ipv6_ndisc_frame(skb) && !need_strict) {
                 skb->dev = vrf_dev;
                 skb->skb_iif = vrf_dev->ifindex;
 
@@ -638,6 +896,10 @@ static struct sk_buff *vrf_ip6_rcv(struct net_device *vrf_dev,
                 IP6CB(skb)->flags |= IP6SKB_L3SLAVE;
         }
 
+        if (need_strict)
+                vrf_ip6_input_dst(skb, vrf_dev, orig_iif);
+
+out:
         return skb;
 }
 
@@ -655,10 +917,19 @@ static struct sk_buff *vrf_ip_rcv(struct net_device *vrf_dev,
         skb->dev = vrf_dev;
         skb->skb_iif = vrf_dev->ifindex;
 
+        /* loopback traffic; do not push through packet taps again.
+         * Reset pkt_type for upper layers to process skb
+         */
+        if (skb->pkt_type == PACKET_LOOPBACK) {
+                skb->pkt_type = PACKET_HOST;
+                goto out;
+        }
+
         skb_push(skb, skb->mac_len);
         dev_queue_xmit_nit(skb, vrf_dev);
         skb_pull(skb, skb->mac_len);
 
+out:
         return skb;
 }
 
@@ -679,13 +950,37 @@ static struct sk_buff *vrf_l3_rcv(struct net_device *vrf_dev,
 
 #if IS_ENABLED(CONFIG_IPV6)
 static struct dst_entry *vrf_get_rt6_dst(const struct net_device *dev,
-                                         const struct flowi6 *fl6)
+                                         struct flowi6 *fl6)
 {
+        bool need_strict = rt6_need_strict(&fl6->daddr);
+        struct net_vrf *vrf = netdev_priv(dev);
+        struct net *net = dev_net(dev);
         struct dst_entry *dst = NULL;
+        struct rt6_info *rt;
 
-        if (!(fl6->flowi6_flags & FLOWI_FLAG_L3MDEV_SRC)) {
-                struct net_vrf *vrf = netdev_priv(dev);
-                struct rt6_info *rt;
+        /* send to link-local or multicast address */
+        if (need_strict) {
+                int flags = RT6_LOOKUP_F_IFACE;
+
+                /* VRF device does not have a link-local address and
+                 * sending packets to link-local or mcast addresses over
+                 * a VRF device does not make sense
+                 */
+                if (fl6->flowi6_oif == dev->ifindex) {
+                        struct dst_entry *dst = &net->ipv6.ip6_null_entry->dst;
+
+                        dst_hold(dst);
+                        return dst;
+                }
+
+                if (!ipv6_addr_any(&fl6->saddr))
+                        flags |= RT6_LOOKUP_F_HAS_SADDR;
+
+                rt = vrf_ip6_route_lookup(net, dev, fl6, fl6->flowi6_oif, flags);
+                if (rt)
+                        dst = &rt->dst;
+
+        } else if (!(fl6->flowi6_flags & FLOWI_FLAG_L3MDEV_SRC)) {
 
                 rcu_read_lock();
 
@@ -698,8 +993,52 @@ static struct dst_entry *vrf_get_rt6_dst(const struct net_device *dev,
                 rcu_read_unlock();
         }
 
+        /* make sure oif is set to VRF device for lookup */
+        if (!need_strict)
+                fl6->flowi6_oif = dev->ifindex;
+
         return dst;
 }
+
+/* called under rcu_read_lock */
+static int vrf_get_saddr6(struct net_device *dev, const struct sock *sk,
+                          struct flowi6 *fl6)
+{
+        struct net *net = dev_net(dev);
+        struct dst_entry *dst;
+        struct rt6_info *rt;
+        int err;
+
+        if (rt6_need_strict(&fl6->daddr)) {
+                rt = vrf_ip6_route_lookup(net, dev, fl6, fl6->flowi6_oif,
+                                          RT6_LOOKUP_F_IFACE);
+                if (unlikely(!rt))
+                        return 0;
+
+                dst = &rt->dst;
+        } else {
+                __u8 flags = fl6->flowi6_flags;
+
+                fl6->flowi6_flags |= FLOWI_FLAG_L3MDEV_SRC;
+                fl6->flowi6_flags |= FLOWI_FLAG_SKIP_NH_OIF;
+
+                dst = ip6_route_output(net, sk, fl6);
+                rt = (struct rt6_info *)dst;
+
+                fl6->flowi6_flags = flags;
+        }
+
+        err = dst->error;
+        if (!err) {
+                err = ip6_route_get_saddr(net, rt, &fl6->daddr,
+                                          sk ? inet6_sk(sk)->srcprefs : 0,
+                                          &fl6->saddr);
+        }
+
+        dst_release(dst);
+
+        return err;
+}
 #endif
 
 static const struct l3mdev_ops vrf_l3mdev_ops = {
@@ -709,6 +1048,7 @@ static const struct l3mdev_ops vrf_l3mdev_ops = {
         .l3mdev_l3_rcv          = vrf_l3_rcv,
 #if IS_ENABLED(CONFIG_IPV6)
         .l3mdev_get_rt6_dst     = vrf_get_rt6_dst,
+        .l3mdev_get_saddr6      = vrf_get_saddr6,
 #endif
 };
 
@@ -723,6 +1063,94 @@ static const struct ethtool_ops vrf_ethtool_ops = {
         .get_drvinfo    = vrf_get_drvinfo,
 };
 
+static inline size_t vrf_fib_rule_nl_size(void)
+{
+        size_t sz;
+
+        sz  = NLMSG_ALIGN(sizeof(struct fib_rule_hdr));
+        sz += nla_total_size(sizeof(u8));       /* FRA_L3MDEV */
+        sz += nla_total_size(sizeof(u32));      /* FRA_PRIORITY */
+
+        return sz;
+}
+
+static int vrf_fib_rule(const struct net_device *dev, __u8 family, bool add_it)
+{
+        struct fib_rule_hdr *frh;
+        struct nlmsghdr *nlh;
+        struct sk_buff *skb;
+        int err;
+
+        if (family == AF_INET6 && !ipv6_mod_enabled())
+                return 0;
+
+        skb = nlmsg_new(vrf_fib_rule_nl_size(), GFP_KERNEL);
+        if (!skb)
+                return -ENOMEM;
+
+        nlh = nlmsg_put(skb, 0, 0, 0, sizeof(*frh), 0);
+        if (!nlh)
+                goto nla_put_failure;
+
+        /* rule only needs to appear once */
+        nlh->nlmsg_flags &= NLM_F_EXCL;
+
+        frh = nlmsg_data(nlh);
+        memset(frh, 0, sizeof(*frh));
+        frh->family = family;
+        frh->action = FR_ACT_TO_TBL;
+
+        if (nla_put_u32(skb, FRA_L3MDEV, 1))
+                goto nla_put_failure;
+
+        if (nla_put_u32(skb, FRA_PRIORITY, FIB_RULE_PREF))
+                goto nla_put_failure;
+
+        nlmsg_end(skb, nlh);
+
+        /* fib_nl_{new,del}rule handling looks for net from skb->sk */
+        skb->sk = dev_net(dev)->rtnl;
+        if (add_it) {
+                err = fib_nl_newrule(skb, nlh);
+                if (err == -EEXIST)
+                        err = 0;
+        } else {
+                err = fib_nl_delrule(skb, nlh);
+                if (err == -ENOENT)
+                        err = 0;
+        }
+        nlmsg_free(skb);
+
+        return err;
+
+nla_put_failure:
+        nlmsg_free(skb);
+
+        return -EMSGSIZE;
+}
+
+static int vrf_add_fib_rules(const struct net_device *dev)
+{
+        int err;
+
+        err = vrf_fib_rule(dev, AF_INET,  true);
+        if (err < 0)
+                goto out_err;
+
+        err = vrf_fib_rule(dev, AF_INET6, true);
+        if (err < 0)
+                goto ipv6_err;
+
+        return 0;
+
+ipv6_err:
+        vrf_fib_rule(dev, AF_INET,  false);
+
+out_err:
+        netdev_err(dev, "Failed to add FIB rules.\n");
+        return err;
+}
+
 static void vrf_setup(struct net_device *dev)
 {
         ether_setup(dev);
@@ -741,6 +1169,20 @@ static void vrf_setup(struct net_device *dev)
 
         /* don't allow vrf devices to change network namespaces. */
         dev->features |= NETIF_F_NETNS_LOCAL;
+
+        /* does not make sense for a VLAN to be added to a vrf device */
+        dev->features   |= NETIF_F_VLAN_CHALLENGED;
+
+        /* enable offload features */
+        dev->features   |= NETIF_F_GSO_SOFTWARE;
+        dev->features   |= NETIF_F_RXCSUM | NETIF_F_HW_CSUM;
+        dev->features   |= NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HIGHDMA;
+
+        dev->hw_features = dev->features;
+        dev->hw_enc_features = dev->features;
+
+        /* default to no qdisc; user can add if desired */
+        dev->priv_flags |= IFF_NO_QUEUE;
 }
 
 static int vrf_validate(struct nlattr *tb[], struct nlattr *data[])
@@ -763,6 +1205,7 @@ static int vrf_newlink(struct net *src_net, struct net_device *dev,
                        struct nlattr *tb[], struct nlattr *data[])
 {
         struct net_vrf *vrf = netdev_priv(dev);
+        int err;
 
         if (!data || !data[IFLA_VRF_TABLE])
                 return -EINVAL;
@@ -771,7 +1214,21 @@ static int vrf_newlink(struct net *src_net, struct net_device *dev,
 
         dev->priv_flags |= IFF_L3MDEV_MASTER;
 
-        return register_netdevice(dev);
+        err = register_netdevice(dev);
+        if (err)
+                goto out;
+
+        if (add_fib_rules) {
+                err = vrf_add_fib_rules(dev);
+                if (err) {
+                        unregister_netdevice(dev);
+                        goto out;
+                }
+                add_fib_rules = false;
+        }
+
+out:
+        return err;
 }
 
 static size_t vrf_nl_getsize(const struct net_device *dev)
diff --git a/drivers/net/vxlan.c b/drivers/net/vxlan.c
index b3b9db68f758..ae7455da1687 100644
--- a/drivers/net/vxlan.c
+++ b/drivers/net/vxlan.c
@@ -11,32 +11,18 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include <linux/kernel.h>
-#include <linux/types.h>
 #include <linux/module.h>
 #include <linux/errno.h>
 #include <linux/slab.h>
-#include <linux/skbuff.h>
-#include <linux/rculist.h>
-#include <linux/netdevice.h>
-#include <linux/in.h>
-#include <linux/ip.h>
 #include <linux/udp.h>
 #include <linux/igmp.h>
-#include <linux/etherdevice.h>
 #include <linux/if_ether.h>
-#include <linux/if_vlan.h>
-#include <linux/hash.h>
 #include <linux/ethtool.h>
 #include <net/arp.h>
 #include <net/ndisc.h>
 #include <net/ip.h>
-#include <net/ip_tunnels.h>
 #include <net/icmp.h>
-#include <net/udp.h>
-#include <net/udp_tunnel.h>
 #include <net/rtnetlink.h>
-#include <net/route.h>
-#include <net/dsfield.h>
 #include <net/inet_ecn.h>
 #include <net/net_namespace.h>
 #include <net/netns/generic.h>
@@ -44,12 +30,9 @@
 #include <net/protocol.h>
 
 #if IS_ENABLED(CONFIG_IPV6)
-#include <net/ipv6.h>
-#include <net/addrconf.h>
 #include <net/ip6_tunnel.h>
 #include <net/ip6_checksum.h>
 #endif
-#include <net/dst_metadata.h>
 
 #define VXLAN_VERSION   "0.1"
 
@@ -619,42 +602,6 @@ static int vxlan_gro_complete(struct sock *sk, struct sk_buff *skb, int nhoff)
         return eth_gro_complete(skb, nhoff + sizeof(struct vxlanhdr));
 }
 
-/* Notify netdevs that UDP port started listening */
-static void vxlan_notify_add_rx_port(struct vxlan_sock *vs)
-{
-        struct net_device *dev;
-        struct sock *sk = vs->sock->sk;
-        struct net *net = sock_net(sk);
-        sa_family_t sa_family = vxlan_get_sk_family(vs);
-        __be16 port = inet_sk(sk)->inet_sport;
-
-        rcu_read_lock();
-        for_each_netdev_rcu(net, dev) {
-                if (dev->netdev_ops->ndo_add_vxlan_port)
-                        dev->netdev_ops->ndo_add_vxlan_port(dev, sa_family,
-                                                            port);
-        }
-        rcu_read_unlock();
-}
-
-/* Notify netdevs that UDP port is no more listening */
-static void vxlan_notify_del_rx_port(struct vxlan_sock *vs)
-{
-        struct net_device *dev;
-        struct sock *sk = vs->sock->sk;
-        struct net *net = sock_net(sk);
-        sa_family_t sa_family = vxlan_get_sk_family(vs);
-        __be16 port = inet_sk(sk)->inet_sport;
-
-        rcu_read_lock();
-        for_each_netdev_rcu(net, dev) {
-                if (dev->netdev_ops->ndo_del_vxlan_port)
-                        dev->netdev_ops->ndo_del_vxlan_port(dev, sa_family,
-                                                            port);
-        }
-        rcu_read_unlock();
-}
-
 /* Add new entry to forwarding table -- assumes lock held */
 static int vxlan_fdb_create(struct vxlan_dev *vxlan,
                             const u8 *mac, union vxlan_addr *ip,
@@ -1050,7 +997,10 @@ static bool __vxlan_sock_release_prep(struct vxlan_sock *vs)
         vn = net_generic(sock_net(vs->sock->sk), vxlan_net_id);
         spin_lock(&vn->sock_lock);
         hlist_del_rcu(&vs->hlist);
-        vxlan_notify_del_rx_port(vs);
+        udp_tunnel_notify_del_rx_port(vs->sock,
+                                      (vs->flags & VXLAN_F_GPE) ?
+                                      UDP_TUNNEL_TYPE_VXLAN_GPE :
+                                      UDP_TUNNEL_TYPE_VXLAN);
         spin_unlock(&vn->sock_lock);
 
         return true;
@@ -2525,30 +2475,24 @@ static struct device_type vxlan_type = {
         .name = "vxlan",
 };
 
-/* Calls the ndo_add_vxlan_port of the caller in order to
+/* Calls the ndo_add_udp_enc_port of the caller in order to
  * supply the listening VXLAN udp ports. Callers are expected
- * to implement the ndo_add_vxlan_port.
+ * to implement the ndo_add_udp_enc_port.
  */
 static void vxlan_push_rx_ports(struct net_device *dev)
 {
         struct vxlan_sock *vs;
         struct net *net = dev_net(dev);
         struct vxlan_net *vn = net_generic(net, vxlan_net_id);
-        sa_family_t sa_family;
-        __be16 port;
         unsigned int i;
 
-        if (!dev->netdev_ops->ndo_add_vxlan_port)
-                return;
-
         spin_lock(&vn->sock_lock);
         for (i = 0; i < PORT_HASH_SIZE; ++i) {
-                hlist_for_each_entry_rcu(vs, &vn->sock_list[i], hlist) {
-                        port = inet_sk(vs->sock->sk)->inet_sport;
-                        sa_family = vxlan_get_sk_family(vs);
-                        dev->netdev_ops->ndo_add_vxlan_port(dev, sa_family,
-                                                            port);
-                }
+                hlist_for_each_entry_rcu(vs, &vn->sock_list[i], hlist)
+                        udp_tunnel_push_rx_port(dev, vs->sock,
+                                                (vs->flags & VXLAN_F_GPE) ?
+                                                UDP_TUNNEL_TYPE_VXLAN_GPE :
+                                                UDP_TUNNEL_TYPE_VXLAN);
         }
         spin_unlock(&vn->sock_lock);
 }
@@ -2750,7 +2694,10 @@ static struct vxlan_sock *vxlan_socket_create(struct net *net, bool ipv6,
 
         spin_lock(&vn->sock_lock);
         hlist_add_head_rcu(&vs->hlist, vs_head(net, port));
-        vxlan_notify_add_rx_port(vs);
+        udp_tunnel_notify_add_rx_port(sock,
+                                      (vs->flags & VXLAN_F_GPE) ?
+                                      UDP_TUNNEL_TYPE_VXLAN_GPE :
+                                      UDP_TUNNEL_TYPE_VXLAN);
         spin_unlock(&vn->sock_lock);
 
         /* Mark socket as an encapsulation socket. */
@@ -3308,7 +3255,7 @@ static int vxlan_netdevice_event(struct notifier_block *unused,
 
         if (event == NETDEV_UNREGISTER)
                 vxlan_handle_lowerdev_unregister(vn, dev);
-        else if (event == NETDEV_OFFLOAD_PUSH_VXLAN)
+        else if (event == NETDEV_UDP_TUNNEL_PUSH_INFO)
                 vxlan_push_rx_ports(dev);
 
         return NOTIFY_DONE;
diff --git a/drivers/net/wan/Kconfig b/drivers/net/wan/Kconfig
index a2fdd15f285a..33ab3345d333 100644
--- a/drivers/net/wan/Kconfig
+++ b/drivers/net/wan/Kconfig
@@ -280,6 +280,28 @@ config DSCC4
           To compile this driver as a module, choose M here: the
           module will be called dscc4.
 
+config FSL_UCC_HDLC
+        tristate "Freescale QUICC Engine HDLC support"
+        depends on HDLC
+        depends on QUICC_ENGINE
+        help
+          Driver for Freescale QUICC Engine HDLC controller. The driver
+          supports HDLC in NMSI and TDM mode.
+
+          To compile this driver as a module, choose M here: the
+          module will be called fsl_ucc_hdlc.
+
+config SLIC_DS26522
+        tristate "Slic Maxim ds26522 card support"
+        depends on SPI
+        depends on FSL_SOC || ARCH_MXC || ARCH_LAYERSCAPE
+        help
+          This module initializes and configures the slic maxim card
+          in T1 or E1 mode.
+
+          To compile this driver as a module, choose M here: the
+          module will be called slic_ds26522.
+
 config DSCC4_PCISYNC
         bool "Etinc PCISYNC features"
         depends on DSCC4
diff --git a/drivers/net/wan/Makefile b/drivers/net/wan/Makefile
index c135ef47cbca..73c2326603fc 100644
--- a/drivers/net/wan/Makefile
+++ b/drivers/net/wan/Makefile
@@ -32,6 +32,8 @@ obj-$(CONFIG_WANXL)		+= wanxl.o
 obj-$(CONFIG_PCI200SYN)         += pci200syn.o
 obj-$(CONFIG_PC300TOO)          += pc300too.o
 obj-$(CONFIG_IXP4XX_HSS)        += ixp4xx_hss.o
+obj-$(CONFIG_FSL_UCC_HDLC)      += fsl_ucc_hdlc.o
+obj-$(CONFIG_SLIC_DS26522)      += slic_ds26522.o
 
 clean-files := wanxlfw.inc
 $(obj)/wanxl.o: $(obj)/wanxlfw.inc
diff --git a/drivers/net/wan/fsl_ucc_hdlc.c b/drivers/net/wan/fsl_ucc_hdlc.c
new file mode 100644
index 000000000000..19174ac1e338
--- /dev/null
+++ b/drivers/net/wan/fsl_ucc_hdlc.c
@@ -0,0 +1,1192 @@
+/* Freescale QUICC Engine HDLC Device Driver
+ *
+ * Copyright 2016 Freescale Semiconductor Inc.
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/hdlc.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/irq.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/sched.h>
+#include <linux/skbuff.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/stddef.h>
+#include <soc/fsl/qe/qe_tdm.h>
+#include <uapi/linux/if_arp.h>
+
+#include "fsl_ucc_hdlc.h"
+
+#define DRV_DESC "Freescale QE UCC HDLC Driver"
+#define DRV_NAME "ucc_hdlc"
+
+#define TDM_PPPOHT_SLIC_MAXIN
+#define BROKEN_FRAME_INFO
+
+static struct ucc_tdm_info utdm_primary_info = {
+        .uf_info = {
+                .tsa = 0,
+                .cdp = 0,
+                .cds = 1,
+                .ctsp = 1,
+                .ctss = 1,
+                .revd = 0,
+                .urfs = 256,
+                .utfs = 256,
+                .urfet = 128,
+                .urfset = 192,
+                .utfet = 128,
+                .utftt = 0x40,
+                .ufpt = 256,
+                .mode = UCC_FAST_PROTOCOL_MODE_HDLC,
+                .ttx_trx = UCC_FAST_GUMR_TRANSPARENT_TTX_TRX_NORMAL,
+                .tenc = UCC_FAST_TX_ENCODING_NRZ,
+                .renc = UCC_FAST_RX_ENCODING_NRZ,
+                .tcrc = UCC_FAST_16_BIT_CRC,
+                .synl = UCC_FAST_SYNC_LEN_NOT_USED,
+        },
+
+        .si_info = {
+#ifdef TDM_PPPOHT_SLIC_MAXIN
+                .simr_rfsd = 1,
+                .simr_tfsd = 2,
+#else
+                .simr_rfsd = 0,
+                .simr_tfsd = 0,
+#endif
+                .simr_crt = 0,
+                .simr_sl = 0,
+                .simr_ce = 1,
+                .simr_fe = 1,
+                .simr_gm = 0,
+        },
+};
+
+static struct ucc_tdm_info utdm_info[MAX_HDLC_NUM];
+
+static int uhdlc_init(struct ucc_hdlc_private *priv)
+{
+        struct ucc_tdm_info *ut_info;
+        struct ucc_fast_info *uf_info;
+        u32 cecr_subblock;
+        u16 bd_status;
+        int ret, i;
+        void *bd_buffer;
+        dma_addr_t bd_dma_addr;
+        u32 riptr;
+        u32 tiptr;
+        u32 gumr;
+
+        ut_info = priv->ut_info;
+        uf_info = &ut_info->uf_info;
+
+        if (priv->tsa) {
+                uf_info->tsa = 1;
+                uf_info->ctsp = 1;
+        }
+        uf_info->uccm_mask = ((UCC_HDLC_UCCE_RXB | UCC_HDLC_UCCE_RXF |
+                                UCC_HDLC_UCCE_TXB) << 16);
+
+        ret = ucc_fast_init(uf_info, &priv->uccf);
+        if (ret) {
+                dev_err(priv->dev, "Failed to init uccf.");
+                return ret;
+        }
+
+        priv->uf_regs = priv->uccf->uf_regs;
+        ucc_fast_disable(priv->uccf, COMM_DIR_RX | COMM_DIR_TX);
+
+        /* Loopback mode */
+        if (priv->loopback) {
+                dev_info(priv->dev, "Loopback Mode\n");
+                gumr = ioread32be(&priv->uf_regs->gumr);
+                gumr |= (UCC_FAST_GUMR_LOOPBACK | UCC_FAST_GUMR_CDS |
+                         UCC_FAST_GUMR_TCI);
+                gumr &= ~(UCC_FAST_GUMR_CTSP | UCC_FAST_GUMR_RSYN);
+                iowrite32be(gumr, &priv->uf_regs->gumr);
+        }
+
+        /* Initialize SI */
+        if (priv->tsa)
+                ucc_tdm_init(priv->utdm, priv->ut_info);
+
+        /* Write to QE CECR, UCCx channel to Stop Transmission */
+        cecr_subblock = ucc_fast_get_qe_cr_subblock(uf_info->ucc_num);
+        ret = qe_issue_cmd(QE_STOP_TX, cecr_subblock,
+                           QE_CR_PROTOCOL_UNSPECIFIED, 0);
+
+        /* Set UPSMR normal mode (need fixed)*/
+        iowrite32be(0, &priv->uf_regs->upsmr);
+
+        priv->rx_ring_size = RX_BD_RING_LEN;
+        priv->tx_ring_size = TX_BD_RING_LEN;
+        /* Alloc Rx BD */
+        priv->rx_bd_base = dma_alloc_coherent(priv->dev,
+                        RX_BD_RING_LEN * sizeof(struct qe_bd *),
+                        &priv->dma_rx_bd, GFP_KERNEL);
+
+        if (!priv->rx_bd_base) {
+                dev_err(priv->dev, "Cannot allocate MURAM memory for RxBDs\n");
+                ret = -ENOMEM;
+                goto rxbd_alloc_error;
+        }
+
+        /* Alloc Tx BD */
+        priv->tx_bd_base = dma_alloc_coherent(priv->dev,
+                        TX_BD_RING_LEN * sizeof(struct qe_bd *),
+                        &priv->dma_tx_bd, GFP_KERNEL);
+
+        if (!priv->tx_bd_base) {
+                dev_err(priv->dev, "Cannot allocate MURAM memory for TxBDs\n");
+                ret = -ENOMEM;
+                goto txbd_alloc_error;
+        }
+
+        /* Alloc parameter ram for ucc hdlc */
+        priv->ucc_pram_offset = qe_muram_alloc(sizeof(priv->ucc_pram),
+                                ALIGNMENT_OF_UCC_HDLC_PRAM);
+
+        if (priv->ucc_pram_offset < 0) {
+                dev_err(priv->dev, "Can not allocate MURAM for hdlc prameter.\n");
+                ret = -ENOMEM;
+                goto pram_alloc_error;
+        }
+
+        priv->rx_skbuff = kzalloc(priv->rx_ring_size * sizeof(*priv->rx_skbuff),
+                                  GFP_KERNEL);
+        if (!priv->rx_skbuff)
+                goto rx_skb_alloc_error;
+
+        priv->tx_skbuff = kzalloc(priv->tx_ring_size * sizeof(*priv->tx_skbuff),
+                                  GFP_KERNEL);
+        if (!priv->tx_skbuff)
+                goto tx_skb_alloc_error;
+
+        priv->skb_curtx = 0;
+        priv->skb_dirtytx = 0;
+        priv->curtx_bd = priv->tx_bd_base;
+        priv->dirty_tx = priv->tx_bd_base;
+        priv->currx_bd = priv->rx_bd_base;
+        priv->currx_bdnum = 0;
+
+        /* init parameter base */
+        cecr_subblock = ucc_fast_get_qe_cr_subblock(uf_info->ucc_num);
+        ret = qe_issue_cmd(QE_ASSIGN_PAGE_TO_DEVICE, cecr_subblock,
+                           QE_CR_PROTOCOL_UNSPECIFIED, priv->ucc_pram_offset);
+
+        priv->ucc_pram = (struct ucc_hdlc_param __iomem *)
+                                        qe_muram_addr(priv->ucc_pram_offset);
+
+        /* Zero out parameter ram */
+        memset_io(priv->ucc_pram, 0, sizeof(struct ucc_hdlc_param));
+
+        /* Alloc riptr, tiptr */
+        riptr = qe_muram_alloc(32, 32);
+        if (riptr < 0) {
+                dev_err(priv->dev, "Cannot allocate MURAM mem for Receive internal temp data pointer\n");
+                ret = -ENOMEM;
+                goto riptr_alloc_error;
+        }
+
+        tiptr = qe_muram_alloc(32, 32);
+        if (tiptr < 0) {
+                dev_err(priv->dev, "Cannot allocate MURAM mem for Transmit internal temp data pointer\n");
+                ret = -ENOMEM;
+                goto tiptr_alloc_error;
+        }
+
+        /* Set RIPTR, TIPTR */
+        iowrite16be(riptr, &priv->ucc_pram->riptr);
+        iowrite16be(tiptr, &priv->ucc_pram->tiptr);
+
+        /* Set MRBLR */
+        iowrite16be(MAX_RX_BUF_LENGTH, &priv->ucc_pram->mrblr);
+
+        /* Set RBASE, TBASE */
+        iowrite32be(priv->dma_rx_bd, &priv->ucc_pram->rbase);
+        iowrite32be(priv->dma_tx_bd, &priv->ucc_pram->tbase);
+
+        /* Set RSTATE, TSTATE */
+        iowrite32be(BMR_GBL | BMR_BIG_ENDIAN, &priv->ucc_pram->rstate);
+        iowrite32be(BMR_GBL | BMR_BIG_ENDIAN, &priv->ucc_pram->tstate);
+
+        /* Set C_MASK, C_PRES for 16bit CRC */
+        iowrite32be(CRC_16BIT_MASK, &priv->ucc_pram->c_mask);
+        iowrite32be(CRC_16BIT_PRES, &priv->ucc_pram->c_pres);
+
+        iowrite16be(MAX_FRAME_LENGTH, &priv->ucc_pram->mflr);
+        iowrite16be(DEFAULT_RFTHR, &priv->ucc_pram->rfthr);
+        iowrite16be(DEFAULT_RFTHR, &priv->ucc_pram->rfcnt);
+        iowrite16be(DEFAULT_ADDR_MASK, &priv->ucc_pram->hmask);
+        iowrite16be(DEFAULT_HDLC_ADDR, &priv->ucc_pram->haddr1);
+        iowrite16be(DEFAULT_HDLC_ADDR, &priv->ucc_pram->haddr2);
+        iowrite16be(DEFAULT_HDLC_ADDR, &priv->ucc_pram->haddr3);
+        iowrite16be(DEFAULT_HDLC_ADDR, &priv->ucc_pram->haddr4);
+
+        /* Get BD buffer */
+        bd_buffer = dma_alloc_coherent(priv->dev,
+                                       (RX_BD_RING_LEN + TX_BD_RING_LEN) *
+                                       MAX_RX_BUF_LENGTH,
+                                       &bd_dma_addr, GFP_KERNEL);
+
+        if (!bd_buffer) {
+                dev_err(priv->dev, "Could not allocate buffer descriptors\n");
+                ret = -ENOMEM;
+                goto bd_alloc_error;
+        }
+
+        memset(bd_buffer, 0, (RX_BD_RING_LEN + TX_BD_RING_LEN)
+                        * MAX_RX_BUF_LENGTH);
+
+        priv->rx_buffer = bd_buffer;
+        priv->tx_buffer = bd_buffer + RX_BD_RING_LEN * MAX_RX_BUF_LENGTH;
+
+        priv->dma_rx_addr = bd_dma_addr;
+        priv->dma_tx_addr = bd_dma_addr + RX_BD_RING_LEN * MAX_RX_BUF_LENGTH;
+
+        for (i = 0; i < RX_BD_RING_LEN; i++) {
+                if (i < (RX_BD_RING_LEN - 1))
+                        bd_status = R_E_S | R_I_S;
+                else
+                        bd_status = R_E_S | R_I_S | R_W_S;
+
+                iowrite16be(bd_status, &priv->rx_bd_base[i].status);
+                iowrite32be(priv->dma_rx_addr + i * MAX_RX_BUF_LENGTH,
+                            &priv->rx_bd_base[i].buf);
+        }
+
+        for (i = 0; i < TX_BD_RING_LEN; i++) {
+                if (i < (TX_BD_RING_LEN - 1))
+                        bd_status =  T_I_S | T_TC_S;
+                else
+                        bd_status =  T_I_S | T_TC_S | T_W_S;
+
+                iowrite16be(bd_status, &priv->tx_bd_base[i].status);
+                iowrite32be(priv->dma_tx_addr + i * MAX_RX_BUF_LENGTH,
+                            &priv->tx_bd_base[i].buf);
+        }
+
+        return 0;
+
+bd_alloc_error:
+        qe_muram_free(tiptr);
+tiptr_alloc_error:
+        qe_muram_free(riptr);
+riptr_alloc_error:
+        kfree(priv->tx_skbuff);
+tx_skb_alloc_error:
+        kfree(priv->rx_skbuff);
+rx_skb_alloc_error:
+        qe_muram_free(priv->ucc_pram_offset);
+pram_alloc_error:
+        dma_free_coherent(priv->dev,
+                          TX_BD_RING_LEN * sizeof(struct qe_bd),
+                          priv->tx_bd_base, priv->dma_tx_bd);
+txbd_alloc_error:
+        dma_free_coherent(priv->dev,
+                          RX_BD_RING_LEN * sizeof(struct qe_bd),
+                          priv->rx_bd_base, priv->dma_rx_bd);
+rxbd_alloc_error:
+        ucc_fast_free(priv->uccf);
+
+        return ret;
+}
+
+static netdev_tx_t ucc_hdlc_tx(struct sk_buff *skb, struct net_device *dev)
+{
+        hdlc_device *hdlc = dev_to_hdlc(dev);
+        struct ucc_hdlc_private *priv = (struct ucc_hdlc_private *)hdlc->priv;
+        struct qe_bd __iomem *bd;
+        u16 bd_status;
+        unsigned long flags;
+        u8 *send_buf;
+        int i;
+        u16 *proto_head;
+
+        switch (dev->type) {
+        case ARPHRD_RAWHDLC:
+                if (skb_headroom(skb) < HDLC_HEAD_LEN) {
+                        dev->stats.tx_dropped++;
+                        dev_kfree_skb(skb);
+                        netdev_err(dev, "No enough space for hdlc head\n");
+                        return -ENOMEM;
+                }
+
+                skb_push(skb, HDLC_HEAD_LEN);
+
+                proto_head = (u16 *)skb->data;
+                *proto_head = htons(DEFAULT_HDLC_HEAD);
+
+                dev->stats.tx_bytes += skb->len;
+                break;
+
+        case ARPHRD_PPP:
+                proto_head = (u16 *)skb->data;
+                if (*proto_head != htons(DEFAULT_PPP_HEAD)) {
+                        dev->stats.tx_dropped++;
+                        dev_kfree_skb(skb);
+                        netdev_err(dev, "Wrong ppp header\n");
+                        return -ENOMEM;
+                }
+
+                dev->stats.tx_bytes += skb->len;
+                break;
+
+        default:
+                dev->stats.tx_dropped++;
+                dev_kfree_skb(skb);
+                return -ENOMEM;
+        }
+
+        pr_info("Tx data skb->len:%d ", skb->len);
+        send_buf = (u8 *)skb->data;
+        pr_info("\nTransmitted data:\n");
+        for (i = 0; i < 16; i++) {
+                if (i == skb->len)
+                        pr_info("++++");
+                else
+                pr_info("%02x\n", send_buf[i]);
+        }
+        spin_lock_irqsave(&priv->lock, flags);
+
+        /* Start from the next BD that should be filled */
+        bd = priv->curtx_bd;
+        bd_status = ioread16be(&bd->status);
+        /* Save the skb pointer so we can free it later */
+        priv->tx_skbuff[priv->skb_curtx] = skb;
+
+        /* Update the current skb pointer (wrapping if this was the last) */
+        priv->skb_curtx =
+            (priv->skb_curtx + 1) & TX_RING_MOD_MASK(TX_BD_RING_LEN);
+
+        /* copy skb data to tx buffer for sdma processing */
+        memcpy(priv->tx_buffer + (be32_to_cpu(bd->buf) - priv->dma_tx_addr),
+               skb->data, skb->len);
+
+        /* set bd status and length */
+        bd_status = (bd_status & T_W_S) | T_R_S | T_I_S | T_L_S | T_TC_S;
+
+        iowrite16be(bd_status, &bd->status);
+        iowrite16be(skb->len, &bd->length);
+
+        /* Move to next BD in the ring */
+        if (!(bd_status & T_W_S))
+                bd += 1;
+        else
+                bd = priv->tx_bd_base;
+
+        if (bd == priv->dirty_tx) {
+                if (!netif_queue_stopped(dev))
+                        netif_stop_queue(dev);
+        }
+
+        priv->curtx_bd = bd;
+
+        spin_unlock_irqrestore(&priv->lock, flags);
+
+        return NETDEV_TX_OK;
+}
+
+static int hdlc_tx_done(struct ucc_hdlc_private *priv)
+{
+        /* Start from the next BD that should be filled */
+        struct net_device *dev = priv->ndev;
+        struct qe_bd *bd;               /* BD pointer */
+        u16 bd_status;
+
+        bd = priv->dirty_tx;
+        bd_status = ioread16be(&bd->status);
+
+        /* Normal processing. */
+        while ((bd_status & T_R_S) == 0) {
+                struct sk_buff *skb;
+
+                /* BD contains already transmitted buffer.   */
+                /* Handle the transmitted buffer and release */
+                /* the BD to be used with the current frame  */
+
+                skb = priv->tx_skbuff[priv->skb_dirtytx];
+                if (!skb)
+                        break;
+                pr_info("TxBD: %x\n", bd_status);
+                dev->stats.tx_packets++;
+                memset(priv->tx_buffer +
+                       (be32_to_cpu(bd->buf) - priv->dma_tx_addr),
+                       0, skb->len);
+                dev_kfree_skb_irq(skb);
+
+                priv->tx_skbuff[priv->skb_dirtytx] = NULL;
+                priv->skb_dirtytx =
+                    (priv->skb_dirtytx +
+                     1) & TX_RING_MOD_MASK(TX_BD_RING_LEN);
+
+                /* We freed a buffer, so now we can restart transmission */
+                if (netif_queue_stopped(dev))
+                        netif_wake_queue(dev);
+
+                /* Advance the confirmation BD pointer */
+                if (!(bd_status & T_W_S))
+                        bd += 1;
+                else
+                        bd = priv->tx_bd_base;
+                bd_status = ioread16be(&bd->status);
+        }
+        priv->dirty_tx = bd;
+
+        return 0;
+}
+
+static int hdlc_rx_done(struct ucc_hdlc_private *priv, int rx_work_limit)
+{
+        struct net_device *dev = priv->ndev;
+        struct sk_buff *skb;
+        hdlc_device *hdlc = dev_to_hdlc(dev);
+        struct qe_bd *bd;
+        u32 bd_status;
+        u16 length, howmany = 0;
+        u8 *bdbuffer;
+        int i;
+        static int entry;
+
+        bd = priv->currx_bd;
+        bd_status = ioread16be(&bd->status);
+
+        /* while there are received buffers and BD is full (~R_E) */
+        while (!((bd_status & (R_E_S)) || (--rx_work_limit < 0))) {
+                if (bd_status & R_OV_S)
+                        dev->stats.rx_over_errors++;
+                if (bd_status & R_CR_S) {
+#ifdef BROKEN_FRAME_INFO
+                        pr_info("Broken Frame with RxBD: %x\n", bd_status);
+#endif
+                        dev->stats.rx_crc_errors++;
+                        dev->stats.rx_dropped++;
+                        goto recycle;
+                }
+                bdbuffer = priv->rx_buffer +
+                        (priv->currx_bdnum * MAX_RX_BUF_LENGTH);
+                length = ioread16be(&bd->length);
+
+                pr_info("Received data length:%d", length);
+                pr_info("while entry times:%d", entry++);
+
+                pr_info("\nReceived data:\n");
+                for (i = 0; (i < 16); i++) {
+                        if (i == length)
+                                pr_info("++++");
+                        else
+                        pr_info("%02x\n", bdbuffer[i]);
+                }
+
+                switch (dev->type) {
+                case ARPHRD_RAWHDLC:
+                        bdbuffer += HDLC_HEAD_LEN;
+                        length -= (HDLC_HEAD_LEN + HDLC_CRC_SIZE);
+
+                        skb = dev_alloc_skb(length);
+                        if (!skb) {
+                                dev->stats.rx_dropped++;
+                                return -ENOMEM;
+                        }
+
+                        skb_put(skb, length);
+                        skb->len = length;
+                        skb->dev = dev;
+                        memcpy(skb->data, bdbuffer, length);
+                        break;
+
+                case ARPHRD_PPP:
+                        length -= HDLC_CRC_SIZE;
+
+                        skb = dev_alloc_skb(length);
+                        if (!skb) {
+                                dev->stats.rx_dropped++;
+                                return -ENOMEM;
+                        }
+
+                        skb_put(skb, length);
+                        skb->len = length;
+                        skb->dev = dev;
+                        memcpy(skb->data, bdbuffer, length);
+                        break;
+                }
+
+                dev->stats.rx_packets++;
+                dev->stats.rx_bytes += skb->len;
+                howmany++;
+                if (hdlc->proto)
+                        skb->protocol = hdlc_type_trans(skb, dev);
+                pr_info("skb->protocol:%x\n", skb->protocol);
+                netif_receive_skb(skb);
+
+recycle:
+                iowrite16be(bd_status | R_E_S | R_I_S, &bd->status);
+
+                /* update to point at the next bd */
+                if (bd_status & R_W_S) {
+                        priv->currx_bdnum = 0;
+                        bd = priv->rx_bd_base;
+                } else {
+                        if (priv->currx_bdnum < (RX_BD_RING_LEN - 1))
+                                priv->currx_bdnum += 1;
+                        else
+                                priv->currx_bdnum = RX_BD_RING_LEN - 1;
+
+                        bd += 1;
+                }
+
+                bd_status = ioread16be(&bd->status);
+        }
+
+        priv->currx_bd = bd;
+        return howmany;
+}
+
+static int ucc_hdlc_poll(struct napi_struct *napi, int budget)
+{
+        struct ucc_hdlc_private *priv = container_of(napi,
+                                                     struct ucc_hdlc_private,
+                                                     napi);
+        int howmany;
+
+        /* Tx event processing */
+        spin_lock(&priv->lock);
+                hdlc_tx_done(priv);
+        spin_unlock(&priv->lock);
+
+        howmany = 0;
+        howmany += hdlc_rx_done(priv, budget - howmany);
+
+        if (howmany < budget) {
+                napi_complete(napi);
+                qe_setbits32(priv->uccf->p_uccm,
+                             (UCCE_HDLC_RX_EVENTS | UCCE_HDLC_TX_EVENTS) << 16);
+        }
+
+        return howmany;
+}
+
+static irqreturn_t ucc_hdlc_irq_handler(int irq, void *dev_id)
+{
+        struct ucc_hdlc_private *priv = (struct ucc_hdlc_private *)dev_id;
+        struct net_device *dev = priv->ndev;
+        struct ucc_fast_private *uccf;
+        struct ucc_tdm_info *ut_info;
+        u32 ucce;
+        u32 uccm;
+
+        ut_info = priv->ut_info;
+        uccf = priv->uccf;
+
+        ucce = ioread32be(uccf->p_ucce);
+        uccm = ioread32be(uccf->p_uccm);
+        ucce &= uccm;
+        iowrite32be(ucce, uccf->p_ucce);
+        pr_info("irq ucce:%x\n", ucce);
+        if (!ucce)
+                return IRQ_NONE;
+
+        if ((ucce >> 16) & (UCCE_HDLC_RX_EVENTS | UCCE_HDLC_TX_EVENTS)) {
+                if (napi_schedule_prep(&priv->napi)) {
+                        uccm &= ~((UCCE_HDLC_RX_EVENTS | UCCE_HDLC_TX_EVENTS)
+                                  << 16);
+                        iowrite32be(uccm, uccf->p_uccm);
+                        __napi_schedule(&priv->napi);
+                }
+        }
+
+        /* Errors and other events */
+        if (ucce >> 16 & UCC_HDLC_UCCE_BSY)
+                dev->stats.rx_errors++;
+        if (ucce >> 16 & UCC_HDLC_UCCE_TXE)
+                dev->stats.tx_errors++;
+
+        return IRQ_HANDLED;
+}
+
+static int uhdlc_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
+{
+        const size_t size = sizeof(te1_settings);
+        te1_settings line;
+        struct ucc_hdlc_private *priv = netdev_priv(dev);
+
+        if (cmd != SIOCWANDEV)
+                return hdlc_ioctl(dev, ifr, cmd);
+
+        switch (ifr->ifr_settings.type) {
+        case IF_GET_IFACE:
+                ifr->ifr_settings.type = IF_IFACE_E1;
+                if (ifr->ifr_settings.size < size) {
+                        ifr->ifr_settings.size = size; /* data size wanted */
+                        return -ENOBUFS;
+                }
+                line.clock_type = priv->clocking;
+                line.clock_rate = 0;
+                line.loopback = 0;
+
+                if (copy_to_user(ifr->ifr_settings.ifs_ifsu.sync, &line, size))
+                        return -EFAULT;
+                return 0;
+
+        default:
+                return hdlc_ioctl(dev, ifr, cmd);
+        }
+}
+
+static int uhdlc_open(struct net_device *dev)
+{
+        u32 cecr_subblock;
+        hdlc_device *hdlc = dev_to_hdlc(dev);
+        struct ucc_hdlc_private *priv = hdlc->priv;
+        struct ucc_tdm *utdm = priv->utdm;
+
+        if (priv->hdlc_busy != 1) {
+                if (request_irq(priv->ut_info->uf_info.irq,
+                                ucc_hdlc_irq_handler, 0, "hdlc", priv))
+                        return -ENODEV;
+
+                cecr_subblock = ucc_fast_get_qe_cr_subblock(
+                                        priv->ut_info->uf_info.ucc_num);
+
+                qe_issue_cmd(QE_INIT_TX_RX, cecr_subblock,
+                             QE_CR_PROTOCOL_UNSPECIFIED, 0);
+
+                ucc_fast_enable(priv->uccf, COMM_DIR_RX | COMM_DIR_TX);
+
+                /* Enable the TDM port */
+                if (priv->tsa)
+                        utdm->si_regs->siglmr1_h |= (0x1 << utdm->tdm_port);
+
+                priv->hdlc_busy = 1;
+                netif_device_attach(priv->ndev);
+                napi_enable(&priv->napi);
+                netif_start_queue(dev);
+                hdlc_open(dev);
+        }
+
+        return 0;
+}
+
+static void uhdlc_memclean(struct ucc_hdlc_private *priv)
+{
+        qe_muram_free(priv->ucc_pram->riptr);
+        qe_muram_free(priv->ucc_pram->tiptr);
+
+        if (priv->rx_bd_base) {
+                dma_free_coherent(priv->dev,
+                                  RX_BD_RING_LEN * sizeof(struct qe_bd),
+                                  priv->rx_bd_base, priv->dma_rx_bd);
+
+                priv->rx_bd_base = NULL;
+                priv->dma_rx_bd = 0;
+        }
+
+        if (priv->tx_bd_base) {
+                dma_free_coherent(priv->dev,
+                                  TX_BD_RING_LEN * sizeof(struct qe_bd),
+                                  priv->tx_bd_base, priv->dma_tx_bd);
+
+                priv->tx_bd_base = NULL;
+                priv->dma_tx_bd = 0;
+        }
+
+        if (priv->ucc_pram) {
+                qe_muram_free(priv->ucc_pram_offset);
+                priv->ucc_pram = NULL;
+                priv->ucc_pram_offset = 0;
+         }
+
+        kfree(priv->rx_skbuff);
+        priv->rx_skbuff = NULL;
+
+        kfree(priv->tx_skbuff);
+        priv->tx_skbuff = NULL;
+
+        if (priv->uf_regs) {
+                iounmap(priv->uf_regs);
+                priv->uf_regs = NULL;
+        }
+
+        if (priv->uccf) {
+                ucc_fast_free(priv->uccf);
+                priv->uccf = NULL;
+        }
+
+        if (priv->rx_buffer) {
+                dma_free_coherent(priv->dev,
+                                  RX_BD_RING_LEN * MAX_RX_BUF_LENGTH,
+                                  priv->rx_buffer, priv->dma_rx_addr);
+                priv->rx_buffer = NULL;
+                priv->dma_rx_addr = 0;
+        }
+
+        if (priv->tx_buffer) {
+                dma_free_coherent(priv->dev,
+                                  TX_BD_RING_LEN * MAX_RX_BUF_LENGTH,
+                                  priv->tx_buffer, priv->dma_tx_addr);
+                priv->tx_buffer = NULL;
+                priv->dma_tx_addr = 0;
+        }
+}
+
+static int uhdlc_close(struct net_device *dev)
+{
+        struct ucc_hdlc_private *priv = dev_to_hdlc(dev)->priv;
+        struct ucc_tdm *utdm = priv->utdm;
+        u32 cecr_subblock;
+
+        napi_disable(&priv->napi);
+        cecr_subblock = ucc_fast_get_qe_cr_subblock(
+                                priv->ut_info->uf_info.ucc_num);
+
+        qe_issue_cmd(QE_GRACEFUL_STOP_TX, cecr_subblock,
+                     (u8)QE_CR_PROTOCOL_UNSPECIFIED, 0);
+        qe_issue_cmd(QE_CLOSE_RX_BD, cecr_subblock,
+                     (u8)QE_CR_PROTOCOL_UNSPECIFIED, 0);
+
+        if (priv->tsa)
+                utdm->si_regs->siglmr1_h &= ~(0x1 << utdm->tdm_port);
+
+        ucc_fast_disable(priv->uccf, COMM_DIR_RX | COMM_DIR_TX);
+
+        free_irq(priv->ut_info->uf_info.irq, priv);
+        netif_stop_queue(dev);
+        priv->hdlc_busy = 0;
+
+        return 0;
+}
+
+static int ucc_hdlc_attach(struct net_device *dev, unsigned short encoding,
+                           unsigned short parity)
+{
+        struct ucc_hdlc_private *priv = dev_to_hdlc(dev)->priv;
+
+        if (encoding != ENCODING_NRZ &&
+            encoding != ENCODING_NRZI)
+                return -EINVAL;
+
+        if (parity != PARITY_NONE &&
+            parity != PARITY_CRC32_PR1_CCITT &&
+            parity != PARITY_CRC16_PR1_CCITT)
+                return -EINVAL;
+
+        priv->encoding = encoding;
+        priv->parity = parity;
+
+        return 0;
+}
+
+#ifdef CONFIG_PM
+static void store_clk_config(struct ucc_hdlc_private *priv)
+{
+        struct qe_mux *qe_mux_reg = &qe_immr->qmx;
+
+        /* store si clk */
+        priv->cmxsi1cr_h = ioread32be(&qe_mux_reg->cmxsi1cr_h);
+        priv->cmxsi1cr_l = ioread32be(&qe_mux_reg->cmxsi1cr_l);
+
+        /* store si sync */
+        priv->cmxsi1syr = ioread32be(&qe_mux_reg->cmxsi1syr);
+
+        /* store ucc clk */
+        memcpy_fromio(priv->cmxucr, qe_mux_reg->cmxucr, 4 * sizeof(u32));
+}
+
+static void resume_clk_config(struct ucc_hdlc_private *priv)
+{
+        struct qe_mux *qe_mux_reg = &qe_immr->qmx;
+
+        memcpy_toio(qe_mux_reg->cmxucr, priv->cmxucr, 4 * sizeof(u32));
+
+        iowrite32be(priv->cmxsi1cr_h, &qe_mux_reg->cmxsi1cr_h);
+        iowrite32be(priv->cmxsi1cr_l, &qe_mux_reg->cmxsi1cr_l);
+
+        iowrite32be(priv->cmxsi1syr, &qe_mux_reg->cmxsi1syr);
+}
+
+static int uhdlc_suspend(struct device *dev)
+{
+        struct ucc_hdlc_private *priv = dev_get_drvdata(dev);
+        struct ucc_tdm_info *ut_info;
+        struct ucc_fast __iomem *uf_regs;
+
+        if (!priv)
+                return -EINVAL;
+
+        if (!netif_running(priv->ndev))
+                return 0;
+
+        netif_device_detach(priv->ndev);
+        napi_disable(&priv->napi);
+
+        ut_info = priv->ut_info;
+        uf_regs = priv->uf_regs;
+
+        /* backup gumr guemr*/
+        priv->gumr = ioread32be(&uf_regs->gumr);
+        priv->guemr = ioread8(&uf_regs->guemr);
+
+        priv->ucc_pram_bak = kmalloc(sizeof(*priv->ucc_pram_bak),
+                                        GFP_KERNEL);
+        if (!priv->ucc_pram_bak)
+                return -ENOMEM;
+
+        /* backup HDLC parameter */
+        memcpy_fromio(priv->ucc_pram_bak, priv->ucc_pram,
+                      sizeof(struct ucc_hdlc_param));
+
+        /* store the clk configuration */
+        store_clk_config(priv);
+
+        /* save power */
+        ucc_fast_disable(priv->uccf, COMM_DIR_RX | COMM_DIR_TX);
+
+        dev_dbg(dev, "ucc hdlc suspend\n");
+        return 0;
+}
+
+static int uhdlc_resume(struct device *dev)
+{
+        struct ucc_hdlc_private *priv = dev_get_drvdata(dev);
+        struct ucc_tdm *utdm = priv->utdm;
+        struct ucc_tdm_info *ut_info;
+        struct ucc_fast __iomem *uf_regs;
+        struct ucc_fast_private *uccf;
+        struct ucc_fast_info *uf_info;
+        int ret, i;
+        u32 cecr_subblock;
+        u16 bd_status;
+
+        if (!priv)
+                return -EINVAL;
+
+        if (!netif_running(priv->ndev))
+                return 0;
+
+        ut_info = priv->ut_info;
+        uf_info = &ut_info->uf_info;
+        uf_regs = priv->uf_regs;
+        uccf = priv->uccf;
+
+        /* restore gumr guemr */
+        iowrite8(priv->guemr, &uf_regs->guemr);
+        iowrite32be(priv->gumr, &uf_regs->gumr);
+
+        /* Set Virtual Fifo registers */
+        iowrite16be(uf_info->urfs, &uf_regs->urfs);
+        iowrite16be(uf_info->urfet, &uf_regs->urfet);
+        iowrite16be(uf_info->urfset, &uf_regs->urfset);
+        iowrite16be(uf_info->utfs, &uf_regs->utfs);
+        iowrite16be(uf_info->utfet, &uf_regs->utfet);
+        iowrite16be(uf_info->utftt, &uf_regs->utftt);
+        /* utfb, urfb are offsets from MURAM base */
+        iowrite32be(uccf->ucc_fast_tx_virtual_fifo_base_offset, &uf_regs->utfb);
+        iowrite32be(uccf->ucc_fast_rx_virtual_fifo_base_offset, &uf_regs->urfb);
+
+        /* Rx Tx and sync clock routing */
+        resume_clk_config(priv);
+
+        iowrite32be(uf_info->uccm_mask, &uf_regs->uccm);
+        iowrite32be(0xffffffff, &uf_regs->ucce);
+
+        ucc_fast_disable(priv->uccf, COMM_DIR_RX | COMM_DIR_TX);
+
+        /* rebuild SIRAM */
+        if (priv->tsa)
+                ucc_tdm_init(priv->utdm, priv->ut_info);
+
+        /* Write to QE CECR, UCCx channel to Stop Transmission */
+        cecr_subblock = ucc_fast_get_qe_cr_subblock(uf_info->ucc_num);
+        ret = qe_issue_cmd(QE_STOP_TX, cecr_subblock,
+                           (u8)QE_CR_PROTOCOL_UNSPECIFIED, 0);
+
+        /* Set UPSMR normal mode */
+        iowrite32be(0, &uf_regs->upsmr);
+
+        /* init parameter base */
+        cecr_subblock = ucc_fast_get_qe_cr_subblock(uf_info->ucc_num);
+        ret = qe_issue_cmd(QE_ASSIGN_PAGE_TO_DEVICE, cecr_subblock,
+                           QE_CR_PROTOCOL_UNSPECIFIED, priv->ucc_pram_offset);
+
+        priv->ucc_pram = (struct ucc_hdlc_param __iomem *)
+                                qe_muram_addr(priv->ucc_pram_offset);
+
+        /* restore ucc parameter */
+        memcpy_toio(priv->ucc_pram, priv->ucc_pram_bak,
+                    sizeof(struct ucc_hdlc_param));
+        kfree(priv->ucc_pram_bak);
+
+        /* rebuild BD entry */
+        for (i = 0; i < RX_BD_RING_LEN; i++) {
+                if (i < (RX_BD_RING_LEN - 1))
+                        bd_status = R_E_S | R_I_S;
+                else
+                        bd_status = R_E_S | R_I_S | R_W_S;
+
+                iowrite16be(bd_status, &priv->rx_bd_base[i].status);
+                iowrite32be(priv->dma_rx_addr + i * MAX_RX_BUF_LENGTH,
+                            &priv->rx_bd_base[i].buf);
+        }
+
+        for (i = 0; i < TX_BD_RING_LEN; i++) {
+                if (i < (TX_BD_RING_LEN - 1))
+                        bd_status =  T_I_S | T_TC_S;
+                else
+                        bd_status =  T_I_S | T_TC_S | T_W_S;
+
+                iowrite16be(bd_status, &priv->tx_bd_base[i].status);
+                iowrite32be(priv->dma_tx_addr + i * MAX_RX_BUF_LENGTH,
+                            &priv->tx_bd_base[i].buf);
+        }
+
+        /* if hdlc is busy enable TX and RX */
+        if (priv->hdlc_busy == 1) {
+                cecr_subblock = ucc_fast_get_qe_cr_subblock(
+                                        priv->ut_info->uf_info.ucc_num);
+
+                qe_issue_cmd(QE_INIT_TX_RX, cecr_subblock,
+                             (u8)QE_CR_PROTOCOL_UNSPECIFIED, 0);
+
+                ucc_fast_enable(priv->uccf, COMM_DIR_RX | COMM_DIR_TX);
+
+                /* Enable the TDM port */
+                if (priv->tsa)
+                        utdm->si_regs->siglmr1_h |= (0x1 << utdm->tdm_port);
+        }
+
+        napi_enable(&priv->napi);
+        netif_device_attach(priv->ndev);
+
+        return 0;
+}
+
+static const struct dev_pm_ops uhdlc_pm_ops = {
+        .suspend = uhdlc_suspend,
+        .resume = uhdlc_resume,
+        .freeze = uhdlc_suspend,
+        .thaw = uhdlc_resume,
+};
+
+#define HDLC_PM_OPS (&uhdlc_pm_ops)
+
+#else
+
+#define HDLC_PM_OPS NULL
+
+#endif
+static const struct net_device_ops uhdlc_ops = {
+        .ndo_open       = uhdlc_open,
+        .ndo_stop       = uhdlc_close,
+        .ndo_change_mtu = hdlc_change_mtu,
+        .ndo_start_xmit = hdlc_start_xmit,
+        .ndo_do_ioctl   = uhdlc_ioctl,
+};
+
+static int ucc_hdlc_probe(struct platform_device *pdev)
+{
+        struct device_node *np = pdev->dev.of_node;
+        struct ucc_hdlc_private *uhdlc_priv = NULL;
+        struct ucc_tdm_info *ut_info;
+        struct ucc_tdm *utdm;
+        struct resource res;
+        struct net_device *dev;
+        hdlc_device *hdlc;
+        int ucc_num;
+        const char *sprop;
+        int ret;
+        u32 val;
+
+        ret = of_property_read_u32_index(np, "cell-index", 0, &val);
+        if (ret) {
+                dev_err(&pdev->dev, "Invalid ucc property\n");
+                return -ENODEV;
+        }
+
+        ucc_num = val - 1;
+        if ((ucc_num > 3) || (ucc_num < 0)) {
+                dev_err(&pdev->dev, ": Invalid UCC num\n");
+                return -EINVAL;
+        }
+
+        memcpy(&utdm_info[ucc_num], &utdm_primary_info,
+               sizeof(utdm_primary_info));
+
+        ut_info = &utdm_info[ucc_num];
+        ut_info->uf_info.ucc_num = ucc_num;
+
+        sprop = of_get_property(np, "rx-clock-name", NULL);
+        if (sprop) {
+                ut_info->uf_info.rx_clock = qe_clock_source(sprop);
+                if ((ut_info->uf_info.rx_clock < QE_CLK_NONE) ||
+                    (ut_info->uf_info.rx_clock > QE_CLK24)) {
+                        dev_err(&pdev->dev, "Invalid rx-clock-name property\n");
+                        return -EINVAL;
+                }
+        } else {
+                dev_err(&pdev->dev, "Invalid rx-clock-name property\n");
+                return -EINVAL;
+        }
+
+        sprop = of_get_property(np, "tx-clock-name", NULL);
+        if (sprop) {
+                ut_info->uf_info.tx_clock = qe_clock_source(sprop);
+                if ((ut_info->uf_info.tx_clock < QE_CLK_NONE) ||
+                    (ut_info->uf_info.tx_clock > QE_CLK24)) {
+                        dev_err(&pdev->dev, "Invalid tx-clock-name property\n");
+                        return -EINVAL;
+                }
+        } else {
+                dev_err(&pdev->dev, "Invalid tx-clock-name property\n");
+                return -EINVAL;
+        }
+
+        /* use the same clock when work in loopback */
+        if (ut_info->uf_info.rx_clock == ut_info->uf_info.tx_clock)
+                qe_setbrg(ut_info->uf_info.rx_clock, 20000000, 1);
+
+        ret = of_address_to_resource(np, 0, &res);
+        if (ret)
+                return -EINVAL;
+
+        ut_info->uf_info.regs = res.start;
+        ut_info->uf_info.irq = irq_of_parse_and_map(np, 0);
+
+        uhdlc_priv = kzalloc(sizeof(*uhdlc_priv), GFP_KERNEL);
+        if (!uhdlc_priv) {
+                ret = -ENOMEM;
+                dev_err(&pdev->dev, "No mem to alloc hdlc private data\n");
+                goto err_alloc_priv;
+        }
+
+        dev_set_drvdata(&pdev->dev, uhdlc_priv);
+        uhdlc_priv->dev = &pdev->dev;
+        uhdlc_priv->ut_info = ut_info;
+
+        if (of_get_property(np, "fsl,tdm-interface", NULL))
+                uhdlc_priv->tsa = 1;
+
+        if (of_get_property(np, "fsl,ucc-internal-loopback", NULL))
+                uhdlc_priv->loopback = 1;
+
+        if (uhdlc_priv->tsa == 1) {
+                utdm = kzalloc(sizeof(*utdm), GFP_KERNEL);
+                if (!utdm) {
+                        ret = -ENOMEM;
+                        dev_err(&pdev->dev, "No mem to alloc ucc tdm data\n");
+                        goto err_alloc_utdm;
+                }
+                uhdlc_priv->utdm = utdm;
+                ret = ucc_of_parse_tdm(np, utdm, ut_info);
+                if (ret)
+                        goto err_miss_tsa_property;
+        }
+
+        ret = uhdlc_init(uhdlc_priv);
+        if (ret) {
+                dev_err(&pdev->dev, "Failed to init uhdlc\n");
+                goto err_hdlc_init;
+        }
+
+        dev = alloc_hdlcdev(uhdlc_priv);
+        if (!dev) {
+                ret = -ENOMEM;
+                pr_err("ucc_hdlc: unable to allocate memory\n");
+                goto err_hdlc_init;
+        }
+
+        uhdlc_priv->ndev = dev;
+        hdlc = dev_to_hdlc(dev);
+        dev->tx_queue_len = 16;
+        dev->netdev_ops = &uhdlc_ops;
+        hdlc->attach = ucc_hdlc_attach;
+        hdlc->xmit = ucc_hdlc_tx;
+        netif_napi_add(dev, &uhdlc_priv->napi, ucc_hdlc_poll, 32);
+        if (register_hdlc_device(dev)) {
+                ret = -ENOBUFS;
+                pr_err("ucc_hdlc: unable to register hdlc device\n");
+                free_netdev(dev);
+                goto err_hdlc_init;
+        }
+
+        return 0;
+
+err_hdlc_init:
+err_miss_tsa_property:
+        kfree(uhdlc_priv);
+        if (uhdlc_priv->tsa)
+                kfree(utdm);
+err_alloc_utdm:
+        kfree(uhdlc_priv);
+err_alloc_priv:
+        return ret;
+}
+
+static int ucc_hdlc_remove(struct platform_device *pdev)
+{
+        struct ucc_hdlc_private *priv = dev_get_drvdata(&pdev->dev);
+
+        uhdlc_memclean(priv);
+
+        if (priv->utdm->si_regs) {
+                iounmap(priv->utdm->si_regs);
+                priv->utdm->si_regs = NULL;
+        }
+
+        if (priv->utdm->siram) {
+                iounmap(priv->utdm->siram);
+                priv->utdm->siram = NULL;
+        }
+        kfree(priv);
+
+        dev_info(&pdev->dev, "UCC based hdlc module removed\n");
+
+        return 0;
+}
+
+static const struct of_device_id fsl_ucc_hdlc_of_match[] = {
+        {
+        .compatible = "fsl,ucc-hdlc",
+        },
+        {},
+};
+
+MODULE_DEVICE_TABLE(of, fsl_ucc_hdlc_of_match);
+
+static struct platform_driver ucc_hdlc_driver = {
+        .probe  = ucc_hdlc_probe,
+        .remove = ucc_hdlc_remove,
+        .driver = {
+                .owner          = THIS_MODULE,
+                .name           = DRV_NAME,
+                .pm             = HDLC_PM_OPS,
+                .of_match_table = fsl_ucc_hdlc_of_match,
+        },
+};
+
+static int __init ucc_hdlc_init(void)
+{
+        return platform_driver_register(&ucc_hdlc_driver);
+}
+
+static void __exit ucc_hdlc_exit(void)
+{
+        platform_driver_unregister(&ucc_hdlc_driver);
+}
+
+module_init(ucc_hdlc_init);
+module_exit(ucc_hdlc_exit);
diff --git a/drivers/net/wan/fsl_ucc_hdlc.h b/drivers/net/wan/fsl_ucc_hdlc.h
new file mode 100644
index 000000000000..881ecdeef076
--- /dev/null
+++ b/drivers/net/wan/fsl_ucc_hdlc.h
@@ -0,0 +1,147 @@
+/* Freescale QUICC Engine HDLC Device Driver
+ *
+ * Copyright 2014 Freescale Semiconductor Inc.
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+
+#ifndef _UCC_HDLC_H_
+#define _UCC_HDLC_H_
+
+#include <linux/kernel.h>
+#include <linux/list.h>
+
+#include <soc/fsl/qe/immap_qe.h>
+#include <soc/fsl/qe/qe.h>
+
+#include <soc/fsl/qe/ucc.h>
+#include <soc/fsl/qe/ucc_fast.h>
+
+/* UCC HDLC event register */
+#define UCCE_HDLC_RX_EVENTS     \
+(UCC_HDLC_UCCE_RXF | UCC_HDLC_UCCE_RXB | UCC_HDLC_UCCE_BSY)
+#define UCCE_HDLC_TX_EVENTS     (UCC_HDLC_UCCE_TXB | UCC_HDLC_UCCE_TXE)
+
+struct ucc_hdlc_param {
+        __be16 riptr;
+        __be16 tiptr;
+        __be16 res0;
+        __be16 mrblr;
+        __be32 rstate;
+        __be32 rbase;
+        __be16 rbdstat;
+        __be16 rbdlen;
+        __be32 rdptr;
+        __be32 tstate;
+        __be32 tbase;
+        __be16 tbdstat;
+        __be16 tbdlen;
+        __be32 tdptr;
+        __be32 rbptr;
+        __be32 tbptr;
+        __be32 rcrc;
+        __be32 res1;
+        __be32 tcrc;
+        __be32 res2;
+        __be32 res3;
+        __be32 c_mask;
+        __be32 c_pres;
+        __be16 disfc;
+        __be16 crcec;
+        __be16 abtsc;
+        __be16 nmarc;
+        __be32 max_cnt;
+        __be16 mflr;
+        __be16 rfthr;
+        __be16 rfcnt;
+        __be16 hmask;
+        __be16 haddr1;
+        __be16 haddr2;
+        __be16 haddr3;
+        __be16 haddr4;
+        __be16 ts_tmp;
+        __be16 tmp_mb;
+};
+
+struct ucc_hdlc_private {
+        struct ucc_tdm  *utdm;
+        struct ucc_tdm_info *ut_info;
+        struct ucc_fast_private *uccf;
+        struct device *dev;
+        struct net_device *ndev;
+        struct napi_struct napi;
+        struct ucc_fast __iomem *uf_regs;       /* UCC Fast registers */
+        struct ucc_hdlc_param __iomem *ucc_pram;
+        u16 tsa;
+        bool hdlc_busy;
+        bool loopback;
+
+        u8 *tx_buffer;
+        u8 *rx_buffer;
+        dma_addr_t dma_tx_addr;
+        dma_addr_t dma_rx_addr;
+
+        struct qe_bd *tx_bd_base;
+        struct qe_bd *rx_bd_base;
+        dma_addr_t dma_tx_bd;
+        dma_addr_t dma_rx_bd;
+        struct qe_bd *curtx_bd;
+        struct qe_bd *currx_bd;
+        struct qe_bd *dirty_tx;
+        u16 currx_bdnum;
+
+        struct sk_buff **tx_skbuff;
+        struct sk_buff **rx_skbuff;
+        u16 skb_curtx;
+        u16 skb_currx;
+        unsigned short skb_dirtytx;
+
+        unsigned short tx_ring_size;
+        unsigned short rx_ring_size;
+        u32 ucc_pram_offset;
+
+        unsigned short encoding;
+        unsigned short parity;
+        u32 clocking;
+        spinlock_t lock;        /* lock for Tx BD and Tx buffer */
+#ifdef CONFIG_PM
+        struct ucc_hdlc_param *ucc_pram_bak;
+        u32 gumr;
+        u8 guemr;
+        u32 cmxsi1cr_l, cmxsi1cr_h;
+        u32 cmxsi1syr;
+        u32 cmxucr[4];
+#endif
+};
+
+#define TX_BD_RING_LEN  0x10
+#define RX_BD_RING_LEN  0x20
+#define RX_CLEAN_MAX    0x10
+#define NUM_OF_BUF      4
+#define MAX_RX_BUF_LENGTH       (48 * 0x20)
+#define MAX_FRAME_LENGTH        (MAX_RX_BUF_LENGTH + 8)
+#define ALIGNMENT_OF_UCC_HDLC_PRAM      64
+#define SI_BANK_SIZE    128
+#define MAX_HDLC_NUM    4
+#define HDLC_HEAD_LEN   2
+#define HDLC_CRC_SIZE   2
+#define TX_RING_MOD_MASK(size) (size - 1)
+#define RX_RING_MOD_MASK(size) (size - 1)
+
+#define HDLC_HEAD_MASK          0x0000
+#define DEFAULT_HDLC_HEAD       0xff44
+#define DEFAULT_ADDR_MASK       0x00ff
+#define DEFAULT_HDLC_ADDR       0x00ff
+
+#define BMR_GBL                 0x20000000
+#define BMR_BIG_ENDIAN          0x10000000
+#define CRC_16BIT_MASK          0x0000F0B8
+#define CRC_16BIT_PRES          0x0000FFFF
+#define DEFAULT_RFTHR           1
+
+#define DEFAULT_PPP_HEAD    0xff03
+
+#endif
diff --git a/drivers/net/wan/slic_ds26522.c b/drivers/net/wan/slic_ds26522.c
new file mode 100644
index 000000000000..d06a887a2352
--- /dev/null
+++ b/drivers/net/wan/slic_ds26522.c
@@ -0,0 +1,255 @@
+/*
+ * drivers/net/wan/slic_ds26522.c
+ *
+ * Copyright (C) 2016 Freescale Semiconductor, Inc.
+ *
+ * Author:Zhao Qiang<qiang.zhao@nxp.com>
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+
+#include <linux/bitrev.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/kthread.h>
+#include <linux/spi/spi.h>
+#include <linux/wait.h>
+#include <linux/param.h>
+#include <linux/delay.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/io.h>
+#include "slic_ds26522.h"
+
+#define DRV_NAME "ds26522"
+
+#define SLIC_TRANS_LEN 1
+#define SLIC_TWO_LEN 2
+#define SLIC_THREE_LEN 3
+
+static struct spi_device *g_spi;
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Zhao Qiang<B45475@freescale.com>");
+
+/* the read/write format of address is
+ * w/r|A13|A12|A11|A10|A9|A8|A7|A6|A5|A4|A3|A2|A1|A0|x
+ */
+static void slic_write(struct spi_device *spi, u16 addr,
+                       u8 data)
+{
+        u8 temp[3];
+
+        addr = bitrev16(addr) >> 1;
+        data = bitrev8(data);
+        temp[0] = (u8)((addr >> 8) & 0x7f);
+        temp[1] = (u8)(addr & 0xfe);
+        temp[2] = data;
+
+        /* write spi addr and value */
+        spi_write(spi, &temp[0], SLIC_THREE_LEN);
+}
+
+static u8 slic_read(struct spi_device *spi, u16 addr)
+{
+        u8 temp[2];
+        u8 data;
+
+        addr = bitrev16(addr) >> 1;
+        temp[0] = (u8)(((addr >> 8) & 0x7f) | 0x80);
+        temp[1] = (u8)(addr & 0xfe);
+
+        spi_write_then_read(spi, &temp[0], SLIC_TWO_LEN, &data,
+                            SLIC_TRANS_LEN);
+
+        data = bitrev8(data);
+        return data;
+}
+
+static bool get_slic_product_code(struct spi_device *spi)
+{
+        u8 device_id;
+
+        device_id = slic_read(spi, DS26522_IDR_ADDR);
+        if ((device_id & 0xf8) == 0x68)
+                return true;
+        else
+                return false;
+}
+
+static void ds26522_e1_spec_config(struct spi_device *spi)
+{
+        /* Receive E1 Mode, Framer Disabled */
+        slic_write(spi, DS26522_RMMR_ADDR, DS26522_RMMR_E1);
+
+        /* Transmit E1 Mode, Framer Disable */
+        slic_write(spi, DS26522_TMMR_ADDR, DS26522_TMMR_E1);
+
+        /* Receive E1 Mode Framer Enable */
+        slic_write(spi, DS26522_RMMR_ADDR,
+                   slic_read(spi, DS26522_RMMR_ADDR) | DS26522_RMMR_FRM_EN);
+
+        /* Transmit E1 Mode Framer Enable */
+        slic_write(spi, DS26522_TMMR_ADDR,
+                   slic_read(spi, DS26522_TMMR_ADDR) | DS26522_TMMR_FRM_EN);
+
+        /* RCR1, receive E1 B8zs & ESF */
+        slic_write(spi, DS26522_RCR1_ADDR,
+                   DS26522_RCR1_E1_HDB3 | DS26522_RCR1_E1_CCS);
+
+        /* RSYSCLK=2.048MHz, RSYNC-Output */
+        slic_write(spi, DS26522_RIOCR_ADDR,
+                   DS26522_RIOCR_2048KHZ | DS26522_RIOCR_RSIO_OUT);
+
+        /* TCR1 Transmit E1 b8zs */
+        slic_write(spi, DS26522_TCR1_ADDR, DS26522_TCR1_TB8ZS);
+
+        /* TSYSCLK=2.048MHz, TSYNC-Output */
+        slic_write(spi, DS26522_TIOCR_ADDR,
+                   DS26522_TIOCR_2048KHZ | DS26522_TIOCR_TSIO_OUT);
+
+        /* Set E1TAF */
+        slic_write(spi, DS26522_E1TAF_ADDR, DS26522_E1TAF_DEFAULT);
+
+        /* Set E1TNAF register */
+        slic_write(spi, DS26522_E1TNAF_ADDR, DS26522_E1TNAF_DEFAULT);
+
+        /* Receive E1 Mode Framer Enable & init Done */
+        slic_write(spi, DS26522_RMMR_ADDR, slic_read(spi, DS26522_RMMR_ADDR) |
+                   DS26522_RMMR_INIT_DONE);
+
+        /* Transmit E1 Mode Framer Enable & init Done */
+        slic_write(spi, DS26522_TMMR_ADDR, slic_read(spi, DS26522_TMMR_ADDR) |
+                   DS26522_TMMR_INIT_DONE);
+
+        /* Configure LIU E1 mode */
+        slic_write(spi, DS26522_LTRCR_ADDR, DS26522_LTRCR_E1);
+
+        /* E1 Mode default 75 ohm w/Transmit Impedance Matlinking */
+        slic_write(spi, DS26522_LTITSR_ADDR,
+                   DS26522_LTITSR_TLIS_75OHM | DS26522_LTITSR_LBOS_75OHM);
+
+        /* E1 Mode default 75 ohm Long Haul w/Receive Impedance Matlinking */
+        slic_write(spi, DS26522_LRISMR_ADDR,
+                   DS26522_LRISMR_75OHM | DS26522_LRISMR_MAX);
+
+        /* Enable Transmit output */
+        slic_write(spi, DS26522_LMCR_ADDR, DS26522_LMCR_TE);
+}
+
+static int slic_ds26522_init_configure(struct spi_device *spi)
+{
+        u16 addr;
+
+        /* set clock */
+        slic_write(spi, DS26522_GTCCR_ADDR, DS26522_GTCCR_BPREFSEL_REFCLKIN |
+                        DS26522_GTCCR_BFREQSEL_2048KHZ |
+                        DS26522_GTCCR_FREQSEL_2048KHZ);
+        slic_write(spi, DS26522_GTCR2_ADDR, DS26522_GTCR2_TSSYNCOUT);
+        slic_write(spi, DS26522_GFCR_ADDR, DS26522_GFCR_BPCLK_2048KHZ);
+
+        /* set gtcr */
+        slic_write(spi, DS26522_GTCR1_ADDR, DS26522_GTCR1);
+
+        /* Global LIU Software Reset Register */
+        slic_write(spi, DS26522_GLSRR_ADDR, DS26522_GLSRR_RESET);
+
+        /* Global Framer and BERT Software Reset Register */
+        slic_write(spi, DS26522_GFSRR_ADDR, DS26522_GFSRR_RESET);
+
+        usleep_range(100, 120);
+
+        slic_write(spi, DS26522_GLSRR_ADDR, DS26522_GLSRR_NORMAL);
+        slic_write(spi, DS26522_GFSRR_ADDR, DS26522_GFSRR_NORMAL);
+
+        /* Perform RX/TX SRESET,Reset receiver */
+        slic_write(spi, DS26522_RMMR_ADDR, DS26522_RMMR_SFTRST);
+
+        /* Reset tranceiver */
+        slic_write(spi, DS26522_TMMR_ADDR, DS26522_TMMR_SFTRST);
+
+        usleep_range(100, 120);
+
+        /* Zero all Framer Registers */
+        for (addr = DS26522_RF_ADDR_START; addr <= DS26522_RF_ADDR_END;
+             addr++)
+                slic_write(spi, addr, 0);
+
+        for (addr = DS26522_TF_ADDR_START; addr <= DS26522_TF_ADDR_END;
+             addr++)
+                slic_write(spi, addr, 0);
+
+        for (addr = DS26522_LIU_ADDR_START; addr <= DS26522_LIU_ADDR_END;
+             addr++)
+                slic_write(spi, addr, 0);
+
+        for (addr = DS26522_BERT_ADDR_START; addr <= DS26522_BERT_ADDR_END;
+             addr++)
+                slic_write(spi, addr, 0);
+
+        /* setup ds26522 for E1 specification */
+        ds26522_e1_spec_config(spi);
+
+        slic_write(spi, DS26522_GTCR1_ADDR, 0x00);
+
+        return 0;
+}
+
+static int slic_ds26522_remove(struct spi_device *spi)
+{
+        pr_info("DS26522 module uninstalled\n");
+        return 0;
+}
+
+static int slic_ds26522_probe(struct spi_device *spi)
+{
+        int ret = 0;
+
+        g_spi = spi;
+        spi->bits_per_word = 8;
+
+        if (!get_slic_product_code(spi))
+                return ret;
+
+        ret = slic_ds26522_init_configure(spi);
+        if (ret == 0)
+                pr_info("DS26522 cs%d configurated\n", spi->chip_select);
+
+        return ret;
+}
+
+static const struct of_device_id slic_ds26522_match[] = {
+        {
+         .compatible = "maxim,ds26522",
+         },
+        {},
+};
+
+static struct spi_driver slic_ds26522_driver = {
+        .driver = {
+                   .name = "ds26522",
+                   .bus = &spi_bus_type,
+                   .owner = THIS_MODULE,
+                   .of_match_table = slic_ds26522_match,
+                   },
+        .probe = slic_ds26522_probe,
+        .remove = slic_ds26522_remove,
+};
+
+static int __init slic_ds26522_init(void)
+{
+        return spi_register_driver(&slic_ds26522_driver);
+}
+
+static void __exit slic_ds26522_exit(void)
+{
+        spi_unregister_driver(&slic_ds26522_driver);
+}
+
+module_init(slic_ds26522_init);
+module_exit(slic_ds26522_exit);
diff --git a/drivers/net/wan/slic_ds26522.h b/drivers/net/wan/slic_ds26522.h
new file mode 100644
index 000000000000..22aa0ecbd9fd
--- /dev/null
+++ b/drivers/net/wan/slic_ds26522.h
@@ -0,0 +1,134 @@
+/*
+ * drivers/tdm/line_ctrl/slic_ds26522.h
+ *
+ * Copyright 2016 Freescale Semiconductor, Inc.
+ *
+ * Author: Zhao Qiang <B45475@freescale.com>
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+
+#define DS26522_RF_ADDR_START   0x00
+#define DS26522_RF_ADDR_END     0xef
+#define DS26522_GLB_ADDR_START  0xf0
+#define DS26522_GLB_ADDR_END    0xff
+#define DS26522_TF_ADDR_START   0x100
+#define DS26522_TF_ADDR_END     0x1ef
+#define DS26522_LIU_ADDR_START  0x1000
+#define DS26522_LIU_ADDR_END    0x101f
+#define DS26522_TEST_ADDR_START 0x1008
+#define DS26522_TEST_ADDR_END   0x101f
+#define DS26522_BERT_ADDR_START 0x1100
+#define DS26522_BERT_ADDR_END   0x110f
+
+#define DS26522_RMMR_ADDR       0x80
+#define DS26522_RCR1_ADDR       0x81
+#define DS26522_RCR3_ADDR       0x83
+#define DS26522_RIOCR_ADDR      0x84
+
+#define DS26522_GTCR1_ADDR      0xf0
+#define DS26522_GFCR_ADDR       0xf1
+#define DS26522_GTCR2_ADDR      0xf2
+#define DS26522_GTCCR_ADDR      0xf3
+#define DS26522_GLSRR_ADDR      0xf5
+#define DS26522_GFSRR_ADDR      0xf6
+#define DS26522_IDR_ADDR        0xf8
+
+#define DS26522_E1TAF_ADDR      0x164
+#define DS26522_E1TNAF_ADDR     0x165
+#define DS26522_TMMR_ADDR       0x180
+#define DS26522_TCR1_ADDR       0x181
+#define DS26522_TIOCR_ADDR      0x184
+
+#define DS26522_LTRCR_ADDR      0x1000
+#define DS26522_LTITSR_ADDR     0x1001
+#define DS26522_LMCR_ADDR       0x1002
+#define DS26522_LRISMR_ADDR     0x1007
+
+#define MAX_NUM_OF_CHANNELS     8
+#define PQ_MDS_8E1T1_BRD_REV    0x00
+#define PQ_MDS_8E1T1_PLD_REV    0x00
+
+#define DS26522_GTCCR_BPREFSEL_REFCLKIN 0xa0
+#define DS26522_GTCCR_BFREQSEL_1544KHZ  0x08
+#define DS26522_GTCCR_FREQSEL_1544KHZ   0x04
+#define DS26522_GTCCR_BFREQSEL_2048KHZ  0x00
+#define DS26522_GTCCR_FREQSEL_2048KHZ   0x00
+
+#define DS26522_GFCR_BPCLK_2048KHZ      0x00
+
+#define DS26522_GTCR2_TSSYNCOUT 0x02
+#define DS26522_GTCR1   0x00
+
+#define DS26522_GFSRR_RESET     0x01
+#define DS26522_GFSRR_NORMAL    0x00
+
+#define DS26522_GLSRR_RESET     0x01
+#define DS26522_GLSRR_NORMAL    0x00
+
+#define DS26522_RMMR_SFTRST     0x02
+#define DS26522_RMMR_FRM_EN     0x80
+#define DS26522_RMMR_INIT_DONE  0x40
+#define DS26522_RMMR_T1         0x00
+#define DS26522_RMMR_E1         0x01
+
+#define DS26522_E1TAF_DEFAULT   0x1b
+#define DS26522_E1TNAF_DEFAULT  0x40
+
+#define DS26522_TMMR_SFTRST     0x02
+#define DS26522_TMMR_FRM_EN     0x80
+#define DS26522_TMMR_INIT_DONE  0x40
+#define DS26522_TMMR_T1         0x00
+#define DS26522_TMMR_E1         0x01
+
+#define DS26522_RCR1_T1_SYNCT   0x80
+#define DS26522_RCR1_T1_RB8ZS   0x40
+#define DS26522_RCR1_T1_SYNCC   0x08
+
+#define DS26522_RCR1_E1_HDB3    0x40
+#define DS26522_RCR1_E1_CCS     0x20
+
+#define DS26522_RIOCR_1544KHZ   0x00
+#define DS26522_RIOCR_2048KHZ   0x10
+#define DS26522_RIOCR_RSIO_OUT  0x00
+
+#define DS26522_RCR3_FLB        0x01
+
+#define DS26522_TIOCR_1544KHZ   0x00
+#define DS26522_TIOCR_2048KHZ   0x10
+#define DS26522_TIOCR_TSIO_OUT  0x04
+
+#define DS26522_TCR1_TB8ZS      0x04
+
+#define DS26522_LTRCR_T1        0x02
+#define DS26522_LTRCR_E1        0x00
+
+#define DS26522_LTITSR_TLIS_75OHM       0x00
+#define DS26522_LTITSR_LBOS_75OHM       0x00
+#define DS26522_LTITSR_TLIS_100OHM      0x10
+#define DS26522_LTITSR_TLIS_0DB_CSU     0x00
+
+#define DS26522_LRISMR_75OHM    0x00
+#define DS26522_LRISMR_100OHM   0x10
+#define DS26522_LRISMR_MAX      0x03
+
+#define DS26522_LMCR_TE 0x01
+
+enum line_rate {
+        LINE_RATE_T1,   /* T1 line rate (1.544 Mbps)      */
+        LINE_RATE_E1    /* E1 line rate (2.048 Mbps)     */
+};
+
+enum tdm_trans_mode {
+        NORMAL = 0,
+        FRAMER_LB
+};
+
+enum card_support_type {
+        LM_CARD = 0,
+        DS26522_CARD,
+        NO_CARD
+};
diff --git a/drivers/net/wireless/ath/ath10k/ahb.c b/drivers/net/wireless/ath/ath10k/ahb.c
index bd62bc19e758..acec16b9cf49 100644
--- a/drivers/net/wireless/ath/ath10k/ahb.c
+++ b/drivers/net/wireless/ath/ath10k/ahb.c
@@ -25,10 +25,9 @@
 #include "ahb.h"
 
 static const struct of_device_id ath10k_ahb_of_match[] = {
-        /* TODO: enable this entry once everything in place.
-         * { .compatible = "qcom,ipq4019-wifi",
-         *   .data = (void *)ATH10K_HW_QCA4019 },
-         */
+        { .compatible = "qcom,ipq4019-wifi",
+          .data = (void *)ATH10K_HW_QCA4019
+        },
         { }
 };
 
@@ -476,6 +475,7 @@ static irqreturn_t ath10k_ahb_interrupt_handler(int irq, void *arg)
 
 static int ath10k_ahb_request_irq_legacy(struct ath10k *ar)
 {
+        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
         struct ath10k_ahb *ar_ahb = ath10k_ahb_priv(ar);
         int ret;
 
@@ -487,6 +487,7 @@ static int ath10k_ahb_request_irq_legacy(struct ath10k *ar)
                             ar_ahb->irq, ret);
                 return ret;
         }
+        ar_pci->oper_irq_mode = ATH10K_PCI_IRQ_LEGACY;
 
         return 0;
 }
@@ -918,8 +919,6 @@ int ath10k_ahb_init(void)
 {
         int ret;
 
-        printk(KERN_ERR "AHB support is still work in progress\n");
-
         ret = platform_driver_register(&ath10k_ahb_driver);
         if (ret)
                 printk(KERN_ERR "failed to register ath10k ahb driver: %d\n",
diff --git a/drivers/net/wireless/ath/ath10k/core.c b/drivers/net/wireless/ath/ath10k/core.c
index a92a0ba829f5..dfb3db0ee5d1 100644
--- a/drivers/net/wireless/ath/ath10k/core.c
+++ b/drivers/net/wireless/ath/ath10k/core.c
@@ -18,6 +18,7 @@
 #include <linux/module.h>
 #include <linux/firmware.h>
 #include <linux/of.h>
+#include <asm/byteorder.h>
 
 #include "core.h"
 #include "mac.h"
@@ -55,7 +56,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
                 .name = "qca988x hw2.0",
                 .patch_load_addr = QCA988X_HW_2_0_PATCH_LOAD_ADDR,
                 .uart_pin = 7,
-                .has_shifted_cc_wraparound = true,
+                .cc_wraparound_type = ATH10K_HW_CC_WRAP_SHIFTED_ALL,
                 .otp_exe_param = 0,
                 .channel_counters_freq_hz = 88000,
                 .max_probe_resp_desc_thres = 0,
@@ -69,6 +70,25 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
                 },
         },
         {
+                .id = QCA9887_HW_1_0_VERSION,
+                .dev_id = QCA9887_1_0_DEVICE_ID,
+                .name = "qca9887 hw1.0",
+                .patch_load_addr = QCA9887_HW_1_0_PATCH_LOAD_ADDR,
+                .uart_pin = 7,
+                .cc_wraparound_type = ATH10K_HW_CC_WRAP_SHIFTED_ALL,
+                .otp_exe_param = 0,
+                .channel_counters_freq_hz = 88000,
+                .max_probe_resp_desc_thres = 0,
+                .hw_4addr_pad = ATH10K_HW_4ADDR_PAD_AFTER,
+                .cal_data_len = 2116,
+                .fw = {
+                        .dir = QCA9887_HW_1_0_FW_DIR,
+                        .board = QCA9887_HW_1_0_BOARD_DATA_FILE,
+                        .board_size = QCA9887_BOARD_DATA_SZ,
+                        .board_ext_size = QCA9887_BOARD_EXT_DATA_SZ,
+                },
+        },
+        {
                 .id = QCA6174_HW_2_1_VERSION,
                 .dev_id = QCA6164_2_1_DEVICE_ID,
                 .name = "qca6164 hw2.1",
@@ -148,6 +168,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
                 .uart_pin = 7,
                 .otp_exe_param = 0x00000700,
                 .continuous_frag_desc = true,
+                .cck_rate_map_rev2 = true,
                 .channel_counters_freq_hz = 150000,
                 .max_probe_resp_desc_thres = 24,
                 .hw_4addr_pad = ATH10K_HW_4ADDR_PAD_BEFORE,
@@ -163,6 +184,29 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
                 },
         },
         {
+                .id = QCA9984_HW_1_0_DEV_VERSION,
+                .dev_id = QCA9984_1_0_DEVICE_ID,
+                .name = "qca9984/qca9994 hw1.0",
+                .patch_load_addr = QCA9984_HW_1_0_PATCH_LOAD_ADDR,
+                .uart_pin = 7,
+                .otp_exe_param = 0x00000700,
+                .continuous_frag_desc = true,
+                .cck_rate_map_rev2 = true,
+                .channel_counters_freq_hz = 150000,
+                .max_probe_resp_desc_thres = 24,
+                .hw_4addr_pad = ATH10K_HW_4ADDR_PAD_BEFORE,
+                .tx_chain_mask = 0xf,
+                .rx_chain_mask = 0xf,
+                .max_spatial_stream = 4,
+                .cal_data_len = 12064,
+                .fw = {
+                        .dir = QCA9984_HW_1_0_FW_DIR,
+                        .board = QCA9984_HW_1_0_BOARD_DATA_FILE,
+                        .board_size = QCA99X0_BOARD_DATA_SZ,
+                        .board_ext_size = QCA99X0_BOARD_EXT_DATA_SZ,
+                },
+        },
+        {
                 .id = QCA9377_HW_1_0_DEV_VERSION,
                 .dev_id = QCA9377_1_0_DEVICE_ID,
                 .name = "qca9377 hw1.0",
@@ -202,9 +246,10 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
                 .name = "qca4019 hw1.0",
                 .patch_load_addr = QCA4019_HW_1_0_PATCH_LOAD_ADDR,
                 .uart_pin = 7,
-                .has_shifted_cc_wraparound = true,
+                .cc_wraparound_type = ATH10K_HW_CC_WRAP_SHIFTED_EACH,
                 .otp_exe_param = 0x0010000,
                 .continuous_frag_desc = true,
+                .cck_rate_map_rev2 = true,
                 .channel_counters_freq_hz = 125000,
                 .max_probe_resp_desc_thres = 24,
                 .hw_4addr_pad = ATH10K_HW_4ADDR_PAD_BEFORE,
@@ -236,6 +281,7 @@ static const char *const ath10k_core_fw_feature_str[] = {
         [ATH10K_FW_FEATURE_SUPPORTS_ADAPTIVE_CCA] = "adaptive-cca",
         [ATH10K_FW_FEATURE_MFP_SUPPORT] = "mfp",
         [ATH10K_FW_FEATURE_PEER_FLOW_CONTROL] = "peer-flow-ctrl",
+        [ATH10K_FW_FEATURE_BTCOEX_PARAM] = "btcoex-param",
 };
 
 static unsigned int ath10k_core_get_fw_feature_str(char *buf,
@@ -531,6 +577,35 @@ out:
         return ret;
 }
 
+static int ath10k_download_cal_eeprom(struct ath10k *ar)
+{
+        size_t data_len;
+        void *data = NULL;
+        int ret;
+
+        ret = ath10k_hif_fetch_cal_eeprom(ar, &data, &data_len);
+        if (ret) {
+                if (ret != -EOPNOTSUPP)
+                        ath10k_warn(ar, "failed to read calibration data from EEPROM: %d\n",
+                                    ret);
+                goto out_free;
+        }
+
+        ret = ath10k_download_board_data(ar, data, data_len);
+        if (ret) {
+                ath10k_warn(ar, "failed to download calibration data from EEPROM: %d\n",
+                            ret);
+                goto out_free;
+        }
+
+        ret = 0;
+
+out_free:
+        kfree(data);
+
+        return ret;
+}
+
 static int ath10k_core_get_board_id_from_otp(struct ath10k *ar)
 {
         u32 result, address;
@@ -1293,7 +1368,17 @@ static int ath10k_download_cal_data(struct ath10k *ar)
         }
 
         ath10k_dbg(ar, ATH10K_DBG_BOOT,
-                   "boot did not find DT entry, try OTP next: %d\n",
+                   "boot did not find DT entry, try target EEPROM next: %d\n",
+                   ret);
+
+        ret = ath10k_download_cal_eeprom(ar);
+        if (ret == 0) {
+                ar->cal_mode = ATH10K_CAL_MODE_EEPROM;
+                goto done;
+        }
+
+        ath10k_dbg(ar, ATH10K_DBG_BOOT,
+                   "boot did not find target EEPROM entry, try OTP next: %d\n",
                    ret);
 
         ret = ath10k_download_and_run_otp(ar);
@@ -1733,6 +1818,16 @@ int ath10k_core_start(struct ath10k *ar, enum ath10k_firmware_mode mode,
                 if (test_bit(WMI_SERVICE_BSS_CHANNEL_INFO_64, ar->wmi.svc_map))
                         val |= WMI_10_4_BSS_CHANNEL_INFO_64;
 
+                /* 10.4 firmware supports BT-Coex without reloading firmware
+                 * via pdev param. To support Bluetooth coexistence pdev param,
+                 * WMI_COEX_GPIO_SUPPORT of extended resource config should be
+                 * enabled always.
+                 */
+                if (test_bit(WMI_SERVICE_COEX_GPIO, ar->wmi.svc_map) &&
+                    test_bit(ATH10K_FW_FEATURE_BTCOEX_PARAM,
+                             ar->running_fw->fw_file.fw_features))
+                        val |= WMI_10_4_COEX_GPIO_SUPPORT;
+
                 status = ath10k_mac_ext_resource_config(ar, val);
                 if (status) {
                         ath10k_err(ar,
@@ -2062,6 +2157,7 @@ struct ath10k *ath10k_core_create(size_t priv_size, struct device *dev,
 
         switch (hw_rev) {
         case ATH10K_HW_QCA988X:
+        case ATH10K_HW_QCA9887:
                 ar->regs = &qca988x_regs;
                 ar->hw_values = &qca988x_values;
                 break;
@@ -2071,6 +2167,7 @@ struct ath10k *ath10k_core_create(size_t priv_size, struct device *dev,
                 ar->hw_values = &qca6174_values;
                 break;
         case ATH10K_HW_QCA99X0:
+        case ATH10K_HW_QCA9984:
                 ar->regs = &qca99x0_regs;
                 ar->hw_values = &qca99x0_values;
                 break;
@@ -2159,5 +2256,5 @@ void ath10k_core_destroy(struct ath10k *ar)
 EXPORT_SYMBOL(ath10k_core_destroy);
 
 MODULE_AUTHOR("Qualcomm Atheros");
-MODULE_DESCRIPTION("Core module for QCA988X PCIe devices.");
+MODULE_DESCRIPTION("Core module for Qualcomm Atheros 802.11ac wireless LAN cards.");
 MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/net/wireless/ath/ath10k/core.h b/drivers/net/wireless/ath/ath10k/core.h
index 1852e0ee3fa1..3da18c9dbd7a 100644
--- a/drivers/net/wireless/ath/ath10k/core.h
+++ b/drivers/net/wireless/ath/ath10k/core.h
@@ -535,6 +535,13 @@ enum ath10k_fw_features {
          */
         ATH10K_FW_FEATURE_PEER_FLOW_CONTROL = 13,
 
+        /* Firmware supports BT-Coex without reloading firmware via pdev param.
+         * To support Bluetooth coexistence pdev param, WMI_COEX_GPIO_SUPPORT of
+         * extended resource config should be enabled always. This firmware IE
+         * is used to configure WMI_COEX_GPIO_SUPPORT.
+         */
+        ATH10K_FW_FEATURE_BTCOEX_PARAM = 14,
+
         /* keep last */
         ATH10K_FW_FEATURE_COUNT,
 };
@@ -571,6 +578,7 @@ enum ath10k_cal_mode {
         ATH10K_CAL_MODE_DT,
         ATH10K_PRE_CAL_MODE_FILE,
         ATH10K_PRE_CAL_MODE_DT,
+        ATH10K_CAL_MODE_EEPROM,
 };
 
 enum ath10k_crypt_mode {
@@ -593,6 +601,8 @@ static inline const char *ath10k_cal_mode_str(enum ath10k_cal_mode mode)
                 return "pre-cal-file";
         case ATH10K_PRE_CAL_MODE_DT:
                 return "pre-cal-dt";
+        case ATH10K_CAL_MODE_EEPROM:
+                return "eeprom";
         }
 
         return "unknown";
@@ -703,12 +713,10 @@ struct ath10k {
                 int uart_pin;
                 u32 otp_exe_param;
 
-                /* This is true if given HW chip has a quirky Cycle Counter
-                 * wraparound which resets to 0x7fffffff instead of 0. All
-                 * other CC related counters (e.g. Rx Clear Count) are divided
-                 * by 2 so they never wraparound themselves.
+                /* Type of hw cycle counter wraparound logic, for more info
+                 * refer enum ath10k_hw_cc_wraparound_type.
                  */
-                bool has_shifted_cc_wraparound;
+                enum ath10k_hw_cc_wraparound_type cc_wraparound_type;
 
                 /* Some of chip expects fragment descriptor to be continuous
                  * memory for any TX operation. Set continuous_frag_desc flag
@@ -716,6 +724,12 @@ struct ath10k {
                  */
                 bool continuous_frag_desc;
 
+                /* CCK hardware rate table mapping for the newer chipsets
+                 * like QCA99X0, QCA4019 got revised. The CCK h/w rate values
+                 * are in a proper order with respect to the rate/preamble
+                 */
+                bool cck_rate_map_rev2;
+
                 u32 channel_counters_freq_hz;
 
                 /* Mgmt tx descriptors threshold for limiting probe response
diff --git a/drivers/net/wireless/ath/ath10k/debug.c b/drivers/net/wireless/ath/ath10k/debug.c
index e2511550fbb8..8fbb8f2c7828 100644
--- a/drivers/net/wireless/ath/ath10k/debug.c
+++ b/drivers/net/wireless/ath/ath10k/debug.c
@@ -609,25 +609,23 @@ static ssize_t ath10k_write_simulate_fw_crash(struct file *file,
         char buf[32];
         int ret;
 
-        mutex_lock(&ar->conf_mutex);
-
         simple_write_to_buffer(buf, sizeof(buf) - 1, ppos, user_buf, count);
 
         /* make sure that buf is null terminated */
         buf[sizeof(buf) - 1] = 0;
 
+        /* drop the possible '\n' from the end */
+        if (buf[count - 1] == '\n')
+                buf[count - 1] = 0;
+
+        mutex_lock(&ar->conf_mutex);
+
         if (ar->state != ATH10K_STATE_ON &&
             ar->state != ATH10K_STATE_RESTARTED) {
                 ret = -ENETDOWN;
                 goto exit;
         }
 
-        /* drop the possible '\n' from the end */
-        if (buf[count - 1] == '\n') {
-                buf[count - 1] = 0;
-                count--;
-        }
-
         if (!strcmp(buf, "soft")) {
                 ath10k_info(ar, "simulating soft firmware crash\n");
                 ret = ath10k_wmi_force_fw_hang(ar, WMI_FORCE_FW_HANG_ASSERT, 0);
@@ -2127,6 +2125,7 @@ static ssize_t ath10k_write_btcoex(struct file *file,
         size_t buf_size;
         int ret;
         bool val;
+        u32 pdev_param;
 
         buf_size = min(count, (sizeof(buf) - 1));
         if (copy_from_user(buf, ubuf, buf_size))
@@ -2150,14 +2149,25 @@ static ssize_t ath10k_write_btcoex(struct file *file,
                 goto exit;
         }
 
+        pdev_param = ar->wmi.pdev_param->enable_btcoex;
+        if (test_bit(ATH10K_FW_FEATURE_BTCOEX_PARAM,
+                     ar->running_fw->fw_file.fw_features)) {
+                ret = ath10k_wmi_pdev_set_param(ar, pdev_param, val);
+                if (ret) {
+                        ath10k_warn(ar, "failed to enable btcoex: %d\n", ret);
+                        ret = count;
+                        goto exit;
+                }
+        } else {
+                ath10k_info(ar, "restarting firmware due to btcoex change");
+                queue_work(ar->workqueue, &ar->restart_work);
+        }
+
         if (val)
                 set_bit(ATH10K_FLAG_BTCOEX, &ar->dev_flags);
         else
                 clear_bit(ATH10K_FLAG_BTCOEX, &ar->dev_flags);
 
-        ath10k_info(ar, "restarting firmware due to btcoex change");
-
-        queue_work(ar->workqueue, &ar->restart_work);
         ret = count;
 
 exit:
diff --git a/drivers/net/wireless/ath/ath10k/hif.h b/drivers/net/wireless/ath/ath10k/hif.h
index 89e7076c919f..b2566b06e1e1 100644
--- a/drivers/net/wireless/ath/ath10k/hif.h
+++ b/drivers/net/wireless/ath/ath10k/hif.h
@@ -87,6 +87,10 @@ struct ath10k_hif_ops {
 
         int (*suspend)(struct ath10k *ar);
         int (*resume)(struct ath10k *ar);
+
+        /* fetch calibration data from target eeprom */
+        int (*fetch_cal_eeprom)(struct ath10k *ar, void **data,
+                                size_t *data_len);
 };
 
 static inline int ath10k_hif_tx_sg(struct ath10k *ar, u8 pipe_id,
@@ -202,4 +206,14 @@ static inline void ath10k_hif_write32(struct ath10k *ar,
         ar->hif.ops->write32(ar, address, data);
 }
 
+static inline int ath10k_hif_fetch_cal_eeprom(struct ath10k *ar,
+                                              void **data,
+                                              size_t *data_len)
+{
+        if (!ar->hif.ops->fetch_cal_eeprom)
+                return -EOPNOTSUPP;
+
+        return ar->hif.ops->fetch_cal_eeprom(ar, data, data_len);
+}
+
 #endif /* _HIF_H_ */
diff --git a/drivers/net/wireless/ath/ath10k/htt.h b/drivers/net/wireless/ath/ath10k/htt.h
index 911c535d0863..430a83e142aa 100644
--- a/drivers/net/wireless/ath/ath10k/htt.h
+++ b/drivers/net/wireless/ath/ath10k/htt.h
@@ -485,10 +485,10 @@ struct htt_mgmt_tx_completion {
         __le32 status;
 } __packed;
 
-#define HTT_RX_INDICATION_INFO0_EXT_TID_MASK  (0x3F)
+#define HTT_RX_INDICATION_INFO0_EXT_TID_MASK  (0x1F)
 #define HTT_RX_INDICATION_INFO0_EXT_TID_LSB   (0)
-#define HTT_RX_INDICATION_INFO0_FLUSH_VALID   (1 << 6)
-#define HTT_RX_INDICATION_INFO0_RELEASE_VALID (1 << 7)
+#define HTT_RX_INDICATION_INFO0_FLUSH_VALID   (1 << 5)
+#define HTT_RX_INDICATION_INFO0_RELEASE_VALID (1 << 6)
 
 #define HTT_RX_INDICATION_INFO1_FLUSH_START_SEQNO_MASK   0x0000003F
 #define HTT_RX_INDICATION_INFO1_FLUSH_START_SEQNO_LSB    0
diff --git a/drivers/net/wireless/ath/ath10k/htt_rx.c b/drivers/net/wireless/ath/ath10k/htt_rx.c
index 813cdd2621a1..80e645302b54 100644
--- a/drivers/net/wireless/ath/ath10k/htt_rx.c
+++ b/drivers/net/wireless/ath/ath10k/htt_rx.c
@@ -748,7 +748,7 @@ ath10k_htt_rx_h_peer_channel(struct ath10k *ar, struct htt_rx_desc *rxd)
         if (WARN_ON_ONCE(!arvif))
                 return NULL;
 
-        if (WARN_ON(ath10k_mac_vif_chan(arvif->vif, &def)))
+        if (WARN_ON_ONCE(ath10k_mac_vif_chan(arvif->vif, &def)))
                 return NULL;
 
         return def.chan;
@@ -939,7 +939,8 @@ static void ath10k_process_rx(struct ath10k *ar,
                    is_multicast_ether_addr(ieee80211_get_DA(hdr)) ?
                                                         "mcast" : "ucast",
                    (__le16_to_cpu(hdr->seq_ctrl) & IEEE80211_SCTL_SEQ) >> 4,
-                   status->flag == 0 ? "legacy" : "",
+                   (status->flag & (RX_FLAG_HT | RX_FLAG_VHT)) == 0 ?
+                                                        "legacy" : "",
                    status->flag & RX_FLAG_HT ? "ht" : "",
                    status->flag & RX_FLAG_VHT ? "vht" : "",
                    status->flag & RX_FLAG_40MHZ ? "40" : "",
@@ -2181,34 +2182,6 @@ static void ath10k_htt_rx_tx_mode_switch_ind(struct ath10k *ar,
         ath10k_mac_tx_push_pending(ar);
 }
 
-static inline enum nl80211_band phy_mode_to_band(u32 phy_mode)
-{
-        enum nl80211_band band;
-
-        switch (phy_mode) {
-        case MODE_11A:
-        case MODE_11NA_HT20:
-        case MODE_11NA_HT40:
-        case MODE_11AC_VHT20:
-        case MODE_11AC_VHT40:
-        case MODE_11AC_VHT80:
-                band = NL80211_BAND_5GHZ;
-                break;
-        case MODE_11G:
-        case MODE_11B:
-        case MODE_11GONLY:
-        case MODE_11NG_HT20:
-        case MODE_11NG_HT40:
-        case MODE_11AC_VHT20_2G:
-        case MODE_11AC_VHT40_2G:
-        case MODE_11AC_VHT80_2G:
-        default:
-                band = NL80211_BAND_2GHZ;
-        }
-
-        return band;
-}
-
 void ath10k_htt_htc_t2h_msg_handler(struct ath10k *ar, struct sk_buff *skb)
 {
         bool release;
@@ -2290,7 +2263,6 @@ bool ath10k_htt_t2h_msg_handler(struct ath10k *ar, struct sk_buff *skb)
                         ath10k_htt_tx_mgmt_dec_pending(htt);
                         spin_unlock_bh(&htt->tx_lock);
                 }
-                ath10k_mac_tx_push_pending(ar);
                 break;
         }
         case HTT_T2H_MSG_TYPE_TX_COMPL_IND:
@@ -2441,8 +2413,6 @@ static void ath10k_htt_txrx_compl_task(unsigned long ptr)
                 dev_kfree_skb_any(skb);
         }
 
-        ath10k_mac_tx_push_pending(ar);
-
         num_mpdus = atomic_read(&htt->num_mpdus_ready);
 
         while (num_mpdus) {
diff --git a/drivers/net/wireless/ath/ath10k/hw.c b/drivers/net/wireless/ath/ath10k/hw.c
index f544d48518c3..bd86e7a38db9 100644
--- a/drivers/net/wireless/ath/ath10k/hw.c
+++ b/drivers/net/wireless/ath/ath10k/hw.c
@@ -179,17 +179,35 @@ void ath10k_hw_fill_survey_time(struct ath10k *ar, struct survey_info *survey,
                                 u32 cc, u32 rcc, u32 cc_prev, u32 rcc_prev)
 {
         u32 cc_fix = 0;
+        u32 rcc_fix = 0;
+        enum ath10k_hw_cc_wraparound_type wraparound_type;
 
         survey->filled |= SURVEY_INFO_TIME |
                           SURVEY_INFO_TIME_BUSY;
 
-        if (ar->hw_params.has_shifted_cc_wraparound && cc < cc_prev) {
-                cc_fix = 0x7fffffff;
-                survey->filled &= ~SURVEY_INFO_TIME_BUSY;
+        wraparound_type = ar->hw_params.cc_wraparound_type;
+
+        if (cc < cc_prev || rcc < rcc_prev) {
+                switch (wraparound_type) {
+                case ATH10K_HW_CC_WRAP_SHIFTED_ALL:
+                        if (cc < cc_prev) {
+                                cc_fix = 0x7fffffff;
+                                survey->filled &= ~SURVEY_INFO_TIME_BUSY;
+                        }
+                        break;
+                case ATH10K_HW_CC_WRAP_SHIFTED_EACH:
+                        if (cc < cc_prev)
+                                cc_fix = 0x7fffffff;
+                        else
+                                rcc_fix = 0x7fffffff;
+                        break;
+                case ATH10K_HW_CC_WRAP_DISABLED:
+                        break;
+                }
         }
 
         cc -= cc_prev - cc_fix;
-        rcc -= rcc_prev;
+        rcc -= rcc_prev - rcc_fix;
 
         survey->time = CCNT_TO_MSEC(ar, cc);
         survey->time_busy = CCNT_TO_MSEC(ar, rcc);
diff --git a/drivers/net/wireless/ath/ath10k/hw.h b/drivers/net/wireless/ath/ath10k/hw.h
index aedd8987040b..f31d3ce42470 100644
--- a/drivers/net/wireless/ath/ath10k/hw.h
+++ b/drivers/net/wireless/ath/ath10k/hw.h
@@ -26,7 +26,9 @@
 #define QCA6164_2_1_DEVICE_ID   (0x0041)
 #define QCA6174_2_1_DEVICE_ID   (0x003e)
 #define QCA99X0_2_0_DEVICE_ID   (0x0040)
+#define QCA9984_1_0_DEVICE_ID   (0x0046)
 #define QCA9377_1_0_DEVICE_ID   (0x0042)
+#define QCA9887_1_0_DEVICE_ID   (0x0050)
 
 /* QCA988X 1.0 definitions (unsupported) */
 #define QCA988X_HW_1_0_CHIP_ID_REV      0x0
@@ -38,6 +40,13 @@
 #define QCA988X_HW_2_0_BOARD_DATA_FILE  "board.bin"
 #define QCA988X_HW_2_0_PATCH_LOAD_ADDR  0x1234
 
+/* QCA9887 1.0 definitions */
+#define QCA9887_HW_1_0_VERSION          0x4100016d
+#define QCA9887_HW_1_0_CHIP_ID_REV      0
+#define QCA9887_HW_1_0_FW_DIR           ATH10K_FW_DIR "/QCA9887/hw1.0"
+#define QCA9887_HW_1_0_BOARD_DATA_FILE  "board.bin"
+#define QCA9887_HW_1_0_PATCH_LOAD_ADDR  0x1234
+
 /* QCA6174 target BMI version signatures */
 #define QCA6174_HW_1_0_VERSION          0x05000000
 #define QCA6174_HW_1_1_VERSION          0x05000001
@@ -91,6 +100,14 @@ enum qca9377_chip_id_rev {
 #define QCA99X0_HW_2_0_BOARD_DATA_FILE "board.bin"
 #define QCA99X0_HW_2_0_PATCH_LOAD_ADDR  0x1234
 
+/* QCA9984 1.0 defines */
+#define QCA9984_HW_1_0_DEV_VERSION      0x1000000
+#define QCA9984_HW_DEV_TYPE             0xa
+#define QCA9984_HW_1_0_CHIP_ID_REV      0x0
+#define QCA9984_HW_1_0_FW_DIR           ATH10K_FW_DIR "/QCA9984/hw1.0"
+#define QCA9984_HW_1_0_BOARD_DATA_FILE "board.bin"
+#define QCA9984_HW_1_0_PATCH_LOAD_ADDR  0x1234
+
 /* QCA9377 1.0 definitions */
 #define QCA9377_HW_1_0_FW_DIR          ATH10K_FW_DIR "/QCA9377/hw1.0"
 #define QCA9377_HW_1_0_BOARD_DATA_FILE "board.bin"
@@ -193,8 +210,10 @@ enum ath10k_hw_rev {
         ATH10K_HW_QCA988X,
         ATH10K_HW_QCA6174,
         ATH10K_HW_QCA99X0,
+        ATH10K_HW_QCA9984,
         ATH10K_HW_QCA9377,
         ATH10K_HW_QCA4019,
+        ATH10K_HW_QCA9887,
 };
 
 struct ath10k_hw_regs {
@@ -247,8 +266,10 @@ void ath10k_hw_fill_survey_time(struct ath10k *ar, struct survey_info *survey,
                                 u32 cc, u32 rcc, u32 cc_prev, u32 rcc_prev);
 
 #define QCA_REV_988X(ar) ((ar)->hw_rev == ATH10K_HW_QCA988X)
+#define QCA_REV_9887(ar) ((ar)->hw_rev == ATH10K_HW_QCA9887)
 #define QCA_REV_6174(ar) ((ar)->hw_rev == ATH10K_HW_QCA6174)
 #define QCA_REV_99X0(ar) ((ar)->hw_rev == ATH10K_HW_QCA99X0)
+#define QCA_REV_9984(ar) ((ar)->hw_rev == ATH10K_HW_QCA9984)
 #define QCA_REV_9377(ar) ((ar)->hw_rev == ATH10K_HW_QCA9377)
 #define QCA_REV_40XX(ar) ((ar)->hw_rev == ATH10K_HW_QCA4019)
 
@@ -315,11 +336,41 @@ enum ath10k_hw_rate_cck {
         ATH10K_HW_RATE_CCK_SP_2M,
 };
 
+enum ath10k_hw_rate_rev2_cck {
+        ATH10K_HW_RATE_REV2_CCK_LP_1M = 1,
+        ATH10K_HW_RATE_REV2_CCK_LP_2M,
+        ATH10K_HW_RATE_REV2_CCK_LP_5_5M,
+        ATH10K_HW_RATE_REV2_CCK_LP_11M,
+        ATH10K_HW_RATE_REV2_CCK_SP_2M,
+        ATH10K_HW_RATE_REV2_CCK_SP_5_5M,
+        ATH10K_HW_RATE_REV2_CCK_SP_11M,
+};
+
 enum ath10k_hw_4addr_pad {
         ATH10K_HW_4ADDR_PAD_AFTER,
         ATH10K_HW_4ADDR_PAD_BEFORE,
 };
 
+enum ath10k_hw_cc_wraparound_type {
+        ATH10K_HW_CC_WRAP_DISABLED = 0,
+
+        /* This type is when the HW chip has a quirky Cycle Counter
+         * wraparound which resets to 0x7fffffff instead of 0. All
+         * other CC related counters (e.g. Rx Clear Count) are divided
+         * by 2 so they never wraparound themselves.
+         */
+        ATH10K_HW_CC_WRAP_SHIFTED_ALL = 1,
+
+        /* Each hw counter wrapsaround independently. When the
+         * counter overflows the repestive counter is right shifted
+         * by 1, i.e reset to 0x7fffffff, and other counters will be
+         * running unaffected. In this type of wraparound, it should
+         * be possible to report accurate Rx busy time unlike the
+         * first type.
+         */
+        ATH10K_HW_CC_WRAP_SHIFTED_EACH = 2,
+};
+
 /* Target specific defines for MAIN firmware */
 #define TARGET_NUM_VDEVS                        8
 #define TARGET_NUM_PEER_AST                     2
@@ -547,7 +598,10 @@ enum ath10k_hw_4addr_pad {
 #define WLAN_SYSTEM_SLEEP_DISABLE_MASK          0x00000001
 
 #define WLAN_GPIO_PIN0_ADDRESS                  0x00000028
+#define WLAN_GPIO_PIN0_CONFIG_LSB               11
 #define WLAN_GPIO_PIN0_CONFIG_MASK              0x00007800
+#define WLAN_GPIO_PIN0_PAD_PULL_LSB             5
+#define WLAN_GPIO_PIN0_PAD_PULL_MASK            0x00000060
 #define WLAN_GPIO_PIN1_ADDRESS                  0x0000002c
 #define WLAN_GPIO_PIN1_CONFIG_MASK              0x00007800
 #define WLAN_GPIO_PIN10_ADDRESS                 0x00000050
@@ -560,6 +614,8 @@ enum ath10k_hw_4addr_pad {
 #define CLOCK_GPIO_BT_CLK_OUT_EN_MASK           0
 
 #define SI_CONFIG_OFFSET                        0x00000000
+#define SI_CONFIG_ERR_INT_LSB                   19
+#define SI_CONFIG_ERR_INT_MASK                  0x00080000
 #define SI_CONFIG_BIDIR_OD_DATA_LSB             18
 #define SI_CONFIG_BIDIR_OD_DATA_MASK            0x00040000
 #define SI_CONFIG_I2C_LSB                       16
@@ -573,7 +629,9 @@ enum ath10k_hw_4addr_pad {
 #define SI_CONFIG_DIVIDER_LSB                   0
 #define SI_CONFIG_DIVIDER_MASK                  0x0000000f
 #define SI_CS_OFFSET                            0x00000004
+#define SI_CS_DONE_ERR_LSB                      10
 #define SI_CS_DONE_ERR_MASK                     0x00000400
+#define SI_CS_DONE_INT_LSB                      9
 #define SI_CS_DONE_INT_MASK                     0x00000200
 #define SI_CS_START_LSB                         8
 #define SI_CS_START_MASK                        0x00000100
@@ -624,7 +682,10 @@ enum ath10k_hw_4addr_pad {
 #define GPIO_BASE_ADDRESS                       WLAN_GPIO_BASE_ADDRESS
 #define GPIO_PIN0_OFFSET                        WLAN_GPIO_PIN0_ADDRESS
 #define GPIO_PIN1_OFFSET                        WLAN_GPIO_PIN1_ADDRESS
+#define GPIO_PIN0_CONFIG_LSB                    WLAN_GPIO_PIN0_CONFIG_LSB
 #define GPIO_PIN0_CONFIG_MASK                   WLAN_GPIO_PIN0_CONFIG_MASK
+#define GPIO_PIN0_PAD_PULL_LSB                  WLAN_GPIO_PIN0_PAD_PULL_LSB
+#define GPIO_PIN0_PAD_PULL_MASK                 WLAN_GPIO_PIN0_PAD_PULL_MASK
 #define GPIO_PIN1_CONFIG_MASK                   WLAN_GPIO_PIN1_CONFIG_MASK
 #define SI_BASE_ADDRESS                         WLAN_SI_BASE_ADDRESS
 #define SCRATCH_BASE_ADDRESS                    SOC_CORE_BASE_ADDRESS
@@ -679,6 +740,18 @@ enum ath10k_hw_4addr_pad {
 #define WINDOW_READ_ADDR_ADDRESS                MISSING
 #define WINDOW_WRITE_ADDR_ADDRESS               MISSING
 
+#define QCA9887_1_0_I2C_SDA_GPIO_PIN            5
+#define QCA9887_1_0_I2C_SDA_PIN_CONFIG          3
+#define QCA9887_1_0_SI_CLK_GPIO_PIN             17
+#define QCA9887_1_0_SI_CLK_PIN_CONFIG           3
+#define QCA9887_1_0_GPIO_ENABLE_W1TS_LOW_ADDRESS 0x00000010
+
+#define QCA9887_EEPROM_SELECT_READ              0xa10000a0
+#define QCA9887_EEPROM_ADDR_HI_MASK             0x0000ff00
+#define QCA9887_EEPROM_ADDR_HI_LSB              8
+#define QCA9887_EEPROM_ADDR_LO_MASK             0x00ff0000
+#define QCA9887_EEPROM_ADDR_LO_LSB              16
+
 #define RTC_STATE_V_GET(x) (((x) & RTC_STATE_V_MASK) >> RTC_STATE_V_LSB)
 
 #endif /* _HW_H_ */
diff --git a/drivers/net/wireless/ath/ath10k/mac.c b/drivers/net/wireless/ath/ath10k/mac.c
index 4040f9413e86..d4b7a168f7c0 100644
--- a/drivers/net/wireless/ath/ath10k/mac.c
+++ b/drivers/net/wireless/ath/ath10k/mac.c
@@ -62,6 +62,32 @@ static struct ieee80211_rate ath10k_rates[] = {
         { .bitrate = 540, .hw_value = ATH10K_HW_RATE_OFDM_54M },
 };
 
+static struct ieee80211_rate ath10k_rates_rev2[] = {
+        { .bitrate = 10,
+          .hw_value = ATH10K_HW_RATE_REV2_CCK_LP_1M },
+        { .bitrate = 20,
+          .hw_value = ATH10K_HW_RATE_REV2_CCK_LP_2M,
+          .hw_value_short = ATH10K_HW_RATE_REV2_CCK_SP_2M,
+          .flags = IEEE80211_RATE_SHORT_PREAMBLE },
+        { .bitrate = 55,
+          .hw_value = ATH10K_HW_RATE_REV2_CCK_LP_5_5M,
+          .hw_value_short = ATH10K_HW_RATE_REV2_CCK_SP_5_5M,
+          .flags = IEEE80211_RATE_SHORT_PREAMBLE },
+        { .bitrate = 110,
+          .hw_value = ATH10K_HW_RATE_REV2_CCK_LP_11M,
+          .hw_value_short = ATH10K_HW_RATE_REV2_CCK_SP_11M,
+          .flags = IEEE80211_RATE_SHORT_PREAMBLE },
+
+        { .bitrate = 60, .hw_value = ATH10K_HW_RATE_OFDM_6M },
+        { .bitrate = 90, .hw_value = ATH10K_HW_RATE_OFDM_9M },
+        { .bitrate = 120, .hw_value = ATH10K_HW_RATE_OFDM_12M },
+        { .bitrate = 180, .hw_value = ATH10K_HW_RATE_OFDM_18M },
+        { .bitrate = 240, .hw_value = ATH10K_HW_RATE_OFDM_24M },
+        { .bitrate = 360, .hw_value = ATH10K_HW_RATE_OFDM_36M },
+        { .bitrate = 480, .hw_value = ATH10K_HW_RATE_OFDM_48M },
+        { .bitrate = 540, .hw_value = ATH10K_HW_RATE_OFDM_54M },
+};
+
 #define ATH10K_MAC_FIRST_OFDM_RATE_IDX 4
 
 #define ath10k_a_rates (ath10k_rates + ATH10K_MAC_FIRST_OFDM_RATE_IDX)
@@ -70,6 +96,9 @@ static struct ieee80211_rate ath10k_rates[] = {
 #define ath10k_g_rates (ath10k_rates + 0)
 #define ath10k_g_rates_size (ARRAY_SIZE(ath10k_rates))
 
+#define ath10k_g_rates_rev2 (ath10k_rates_rev2 + 0)
+#define ath10k_g_rates_rev2_size (ARRAY_SIZE(ath10k_rates_rev2))
+
 static bool ath10k_mac_bitrate_is_cck(int bitrate)
 {
         switch (bitrate) {
@@ -3781,6 +3810,9 @@ void ath10k_mac_tx_push_pending(struct ath10k *ar)
         int ret;
         int max;
 
+        if (ar->htt.num_pending_tx >= (ar->htt.max_num_pending_tx / 2))
+                return;
+
         spin_lock_bh(&ar->txqs_lock);
         rcu_read_lock();
 
@@ -4051,9 +4083,7 @@ static void ath10k_mac_op_wake_tx_queue(struct ieee80211_hw *hw,
                 list_add_tail(&artxq->list, &ar->txqs);
         spin_unlock_bh(&ar->txqs_lock);
 
-        if (ath10k_mac_tx_can_push(hw, txq))
-                tasklet_schedule(&ar->htt.txrx_compl_task);
-
+        ath10k_mac_tx_push_pending(ar);
         ath10k_htt_tx_txq_update(hw, txq);
 }
 
@@ -4467,6 +4497,19 @@ static int ath10k_start(struct ieee80211_hw *hw)
                 }
         }
 
+        param = ar->wmi.pdev_param->enable_btcoex;
+        if (test_bit(WMI_SERVICE_COEX_GPIO, ar->wmi.svc_map) &&
+            test_bit(ATH10K_FW_FEATURE_BTCOEX_PARAM,
+                     ar->running_fw->fw_file.fw_features)) {
+                ret = ath10k_wmi_pdev_set_param(ar, param, 0);
+                if (ret) {
+                        ath10k_warn(ar,
+                                    "failed to set btcoex param: %d\n", ret);
+                        goto err_core_stop;
+                }
+                clear_bit(ATH10K_FLAG_BTCOEX, &ar->dev_flags);
+        }
+
         ar->num_started_vdevs = 0;
         ath10k_regd_update(ar);
 
@@ -7695,8 +7738,14 @@ int ath10k_mac_register(struct ath10k *ar)
                 band = &ar->mac.sbands[NL80211_BAND_2GHZ];
                 band->n_channels = ARRAY_SIZE(ath10k_2ghz_channels);
                 band->channels = channels;
-                band->n_bitrates = ath10k_g_rates_size;
-                band->bitrates = ath10k_g_rates;
+
+                if (ar->hw_params.cck_rate_map_rev2) {
+                        band->n_bitrates = ath10k_g_rates_rev2_size;
+                        band->bitrates = ath10k_g_rates_rev2;
+                } else {
+                        band->n_bitrates = ath10k_g_rates_size;
+                        band->bitrates = ath10k_g_rates;
+                }
 
                 ar->hw->wiphy->bands[NL80211_BAND_2GHZ] = band;
         }
diff --git a/drivers/net/wireless/ath/ath10k/pci.c b/drivers/net/wireless/ath/ath10k/pci.c
index 8133d7b5b956..f06dd3941bac 100644
--- a/drivers/net/wireless/ath/ath10k/pci.c
+++ b/drivers/net/wireless/ath/ath10k/pci.c
@@ -56,7 +56,9 @@ static const struct pci_device_id ath10k_pci_id_table[] = {
         { PCI_VDEVICE(ATHEROS, QCA6164_2_1_DEVICE_ID) }, /* PCI-E QCA6164 V2.1 */
         { PCI_VDEVICE(ATHEROS, QCA6174_2_1_DEVICE_ID) }, /* PCI-E QCA6174 V2.1 */
         { PCI_VDEVICE(ATHEROS, QCA99X0_2_0_DEVICE_ID) }, /* PCI-E QCA99X0 V2 */
+        { PCI_VDEVICE(ATHEROS, QCA9984_1_0_DEVICE_ID) }, /* PCI-E QCA9984 V1 */
         { PCI_VDEVICE(ATHEROS, QCA9377_1_0_DEVICE_ID) }, /* PCI-E QCA9377 V1 */
+        { PCI_VDEVICE(ATHEROS, QCA9887_1_0_DEVICE_ID) }, /* PCI-E QCA9887 */
         {0}
 };
 
@@ -81,8 +83,12 @@ static const struct ath10k_pci_supp_chip ath10k_pci_supp_chips[] = {
 
         { QCA99X0_2_0_DEVICE_ID, QCA99X0_HW_2_0_CHIP_ID_REV },
 
+        { QCA9984_1_0_DEVICE_ID, QCA9984_HW_1_0_CHIP_ID_REV },
+
         { QCA9377_1_0_DEVICE_ID, QCA9377_HW_1_0_CHIP_ID_REV },
         { QCA9377_1_0_DEVICE_ID, QCA9377_HW_1_1_CHIP_ID_REV },
+
+        { QCA9887_1_0_DEVICE_ID, QCA9887_HW_1_0_CHIP_ID_REV },
 };
 
 static void ath10k_pci_buffer_cleanup(struct ath10k *ar);
@@ -837,6 +843,7 @@ static u32 ath10k_pci_targ_cpu_to_ce_addr(struct ath10k *ar, u32 addr)
 
         switch (ar->hw_rev) {
         case ATH10K_HW_QCA988X:
+        case ATH10K_HW_QCA9887:
         case ATH10K_HW_QCA6174:
         case ATH10K_HW_QCA9377:
                 val = (ath10k_pci_read32(ar, SOC_CORE_BASE_ADDRESS +
@@ -844,6 +851,7 @@ static u32 ath10k_pci_targ_cpu_to_ce_addr(struct ath10k *ar, u32 addr)
                        0x7ff) << 21;
                 break;
         case ATH10K_HW_QCA99X0:
+        case ATH10K_HW_QCA9984:
         case ATH10K_HW_QCA4019:
                 val = ath10k_pci_read32(ar, PCIE_BAR_REG_ADDRESS);
                 break;
@@ -864,7 +872,7 @@ static int ath10k_pci_diag_read_mem(struct ath10k *ar, u32 address, void *data,
         struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
         int ret = 0;
         u32 *buf;
-        unsigned int completed_nbytes, orig_nbytes, remaining_bytes;
+        unsigned int completed_nbytes, alloc_nbytes, remaining_bytes;
         struct ath10k_ce_pipe *ce_diag;
         /* Host buffer address in CE space */
         u32 ce_data;
@@ -882,9 +890,10 @@ static int ath10k_pci_diag_read_mem(struct ath10k *ar, u32 address, void *data,
          *   1) 4-byte alignment
          *   2) Buffer in DMA-able space
          */
-        orig_nbytes = nbytes;
+        alloc_nbytes = min_t(unsigned int, nbytes, DIAG_TRANSFER_LIMIT);
+
         data_buf = (unsigned char *)dma_alloc_coherent(ar->dev,
-                                                       orig_nbytes,
+                                                       alloc_nbytes,
                                                        &ce_data_base,
                                                        GFP_ATOMIC);
 
@@ -892,9 +901,9 @@ static int ath10k_pci_diag_read_mem(struct ath10k *ar, u32 address, void *data,
                 ret = -ENOMEM;
                 goto done;
         }
-        memset(data_buf, 0, orig_nbytes);
+        memset(data_buf, 0, alloc_nbytes);
 
-        remaining_bytes = orig_nbytes;
+        remaining_bytes = nbytes;
         ce_data = ce_data_base;
         while (remaining_bytes) {
                 nbytes = min_t(unsigned int, remaining_bytes,
@@ -954,19 +963,22 @@ static int ath10k_pci_diag_read_mem(struct ath10k *ar, u32 address, void *data,
                 }
 
                 remaining_bytes -= nbytes;
+
+                if (ret) {
+                        ath10k_warn(ar, "failed to read diag value at 0x%x: %d\n",
+                                    address, ret);
+                        break;
+                }
+                memcpy(data, data_buf, nbytes);
+
                 address += nbytes;
-                ce_data += nbytes;
+                data += nbytes;
         }
 
 done:
-        if (ret == 0)
-                memcpy(data, data_buf, orig_nbytes);
-        else
-                ath10k_warn(ar, "failed to read diag value at 0x%x: %d\n",
-                            address, ret);
 
         if (data_buf)
-                dma_free_coherent(ar->dev, orig_nbytes, data_buf,
+                dma_free_coherent(ar->dev, alloc_nbytes, data_buf,
                                   ce_data_base);
 
         spin_unlock_bh(&ar_pci->ce_lock);
@@ -1560,6 +1572,7 @@ static void ath10k_pci_irq_msi_fw_mask(struct ath10k *ar)
 
         switch (ar->hw_rev) {
         case ATH10K_HW_QCA988X:
+        case ATH10K_HW_QCA9887:
         case ATH10K_HW_QCA6174:
         case ATH10K_HW_QCA9377:
                 val = ath10k_pci_read32(ar, SOC_CORE_BASE_ADDRESS +
@@ -1569,6 +1582,7 @@ static void ath10k_pci_irq_msi_fw_mask(struct ath10k *ar)
                                    CORE_CTRL_ADDRESS, val);
                 break;
         case ATH10K_HW_QCA99X0:
+        case ATH10K_HW_QCA9984:
         case ATH10K_HW_QCA4019:
                 /* TODO: Find appropriate register configuration for QCA99X0
                  *  to mask irq/MSI.
@@ -1583,6 +1597,7 @@ static void ath10k_pci_irq_msi_fw_unmask(struct ath10k *ar)
 
         switch (ar->hw_rev) {
         case ATH10K_HW_QCA988X:
+        case ATH10K_HW_QCA9887:
         case ATH10K_HW_QCA6174:
         case ATH10K_HW_QCA9377:
                 val = ath10k_pci_read32(ar, SOC_CORE_BASE_ADDRESS +
@@ -1592,6 +1607,7 @@ static void ath10k_pci_irq_msi_fw_unmask(struct ath10k *ar)
                                    CORE_CTRL_ADDRESS, val);
                 break;
         case ATH10K_HW_QCA99X0:
+        case ATH10K_HW_QCA9984:
         case ATH10K_HW_QCA4019:
                 /* TODO: Find appropriate register configuration for QCA99X0
                  *  to unmask irq/MSI.
@@ -1932,6 +1948,8 @@ static int ath10k_pci_get_num_banks(struct ath10k *ar)
         switch (ar_pci->pdev->device) {
         case QCA988X_2_0_DEVICE_ID:
         case QCA99X0_2_0_DEVICE_ID:
+        case QCA9984_1_0_DEVICE_ID:
+        case QCA9887_1_0_DEVICE_ID:
                 return 1;
         case QCA6164_2_1_DEVICE_ID:
         case QCA6174_2_1_DEVICE_ID:
@@ -2293,16 +2311,20 @@ static int ath10k_pci_warm_reset(struct ath10k *ar)
         return 0;
 }
 
+static int ath10k_pci_qca99x0_soft_chip_reset(struct ath10k *ar)
+{
+        ath10k_pci_irq_disable(ar);
+        return ath10k_pci_qca99x0_chip_reset(ar);
+}
+
 static int ath10k_pci_safe_chip_reset(struct ath10k *ar)
 {
-        if (QCA_REV_988X(ar) || QCA_REV_6174(ar)) {
-                return ath10k_pci_warm_reset(ar);
-        } else if (QCA_REV_99X0(ar)) {
-                ath10k_pci_irq_disable(ar);
-                return ath10k_pci_qca99x0_chip_reset(ar);
-        } else {
+        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
+
+        if (!ar_pci->pci_soft_reset)
                 return -ENOTSUPP;
-        }
+
+        return ar_pci->pci_soft_reset(ar);
 }
 
 static int ath10k_pci_qca988x_chip_reset(struct ath10k *ar)
@@ -2437,16 +2459,12 @@ static int ath10k_pci_qca99x0_chip_reset(struct ath10k *ar)
 
 static int ath10k_pci_chip_reset(struct ath10k *ar)
 {
-        if (QCA_REV_988X(ar))
-                return ath10k_pci_qca988x_chip_reset(ar);
-        else if (QCA_REV_6174(ar))
-                return ath10k_pci_qca6174_chip_reset(ar);
-        else if (QCA_REV_9377(ar))
-                return ath10k_pci_qca6174_chip_reset(ar);
-        else if (QCA_REV_99X0(ar))
-                return ath10k_pci_qca99x0_chip_reset(ar);
-        else
+        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
+
+        if (WARN_ON(!ar_pci->pci_hard_reset))
                 return -ENOTSUPP;
+
+        return ar_pci->pci_hard_reset(ar);
 }
 
 static int ath10k_pci_hif_power_up(struct ath10k *ar)
@@ -2559,6 +2577,144 @@ static int ath10k_pci_hif_resume(struct ath10k *ar)
 }
 #endif
 
+static bool ath10k_pci_validate_cal(void *data, size_t size)
+{
+        __le16 *cal_words = data;
+        u16 checksum = 0;
+        size_t i;
+
+        if (size % 2 != 0)
+                return false;
+
+        for (i = 0; i < size / 2; i++)
+                checksum ^= le16_to_cpu(cal_words[i]);
+
+        return checksum == 0xffff;
+}
+
+static void ath10k_pci_enable_eeprom(struct ath10k *ar)
+{
+        /* Enable SI clock */
+        ath10k_pci_soc_write32(ar, CLOCK_CONTROL_OFFSET, 0x0);
+
+        /* Configure GPIOs for I2C operation */
+        ath10k_pci_write32(ar,
+                           GPIO_BASE_ADDRESS + GPIO_PIN0_OFFSET +
+                           4 * QCA9887_1_0_I2C_SDA_GPIO_PIN,
+                           SM(QCA9887_1_0_I2C_SDA_PIN_CONFIG,
+                              GPIO_PIN0_CONFIG) |
+                           SM(1, GPIO_PIN0_PAD_PULL));
+
+        ath10k_pci_write32(ar,
+                           GPIO_BASE_ADDRESS + GPIO_PIN0_OFFSET +
+                           4 * QCA9887_1_0_SI_CLK_GPIO_PIN,
+                           SM(QCA9887_1_0_SI_CLK_PIN_CONFIG, GPIO_PIN0_CONFIG) |
+                           SM(1, GPIO_PIN0_PAD_PULL));
+
+        ath10k_pci_write32(ar,
+                           GPIO_BASE_ADDRESS +
+                           QCA9887_1_0_GPIO_ENABLE_W1TS_LOW_ADDRESS,
+                           1u << QCA9887_1_0_SI_CLK_GPIO_PIN);
+
+        /* In Swift ASIC - EEPROM clock will be (110MHz/512) = 214KHz */
+        ath10k_pci_write32(ar,
+                           SI_BASE_ADDRESS + SI_CONFIG_OFFSET,
+                           SM(1, SI_CONFIG_ERR_INT) |
+                           SM(1, SI_CONFIG_BIDIR_OD_DATA) |
+                           SM(1, SI_CONFIG_I2C) |
+                           SM(1, SI_CONFIG_POS_SAMPLE) |
+                           SM(1, SI_CONFIG_INACTIVE_DATA) |
+                           SM(1, SI_CONFIG_INACTIVE_CLK) |
+                           SM(8, SI_CONFIG_DIVIDER));
+}
+
+static int ath10k_pci_read_eeprom(struct ath10k *ar, u16 addr, u8 *out)
+{
+        u32 reg;
+        int wait_limit;
+
+        /* set device select byte and for the read operation */
+        reg = QCA9887_EEPROM_SELECT_READ |
+              SM(addr, QCA9887_EEPROM_ADDR_LO) |
+              SM(addr >> 8, QCA9887_EEPROM_ADDR_HI);
+        ath10k_pci_write32(ar, SI_BASE_ADDRESS + SI_TX_DATA0_OFFSET, reg);
+
+        /* write transmit data, transfer length, and START bit */
+        ath10k_pci_write32(ar, SI_BASE_ADDRESS + SI_CS_OFFSET,
+                           SM(1, SI_CS_START) | SM(1, SI_CS_RX_CNT) |
+                           SM(4, SI_CS_TX_CNT));
+
+        /* wait max 1 sec */
+        wait_limit = 100000;
+
+        /* wait for SI_CS_DONE_INT */
+        do {
+                reg = ath10k_pci_read32(ar, SI_BASE_ADDRESS + SI_CS_OFFSET);
+                if (MS(reg, SI_CS_DONE_INT))
+                        break;
+
+                wait_limit--;
+                udelay(10);
+        } while (wait_limit > 0);
+
+        if (!MS(reg, SI_CS_DONE_INT)) {
+                ath10k_err(ar, "timeout while reading device EEPROM at %04x\n",
+                           addr);
+                return -ETIMEDOUT;
+        }
+
+        /* clear SI_CS_DONE_INT */
+        ath10k_pci_write32(ar, SI_BASE_ADDRESS + SI_CS_OFFSET, reg);
+
+        if (MS(reg, SI_CS_DONE_ERR)) {
+                ath10k_err(ar, "failed to read device EEPROM at %04x\n", addr);
+                return -EIO;
+        }
+
+        /* extract receive data */
+        reg = ath10k_pci_read32(ar, SI_BASE_ADDRESS + SI_RX_DATA0_OFFSET);
+        *out = reg;
+
+        return 0;
+}
+
+static int ath10k_pci_hif_fetch_cal_eeprom(struct ath10k *ar, void **data,
+                                           size_t *data_len)
+{
+        u8 *caldata = NULL;
+        size_t calsize, i;
+        int ret;
+
+        if (!QCA_REV_9887(ar))
+                return -EOPNOTSUPP;
+
+        calsize = ar->hw_params.cal_data_len;
+        caldata = kmalloc(calsize, GFP_KERNEL);
+        if (!caldata)
+                return -ENOMEM;
+
+        ath10k_pci_enable_eeprom(ar);
+
+        for (i = 0; i < calsize; i++) {
+                ret = ath10k_pci_read_eeprom(ar, i, &caldata[i]);
+                if (ret)
+                        goto err_free;
+        }
+
+        if (!ath10k_pci_validate_cal(caldata, calsize))
+                goto err_free;
+
+        *data = caldata;
+        *data_len = calsize;
+
+        return 0;
+
+err_free:
+        kfree(data);
+
+        return -EINVAL;
+}
+
 static const struct ath10k_hif_ops ath10k_pci_hif_ops = {
         .tx_sg                  = ath10k_pci_hif_tx_sg,
         .diag_read              = ath10k_pci_hif_diag_read,
@@ -2578,6 +2734,7 @@ static const struct ath10k_hif_ops ath10k_pci_hif_ops = {
         .suspend                = ath10k_pci_hif_suspend,
         .resume                 = ath10k_pci_hif_resume,
 #endif
+        .fetch_cal_eeprom       = ath10k_pci_hif_fetch_cal_eeprom,
 };
 
 /*
@@ -2976,24 +3133,47 @@ static int ath10k_pci_probe(struct pci_dev *pdev,
         enum ath10k_hw_rev hw_rev;
         u32 chip_id;
         bool pci_ps;
+        int (*pci_soft_reset)(struct ath10k *ar);
+        int (*pci_hard_reset)(struct ath10k *ar);
 
         switch (pci_dev->device) {
         case QCA988X_2_0_DEVICE_ID:
                 hw_rev = ATH10K_HW_QCA988X;
                 pci_ps = false;
+                pci_soft_reset = ath10k_pci_warm_reset;
+                pci_hard_reset = ath10k_pci_qca988x_chip_reset;
+                break;
+        case QCA9887_1_0_DEVICE_ID:
+                dev_warn(&pdev->dev, "QCA9887 support is still experimental, there are likely bugs. You have been warned.\n");
+                hw_rev = ATH10K_HW_QCA9887;
+                pci_ps = false;
+                pci_soft_reset = ath10k_pci_warm_reset;
+                pci_hard_reset = ath10k_pci_qca988x_chip_reset;
                 break;
         case QCA6164_2_1_DEVICE_ID:
         case QCA6174_2_1_DEVICE_ID:
                 hw_rev = ATH10K_HW_QCA6174;
                 pci_ps = true;
+                pci_soft_reset = ath10k_pci_warm_reset;
+                pci_hard_reset = ath10k_pci_qca6174_chip_reset;
                 break;
         case QCA99X0_2_0_DEVICE_ID:
                 hw_rev = ATH10K_HW_QCA99X0;
                 pci_ps = false;
+                pci_soft_reset = ath10k_pci_qca99x0_soft_chip_reset;
+                pci_hard_reset = ath10k_pci_qca99x0_chip_reset;
+                break;
+        case QCA9984_1_0_DEVICE_ID:
+                hw_rev = ATH10K_HW_QCA9984;
+                pci_ps = false;
+                pci_soft_reset = ath10k_pci_qca99x0_soft_chip_reset;
+                pci_hard_reset = ath10k_pci_qca99x0_chip_reset;
                 break;
         case QCA9377_1_0_DEVICE_ID:
                 hw_rev = ATH10K_HW_QCA9377;
                 pci_ps = true;
+                pci_soft_reset = NULL;
+                pci_hard_reset = ath10k_pci_qca6174_chip_reset;
                 break;
         default:
                 WARN_ON(1);
@@ -3018,6 +3198,8 @@ static int ath10k_pci_probe(struct pci_dev *pdev,
         ar->dev_id = pci_dev->device;
         ar_pci->pci_ps = pci_ps;
         ar_pci->bus_ops = &ath10k_pci_bus_ops;
+        ar_pci->pci_soft_reset = pci_soft_reset;
+        ar_pci->pci_hard_reset = pci_hard_reset;
 
         ar->id.vendor = pdev->vendor;
         ar->id.device = pdev->device;
@@ -3169,7 +3351,7 @@ static void __exit ath10k_pci_exit(void)
 module_exit(ath10k_pci_exit);
 
 MODULE_AUTHOR("Qualcomm Atheros");
-MODULE_DESCRIPTION("Driver support for Atheros QCA988X PCIe devices");
+MODULE_DESCRIPTION("Driver support for Qualcomm Atheros 802.11ac WLAN PCIe/AHB devices");
 MODULE_LICENSE("Dual BSD/GPL");
 
 /* QCA988x 2.0 firmware files */
@@ -3180,6 +3362,11 @@ MODULE_FIRMWARE(QCA988X_HW_2_0_FW_DIR "/" ATH10K_FW_API5_FILE);
 MODULE_FIRMWARE(QCA988X_HW_2_0_FW_DIR "/" QCA988X_HW_2_0_BOARD_DATA_FILE);
 MODULE_FIRMWARE(QCA988X_HW_2_0_FW_DIR "/" ATH10K_BOARD_API2_FILE);
 
+/* QCA9887 1.0 firmware files */
+MODULE_FIRMWARE(QCA9887_HW_1_0_FW_DIR "/" ATH10K_FW_API5_FILE);
+MODULE_FIRMWARE(QCA9887_HW_1_0_FW_DIR "/" QCA9887_HW_1_0_BOARD_DATA_FILE);
+MODULE_FIRMWARE(QCA9887_HW_1_0_FW_DIR "/" ATH10K_BOARD_API2_FILE);
+
 /* QCA6174 2.1 firmware files */
 MODULE_FIRMWARE(QCA6174_HW_2_1_FW_DIR "/" ATH10K_FW_API4_FILE);
 MODULE_FIRMWARE(QCA6174_HW_2_1_FW_DIR "/" ATH10K_FW_API5_FILE);
diff --git a/drivers/net/wireless/ath/ath10k/pci.h b/drivers/net/wireless/ath/ath10k/pci.h
index 959dc321b75e..6eca1df2ce60 100644
--- a/drivers/net/wireless/ath/ath10k/pci.h
+++ b/drivers/net/wireless/ath/ath10k/pci.h
@@ -234,6 +234,12 @@ struct ath10k_pci {
 
         const struct ath10k_bus_ops *bus_ops;
 
+        /* Chip specific pci reset routine used to do a safe reset */
+        int (*pci_soft_reset)(struct ath10k *ar);
+
+        /* Chip specific pci full reset function */
+        int (*pci_hard_reset)(struct ath10k *ar);
+
         /* Keep this entry in the last, memory for struct ath10k_ahb is
          * allocated (ahb support enabled case) in the continuation of
          * this struct.
diff --git a/drivers/net/wireless/ath/ath10k/rx_desc.h b/drivers/net/wireless/ath/ath10k/rx_desc.h
index ca8d16884af1..034e7a54c5b2 100644
--- a/drivers/net/wireless/ath/ath10k/rx_desc.h
+++ b/drivers/net/wireless/ath/ath10k/rx_desc.h
@@ -656,26 +656,6 @@ struct rx_msdu_end {
  *              Reserved: HW should fill with zero.  FW should ignore.
  */
 
-#define RX_PPDU_START_SIG_RATE_SELECT_OFDM 0
-#define RX_PPDU_START_SIG_RATE_SELECT_CCK  1
-
-#define RX_PPDU_START_SIG_RATE_OFDM_48 0
-#define RX_PPDU_START_SIG_RATE_OFDM_24 1
-#define RX_PPDU_START_SIG_RATE_OFDM_12 2
-#define RX_PPDU_START_SIG_RATE_OFDM_6  3
-#define RX_PPDU_START_SIG_RATE_OFDM_54 4
-#define RX_PPDU_START_SIG_RATE_OFDM_36 5
-#define RX_PPDU_START_SIG_RATE_OFDM_18 6
-#define RX_PPDU_START_SIG_RATE_OFDM_9  7
-
-#define RX_PPDU_START_SIG_RATE_CCK_LP_11  0
-#define RX_PPDU_START_SIG_RATE_CCK_LP_5_5 1
-#define RX_PPDU_START_SIG_RATE_CCK_LP_2   2
-#define RX_PPDU_START_SIG_RATE_CCK_LP_1   3
-#define RX_PPDU_START_SIG_RATE_CCK_SP_11  4
-#define RX_PPDU_START_SIG_RATE_CCK_SP_5_5 5
-#define RX_PPDU_START_SIG_RATE_CCK_SP_2   6
-
 #define HTT_RX_PPDU_START_PREAMBLE_LEGACY        0x04
 #define HTT_RX_PPDU_START_PREAMBLE_HT            0x08
 #define HTT_RX_PPDU_START_PREAMBLE_HT_WITH_TXBF  0x09
@@ -711,25 +691,6 @@ struct rx_msdu_end {
 /* No idea what this flag means. It seems to be always set in rate. */
 #define RX_PPDU_START_RATE_FLAG BIT(3)
 
-enum rx_ppdu_start_rate {
-        RX_PPDU_START_RATE_OFDM_48M = RX_PPDU_START_RATE_FLAG | ATH10K_HW_RATE_OFDM_48M,
-        RX_PPDU_START_RATE_OFDM_24M = RX_PPDU_START_RATE_FLAG | ATH10K_HW_RATE_OFDM_24M,
-        RX_PPDU_START_RATE_OFDM_12M = RX_PPDU_START_RATE_FLAG | ATH10K_HW_RATE_OFDM_12M,
-        RX_PPDU_START_RATE_OFDM_6M  = RX_PPDU_START_RATE_FLAG | ATH10K_HW_RATE_OFDM_6M,
-        RX_PPDU_START_RATE_OFDM_54M = RX_PPDU_START_RATE_FLAG | ATH10K_HW_RATE_OFDM_54M,
-        RX_PPDU_START_RATE_OFDM_36M = RX_PPDU_START_RATE_FLAG | ATH10K_HW_RATE_OFDM_36M,
-        RX_PPDU_START_RATE_OFDM_18M = RX_PPDU_START_RATE_FLAG | ATH10K_HW_RATE_OFDM_18M,
-        RX_PPDU_START_RATE_OFDM_9M  = RX_PPDU_START_RATE_FLAG | ATH10K_HW_RATE_OFDM_9M,
-
-        RX_PPDU_START_RATE_CCK_LP_11M  = RX_PPDU_START_RATE_FLAG | ATH10K_HW_RATE_CCK_LP_11M,
-        RX_PPDU_START_RATE_CCK_LP_5_5M = RX_PPDU_START_RATE_FLAG | ATH10K_HW_RATE_CCK_LP_5_5M,
-        RX_PPDU_START_RATE_CCK_LP_2M   = RX_PPDU_START_RATE_FLAG | ATH10K_HW_RATE_CCK_LP_2M,
-        RX_PPDU_START_RATE_CCK_LP_1M   = RX_PPDU_START_RATE_FLAG | ATH10K_HW_RATE_CCK_LP_1M,
-        RX_PPDU_START_RATE_CCK_SP_11M  = RX_PPDU_START_RATE_FLAG | ATH10K_HW_RATE_CCK_SP_11M,
-        RX_PPDU_START_RATE_CCK_SP_5_5M = RX_PPDU_START_RATE_FLAG | ATH10K_HW_RATE_CCK_SP_5_5M,
-        RX_PPDU_START_RATE_CCK_SP_2M   = RX_PPDU_START_RATE_FLAG | ATH10K_HW_RATE_CCK_SP_2M,
-};
-
 struct rx_ppdu_start {
         struct {
                 u8 pri20_mhz;
@@ -994,7 +955,41 @@ struct rx_pkt_end {
         __le32 info0; /* %RX_PKT_END_INFO0_ */
         __le32 phy_timestamp_1;
         __le32 phy_timestamp_2;
-        __le32 rx_location_info; /* %RX_LOCATION_INFO_ */
+} __packed;
+
+#define RX_LOCATION_INFO0_RTT_FAC_LEGACY_MASK           0x00003fff
+#define RX_LOCATION_INFO0_RTT_FAC_LEGACY_LSB            0
+#define RX_LOCATION_INFO0_RTT_FAC_VHT_MASK              0x1fff8000
+#define RX_LOCATION_INFO0_RTT_FAC_VHT_LSB               15
+#define RX_LOCATION_INFO0_RTT_STRONGEST_CHAIN_MASK      0xc0000000
+#define RX_LOCATION_INFO0_RTT_STRONGEST_CHAIN_LSB       30
+#define RX_LOCATION_INFO0_RTT_FAC_LEGACY_STATUS         BIT(14)
+#define RX_LOCATION_INFO0_RTT_FAC_VHT_STATUS            BIT(29)
+
+#define RX_LOCATION_INFO1_RTT_PREAMBLE_TYPE_MASK        0x0000000c
+#define RX_LOCATION_INFO1_RTT_PREAMBLE_TYPE_LSB         2
+#define RX_LOCATION_INFO1_PKT_BW_MASK                   0x00000030
+#define RX_LOCATION_INFO1_PKT_BW_LSB                    4
+#define RX_LOCATION_INFO1_SKIP_P_SKIP_BTCF_MASK         0x0000ff00
+#define RX_LOCATION_INFO1_SKIP_P_SKIP_BTCF_LSB          8
+#define RX_LOCATION_INFO1_RTT_MSC_RATE_MASK             0x000f0000
+#define RX_LOCATION_INFO1_RTT_MSC_RATE_LSB              16
+#define RX_LOCATION_INFO1_RTT_PBD_LEG_BW_MASK           0x00300000
+#define RX_LOCATION_INFO1_RTT_PBD_LEG_BW_LSB            20
+#define RX_LOCATION_INFO1_TIMING_BACKOFF_MASK           0x07c00000
+#define RX_LOCATION_INFO1_TIMING_BACKOFF_LSB            22
+#define RX_LOCATION_INFO1_RTT_TX_FRAME_PHASE_MASK       0x18000000
+#define RX_LOCATION_INFO1_RTT_TX_FRAME_PHASE_LSB        27
+#define RX_LOCATION_INFO1_RTT_CFR_STATUS                BIT(0)
+#define RX_LOCATION_INFO1_RTT_CIR_STATUS                BIT(1)
+#define RX_LOCATION_INFO1_RTT_GI_TYPE                   BIT(7)
+#define RX_LOCATION_INFO1_RTT_MAC_PHY_PHASE             BIT(29)
+#define RX_LOCATION_INFO1_RTT_TX_DATA_START_X_PHASE     BIT(30)
+#define RX_LOCATION_INFO1_RX_LOCATION_VALID             BIT(31)
+
+struct rx_location_info {
+        __le32 rx_location_info0; /* %RX_LOCATION_INFO0_ */
+        __le32 rx_location_info1; /* %RX_LOCATION_INFO1_ */
 } __packed;
 
 enum rx_phy_ppdu_end_info0 {
@@ -1067,6 +1062,17 @@ struct rx_phy_ppdu_end {
 
 struct rx_ppdu_end_qca99x0 {
         struct rx_pkt_end rx_pkt_end;
+        __le32 rx_location_info; /* %RX_LOCATION_INFO_ */
+        struct rx_phy_ppdu_end rx_phy_ppdu_end;
+        __le32 rx_timing_offset; /* %RX_PPDU_END_RX_TIMING_OFFSET_ */
+        __le32 rx_info; /* %RX_PPDU_END_RX_INFO_ */
+        __le16 bb_length;
+        __le16 info1; /* %RX_PPDU_END_INFO1_ */
+} __packed;
+
+struct rx_ppdu_end_qca9984 {
+        struct rx_pkt_end rx_pkt_end;
+        struct rx_location_info rx_location_info;
         struct rx_phy_ppdu_end rx_phy_ppdu_end;
         __le32 rx_timing_offset; /* %RX_PPDU_END_RX_TIMING_OFFSET_ */
         __le32 rx_info; /* %RX_PPDU_END_RX_INFO_ */
@@ -1080,6 +1086,7 @@ struct rx_ppdu_end {
                 struct rx_ppdu_end_qca988x qca988x;
                 struct rx_ppdu_end_qca6174 qca6174;
                 struct rx_ppdu_end_qca99x0 qca99x0;
+                struct rx_ppdu_end_qca9984 qca9984;
         } __packed;
 } __packed;
 
diff --git a/drivers/net/wireless/ath/ath10k/targaddrs.h b/drivers/net/wireless/ath/ath10k/targaddrs.h
index 8e24099fa936..aaf53a81e78b 100644
--- a/drivers/net/wireless/ath/ath10k/targaddrs.h
+++ b/drivers/net/wireless/ath/ath10k/targaddrs.h
@@ -447,6 +447,9 @@ Fw Mode/SubMode Mask
 #define QCA988X_BOARD_DATA_SZ     7168
 #define QCA988X_BOARD_EXT_DATA_SZ 0
 
+#define QCA9887_BOARD_DATA_SZ     7168
+#define QCA9887_BOARD_EXT_DATA_SZ 0
+
 #define QCA6174_BOARD_DATA_SZ     8192
 #define QCA6174_BOARD_EXT_DATA_SZ 0
 
diff --git a/drivers/net/wireless/ath/ath10k/txrx.c b/drivers/net/wireless/ath/ath10k/txrx.c
index 576e7c42ed65..1966c787998b 100644
--- a/drivers/net/wireless/ath/ath10k/txrx.c
+++ b/drivers/net/wireless/ath/ath10k/txrx.c
@@ -117,6 +117,9 @@ int ath10k_txrx_tx_unref(struct ath10k_htt *htt,
 
         ieee80211_tx_status(htt->ar->hw, msdu);
         /* we do not own the msdu anymore */
+
+        ath10k_mac_tx_push_pending(ar);
+
         return 0;
 }
 
diff --git a/drivers/net/wireless/ath/ath10k/wmi.c b/drivers/net/wireless/ath/ath10k/wmi.c
index 2c300329ebc3..6279ab4a760e 100644
--- a/drivers/net/wireless/ath/ath10k/wmi.c
+++ b/drivers/net/wireless/ath/ath10k/wmi.c
@@ -1104,6 +1104,7 @@ static struct wmi_pdev_param_map wmi_pdev_param_map = {
         .wapi_mbssid_offset = WMI_PDEV_PARAM_UNSUPPORTED,
         .arp_srcaddr = WMI_PDEV_PARAM_UNSUPPORTED,
         .arp_dstaddr = WMI_PDEV_PARAM_UNSUPPORTED,
+        .enable_btcoex = WMI_PDEV_PARAM_UNSUPPORTED,
 };
 
 static struct wmi_pdev_param_map wmi_10x_pdev_param_map = {
@@ -1199,6 +1200,7 @@ static struct wmi_pdev_param_map wmi_10x_pdev_param_map = {
         .wapi_mbssid_offset = WMI_PDEV_PARAM_UNSUPPORTED,
         .arp_srcaddr = WMI_PDEV_PARAM_UNSUPPORTED,
         .arp_dstaddr = WMI_PDEV_PARAM_UNSUPPORTED,
+        .enable_btcoex = WMI_PDEV_PARAM_UNSUPPORTED,
 };
 
 static struct wmi_pdev_param_map wmi_10_2_4_pdev_param_map = {
@@ -1294,6 +1296,7 @@ static struct wmi_pdev_param_map wmi_10_2_4_pdev_param_map = {
         .wapi_mbssid_offset = WMI_PDEV_PARAM_UNSUPPORTED,
         .arp_srcaddr = WMI_PDEV_PARAM_UNSUPPORTED,
         .arp_dstaddr = WMI_PDEV_PARAM_UNSUPPORTED,
+        .enable_btcoex = WMI_PDEV_PARAM_UNSUPPORTED,
 };
 
 /* firmware 10.2 specific mappings */
@@ -1550,6 +1553,7 @@ static struct wmi_pdev_param_map wmi_10_4_pdev_param_map = {
         .wapi_mbssid_offset = WMI_10_4_PDEV_PARAM_WAPI_MBSSID_OFFSET,
         .arp_srcaddr = WMI_10_4_PDEV_PARAM_ARP_SRCADDR,
         .arp_dstaddr = WMI_10_4_PDEV_PARAM_ARP_DSTADDR,
+        .enable_btcoex = WMI_10_4_PDEV_PARAM_ENABLE_BTCOEX,
 };
 
 static const struct wmi_peer_flags_map wmi_peer_flags_map = {
diff --git a/drivers/net/wireless/ath/ath10k/wmi.h b/drivers/net/wireless/ath/ath10k/wmi.h
index 9fdf47ea27d0..90f594e89f94 100644
--- a/drivers/net/wireless/ath/ath10k/wmi.h
+++ b/drivers/net/wireless/ath/ath10k/wmi.h
@@ -3447,6 +3447,7 @@ struct wmi_pdev_param_map {
         u32 wapi_mbssid_offset;
         u32 arp_srcaddr;
         u32 arp_dstaddr;
+        u32 enable_btcoex;
 };
 
 #define WMI_PDEV_PARAM_UNSUPPORTED 0
@@ -3760,6 +3761,9 @@ enum wmi_10_4_pdev_param {
         WMI_10_4_PDEV_PARAM_ATF_OBSS_NOISE_SCH,
         WMI_10_4_PDEV_PARAM_ATF_OBSS_NOISE_SCALING_FACTOR,
         WMI_10_4_PDEV_PARAM_CUST_TXPOWER_SCALE,
+        WMI_10_4_PDEV_PARAM_ATF_DYNAMIC_ENABLE,
+        WMI_10_4_PDEV_PARAM_ATF_SSID_GROUP_POLICY,
+        WMI_10_4_PDEV_PARAM_ENABLE_BTCOEX,
 };
 
 struct wmi_pdev_set_param_cmd {
diff --git a/drivers/net/wireless/ath/ath5k/pcu.c b/drivers/net/wireless/ath/ath5k/pcu.c
index fc47b70988b1..f23c851765df 100644
--- a/drivers/net/wireless/ath/ath5k/pcu.c
+++ b/drivers/net/wireless/ath/ath5k/pcu.c
@@ -219,8 +219,8 @@ ath5k_hw_get_default_sifs(struct ath5k_hw *ah)
                 sifs = AR5K_INIT_SIFS_QUARTER_RATE;
                 break;
         case AR5K_BWMODE_DEFAULT:
-                sifs = AR5K_INIT_SIFS_DEFAULT_BG;
         default:
+                sifs = AR5K_INIT_SIFS_DEFAULT_BG;
                 if (channel->band == NL80211_BAND_5GHZ)
                         sifs = AR5K_INIT_SIFS_DEFAULT_A;
                 break;
diff --git a/drivers/net/wireless/ath/ath6kl/core.h b/drivers/net/wireless/ath/ath6kl/core.h
index 7a1970e484a6..ac25f1781b42 100644
--- a/drivers/net/wireless/ath/ath6kl/core.h
+++ b/drivers/net/wireless/ath/ath6kl/core.h
@@ -148,7 +148,7 @@ enum ath6kl_fw_capability {
         /* ratetable is the 2 stream version (max MCS15) */
         ATH6KL_FW_CAPABILITY_RATETABLE_MCS15,
 
-        /* firmare doesn't support IP checksumming */
+        /* firmware doesn't support IP checksumming */
         ATH6KL_FW_CAPABILITY_NO_IP_CHECKSUM,
 
         /* this needs to be last */
diff --git a/drivers/net/wireless/ath/ath6kl/wmi.c b/drivers/net/wireless/ath/ath6kl/wmi.c
index 631c3a0c572b..b8cf04d11975 100644
--- a/drivers/net/wireless/ath/ath6kl/wmi.c
+++ b/drivers/net/wireless/ath/ath6kl/wmi.c
@@ -2544,8 +2544,7 @@ int ath6kl_wmi_create_pstream_cmd(struct wmi *wmi, u8 if_idx,
         s32 nominal_phy = 0;
         int ret;
 
-        if (!((params->user_pri < 8) &&
-              (params->user_pri <= 0x7) &&
+        if (!((params->user_pri <= 0x7) &&
               (up_to_ac[params->user_pri & 0x7] == params->traffic_class) &&
               (params->traffic_direc == UPLINK_TRAFFIC ||
                params->traffic_direc == DNLINK_TRAFFIC ||
diff --git a/drivers/net/wireless/ath/ath9k/ar9003_eeprom.c b/drivers/net/wireless/ath/ath9k/ar9003_eeprom.c
index dec1a317a070..d0224fc58e78 100644
--- a/drivers/net/wireless/ath/ath9k/ar9003_eeprom.c
+++ b/drivers/net/wireless/ath/ath9k/ar9003_eeprom.c
@@ -3202,8 +3202,7 @@ static int ar9300_compress_decision(struct ath_hw *ah,
                         it, length);
                 break;
         case _CompressBlock:
-                if (reference == 0) {
-                } else {
+                if (reference != 0) {
                         eep = ar9003_eeprom_struct_find_by_id(reference);
                         if (eep == NULL) {
                                 ath_dbg(common, EEPROM,
diff --git a/drivers/net/wireless/ath/ath9k/tx99.c b/drivers/net/wireless/ath/ath9k/tx99.c
index ac4781f37e78..16aca9e28b77 100644
--- a/drivers/net/wireless/ath/ath9k/tx99.c
+++ b/drivers/net/wireless/ath/ath9k/tx99.c
@@ -132,7 +132,6 @@ static int ath9k_tx99_init(struct ath_softc *sc)
         ath9k_ps_wakeup(sc);
 
         ath9k_hw_disable_interrupts(ah);
-        atomic_set(&ah->intr_ref_cnt, -1);
         ath_drain_all_txq(sc);
         ath_stoprecv(sc);
 
@@ -266,7 +265,7 @@ static const struct file_operations fops_tx99_power = {
 
 void ath9k_tx99_init_debug(struct ath_softc *sc)
 {
-        if (!AR_SREV_9300_20_OR_LATER(sc->sc_ah))
+        if (!AR_SREV_9280_20_OR_LATER(sc->sc_ah))
                 return;
 
         debugfs_create_file("tx99", S_IRUSR | S_IWUSR,
diff --git a/drivers/net/wireless/ath/carl9170/Kconfig b/drivers/net/wireless/ath/carl9170/Kconfig
index 1a796e5f69ec..2e34baeaf764 100644
--- a/drivers/net/wireless/ath/carl9170/Kconfig
+++ b/drivers/net/wireless/ath/carl9170/Kconfig
@@ -5,12 +5,10 @@ config CARL9170
         select FW_LOADER
         select CRC32
         help
-          This is another driver for the Atheros "otus" 802.11n USB devices.
+          This is the mainline driver for the Atheros "otus" 802.11n USB devices.
 
-          This driver provides more features than the original,
-          but it needs a special firmware (carl9170-1.fw) to do that.
-
-          The firmware can be downloaded from our wiki here:
+          It needs a special firmware (carl9170-1.fw), which can be downloaded
+          from our wiki here:
           <http://wireless.kernel.org/en/users/Drivers/carl9170>
 
           If you choose to build a module, it'll be called carl9170.
diff --git a/drivers/net/wireless/ath/wil6210/cfg80211.c b/drivers/net/wireless/ath/wil6210/cfg80211.c
index 5769811291bf..62bf9331bd7f 100644
--- a/drivers/net/wireless/ath/wil6210/cfg80211.c
+++ b/drivers/net/wireless/ath/wil6210/cfg80211.c
@@ -378,6 +378,10 @@ static int wil_cfg80211_scan(struct wiphy *wiphy,
         /* social scan on P2P_DEVICE is handled as p2p search */
         if (wdev->iftype == NL80211_IFTYPE_P2P_DEVICE &&
             wil_p2p_is_social_scan(request)) {
+                if (!wil->p2p.p2p_dev_started) {
+                        wil_err(wil, "P2P search requested on stopped P2P device\n");
+                        return -EIO;
+                }
                 wil->scan_request = request;
                 wil->radio_wdev = wdev;
                 rc = wil_p2p_search(wil, request);
@@ -1351,6 +1355,7 @@ static int wil_cfg80211_start_p2p_device(struct wiphy *wiphy,
         struct wil6210_priv *wil = wiphy_to_wil(wiphy);
 
         wil_dbg_misc(wil, "%s: entered\n", __func__);
+        wil->p2p.p2p_dev_started = 1;
         return 0;
 }
 
@@ -1358,8 +1363,19 @@ static void wil_cfg80211_stop_p2p_device(struct wiphy *wiphy,
                                          struct wireless_dev *wdev)
 {
         struct wil6210_priv *wil = wiphy_to_wil(wiphy);
+        u8 started;
 
         wil_dbg_misc(wil, "%s: entered\n", __func__);
+        mutex_lock(&wil->mutex);
+        started = wil_p2p_stop_discovery(wil);
+        if (started && wil->scan_request) {
+                cfg80211_scan_done(wil->scan_request, 1);
+                wil->scan_request = NULL;
+                wil->radio_wdev = wil->wdev;
+        }
+        mutex_unlock(&wil->mutex);
+
+        wil->p2p.p2p_dev_started = 0;
 }
 
 static struct cfg80211_ops wil_cfg80211_ops = {
diff --git a/drivers/net/wireless/ath/wil6210/debug.c b/drivers/net/wireless/ath/wil6210/debug.c
index c312a667c12a..217a4591bde4 100644
--- a/drivers/net/wireless/ath/wil6210/debug.c
+++ b/drivers/net/wireless/ath/wil6210/debug.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2013,2016 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -19,34 +19,31 @@
 
 void __wil_err(struct wil6210_priv *wil, const char *fmt, ...)
 {
-        struct net_device *ndev = wil_to_ndev(wil);
-        struct va_format vaf = {
-                .fmt = fmt,
-        };
+        struct va_format vaf;
         va_list args;
 
         va_start(args, fmt);
+        vaf.fmt = fmt;
         vaf.va = &args;
-        netdev_err(ndev, "%pV", &vaf);
+        netdev_err(wil_to_ndev(wil), "%pV", &vaf);
         trace_wil6210_log_err(&vaf);
         va_end(args);
 }
 
 void __wil_err_ratelimited(struct wil6210_priv *wil, const char *fmt, ...)
 {
-        if (net_ratelimit()) {
-                struct net_device *ndev = wil_to_ndev(wil);
-                struct va_format vaf = {
-                        .fmt = fmt,
-                };
-                va_list args;
+        struct va_format vaf;
+        va_list args;
 
-                va_start(args, fmt);
-                vaf.va = &args;
-                netdev_err(ndev, "%pV", &vaf);
-                trace_wil6210_log_err(&vaf);
-                va_end(args);
-        }
+        if (!net_ratelimit())
+                return;
+
+        va_start(args, fmt);
+        vaf.fmt = fmt;
+        vaf.va = &args;
+        netdev_err(wil_to_ndev(wil), "%pV", &vaf);
+        trace_wil6210_log_err(&vaf);
+        va_end(args);
 }
 
 void wil_dbg_ratelimited(const struct wil6210_priv *wil, const char *fmt, ...)
@@ -67,27 +64,24 @@ void wil_dbg_ratelimited(const struct wil6210_priv *wil, const char *fmt, ...)
 
 void __wil_info(struct wil6210_priv *wil, const char *fmt, ...)
 {
-        struct net_device *ndev = wil_to_ndev(wil);
-        struct va_format vaf = {
-                .fmt = fmt,
-        };
+        struct va_format vaf;
         va_list args;
 
         va_start(args, fmt);
+        vaf.fmt = fmt;
         vaf.va = &args;
-        netdev_info(ndev, "%pV", &vaf);
+        netdev_info(wil_to_ndev(wil), "%pV", &vaf);
         trace_wil6210_log_info(&vaf);
         va_end(args);
 }
 
 void wil_dbg_trace(struct wil6210_priv *wil, const char *fmt, ...)
 {
-        struct va_format vaf = {
-                .fmt = fmt,
-        };
+        struct va_format vaf;
         va_list args;
 
         va_start(args, fmt);
+        vaf.fmt = fmt;
         vaf.va = &args;
         trace_wil6210_log_dbg(&vaf);
         va_end(args);
diff --git a/drivers/net/wireless/ath/wil6210/p2p.c b/drivers/net/wireless/ath/wil6210/p2p.c
index 1c9153894dca..213b8259638c 100644
--- a/drivers/net/wireless/ath/wil6210/p2p.c
+++ b/drivers/net/wireless/ath/wil6210/p2p.c
@@ -114,8 +114,10 @@ int wil_p2p_listen(struct wil6210_priv *wil, unsigned int duration,
         u8 channel = P2P_DMG_SOCIAL_CHANNEL;
         int rc;
 
-        if (chan)
-                channel = chan->hw_value;
+        if (!chan)
+                return -EINVAL;
+
+        channel = chan->hw_value;
 
         wil_dbg_misc(wil, "%s: duration %d\n", __func__, duration);
 
diff --git a/drivers/net/wireless/ath/wil6210/pcie_bus.c b/drivers/net/wireless/ath/wil6210/pcie_bus.c
index aeb72c438e44..7b5c4222bc33 100644
--- a/drivers/net/wireless/ath/wil6210/pcie_bus.c
+++ b/drivers/net/wireless/ath/wil6210/pcie_bus.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012-2015 Qualcomm Atheros, Inc.
+ * Copyright (c) 2012-2016 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -18,13 +18,20 @@
 #include <linux/pci.h>
 #include <linux/moduleparam.h>
 #include <linux/interrupt.h>
-
+#include <linux/suspend.h>
 #include "wil6210.h"
 
 static bool use_msi = true;
 module_param(use_msi, bool, S_IRUGO);
 MODULE_PARM_DESC(use_msi, " Use MSI interrupt, default - true");
 
+#ifdef CONFIG_PM
+#ifdef CONFIG_PM_SLEEP
+static int wil6210_pm_notify(struct notifier_block *notify_block,
+                             unsigned long mode, void *unused);
+#endif /* CONFIG_PM_SLEEP */
+#endif /* CONFIG_PM */
+
 static
 void wil_set_capabilities(struct wil6210_priv *wil)
 {
@@ -238,6 +245,18 @@ static int wil_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *id)
                 goto bus_disable;
         }
 
+#ifdef CONFIG_PM
+#ifdef CONFIG_PM_SLEEP
+        wil->pm_notify.notifier_call = wil6210_pm_notify;
+        rc = register_pm_notifier(&wil->pm_notify);
+        if (rc)
+                /* Do not fail the driver initialization, as suspend can
+                 * be prevented in a later phase if needed
+                 */
+                wil_err(wil, "register_pm_notifier failed: %d\n", rc);
+#endif /* CONFIG_PM_SLEEP */
+#endif /* CONFIG_PM */
+
         wil6210_debugfs_init(wil);
 
 
@@ -267,6 +286,12 @@ static void wil_pcie_remove(struct pci_dev *pdev)
 
         wil_dbg_misc(wil, "%s()\n", __func__);
 
+#ifdef CONFIG_PM
+#ifdef CONFIG_PM_SLEEP
+        unregister_pm_notifier(&wil->pm_notify);
+#endif /* CONFIG_PM_SLEEP */
+#endif /* CONFIG_PM */
+
         wil6210_debugfs_remove(wil);
         wil_if_remove(wil);
         wil_if_pcie_disable(wil);
@@ -335,6 +360,45 @@ static int wil6210_resume(struct device *dev, bool is_runtime)
         return rc;
 }
 
+static int wil6210_pm_notify(struct notifier_block *notify_block,
+                             unsigned long mode, void *unused)
+{
+        struct wil6210_priv *wil = container_of(
+                notify_block, struct wil6210_priv, pm_notify);
+        int rc = 0;
+        enum wil_platform_event evt;
+
+        wil_dbg_pm(wil, "%s: mode (%ld)\n", __func__, mode);
+
+        switch (mode) {
+        case PM_HIBERNATION_PREPARE:
+        case PM_SUSPEND_PREPARE:
+        case PM_RESTORE_PREPARE:
+                rc = wil_can_suspend(wil, false);
+                if (rc)
+                        break;
+                evt = WIL_PLATFORM_EVT_PRE_SUSPEND;
+                if (wil->platform_ops.notify)
+                        rc = wil->platform_ops.notify(wil->platform_handle,
+                                                      evt);
+                break;
+        case PM_POST_SUSPEND:
+        case PM_POST_HIBERNATION:
+        case PM_POST_RESTORE:
+                evt = WIL_PLATFORM_EVT_POST_SUSPEND;
+                if (wil->platform_ops.notify)
+                        rc = wil->platform_ops.notify(wil->platform_handle,
+                                                      evt);
+                break;
+        default:
+                wil_dbg_pm(wil, "unhandled notify mode %ld\n", mode);
+                break;
+        }
+
+        wil_dbg_pm(wil, "notification mode %ld: rc (%d)\n", mode, rc);
+        return rc;
+}
+
 static int wil6210_pm_suspend(struct device *dev)
 {
         return wil6210_suspend(dev, false);
diff --git a/drivers/net/wireless/ath/wil6210/pm.c b/drivers/net/wireless/ath/wil6210/pm.c
index 0b7ecbcac19c..11ee24d509e5 100644
--- a/drivers/net/wireless/ath/wil6210/pm.c
+++ b/drivers/net/wireless/ath/wil6210/pm.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014 Qualcomm Atheros, Inc.
+ * Copyright (c) 2014,2016 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -24,10 +24,32 @@ int wil_can_suspend(struct wil6210_priv *wil, bool is_runtime)
         wil_dbg_pm(wil, "%s(%s)\n", __func__,
                    is_runtime ? "runtime" : "system");
 
+        if (!netif_running(wil_to_ndev(wil))) {
+                /* can always sleep when down */
+                wil_dbg_pm(wil, "Interface is down\n");
+                goto out;
+        }
+        if (test_bit(wil_status_resetting, wil->status)) {
+                wil_dbg_pm(wil, "Delay suspend when resetting\n");
+                rc = -EBUSY;
+                goto out;
+        }
+        if (wil->recovery_state != fw_recovery_idle) {
+                wil_dbg_pm(wil, "Delay suspend during recovery\n");
+                rc = -EBUSY;
+                goto out;
+        }
+
+        /* interface is running */
         switch (wdev->iftype) {
         case NL80211_IFTYPE_MONITOR:
         case NL80211_IFTYPE_STATION:
         case NL80211_IFTYPE_P2P_CLIENT:
+                if (test_bit(wil_status_fwconnecting, wil->status)) {
+                        wil_dbg_pm(wil, "Delay suspend when connecting\n");
+                        rc = -EBUSY;
+                        goto out;
+                }
                 break;
         /* AP-like interface - can't suspend */
         default:
@@ -36,6 +58,7 @@ int wil_can_suspend(struct wil6210_priv *wil, bool is_runtime)
                 break;
         }
 
+out:
         wil_dbg_pm(wil, "%s(%s) => %s (%d)\n", __func__,
                    is_runtime ? "runtime" : "system", rc ? "No" : "Yes", rc);
 
diff --git a/drivers/net/wireless/ath/wil6210/txrx.c b/drivers/net/wireless/ath/wil6210/txrx.c
index a4e43796addb..f2f6a404d3d1 100644
--- a/drivers/net/wireless/ath/wil6210/txrx.c
+++ b/drivers/net/wireless/ath/wil6210/txrx.c
@@ -184,6 +184,13 @@ static void wil_vring_free(struct wil6210_priv *wil, struct vring *vring,
                                         &vring->va[vring->swtail].tx;
 
                         ctx = &vring->ctx[vring->swtail];
+                        if (!ctx) {
+                                wil_dbg_txrx(wil,
+                                             "ctx(%d) was already completed\n",
+                                             vring->swtail);
+                                vring->swtail = wil_vring_next_tail(vring);
+                                continue;
+                        }
                         *d = *_d;
                         wil_txdesc_unmap(dev, d, ctx);
                         if (ctx->skb)
@@ -544,6 +551,12 @@ static int wil_rx_refill(struct wil6210_priv *wil, int count)
                         break;
                 }
         }
+
+        /* make sure all writes to descriptors (shared memory) are done before
+         * committing them to HW
+         */
+        wmb();
+
         wil_w(wil, v->hwtail, v->swtail);
 
         return rc;
@@ -969,6 +982,13 @@ void wil_vring_fini_tx(struct wil6210_priv *wil, int id)
         txdata->dot1x_open = false;
         txdata->enabled = 0; /* no Tx can be in progress or start anew */
         spin_unlock_bh(&txdata->lock);
+        /* napi_synchronize waits for completion of the current NAPI but will
+         * not prevent the next NAPI run.
+         * Add a memory barrier to guarantee that txdata->enabled is zeroed
+         * before napi_synchronize so that the next scheduled NAPI will not
+         * handle this vring
+         */
+        wmb();
         /* make sure NAPI won't touch this vring */
         if (test_bit(wil_status_napi_en, wil->status))
                 napi_synchronize(&wil->napi_tx);
@@ -1551,6 +1571,13 @@ static int __wil_tx_vring_tso(struct wil6210_priv *wil, struct vring *vring,
                              vring_index, used, used + descs_used);
         }
 
+        /* Make sure to advance the head only after descriptor update is done.
+         * This will prevent a race condition where the completion thread
+         * will see the DU bit set from previous run and will handle the
+         * skb before it was completed.
+         */
+        wmb();
+
         /* advance swhead */
         wil_vring_advance_head(vring, descs_used);
         wil_dbg_txrx(wil, "TSO: Tx swhead %d -> %d\n", swhead, vring->swhead);
@@ -1567,7 +1594,7 @@ mem_error:
         while (descs_used > 0) {
                 struct wil_ctx *ctx;
 
-                i = (swhead + descs_used) % vring->size;
+                i = (swhead + descs_used - 1) % vring->size;
                 d = (struct vring_tx_desc *)&vring->va[i].tx;
                 _desc = &vring->va[i].tx;
                 *d = *_desc;
@@ -1691,6 +1718,13 @@ static int __wil_tx_vring(struct wil6210_priv *wil, struct vring *vring,
                              vring_index, used, used + nr_frags + 1);
         }
 
+        /* Make sure to advance the head only after descriptor update is done.
+         * This will prevent a race condition where the completion thread
+         * will see the DU bit set from previous run and will handle the
+         * skb before it was completed.
+         */
+        wmb();
+
         /* advance swhead */
         wil_vring_advance_head(vring, nr_frags + 1);
         wil_dbg_txrx(wil, "Tx[%2d] swhead %d -> %d\n", vring_index, swhead,
@@ -1914,6 +1948,12 @@ int wil_tx_complete(struct wil6210_priv *wil, int ringid)
                                 wil_consume_skb(skb, d->dma.error == 0);
                         }
                         memset(ctx, 0, sizeof(*ctx));
+                        /* Make sure the ctx is zeroed before updating the tail
+                         * to prevent a case where wil_tx_vring will see
+                         * this descriptor as used and handle it before ctx zero
+                         * is completed.
+                         */
+                        wmb();
                         /* There is no need to touch HW descriptor:
                          * - ststus bit TX_DMA_STATUS_DU is set by design,
                          *   so hardware will not try to process this desc.,
diff --git a/drivers/net/wireless/ath/wil6210/wil6210.h b/drivers/net/wireless/ath/wil6210/wil6210.h
index aa09cbcce47c..ecab4af90602 100644
--- a/drivers/net/wireless/ath/wil6210/wil6210.h
+++ b/drivers/net/wireless/ath/wil6210/wil6210.h
@@ -458,6 +458,7 @@ struct wil_tid_crypto_rx {
 struct wil_p2p_info {
         struct ieee80211_channel listen_chan;
         u8 discovery_started;
+        u8 p2p_dev_started;
         u64 cookie;
         struct timer_list discovery_timer; /* listen/search duration */
         struct work_struct discovery_expired_work; /* listen/search expire */
@@ -662,6 +663,11 @@ struct wil6210_priv {
         /* High Access Latency Policy voting */
         struct wil_halp halp;
 
+#ifdef CONFIG_PM
+#ifdef CONFIG_PM_SLEEP
+        struct notifier_block pm_notify;
+#endif /* CONFIG_PM_SLEEP */
+#endif /* CONFIG_PM */
 };
 
 #define wil_to_wiphy(i) (i->wdev->wiphy)
diff --git a/drivers/net/wireless/ath/wil6210/wil_platform.h b/drivers/net/wireless/ath/wil6210/wil_platform.h
index 33d4a34b3b1c..f8c41172a3f4 100644
--- a/drivers/net/wireless/ath/wil6210/wil_platform.h
+++ b/drivers/net/wireless/ath/wil6210/wil_platform.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2015 Qualcomm Atheros, Inc.
+ * Copyright (c) 2014-2016 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -23,6 +23,8 @@ enum wil_platform_event {
         WIL_PLATFORM_EVT_FW_CRASH = 0,
         WIL_PLATFORM_EVT_PRE_RESET = 1,
         WIL_PLATFORM_EVT_FW_RDY = 2,
+        WIL_PLATFORM_EVT_PRE_SUSPEND = 3,
+        WIL_PLATFORM_EVT_POST_SUSPEND = 4,
 };
 
 /**
diff --git a/drivers/net/wireless/broadcom/b43/Makefile b/drivers/net/wireless/broadcom/b43/Makefile
index ddc4df46656f..27fab958e3d5 100644
--- a/drivers/net/wireless/broadcom/b43/Makefile
+++ b/drivers/net/wireless/broadcom/b43/Makefile
@@ -1,6 +1,6 @@
 b43-y                           += main.o
 b43-y                           += bus.o
-b43-$(CONFIG_B43_PHY_G)         += phy_a.o phy_g.o tables.o lo.o wa.o
+b43-$(CONFIG_B43_PHY_G)         += phy_g.o tables.o lo.o wa.o
 b43-$(CONFIG_B43_PHY_N)         += tables_nphy.o
 b43-$(CONFIG_B43_PHY_N)         += radio_2055.o
 b43-$(CONFIG_B43_PHY_N)         += radio_2056.o
diff --git a/drivers/net/wireless/broadcom/b43/leds.c b/drivers/net/wireless/broadcom/b43/leds.c
index d79ab2a227e1..cb987c2ecc6b 100644
--- a/drivers/net/wireless/broadcom/b43/leds.c
+++ b/drivers/net/wireless/broadcom/b43/leds.c
@@ -222,7 +222,7 @@ static void b43_led_get_sprominfo(struct b43_wldev *dev,
         sprom[2] = dev->dev->bus_sprom->gpio2;
         sprom[3] = dev->dev->bus_sprom->gpio3;
 
-        if (sprom[led_index] == 0xFF) {
+        if ((sprom[0] & sprom[1] & sprom[2] & sprom[3]) == 0xff) {
                 /* There is no LED information in the SPROM
                  * for this LED. Hardcode it here. */
                 *activelow = false;
@@ -250,7 +250,11 @@ static void b43_led_get_sprominfo(struct b43_wldev *dev,
                         return;
                 }
         } else {
-                *behaviour = sprom[led_index] & B43_LED_BEHAVIOUR;
+                /* keep LED disabled if no mapping is defined */
+                if (sprom[led_index] == 0xff)
+                        *behaviour = B43_LED_OFF;
+                else
+                        *behaviour = sprom[led_index] & B43_LED_BEHAVIOUR;
                 *activelow = !!(sprom[led_index] & B43_LED_ACTIVELOW);
         }
 }
diff --git a/drivers/net/wireless/broadcom/b43/main.c b/drivers/net/wireless/broadcom/b43/main.c
index 4ee5c5853f9f..6e5d9095b195 100644
--- a/drivers/net/wireless/broadcom/b43/main.c
+++ b/drivers/net/wireless/broadcom/b43/main.c
@@ -3180,7 +3180,6 @@ static void b43_rate_memory_write(struct b43_wldev *dev, u16 rate, int is_ofdm)
 static void b43_rate_memory_init(struct b43_wldev *dev)
 {
         switch (dev->phy.type) {
-        case B43_PHYTYPE_A:
         case B43_PHYTYPE_G:
         case B43_PHYTYPE_N:
         case B43_PHYTYPE_LP:
@@ -3194,8 +3193,6 @@ static void b43_rate_memory_init(struct b43_wldev *dev)
                 b43_rate_memory_write(dev, B43_OFDM_RATE_36MB, 1);
                 b43_rate_memory_write(dev, B43_OFDM_RATE_48MB, 1);
                 b43_rate_memory_write(dev, B43_OFDM_RATE_54MB, 1);
-                if (dev->phy.type == B43_PHYTYPE_A)
-                        break;
                 /* fallthrough */
         case B43_PHYTYPE_B:
                 b43_rate_memory_write(dev, B43_CCK_RATE_1MB, 0);
@@ -4604,14 +4601,6 @@ static int b43_phy_versioning(struct b43_wldev *dev)
         if (radio_manuf != 0x17F /* Broadcom */)
                 unsupported = 1;
         switch (phy_type) {
-        case B43_PHYTYPE_A:
-                if (radio_id != 0x2060)
-                        unsupported = 1;
-                if (radio_rev != 1)
-                        unsupported = 1;
-                if (radio_manuf != 0x17F)
-                        unsupported = 1;
-                break;
         case B43_PHYTYPE_B:
                 if ((radio_id & 0xFFF0) != 0x2050)
                         unsupported = 1;
@@ -4766,10 +4755,7 @@ static void b43_set_synth_pu_delay(struct b43_wldev *dev, bool idle)
         u16 pu_delay;
 
         /* The time value is in microseconds. */
-        if (dev->phy.type == B43_PHYTYPE_A)
-                pu_delay = 3700;
-        else
-                pu_delay = 1050;
+        pu_delay = 1050;
         if (b43_is_mode(dev->wl, NL80211_IFTYPE_ADHOC) || idle)
                 pu_delay = 500;
         if ((dev->phy.radio_ver == 0x2050) && (dev->phy.radio_rev == 8))
@@ -4784,14 +4770,10 @@ static void b43_set_pretbtt(struct b43_wldev *dev)
         u16 pretbtt;
 
         /* The time value is in microseconds. */
-        if (b43_is_mode(dev->wl, NL80211_IFTYPE_ADHOC)) {
+        if (b43_is_mode(dev->wl, NL80211_IFTYPE_ADHOC))
                 pretbtt = 2;
-        } else {
-                if (dev->phy.type == B43_PHYTYPE_A)
-                        pretbtt = 120;
-                else
-                        pretbtt = 250;
-        }
+        else
+                pretbtt = 250;
         b43_shm_write16(dev, B43_SHM_SHARED, B43_SHM_SH_PRETBTT, pretbtt);
         b43_write16(dev, B43_MMIO_TSF_CFP_PRETBTT, pretbtt);
 }
@@ -5380,10 +5362,6 @@ static void b43_supported_bands(struct b43_wldev *dev, bool *have_2ghz_phy,
 
         /* As a fallback, try to guess using PHY type */
         switch (dev->phy.type) {
-        case B43_PHYTYPE_A:
-                *have_2ghz_phy = false;
-                *have_5ghz_phy = true;
-                return;
         case B43_PHYTYPE_G:
         case B43_PHYTYPE_N:
         case B43_PHYTYPE_LP:
@@ -5455,7 +5433,6 @@ static int b43_wireless_core_attach(struct b43_wldev *dev)
         /* We don't support 5 GHz on some PHYs yet */
         if (have_5ghz_phy) {
                 switch (dev->phy.type) {
-                case B43_PHYTYPE_A:
                 case B43_PHYTYPE_G:
                 case B43_PHYTYPE_LP:
                 case B43_PHYTYPE_HT:
diff --git a/drivers/net/wireless/broadcom/b43/phy_a.c b/drivers/net/wireless/broadcom/b43/phy_a.c
deleted file mode 100644
index 99c036f5ecb7..000000000000
--- a/drivers/net/wireless/broadcom/b43/phy_a.c
+++ /dev/null
@@ -1,595 +0,0 @@
-/*
-
-  Broadcom B43 wireless driver
-  IEEE 802.11a PHY driver
-
-  Copyright (c) 2005 Martin Langer <martin-langer@gmx.de>,
-  Copyright (c) 2005-2007 Stefano Brivio <stefano.brivio@polimi.it>
-  Copyright (c) 2005-2008 Michael Buesch <m@bues.ch>
-  Copyright (c) 2005, 2006 Danny van Dyk <kugelfang@gentoo.org>
-  Copyright (c) 2005, 2006 Andreas Jaggi <andreas.jaggi@waterwave.ch>
-
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published by
-  the Free Software Foundation; either version 2 of the License, or
-  (at your option) any later version.
-
-  This program is distributed in the hope that it will be useful,
-  but WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-  GNU General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program; see the file COPYING.  If not, write to
-  the Free Software Foundation, Inc., 51 Franklin Steet, Fifth Floor,
-  Boston, MA 02110-1301, USA.
-
-*/
-
-#include <linux/slab.h>
-
-#include "b43.h"
-#include "phy_a.h"
-#include "phy_common.h"
-#include "wa.h"
-#include "tables.h"
-#include "main.h"
-
-
-/* Get the freq, as it has to be written to the device. */
-static inline u16 channel2freq_a(u8 channel)
-{
-        B43_WARN_ON(channel > 200);
-
-        return (5000 + 5 * channel);
-}
-
-static inline u16 freq_r3A_value(u16 frequency)
-{
-        u16 value;
-
-        if (frequency < 5091)
-                value = 0x0040;
-        else if (frequency < 5321)
-                value = 0x0000;
-        else if (frequency < 5806)
-                value = 0x0080;
-        else
-                value = 0x0040;
-
-        return value;
-}
-
-#if 0
-/* This function converts a TSSI value to dBm in Q5.2 */
-static s8 b43_aphy_estimate_power_out(struct b43_wldev *dev, s8 tssi)
-{
-        struct b43_phy *phy = &dev->phy;
-        struct b43_phy_a *aphy = phy->a;
-        s8 dbm = 0;
-        s32 tmp;
-
-        tmp = (aphy->tgt_idle_tssi - aphy->cur_idle_tssi + tssi);
-        tmp += 0x80;
-        tmp = clamp_val(tmp, 0x00, 0xFF);
-        dbm = aphy->tssi2dbm[tmp];
-        //TODO: There's a FIXME on the specs
-
-        return dbm;
-}
-#endif
-
-static void b43_radio_set_tx_iq(struct b43_wldev *dev)
-{
-        static const u8 data_high[5] = { 0x00, 0x40, 0x80, 0x90, 0xD0 };
-        static const u8 data_low[5] = { 0x00, 0x01, 0x05, 0x06, 0x0A };
-        u16 tmp = b43_radio_read16(dev, 0x001E);
-        int i, j;
-
-        for (i = 0; i < 5; i++) {
-                for (j = 0; j < 5; j++) {
-                        if (tmp == (data_high[i] << 4 | data_low[j])) {
-                                b43_phy_write(dev, 0x0069,
-                                              (i - j) << 8 | 0x00C0);
-                                return;
-                        }
-                }
-        }
-}
-
-static void aphy_channel_switch(struct b43_wldev *dev, unsigned int channel)
-{
-        u16 freq, r8, tmp;
-
-        freq = channel2freq_a(channel);
-
-        r8 = b43_radio_read16(dev, 0x0008);
-        b43_write16(dev, 0x03F0, freq);
-        b43_radio_write16(dev, 0x0008, r8);
-
-        //TODO: write max channel TX power? to Radio 0x2D
-        tmp = b43_radio_read16(dev, 0x002E);
-        tmp &= 0x0080;
-        //TODO: OR tmp with the Power out estimation for this channel?
-        b43_radio_write16(dev, 0x002E, tmp);
-
-        if (freq >= 4920 && freq <= 5500) {
-                /*
-                 * r8 = (((freq * 15 * 0xE1FC780F) >> 32) / 29) & 0x0F;
-                 *    = (freq * 0.025862069
-                 */
-                r8 = 3 * freq / 116;    /* is equal to r8 = freq * 0.025862 */
-        }
-        b43_radio_write16(dev, 0x0007, (r8 << 4) | r8);
-        b43_radio_write16(dev, 0x0020, (r8 << 4) | r8);
-        b43_radio_write16(dev, 0x0021, (r8 << 4) | r8);
-        b43_radio_maskset(dev, 0x0022, 0x000F, (r8 << 4));
-        b43_radio_write16(dev, 0x002A, (r8 << 4));
-        b43_radio_write16(dev, 0x002B, (r8 << 4));
-        b43_radio_maskset(dev, 0x0008, 0x00F0, (r8 << 4));
-        b43_radio_maskset(dev, 0x0029, 0xFF0F, 0x00B0);
-        b43_radio_write16(dev, 0x0035, 0x00AA);
-        b43_radio_write16(dev, 0x0036, 0x0085);
-        b43_radio_maskset(dev, 0x003A, 0xFF20, freq_r3A_value(freq));
-        b43_radio_mask(dev, 0x003D, 0x00FF);
-        b43_radio_maskset(dev, 0x0081, 0xFF7F, 0x0080);
-        b43_radio_mask(dev, 0x0035, 0xFFEF);
-        b43_radio_maskset(dev, 0x0035, 0xFFEF, 0x0010);
-        b43_radio_set_tx_iq(dev);
-        //TODO: TSSI2dbm workaround
-//FIXME b43_phy_xmitpower(dev);
-}
-
-static void b43_radio_init2060(struct b43_wldev *dev)
-{
-        b43_radio_write16(dev, 0x0004, 0x00C0);
-        b43_radio_write16(dev, 0x0005, 0x0008);
-        b43_radio_write16(dev, 0x0009, 0x0040);
-        b43_radio_write16(dev, 0x0005, 0x00AA);
-        b43_radio_write16(dev, 0x0032, 0x008F);
-        b43_radio_write16(dev, 0x0006, 0x008F);
-        b43_radio_write16(dev, 0x0034, 0x008F);
-        b43_radio_write16(dev, 0x002C, 0x0007);
-        b43_radio_write16(dev, 0x0082, 0x0080);
-        b43_radio_write16(dev, 0x0080, 0x0000);
-        b43_radio_write16(dev, 0x003F, 0x00DA);
-        b43_radio_mask(dev, 0x0005, ~0x0008);
-        b43_radio_mask(dev, 0x0081, ~0x0010);
-        b43_radio_mask(dev, 0x0081, ~0x0020);
-        b43_radio_mask(dev, 0x0081, ~0x0020);
-        msleep(1);              /* delay 400usec */
-
-        b43_radio_maskset(dev, 0x0081, ~0x0020, 0x0010);
-        msleep(1);              /* delay 400usec */
-
-        b43_radio_maskset(dev, 0x0005, ~0x0008, 0x0008);
-        b43_radio_mask(dev, 0x0085, ~0x0010);
-        b43_radio_mask(dev, 0x0005, ~0x0008);
-        b43_radio_mask(dev, 0x0081, ~0x0040);
-        b43_radio_maskset(dev, 0x0081, ~0x0040, 0x0040);
-        b43_radio_write16(dev, 0x0005,
-                          (b43_radio_read16(dev, 0x0081) & ~0x0008) | 0x0008);
-        b43_phy_write(dev, 0x0063, 0xDDC6);
-        b43_phy_write(dev, 0x0069, 0x07BE);
-        b43_phy_write(dev, 0x006A, 0x0000);
-
-        aphy_channel_switch(dev, dev->phy.ops->get_default_chan(dev));
-
-        msleep(1);
-}
-
-static void b43_phy_rssiagc(struct b43_wldev *dev, u8 enable)
-{
-        int i;
-
-        if (dev->phy.rev < 3) {
-                if (enable)
-                        for (i = 0; i < B43_TAB_RSSIAGC1_SIZE; i++) {
-                                b43_ofdmtab_write16(dev,
-                                        B43_OFDMTAB_LNAHPFGAIN1, i, 0xFFF8);
-                                b43_ofdmtab_write16(dev,
-                                        B43_OFDMTAB_WRSSI, i, 0xFFF8);
-                        }
-                else
-                        for (i = 0; i < B43_TAB_RSSIAGC1_SIZE; i++) {
-                                b43_ofdmtab_write16(dev,
-                                        B43_OFDMTAB_LNAHPFGAIN1, i, b43_tab_rssiagc1[i]);
-                                b43_ofdmtab_write16(dev,
-                                        B43_OFDMTAB_WRSSI, i, b43_tab_rssiagc1[i]);
-                        }
-        } else {
-                if (enable)
-                        for (i = 0; i < B43_TAB_RSSIAGC1_SIZE; i++)
-                                b43_ofdmtab_write16(dev,
-                                        B43_OFDMTAB_WRSSI, i, 0x0820);
-                else
-                        for (i = 0; i < B43_TAB_RSSIAGC2_SIZE; i++)
-                                b43_ofdmtab_write16(dev,
-                                        B43_OFDMTAB_WRSSI, i, b43_tab_rssiagc2[i]);
-        }
-}
-
-static void b43_phy_ww(struct b43_wldev *dev)
-{
-        u16 b, curr_s, best_s = 0xFFFF;
-        int i;
-
-        b43_phy_mask(dev, B43_PHY_CRS0, ~B43_PHY_CRS0_EN);
-        b43_phy_set(dev, B43_PHY_OFDM(0x1B), 0x1000);
-        b43_phy_maskset(dev, B43_PHY_OFDM(0x82), 0xF0FF, 0x0300);
-        b43_radio_set(dev, 0x0009, 0x0080);
-        b43_radio_maskset(dev, 0x0012, 0xFFFC, 0x0002);
-        b43_wa_initgains(dev);
-        b43_phy_write(dev, B43_PHY_OFDM(0xBA), 0x3ED5);
-        b = b43_phy_read(dev, B43_PHY_PWRDOWN);
-        b43_phy_write(dev, B43_PHY_PWRDOWN, (b & 0xFFF8) | 0x0005);
-        b43_radio_set(dev, 0x0004, 0x0004);
-        for (i = 0x10; i <= 0x20; i++) {
-                b43_radio_write16(dev, 0x0013, i);
-                curr_s = b43_phy_read(dev, B43_PHY_OTABLEQ) & 0x00FF;
-                if (!curr_s) {
-                        best_s = 0x0000;
-                        break;
-                } else if (curr_s >= 0x0080)
-                        curr_s = 0x0100 - curr_s;
-                if (curr_s < best_s)
-                        best_s = curr_s;
-        }
-        b43_phy_write(dev, B43_PHY_PWRDOWN, b);
-        b43_radio_mask(dev, 0x0004, 0xFFFB);
-        b43_radio_write16(dev, 0x0013, best_s);
-        b43_ofdmtab_write16(dev, B43_OFDMTAB_AGC1_R1, 0, 0xFFEC);
-        b43_phy_write(dev, B43_PHY_OFDM(0xB7), 0x1E80);
-        b43_phy_write(dev, B43_PHY_OFDM(0xB6), 0x1C00);
-        b43_phy_write(dev, B43_PHY_OFDM(0xB5), 0x0EC0);
-        b43_phy_write(dev, B43_PHY_OFDM(0xB2), 0x00C0);
-        b43_phy_write(dev, B43_PHY_OFDM(0xB9), 0x1FFF);
-        b43_phy_maskset(dev, B43_PHY_OFDM(0xBB), 0xF000, 0x0053);
-        b43_phy_maskset(dev, B43_PHY_OFDM61, 0xFE1F, 0x0120);
-        b43_phy_maskset(dev, B43_PHY_OFDM(0x13), 0x0FFF, 0x3000);
-        b43_phy_maskset(dev, B43_PHY_OFDM(0x14), 0x0FFF, 0x3000);
-        b43_ofdmtab_write16(dev, B43_OFDMTAB_AGC1, 6, 0x0017);
-        for (i = 0; i < 6; i++)
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_AGC1, i, 0x000F);
-        b43_ofdmtab_write16(dev, B43_OFDMTAB_AGC1, 0x0D, 0x000E);
-        b43_ofdmtab_write16(dev, B43_OFDMTAB_AGC1, 0x0E, 0x0011);
-        b43_ofdmtab_write16(dev, B43_OFDMTAB_AGC1, 0x0F, 0x0013);
-        b43_phy_write(dev, B43_PHY_OFDM(0x33), 0x5030);
-        b43_phy_set(dev, B43_PHY_CRS0, B43_PHY_CRS0_EN);
-}
-
-static void hardware_pctl_init_aphy(struct b43_wldev *dev)
-{
-        //TODO
-}
-
-void b43_phy_inita(struct b43_wldev *dev)
-{
-        struct b43_phy *phy = &dev->phy;
-
-        /* This lowlevel A-PHY init is also called from G-PHY init.
-         * So we must not access phy->a, if called from G-PHY code.
-         */
-        B43_WARN_ON((phy->type != B43_PHYTYPE_A) &&
-                    (phy->type != B43_PHYTYPE_G));
-
-        might_sleep();
-
-        if (phy->rev >= 6) {
-                if (phy->type == B43_PHYTYPE_A)
-                        b43_phy_mask(dev, B43_PHY_OFDM(0x1B), ~0x1000);
-                if (b43_phy_read(dev, B43_PHY_ENCORE) & B43_PHY_ENCORE_EN)
-                        b43_phy_set(dev, B43_PHY_ENCORE, 0x0010);
-                else
-                        b43_phy_mask(dev, B43_PHY_ENCORE, ~0x1010);
-        }
-
-        b43_wa_all(dev);
-
-        if (phy->type == B43_PHYTYPE_A) {
-                if (phy->gmode && (phy->rev < 3))
-                        b43_phy_set(dev, 0x0034, 0x0001);
-                b43_phy_rssiagc(dev, 0);
-
-                b43_phy_set(dev, B43_PHY_CRS0, B43_PHY_CRS0_EN);
-
-                b43_radio_init2060(dev);
-
-                if ((dev->dev->board_vendor == SSB_BOARDVENDOR_BCM) &&
-                    ((dev->dev->board_type == SSB_BOARD_BU4306) ||
-                     (dev->dev->board_type == SSB_BOARD_BU4309))) {
-                        ; //TODO: A PHY LO
-                }
-
-                if (phy->rev >= 3)
-                        b43_phy_ww(dev);
-
-                hardware_pctl_init_aphy(dev);
-
-                //TODO: radar detection
-        }
-
-        if ((phy->type == B43_PHYTYPE_G) &&
-            (dev->dev->bus_sprom->boardflags_lo & B43_BFL_PACTRL)) {
-                b43_phy_maskset(dev, B43_PHY_OFDM(0x6E), 0xE000, 0x3CF);
-        }
-}
-
-/* Initialise the TSSI->dBm lookup table */
-static int b43_aphy_init_tssi2dbm_table(struct b43_wldev *dev)
-{
-        struct b43_phy *phy = &dev->phy;
-        struct b43_phy_a *aphy = phy->a;
-        s16 pab0, pab1, pab2;
-
-        pab0 = (s16) (dev->dev->bus_sprom->pa1b0);
-        pab1 = (s16) (dev->dev->bus_sprom->pa1b1);
-        pab2 = (s16) (dev->dev->bus_sprom->pa1b2);
-
-        if (pab0 != 0 && pab1 != 0 && pab2 != 0 &&
-            pab0 != -1 && pab1 != -1 && pab2 != -1) {
-                /* The pabX values are set in SPROM. Use them. */
-                if ((s8) dev->dev->bus_sprom->itssi_a != 0 &&
-                    (s8) dev->dev->bus_sprom->itssi_a != -1)
-                        aphy->tgt_idle_tssi =
-                            (s8) (dev->dev->bus_sprom->itssi_a);
-                else
-                        aphy->tgt_idle_tssi = 62;
-                aphy->tssi2dbm = b43_generate_dyn_tssi2dbm_tab(dev, pab0,
-                                                               pab1, pab2);
-                if (!aphy->tssi2dbm)
-                        return -ENOMEM;
-        } else {
-                /* pabX values not set in SPROM,
-                 * but APHY needs a generated table. */
-                aphy->tssi2dbm = NULL;
-                b43err(dev->wl, "Could not generate tssi2dBm "
-                       "table (wrong SPROM info)!\n");
-                return -ENODEV;
-        }
-
-        return 0;
-}
-
-static int b43_aphy_op_allocate(struct b43_wldev *dev)
-{
-        struct b43_phy_a *aphy;
-        int err;
-
-        aphy = kzalloc(sizeof(*aphy), GFP_KERNEL);
-        if (!aphy)
-                return -ENOMEM;
-        dev->phy.a = aphy;
-
-        err = b43_aphy_init_tssi2dbm_table(dev);
-        if (err)
-                goto err_free_aphy;
-
-        return 0;
-
-err_free_aphy:
-        kfree(aphy);
-        dev->phy.a = NULL;
-
-        return err;
-}
-
-static void b43_aphy_op_prepare_structs(struct b43_wldev *dev)
-{
-        struct b43_phy *phy = &dev->phy;
-        struct b43_phy_a *aphy = phy->a;
-        const void *tssi2dbm;
-        int tgt_idle_tssi;
-
-        /* tssi2dbm table is constant, so it is initialized at alloc time.
-         * Save a copy of the pointer. */
-        tssi2dbm = aphy->tssi2dbm;
-        tgt_idle_tssi = aphy->tgt_idle_tssi;
-
-        /* Zero out the whole PHY structure. */
-        memset(aphy, 0, sizeof(*aphy));
-
-        aphy->tssi2dbm = tssi2dbm;
-        aphy->tgt_idle_tssi = tgt_idle_tssi;
-
-        //TODO init struct b43_phy_a
-
-}
-
-static void b43_aphy_op_free(struct b43_wldev *dev)
-{
-        struct b43_phy *phy = &dev->phy;
-        struct b43_phy_a *aphy = phy->a;
-
-        kfree(aphy->tssi2dbm);
-        aphy->tssi2dbm = NULL;
-
-        kfree(aphy);
-        dev->phy.a = NULL;
-}
-
-static int b43_aphy_op_init(struct b43_wldev *dev)
-{
-        b43_phy_inita(dev);
-
-        return 0;
-}
-
-static inline u16 adjust_phyreg(struct b43_wldev *dev, u16 offset)
-{
-        /* OFDM registers are base-registers for the A-PHY. */
-        if ((offset & B43_PHYROUTE) == B43_PHYROUTE_OFDM_GPHY) {
-                offset &= ~B43_PHYROUTE;
-                offset |= B43_PHYROUTE_BASE;
-        }
-
-#if B43_DEBUG
-        if ((offset & B43_PHYROUTE) == B43_PHYROUTE_EXT_GPHY) {
-                /* Ext-G registers are only available on G-PHYs */
-                b43err(dev->wl, "Invalid EXT-G PHY access at "
-                       "0x%04X on A-PHY\n", offset);
-                dump_stack();
-        }
-        if ((offset & B43_PHYROUTE) == B43_PHYROUTE_N_BMODE) {
-                /* N-BMODE registers are only available on N-PHYs */
-                b43err(dev->wl, "Invalid N-BMODE PHY access at "
-                       "0x%04X on A-PHY\n", offset);
-                dump_stack();
-        }
-#endif /* B43_DEBUG */
-
-        return offset;
-}
-
-static u16 b43_aphy_op_read(struct b43_wldev *dev, u16 reg)
-{
-        reg = adjust_phyreg(dev, reg);
-        b43_write16f(dev, B43_MMIO_PHY_CONTROL, reg);
-        return b43_read16(dev, B43_MMIO_PHY_DATA);
-}
-
-static void b43_aphy_op_write(struct b43_wldev *dev, u16 reg, u16 value)
-{
-        reg = adjust_phyreg(dev, reg);
-        b43_write16f(dev, B43_MMIO_PHY_CONTROL, reg);
-        b43_write16(dev, B43_MMIO_PHY_DATA, value);
-}
-
-static u16 b43_aphy_op_radio_read(struct b43_wldev *dev, u16 reg)
-{
-        /* Register 1 is a 32-bit register. */
-        B43_WARN_ON(reg == 1);
-        /* A-PHY needs 0x40 for read access */
-        reg |= 0x40;
-
-        b43_write16(dev, B43_MMIO_RADIO_CONTROL, reg);
-        return b43_read16(dev, B43_MMIO_RADIO_DATA_LOW);
-}
-
-static void b43_aphy_op_radio_write(struct b43_wldev *dev, u16 reg, u16 value)
-{
-        /* Register 1 is a 32-bit register. */
-        B43_WARN_ON(reg == 1);
-
-        b43_write16(dev, B43_MMIO_RADIO_CONTROL, reg);
-        b43_write16(dev, B43_MMIO_RADIO_DATA_LOW, value);
-}
-
-static bool b43_aphy_op_supports_hwpctl(struct b43_wldev *dev)
-{
-        return (dev->phy.rev >= 5);
-}
-
-static void b43_aphy_op_software_rfkill(struct b43_wldev *dev,
-                                        bool blocked)
-{
-        struct b43_phy *phy = &dev->phy;
-
-        if (!blocked) {
-                if (phy->radio_on)
-                        return;
-                b43_radio_write16(dev, 0x0004, 0x00C0);
-                b43_radio_write16(dev, 0x0005, 0x0008);
-                b43_phy_mask(dev, 0x0010, 0xFFF7);
-                b43_phy_mask(dev, 0x0011, 0xFFF7);
-                b43_radio_init2060(dev);
-        } else {
-                b43_radio_write16(dev, 0x0004, 0x00FF);
-                b43_radio_write16(dev, 0x0005, 0x00FB);
-                b43_phy_set(dev, 0x0010, 0x0008);
-                b43_phy_set(dev, 0x0011, 0x0008);
-        }
-}
-
-static int b43_aphy_op_switch_channel(struct b43_wldev *dev,
-                                      unsigned int new_channel)
-{
-        if (new_channel > 200)
-                return -EINVAL;
-        aphy_channel_switch(dev, new_channel);
-
-        return 0;
-}
-
-static unsigned int b43_aphy_op_get_default_chan(struct b43_wldev *dev)
-{
-        return 36; /* Default to channel 36 */
-}
-
-static void b43_aphy_op_set_rx_antenna(struct b43_wldev *dev, int antenna)
-{//TODO
-        struct b43_phy *phy = &dev->phy;
-        u16 tmp;
-        int autodiv = 0;
-
-        if (antenna == B43_ANTENNA_AUTO0 || antenna == B43_ANTENNA_AUTO1)
-                autodiv = 1;
-
-        b43_hf_write(dev, b43_hf_read(dev) & ~B43_HF_ANTDIVHELP);
-
-        b43_phy_maskset(dev, B43_PHY_BBANDCFG, ~B43_PHY_BBANDCFG_RXANT,
-                        (autodiv ? B43_ANTENNA_AUTO1 : antenna) <<
-                        B43_PHY_BBANDCFG_RXANT_SHIFT);
-
-        if (autodiv) {
-                tmp = b43_phy_read(dev, B43_PHY_ANTDWELL);
-                if (antenna == B43_ANTENNA_AUTO1)
-                        tmp &= ~B43_PHY_ANTDWELL_AUTODIV1;
-                else
-                        tmp |= B43_PHY_ANTDWELL_AUTODIV1;
-                b43_phy_write(dev, B43_PHY_ANTDWELL, tmp);
-        }
-        if (phy->rev < 3)
-                b43_phy_maskset(dev, B43_PHY_ANTDWELL, 0xFF00, 0x24);
-        else {
-                b43_phy_set(dev, B43_PHY_OFDM61, 0x10);
-                if (phy->rev == 3) {
-                        b43_phy_write(dev, B43_PHY_CLIPPWRDOWNT, 0x1D);
-                        b43_phy_write(dev, B43_PHY_ADIVRELATED, 8);
-                } else {
-                        b43_phy_write(dev, B43_PHY_CLIPPWRDOWNT, 0x3A);
-                        b43_phy_maskset(dev, B43_PHY_ADIVRELATED, 0xFF00, 8);
-                }
-        }
-
-        b43_hf_write(dev, b43_hf_read(dev) | B43_HF_ANTDIVHELP);
-}
-
-static void b43_aphy_op_adjust_txpower(struct b43_wldev *dev)
-{//TODO
-}
-
-static enum b43_txpwr_result b43_aphy_op_recalc_txpower(struct b43_wldev *dev,
-                                                        bool ignore_tssi)
-{//TODO
-        return B43_TXPWR_RES_DONE;
-}
-
-static void b43_aphy_op_pwork_15sec(struct b43_wldev *dev)
-{//TODO
-}
-
-static void b43_aphy_op_pwork_60sec(struct b43_wldev *dev)
-{//TODO
-}
-
-static const struct b43_phy_operations b43_phyops_a = {
-        .allocate               = b43_aphy_op_allocate,
-        .free                   = b43_aphy_op_free,
-        .prepare_structs        = b43_aphy_op_prepare_structs,
-        .init                   = b43_aphy_op_init,
-        .phy_read               = b43_aphy_op_read,
-        .phy_write              = b43_aphy_op_write,
-        .radio_read             = b43_aphy_op_radio_read,
-        .radio_write            = b43_aphy_op_radio_write,
-        .supports_hwpctl        = b43_aphy_op_supports_hwpctl,
-        .software_rfkill        = b43_aphy_op_software_rfkill,
-        .switch_analog          = b43_phyop_switch_analog_generic,
-        .switch_channel         = b43_aphy_op_switch_channel,
-        .get_default_chan       = b43_aphy_op_get_default_chan,
-        .set_rx_antenna         = b43_aphy_op_set_rx_antenna,
-        .recalc_txpower         = b43_aphy_op_recalc_txpower,
-        .adjust_txpower         = b43_aphy_op_adjust_txpower,
-        .pwork_15sec            = b43_aphy_op_pwork_15sec,
-        .pwork_60sec            = b43_aphy_op_pwork_60sec,
-};
diff --git a/drivers/net/wireless/broadcom/b43/phy_a.h b/drivers/net/wireless/broadcom/b43/phy_a.h
index f7d0d929a374..0a92d01c21f9 100644
--- a/drivers/net/wireless/broadcom/b43/phy_a.h
+++ b/drivers/net/wireless/broadcom/b43/phy_a.h
@@ -101,26 +101,4 @@ u32 b43_ofdmtab_read32(struct b43_wldev *dev, u16 table, u16 offset);
 void b43_ofdmtab_write32(struct b43_wldev *dev, u16 table,
                          u16 offset, u32 value);
 
-
-struct b43_phy_a {
-        /* Pointer to the table used to convert a
-         * TSSI value to dBm-Q5.2 */
-        const s8 *tssi2dbm;
-        /* Target idle TSSI */
-        int tgt_idle_tssi;
-        /* Current idle TSSI */
-        int cur_idle_tssi;//FIXME value currently not set
-
-        /* A-PHY TX Power control value. */
-        u16 txpwr_offset;
-
-        //TODO lots of missing stuff
-};
-
-/**
- * b43_phy_inita - Lowlevel A-PHY init routine.
- * This is _only_ used by the G-PHY code.
- */
-void b43_phy_inita(struct b43_wldev *dev);
-
 #endif /* LINUX_B43_PHY_A_H_ */
diff --git a/drivers/net/wireless/broadcom/b43/phy_common.h b/drivers/net/wireless/broadcom/b43/phy_common.h
index 78d86526799e..ced054a9850c 100644
--- a/drivers/net/wireless/broadcom/b43/phy_common.h
+++ b/drivers/net/wireless/broadcom/b43/phy_common.h
@@ -190,7 +190,6 @@ struct b43_phy_operations {
         void (*pwork_60sec)(struct b43_wldev *dev);
 };
 
-struct b43_phy_a;
 struct b43_phy_g;
 struct b43_phy_n;
 struct b43_phy_lp;
@@ -210,8 +209,6 @@ struct b43_phy {
 #else
         union {
 #endif
-                /* A-PHY specific information */
-                struct b43_phy_a *a;
                 /* G-PHY specific information */
                 struct b43_phy_g *g;
                 /* N-PHY specific information */
diff --git a/drivers/net/wireless/broadcom/b43/phy_g.c b/drivers/net/wireless/broadcom/b43/phy_g.c
index 462310e6e88f..822dcaa8ace6 100644
--- a/drivers/net/wireless/broadcom/b43/phy_g.c
+++ b/drivers/net/wireless/broadcom/b43/phy_g.c
@@ -31,6 +31,7 @@
 #include "phy_common.h"
 #include "lo.h"
 #include "main.h"
+#include "wa.h"
 
 #include <linux/bitrev.h>
 #include <linux/slab.h>
@@ -1987,6 +1988,25 @@ static void b43_phy_init_pctl(struct b43_wldev *dev)
         b43_shm_clear_tssi(dev);
 }
 
+static void b43_phy_inita(struct b43_wldev *dev)
+{
+        struct b43_phy *phy = &dev->phy;
+
+        might_sleep();
+
+        if (phy->rev >= 6) {
+                if (b43_phy_read(dev, B43_PHY_ENCORE) & B43_PHY_ENCORE_EN)
+                        b43_phy_set(dev, B43_PHY_ENCORE, 0x0010);
+                else
+                        b43_phy_mask(dev, B43_PHY_ENCORE, ~0x1010);
+        }
+
+        b43_wa_all(dev);
+
+        if (dev->dev->bus_sprom->boardflags_lo & B43_BFL_PACTRL)
+                b43_phy_maskset(dev, B43_PHY_OFDM(0x6E), 0xE000, 0x3CF);
+}
+
 static void b43_phy_initg(struct b43_wldev *dev)
 {
         struct b43_phy *phy = &dev->phy;
@@ -2150,11 +2170,6 @@ static void default_radio_attenuation(struct b43_wldev *dev,
                 }
         }
 
-        if (phy->type == B43_PHYTYPE_A) {
-                rf->att = 0x60;
-                return;
-        }
-
         switch (phy->radio_ver) {
         case 0x2053:
                 switch (phy->radio_rev) {
diff --git a/drivers/net/wireless/broadcom/b43/wa.c b/drivers/net/wireless/broadcom/b43/wa.c
index c218c08fb2f5..0e96c08d1e17 100644
--- a/drivers/net/wireless/broadcom/b43/wa.c
+++ b/drivers/net/wireless/broadcom/b43/wa.c
@@ -30,33 +30,6 @@
 #include "phy_common.h"
 #include "wa.h"
 
-static void b43_wa_papd(struct b43_wldev *dev)
-{
-        u16 backup;
-
-        backup = b43_ofdmtab_read16(dev, B43_OFDMTAB_PWRDYN2, 0);
-        b43_ofdmtab_write16(dev, B43_OFDMTAB_PWRDYN2, 0, 7);
-        b43_ofdmtab_write16(dev, B43_OFDMTAB_UNKNOWN_APHY, 0, 0);
-        b43_dummy_transmission(dev, true, true);
-        b43_ofdmtab_write16(dev, B43_OFDMTAB_PWRDYN2, 0, backup);
-}
-
-static void b43_wa_auxclipthr(struct b43_wldev *dev)
-{
-        b43_phy_write(dev, B43_PHY_OFDM(0x8E), 0x3800);
-}
-
-static void b43_wa_afcdac(struct b43_wldev *dev)
-{
-        b43_phy_write(dev, 0x0035, 0x03FF);
-        b43_phy_write(dev, 0x0036, 0x0400);
-}
-
-static void b43_wa_txdc_offset(struct b43_wldev *dev)
-{
-        b43_ofdmtab_write16(dev, B43_OFDMTAB_DC, 0, 0x0051);
-}
-
 void b43_wa_initgains(struct b43_wldev *dev)
 {
         struct b43_phy *phy = &dev->phy;
@@ -81,41 +54,6 @@ void b43_wa_initgains(struct b43_wldev *dev)
                 b43_phy_write(dev, 0x00BA, 0x3ED5);
 }
 
-static void b43_wa_divider(struct b43_wldev *dev)
-{
-        b43_phy_mask(dev, 0x002B, ~0x0100);
-        b43_phy_write(dev, 0x008E, 0x58C1);
-}
-
-static void b43_wa_gt(struct b43_wldev *dev) /* Gain table. */
-{
-        if (dev->phy.rev <= 2) {
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_GAIN2, 0, 15);
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_GAIN2, 1, 31);
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_GAIN2, 2, 42);
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_GAIN2, 3, 48);
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_GAIN2, 4, 58);
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_GAIN0, 0, 19);
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_GAIN0, 1, 19);
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_GAIN0, 2, 19);
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_GAIN0, 3, 19);
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_GAIN0, 4, 21);
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_GAIN0, 5, 21);
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_GAIN0, 6, 25);
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_GAIN1, 0, 3);
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_GAIN1, 1, 3);
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_GAIN1, 2, 7);
-        } else {
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_GAIN0, 0, 19);
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_GAIN0, 1, 19);
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_GAIN0, 2, 19);
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_GAIN0, 3, 19);
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_GAIN0, 4, 21);
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_GAIN0, 5, 21);
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_GAIN0, 6, 25);
-        }
-}
-
 static void b43_wa_rssi_lt(struct b43_wldev *dev) /* RSSI lookup table */
 {
         int i;
@@ -133,15 +71,11 @@ static void b43_wa_rssi_lt(struct b43_wldev *dev) /* RSSI lookup table */
 
 static void b43_wa_analog(struct b43_wldev *dev)
 {
-        struct b43_phy *phy = &dev->phy;
         u16 ofdmrev;
 
         ofdmrev = b43_phy_read(dev, B43_PHY_VERSION_OFDM) & B43_PHYVER_VERSION;
         if (ofdmrev > 2) {
-                if (phy->type == B43_PHYTYPE_A)
-                        b43_phy_write(dev, B43_PHY_PWRDOWN, 0x1808);
-                else
-                        b43_phy_write(dev, B43_PHY_PWRDOWN, 0x1000);
+                b43_phy_write(dev, B43_PHY_PWRDOWN, 0x1000);
         } else {
                 b43_ofdmtab_write16(dev, B43_OFDMTAB_DAC, 3, 0x1044);
                 b43_ofdmtab_write16(dev, B43_OFDMTAB_DAC, 4, 0x7201);
@@ -149,26 +83,13 @@ static void b43_wa_analog(struct b43_wldev *dev)
         }
 }
 
-static void b43_wa_dac(struct b43_wldev *dev)
-{
-        if (dev->phy.analog == 1)
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_DAC, 1,
-                        (b43_ofdmtab_read16(dev, B43_OFDMTAB_DAC, 1) & ~0x0034) | 0x0008);
-        else
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_DAC, 1,
-                        (b43_ofdmtab_read16(dev, B43_OFDMTAB_DAC, 1) & ~0x0078) | 0x0010);
-}
-
 static void b43_wa_fft(struct b43_wldev *dev) /* Fine frequency table */
 {
         int i;
 
-        if (dev->phy.type == B43_PHYTYPE_A)
-                for (i = 0; i < B43_TAB_FINEFREQA_SIZE; i++)
-                        b43_ofdmtab_write16(dev, B43_OFDMTAB_DACRFPABB, i, b43_tab_finefreqa[i]);
-        else
-                for (i = 0; i < B43_TAB_FINEFREQG_SIZE; i++)
-                        b43_ofdmtab_write16(dev, B43_OFDMTAB_DACRFPABB, i, b43_tab_finefreqg[i]);
+        for (i = 0; i < B43_TAB_FINEFREQG_SIZE; i++)
+                b43_ofdmtab_write16(dev, B43_OFDMTAB_DACRFPABB, i,
+                                    b43_tab_finefreqg[i]);
 }
 
 static void b43_wa_nft(struct b43_wldev *dev) /* Noise figure table */
@@ -176,21 +97,14 @@ static void b43_wa_nft(struct b43_wldev *dev) /* Noise figure table */
         struct b43_phy *phy = &dev->phy;
         int i;
 
-        if (phy->type == B43_PHYTYPE_A) {
-                if (phy->rev == 2)
-                        for (i = 0; i < B43_TAB_NOISEA2_SIZE; i++)
-                                b43_ofdmtab_write16(dev, B43_OFDMTAB_AGC2, i, b43_tab_noisea2[i]);
-                else
-                        for (i = 0; i < B43_TAB_NOISEA3_SIZE; i++)
-                                b43_ofdmtab_write16(dev, B43_OFDMTAB_AGC2, i, b43_tab_noisea3[i]);
-        } else {
-                if (phy->rev == 1)
-                        for (i = 0; i < B43_TAB_NOISEG1_SIZE; i++)
-                                b43_ofdmtab_write16(dev, B43_OFDMTAB_AGC2, i, b43_tab_noiseg1[i]);
-                else
-                        for (i = 0; i < B43_TAB_NOISEG2_SIZE; i++)
-                                b43_ofdmtab_write16(dev, B43_OFDMTAB_AGC2, i, b43_tab_noiseg2[i]);
-        }
+        if (phy->rev == 1)
+                for (i = 0; i < B43_TAB_NOISEG1_SIZE; i++)
+                        b43_ofdmtab_write16(dev, B43_OFDMTAB_AGC2, i,
+                                            b43_tab_noiseg1[i]);
+        else
+                for (i = 0; i < B43_TAB_NOISEG2_SIZE; i++)
+                        b43_ofdmtab_write16(dev, B43_OFDMTAB_AGC2, i,
+                                            b43_tab_noiseg2[i]);
 }
 
 static void b43_wa_rt(struct b43_wldev *dev) /* Rotor table */
@@ -201,14 +115,6 @@ static void b43_wa_rt(struct b43_wldev *dev) /* Rotor table */
                 b43_ofdmtab_write32(dev, B43_OFDMTAB_ROTOR, i, b43_tab_rotor[i]);
 }
 
-static void b43_write_null_nst(struct b43_wldev *dev)
-{
-        int i;
-
-        for (i = 0; i < B43_TAB_NOISESCALE_SIZE; i++)
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_NOISESCALE, i, 0);
-}
-
 static void b43_write_nst(struct b43_wldev *dev, const u16 *nst)
 {
         int i;
@@ -221,24 +127,13 @@ static void b43_wa_nst(struct b43_wldev *dev) /* Noise scale table */
 {
         struct b43_phy *phy = &dev->phy;
 
-        if (phy->type == B43_PHYTYPE_A) {
-                if (phy->rev <= 1)
-                        b43_write_null_nst(dev);
-                else if (phy->rev == 2)
-                        b43_write_nst(dev, b43_tab_noisescalea2);
-                else if (phy->rev == 3)
-                        b43_write_nst(dev, b43_tab_noisescalea3);
-                else
+        if (phy->rev >= 6) {
+                if (b43_phy_read(dev, B43_PHY_ENCORE) & B43_PHY_ENCORE_EN)
                         b43_write_nst(dev, b43_tab_noisescaleg3);
+                else
+                        b43_write_nst(dev, b43_tab_noisescaleg2);
         } else {
-                if (phy->rev >= 6) {
-                        if (b43_phy_read(dev, B43_PHY_ENCORE) & B43_PHY_ENCORE_EN)
-                                b43_write_nst(dev, b43_tab_noisescaleg3);
-                        else
-                                b43_write_nst(dev, b43_tab_noisescaleg2);
-                } else {
-                        b43_write_nst(dev, b43_tab_noisescaleg1);
-                }
+                b43_write_nst(dev, b43_tab_noisescaleg1);
         }
 }
 
@@ -251,41 +146,13 @@ static void b43_wa_art(struct b43_wldev *dev) /* ADV retard table */
                                 i, b43_tab_retard[i]);
 }
 
-static void b43_wa_txlna_gain(struct b43_wldev *dev)
-{
-        b43_ofdmtab_write16(dev, B43_OFDMTAB_DC, 13, 0x0000);
-}
-
-static void b43_wa_crs_reset(struct b43_wldev *dev)
-{
-        b43_phy_write(dev, 0x002C, 0x0064);
-}
-
-static void b43_wa_2060txlna_gain(struct b43_wldev *dev)
-{
-        b43_hf_write(dev, b43_hf_read(dev) |
-                         B43_HF_2060W);
-}
-
-static void b43_wa_lms(struct b43_wldev *dev)
-{
-        b43_phy_maskset(dev, 0x0055, 0xFFC0, 0x0004);
-}
-
-static void b43_wa_mixedsignal(struct b43_wldev *dev)
-{
-        b43_ofdmtab_write16(dev, B43_OFDMTAB_DAC, 1, 3);
-}
-
 static void b43_wa_msst(struct b43_wldev *dev) /* Min sigma square table */
 {
         struct b43_phy *phy = &dev->phy;
         int i;
         const u16 *tab;
 
-        if (phy->type == B43_PHYTYPE_A) {
-                tab = b43_tab_sigmasqr1;
-        } else if (phy->type == B43_PHYTYPE_G) {
+        if (phy->type == B43_PHYTYPE_G) {
                 tab = b43_tab_sigmasqr2;
         } else {
                 B43_WARN_ON(1);
@@ -298,13 +165,6 @@ static void b43_wa_msst(struct b43_wldev *dev) /* Min sigma square table */
         }
 }
 
-static void b43_wa_iqadc(struct b43_wldev *dev)
-{
-        if (dev->phy.analog == 4)
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_DAC, 0,
-                        b43_ofdmtab_read16(dev, B43_OFDMTAB_DAC, 0) & ~0xF000);
-}
-
 static void b43_wa_crs_ed(struct b43_wldev *dev)
 {
         struct b43_phy *phy = &dev->phy;
@@ -450,38 +310,6 @@ static void b43_wa_cpll_nonpilot(struct b43_wldev *dev)
         b43_ofdmtab_write16(dev, B43_OFDMTAB_UNKNOWN_11, 1, 0);
 }
 
-static void b43_wa_rssi_adc(struct b43_wldev *dev)
-{
-        if (dev->phy.analog == 4)
-                b43_phy_write(dev, 0x00DC, 0x7454);
-}
-
-static void b43_wa_boards_a(struct b43_wldev *dev)
-{
-        if (dev->dev->board_vendor == SSB_BOARDVENDOR_BCM &&
-            dev->dev->board_type == SSB_BOARD_BU4306 &&
-            dev->dev->board_rev < 0x30) {
-                b43_phy_write(dev, 0x0010, 0xE000);
-                b43_phy_write(dev, 0x0013, 0x0140);
-                b43_phy_write(dev, 0x0014, 0x0280);
-        } else {
-                if (dev->dev->board_type == SSB_BOARD_MP4318 &&
-                    dev->dev->board_rev < 0x20) {
-                        b43_phy_write(dev, 0x0013, 0x0210);
-                        b43_phy_write(dev, 0x0014, 0x0840);
-                } else {
-                        b43_phy_write(dev, 0x0013, 0x0140);
-                        b43_phy_write(dev, 0x0014, 0x0280);
-                }
-                if (dev->phy.rev <= 4)
-                        b43_phy_write(dev, 0x0010, 0xE000);
-                else
-                        b43_phy_write(dev, 0x0010, 0x2000);
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_DC, 1, 0x0039);
-                b43_ofdmtab_write16(dev, B43_OFDMTAB_UNKNOWN_APHY, 7, 0x0040);
-        }
-}
-
 static void b43_wa_boards_g(struct b43_wldev *dev)
 {
         struct ssb_sprom *sprom = dev->dev->bus_sprom;
@@ -518,80 +346,7 @@ void b43_wa_all(struct b43_wldev *dev)
 {
         struct b43_phy *phy = &dev->phy;
 
-        if (phy->type == B43_PHYTYPE_A) {
-                switch (phy->rev) {
-                case 2:
-                        b43_wa_papd(dev);
-                        b43_wa_auxclipthr(dev);
-                        b43_wa_afcdac(dev);
-                        b43_wa_txdc_offset(dev);
-                        b43_wa_initgains(dev);
-                        b43_wa_divider(dev);
-                        b43_wa_gt(dev);
-                        b43_wa_rssi_lt(dev);
-                        b43_wa_analog(dev);
-                        b43_wa_dac(dev);
-                        b43_wa_fft(dev);
-                        b43_wa_nft(dev);
-                        b43_wa_rt(dev);
-                        b43_wa_nst(dev);
-                        b43_wa_art(dev);
-                        b43_wa_txlna_gain(dev);
-                        b43_wa_crs_reset(dev);
-                        b43_wa_2060txlna_gain(dev);
-                        b43_wa_lms(dev);
-                        break;
-                case 3:
-                        b43_wa_papd(dev);
-                        b43_wa_mixedsignal(dev);
-                        b43_wa_rssi_lt(dev);
-                        b43_wa_txdc_offset(dev);
-                        b43_wa_initgains(dev);
-                        b43_wa_dac(dev);
-                        b43_wa_nft(dev);
-                        b43_wa_nst(dev);
-                        b43_wa_msst(dev);
-                        b43_wa_analog(dev);
-                        b43_wa_gt(dev);
-                        b43_wa_txpuoff_rxpuon(dev);
-                        b43_wa_txlna_gain(dev);
-                        break;
-                case 5:
-                        b43_wa_iqadc(dev);
-                case 6:
-                        b43_wa_papd(dev);
-                        b43_wa_rssi_lt(dev);
-                        b43_wa_txdc_offset(dev);
-                        b43_wa_initgains(dev);
-                        b43_wa_dac(dev);
-                        b43_wa_nft(dev);
-                        b43_wa_nst(dev);
-                        b43_wa_msst(dev);
-                        b43_wa_analog(dev);
-                        b43_wa_gt(dev);
-                        b43_wa_txpuoff_rxpuon(dev);
-                        b43_wa_txlna_gain(dev);
-                        break;
-                case 7:
-                        b43_wa_iqadc(dev);
-                        b43_wa_papd(dev);
-                        b43_wa_rssi_lt(dev);
-                        b43_wa_txdc_offset(dev);
-                        b43_wa_initgains(dev);
-                        b43_wa_dac(dev);
-                        b43_wa_nft(dev);
-                        b43_wa_nst(dev);
-                        b43_wa_msst(dev);
-                        b43_wa_analog(dev);
-                        b43_wa_gt(dev);
-                        b43_wa_txpuoff_rxpuon(dev);
-                        b43_wa_txlna_gain(dev);
-                        b43_wa_rssi_adc(dev);
-                default:
-                        B43_WARN_ON(1);
-                }
-                b43_wa_boards_a(dev);
-        } else if (phy->type == B43_PHYTYPE_G) {
+        if (phy->type == B43_PHYTYPE_G) {
                 switch (phy->rev) {
                 case 1://XXX review rev1
                         b43_wa_crs_ed(dev);
diff --git a/drivers/net/wireless/broadcom/b43/xmit.c b/drivers/net/wireless/broadcom/b43/xmit.c
index f6201264de49..b068d5aeee24 100644
--- a/drivers/net/wireless/broadcom/b43/xmit.c
+++ b/drivers/net/wireless/broadcom/b43/xmit.c
@@ -205,7 +205,7 @@ static u16 b43_generate_tx_phy_ctl1(struct b43_wldev *dev, u8 bitrate)
         return control;
 }
 
-static u8 b43_calc_fallback_rate(u8 bitrate)
+static u8 b43_calc_fallback_rate(u8 bitrate, int gmode)
 {
         switch (bitrate) {
         case B43_CCK_RATE_1MB:
@@ -216,8 +216,15 @@ static u8 b43_calc_fallback_rate(u8 bitrate)
                 return B43_CCK_RATE_2MB;
         case B43_CCK_RATE_11MB:
                 return B43_CCK_RATE_5MB;
+        /*
+         * Don't just fallback to CCK; it may be in 5GHz operation
+         * and falling back to CCK won't work out very well.
+         */
         case B43_OFDM_RATE_6MB:
-                return B43_CCK_RATE_5MB;
+                if (gmode)
+                        return B43_CCK_RATE_5MB;
+                else
+                        return B43_OFDM_RATE_6MB;
         case B43_OFDM_RATE_9MB:
                 return B43_OFDM_RATE_6MB;
         case B43_OFDM_RATE_12MB:
@@ -438,7 +445,7 @@ int b43_generate_txhdr(struct b43_wldev *dev,
 
                 rts_rate = rts_cts_rate ? rts_cts_rate->hw_value : B43_CCK_RATE_1MB;
                 rts_rate_ofdm = b43_is_ofdm_rate(rts_rate);
-                rts_rate_fb = b43_calc_fallback_rate(rts_rate);
+                rts_rate_fb = b43_calc_fallback_rate(rts_rate, phy->gmode);
                 rts_rate_fb_ofdm = b43_is_ofdm_rate(rts_rate_fb);
 
                 if (rates[0].flags & IEEE80211_TX_RC_USE_CTS_PROTECT) {
@@ -642,11 +649,7 @@ static s8 b43_rssinoise_postprocess(struct b43_wldev *dev, u8 in_rssi)
         struct b43_phy *phy = &dev->phy;
         s8 ret;
 
-        if (phy->type == B43_PHYTYPE_A) {
-                //TODO: Incomplete specs.
-                ret = 0;
-        } else
-                ret = b43_rssi_postprocess(dev, in_rssi, 0, 1, 1);
+        ret = b43_rssi_postprocess(dev, in_rssi, 0, 1, 1);
 
         return ret;
 }
@@ -663,7 +666,6 @@ void b43_rx(struct b43_wldev *dev, struct sk_buff *skb, const void *_rxhdr)
         u16 uninitialized_var(chanstat), uninitialized_var(mactime);
         u32 uninitialized_var(macstat);
         u16 chanid;
-        u16 phytype;
         int padding, rate_idx;
 
         memset(&status, 0, sizeof(status));
@@ -684,7 +686,6 @@ void b43_rx(struct b43_wldev *dev, struct sk_buff *skb, const void *_rxhdr)
                 chanstat = le16_to_cpu(rxhdr->format_351.channel);
                 break;
         }
-        phytype = chanstat & B43_RX_CHAN_PHYTYPE;
 
         if (unlikely(macstat & B43_RX_MAC_FCSERR)) {
                 dev->wl->ieee_stats.dot11FCSErrorCount++;
@@ -755,7 +756,6 @@ void b43_rx(struct b43_wldev *dev, struct sk_buff *skb, const void *_rxhdr)
                 else
                         status.signal = max(rxhdr->power0, rxhdr->power1);
                 break;
-        case B43_PHYTYPE_A:
         case B43_PHYTYPE_B:
         case B43_PHYTYPE_G:
         case B43_PHYTYPE_LP:
@@ -802,14 +802,6 @@ void b43_rx(struct b43_wldev *dev, struct sk_buff *skb, const void *_rxhdr)
 
         chanid = (chanstat & B43_RX_CHAN_ID) >> B43_RX_CHAN_ID_SHIFT;
         switch (chanstat & B43_RX_CHAN_PHYTYPE) {
-        case B43_PHYTYPE_A:
-                status.band = NL80211_BAND_5GHZ;
-                B43_WARN_ON(1);
-                /* FIXME: We don't really know which value the "chanid" contains.
-                 *        So the following assignment might be wrong. */
-                status.freq =
-                        ieee80211_channel_to_frequency(chanid, status.band);
-                break;
         case B43_PHYTYPE_G:
                 status.band = NL80211_BAND_2GHZ;
                 /* Somewhere between 478.104 and 508.1084 firmware for G-PHY
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c
index c7550dab6a23..c4b89d27e2e8 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c
@@ -166,41 +166,45 @@ int brcmf_sdiod_intr_register(struct brcmf_sdio_dev *sdiodev)
                 sdio_claim_irq(sdiodev->func[1], brcmf_sdiod_ib_irqhandler);
                 sdio_claim_irq(sdiodev->func[2], brcmf_sdiod_dummy_irqhandler);
                 sdio_release_host(sdiodev->func[1]);
+                sdiodev->sd_irq_requested = true;
         }
 
         return 0;
 }
 
-int brcmf_sdiod_intr_unregister(struct brcmf_sdio_dev *sdiodev)
+void brcmf_sdiod_intr_unregister(struct brcmf_sdio_dev *sdiodev)
 {
-        struct brcmfmac_sdio_pd *pdata;
 
-        brcmf_dbg(SDIO, "Entering\n");
+        brcmf_dbg(SDIO, "Entering oob=%d sd=%d\n",
+                  sdiodev->oob_irq_requested,
+                  sdiodev->sd_irq_requested);
 
-        pdata = &sdiodev->settings->bus.sdio;
-        if (pdata->oob_irq_supported) {
+        if (sdiodev->oob_irq_requested) {
+                struct brcmfmac_sdio_pd *pdata;
+
+                pdata = &sdiodev->settings->bus.sdio;
                 sdio_claim_host(sdiodev->func[1]);
                 brcmf_sdiod_regwb(sdiodev, SDIO_CCCR_BRCM_SEPINT, 0, NULL);
                 brcmf_sdiod_regwb(sdiodev, SDIO_CCCR_IENx, 0, NULL);
                 sdio_release_host(sdiodev->func[1]);
 
-                if (sdiodev->oob_irq_requested) {
-                        sdiodev->oob_irq_requested = false;
-                        if (sdiodev->irq_wake) {
-                                disable_irq_wake(pdata->oob_irq_nr);
-                                sdiodev->irq_wake = false;
-                        }
-                        free_irq(pdata->oob_irq_nr, &sdiodev->func[1]->dev);
-                        sdiodev->irq_en = false;
+                sdiodev->oob_irq_requested = false;
+                if (sdiodev->irq_wake) {
+                        disable_irq_wake(pdata->oob_irq_nr);
+                        sdiodev->irq_wake = false;
                 }
-        } else {
+                free_irq(pdata->oob_irq_nr, &sdiodev->func[1]->dev);
+                sdiodev->irq_en = false;
+                sdiodev->oob_irq_requested = false;
+        }
+
+        if (sdiodev->sd_irq_requested) {
                 sdio_claim_host(sdiodev->func[1]);
                 sdio_release_irq(sdiodev->func[2]);
                 sdio_release_irq(sdiodev->func[1]);
                 sdio_release_host(sdiodev->func[1]);
+                sdiodev->sd_irq_requested = false;
         }
-
-        return 0;
 }
 
 void brcmf_sdiod_change_state(struct brcmf_sdio_dev *sdiodev,
@@ -1197,12 +1201,17 @@ static void brcmf_ops_sdio_remove(struct sdio_func *func)
         brcmf_dbg(SDIO, "sdio device ID: 0x%04x\n", func->device);
         brcmf_dbg(SDIO, "Function: %d\n", func->num);
 
-        if (func->num != 1)
-                return;
-
         bus_if = dev_get_drvdata(&func->dev);
         if (bus_if) {
                 sdiodev = bus_if->bus_priv.sdio;
+
+                /* start by unregistering irqs */
+                brcmf_sdiod_intr_unregister(sdiodev);
+
+                if (func->num != 1)
+                        return;
+
+                /* only proceed with rest of cleanup if func 1 */
                 brcmf_sdiod_remove(sdiodev);
 
                 dev_set_drvdata(&sdiodev->func[1]->dev, NULL);
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
index 62f475e31077..264bd638a3d9 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
@@ -541,6 +541,21 @@ brcmf_cfg80211_update_proto_addr_mode(struct wireless_dev *wdev)
                                                 ADDR_INDIRECT);
 }
 
+static int brcmf_get_first_free_bsscfgidx(struct brcmf_pub *drvr)
+{
+        int bsscfgidx;
+
+        for (bsscfgidx = 0; bsscfgidx < BRCMF_MAX_IFS; bsscfgidx++) {
+                /* bsscfgidx 1 is reserved for legacy P2P */
+                if (bsscfgidx == 1)
+                        continue;
+                if (!drvr->iflist[bsscfgidx])
+                        return bsscfgidx;
+        }
+
+        return -ENOMEM;
+}
+
 static int brcmf_cfg80211_request_ap_if(struct brcmf_if *ifp)
 {
         struct brcmf_mbss_ssid_le mbss_ssid_le;
@@ -548,7 +563,7 @@ static int brcmf_cfg80211_request_ap_if(struct brcmf_if *ifp)
         int err;
 
         memset(&mbss_ssid_le, 0, sizeof(mbss_ssid_le));
-        bsscfgidx = brcmf_get_next_free_bsscfgidx(ifp->drvr);
+        bsscfgidx = brcmf_get_first_free_bsscfgidx(ifp->drvr);
         if (bsscfgidx < 0)
                 return bsscfgidx;
 
@@ -586,7 +601,7 @@ struct wireless_dev *brcmf_ap_add_vif(struct wiphy *wiphy, const char *name,
 
         brcmf_dbg(INFO, "Adding vif \"%s\"\n", name);
 
-        vif = brcmf_alloc_vif(cfg, NL80211_IFTYPE_AP, false);
+        vif = brcmf_alloc_vif(cfg, NL80211_IFTYPE_AP);
         if (IS_ERR(vif))
                 return (struct wireless_dev *)vif;
 
@@ -669,20 +684,24 @@ static struct wireless_dev *brcmf_cfg80211_add_iface(struct wiphy *wiphy,
                 return ERR_PTR(-EOPNOTSUPP);
         case NL80211_IFTYPE_AP:
                 wdev = brcmf_ap_add_vif(wiphy, name, flags, params);
-                if (!IS_ERR(wdev))
-                        brcmf_cfg80211_update_proto_addr_mode(wdev);
-                return wdev;
+                break;
         case NL80211_IFTYPE_P2P_CLIENT:
         case NL80211_IFTYPE_P2P_GO:
         case NL80211_IFTYPE_P2P_DEVICE:
                 wdev = brcmf_p2p_add_vif(wiphy, name, name_assign_type, type, flags, params);
-                if (!IS_ERR(wdev))
-                        brcmf_cfg80211_update_proto_addr_mode(wdev);
-                return wdev;
+                break;
         case NL80211_IFTYPE_UNSPECIFIED:
         default:
                 return ERR_PTR(-EINVAL);
         }
+
+        if (IS_ERR(wdev))
+                brcmf_err("add iface %s type %d failed: err=%d\n",
+                          name, type, (int)PTR_ERR(wdev));
+        else
+                brcmf_cfg80211_update_proto_addr_mode(wdev);
+
+        return wdev;
 }
 
 static void brcmf_scan_config_mpc(struct brcmf_if *ifp, int mpc)
@@ -2750,7 +2769,7 @@ static s32 brcmf_inform_single_bss(struct brcmf_cfg80211_info *cfg,
         if (!bi->ctl_ch) {
                 ch.chspec = le16_to_cpu(bi->chanspec);
                 cfg->d11inf.decchspec(&ch);
-                bi->ctl_ch = ch.chnum;
+                bi->ctl_ch = ch.control_ch_num;
         }
         channel = bi->ctl_ch;
 
@@ -2868,7 +2887,7 @@ static s32 brcmf_inform_ibss(struct brcmf_cfg80211_info *cfg,
         else
                 band = wiphy->bands[NL80211_BAND_5GHZ];
 
-        freq = ieee80211_channel_to_frequency(ch.chnum, band->band);
+        freq = ieee80211_channel_to_frequency(ch.control_ch_num, band->band);
         cfg->channel = freq;
         notify_channel = ieee80211_get_channel(wiphy, freq);
 
@@ -2878,7 +2897,7 @@ static s32 brcmf_inform_ibss(struct brcmf_cfg80211_info *cfg,
         notify_ielen = le32_to_cpu(bi->ie_length);
         notify_signal = (s16)le16_to_cpu(bi->RSSI) * 100;
 
-        brcmf_dbg(CONN, "channel: %d(%d)\n", ch.chnum, freq);
+        brcmf_dbg(CONN, "channel: %d(%d)\n", ch.control_ch_num, freq);
         brcmf_dbg(CONN, "capability: %X\n", notify_capability);
         brcmf_dbg(CONN, "beacon interval: %d\n", notify_interval);
         brcmf_dbg(CONN, "signal: %d\n", notify_signal);
@@ -4439,7 +4458,7 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev,
         struct brcmf_join_params join_params;
         enum nl80211_iftype dev_role;
         struct brcmf_fil_bss_enable_le bss_enable;
-        u16 chanspec;
+        u16 chanspec = chandef_to_chanspec(&cfg->d11inf, &settings->chandef);
         bool mbss;
         int is_11d;
 
@@ -4515,16 +4534,8 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev,
 
         brcmf_config_ap_mgmt_ie(ifp->vif, &settings->beacon);
 
+        /* Parameters shared by all radio interfaces */
         if (!mbss) {
-                chanspec = chandef_to_chanspec(&cfg->d11inf,
-                                               &settings->chandef);
-                err = brcmf_fil_iovar_int_set(ifp, "chanspec", chanspec);
-                if (err < 0) {
-                        brcmf_err("Set Channel failed: chspec=%d, %d\n",
-                                  chanspec, err);
-                        goto exit;
-                }
-
                 if (is_11d != ifp->vif->is_11d) {
                         err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_REGULATORY,
                                                     is_11d);
@@ -4572,6 +4583,8 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev,
                 err = -EINVAL;
                 goto exit;
         }
+
+        /* Interface specific setup */
         if (dev_role == NL80211_IFTYPE_AP) {
                 if ((brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MBSS)) && (!mbss))
                         brcmf_fil_iovar_int_set(ifp, "mbss", 1);
@@ -4581,6 +4594,17 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev,
                         brcmf_err("setting AP mode failed %d\n", err);
                         goto exit;
                 }
+                if (!mbss) {
+                        /* Firmware 10.x requires setting channel after enabling
+                         * AP and before bringing interface up.
+                         */
+                        err = brcmf_fil_iovar_int_set(ifp, "chanspec", chanspec);
+                        if (err < 0) {
+                                brcmf_err("Set Channel failed: chspec=%d, %d\n",
+                                          chanspec, err);
+                                goto exit;
+                        }
+                }
                 err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_UP, 1);
                 if (err < 0) {
                         brcmf_err("BRCMF_C_UP error (%d)\n", err);
@@ -4602,7 +4626,13 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev,
                         goto exit;
                 }
                 brcmf_dbg(TRACE, "AP mode configuration complete\n");
-        } else {
+        } else if (dev_role == NL80211_IFTYPE_P2P_GO) {
+                err = brcmf_fil_iovar_int_set(ifp, "chanspec", chanspec);
+                if (err < 0) {
+                        brcmf_err("Set Channel failed: chspec=%d, %d\n",
+                                  chanspec, err);
+                        goto exit;
+                }
                 err = brcmf_fil_bsscfg_data_set(ifp, "ssid", &ssid_le,
                                                 sizeof(ssid_le));
                 if (err < 0) {
@@ -4619,7 +4649,10 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev,
                 }
 
                 brcmf_dbg(TRACE, "GO mode configuration complete\n");
+        } else {
+                WARN_ON(1);
         }
+
         set_bit(BRCMF_VIF_STATUS_AP_CREATED, &ifp->vif->sme_state);
         brcmf_net_setcarrier(ifp, true);
 
@@ -4908,6 +4941,68 @@ exit:
         return err;
 }
 
+static int brcmf_cfg80211_get_channel(struct wiphy *wiphy,
+                                      struct wireless_dev *wdev,
+                                      struct cfg80211_chan_def *chandef)
+{
+        struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy);
+        struct net_device *ndev = wdev->netdev;
+        struct brcmf_if *ifp;
+        struct brcmu_chan ch;
+        enum nl80211_band band = 0;
+        enum nl80211_chan_width width = 0;
+        u32 chanspec;
+        int freq, err;
+
+        if (!ndev)
+                return -ENODEV;
+        ifp = netdev_priv(ndev);
+
+        err = brcmf_fil_iovar_int_get(ifp, "chanspec", &chanspec);
+        if (err) {
+                brcmf_err("chanspec failed (%d)\n", err);
+                return err;
+        }
+
+        ch.chspec = chanspec;
+        cfg->d11inf.decchspec(&ch);
+
+        switch (ch.band) {
+        case BRCMU_CHAN_BAND_2G:
+                band = NL80211_BAND_2GHZ;
+                break;
+        case BRCMU_CHAN_BAND_5G:
+                band = NL80211_BAND_5GHZ;
+                break;
+        }
+
+        switch (ch.bw) {
+        case BRCMU_CHAN_BW_80:
+                width = NL80211_CHAN_WIDTH_80;
+                break;
+        case BRCMU_CHAN_BW_40:
+                width = NL80211_CHAN_WIDTH_40;
+                break;
+        case BRCMU_CHAN_BW_20:
+                width = NL80211_CHAN_WIDTH_20;
+                break;
+        case BRCMU_CHAN_BW_80P80:
+                width = NL80211_CHAN_WIDTH_80P80;
+                break;
+        case BRCMU_CHAN_BW_160:
+                width = NL80211_CHAN_WIDTH_160;
+                break;
+        }
+
+        freq = ieee80211_channel_to_frequency(ch.control_ch_num, band);
+        chandef->chan = ieee80211_get_channel(wiphy, freq);
+        chandef->width = width;
+        chandef->center_freq1 = ieee80211_channel_to_frequency(ch.chnum, band);
+        chandef->center_freq2 = 0;
+
+        return 0;
+}
+
 static int brcmf_cfg80211_crit_proto_start(struct wiphy *wiphy,
                                            struct wireless_dev *wdev,
                                            enum nl80211_crit_proto_id proto,
@@ -5070,6 +5165,7 @@ static struct cfg80211_ops brcmf_cfg80211_ops = {
         .mgmt_tx = brcmf_cfg80211_mgmt_tx,
         .remain_on_channel = brcmf_p2p_remain_on_channel,
         .cancel_remain_on_channel = brcmf_cfg80211_cancel_remain_on_channel,
+        .get_channel = brcmf_cfg80211_get_channel,
         .start_p2p_device = brcmf_p2p_start_device,
         .stop_p2p_device = brcmf_p2p_stop_device,
         .crit_proto_start = brcmf_cfg80211_crit_proto_start,
@@ -5078,8 +5174,7 @@ static struct cfg80211_ops brcmf_cfg80211_ops = {
 };
 
 struct brcmf_cfg80211_vif *brcmf_alloc_vif(struct brcmf_cfg80211_info *cfg,
-                                           enum nl80211_iftype type,
-                                           bool pm_block)
+                                           enum nl80211_iftype type)
 {
         struct brcmf_cfg80211_vif *vif_walk;
         struct brcmf_cfg80211_vif *vif;
@@ -5094,8 +5189,6 @@ struct brcmf_cfg80211_vif *brcmf_alloc_vif(struct brcmf_cfg80211_info *cfg,
         vif->wdev.wiphy = cfg->wiphy;
         vif->wdev.iftype = type;
 
-        vif->pm_block = pm_block;
-
         brcmf_init_prof(&vif->profile);
 
         if (type == NL80211_IFTYPE_AP) {
@@ -5296,7 +5389,7 @@ brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg,
         else
                 band = wiphy->bands[NL80211_BAND_5GHZ];
 
-        freq = ieee80211_channel_to_frequency(ch.chnum, band->band);
+        freq = ieee80211_channel_to_frequency(ch.control_ch_num, band->band);
         notify_channel = ieee80211_get_channel(wiphy, freq);
 
 done:
@@ -5352,7 +5445,6 @@ brcmf_notify_connect_status_ap(struct brcmf_cfg80211_info *cfg,
                                struct net_device *ndev,
                                const struct brcmf_event_msg *e, void *data)
 {
-        struct brcmf_if *ifp = netdev_priv(ndev);
         static int generation;
         u32 event = e->event_code;
         u32 reason = e->reason;
@@ -5363,8 +5455,6 @@ brcmf_notify_connect_status_ap(struct brcmf_cfg80211_info *cfg,
             ndev != cfg_to_ndev(cfg)) {
                 brcmf_dbg(CONN, "AP mode link down\n");
                 complete(&cfg->vif_disabled);
-                if (ifp->vif->mbss)
-                        brcmf_remove_interface(ifp);
                 return 0;
         }
 
@@ -5818,14 +5908,15 @@ static int brcmf_construct_chaninfo(struct brcmf_cfg80211_info *cfg,
                 channel = band->channels;
                 index = band->n_channels;
                 for (j = 0; j < band->n_channels; j++) {
-                        if (channel[j].hw_value == ch.chnum) {
+                        if (channel[j].hw_value == ch.control_ch_num) {
                                 index = j;
                                 break;
                         }
                 }
                 channel[index].center_freq =
-                        ieee80211_channel_to_frequency(ch.chnum, band->band);
-                channel[index].hw_value = ch.chnum;
+                        ieee80211_channel_to_frequency(ch.control_ch_num,
+                                                       band->band);
+                channel[index].hw_value = ch.control_ch_num;
 
                 /* assuming the chanspecs order is HT20,
                  * HT40 upper, HT40 lower, and VHT80.
@@ -5927,7 +6018,7 @@ static int brcmf_enable_bw40_2g(struct brcmf_cfg80211_info *cfg)
                         if (WARN_ON(ch.bw != BRCMU_CHAN_BW_40))
                                 continue;
                         for (j = 0; j < band->n_channels; j++) {
-                                if (band->channels[j].hw_value == ch.chnum)
+                                if (band->channels[j].hw_value == ch.control_ch_num)
                                         break;
                         }
                         if (WARN_ON(j == band->n_channels))
@@ -6715,11 +6806,10 @@ struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr,
                 return NULL;
         }
 
-        ops = kzalloc(sizeof(*ops), GFP_KERNEL);
+        ops = kmemdup(&brcmf_cfg80211_ops, sizeof(*ops), GFP_KERNEL);
         if (!ops)
                 return NULL;
 
-        memcpy(ops, &brcmf_cfg80211_ops, sizeof(*ops));
         ifp = netdev_priv(ndev);
 #ifdef CONFIG_PM
         if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_WOWL_GTK))
@@ -6740,7 +6830,7 @@ struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr,
         init_vif_event(&cfg->vif_event);
         INIT_LIST_HEAD(&cfg->vif_list);
 
-        vif = brcmf_alloc_vif(cfg, NL80211_IFTYPE_STATION, false);
+        vif = brcmf_alloc_vif(cfg, NL80211_IFTYPE_STATION);
         if (IS_ERR(vif))
                 goto wiphy_out;
 
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h
index 95e35bcc16ce..04bfc7e3ecde 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h
@@ -20,6 +20,9 @@
 /* for brcmu_d11inf */
 #include <brcmu_d11.h>
 
+#include "fwil_types.h"
+#include "p2p.h"
+
 #define WL_NUM_SCAN_MAX                 10
 #define WL_TLV_INFO_MAX                 1024
 #define WL_BSS_INFO_MAX                 2048
@@ -167,7 +170,6 @@ struct vif_saved_ie {
  * @wdev: wireless device.
  * @profile: profile information.
  * @sme_state: SME state using enum brcmf_vif_status bits.
- * @pm_block: power-management blocked.
  * @list: linked list.
  * @mgmt_rx_reg: registered rx mgmt frame types.
  * @mbss: Multiple BSS type, set if not first AP (not relevant for P2P).
@@ -177,7 +179,6 @@ struct brcmf_cfg80211_vif {
         struct wireless_dev wdev;
         struct brcmf_cfg80211_profile profile;
         unsigned long sme_state;
-        bool pm_block;
         struct vif_saved_ie saved_ie;
         struct list_head list;
         u16 mgmt_rx_reg;
@@ -388,8 +389,7 @@ s32 brcmf_cfg80211_down(struct net_device *ndev);
 enum nl80211_iftype brcmf_cfg80211_get_iftype(struct brcmf_if *ifp);
 
 struct brcmf_cfg80211_vif *brcmf_alloc_vif(struct brcmf_cfg80211_info *cfg,
-                                           enum nl80211_iftype type,
-                                           bool pm_block);
+                                           enum nl80211_iftype type);
 void brcmf_free_vif(struct brcmf_cfg80211_vif *vif);
 
 s32 brcmf_vif_set_mgmt_ie(struct brcmf_cfg80211_vif *vif, s32 pktflag,
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c
index d3fd6b1db1d9..05f22ff81d60 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c
@@ -685,6 +685,8 @@ static u32 brcmf_chip_tcm_rambase(struct brcmf_chip_priv *ci)
         case BRCM_CC_43602_CHIP_ID:
         case BRCM_CC_4371_CHIP_ID:
                 return 0x180000;
+        case BRCM_CC_43465_CHIP_ID:
+        case BRCM_CC_43525_CHIP_ID:
         case BRCM_CC_4365_CHIP_ID:
         case BRCM_CC_4366_CHIP_ID:
                 return 0x200000;
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c
index b590499f6883..faf4e46bd65b 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c
@@ -516,7 +516,7 @@ int brcmf_net_attach(struct brcmf_if *ifp, bool rtnl_locked)
         /* set appropriate operations */
         ndev->netdev_ops = &brcmf_netdev_ops_pri;
 
-        ndev->hard_header_len += drvr->hdrlen;
+        ndev->needed_headroom += drvr->hdrlen;
         ndev->ethtool_ops = &brcmf_ethtool_ops;
 
         drvr->rxsz = ndev->mtu + ndev->hard_header_len +
@@ -753,30 +753,6 @@ void brcmf_remove_interface(struct brcmf_if *ifp)
         brcmf_del_if(ifp->drvr, ifp->bsscfgidx);
 }
 
-int brcmf_get_next_free_bsscfgidx(struct brcmf_pub *drvr)
-{
-        int ifidx;
-        int bsscfgidx;
-        bool available;
-        int highest;
-
-        available = false;
-        bsscfgidx = 2;
-        highest = 2;
-        for (ifidx = 0; ifidx < BRCMF_MAX_IFS; ifidx++) {
-                if (drvr->iflist[ifidx]) {
-                        if (drvr->iflist[ifidx]->bsscfgidx == bsscfgidx)
-                                bsscfgidx = highest + 1;
-                        else if (drvr->iflist[ifidx]->bsscfgidx > highest)
-                                highest = drvr->iflist[ifidx]->bsscfgidx;
-                } else {
-                        available = true;
-                }
-        }
-
-        return available ? bsscfgidx : -ENOMEM;
-}
-
 #ifdef CONFIG_INET
 #define ARPOL_MAX_ENTRIES       8
 static int brcmf_inetaddr_changed(struct notifier_block *nb,
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h
index 647d3cc2a4dc..2a075c5f6f8b 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h
@@ -217,7 +217,6 @@ int brcmf_net_attach(struct brcmf_if *ifp, bool rtnl_locked);
 struct brcmf_if *brcmf_add_if(struct brcmf_pub *drvr, s32 bsscfgidx, s32 ifidx,
                               bool is_p2pdev, char *name, u8 *mac_addr);
 void brcmf_remove_interface(struct brcmf_if *ifp);
-int brcmf_get_next_free_bsscfgidx(struct brcmf_pub *drvr);
 void brcmf_txflowblock_if(struct brcmf_if *ifp,
                           enum brcmf_netif_stop_reason reason, bool state);
 void brcmf_txfinalize(struct brcmf_if *ifp, struct sk_buff *txp, bool success);
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c
index 5b30922b67ec..cd221ab55062 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c
@@ -2101,7 +2101,7 @@ int brcmf_fws_process_skb(struct brcmf_if *ifp, struct sk_buff *skb)
 
         brcmf_dbg(DATA, "tx proto=0x%X\n", ntohs(eh->h_proto));
         /* determine the priority */
-        if (!skb->priority)
+        if ((skb->priority == 0) || (skb->priority > 7))
                 skb->priority = cfg80211_classify8021d(skb, NULL);
 
         drvr->tx_multicast += !!multicast;
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c
index a70cda6c0592..f38a82133540 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c
@@ -1246,7 +1246,7 @@ bool brcmf_p2p_scan_finding_common_channel(struct brcmf_cfg80211_info *cfg,
                 if (!bi->ctl_ch) {
                         ch.chspec = le16_to_cpu(bi->chanspec);
                         cfg->d11inf.decchspec(&ch);
-                        bi->ctl_ch = ch.chnum;
+                        bi->ctl_ch = ch.control_ch_num;
                 }
                 afx_hdl->peer_chan = bi->ctl_ch;
                 brcmf_dbg(TRACE, "ACTION FRAME SCAN : Peer %pM found, channel : %d\n",
@@ -1385,7 +1385,7 @@ int brcmf_p2p_notify_action_frame_rx(struct brcmf_if *ifp,
                         if (test_bit(BRCMF_P2P_STATUS_FINDING_COMMON_CHANNEL,
                                      &p2p->status) &&
                             (ether_addr_equal(afx_hdl->tx_dst_addr, e->addr))) {
-                                afx_hdl->peer_chan = ch.chnum;
+                                afx_hdl->peer_chan = ch.control_ch_num;
                                 brcmf_dbg(INFO, "GON request: Peer found, channel=%d\n",
                                           afx_hdl->peer_chan);
                                 complete(&afx_hdl->act_frm_scan);
@@ -1428,7 +1428,7 @@ int brcmf_p2p_notify_action_frame_rx(struct brcmf_if *ifp,
         memcpy(&mgmt_frame->u, frame, mgmt_frame_len);
         mgmt_frame_len += offsetof(struct ieee80211_mgmt, u);
 
-        freq = ieee80211_channel_to_frequency(ch.chnum,
+        freq = ieee80211_channel_to_frequency(ch.control_ch_num,
                                               ch.band == BRCMU_CHAN_BAND_2G ?
                                               NL80211_BAND_2GHZ :
                                               NL80211_BAND_5GHZ);
@@ -1873,7 +1873,7 @@ s32 brcmf_p2p_notify_rx_mgmt_p2p_probereq(struct brcmf_if *ifp,
 
         if (test_bit(BRCMF_P2P_STATUS_FINDING_COMMON_CHANNEL, &p2p->status) &&
             (ether_addr_equal(afx_hdl->tx_dst_addr, e->addr))) {
-                afx_hdl->peer_chan = ch.chnum;
+                afx_hdl->peer_chan = ch.control_ch_num;
                 brcmf_dbg(INFO, "PROBE REQUEST: Peer found, channel=%d\n",
                           afx_hdl->peer_chan);
                 complete(&afx_hdl->act_frm_scan);
@@ -1898,7 +1898,7 @@ s32 brcmf_p2p_notify_rx_mgmt_p2p_probereq(struct brcmf_if *ifp,
 
         mgmt_frame = (u8 *)(rxframe + 1);
         mgmt_frame_len = e->datalen - sizeof(*rxframe);
-        freq = ieee80211_channel_to_frequency(ch.chnum,
+        freq = ieee80211_channel_to_frequency(ch.control_ch_num,
                                               ch.band == BRCMU_CHAN_BAND_2G ?
                                               NL80211_BAND_2GHZ :
                                               NL80211_BAND_5GHZ);
@@ -2030,8 +2030,6 @@ static int brcmf_p2p_request_p2p_if(struct brcmf_p2p_info *p2p,
 
         err = brcmf_fil_iovar_data_set(ifp, "p2p_ifadd", &if_request,
                                        sizeof(if_request));
-        if (err)
-                return err;
 
         return err;
 }
@@ -2076,8 +2074,7 @@ static struct wireless_dev *brcmf_p2p_create_p2pdev(struct brcmf_p2p_info *p2p,
         if (p2p->bss_idx[P2PAPI_BSSCFG_DEVICE].vif)
                 return ERR_PTR(-ENOSPC);
 
-        p2p_vif = brcmf_alloc_vif(p2p->cfg, NL80211_IFTYPE_P2P_DEVICE,
-                                  false);
+        p2p_vif = brcmf_alloc_vif(p2p->cfg, NL80211_IFTYPE_P2P_DEVICE);
         if (IS_ERR(p2p_vif)) {
                 brcmf_err("could not create discovery vif\n");
                 return (struct wireless_dev *)p2p_vif;
@@ -2177,7 +2174,7 @@ struct wireless_dev *brcmf_p2p_add_vif(struct wiphy *wiphy, const char *name,
                 return ERR_PTR(-EOPNOTSUPP);
         }
 
-        vif = brcmf_alloc_vif(cfg, type, false);
+        vif = brcmf_alloc_vif(cfg, type);
         if (IS_ERR(vif))
                 return (struct wireless_dev *)vif;
         brcmf_cfg80211_arm_vif_event(cfg, vif);
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
index 0af8db82da0c..3deba90c7eb5 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
@@ -54,21 +54,25 @@ BRCMF_FW_NVRAM_DEF(43570, "brcmfmac43570-pcie.bin", "brcmfmac43570-pcie.txt");
 BRCMF_FW_NVRAM_DEF(4358, "brcmfmac4358-pcie.bin", "brcmfmac4358-pcie.txt");
 BRCMF_FW_NVRAM_DEF(4359, "brcmfmac4359-pcie.bin", "brcmfmac4359-pcie.txt");
 BRCMF_FW_NVRAM_DEF(4365B, "brcmfmac4365b-pcie.bin", "brcmfmac4365b-pcie.txt");
+BRCMF_FW_NVRAM_DEF(4365C, "brcmfmac4365c-pcie.bin", "brcmfmac4365c-pcie.txt");
 BRCMF_FW_NVRAM_DEF(4366B, "brcmfmac4366b-pcie.bin", "brcmfmac4366b-pcie.txt");
 BRCMF_FW_NVRAM_DEF(4366C, "brcmfmac4366c-pcie.bin", "brcmfmac4366c-pcie.txt");
 BRCMF_FW_NVRAM_DEF(4371, "brcmfmac4371-pcie.bin", "brcmfmac4371-pcie.txt");
 
 static struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = {
         BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43602_CHIP_ID, 0xFFFFFFFF, 43602),
+        BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43465_CHIP_ID, 0xFFFFFFF0, 4366C),
         BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4350_CHIP_ID, 0x000000FF, 4350C),
         BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4350_CHIP_ID, 0xFFFFFF00, 4350),
+        BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43525_CHIP_ID, 0xFFFFFFF0, 4365C),
         BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4356_CHIP_ID, 0xFFFFFFFF, 4356),
         BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43567_CHIP_ID, 0xFFFFFFFF, 43570),
         BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43569_CHIP_ID, 0xFFFFFFFF, 43570),
         BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43570_CHIP_ID, 0xFFFFFFFF, 43570),
         BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4358_CHIP_ID, 0xFFFFFFFF, 4358),
         BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4359_CHIP_ID, 0xFFFFFFFF, 4359),
-        BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4365_CHIP_ID, 0xFFFFFFFF, 4365B),
+        BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4365_CHIP_ID, 0x0000000F, 4365B),
+        BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4365_CHIP_ID, 0xFFFFFFF0, 4365C),
         BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4366_CHIP_ID, 0x0000000F, 4366B),
         BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4366_CHIP_ID, 0xFFFFFFF0, 4366C),
         BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4371_CHIP_ID, 0xFFFFFFFF, 4371),
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
index 67e69bff2545..5fb8b91b9326 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
@@ -1384,8 +1384,7 @@ static int brcmf_sdio_hdparse(struct brcmf_sdio *bus, u8 *header,
                 return -ENXIO;
         }
         if (rd->seq_num != rx_seq) {
-                brcmf_err("seq %d: sequence number error, expect %d\n",
-                          rx_seq, rd->seq_num);
+                brcmf_dbg(SDIO, "seq %d, expected %d\n", rx_seq, rd->seq_num);
                 bus->sdcnt.rx_badseq++;
                 rd->seq_num = rx_seq;
         }
@@ -3666,7 +3665,7 @@ brcmf_sdio_drivestrengthinit(struct brcmf_sdio_dev *sdiodev,
                 str_shift = 11;
                 break;
         default:
-                brcmf_err("No SDIO Drive strength init done for chip %s rev %d pmurev %d\n",
+                brcmf_dbg(INFO, "No SDIO driver strength init needed for chip %s rev %d pmurev %d\n",
                           ci->name, ci->chiprev, ci->pmurev);
                 break;
         }
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h
index dcf0ce8cd2c1..f3da32fc6360 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h
@@ -186,6 +186,7 @@ struct brcmf_sdio_dev {
         struct brcmf_bus *bus_if;
         struct brcmf_mp_device *settings;
         bool oob_irq_requested;
+        bool sd_irq_requested;
         bool irq_en;                    /* irq enable flags */
         spinlock_t irq_en_lock;
         bool irq_wake;                  /* irq wake enable flags */
@@ -293,7 +294,7 @@ struct sdpcmd_regs {
 
 /* Register/deregister interrupt handler. */
 int brcmf_sdiod_intr_register(struct brcmf_sdio_dev *sdiodev);
-int brcmf_sdiod_intr_unregister(struct brcmf_sdio_dev *sdiodev);
+void brcmf_sdiod_intr_unregister(struct brcmf_sdio_dev *sdiodev);
 
 /* sdio device register access interface */
 u8 brcmf_sdiod_regrb(struct brcmf_sdio_dev *sdiodev, u32 addr, int *ret);
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_n.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_n.c
index 99dac9b8a082..b3aab2fe96eb 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_n.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_n.c
@@ -27017,7 +27017,7 @@ wlc_phy_rxcal_gainctrl_nphy_rev5(struct brcms_phy *pi, u8 rx_core,
                 tx_core = 1 - rx_core;
 
         num_samps = 1024;
-        desired_log2_pwr = (cal_type == 0) ? 13 : 13;
+        desired_log2_pwr = 13;
 
         wlc_phy_rx_iq_coeffs_nphy(pi, 0, &save_comp);
         zero_comp.a0 = zero_comp.b0 = zero_comp.a1 = zero_comp.b1 = 0x0;
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c b/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c
index 2b2522bdd8eb..d8b79cb72b58 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c
@@ -107,6 +107,7 @@ static void brcmu_d11n_decchspec(struct brcmu_chan *ch)
         u16 val;
 
         ch->chnum = (u8)(ch->chspec & BRCMU_CHSPEC_CH_MASK);
+        ch->control_ch_num = ch->chnum;
 
         switch (ch->chspec & BRCMU_CHSPEC_D11N_BW_MASK) {
         case BRCMU_CHSPEC_D11N_BW_20:
@@ -118,10 +119,10 @@ static void brcmu_d11n_decchspec(struct brcmu_chan *ch)
                 val = ch->chspec & BRCMU_CHSPEC_D11N_SB_MASK;
                 if (val == BRCMU_CHSPEC_D11N_SB_L) {
                         ch->sb = BRCMU_CHAN_SB_L;
-                        ch->chnum -= CH_10MHZ_APART;
+                        ch->control_ch_num -= CH_10MHZ_APART;
                 } else {
                         ch->sb = BRCMU_CHAN_SB_U;
-                        ch->chnum += CH_10MHZ_APART;
+                        ch->control_ch_num += CH_10MHZ_APART;
                 }
                 break;
         default:
@@ -147,6 +148,7 @@ static void brcmu_d11ac_decchspec(struct brcmu_chan *ch)
         u16 val;
 
         ch->chnum = (u8)(ch->chspec & BRCMU_CHSPEC_CH_MASK);
+        ch->control_ch_num = ch->chnum;
 
         switch (ch->chspec & BRCMU_CHSPEC_D11AC_BW_MASK) {
         case BRCMU_CHSPEC_D11AC_BW_20:
@@ -158,10 +160,10 @@ static void brcmu_d11ac_decchspec(struct brcmu_chan *ch)
                 val = ch->chspec & BRCMU_CHSPEC_D11AC_SB_MASK;
                 if (val == BRCMU_CHSPEC_D11AC_SB_L) {
                         ch->sb = BRCMU_CHAN_SB_L;
-                        ch->chnum -= CH_10MHZ_APART;
+                        ch->control_ch_num -= CH_10MHZ_APART;
                 } else if (val == BRCMU_CHSPEC_D11AC_SB_U) {
                         ch->sb = BRCMU_CHAN_SB_U;
-                        ch->chnum += CH_10MHZ_APART;
+                        ch->control_ch_num += CH_10MHZ_APART;
                 } else {
                         WARN_ON_ONCE(1);
                 }
@@ -172,16 +174,16 @@ static void brcmu_d11ac_decchspec(struct brcmu_chan *ch)
                                          BRCMU_CHSPEC_D11AC_SB_SHIFT);
                 switch (ch->sb) {
                 case BRCMU_CHAN_SB_LL:
-                        ch->chnum -= CH_30MHZ_APART;
+                        ch->control_ch_num -= CH_30MHZ_APART;
                         break;
                 case BRCMU_CHAN_SB_LU:
-                        ch->chnum -= CH_10MHZ_APART;
+                        ch->control_ch_num -= CH_10MHZ_APART;
                         break;
                 case BRCMU_CHAN_SB_UL:
-                        ch->chnum += CH_10MHZ_APART;
+                        ch->control_ch_num += CH_10MHZ_APART;
                         break;
                 case BRCMU_CHAN_SB_UU:
-                        ch->chnum += CH_30MHZ_APART;
+                        ch->control_ch_num += CH_30MHZ_APART;
                         break;
                 default:
                         WARN_ON_ONCE(1);
diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h
index 699f2c2782ee..3cc42bef6245 100644
--- a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h
+++ b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h
@@ -40,7 +40,9 @@
 #define BRCM_CC_4339_CHIP_ID            0x4339
 #define BRCM_CC_43430_CHIP_ID           43430
 #define BRCM_CC_4345_CHIP_ID            0x4345
+#define BRCM_CC_43465_CHIP_ID           43465
 #define BRCM_CC_4350_CHIP_ID            0x4350
+#define BRCM_CC_43525_CHIP_ID           43525
 #define BRCM_CC_4354_CHIP_ID            0x4354
 #define BRCM_CC_4356_CHIP_ID            0x4356
 #define BRCM_CC_43566_CHIP_ID           43566
diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_d11.h b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_d11.h
index f9745ea8b3e0..8b8b2ecb3199 100644
--- a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_d11.h
+++ b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_d11.h
@@ -125,14 +125,36 @@ enum brcmu_chan_sb {
         BRCMU_CHAN_SB_UU = BRCMU_CHAN_SB_LUU,
 };
 
+/**
+ * struct brcmu_chan - stores channel formats
+ *
+ * This structure can be used with functions translating chanspec into generic
+ * channel info and the other way.
+ *
+ * @chspec: firmware specific format
+ * @chnum: center channel number
+ * @control_ch_num: control channel number
+ * @band: frequency band
+ * @bw: channel width
+ * @sb: control sideband (location of control channel against the center one)
+ */
 struct brcmu_chan {
         u16 chspec;
         u8 chnum;
+        u8 control_ch_num;
         u8 band;
         enum brcmu_chan_bw bw;
         enum brcmu_chan_sb sb;
 };
 
+/**
+ * struct brcmu_d11inf - provides functions translating channel format
+ *
+ * @io_type: determines version of channel format used by firmware
+ * @encchspec: encodes channel info into a chanspec, requires center channel
+ *      number, ignores control one
+ * @decchspec: decodes chanspec into generic info
+ */
 struct brcmu_d11inf {
         u8 io_type;
 
diff --git a/drivers/net/wireless/intel/iwlegacy/3945.c b/drivers/net/wireless/intel/iwlegacy/3945.c
index 7bcedbb53d94..209dc9988455 100644
--- a/drivers/net/wireless/intel/iwlegacy/3945.c
+++ b/drivers/net/wireless/intel/iwlegacy/3945.c
@@ -1019,12 +1019,13 @@ il3945_hw_txq_ctx_free(struct il_priv *il)
         int txq_id;
 
         /* Tx queues */
-        if (il->txq)
+        if (il->txq) {
                 for (txq_id = 0; txq_id < il->hw_params.max_txq_num; txq_id++)
                         if (txq_id == IL39_CMD_QUEUE_NUM)
                                 il_cmd_queue_free(il);
                         else
                                 il_tx_queue_free(il, txq_id);
+        }
 
         /* free tx queue structure */
         il_free_txq_mem(il);
diff --git a/drivers/net/wireless/mac80211_hwsim.c b/drivers/net/wireless/mac80211_hwsim.c
index 4dd5adcdd29b..a1e28a4fd658 100644
--- a/drivers/net/wireless/mac80211_hwsim.c
+++ b/drivers/net/wireless/mac80211_hwsim.c
@@ -30,6 +30,8 @@
 #include <linux/module.h>
 #include <linux/ktime.h>
 #include <net/genetlink.h>
+#include <net/net_namespace.h>
+#include <net/netns/generic.h>
 #include "mac80211_hwsim.h"
 
 #define WARN_QUEUE 100
@@ -250,6 +252,28 @@ static inline void hwsim_clear_chanctx_magic(struct ieee80211_chanctx_conf *c)
         cp->magic = 0;
 }
 
+static unsigned int hwsim_net_id;
+
+static int hwsim_netgroup;
+
+struct hwsim_net {
+        int netgroup;
+};
+
+static inline int hwsim_net_get_netgroup(struct net *net)
+{
+        struct hwsim_net *hwsim_net = net_generic(net, hwsim_net_id);
+
+        return hwsim_net->netgroup;
+}
+
+static inline void hwsim_net_set_netgroup(struct net *net)
+{
+        struct hwsim_net *hwsim_net = net_generic(net, hwsim_net_id);
+
+        hwsim_net->netgroup = hwsim_netgroup++;
+}
+
 static struct class *hwsim_class;
 
 static struct net_device *hwsim_mon; /* global monitor netdev */
@@ -526,6 +550,9 @@ struct mac80211_hwsim_data {
          */
         u64 group;
 
+        /* group shared by radios created in the same netns */
+        int netgroup;
+
         int power_level;
 
         /* difference between this hw's clock and the real clock, in usecs */
@@ -568,6 +595,7 @@ static struct genl_family hwsim_genl_family = {
         .name = "MAC80211_HWSIM",
         .version = 1,
         .maxattr = HWSIM_ATTR_MAX,
+        .netnsok = true,
 };
 
 enum hwsim_multicast_groups {
@@ -1202,6 +1230,9 @@ static bool mac80211_hwsim_tx_frame_no_nl(struct ieee80211_hw *hw,
                 if (!(data->group & data2->group))
                         continue;
 
+                if (data->netgroup != data2->netgroup)
+                        continue;
+
                 if (!hwsim_chans_compat(chan, data2->tmp_chan) &&
                     !hwsim_chans_compat(chan, data2->channel)) {
                         ieee80211_iterate_active_interfaces_atomic(
@@ -2349,6 +2380,7 @@ static int mac80211_hwsim_new_radio(struct genl_info *info,
         struct ieee80211_hw *hw;
         enum nl80211_band band;
         const struct ieee80211_ops *ops = &mac80211_hwsim_ops;
+        struct net *net;
         int idx;
 
         if (WARN_ON(param->channels > 1 && !param->use_chanctx))
@@ -2366,6 +2398,13 @@ static int mac80211_hwsim_new_radio(struct genl_info *info,
                 err = -ENOMEM;
                 goto failed;
         }
+
+        if (info)
+                net = genl_info_net(info);
+        else
+                net = &init_net;
+        wiphy_net_set(hw->wiphy, net);
+
         data = hw->priv;
         data->hw = hw;
 
@@ -2541,6 +2580,8 @@ static int mac80211_hwsim_new_radio(struct genl_info *info,
         data->group = 1;
         mutex_init(&data->mutex);
 
+        data->netgroup = hwsim_net_get_netgroup(net);
+
         /* Enable frame retransmissions for lossy channels */
         hw->max_rates = 4;
         hw->max_rate_tries = 11;
@@ -3014,6 +3055,9 @@ static int hwsim_del_radio_nl(struct sk_buff *msg, struct genl_info *info)
                                 continue;
                 }
 
+                if (!net_eq(wiphy_net(data->hw->wiphy), genl_info_net(info)))
+                        continue;
+
                 list_del(&data->list);
                 spin_unlock_bh(&hwsim_radio_lock);
                 mac80211_hwsim_del_radio(data, wiphy_name(data->hw->wiphy),
@@ -3040,6 +3084,9 @@ static int hwsim_get_radio_nl(struct sk_buff *msg, struct genl_info *info)
                 if (data->idx != idx)
                         continue;
 
+                if (!net_eq(wiphy_net(data->hw->wiphy), genl_info_net(info)))
+                        continue;
+
                 skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
                 if (!skb) {
                         res = -ENOMEM;
@@ -3079,6 +3126,9 @@ static int hwsim_dump_radio_nl(struct sk_buff *skb,
                 if (data->idx < idx)
                         continue;
 
+                if (!net_eq(wiphy_net(data->hw->wiphy), sock_net(skb->sk)))
+                        continue;
+
                 res = mac80211_hwsim_get_radio(skb, data,
                                                NETLINK_CB(cb->skb).portid,
                                                cb->nlh->nlmsg_seq, cb,
@@ -3118,13 +3168,13 @@ static const struct genl_ops hwsim_ops[] = {
                 .cmd = HWSIM_CMD_NEW_RADIO,
                 .policy = hwsim_genl_policy,
                 .doit = hwsim_new_radio_nl,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
         },
         {
                 .cmd = HWSIM_CMD_DEL_RADIO,
                 .policy = hwsim_genl_policy,
                 .doit = hwsim_del_radio_nl,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
         },
         {
                 .cmd = HWSIM_CMD_GET_RADIO,
@@ -3206,6 +3256,40 @@ failure:
         return -EINVAL;
 }
 
+static __net_init int hwsim_init_net(struct net *net)
+{
+        hwsim_net_set_netgroup(net);
+
+        return 0;
+}
+
+static void __net_exit hwsim_exit_net(struct net *net)
+{
+        struct mac80211_hwsim_data *data, *tmp;
+
+        spin_lock_bh(&hwsim_radio_lock);
+        list_for_each_entry_safe(data, tmp, &hwsim_radios, list) {
+                if (!net_eq(wiphy_net(data->hw->wiphy), net))
+                        continue;
+
+                /* Radios created in init_net are returned to init_net. */
+                if (data->netgroup == hwsim_net_get_netgroup(&init_net))
+                        continue;
+
+                list_del(&data->list);
+                INIT_WORK(&data->destroy_work, destroy_radio);
+                schedule_work(&data->destroy_work);
+        }
+        spin_unlock_bh(&hwsim_radio_lock);
+}
+
+static struct pernet_operations hwsim_net_ops = {
+        .init = hwsim_init_net,
+        .exit = hwsim_exit_net,
+        .id   = &hwsim_net_id,
+        .size = sizeof(struct hwsim_net),
+};
+
 static void hwsim_exit_netlink(void)
 {
         /* unregister the notifier */
@@ -3242,10 +3326,14 @@ static int __init init_mac80211_hwsim(void)
         spin_lock_init(&hwsim_radio_lock);
         INIT_LIST_HEAD(&hwsim_radios);
 
-        err = platform_driver_register(&mac80211_hwsim_driver);
+        err = register_pernet_device(&hwsim_net_ops);
         if (err)
                 return err;
 
+        err = platform_driver_register(&mac80211_hwsim_driver);
+        if (err)
+                goto out_unregister_pernet;
+
         hwsim_class = class_create(THIS_MODULE, "mac80211_hwsim");
         if (IS_ERR(hwsim_class)) {
                 err = PTR_ERR(hwsim_class);
@@ -3363,6 +3451,8 @@ out_free_radios:
         mac80211_hwsim_free();
 out_unregister_driver:
         platform_driver_unregister(&mac80211_hwsim_driver);
+out_unregister_pernet:
+        unregister_pernet_device(&hwsim_net_ops);
         return err;
 }
 module_init(init_mac80211_hwsim);
@@ -3376,5 +3466,6 @@ static void __exit exit_mac80211_hwsim(void)
         mac80211_hwsim_free();
         unregister_netdev(hwsim_mon);
         platform_driver_unregister(&mac80211_hwsim_driver);
+        unregister_pernet_device(&hwsim_net_ops);
 }
 module_exit(exit_mac80211_hwsim);
diff --git a/drivers/net/wireless/marvell/libertas/if_sdio.c b/drivers/net/wireless/marvell/libertas/if_sdio.c
index 13eae9ff8c35..47f4a14c84fe 100644
--- a/drivers/net/wireless/marvell/libertas/if_sdio.c
+++ b/drivers/net/wireless/marvell/libertas/if_sdio.c
@@ -1228,7 +1228,7 @@ static int if_sdio_probe(struct sdio_func *func,
         }
 
         spin_lock_init(&card->lock);
-        card->workqueue = create_workqueue("libertas_sdio");
+        card->workqueue = alloc_workqueue("libertas_sdio", WQ_MEM_RECLAIM, 0);
         INIT_WORK(&card->packet_worker, if_sdio_host_to_card_worker);
         init_waitqueue_head(&card->pwron_waitq);
 
@@ -1326,7 +1326,6 @@ static void if_sdio_remove(struct sdio_func *func)
         lbs_stop_card(card->priv);
         lbs_remove_card(card->priv);
 
-        flush_workqueue(card->workqueue);
         destroy_workqueue(card->workqueue);
 
         while (card->packets) {
diff --git a/drivers/net/wireless/marvell/libertas/if_spi.c b/drivers/net/wireless/marvell/libertas/if_spi.c
index 82c0796377aa..c3a53cd6988e 100644
--- a/drivers/net/wireless/marvell/libertas/if_spi.c
+++ b/drivers/net/wireless/marvell/libertas/if_spi.c
@@ -1180,7 +1180,7 @@ static int if_spi_probe(struct spi_device *spi)
         priv->fw_ready = 1;
 
         /* Initialize interrupt handling stuff. */
-        card->workqueue = create_workqueue("libertas_spi");
+        card->workqueue = alloc_workqueue("libertas_spi", WQ_MEM_RECLAIM, 0);
         INIT_WORK(&card->packet_work, if_spi_host_to_card_worker);
         INIT_WORK(&card->resume_work, if_spi_resume_worker);
 
@@ -1208,7 +1208,6 @@ static int if_spi_probe(struct spi_device *spi)
 release_irq:
         free_irq(spi->irq, card);
 terminate_workqueue:
-        flush_workqueue(card->workqueue);
         destroy_workqueue(card->workqueue);
         lbs_remove_card(priv); /* will call free_netdev */
 free_card:
@@ -1235,7 +1234,6 @@ static int libertas_spi_remove(struct spi_device *spi)
         lbs_remove_card(priv); /* will call free_netdev */
 
         free_irq(spi->irq, card);
-        flush_workqueue(card->workqueue);
         destroy_workqueue(card->workqueue);
         if (card->pdata->teardown)
                 card->pdata->teardown(spi);
diff --git a/drivers/net/wireless/marvell/libertas_tf/main.c b/drivers/net/wireless/marvell/libertas_tf/main.c
index 0bf8916a02cf..75bf0c8a2f6f 100644
--- a/drivers/net/wireless/marvell/libertas_tf/main.c
+++ b/drivers/net/wireless/marvell/libertas_tf/main.c
@@ -16,7 +16,6 @@
 #include <linux/module.h>
 #include "libertas_tf.h"
 
-#define DRIVER_RELEASE_VERSION "004.p0"
 /* thinfirm version: 5.132.X.pX */
 #define LBTF_FW_VER_MIN         0x05840300
 #define LBTF_FW_VER_MAX         0x0584ffff
@@ -27,12 +26,6 @@ unsigned int lbtf_debug;
 EXPORT_SYMBOL_GPL(lbtf_debug);
 module_param_named(libertas_tf_debug, lbtf_debug, int, 0644);
 
-static const char lbtf_driver_version[] = "THINFIRM-USB8388-" DRIVER_RELEASE_VERSION
-#ifdef DEBUG
-        "-dbg"
-#endif
-        "";
-
 struct workqueue_struct *lbtf_wq;
 
 static const struct ieee80211_channel lbtf_channels[] = {
diff --git a/drivers/net/wireless/marvell/mwifiex/11n_aggr.c b/drivers/net/wireless/marvell/mwifiex/11n_aggr.c
index 1efef3b8273d..dc49c3de1f25 100644
--- a/drivers/net/wireless/marvell/mwifiex/11n_aggr.c
+++ b/drivers/net/wireless/marvell/mwifiex/11n_aggr.c
@@ -184,7 +184,7 @@ mwifiex_11n_aggregate_pkt(struct mwifiex_private *priv,
 
         tx_info_src = MWIFIEX_SKB_TXCB(skb_src);
         skb_aggr = mwifiex_alloc_dma_align_buf(adapter->tx_buf_size,
-                                               GFP_ATOMIC | GFP_DMA);
+                                               GFP_ATOMIC);
         if (!skb_aggr) {
                 spin_unlock_irqrestore(&priv->wmm.ra_list_spinlock,
                                        ra_list_flags);
diff --git a/drivers/net/wireless/marvell/mwifiex/init.c b/drivers/net/wireless/marvell/mwifiex/init.c
index 78c532f0d286..a6d86d4ccd22 100644
--- a/drivers/net/wireless/marvell/mwifiex/init.c
+++ b/drivers/net/wireless/marvell/mwifiex/init.c
@@ -788,3 +788,4 @@ poll_fw:
 
         return ret;
 }
+EXPORT_SYMBOL_GPL(mwifiex_dnld_fw);
diff --git a/drivers/net/wireless/marvell/mwifiex/join.c b/drivers/net/wireless/marvell/mwifiex/join.c
index 62211fca91b7..a4b773d102b3 100644
--- a/drivers/net/wireless/marvell/mwifiex/join.c
+++ b/drivers/net/wireless/marvell/mwifiex/join.c
@@ -1281,7 +1281,7 @@ int mwifiex_ret_802_11_ad_hoc(struct mwifiex_private *priv,
         if (result) {
                 mwifiex_dbg(priv->adapter, ERROR, "ADHOC_RESP: failed\n");
                 if (priv->media_connected)
-                        mwifiex_reset_connect_state(priv, result);
+                        mwifiex_reset_connect_state(priv, result, true);
 
                 memset(&priv->curr_bss_params.bss_descriptor,
                        0x00, sizeof(struct mwifiex_bssdescriptor));
diff --git a/drivers/net/wireless/marvell/mwifiex/main.c b/drivers/net/wireless/marvell/mwifiex/main.c
index 8b67a552a690..0e280f879b58 100644
--- a/drivers/net/wireless/marvell/mwifiex/main.c
+++ b/drivers/net/wireless/marvell/mwifiex/main.c
@@ -526,10 +526,12 @@ static void mwifiex_fw_dpc(const struct firmware *firmware, void *context)
         fw.fw_buf = (u8 *) adapter->firmware->data;
         fw.fw_len = adapter->firmware->size;
 
-        if (adapter->if_ops.dnld_fw)
+        if (adapter->if_ops.dnld_fw) {
                 ret = adapter->if_ops.dnld_fw(adapter, &fw);
-        else
+        } else {
                 ret = mwifiex_dnld_fw(adapter, &fw);
+        }
+
         if (ret == -1)
                 goto err_dnld_fw;
 
diff --git a/drivers/net/wireless/marvell/mwifiex/main.h b/drivers/net/wireless/marvell/mwifiex/main.h
index 0207af00be42..f0cd055c0b2c 100644
--- a/drivers/net/wireless/marvell/mwifiex/main.h
+++ b/drivers/net/wireless/marvell/mwifiex/main.h
@@ -1128,7 +1128,8 @@ int mwifiex_cmd_802_11_associate(struct mwifiex_private *priv,
                                  struct mwifiex_bssdescriptor *bss_desc);
 int mwifiex_ret_802_11_associate(struct mwifiex_private *priv,
                                  struct host_cmd_ds_command *resp);
-void mwifiex_reset_connect_state(struct mwifiex_private *priv, u16 reason);
+void mwifiex_reset_connect_state(struct mwifiex_private *priv, u16 reason,
+                                 bool from_ap);
 u8 mwifiex_band_to_radio_type(u8 band);
 int mwifiex_deauthenticate(struct mwifiex_private *priv, u8 *mac);
 void mwifiex_deauthenticate_all(struct mwifiex_adapter *adapter);
diff --git a/drivers/net/wireless/marvell/mwifiex/pcie.c b/drivers/net/wireless/marvell/mwifiex/pcie.c
index 0c7937eb6b77..1b1e266ce00f 100644
--- a/drivers/net/wireless/marvell/mwifiex/pcie.c
+++ b/drivers/net/wireless/marvell/mwifiex/pcie.c
@@ -507,7 +507,7 @@ static int mwifiex_init_rxq_ring(struct mwifiex_adapter *adapter)
         for (i = 0; i < MWIFIEX_MAX_TXRX_BD; i++) {
                 /* Allocate skb here so that firmware can DMA data from it */
                 skb = mwifiex_alloc_dma_align_buf(MWIFIEX_RX_DATA_BUF_SIZE,
-                                                  GFP_KERNEL | GFP_DMA);
+                                                  GFP_KERNEL);
                 if (!skb) {
                         mwifiex_dbg(adapter, ERROR,
                                     "Unable to allocate skb for RX ring.\n");
@@ -1319,7 +1319,7 @@ static int mwifiex_pcie_process_recv_data(struct mwifiex_adapter *adapter)
                 }
 
                 skb_tmp = mwifiex_alloc_dma_align_buf(MWIFIEX_RX_DATA_BUF_SIZE,
-                                                      GFP_KERNEL | GFP_DMA);
+                                                      GFP_KERNEL);
                 if (!skb_tmp) {
                         mwifiex_dbg(adapter, ERROR,
                                     "Unable to allocate skb.\n");
@@ -2804,7 +2804,7 @@ static int mwifiex_pcie_request_irq(struct mwifiex_adapter *adapter)
 }
 
 /*
- * This function get firmare name for downloading by revision id
+ * This function gets the firmware name for downloading by revision id
  *
  * Read revision id register to get revision id
  */
@@ -2901,10 +2901,11 @@ static void mwifiex_unregister_dev(struct mwifiex_adapter *adapter)
 {
         struct pcie_service_card *card = adapter->card;
         const struct mwifiex_pcie_card_reg *reg;
-        struct pci_dev *pdev = card->dev;
+        struct pci_dev *pdev;
         int i;
 
         if (card) {
+                pdev = card->dev;
                 if (card->msix_enable) {
                         for (i = 0; i < MWIFIEX_NUM_MSIX_VECTORS; i++)
                                 synchronize_irq(card->msix_entries[i].vector);
diff --git a/drivers/net/wireless/marvell/mwifiex/sdio.c b/drivers/net/wireless/marvell/mwifiex/sdio.c
index bdc51ffd43ec..d3e1561ca075 100644
--- a/drivers/net/wireless/marvell/mwifiex/sdio.c
+++ b/drivers/net/wireless/marvell/mwifiex/sdio.c
@@ -102,10 +102,9 @@ static int mwifiex_sdio_probe_of(struct device *dev, struct sdio_mmc_card *card)
         struct mwifiex_plt_wake_cfg *cfg;
         int ret;
 
-        if (!dev->of_node ||
-            !of_match_node(mwifiex_sdio_of_match_table, dev->of_node)) {
-                dev_err(dev, "sdio platform data not available\n");
-                return -1;
+        if (!of_match_node(mwifiex_sdio_of_match_table, dev->of_node)) {
+                dev_err(dev, "required compatible string missing\n");
+                return -EINVAL;
         }
 
         card->plt_of_node = dev->of_node;
@@ -115,7 +114,7 @@ static int mwifiex_sdio_probe_of(struct device *dev, struct sdio_mmc_card *card)
         if (cfg && card->plt_of_node) {
                 cfg->irq_wifi = irq_of_parse_and_map(card->plt_of_node, 0);
                 if (!cfg->irq_wifi) {
-                        dev_err(dev,
+                        dev_dbg(dev,
                                 "fail to parse irq_wifi from device tree\n");
                 } else {
                         ret = devm_request_irq(dev, cfg->irq_wifi,
@@ -183,24 +182,35 @@ mwifiex_sdio_probe(struct sdio_func *func, const struct sdio_device_id *id)
         sdio_release_host(func);
 
         if (ret) {
-                pr_err("%s: failed to enable function\n", __func__);
-                kfree(card);
-                return -EIO;
+                dev_err(&func->dev, "failed to enable function\n");
+                goto err_free;
         }
 
         /* device tree node parsing and platform specific configuration*/
-        mwifiex_sdio_probe_of(&func->dev, card);
-
-        if (mwifiex_add_card(card, &add_remove_card_sem, &sdio_ops,
-                             MWIFIEX_SDIO)) {
-                pr_err("%s: add card failed\n", __func__);
-                kfree(card);
-                sdio_claim_host(func);
-                ret = sdio_disable_func(func);
-                sdio_release_host(func);
-                ret = -1;
+        if (func->dev.of_node) {
+                ret = mwifiex_sdio_probe_of(&func->dev, card);
+                if (ret) {
+                        dev_err(&func->dev, "SDIO dt node parse failed\n");
+                        goto err_disable;
+                }
+        }
+
+        ret = mwifiex_add_card(card, &add_remove_card_sem, &sdio_ops,
+                               MWIFIEX_SDIO);
+        if (ret) {
+                dev_err(&func->dev, "add card failed\n");
+                goto err_disable;
         }
 
+        return 0;
+
+err_disable:
+        sdio_claim_host(func);
+        sdio_disable_func(func);
+        sdio_release_host(func);
+err_free:
+        kfree(card);
+
         return ret;
 }
 
@@ -544,6 +554,19 @@ static int mwifiex_pm_wakeup_card_complete(struct mwifiex_adapter *adapter)
         return mwifiex_write_reg(adapter, CONFIGURATION_REG, 0);
 }
 
+static int mwifiex_sdio_dnld_fw(struct mwifiex_adapter *adapter,
+                        struct mwifiex_fw_image *fw)
+{
+        struct sdio_mmc_card *card = adapter->card;
+        int ret;
+
+        sdio_claim_host(card->func);
+        ret = mwifiex_dnld_fw(adapter, fw);
+        sdio_release_host(card->func);
+
+        return ret;
+}
+
 /*
  * This function is used to initialize IO ports for the
  * chipsets supporting SDIO new mode eg SD8897.
@@ -1492,7 +1515,7 @@ rx_curr_single:
                 mwifiex_dbg(adapter, INFO, "info: RX: port: %d, rx_len: %d\n",
                             port, rx_len);
 
-                skb = mwifiex_alloc_dma_align_buf(rx_len, GFP_KERNEL | GFP_DMA);
+                skb = mwifiex_alloc_dma_align_buf(rx_len, GFP_KERNEL);
                 if (!skb) {
                         mwifiex_dbg(adapter, ERROR,
                                     "single skb allocated fail,\t"
@@ -1597,7 +1620,7 @@ static int mwifiex_process_int_status(struct mwifiex_adapter *adapter)
                 rx_len = (u16) (rx_blocks * MWIFIEX_SDIO_BLOCK_SIZE);
                 mwifiex_dbg(adapter, INFO, "info: rx_len = %d\n", rx_len);
 
-                skb = mwifiex_alloc_dma_align_buf(rx_len, GFP_KERNEL | GFP_DMA);
+                skb = mwifiex_alloc_dma_align_buf(rx_len, GFP_KERNEL);
                 if (!skb)
                         return -1;
 
@@ -2732,6 +2755,7 @@ static struct mwifiex_if_ops sdio_ops = {
         .cleanup_mpa_buf = mwifiex_cleanup_mpa_buf,
         .cmdrsp_complete = mwifiex_sdio_cmdrsp_complete,
         .event_complete = mwifiex_sdio_event_complete,
+        .dnld_fw = mwifiex_sdio_dnld_fw,
         .card_reset = mwifiex_sdio_card_reset,
         .reg_dump = mwifiex_sdio_reg_dump,
         .device_dump = mwifiex_sdio_device_dump,
diff --git a/drivers/net/wireless/marvell/mwifiex/sta_cmdresp.c b/drivers/net/wireless/marvell/mwifiex/sta_cmdresp.c
index d18c7979d723..bcfd4b743145 100644
--- a/drivers/net/wireless/marvell/mwifiex/sta_cmdresp.c
+++ b/drivers/net/wireless/marvell/mwifiex/sta_cmdresp.c
@@ -553,7 +553,8 @@ static int mwifiex_ret_802_11_deauthenticate(struct mwifiex_private *priv,
         if (!memcmp(resp->params.deauth.mac_addr,
                     &priv->curr_bss_params.bss_descriptor.mac_address,
                     sizeof(resp->params.deauth.mac_addr)))
-                mwifiex_reset_connect_state(priv, WLAN_REASON_DEAUTH_LEAVING);
+                mwifiex_reset_connect_state(priv, WLAN_REASON_DEAUTH_LEAVING,
+                                            false);
 
         return 0;
 }
@@ -566,7 +567,7 @@ static int mwifiex_ret_802_11_deauthenticate(struct mwifiex_private *priv,
 static int mwifiex_ret_802_11_ad_hoc_stop(struct mwifiex_private *priv,
                                           struct host_cmd_ds_command *resp)
 {
-        mwifiex_reset_connect_state(priv, WLAN_REASON_DEAUTH_LEAVING);
+        mwifiex_reset_connect_state(priv, WLAN_REASON_DEAUTH_LEAVING, false);
         return 0;
 }
 
diff --git a/drivers/net/wireless/marvell/mwifiex/sta_event.c b/drivers/net/wireless/marvell/mwifiex/sta_event.c
index 0104108b4ea2..0cefd40b2762 100644
--- a/drivers/net/wireless/marvell/mwifiex/sta_event.c
+++ b/drivers/net/wireless/marvell/mwifiex/sta_event.c
@@ -40,8 +40,8 @@
  *      - Erases current SSID and BSSID information
  *      - Sends a disconnect event to upper layers/applications.
  */
-void
-mwifiex_reset_connect_state(struct mwifiex_private *priv, u16 reason_code)
+void mwifiex_reset_connect_state(struct mwifiex_private *priv, u16 reason_code,
+                                 bool from_ap)
 {
         struct mwifiex_adapter *adapter = priv->adapter;
 
@@ -140,7 +140,7 @@ mwifiex_reset_connect_state(struct mwifiex_private *priv, u16 reason_code)
         if (priv->bss_mode == NL80211_IFTYPE_STATION ||
             priv->bss_mode == NL80211_IFTYPE_P2P_CLIENT) {
                 cfg80211_disconnected(priv->netdev, reason_code, NULL, 0,
-                                      false, GFP_KERNEL);
+                                      !from_ap, GFP_KERNEL);
         }
         eth_zero_addr(priv->cfg_bssid);
 
@@ -574,7 +574,7 @@ int mwifiex_process_sta_event(struct mwifiex_private *priv)
                 if (priv->media_connected) {
                         reason_code =
                                 le16_to_cpu(*(__le16 *)adapter->event_body);
-                        mwifiex_reset_connect_state(priv, reason_code);
+                        mwifiex_reset_connect_state(priv, reason_code, true);
                 }
                 break;
 
@@ -589,7 +589,7 @@ int mwifiex_process_sta_event(struct mwifiex_private *priv)
                 if (priv->media_connected) {
                         reason_code =
                                 le16_to_cpu(*(__le16 *)adapter->event_body);
-                        mwifiex_reset_connect_state(priv, reason_code);
+                        mwifiex_reset_connect_state(priv, reason_code, true);
                 }
                 break;
 
@@ -599,7 +599,7 @@ int mwifiex_process_sta_event(struct mwifiex_private *priv)
                 if (priv->media_connected) {
                         reason_code =
                                 le16_to_cpu(*(__le16 *)adapter->event_body);
-                        mwifiex_reset_connect_state(priv, reason_code);
+                        mwifiex_reset_connect_state(priv, reason_code, true);
                 }
                 break;
 
diff --git a/drivers/net/wireless/marvell/mwifiex/uap_txrx.c b/drivers/net/wireless/marvell/mwifiex/uap_txrx.c
index 666e91af59d7..bf5660eb27d3 100644
--- a/drivers/net/wireless/marvell/mwifiex/uap_txrx.c
+++ b/drivers/net/wireless/marvell/mwifiex/uap_txrx.c
@@ -272,7 +272,7 @@ int mwifiex_handle_uap_rx_forward(struct mwifiex_private *priv,
 int mwifiex_uap_recv_packet(struct mwifiex_private *priv,
                             struct sk_buff *skb)
 {
-        struct mwifiex_adapter *adapter = adapter;
+        struct mwifiex_adapter *adapter = priv->adapter;
         struct mwifiex_sta_node *src_node;
         struct ethhdr *p_ethhdr;
         struct sk_buff *skb_uap;
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/hw.c
index 8ee83b093c0d..e26a233684bb 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/hw.c
@@ -1839,20 +1839,22 @@ static void _rtl88ee_read_adapter_info(struct ieee80211_hw *hw)
         u8 hwinfo[HWSET_MAX_SIZE];
         u16 eeprom_id;
 
-        if (rtlefuse->epromtype == EEPROM_BOOT_EFUSE) {
+        switch (rtlefuse->epromtype) {
+        case EEPROM_BOOT_EFUSE:
                 rtl_efuse_shadow_map_update(hw);
+                break;
 
-                memcpy(hwinfo, &rtlefuse->efuse_map[EFUSE_INIT_MAP][0],
-                       HWSET_MAX_SIZE);
-        } else if (rtlefuse->epromtype == EEPROM_93C46) {
+        case EEPROM_93C46:
                 RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
                          "RTL819X Not boot from eeprom, check it !!");
                 return;
-        } else {
+
+        default:
                 RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
                          "boot from neither eeprom nor efuse, check it !!");
                 return;
         }
+        memcpy(hwinfo, &rtlefuse->efuse_map[EFUSE_INIT_MAP][0], HWSET_MAX_SIZE);
 
         RT_PRINT_DATA(rtlpriv, COMP_INIT, DBG_DMESG, "MAP\n",
                       hwinfo, HWSET_MAX_SIZE);
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/hw.c
index 04eb5c3f8464..58b7ac6899ef 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/hw.c
@@ -1680,21 +1680,28 @@ static void _rtl92ce_read_adapter_info(struct ieee80211_hw *hw)
         struct rtl_priv *rtlpriv = rtl_priv(hw);
         struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
         struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+        struct device *dev = &rtl_pcipriv(hw)->dev.pdev->dev;
         u16 i, usvalue;
         u8 hwinfo[HWSET_MAX_SIZE];
         u16 eeprom_id;
 
-        if (rtlefuse->epromtype == EEPROM_BOOT_EFUSE) {
+        switch (rtlefuse->epromtype) {
+        case EEPROM_BOOT_EFUSE:
                 rtl_efuse_shadow_map_update(hw);
+                break;
 
-                memcpy((void *)hwinfo,
-                       (void *)&rtlefuse->efuse_map[EFUSE_INIT_MAP][0],
-                       HWSET_MAX_SIZE);
-        } else if (rtlefuse->epromtype == EEPROM_93C46) {
+        case EEPROM_93C46:
                 RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
                          "RTL819X Not boot from eeprom, check it !!");
+                return;
+
+        default:
+                dev_warn(dev, "no efuse data\n");
+                return;
         }
 
+        memcpy(hwinfo, &rtlefuse->efuse_map[EFUSE_INIT_MAP][0], HWSET_MAX_SIZE);
+
         RT_PRINT_DATA(rtlpriv, COMP_INIT, DBG_DMESG, "MAP",
                       hwinfo, HWSET_MAX_SIZE);
 
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/hw.c
index 34ce06441d1b..ae1129f916d5 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/hw.c
@@ -351,15 +351,21 @@ static void _rtl92cu_read_adapter_info(struct ieee80211_hw *hw)
         u8 hwinfo[HWSET_MAX_SIZE] = {0};
         u16 eeprom_id;
 
-        if (rtlefuse->epromtype == EEPROM_BOOT_EFUSE) {
+        switch (rtlefuse->epromtype) {
+        case EEPROM_BOOT_EFUSE:
                 rtl_efuse_shadow_map_update(hw);
-                memcpy((void *)hwinfo,
-                       (void *)&rtlefuse->efuse_map[EFUSE_INIT_MAP][0],
-                       HWSET_MAX_SIZE);
-        } else if (rtlefuse->epromtype == EEPROM_93C46) {
+                break;
+
+        case EEPROM_93C46:
                 RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
                          "RTL819X Not boot from eeprom, check it !!\n");
+                return;
+
+        default:
+                pr_warn("rtl92cu: no efuse data\n\n");
+                return;
         }
+        memcpy(hwinfo, &rtlefuse->efuse_map[EFUSE_INIT_MAP][0], HWSET_MAX_SIZE);
         RT_PRINT_DATA(rtlpriv, COMP_INIT, DBG_LOUD, "MAP",
                       hwinfo, HWSET_MAX_SIZE);
         eeprom_id = le16_to_cpu(*((__le16 *)&hwinfo[0]));
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/hw.c
index f49b60d31450..8618c322a3f8 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/hw.c
@@ -1744,23 +1744,29 @@ static void _rtl92de_read_adapter_info(struct ieee80211_hw *hw)
         struct rtl_priv *rtlpriv = rtl_priv(hw);
         struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
         struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+        struct device *dev = &rtl_pcipriv(hw)->dev.pdev->dev;
         u16 i, usvalue;
         u8 hwinfo[HWSET_MAX_SIZE];
         u16 eeprom_id;
         unsigned long flags;
 
-        if (rtlefuse->epromtype == EEPROM_BOOT_EFUSE) {
+        switch (rtlefuse->epromtype) {
+        case EEPROM_BOOT_EFUSE:
                 spin_lock_irqsave(&globalmutex_for_power_and_efuse, flags);
                 rtl_efuse_shadow_map_update(hw);
                 _rtl92de_efuse_update_chip_version(hw);
                 spin_unlock_irqrestore(&globalmutex_for_power_and_efuse, flags);
-                memcpy((void *)hwinfo, (void *)&rtlefuse->efuse_map
-                       [EFUSE_INIT_MAP][0],
-                       HWSET_MAX_SIZE);
-        } else if (rtlefuse->epromtype == EEPROM_93C46) {
+                break;
+        case EEPROM_93C46:
                 RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
                          "RTL819X Not boot from eeprom, check it !!\n");
+                return;
+        default:
+                dev_warn(dev, "no efuse data\n");
+                return;
         }
+
+        memcpy(hwinfo, &rtlefuse->efuse_map[EFUSE_INIT_MAP][0], HWSET_MAX_SIZE);
         RT_PRINT_DATA(rtlpriv, COMP_INIT, DBG_DMESG, "MAP",
                       hwinfo, HWSET_MAX_SIZE);
 
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.c
index 9fd3f1b6e4a8..28c260dd11ea 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.c
@@ -2102,20 +2102,22 @@ static void _rtl92ee_read_adapter_info(struct ieee80211_hw *hw)
         u8 hwinfo[HWSET_MAX_SIZE];
         u16 eeprom_id;
 
-        if (rtlefuse->epromtype == EEPROM_BOOT_EFUSE) {
+        switch (rtlefuse->epromtype) {
+        case EEPROM_BOOT_EFUSE:
                 rtl_efuse_shadow_map_update(hw);
+                break;
 
-                memcpy(hwinfo, &rtlefuse->efuse_map[EFUSE_INIT_MAP][0],
-                       HWSET_MAX_SIZE);
-        } else if (rtlefuse->epromtype == EEPROM_93C46) {
+        case EEPROM_93C46:
                 RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
                          "RTL819X Not boot from eeprom, check it !!");
                 return;
-        } else {
+
+        default:
                 RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
                          "boot from neither eeprom nor efuse, check it !!");
                 return;
         }
+        memcpy(hwinfo, &rtlefuse->efuse_map[EFUSE_INIT_MAP][0], HWSET_MAX_SIZE);
 
         RT_PRINT_DATA(rtlpriv, COMP_INIT, DBG_DMESG, "MAP\n",
                       hwinfo, HWSET_MAX_SIZE);
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/phy.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/phy.c
index 018340aedf09..c2bf8d1a7af3 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/phy.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/phy.c
@@ -2414,19 +2414,10 @@ static void _rtl92ee_phy_reload_mac_registers(struct ieee80211_hw *hw,
 static void _rtl92ee_phy_path_adda_on(struct ieee80211_hw *hw, u32 *addareg,
                                       bool is_patha_on, bool is2t)
 {
-        u32 pathon;
         u32 i;
 
-        pathon = is_patha_on ? 0x0fc01616 : 0x0fc01616;
-        if (!is2t) {
-                pathon = 0x0fc01616;
-                rtl_set_bbreg(hw, addareg[0], MASKDWORD, 0x0fc01616);
-        } else {
-                rtl_set_bbreg(hw, addareg[0], MASKDWORD, pathon);
-        }
-
-        for (i = 1; i < IQK_ADDA_REG_NUM; i++)
-                rtl_set_bbreg(hw, addareg[i], MASKDWORD, pathon);
+        for (i = 0; i < IQK_ADDA_REG_NUM; i++)
+                rtl_set_bbreg(hw, addareg[i], MASKDWORD, 0x0fc01616);
 }
 
 static void _rtl92ee_phy_mac_setting_calibration(struct ieee80211_hw *hw,
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/hw.c
index 12b0978ba4fa..442f2b68ee58 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/hw.c
@@ -1673,23 +1673,31 @@ static void _rtl92se_read_adapter_info(struct ieee80211_hw *hw)
         struct rtl_priv *rtlpriv = rtl_priv(hw);
         struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
         struct rtl_phy *rtlphy = &(rtlpriv->phy);
+        struct device *dev = &rtl_pcipriv(hw)->dev.pdev->dev;
         u16 i, usvalue;
         u16     eeprom_id;
         u8 tempval;
         u8 hwinfo[HWSET_MAX_SIZE_92S];
         u8 rf_path, index;
 
-        if (rtlefuse->epromtype == EEPROM_93C46) {
+        switch (rtlefuse->epromtype) {
+        case EEPROM_BOOT_EFUSE:
+                rtl_efuse_shadow_map_update(hw);
+                break;
+
+        case EEPROM_93C46:
                 RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
                          "RTL819X Not boot from eeprom, check it !!\n");
-        } else if (rtlefuse->epromtype == EEPROM_BOOT_EFUSE) {
-                rtl_efuse_shadow_map_update(hw);
+                return;
 
-                memcpy((void *)hwinfo, (void *)
-                        &rtlefuse->efuse_map[EFUSE_INIT_MAP][0],
-                        HWSET_MAX_SIZE_92S);
+        default:
+                dev_warn(dev, "no efuse data\n");
+                return;
         }
 
+        memcpy(hwinfo, &rtlefuse->efuse_map[EFUSE_INIT_MAP][0],
+               HWSET_MAX_SIZE_92S);
+
         RT_PRINT_DATA(rtlpriv, COMP_INIT, DBG_DMESG, "MAP",
                       hwinfo, HWSET_MAX_SIZE_92S);
 
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/hw.c
index a4b7eac6856f..57a1ba8822b1 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/hw.c
@@ -1630,6 +1630,7 @@ static void _rtl8723e_read_adapter_info(struct ieee80211_hw *hw,
         struct rtl_priv *rtlpriv = rtl_priv(hw);
         struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
         struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+        struct device *dev = &rtl_pcipriv(hw)->dev.pdev->dev;
         u16 i, usvalue;
         u8 hwinfo[HWSET_MAX_SIZE];
         u16 eeprom_id;
@@ -1638,15 +1639,19 @@ static void _rtl8723e_read_adapter_info(struct ieee80211_hw *hw,
                 /* need add */
                 return;
         }
-        if (rtlefuse->epromtype == EEPROM_BOOT_EFUSE) {
+        switch (rtlefuse->epromtype) {
+        case EEPROM_BOOT_EFUSE:
                 rtl_efuse_shadow_map_update(hw);
 
-                memcpy(hwinfo, &rtlefuse->efuse_map[EFUSE_INIT_MAP][0],
-                       HWSET_MAX_SIZE);
-        } else if (rtlefuse->epromtype == EEPROM_93C46) {
+        case EEPROM_93C46:
                 RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
                          "RTL819X Not boot from eeprom, check it !!");
+                return;
+
+        default:
+                dev_warn(dev, "no efuse data\n");
         }
+        memcpy(hwinfo, &rtlefuse->efuse_map[EFUSE_INIT_MAP][0], HWSET_MAX_SIZE);
 
         RT_PRINT_DATA(rtlpriv, COMP_INIT, DBG_DMESG, "MAP\n",
                       hwinfo, HWSET_MAX_SIZE);
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/hw.c
index 5a3df9198ddf..08288ac9020a 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/hw.c
@@ -2026,6 +2026,7 @@ static void _rtl8723be_read_adapter_info(struct ieee80211_hw *hw,
         struct rtl_priv *rtlpriv = rtl_priv(hw);
         struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
         struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+        struct device *dev = &rtl_pcipriv(hw)->dev.pdev->dev;
         u16 i, usvalue;
         u8 hwinfo[HWSET_MAX_SIZE];
         u16 eeprom_id;
@@ -2055,15 +2056,22 @@ static void _rtl8723be_read_adapter_info(struct ieee80211_hw *hw,
                 /* needs to be added */
                 return;
         }
-        if (rtlefuse->epromtype == EEPROM_BOOT_EFUSE) {
+
+        switch (rtlefuse->epromtype) {
+        case EEPROM_BOOT_EFUSE:
                 rtl_efuse_shadow_map_update(hw);
+                break;
 
-                memcpy(hwinfo, &rtlefuse->efuse_map[EFUSE_INIT_MAP][0],
-                       HWSET_MAX_SIZE);
-        } else if (rtlefuse->epromtype == EEPROM_93C46) {
+        case EEPROM_93C46:
                 RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
                          "RTL819X Not boot from eeprom, check it !!");
+                return;
+
+        default:
+                dev_warn(dev, "no efuse data\n");
+                return;
         }
+        memcpy(hwinfo, &rtlefuse->efuse_map[EFUSE_INIT_MAP][0], HWSET_MAX_SIZE);
         RT_PRINT_DATA(rtlpriv, COMP_INIT, DBG_DMESG, ("MAP\n"),
                       hwinfo, HWSET_MAX_SIZE);
 
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/phy.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/phy.c
index 445f681d08c0..c5ca9dfb445c 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/phy.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/phy.c
@@ -1019,7 +1019,7 @@ static u8 _rtl8723be_get_txpower_index(struct ieee80211_hw *hw, u8 path,
         struct rtl_priv *rtlpriv = rtl_priv(hw);
         struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
         u8 index = (channel - 1);
-        u8 txpower;
+        u8 txpower = 0;
         u8 power_diff_byrate = 0;
 
         if (channel > 14 || channel < 1) {
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/hw.c
index 71e4dd9965bb..b9436df9e1ec 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/hw.c
@@ -3101,6 +3101,7 @@ static void _rtl8821ae_read_adapter_info(struct ieee80211_hw *hw, bool b_pseudo_
         struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
         struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
         struct rtl_pci_priv *pcipriv = rtl_pcipriv(hw);
+        struct device *dev = &rtl_pcipriv(hw)->dev.pdev->dev;
         u16 i, usvalue;
         u8 hwinfo[HWSET_MAX_SIZE];
         u16 eeprom_id;
@@ -3109,14 +3110,20 @@ static void _rtl8821ae_read_adapter_info(struct ieee80211_hw *hw, bool b_pseudo_
                 ;/* need add */
         }
 
-        if (rtlefuse->epromtype == EEPROM_BOOT_EFUSE) {
+        switch (rtlefuse->epromtype) {
+        case EEPROM_BOOT_EFUSE:
                 rtl_efuse_shadow_map_update(hw);
-                memcpy(hwinfo, &rtlefuse->efuse_map[EFUSE_INIT_MAP][0],
-                       HWSET_MAX_SIZE);
-        } else if (rtlefuse->epromtype == EEPROM_93C46) {
+                break;
+
+        case EEPROM_93C46:
                 RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
                          "RTL819X Not boot from eeprom, check it !!");
+                return;
+
+        default:
+                dev_warn(dev, "no efuse data\n");
         }
+        memcpy(hwinfo, &rtlefuse->efuse_map[EFUSE_INIT_MAP][0], HWSET_MAX_SIZE);
 
         RT_PRINT_DATA(rtlpriv, COMP_INIT, DBG_DMESG, "MAP\n",
                       hwinfo, HWSET_MAX_SIZE);
diff --git a/drivers/net/wireless/rsi/rsi_91x_mgmt.c b/drivers/net/wireless/rsi/rsi_91x_mgmt.c
index 40658b62d077..35c14cc3f0d2 100644
--- a/drivers/net/wireless/rsi/rsi_91x_mgmt.c
+++ b/drivers/net/wireless/rsi/rsi_91x_mgmt.c
@@ -398,7 +398,7 @@ static int rsi_mgmt_pkt_to_core(struct rsi_common *common,
                         return -ENOLINK;
 
                 msg_len -= pad_bytes;
-                if ((msg_len <= 0) || (!msg)) {
+                if (msg_len <= 0) {
                         rsi_dbg(MGMT_RX_ZONE,
                                 "%s: Invalid rx msg of len = %d\n",
                                 __func__, msg_len);
diff --git a/drivers/net/wireless/wl3501_cs.c b/drivers/net/wireless/wl3501_cs.c
index 13fd734b61ec..82d94f83b6b4 100644
--- a/drivers/net/wireless/wl3501_cs.c
+++ b/drivers/net/wireless/wl3501_cs.c
@@ -378,8 +378,7 @@ static int wl3501_esbq_exec(struct wl3501_card *this, void *sig, int sig_size)
         return rc;
 }
 
-static int wl3501_get_mib_value(struct wl3501_card *this, u8 index,
-                                void *bf, int size)
+static int wl3501_request_mib(struct wl3501_card *this, u8 index, void *bf)
 {
         struct wl3501_get_req sig = {
                 .sig_id     = WL3501_SIG_GET_REQ,
@@ -395,20 +394,32 @@ static int wl3501_get_mib_value(struct wl3501_card *this, u8 index,
                         wl3501_set_to_wla(this, ptr, &sig, sizeof(sig));
                         wl3501_esbq_req(this, &ptr);
                         this->sig_get_confirm.mib_status = 255;
-                        spin_unlock_irqrestore(&this->lock, flags);
-                        rc = wait_event_interruptible(this->wait,
-                                this->sig_get_confirm.mib_status != 255);
-                        if (!rc)
-                                memcpy(bf, this->sig_get_confirm.mib_value,
-                                       size);
-                        goto out;
+                        rc = 0;
                 }
         }
         spin_unlock_irqrestore(&this->lock, flags);
-out:
+
         return rc;
 }
 
+static int wl3501_get_mib_value(struct wl3501_card *this, u8 index,
+                                void *bf, int size)
+{
+        int rc;
+
+        rc = wl3501_request_mib(this, index, bf);
+        if (rc)
+                return rc;
+
+        rc = wait_event_interruptible(this->wait,
+                this->sig_get_confirm.mib_status != 255);
+        if (rc)
+                return rc;
+
+        memcpy(bf, this->sig_get_confirm.mib_value, size);
+        return 0;
+}
+
 static int wl3501_pwr_mgmt(struct wl3501_card *this, int suspend)
 {
         struct wl3501_pwr_mgmt_req sig = {
diff --git a/drivers/of/Kconfig b/drivers/of/Kconfig
index b3bec3aaa45d..bc07ad30c9bf 100644
--- a/drivers/of/Kconfig
+++ b/drivers/of/Kconfig
@@ -74,6 +74,7 @@ config OF_NET
 config OF_MDIO
         def_tristate PHYLIB
         depends on PHYLIB
+        select FIXED_PHY
         help
           OpenFirmware MDIO bus (Ethernet PHY) accessors
 
diff --git a/drivers/of/of_mdio.c b/drivers/of/of_mdio.c
index e051e1b57609..e2b50bc12f23 100644
--- a/drivers/of/of_mdio.c
+++ b/drivers/of/of_mdio.c
@@ -361,7 +361,6 @@ struct phy_device *of_phy_attach(struct net_device *dev,
 }
 EXPORT_SYMBOL(of_phy_attach);
 
-#if defined(CONFIG_FIXED_PHY)
 /*
  * of_phy_is_fixed_link() and of_phy_register_fixed_link() must
  * support two DT bindings:
@@ -451,4 +450,3 @@ int of_phy_register_fixed_link(struct device_node *np)
         return -ENODEV;
 }
 EXPORT_SYMBOL(of_phy_register_fixed_link);
-#endif
diff --git a/drivers/phy/Kconfig b/drivers/phy/Kconfig
index b869b98835f4..01fb93b4b1e4 100644
--- a/drivers/phy/Kconfig
+++ b/drivers/phy/Kconfig
@@ -434,4 +434,12 @@ config PHY_CYGNUS_PCIE
 
 source "drivers/phy/tegra/Kconfig"
 
+config PHY_NS2_PCIE
+        tristate "Broadcom Northstar2 PCIe PHY driver"
+        depends on OF && MDIO_BUS_MUX_BCM_IPROC
+        select GENERIC_PHY
+        default ARCH_BCM_IPROC
+        help
+          Enable this to support the Broadcom Northstar2 PCIe PHY.
+          If unsure, say N.
 endmenu
diff --git a/drivers/phy/Makefile b/drivers/phy/Makefile
index 9c3e73ccabc4..7aea094da3e2 100644
--- a/drivers/phy/Makefile
+++ b/drivers/phy/Makefile
@@ -53,5 +53,5 @@ obj-$(CONFIG_PHY_TUSB1210)		+= phy-tusb1210.o
 obj-$(CONFIG_PHY_BRCM_SATA)             += phy-brcm-sata.o
 obj-$(CONFIG_PHY_PISTACHIO_USB)         += phy-pistachio-usb.o
 obj-$(CONFIG_PHY_CYGNUS_PCIE)           += phy-bcm-cygnus-pcie.o
-
 obj-$(CONFIG_ARCH_TEGRA) += tegra/
+obj-$(CONFIG_PHY_NS2_PCIE)              += phy-bcm-ns2-pcie.o
diff --git a/drivers/phy/phy-bcm-ns2-pcie.c b/drivers/phy/phy-bcm-ns2-pcie.c
new file mode 100644
index 000000000000..9513f7ab1eaa
--- /dev/null
+++ b/drivers/phy/phy-bcm-ns2-pcie.c
@@ -0,0 +1,115 @@
+/*
+ * Copyright (C) 2016 Broadcom
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/of_mdio.h>
+#include <linux/mdio.h>
+#include <linux/phy.h>
+#include <linux/phy/phy.h>
+
+struct ns2_pci_phy {
+        struct mdio_device *mdiodev;
+        struct phy *phy;
+};
+
+#define BLK_ADDR_REG_OFFSET     0x1f
+#define PLL_AFE1_100MHZ_BLK     0x2100
+#define PLL_CLK_AMP_OFFSET      0x03
+#define PLL_CLK_AMP_2P05V       0x2b18
+
+static int ns2_pci_phy_init(struct phy *p)
+{
+        struct ns2_pci_phy *phy = phy_get_drvdata(p);
+        int rc;
+
+        /* select the AFE 100MHz block page */
+        rc = mdiobus_write(phy->mdiodev->bus, phy->mdiodev->addr,
+                           BLK_ADDR_REG_OFFSET, PLL_AFE1_100MHZ_BLK);
+        if (rc)
+                goto err;
+
+        /* set the 100 MHz reference clock amplitude to 2.05 v */
+        rc = mdiobus_write(phy->mdiodev->bus, phy->mdiodev->addr,
+                           PLL_CLK_AMP_OFFSET, PLL_CLK_AMP_2P05V);
+        if (rc)
+                goto err;
+
+        return 0;
+
+err:
+        dev_err(&phy->mdiodev->dev, "Error %d writing to phy\n", rc);
+        return rc;
+}
+
+static struct phy_ops ns2_pci_phy_ops = {
+        .init = ns2_pci_phy_init,
+};
+
+static int ns2_pci_phy_probe(struct mdio_device *mdiodev)
+{
+        struct device *dev = &mdiodev->dev;
+        struct phy_provider *provider;
+        struct ns2_pci_phy *p;
+        struct phy *phy;
+
+        phy = devm_phy_create(dev, dev->of_node, &ns2_pci_phy_ops);
+        if (IS_ERR(phy)) {
+                dev_err(dev, "failed to create Phy\n");
+                return PTR_ERR(phy);
+        }
+
+        p = devm_kmalloc(dev, sizeof(struct ns2_pci_phy),
+                         GFP_KERNEL);
+        if (!p)
+                return -ENOMEM;
+
+        p->mdiodev = mdiodev;
+        dev_set_drvdata(dev, p);
+
+        p->phy = phy;
+        phy_set_drvdata(phy, p);
+
+        provider = devm_of_phy_provider_register(&phy->dev,
+                                                 of_phy_simple_xlate);
+        if (IS_ERR(provider)) {
+                dev_err(dev, "failed to register Phy provider\n");
+                return PTR_ERR(provider);
+        }
+
+        dev_info(dev, "%s PHY registered\n", dev_name(dev));
+
+        return 0;
+}
+
+static const struct of_device_id ns2_pci_phy_of_match[] = {
+        { .compatible = "brcm,ns2-pcie-phy", },
+        { /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, ns2_pci_phy_of_match);
+
+static struct mdio_driver ns2_pci_phy_driver = {
+        .mdiodrv = {
+                .driver = {
+                        .name = "phy-bcm-ns2-pci",
+                        .of_match_table = ns2_pci_phy_of_match,
+                },
+        },
+        .probe = ns2_pci_phy_probe,
+};
+mdio_module_driver(ns2_pci_phy_driver);
+
+MODULE_AUTHOR("Broadcom");
+MODULE_DESCRIPTION("Broadcom Northstar2 PCI Phy driver");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:phy-bcm-ns2-pci");
diff --git a/drivers/s390/net/qeth_core.h b/drivers/s390/net/qeth_core.h
index ec2e014e885c..bf40063de202 100644
--- a/drivers/s390/net/qeth_core.h
+++ b/drivers/s390/net/qeth_core.h
@@ -19,6 +19,7 @@
 #include <linux/seq_file.h>
 #include <linux/ethtool.h>
 #include <linux/hashtable.h>
+#include <linux/ip.h>
 
 #include <net/ipv6.h>
 #include <net/if_inet6.h>
@@ -144,6 +145,7 @@ struct qeth_perf_stats {
         unsigned int sg_alloc_page_rx;
         unsigned int tx_csum;
         unsigned int tx_lin;
+        unsigned int tx_linfail;
 };
 
 /* Routing stuff */
@@ -559,7 +561,6 @@ enum qeth_ip_types {
         QETH_IP_TYPE_NORMAL,
         QETH_IP_TYPE_VIPA,
         QETH_IP_TYPE_RXIP,
-        QETH_IP_TYPE_DEL_ALL_MC,
 };
 
 enum qeth_cmd_buffer_state {
@@ -740,17 +741,10 @@ struct qeth_vlan_vid {
         unsigned short vid;
 };
 
-enum qeth_mac_disposition {
-        QETH_DISP_MAC_DELETE = 0,
-        QETH_DISP_MAC_DO_NOTHING = 1,
-        QETH_DISP_MAC_ADD = 2,
-};
-
-struct qeth_mac {
-        u8 mac_addr[OSA_ADDR_LEN];
-        u8 is_uc:1;
-        u8 disp_flag:2;
-        struct hlist_node hnode;
+enum qeth_addr_disposition {
+        QETH_DISP_ADDR_DELETE = 0,
+        QETH_DISP_ADDR_DO_NOTHING = 1,
+        QETH_DISP_ADDR_ADD = 2,
 };
 
 struct qeth_rx {
@@ -798,6 +792,8 @@ struct qeth_card {
         unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
         struct list_head vid_list;
         DECLARE_HASHTABLE(mac_htable, 4);
+        DECLARE_HASHTABLE(ip_htable, 4);
+        DECLARE_HASHTABLE(ip_mc_htable, 4);
         struct work_struct kernel_thread_starter;
         spinlock_t thread_mask_lock;
         unsigned long thread_start_mask;
@@ -805,8 +801,6 @@ struct qeth_card {
         unsigned long thread_running_mask;
         struct task_struct *recovery_task;
         spinlock_t ip_lock;
-        struct list_head ip_list;
-        struct list_head *ip_tbd_list;
         struct qeth_ipato ipato;
         struct list_head cmd_waiter_list;
         /* QDIO buffer handling */
@@ -844,6 +838,19 @@ struct qeth_trap_id {
 /*some helper functions*/
 #define QETH_CARD_IFNAME(card) (((card)->dev)? (card)->dev->name : "")
 
+/**
+ * qeth_get_elements_for_range() -      find number of SBALEs to cover range.
+ * @start:                              Start of the address range.
+ * @end:                                Address after the end of the range.
+ *
+ * Returns the number of pages, and thus QDIO buffer elements, needed to cover
+ * the specified address range.
+ */
+static inline int qeth_get_elements_for_range(addr_t start, addr_t end)
+{
+        return PFN_UP(end - 1) - PFN_DOWN(start);
+}
+
 static inline int qeth_get_micros(void)
 {
         return (int) (get_tod_clock() >> 12);
@@ -865,6 +872,11 @@ static inline int qeth_get_ip_version(struct sk_buff *skb)
         }
 }
 
+static inline int qeth_get_ip_protocol(struct sk_buff *skb)
+{
+        return ip_hdr(skb)->protocol;
+}
+
 static inline void qeth_put_buffer_pool_entry(struct qeth_card *card,
                 struct qeth_buffer_pool_entry *entry)
 {
@@ -981,12 +993,13 @@ int qeth_send_setassparms(struct qeth_card *, struct qeth_cmd_buffer *, __u16,
                           int (*reply_cb)(struct qeth_card *,
                                           struct qeth_reply *, unsigned long),
                           void *);
+int qeth_setassparms_cb(struct qeth_card *, struct qeth_reply *, unsigned long);
 struct qeth_cmd_buffer *qeth_get_setassparms_cmd(struct qeth_card *,
                                                  enum qeth_ipa_funcs,
                                                  __u16, __u16,
                                                  enum qeth_prot_versions);
-int qeth_start_ipa_tx_checksum(struct qeth_card *);
-int qeth_set_rx_csum(struct qeth_card *, int);
+int qeth_set_features(struct net_device *, netdev_features_t);
+netdev_features_t qeth_fix_features(struct net_device *, netdev_features_t);
 
 /* exports for OSN */
 int qeth_osn_assist(struct net_device *, void *, int);
diff --git a/drivers/s390/net/qeth_core_main.c b/drivers/s390/net/qeth_core_main.c
index b7b74776e2ff..7dba6c8537a1 100644
--- a/drivers/s390/net/qeth_core_main.c
+++ b/drivers/s390/net/qeth_core_main.c
@@ -1464,8 +1464,6 @@ static int qeth_setup_card(struct qeth_card *card)
         card->thread_allowed_mask = 0;
         card->thread_running_mask = 0;
         INIT_WORK(&card->kernel_thread_starter, qeth_start_kernel_thread);
-        INIT_LIST_HEAD(&card->ip_list);
-        INIT_LIST_HEAD(card->ip_tbd_list);
         INIT_LIST_HEAD(&card->cmd_waiter_list);
         init_waitqueue_head(&card->wait_q);
         /* initial options */
@@ -1500,11 +1498,6 @@ static struct qeth_card *qeth_alloc_card(void)
         if (!card)
                 goto out;
         QETH_DBF_HEX(SETUP, 2, &card, sizeof(void *));
-        card->ip_tbd_list = kzalloc(sizeof(struct list_head), GFP_KERNEL);
-        if (!card->ip_tbd_list) {
-                QETH_DBF_TEXT(SETUP, 0, "iptbdnom");
-                goto out_card;
-        }
         if (qeth_setup_channel(&card->read))
                 goto out_ip;
         if (qeth_setup_channel(&card->write))
@@ -1517,8 +1510,6 @@ static struct qeth_card *qeth_alloc_card(void)
 out_channel:
         qeth_clean_channel(&card->read);
 out_ip:
-        kfree(card->ip_tbd_list);
-out_card:
         kfree(card);
 out:
         return NULL;
@@ -3757,6 +3748,14 @@ void qeth_qdio_output_handler(struct ccw_device *ccwdev,
 }
 EXPORT_SYMBOL_GPL(qeth_qdio_output_handler);
 
+/* We cannot use outbound queue 3 for unicast packets on HiperSockets */
+static inline int qeth_cut_iqd_prio(struct qeth_card *card, int queue_num)
+{
+        if ((card->info.type == QETH_CARD_TYPE_IQD) && (queue_num == 3))
+                return 2;
+        return queue_num;
+}
+
 /**
  * Note: Function assumes that we have 4 outbound queues.
  */
@@ -3784,9 +3783,9 @@ int qeth_get_priority_queue(struct qeth_card *card, struct sk_buff *skb,
                         return card->qdio.default_out_queue;
                 }
                 if (card->qdio.do_prio_queueing == QETH_PRIO_Q_ING_PREC)
-                        return ~tos >> 6 & 3;
+                        return qeth_cut_iqd_prio(card, ~tos >> 6 & 3);
                 if (tos & IPTOS_MINCOST)
-                        return 3;
+                        return qeth_cut_iqd_prio(card, 3);
                 if (tos & IPTOS_RELIABILITY)
                         return 2;
                 if (tos & IPTOS_THROUGHPUT)
@@ -3797,11 +3796,12 @@ int qeth_get_priority_queue(struct qeth_card *card, struct sk_buff *skb,
         case QETH_PRIO_Q_ING_SKB:
                 if (skb->priority > 5)
                         return 0;
-                return ~skb->priority >> 1 & 3;
+                return qeth_cut_iqd_prio(card, ~skb->priority >> 1 & 3);
         case QETH_PRIO_Q_ING_VLAN:
                 tci = &((struct ethhdr *)skb->data)->h_proto;
                 if (*tci == ETH_P_8021Q)
-                        return ~*(tci + 1) >> (VLAN_PRIO_SHIFT + 1) & 3;
+                        return qeth_cut_iqd_prio(card, ~*(tci + 1) >>
+                        (VLAN_PRIO_SHIFT + 1) & 3);
                 break;
         default:
                 break;
@@ -3810,41 +3810,54 @@ int qeth_get_priority_queue(struct qeth_card *card, struct sk_buff *skb,
 }
 EXPORT_SYMBOL_GPL(qeth_get_priority_queue);
 
+/**
+ * qeth_get_elements_for_frags() -      find number of SBALEs for skb frags.
+ * @skb:                                SKB address
+ *
+ * Returns the number of pages, and thus QDIO buffer elements, needed to cover
+ * fragmented part of the SKB. Returns zero for linear SKB.
+ */
 int qeth_get_elements_for_frags(struct sk_buff *skb)
 {
-        int cnt, length, e, elements = 0;
-        struct skb_frag_struct *frag;
-        char *data;
+        int cnt, elements = 0;
 
         for (cnt = 0; cnt < skb_shinfo(skb)->nr_frags; cnt++) {
-                frag = &skb_shinfo(skb)->frags[cnt];
-                data = (char *)page_to_phys(skb_frag_page(frag)) +
-                        frag->page_offset;
-                length = frag->size;
-                e = PFN_UP((unsigned long)data + length - 1) -
-                        PFN_DOWN((unsigned long)data);
-                elements += e;
+                struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[cnt];
+
+                elements += qeth_get_elements_for_range(
+                        (addr_t)skb_frag_address(frag),
+                        (addr_t)skb_frag_address(frag) + skb_frag_size(frag));
         }
         return elements;
 }
 EXPORT_SYMBOL_GPL(qeth_get_elements_for_frags);
 
+/**
+ * qeth_get_elements_no() -     find number of SBALEs for skb data, inc. frags.
+ * @card:                       qeth card structure, to check max. elems.
+ * @skb:                        SKB address
+ * @extra_elems:                extra elems needed, to check against max.
+ *
+ * Returns the number of pages, and thus QDIO buffer elements, needed to cover
+ * skb data, including linear part and fragments. Checks if the result plus
+ * extra_elems fits under the limit for the card. Returns 0 if it does not.
+ * Note: extra_elems is not included in the returned result.
+ */
 int qeth_get_elements_no(struct qeth_card *card,
-                     struct sk_buff *skb, int elems)
+                     struct sk_buff *skb, int extra_elems)
 {
-        int dlen = skb->len - skb->data_len;
-        int elements_needed = PFN_UP((unsigned long)skb->data + dlen - 1) -
-                PFN_DOWN((unsigned long)skb->data);
-
-        elements_needed += qeth_get_elements_for_frags(skb);
+        int elements = qeth_get_elements_for_range(
+                                (addr_t)skb->data,
+                                (addr_t)skb->data + skb_headlen(skb)) +
+                        qeth_get_elements_for_frags(skb);
 
-        if ((elements_needed + elems) > QETH_MAX_BUFFER_ELEMENTS(card)) {
+        if ((elements + extra_elems) > QETH_MAX_BUFFER_ELEMENTS(card)) {
                 QETH_DBF_MESSAGE(2, "Invalid size of IP packet "
                         "(Number=%d / Length=%d). Discarded.\n",
-                        (elements_needed+elems), skb->len);
+                        elements + extra_elems, skb->len);
                 return 0;
         }
-        return elements_needed;
+        return elements;
 }
 EXPORT_SYMBOL_GPL(qeth_get_elements_no);
 
@@ -3859,7 +3872,7 @@ int qeth_hdr_chk_and_bounce(struct sk_buff *skb, struct qeth_hdr **hdr, int len)
                 rest = len - inpage;
                 if (rest > hroom)
                         return 1;
-                memmove(skb->data - rest, skb->data, skb->len - skb->data_len);
+                memmove(skb->data - rest, skb->data, skb_headlen(skb));
                 skb->data -= rest;
                 skb->tail -= rest;
                 *hdr = (struct qeth_hdr *)skb->data;
@@ -3873,7 +3886,7 @@ static inline void __qeth_fill_buffer(struct sk_buff *skb,
         struct qdio_buffer *buffer, int is_tso, int *next_element_to_fill,
         int offset)
 {
-        int length = skb->len - skb->data_len;
+        int length = skb_headlen(skb);
         int length_here;
         int element;
         char *data;
@@ -4967,7 +4980,6 @@ static void qeth_core_free_card(struct qeth_card *card)
         qeth_clean_channel(&card->write);
         if (card->dev)
                 free_netdev(card->dev);
-        kfree(card->ip_tbd_list);
         qeth_free_qdio_buffers(card);
         unregister_service_level(&card->qeth_service_level);
         kfree(card);
@@ -5265,8 +5277,8 @@ no_mem:
 }
 EXPORT_SYMBOL_GPL(qeth_core_get_next_skb);
 
-static int qeth_setassparms_cb(struct qeth_card *card,
-                               struct qeth_reply *reply, unsigned long data)
+int qeth_setassparms_cb(struct qeth_card *card,
+                        struct qeth_reply *reply, unsigned long data)
 {
         struct qeth_ipa_cmd *cmd;
 
@@ -5294,6 +5306,7 @@ static int qeth_setassparms_cb(struct qeth_card *card,
 
         return 0;
 }
+EXPORT_SYMBOL_GPL(qeth_setassparms_cb);
 
 struct qeth_cmd_buffer *qeth_get_setassparms_cmd(struct qeth_card *card,
                                                  enum qeth_ipa_funcs ipa_func,
@@ -5788,6 +5801,7 @@ static struct {
         {"tx do_QDIO count"},
         {"tx csum"},
         {"tx lin"},
+        {"tx linfail"},
         {"cq handler count"},
         {"cq handler time"}
 };
@@ -5848,8 +5862,9 @@ void qeth_core_get_ethtool_stats(struct net_device *dev,
         data[32] = card->perf_stats.outbound_do_qdio_cnt;
         data[33] = card->perf_stats.tx_csum;
         data[34] = card->perf_stats.tx_lin;
-        data[35] = card->perf_stats.cq_cnt;
-        data[36] = card->perf_stats.cq_time;
+        data[35] = card->perf_stats.tx_linfail;
+        data[36] = card->perf_stats.cq_cnt;
+        data[37] = card->perf_stats.cq_time;
 }
 EXPORT_SYMBOL_GPL(qeth_core_get_ethtool_stats);
 
@@ -6048,74 +6063,136 @@ int qeth_core_ethtool_get_settings(struct net_device *netdev,
 }
 EXPORT_SYMBOL_GPL(qeth_core_ethtool_get_settings);
 
-static int qeth_send_checksum_command(struct qeth_card *card)
+static int qeth_send_checksum_on(struct qeth_card *card, int cstype)
 {
+        long rxtx_arg;
         int rc;
 
-        rc = qeth_send_simple_setassparms(card, IPA_INBOUND_CHECKSUM,
-                                          IPA_CMD_ASS_START, 0);
+        rc = qeth_send_simple_setassparms(card, cstype, IPA_CMD_ASS_START, 0);
         if (rc) {
-                dev_warn(&card->gdev->dev, "Starting HW checksumming for %s "
-                        "failed, using SW checksumming\n",
-                        QETH_CARD_IFNAME(card));
+                dev_warn(&card->gdev->dev,
+                         "Starting HW checksumming for %s failed, using SW checksumming\n",
+                         QETH_CARD_IFNAME(card));
                 return rc;
         }
-        rc = qeth_send_simple_setassparms(card, IPA_INBOUND_CHECKSUM,
-                                          IPA_CMD_ASS_ENABLE,
-                                          card->info.csum_mask);
+        rxtx_arg = (cstype == IPA_OUTBOUND_CHECKSUM) ? card->info.tx_csum_mask
+                                                     : card->info.csum_mask;
+        rc = qeth_send_simple_setassparms(card, cstype, IPA_CMD_ASS_ENABLE,
+                                          rxtx_arg);
         if (rc) {
-                dev_warn(&card->gdev->dev, "Enabling HW checksumming for %s "
-                        "failed, using SW checksumming\n",
-                        QETH_CARD_IFNAME(card));
+                dev_warn(&card->gdev->dev,
+                         "Enabling HW checksumming for %s failed, using SW checksumming\n",
+                         QETH_CARD_IFNAME(card));
                 return rc;
         }
+
+        dev_info(&card->gdev->dev, "HW Checksumming (%sbound) enabled\n",
+                 cstype == IPA_INBOUND_CHECKSUM ? "in" : "out");
         return 0;
 }
 
-int qeth_set_rx_csum(struct qeth_card *card, int on)
+static int qeth_set_ipa_csum(struct qeth_card *card, int on, int cstype)
 {
         int rc;
 
         if (on) {
-                rc = qeth_send_checksum_command(card);
+                rc = qeth_send_checksum_on(card, cstype);
                 if (rc)
                         return -EIO;
-                dev_info(&card->gdev->dev,
-                        "HW Checksumming (inbound) enabled\n");
         } else {
-                rc = qeth_send_simple_setassparms(card,
-                        IPA_INBOUND_CHECKSUM, IPA_CMD_ASS_STOP, 0);
+                rc = qeth_send_simple_setassparms(card, cstype,
+                                                  IPA_CMD_ASS_STOP, 0);
                 if (rc)
                         return -EIO;
         }
         return 0;
 }
-EXPORT_SYMBOL_GPL(qeth_set_rx_csum);
 
-int qeth_start_ipa_tx_checksum(struct qeth_card *card)
+static int qeth_set_ipa_tso(struct qeth_card *card, int on)
 {
-        int rc = 0;
+        int rc;
 
-        if (!qeth_is_supported(card, IPA_OUTBOUND_CHECKSUM))
-                return rc;
-        rc = qeth_send_simple_setassparms(card, IPA_OUTBOUND_CHECKSUM,
-                                          IPA_CMD_ASS_START, 0);
-        if (rc)
-                goto err_out;
-        rc = qeth_send_simple_setassparms(card, IPA_OUTBOUND_CHECKSUM,
-                                          IPA_CMD_ASS_ENABLE,
-                                          card->info.tx_csum_mask);
-        if (rc)
-                goto err_out;
+        QETH_CARD_TEXT(card, 3, "sttso");
 
-        dev_info(&card->gdev->dev, "HW TX Checksumming enabled\n");
-        return rc;
-err_out:
-        dev_warn(&card->gdev->dev, "Enabling HW TX checksumming for %s "
-                "failed, using SW TX checksumming\n", QETH_CARD_IFNAME(card));
+        if (on) {
+                rc = qeth_send_simple_setassparms(card, IPA_OUTBOUND_TSO,
+                                                  IPA_CMD_ASS_START, 0);
+                if (rc) {
+                        dev_warn(&card->gdev->dev,
+                                 "Starting outbound TCP segmentation offload for %s failed\n",
+                                 QETH_CARD_IFNAME(card));
+                        return -EIO;
+                }
+                dev_info(&card->gdev->dev, "Outbound TSO enabled\n");
+        } else {
+                rc = qeth_send_simple_setassparms(card, IPA_OUTBOUND_TSO,
+                                                  IPA_CMD_ASS_STOP, 0);
+        }
         return rc;
 }
-EXPORT_SYMBOL_GPL(qeth_start_ipa_tx_checksum);
+
+int qeth_set_features(struct net_device *dev, netdev_features_t features)
+{
+        struct qeth_card *card = dev->ml_priv;
+        netdev_features_t changed = dev->features ^ features;
+        int rc = 0;
+
+        QETH_DBF_TEXT(SETUP, 2, "setfeat");
+        QETH_DBF_HEX(SETUP, 2, &features, sizeof(features));
+
+        if ((changed & NETIF_F_IP_CSUM)) {
+                rc = qeth_set_ipa_csum(card,
+                                       features & NETIF_F_IP_CSUM ? 1 : 0,
+                                       IPA_OUTBOUND_CHECKSUM);
+                if (rc)
+                        changed ^= NETIF_F_IP_CSUM;
+        }
+        if ((changed & NETIF_F_RXCSUM)) {
+                rc = qeth_set_ipa_csum(card,
+                                        features & NETIF_F_RXCSUM ? 1 : 0,
+                                        IPA_INBOUND_CHECKSUM);
+                if (rc)
+                        changed ^= NETIF_F_RXCSUM;
+        }
+        if ((changed & NETIF_F_TSO)) {
+                rc = qeth_set_ipa_tso(card, features & NETIF_F_TSO ? 1 : 0);
+                if (rc)
+                        changed ^= NETIF_F_TSO;
+        }
+
+        /* everything changed successfully? */
+        if ((dev->features ^ features) == changed)
+                return 0;
+        /* something went wrong. save changed features and return error */
+        dev->features ^= changed;
+        return -EIO;
+}
+EXPORT_SYMBOL_GPL(qeth_set_features);
+
+netdev_features_t qeth_fix_features(struct net_device *dev,
+                                    netdev_features_t features)
+{
+        struct qeth_card *card = dev->ml_priv;
+
+        QETH_DBF_TEXT(SETUP, 2, "fixfeat");
+        if (!qeth_is_supported(card, IPA_OUTBOUND_CHECKSUM))
+                features &= ~NETIF_F_IP_CSUM;
+        if (!qeth_is_supported(card, IPA_INBOUND_CHECKSUM))
+                features &= ~NETIF_F_RXCSUM;
+        if (!qeth_is_supported(card, IPA_OUTBOUND_TSO)) {
+                features &= ~NETIF_F_TSO;
+                dev_info(&card->gdev->dev, "Outbound TSO not supported on %s\n",
+                         QETH_CARD_IFNAME(card));
+        }
+        /* if the card isn't up, remove features that require hw changes */
+        if (card->state == CARD_STATE_DOWN ||
+            card->state == CARD_STATE_RECOVER)
+                features = features & ~(NETIF_F_IP_CSUM | NETIF_F_RXCSUM |
+                                        NETIF_F_TSO);
+        QETH_DBF_HEX(SETUP, 2, &features, sizeof(features));
+        return features;
+}
+EXPORT_SYMBOL_GPL(qeth_fix_features);
 
 static int __init qeth_core_init(void)
 {
diff --git a/drivers/s390/net/qeth_core_sys.c b/drivers/s390/net/qeth_core_sys.c
index e6e5b9671bf2..75b29fd2fcf4 100644
--- a/drivers/s390/net/qeth_core_sys.c
+++ b/drivers/s390/net/qeth_core_sys.c
@@ -243,6 +243,10 @@ static ssize_t qeth_dev_prioqing_store(struct device *dev,
                 card->qdio.do_prio_queueing = QETH_NO_PRIO_QUEUEING;
                 card->qdio.default_out_queue = 2;
         } else if (sysfs_streq(buf, "no_prio_queueing:3")) {
+                if (card->info.type == QETH_CARD_TYPE_IQD) {
+                        rc = -EPERM;
+                        goto out;
+                }
                 card->qdio.do_prio_queueing = QETH_NO_PRIO_QUEUEING;
                 card->qdio.default_out_queue = 3;
         } else if (sysfs_streq(buf, "no_prio_queueing")) {
diff --git a/drivers/s390/net/qeth_l2.h b/drivers/s390/net/qeth_l2.h
index 0767556404bd..29d9fb3890ad 100644
--- a/drivers/s390/net/qeth_l2.h
+++ b/drivers/s390/net/qeth_l2.h
@@ -12,4 +12,11 @@ int qeth_l2_create_device_attributes(struct device *);
 void qeth_l2_remove_device_attributes(struct device *);
 void qeth_l2_setup_bridgeport_attrs(struct qeth_card *card);
 
+struct qeth_mac {
+        u8 mac_addr[OSA_ADDR_LEN];
+        u8 is_uc:1;
+        u8 disp_flag:2;
+        struct hlist_node hnode;
+};
+
 #endif /* __QETH_L2_H__ */
diff --git a/drivers/s390/net/qeth_l2_main.c b/drivers/s390/net/qeth_l2_main.c
index 80b1979e8d95..9fd48de38a4c 100644
--- a/drivers/s390/net/qeth_l2_main.c
+++ b/drivers/s390/net/qeth_l2_main.c
@@ -404,38 +404,6 @@ static int qeth_l2_vlan_rx_kill_vid(struct net_device *dev,
         return rc;
 }
 
-static netdev_features_t qeth_l2_fix_features(struct net_device *dev,
-                                              netdev_features_t features)
-{
-        struct qeth_card *card = dev->ml_priv;
-
-        QETH_DBF_TEXT(SETUP, 2, "fixfeat");
-        if (!qeth_is_supported(card, IPA_OUTBOUND_CHECKSUM))
-                features &= ~NETIF_F_IP_CSUM;
-        if (!qeth_is_supported(card, IPA_INBOUND_CHECKSUM))
-                features &= ~NETIF_F_RXCSUM;
-        QETH_DBF_HEX(SETUP, 2, &features, sizeof(features));
-        return features;
-}
-
-static int qeth_l2_set_features(struct net_device *dev,
-                                netdev_features_t features)
-{
-        struct qeth_card *card = dev->ml_priv;
-        netdev_features_t changed = dev->features ^ features;
-
-        QETH_DBF_TEXT(SETUP, 2, "setfeat");
-        QETH_DBF_HEX(SETUP, 2, &features, sizeof(features));
-
-        if (card->state == CARD_STATE_DOWN ||
-            card->state == CARD_STATE_RECOVER)
-                return 0;
-
-        if (!(changed & NETIF_F_RXCSUM))
-                return 0;
-        return qeth_set_rx_csum(card, features & NETIF_F_RXCSUM ? 1 : 0);
-}
-
 static void qeth_l2_stop_card(struct qeth_card *card, int recovery_mode)
 {
         QETH_DBF_TEXT(SETUP , 2, "stopcard");
@@ -780,7 +748,7 @@ qeth_l2_add_mac(struct qeth_card *card, struct netdev_hw_addr *ha, u8 is_uc)
                         qeth_l2_mac_hash(ha->addr)) {
                 if (is_uc == mac->is_uc &&
                     !memcmp(ha->addr, mac->mac_addr, OSA_ADDR_LEN)) {
-                        mac->disp_flag = QETH_DISP_MAC_DO_NOTHING;
+                        mac->disp_flag = QETH_DISP_ADDR_DO_NOTHING;
                         return;
                 }
         }
@@ -792,7 +760,7 @@ qeth_l2_add_mac(struct qeth_card *card, struct netdev_hw_addr *ha, u8 is_uc)
 
         memcpy(mac->mac_addr, ha->addr, OSA_ADDR_LEN);
         mac->is_uc = is_uc;
-        mac->disp_flag = QETH_DISP_MAC_ADD;
+        mac->disp_flag = QETH_DISP_ADDR_ADD;
 
         hash_add(card->mac_htable, &mac->hnode,
                         qeth_l2_mac_hash(mac->mac_addr));
@@ -825,7 +793,7 @@ static void qeth_l2_set_rx_mode(struct net_device *dev)
                 qeth_l2_add_mac(card, ha, 1);
 
         hash_for_each_safe(card->mac_htable, i, tmp, mac, hnode) {
-                if (mac->disp_flag == QETH_DISP_MAC_DELETE) {
+                if (mac->disp_flag == QETH_DISP_ADDR_DELETE) {
                         if (!mac->is_uc)
                                 rc = qeth_l2_send_delgroupmac(card,
                                                 mac->mac_addr);
@@ -837,15 +805,15 @@ static void qeth_l2_set_rx_mode(struct net_device *dev)
                         hash_del(&mac->hnode);
                         kfree(mac);
 
-                } else if (mac->disp_flag == QETH_DISP_MAC_ADD) {
+                } else if (mac->disp_flag == QETH_DISP_ADDR_ADD) {
                         rc = qeth_l2_write_mac(card, mac);
                         if (rc) {
                                 hash_del(&mac->hnode);
                                 kfree(mac);
                         } else
-                                mac->disp_flag = QETH_DISP_MAC_DELETE;
+                                mac->disp_flag = QETH_DISP_ADDR_DELETE;
                 } else
-                        mac->disp_flag = QETH_DISP_MAC_DELETE;
+                        mac->disp_flag = QETH_DISP_ADDR_DELETE;
         }
 
         spin_unlock_bh(&card->mclock);
@@ -869,6 +837,7 @@ static int qeth_l2_hard_start_xmit(struct sk_buff *skb, struct net_device *dev)
         int data_offset = -1;
         int elements_needed = 0;
         int hd_len = 0;
+        int nr_frags;
 
         if (card->qdio.do_prio_queueing || (cast_type &&
                                         card->info.is_multicast_different))
@@ -892,6 +861,23 @@ static int qeth_l2_hard_start_xmit(struct sk_buff *skb, struct net_device *dev)
         }
         netif_stop_queue(dev);
 
+        /* fix hardware limitation: as long as we do not have sbal
+         * chaining we can not send long frag lists
+         */
+        if ((card->info.type != QETH_CARD_TYPE_IQD) &&
+            !qeth_get_elements_no(card, new_skb, 0)) {
+                int lin_rc = skb_linearize(new_skb);
+
+                if (card->options.performance_stats) {
+                        if (lin_rc)
+                                card->perf_stats.tx_linfail++;
+                        else
+                                card->perf_stats.tx_lin++;
+                }
+                if (lin_rc)
+                        goto tx_drop;
+        }
+
         if (card->info.type == QETH_CARD_TYPE_OSN)
                 hdr = (struct qeth_hdr *)skb->data;
         else {
@@ -943,6 +929,14 @@ static int qeth_l2_hard_start_xmit(struct sk_buff *skb, struct net_device *dev)
         if (!rc) {
                 card->stats.tx_packets++;
                 card->stats.tx_bytes += tx_bytes;
+                if (card->options.performance_stats) {
+                        nr_frags = skb_shinfo(new_skb)->nr_frags;
+                        if (nr_frags) {
+                                card->perf_stats.sg_skbs_sent++;
+                                /* nr_frags + skb->data */
+                                card->perf_stats.sg_frags_sent += nr_frags + 1;
+                        }
+                }
                 if (new_skb != skb)
                         dev_kfree_skb_any(skb);
                 rc = NETDEV_TX_OK;
@@ -1086,8 +1080,8 @@ static const struct net_device_ops qeth_l2_netdev_ops = {
         .ndo_vlan_rx_add_vid    = qeth_l2_vlan_rx_add_vid,
         .ndo_vlan_rx_kill_vid   = qeth_l2_vlan_rx_kill_vid,
         .ndo_tx_timeout         = qeth_tx_timeout,
-        .ndo_fix_features       = qeth_l2_fix_features,
-        .ndo_set_features       = qeth_l2_set_features
+        .ndo_fix_features       = qeth_fix_features,
+        .ndo_set_features       = qeth_set_features
 };
 
 static int qeth_l2_setup_netdev(struct qeth_card *card)
@@ -1118,12 +1112,25 @@ static int qeth_l2_setup_netdev(struct qeth_card *card)
                 &qeth_l2_ethtool_ops : &qeth_l2_osn_ops;
         card->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
         if (card->info.type == QETH_CARD_TYPE_OSD && !card->info.guestlan) {
-                card->dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_RXCSUM;
-                /* Turn on RX offloading per default */
-                card->dev->features |= NETIF_F_RXCSUM;
+                card->dev->hw_features = NETIF_F_SG;
+                card->dev->vlan_features = NETIF_F_SG;
+                /* OSA 3S and earlier has no RX/TX support */
+                if (qeth_is_supported(card, IPA_OUTBOUND_CHECKSUM)) {
+                        card->dev->hw_features |= NETIF_F_IP_CSUM;
+                        card->dev->vlan_features |= NETIF_F_IP_CSUM;
+                }
+                if (qeth_is_supported(card, IPA_INBOUND_CHECKSUM)) {
+                        card->dev->hw_features |= NETIF_F_RXCSUM;
+                        card->dev->vlan_features |= NETIF_F_RXCSUM;
+                }
+                /* Turn on SG per default */
+                card->dev->features |= NETIF_F_SG;
         }
         card->info.broadcast_capable = 1;
         qeth_l2_request_initial_mac(card);
+        card->dev->gso_max_size = (QETH_MAX_BUFFER_ELEMENTS(card) - 1) *
+                                  PAGE_SIZE;
+        card->dev->gso_max_segs = (QETH_MAX_BUFFER_ELEMENTS(card) - 1);
         SET_NETDEV_DEV(card->dev, &card->gdev->dev);
         netif_napi_add(card->dev, &card->napi, qeth_l2_poll, QETH_NAPI_WEIGHT);
         netif_carrier_off(card->dev);
@@ -1135,9 +1142,6 @@ static int qeth_l2_start_ipassists(struct qeth_card *card)
         /* configure isolation level */
         if (qeth_set_access_ctrl_online(card, 0))
                 return -ENODEV;
-        if (qeth_is_supported(card, IPA_INBOUND_CHECKSUM))
-                qeth_set_rx_csum(card, 1);
-        qeth_start_ipa_tx_checksum(card);
         return 0;
 }
 
@@ -1206,7 +1210,8 @@ static int __qeth_l2_set_online(struct ccwgroup_device *gdev, int recovery_mode)
 contin:
         if ((card->info.type == QETH_CARD_TYPE_OSD) ||
             (card->info.type == QETH_CARD_TYPE_OSX)) {
-                if (qeth_l2_start_ipassists(card))
+                rc = qeth_l2_start_ipassists(card);
+                if (rc)
                         goto out_remove;
         }
 
@@ -1800,6 +1805,12 @@ static int qeth_bridgeport_makerc(struct qeth_card *card,
                         dev_err(&card->gdev->dev,
         "The device is not configured as a Bridge Port\n");
                         break;
+                case 0x2B10:
+                case 0x0010: /* OS mismatch */
+                        rc = -EPERM;
+                        dev_err(&card->gdev->dev,
+        "A Bridge Port is already configured by a different operating system\n");
+                        break;
                 case 0x2B14:
                 case 0x0014: /* Another device is Primary */
                         switch (setcmd) {
diff --git a/drivers/s390/net/qeth_l3.h b/drivers/s390/net/qeth_l3.h
index 551a4b4c03fd..26f79533e62e 100644
--- a/drivers/s390/net/qeth_l3.h
+++ b/drivers/s390/net/qeth_l3.h
@@ -10,16 +10,23 @@
 #define __QETH_L3_H__
 
 #include "qeth_core.h"
+#include <linux/hashtable.h>
 
 #define QETH_SNIFF_AVAIL        0x0008
 
 struct qeth_ipaddr {
-        struct list_head entry;
+        struct hlist_node hnode;
         enum qeth_ip_types type;
         enum qeth_ipa_setdelip_flags set_flags;
         enum qeth_ipa_setdelip_flags del_flags;
-        int is_multicast;
-        int users;
+        u8 is_multicast:1;
+        u8 in_progress:1;
+        u8 disp_flag:2;
+
+        /* is changed only for normal ip addresses
+         * for non-normal addresses it always is  1
+         */
+        int  ref_counter;
         enum qeth_prot_versions proto;
         unsigned char mac[OSA_ADDR_LEN];
         union {
@@ -32,7 +39,24 @@ struct qeth_ipaddr {
                         unsigned int pfxlen;
                 } a6;
         } u;
+
 };
+static inline  u64 qeth_l3_ipaddr_hash(struct qeth_ipaddr *addr)
+{
+        u64  ret = 0;
+        u8 *point;
+
+        if (addr->proto == QETH_PROT_IPV6) {
+                point = (u8 *) &addr->u.a6.addr;
+                ret = get_unaligned((u64 *)point) ^
+                        get_unaligned((u64 *) (point + 8));
+        }
+        if (addr->proto == QETH_PROT_IPV4) {
+                point = (u8 *) &addr->u.a4.addr;
+                ret = get_unaligned((u32 *) point);
+        }
+        return ret;
+}
 
 struct qeth_ipato_entry {
         struct list_head entry;
@@ -60,6 +84,5 @@ int qeth_l3_is_addr_covered_by_ipato(struct qeth_card *, struct qeth_ipaddr *);
 struct qeth_ipaddr *qeth_l3_get_addr_buffer(enum qeth_prot_versions);
 int qeth_l3_add_ip(struct qeth_card *, struct qeth_ipaddr *);
 int qeth_l3_delete_ip(struct qeth_card *, struct qeth_ipaddr *);
-void qeth_l3_set_ip_addr_list(struct qeth_card *);
 
 #endif /* __QETH_L3_H__ */
diff --git a/drivers/s390/net/qeth_l3_main.c b/drivers/s390/net/qeth_l3_main.c
index ac544330daeb..bcd324e054a9 100644
--- a/drivers/s390/net/qeth_l3_main.c
+++ b/drivers/s390/net/qeth_l3_main.c
@@ -30,6 +30,7 @@
 #include <net/ip6_fib.h>
 #include <net/ip6_checksum.h>
 #include <net/iucv/af_iucv.h>
+#include <linux/hashtable.h>
 
 #include "qeth_l3.h"
 
@@ -57,7 +58,7 @@ static int qeth_l3_isxdigit(char *buf)
 
 static void qeth_l3_ipaddr4_to_string(const __u8 *addr, char *buf)
 {
-        sprintf(buf, "%i.%i.%i.%i", addr[0], addr[1], addr[2], addr[3]);
+        sprintf(buf, "%pI4", addr);
 }
 
 static int qeth_l3_string_to_ipaddr4(const char *buf, __u8 *addr)
@@ -204,104 +205,129 @@ int qeth_l3_is_addr_covered_by_ipato(struct qeth_card *card,
         return rc;
 }
 
-/*
- * Add IP to be added to todo list. If there is already an "add todo"
- * in this list we just incremenent the reference count.
- * Returns 0 if we  just incremented reference count.
- */
-static int __qeth_l3_insert_ip_todo(struct qeth_card *card,
-                                        struct qeth_ipaddr *addr, int add)
+inline int
+qeth_l3_ipaddrs_is_equal(struct qeth_ipaddr *addr1, struct qeth_ipaddr *addr2)
 {
-        struct qeth_ipaddr *tmp, *t;
-        int found = 0;
+        return addr1->proto == addr2->proto &&
+                !memcmp(&addr1->u, &addr2->u, sizeof(addr1->u))  &&
+                !memcmp(&addr1->mac, &addr2->mac, sizeof(addr1->mac));
+}
 
-        if (card->options.sniffer)
-                return 0;
-        list_for_each_entry_safe(tmp, t, card->ip_tbd_list, entry) {
-                if ((addr->type == QETH_IP_TYPE_DEL_ALL_MC) &&
-                    (tmp->type == QETH_IP_TYPE_DEL_ALL_MC))
-                        return 0;
-                if ((tmp->proto        == QETH_PROT_IPV4)     &&
-                    (addr->proto       == QETH_PROT_IPV4)     &&
-                    (tmp->type         == addr->type)         &&
-                    (tmp->is_multicast == addr->is_multicast) &&
-                    (tmp->u.a4.addr    == addr->u.a4.addr)    &&
-                    (tmp->u.a4.mask    == addr->u.a4.mask)) {
-                        found = 1;
-                        break;
-                }
-                if ((tmp->proto        == QETH_PROT_IPV6)      &&
-                    (addr->proto       == QETH_PROT_IPV6)      &&
-                    (tmp->type         == addr->type)          &&
-                    (tmp->is_multicast == addr->is_multicast)  &&
-                    (tmp->u.a6.pfxlen  == addr->u.a6.pfxlen)   &&
-                    (memcmp(&tmp->u.a6.addr, &addr->u.a6.addr,
-                            sizeof(struct in6_addr)) == 0)) {
-                        found = 1;
-                        break;
-                }
-        }
-        if (found) {
-                if (addr->users != 0)
-                        tmp->users += addr->users;
-                else
-                        tmp->users += add ? 1 : -1;
-                if (tmp->users == 0) {
-                        list_del(&tmp->entry);
-                        kfree(tmp);
-                }
-                return 0;
+static struct qeth_ipaddr *
+qeth_l3_ip_from_hash(struct qeth_card *card, struct qeth_ipaddr *tmp_addr)
+{
+        struct qeth_ipaddr *addr;
+
+        if (tmp_addr->is_multicast) {
+                hash_for_each_possible(card->ip_mc_htable,  addr,
+                                hnode, qeth_l3_ipaddr_hash(tmp_addr))
+                        if (qeth_l3_ipaddrs_is_equal(tmp_addr, addr))
+                                return addr;
         } else {
-                if (addr->type == QETH_IP_TYPE_DEL_ALL_MC)
-                        list_add(&addr->entry, card->ip_tbd_list);
-                else {
-                        if (addr->users == 0)
-                                addr->users += add ? 1 : -1;
-                        if (add && (addr->type == QETH_IP_TYPE_NORMAL) &&
-                            qeth_l3_is_addr_covered_by_ipato(card, addr)) {
-                                QETH_CARD_TEXT(card, 2, "tkovaddr");
-                                addr->set_flags |= QETH_IPA_SETIP_TAKEOVER_FLAG;
-                        }
-                        list_add_tail(&addr->entry, card->ip_tbd_list);
-                }
-                return 1;
+                hash_for_each_possible(card->ip_htable,  addr,
+                                hnode, qeth_l3_ipaddr_hash(tmp_addr))
+                        if (qeth_l3_ipaddrs_is_equal(tmp_addr, addr))
+                                return addr;
         }
+
+        return NULL;
 }
 
-int qeth_l3_delete_ip(struct qeth_card *card, struct qeth_ipaddr *addr)
+int qeth_l3_delete_ip(struct qeth_card *card, struct qeth_ipaddr *tmp_addr)
 {
-        unsigned long flags;
         int rc = 0;
+        struct qeth_ipaddr *addr;
 
         QETH_CARD_TEXT(card, 4, "delip");
 
-        if (addr->proto == QETH_PROT_IPV4)
-                QETH_CARD_HEX(card, 4, &addr->u.a4.addr, 4);
+        if (tmp_addr->proto == QETH_PROT_IPV4)
+                QETH_CARD_HEX(card, 4, &tmp_addr->u.a4.addr, 4);
         else {
-                QETH_CARD_HEX(card, 4, &addr->u.a6.addr, 8);
-                QETH_CARD_HEX(card, 4, ((char *)&addr->u.a6.addr) + 8, 8);
+                QETH_CARD_HEX(card, 4, &tmp_addr->u.a6.addr, 8);
+                QETH_CARD_HEX(card, 4, ((char *)&tmp_addr->u.a6.addr) + 8, 8);
         }
-        spin_lock_irqsave(&card->ip_lock, flags);
-        rc = __qeth_l3_insert_ip_todo(card, addr, 0);
-        spin_unlock_irqrestore(&card->ip_lock, flags);
+
+        addr = qeth_l3_ip_from_hash(card, tmp_addr);
+        if (!addr)
+                return -ENOENT;
+
+        addr->ref_counter--;
+        if (addr->type == QETH_IP_TYPE_NORMAL && addr->ref_counter > 0)
+                return rc;
+        if (addr->in_progress)
+                return -EINPROGRESS;
+
+        rc = qeth_l3_deregister_addr_entry(card, addr);
+
+        hash_del(&addr->hnode);
+        kfree(addr);
+
         return rc;
 }
 
-int qeth_l3_add_ip(struct qeth_card *card, struct qeth_ipaddr *addr)
+int qeth_l3_add_ip(struct qeth_card *card, struct qeth_ipaddr *tmp_addr)
 {
-        unsigned long flags;
         int rc = 0;
+        struct qeth_ipaddr *addr;
 
         QETH_CARD_TEXT(card, 4, "addip");
-        if (addr->proto == QETH_PROT_IPV4)
-                QETH_CARD_HEX(card, 4, &addr->u.a4.addr, 4);
+
+        if (tmp_addr->proto == QETH_PROT_IPV4)
+                QETH_CARD_HEX(card, 4, &tmp_addr->u.a4.addr, 4);
         else {
-                QETH_CARD_HEX(card, 4, &addr->u.a6.addr, 8);
-                QETH_CARD_HEX(card, 4, ((char *)&addr->u.a6.addr) + 8, 8);
+                QETH_CARD_HEX(card, 4, &tmp_addr->u.a6.addr, 8);
+                QETH_CARD_HEX(card, 4, ((char *)&tmp_addr->u.a6.addr) + 8, 8);
         }
-        spin_lock_irqsave(&card->ip_lock, flags);
-        rc = __qeth_l3_insert_ip_todo(card, addr, 1);
-        spin_unlock_irqrestore(&card->ip_lock, flags);
+
+        addr = qeth_l3_ip_from_hash(card, tmp_addr);
+        if (!addr) {
+                addr = qeth_l3_get_addr_buffer(tmp_addr->proto);
+                if (!addr)
+                        return -ENOMEM;
+
+                memcpy(addr, tmp_addr, sizeof(struct qeth_ipaddr));
+                addr->ref_counter = 1;
+
+                if (addr->type == QETH_IP_TYPE_NORMAL  &&
+                                qeth_l3_is_addr_covered_by_ipato(card, addr)) {
+                        QETH_CARD_TEXT(card, 2, "tkovaddr");
+                        addr->set_flags |= QETH_IPA_SETIP_TAKEOVER_FLAG;
+                }
+                hash_add(card->ip_htable, &addr->hnode,
+                                qeth_l3_ipaddr_hash(addr));
+
+                /* qeth_l3_register_addr_entry can go to sleep
+                 * if we add a IPV4 addr. It is caused by the reason
+                 * that SETIP ipa cmd starts ARP staff for IPV4 addr.
+                 * Thus we should unlock spinlock, and make a protection
+                 * using in_progress variable to indicate that there is
+                 * an hardware operation with this IPV4 address
+                 */
+                if (addr->proto == QETH_PROT_IPV4) {
+                        addr->in_progress = 1;
+                        spin_unlock_bh(&card->ip_lock);
+                        rc = qeth_l3_register_addr_entry(card, addr);
+                        spin_lock_bh(&card->ip_lock);
+                        addr->in_progress = 0;
+                } else
+                        rc = qeth_l3_register_addr_entry(card, addr);
+
+                if (!rc || (rc == IPA_RC_DUPLICATE_IP_ADDRESS) ||
+                                (rc == IPA_RC_LAN_OFFLINE)) {
+                        addr->disp_flag = QETH_DISP_ADDR_DO_NOTHING;
+                        if (addr->ref_counter < 1) {
+                                qeth_l3_delete_ip(card, addr);
+                                kfree(addr);
+                        }
+                } else {
+                        hash_del(&addr->hnode);
+                        kfree(addr);
+                }
+        } else {
+                        if (addr->type == QETH_IP_TYPE_NORMAL)
+                                addr->ref_counter++;
+        }
+
         return rc;
 }
 
@@ -312,229 +338,88 @@ struct qeth_ipaddr *qeth_l3_get_addr_buffer(
         struct qeth_ipaddr *addr;
 
         addr = kzalloc(sizeof(struct qeth_ipaddr), GFP_ATOMIC);
-        if (addr == NULL) {
+        if (!addr)
                 return NULL;
-        }
+
         addr->type = QETH_IP_TYPE_NORMAL;
+        addr->disp_flag = QETH_DISP_ADDR_DO_NOTHING;
         addr->proto = prot;
+
         return addr;
 }
 
-static void qeth_l3_delete_mc_addresses(struct qeth_card *card)
-{
-        struct qeth_ipaddr *iptodo;
-        unsigned long flags;
-
-        QETH_CARD_TEXT(card, 4, "delmc");
-        iptodo = qeth_l3_get_addr_buffer(QETH_PROT_IPV4);
-        if (!iptodo) {
-                QETH_CARD_TEXT(card, 2, "dmcnomem");
-                return;
-        }
-        iptodo->type = QETH_IP_TYPE_DEL_ALL_MC;
-        spin_lock_irqsave(&card->ip_lock, flags);
-        if (!__qeth_l3_insert_ip_todo(card, iptodo, 0))
-                kfree(iptodo);
-        spin_unlock_irqrestore(&card->ip_lock, flags);
-}
-
-/*
- * Add/remove address to/from card's ip list, i.e. try to add or remove
- * reference to/from an IP address that is already registered on the card.
- * Returns:
- *      0  address was on card and its reference count has been adjusted,
- *         but is still > 0, so nothing has to be done
- *         also returns 0 if card was not on card and the todo was to delete
- *         the address -> there is also nothing to be done
- *      1  address was not on card and the todo is to add it to the card's ip
- *         list
- *      -1 address was on card and its reference count has been decremented
- *         to <= 0 by the todo -> address must be removed from card
- */
-static int __qeth_l3_ref_ip_on_card(struct qeth_card *card,
-                struct qeth_ipaddr *todo, struct qeth_ipaddr **__addr)
+static void qeth_l3_clear_ip_htable(struct qeth_card *card, int recover)
 {
         struct qeth_ipaddr *addr;
-        int found = 0;
-
-        list_for_each_entry(addr, &card->ip_list, entry) {
-                if ((addr->proto == QETH_PROT_IPV4) &&
-                    (todo->proto == QETH_PROT_IPV4) &&
-                    (addr->type == todo->type) &&
-                    (addr->u.a4.addr == todo->u.a4.addr) &&
-                    (addr->u.a4.mask == todo->u.a4.mask)) {
-                        found = 1;
-                        break;
-                }
-                if ((addr->proto == QETH_PROT_IPV6) &&
-                    (todo->proto == QETH_PROT_IPV6) &&
-                    (addr->type == todo->type) &&
-                    (addr->u.a6.pfxlen == todo->u.a6.pfxlen) &&
-                    (memcmp(&addr->u.a6.addr, &todo->u.a6.addr,
-                            sizeof(struct in6_addr)) == 0)) {
-                        found = 1;
-                        break;
-                }
-        }
-        if (found) {
-                addr->users += todo->users;
-                if (addr->users <= 0) {
-                        *__addr = addr;
-                        return -1;
-                } else {
-                        /* for VIPA and RXIP limit refcount to 1 */
-                        if (addr->type != QETH_IP_TYPE_NORMAL)
-                                addr->users = 1;
-                        return 0;
-                }
-        }
-        if (todo->users > 0) {
-                /* for VIPA and RXIP limit refcount to 1 */
-                if (todo->type != QETH_IP_TYPE_NORMAL)
-                        todo->users = 1;
-                return 1;
-        } else
-                return 0;
-}
-
-static void __qeth_l3_delete_all_mc(struct qeth_card *card,
-                                        unsigned long *flags)
-{
-        struct list_head fail_list;
-        struct qeth_ipaddr *addr, *tmp;
-        int rc;
-
-        INIT_LIST_HEAD(&fail_list);
-again:
-        list_for_each_entry_safe(addr, tmp, &card->ip_list, entry) {
-                if (addr->is_multicast) {
-                        list_del(&addr->entry);
-                        spin_unlock_irqrestore(&card->ip_lock, *flags);
-                        rc = qeth_l3_deregister_addr_entry(card, addr);
-                        spin_lock_irqsave(&card->ip_lock, *flags);
-                        if (!rc || (rc == IPA_RC_MC_ADDR_NOT_FOUND))
-                                kfree(addr);
-                        else
-                                list_add_tail(&addr->entry, &fail_list);
-                        goto again;
-                }
-        }
-        list_splice(&fail_list, &card->ip_list);
-}
-
-void qeth_l3_set_ip_addr_list(struct qeth_card *card)
-{
-        struct list_head *tbd_list;
-        struct qeth_ipaddr *todo, *addr;
-        unsigned long flags;
-        int rc;
+        struct hlist_node *tmp;
+        int i;
 
-        QETH_CARD_TEXT(card, 2, "sdiplist");
-        QETH_CARD_HEX(card, 2, &card, sizeof(void *));
+        QETH_CARD_TEXT(card, 4, "clearip");
 
-        if (!qeth_card_hw_is_reachable(card) || card->options.sniffer)
+        if (recover && card->options.sniffer)
                 return;
 
-        spin_lock_irqsave(&card->ip_lock, flags);
-        tbd_list = card->ip_tbd_list;
-        card->ip_tbd_list = kzalloc(sizeof(struct list_head), GFP_ATOMIC);
-        if (!card->ip_tbd_list) {
-                QETH_CARD_TEXT(card, 0, "silnomem");
-                card->ip_tbd_list = tbd_list;
-                spin_unlock_irqrestore(&card->ip_lock, flags);
-                return;
-        } else
-                INIT_LIST_HEAD(card->ip_tbd_list);
-
-        while (!list_empty(tbd_list)) {
-                todo = list_entry(tbd_list->next, struct qeth_ipaddr, entry);
-                list_del(&todo->entry);
-                if (todo->type == QETH_IP_TYPE_DEL_ALL_MC) {
-                        __qeth_l3_delete_all_mc(card, &flags);
-                        kfree(todo);
+        spin_lock_bh(&card->ip_lock);
+
+        hash_for_each_safe(card->ip_htable, i, tmp, addr, hnode) {
+                if (!recover) {
+                        hash_del(&addr->hnode);
+                        kfree(addr);
                         continue;
                 }
-                rc = __qeth_l3_ref_ip_on_card(card, todo, &addr);
-                if (rc == 0) {
-                        /* nothing to be done; only adjusted refcount */
-                        kfree(todo);
-                } else if (rc == 1) {
-                        /* new entry to be added to on-card list */
-                        spin_unlock_irqrestore(&card->ip_lock, flags);
-                        rc = qeth_l3_register_addr_entry(card, todo);
-                        spin_lock_irqsave(&card->ip_lock, flags);
-                        if (!rc || (rc == IPA_RC_LAN_OFFLINE))
-                                list_add_tail(&todo->entry, &card->ip_list);
-                        else
-                                kfree(todo);
-                } else if (rc == -1) {
-                        /* on-card entry to be removed */
-                        list_del_init(&addr->entry);
-                        spin_unlock_irqrestore(&card->ip_lock, flags);
-                        rc = qeth_l3_deregister_addr_entry(card, addr);
-                        spin_lock_irqsave(&card->ip_lock, flags);
-                        if (!rc || (rc == IPA_RC_IP_ADDRESS_NOT_DEFINED))
-                                kfree(addr);
-                        else
-                                list_add_tail(&addr->entry, &card->ip_list);
-                        kfree(todo);
-                }
+                addr->disp_flag = QETH_DISP_ADDR_ADD;
         }
-        spin_unlock_irqrestore(&card->ip_lock, flags);
-        kfree(tbd_list);
-}
 
-static void qeth_l3_clear_ip_list(struct qeth_card *card, int recover)
-{
-        struct qeth_ipaddr *addr, *tmp;
-        unsigned long flags;
+        spin_unlock_bh(&card->ip_lock);
 
-        QETH_CARD_TEXT(card, 4, "clearip");
-        if (recover && card->options.sniffer)
-                return;
-        spin_lock_irqsave(&card->ip_lock, flags);
-        /* clear todo list */
-        list_for_each_entry_safe(addr, tmp, card->ip_tbd_list, entry) {
-                list_del(&addr->entry);
+        spin_lock_bh(&card->mclock);
+
+        hash_for_each_safe(card->ip_mc_htable, i, tmp, addr, hnode) {
+                hash_del(&addr->hnode);
                 kfree(addr);
         }
 
-        while (!list_empty(&card->ip_list)) {
-                addr = list_entry(card->ip_list.next,
-                                  struct qeth_ipaddr, entry);
-                list_del_init(&addr->entry);
-                if (!recover || addr->is_multicast) {
-                        kfree(addr);
-                        continue;
-                }
-                list_add_tail(&addr->entry, card->ip_tbd_list);
-        }
-        spin_unlock_irqrestore(&card->ip_lock, flags);
-}
+        spin_unlock_bh(&card->mclock);
+
 
-static int qeth_l3_address_exists_in_list(struct list_head *list,
-                struct qeth_ipaddr *addr, int same_type)
+}
+static void qeth_l3_recover_ip(struct qeth_card *card)
 {
-        struct qeth_ipaddr *tmp;
+        struct qeth_ipaddr *addr;
+        struct hlist_node *tmp;
+        int i;
+        int rc;
 
-        list_for_each_entry(tmp, list, entry) {
-                if ((tmp->proto == QETH_PROT_IPV4) &&
-                    (addr->proto == QETH_PROT_IPV4) &&
-                    ((same_type && (tmp->type == addr->type)) ||
-                    (!same_type && (tmp->type != addr->type))) &&
-                    (tmp->u.a4.addr == addr->u.a4.addr))
-                        return 1;
+        QETH_CARD_TEXT(card, 4, "recoverip");
+
+        spin_lock_bh(&card->ip_lock);
+
+        hash_for_each_safe(card->ip_htable, i, tmp, addr, hnode) {
+                if (addr->disp_flag == QETH_DISP_ADDR_ADD) {
+                        if (addr->proto == QETH_PROT_IPV4) {
+                                addr->in_progress = 1;
+                                spin_unlock_bh(&card->ip_lock);
+                                rc = qeth_l3_register_addr_entry(card, addr);
+                                spin_lock_bh(&card->ip_lock);
+                                addr->in_progress = 0;
+                        } else
+                                rc = qeth_l3_register_addr_entry(card, addr);
+
+                        if (!rc) {
+                                addr->disp_flag = QETH_DISP_ADDR_DO_NOTHING;
+                                if (addr->ref_counter < 1) {
+                                        qeth_l3_delete_ip(card, addr);
+                                        kfree(addr);
+                                }
+                        } else {
+                                hash_del(&addr->hnode);
+                                kfree(addr);
+                        }
+                }
+        }
 
-                if ((tmp->proto == QETH_PROT_IPV6) &&
-                    (addr->proto == QETH_PROT_IPV6) &&
-                    ((same_type && (tmp->type == addr->type)) ||
-                    (!same_type && (tmp->type != addr->type))) &&
-                    (memcmp(&tmp->u.a6.addr, &addr->u.a6.addr,
-                            sizeof(struct in6_addr)) == 0))
-                        return 1;
+        spin_unlock_bh(&card->ip_lock);
 
-        }
-        return 0;
 }
 
 static int qeth_l3_send_setdelmc(struct qeth_card *card,
@@ -712,27 +597,28 @@ int qeth_l3_setrouting_v6(struct qeth_card *card)
  */
 static void qeth_l3_clear_ipato_list(struct qeth_card *card)
 {
-
         struct qeth_ipato_entry *ipatoe, *tmp;
-        unsigned long flags;
 
-        spin_lock_irqsave(&card->ip_lock, flags);
+        spin_lock_bh(&card->ip_lock);
+
         list_for_each_entry_safe(ipatoe, tmp, &card->ipato.entries, entry) {
                 list_del(&ipatoe->entry);
                 kfree(ipatoe);
         }
-        spin_unlock_irqrestore(&card->ip_lock, flags);
+
+        spin_unlock_bh(&card->ip_lock);
 }
 
 int qeth_l3_add_ipato_entry(struct qeth_card *card,
                                 struct qeth_ipato_entry *new)
 {
         struct qeth_ipato_entry *ipatoe;
-        unsigned long flags;
         int rc = 0;
 
         QETH_CARD_TEXT(card, 2, "addipato");
-        spin_lock_irqsave(&card->ip_lock, flags);
+
+        spin_lock_bh(&card->ip_lock);
+
         list_for_each_entry(ipatoe, &card->ipato.entries, entry) {
                 if (ipatoe->proto != new->proto)
                         continue;
@@ -743,10 +629,12 @@ int qeth_l3_add_ipato_entry(struct qeth_card *card,
                         break;
                 }
         }
+
         if (!rc)
                 list_add_tail(&new->entry, &card->ipato.entries);
 
-        spin_unlock_irqrestore(&card->ip_lock, flags);
+        spin_unlock_bh(&card->ip_lock);
+
         return rc;
 }
 
@@ -754,10 +642,11 @@ void qeth_l3_del_ipato_entry(struct qeth_card *card,
                 enum qeth_prot_versions proto, u8 *addr, int mask_bits)
 {
         struct qeth_ipato_entry *ipatoe, *tmp;
-        unsigned long flags;
 
         QETH_CARD_TEXT(card, 2, "delipato");
-        spin_lock_irqsave(&card->ip_lock, flags);
+
+        spin_lock_bh(&card->ip_lock);
+
         list_for_each_entry_safe(ipatoe, tmp, &card->ipato.entries, entry) {
                 if (ipatoe->proto != proto)
                         continue;
@@ -768,7 +657,8 @@ void qeth_l3_del_ipato_entry(struct qeth_card *card,
                         kfree(ipatoe);
                 }
         }
-        spin_unlock_irqrestore(&card->ip_lock, flags);
+
+        spin_unlock_bh(&card->ip_lock);
 }
 
 /*
@@ -778,7 +668,6 @@ int qeth_l3_add_vipa(struct qeth_card *card, enum qeth_prot_versions proto,
               const u8 *addr)
 {
         struct qeth_ipaddr *ipaddr;
-        unsigned long flags;
         int rc = 0;
 
         ipaddr = qeth_l3_get_addr_buffer(proto);
@@ -797,18 +686,18 @@ int qeth_l3_add_vipa(struct qeth_card *card, enum qeth_prot_versions proto,
                 ipaddr->del_flags = QETH_IPA_DELIP_VIPA_FLAG;
         } else
                 return -ENOMEM;
-        spin_lock_irqsave(&card->ip_lock, flags);
-        if (qeth_l3_address_exists_in_list(&card->ip_list, ipaddr, 0) ||
-            qeth_l3_address_exists_in_list(card->ip_tbd_list, ipaddr, 0))
+
+        spin_lock_bh(&card->ip_lock);
+
+        if (!qeth_l3_ip_from_hash(card, ipaddr))
                 rc = -EEXIST;
-        spin_unlock_irqrestore(&card->ip_lock, flags);
-        if (rc) {
-                kfree(ipaddr);
-                return rc;
-        }
-        if (!qeth_l3_add_ip(card, ipaddr))
-                kfree(ipaddr);
-        qeth_l3_set_ip_addr_list(card);
+        else
+                qeth_l3_add_ip(card, ipaddr);
+
+        spin_unlock_bh(&card->ip_lock);
+
+        kfree(ipaddr);
+
         return rc;
 }
 
@@ -831,9 +720,12 @@ void qeth_l3_del_vipa(struct qeth_card *card, enum qeth_prot_versions proto,
                 ipaddr->type = QETH_IP_TYPE_VIPA;
         } else
                 return;
-        if (!qeth_l3_delete_ip(card, ipaddr))
-                kfree(ipaddr);
-        qeth_l3_set_ip_addr_list(card);
+
+        spin_lock_bh(&card->ip_lock);
+        qeth_l3_delete_ip(card, ipaddr);
+        spin_unlock_bh(&card->ip_lock);
+
+        kfree(ipaddr);
 }
 
 /*
@@ -843,7 +735,6 @@ int qeth_l3_add_rxip(struct qeth_card *card, enum qeth_prot_versions proto,
               const u8 *addr)
 {
         struct qeth_ipaddr *ipaddr;
-        unsigned long flags;
         int rc = 0;
 
         ipaddr = qeth_l3_get_addr_buffer(proto);
@@ -857,24 +748,25 @@ int qeth_l3_add_rxip(struct qeth_card *card, enum qeth_prot_versions proto,
                         memcpy(&ipaddr->u.a6.addr, addr, 16);
                         ipaddr->u.a6.pfxlen = 0;
                 }
+
                 ipaddr->type = QETH_IP_TYPE_RXIP;
                 ipaddr->set_flags = QETH_IPA_SETIP_TAKEOVER_FLAG;
                 ipaddr->del_flags = 0;
         } else
                 return -ENOMEM;
-        spin_lock_irqsave(&card->ip_lock, flags);
-        if (qeth_l3_address_exists_in_list(&card->ip_list, ipaddr, 0) ||
-            qeth_l3_address_exists_in_list(card->ip_tbd_list, ipaddr, 0))
+
+        spin_lock_bh(&card->ip_lock);
+
+        if (!qeth_l3_ip_from_hash(card, ipaddr))
                 rc = -EEXIST;
-        spin_unlock_irqrestore(&card->ip_lock, flags);
-        if (rc) {
-                kfree(ipaddr);
-                return rc;
-        }
-        if (!qeth_l3_add_ip(card, ipaddr))
-                kfree(ipaddr);
-        qeth_l3_set_ip_addr_list(card);
-        return 0;
+        else
+                qeth_l3_add_ip(card, ipaddr);
+
+        spin_unlock_bh(&card->ip_lock);
+
+        kfree(ipaddr);
+
+        return rc;
 }
 
 void qeth_l3_del_rxip(struct qeth_card *card, enum qeth_prot_versions proto,
@@ -896,9 +788,12 @@ void qeth_l3_del_rxip(struct qeth_card *card, enum qeth_prot_versions proto,
                 ipaddr->type = QETH_IP_TYPE_RXIP;
         } else
                 return;
-        if (!qeth_l3_delete_ip(card, ipaddr))
-                kfree(ipaddr);
-        qeth_l3_set_ip_addr_list(card);
+
+        spin_lock_bh(&card->ip_lock);
+        qeth_l3_delete_ip(card, ipaddr);
+        spin_unlock_bh(&card->ip_lock);
+
+        kfree(ipaddr);
 }
 
 static int qeth_l3_register_addr_entry(struct qeth_card *card,
@@ -908,6 +803,7 @@ static int qeth_l3_register_addr_entry(struct qeth_card *card,
         int rc = 0;
         int cnt = 3;
 
+
         if (addr->proto == QETH_PROT_IPV4) {
                 QETH_CARD_TEXT(card, 2, "setaddr4");
                 QETH_CARD_HEX(card, 3, &addr->u.a4.addr, sizeof(int));
@@ -1013,36 +909,6 @@ static int qeth_l3_setadapter_parms(struct qeth_card *card)
         return rc;
 }
 
-static int qeth_l3_default_setassparms_cb(struct qeth_card *card,
-                        struct qeth_reply *reply, unsigned long data)
-{
-        struct qeth_ipa_cmd *cmd;
-
-        QETH_CARD_TEXT(card, 4, "defadpcb");
-
-        cmd = (struct qeth_ipa_cmd *) data;
-        if (cmd->hdr.return_code == 0) {
-                cmd->hdr.return_code = cmd->data.setassparms.hdr.return_code;
-                if (cmd->hdr.prot_version == QETH_PROT_IPV4)
-                        card->options.ipa4.enabled_funcs = cmd->hdr.ipa_enabled;
-                if (cmd->hdr.prot_version == QETH_PROT_IPV6)
-                        card->options.ipa6.enabled_funcs = cmd->hdr.ipa_enabled;
-        }
-        if (cmd->data.setassparms.hdr.assist_no == IPA_INBOUND_CHECKSUM &&
-            cmd->data.setassparms.hdr.command_code == IPA_CMD_ASS_START) {
-                card->info.csum_mask = cmd->data.setassparms.data.flags_32bit;
-                QETH_CARD_TEXT_(card, 3, "csum:%d", card->info.csum_mask);
-        }
-        if (cmd->data.setassparms.hdr.assist_no == IPA_OUTBOUND_CHECKSUM &&
-            cmd->data.setassparms.hdr.command_code == IPA_CMD_ASS_START) {
-                card->info.tx_csum_mask =
-                        cmd->data.setassparms.data.flags_32bit;
-                QETH_CARD_TEXT_(card, 3, "tcsu:%d", card->info.tx_csum_mask);
-        }
-
-        return 0;
-}
-
 #ifdef CONFIG_QETH_IPV6
 static int qeth_l3_send_simple_setassparms_ipv6(struct qeth_card *card,
                 enum qeth_ipa_funcs ipa_func, __u16 cmd_code)
@@ -1056,7 +922,7 @@ static int qeth_l3_send_simple_setassparms_ipv6(struct qeth_card *card,
         if (!iob)
                 return -ENOMEM;
         rc = qeth_send_setassparms(card, iob, 0, 0,
-                                   qeth_l3_default_setassparms_cb, NULL);
+                                   qeth_setassparms_cb, NULL);
         return rc;
 }
 #endif
@@ -1291,47 +1157,6 @@ out:
         return rc;
 }
 
-static void qeth_l3_start_ipa_checksum(struct qeth_card *card)
-{
-        QETH_CARD_TEXT(card, 3, "strtcsum");
-        if (qeth_is_supported(card, IPA_INBOUND_CHECKSUM)
-            && (card->dev->features & NETIF_F_RXCSUM))
-                qeth_set_rx_csum(card, 1);
-}
-
-static void qeth_l3_start_ipa_tx_checksum(struct qeth_card *card)
-{
-        QETH_CARD_TEXT(card, 3, "strttxcs");
-        qeth_start_ipa_tx_checksum(card);
-}
-
-static int qeth_l3_start_ipa_tso(struct qeth_card *card)
-{
-        int rc;
-
-        QETH_CARD_TEXT(card, 3, "sttso");
-
-        if (!qeth_is_supported(card, IPA_OUTBOUND_TSO)) {
-                dev_info(&card->gdev->dev,
-                        "Outbound TSO not supported on %s\n",
-                        QETH_CARD_IFNAME(card));
-                rc = -EOPNOTSUPP;
-        } else {
-                rc = qeth_send_simple_setassparms(card, IPA_OUTBOUND_TSO,
-                                                  IPA_CMD_ASS_START, 0);
-                if (rc)
-                        dev_warn(&card->gdev->dev, "Starting outbound TCP "
-                                "segmentation offload for %s failed\n",
-                                QETH_CARD_IFNAME(card));
-                else
-                        dev_info(&card->gdev->dev,
-                                "Outbound TSO enabled\n");
-        }
-        if (rc)
-                card->dev->features &= ~NETIF_F_TSO;
-        return rc;
-}
-
 static int qeth_l3_start_ipassists(struct qeth_card *card)
 {
         QETH_CARD_TEXT(card, 3, "strtipas");
@@ -1345,9 +1170,6 @@ static int qeth_l3_start_ipassists(struct qeth_card *card)
         qeth_l3_start_ipa_multicast(card);              /* go on*/
         qeth_l3_start_ipa_ipv6(card);           /* go on*/
         qeth_l3_start_ipa_broadcast(card);              /* go on*/
-        qeth_l3_start_ipa_checksum(card);               /* go on*/
-        qeth_l3_start_ipa_tx_checksum(card);
-        qeth_l3_start_ipa_tso(card);            /* go on*/
         return 0;
 }
 
@@ -1507,31 +1329,99 @@ qeth_diags_trace(struct qeth_card *card, enum qeth_diags_trace_cmds diags_cmd)
         return qeth_send_ipa_cmd(card, iob, qeth_diags_trace_cb, NULL);
 }
 
-static void qeth_l3_get_mac_for_ipm(__u32 ipm, char *mac,
-                                struct net_device *dev)
+static void qeth_l3_get_mac_for_ipm(__u32 ipm, char *mac)
 {
         ip_eth_mc_map(ipm, mac);
 }
 
-static void qeth_l3_add_mc(struct qeth_card *card, struct in_device *in4_dev)
+static void qeth_l3_mark_all_mc_to_be_deleted(struct qeth_card *card)
+{
+        struct qeth_ipaddr *addr;
+        int i;
+
+        hash_for_each(card->ip_mc_htable, i, addr, hnode)
+                addr->disp_flag = QETH_DISP_ADDR_DELETE;
+
+}
+
+static void qeth_l3_add_all_new_mc(struct qeth_card *card)
+{
+        struct qeth_ipaddr *addr;
+        struct hlist_node *tmp;
+        int i;
+        int rc;
+
+        hash_for_each_safe(card->ip_mc_htable, i, tmp, addr, hnode) {
+                if (addr->disp_flag == QETH_DISP_ADDR_ADD) {
+                        rc = qeth_l3_register_addr_entry(card, addr);
+                        if (!rc || (rc == IPA_RC_LAN_OFFLINE))
+                                addr->ref_counter = 1;
+                        else {
+                                hash_del(&addr->hnode);
+                                kfree(addr);
+                        }
+                }
+        }
+
+}
+
+static void qeth_l3_delete_nonused_mc(struct qeth_card *card)
+{
+        struct qeth_ipaddr *addr;
+        struct hlist_node *tmp;
+        int i;
+        int rc;
+
+        hash_for_each_safe(card->ip_mc_htable, i, tmp, addr, hnode) {
+                if (addr->disp_flag == QETH_DISP_ADDR_DELETE) {
+                        rc = qeth_l3_deregister_addr_entry(card, addr);
+                        if (!rc || (rc == IPA_RC_MC_ADDR_NOT_FOUND)) {
+                                hash_del(&addr->hnode);
+                                kfree(addr);
+                        }
+                }
+        }
+
+}
+
+
+static void
+qeth_l3_add_mc_to_hash(struct qeth_card *card, struct in_device *in4_dev)
 {
-        struct qeth_ipaddr *ipm;
         struct ip_mc_list *im4;
+        struct qeth_ipaddr *tmp, *ipm;
         char buf[MAX_ADDR_LEN];
 
         QETH_CARD_TEXT(card, 4, "addmc");
+
+        tmp = qeth_l3_get_addr_buffer(QETH_PROT_IPV4);
+                if (!tmp)
+                        return;
+
         for (im4 = rcu_dereference(in4_dev->mc_list); im4 != NULL;
              im4 = rcu_dereference(im4->next_rcu)) {
-                qeth_l3_get_mac_for_ipm(im4->multiaddr, buf, in4_dev->dev);
-                ipm = qeth_l3_get_addr_buffer(QETH_PROT_IPV4);
-                if (!ipm)
-                        continue;
-                ipm->u.a4.addr = im4->multiaddr;
-                memcpy(ipm->mac, buf, OSA_ADDR_LEN);
-                ipm->is_multicast = 1;
-                if (!qeth_l3_add_ip(card, ipm))
-                        kfree(ipm);
+                qeth_l3_get_mac_for_ipm(im4->multiaddr, buf);
+
+                tmp->u.a4.addr = im4->multiaddr;
+                memcpy(tmp->mac, buf, sizeof(tmp->mac));
+
+                ipm = qeth_l3_ip_from_hash(card, tmp);
+                if (ipm) {
+                        ipm->disp_flag = QETH_DISP_ADDR_DO_NOTHING;
+                } else {
+                        ipm = qeth_l3_get_addr_buffer(QETH_PROT_IPV4);
+                        if (!ipm)
+                                continue;
+                        memcpy(ipm->mac, buf, sizeof(tmp->mac));
+                        ipm->u.a4.addr = im4->multiaddr;
+                        ipm->is_multicast = 1;
+                        ipm->disp_flag = QETH_DISP_ADDR_ADD;
+                        hash_add(card->ip_mc_htable,
+                                        &ipm->hnode, qeth_l3_ipaddr_hash(ipm));
+                }
         }
+
+        kfree(tmp);
 }
 
 /* called with rcu_read_lock */
@@ -1541,6 +1431,7 @@ static void qeth_l3_add_vlan_mc(struct qeth_card *card)
         u16 vid;
 
         QETH_CARD_TEXT(card, 4, "addmcvl");
+
         if (!qeth_is_supported(card, IPA_FULL_VLAN))
                 return;
 
@@ -1555,7 +1446,7 @@ static void qeth_l3_add_vlan_mc(struct qeth_card *card)
                 in_dev = __in_dev_get_rcu(netdev);
                 if (!in_dev)
                         continue;
-                qeth_l3_add_mc(card, in_dev);
+                qeth_l3_add_mc_to_hash(card, in_dev);
         }
 }
 
@@ -1564,36 +1455,60 @@ static void qeth_l3_add_multicast_ipv4(struct qeth_card *card)
         struct in_device *in4_dev;
 
         QETH_CARD_TEXT(card, 4, "chkmcv4");
+
         rcu_read_lock();
         in4_dev = __in_dev_get_rcu(card->dev);
         if (in4_dev == NULL)
                 goto unlock;
-        qeth_l3_add_mc(card, in4_dev);
+        qeth_l3_add_mc_to_hash(card, in4_dev);
         qeth_l3_add_vlan_mc(card);
 unlock:
         rcu_read_unlock();
 }
 
 #ifdef CONFIG_QETH_IPV6
-static void qeth_l3_add_mc6(struct qeth_card *card, struct inet6_dev *in6_dev)
+static void
+qeth_l3_add_mc6_to_hash(struct qeth_card *card, struct inet6_dev *in6_dev)
 {
         struct qeth_ipaddr *ipm;
         struct ifmcaddr6 *im6;
+        struct qeth_ipaddr *tmp;
         char buf[MAX_ADDR_LEN];
 
         QETH_CARD_TEXT(card, 4, "addmc6");
+
+        tmp = qeth_l3_get_addr_buffer(QETH_PROT_IPV6);
+                if (!tmp)
+                        return;
+
         for (im6 = in6_dev->mc_list; im6 != NULL; im6 = im6->next) {
                 ndisc_mc_map(&im6->mca_addr, buf, in6_dev->dev, 0);
+
+                memcpy(tmp->mac, buf, sizeof(tmp->mac));
+                memcpy(&tmp->u.a6.addr, &im6->mca_addr.s6_addr,
+                       sizeof(struct in6_addr));
+                tmp->is_multicast = 1;
+
+                ipm = qeth_l3_ip_from_hash(card, tmp);
+                if (ipm) {
+                        ipm->disp_flag = QETH_DISP_ADDR_DO_NOTHING;
+                        continue;
+                }
+
                 ipm = qeth_l3_get_addr_buffer(QETH_PROT_IPV6);
                 if (!ipm)
                         continue;
-                ipm->is_multicast = 1;
+
                 memcpy(ipm->mac, buf, OSA_ADDR_LEN);
                 memcpy(&ipm->u.a6.addr, &im6->mca_addr.s6_addr,
                        sizeof(struct in6_addr));
-                if (!qeth_l3_add_ip(card, ipm))
-                        kfree(ipm);
+                ipm->is_multicast = 1;
+                ipm->disp_flag = QETH_DISP_ADDR_ADD;
+                hash_add(card->ip_mc_htable,
+                                &ipm->hnode, qeth_l3_ipaddr_hash(ipm));
+
         }
+        kfree(tmp);
 }
 
 /* called with rcu_read_lock */
@@ -1603,6 +1518,7 @@ static void qeth_l3_add_vlan_mc6(struct qeth_card *card)
         u16 vid;
 
         QETH_CARD_TEXT(card, 4, "admc6vl");
+
         if (!qeth_is_supported(card, IPA_FULL_VLAN))
                 return;
 
@@ -1618,7 +1534,7 @@ static void qeth_l3_add_vlan_mc6(struct qeth_card *card)
                 if (!in_dev)
                         continue;
                 read_lock_bh(&in_dev->lock);
-                qeth_l3_add_mc6(card, in_dev);
+                qeth_l3_add_mc6_to_hash(card, in_dev);
                 read_unlock_bh(&in_dev->lock);
                 in6_dev_put(in_dev);
         }
@@ -1629,14 +1545,16 @@ static void qeth_l3_add_multicast_ipv6(struct qeth_card *card)
         struct inet6_dev *in6_dev;
 
         QETH_CARD_TEXT(card, 4, "chkmcv6");
+
         if (!qeth_is_supported(card, IPA_IPV6))
                 return ;
         in6_dev = in6_dev_get(card->dev);
-        if (in6_dev == NULL)
+        if (!in6_dev)
                 return;
+
         rcu_read_lock();
         read_lock_bh(&in6_dev->lock);
-        qeth_l3_add_mc6(card, in6_dev);
+        qeth_l3_add_mc6_to_hash(card, in6_dev);
         qeth_l3_add_vlan_mc6(card);
         read_unlock_bh(&in6_dev->lock);
         rcu_read_unlock();
@@ -1660,16 +1578,23 @@ static void qeth_l3_free_vlan_addresses4(struct qeth_card *card,
         in_dev = in_dev_get(netdev);
         if (!in_dev)
                 return;
+
+        addr = qeth_l3_get_addr_buffer(QETH_PROT_IPV4);
+        if (!addr)
+                return;
+
+        spin_lock_bh(&card->ip_lock);
+
         for (ifa = in_dev->ifa_list; ifa; ifa = ifa->ifa_next) {
-                addr = qeth_l3_get_addr_buffer(QETH_PROT_IPV4);
-                if (addr) {
-                        addr->u.a4.addr = ifa->ifa_address;
-                        addr->u.a4.mask = ifa->ifa_mask;
-                        addr->type = QETH_IP_TYPE_NORMAL;
-                        if (!qeth_l3_delete_ip(card, addr))
-                                kfree(addr);
-                }
+                addr->u.a4.addr = ifa->ifa_address;
+                addr->u.a4.mask = ifa->ifa_mask;
+                addr->type = QETH_IP_TYPE_NORMAL;
+                qeth_l3_delete_ip(card, addr);
         }
+
+        spin_unlock_bh(&card->ip_lock);
+
+        kfree(addr);
         in_dev_put(in_dev);
 }
 
@@ -1687,20 +1612,28 @@ static void qeth_l3_free_vlan_addresses6(struct qeth_card *card,
         netdev = __vlan_find_dev_deep_rcu(card->dev, htons(ETH_P_8021Q), vid);
         if (!netdev)
                 return;
+
         in6_dev = in6_dev_get(netdev);
         if (!in6_dev)
                 return;
+
+        addr = qeth_l3_get_addr_buffer(QETH_PROT_IPV6);
+        if (!addr)
+                return;
+
+        spin_lock_bh(&card->ip_lock);
+
         list_for_each_entry(ifa, &in6_dev->addr_list, if_list) {
-                addr = qeth_l3_get_addr_buffer(QETH_PROT_IPV6);
-                if (addr) {
-                        memcpy(&addr->u.a6.addr, &ifa->addr,
-                               sizeof(struct in6_addr));
-                        addr->u.a6.pfxlen = ifa->prefix_len;
-                        addr->type = QETH_IP_TYPE_NORMAL;
-                        if (!qeth_l3_delete_ip(card, addr))
-                                kfree(addr);
-                }
+                memcpy(&addr->u.a6.addr, &ifa->addr,
+                       sizeof(struct in6_addr));
+                addr->u.a6.pfxlen = ifa->prefix_len;
+                addr->type = QETH_IP_TYPE_NORMAL;
+                qeth_l3_delete_ip(card, addr);
         }
+
+        spin_unlock_bh(&card->ip_lock);
+
+        kfree(addr);
         in6_dev_put(in6_dev);
 #endif /* CONFIG_QETH_IPV6 */
 }
@@ -1727,18 +1660,16 @@ static int qeth_l3_vlan_rx_kill_vid(struct net_device *dev,
                                     __be16 proto, u16 vid)
 {
         struct qeth_card *card = dev->ml_priv;
-        unsigned long flags;
 
         QETH_CARD_TEXT_(card, 4, "kid:%d", vid);
+
         if (qeth_wait_for_threads(card, QETH_RECOVER_THREAD)) {
                 QETH_CARD_TEXT(card, 3, "kidREC");
                 return 0;
         }
-        spin_lock_irqsave(&card->vlanlock, flags);
         /* unregister IP addresses of vlan device */
         qeth_l3_free_vlan_addresses(card, vid);
         clear_bit(vid, card->active_vlans);
-        spin_unlock_irqrestore(&card->vlanlock, flags);
         qeth_l3_set_multicast_list(card->dev);
         return 0;
 }
@@ -1994,8 +1925,8 @@ static int qeth_l3_verify_vlan_dev(struct net_device *dev,
 static int qeth_l3_verify_dev(struct net_device *dev)
 {
         struct qeth_card *card;
-        unsigned long flags;
         int rc = 0;
+        unsigned long flags;
 
         read_lock_irqsave(&qeth_core_card_list.rwlock, flags);
         list_for_each_entry(card, &qeth_core_card_list.list, list) {
@@ -2051,7 +1982,7 @@ static void qeth_l3_stop_card(struct qeth_card *card, int recovery_mode)
                 card->state = CARD_STATE_SOFTSETUP;
         }
         if (card->state == CARD_STATE_SOFTSETUP) {
-                qeth_l3_clear_ip_list(card, 1);
+                qeth_l3_clear_ip_htable(card, 1);
                 qeth_clear_ipacmd_list(card);
                 card->state = CARD_STATE_HARDSETUP;
         }
@@ -2106,12 +2037,20 @@ static void qeth_l3_set_multicast_list(struct net_device *dev)
             (card->state != CARD_STATE_UP))
                 return;
         if (!card->options.sniffer) {
-                qeth_l3_delete_mc_addresses(card);
+
+                spin_lock_bh(&card->mclock);
+
+                qeth_l3_mark_all_mc_to_be_deleted(card);
+
                 qeth_l3_add_multicast_ipv4(card);
 #ifdef CONFIG_QETH_IPV6
                 qeth_l3_add_multicast_ipv6(card);
 #endif
-                qeth_l3_set_ip_addr_list(card);
+                qeth_l3_delete_nonused_mc(card);
+                qeth_l3_add_all_new_mc(card);
+
+                spin_unlock_bh(&card->mclock);
+
                 if (!qeth_adp_supported(card, IPA_SETADP_SET_PROMISC_MODE))
                         return;
         }
@@ -2375,22 +2314,21 @@ static int qeth_l3_arp_query(struct qeth_card *card, char __user *udata)
         if (rc) {
                 if (copy_to_user(udata, qinfo.udata, 4))
                         rc = -EFAULT;
-                        goto free_and_out;
-        } else {
+                goto free_and_out;
+        }
 #ifdef CONFIG_QETH_IPV6
-                if (qinfo.mask_bits & QETH_QARP_WITH_IPV6) {
-                        /* fails in case of GuestLAN QDIO mode */
-                        qeth_l3_query_arp_cache_info(card, QETH_PROT_IPV6,
-                                &qinfo);
-                }
+        if (qinfo.mask_bits & QETH_QARP_WITH_IPV6) {
+                /* fails in case of GuestLAN QDIO mode */
+                qeth_l3_query_arp_cache_info(card, QETH_PROT_IPV6, &qinfo);
+        }
 #endif
-                if (copy_to_user(udata, qinfo.udata, qinfo.udata_len)) {
-                        QETH_CARD_TEXT(card, 4, "qactf");
-                        rc = -EFAULT;
-                        goto free_and_out;
-                }
-                QETH_CARD_TEXT(card, 4, "qacts");
+        if (copy_to_user(udata, qinfo.udata, qinfo.udata_len)) {
+                QETH_CARD_TEXT(card, 4, "qactf");
+                rc = -EFAULT;
+                goto free_and_out;
         }
+        QETH_CARD_TEXT(card, 4, "qacts");
+
 free_and_out:
         kfree(qinfo.udata);
 out:
@@ -2427,7 +2365,7 @@ static int qeth_l3_arp_add_entry(struct qeth_card *card,
         rc = qeth_send_setassparms(card, iob,
                                    sizeof(struct qeth_arp_cache_entry),
                                    (unsigned long) entry,
-                                   qeth_l3_default_setassparms_cb, NULL);
+                                   qeth_setassparms_cb, NULL);
         if (rc) {
                 tmp = rc;
                 qeth_l3_ipaddr4_to_string((u8 *)entry->ipaddr, buf);
@@ -2467,7 +2405,7 @@ static int qeth_l3_arp_remove_entry(struct qeth_card *card,
                 return -ENOMEM;
         rc = qeth_send_setassparms(card, iob,
                                    12, (unsigned long)buf,
-                                   qeth_l3_default_setassparms_cb, NULL);
+                                   qeth_setassparms_cb, NULL);
         if (rc) {
                 tmp = rc;
                 memset(buf, 0, 16);
@@ -2793,15 +2731,34 @@ static void qeth_tso_fill_header(struct qeth_card *card,
         }
 }
 
-static inline int qeth_l3_tso_elements(struct sk_buff *skb)
+/**
+ * qeth_l3_get_elements_no_tso() - find number of SBALEs for skb data for tso
+ * @card:                          qeth card structure, to check max. elems.
+ * @skb:                           SKB address
+ * @extra_elems:                   extra elems needed, to check against max.
+ *
+ * Returns the number of pages, and thus QDIO buffer elements, needed to cover
+ * skb data, including linear part and fragments, but excluding TCP header.
+ * (Exclusion of TCP header distinguishes it from qeth_get_elements_no().)
+ * Checks if the result plus extra_elems fits under the limit for the card.
+ * Returns 0 if it does not.
+ * Note: extra_elems is not included in the returned result.
+ */
+static int qeth_l3_get_elements_no_tso(struct qeth_card *card,
+                        struct sk_buff *skb, int extra_elems)
 {
-        unsigned long tcpd = (unsigned long)tcp_hdr(skb) +
-                tcp_hdr(skb)->doff * 4;
-        int tcpd_len = skb_headlen(skb) - (tcpd - (unsigned long)skb->data);
-        int elements = PFN_UP(tcpd + tcpd_len - 1) - PFN_DOWN(tcpd);
-
-        elements += qeth_get_elements_for_frags(skb);
+        addr_t tcpdptr = (addr_t)tcp_hdr(skb) + tcp_hdrlen(skb);
+        int elements = qeth_get_elements_for_range(
+                                tcpdptr,
+                                (addr_t)skb->data + skb_headlen(skb)) +
+                                qeth_get_elements_for_frags(skb);
 
+        if ((elements + extra_elems) > QETH_MAX_BUFFER_ELEMENTS(card)) {
+                QETH_DBF_MESSAGE(2,
+        "Invalid size of TSO IP packet (Number=%d / Length=%d). Discarded.\n",
+                                elements + extra_elems, skb->len);
+                return 0;
+        }
         return elements;
 }
 
@@ -2810,8 +2767,8 @@ static int qeth_l3_hard_start_xmit(struct sk_buff *skb, struct net_device *dev)
         int rc;
         u16 *tag;
         struct qeth_hdr *hdr = NULL;
-        int elements_needed = 0;
-        int elems;
+        int hdr_elements = 0;
+        int elements;
         struct qeth_card *card = dev->ml_priv;
         struct sk_buff *new_skb = NULL;
         int ipv = qeth_get_ip_version(skb);
@@ -2822,7 +2779,7 @@ static int qeth_l3_hard_start_xmit(struct sk_buff *skb, struct net_device *dev)
                         qeth_get_priority_queue(card, skb, ipv, cast_type) :
                         card->qdio.default_out_queue];
         int tx_bytes = skb->len;
-        bool large_send;
+        bool use_tso;
         int data_offset = -1;
         int nr_frags;
 
@@ -2847,10 +2804,12 @@ static int qeth_l3_hard_start_xmit(struct sk_buff *skb, struct net_device *dev)
                 card->perf_stats.outbound_start_time = qeth_get_micros();
         }
 
-        large_send = skb_is_gso(skb);
+        /* Ignore segment size from skb_is_gso(), 1 page is always used. */
+        use_tso = skb_is_gso(skb) &&
+                  (qeth_get_ip_protocol(skb) == IPPROTO_TCP) && (ipv == 4);
 
-        if ((card->info.type == QETH_CARD_TYPE_IQD) && (!large_send) &&
-            (skb_shinfo(skb)->nr_frags == 0)) {
+        if ((card->info.type == QETH_CARD_TYPE_IQD) &&
+            !skb_is_nonlinear(skb)) {
                 new_skb = skb;
                 if (new_skb->protocol == ETH_P_AF_IUCV)
                         data_offset = 0;
@@ -2859,7 +2818,7 @@ static int qeth_l3_hard_start_xmit(struct sk_buff *skb, struct net_device *dev)
                 hdr = kmem_cache_alloc(qeth_core_header_cache, GFP_ATOMIC);
                 if (!hdr)
                         goto tx_drop;
-                elements_needed++;
+                hdr_elements++;
         } else {
                 /* create a clone with writeable headroom */
                 new_skb = skb_realloc_headroom(skb, sizeof(struct qeth_hdr_tso)
@@ -2894,22 +2853,28 @@ static int qeth_l3_hard_start_xmit(struct sk_buff *skb, struct net_device *dev)
         /* fix hardware limitation: as long as we do not have sbal
          * chaining we can not send long frag lists
          */
-        if (large_send) {
-                if (qeth_l3_tso_elements(new_skb) + 1 > 16) {
-                        if (skb_linearize(new_skb))
-                                goto tx_drop;
-                        if (card->options.performance_stats)
+        if ((card->info.type != QETH_CARD_TYPE_IQD) &&
+            ((use_tso && !qeth_l3_get_elements_no_tso(card, new_skb, 1)) ||
+             (!use_tso && !qeth_get_elements_no(card, new_skb, 0)))) {
+                int lin_rc = skb_linearize(new_skb);
+
+                if (card->options.performance_stats) {
+                        if (lin_rc)
+                                card->perf_stats.tx_linfail++;
+                        else
                                 card->perf_stats.tx_lin++;
                 }
+                if (lin_rc)
+                        goto tx_drop;
         }
 
-        if (large_send && (cast_type == RTN_UNSPEC)) {
+        if (use_tso) {
                 hdr = (struct qeth_hdr *)skb_push(new_skb,
                                                 sizeof(struct qeth_hdr_tso));
                 memset(hdr, 0, sizeof(struct qeth_hdr_tso));
                 qeth_l3_fill_header(card, hdr, new_skb, ipv, cast_type);
                 qeth_tso_fill_header(card, hdr, new_skb);
-                elements_needed++;
+                hdr_elements++;
         } else {
                 if (data_offset < 0) {
                         hdr = (struct qeth_hdr *)skb_push(new_skb,
@@ -2930,31 +2895,31 @@ static int qeth_l3_hard_start_xmit(struct sk_buff *skb, struct net_device *dev)
                         qeth_l3_hdr_csum(card, hdr, new_skb);
         }
 
-        elems = qeth_get_elements_no(card, new_skb, elements_needed);
-        if (!elems) {
+        elements = use_tso ?
+                   qeth_l3_get_elements_no_tso(card, new_skb, hdr_elements) :
+                   qeth_get_elements_no(card, new_skb, hdr_elements);
+        if (!elements) {
                 if (data_offset >= 0)
                         kmem_cache_free(qeth_core_header_cache, hdr);
                 goto tx_drop;
         }
-        elements_needed += elems;
-        nr_frags = skb_shinfo(new_skb)->nr_frags;
+        elements += hdr_elements;
 
         if (card->info.type != QETH_CARD_TYPE_IQD) {
                 int len;
-                if (large_send)
+                if (use_tso)
                         len = ((unsigned long)tcp_hdr(new_skb) +
-                                tcp_hdr(new_skb)->doff * 4) -
+                                tcp_hdrlen(new_skb)) -
                                 (unsigned long)new_skb->data;
                 else
                         len = sizeof(struct qeth_hdr_layer3);
 
                 if (qeth_hdr_chk_and_bounce(new_skb, &hdr, len))
                         goto tx_drop;
-                rc = qeth_do_send_packet(card, queue, new_skb, hdr,
-                                         elements_needed);
+                rc = qeth_do_send_packet(card, queue, new_skb, hdr, elements);
         } else
                 rc = qeth_do_send_packet_fast(card, queue, new_skb, hdr,
-                                        elements_needed, data_offset, 0);
+                                        elements, data_offset, 0);
 
         if (!rc) {
                 card->stats.tx_packets++;
@@ -2962,7 +2927,8 @@ static int qeth_l3_hard_start_xmit(struct sk_buff *skb, struct net_device *dev)
                 if (new_skb != skb)
                         dev_kfree_skb_any(skb);
                 if (card->options.performance_stats) {
-                        if (large_send) {
+                        nr_frags = skb_shinfo(new_skb)->nr_frags;
+                        if (use_tso) {
                                 card->perf_stats.large_send_bytes += tx_bytes;
                                 card->perf_stats.large_send_cnt++;
                         }
@@ -3048,36 +3014,6 @@ static int qeth_l3_stop(struct net_device *dev)
         return 0;
 }
 
-static netdev_features_t qeth_l3_fix_features(struct net_device *dev,
-        netdev_features_t features)
-{
-        struct qeth_card *card = dev->ml_priv;
-
-        if (!qeth_is_supported(card, IPA_OUTBOUND_CHECKSUM))
-                features &= ~NETIF_F_IP_CSUM;
-        if (!qeth_is_supported(card, IPA_OUTBOUND_TSO))
-                features &= ~NETIF_F_TSO;
-        if (!qeth_is_supported(card, IPA_INBOUND_CHECKSUM))
-                features &= ~NETIF_F_RXCSUM;
-        return features;
-}
-
-static int qeth_l3_set_features(struct net_device *dev,
-        netdev_features_t features)
-{
-        struct qeth_card *card = dev->ml_priv;
-        netdev_features_t changed = dev->features ^ features;
-
-        if (!(changed & NETIF_F_RXCSUM))
-                return 0;
-
-        if (card->state == CARD_STATE_DOWN ||
-            card->state == CARD_STATE_RECOVER)
-                return 0;
-
-        return qeth_set_rx_csum(card, features & NETIF_F_RXCSUM ? 1 : 0);
-}
-
 static const struct ethtool_ops qeth_l3_ethtool_ops = {
         .get_link = ethtool_op_get_link,
         .get_strings = qeth_core_get_strings,
@@ -3120,8 +3056,8 @@ static const struct net_device_ops qeth_l3_netdev_ops = {
         .ndo_set_rx_mode        = qeth_l3_set_multicast_list,
         .ndo_do_ioctl           = qeth_l3_do_ioctl,
         .ndo_change_mtu         = qeth_change_mtu,
-        .ndo_fix_features       = qeth_l3_fix_features,
-        .ndo_set_features       = qeth_l3_set_features,
+        .ndo_fix_features       = qeth_fix_features,
+        .ndo_set_features       = qeth_set_features,
         .ndo_vlan_rx_add_vid    = qeth_l3_vlan_rx_add_vid,
         .ndo_vlan_rx_kill_vid   = qeth_l3_vlan_rx_kill_vid,
         .ndo_tx_timeout         = qeth_tx_timeout,
@@ -3136,8 +3072,8 @@ static const struct net_device_ops qeth_l3_osa_netdev_ops = {
         .ndo_set_rx_mode        = qeth_l3_set_multicast_list,
         .ndo_do_ioctl           = qeth_l3_do_ioctl,
         .ndo_change_mtu         = qeth_change_mtu,
-        .ndo_fix_features       = qeth_l3_fix_features,
-        .ndo_set_features       = qeth_l3_set_features,
+        .ndo_fix_features       = qeth_fix_features,
+        .ndo_set_features       = qeth_set_features,
         .ndo_vlan_rx_add_vid    = qeth_l3_vlan_rx_add_vid,
         .ndo_vlan_rx_kill_vid   = qeth_l3_vlan_rx_kill_vid,
         .ndo_tx_timeout         = qeth_tx_timeout,
@@ -3169,7 +3105,10 @@ static int qeth_l3_setup_netdev(struct qeth_card *card)
                                 card->dev->hw_features = NETIF_F_SG |
                                         NETIF_F_RXCSUM | NETIF_F_IP_CSUM |
                                         NETIF_F_TSO;
-                                card->dev->features = NETIF_F_RXCSUM;
+                                card->dev->vlan_features = NETIF_F_SG |
+                                        NETIF_F_RXCSUM | NETIF_F_IP_CSUM |
+                                        NETIF_F_TSO;
+                                card->dev->features = NETIF_F_SG;
                         }
                 }
         } else if (card->info.type == QETH_CARD_TYPE_IQD) {
@@ -3195,7 +3134,9 @@ static int qeth_l3_setup_netdev(struct qeth_card *card)
                                 NETIF_F_HW_VLAN_CTAG_RX |
                                 NETIF_F_HW_VLAN_CTAG_FILTER;
         netif_keep_dst(card->dev);
-        card->dev->gso_max_size = 15 * PAGE_SIZE;
+        card->dev->gso_max_size = (QETH_MAX_BUFFER_ELEMENTS(card) - 1) *
+                                  PAGE_SIZE;
+        card->dev->gso_max_segs = (QETH_MAX_BUFFER_ELEMENTS(card) - 1);
 
         SET_NETDEV_DEV(card->dev, &card->gdev->dev);
         netif_napi_add(card->dev, &card->napi, qeth_l3_poll, QETH_NAPI_WEIGHT);
@@ -3230,7 +3171,7 @@ static void qeth_l3_remove_device(struct ccwgroup_device *cgdev)
                 card->dev = NULL;
         }
 
-        qeth_l3_clear_ip_list(card, 0);
+        qeth_l3_clear_ip_htable(card, 0);
         qeth_l3_clear_ipato_list(card);
         return;
 }
@@ -3315,7 +3256,7 @@ contin:
         card->state = CARD_STATE_SOFTSETUP;
 
         qeth_set_allowed_threads(card, 0xffffffff, 0);
-        qeth_l3_set_ip_addr_list(card);
+        qeth_l3_recover_ip(card);
         if (card->lan_online)
                 netif_carrier_on(card->dev);
         else
@@ -3516,6 +3457,7 @@ EXPORT_SYMBOL_GPL(qeth_l3_discipline);
 static int qeth_l3_ip_event(struct notifier_block *this,
                             unsigned long event, void *ptr)
 {
+
         struct in_ifaddr *ifa = (struct in_ifaddr *)ptr;
         struct net_device *dev = (struct net_device *)ifa->ifa_dev->dev;
         struct qeth_ipaddr *addr;
@@ -3530,27 +3472,27 @@ static int qeth_l3_ip_event(struct notifier_block *this,
         QETH_CARD_TEXT(card, 3, "ipevent");
 
         addr = qeth_l3_get_addr_buffer(QETH_PROT_IPV4);
-        if (addr != NULL) {
+        if (addr) {
                 addr->u.a4.addr = ifa->ifa_address;
                 addr->u.a4.mask = ifa->ifa_mask;
                 addr->type = QETH_IP_TYPE_NORMAL;
         } else
-                goto out;
+                return NOTIFY_DONE;
 
         switch (event) {
         case NETDEV_UP:
-                if (!qeth_l3_add_ip(card, addr))
-                        kfree(addr);
+                spin_lock_bh(&card->ip_lock);
+                qeth_l3_add_ip(card, addr);
+                spin_unlock_bh(&card->ip_lock);
                 break;
         case NETDEV_DOWN:
-                if (!qeth_l3_delete_ip(card, addr))
-                        kfree(addr);
-                break;
-        default:
+                spin_lock_bh(&card->ip_lock);
+                qeth_l3_delete_ip(card, addr);
+                spin_unlock_bh(&card->ip_lock);
                 break;
         }
-        qeth_l3_set_ip_addr_list(card);
-out:
+
+        kfree(addr);
         return NOTIFY_DONE;
 }
 
@@ -3579,27 +3521,27 @@ static int qeth_l3_ip6_event(struct notifier_block *this,
                 return NOTIFY_DONE;
 
         addr = qeth_l3_get_addr_buffer(QETH_PROT_IPV6);
-        if (addr != NULL) {
+        if (addr) {
                 memcpy(&addr->u.a6.addr, &ifa->addr, sizeof(struct in6_addr));
                 addr->u.a6.pfxlen = ifa->prefix_len;
                 addr->type = QETH_IP_TYPE_NORMAL;
         } else
-                goto out;
+                return NOTIFY_DONE;
 
         switch (event) {
         case NETDEV_UP:
-                if (!qeth_l3_add_ip(card, addr))
-                        kfree(addr);
+                spin_lock_bh(&card->ip_lock);
+                qeth_l3_add_ip(card, addr);
+                spin_unlock_bh(&card->ip_lock);
                 break;
         case NETDEV_DOWN:
-                if (!qeth_l3_delete_ip(card, addr))
-                        kfree(addr);
-                break;
-        default:
+                spin_lock_bh(&card->ip_lock);
+                qeth_l3_delete_ip(card, addr);
+                spin_unlock_bh(&card->ip_lock);
                 break;
         }
-        qeth_l3_set_ip_addr_list(card);
-out:
+
+        kfree(addr);
         return NOTIFY_DONE;
 }
 
diff --git a/drivers/s390/net/qeth_l3_sys.c b/drivers/s390/net/qeth_l3_sys.c
index 386eb7b89b1e..65645b11fc19 100644
--- a/drivers/s390/net/qeth_l3_sys.c
+++ b/drivers/s390/net/qeth_l3_sys.c
@@ -8,6 +8,7 @@
 
 #include <linux/slab.h>
 #include <asm/ebcdic.h>
+#include <linux/hashtable.h>
 #include "qeth_l3.h"
 
 #define QETH_DEVICE_ATTR(_id, _name, _mode, _show, _store) \
@@ -285,19 +286,19 @@ static ssize_t qeth_l3_dev_hsuid_store(struct device *dev,
         if (card->options.hsuid[0]) {
                 /* delete old ip address */
                 addr = qeth_l3_get_addr_buffer(QETH_PROT_IPV6);
-                if (addr != NULL) {
-                        addr->u.a6.addr.s6_addr32[0] = 0xfe800000;
-                        addr->u.a6.addr.s6_addr32[1] = 0x00000000;
-                        for (i = 8; i < 16; i++)
-                                addr->u.a6.addr.s6_addr[i] =
-                                        card->options.hsuid[i - 8];
-                        addr->u.a6.pfxlen = 0;
-                        addr->type = QETH_IP_TYPE_NORMAL;
-                } else
+                if (!addr)
                         return -ENOMEM;
-                if (!qeth_l3_delete_ip(card, addr))
-                        kfree(addr);
-                qeth_l3_set_ip_addr_list(card);
+
+                addr->u.a6.addr.s6_addr32[0] = 0xfe800000;
+                addr->u.a6.addr.s6_addr32[1] = 0x00000000;
+                for (i = 8; i < 16; i++)
+                        addr->u.a6.addr.s6_addr[i] =
+                                card->options.hsuid[i - 8];
+                addr->u.a6.pfxlen = 0;
+                addr->type = QETH_IP_TYPE_NORMAL;
+
+                qeth_l3_delete_ip(card, addr);
+                kfree(addr);
         }
 
         if (strlen(tmp) == 0) {
@@ -328,9 +329,8 @@ static ssize_t qeth_l3_dev_hsuid_store(struct device *dev,
                 addr->type = QETH_IP_TYPE_NORMAL;
         } else
                 return -ENOMEM;
-        if (!qeth_l3_add_ip(card, addr))
-                kfree(addr);
-        qeth_l3_set_ip_addr_list(card);
+        qeth_l3_add_ip(card, addr);
+        kfree(addr);
 
         return count;
 }
@@ -367,8 +367,8 @@ static ssize_t qeth_l3_dev_ipato_enable_store(struct device *dev,
                 struct device_attribute *attr, const char *buf, size_t count)
 {
         struct qeth_card *card = dev_get_drvdata(dev);
-        struct qeth_ipaddr *tmpipa, *t;
-        int rc = 0;
+        struct qeth_ipaddr *addr;
+        int i, rc = 0;
 
         if (!card)
                 return -EINVAL;
@@ -384,21 +384,20 @@ static ssize_t qeth_l3_dev_ipato_enable_store(struct device *dev,
                 card->ipato.enabled = (card->ipato.enabled)? 0 : 1;
         } else if (sysfs_streq(buf, "1")) {
                 card->ipato.enabled = 1;
-                list_for_each_entry_safe(tmpipa, t, card->ip_tbd_list, entry) {
-                        if ((tmpipa->type == QETH_IP_TYPE_NORMAL) &&
-                                qeth_l3_is_addr_covered_by_ipato(card, tmpipa))
-                                tmpipa->set_flags |=
+                hash_for_each(card->ip_htable, i, addr, hnode) {
+                                if ((addr->type == QETH_IP_TYPE_NORMAL) &&
+                                qeth_l3_is_addr_covered_by_ipato(card, addr))
+                                        addr->set_flags |=
                                         QETH_IPA_SETIP_TAKEOVER_FLAG;
-                }
-
+                        }
         } else if (sysfs_streq(buf, "0")) {
                 card->ipato.enabled = 0;
-                list_for_each_entry_safe(tmpipa, t, card->ip_tbd_list, entry) {
-                        if (tmpipa->set_flags &
-                                QETH_IPA_SETIP_TAKEOVER_FLAG)
-                                tmpipa->set_flags &=
-                                        ~QETH_IPA_SETIP_TAKEOVER_FLAG;
-                }
+                hash_for_each(card->ip_htable, i, addr, hnode) {
+                        if (addr->set_flags &
+                        QETH_IPA_SETIP_TAKEOVER_FLAG)
+                                addr->set_flags &=
+                                ~QETH_IPA_SETIP_TAKEOVER_FLAG;
+                        }
         } else
                 rc = -EINVAL;
 out:
@@ -452,7 +451,6 @@ static ssize_t qeth_l3_dev_ipato_add_show(char *buf, struct qeth_card *card,
                         enum qeth_prot_versions proto)
 {
         struct qeth_ipato_entry *ipatoe;
-        unsigned long flags;
         char addr_str[40];
         int entry_len; /* length of 1 entry string, differs between v4 and v6 */
         int i = 0;
@@ -460,7 +458,7 @@ static ssize_t qeth_l3_dev_ipato_add_show(char *buf, struct qeth_card *card,
         entry_len = (proto == QETH_PROT_IPV4)? 12 : 40;
         /* add strlen for "/<mask>\n" */
         entry_len += (proto == QETH_PROT_IPV4)? 5 : 6;
-        spin_lock_irqsave(&card->ip_lock, flags);
+        spin_lock_bh(&card->ip_lock);
         list_for_each_entry(ipatoe, &card->ipato.entries, entry) {
                 if (ipatoe->proto != proto)
                         continue;
@@ -473,7 +471,7 @@ static ssize_t qeth_l3_dev_ipato_add_show(char *buf, struct qeth_card *card,
                 i += snprintf(buf + i, PAGE_SIZE - i,
                               "%s/%i\n", addr_str, ipatoe->mask_bits);
         }
-        spin_unlock_irqrestore(&card->ip_lock, flags);
+        spin_unlock_bh(&card->ip_lock);
         i += snprintf(buf + i, PAGE_SIZE - i, "\n");
 
         return i;
@@ -689,15 +687,15 @@ static ssize_t qeth_l3_dev_vipa_add_show(char *buf, struct qeth_card *card,
                         enum qeth_prot_versions proto)
 {
         struct qeth_ipaddr *ipaddr;
+        struct hlist_node  *tmp;
         char addr_str[40];
         int entry_len; /* length of 1 entry string, differs between v4 and v6 */
-        unsigned long flags;
         int i = 0;
 
         entry_len = (proto == QETH_PROT_IPV4)? 12 : 40;
         entry_len += 2; /* \n + terminator */
-        spin_lock_irqsave(&card->ip_lock, flags);
-        list_for_each_entry(ipaddr, &card->ip_list, entry) {
+        spin_lock_bh(&card->ip_lock);
+        hash_for_each_safe(card->ip_htable, i, tmp, ipaddr, hnode) {
                 if (ipaddr->proto != proto)
                         continue;
                 if (ipaddr->type != QETH_IP_TYPE_VIPA)
@@ -711,7 +709,7 @@ static ssize_t qeth_l3_dev_vipa_add_show(char *buf, struct qeth_card *card,
                         addr_str);
                 i += snprintf(buf + i, PAGE_SIZE - i, "%s\n", addr_str);
         }
-        spin_unlock_irqrestore(&card->ip_lock, flags);
+        spin_unlock_bh(&card->ip_lock);
         i += snprintf(buf + i, PAGE_SIZE - i, "\n");
 
         return i;
@@ -851,15 +849,15 @@ static ssize_t qeth_l3_dev_rxip_add_show(char *buf, struct qeth_card *card,
                        enum qeth_prot_versions proto)
 {
         struct qeth_ipaddr *ipaddr;
+        struct hlist_node *tmp;
         char addr_str[40];
         int entry_len; /* length of 1 entry string, differs between v4 and v6 */
-        unsigned long flags;
         int i = 0;
 
         entry_len = (proto == QETH_PROT_IPV4)? 12 : 40;
         entry_len += 2; /* \n + terminator */
-        spin_lock_irqsave(&card->ip_lock, flags);
-        list_for_each_entry(ipaddr, &card->ip_list, entry) {
+        spin_lock_bh(&card->ip_lock);
+        hash_for_each_safe(card->ip_htable, i, tmp, ipaddr, hnode) {
                 if (ipaddr->proto != proto)
                         continue;
                 if (ipaddr->type != QETH_IP_TYPE_RXIP)
@@ -873,7 +871,7 @@ static ssize_t qeth_l3_dev_rxip_add_show(char *buf, struct qeth_card *card,
                         addr_str);
                 i += snprintf(buf + i, PAGE_SIZE - i, "%s\n", addr_str);
         }
-        spin_unlock_irqrestore(&card->ip_lock, flags);
+        spin_unlock_bh(&card->ip_lock);
         i += snprintf(buf + i, PAGE_SIZE - i, "\n");
 
         return i;
diff --git a/drivers/soc/fsl/qe/Kconfig b/drivers/soc/fsl/qe/Kconfig
index 20978f2058a6..73a2e08b47ef 100644
--- a/drivers/soc/fsl/qe/Kconfig
+++ b/drivers/soc/fsl/qe/Kconfig
@@ -22,7 +22,7 @@ config UCC_SLOW
 
 config UCC_FAST
         bool
-        default y if UCC_GETH
+        default y if UCC_GETH || QE_TDM
         help
           This option provides qe_lib support to UCC fast
           protocols: HDLC, Ethernet, ATM, transparent
@@ -31,6 +31,10 @@ config UCC
         bool
         default y if UCC_FAST || UCC_SLOW
 
+config QE_TDM
+        bool
+        default y if FSL_UCC_HDLC
+
 config QE_USB
         bool
         default y if USB_FSL_QE
diff --git a/drivers/soc/fsl/qe/Makefile b/drivers/soc/fsl/qe/Makefile
index ffac5410c5c7..2031d385bc7e 100644
--- a/drivers/soc/fsl/qe/Makefile
+++ b/drivers/soc/fsl/qe/Makefile
@@ -6,5 +6,6 @@ obj-$(CONFIG_CPM)	+= qe_common.o
 obj-$(CONFIG_UCC)       += ucc.o
 obj-$(CONFIG_UCC_SLOW)  += ucc_slow.o
 obj-$(CONFIG_UCC_FAST)  += ucc_fast.o
+obj-$(CONFIG_QE_TDM)    += qe_tdm.o
 obj-$(CONFIG_QE_USB)    += usb.o
 obj-$(CONFIG_QE_GPIO)   += gpio.o
diff --git a/drivers/soc/fsl/qe/qe.c b/drivers/soc/fsl/qe/qe.c
index 709fc63809e5..7026507e6f1d 100644
--- a/drivers/soc/fsl/qe/qe.c
+++ b/drivers/soc/fsl/qe/qe.c
@@ -239,6 +239,12 @@ enum qe_clock qe_clock_source(const char *source)
         if (strcasecmp(source, "none") == 0)
                 return QE_CLK_NONE;
 
+        if (strcmp(source, "tsync_pin") == 0)
+                return QE_TSYNC_PIN;
+
+        if (strcmp(source, "rsync_pin") == 0)
+                return QE_RSYNC_PIN;
+
         if (strncasecmp(source, "brg", 3) == 0) {
                 i = simple_strtoul(source + 3, NULL, 10);
                 if ((i >= 1) && (i <= 16))
diff --git a/drivers/soc/fsl/qe/qe_tdm.c b/drivers/soc/fsl/qe/qe_tdm.c
new file mode 100644
index 000000000000..5e48b1470178
--- /dev/null
+++ b/drivers/soc/fsl/qe/qe_tdm.c
@@ -0,0 +1,276 @@
+/*
+ * Copyright (C) 2015 Freescale Semiconductor, Inc. All rights reserved.
+ *
+ * Authors:     Zhao Qiang <qiang.zhao@nxp.com>
+ *
+ * Description:
+ * QE TDM API Set - TDM specific routines implementations.
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/of_platform.h>
+#include <soc/fsl/qe/qe_tdm.h>
+
+static int set_tdm_framer(const char *tdm_framer_type)
+{
+        if (strcmp(tdm_framer_type, "e1") == 0)
+                return TDM_FRAMER_E1;
+        else if (strcmp(tdm_framer_type, "t1") == 0)
+                return TDM_FRAMER_T1;
+        else
+                return -EINVAL;
+}
+
+static void set_si_param(struct ucc_tdm *utdm, struct ucc_tdm_info *ut_info)
+{
+        struct si_mode_info *si_info = &ut_info->si_info;
+
+        if (utdm->tdm_mode == TDM_INTERNAL_LOOPBACK) {
+                si_info->simr_crt = 1;
+                si_info->simr_rfsd = 0;
+        }
+}
+
+int ucc_of_parse_tdm(struct device_node *np, struct ucc_tdm *utdm,
+                     struct ucc_tdm_info *ut_info)
+{
+        const char *sprop;
+        int ret = 0;
+        u32 val;
+        struct resource *res;
+        struct device_node *np2;
+        static int siram_init_flag;
+        struct platform_device *pdev;
+
+        sprop = of_get_property(np, "fsl,rx-sync-clock", NULL);
+        if (sprop) {
+                ut_info->uf_info.rx_sync = qe_clock_source(sprop);
+                if ((ut_info->uf_info.rx_sync < QE_CLK_NONE) ||
+                    (ut_info->uf_info.rx_sync > QE_RSYNC_PIN)) {
+                        pr_err("QE-TDM: Invalid rx-sync-clock property\n");
+                        return -EINVAL;
+                }
+        } else {
+                pr_err("QE-TDM: Invalid rx-sync-clock property\n");
+                return -EINVAL;
+        }
+
+        sprop = of_get_property(np, "fsl,tx-sync-clock", NULL);
+        if (sprop) {
+                ut_info->uf_info.tx_sync = qe_clock_source(sprop);
+                if ((ut_info->uf_info.tx_sync < QE_CLK_NONE) ||
+                    (ut_info->uf_info.tx_sync > QE_TSYNC_PIN)) {
+                        pr_err("QE-TDM: Invalid tx-sync-clock property\n");
+                return -EINVAL;
+                }
+        } else {
+                pr_err("QE-TDM: Invalid tx-sync-clock property\n");
+                return -EINVAL;
+        }
+
+        ret = of_property_read_u32_index(np, "fsl,tx-timeslot-mask", 0, &val);
+        if (ret) {
+                pr_err("QE-TDM: Invalid tx-timeslot-mask property\n");
+                return -EINVAL;
+        }
+        utdm->tx_ts_mask = val;
+
+        ret = of_property_read_u32_index(np, "fsl,rx-timeslot-mask", 0, &val);
+        if (ret) {
+                ret = -EINVAL;
+                pr_err("QE-TDM: Invalid rx-timeslot-mask property\n");
+                return ret;
+        }
+        utdm->rx_ts_mask = val;
+
+        ret = of_property_read_u32_index(np, "fsl,tdm-id", 0, &val);
+        if (ret) {
+                ret = -EINVAL;
+                pr_err("QE-TDM: No fsl,tdm-id property for this UCC\n");
+                return ret;
+        }
+        utdm->tdm_port = val;
+        ut_info->uf_info.tdm_num = utdm->tdm_port;
+
+        if (of_get_property(np, "fsl,tdm-internal-loopback", NULL))
+                utdm->tdm_mode = TDM_INTERNAL_LOOPBACK;
+        else
+                utdm->tdm_mode = TDM_NORMAL;
+
+        sprop = of_get_property(np, "fsl,tdm-framer-type", NULL);
+        if (!sprop) {
+                ret = -EINVAL;
+                pr_err("QE-TDM: No tdm-framer-type property for UCC\n");
+                return ret;
+        }
+        ret = set_tdm_framer(sprop);
+        if (ret < 0)
+                return -EINVAL;
+        utdm->tdm_framer_type = ret;
+
+        ret = of_property_read_u32_index(np, "fsl,siram-entry-id", 0, &val);
+        if (ret) {
+                ret = -EINVAL;
+                pr_err("QE-TDM: No siram entry id for UCC\n");
+                return ret;
+        }
+        utdm->siram_entry_id = val;
+
+        set_si_param(utdm, ut_info);
+
+        np2 = of_find_compatible_node(NULL, NULL, "fsl,t1040-qe-si");
+        if (!np2)
+                return -EINVAL;
+
+        pdev = of_find_device_by_node(np2);
+        if (!pdev) {
+                pr_err("%s: failed to lookup pdev\n", np2->name);
+                of_node_put(np2);
+                return -EINVAL;
+        }
+
+        of_node_put(np2);
+        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+        utdm->si_regs = devm_ioremap_resource(&pdev->dev, res);
+        if (IS_ERR(utdm->si_regs)) {
+                ret = PTR_ERR(utdm->si_regs);
+                goto err_miss_siram_property;
+        }
+
+        np2 = of_find_compatible_node(NULL, NULL, "fsl,t1040-qe-siram");
+        if (!np2) {
+                ret = -EINVAL;
+                goto err_miss_siram_property;
+        }
+
+        pdev = of_find_device_by_node(np2);
+        if (!pdev) {
+                ret = -EINVAL;
+                pr_err("%s: failed to lookup pdev\n", np2->name);
+                of_node_put(np2);
+                goto err_miss_siram_property;
+        }
+
+        of_node_put(np2);
+        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+        utdm->siram = devm_ioremap_resource(&pdev->dev, res);
+        if (IS_ERR(utdm->siram)) {
+                ret = PTR_ERR(utdm->siram);
+                goto err_miss_siram_property;
+        }
+
+        if (siram_init_flag == 0) {
+                memset_io(utdm->siram, 0,  res->end - res->start + 1);
+                siram_init_flag = 1;
+        }
+
+        return ret;
+
+err_miss_siram_property:
+        devm_iounmap(&pdev->dev, utdm->si_regs);
+        return ret;
+}
+
+void ucc_tdm_init(struct ucc_tdm *utdm, struct ucc_tdm_info *ut_info)
+{
+        struct si1 __iomem *si_regs;
+        u16 __iomem *siram;
+        u16 siram_entry_valid;
+        u16 siram_entry_closed;
+        u16 ucc_num;
+        u8 csel;
+        u16 sixmr;
+        u16 tdm_port;
+        u32 siram_entry_id;
+        u32 mask;
+        int i;
+
+        si_regs = utdm->si_regs;
+        siram = utdm->siram;
+        ucc_num = ut_info->uf_info.ucc_num;
+        tdm_port = utdm->tdm_port;
+        siram_entry_id = utdm->siram_entry_id;
+
+        if (utdm->tdm_framer_type == TDM_FRAMER_T1)
+                utdm->num_of_ts = 24;
+        if (utdm->tdm_framer_type == TDM_FRAMER_E1)
+                utdm->num_of_ts = 32;
+
+        /* set siram table */
+        csel = (ucc_num < 4) ? ucc_num + 9 : ucc_num - 3;
+
+        siram_entry_valid = SIR_CSEL(csel) | SIR_BYTE | SIR_CNT(0);
+        siram_entry_closed = SIR_IDLE | SIR_BYTE | SIR_CNT(0);
+
+        for (i = 0; i < utdm->num_of_ts; i++) {
+                mask = 0x01 << i;
+
+                if (utdm->tx_ts_mask & mask)
+                        iowrite16be(siram_entry_valid,
+                                    &siram[siram_entry_id * 32 + i]);
+                else
+                        iowrite16be(siram_entry_closed,
+                                    &siram[siram_entry_id * 32 + i]);
+
+                if (utdm->rx_ts_mask & mask)
+                        iowrite16be(siram_entry_valid,
+                                    &siram[siram_entry_id * 32 + 0x200 +  i]);
+                else
+                        iowrite16be(siram_entry_closed,
+                                    &siram[siram_entry_id * 32 + 0x200 +  i]);
+        }
+
+        setbits16(&siram[(siram_entry_id * 32) + (utdm->num_of_ts - 1)],
+                  SIR_LAST);
+        setbits16(&siram[(siram_entry_id * 32) + 0x200 + (utdm->num_of_ts - 1)],
+                  SIR_LAST);
+
+        /* Set SIxMR register */
+        sixmr = SIMR_SAD(siram_entry_id);
+
+        sixmr &= ~SIMR_SDM_MASK;
+
+        if (utdm->tdm_mode == TDM_INTERNAL_LOOPBACK)
+                sixmr |= SIMR_SDM_INTERNAL_LOOPBACK;
+        else
+                sixmr |= SIMR_SDM_NORMAL;
+
+        sixmr |= SIMR_RFSD(ut_info->si_info.simr_rfsd) |
+                        SIMR_TFSD(ut_info->si_info.simr_tfsd);
+
+        if (ut_info->si_info.simr_crt)
+                sixmr |= SIMR_CRT;
+        if (ut_info->si_info.simr_sl)
+                sixmr |= SIMR_SL;
+        if (ut_info->si_info.simr_ce)
+                sixmr |= SIMR_CE;
+        if (ut_info->si_info.simr_fe)
+                sixmr |= SIMR_FE;
+        if (ut_info->si_info.simr_gm)
+                sixmr |= SIMR_GM;
+
+        switch (tdm_port) {
+        case 0:
+                iowrite16be(sixmr, &si_regs->sixmr1[0]);
+                break;
+        case 1:
+                iowrite16be(sixmr, &si_regs->sixmr1[1]);
+                break;
+        case 2:
+                iowrite16be(sixmr, &si_regs->sixmr1[2]);
+                break;
+        case 3:
+                iowrite16be(sixmr, &si_regs->sixmr1[3]);
+                break;
+        default:
+                pr_err("QE-TDM: can not find tdm sixmr reg\n");
+                break;
+        }
+}
diff --git a/drivers/soc/fsl/qe/ucc.c b/drivers/soc/fsl/qe/ucc.c
index b59d3358f9bd..c646d8713861 100644
--- a/drivers/soc/fsl/qe/ucc.c
+++ b/drivers/soc/fsl/qe/ucc.c
@@ -25,6 +25,12 @@
 #include <soc/fsl/qe/qe.h>
 #include <soc/fsl/qe/ucc.h>
 
+#define UCC_TDM_NUM 8
+#define RX_SYNC_SHIFT_BASE 30
+#define TX_SYNC_SHIFT_BASE 14
+#define RX_CLK_SHIFT_BASE 28
+#define TX_CLK_SHIFT_BASE 12
+
 int ucc_set_qe_mux_mii_mng(unsigned int ucc_num)
 {
         unsigned long flags;
@@ -210,3 +216,447 @@ int ucc_set_qe_mux_rxtx(unsigned int ucc_num, enum qe_clock clock,
 
         return 0;
 }
+
+static int ucc_get_tdm_common_clk(u32 tdm_num, enum qe_clock clock)
+{
+        int clock_bits = -EINVAL;
+
+        /*
+         * for TDM[0, 1, 2, 3], TX and RX use  common
+         * clock source BRG3,4 and CLK1,2
+         * for TDM[4, 5, 6, 7], TX and RX use  common
+         * clock source BRG12,13 and CLK23,24
+         */
+        switch (tdm_num) {
+        case 0:
+        case 1:
+        case 2:
+        case 3:
+                switch (clock) {
+                case QE_BRG3:
+                        clock_bits = 1;
+                        break;
+                case QE_BRG4:
+                        clock_bits = 2;
+                        break;
+                case QE_CLK1:
+                        clock_bits = 4;
+                        break;
+                case QE_CLK2:
+                        clock_bits = 5;
+                        break;
+                default:
+                        break;
+                }
+                break;
+        case 4:
+        case 5:
+        case 6:
+        case 7:
+                switch (clock) {
+                case QE_BRG12:
+                        clock_bits = 1;
+                        break;
+                case QE_BRG13:
+                        clock_bits = 2;
+                        break;
+                case QE_CLK23:
+                        clock_bits = 4;
+                        break;
+                case QE_CLK24:
+                        clock_bits = 5;
+                        break;
+                default:
+                        break;
+                }
+                break;
+        default:
+                break;
+        }
+
+        return clock_bits;
+}
+
+static int ucc_get_tdm_rx_clk(u32 tdm_num, enum qe_clock clock)
+{
+        int clock_bits = -EINVAL;
+
+        switch (tdm_num) {
+        case 0:
+                switch (clock) {
+                case QE_CLK3:
+                        clock_bits = 6;
+                        break;
+                case QE_CLK8:
+                        clock_bits = 7;
+                        break;
+                default:
+                        break;
+                }
+                break;
+        case 1:
+                switch (clock) {
+                case QE_CLK5:
+                        clock_bits = 6;
+                        break;
+                case QE_CLK10:
+                        clock_bits = 7;
+                        break;
+                default:
+                        break;
+                }
+                break;
+        case 2:
+                switch (clock) {
+                case QE_CLK7:
+                        clock_bits = 6;
+                        break;
+                case QE_CLK12:
+                        clock_bits = 7;
+                        break;
+                default:
+                        break;
+                }
+                break;
+        case 3:
+                switch (clock) {
+                case QE_CLK9:
+                        clock_bits = 6;
+                        break;
+                case QE_CLK14:
+                        clock_bits = 7;
+                        break;
+                default:
+                        break;
+                }
+                break;
+        case 4:
+                switch (clock) {
+                case QE_CLK11:
+                        clock_bits = 6;
+                        break;
+                case QE_CLK16:
+                        clock_bits = 7;
+                        break;
+                default:
+                        break;
+                }
+                break;
+        case 5:
+                switch (clock) {
+                case QE_CLK13:
+                        clock_bits = 6;
+                        break;
+                case QE_CLK18:
+                        clock_bits = 7;
+                        break;
+                default:
+                        break;
+                }
+                break;
+        case 6:
+                switch (clock) {
+                case QE_CLK15:
+                        clock_bits = 6;
+                        break;
+                case QE_CLK20:
+                        clock_bits = 7;
+                        break;
+                default:
+                        break;
+                }
+                break;
+        case 7:
+                switch (clock) {
+                case QE_CLK17:
+                        clock_bits = 6;
+                        break;
+                case QE_CLK22:
+                        clock_bits = 7;
+                        break;
+                default:
+                        break;
+                }
+                break;
+        }
+
+        return clock_bits;
+}
+
+static int ucc_get_tdm_tx_clk(u32 tdm_num, enum qe_clock clock)
+{
+        int clock_bits = -EINVAL;
+
+        switch (tdm_num) {
+        case 0:
+                switch (clock) {
+                case QE_CLK4:
+                        clock_bits = 6;
+                        break;
+                case QE_CLK9:
+                        clock_bits = 7;
+                        break;
+                default:
+                        break;
+                }
+                break;
+        case 1:
+                switch (clock) {
+                case QE_CLK6:
+                        clock_bits = 6;
+                        break;
+                case QE_CLK11:
+                        clock_bits = 7;
+                        break;
+                default:
+                        break;
+                }
+                break;
+        case 2:
+                switch (clock) {
+                case QE_CLK8:
+                        clock_bits = 6;
+                        break;
+                case QE_CLK13:
+                        clock_bits = 7;
+                        break;
+                default:
+                        break;
+                }
+                break;
+        case 3:
+                switch (clock) {
+                case QE_CLK10:
+                        clock_bits = 6;
+                        break;
+                case QE_CLK15:
+                        clock_bits = 7;
+                        break;
+                default:
+                        break;
+                }
+                break;
+        case 4:
+                switch (clock) {
+                case QE_CLK12:
+                        clock_bits = 6;
+                        break;
+                case QE_CLK17:
+                        clock_bits = 7;
+                        break;
+                default:
+                        break;
+                }
+                break;
+        case 5:
+                switch (clock) {
+                case QE_CLK14:
+                        clock_bits = 6;
+                        break;
+                case QE_CLK19:
+                        clock_bits = 7;
+                        break;
+                default:
+                        break;
+                }
+                break;
+        case 6:
+                switch (clock) {
+                case QE_CLK16:
+                        clock_bits = 6;
+                        break;
+                case QE_CLK21:
+                        clock_bits = 7;
+                        break;
+                default:
+                        break;
+                }
+                break;
+        case 7:
+                switch (clock) {
+                case QE_CLK18:
+                        clock_bits = 6;
+                        break;
+                case QE_CLK3:
+                        clock_bits = 7;
+                        break;
+                default:
+                        break;
+                }
+                break;
+        }
+
+        return clock_bits;
+}
+
+/* tdm_num: TDM A-H port num is 0-7 */
+static int ucc_get_tdm_rxtx_clk(enum comm_dir mode, u32 tdm_num,
+                                enum qe_clock clock)
+{
+        int clock_bits;
+
+        clock_bits = ucc_get_tdm_common_clk(tdm_num, clock);
+        if (clock_bits > 0)
+                return clock_bits;
+        if (mode == COMM_DIR_RX)
+                clock_bits = ucc_get_tdm_rx_clk(tdm_num, clock);
+        if (mode == COMM_DIR_TX)
+                clock_bits = ucc_get_tdm_tx_clk(tdm_num, clock);
+        return clock_bits;
+}
+
+static u32 ucc_get_tdm_clk_shift(enum comm_dir mode, u32 tdm_num)
+{
+        u32 shift;
+
+        shift = (mode == COMM_DIR_RX) ? RX_CLK_SHIFT_BASE : TX_CLK_SHIFT_BASE;
+        if (tdm_num < 4)
+                shift -= tdm_num * 4;
+        else
+                shift -= (tdm_num - 4) * 4;
+
+        return shift;
+}
+
+int ucc_set_tdm_rxtx_clk(u32 tdm_num, enum qe_clock clock,
+                         enum comm_dir mode)
+{
+        int clock_bits;
+        u32 shift;
+        struct qe_mux __iomem *qe_mux_reg;
+         __be32 __iomem *cmxs1cr;
+
+        qe_mux_reg = &qe_immr->qmx;
+
+        if (tdm_num > 7 || tdm_num < 0)
+                return -EINVAL;
+
+        /* The communications direction must be RX or TX */
+        if (mode != COMM_DIR_RX && mode != COMM_DIR_TX)
+                return -EINVAL;
+
+        clock_bits = ucc_get_tdm_rxtx_clk(mode, tdm_num, clock);
+        if (clock_bits < 0)
+                return -EINVAL;
+
+        shift = ucc_get_tdm_clk_shift(mode, tdm_num);
+
+        cmxs1cr = (tdm_num < 4) ? &qe_mux_reg->cmxsi1cr_l :
+                                  &qe_mux_reg->cmxsi1cr_h;
+
+        qe_clrsetbits32(cmxs1cr, QE_CMXUCR_TX_CLK_SRC_MASK << shift,
+                        clock_bits << shift);
+
+        return 0;
+}
+
+static int ucc_get_tdm_sync_source(u32 tdm_num, enum qe_clock clock,
+                                   enum comm_dir mode)
+{
+        int source = -EINVAL;
+
+        if (mode == COMM_DIR_RX && clock == QE_RSYNC_PIN) {
+                source = 0;
+                return source;
+        }
+        if (mode == COMM_DIR_TX && clock == QE_TSYNC_PIN) {
+                source = 0;
+                return source;
+        }
+
+        switch (tdm_num) {
+        case 0:
+        case 1:
+                switch (clock) {
+                case QE_BRG9:
+                        source = 1;
+                        break;
+                case QE_BRG10:
+                        source = 2;
+                        break;
+                default:
+                        break;
+                }
+                break;
+        case 2:
+        case 3:
+                switch (clock) {
+                case QE_BRG9:
+                        source = 1;
+                        break;
+                case QE_BRG11:
+                        source = 2;
+                        break;
+                default:
+                        break;
+                }
+                break;
+        case 4:
+        case 5:
+                switch (clock) {
+                case QE_BRG13:
+                        source = 1;
+                        break;
+                case QE_BRG14:
+                        source = 2;
+                        break;
+                default:
+                        break;
+                }
+                break;
+        case 6:
+        case 7:
+                switch (clock) {
+                case QE_BRG13:
+                        source = 1;
+                        break;
+                case QE_BRG15:
+                        source = 2;
+                        break;
+                default:
+                        break;
+                }
+                break;
+        }
+
+        return source;
+}
+
+static u32 ucc_get_tdm_sync_shift(enum comm_dir mode, u32 tdm_num)
+{
+        u32 shift;
+
+        shift = (mode == COMM_DIR_RX) ? RX_SYNC_SHIFT_BASE : RX_SYNC_SHIFT_BASE;
+        shift -= tdm_num * 2;
+
+        return shift;
+}
+
+int ucc_set_tdm_rxtx_sync(u32 tdm_num, enum qe_clock clock,
+                          enum comm_dir mode)
+{
+        int source;
+        u32 shift;
+        struct qe_mux *qe_mux_reg;
+
+        qe_mux_reg = &qe_immr->qmx;
+
+        if (tdm_num >= UCC_TDM_NUM)
+                return -EINVAL;
+
+        /* The communications direction must be RX or TX */
+        if (mode != COMM_DIR_RX && mode != COMM_DIR_TX)
+                return -EINVAL;
+
+        source = ucc_get_tdm_sync_source(tdm_num, clock, mode);
+        if (source < 0)
+                return -EINVAL;
+
+        shift = ucc_get_tdm_sync_shift(mode, tdm_num);
+
+        qe_clrsetbits32(&qe_mux_reg->cmxsi1syr,
+                        QE_CMXUCR_TX_CLK_SRC_MASK << shift,
+                        source << shift);
+
+        return 0;
+}
diff --git a/drivers/soc/fsl/qe/ucc_fast.c b/drivers/soc/fsl/qe/ucc_fast.c
index a7689310fe40..83d8d16e3a69 100644
--- a/drivers/soc/fsl/qe/ucc_fast.c
+++ b/drivers/soc/fsl/qe/ucc_fast.c
@@ -327,6 +327,42 @@ int ucc_fast_init(struct ucc_fast_info * uf_info, struct ucc_fast_private ** ucc
                         ucc_fast_free(uccf);
                         return -EINVAL;
                 }
+        } else {
+                /* tdm Rx clock routing */
+                if ((uf_info->rx_clock != QE_CLK_NONE) &&
+                    ucc_set_tdm_rxtx_clk(uf_info->tdm_num, uf_info->rx_clock,
+                                         COMM_DIR_RX)) {
+                        pr_err("%s: illegal value for RX clock", __func__);
+                        ucc_fast_free(uccf);
+                        return -EINVAL;
+                }
+
+                /* tdm Tx clock routing */
+                if ((uf_info->tx_clock != QE_CLK_NONE) &&
+                    ucc_set_tdm_rxtx_clk(uf_info->tdm_num, uf_info->tx_clock,
+                                         COMM_DIR_TX)) {
+                        pr_err("%s: illegal value for TX clock", __func__);
+                        ucc_fast_free(uccf);
+                        return -EINVAL;
+                }
+
+                /* tdm Rx sync clock routing */
+                if ((uf_info->rx_sync != QE_CLK_NONE) &&
+                    ucc_set_tdm_rxtx_sync(uf_info->tdm_num, uf_info->rx_sync,
+                                          COMM_DIR_RX)) {
+                        pr_err("%s: illegal value for RX clock", __func__);
+                        ucc_fast_free(uccf);
+                        return -EINVAL;
+                }
+
+                /* tdm Tx sync clock routing */
+                if ((uf_info->tx_sync != QE_CLK_NONE) &&
+                    ucc_set_tdm_rxtx_sync(uf_info->tdm_num, uf_info->tx_sync,
+                                          COMM_DIR_TX)) {
+                        pr_err("%s: illegal value for TX clock", __func__);
+                        ucc_fast_free(uccf);
+                        return -EINVAL;
+                }
         }
 
         /* Set interrupt mask register at UCC level. */
diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
index f744eeb3e2b4..1d3e45f84549 100644
--- a/drivers/vhost/net.c
+++ b/drivers/vhost/net.c
@@ -301,6 +301,32 @@ static bool vhost_can_busy_poll(struct vhost_dev *dev,
                !vhost_has_work(dev);
 }
 
+static void vhost_net_disable_vq(struct vhost_net *n,
+                                 struct vhost_virtqueue *vq)
+{
+        struct vhost_net_virtqueue *nvq =
+                container_of(vq, struct vhost_net_virtqueue, vq);
+        struct vhost_poll *poll = n->poll + (nvq - n->vqs);
+        if (!vq->private_data)
+                return;
+        vhost_poll_stop(poll);
+}
+
+static int vhost_net_enable_vq(struct vhost_net *n,
+                                struct vhost_virtqueue *vq)
+{
+        struct vhost_net_virtqueue *nvq =
+                container_of(vq, struct vhost_net_virtqueue, vq);
+        struct vhost_poll *poll = n->poll + (nvq - n->vqs);
+        struct socket *sock;
+
+        sock = vq->private_data;
+        if (!sock)
+                return 0;
+
+        return vhost_poll_start(poll, sock->file);
+}
+
 static int vhost_net_tx_get_vq_desc(struct vhost_net *net,
                                     struct vhost_virtqueue *vq,
                                     struct iovec iov[], unsigned int iov_size,
@@ -613,6 +639,7 @@ static void handle_rx(struct vhost_net *net)
         if (!sock)
                 goto out;
         vhost_disable_notify(&net->dev, vq);
+        vhost_net_disable_vq(net, vq);
 
         vhost_hlen = nvq->vhost_hlen;
         sock_hlen = nvq->sock_hlen;
@@ -629,7 +656,7 @@ static void handle_rx(struct vhost_net *net)
                                         likely(mergeable) ? UIO_MAXIOV : 1);
                 /* On error, stop handling until the next kick. */
                 if (unlikely(headcount < 0))
-                        break;
+                        goto out;
                 /* On overrun, truncate and discard */
                 if (unlikely(headcount > UIO_MAXIOV)) {
                         iov_iter_init(&msg.msg_iter, READ, vq->iov, 1, 1);
@@ -648,7 +675,7 @@ static void handle_rx(struct vhost_net *net)
                         }
                         /* Nothing new?  Wait for eventfd to tell us
                          * they refilled. */
-                        break;
+                        goto out;
                 }
                 /* We don't need to be notified again. */
                 iov_iter_init(&msg.msg_iter, READ, vq->iov, in, vhost_len);
@@ -676,7 +703,7 @@ static void handle_rx(struct vhost_net *net)
                                          &fixup) != sizeof(hdr)) {
                                 vq_err(vq, "Unable to write vnet_hdr "
                                        "at addr %p\n", vq->iov->iov_base);
-                                break;
+                                goto out;
                         }
                 } else {
                         /* Header came from socket; we'll need to patch
@@ -692,7 +719,7 @@ static void handle_rx(struct vhost_net *net)
                                  &fixup) != sizeof num_buffers) {
                         vq_err(vq, "Failed num_buffers write");
                         vhost_discard_vq_desc(vq, headcount);
-                        break;
+                        goto out;
                 }
                 vhost_add_used_and_signal_n(&net->dev, vq, vq->heads,
                                             headcount);
@@ -701,9 +728,10 @@ static void handle_rx(struct vhost_net *net)
                 total_len += vhost_len;
                 if (unlikely(total_len >= VHOST_NET_WEIGHT)) {
                         vhost_poll_queue(&vq->poll);
-                        break;
+                        goto out;
                 }
         }
+        vhost_net_enable_vq(net, vq);
 out:
         mutex_unlock(&vq->mutex);
 }
@@ -782,32 +810,6 @@ static int vhost_net_open(struct inode *inode, struct file *f)
         return 0;
 }
 
-static void vhost_net_disable_vq(struct vhost_net *n,
-                                 struct vhost_virtqueue *vq)
-{
-        struct vhost_net_virtqueue *nvq =
-                container_of(vq, struct vhost_net_virtqueue, vq);
-        struct vhost_poll *poll = n->poll + (nvq - n->vqs);
-        if (!vq->private_data)
-                return;
-        vhost_poll_stop(poll);
-}
-
-static int vhost_net_enable_vq(struct vhost_net *n,
-                                struct vhost_virtqueue *vq)
-{
-        struct vhost_net_virtqueue *nvq =
-                container_of(vq, struct vhost_net_virtqueue, vq);
-        struct vhost_poll *poll = n->poll + (nvq - n->vqs);
-        struct socket *sock;
-
-        sock = vq->private_data;
-        if (!sock)
-                return 0;
-
-        return vhost_poll_start(poll, sock->file);
-}
-
 static struct socket *vhost_net_stop_vq(struct vhost_net *n,
                                         struct vhost_virtqueue *vq)
 {
diff --git a/fs/afs/rxrpc.c b/fs/afs/rxrpc.c
index 63cd9f939f19..4832de84d52c 100644
--- a/fs/afs/rxrpc.c
+++ b/fs/afs/rxrpc.c
@@ -85,18 +85,14 @@ int afs_open_socket(void)
 
         skb_queue_head_init(&afs_incoming_calls);
 
+        ret = -ENOMEM;
         afs_async_calls = create_singlethread_workqueue("kafsd");
-        if (!afs_async_calls) {
-                _leave(" = -ENOMEM [wq]");
-                return -ENOMEM;
-        }
+        if (!afs_async_calls)
+                goto error_0;
 
         ret = sock_create_kern(&init_net, AF_RXRPC, SOCK_DGRAM, PF_INET, &socket);
-        if (ret < 0) {
-                destroy_workqueue(afs_async_calls);
-                _leave(" = %d [socket]", ret);
-                return ret;
-        }
+        if (ret < 0)
+                goto error_1;
 
         socket->sk->sk_allocation = GFP_NOFS;
 
@@ -111,18 +107,26 @@ int afs_open_socket(void)
                sizeof(srx.transport.sin.sin_addr));
 
         ret = kernel_bind(socket, (struct sockaddr *) &srx, sizeof(srx));
-        if (ret < 0) {
-                sock_release(socket);
-                destroy_workqueue(afs_async_calls);
-                _leave(" = %d [bind]", ret);
-                return ret;
-        }
+        if (ret < 0)
+                goto error_2;
+
+        ret = kernel_listen(socket, INT_MAX);
+        if (ret < 0)
+                goto error_2;
 
         rxrpc_kernel_intercept_rx_messages(socket, afs_rx_interceptor);
 
         afs_socket = socket;
         _leave(" = 0");
         return 0;
+
+error_2:
+        sock_release(socket);
+error_1:
+        destroy_workqueue(afs_async_calls);
+error_0:
+        _leave(" = %d", ret);
+        return ret;
 }
 
 /*
diff --git a/include/linux/acpi.h b/include/linux/acpi.h
index 288fac5294f5..4d4bb4955682 100644
--- a/include/linux/acpi.h
+++ b/include/linux/acpi.h
@@ -543,6 +543,11 @@ struct platform_device *acpi_create_platform_device(struct acpi_device *);
 
 struct fwnode_handle;
 
+static inline bool acpi_dev_found(const char *hid)
+{
+        return false;
+}
+
 static inline bool is_acpi_node(struct fwnode_handle *fwnode)
 {
         return false;
@@ -654,6 +659,14 @@ static inline bool acpi_driver_match_device(struct device *dev,
         return false;
 }
 
+static inline union acpi_object *acpi_evaluate_dsm(acpi_handle handle,
+                                                   const u8 *uuid,
+                                                   int rev, int func,
+                                                   union acpi_object *argv4)
+{
+        return NULL;
+}
+
 static inline int acpi_device_uevent_modalias(struct device *dev,
                                 struct kobj_uevent_env *env)
 {
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 0de4de6dd43e..8411032ac90d 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -11,14 +11,17 @@
 #include <linux/workqueue.h>
 #include <linux/file.h>
 #include <linux/percpu.h>
+#include <linux/err.h>
 
+struct perf_event;
 struct bpf_map;
 
 /* map is generic key/value storage optionally accesible by eBPF programs */
 struct bpf_map_ops {
         /* funcs callable from userspace (via syscall) */
         struct bpf_map *(*map_alloc)(union bpf_attr *attr);
-        void (*map_free)(struct bpf_map *);
+        void (*map_release)(struct bpf_map *map, struct file *map_file);
+        void (*map_free)(struct bpf_map *map);
         int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key);
 
         /* funcs callable from userspace and from eBPF programs */
@@ -27,8 +30,9 @@ struct bpf_map_ops {
         int (*map_delete_elem)(struct bpf_map *map, void *key);
 
         /* funcs called by prog_array and perf_event_array map */
-        void *(*map_fd_get_ptr) (struct bpf_map *map, int fd);
-        void (*map_fd_put_ptr) (void *ptr);
+        void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file,
+                                int fd);
+        void (*map_fd_put_ptr)(void *ptr);
 };
 
 struct bpf_map {
@@ -189,11 +193,19 @@ struct bpf_array {
                 void __percpu *pptrs[0] __aligned(8);
         };
 };
+
 #define MAX_TAIL_CALL_CNT 32
 
+struct bpf_event_entry {
+        struct perf_event *event;
+        struct file *perf_file;
+        struct file *map_file;
+        struct rcu_head rcu;
+};
+
 u64 bpf_tail_call(u64 ctx, u64 r2, u64 index, u64 r4, u64 r5);
 u64 bpf_get_stackid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
-void bpf_fd_array_map_clear(struct bpf_map *map);
+
 bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp);
 
 const struct bpf_func_proto *bpf_get_trace_printk_proto(void);
@@ -231,8 +243,13 @@ int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value,
                            u64 flags);
 int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
                             u64 flags);
+
 int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value);
 
+int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,
+                                 void *key, void *value, u64 map_flags);
+void bpf_fd_array_map_clear(struct bpf_map *map);
+
 /* memcpy that is used with 8-byte aligned pointers, power-of-8 size and
  * forced to use 'long' read/writes to try to atomically copy long counters.
  * Best-effort only.  No barriers here, since it _will_ race with concurrent
diff --git a/include/linux/icmpv6.h b/include/linux/icmpv6.h
index 630f45335c73..57086e9fc64c 100644
--- a/include/linux/icmpv6.h
+++ b/include/linux/icmpv6.h
@@ -14,9 +14,12 @@ static inline struct icmp6hdr *icmp6_hdr(const struct sk_buff *skb)
 #if IS_ENABLED(CONFIG_IPV6)
 extern void icmpv6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info);
 
-typedef void ip6_icmp_send_t(struct sk_buff *skb, u8 type, u8 code, __u32 info);
+typedef void ip6_icmp_send_t(struct sk_buff *skb, u8 type, u8 code, __u32 info,
+                             const struct in6_addr *force_saddr);
 extern int inet6_register_icmp_sender(ip6_icmp_send_t *fn);
 extern int inet6_unregister_icmp_sender(ip6_icmp_send_t *fn);
+int ip6_err_gen_icmpv6_unreach(struct sk_buff *skb, int nhs, int type,
+                               unsigned int data_len);
 
 #else
 
diff --git a/include/linux/ipv6.h b/include/linux/ipv6.h
index 5c91b0b055d4..c6dbcd84a2c7 100644
--- a/include/linux/ipv6.h
+++ b/include/linux/ipv6.h
@@ -283,6 +283,8 @@ struct tcp6_timewait_sock {
 };
 
 #if IS_ENABLED(CONFIG_IPV6)
+bool ipv6_mod_enabled(void);
+
 static inline struct ipv6_pinfo *inet6_sk(const struct sock *__sk)
 {
         return sk_fullsock(__sk) ? inet_sk(__sk)->pinet6 : NULL;
@@ -326,6 +328,11 @@ static inline int inet_v6_ipv6only(const struct sock *sk)
 #define ipv6_only_sock(sk)      0
 #define ipv6_sk_rxinfo(sk)      0
 
+static inline bool ipv6_mod_enabled(void)
+{
+        return false;
+}
+
 static inline struct ipv6_pinfo * inet6_sk(const struct sock *__sk)
 {
         return NULL;
diff --git a/include/linux/mdio-mux.h b/include/linux/mdio-mux.h
index a243dbba8659..61f5b21b31c7 100644
--- a/include/linux/mdio-mux.h
+++ b/include/linux/mdio-mux.h
@@ -10,11 +10,13 @@
 #ifndef __LINUX_MDIO_MUX_H
 #define __LINUX_MDIO_MUX_H
 #include <linux/device.h>
+#include <linux/phy.h>
 
 int mdio_mux_init(struct device *dev,
                   int (*switch_fn) (int cur, int desired, void *data),
                   void **mux_handle,
-                  void *data);
+                  void *data,
+                  struct mii_bus *mux_bus);
 
 void mdio_mux_uninit(void *mux_handle);
 
diff --git a/include/linux/mlx4/device.h b/include/linux/mlx4/device.h
index d46a0e7f144d..e6f6910278f3 100644
--- a/include/linux/mlx4/device.h
+++ b/include/linux/mlx4/device.h
@@ -536,6 +536,7 @@ struct mlx4_caps {
         int                     max_rq_desc_sz;
         int                     max_qp_init_rdma;
         int                     max_qp_dest_rdma;
+        int                     max_tc_eth;
         u32                     *qp0_qkey;
         u32                     *qp0_proxy;
         u32                     *qp1_proxy;
@@ -1495,6 +1496,7 @@ int mlx4_mr_rereg_mem_write(struct mlx4_dev *dev, struct mlx4_mr *mr,
 
 int mlx4_get_module_info(struct mlx4_dev *dev, u8 port,
                          u16 offset, u16 size, u8 *data);
+int mlx4_max_tc(struct mlx4_dev *dev);
 
 /* Returns true if running in low memory profile (kdump kernel) */
 static inline bool mlx4_low_memory_profile(void)
diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h
index 73a48479892d..e0a3ed758287 100644
--- a/include/linux/mlx5/device.h
+++ b/include/linux/mlx5/device.h
@@ -1330,6 +1330,7 @@ enum mlx5_cap_type {
         MLX5_CAP_ESWITCH,
         MLX5_CAP_RESERVED,
         MLX5_CAP_VECTOR_CALC,
+        MLX5_CAP_QOS,
         /* NUM OF CAP Types */
         MLX5_CAP_NUM
 };
@@ -1414,6 +1415,9 @@ enum mlx5_cap_type {
         MLX5_GET(vector_calc_cap, \
                  mdev->hca_caps_cur[MLX5_CAP_VECTOR_CALC], cap)
 
+#define MLX5_CAP_QOS(mdev, cap)\
+        MLX5_GET(qos_cap, mdev->hca_caps_cur[MLX5_CAP_QOS], cap)
+
 enum {
         MLX5_CMD_STAT_OK                        = 0x0,
         MLX5_CMD_STAT_INT_ERR                   = 0x1,
diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index 80776d0c52dc..46260fdc5305 100644
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -481,6 +481,21 @@ struct mlx5_fc_stats {
 
 struct mlx5_eswitch;
 
+struct mlx5_rl_entry {
+        u32                     rate;
+        u16                     index;
+        u16                     refcount;
+};
+
+struct mlx5_rl_table {
+        /* protect rate limit table */
+        struct mutex            rl_lock;
+        u16                     max_size;
+        u32                     max_rate;
+        u32                     min_rate;
+        struct mlx5_rl_entry   *rl_entry;
+};
+
 struct mlx5_priv {
         char                    name[MLX5_MAX_NAME_LEN];
         struct mlx5_eq_table    eq_table;
@@ -544,6 +559,7 @@ struct mlx5_priv {
         struct mlx5_flow_root_namespace *esw_ingress_root_ns;
 
         struct mlx5_fc_stats            fc_stats;
+        struct mlx5_rl_table            rl_table;
 };
 
 enum mlx5_device_state {
@@ -861,6 +877,12 @@ int mlx5_query_odp_caps(struct mlx5_core_dev *dev,
 int mlx5_core_query_ib_ppcnt(struct mlx5_core_dev *dev,
                              u8 port_num, void *out, size_t sz);
 
+int mlx5_init_rl_table(struct mlx5_core_dev *dev);
+void mlx5_cleanup_rl_table(struct mlx5_core_dev *dev);
+int mlx5_rl_add_rate(struct mlx5_core_dev *dev, u32 rate, u16 *index);
+void mlx5_rl_remove_rate(struct mlx5_core_dev *dev, u32 rate);
+bool mlx5_rl_is_in_range(struct mlx5_core_dev *dev, u32 rate);
+
 static inline int fw_initializing(struct mlx5_core_dev *dev)
 {
         return ioread32be(&dev->iseg->initializing) >> 31;
@@ -938,6 +960,11 @@ static inline int mlx5_get_gid_table_len(u16 param)
         return 8 * (1 << param);
 }
 
+static inline bool mlx5_rl_is_supported(struct mlx5_core_dev *dev)
+{
+        return !!(dev->priv.rl_table.max_size);
+}
+
 enum {
         MLX5_TRIGGERED_CMD_COMP = (u64)1 << 32,
 };
diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h
index e955a2859009..152421cc6f44 100644
--- a/include/linux/mlx5/mlx5_ifc.h
+++ b/include/linux/mlx5/mlx5_ifc.h
@@ -123,6 +123,10 @@ enum {
         MLX5_CMD_OP_DRAIN_DCT                     = 0x712,
         MLX5_CMD_OP_QUERY_DCT                     = 0x713,
         MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION     = 0x714,
+        MLX5_CMD_OP_CREATE_XRQ                    = 0x717,
+        MLX5_CMD_OP_DESTROY_XRQ                   = 0x718,
+        MLX5_CMD_OP_QUERY_XRQ                     = 0x719,
+        MLX5_CMD_OP_ARM_XRQ                       = 0x71a,
         MLX5_CMD_OP_QUERY_VPORT_STATE             = 0x750,
         MLX5_CMD_OP_MODIFY_VPORT_STATE            = 0x751,
         MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT       = 0x752,
@@ -139,6 +143,8 @@ enum {
         MLX5_CMD_OP_ALLOC_Q_COUNTER               = 0x771,
         MLX5_CMD_OP_DEALLOC_Q_COUNTER             = 0x772,
         MLX5_CMD_OP_QUERY_Q_COUNTER               = 0x773,
+        MLX5_CMD_OP_SET_RATE_LIMIT                = 0x780,
+        MLX5_CMD_OP_QUERY_RATE_LIMIT              = 0x781,
         MLX5_CMD_OP_ALLOC_PD                      = 0x800,
         MLX5_CMD_OP_DEALLOC_PD                    = 0x801,
         MLX5_CMD_OP_ALLOC_UAR                     = 0x802,
@@ -362,7 +368,8 @@ struct mlx5_ifc_fte_match_set_lyr_2_4_bits {
 };
 
 struct mlx5_ifc_fte_match_set_misc_bits {
-        u8         reserved_at_0[0x20];
+        u8         reserved_at_0[0x8];
+        u8         source_sqn[0x18];
 
         u8         reserved_at_20[0x10];
         u8         source_port[0x10];
@@ -508,6 +515,17 @@ struct mlx5_ifc_e_switch_cap_bits {
         u8         reserved_at_20[0x7e0];
 };
 
+struct mlx5_ifc_qos_cap_bits {
+        u8         packet_pacing[0x1];
+        u8         reserved_0[0x1f];
+        u8         reserved_1[0x20];
+        u8         packet_pacing_max_rate[0x20];
+        u8         packet_pacing_min_rate[0x20];
+        u8         reserved_2[0x10];
+        u8         packet_pacing_rate_table_size[0x10];
+        u8         reserved_3[0x760];
+};
+
 struct mlx5_ifc_per_protocol_networking_offload_caps_bits {
         u8         csum_cap[0x1];
         u8         vlan_cap[0x1];
@@ -747,7 +765,8 @@ struct mlx5_ifc_cmd_hca_cap_bits {
 
         u8         out_of_seq_cnt[0x1];
         u8         vport_counters[0x1];
-        u8         reserved_at_182[0x4];
+        u8         retransmission_q_counters[0x1];
+        u8         reserved_at_183[0x3];
         u8         max_qp_cnt[0xa];
         u8         pkey_table_size[0x10];
 
@@ -774,7 +793,9 @@ struct mlx5_ifc_cmd_hca_cap_bits {
         u8         log_max_msg[0x5];
         u8         reserved_at_1c8[0x4];
         u8         max_tc[0x4];
-        u8         reserved_at_1d0[0x6];
+        u8         reserved_at_1d0[0x1];
+        u8         dcbx[0x1];
+        u8         reserved_at_1d2[0x4];
         u8         rol_s[0x1];
         u8         rol_g[0x1];
         u8         reserved_at_1d8[0x1];
@@ -806,7 +827,7 @@ struct mlx5_ifc_cmd_hca_cap_bits {
         u8         tph[0x1];
         u8         rf[0x1];
         u8         dct[0x1];
-        u8         reserved_at_21b[0x1];
+        u8         qos[0x1];
         u8         eth_net_offloads[0x1];
         u8         roce[0x1];
         u8         atomic[0x1];
@@ -932,7 +953,15 @@ struct mlx5_ifc_cmd_hca_cap_bits {
         u8         cqe_compression_timeout[0x10];
         u8         cqe_compression_max_num[0x10];
 
-        u8         reserved_at_5e0[0x220];
+        u8         reserved_at_5e0[0x10];
+        u8         tag_matching[0x1];
+        u8         rndv_offload_rc[0x1];
+        u8         rndv_offload_dc[0x1];
+        u8         log_tag_matching_list_sz[0x5];
+        u8         reserved_at_5e8[0x3];
+        u8         log_max_xrq[0x5];
+
+        u8         reserved_at_5f0[0x200];
 };
 
 enum mlx5_flow_destination_type {
@@ -1970,7 +1999,7 @@ struct mlx5_ifc_qpc_bits {
 
         u8         reserved_at_560[0x5];
         u8         rq_type[0x3];
-        u8         srqn_rmpn[0x18];
+        u8         srqn_rmpn_xrqn[0x18];
 
         u8         reserved_at_580[0x8];
         u8         rmsn[0x18];
@@ -2021,6 +2050,7 @@ union mlx5_ifc_hca_cap_union_bits {
         struct mlx5_ifc_flow_table_eswitch_cap_bits flow_table_eswitch_cap;
         struct mlx5_ifc_e_switch_cap_bits e_switch_cap;
         struct mlx5_ifc_vector_calc_cap_bits vector_calc_cap;
+        struct mlx5_ifc_qos_cap_bits qos_cap;
         u8         reserved_at_0[0x8000];
 };
 
@@ -2247,8 +2277,9 @@ struct mlx5_ifc_sqc_bits {
         u8         reserved_at_40[0x8];
         u8         cqn[0x18];
 
-        u8         reserved_at_60[0xa0];
+        u8         reserved_at_60[0x90];
 
+        u8         packet_pacing_rate_limit_index[0x10];
         u8         tis_lst_sz[0x10];
         u8         reserved_at_110[0x10];
 
@@ -2596,7 +2627,7 @@ struct mlx5_ifc_dctc_bits {
         u8         reserved_at_98[0x8];
 
         u8         reserved_at_a0[0x8];
-        u8         srqn[0x18];
+        u8         srqn_xrqn[0x18];
 
         u8         reserved_at_c0[0x8];
         u8         pd[0x18];
@@ -2648,6 +2679,7 @@ enum {
 enum {
         MLX5_CQ_PERIOD_MODE_START_FROM_EQE = 0x0,
         MLX5_CQ_PERIOD_MODE_START_FROM_CQE = 0x1,
+        MLX5_CQ_PERIOD_NUM_MODES
 };
 
 struct mlx5_ifc_cqc_bits {
@@ -2725,6 +2757,54 @@ struct mlx5_ifc_query_adapter_param_block_bits {
         u8         vsd_contd_psid[16][0x8];
 };
 
+enum {
+        MLX5_XRQC_STATE_GOOD   = 0x0,
+        MLX5_XRQC_STATE_ERROR  = 0x1,
+};
+
+enum {
+        MLX5_XRQC_TOPOLOGY_NO_SPECIAL_TOPOLOGY = 0x0,
+        MLX5_XRQC_TOPOLOGY_TAG_MATCHING        = 0x1,
+};
+
+enum {
+        MLX5_XRQC_OFFLOAD_RNDV = 0x1,
+};
+
+struct mlx5_ifc_tag_matching_topology_context_bits {
+        u8         log_matching_list_sz[0x4];
+        u8         reserved_at_4[0xc];
+        u8         append_next_index[0x10];
+
+        u8         sw_phase_cnt[0x10];
+        u8         hw_phase_cnt[0x10];
+
+        u8         reserved_at_40[0x40];
+};
+
+struct mlx5_ifc_xrqc_bits {
+        u8         state[0x4];
+        u8         rlkey[0x1];
+        u8         reserved_at_5[0xf];
+        u8         topology[0x4];
+        u8         reserved_at_18[0x4];
+        u8         offload[0x4];
+
+        u8         reserved_at_20[0x8];
+        u8         user_index[0x18];
+
+        u8         reserved_at_40[0x8];
+        u8         cqn[0x18];
+
+        u8         reserved_at_60[0xa0];
+
+        struct mlx5_ifc_tag_matching_topology_context_bits tag_matching_topology_context;
+
+        u8         reserved_at_180[0x180];
+
+        struct mlx5_ifc_wq_bits wq;
+};
+
 union mlx5_ifc_modify_field_select_resize_field_select_auto_bits {
         struct mlx5_ifc_modify_field_select_bits modify_field_select;
         struct mlx5_ifc_resize_field_select_bits resize_field_select;
@@ -3147,6 +3227,30 @@ struct mlx5_ifc_rst2init_qp_in_bits {
         u8         reserved_at_800[0x80];
 };
 
+struct mlx5_ifc_query_xrq_out_bits {
+        u8         status[0x8];
+        u8         reserved_at_8[0x18];
+
+        u8         syndrome[0x20];
+
+        u8         reserved_at_40[0x40];
+
+        struct mlx5_ifc_xrqc_bits xrq_context;
+};
+
+struct mlx5_ifc_query_xrq_in_bits {
+        u8         opcode[0x10];
+        u8         reserved_at_10[0x10];
+
+        u8         reserved_at_20[0x10];
+        u8         op_mod[0x10];
+
+        u8         reserved_at_40[0x8];
+        u8         xrqn[0x18];
+
+        u8         reserved_at_60[0x20];
+};
+
 struct mlx5_ifc_query_xrc_srq_out_bits {
         u8         status[0x8];
         u8         reserved_at_8[0x18];
@@ -3550,7 +3654,27 @@ struct mlx5_ifc_query_q_counter_out_bits {
 
         u8         out_of_sequence[0x20];
 
-        u8         reserved_at_1e0[0x620];
+        u8         reserved_at_1e0[0x20];
+
+        u8         duplicate_request[0x20];
+
+        u8         reserved_at_220[0x20];
+
+        u8         rnr_nak_retry_err[0x20];
+
+        u8         reserved_at_260[0x20];
+
+        u8         packet_seq_err[0x20];
+
+        u8         reserved_at_2a0[0x20];
+
+        u8         implied_nak_seq_err[0x20];
+
+        u8         reserved_at_2e0[0x20];
+
+        u8         local_ack_timeout_err[0x20];
+
+        u8         reserved_at_320[0x4e0];
 };
 
 struct mlx5_ifc_query_q_counter_in_bits {
@@ -5004,6 +5128,28 @@ struct mlx5_ifc_detach_from_mcg_in_bits {
         u8         multicast_gid[16][0x8];
 };
 
+struct mlx5_ifc_destroy_xrq_out_bits {
+        u8         status[0x8];
+        u8         reserved_at_8[0x18];
+
+        u8         syndrome[0x20];
+
+        u8         reserved_at_40[0x40];
+};
+
+struct mlx5_ifc_destroy_xrq_in_bits {
+        u8         opcode[0x10];
+        u8         reserved_at_10[0x10];
+
+        u8         reserved_at_20[0x10];
+        u8         op_mod[0x10];
+
+        u8         reserved_at_40[0x8];
+        u8         xrqn[0x18];
+
+        u8         reserved_at_60[0x20];
+};
+
 struct mlx5_ifc_destroy_xrc_srq_out_bits {
         u8         status[0x8];
         u8         reserved_at_8[0x18];
@@ -5589,6 +5735,30 @@ struct mlx5_ifc_dealloc_flow_counter_in_bits {
         u8         reserved_at_60[0x20];
 };
 
+struct mlx5_ifc_create_xrq_out_bits {
+        u8         status[0x8];
+        u8         reserved_at_8[0x18];
+
+        u8         syndrome[0x20];
+
+        u8         reserved_at_40[0x8];
+        u8         xrqn[0x18];
+
+        u8         reserved_at_60[0x20];
+};
+
+struct mlx5_ifc_create_xrq_in_bits {
+        u8         opcode[0x10];
+        u8         reserved_at_10[0x10];
+
+        u8         reserved_at_20[0x10];
+        u8         op_mod[0x10];
+
+        u8         reserved_at_40[0x40];
+
+        struct mlx5_ifc_xrqc_bits xrq_context;
+};
+
 struct mlx5_ifc_create_xrc_srq_out_bits {
         u8         status[0x8];
         u8         reserved_at_8[0x18];
@@ -6130,6 +6300,29 @@ struct mlx5_ifc_attach_to_mcg_in_bits {
         u8         multicast_gid[16][0x8];
 };
 
+struct mlx5_ifc_arm_xrq_out_bits {
+        u8         status[0x8];
+        u8         reserved_at_8[0x18];
+
+        u8         syndrome[0x20];
+
+        u8         reserved_at_40[0x40];
+};
+
+struct mlx5_ifc_arm_xrq_in_bits {
+        u8         opcode[0x10];
+        u8         reserved_at_10[0x10];
+
+        u8         reserved_at_20[0x10];
+        u8         op_mod[0x10];
+
+        u8         reserved_at_40[0x8];
+        u8         xrqn[0x18];
+
+        u8         reserved_at_60[0x10];
+        u8         lwm[0x10];
+};
+
 struct mlx5_ifc_arm_xrc_srq_out_bits {
         u8         status[0x8];
         u8         reserved_at_8[0x18];
@@ -6167,7 +6360,8 @@ struct mlx5_ifc_arm_rq_out_bits {
 };
 
 enum {
-        MLX5_ARM_RQ_IN_OP_MOD_SRQ_  = 0x1,
+        MLX5_ARM_RQ_IN_OP_MOD_SRQ = 0x1,
+        MLX5_ARM_RQ_IN_OP_MOD_XRQ = 0x2,
 };
 
 struct mlx5_ifc_arm_rq_in_bits {
@@ -6360,6 +6554,30 @@ struct mlx5_ifc_add_vxlan_udp_dport_in_bits {
         u8         vxlan_udp_port[0x10];
 };
 
+struct mlx5_ifc_set_rate_limit_out_bits {
+        u8         status[0x8];
+        u8         reserved_at_8[0x18];
+
+        u8         syndrome[0x20];
+
+        u8         reserved_at_40[0x40];
+};
+
+struct mlx5_ifc_set_rate_limit_in_bits {
+        u8         opcode[0x10];
+        u8         reserved_at_10[0x10];
+
+        u8         reserved_at_20[0x10];
+        u8         op_mod[0x10];
+
+        u8         reserved_at_40[0x10];
+        u8         rate_limit_index[0x10];
+
+        u8         reserved_at_60[0x20];
+
+        u8         rate_limit[0x20];
+};
+
 struct mlx5_ifc_access_register_out_bits {
         u8         status[0x8];
         u8         reserved_at_8[0x18];
@@ -6484,12 +6702,15 @@ struct mlx5_ifc_pude_reg_bits {
 };
 
 struct mlx5_ifc_ptys_reg_bits {
-        u8         reserved_at_0[0x8];
+        u8         an_disable_cap[0x1];
+        u8         an_disable_admin[0x1];
+        u8         reserved_at_2[0x6];
         u8         local_port[0x8];
         u8         reserved_at_10[0xd];
         u8         proto_mask[0x3];
 
-        u8         reserved_at_20[0x40];
+        u8         an_status[0x4];
+        u8         reserved_at_24[0x3c];
 
         u8         eth_proto_capability[0x20];
 
@@ -7450,4 +7671,34 @@ struct mlx5_ifc_mcia_reg_bits {
         u8         dword_11[0x20];
 };
 
+struct mlx5_ifc_dcbx_param_bits {
+        u8         dcbx_cee_cap[0x1];
+        u8         dcbx_ieee_cap[0x1];
+        u8         dcbx_standby_cap[0x1];
+        u8         reserved_at_0[0x5];
+        u8         port_number[0x8];
+        u8         reserved_at_10[0xa];
+        u8         max_application_table_size[6];
+        u8         reserved_at_20[0x15];
+        u8         version_oper[0x3];
+        u8         reserved_at_38[5];
+        u8         version_admin[0x3];
+        u8         willing_admin[0x1];
+        u8         reserved_at_41[0x3];
+        u8         pfc_cap_oper[0x4];
+        u8         reserved_at_48[0x4];
+        u8         pfc_cap_admin[0x4];
+        u8         reserved_at_50[0x4];
+        u8         num_of_tc_oper[0x4];
+        u8         reserved_at_58[0x4];
+        u8         num_of_tc_admin[0x4];
+        u8         remote_willing[0x1];
+        u8         reserved_at_61[3];
+        u8         remote_pfc_cap[4];
+        u8         reserved_at_68[0x14];
+        u8         remote_num_of_tc[0x4];
+        u8         reserved_at_80[0x18];
+        u8         error[0x8];
+        u8         reserved_at_a0[0x160];
+};
 #endif /* MLX5_IFC_H */
diff --git a/include/linux/mlx5/port.h b/include/linux/mlx5/port.h
index 9851862c0ec5..e3012cc64b8a 100644
--- a/include/linux/mlx5/port.h
+++ b/include/linux/mlx5/port.h
@@ -47,6 +47,14 @@ enum mlx5_module_id {
         MLX5_MODULE_ID_QSFP28           = 0x11,
 };
 
+enum mlx5_an_status {
+        MLX5_AN_UNAVAILABLE = 0,
+        MLX5_AN_COMPLETE    = 1,
+        MLX5_AN_FAILED      = 2,
+        MLX5_AN_LINK_UP     = 3,
+        MLX5_AN_LINK_DOWN   = 4,
+};
+
 #define MLX5_EEPROM_MAX_BYTES                   32
 #define MLX5_EEPROM_IDENTIFIER_BYTE_MASK        0x000000ff
 #define MLX5_I2C_ADDR_LOW               0x50
@@ -65,13 +73,17 @@ int mlx5_query_port_link_width_oper(struct mlx5_core_dev *dev,
 int mlx5_query_port_proto_oper(struct mlx5_core_dev *dev,
                                u8 *proto_oper, int proto_mask,
                                u8 local_port);
-int mlx5_set_port_proto(struct mlx5_core_dev *dev, u32 proto_admin,
-                        int proto_mask);
+int mlx5_set_port_ptys(struct mlx5_core_dev *dev, bool an_disable,
+                       u32 proto_admin, int proto_mask);
+void mlx5_toggle_port_link(struct mlx5_core_dev *dev);
 int mlx5_set_port_admin_status(struct mlx5_core_dev *dev,
                                enum mlx5_port_status status);
 int mlx5_query_port_admin_status(struct mlx5_core_dev *dev,
                                  enum mlx5_port_status *status);
 int mlx5_set_port_beacon(struct mlx5_core_dev *dev, u16 beacon_duration);
+void mlx5_query_port_autoneg(struct mlx5_core_dev *dev, int proto_mask,
+                             u8 *an_status,
+                             u8 *an_disable_cap, u8 *an_disable_admin);
 
 int mlx5_set_port_mtu(struct mlx5_core_dev *dev, u16 mtu, u8 port);
 void mlx5_query_port_max_mtu(struct mlx5_core_dev *dev, u16 *max_mtu, u8 port);
diff --git a/include/linux/netdev_features.h b/include/linux/netdev_features.h
index aa7b2400f98c..9c6c8ef2e9e7 100644
--- a/include/linux/netdev_features.h
+++ b/include/linux/netdev_features.h
@@ -53,8 +53,9 @@ enum {
                                          *     headers in software.
                                          */
         NETIF_F_GSO_TUNNEL_REMCSUM_BIT, /* ... TUNNEL with TSO & REMCSUM */
+        NETIF_F_GSO_SCTP_BIT,           /* ... SCTP fragmentation */
         /**/NETIF_F_GSO_LAST =          /* last bit, see GSO_MASK */
-                NETIF_F_GSO_TUNNEL_REMCSUM_BIT,
+                NETIF_F_GSO_SCTP_BIT,
 
         NETIF_F_FCOE_CRC_BIT,           /* FCoE CRC32 */
         NETIF_F_SCTP_CRC_BIT,           /* SCTP checksum offload */
@@ -128,6 +129,7 @@ enum {
 #define NETIF_F_TSO_MANGLEID    __NETIF_F(TSO_MANGLEID)
 #define NETIF_F_GSO_PARTIAL      __NETIF_F(GSO_PARTIAL)
 #define NETIF_F_GSO_TUNNEL_REMCSUM __NETIF_F(GSO_TUNNEL_REMCSUM)
+#define NETIF_F_GSO_SCTP        __NETIF_F(GSO_SCTP)
 #define NETIF_F_HW_VLAN_STAG_FILTER __NETIF_F(HW_VLAN_STAG_FILTER)
 #define NETIF_F_HW_VLAN_STAG_RX __NETIF_F(HW_VLAN_STAG_RX)
 #define NETIF_F_HW_VLAN_STAG_TX __NETIF_F(HW_VLAN_STAG_TX)
@@ -166,7 +168,8 @@ enum {
                                  NETIF_F_FSO)
 
 /* List of features with software fallbacks. */
-#define NETIF_F_GSO_SOFTWARE    (NETIF_F_ALL_TSO | NETIF_F_UFO)
+#define NETIF_F_GSO_SOFTWARE    (NETIF_F_ALL_TSO | NETIF_F_UFO | \
+                                 NETIF_F_GSO_SCTP)
 
 /*
  * If one device supports one of these features, then enable them
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index f45929ce8157..e84d9d23c2d5 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -61,6 +61,8 @@ struct wireless_dev;
 /* 802.15.4 specific */
 struct wpan_dev;
 struct mpls_dev;
+/* UDP Tunnel offloads */
+struct udp_tunnel_info;
 
 void netdev_set_default_ethtool_ops(struct net_device *dev,
                                     const struct ethtool_ops *ops);
@@ -90,7 +92,6 @@ void netdev_set_default_ethtool_ops(struct net_device *dev,
 #define NET_XMIT_SUCCESS        0x00
 #define NET_XMIT_DROP           0x01    /* skb dropped                  */
 #define NET_XMIT_CN             0x02    /* congestion notification      */
-#define NET_XMIT_POLICED        0x03    /* skb is shot by police        */
 #define NET_XMIT_MASK           0x0f    /* qdisc flags in net/sch_generic.h */
 
 /* NET_XMIT_CN is special. It does not guarantee that this packet is lost. It
@@ -1025,31 +1026,18 @@ struct tc_to_netdev {
  *      not implement this, it is assumed that the hw is not able to have
  *      multiple net devices on single physical port.
  *
- * void (*ndo_add_vxlan_port)(struct  net_device *dev,
- *                            sa_family_t sa_family, __be16 port);
- *      Called by vxlan to notify a driver about the UDP port and socket
- *      address family that vxlan is listening to. It is called only when
- *      a new port starts listening. The operation is protected by the
- *      vxlan_net->sock_lock.
- *
- * void (*ndo_add_geneve_port)(struct net_device *dev,
- *                             sa_family_t sa_family, __be16 port);
- *      Called by geneve to notify a driver about the UDP port and socket
- *      address family that geneve is listnening to. It is called only when
- *      a new port starts listening. The operation is protected by the
- *      geneve_net->sock_lock.
- *
- * void (*ndo_del_geneve_port)(struct net_device *dev,
- *                             sa_family_t sa_family, __be16 port);
- *      Called by geneve to notify the driver about a UDP port and socket
- *      address family that geneve is not listening to anymore. The operation
- *      is protected by the geneve_net->sock_lock.
- *
- * void (*ndo_del_vxlan_port)(struct  net_device *dev,
- *                            sa_family_t sa_family, __be16 port);
- *      Called by vxlan to notify the driver about a UDP port and socket
- *      address family that vxlan is not listening to anymore. The operation
- *      is protected by the vxlan_net->sock_lock.
+ * void (*ndo_udp_tunnel_add)(struct net_device *dev,
+ *                            struct udp_tunnel_info *ti);
+ *      Called by UDP tunnel to notify a driver about the UDP port and socket
+ *      address family that a UDP tunnel is listnening to. It is called only
+ *      when a new port starts listening. The operation is protected by the
+ *      RTNL.
+ *
+ * void (*ndo_udp_tunnel_del)(struct net_device *dev,
+ *                            struct udp_tunnel_info *ti);
+ *      Called by UDP tunnel to notify the driver about a UDP port and socket
+ *      address family that the UDP tunnel is not listening to anymore. The
+ *      operation is protected by the RTNL.
  *
  * void* (*ndo_dfwd_add_station)(struct net_device *pdev,
  *                               struct net_device *dev)
@@ -1258,18 +1246,10 @@ struct net_device_ops {
                                                         struct netdev_phys_item_id *ppid);
         int                     (*ndo_get_phys_port_name)(struct net_device *dev,
                                                           char *name, size_t len);
-        void                    (*ndo_add_vxlan_port)(struct  net_device *dev,
-                                                      sa_family_t sa_family,
-                                                      __be16 port);
-        void                    (*ndo_del_vxlan_port)(struct  net_device *dev,
-                                                      sa_family_t sa_family,
-                                                      __be16 port);
-        void                    (*ndo_add_geneve_port)(struct  net_device *dev,
-                                                       sa_family_t sa_family,
-                                                       __be16 port);
-        void                    (*ndo_del_geneve_port)(struct  net_device *dev,
-                                                       sa_family_t sa_family,
-                                                       __be16 port);
+        void                    (*ndo_udp_tunnel_add)(struct net_device *dev,
+                                                      struct udp_tunnel_info *ti);
+        void                    (*ndo_udp_tunnel_del)(struct net_device *dev,
+                                                      struct udp_tunnel_info *ti);
         void*                   (*ndo_dfwd_add_station)(struct net_device *pdev,
                                                         struct net_device *dev);
         void                    (*ndo_dfwd_del_station)(struct net_device *pdev,
@@ -1457,6 +1437,8 @@ enum netdev_priv_flags {
  *      @netdev_ops:    Includes several pointers to callbacks,
  *                      if one wants to override the ndo_*() functions
  *      @ethtool_ops:   Management operations
+ *      @ndisc_ops:     Includes callbacks for different IPv6 neighbour
+ *                      discovery handling. Necessary for e.g. 6LoWPAN.
  *      @header_ops:    Includes callbacks for creating,parsing,caching,etc
  *                      of Layer 2 headers.
  *
@@ -1484,8 +1466,7 @@ enum netdev_priv_flags {
  *      @perm_addr:             Permanent hw address
  *      @addr_assign_type:      Hw address assignment type
  *      @addr_len:              Hardware address length
- *      @neigh_priv_len;        Used in neigh_alloc(),
- *                              initialized only in atm/clip.c
+ *      @neigh_priv_len:        Used in neigh_alloc()
  *      @dev_id:                Used to differentiate devices that share
  *                              the same link layer address
  *      @dev_port:              Used to differentiate devices that share
@@ -1594,7 +1575,8 @@ enum netdev_priv_flags {
  *      @phydev:        Physical device may attach itself
  *                      for hardware timestamping
  *
- *      @qdisc_tx_busylock:     XXX: need comments on this one
+ *      @qdisc_tx_busylock: lockdep class annotating Qdisc->busylock spinlock
+ *      @qdisc_running_key: lockdep class annotating Qdisc->running seqcount
  *
  *      @proto_down:    protocol port state information can be sent to the
  *                      switch driver and used to set the phys state of the
@@ -1673,6 +1655,9 @@ struct net_device {
 #ifdef CONFIG_NET_L3_MASTER_DEV
         const struct l3mdev_ops *l3mdev_ops;
 #endif
+#if IS_ENABLED(CONFIG_IPV6)
+        const struct ndisc_ops *ndisc_ops;
+#endif
 
         const struct header_ops *header_ops;
 
@@ -1862,6 +1847,7 @@ struct net_device {
 #endif
         struct phy_device       *phydev;
         struct lock_class_key   *qdisc_tx_busylock;
+        struct lock_class_key   *qdisc_running_key;
         bool                    proto_down;
 };
 #define to_net_dev(d) container_of(d, struct net_device, dev)
@@ -1944,6 +1930,23 @@ static inline void netdev_for_each_tx_queue(struct net_device *dev,
                 f(dev, &dev->_tx[i], arg);
 }
 
+#define netdev_lockdep_set_classes(dev)                         \
+{                                                               \
+        static struct lock_class_key qdisc_tx_busylock_key;     \
+        static struct lock_class_key qdisc_running_key;         \
+        static struct lock_class_key qdisc_xmit_lock_key;       \
+        static struct lock_class_key dev_addr_list_lock_key;    \
+        unsigned int i;                                         \
+                                                                \
+        (dev)->qdisc_tx_busylock = &qdisc_tx_busylock_key;      \
+        (dev)->qdisc_running_key = &qdisc_running_key;          \
+        lockdep_set_class(&(dev)->addr_list_lock,               \
+                          &dev_addr_list_lock_key);             \
+        for (i = 0; i < (dev)->num_tx_queues; i++)              \
+                lockdep_set_class(&(dev)->_tx[i]._xmit_lock,    \
+                                  &qdisc_xmit_lock_key);        \
+}
+
 struct netdev_queue *netdev_pick_tx(struct net_device *dev,
                                     struct sk_buff *skb,
                                     void *accel_priv);
@@ -2233,8 +2236,7 @@ struct netdev_lag_lower_state_info {
 #define NETDEV_BONDING_INFO     0x0019
 #define NETDEV_PRECHANGEUPPER   0x001A
 #define NETDEV_CHANGELOWERSTATE 0x001B
-#define NETDEV_OFFLOAD_PUSH_VXLAN       0x001C
-#define NETDEV_OFFLOAD_PUSH_GENEVE      0x001D
+#define NETDEV_UDP_TUNNEL_PUSH_INFO     0x001C
 
 int register_netdevice_notifier(struct notifier_block *nb);
 int unregister_netdevice_notifier(struct notifier_block *nb);
@@ -2370,6 +2372,8 @@ void synchronize_net(void);
 int init_dummy_netdev(struct net_device *dev);
 
 DECLARE_PER_CPU(int, xmit_recursion);
+#define XMIT_RECURSION_LIMIT    10
+
 static inline int dev_recursion_level(void)
 {
         return this_cpu_read(xmit_recursion);
@@ -4012,6 +4016,7 @@ static inline bool net_gso_ok(netdev_features_t features, int gso_type)
         BUILD_BUG_ON(SKB_GSO_UDP_TUNNEL_CSUM != (NETIF_F_GSO_UDP_TUNNEL_CSUM >> NETIF_F_GSO_SHIFT));
         BUILD_BUG_ON(SKB_GSO_PARTIAL != (NETIF_F_GSO_PARTIAL >> NETIF_F_GSO_SHIFT));
         BUILD_BUG_ON(SKB_GSO_TUNNEL_REMCSUM != (NETIF_F_GSO_TUNNEL_REMCSUM >> NETIF_F_GSO_SHIFT));
+        BUILD_BUG_ON(SKB_GSO_SCTP    != (NETIF_F_GSO_SCTP >> NETIF_F_GSO_SHIFT));
 
         return (features & feature) == feature;
 }
diff --git a/include/linux/of_mdio.h b/include/linux/of_mdio.h
index 8f2237eb3485..4b04587d0441 100644
--- a/include/linux/of_mdio.h
+++ b/include/linux/of_mdio.h
@@ -25,6 +25,8 @@ struct phy_device *of_phy_attach(struct net_device *dev,
 
 extern struct mii_bus *of_mdio_find_bus(struct device_node *mdio_np);
 extern int of_mdio_parse_addr(struct device *dev, const struct device_node *np);
+extern int of_phy_register_fixed_link(struct device_node *np);
+extern bool of_phy_is_fixed_link(struct device_node *np);
 
 #else /* CONFIG_OF */
 static inline int of_mdiobus_register(struct mii_bus *mdio, struct device_node *np)
@@ -67,12 +69,6 @@ static inline int of_mdio_parse_addr(struct device *dev,
 {
         return -ENOSYS;
 }
-#endif /* CONFIG_OF */
-
-#if defined(CONFIG_OF) && defined(CONFIG_FIXED_PHY)
-extern int of_phy_register_fixed_link(struct device_node *np);
-extern bool of_phy_is_fixed_link(struct device_node *np);
-#else
 static inline int of_phy_register_fixed_link(struct device_node *np)
 {
         return -ENOSYS;
diff --git a/include/linux/platform_data/b53.h b/include/linux/platform_data/b53.h
new file mode 100644
index 000000000000..69d279c0da96
--- /dev/null
+++ b/include/linux/platform_data/b53.h
@@ -0,0 +1,33 @@
+/*
+ * B53 platform data
+ *
+ * Copyright (C) 2013 Jonas Gorski <jogo@openwrt.org>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef __B53_H
+#define __B53_H
+
+#include <linux/kernel.h>
+
+struct b53_platform_data {
+        u32 chip_id;
+        u16 enabled_ports;
+
+        /* only used by MMAP'd driver */
+        unsigned big_endian:1;
+        void __iomem *regs;
+};
+
+#endif
diff --git a/include/linux/ptr_ring.h b/include/linux/ptr_ring.h
new file mode 100644
index 000000000000..562a65e8bcc0
--- /dev/null
+++ b/include/linux/ptr_ring.h
@@ -0,0 +1,393 @@
+/*
+ *      Definitions for the 'struct ptr_ring' datastructure.
+ *
+ *      Author:
+ *              Michael S. Tsirkin <mst@redhat.com>
+ *
+ *      Copyright (C) 2016 Red Hat, Inc.
+ *
+ *      This program is free software; you can redistribute it and/or modify it
+ *      under the terms of the GNU General Public License as published by the
+ *      Free Software Foundation; either version 2 of the License, or (at your
+ *      option) any later version.
+ *
+ *      This is a limited-size FIFO maintaining pointers in FIFO order, with
+ *      one CPU producing entries and another consuming entries from a FIFO.
+ *
+ *      This implementation tries to minimize cache-contention when there is a
+ *      single producer and a single consumer CPU.
+ */
+
+#ifndef _LINUX_PTR_RING_H
+#define _LINUX_PTR_RING_H 1
+
+#ifdef __KERNEL__
+#include <linux/spinlock.h>
+#include <linux/cache.h>
+#include <linux/types.h>
+#include <linux/compiler.h>
+#include <linux/cache.h>
+#include <linux/slab.h>
+#include <asm/errno.h>
+#endif
+
+struct ptr_ring {
+        int producer ____cacheline_aligned_in_smp;
+        spinlock_t producer_lock;
+        int consumer ____cacheline_aligned_in_smp;
+        spinlock_t consumer_lock;
+        /* Shared consumer/producer data */
+        /* Read-only by both the producer and the consumer */
+        int size ____cacheline_aligned_in_smp; /* max entries in queue */
+        void **queue;
+};
+
+/* Note: callers invoking this in a loop must use a compiler barrier,
+ * for example cpu_relax().  If ring is ever resized, callers must hold
+ * producer_lock - see e.g. ptr_ring_full.  Otherwise, if callers don't hold
+ * producer_lock, the next call to __ptr_ring_produce may fail.
+ */
+static inline bool __ptr_ring_full(struct ptr_ring *r)
+{
+        return r->queue[r->producer];
+}
+
+static inline bool ptr_ring_full(struct ptr_ring *r)
+{
+        bool ret;
+
+        spin_lock(&r->producer_lock);
+        ret = __ptr_ring_full(r);
+        spin_unlock(&r->producer_lock);
+
+        return ret;
+}
+
+static inline bool ptr_ring_full_irq(struct ptr_ring *r)
+{
+        bool ret;
+
+        spin_lock_irq(&r->producer_lock);
+        ret = __ptr_ring_full(r);
+        spin_unlock_irq(&r->producer_lock);
+
+        return ret;
+}
+
+static inline bool ptr_ring_full_any(struct ptr_ring *r)
+{
+        unsigned long flags;
+        bool ret;
+
+        spin_lock_irqsave(&r->producer_lock, flags);
+        ret = __ptr_ring_full(r);
+        spin_unlock_irqrestore(&r->producer_lock, flags);
+
+        return ret;
+}
+
+static inline bool ptr_ring_full_bh(struct ptr_ring *r)
+{
+        bool ret;
+
+        spin_lock_bh(&r->producer_lock);
+        ret = __ptr_ring_full(r);
+        spin_unlock_bh(&r->producer_lock);
+
+        return ret;
+}
+
+/* Note: callers invoking this in a loop must use a compiler barrier,
+ * for example cpu_relax(). Callers must hold producer_lock.
+ */
+static inline int __ptr_ring_produce(struct ptr_ring *r, void *ptr)
+{
+        if (r->queue[r->producer])
+                return -ENOSPC;
+
+        r->queue[r->producer++] = ptr;
+        if (unlikely(r->producer >= r->size))
+                r->producer = 0;
+        return 0;
+}
+
+static inline int ptr_ring_produce(struct ptr_ring *r, void *ptr)
+{
+        int ret;
+
+        spin_lock(&r->producer_lock);
+        ret = __ptr_ring_produce(r, ptr);
+        spin_unlock(&r->producer_lock);
+
+        return ret;
+}
+
+static inline int ptr_ring_produce_irq(struct ptr_ring *r, void *ptr)
+{
+        int ret;
+
+        spin_lock_irq(&r->producer_lock);
+        ret = __ptr_ring_produce(r, ptr);
+        spin_unlock_irq(&r->producer_lock);
+
+        return ret;
+}
+
+static inline int ptr_ring_produce_any(struct ptr_ring *r, void *ptr)
+{
+        unsigned long flags;
+        int ret;
+
+        spin_lock_irqsave(&r->producer_lock, flags);
+        ret = __ptr_ring_produce(r, ptr);
+        spin_unlock_irqrestore(&r->producer_lock, flags);
+
+        return ret;
+}
+
+static inline int ptr_ring_produce_bh(struct ptr_ring *r, void *ptr)
+{
+        int ret;
+
+        spin_lock_bh(&r->producer_lock);
+        ret = __ptr_ring_produce(r, ptr);
+        spin_unlock_bh(&r->producer_lock);
+
+        return ret;
+}
+
+/* Note: callers invoking this in a loop must use a compiler barrier,
+ * for example cpu_relax(). Callers must take consumer_lock
+ * if they dereference the pointer - see e.g. PTR_RING_PEEK_CALL.
+ * If ring is never resized, and if the pointer is merely
+ * tested, there's no need to take the lock - see e.g.  __ptr_ring_empty.
+ */
+static inline void *__ptr_ring_peek(struct ptr_ring *r)
+{
+        return r->queue[r->consumer];
+}
+
+/* Note: callers invoking this in a loop must use a compiler barrier,
+ * for example cpu_relax(). Callers must take consumer_lock
+ * if the ring is ever resized - see e.g. ptr_ring_empty.
+ */
+static inline bool __ptr_ring_empty(struct ptr_ring *r)
+{
+        return !__ptr_ring_peek(r);
+}
+
+static inline bool ptr_ring_empty(struct ptr_ring *r)
+{
+        bool ret;
+
+        spin_lock(&r->consumer_lock);
+        ret = __ptr_ring_empty(r);
+        spin_unlock(&r->consumer_lock);
+
+        return ret;
+}
+
+static inline bool ptr_ring_empty_irq(struct ptr_ring *r)
+{
+        bool ret;
+
+        spin_lock_irq(&r->consumer_lock);
+        ret = __ptr_ring_empty(r);
+        spin_unlock_irq(&r->consumer_lock);
+
+        return ret;
+}
+
+static inline bool ptr_ring_empty_any(struct ptr_ring *r)
+{
+        unsigned long flags;
+        bool ret;
+
+        spin_lock_irqsave(&r->consumer_lock, flags);
+        ret = __ptr_ring_empty(r);
+        spin_unlock_irqrestore(&r->consumer_lock, flags);
+
+        return ret;
+}
+
+static inline bool ptr_ring_empty_bh(struct ptr_ring *r)
+{
+        bool ret;
+
+        spin_lock_bh(&r->consumer_lock);
+        ret = __ptr_ring_empty(r);
+        spin_unlock_bh(&r->consumer_lock);
+
+        return ret;
+}
+
+/* Must only be called after __ptr_ring_peek returned !NULL */
+static inline void __ptr_ring_discard_one(struct ptr_ring *r)
+{
+        r->queue[r->consumer++] = NULL;
+        if (unlikely(r->consumer >= r->size))
+                r->consumer = 0;
+}
+
+static inline void *__ptr_ring_consume(struct ptr_ring *r)
+{
+        void *ptr;
+
+        ptr = __ptr_ring_peek(r);
+        if (ptr)
+                __ptr_ring_discard_one(r);
+
+        return ptr;
+}
+
+static inline void *ptr_ring_consume(struct ptr_ring *r)
+{
+        void *ptr;
+
+        spin_lock(&r->consumer_lock);
+        ptr = __ptr_ring_consume(r);
+        spin_unlock(&r->consumer_lock);
+
+        return ptr;
+}
+
+static inline void *ptr_ring_consume_irq(struct ptr_ring *r)
+{
+        void *ptr;
+
+        spin_lock_irq(&r->consumer_lock);
+        ptr = __ptr_ring_consume(r);
+        spin_unlock_irq(&r->consumer_lock);
+
+        return ptr;
+}
+
+static inline void *ptr_ring_consume_any(struct ptr_ring *r)
+{
+        unsigned long flags;
+        void *ptr;
+
+        spin_lock_irqsave(&r->consumer_lock, flags);
+        ptr = __ptr_ring_consume(r);
+        spin_unlock_irqrestore(&r->consumer_lock, flags);
+
+        return ptr;
+}
+
+static inline void *ptr_ring_consume_bh(struct ptr_ring *r)
+{
+        void *ptr;
+
+        spin_lock_bh(&r->consumer_lock);
+        ptr = __ptr_ring_consume(r);
+        spin_unlock_bh(&r->consumer_lock);
+
+        return ptr;
+}
+
+/* Cast to structure type and call a function without discarding from FIFO.
+ * Function must return a value.
+ * Callers must take consumer_lock.
+ */
+#define __PTR_RING_PEEK_CALL(r, f) ((f)(__ptr_ring_peek(r)))
+
+#define PTR_RING_PEEK_CALL(r, f) ({ \
+        typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \
+        \
+        spin_lock(&(r)->consumer_lock); \
+        __PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \
+        spin_unlock(&(r)->consumer_lock); \
+        __PTR_RING_PEEK_CALL_v; \
+})
+
+#define PTR_RING_PEEK_CALL_IRQ(r, f) ({ \
+        typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \
+        \
+        spin_lock_irq(&(r)->consumer_lock); \
+        __PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \
+        spin_unlock_irq(&(r)->consumer_lock); \
+        __PTR_RING_PEEK_CALL_v; \
+})
+
+#define PTR_RING_PEEK_CALL_BH(r, f) ({ \
+        typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \
+        \
+        spin_lock_bh(&(r)->consumer_lock); \
+        __PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \
+        spin_unlock_bh(&(r)->consumer_lock); \
+        __PTR_RING_PEEK_CALL_v; \
+})
+
+#define PTR_RING_PEEK_CALL_ANY(r, f) ({ \
+        typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \
+        unsigned long __PTR_RING_PEEK_CALL_f;\
+        \
+        spin_lock_irqsave(&(r)->consumer_lock, __PTR_RING_PEEK_CALL_f); \
+        __PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \
+        spin_unlock_irqrestore(&(r)->consumer_lock, __PTR_RING_PEEK_CALL_f); \
+        __PTR_RING_PEEK_CALL_v; \
+})
+
+static inline void **__ptr_ring_init_queue_alloc(int size, gfp_t gfp)
+{
+        return kzalloc(ALIGN(size * sizeof(void *), SMP_CACHE_BYTES), gfp);
+}
+
+static inline int ptr_ring_init(struct ptr_ring *r, int size, gfp_t gfp)
+{
+        r->queue = __ptr_ring_init_queue_alloc(size, gfp);
+        if (!r->queue)
+                return -ENOMEM;
+
+        r->size = size;
+        r->producer = r->consumer = 0;
+        spin_lock_init(&r->producer_lock);
+        spin_lock_init(&r->consumer_lock);
+
+        return 0;
+}
+
+static inline int ptr_ring_resize(struct ptr_ring *r, int size, gfp_t gfp,
+                                  void (*destroy)(void *))
+{
+        unsigned long flags;
+        int producer = 0;
+        void **queue = __ptr_ring_init_queue_alloc(size, gfp);
+        void **old;
+        void *ptr;
+
+        if (!queue)
+                return -ENOMEM;
+
+        spin_lock_irqsave(&(r)->producer_lock, flags);
+
+        while ((ptr = ptr_ring_consume(r)))
+                if (producer < size)
+                        queue[producer++] = ptr;
+                else if (destroy)
+                        destroy(ptr);
+
+        r->size = size;
+        r->producer = producer;
+        r->consumer = 0;
+        old = r->queue;
+        r->queue = queue;
+
+        spin_unlock_irqrestore(&(r)->producer_lock, flags);
+
+        kfree(old);
+
+        return 0;
+}
+
+static inline void ptr_ring_cleanup(struct ptr_ring *r, void (*destroy)(void *))
+{
+        void *ptr;
+
+        if (destroy)
+                while ((ptr = ptr_ring_consume(r)))
+                        destroy(ptr);
+        kfree(r->queue);
+}
+
+#endif /* _LINUX_PTR_RING_H  */
diff --git a/include/linux/qed/common_hsi.h b/include/linux/qed/common_hsi.h
index 3f14c7efe68f..40c0ada01806 100644
--- a/include/linux/qed/common_hsi.h
+++ b/include/linux/qed/common_hsi.h
@@ -12,10 +12,21 @@
 #define CORE_SPQE_PAGE_SIZE_BYTES                       4096
 
 #define X_FINAL_CLEANUP_AGG_INT 1
+#define NUM_OF_GLOBAL_QUEUES                            128
+
+/* Queue Zone sizes in bytes */
+#define TSTORM_QZONE_SIZE 8
+#define MSTORM_QZONE_SIZE 0
+#define USTORM_QZONE_SIZE 8
+#define XSTORM_QZONE_SIZE 8
+#define YSTORM_QZONE_SIZE 0
+#define PSTORM_QZONE_SIZE 0
+
+#define ETH_MAX_NUM_RX_QUEUES_PER_VF 16
 
 #define FW_MAJOR_VERSION        8
-#define FW_MINOR_VERSION        7
-#define FW_REVISION_VERSION     3
+#define FW_MINOR_VERSION        10
+#define FW_REVISION_VERSION     5
 #define FW_ENGINEERING_VERSION  0
 
 /***********************/
@@ -97,45 +108,86 @@
 #define DQ_XCM_AGG_VAL_SEL_REG6   7
 
 /* XCM agg val selection */
-#define DQ_XCM_ETH_EDPM_NUM_BDS_CMD \
-        DQ_XCM_AGG_VAL_SEL_WORD2
-#define DQ_XCM_ETH_TX_BD_CONS_CMD \
-        DQ_XCM_AGG_VAL_SEL_WORD3
-#define DQ_XCM_CORE_TX_BD_CONS_CMD \
-        DQ_XCM_AGG_VAL_SEL_WORD3
-#define DQ_XCM_ETH_TX_BD_PROD_CMD \
-        DQ_XCM_AGG_VAL_SEL_WORD4
-#define DQ_XCM_CORE_TX_BD_PROD_CMD \
-        DQ_XCM_AGG_VAL_SEL_WORD4
-#define DQ_XCM_CORE_SPQ_PROD_CMD \
-        DQ_XCM_AGG_VAL_SEL_WORD4
-#define DQ_XCM_ETH_GO_TO_BD_CONS_CMD            DQ_XCM_AGG_VAL_SEL_WORD5
+#define DQ_XCM_CORE_TX_BD_CONS_CMD      DQ_XCM_AGG_VAL_SEL_WORD3
+#define DQ_XCM_CORE_TX_BD_PROD_CMD      DQ_XCM_AGG_VAL_SEL_WORD4
+#define DQ_XCM_CORE_SPQ_PROD_CMD        DQ_XCM_AGG_VAL_SEL_WORD4
+#define DQ_XCM_ETH_EDPM_NUM_BDS_CMD     DQ_XCM_AGG_VAL_SEL_WORD2
+#define DQ_XCM_ETH_TX_BD_CONS_CMD       DQ_XCM_AGG_VAL_SEL_WORD3
+#define DQ_XCM_ETH_TX_BD_PROD_CMD       DQ_XCM_AGG_VAL_SEL_WORD4
+#define DQ_XCM_ETH_GO_TO_BD_CONS_CMD    DQ_XCM_AGG_VAL_SEL_WORD5
+
+/* UCM agg val selection (HW) */
+#define DQ_UCM_AGG_VAL_SEL_WORD0        0
+#define DQ_UCM_AGG_VAL_SEL_WORD1        1
+#define DQ_UCM_AGG_VAL_SEL_WORD2        2
+#define DQ_UCM_AGG_VAL_SEL_WORD3        3
+#define DQ_UCM_AGG_VAL_SEL_REG0 4
+#define DQ_UCM_AGG_VAL_SEL_REG1 5
+#define DQ_UCM_AGG_VAL_SEL_REG2 6
+#define DQ_UCM_AGG_VAL_SEL_REG3 7
+
+/* UCM agg val selection (FW) */
+#define DQ_UCM_ETH_PMD_TX_CONS_CMD      DQ_UCM_AGG_VAL_SEL_WORD2
+#define DQ_UCM_ETH_PMD_RX_CONS_CMD      DQ_UCM_AGG_VAL_SEL_WORD3
+#define DQ_UCM_ROCE_CQ_CONS_CMD         DQ_UCM_AGG_VAL_SEL_REG0
+#define DQ_UCM_ROCE_CQ_PROD_CMD         DQ_UCM_AGG_VAL_SEL_REG2
+
+/* TCM agg val selection (HW) */
+#define DQ_TCM_AGG_VAL_SEL_WORD0        0
+#define DQ_TCM_AGG_VAL_SEL_WORD1        1
+#define DQ_TCM_AGG_VAL_SEL_WORD2        2
+#define DQ_TCM_AGG_VAL_SEL_WORD3        3
+#define DQ_TCM_AGG_VAL_SEL_REG1         4
+#define DQ_TCM_AGG_VAL_SEL_REG2         5
+#define DQ_TCM_AGG_VAL_SEL_REG6         6
+#define DQ_TCM_AGG_VAL_SEL_REG9         7
+
+/* TCM agg val selection (FW) */
+#define DQ_TCM_L2B_BD_PROD_CMD \
+        DQ_TCM_AGG_VAL_SEL_WORD1
+#define DQ_TCM_ROCE_RQ_PROD_CMD \
+        DQ_TCM_AGG_VAL_SEL_WORD0
 
 /* XCM agg counter flag selection */
-#define DQ_XCM_AGG_FLG_SHIFT_BIT14  0
-#define DQ_XCM_AGG_FLG_SHIFT_BIT15  1
-#define DQ_XCM_AGG_FLG_SHIFT_CF12   2
-#define DQ_XCM_AGG_FLG_SHIFT_CF13   3
-#define DQ_XCM_AGG_FLG_SHIFT_CF18   4
-#define DQ_XCM_AGG_FLG_SHIFT_CF19   5
-#define DQ_XCM_AGG_FLG_SHIFT_CF22   6
-#define DQ_XCM_AGG_FLG_SHIFT_CF23   7
+#define DQ_XCM_AGG_FLG_SHIFT_BIT14      0
+#define DQ_XCM_AGG_FLG_SHIFT_BIT15      1
+#define DQ_XCM_AGG_FLG_SHIFT_CF12       2
+#define DQ_XCM_AGG_FLG_SHIFT_CF13       3
+#define DQ_XCM_AGG_FLG_SHIFT_CF18       4
+#define DQ_XCM_AGG_FLG_SHIFT_CF19       5
+#define DQ_XCM_AGG_FLG_SHIFT_CF22       6
+#define DQ_XCM_AGG_FLG_SHIFT_CF23       7
 
 /* XCM agg counter flag selection */
-#define DQ_XCM_ETH_DQ_CF_CMD            (1 << \
-                                        DQ_XCM_AGG_FLG_SHIFT_CF18)
-#define DQ_XCM_CORE_DQ_CF_CMD           (1 << \
-                                        DQ_XCM_AGG_FLG_SHIFT_CF18)
-#define DQ_XCM_ETH_TERMINATE_CMD        (1 << \
-                                        DQ_XCM_AGG_FLG_SHIFT_CF19)
-#define DQ_XCM_CORE_TERMINATE_CMD       (1 << \
-                                        DQ_XCM_AGG_FLG_SHIFT_CF19)
-#define DQ_XCM_ETH_SLOW_PATH_CMD        (1 << \
-                                        DQ_XCM_AGG_FLG_SHIFT_CF22)
-#define DQ_XCM_CORE_SLOW_PATH_CMD       (1 << \
-                                        DQ_XCM_AGG_FLG_SHIFT_CF22)
-#define DQ_XCM_ETH_TPH_EN_CMD           (1 << \
-                                        DQ_XCM_AGG_FLG_SHIFT_CF23)
+#define DQ_XCM_CORE_DQ_CF_CMD           (1 << DQ_XCM_AGG_FLG_SHIFT_CF18)
+#define DQ_XCM_CORE_TERMINATE_CMD       (1 << DQ_XCM_AGG_FLG_SHIFT_CF19)
+#define DQ_XCM_CORE_SLOW_PATH_CMD       (1 << DQ_XCM_AGG_FLG_SHIFT_CF22)
+#define DQ_XCM_ETH_DQ_CF_CMD            (1 << DQ_XCM_AGG_FLG_SHIFT_CF18)
+#define DQ_XCM_ETH_TERMINATE_CMD        (1 << DQ_XCM_AGG_FLG_SHIFT_CF19)
+#define DQ_XCM_ETH_SLOW_PATH_CMD        (1 << DQ_XCM_AGG_FLG_SHIFT_CF22)
+#define DQ_XCM_ETH_TPH_EN_CMD           (1 << DQ_XCM_AGG_FLG_SHIFT_CF23)
+
+/* UCM agg counter flag selection (HW) */
+#define DQ_UCM_AGG_FLG_SHIFT_CF0        0
+#define DQ_UCM_AGG_FLG_SHIFT_CF1        1
+#define DQ_UCM_AGG_FLG_SHIFT_CF3        2
+#define DQ_UCM_AGG_FLG_SHIFT_CF4        3
+#define DQ_UCM_AGG_FLG_SHIFT_CF5        4
+#define DQ_UCM_AGG_FLG_SHIFT_CF6        5
+#define DQ_UCM_AGG_FLG_SHIFT_RULE0EN    6
+#define DQ_UCM_AGG_FLG_SHIFT_RULE1EN    7
+
+/* UCM agg counter flag selection (FW) */
+#define DQ_UCM_ETH_PMD_TX_ARM_CMD       (1 << DQ_UCM_AGG_FLG_SHIFT_CF4)
+#define DQ_UCM_ETH_PMD_RX_ARM_CMD       (1 << DQ_UCM_AGG_FLG_SHIFT_CF5)
+
+#define DQ_REGION_SHIFT (12)
+
+/* DPM */
+#define DQ_DPM_WQE_BUFF_SIZE    (320)
+
+/* Conn type ranges */
+#define DQ_CONN_TYPE_RANGE_SHIFT        (4)
 
 /*****************/
 /* QM CONSTANTS  */
@@ -282,8 +334,6 @@
         (PXP_EXTERNAL_BAR_GLOBAL_WINDOW_START + \
          PXP_EXTERNAL_BAR_GLOBAL_WINDOW_LENGTH - 1)
 
-#define PXP_ILT_PAGE_SIZE_NUM_BITS_MIN  12
-#define PXP_ILT_BLOCK_FACTOR_MULTIPLIER 1024
 
 #define PXP_VF_BAR0_START_IGU                   0
 #define PXP_VF_BAR0_IGU_LENGTH                  0x3000
@@ -342,6 +392,9 @@
 
 #define PXP_VF_BAR0_GRC_WINDOW_LENGTH           32
 
+#define PXP_ILT_PAGE_SIZE_NUM_BITS_MIN          12
+#define PXP_ILT_BLOCK_FACTOR_MULTIPLIER         1024
+
 /* ILT Records */
 #define PXP_NUM_ILT_RECORDS_BB 7600
 #define PXP_NUM_ILT_RECORDS_K2 11000
@@ -379,6 +432,38 @@ struct async_data {
         u8      fw_debug_param;
 };
 
+struct coalescing_timeset {
+        u8 value;
+#define COALESCING_TIMESET_TIMESET_MASK         0x7F
+#define COALESCING_TIMESET_TIMESET_SHIFT        0
+#define COALESCING_TIMESET_VALID_MASK           0x1
+#define COALESCING_TIMESET_VALID_SHIFT          7
+};
+
+struct common_prs_pf_msg_info {
+        __le32 value;
+#define COMMON_PRS_PF_MSG_INFO_NPAR_DEFAULT_PF_MASK     0x1
+#define COMMON_PRS_PF_MSG_INFO_NPAR_DEFAULT_PF_SHIFT    0
+#define COMMON_PRS_PF_MSG_INFO_FW_DEBUG_1_MASK          0x1
+#define COMMON_PRS_PF_MSG_INFO_FW_DEBUG_1_SHIFT         1
+#define COMMON_PRS_PF_MSG_INFO_FW_DEBUG_2_MASK          0x1
+#define COMMON_PRS_PF_MSG_INFO_FW_DEBUG_2_SHIFT         2
+#define COMMON_PRS_PF_MSG_INFO_FW_DEBUG_3_MASK          0x1
+#define COMMON_PRS_PF_MSG_INFO_FW_DEBUG_3_SHIFT         3
+#define COMMON_PRS_PF_MSG_INFO_RESERVED_MASK            0xFFFFFFF
+#define COMMON_PRS_PF_MSG_INFO_RESERVED_SHIFT           4
+};
+
+struct common_queue_zone {
+        __le16 ring_drv_data_consumer;
+        __le16 reserved;
+};
+
+struct eth_rx_prod_data {
+        __le16 bd_prod;
+        __le16 cqe_prod;
+};
+
 struct regpair {
         __le32  lo;
         __le32  hi;
@@ -388,11 +473,23 @@ struct vf_pf_channel_eqe_data {
         struct regpair msg_addr;
 };
 
+struct malicious_vf_eqe_data {
+        u8 vf_id;
+        u8 err_id;
+        __le16 reserved[3];
+};
+
+struct initial_cleanup_eqe_data {
+        u8 vf_id;
+        u8 reserved[7];
+};
+
 /* Event Data Union */
 union event_ring_data {
-        u8                              bytes[8];
-        struct vf_pf_channel_eqe_data   vf_pf_channel;
-        struct async_data               async_info;
+        u8 bytes[8];
+        struct vf_pf_channel_eqe_data vf_pf_channel;
+        struct malicious_vf_eqe_data malicious_vf;
+        struct initial_cleanup_eqe_data vf_init_cleanup;
 };
 
 /* Event Ring Entry */
@@ -420,9 +517,9 @@ enum mf_mode {
 
 /* Per-protocol connection types */
 enum protocol_type {
-        PROTOCOLID_RESERVED1,
+        PROTOCOLID_ISCSI,
         PROTOCOLID_RESERVED2,
-        PROTOCOLID_RESERVED3,
+        PROTOCOLID_ROCE,
         PROTOCOLID_CORE,
         PROTOCOLID_ETH,
         PROTOCOLID_RESERVED4,
@@ -433,6 +530,16 @@ enum protocol_type {
         MAX_PROTOCOL_TYPE
 };
 
+struct ustorm_eth_queue_zone {
+        struct coalescing_timeset int_coalescing_timeset;
+        u8 reserved[3];
+};
+
+struct ustorm_queue_zone {
+        struct ustorm_eth_queue_zone eth;
+        struct common_queue_zone common;
+};
+
 /* status block structure */
 struct cau_pi_entry {
         u32 prod;
@@ -588,7 +695,10 @@ struct parsing_and_err_flags {
 #define PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMERROR_SHIFT         15
 };
 
-/* Concrete Function ID. */
+struct pb_context {
+        __le32 crc[4];
+};
+
 struct pxp_concrete_fid {
         __le16 fid;
 #define PXP_CONCRETE_FID_PFID_MASK     0xF
@@ -655,6 +765,72 @@ struct pxp_ptt_entry {
 };
 
 /* RSS hash type */
+struct rdif_task_context {
+        __le32 initial_ref_tag;
+        __le16 app_tag_value;
+        __le16 app_tag_mask;
+        u8 flags0;
+#define RDIF_TASK_CONTEXT_IGNOREAPPTAG_MASK            0x1
+#define RDIF_TASK_CONTEXT_IGNOREAPPTAG_SHIFT           0
+#define RDIF_TASK_CONTEXT_INITIALREFTAGVALID_MASK      0x1
+#define RDIF_TASK_CONTEXT_INITIALREFTAGVALID_SHIFT     1
+#define RDIF_TASK_CONTEXT_HOSTGUARDTYPE_MASK           0x1
+#define RDIF_TASK_CONTEXT_HOSTGUARDTYPE_SHIFT          2
+#define RDIF_TASK_CONTEXT_SETERRORWITHEOP_MASK         0x1
+#define RDIF_TASK_CONTEXT_SETERRORWITHEOP_SHIFT        3
+#define RDIF_TASK_CONTEXT_PROTECTIONTYPE_MASK          0x3
+#define RDIF_TASK_CONTEXT_PROTECTIONTYPE_SHIFT         4
+#define RDIF_TASK_CONTEXT_CRC_SEED_MASK                0x1
+#define RDIF_TASK_CONTEXT_CRC_SEED_SHIFT               6
+#define RDIF_TASK_CONTEXT_KEEPREFTAGCONST_MASK         0x1
+#define RDIF_TASK_CONTEXT_KEEPREFTAGCONST_SHIFT        7
+        u8 partial_dif_data[7];
+        __le16 partial_crc_value;
+        __le16 partial_checksum_value;
+        __le32 offset_in_io;
+        __le16 flags1;
+#define RDIF_TASK_CONTEXT_VALIDATEGUARD_MASK           0x1
+#define RDIF_TASK_CONTEXT_VALIDATEGUARD_SHIFT          0
+#define RDIF_TASK_CONTEXT_VALIDATEAPPTAG_MASK          0x1
+#define RDIF_TASK_CONTEXT_VALIDATEAPPTAG_SHIFT         1
+#define RDIF_TASK_CONTEXT_VALIDATEREFTAG_MASK          0x1
+#define RDIF_TASK_CONTEXT_VALIDATEREFTAG_SHIFT         2
+#define RDIF_TASK_CONTEXT_FORWARDGUARD_MASK            0x1
+#define RDIF_TASK_CONTEXT_FORWARDGUARD_SHIFT           3
+#define RDIF_TASK_CONTEXT_FORWARDAPPTAG_MASK           0x1
+#define RDIF_TASK_CONTEXT_FORWARDAPPTAG_SHIFT          4
+#define RDIF_TASK_CONTEXT_FORWARDREFTAG_MASK           0x1
+#define RDIF_TASK_CONTEXT_FORWARDREFTAG_SHIFT          5
+#define RDIF_TASK_CONTEXT_INTERVALSIZE_MASK            0x7
+#define RDIF_TASK_CONTEXT_INTERVALSIZE_SHIFT           6
+#define RDIF_TASK_CONTEXT_HOSTINTERFACE_MASK           0x3
+#define RDIF_TASK_CONTEXT_HOSTINTERFACE_SHIFT          9
+#define RDIF_TASK_CONTEXT_DIFBEFOREDATA_MASK           0x1
+#define RDIF_TASK_CONTEXT_DIFBEFOREDATA_SHIFT          11
+#define RDIF_TASK_CONTEXT_RESERVED0_MASK               0x1
+#define RDIF_TASK_CONTEXT_RESERVED0_SHIFT              12
+#define RDIF_TASK_CONTEXT_NETWORKINTERFACE_MASK        0x1
+#define RDIF_TASK_CONTEXT_NETWORKINTERFACE_SHIFT       13
+#define RDIF_TASK_CONTEXT_FORWARDAPPTAGWITHMASK_MASK   0x1
+#define RDIF_TASK_CONTEXT_FORWARDAPPTAGWITHMASK_SHIFT  14
+#define RDIF_TASK_CONTEXT_FORWARDREFTAGWITHMASK_MASK   0x1
+#define RDIF_TASK_CONTEXT_FORWARDREFTAGWITHMASK_SHIFT  15
+        __le16 state;
+#define RDIF_TASK_CONTEXT_RECEIVEDDIFBYTESLEFT_MASK    0xF
+#define RDIF_TASK_CONTEXT_RECEIVEDDIFBYTESLEFT_SHIFT   0
+#define RDIF_TASK_CONTEXT_TRANSMITEDDIFBYTESLEFT_MASK  0xF
+#define RDIF_TASK_CONTEXT_TRANSMITEDDIFBYTESLEFT_SHIFT 4
+#define RDIF_TASK_CONTEXT_ERRORINIO_MASK               0x1
+#define RDIF_TASK_CONTEXT_ERRORINIO_SHIFT              8
+#define RDIF_TASK_CONTEXT_CHECKSUMOVERFLOW_MASK        0x1
+#define RDIF_TASK_CONTEXT_CHECKSUMOVERFLOW_SHIFT       9
+#define RDIF_TASK_CONTEXT_REFTAGMASK_MASK              0xF
+#define RDIF_TASK_CONTEXT_REFTAGMASK_SHIFT             10
+#define RDIF_TASK_CONTEXT_RESERVED1_MASK               0x3
+#define RDIF_TASK_CONTEXT_RESERVED1_SHIFT              14
+        __le32 reserved2;
+};
+
 enum rss_hash_type {
         RSS_HASH_TYPE_DEFAULT   = 0,
         RSS_HASH_TYPE_IPV4      = 1,
@@ -683,19 +859,122 @@ struct status_block {
 #define STATUS_BLOCK_ZERO_PAD3_SHIFT  24
 };
 
-struct tunnel_parsing_flags {
-        u8 flags;
-#define TUNNEL_PARSING_FLAGS_TYPE_MASK              0x3
-#define TUNNEL_PARSING_FLAGS_TYPE_SHIFT             0
-#define TUNNEL_PARSING_FLAGS_TENNANT_ID_EXIST_MASK  0x1
-#define TUNNEL_PARSING_FLAGS_TENNANT_ID_EXIST_SHIFT 2
-#define TUNNEL_PARSING_FLAGS_NEXT_PROTOCOL_MASK     0x3
-#define TUNNEL_PARSING_FLAGS_NEXT_PROTOCOL_SHIFT    3
-#define TUNNEL_PARSING_FLAGS_FIRSTHDRIPMATCH_MASK   0x1
-#define TUNNEL_PARSING_FLAGS_FIRSTHDRIPMATCH_SHIFT  5
-#define TUNNEL_PARSING_FLAGS_IPV4_FRAGMENT_MASK     0x1
-#define TUNNEL_PARSING_FLAGS_IPV4_FRAGMENT_SHIFT    6
-#define TUNNEL_PARSING_FLAGS_IPV4_OPTIONS_MASK      0x1
-#define TUNNEL_PARSING_FLAGS_IPV4_OPTIONS_SHIFT     7
+struct tdif_task_context {
+        __le32 initial_ref_tag;
+        __le16 app_tag_value;
+        __le16 app_tag_mask;
+        __le16 partial_crc_valueB;
+        __le16 partial_checksum_valueB;
+        __le16 stateB;
+#define TDIF_TASK_CONTEXT_RECEIVEDDIFBYTESLEFTB_MASK    0xF
+#define TDIF_TASK_CONTEXT_RECEIVEDDIFBYTESLEFTB_SHIFT   0
+#define TDIF_TASK_CONTEXT_TRANSMITEDDIFBYTESLEFTB_MASK  0xF
+#define TDIF_TASK_CONTEXT_TRANSMITEDDIFBYTESLEFTB_SHIFT 4
+#define TDIF_TASK_CONTEXT_ERRORINIOB_MASK               0x1
+#define TDIF_TASK_CONTEXT_ERRORINIOB_SHIFT              8
+#define TDIF_TASK_CONTEXT_CHECKSUMOVERFLOW_MASK         0x1
+#define TDIF_TASK_CONTEXT_CHECKSUMOVERFLOW_SHIFT        9
+#define TDIF_TASK_CONTEXT_RESERVED0_MASK                0x3F
+#define TDIF_TASK_CONTEXT_RESERVED0_SHIFT               10
+        u8 reserved1;
+        u8 flags0;
+#define TDIF_TASK_CONTEXT_IGNOREAPPTAG_MASK             0x1
+#define TDIF_TASK_CONTEXT_IGNOREAPPTAG_SHIFT            0
+#define TDIF_TASK_CONTEXT_INITIALREFTAGVALID_MASK       0x1
+#define TDIF_TASK_CONTEXT_INITIALREFTAGVALID_SHIFT      1
+#define TDIF_TASK_CONTEXT_HOSTGUARDTYPE_MASK            0x1
+#define TDIF_TASK_CONTEXT_HOSTGUARDTYPE_SHIFT           2
+#define TDIF_TASK_CONTEXT_SETERRORWITHEOP_MASK          0x1
+#define TDIF_TASK_CONTEXT_SETERRORWITHEOP_SHIFT         3
+#define TDIF_TASK_CONTEXT_PROTECTIONTYPE_MASK           0x3
+#define TDIF_TASK_CONTEXT_PROTECTIONTYPE_SHIFT          4
+#define TDIF_TASK_CONTEXT_CRC_SEED_MASK                 0x1
+#define TDIF_TASK_CONTEXT_CRC_SEED_SHIFT                6
+#define TDIF_TASK_CONTEXT_RESERVED2_MASK                0x1
+#define TDIF_TASK_CONTEXT_RESERVED2_SHIFT               7
+        __le32 flags1;
+#define TDIF_TASK_CONTEXT_VALIDATEGUARD_MASK            0x1
+#define TDIF_TASK_CONTEXT_VALIDATEGUARD_SHIFT           0
+#define TDIF_TASK_CONTEXT_VALIDATEAPPTAG_MASK           0x1
+#define TDIF_TASK_CONTEXT_VALIDATEAPPTAG_SHIFT          1
+#define TDIF_TASK_CONTEXT_VALIDATEREFTAG_MASK           0x1
+#define TDIF_TASK_CONTEXT_VALIDATEREFTAG_SHIFT          2
+#define TDIF_TASK_CONTEXT_FORWARDGUARD_MASK             0x1
+#define TDIF_TASK_CONTEXT_FORWARDGUARD_SHIFT            3
+#define TDIF_TASK_CONTEXT_FORWARDAPPTAG_MASK            0x1
+#define TDIF_TASK_CONTEXT_FORWARDAPPTAG_SHIFT           4
+#define TDIF_TASK_CONTEXT_FORWARDREFTAG_MASK            0x1
+#define TDIF_TASK_CONTEXT_FORWARDREFTAG_SHIFT           5
+#define TDIF_TASK_CONTEXT_INTERVALSIZE_MASK             0x7
+#define TDIF_TASK_CONTEXT_INTERVALSIZE_SHIFT            6
+#define TDIF_TASK_CONTEXT_HOSTINTERFACE_MASK            0x3
+#define TDIF_TASK_CONTEXT_HOSTINTERFACE_SHIFT           9
+#define TDIF_TASK_CONTEXT_DIFBEFOREDATA_MASK            0x1
+#define TDIF_TASK_CONTEXT_DIFBEFOREDATA_SHIFT           11
+#define TDIF_TASK_CONTEXT_RESERVED3_MASK                0x1
+#define TDIF_TASK_CONTEXT_RESERVED3_SHIFT               12
+#define TDIF_TASK_CONTEXT_NETWORKINTERFACE_MASK         0x1
+#define TDIF_TASK_CONTEXT_NETWORKINTERFACE_SHIFT        13
+#define TDIF_TASK_CONTEXT_RECEIVEDDIFBYTESLEFTA_MASK    0xF
+#define TDIF_TASK_CONTEXT_RECEIVEDDIFBYTESLEFTA_SHIFT   14
+#define TDIF_TASK_CONTEXT_TRANSMITEDDIFBYTESLEFTA_MASK  0xF
+#define TDIF_TASK_CONTEXT_TRANSMITEDDIFBYTESLEFTA_SHIFT 18
+#define TDIF_TASK_CONTEXT_ERRORINIOA_MASK               0x1
+#define TDIF_TASK_CONTEXT_ERRORINIOA_SHIFT              22
+#define TDIF_TASK_CONTEXT_CHECKSUMOVERFLOWA_MASK        0x1
+#define TDIF_TASK_CONTEXT_CHECKSUMOVERFLOWA_SHIFT       23
+#define TDIF_TASK_CONTEXT_REFTAGMASK_MASK               0xF
+#define TDIF_TASK_CONTEXT_REFTAGMASK_SHIFT              24
+#define TDIF_TASK_CONTEXT_FORWARDAPPTAGWITHMASK_MASK    0x1
+#define TDIF_TASK_CONTEXT_FORWARDAPPTAGWITHMASK_SHIFT   28
+#define TDIF_TASK_CONTEXT_FORWARDREFTAGWITHMASK_MASK    0x1
+#define TDIF_TASK_CONTEXT_FORWARDREFTAGWITHMASK_SHIFT   29
+#define TDIF_TASK_CONTEXT_KEEPREFTAGCONST_MASK          0x1
+#define TDIF_TASK_CONTEXT_KEEPREFTAGCONST_SHIFT         30
+#define TDIF_TASK_CONTEXT_RESERVED4_MASK                0x1
+#define TDIF_TASK_CONTEXT_RESERVED4_SHIFT               31
+        __le32 offset_in_iob;
+        __le16 partial_crc_value_a;
+        __le16 partial_checksum_valuea_;
+        __le32 offset_in_ioa;
+        u8 partial_dif_data_a[8];
+        u8 partial_dif_data_b[8];
+};
+
+struct timers_context {
+        __le32 logical_client0;
+#define TIMERS_CONTEXT_EXPIRATIONTIMELC0_MASK     0xFFFFFFF
+#define TIMERS_CONTEXT_EXPIRATIONTIMELC0_SHIFT    0
+#define TIMERS_CONTEXT_VALIDLC0_MASK              0x1
+#define TIMERS_CONTEXT_VALIDLC0_SHIFT             28
+#define TIMERS_CONTEXT_ACTIVELC0_MASK             0x1
+#define TIMERS_CONTEXT_ACTIVELC0_SHIFT            29
+#define TIMERS_CONTEXT_RESERVED0_MASK             0x3
+#define TIMERS_CONTEXT_RESERVED0_SHIFT            30
+        __le32 logical_client1;
+#define TIMERS_CONTEXT_EXPIRATIONTIMELC1_MASK     0xFFFFFFF
+#define TIMERS_CONTEXT_EXPIRATIONTIMELC1_SHIFT    0
+#define TIMERS_CONTEXT_VALIDLC1_MASK              0x1
+#define TIMERS_CONTEXT_VALIDLC1_SHIFT             28
+#define TIMERS_CONTEXT_ACTIVELC1_MASK             0x1
+#define TIMERS_CONTEXT_ACTIVELC1_SHIFT            29
+#define TIMERS_CONTEXT_RESERVED1_MASK             0x3
+#define TIMERS_CONTEXT_RESERVED1_SHIFT            30
+        __le32 logical_client2;
+#define TIMERS_CONTEXT_EXPIRATIONTIMELC2_MASK     0xFFFFFFF
+#define TIMERS_CONTEXT_EXPIRATIONTIMELC2_SHIFT    0
+#define TIMERS_CONTEXT_VALIDLC2_MASK              0x1
+#define TIMERS_CONTEXT_VALIDLC2_SHIFT             28
+#define TIMERS_CONTEXT_ACTIVELC2_MASK             0x1
+#define TIMERS_CONTEXT_ACTIVELC2_SHIFT            29
+#define TIMERS_CONTEXT_RESERVED2_MASK             0x3
+#define TIMERS_CONTEXT_RESERVED2_SHIFT            30
+        __le32 host_expiration_fields;
+#define TIMERS_CONTEXT_HOSTEXPRIRATIONVALUE_MASK  0xFFFFFFF
+#define TIMERS_CONTEXT_HOSTEXPRIRATIONVALUE_SHIFT 0
+#define TIMERS_CONTEXT_HOSTEXPRIRATIONVALID_MASK  0x1
+#define TIMERS_CONTEXT_HOSTEXPRIRATIONVALID_SHIFT 28
+#define TIMERS_CONTEXT_RESERVED3_MASK             0x7
+#define TIMERS_CONTEXT_RESERVED3_SHIFT            29
 };
 #endif /* __COMMON_HSI__ */
diff --git a/include/linux/qed/eth_common.h b/include/linux/qed/eth_common.h
index 092cb0c1afcb..b5ebc697d05f 100644
--- a/include/linux/qed/eth_common.h
+++ b/include/linux/qed/eth_common.h
@@ -12,6 +12,8 @@
 /********************/
 /* ETH FW CONSTANTS */
 /********************/
+#define ETH_HSI_VER_MAJOR                   3
+#define ETH_HSI_VER_MINOR                   0
 #define ETH_CACHE_LINE_SIZE                 64
 
 #define ETH_MAX_RAMROD_PER_CON                          8
@@ -57,19 +59,6 @@
 #define ETH_TPA_CQE_CONT_LEN_LIST_SIZE    6
 #define ETH_TPA_CQE_END_LEN_LIST_SIZE     4
 
-/* Queue Zone sizes */
-#define TSTORM_QZONE_SIZE    0
-#define MSTORM_QZONE_SIZE    sizeof(struct mstorm_eth_queue_zone)
-#define USTORM_QZONE_SIZE    sizeof(struct ustorm_eth_queue_zone)
-#define XSTORM_QZONE_SIZE    0
-#define YSTORM_QZONE_SIZE    sizeof(struct ystorm_eth_queue_zone)
-#define PSTORM_QZONE_SIZE    0
-
-/* Interrupt coalescing TimeSet */
-struct coalescing_timeset {
-        u8      timeset;
-        u8      valid;
-};
 
 struct eth_tx_1st_bd_flags {
         u8 bitfields;
@@ -97,12 +86,12 @@ struct eth_tx_data_1st_bd {
         u8                              nbds;
         struct eth_tx_1st_bd_flags      bd_flags;
         __le16                          bitfields;
-#define ETH_TX_DATA_1ST_BD_TUNN_CFG_OVERRIDE_MASK  0x1
-#define ETH_TX_DATA_1ST_BD_TUNN_CFG_OVERRIDE_SHIFT 0
+#define ETH_TX_DATA_1ST_BD_TUNN_FLAG_MASK  0x1
+#define ETH_TX_DATA_1ST_BD_TUNN_FLAG_SHIFT 0
 #define ETH_TX_DATA_1ST_BD_RESERVED0_MASK          0x1
 #define ETH_TX_DATA_1ST_BD_RESERVED0_SHIFT         1
-#define ETH_TX_DATA_1ST_BD_FW_USE_ONLY_MASK        0x3FFF
-#define ETH_TX_DATA_1ST_BD_FW_USE_ONLY_SHIFT       2
+#define ETH_TX_DATA_1ST_BD_PKT_LEN_MASK    0x3FFF
+#define ETH_TX_DATA_1ST_BD_PKT_LEN_SHIFT   2
 };
 
 /* The parsing information data for the second tx bd of a given packet. */
@@ -136,28 +125,51 @@ struct eth_tx_data_2nd_bd {
 #define ETH_TX_DATA_2ND_BD_RESERVED0_SHIFT                13
 };
 
+struct eth_fast_path_cqe_fw_debug {
+        u8 reserved0;
+        u8 reserved1;
+        __le16 reserved2;
+};
+
+/*  tunneling parsing flags */
+struct eth_tunnel_parsing_flags {
+        u8 flags;
+#define ETH_TUNNEL_PARSING_FLAGS_TYPE_MASK              0x3
+#define ETH_TUNNEL_PARSING_FLAGS_TYPE_SHIFT             0
+#define ETH_TUNNEL_PARSING_FLAGS_TENNANT_ID_EXIST_MASK  0x1
+#define ETH_TUNNEL_PARSING_FLAGS_TENNANT_ID_EXIST_SHIFT 2
+#define ETH_TUNNEL_PARSING_FLAGS_NEXT_PROTOCOL_MASK     0x3
+#define ETH_TUNNEL_PARSING_FLAGS_NEXT_PROTOCOL_SHIFT    3
+#define ETH_TUNNEL_PARSING_FLAGS_FIRSTHDRIPMATCH_MASK   0x1
+#define ETH_TUNNEL_PARSING_FLAGS_FIRSTHDRIPMATCH_SHIFT  5
+#define ETH_TUNNEL_PARSING_FLAGS_IPV4_FRAGMENT_MASK     0x1
+#define ETH_TUNNEL_PARSING_FLAGS_IPV4_FRAGMENT_SHIFT    6
+#define ETH_TUNNEL_PARSING_FLAGS_IPV4_OPTIONS_MASK      0x1
+#define ETH_TUNNEL_PARSING_FLAGS_IPV4_OPTIONS_SHIFT     7
+};
+
 /* Regular ETH Rx FP CQE. */
 struct eth_fast_path_rx_reg_cqe {
-        u8      type;
-        u8      bitfields;
+        u8 type;
+        u8 bitfields;
 #define ETH_FAST_PATH_RX_REG_CQE_RSS_HASH_TYPE_MASK  0x7
 #define ETH_FAST_PATH_RX_REG_CQE_RSS_HASH_TYPE_SHIFT 0
 #define ETH_FAST_PATH_RX_REG_CQE_TC_MASK             0xF
 #define ETH_FAST_PATH_RX_REG_CQE_TC_SHIFT            3
 #define ETH_FAST_PATH_RX_REG_CQE_RESERVED0_MASK      0x1
 #define ETH_FAST_PATH_RX_REG_CQE_RESERVED0_SHIFT     7
-        __le16                          pkt_len;
-        struct parsing_and_err_flags    pars_flags;
-        __le16                          vlan_tag;
-        __le32                          rss_hash;
-        __le16                          len_on_first_bd;
-        u8                              placement_offset;
-        struct tunnel_parsing_flags     tunnel_pars_flags;
-        u8                              bd_num;
-        u8                              reserved[7];
-        u32                             fw_debug;
-        u8                              reserved1[3];
-        u8                              flags;
+        __le16 pkt_len;
+        struct parsing_and_err_flags pars_flags;
+        __le16 vlan_tag;
+        __le32 rss_hash;
+        __le16 len_on_first_bd;
+        u8 placement_offset;
+        struct eth_tunnel_parsing_flags tunnel_pars_flags;
+        u8 bd_num;
+        u8 reserved[7];
+        struct eth_fast_path_cqe_fw_debug fw_debug;
+        u8 reserved1[3];
+        u8 flags;
 #define ETH_FAST_PATH_RX_REG_CQE_VALID_MASK          0x1
 #define ETH_FAST_PATH_RX_REG_CQE_VALID_SHIFT         0
 #define ETH_FAST_PATH_RX_REG_CQE_VALID_TOGGLE_MASK   0x1
@@ -207,11 +219,11 @@ struct eth_fast_path_rx_tpa_start_cqe {
         __le32  rss_hash;
         __le16  len_on_first_bd;
         u8      placement_offset;
-        struct tunnel_parsing_flags tunnel_pars_flags;
+        struct eth_tunnel_parsing_flags tunnel_pars_flags;
         u8      tpa_agg_index;
         u8      header_len;
         __le16  ext_bd_len_list[ETH_TPA_CQE_START_LEN_LIST_SIZE];
-        u32     fw_debug;
+        struct eth_fast_path_cqe_fw_debug fw_debug;
 };
 
 /* The L4 pseudo checksum mode for Ethernet */
@@ -264,12 +276,25 @@ enum eth_rx_cqe_type {
         MAX_ETH_RX_CQE_TYPE
 };
 
-/* ETH Rx producers data */
-struct eth_rx_prod_data {
-        __le16  bd_prod;
-        __le16  cqe_prod;
-        __le16  reserved;
-        __le16  reserved1;
+enum eth_rx_tunn_type {
+        ETH_RX_NO_TUNN,
+        ETH_RX_TUNN_GENEVE,
+        ETH_RX_TUNN_GRE,
+        ETH_RX_TUNN_VXLAN,
+        MAX_ETH_RX_TUNN_TYPE
+};
+
+/*  Aggregation end reason. */
+enum eth_tpa_end_reason {
+        ETH_AGG_END_UNUSED,
+        ETH_AGG_END_SP_UPDATE,
+        ETH_AGG_END_MAX_LEN,
+        ETH_AGG_END_LAST_SEG,
+        ETH_AGG_END_TIMEOUT,
+        ETH_AGG_END_NOT_CONSISTENT,
+        ETH_AGG_END_OUT_OF_ORDER,
+        ETH_AGG_END_NON_TPA_SEG,
+        MAX_ETH_TPA_END_REASON
 };
 
 /* The first tx bd of a given packet */
@@ -337,21 +362,18 @@ union eth_tx_bd_types {
 };
 
 /* Mstorm Queue Zone */
-struct mstorm_eth_queue_zone {
-        struct eth_rx_prod_data rx_producers;
-        __le32                  reserved[2];
-};
-
-/* Ustorm Queue Zone */
-struct ustorm_eth_queue_zone {
-        struct coalescing_timeset       int_coalescing_timeset;
-        __le16                          reserved[3];
+enum eth_tx_tunn_type {
+        ETH_TX_TUNN_GENEVE,
+        ETH_TX_TUNN_TTAG,
+        ETH_TX_TUNN_GRE,
+        ETH_TX_TUNN_VXLAN,
+        MAX_ETH_TX_TUNN_TYPE
 };
 
 /* Ystorm Queue Zone */
-struct ystorm_eth_queue_zone {
-        struct coalescing_timeset       int_coalescing_timeset;
-        __le16                          reserved[3];
+struct xstorm_eth_queue_zone {
+        struct coalescing_timeset int_coalescing_timeset;
+        u8 reserved[7];
 };
 
 /* ETH doorbell data */
diff --git a/include/linux/qed/iscsi_common.h b/include/linux/qed/iscsi_common.h
new file mode 100644
index 000000000000..b3c0feb15ae9
--- /dev/null
+++ b/include/linux/qed/iscsi_common.h
@@ -0,0 +1,1439 @@
+/* QLogic qed NIC Driver
+ * Copyright (c) 2015 QLogic Corporation
+ *
+ * This software is available under the terms of the GNU General Public License
+ * (GPL) Version 2, available from the file COPYING in the main directory of
+ * this source tree.
+ */
+
+#ifndef __ISCSI_COMMON__
+#define __ISCSI_COMMON__
+/**********************/
+/* ISCSI FW CONSTANTS */
+/**********************/
+
+/* iSCSI HSI constants */
+#define ISCSI_DEFAULT_MTU       (1500)
+
+/* Current iSCSI HSI version number composed of two fields (16 bit) */
+#define ISCSI_HSI_MAJOR_VERSION (0)
+#define ISCSI_HSI_MINOR_VERSION (0)
+
+/* KWQ (kernel work queue) layer codes */
+#define ISCSI_SLOW_PATH_LAYER_CODE   (6)
+
+/* CQE completion status */
+#define ISCSI_EQE_COMPLETION_SUCCESS (0x0)
+#define ISCSI_EQE_RST_CONN_RCVD (0x1)
+
+/* iSCSI parameter defaults */
+#define ISCSI_DEFAULT_HEADER_DIGEST         (0)
+#define ISCSI_DEFAULT_DATA_DIGEST           (0)
+#define ISCSI_DEFAULT_INITIAL_R2T           (1)
+#define ISCSI_DEFAULT_IMMEDIATE_DATA        (1)
+#define ISCSI_DEFAULT_MAX_PDU_LENGTH        (0x2000)
+#define ISCSI_DEFAULT_FIRST_BURST_LENGTH    (0x10000)
+#define ISCSI_DEFAULT_MAX_BURST_LENGTH      (0x40000)
+#define ISCSI_DEFAULT_MAX_OUTSTANDING_R2T   (1)
+
+/* iSCSI parameter limits */
+#define ISCSI_MIN_VAL_MAX_PDU_LENGTH        (0x200)
+#define ISCSI_MAX_VAL_MAX_PDU_LENGTH        (0xffffff)
+#define ISCSI_MIN_VAL_BURST_LENGTH          (0x200)
+#define ISCSI_MAX_VAL_BURST_LENGTH          (0xffffff)
+#define ISCSI_MIN_VAL_MAX_OUTSTANDING_R2T   (1)
+#define ISCSI_MAX_VAL_MAX_OUTSTANDING_R2T   (0xff)
+
+/* iSCSI reserved params */
+#define ISCSI_ITT_ALL_ONES      (0xffffffff)
+#define ISCSI_TTT_ALL_ONES      (0xffffffff)
+
+#define ISCSI_OPTION_1_OFF_CHIP_TCP 1
+#define ISCSI_OPTION_2_ON_CHIP_TCP 2
+
+#define ISCSI_INITIATOR_MODE 0
+#define ISCSI_TARGET_MODE 1
+
+/* iSCSI request op codes */
+#define ISCSI_OPCODE_NOP_OUT_NO_IMM                     (0)
+#define ISCSI_OPCODE_NOP_OUT                            ( \
+                ISCSI_OPCODE_NOP_OUT_NO_IMM | 0x40)
+#define ISCSI_OPCODE_SCSI_CMD_NO_IMM            (1)
+#define ISCSI_OPCODE_SCSI_CMD                           ( \
+                ISCSI_OPCODE_SCSI_CMD_NO_IMM | 0x40)
+#define ISCSI_OPCODE_TMF_REQUEST_NO_IMM         (2)
+#define ISCSI_OPCODE_TMF_REQUEST                        ( \
+                ISCSI_OPCODE_TMF_REQUEST_NO_IMM | 0x40)
+#define ISCSI_OPCODE_LOGIN_REQUEST_NO_IMM       (3)
+#define ISCSI_OPCODE_LOGIN_REQUEST                      ( \
+                ISCSI_OPCODE_LOGIN_REQUEST_NO_IMM | 0x40)
+#define ISCSI_OPCODE_TEXT_REQUEST_NO_IMM        (4)
+#define ISCSI_OPCODE_TEXT_REQUEST                       ( \
+                ISCSI_OPCODE_TEXT_REQUEST_NO_IMM | 0x40)
+#define ISCSI_OPCODE_DATA_OUT                           (5)
+#define ISCSI_OPCODE_LOGOUT_REQUEST_NO_IMM      (6)
+#define ISCSI_OPCODE_LOGOUT_REQUEST                     ( \
+                ISCSI_OPCODE_LOGOUT_REQUEST_NO_IMM | 0x40)
+
+/* iSCSI response/messages op codes */
+#define ISCSI_OPCODE_NOP_IN             (0x20)
+#define ISCSI_OPCODE_SCSI_RESPONSE      (0x21)
+#define ISCSI_OPCODE_TMF_RESPONSE       (0x22)
+#define ISCSI_OPCODE_LOGIN_RESPONSE     (0x23)
+#define ISCSI_OPCODE_TEXT_RESPONSE      (0x24)
+#define ISCSI_OPCODE_DATA_IN            (0x25)
+#define ISCSI_OPCODE_LOGOUT_RESPONSE    (0x26)
+#define ISCSI_OPCODE_R2T                (0x31)
+#define ISCSI_OPCODE_ASYNC_MSG          (0x32)
+#define ISCSI_OPCODE_REJECT             (0x3f)
+
+/* iSCSI stages */
+#define ISCSI_STAGE_SECURITY_NEGOTIATION            (0)
+#define ISCSI_STAGE_LOGIN_OPERATIONAL_NEGOTIATION   (1)
+#define ISCSI_STAGE_FULL_FEATURE_PHASE              (3)
+
+/* iSCSI CQE errors */
+#define CQE_ERROR_BITMAP_DATA_DIGEST          (0x08)
+#define CQE_ERROR_BITMAP_RCV_ON_INVALID_CONN  (0x10)
+#define CQE_ERROR_BITMAP_DATA_TRUNCATED       (0x20)
+
+struct cqe_error_bitmap {
+        u8 cqe_error_status_bits;
+#define CQE_ERROR_BITMAP_DIF_ERR_BITS_MASK         0x7
+#define CQE_ERROR_BITMAP_DIF_ERR_BITS_SHIFT        0
+#define CQE_ERROR_BITMAP_DATA_DIGEST_ERR_MASK      0x1
+#define CQE_ERROR_BITMAP_DATA_DIGEST_ERR_SHIFT     3
+#define CQE_ERROR_BITMAP_RCV_ON_INVALID_CONN_MASK  0x1
+#define CQE_ERROR_BITMAP_RCV_ON_INVALID_CONN_SHIFT 4
+#define CQE_ERROR_BITMAP_DATA_TRUNCATED_ERR_MASK   0x1
+#define CQE_ERROR_BITMAP_DATA_TRUNCATED_ERR_SHIFT  5
+#define CQE_ERROR_BITMAP_UNDER_RUN_ERR_MASK        0x1
+#define CQE_ERROR_BITMAP_UNDER_RUN_ERR_SHIFT       6
+#define CQE_ERROR_BITMAP_RESERVED2_MASK            0x1
+#define CQE_ERROR_BITMAP_RESERVED2_SHIFT           7
+};
+
+union cqe_error_status {
+        u8 error_status;
+        struct cqe_error_bitmap error_bits;
+};
+
+struct data_hdr {
+        __le32 data[12];
+};
+
+struct iscsi_async_msg_hdr {
+        __le16 reserved0;
+        u8 flags_attr;
+#define ISCSI_ASYNC_MSG_HDR_RSRV_MASK           0x7F
+#define ISCSI_ASYNC_MSG_HDR_RSRV_SHIFT          0
+#define ISCSI_ASYNC_MSG_HDR_CONST1_MASK         0x1
+#define ISCSI_ASYNC_MSG_HDR_CONST1_SHIFT        7
+        u8 opcode;
+        __le32 hdr_second_dword;
+#define ISCSI_ASYNC_MSG_HDR_DATA_SEG_LEN_MASK   0xFFFFFF
+#define ISCSI_ASYNC_MSG_HDR_DATA_SEG_LEN_SHIFT  0
+#define ISCSI_ASYNC_MSG_HDR_TOTAL_AHS_LEN_MASK  0xFF
+#define ISCSI_ASYNC_MSG_HDR_TOTAL_AHS_LEN_SHIFT 24
+        struct regpair lun;
+        __le32 all_ones;
+        __le32 reserved1;
+        __le32 stat_sn;
+        __le32 exp_cmd_sn;
+        __le32 max_cmd_sn;
+        __le16 param1_rsrv;
+        u8 async_vcode;
+        u8 async_event;
+        __le16 param3_rsrv;
+        __le16 param2_rsrv;
+        __le32 reserved7;
+};
+
+struct iscsi_sge {
+        struct regpair sge_addr;
+        __le16 sge_len;
+        __le16 reserved0;
+        __le32 reserved1;
+};
+
+struct iscsi_cached_sge_ctx {
+        struct iscsi_sge sge;
+        struct regpair reserved;
+        __le32 dsgl_curr_offset[2];
+};
+
+struct iscsi_cmd_hdr {
+        __le16 reserved1;
+        u8 flags_attr;
+#define ISCSI_CMD_HDR_ATTR_MASK           0x7
+#define ISCSI_CMD_HDR_ATTR_SHIFT          0
+#define ISCSI_CMD_HDR_RSRV_MASK           0x3
+#define ISCSI_CMD_HDR_RSRV_SHIFT          3
+#define ISCSI_CMD_HDR_WRITE_MASK          0x1
+#define ISCSI_CMD_HDR_WRITE_SHIFT         5
+#define ISCSI_CMD_HDR_READ_MASK           0x1
+#define ISCSI_CMD_HDR_READ_SHIFT          6
+#define ISCSI_CMD_HDR_FINAL_MASK          0x1
+#define ISCSI_CMD_HDR_FINAL_SHIFT         7
+        u8 opcode;
+        __le32 hdr_second_dword;
+#define ISCSI_CMD_HDR_DATA_SEG_LEN_MASK   0xFFFFFF
+#define ISCSI_CMD_HDR_DATA_SEG_LEN_SHIFT  0
+#define ISCSI_CMD_HDR_TOTAL_AHS_LEN_MASK  0xFF
+#define ISCSI_CMD_HDR_TOTAL_AHS_LEN_SHIFT 24
+        struct regpair lun;
+        __le32 itt;
+        __le32 expected_transfer_length;
+        __le32 cmd_sn;
+        __le32 exp_stat_sn;
+        __le32 cdb[4];
+};
+
+struct iscsi_common_hdr {
+        u8 hdr_status;
+        u8 hdr_response;
+        u8 hdr_flags;
+        u8 hdr_first_byte;
+#define ISCSI_COMMON_HDR_OPCODE_MASK         0x3F
+#define ISCSI_COMMON_HDR_OPCODE_SHIFT        0
+#define ISCSI_COMMON_HDR_IMM_MASK            0x1
+#define ISCSI_COMMON_HDR_IMM_SHIFT           6
+#define ISCSI_COMMON_HDR_RSRV_MASK           0x1
+#define ISCSI_COMMON_HDR_RSRV_SHIFT          7
+        __le32 hdr_second_dword;
+#define ISCSI_COMMON_HDR_DATA_SEG_LEN_MASK   0xFFFFFF
+#define ISCSI_COMMON_HDR_DATA_SEG_LEN_SHIFT  0
+#define ISCSI_COMMON_HDR_TOTAL_AHS_LEN_MASK  0xFF
+#define ISCSI_COMMON_HDR_TOTAL_AHS_LEN_SHIFT 24
+        __le32 lun_reserved[4];
+        __le32 data[6];
+};
+
+struct iscsi_conn_offload_params {
+        struct regpair sq_pbl_addr;
+        struct regpair r2tq_pbl_addr;
+        struct regpair xhq_pbl_addr;
+        struct regpair uhq_pbl_addr;
+        __le32 initial_ack;
+        __le16 physical_q0;
+        __le16 physical_q1;
+        u8 flags;
+#define ISCSI_CONN_OFFLOAD_PARAMS_TCP_ON_CHIP_1B_MASK  0x1
+#define ISCSI_CONN_OFFLOAD_PARAMS_TCP_ON_CHIP_1B_SHIFT 0
+#define ISCSI_CONN_OFFLOAD_PARAMS_TARGET_MODE_MASK     0x1
+#define ISCSI_CONN_OFFLOAD_PARAMS_TARGET_MODE_SHIFT    1
+#define ISCSI_CONN_OFFLOAD_PARAMS_RESERVED1_MASK       0x3F
+#define ISCSI_CONN_OFFLOAD_PARAMS_RESERVED1_SHIFT      2
+        u8 pbl_page_size_log;
+        u8 pbe_page_size_log;
+        u8 default_cq;
+        __le32 stat_sn;
+};
+
+struct iscsi_slow_path_hdr {
+        u8 op_code;
+        u8 flags;
+#define ISCSI_SLOW_PATH_HDR_RESERVED0_MASK   0xF
+#define ISCSI_SLOW_PATH_HDR_RESERVED0_SHIFT  0
+#define ISCSI_SLOW_PATH_HDR_LAYER_CODE_MASK  0x7
+#define ISCSI_SLOW_PATH_HDR_LAYER_CODE_SHIFT 4
+#define ISCSI_SLOW_PATH_HDR_RESERVED1_MASK   0x1
+#define ISCSI_SLOW_PATH_HDR_RESERVED1_SHIFT  7
+};
+
+struct iscsi_conn_update_ramrod_params {
+        struct iscsi_slow_path_hdr hdr;
+        __le16 conn_id;
+        __le32 fw_cid;
+        u8 flags;
+#define ISCSI_CONN_UPDATE_RAMROD_PARAMS_HD_EN_MASK           0x1
+#define ISCSI_CONN_UPDATE_RAMROD_PARAMS_HD_EN_SHIFT          0
+#define ISCSI_CONN_UPDATE_RAMROD_PARAMS_DD_EN_MASK           0x1
+#define ISCSI_CONN_UPDATE_RAMROD_PARAMS_DD_EN_SHIFT          1
+#define ISCSI_CONN_UPDATE_RAMROD_PARAMS_INITIAL_R2T_MASK     0x1
+#define ISCSI_CONN_UPDATE_RAMROD_PARAMS_INITIAL_R2T_SHIFT    2
+#define ISCSI_CONN_UPDATE_RAMROD_PARAMS_IMMEDIATE_DATA_MASK  0x1
+#define ISCSI_CONN_UPDATE_RAMROD_PARAMS_IMMEDIATE_DATA_SHIFT 3
+#define ISCSI_CONN_UPDATE_RAMROD_PARAMS_RESERVED1_MASK       0xF
+#define ISCSI_CONN_UPDATE_RAMROD_PARAMS_RESERVED1_SHIFT      4
+        u8 reserved0[3];
+        __le32 max_seq_size;
+        __le32 max_send_pdu_length;
+        __le32 max_recv_pdu_length;
+        __le32 first_seq_length;
+        __le32 exp_stat_sn;
+};
+
+struct iscsi_ext_cdb_cmd_hdr {
+        __le16 reserved1;
+        u8 flags_attr;
+#define ISCSI_EXT_CDB_CMD_HDR_ATTR_MASK          0x7
+#define ISCSI_EXT_CDB_CMD_HDR_ATTR_SHIFT         0
+#define ISCSI_EXT_CDB_CMD_HDR_RSRV_MASK          0x3
+#define ISCSI_EXT_CDB_CMD_HDR_RSRV_SHIFT         3
+#define ISCSI_EXT_CDB_CMD_HDR_WRITE_MASK         0x1
+#define ISCSI_EXT_CDB_CMD_HDR_WRITE_SHIFT        5
+#define ISCSI_EXT_CDB_CMD_HDR_READ_MASK          0x1
+#define ISCSI_EXT_CDB_CMD_HDR_READ_SHIFT         6
+#define ISCSI_EXT_CDB_CMD_HDR_FINAL_MASK         0x1
+#define ISCSI_EXT_CDB_CMD_HDR_FINAL_SHIFT        7
+        u8 opcode;
+        __le32 hdr_second_dword;
+#define ISCSI_EXT_CDB_CMD_HDR_DATA_SEG_LEN_MASK  0xFFFFFF
+#define ISCSI_EXT_CDB_CMD_HDR_DATA_SEG_LEN_SHIFT 0
+#define ISCSI_EXT_CDB_CMD_HDR_CDB_SIZE_MASK      0xFF
+#define ISCSI_EXT_CDB_CMD_HDR_CDB_SIZE_SHIFT     24
+        struct regpair lun;
+        __le32 itt;
+        __le32 expected_transfer_length;
+        __le32 cmd_sn;
+        __le32 exp_stat_sn;
+        struct iscsi_sge cdb_sge;
+};
+
+struct iscsi_login_req_hdr {
+        u8 version_min;
+        u8 version_max;
+        u8 flags_attr;
+#define ISCSI_LOGIN_REQ_HDR_NSG_MASK            0x3
+#define ISCSI_LOGIN_REQ_HDR_NSG_SHIFT           0
+#define ISCSI_LOGIN_REQ_HDR_CSG_MASK            0x3
+#define ISCSI_LOGIN_REQ_HDR_CSG_SHIFT           2
+#define ISCSI_LOGIN_REQ_HDR_RSRV_MASK           0x3
+#define ISCSI_LOGIN_REQ_HDR_RSRV_SHIFT          4
+#define ISCSI_LOGIN_REQ_HDR_C_MASK              0x1
+#define ISCSI_LOGIN_REQ_HDR_C_SHIFT             6
+#define ISCSI_LOGIN_REQ_HDR_T_MASK              0x1
+#define ISCSI_LOGIN_REQ_HDR_T_SHIFT             7
+        u8 opcode;
+        __le32 hdr_second_dword;
+#define ISCSI_LOGIN_REQ_HDR_DATA_SEG_LEN_MASK   0xFFFFFF
+#define ISCSI_LOGIN_REQ_HDR_DATA_SEG_LEN_SHIFT  0
+#define ISCSI_LOGIN_REQ_HDR_TOTAL_AHS_LEN_MASK  0xFF
+#define ISCSI_LOGIN_REQ_HDR_TOTAL_AHS_LEN_SHIFT 24
+        __le32 isid_TABC;
+        __le16 tsih;
+        __le16 isid_d;
+        __le32 itt;
+        __le16 reserved1;
+        __le16 cid;
+        __le32 cmd_sn;
+        __le32 exp_stat_sn;
+        __le32 reserved2[4];
+};
+
+struct iscsi_logout_req_hdr {
+        __le16 reserved0;
+        u8 reason_code;
+        u8 opcode;
+        __le32 reserved1;
+        __le32 reserved2[2];
+        __le32 itt;
+        __le16 reserved3;
+        __le16 cid;
+        __le32 cmd_sn;
+        __le32 exp_stat_sn;
+        __le32 reserved4[4];
+};
+
+struct iscsi_data_out_hdr {
+        __le16 reserved1;
+        u8 flags_attr;
+#define ISCSI_DATA_OUT_HDR_RSRV_MASK   0x7F
+#define ISCSI_DATA_OUT_HDR_RSRV_SHIFT  0
+#define ISCSI_DATA_OUT_HDR_FINAL_MASK  0x1
+#define ISCSI_DATA_OUT_HDR_FINAL_SHIFT 7
+        u8 opcode;
+        __le32 reserved2;
+        struct regpair lun;
+        __le32 itt;
+        __le32 ttt;
+        __le32 reserved3;
+        __le32 exp_stat_sn;
+        __le32 reserved4;
+        __le32 data_sn;
+        __le32 buffer_offset;
+        __le32 reserved5;
+};
+
+struct iscsi_data_in_hdr {
+        u8 status_rsvd;
+        u8 reserved1;
+        u8 flags;
+#define ISCSI_DATA_IN_HDR_STATUS_MASK     0x1
+#define ISCSI_DATA_IN_HDR_STATUS_SHIFT    0
+#define ISCSI_DATA_IN_HDR_UNDERFLOW_MASK  0x1
+#define ISCSI_DATA_IN_HDR_UNDERFLOW_SHIFT 1
+#define ISCSI_DATA_IN_HDR_OVERFLOW_MASK   0x1
+#define ISCSI_DATA_IN_HDR_OVERFLOW_SHIFT  2
+#define ISCSI_DATA_IN_HDR_RSRV_MASK       0x7
+#define ISCSI_DATA_IN_HDR_RSRV_SHIFT      3
+#define ISCSI_DATA_IN_HDR_ACK_MASK        0x1
+#define ISCSI_DATA_IN_HDR_ACK_SHIFT       6
+#define ISCSI_DATA_IN_HDR_FINAL_MASK      0x1
+#define ISCSI_DATA_IN_HDR_FINAL_SHIFT     7
+        u8 opcode;
+        __le32 reserved2;
+        struct regpair lun;
+        __le32 itt;
+        __le32 ttt;
+        __le32 stat_sn;
+        __le32 exp_cmd_sn;
+        __le32 max_cmd_sn;
+        __le32 data_sn;
+        __le32 buffer_offset;
+        __le32 residual_count;
+};
+
+struct iscsi_r2t_hdr {
+        u8 reserved0[3];
+        u8 opcode;
+        __le32 reserved2;
+        struct regpair lun;
+        __le32 itt;
+        __le32 ttt;
+        __le32 stat_sn;
+        __le32 exp_cmd_sn;
+        __le32 max_cmd_sn;
+        __le32 r2t_sn;
+        __le32 buffer_offset;
+        __le32 desired_data_trns_len;
+};
+
+struct iscsi_nop_out_hdr {
+        __le16 reserved1;
+        u8 flags_attr;
+#define ISCSI_NOP_OUT_HDR_RSRV_MASK    0x7F
+#define ISCSI_NOP_OUT_HDR_RSRV_SHIFT   0
+#define ISCSI_NOP_OUT_HDR_CONST1_MASK  0x1
+#define ISCSI_NOP_OUT_HDR_CONST1_SHIFT 7
+        u8 opcode;
+        __le32 reserved2;
+        struct regpair lun;
+        __le32 itt;
+        __le32 ttt;
+        __le32 cmd_sn;
+        __le32 exp_stat_sn;
+        __le32 reserved3;
+        __le32 reserved4;
+        __le32 reserved5;
+        __le32 reserved6;
+};
+
+struct iscsi_nop_in_hdr {
+        __le16 reserved0;
+        u8 flags_attr;
+#define ISCSI_NOP_IN_HDR_RSRV_MASK           0x7F
+#define ISCSI_NOP_IN_HDR_RSRV_SHIFT          0
+#define ISCSI_NOP_IN_HDR_CONST1_MASK         0x1
+#define ISCSI_NOP_IN_HDR_CONST1_SHIFT        7
+        u8 opcode;
+        __le32 hdr_second_dword;
+#define ISCSI_NOP_IN_HDR_DATA_SEG_LEN_MASK   0xFFFFFF
+#define ISCSI_NOP_IN_HDR_DATA_SEG_LEN_SHIFT  0
+#define ISCSI_NOP_IN_HDR_TOTAL_AHS_LEN_MASK  0xFF
+#define ISCSI_NOP_IN_HDR_TOTAL_AHS_LEN_SHIFT 24
+        struct regpair lun;
+        __le32 itt;
+        __le32 ttt;
+        __le32 stat_sn;
+        __le32 exp_cmd_sn;
+        __le32 max_cmd_sn;
+        __le32 reserved5;
+        __le32 reserved6;
+        __le32 reserved7;
+};
+
+struct iscsi_login_response_hdr {
+        u8 version_active;
+        u8 version_max;
+        u8 flags_attr;
+#define ISCSI_LOGIN_RESPONSE_HDR_NSG_MASK            0x3
+#define ISCSI_LOGIN_RESPONSE_HDR_NSG_SHIFT           0
+#define ISCSI_LOGIN_RESPONSE_HDR_CSG_MASK            0x3
+#define ISCSI_LOGIN_RESPONSE_HDR_CSG_SHIFT           2
+#define ISCSI_LOGIN_RESPONSE_HDR_RSRV_MASK           0x3
+#define ISCSI_LOGIN_RESPONSE_HDR_RSRV_SHIFT          4
+#define ISCSI_LOGIN_RESPONSE_HDR_C_MASK              0x1
+#define ISCSI_LOGIN_RESPONSE_HDR_C_SHIFT             6
+#define ISCSI_LOGIN_RESPONSE_HDR_T_MASK              0x1
+#define ISCSI_LOGIN_RESPONSE_HDR_T_SHIFT             7
+        u8 opcode;
+        __le32 hdr_second_dword;
+#define ISCSI_LOGIN_RESPONSE_HDR_DATA_SEG_LEN_MASK   0xFFFFFF
+#define ISCSI_LOGIN_RESPONSE_HDR_DATA_SEG_LEN_SHIFT  0
+#define ISCSI_LOGIN_RESPONSE_HDR_TOTAL_AHS_LEN_MASK  0xFF
+#define ISCSI_LOGIN_RESPONSE_HDR_TOTAL_AHS_LEN_SHIFT 24
+        __le32 isid_TABC;
+        __le16 tsih;
+        __le16 isid_d;
+        __le32 itt;
+        __le32 reserved1;
+        __le32 stat_sn;
+        __le32 exp_cmd_sn;
+        __le32 max_cmd_sn;
+        __le16 reserved2;
+        u8 status_detail;
+        u8 status_class;
+        __le32 reserved4[2];
+};
+
+struct iscsi_logout_response_hdr {
+        u8 reserved1;
+        u8 response;
+        u8 flags;
+        u8 opcode;
+        __le32 hdr_second_dword;
+#define ISCSI_LOGOUT_RESPONSE_HDR_DATA_SEG_LEN_MASK   0xFFFFFF
+#define ISCSI_LOGOUT_RESPONSE_HDR_DATA_SEG_LEN_SHIFT  0
+#define ISCSI_LOGOUT_RESPONSE_HDR_TOTAL_AHS_LEN_MASK  0xFF
+#define ISCSI_LOGOUT_RESPONSE_HDR_TOTAL_AHS_LEN_SHIFT 24
+        __le32 reserved2[2];
+        __le32 itt;
+        __le32 reserved3;
+        __le32 stat_sn;
+        __le32 exp_cmd_sn;
+        __le32 max_cmd_sn;
+        __le32 reserved4;
+        __le16 time2retain;
+        __le16 time2wait;
+        __le32 reserved5[1];
+};
+
+struct iscsi_text_request_hdr {
+        __le16 reserved0;
+        u8 flags_attr;
+#define ISCSI_TEXT_REQUEST_HDR_RSRV_MASK           0x3F
+#define ISCSI_TEXT_REQUEST_HDR_RSRV_SHIFT          0
+#define ISCSI_TEXT_REQUEST_HDR_C_MASK              0x1
+#define ISCSI_TEXT_REQUEST_HDR_C_SHIFT             6
+#define ISCSI_TEXT_REQUEST_HDR_F_MASK              0x1
+#define ISCSI_TEXT_REQUEST_HDR_F_SHIFT             7
+        u8 opcode;
+        __le32 hdr_second_dword;
+#define ISCSI_TEXT_REQUEST_HDR_DATA_SEG_LEN_MASK   0xFFFFFF
+#define ISCSI_TEXT_REQUEST_HDR_DATA_SEG_LEN_SHIFT  0
+#define ISCSI_TEXT_REQUEST_HDR_TOTAL_AHS_LEN_MASK  0xFF
+#define ISCSI_TEXT_REQUEST_HDR_TOTAL_AHS_LEN_SHIFT 24
+        struct regpair lun;
+        __le32 itt;
+        __le32 ttt;
+        __le32 cmd_sn;
+        __le32 exp_stat_sn;
+        __le32 reserved4[4];
+};
+
+struct iscsi_text_response_hdr {
+        __le16 reserved1;
+        u8 flags;
+#define ISCSI_TEXT_RESPONSE_HDR_RSRV_MASK           0x3F
+#define ISCSI_TEXT_RESPONSE_HDR_RSRV_SHIFT          0
+#define ISCSI_TEXT_RESPONSE_HDR_C_MASK              0x1
+#define ISCSI_TEXT_RESPONSE_HDR_C_SHIFT             6
+#define ISCSI_TEXT_RESPONSE_HDR_F_MASK              0x1
+#define ISCSI_TEXT_RESPONSE_HDR_F_SHIFT             7
+        u8 opcode;
+        __le32 hdr_second_dword;
+#define ISCSI_TEXT_RESPONSE_HDR_DATA_SEG_LEN_MASK   0xFFFFFF
+#define ISCSI_TEXT_RESPONSE_HDR_DATA_SEG_LEN_SHIFT  0
+#define ISCSI_TEXT_RESPONSE_HDR_TOTAL_AHS_LEN_MASK  0xFF
+#define ISCSI_TEXT_RESPONSE_HDR_TOTAL_AHS_LEN_SHIFT 24
+        struct regpair lun;
+        __le32 itt;
+        __le32 ttt;
+        __le32 stat_sn;
+        __le32 exp_cmd_sn;
+        __le32 max_cmd_sn;
+        __le32 reserved4[3];
+};
+
+struct iscsi_tmf_request_hdr {
+        __le16 reserved0;
+        u8 function;
+        u8 opcode;
+        __le32 hdr_second_dword;
+#define ISCSI_TMF_REQUEST_HDR_DATA_SEG_LEN_MASK   0xFFFFFF
+#define ISCSI_TMF_REQUEST_HDR_DATA_SEG_LEN_SHIFT  0
+#define ISCSI_TMF_REQUEST_HDR_TOTAL_AHS_LEN_MASK  0xFF
+#define ISCSI_TMF_REQUEST_HDR_TOTAL_AHS_LEN_SHIFT 24
+        struct regpair lun;
+        __le32 itt;
+        __le32 rtt;
+        __le32 cmd_sn;
+        __le32 exp_stat_sn;
+        __le32 ref_cmd_sn;
+        __le32 exp_data_sn;
+        __le32 reserved4[2];
+};
+
+struct iscsi_tmf_response_hdr {
+        u8 reserved2;
+        u8 hdr_response;
+        u8 hdr_flags;
+        u8 opcode;
+        __le32 hdr_second_dword;
+#define ISCSI_TMF_RESPONSE_HDR_DATA_SEG_LEN_MASK   0xFFFFFF
+#define ISCSI_TMF_RESPONSE_HDR_DATA_SEG_LEN_SHIFT  0
+#define ISCSI_TMF_RESPONSE_HDR_TOTAL_AHS_LEN_MASK  0xFF
+#define ISCSI_TMF_RESPONSE_HDR_TOTAL_AHS_LEN_SHIFT 24
+        struct regpair reserved0;
+        __le32 itt;
+        __le32 rtt;
+        __le32 stat_sn;
+        __le32 exp_cmd_sn;
+        __le32 max_cmd_sn;
+        __le32 reserved4[3];
+};
+
+struct iscsi_response_hdr {
+        u8 hdr_status;
+        u8 hdr_response;
+        u8 hdr_flags;
+        u8 opcode;
+        __le32 hdr_second_dword;
+#define ISCSI_RESPONSE_HDR_DATA_SEG_LEN_MASK   0xFFFFFF
+#define ISCSI_RESPONSE_HDR_DATA_SEG_LEN_SHIFT  0
+#define ISCSI_RESPONSE_HDR_TOTAL_AHS_LEN_MASK  0xFF
+#define ISCSI_RESPONSE_HDR_TOTAL_AHS_LEN_SHIFT 24
+        struct regpair lun;
+        __le32 itt;
+        __le32 snack_tag;
+        __le32 stat_sn;
+        __le32 exp_cmd_sn;
+        __le32 max_cmd_sn;
+        __le32 exp_data_sn;
+        __le32 bi_residual_count;
+        __le32 residual_count;
+};
+
+struct iscsi_reject_hdr {
+        u8 reserved4;
+        u8 hdr_reason;
+        u8 hdr_flags;
+        u8 opcode;
+        __le32 hdr_second_dword;
+#define ISCSI_REJECT_HDR_DATA_SEG_LEN_MASK   0xFFFFFF
+#define ISCSI_REJECT_HDR_DATA_SEG_LEN_SHIFT  0
+#define ISCSI_REJECT_HDR_TOTAL_AHS_LEN_MASK  0xFF
+#define ISCSI_REJECT_HDR_TOTAL_AHS_LEN_SHIFT 24
+        struct regpair reserved0;
+        __le32 reserved1;
+        __le32 reserved2;
+        __le32 stat_sn;
+        __le32 exp_cmd_sn;
+        __le32 max_cmd_sn;
+        __le32 data_sn;
+        __le32 reserved3[2];
+};
+
+union iscsi_task_hdr {
+        struct iscsi_common_hdr common;
+        struct data_hdr data;
+        struct iscsi_cmd_hdr cmd;
+        struct iscsi_ext_cdb_cmd_hdr ext_cdb_cmd;
+        struct iscsi_login_req_hdr login_req;
+        struct iscsi_logout_req_hdr logout_req;
+        struct iscsi_data_out_hdr data_out;
+        struct iscsi_data_in_hdr data_in;
+        struct iscsi_r2t_hdr r2t;
+        struct iscsi_nop_out_hdr nop_out;
+        struct iscsi_nop_in_hdr nop_in;
+        struct iscsi_login_response_hdr login_response;
+        struct iscsi_logout_response_hdr logout_response;
+        struct iscsi_text_request_hdr text_request;
+        struct iscsi_text_response_hdr text_response;
+        struct iscsi_tmf_request_hdr tmf_request;
+        struct iscsi_tmf_response_hdr tmf_response;
+        struct iscsi_response_hdr response;
+        struct iscsi_reject_hdr reject;
+        struct iscsi_async_msg_hdr async_msg;
+};
+
+struct iscsi_cqe_common {
+        __le16 conn_id;
+        u8 cqe_type;
+        union cqe_error_status error_bitmap;
+        __le32 reserved[3];
+        union iscsi_task_hdr iscsi_hdr;
+};
+
+struct iscsi_cqe_solicited {
+        __le16 conn_id;
+        u8 cqe_type;
+        union cqe_error_status error_bitmap;
+        __le16 itid;
+        u8 task_type;
+        u8 fw_dbg_field;
+        __le32 reserved1[2];
+        union iscsi_task_hdr iscsi_hdr;
+};
+
+struct iscsi_cqe_unsolicited {
+        __le16 conn_id;
+        u8 cqe_type;
+        union cqe_error_status error_bitmap;
+        __le16 reserved0;
+        u8 reserved1;
+        u8 unsol_cqe_type;
+        struct regpair rqe_opaque;
+        union iscsi_task_hdr iscsi_hdr;
+};
+
+union iscsi_cqe {
+        struct iscsi_cqe_common cqe_common;
+        struct iscsi_cqe_solicited cqe_solicited;
+        struct iscsi_cqe_unsolicited cqe_unsolicited;
+};
+
+enum iscsi_cqes_type {
+        ISCSI_CQE_TYPE_SOLICITED = 1,
+        ISCSI_CQE_TYPE_UNSOLICITED,
+        ISCSI_CQE_TYPE_SOLICITED_WITH_SENSE
+           ,
+        ISCSI_CQE_TYPE_TASK_CLEANUP,
+        ISCSI_CQE_TYPE_DUMMY,
+        MAX_ISCSI_CQES_TYPE
+};
+
+enum iscsi_cqe_unsolicited_type {
+        ISCSI_CQE_UNSOLICITED_NONE,
+        ISCSI_CQE_UNSOLICITED_SINGLE,
+        ISCSI_CQE_UNSOLICITED_FIRST,
+        ISCSI_CQE_UNSOLICITED_MIDDLE,
+        ISCSI_CQE_UNSOLICITED_LAST,
+        MAX_ISCSI_CQE_UNSOLICITED_TYPE
+};
+
+struct iscsi_virt_sgl_ctx {
+        struct regpair sgl_base;
+        struct regpair dsgl_base;
+        __le32 sgl_initial_offset;
+        __le32 dsgl_initial_offset;
+        __le32 dsgl_curr_offset[2];
+};
+
+struct iscsi_sgl_var_params {
+        u8 sgl_ptr;
+        u8 dsgl_ptr;
+        __le16 sge_offset;
+        __le16 dsge_offset;
+};
+
+struct iscsi_phys_sgl_ctx {
+        struct regpair sgl_base;
+        struct regpair dsgl_base;
+        u8 sgl_size;
+        u8 dsgl_size;
+        __le16 reserved;
+        struct iscsi_sgl_var_params var_params[2];
+};
+
+union iscsi_data_desc_ctx {
+        struct iscsi_virt_sgl_ctx virt_sgl;
+        struct iscsi_phys_sgl_ctx phys_sgl;
+        struct iscsi_cached_sge_ctx cached_sge;
+};
+
+struct iscsi_debug_modes {
+        u8 flags;
+#define ISCSI_DEBUG_MODES_ASSERT_IF_RX_CONN_ERROR_MASK         0x1
+#define ISCSI_DEBUG_MODES_ASSERT_IF_RX_CONN_ERROR_SHIFT        0
+#define ISCSI_DEBUG_MODES_ASSERT_IF_RECV_RESET_MASK            0x1
+#define ISCSI_DEBUG_MODES_ASSERT_IF_RECV_RESET_SHIFT           1
+#define ISCSI_DEBUG_MODES_ASSERT_IF_RECV_FIN_MASK              0x1
+#define ISCSI_DEBUG_MODES_ASSERT_IF_RECV_FIN_SHIFT             2
+#define ISCSI_DEBUG_MODES_ASSERT_IF_RECV_CLEANUP_MASK          0x1
+#define ISCSI_DEBUG_MODES_ASSERT_IF_RECV_CLEANUP_SHIFT         3
+#define ISCSI_DEBUG_MODES_ASSERT_IF_RECV_REJECT_OR_ASYNC_MASK  0x1
+#define ISCSI_DEBUG_MODES_ASSERT_IF_RECV_REJECT_OR_ASYNC_SHIFT 4
+#define ISCSI_DEBUG_MODES_ASSERT_IF_RECV_NOP_MASK              0x1
+#define ISCSI_DEBUG_MODES_ASSERT_IF_RECV_NOP_SHIFT             5
+#define ISCSI_DEBUG_MODES_RESERVED0_MASK                       0x3
+#define ISCSI_DEBUG_MODES_RESERVED0_SHIFT                      6
+};
+
+struct iscsi_dif_flags {
+        u8 flags;
+#define ISCSI_DIF_FLAGS_PROT_INTERVAL_SIZE_LOG_MASK  0xF
+#define ISCSI_DIF_FLAGS_PROT_INTERVAL_SIZE_LOG_SHIFT 0
+#define ISCSI_DIF_FLAGS_DIF_TO_PEER_MASK             0x1
+#define ISCSI_DIF_FLAGS_DIF_TO_PEER_SHIFT            4
+#define ISCSI_DIF_FLAGS_HOST_INTERFACE_MASK          0x7
+#define ISCSI_DIF_FLAGS_HOST_INTERFACE_SHIFT         5
+};
+
+enum iscsi_eqe_opcode {
+        ISCSI_EVENT_TYPE_INIT_FUNC = 0,
+        ISCSI_EVENT_TYPE_DESTROY_FUNC,
+        ISCSI_EVENT_TYPE_OFFLOAD_CONN,
+        ISCSI_EVENT_TYPE_UPDATE_CONN,
+        ISCSI_EVENT_TYPE_CLEAR_SQ,
+        ISCSI_EVENT_TYPE_TERMINATE_CONN,
+        ISCSI_EVENT_TYPE_ASYN_CONNECT_COMPLETE,
+        ISCSI_EVENT_TYPE_ASYN_TERMINATE_DONE,
+        RESERVED8,
+        RESERVED9,
+        ISCSI_EVENT_TYPE_START_OF_ERROR_TYPES = 10,
+        ISCSI_EVENT_TYPE_ASYN_ABORT_RCVD,
+        ISCSI_EVENT_TYPE_ASYN_CLOSE_RCVD,
+        ISCSI_EVENT_TYPE_ASYN_SYN_RCVD,
+        ISCSI_EVENT_TYPE_ASYN_MAX_RT_TIME,
+        ISCSI_EVENT_TYPE_ASYN_MAX_RT_CNT,
+        ISCSI_EVENT_TYPE_ASYN_MAX_KA_PROBES_CNT,
+        ISCSI_EVENT_TYPE_ASYN_FIN_WAIT2,
+        ISCSI_EVENT_TYPE_ISCSI_CONN_ERROR,
+        ISCSI_EVENT_TYPE_TCP_CONN_ERROR,
+        ISCSI_EVENT_TYPE_ASYN_DELETE_OOO_ISLES,
+        MAX_ISCSI_EQE_OPCODE
+};
+
+enum iscsi_error_types {
+        ISCSI_STATUS_NONE = 0,
+        ISCSI_CQE_ERROR_UNSOLICITED_RCV_ON_INVALID_CONN = 1,
+        ISCSI_CONN_ERROR_TASK_CID_MISMATCH,
+        ISCSI_CONN_ERROR_TASK_NOT_VALID,
+        ISCSI_CONN_ERROR_RQ_RING_IS_FULL,
+        ISCSI_CONN_ERROR_CMDQ_RING_IS_FULL,
+        ISCSI_CONN_ERROR_HQE_CACHING_FAILED,
+        ISCSI_CONN_ERROR_HEADER_DIGEST_ERROR,
+        ISCSI_CONN_ERROR_LOCAL_COMPLETION_ERROR,
+        ISCSI_CONN_ERROR_DATA_OVERRUN,
+        ISCSI_CONN_ERROR_OUT_OF_SGES_ERROR,
+        ISCSI_CONN_ERROR_TCP_SEG_PROC_URG_ERROR,
+        ISCSI_CONN_ERROR_TCP_SEG_PROC_IP_OPTIONS_ERROR,
+        ISCSI_CONN_ERROR_TCP_SEG_PROC_CONNECT_INVALID_WS_OPTION,
+        ISCSI_CONN_ERROR_TCP_IP_FRAGMENT_ERROR,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_AHS_LEN,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_AHS_TYPE,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_ITT_OUT_OF_RANGE,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_TTT_OUT_OF_RANGE,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_DATA_SEG_LEN_EXCEEDS_PDU_SIZE,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_INVALID_OPCODE,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_INVALID_OPCODE_BEFORE_UPDATE,
+        ISCSI_CONN_ERROR_UNVALID_NOPIN_DSL,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_R2T_CARRIES_NO_DATA,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_DATA_SN,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_DATA_IN_TTT,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_DATA_OUT_ITT,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_R2T_TTT,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_R2T_BUFFER_OFFSET,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_BUFFER_OFFSET_OOO,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_R2T_SN,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_DESIRED_DATA_TRNS_LEN_0,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_DESIRED_DATA_TRNS_LEN_1,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_DESIRED_DATA_TRNS_LEN_2,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_LUN,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_F_BIT_ZERO,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_F_BIT_ZERO_S_BIT_ONE,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_EXP_STAT_SN,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_DSL_NOT_ZERO,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_INVALID_DSL,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_DATA_SEG_LEN_TOO_BIG,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_OUTSTANDING_R2T_COUNT,
+        ISCSI_CONN_ERROR_PROTOCOL_ERR_DIF_TX,
+        ISCSI_CONN_ERROR_SENSE_DATA_LENGTH,
+        ISCSI_CONN_ERROR_DATA_PLACEMENT_ERROR,
+        ISCSI_ERROR_UNKNOWN,
+        MAX_ISCSI_ERROR_TYPES
+};
+
+struct iscsi_mflags {
+        u8 mflags;
+#define ISCSI_MFLAGS_SLOW_IO_MASK     0x1
+#define ISCSI_MFLAGS_SLOW_IO_SHIFT    0
+#define ISCSI_MFLAGS_SINGLE_SGE_MASK  0x1
+#define ISCSI_MFLAGS_SINGLE_SGE_SHIFT 1
+#define ISCSI_MFLAGS_RESERVED_MASK    0x3F
+#define ISCSI_MFLAGS_RESERVED_SHIFT   2
+};
+
+struct iscsi_sgl {
+        struct regpair sgl_addr;
+        __le16 updated_sge_size;
+        __le16 updated_sge_offset;
+        __le32 byte_offset;
+};
+
+union iscsi_mstorm_sgl {
+        struct iscsi_sgl sgl_struct;
+        struct iscsi_sge single_sge;
+};
+
+enum iscsi_ramrod_cmd_id {
+        ISCSI_RAMROD_CMD_ID_UNUSED = 0,
+        ISCSI_RAMROD_CMD_ID_INIT_FUNC = 1,
+        ISCSI_RAMROD_CMD_ID_DESTROY_FUNC = 2,
+        ISCSI_RAMROD_CMD_ID_OFFLOAD_CONN = 3,
+        ISCSI_RAMROD_CMD_ID_UPDATE_CONN = 4,
+        ISCSI_RAMROD_CMD_ID_TERMINATION_CONN = 5,
+        ISCSI_RAMROD_CMD_ID_CLEAR_SQ = 6,
+        MAX_ISCSI_RAMROD_CMD_ID
+};
+
+struct iscsi_reg1 {
+        __le32 reg1_map;
+#define ISCSI_REG1_NUM_FAST_SGES_MASK  0x7
+#define ISCSI_REG1_NUM_FAST_SGES_SHIFT 0
+#define ISCSI_REG1_RESERVED1_MASK      0x1FFFFFFF
+#define ISCSI_REG1_RESERVED1_SHIFT     3
+};
+
+union iscsi_seq_num {
+        __le16 data_sn;
+        __le16 r2t_sn;
+};
+
+struct iscsi_spe_conn_offload {
+        struct iscsi_slow_path_hdr hdr;
+        __le16 conn_id;
+        __le32 fw_cid;
+        struct iscsi_conn_offload_params iscsi;
+        struct tcp_offload_params tcp;
+};
+
+struct iscsi_spe_conn_offload_option2 {
+        struct iscsi_slow_path_hdr hdr;
+        __le16 conn_id;
+        __le32 fw_cid;
+        struct iscsi_conn_offload_params iscsi;
+        struct tcp_offload_params_opt2 tcp;
+};
+
+struct iscsi_spe_conn_termination {
+        struct iscsi_slow_path_hdr hdr;
+        __le16 conn_id;
+        __le32 fw_cid;
+        u8 abortive;
+        u8 reserved0[7];
+        struct regpair queue_cnts_addr;
+        struct regpair query_params_addr;
+};
+
+struct iscsi_spe_func_dstry {
+        struct iscsi_slow_path_hdr hdr;
+        __le16 reserved0;
+        __le32 reserved1;
+};
+
+struct iscsi_spe_func_init {
+        struct iscsi_slow_path_hdr hdr;
+        __le16 half_way_close_timeout;
+        u8 num_sq_pages_in_ring;
+        u8 num_r2tq_pages_in_ring;
+        u8 num_uhq_pages_in_ring;
+        u8 ll2_rx_queue_id;
+        u8 ooo_enable;
+        struct iscsi_debug_modes debug_mode;
+        __le16 reserved1;
+        __le32 reserved2;
+        __le32 reserved3;
+        __le32 reserved4;
+        struct scsi_init_func_params func_params;
+        struct scsi_init_func_queues q_params;
+};
+
+struct ystorm_iscsi_task_state {
+        union iscsi_data_desc_ctx sgl_ctx_union;
+        __le32 buffer_offset[2];
+        __le16 bytes_nxt_dif;
+        __le16 rxmit_bytes_nxt_dif;
+        union iscsi_seq_num seq_num_union;
+        u8 dif_bytes_leftover;
+        u8 rxmit_dif_bytes_leftover;
+        __le16 reuse_count;
+        struct iscsi_dif_flags dif_flags;
+        u8 local_comp;
+        __le32 exp_r2t_sn;
+        __le32 sgl_offset[2];
+};
+
+struct ystorm_iscsi_task_st_ctx {
+        struct ystorm_iscsi_task_state state;
+        union iscsi_task_hdr pdu_hdr;
+};
+
+struct ystorm_iscsi_task_ag_ctx {
+        u8 reserved;
+        u8 byte1;
+        __le16 word0;
+        u8 flags0;
+#define YSTORM_ISCSI_TASK_AG_CTX_NIBBLE0_MASK     0xF
+#define YSTORM_ISCSI_TASK_AG_CTX_NIBBLE0_SHIFT    0
+#define YSTORM_ISCSI_TASK_AG_CTX_BIT0_MASK        0x1
+#define YSTORM_ISCSI_TASK_AG_CTX_BIT0_SHIFT       4
+#define YSTORM_ISCSI_TASK_AG_CTX_BIT1_MASK        0x1
+#define YSTORM_ISCSI_TASK_AG_CTX_BIT1_SHIFT       5
+#define YSTORM_ISCSI_TASK_AG_CTX_VALID_MASK       0x1
+#define YSTORM_ISCSI_TASK_AG_CTX_VALID_SHIFT      6
+#define YSTORM_ISCSI_TASK_AG_CTX_BIT3_MASK        0x1
+#define YSTORM_ISCSI_TASK_AG_CTX_BIT3_SHIFT       7
+        u8 flags1;
+#define YSTORM_ISCSI_TASK_AG_CTX_CF0_MASK         0x3
+#define YSTORM_ISCSI_TASK_AG_CTX_CF0_SHIFT        0
+#define YSTORM_ISCSI_TASK_AG_CTX_CF1_MASK         0x3
+#define YSTORM_ISCSI_TASK_AG_CTX_CF1_SHIFT        2
+#define YSTORM_ISCSI_TASK_AG_CTX_CF2SPECIAL_MASK  0x3
+#define YSTORM_ISCSI_TASK_AG_CTX_CF2SPECIAL_SHIFT 4
+#define YSTORM_ISCSI_TASK_AG_CTX_CF0EN_MASK       0x1
+#define YSTORM_ISCSI_TASK_AG_CTX_CF0EN_SHIFT      6
+#define YSTORM_ISCSI_TASK_AG_CTX_CF1EN_MASK       0x1
+#define YSTORM_ISCSI_TASK_AG_CTX_CF1EN_SHIFT      7
+        u8 flags2;
+#define YSTORM_ISCSI_TASK_AG_CTX_BIT4_MASK        0x1
+#define YSTORM_ISCSI_TASK_AG_CTX_BIT4_SHIFT       0
+#define YSTORM_ISCSI_TASK_AG_CTX_RULE0EN_MASK     0x1
+#define YSTORM_ISCSI_TASK_AG_CTX_RULE0EN_SHIFT    1
+#define YSTORM_ISCSI_TASK_AG_CTX_RULE1EN_MASK     0x1
+#define YSTORM_ISCSI_TASK_AG_CTX_RULE1EN_SHIFT    2
+#define YSTORM_ISCSI_TASK_AG_CTX_RULE2EN_MASK     0x1
+#define YSTORM_ISCSI_TASK_AG_CTX_RULE2EN_SHIFT    3
+#define YSTORM_ISCSI_TASK_AG_CTX_RULE3EN_MASK     0x1
+#define YSTORM_ISCSI_TASK_AG_CTX_RULE3EN_SHIFT    4
+#define YSTORM_ISCSI_TASK_AG_CTX_RULE4EN_MASK     0x1
+#define YSTORM_ISCSI_TASK_AG_CTX_RULE4EN_SHIFT    5
+#define YSTORM_ISCSI_TASK_AG_CTX_RULE5EN_MASK     0x1
+#define YSTORM_ISCSI_TASK_AG_CTX_RULE5EN_SHIFT    6
+#define YSTORM_ISCSI_TASK_AG_CTX_RULE6EN_MASK     0x1
+#define YSTORM_ISCSI_TASK_AG_CTX_RULE6EN_SHIFT    7
+        u8 byte2;
+        __le32 TTT;
+        u8 byte3;
+        u8 byte4;
+        __le16 word1;
+};
+
+struct mstorm_iscsi_task_ag_ctx {
+        u8 cdu_validation;
+        u8 byte1;
+        __le16 task_cid;
+        u8 flags0;
+#define MSTORM_ISCSI_TASK_AG_CTX_CONNECTION_TYPE_MASK     0xF
+#define MSTORM_ISCSI_TASK_AG_CTX_CONNECTION_TYPE_SHIFT    0
+#define MSTORM_ISCSI_TASK_AG_CTX_EXIST_IN_QM0_MASK        0x1
+#define MSTORM_ISCSI_TASK_AG_CTX_EXIST_IN_QM0_SHIFT       4
+#define MSTORM_ISCSI_TASK_AG_CTX_BIT1_MASK                0x1
+#define MSTORM_ISCSI_TASK_AG_CTX_BIT1_SHIFT               5
+#define MSTORM_ISCSI_TASK_AG_CTX_VALID_MASK               0x1
+#define MSTORM_ISCSI_TASK_AG_CTX_VALID_SHIFT              6
+#define MSTORM_ISCSI_TASK_AG_CTX_TASK_CLEANUP_FLAG_MASK   0x1
+#define MSTORM_ISCSI_TASK_AG_CTX_TASK_CLEANUP_FLAG_SHIFT  7
+        u8 flags1;
+#define MSTORM_ISCSI_TASK_AG_CTX_TASK_CLEANUP_CF_MASK     0x3
+#define MSTORM_ISCSI_TASK_AG_CTX_TASK_CLEANUP_CF_SHIFT    0
+#define MSTORM_ISCSI_TASK_AG_CTX_CF1_MASK                 0x3
+#define MSTORM_ISCSI_TASK_AG_CTX_CF1_SHIFT                2
+#define MSTORM_ISCSI_TASK_AG_CTX_CF2_MASK                 0x3
+#define MSTORM_ISCSI_TASK_AG_CTX_CF2_SHIFT                4
+#define MSTORM_ISCSI_TASK_AG_CTX_TASK_CLEANUP_CF_EN_MASK  0x1
+#define MSTORM_ISCSI_TASK_AG_CTX_TASK_CLEANUP_CF_EN_SHIFT 6
+#define MSTORM_ISCSI_TASK_AG_CTX_CF1EN_MASK               0x1
+#define MSTORM_ISCSI_TASK_AG_CTX_CF1EN_SHIFT              7
+        u8 flags2;
+#define MSTORM_ISCSI_TASK_AG_CTX_CF2EN_MASK               0x1
+#define MSTORM_ISCSI_TASK_AG_CTX_CF2EN_SHIFT              0
+#define MSTORM_ISCSI_TASK_AG_CTX_RULE0EN_MASK             0x1
+#define MSTORM_ISCSI_TASK_AG_CTX_RULE0EN_SHIFT            1
+#define MSTORM_ISCSI_TASK_AG_CTX_RULE1EN_MASK             0x1
+#define MSTORM_ISCSI_TASK_AG_CTX_RULE1EN_SHIFT            2
+#define MSTORM_ISCSI_TASK_AG_CTX_RULE2EN_MASK             0x1
+#define MSTORM_ISCSI_TASK_AG_CTX_RULE2EN_SHIFT            3
+#define MSTORM_ISCSI_TASK_AG_CTX_RULE3EN_MASK             0x1
+#define MSTORM_ISCSI_TASK_AG_CTX_RULE3EN_SHIFT            4
+#define MSTORM_ISCSI_TASK_AG_CTX_RULE4EN_MASK             0x1
+#define MSTORM_ISCSI_TASK_AG_CTX_RULE4EN_SHIFT            5
+#define MSTORM_ISCSI_TASK_AG_CTX_RULE5EN_MASK             0x1
+#define MSTORM_ISCSI_TASK_AG_CTX_RULE5EN_SHIFT            6
+#define MSTORM_ISCSI_TASK_AG_CTX_RULE6EN_MASK             0x1
+#define MSTORM_ISCSI_TASK_AG_CTX_RULE6EN_SHIFT            7
+        u8 byte2;
+        __le32 reg0;
+        u8 byte3;
+        u8 byte4;
+        __le16 word1;
+};
+
+struct ustorm_iscsi_task_ag_ctx {
+        u8 reserved;
+        u8 state;
+        __le16 icid;
+        u8 flags0;
+#define USTORM_ISCSI_TASK_AG_CTX_CONNECTION_TYPE_MASK        0xF
+#define USTORM_ISCSI_TASK_AG_CTX_CONNECTION_TYPE_SHIFT       0
+#define USTORM_ISCSI_TASK_AG_CTX_EXIST_IN_QM0_MASK           0x1
+#define USTORM_ISCSI_TASK_AG_CTX_EXIST_IN_QM0_SHIFT          4
+#define USTORM_ISCSI_TASK_AG_CTX_BIT1_MASK                   0x1
+#define USTORM_ISCSI_TASK_AG_CTX_BIT1_SHIFT                  5
+#define USTORM_ISCSI_TASK_AG_CTX_HQ_SCANNED_CF_MASK          0x3
+#define USTORM_ISCSI_TASK_AG_CTX_HQ_SCANNED_CF_SHIFT         6
+        u8 flags1;
+#define USTORM_ISCSI_TASK_AG_CTX_RESERVED1_MASK              0x3
+#define USTORM_ISCSI_TASK_AG_CTX_RESERVED1_SHIFT             0
+#define USTORM_ISCSI_TASK_AG_CTX_R2T2RECV_MASK               0x3
+#define USTORM_ISCSI_TASK_AG_CTX_R2T2RECV_SHIFT              2
+#define USTORM_ISCSI_TASK_AG_CTX_CF3_MASK                    0x3
+#define USTORM_ISCSI_TASK_AG_CTX_CF3_SHIFT                   4
+#define USTORM_ISCSI_TASK_AG_CTX_DIF_ERROR_CF_MASK           0x3
+#define USTORM_ISCSI_TASK_AG_CTX_DIF_ERROR_CF_SHIFT          6
+        u8 flags2;
+#define USTORM_ISCSI_TASK_AG_CTX_HQ_SCANNED_CF_EN_MASK       0x1
+#define USTORM_ISCSI_TASK_AG_CTX_HQ_SCANNED_CF_EN_SHIFT      0
+#define USTORM_ISCSI_TASK_AG_CTX_DISABLE_DATA_ACKED_MASK     0x1
+#define USTORM_ISCSI_TASK_AG_CTX_DISABLE_DATA_ACKED_SHIFT    1
+#define USTORM_ISCSI_TASK_AG_CTX_R2T2RECV_EN_MASK            0x1
+#define USTORM_ISCSI_TASK_AG_CTX_R2T2RECV_EN_SHIFT           2
+#define USTORM_ISCSI_TASK_AG_CTX_CF3EN_MASK                  0x1
+#define USTORM_ISCSI_TASK_AG_CTX_CF3EN_SHIFT                 3
+#define USTORM_ISCSI_TASK_AG_CTX_DIF_ERROR_CF_EN_MASK        0x1
+#define USTORM_ISCSI_TASK_AG_CTX_DIF_ERROR_CF_EN_SHIFT       4
+#define USTORM_ISCSI_TASK_AG_CTX_CMP_DATA_TOTAL_EXP_EN_MASK  0x1
+#define USTORM_ISCSI_TASK_AG_CTX_CMP_DATA_TOTAL_EXP_EN_SHIFT 5
+#define USTORM_ISCSI_TASK_AG_CTX_RULE1EN_MASK                0x1
+#define USTORM_ISCSI_TASK_AG_CTX_RULE1EN_SHIFT               6
+#define USTORM_ISCSI_TASK_AG_CTX_CMP_CONT_RCV_EXP_EN_MASK    0x1
+#define USTORM_ISCSI_TASK_AG_CTX_CMP_CONT_RCV_EXP_EN_SHIFT   7
+        u8 flags3;
+#define USTORM_ISCSI_TASK_AG_CTX_RULE3EN_MASK                0x1
+#define USTORM_ISCSI_TASK_AG_CTX_RULE3EN_SHIFT               0
+#define USTORM_ISCSI_TASK_AG_CTX_RULE4EN_MASK                0x1
+#define USTORM_ISCSI_TASK_AG_CTX_RULE4EN_SHIFT               1
+#define USTORM_ISCSI_TASK_AG_CTX_RULE5EN_MASK                0x1
+#define USTORM_ISCSI_TASK_AG_CTX_RULE5EN_SHIFT               2
+#define USTORM_ISCSI_TASK_AG_CTX_RULE6EN_MASK                0x1
+#define USTORM_ISCSI_TASK_AG_CTX_RULE6EN_SHIFT               3
+#define USTORM_ISCSI_TASK_AG_CTX_DIF_ERROR_TYPE_MASK         0xF
+#define USTORM_ISCSI_TASK_AG_CTX_DIF_ERROR_TYPE_SHIFT        4
+        __le32 dif_err_intervals;
+        __le32 dif_error_1st_interval;
+        __le32 rcv_cont_len;
+        __le32 exp_cont_len;
+        __le32 total_data_acked;
+        __le32 exp_data_acked;
+        u8 next_tid_valid;
+        u8 byte3;
+        __le16 word1;
+        __le16 next_tid;
+        __le16 word3;
+        __le32 hdr_residual_count;
+        __le32 exp_r2t_sn;
+};
+
+struct mstorm_iscsi_task_st_ctx {
+        union iscsi_mstorm_sgl sgl_union;
+        struct iscsi_dif_flags dif_flags;
+        struct iscsi_mflags flags;
+        u8 sgl_size;
+        u8 host_sge_index;
+        __le16 dix_cur_sge_offset;
+        __le16 dix_cur_sge_size;
+        __le32 data_offset_rtid;
+        u8 dif_offset;
+        u8 dix_sgl_size;
+        u8 dix_sge_index;
+        u8 task_type;
+        struct regpair sense_db;
+        struct regpair dix_sgl_cur_sge;
+        __le32 rem_task_size;
+        __le16 reuse_count;
+        __le16 dif_data_residue;
+        u8 reserved0[4];
+        __le32 reserved1[1];
+};
+
+struct ustorm_iscsi_task_st_ctx {
+        __le32 rem_rcv_len;
+        __le32 exp_data_transfer_len;
+        __le32 exp_data_sn;
+        struct regpair lun;
+        struct iscsi_reg1 reg1;
+        u8 flags2;
+#define USTORM_ISCSI_TASK_ST_CTX_AHS_EXIST_MASK             0x1
+#define USTORM_ISCSI_TASK_ST_CTX_AHS_EXIST_SHIFT            0
+#define USTORM_ISCSI_TASK_ST_CTX_RESERVED1_MASK             0x7F
+#define USTORM_ISCSI_TASK_ST_CTX_RESERVED1_SHIFT            1
+        u8 reserved2;
+        __le16 reserved3;
+        __le32 reserved4;
+        __le32 reserved5;
+        __le32 reserved6;
+        __le32 reserved7;
+        u8 task_type;
+        u8 error_flags;
+#define USTORM_ISCSI_TASK_ST_CTX_DATA_DIGEST_ERROR_MASK     0x1
+#define USTORM_ISCSI_TASK_ST_CTX_DATA_DIGEST_ERROR_SHIFT    0
+#define USTORM_ISCSI_TASK_ST_CTX_DATA_TRUNCATED_ERROR_MASK  0x1
+#define USTORM_ISCSI_TASK_ST_CTX_DATA_TRUNCATED_ERROR_SHIFT 1
+#define USTORM_ISCSI_TASK_ST_CTX_UNDER_RUN_ERROR_MASK       0x1
+#define USTORM_ISCSI_TASK_ST_CTX_UNDER_RUN_ERROR_SHIFT      2
+#define USTORM_ISCSI_TASK_ST_CTX_RESERVED8_MASK             0x1F
+#define USTORM_ISCSI_TASK_ST_CTX_RESERVED8_SHIFT            3
+        u8 flags;
+#define USTORM_ISCSI_TASK_ST_CTX_CQE_WRITE_MASK             0x3
+#define USTORM_ISCSI_TASK_ST_CTX_CQE_WRITE_SHIFT            0
+#define USTORM_ISCSI_TASK_ST_CTX_LOCAL_COMP_MASK            0x1
+#define USTORM_ISCSI_TASK_ST_CTX_LOCAL_COMP_SHIFT           2
+#define USTORM_ISCSI_TASK_ST_CTX_Q0_R2TQE_WRITE_MASK        0x1
+#define USTORM_ISCSI_TASK_ST_CTX_Q0_R2TQE_WRITE_SHIFT       3
+#define USTORM_ISCSI_TASK_ST_CTX_TOTALDATAACKED_DONE_MASK   0x1
+#define USTORM_ISCSI_TASK_ST_CTX_TOTALDATAACKED_DONE_SHIFT  4
+#define USTORM_ISCSI_TASK_ST_CTX_HQSCANNED_DONE_MASK        0x1
+#define USTORM_ISCSI_TASK_ST_CTX_HQSCANNED_DONE_SHIFT       5
+#define USTORM_ISCSI_TASK_ST_CTX_R2T2RECV_DONE_MASK         0x1
+#define USTORM_ISCSI_TASK_ST_CTX_R2T2RECV_DONE_SHIFT        6
+#define USTORM_ISCSI_TASK_ST_CTX_RESERVED0_MASK             0x1
+#define USTORM_ISCSI_TASK_ST_CTX_RESERVED0_SHIFT            7
+        u8 cq_rss_number;
+};
+
+struct iscsi_task_context {
+        struct ystorm_iscsi_task_st_ctx ystorm_st_context;
+        struct regpair ystorm_st_padding[2];
+        struct ystorm_iscsi_task_ag_ctx ystorm_ag_context;
+        struct regpair ystorm_ag_padding[2];
+        struct tdif_task_context tdif_context;
+        struct mstorm_iscsi_task_ag_ctx mstorm_ag_context;
+        struct regpair mstorm_ag_padding[2];
+        struct ustorm_iscsi_task_ag_ctx ustorm_ag_context;
+        struct mstorm_iscsi_task_st_ctx mstorm_st_context;
+        struct ustorm_iscsi_task_st_ctx ustorm_st_context;
+        struct rdif_task_context rdif_context;
+};
+
+enum iscsi_task_type {
+        ISCSI_TASK_TYPE_INITIATOR_WRITE,
+        ISCSI_TASK_TYPE_INITIATOR_READ,
+        ISCSI_TASK_TYPE_MIDPATH,
+        ISCSI_TASK_TYPE_UNSOLIC,
+        ISCSI_TASK_TYPE_EXCHCLEANUP,
+        ISCSI_TASK_TYPE_IRRELEVANT,
+        ISCSI_TASK_TYPE_TARGET_WRITE,
+        ISCSI_TASK_TYPE_TARGET_READ,
+        ISCSI_TASK_TYPE_TARGET_RESPONSE,
+        ISCSI_TASK_TYPE_LOGIN_RESPONSE,
+        MAX_ISCSI_TASK_TYPE
+};
+
+union iscsi_ttt_txlen_union {
+        __le32 desired_tx_len;
+        __le32 ttt;
+};
+
+struct iscsi_uhqe {
+        __le32 reg1;
+#define ISCSI_UHQE_PDU_PAYLOAD_LEN_MASK     0xFFFFF
+#define ISCSI_UHQE_PDU_PAYLOAD_LEN_SHIFT    0
+#define ISCSI_UHQE_LOCAL_COMP_MASK          0x1
+#define ISCSI_UHQE_LOCAL_COMP_SHIFT         20
+#define ISCSI_UHQE_TOGGLE_BIT_MASK          0x1
+#define ISCSI_UHQE_TOGGLE_BIT_SHIFT         21
+#define ISCSI_UHQE_PURE_PAYLOAD_MASK        0x1
+#define ISCSI_UHQE_PURE_PAYLOAD_SHIFT       22
+#define ISCSI_UHQE_LOGIN_RESPONSE_PDU_MASK  0x1
+#define ISCSI_UHQE_LOGIN_RESPONSE_PDU_SHIFT 23
+#define ISCSI_UHQE_TASK_ID_HI_MASK          0xFF
+#define ISCSI_UHQE_TASK_ID_HI_SHIFT         24
+        __le32 reg2;
+#define ISCSI_UHQE_BUFFER_OFFSET_MASK       0xFFFFFF
+#define ISCSI_UHQE_BUFFER_OFFSET_SHIFT      0
+#define ISCSI_UHQE_TASK_ID_LO_MASK          0xFF
+#define ISCSI_UHQE_TASK_ID_LO_SHIFT         24
+};
+
+struct iscsi_wqe_field {
+        __le32 contlen_cdbsize_field;
+#define ISCSI_WQE_FIELD_CONT_LEN_MASK  0xFFFFFF
+#define ISCSI_WQE_FIELD_CONT_LEN_SHIFT 0
+#define ISCSI_WQE_FIELD_CDB_SIZE_MASK  0xFF
+#define ISCSI_WQE_FIELD_CDB_SIZE_SHIFT 24
+};
+
+union iscsi_wqe_field_union {
+        struct iscsi_wqe_field cont_field;
+        __le32 prev_tid;
+};
+
+struct iscsi_wqe {
+        __le16 task_id;
+        u8 flags;
+#define ISCSI_WQE_WQE_TYPE_MASK        0x7
+#define ISCSI_WQE_WQE_TYPE_SHIFT       0
+#define ISCSI_WQE_NUM_FAST_SGES_MASK   0x7
+#define ISCSI_WQE_NUM_FAST_SGES_SHIFT  3
+#define ISCSI_WQE_PTU_INVALIDATE_MASK  0x1
+#define ISCSI_WQE_PTU_INVALIDATE_SHIFT 6
+#define ISCSI_WQE_RESPONSE_MASK        0x1
+#define ISCSI_WQE_RESPONSE_SHIFT       7
+        struct iscsi_dif_flags prot_flags;
+        union iscsi_wqe_field_union cont_prevtid_union;
+};
+
+enum iscsi_wqe_type {
+        ISCSI_WQE_TYPE_NORMAL,
+        ISCSI_WQE_TYPE_TASK_CLEANUP,
+        ISCSI_WQE_TYPE_MIDDLE_PATH,
+        ISCSI_WQE_TYPE_LOGIN,
+        ISCSI_WQE_TYPE_FIRST_R2T_CONT,
+        ISCSI_WQE_TYPE_NONFIRST_R2T_CONT,
+        ISCSI_WQE_TYPE_RESPONSE,
+        MAX_ISCSI_WQE_TYPE
+};
+
+struct iscsi_xhqe {
+        union iscsi_ttt_txlen_union ttt_or_txlen;
+        __le32 exp_stat_sn;
+        struct iscsi_dif_flags prot_flags;
+        u8 total_ahs_length;
+        u8 opcode;
+        u8 flags;
+#define ISCSI_XHQE_NUM_FAST_SGES_MASK  0x7
+#define ISCSI_XHQE_NUM_FAST_SGES_SHIFT 0
+#define ISCSI_XHQE_FINAL_MASK          0x1
+#define ISCSI_XHQE_FINAL_SHIFT         3
+#define ISCSI_XHQE_SUPER_IO_MASK       0x1
+#define ISCSI_XHQE_SUPER_IO_SHIFT      4
+#define ISCSI_XHQE_STATUS_BIT_MASK     0x1
+#define ISCSI_XHQE_STATUS_BIT_SHIFT    5
+#define ISCSI_XHQE_RESERVED_MASK       0x3
+#define ISCSI_XHQE_RESERVED_SHIFT      6
+        union iscsi_seq_num seq_num_union;
+        __le16 reserved1;
+};
+
+struct mstorm_iscsi_stats_drv {
+        struct regpair iscsi_rx_dropped_pdus_task_not_valid;
+};
+
+struct ooo_opaque {
+        __le32 cid;
+        u8 drop_isle;
+        u8 drop_size;
+        u8 ooo_opcode;
+        u8 ooo_isle;
+};
+
+struct pstorm_iscsi_stats_drv {
+        struct regpair iscsi_tx_bytes_cnt;
+        struct regpair iscsi_tx_packet_cnt;
+};
+
+struct tstorm_iscsi_stats_drv {
+        struct regpair iscsi_rx_bytes_cnt;
+        struct regpair iscsi_rx_packet_cnt;
+        struct regpair iscsi_rx_new_ooo_isle_events_cnt;
+        __le32 iscsi_cmdq_threshold_cnt;
+        __le32 iscsi_rq_threshold_cnt;
+        __le32 iscsi_immq_threshold_cnt;
+};
+
+struct ustorm_iscsi_stats_drv {
+        struct regpair iscsi_rx_data_pdu_cnt;
+        struct regpair iscsi_rx_r2t_pdu_cnt;
+        struct regpair iscsi_rx_total_pdu_cnt;
+};
+
+struct xstorm_iscsi_stats_drv {
+        struct regpair iscsi_tx_go_to_slow_start_event_cnt;
+        struct regpair iscsi_tx_fast_retransmit_event_cnt;
+};
+
+struct ystorm_iscsi_stats_drv {
+        struct regpair iscsi_tx_data_pdu_cnt;
+        struct regpair iscsi_tx_r2t_pdu_cnt;
+        struct regpair iscsi_tx_total_pdu_cnt;
+};
+
+struct iscsi_db_data {
+        u8 params;
+#define ISCSI_DB_DATA_DEST_MASK         0x3
+#define ISCSI_DB_DATA_DEST_SHIFT        0
+#define ISCSI_DB_DATA_AGG_CMD_MASK      0x3
+#define ISCSI_DB_DATA_AGG_CMD_SHIFT     2
+#define ISCSI_DB_DATA_BYPASS_EN_MASK    0x1
+#define ISCSI_DB_DATA_BYPASS_EN_SHIFT   4
+#define ISCSI_DB_DATA_RESERVED_MASK     0x1
+#define ISCSI_DB_DATA_RESERVED_SHIFT    5
+#define ISCSI_DB_DATA_AGG_VAL_SEL_MASK  0x3
+#define ISCSI_DB_DATA_AGG_VAL_SEL_SHIFT 6
+        u8 agg_flags;
+        __le16 sq_prod;
+};
+
+struct tstorm_iscsi_task_ag_ctx {
+        u8 byte0;
+        u8 byte1;
+        __le16 word0;
+        u8 flags0;
+#define TSTORM_ISCSI_TASK_AG_CTX_NIBBLE0_MASK  0xF
+#define TSTORM_ISCSI_TASK_AG_CTX_NIBBLE0_SHIFT 0
+#define TSTORM_ISCSI_TASK_AG_CTX_BIT0_MASK     0x1
+#define TSTORM_ISCSI_TASK_AG_CTX_BIT0_SHIFT    4
+#define TSTORM_ISCSI_TASK_AG_CTX_BIT1_MASK     0x1
+#define TSTORM_ISCSI_TASK_AG_CTX_BIT1_SHIFT    5
+#define TSTORM_ISCSI_TASK_AG_CTX_BIT2_MASK     0x1
+#define TSTORM_ISCSI_TASK_AG_CTX_BIT2_SHIFT    6
+#define TSTORM_ISCSI_TASK_AG_CTX_BIT3_MASK     0x1
+#define TSTORM_ISCSI_TASK_AG_CTX_BIT3_SHIFT    7
+        u8 flags1;
+#define TSTORM_ISCSI_TASK_AG_CTX_BIT4_MASK     0x1
+#define TSTORM_ISCSI_TASK_AG_CTX_BIT4_SHIFT    0
+#define TSTORM_ISCSI_TASK_AG_CTX_BIT5_MASK     0x1
+#define TSTORM_ISCSI_TASK_AG_CTX_BIT5_SHIFT    1
+#define TSTORM_ISCSI_TASK_AG_CTX_CF0_MASK      0x3
+#define TSTORM_ISCSI_TASK_AG_CTX_CF0_SHIFT     2
+#define TSTORM_ISCSI_TASK_AG_CTX_CF1_MASK      0x3
+#define TSTORM_ISCSI_TASK_AG_CTX_CF1_SHIFT     4
+#define TSTORM_ISCSI_TASK_AG_CTX_CF2_MASK      0x3
+#define TSTORM_ISCSI_TASK_AG_CTX_CF2_SHIFT     6
+        u8 flags2;
+#define TSTORM_ISCSI_TASK_AG_CTX_CF3_MASK      0x3
+#define TSTORM_ISCSI_TASK_AG_CTX_CF3_SHIFT     0
+#define TSTORM_ISCSI_TASK_AG_CTX_CF4_MASK      0x3
+#define TSTORM_ISCSI_TASK_AG_CTX_CF4_SHIFT     2
+#define TSTORM_ISCSI_TASK_AG_CTX_CF5_MASK      0x3
+#define TSTORM_ISCSI_TASK_AG_CTX_CF5_SHIFT     4
+#define TSTORM_ISCSI_TASK_AG_CTX_CF6_MASK      0x3
+#define TSTORM_ISCSI_TASK_AG_CTX_CF6_SHIFT     6
+        u8 flags3;
+#define TSTORM_ISCSI_TASK_AG_CTX_CF7_MASK      0x3
+#define TSTORM_ISCSI_TASK_AG_CTX_CF7_SHIFT     0
+#define TSTORM_ISCSI_TASK_AG_CTX_CF0EN_MASK    0x1
+#define TSTORM_ISCSI_TASK_AG_CTX_CF0EN_SHIFT   2
+#define TSTORM_ISCSI_TASK_AG_CTX_CF1EN_MASK    0x1
+#define TSTORM_ISCSI_TASK_AG_CTX_CF1EN_SHIFT   3
+#define TSTORM_ISCSI_TASK_AG_CTX_CF2EN_MASK    0x1
+#define TSTORM_ISCSI_TASK_AG_CTX_CF2EN_SHIFT   4
+#define TSTORM_ISCSI_TASK_AG_CTX_CF3EN_MASK    0x1
+#define TSTORM_ISCSI_TASK_AG_CTX_CF3EN_SHIFT   5
+#define TSTORM_ISCSI_TASK_AG_CTX_CF4EN_MASK    0x1
+#define TSTORM_ISCSI_TASK_AG_CTX_CF4EN_SHIFT   6
+#define TSTORM_ISCSI_TASK_AG_CTX_CF5EN_MASK    0x1
+#define TSTORM_ISCSI_TASK_AG_CTX_CF5EN_SHIFT   7
+        u8 flags4;
+#define TSTORM_ISCSI_TASK_AG_CTX_CF6EN_MASK    0x1
+#define TSTORM_ISCSI_TASK_AG_CTX_CF6EN_SHIFT   0
+#define TSTORM_ISCSI_TASK_AG_CTX_CF7EN_MASK    0x1
+#define TSTORM_ISCSI_TASK_AG_CTX_CF7EN_SHIFT   1
+#define TSTORM_ISCSI_TASK_AG_CTX_RULE0EN_MASK  0x1
+#define TSTORM_ISCSI_TASK_AG_CTX_RULE0EN_SHIFT 2
+#define TSTORM_ISCSI_TASK_AG_CTX_RULE1EN_MASK  0x1
+#define TSTORM_ISCSI_TASK_AG_CTX_RULE1EN_SHIFT 3
+#define TSTORM_ISCSI_TASK_AG_CTX_RULE2EN_MASK  0x1
+#define TSTORM_ISCSI_TASK_AG_CTX_RULE2EN_SHIFT 4
+#define TSTORM_ISCSI_TASK_AG_CTX_RULE3EN_MASK  0x1
+#define TSTORM_ISCSI_TASK_AG_CTX_RULE3EN_SHIFT 5
+#define TSTORM_ISCSI_TASK_AG_CTX_RULE4EN_MASK  0x1
+#define TSTORM_ISCSI_TASK_AG_CTX_RULE4EN_SHIFT 6
+#define TSTORM_ISCSI_TASK_AG_CTX_RULE5EN_MASK  0x1
+#define TSTORM_ISCSI_TASK_AG_CTX_RULE5EN_SHIFT 7
+        u8 byte2;
+        __le16 word1;
+        __le32 reg0;
+        u8 byte3;
+        u8 byte4;
+        __le16 word2;
+        __le16 word3;
+        __le16 word4;
+        __le32 reg1;
+        __le32 reg2;
+};
+
+#endif /* __ISCSI_COMMON__ */
diff --git a/include/linux/qed/qed_chain.h b/include/linux/qed/qed_chain.h
index 5f8fcaaa6504..7e441bdeabdc 100644
--- a/include/linux/qed/qed_chain.h
+++ b/include/linux/qed/qed_chain.h
@@ -25,10 +25,9 @@
                                 } while (0)
 
 #define HILO_GEN(hi, lo, type)  ((((type)(hi)) << 32) + (lo))
-#define HILO_DMA(hi, lo)        HILO_GEN(hi, lo, dma_addr_t)
 #define HILO_64(hi, lo) HILO_GEN((le32_to_cpu(hi)), (le32_to_cpu(lo)), u64)
-#define HILO_DMA_REGPAIR(regpair)       (HILO_DMA(regpair.hi, regpair.lo))
 #define HILO_64_REGPAIR(regpair)        (HILO_64(regpair.hi, regpair.lo))
+#define HILO_DMA_REGPAIR(regpair)       ((dma_addr_t)HILO_64_REGPAIR(regpair))
 
 enum qed_chain_mode {
         /* Each Page contains a next pointer at its end */
@@ -47,16 +46,56 @@ enum qed_chain_use_mode {
         QED_CHAIN_USE_TO_CONSUME_PRODUCE,       /* Chain starts empty */
 };
 
+enum qed_chain_cnt_type {
+        /* The chain's size/prod/cons are kept in 16-bit variables */
+        QED_CHAIN_CNT_TYPE_U16,
+
+        /* The chain's size/prod/cons are kept in 32-bit variables  */
+        QED_CHAIN_CNT_TYPE_U32,
+};
+
 struct qed_chain_next {
         struct regpair  next_phys;
         void            *next_virt;
 };
 
+struct qed_chain_pbl_u16 {
+        u16 prod_page_idx;
+        u16 cons_page_idx;
+};
+
+struct qed_chain_pbl_u32 {
+        u32 prod_page_idx;
+        u32 cons_page_idx;
+};
+
 struct qed_chain_pbl {
+        /* Base address of a pre-allocated buffer for pbl */
         dma_addr_t      p_phys_table;
         void            *p_virt_table;
-        u16             prod_page_idx;
-        u16             cons_page_idx;
+
+        /* Table for keeping the virtual addresses of the chain pages,
+         * respectively to the physical addresses in the pbl table.
+         */
+        void **pp_virt_addr_tbl;
+
+        /* Index to current used page by producer/consumer */
+        union {
+                struct qed_chain_pbl_u16 pbl16;
+                struct qed_chain_pbl_u32 pbl32;
+        } u;
+};
+
+struct qed_chain_u16 {
+        /* Cyclic index of next element to produce/consme */
+        u16 prod_idx;
+        u16 cons_idx;
+};
+
+struct qed_chain_u32 {
+        /* Cyclic index of next element to produce/consme */
+        u32 prod_idx;
+        u32 cons_idx;
 };
 
 struct qed_chain {
@@ -64,13 +103,25 @@ struct qed_chain {
         dma_addr_t              p_phys_addr;
         void                    *p_prod_elem;
         void                    *p_cons_elem;
-        u16                     page_cnt;
+
         enum qed_chain_mode     mode;
         enum qed_chain_use_mode intended_use; /* used to produce/consume */
-        u16                     capacity; /*< number of _usable_ elements */
-        u16                     size; /* number of elements */
-        u16                     prod_idx;
-        u16                     cons_idx;
+        enum qed_chain_cnt_type cnt_type;
+
+        union {
+                struct qed_chain_u16 chain16;
+                struct qed_chain_u32 chain32;
+        } u;
+
+        u32 page_cnt;
+
+        /* Number of elements - capacity is for usable elements only,
+         * while size will contain total number of elements [for entire chain].
+         */
+        u32 capacity;
+        u32 size;
+
+        /* Elements information for fast calculations */
         u16                     elem_per_page;
         u16                     elem_per_page_mask;
         u16                     elem_unusable;
@@ -96,66 +147,69 @@ struct qed_chain {
 #define QED_CHAIN_PAGE_CNT(elem_cnt, elem_size, mode) \
         DIV_ROUND_UP(elem_cnt, USABLE_ELEMS_PER_PAGE(elem_size, mode))
 
+#define is_chain_u16(p) ((p)->cnt_type == QED_CHAIN_CNT_TYPE_U16)
+#define is_chain_u32(p) ((p)->cnt_type == QED_CHAIN_CNT_TYPE_U32)
+
 /* Accessors */
 static inline u16 qed_chain_get_prod_idx(struct qed_chain *p_chain)
 {
-        return p_chain->prod_idx;
+        return p_chain->u.chain16.prod_idx;
 }
 
 static inline u16 qed_chain_get_cons_idx(struct qed_chain *p_chain)
 {
-        return p_chain->cons_idx;
+        return p_chain->u.chain16.cons_idx;
+}
+
+static inline u32 qed_chain_get_cons_idx_u32(struct qed_chain *p_chain)
+{
+        return p_chain->u.chain32.cons_idx;
 }
 
 static inline u16 qed_chain_get_elem_left(struct qed_chain *p_chain)
 {
         u16 used;
 
-        /* we don't need to trancate upon assignmet, as we assign u32->u16 */
-        used = ((u32)0x10000u + (u32)(p_chain->prod_idx)) -
-                (u32)p_chain->cons_idx;
+        used = (u16) (((u32)0x10000 +
+                       (u32)p_chain->u.chain16.prod_idx) -
+                      (u32)p_chain->u.chain16.cons_idx);
         if (p_chain->mode == QED_CHAIN_MODE_NEXT_PTR)
-                used -= p_chain->prod_idx / p_chain->elem_per_page -
-                        p_chain->cons_idx / p_chain->elem_per_page;
+                used -= p_chain->u.chain16.prod_idx / p_chain->elem_per_page -
+                    p_chain->u.chain16.cons_idx / p_chain->elem_per_page;
 
-        return p_chain->capacity - used;
+        return (u16)(p_chain->capacity - used);
 }
 
-static inline u8 qed_chain_is_full(struct qed_chain *p_chain)
+static inline u32 qed_chain_get_elem_left_u32(struct qed_chain *p_chain)
 {
-        return qed_chain_get_elem_left(p_chain) == p_chain->capacity;
-}
+        u32 used;
 
-static inline u8 qed_chain_is_empty(struct qed_chain *p_chain)
-{
-        return qed_chain_get_elem_left(p_chain) == 0;
-}
+        used = (u32) (((u64)0x100000000ULL +
+                       (u64)p_chain->u.chain32.prod_idx) -
+                      (u64)p_chain->u.chain32.cons_idx);
+        if (p_chain->mode == QED_CHAIN_MODE_NEXT_PTR)
+                used -= p_chain->u.chain32.prod_idx / p_chain->elem_per_page -
+                    p_chain->u.chain32.cons_idx / p_chain->elem_per_page;
 
-static inline u16 qed_chain_get_elem_per_page(
-        struct qed_chain *p_chain)
-{
-        return p_chain->elem_per_page;
+        return p_chain->capacity - used;
 }
 
-static inline u16 qed_chain_get_usable_per_page(
-        struct qed_chain *p_chain)
+static inline u16 qed_chain_get_usable_per_page(struct qed_chain *p_chain)
 {
         return p_chain->usable_per_page;
 }
 
-static inline u16 qed_chain_get_unusable_per_page(
-        struct qed_chain *p_chain)
+static inline u16 qed_chain_get_unusable_per_page(struct qed_chain *p_chain)
 {
         return p_chain->elem_unusable;
 }
 
-static inline u16 qed_chain_get_size(struct qed_chain *p_chain)
+static inline u32 qed_chain_get_page_cnt(struct qed_chain *p_chain)
 {
-        return p_chain->size;
+        return p_chain->page_cnt;
 }
 
-static inline dma_addr_t
-qed_chain_get_pbl_phys(struct qed_chain *p_chain)
+static inline dma_addr_t qed_chain_get_pbl_phys(struct qed_chain *p_chain)
 {
         return p_chain->pbl.p_phys_table;
 }
@@ -172,65 +226,63 @@ qed_chain_get_pbl_phys(struct qed_chain *p_chain)
  */
 static inline void
 qed_chain_advance_page(struct qed_chain *p_chain,
-                       void **p_next_elem,
-                       u16 *idx_to_inc,
-                       u16 *page_to_inc)
+                       void **p_next_elem, void *idx_to_inc, void *page_to_inc)
 
 {
+        struct qed_chain_next *p_next = NULL;
+        u32 page_index = 0;
         switch (p_chain->mode) {
         case QED_CHAIN_MODE_NEXT_PTR:
-        {
-                struct qed_chain_next *p_next = *p_next_elem;
+                p_next = *p_next_elem;
                 *p_next_elem = p_next->next_virt;
-                *idx_to_inc += p_chain->elem_unusable;
+                if (is_chain_u16(p_chain))
+                        *(u16 *)idx_to_inc += p_chain->elem_unusable;
+                else
+                        *(u32 *)idx_to_inc += p_chain->elem_unusable;
                 break;
-        }
         case QED_CHAIN_MODE_SINGLE:
                 *p_next_elem = p_chain->p_virt_addr;
                 break;
 
         case QED_CHAIN_MODE_PBL:
-                /* It is assumed pages are sequential, next element needs
-                 * to change only when passing going back to first from last.
-                 */
-                if (++(*page_to_inc) == p_chain->page_cnt) {
-                        *page_to_inc = 0;
-                        *p_next_elem = p_chain->p_virt_addr;
+                if (is_chain_u16(p_chain)) {
+                        if (++(*(u16 *)page_to_inc) == p_chain->page_cnt)
+                                *(u16 *)page_to_inc = 0;
+                        page_index = *(u16 *)page_to_inc;
+                } else {
+                        if (++(*(u32 *)page_to_inc) == p_chain->page_cnt)
+                                *(u32 *)page_to_inc = 0;
+                        page_index = *(u32 *)page_to_inc;
                 }
+                *p_next_elem = p_chain->pbl.pp_virt_addr_tbl[page_index];
         }
 }
 
 #define is_unusable_idx(p, idx) \
-        (((p)->idx & (p)->elem_per_page_mask) == (p)->usable_per_page)
+        (((p)->u.chain16.idx & (p)->elem_per_page_mask) == (p)->usable_per_page)
+
+#define is_unusable_idx_u32(p, idx) \
+        (((p)->u.chain32.idx & (p)->elem_per_page_mask) == (p)->usable_per_page)
+#define is_unusable_next_idx(p, idx)                             \
+        ((((p)->u.chain16.idx + 1) & (p)->elem_per_page_mask) == \
+         (p)->usable_per_page)
 
-#define is_unusable_next_idx(p, idx) \
-        ((((p)->idx + 1) & (p)->elem_per_page_mask) == (p)->usable_per_page)
+#define is_unusable_next_idx_u32(p, idx)                         \
+        ((((p)->u.chain32.idx + 1) & (p)->elem_per_page_mask) == \
+         (p)->usable_per_page)
 
-#define test_ans_skip(p, idx)                           \
+#define test_and_skip(p, idx)                                              \
         do {                                            \
-                if (is_unusable_idx(p, idx)) {          \
-                        (p)->idx += (p)->elem_unusable; \
+                if (is_chain_u16(p)) {                                     \
+                        if (is_unusable_idx(p, idx))                       \
+                                (p)->u.chain16.idx += (p)->elem_unusable;  \
+                } else {                                                   \
+                        if (is_unusable_idx_u32(p, idx))                   \
+                                (p)->u.chain32.idx += (p)->elem_unusable;  \
                 }                                       \
         } while (0)
 
 /**
- * @brief qed_chain_return_multi_produced -
- *
- * A chain in which the driver "Produces" elements should use this API
- * to indicate previous produced elements are now consumed.
- *
- * @param p_chain
- * @param num
- */
-static inline void
-qed_chain_return_multi_produced(struct qed_chain *p_chain,
-                                u16 num)
-{
-        p_chain->cons_idx += num;
-        test_ans_skip(p_chain, cons_idx);
-}
-
-/**
  * @brief qed_chain_return_produced -
  *
  * A chain in which the driver "Produces" elements should use this API
@@ -240,8 +292,11 @@ qed_chain_return_multi_produced(struct qed_chain *p_chain,
  */
 static inline void qed_chain_return_produced(struct qed_chain *p_chain)
 {
-        p_chain->cons_idx++;
-        test_ans_skip(p_chain, cons_idx);
+        if (is_chain_u16(p_chain))
+                p_chain->u.chain16.cons_idx++;
+        else
+                p_chain->u.chain32.cons_idx++;
+        test_and_skip(p_chain, cons_idx);
 }
 
 /**
@@ -257,21 +312,33 @@ static inline void qed_chain_return_produced(struct qed_chain *p_chain)
  */
 static inline void *qed_chain_produce(struct qed_chain *p_chain)
 {
-        void *ret = NULL;
-
-        if ((p_chain->prod_idx & p_chain->elem_per_page_mask) ==
-            p_chain->next_page_mask) {
-                qed_chain_advance_page(p_chain, &p_chain->p_prod_elem,
-                                       &p_chain->prod_idx,
-                                       &p_chain->pbl.prod_page_idx);
+        void *p_ret = NULL, *p_prod_idx, *p_prod_page_idx;
+
+        if (is_chain_u16(p_chain)) {
+                if ((p_chain->u.chain16.prod_idx &
+                     p_chain->elem_per_page_mask) == p_chain->next_page_mask) {
+                        p_prod_idx = &p_chain->u.chain16.prod_idx;
+                        p_prod_page_idx = &p_chain->pbl.u.pbl16.prod_page_idx;
+                        qed_chain_advance_page(p_chain, &p_chain->p_prod_elem,
+                                               p_prod_idx, p_prod_page_idx);
+                }
+                p_chain->u.chain16.prod_idx++;
+        } else {
+                if ((p_chain->u.chain32.prod_idx &
+                     p_chain->elem_per_page_mask) == p_chain->next_page_mask) {
+                        p_prod_idx = &p_chain->u.chain32.prod_idx;
+                        p_prod_page_idx = &p_chain->pbl.u.pbl32.prod_page_idx;
+                        qed_chain_advance_page(p_chain, &p_chain->p_prod_elem,
+                                               p_prod_idx, p_prod_page_idx);
+                }
+                p_chain->u.chain32.prod_idx++;
         }
 
-        ret = p_chain->p_prod_elem;
-        p_chain->prod_idx++;
+        p_ret = p_chain->p_prod_elem;
         p_chain->p_prod_elem = (void *)(((u8 *)p_chain->p_prod_elem) +
                                         p_chain->elem_size);
 
-        return ret;
+        return p_ret;
 }
 
 /**
@@ -282,9 +349,9 @@ static inline void *qed_chain_produce(struct qed_chain *p_chain)
  * @param p_chain
  * @param num
  *
- * @return u16, number of unusable BDs
+ * @return number of unusable BDs
  */
-static inline u16 qed_chain_get_capacity(struct qed_chain *p_chain)
+static inline u32 qed_chain_get_capacity(struct qed_chain *p_chain)
 {
         return p_chain->capacity;
 }
@@ -297,11 +364,13 @@ static inline u16 qed_chain_get_capacity(struct qed_chain *p_chain)
  *
  * @param p_chain
  */
-static inline void
-qed_chain_recycle_consumed(struct qed_chain *p_chain)
+static inline void qed_chain_recycle_consumed(struct qed_chain *p_chain)
 {
-        test_ans_skip(p_chain, prod_idx);
-        p_chain->prod_idx++;
+        test_and_skip(p_chain, prod_idx);
+        if (is_chain_u16(p_chain))
+                p_chain->u.chain16.prod_idx++;
+        else
+                p_chain->u.chain32.prod_idx++;
 }
 
 /**
@@ -316,21 +385,33 @@ qed_chain_recycle_consumed(struct qed_chain *p_chain)
  */
 static inline void *qed_chain_consume(struct qed_chain *p_chain)
 {
-        void *ret = NULL;
-
-        if ((p_chain->cons_idx & p_chain->elem_per_page_mask) ==
-            p_chain->next_page_mask) {
+        void *p_ret = NULL, *p_cons_idx, *p_cons_page_idx;
+
+        if (is_chain_u16(p_chain)) {
+                if ((p_chain->u.chain16.cons_idx &
+                     p_chain->elem_per_page_mask) == p_chain->next_page_mask) {
+                        p_cons_idx = &p_chain->u.chain16.cons_idx;
+                        p_cons_page_idx = &p_chain->pbl.u.pbl16.cons_page_idx;
+                        qed_chain_advance_page(p_chain, &p_chain->p_cons_elem,
+                                               p_cons_idx, p_cons_page_idx);
+                }
+                p_chain->u.chain16.cons_idx++;
+        } else {
+                if ((p_chain->u.chain32.cons_idx &
+                     p_chain->elem_per_page_mask) == p_chain->next_page_mask) {
+                        p_cons_idx = &p_chain->u.chain32.cons_idx;
+                        p_cons_page_idx = &p_chain->pbl.u.pbl32.cons_page_idx;
                 qed_chain_advance_page(p_chain, &p_chain->p_cons_elem,
-                                       &p_chain->cons_idx,
-                                       &p_chain->pbl.cons_page_idx);
+                                               p_cons_idx, p_cons_page_idx);
+                }
+                p_chain->u.chain32.cons_idx++;
         }
 
-        ret = p_chain->p_cons_elem;
-        p_chain->cons_idx++;
+        p_ret = p_chain->p_cons_elem;
         p_chain->p_cons_elem = (void *)(((u8 *)p_chain->p_cons_elem) +
                                         p_chain->elem_size);
 
-        return ret;
+        return p_ret;
 }
 
 /**
@@ -340,16 +421,33 @@ static inline void *qed_chain_consume(struct qed_chain *p_chain)
  */
 static inline void qed_chain_reset(struct qed_chain *p_chain)
 {
-        int i;
-
-        p_chain->prod_idx       = 0;
-        p_chain->cons_idx       = 0;
-        p_chain->p_cons_elem    = p_chain->p_virt_addr;
-        p_chain->p_prod_elem    = p_chain->p_virt_addr;
+        u32 i;
+
+        if (is_chain_u16(p_chain)) {
+                p_chain->u.chain16.prod_idx = 0;
+                p_chain->u.chain16.cons_idx = 0;
+        } else {
+                p_chain->u.chain32.prod_idx = 0;
+                p_chain->u.chain32.cons_idx = 0;
+        }
+        p_chain->p_cons_elem = p_chain->p_virt_addr;
+        p_chain->p_prod_elem = p_chain->p_virt_addr;
 
         if (p_chain->mode == QED_CHAIN_MODE_PBL) {
-                p_chain->pbl.prod_page_idx      = p_chain->page_cnt - 1;
-                p_chain->pbl.cons_page_idx      = p_chain->page_cnt - 1;
+                /* Use (page_cnt - 1) as a reset value for the prod/cons page's
+                 * indices, to avoid unnecessary page advancing on the first
+                 * call to qed_chain_produce/consume. Instead, the indices
+                 * will be advanced to page_cnt and then will be wrapped to 0.
+                 */
+                u32 reset_val = p_chain->page_cnt - 1;
+
+                if (is_chain_u16(p_chain)) {
+                        p_chain->pbl.u.pbl16.prod_page_idx = (u16)reset_val;
+                        p_chain->pbl.u.pbl16.cons_page_idx = (u16)reset_val;
+                } else {
+                        p_chain->pbl.u.pbl32.prod_page_idx = reset_val;
+                        p_chain->pbl.u.pbl32.cons_page_idx = reset_val;
+                }
         }
 
         switch (p_chain->intended_use) {
@@ -377,168 +475,184 @@ static inline void qed_chain_reset(struct qed_chain *p_chain)
  * @param intended_use
  * @param mode
  */
-static inline void qed_chain_init(struct qed_chain *p_chain,
-                                  void *p_virt_addr,
-                                  dma_addr_t p_phys_addr,
-                                  u16 page_cnt,
-                                  u8 elem_size,
-                                  enum qed_chain_use_mode intended_use,
-                                  enum qed_chain_mode mode)
+static inline void qed_chain_init_params(struct qed_chain *p_chain,
+                                         u32 page_cnt,
+                                         u8 elem_size,
+                                         enum qed_chain_use_mode intended_use,
+                                         enum qed_chain_mode mode,
+                                         enum qed_chain_cnt_type cnt_type)
 {
         /* chain fixed parameters */
-        p_chain->p_virt_addr    = p_virt_addr;
-        p_chain->p_phys_addr    = p_phys_addr;
+        p_chain->p_virt_addr = NULL;
+        p_chain->p_phys_addr = 0;
         p_chain->elem_size      = elem_size;
-        p_chain->page_cnt       = page_cnt;
+        p_chain->intended_use = intended_use;
         p_chain->mode           = mode;
+        p_chain->cnt_type = cnt_type;
 
-        p_chain->intended_use           = intended_use;
         p_chain->elem_per_page          = ELEMS_PER_PAGE(elem_size);
-        p_chain->usable_per_page =
-                USABLE_ELEMS_PER_PAGE(elem_size, mode);
-        p_chain->capacity               = p_chain->usable_per_page * page_cnt;
-        p_chain->size                   = p_chain->elem_per_page * page_cnt;
+        p_chain->usable_per_page = USABLE_ELEMS_PER_PAGE(elem_size, mode);
         p_chain->elem_per_page_mask     = p_chain->elem_per_page - 1;
-
         p_chain->elem_unusable = UNUSABLE_ELEMS_PER_PAGE(elem_size, mode);
-
         p_chain->next_page_mask = (p_chain->usable_per_page &
                                    p_chain->elem_per_page_mask);
 
-        if (mode == QED_CHAIN_MODE_NEXT_PTR) {
-                struct qed_chain_next   *p_next;
-                u16                     i;
-
-                for (i = 0; i < page_cnt - 1; i++) {
-                        /* Increment mem_phy to the next page. */
-                        p_phys_addr += QED_CHAIN_PAGE_SIZE;
-
-                        /* Initialize the physical address of the next page. */
-                        p_next = (struct qed_chain_next *)((u8 *)p_virt_addr +
-                                                           elem_size *
-                                                           p_chain->
-                                                           usable_per_page);
-
-                        p_next->next_phys.lo    = DMA_LO_LE(p_phys_addr);
-                        p_next->next_phys.hi    = DMA_HI_LE(p_phys_addr);
-
-                        /* Initialize the virtual address of the next page. */
-                        p_next->next_virt = (void *)((u8 *)p_virt_addr +
-                                                     QED_CHAIN_PAGE_SIZE);
-
-                        /* Move to the next page. */
-                        p_virt_addr = p_next->next_virt;
-                }
-
-                /* Last page's next should point to beginning of the chain */
-                p_next = (struct qed_chain_next *)((u8 *)p_virt_addr +
-                                                   elem_size *
-                                                   p_chain->usable_per_page);
+        p_chain->page_cnt = page_cnt;
+        p_chain->capacity = p_chain->usable_per_page * page_cnt;
+        p_chain->size = p_chain->elem_per_page * page_cnt;
 
-                p_next->next_phys.lo    = DMA_LO_LE(p_chain->p_phys_addr);
-                p_next->next_phys.hi    = DMA_HI_LE(p_chain->p_phys_addr);
-                p_next->next_virt       = p_chain->p_virt_addr;
-        }
-        qed_chain_reset(p_chain);
+        p_chain->pbl.p_phys_table = 0;
+        p_chain->pbl.p_virt_table = NULL;
+        p_chain->pbl.pp_virt_addr_tbl = NULL;
 }
 
 /**
- * @brief qed_chain_pbl_init - Initalizes a basic pbl chain
- *        struct
+ * @brief qed_chain_init_mem -
+ *
+ * Initalizes a basic chain struct with its chain buffers
+ *
  * @param p_chain
  * @param p_virt_addr   virtual address of allocated buffer's beginning
  * @param p_phys_addr   physical address of allocated buffer's beginning
- * @param page_cnt      number of pages in the allocated buffer
- * @param elem_size     size of each element in the chain
- * @param use_mode
- * @param p_phys_pbl    pointer to a pre-allocated side table
- *                      which will hold physical page addresses.
- * @param p_virt_pbl    pointer to a pre allocated side table
- *                      which will hold virtual page addresses.
+ *
  */
-static inline void
-qed_chain_pbl_init(struct qed_chain *p_chain,
-                   void *p_virt_addr,
-                   dma_addr_t p_phys_addr,
-                   u16 page_cnt,
-                   u8 elem_size,
-                   enum qed_chain_use_mode use_mode,
-                   dma_addr_t p_phys_pbl,
-                   dma_addr_t *p_virt_pbl)
+static inline void qed_chain_init_mem(struct qed_chain *p_chain,
+                                      void *p_virt_addr, dma_addr_t p_phys_addr)
 {
-        dma_addr_t *p_pbl_dma = p_virt_pbl;
-        int i;
-
-        qed_chain_init(p_chain, p_virt_addr, p_phys_addr, page_cnt,
-                       elem_size, use_mode, QED_CHAIN_MODE_PBL);
+        p_chain->p_virt_addr = p_virt_addr;
+        p_chain->p_phys_addr = p_phys_addr;
+}
 
+/**
+ * @brief qed_chain_init_pbl_mem -
+ *
+ * Initalizes a basic chain struct with its pbl buffers
+ *
+ * @param p_chain
+ * @param p_virt_pbl    pointer to a pre allocated side table which will hold
+ *                      virtual page addresses.
+ * @param p_phys_pbl    pointer to a pre-allocated side table which will hold
+ *                      physical page addresses.
+ * @param pp_virt_addr_tbl
+ *                      pointer to a pre-allocated side table which will hold
+ *                      the virtual addresses of the chain pages.
+ *
+ */
+static inline void qed_chain_init_pbl_mem(struct qed_chain *p_chain,
+                                          void *p_virt_pbl,
+                                          dma_addr_t p_phys_pbl,
+                                          void **pp_virt_addr_tbl)
+{
         p_chain->pbl.p_phys_table = p_phys_pbl;
         p_chain->pbl.p_virt_table = p_virt_pbl;
-
-        /* Fill the PBL with physical addresses*/
-        for (i = 0; i < page_cnt; i++) {
-                *p_pbl_dma = p_phys_addr;
-                p_phys_addr += QED_CHAIN_PAGE_SIZE;
-                p_pbl_dma++;
-        }
+        p_chain->pbl.pp_virt_addr_tbl = pp_virt_addr_tbl;
 }
 
 /**
- * @brief qed_chain_set_prod - sets the prod to the given
- *        value
+ * @brief qed_chain_init_next_ptr_elem -
+ *
+ * Initalizes a next pointer element
+ *
+ * @param p_chain
+ * @param p_virt_curr   virtual address of a chain page of which the next
+ *                      pointer element is initialized
+ * @param p_virt_next   virtual address of the next chain page
+ * @param p_phys_next   physical address of the next chain page
  *
- * @param prod_idx
- * @param p_prod_elem
  */
-static inline void qed_chain_set_prod(struct qed_chain *p_chain,
-                                      u16 prod_idx,
-                                      void *p_prod_elem)
+static inline void
+qed_chain_init_next_ptr_elem(struct qed_chain *p_chain,
+                             void *p_virt_curr,
+                             void *p_virt_next, dma_addr_t p_phys_next)
 {
-        p_chain->prod_idx       = prod_idx;
-        p_chain->p_prod_elem    = p_prod_elem;
+        struct qed_chain_next *p_next;
+        u32 size;
+
+        size = p_chain->elem_size * p_chain->usable_per_page;
+        p_next = (struct qed_chain_next *)((u8 *)p_virt_curr + size);
+
+        DMA_REGPAIR_LE(p_next->next_phys, p_phys_next);
+
+        p_next->next_virt = p_virt_next;
 }
 
 /**
- * @brief qed_chain_get_elem -
+ * @brief qed_chain_get_last_elem -
  *
- * get a pointer to an element represented by absolute idx
+ * Returns a pointer to the last element of the chain
  *
  * @param p_chain
- * @assumption p_chain->size is a power of 2
  *
- * @return void*, a pointer to next element
+ * @return void*
  */
-static inline void *qed_chain_sge_get_elem(struct qed_chain *p_chain,
-                                           u16 idx)
+static inline void *qed_chain_get_last_elem(struct qed_chain *p_chain)
 {
-        void *ret = NULL;
-
-        if (idx >= p_chain->size)
-                return NULL;
+        struct qed_chain_next *p_next = NULL;
+        void *p_virt_addr = NULL;
+        u32 size, last_page_idx;
 
-        ret = (u8 *)p_chain->p_virt_addr + p_chain->elem_size * idx;
+        if (!p_chain->p_virt_addr)
+                goto out;
 
-        return ret;
+        switch (p_chain->mode) {
+        case QED_CHAIN_MODE_NEXT_PTR:
+                size = p_chain->elem_size * p_chain->usable_per_page;
+                p_virt_addr = p_chain->p_virt_addr;
+                p_next = (struct qed_chain_next *)((u8 *)p_virt_addr + size);
+                while (p_next->next_virt != p_chain->p_virt_addr) {
+                        p_virt_addr = p_next->next_virt;
+                        p_next = (struct qed_chain_next *)((u8 *)p_virt_addr +
+                                                           size);
+                }
+                break;
+        case QED_CHAIN_MODE_SINGLE:
+                p_virt_addr = p_chain->p_virt_addr;
+                break;
+        case QED_CHAIN_MODE_PBL:
+                last_page_idx = p_chain->page_cnt - 1;
+                p_virt_addr = p_chain->pbl.pp_virt_addr_tbl[last_page_idx];
+                break;
+        }
+        /* p_virt_addr points at this stage to the last page of the chain */
+        size = p_chain->elem_size * (p_chain->usable_per_page - 1);
+        p_virt_addr = (u8 *)p_virt_addr + size;
+out:
+        return p_virt_addr;
 }
 
 /**
- * @brief qed_chain_sge_inc_cons_prod
+ * @brief qed_chain_set_prod - sets the prod to the given value
  *
- * for sge chains, producer isn't increased serially, the ring
- * is expected to be full at all times. Once elements are
- * consumed, they are immediately produced.
+ * @param prod_idx
+ * @param p_prod_elem
+ */
+static inline void qed_chain_set_prod(struct qed_chain *p_chain,
+                                      u32 prod_idx, void *p_prod_elem)
+{
+        if (is_chain_u16(p_chain))
+                p_chain->u.chain16.prod_idx = (u16) prod_idx;
+        else
+                p_chain->u.chain32.prod_idx = prod_idx;
+        p_chain->p_prod_elem = p_prod_elem;
+}
+
+/**
+ * @brief qed_chain_pbl_zero_mem - set chain memory to 0
  *
  * @param p_chain
- * @param cnt
- *
- * @return inline void
  */
-static inline void
-qed_chain_sge_inc_cons_prod(struct qed_chain *p_chain,
-                            u16 cnt)
+static inline void qed_chain_pbl_zero_mem(struct qed_chain *p_chain)
 {
-        p_chain->prod_idx += cnt;
-        p_chain->cons_idx += cnt;
+        u32 i, page_cnt;
+
+        if (p_chain->mode != QED_CHAIN_MODE_PBL)
+                return;
+
+        page_cnt = qed_chain_get_page_cnt(p_chain);
+
+        for (i = 0; i < page_cnt; i++)
+                memset(p_chain->pbl.pp_virt_addr_tbl[i], 0,
+                       QED_CHAIN_PAGE_SIZE);
 }
 
 #endif
diff --git a/include/linux/qed/qed_eth_if.h b/include/linux/qed/qed_eth_if.h
index 6c876a63558d..4475a9d8ae15 100644
--- a/include/linux/qed/qed_eth_if.h
+++ b/include/linux/qed/qed_eth_if.h
@@ -114,6 +114,7 @@ struct qed_queue_start_common_params {
         u8 vport_id;
         u16 sb;
         u16 sb_idx;
+        u16 vf_qid;
 };
 
 struct qed_tunn_params {
@@ -128,11 +129,73 @@ struct qed_eth_cb_ops {
         void (*force_mac) (void *dev, u8 *mac);
 };
 
+#ifdef CONFIG_DCB
+/* Prototype declaration of qed_eth_dcbnl_ops should match with the declaration
+ * of dcbnl_rtnl_ops structure.
+ */
+struct qed_eth_dcbnl_ops {
+        /* IEEE 802.1Qaz std */
+        int (*ieee_getpfc)(struct qed_dev *cdev, struct ieee_pfc *pfc);
+        int (*ieee_setpfc)(struct qed_dev *cdev, struct ieee_pfc *pfc);
+        int (*ieee_getets)(struct qed_dev *cdev, struct ieee_ets *ets);
+        int (*ieee_setets)(struct qed_dev *cdev, struct ieee_ets *ets);
+        int (*ieee_peer_getets)(struct qed_dev *cdev, struct ieee_ets *ets);
+        int (*ieee_peer_getpfc)(struct qed_dev *cdev, struct ieee_pfc *pfc);
+        int (*ieee_getapp)(struct qed_dev *cdev, struct dcb_app *app);
+        int (*ieee_setapp)(struct qed_dev *cdev, struct dcb_app *app);
+
+        /* CEE std */
+        u8 (*getstate)(struct qed_dev *cdev);
+        u8 (*setstate)(struct qed_dev *cdev, u8 state);
+        void (*getpgtccfgtx)(struct qed_dev *cdev, int prio, u8 *prio_type,
+                             u8 *pgid, u8 *bw_pct, u8 *up_map);
+        void (*getpgbwgcfgtx)(struct qed_dev *cdev, int pgid, u8 *bw_pct);
+        void (*getpgtccfgrx)(struct qed_dev *cdev, int prio, u8 *prio_type,
+                             u8 *pgid, u8 *bw_pct, u8 *up_map);
+        void (*getpgbwgcfgrx)(struct qed_dev *cdev, int pgid, u8 *bw_pct);
+        void (*getpfccfg)(struct qed_dev *cdev, int prio, u8 *setting);
+        void (*setpfccfg)(struct qed_dev *cdev, int prio, u8 setting);
+        u8 (*getcap)(struct qed_dev *cdev, int capid, u8 *cap);
+        int (*getnumtcs)(struct qed_dev *cdev, int tcid, u8 *num);
+        u8 (*getpfcstate)(struct qed_dev *cdev);
+        int (*getapp)(struct qed_dev *cdev, u8 idtype, u16 id);
+        u8 (*getfeatcfg)(struct qed_dev *cdev, int featid, u8 *flags);
+
+        /* DCBX configuration */
+        u8 (*getdcbx)(struct qed_dev *cdev);
+        void (*setpgtccfgtx)(struct qed_dev *cdev, int prio,
+                             u8 pri_type, u8 pgid, u8 bw_pct, u8 up_map);
+        void (*setpgtccfgrx)(struct qed_dev *cdev, int prio,
+                             u8 pri_type, u8 pgid, u8 bw_pct, u8 up_map);
+        void (*setpgbwgcfgtx)(struct qed_dev *cdev, int pgid, u8 bw_pct);
+        void (*setpgbwgcfgrx)(struct qed_dev *cdev, int pgid, u8 bw_pct);
+        u8 (*setall)(struct qed_dev *cdev);
+        int (*setnumtcs)(struct qed_dev *cdev, int tcid, u8 num);
+        void (*setpfcstate)(struct qed_dev *cdev, u8 state);
+        int (*setapp)(struct qed_dev *cdev, u8 idtype, u16 idval, u8 up);
+        u8 (*setdcbx)(struct qed_dev *cdev, u8 state);
+        u8 (*setfeatcfg)(struct qed_dev *cdev, int featid, u8 flags);
+
+        /* Peer apps */
+        int (*peer_getappinfo)(struct qed_dev *cdev,
+                               struct dcb_peer_app_info *info,
+                               u16 *app_count);
+        int (*peer_getapptable)(struct qed_dev *cdev, struct dcb_app *table);
+
+        /* CEE peer */
+        int (*cee_peer_getpfc)(struct qed_dev *cdev, struct cee_pfc *pfc);
+        int (*cee_peer_getpg)(struct qed_dev *cdev, struct cee_pg *pg);
+};
+#endif
+
 struct qed_eth_ops {
         const struct qed_common_ops *common;
 #ifdef CONFIG_QED_SRIOV
         const struct qed_iov_hv_ops *iov;
 #endif
+#ifdef CONFIG_DCB
+        const struct qed_eth_dcbnl_ops *dcb;
+#endif
 
         int (*fill_dev_info)(struct qed_dev *cdev,
                              struct qed_dev_eth_info *info);
diff --git a/include/linux/qed/qed_if.h b/include/linux/qed/qed_if.h
index 4c29439f54bf..b1e3c57c7117 100644
--- a/include/linux/qed/qed_if.h
+++ b/include/linux/qed/qed_if.h
@@ -34,6 +34,96 @@ enum dcbx_protocol_type {
         DCBX_MAX_PROTOCOL_TYPE
 };
 
+#ifdef CONFIG_DCB
+#define QED_LLDP_CHASSIS_ID_STAT_LEN 4
+#define QED_LLDP_PORT_ID_STAT_LEN 4
+#define QED_DCBX_MAX_APP_PROTOCOL 32
+#define QED_MAX_PFC_PRIORITIES 8
+#define QED_DCBX_DSCP_SIZE 64
+
+struct qed_dcbx_lldp_remote {
+        u32 peer_chassis_id[QED_LLDP_CHASSIS_ID_STAT_LEN];
+        u32 peer_port_id[QED_LLDP_PORT_ID_STAT_LEN];
+        bool enable_rx;
+        bool enable_tx;
+        u32 tx_interval;
+        u32 max_credit;
+};
+
+struct qed_dcbx_lldp_local {
+        u32 local_chassis_id[QED_LLDP_CHASSIS_ID_STAT_LEN];
+        u32 local_port_id[QED_LLDP_PORT_ID_STAT_LEN];
+};
+
+struct qed_dcbx_app_prio {
+        u8 roce;
+        u8 roce_v2;
+        u8 fcoe;
+        u8 iscsi;
+        u8 eth;
+};
+
+struct qed_dbcx_pfc_params {
+        bool willing;
+        bool enabled;
+        u8 prio[QED_MAX_PFC_PRIORITIES];
+        u8 max_tc;
+};
+
+struct qed_app_entry {
+        bool ethtype;
+        bool enabled;
+        u8 prio;
+        u16 proto_id;
+        enum dcbx_protocol_type proto_type;
+};
+
+struct qed_dcbx_params {
+        struct qed_app_entry app_entry[QED_DCBX_MAX_APP_PROTOCOL];
+        u16 num_app_entries;
+        bool app_willing;
+        bool app_valid;
+        bool app_error;
+        bool ets_willing;
+        bool ets_enabled;
+        bool ets_cbs;
+        bool valid;
+        u8 ets_pri_tc_tbl[QED_MAX_PFC_PRIORITIES];
+        u8 ets_tc_bw_tbl[QED_MAX_PFC_PRIORITIES];
+        u8 ets_tc_tsa_tbl[QED_MAX_PFC_PRIORITIES];
+        struct qed_dbcx_pfc_params pfc;
+        u8 max_ets_tc;
+};
+
+struct qed_dcbx_admin_params {
+        struct qed_dcbx_params params;
+        bool valid;
+};
+
+struct qed_dcbx_remote_params {
+        struct qed_dcbx_params params;
+        bool valid;
+};
+
+struct qed_dcbx_operational_params {
+        struct qed_dcbx_app_prio app_prio;
+        struct qed_dcbx_params params;
+        bool valid;
+        bool enabled;
+        bool ieee;
+        bool cee;
+        u32 err;
+};
+
+struct qed_dcbx_get {
+        struct qed_dcbx_operational_params operational;
+        struct qed_dcbx_lldp_remote lldp_remote;
+        struct qed_dcbx_lldp_local lldp_local;
+        struct qed_dcbx_remote_params remote;
+        struct qed_dcbx_admin_params local;
+};
+#endif
+
 enum qed_led_mode {
         QED_LED_MODE_OFF,
         QED_LED_MODE_ON,
@@ -58,8 +148,70 @@ struct qed_eth_pf_params {
         u16 num_cons;
 };
 
+/* Most of the the parameters below are described in the FW iSCSI / TCP HSI */
+struct qed_iscsi_pf_params {
+        u64 glbl_q_params_addr;
+        u64 bdq_pbl_base_addr[2];
+        u32 max_cwnd;
+        u16 cq_num_entries;
+        u16 cmdq_num_entries;
+        u16 dup_ack_threshold;
+        u16 tx_sws_timer;
+        u16 min_rto;
+        u16 min_rto_rt;
+        u16 max_rto;
+
+        /* The following parameters are used during HW-init
+         * and these parameters need to be passed as arguments
+         * to update_pf_params routine invoked before slowpath start
+         */
+        u16 num_cons;
+        u16 num_tasks;
+
+        /* The following parameters are used during protocol-init */
+        u16 half_way_close_timeout;
+        u16 bdq_xoff_threshold[2];
+        u16 bdq_xon_threshold[2];
+        u16 cmdq_xoff_threshold;
+        u16 cmdq_xon_threshold;
+        u16 rq_buffer_size;
+
+        u8 num_sq_pages_in_ring;
+        u8 num_r2tq_pages_in_ring;
+        u8 num_uhq_pages_in_ring;
+        u8 num_queues;
+        u8 log_page_size;
+        u8 rqe_log_size;
+        u8 max_fin_rt;
+        u8 gl_rq_pi;
+        u8 gl_cmd_pi;
+        u8 debug_mode;
+        u8 ll2_ooo_queue_id;
+        u8 ooo_enable;
+
+        u8 is_target;
+        u8 bdq_pbl_num_entries[2];
+};
+
+struct qed_rdma_pf_params {
+        /* Supplied to QED during resource allocation (may affect the ILT and
+         * the doorbell BAR).
+         */
+        u32 min_dpis;           /* number of requested DPIs */
+        u32 num_mrs;            /* number of requested memory regions */
+        u32 num_qps;            /* number of requested Queue Pairs */
+        u32 num_srqs;           /* number of requested SRQ */
+        u8 roce_edpm_mode;      /* see QED_ROCE_EDPM_MODE_ENABLE */
+        u8 gl_pi;               /* protocol index */
+
+        /* Will allocate rate limiters to be used with QPs */
+        u8 enable_dcqcn;
+};
+
 struct qed_pf_params {
         struct qed_eth_pf_params eth_pf_params;
+        struct qed_iscsi_pf_params iscsi_pf_params;
+        struct qed_rdma_pf_params rdma_pf_params;
 };
 
 enum qed_int_mode {
@@ -100,6 +252,8 @@ struct qed_dev_info {
         /* MFW version */
         u32             mfw_rev;
 
+        bool rdma_supported;
+
         u32             flash_size;
         u8              mf_mode;
         bool            tx_switching;
@@ -111,6 +265,7 @@ enum qed_sb_type {
 
 enum qed_protocol {
         QED_PROTOCOL_ETH,
+        QED_PROTOCOL_ISCSI,
 };
 
 struct qed_link_params {
@@ -325,7 +480,8 @@ struct qed_common_ops {
         int             (*chain_alloc)(struct qed_dev *cdev,
                                        enum qed_chain_use_mode intended_use,
                                        enum qed_chain_mode mode,
-                                       u16 num_elems,
+                                       enum qed_chain_cnt_type cnt_type,
+                                       u32 num_elems,
                                        size_t elem_size,
                                        struct qed_chain *p_chain);
 
@@ -333,6 +489,30 @@ struct qed_common_ops {
                                       struct qed_chain *p_chain);
 
 /**
+ * @brief get_coalesce - Get coalesce parameters in usec
+ *
+ * @param cdev
+ * @param rx_coal - Rx coalesce value in usec
+ * @param tx_coal - Tx coalesce value in usec
+ *
+ */
+        void (*get_coalesce)(struct qed_dev *cdev, u16 *rx_coal, u16 *tx_coal);
+
+/**
+ * @brief set_coalesce - Configure Rx coalesce value in usec
+ *
+ * @param cdev
+ * @param rx_coal - Rx coalesce value in usec
+ * @param tx_coal - Tx coalesce value in usec
+ * @param qid - Queue index
+ * @param sb_id - Status Block Id
+ *
+ * @return 0 on success, error otherwise.
+ */
+        int (*set_coalesce)(struct qed_dev *cdev, u16 rx_coal, u16 tx_coal,
+                            u8 qid, u16 sb_id);
+
+/**
  * @brief set_led - Configure LED mode
  *
  * @param cdev
diff --git a/include/linux/qed/rdma_common.h b/include/linux/qed/rdma_common.h
new file mode 100644
index 000000000000..187991c1f439
--- /dev/null
+++ b/include/linux/qed/rdma_common.h
@@ -0,0 +1,44 @@
+/* QLogic qed NIC Driver
+ * Copyright (c) 2015 QLogic Corporation
+ *
+ * This software is available under the terms of the GNU General Public License
+ * (GPL) Version 2, available from the file COPYING in the main directory of
+ * this source tree.
+ */
+
+#ifndef __RDMA_COMMON__
+#define __RDMA_COMMON__
+/************************/
+/* RDMA FW CONSTANTS */
+/************************/
+
+#define RDMA_RESERVED_LKEY                      (0)
+#define RDMA_RING_PAGE_SIZE                     (0x1000)
+
+#define RDMA_MAX_SGE_PER_SQ_WQE         (4)
+#define RDMA_MAX_SGE_PER_RQ_WQE         (4)
+
+#define RDMA_MAX_DATA_SIZE_IN_WQE       (0x7FFFFFFF)
+
+#define RDMA_REQ_RD_ATOMIC_ELM_SIZE             (0x50)
+#define RDMA_RESP_RD_ATOMIC_ELM_SIZE    (0x20)
+
+#define RDMA_MAX_CQS                            (64 * 1024)
+#define RDMA_MAX_TIDS                           (128 * 1024 - 1)
+#define RDMA_MAX_PDS                            (64 * 1024)
+
+#define RDMA_NUM_STATISTIC_COUNTERS                     MAX_NUM_VPORTS
+
+#define RDMA_TASK_TYPE (PROTOCOLID_ROCE)
+
+struct rdma_srq_id {
+        __le16 srq_idx;
+        __le16 opaque_fid;
+};
+
+struct rdma_srq_producers {
+        __le32 sge_prod;
+        __le32 wqe_prod;
+};
+
+#endif /* __RDMA_COMMON__ */
diff --git a/include/linux/qed/roce_common.h b/include/linux/qed/roce_common.h
new file mode 100644
index 000000000000..2eeaf3dc6646
--- /dev/null
+++ b/include/linux/qed/roce_common.h
@@ -0,0 +1,17 @@
+/* QLogic qed NIC Driver
+ * Copyright (c) 2015 QLogic Corporation
+ *
+ * This software is available under the terms of the GNU General Public License
+ * (GPL) Version 2, available from the file COPYING in the main directory of
+ * this source tree.
+ */
+
+#ifndef __ROCE_COMMON__
+#define __ROCE_COMMON__
+
+#define ROCE_REQ_MAX_INLINE_DATA_SIZE (256)
+#define ROCE_REQ_MAX_SINGLE_SQ_WQE_SIZE (288)
+
+#define ROCE_MAX_QPS    (32 * 1024)
+
+#endif /* __ROCE_COMMON__ */
diff --git a/include/linux/qed/storage_common.h b/include/linux/qed/storage_common.h
new file mode 100644
index 000000000000..3b8e1efd9bc2
--- /dev/null
+++ b/include/linux/qed/storage_common.h
@@ -0,0 +1,91 @@
+/* QLogic qed NIC Driver
+ * Copyright (c) 2015 QLogic Corporation
+ *
+ * This software is available under the terms of the GNU General Public License
+ * (GPL) Version 2, available from the file COPYING in the main directory of
+ * this source tree.
+ */
+
+#ifndef __STORAGE_COMMON__
+#define __STORAGE_COMMON__
+
+#define NUM_OF_CMDQS_CQS (NUM_OF_GLOBAL_QUEUES / 2)
+#define BDQ_NUM_RESOURCES (4)
+
+#define BDQ_ID_RQ                        (0)
+#define BDQ_ID_IMM_DATA          (1)
+#define BDQ_NUM_IDS          (2)
+
+#define BDQ_MAX_EXTERNAL_RING_SIZE (1 << 15)
+
+struct scsi_bd {
+        struct regpair address;
+        struct regpair opaque;
+};
+
+struct scsi_bdq_ram_drv_data {
+        __le16 external_producer;
+        __le16 reserved0[3];
+};
+
+struct scsi_drv_cmdq {
+        __le16 cmdq_cons;
+        __le16 reserved0;
+        __le32 reserved1;
+};
+
+struct scsi_init_func_params {
+        __le16 num_tasks;
+        u8 log_page_size;
+        u8 debug_mode;
+        u8 reserved2[12];
+};
+
+struct scsi_init_func_queues {
+        struct regpair glbl_q_params_addr;
+        __le16 rq_buffer_size;
+        __le16 cq_num_entries;
+        __le16 cmdq_num_entries;
+        u8 bdq_resource_id;
+        u8 q_validity;
+#define SCSI_INIT_FUNC_QUEUES_RQ_VALID_MASK        0x1
+#define SCSI_INIT_FUNC_QUEUES_RQ_VALID_SHIFT       0
+#define SCSI_INIT_FUNC_QUEUES_IMM_DATA_VALID_MASK  0x1
+#define SCSI_INIT_FUNC_QUEUES_IMM_DATA_VALID_SHIFT 1
+#define SCSI_INIT_FUNC_QUEUES_CMD_VALID_MASK       0x1
+#define SCSI_INIT_FUNC_QUEUES_CMD_VALID_SHIFT      2
+#define SCSI_INIT_FUNC_QUEUES_RESERVED_VALID_MASK  0x1F
+#define SCSI_INIT_FUNC_QUEUES_RESERVED_VALID_SHIFT 3
+        u8 num_queues;
+        u8 queue_relative_offset;
+        u8 cq_sb_pi;
+        u8 cmdq_sb_pi;
+        __le16 cq_cmdq_sb_num_arr[NUM_OF_CMDQS_CQS];
+        __le16 reserved0;
+        u8 bdq_pbl_num_entries[BDQ_NUM_IDS];
+        struct regpair bdq_pbl_base_address[BDQ_NUM_IDS];
+        __le16 bdq_xoff_threshold[BDQ_NUM_IDS];
+        __le16 bdq_xon_threshold[BDQ_NUM_IDS];
+        __le16 cmdq_xoff_threshold;
+        __le16 cmdq_xon_threshold;
+        __le32 reserved1;
+};
+
+struct scsi_ram_per_bdq_resource_drv_data {
+        struct scsi_bdq_ram_drv_data drv_data_per_bdq_id[BDQ_NUM_IDS];
+};
+
+struct scsi_sge {
+        struct regpair sge_addr;
+        __le16 sge_len;
+        __le16 reserved0;
+        __le32 reserved1;
+};
+
+struct scsi_terminate_extra_params {
+        __le16 unsolicited_cq_count;
+        __le16 cmdq_count;
+        u8 reserved[4];
+};
+
+#endif /* __STORAGE_COMMON__ */
diff --git a/include/linux/qed/tcp_common.h b/include/linux/qed/tcp_common.h
new file mode 100644
index 000000000000..accba0e6b704
--- /dev/null
+++ b/include/linux/qed/tcp_common.h
@@ -0,0 +1,226 @@
+/* QLogic qed NIC Driver
+ * Copyright (c) 2015 QLogic Corporation
+ *
+ * This software is available under the terms of the GNU General Public License
+ * (GPL) Version 2, available from the file COPYING in the main directory of
+ * this source tree.
+ */
+
+#ifndef __TCP_COMMON__
+#define __TCP_COMMON__
+
+#define TCP_INVALID_TIMEOUT_VAL -1
+
+enum tcp_connect_mode {
+        TCP_CONNECT_ACTIVE,
+        TCP_CONNECT_PASSIVE,
+        MAX_TCP_CONNECT_MODE
+};
+
+struct tcp_init_params {
+        __le32 max_cwnd;
+        __le16 dup_ack_threshold;
+        __le16 tx_sws_timer;
+        __le16 min_rto;
+        __le16 min_rto_rt;
+        __le16 max_rto;
+        u8 maxfinrt;
+        u8 reserved[1];
+};
+
+enum tcp_ip_version {
+        TCP_IPV4,
+        TCP_IPV6,
+        MAX_TCP_IP_VERSION
+};
+
+struct tcp_offload_params {
+        __le16 local_mac_addr_lo;
+        __le16 local_mac_addr_mid;
+        __le16 local_mac_addr_hi;
+        __le16 remote_mac_addr_lo;
+        __le16 remote_mac_addr_mid;
+        __le16 remote_mac_addr_hi;
+        __le16 vlan_id;
+        u8 flags;
+#define TCP_OFFLOAD_PARAMS_TS_EN_MASK         0x1
+#define TCP_OFFLOAD_PARAMS_TS_EN_SHIFT        0
+#define TCP_OFFLOAD_PARAMS_DA_EN_MASK         0x1
+#define TCP_OFFLOAD_PARAMS_DA_EN_SHIFT        1
+#define TCP_OFFLOAD_PARAMS_KA_EN_MASK         0x1
+#define TCP_OFFLOAD_PARAMS_KA_EN_SHIFT        2
+#define TCP_OFFLOAD_PARAMS_NAGLE_EN_MASK      0x1
+#define TCP_OFFLOAD_PARAMS_NAGLE_EN_SHIFT     3
+#define TCP_OFFLOAD_PARAMS_DA_CNT_EN_MASK     0x1
+#define TCP_OFFLOAD_PARAMS_DA_CNT_EN_SHIFT    4
+#define TCP_OFFLOAD_PARAMS_FIN_SENT_MASK      0x1
+#define TCP_OFFLOAD_PARAMS_FIN_SENT_SHIFT     5
+#define TCP_OFFLOAD_PARAMS_FIN_RECEIVED_MASK  0x1
+#define TCP_OFFLOAD_PARAMS_FIN_RECEIVED_SHIFT 6
+#define TCP_OFFLOAD_PARAMS_RESERVED0_MASK     0x1
+#define TCP_OFFLOAD_PARAMS_RESERVED0_SHIFT    7
+        u8 ip_version;
+        __le32 remote_ip[4];
+        __le32 local_ip[4];
+        __le32 flow_label;
+        u8 ttl;
+        u8 tos_or_tc;
+        __le16 remote_port;
+        __le16 local_port;
+        __le16 mss;
+        u8 rcv_wnd_scale;
+        u8 connect_mode;
+        __le16 srtt;
+        __le32 cwnd;
+        __le32 ss_thresh;
+        __le16 reserved1;
+        u8 ka_max_probe_cnt;
+        u8 dup_ack_theshold;
+        __le32 rcv_next;
+        __le32 snd_una;
+        __le32 snd_next;
+        __le32 snd_max;
+        __le32 snd_wnd;
+        __le32 rcv_wnd;
+        __le32 snd_wl1;
+        __le32 ts_time;
+        __le32 ts_recent;
+        __le32 ts_recent_age;
+        __le32 total_rt;
+        __le32 ka_timeout_delta;
+        __le32 rt_timeout_delta;
+        u8 dup_ack_cnt;
+        u8 snd_wnd_probe_cnt;
+        u8 ka_probe_cnt;
+        u8 rt_cnt;
+        __le16 rtt_var;
+        __le16 reserved2;
+        __le32 ka_timeout;
+        __le32 ka_interval;
+        __le32 max_rt_time;
+        __le32 initial_rcv_wnd;
+        u8 snd_wnd_scale;
+        u8 ack_frequency;
+        __le16 da_timeout_value;
+        __le32 ts_ticks_per_second;
+};
+
+struct tcp_offload_params_opt2 {
+        __le16 local_mac_addr_lo;
+        __le16 local_mac_addr_mid;
+        __le16 local_mac_addr_hi;
+        __le16 remote_mac_addr_lo;
+        __le16 remote_mac_addr_mid;
+        __le16 remote_mac_addr_hi;
+        __le16 vlan_id;
+        u8 flags;
+#define TCP_OFFLOAD_PARAMS_OPT2_TS_EN_MASK      0x1
+#define TCP_OFFLOAD_PARAMS_OPT2_TS_EN_SHIFT     0
+#define TCP_OFFLOAD_PARAMS_OPT2_DA_EN_MASK      0x1
+#define TCP_OFFLOAD_PARAMS_OPT2_DA_EN_SHIFT     1
+#define TCP_OFFLOAD_PARAMS_OPT2_KA_EN_MASK      0x1
+#define TCP_OFFLOAD_PARAMS_OPT2_KA_EN_SHIFT     2
+#define TCP_OFFLOAD_PARAMS_OPT2_RESERVED0_MASK  0x1F
+#define TCP_OFFLOAD_PARAMS_OPT2_RESERVED0_SHIFT 3
+        u8 ip_version;
+        __le32 remote_ip[4];
+        __le32 local_ip[4];
+        __le32 flow_label;
+        u8 ttl;
+        u8 tos_or_tc;
+        __le16 remote_port;
+        __le16 local_port;
+        __le16 mss;
+        u8 rcv_wnd_scale;
+        u8 connect_mode;
+        __le16 syn_ip_payload_length;
+        __le32 syn_phy_addr_lo;
+        __le32 syn_phy_addr_hi;
+        __le32 reserved1[22];
+};
+
+enum tcp_seg_placement_event {
+        TCP_EVENT_ADD_PEN,
+        TCP_EVENT_ADD_NEW_ISLE,
+        TCP_EVENT_ADD_ISLE_RIGHT,
+        TCP_EVENT_ADD_ISLE_LEFT,
+        TCP_EVENT_JOIN,
+        TCP_EVENT_NOP,
+        MAX_TCP_SEG_PLACEMENT_EVENT
+};
+
+struct tcp_update_params {
+        __le16 flags;
+#define TCP_UPDATE_PARAMS_REMOTE_MAC_ADDR_CHANGED_MASK   0x1
+#define TCP_UPDATE_PARAMS_REMOTE_MAC_ADDR_CHANGED_SHIFT  0
+#define TCP_UPDATE_PARAMS_MSS_CHANGED_MASK               0x1
+#define TCP_UPDATE_PARAMS_MSS_CHANGED_SHIFT              1
+#define TCP_UPDATE_PARAMS_TTL_CHANGED_MASK               0x1
+#define TCP_UPDATE_PARAMS_TTL_CHANGED_SHIFT              2
+#define TCP_UPDATE_PARAMS_TOS_OR_TC_CHANGED_MASK         0x1
+#define TCP_UPDATE_PARAMS_TOS_OR_TC_CHANGED_SHIFT        3
+#define TCP_UPDATE_PARAMS_KA_TIMEOUT_CHANGED_MASK        0x1
+#define TCP_UPDATE_PARAMS_KA_TIMEOUT_CHANGED_SHIFT       4
+#define TCP_UPDATE_PARAMS_KA_INTERVAL_CHANGED_MASK       0x1
+#define TCP_UPDATE_PARAMS_KA_INTERVAL_CHANGED_SHIFT      5
+#define TCP_UPDATE_PARAMS_MAX_RT_TIME_CHANGED_MASK       0x1
+#define TCP_UPDATE_PARAMS_MAX_RT_TIME_CHANGED_SHIFT      6
+#define TCP_UPDATE_PARAMS_FLOW_LABEL_CHANGED_MASK        0x1
+#define TCP_UPDATE_PARAMS_FLOW_LABEL_CHANGED_SHIFT       7
+#define TCP_UPDATE_PARAMS_INITIAL_RCV_WND_CHANGED_MASK   0x1
+#define TCP_UPDATE_PARAMS_INITIAL_RCV_WND_CHANGED_SHIFT  8
+#define TCP_UPDATE_PARAMS_KA_MAX_PROBE_CNT_CHANGED_MASK  0x1
+#define TCP_UPDATE_PARAMS_KA_MAX_PROBE_CNT_CHANGED_SHIFT 9
+#define TCP_UPDATE_PARAMS_KA_EN_CHANGED_MASK             0x1
+#define TCP_UPDATE_PARAMS_KA_EN_CHANGED_SHIFT            10
+#define TCP_UPDATE_PARAMS_NAGLE_EN_CHANGED_MASK          0x1
+#define TCP_UPDATE_PARAMS_NAGLE_EN_CHANGED_SHIFT         11
+#define TCP_UPDATE_PARAMS_KA_EN_MASK                     0x1
+#define TCP_UPDATE_PARAMS_KA_EN_SHIFT                    12
+#define TCP_UPDATE_PARAMS_NAGLE_EN_MASK                  0x1
+#define TCP_UPDATE_PARAMS_NAGLE_EN_SHIFT                 13
+#define TCP_UPDATE_PARAMS_KA_RESTART_MASK                0x1
+#define TCP_UPDATE_PARAMS_KA_RESTART_SHIFT               14
+#define TCP_UPDATE_PARAMS_RETRANSMIT_RESTART_MASK        0x1
+#define TCP_UPDATE_PARAMS_RETRANSMIT_RESTART_SHIFT       15
+        __le16 remote_mac_addr_lo;
+        __le16 remote_mac_addr_mid;
+        __le16 remote_mac_addr_hi;
+        __le16 mss;
+        u8 ttl;
+        u8 tos_or_tc;
+        __le32 ka_timeout;
+        __le32 ka_interval;
+        __le32 max_rt_time;
+        __le32 flow_label;
+        __le32 initial_rcv_wnd;
+        u8 ka_max_probe_cnt;
+        u8 reserved1[7];
+};
+
+struct tcp_upload_params {
+        __le32 rcv_next;
+        __le32 snd_una;
+        __le32 snd_next;
+        __le32 snd_max;
+        __le32 snd_wnd;
+        __le32 rcv_wnd;
+        __le32 snd_wl1;
+        __le32 cwnd;
+        __le32 ss_thresh;
+        __le16 srtt;
+        __le16 rtt_var;
+        __le32 ts_time;
+        __le32 ts_recent;
+        __le32 ts_recent_age;
+        __le32 total_rt;
+        __le32 ka_timeout_delta;
+        __le32 rt_timeout_delta;
+        u8 dup_ack_cnt;
+        u8 snd_wnd_probe_cnt;
+        u8 ka_probe_cnt;
+        u8 rt_cnt;
+        __le32 reserved;
+};
+
+#endif /* __TCP_COMMON__ */
diff --git a/include/linux/rtnetlink.h b/include/linux/rtnetlink.h
index c006cc900c44..2daece8979f7 100644
--- a/include/linux/rtnetlink.h
+++ b/include/linux/rtnetlink.h
@@ -89,8 +89,9 @@ void net_inc_egress_queue(void);
 void net_dec_egress_queue(void);
 #endif
 
-extern void rtnetlink_init(void);
-extern void __rtnl_unlock(void);
+void rtnetlink_init(void);
+void __rtnl_unlock(void);
+void rtnl_kfree_skbs(struct sk_buff *head, struct sk_buff *tail);
 
 #define ASSERT_RTNL() do { \
         if (unlikely(!rtnl_is_locked())) { \
diff --git a/include/linux/rxrpc.h b/include/linux/rxrpc.h
index a53915cd5581..c68307bc306f 100644
--- a/include/linux/rxrpc.h
+++ b/include/linux/rxrpc.h
@@ -35,21 +35,24 @@ struct sockaddr_rxrpc {
  */
 #define RXRPC_SECURITY_KEY              1       /* [clnt] set client security key */
 #define RXRPC_SECURITY_KEYRING          2       /* [srvr] set ring of server security keys */
-#define RXRPC_EXCLUSIVE_CONNECTION      3       /* [clnt] use exclusive RxRPC connection */
+#define RXRPC_EXCLUSIVE_CONNECTION      3       /* Deprecated; use RXRPC_EXCLUSIVE_CALL instead */
 #define RXRPC_MIN_SECURITY_LEVEL        4       /* minimum security level */
 
 /*
  * RxRPC control messages
+ * - If neither abort or accept are specified, the message is a data message.
  * - terminal messages mean that a user call ID tag can be recycled
+ * - s/r/- indicate whether these are applicable to sendmsg() and/or recvmsg()
  */
-#define RXRPC_USER_CALL_ID      1       /* user call ID specifier */
-#define RXRPC_ABORT             2       /* abort request / notification [terminal] */
-#define RXRPC_ACK               3       /* [Server] RPC op final ACK received [terminal] */
-#define RXRPC_NET_ERROR         5       /* network error received [terminal] */
-#define RXRPC_BUSY              6       /* server busy received [terminal] */
-#define RXRPC_LOCAL_ERROR       7       /* local error generated [terminal] */
-#define RXRPC_NEW_CALL          8       /* [Server] new incoming call notification */
-#define RXRPC_ACCEPT            9       /* [Server] accept request */
+#define RXRPC_USER_CALL_ID      1       /* sr: user call ID specifier */
+#define RXRPC_ABORT             2       /* sr: abort request / notification [terminal] */
+#define RXRPC_ACK               3       /* -r: [Service] RPC op final ACK received [terminal] */
+#define RXRPC_NET_ERROR         5       /* -r: network error received [terminal] */
+#define RXRPC_BUSY              6       /* -r: server busy received [terminal] */
+#define RXRPC_LOCAL_ERROR       7       /* -r: local error generated [terminal] */
+#define RXRPC_NEW_CALL          8       /* -r: [Service] new incoming call notification */
+#define RXRPC_ACCEPT            9       /* s-: [Service] accept request */
+#define RXRPC_EXCLUSIVE_CALL    10      /* s-: Call should be on exclusive connection */
 
 /*
  * RxRPC security levels
diff --git a/include/linux/skb_array.h b/include/linux/skb_array.h
new file mode 100644
index 000000000000..678bfbf78ac4
--- /dev/null
+++ b/include/linux/skb_array.h
@@ -0,0 +1,169 @@
+/*
+ *      Definitions for the 'struct skb_array' datastructure.
+ *
+ *      Author:
+ *              Michael S. Tsirkin <mst@redhat.com>
+ *
+ *      Copyright (C) 2016 Red Hat, Inc.
+ *
+ *      This program is free software; you can redistribute it and/or modify it
+ *      under the terms of the GNU General Public License as published by the
+ *      Free Software Foundation; either version 2 of the License, or (at your
+ *      option) any later version.
+ *
+ *      Limited-size FIFO of skbs. Can be used more or less whenever
+ *      sk_buff_head can be used, except you need to know the queue size in
+ *      advance.
+ *      Implemented as a type-safe wrapper around ptr_ring.
+ */
+
+#ifndef _LINUX_SKB_ARRAY_H
+#define _LINUX_SKB_ARRAY_H 1
+
+#ifdef __KERNEL__
+#include <linux/ptr_ring.h>
+#include <linux/skbuff.h>
+#include <linux/if_vlan.h>
+#endif
+
+struct skb_array {
+        struct ptr_ring ring;
+};
+
+/* Might be slightly faster than skb_array_full below, but callers invoking
+ * this in a loop must use a compiler barrier, for example cpu_relax().
+ */
+static inline bool __skb_array_full(struct skb_array *a)
+{
+        return __ptr_ring_full(&a->ring);
+}
+
+static inline bool skb_array_full(struct skb_array *a)
+{
+        return ptr_ring_full(&a->ring);
+}
+
+static inline int skb_array_produce(struct skb_array *a, struct sk_buff *skb)
+{
+        return ptr_ring_produce(&a->ring, skb);
+}
+
+static inline int skb_array_produce_irq(struct skb_array *a, struct sk_buff *skb)
+{
+        return ptr_ring_produce_irq(&a->ring, skb);
+}
+
+static inline int skb_array_produce_bh(struct skb_array *a, struct sk_buff *skb)
+{
+        return ptr_ring_produce_bh(&a->ring, skb);
+}
+
+static inline int skb_array_produce_any(struct skb_array *a, struct sk_buff *skb)
+{
+        return ptr_ring_produce_any(&a->ring, skb);
+}
+
+/* Might be slightly faster than skb_array_empty below, but only safe if the
+ * array is never resized. Also, callers invoking this in a loop must take care
+ * to use a compiler barrier, for example cpu_relax().
+ */
+static inline bool __skb_array_empty(struct skb_array *a)
+{
+        return !__ptr_ring_peek(&a->ring);
+}
+
+static inline bool skb_array_empty(struct skb_array *a)
+{
+        return ptr_ring_empty(&a->ring);
+}
+
+static inline bool skb_array_empty_bh(struct skb_array *a)
+{
+        return ptr_ring_empty_bh(&a->ring);
+}
+
+static inline bool skb_array_empty_irq(struct skb_array *a)
+{
+        return ptr_ring_empty_irq(&a->ring);
+}
+
+static inline bool skb_array_empty_any(struct skb_array *a)
+{
+        return ptr_ring_empty_any(&a->ring);
+}
+
+static inline struct sk_buff *skb_array_consume(struct skb_array *a)
+{
+        return ptr_ring_consume(&a->ring);
+}
+
+static inline struct sk_buff *skb_array_consume_irq(struct skb_array *a)
+{
+        return ptr_ring_consume_irq(&a->ring);
+}
+
+static inline struct sk_buff *skb_array_consume_any(struct skb_array *a)
+{
+        return ptr_ring_consume_any(&a->ring);
+}
+
+static inline struct sk_buff *skb_array_consume_bh(struct skb_array *a)
+{
+        return ptr_ring_consume_bh(&a->ring);
+}
+
+static inline int __skb_array_len_with_tag(struct sk_buff *skb)
+{
+        if (likely(skb)) {
+                int len = skb->len;
+
+                if (skb_vlan_tag_present(skb))
+                        len += VLAN_HLEN;
+
+                return len;
+        } else {
+                return 0;
+        }
+}
+
+static inline int skb_array_peek_len(struct skb_array *a)
+{
+        return PTR_RING_PEEK_CALL(&a->ring, __skb_array_len_with_tag);
+}
+
+static inline int skb_array_peek_len_irq(struct skb_array *a)
+{
+        return PTR_RING_PEEK_CALL_IRQ(&a->ring, __skb_array_len_with_tag);
+}
+
+static inline int skb_array_peek_len_bh(struct skb_array *a)
+{
+        return PTR_RING_PEEK_CALL_BH(&a->ring, __skb_array_len_with_tag);
+}
+
+static inline int skb_array_peek_len_any(struct skb_array *a)
+{
+        return PTR_RING_PEEK_CALL_ANY(&a->ring, __skb_array_len_with_tag);
+}
+
+static inline int skb_array_init(struct skb_array *a, int size, gfp_t gfp)
+{
+        return ptr_ring_init(&a->ring, size, gfp);
+}
+
+void __skb_array_destroy_skb(void *ptr)
+{
+        kfree_skb(ptr);
+}
+
+int skb_array_resize(struct skb_array *a, int size, gfp_t gfp)
+{
+        return ptr_ring_resize(&a->ring, size, gfp, __skb_array_destroy_skb);
+}
+
+static inline void skb_array_cleanup(struct skb_array *a)
+{
+        ptr_ring_cleanup(&a->ring, __skb_array_destroy_skb);
+}
+
+#endif /* _LINUX_SKB_ARRAY_H  */
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index ee38a4127475..dc0fca747c5e 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -301,6 +301,11 @@ struct sk_buff;
 #endif
 extern int sysctl_max_skb_frags;
 
+/* Set skb_shinfo(skb)->gso_size to this in case you want skb_segment to
+ * segment using its current segmentation instead.
+ */
+#define GSO_BY_FRAGS    0xFFFF
+
 typedef struct skb_frag_struct skb_frag_t;
 
 struct skb_frag_struct {
@@ -482,6 +487,8 @@ enum {
         SKB_GSO_PARTIAL = 1 << 13,
 
         SKB_GSO_TUNNEL_REMCSUM = 1 << 14,
+
+        SKB_GSO_SCTP = 1 << 15,
 };
 
 #if BITS_PER_LONG > 32
@@ -2987,6 +2994,7 @@ void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len);
 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen);
 void skb_scrub_packet(struct sk_buff *skb, bool xnet);
 unsigned int skb_gso_transport_seglen(const struct sk_buff *skb);
+bool skb_gso_validate_mtu(const struct sk_buff *skb, unsigned int mtu);
 struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features);
 struct sk_buff *skb_vlan_untag(struct sk_buff *skb);
 int skb_ensure_writable(struct sk_buff *skb, int write_len);
diff --git a/include/linux/stmmac.h b/include/linux/stmmac.h
index ffdaca9c01af..705840e0438f 100644
--- a/include/linux/stmmac.h
+++ b/include/linux/stmmac.h
@@ -135,9 +135,12 @@ struct plat_stmmacenet_data {
         void (*bus_setup)(void __iomem *ioaddr);
         int (*init)(struct platform_device *pdev, void *priv);
         void (*exit)(struct platform_device *pdev, void *priv);
+        void (*suspend)(struct platform_device *pdev, void *priv);
+        void (*resume)(struct platform_device *pdev, void *priv);
         void *bsp_priv;
         struct stmmac_axi *axi;
         int has_gmac4;
         bool tso_en;
+        int mac_port_sel_speed;
 };
 #endif
diff --git a/include/linux/virtio_net.h b/include/linux/virtio_net.h
new file mode 100644
index 000000000000..1c912f85e041
--- /dev/null
+++ b/include/linux/virtio_net.h
@@ -0,0 +1,101 @@
+#ifndef _LINUX_VIRTIO_NET_H
+#define _LINUX_VIRTIO_NET_H
+
+#include <linux/if_vlan.h>
+#include <uapi/linux/virtio_net.h>
+
+static inline int virtio_net_hdr_to_skb(struct sk_buff *skb,
+                                        const struct virtio_net_hdr *hdr,
+                                        bool little_endian)
+{
+        unsigned short gso_type = 0;
+
+        if (hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
+                switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
+                case VIRTIO_NET_HDR_GSO_TCPV4:
+                        gso_type = SKB_GSO_TCPV4;
+                        break;
+                case VIRTIO_NET_HDR_GSO_TCPV6:
+                        gso_type = SKB_GSO_TCPV6;
+                        break;
+                case VIRTIO_NET_HDR_GSO_UDP:
+                        gso_type = SKB_GSO_UDP;
+                        break;
+                default:
+                        return -EINVAL;
+                }
+
+                if (hdr->gso_type & VIRTIO_NET_HDR_GSO_ECN)
+                        gso_type |= SKB_GSO_TCP_ECN;
+
+                if (hdr->gso_size == 0)
+                        return -EINVAL;
+        }
+
+        if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
+                u16 start = __virtio16_to_cpu(little_endian, hdr->csum_start);
+                u16 off = __virtio16_to_cpu(little_endian, hdr->csum_offset);
+
+                if (!skb_partial_csum_set(skb, start, off))
+                        return -EINVAL;
+        }
+
+        if (hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
+                u16 gso_size = __virtio16_to_cpu(little_endian, hdr->gso_size);
+
+                skb_shinfo(skb)->gso_size = gso_size;
+                skb_shinfo(skb)->gso_type = gso_type;
+
+                /* Header must be checked, and gso_segs computed. */
+                skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
+                skb_shinfo(skb)->gso_segs = 0;
+        }
+
+        return 0;
+}
+
+static inline int virtio_net_hdr_from_skb(const struct sk_buff *skb,
+                                          struct virtio_net_hdr *hdr,
+                                          bool little_endian)
+{
+        memset(hdr, 0, sizeof(*hdr));
+
+        if (skb_is_gso(skb)) {
+                struct skb_shared_info *sinfo = skb_shinfo(skb);
+
+                /* This is a hint as to how much should be linear. */
+                hdr->hdr_len = __cpu_to_virtio16(little_endian,
+                                                 skb_headlen(skb));
+                hdr->gso_size = __cpu_to_virtio16(little_endian,
+                                                  sinfo->gso_size);
+                if (sinfo->gso_type & SKB_GSO_TCPV4)
+                        hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
+                else if (sinfo->gso_type & SKB_GSO_TCPV6)
+                        hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
+                else if (sinfo->gso_type & SKB_GSO_UDP)
+                        hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP;
+                else
+                        return -EINVAL;
+                if (sinfo->gso_type & SKB_GSO_TCP_ECN)
+                        hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN;
+        } else
+                hdr->gso_type = VIRTIO_NET_HDR_GSO_NONE;
+
+        if (skb->ip_summed == CHECKSUM_PARTIAL) {
+                hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
+                if (skb_vlan_tag_present(skb))
+                        hdr->csum_start = __cpu_to_virtio16(little_endian,
+                                skb_checksum_start_offset(skb) + VLAN_HLEN);
+                else
+                        hdr->csum_start = __cpu_to_virtio16(little_endian,
+                                skb_checksum_start_offset(skb));
+                hdr->csum_offset = __cpu_to_virtio16(little_endian,
+                                skb->csum_offset);
+        } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
+                hdr->flags = VIRTIO_NET_HDR_F_DATA_VALID;
+        } /* else everything is zero */
+
+        return 0;
+}
+
+#endif /* _LINUX_VIRTIO_BYTEORDER */
diff --git a/include/net/6lowpan.h b/include/net/6lowpan.h
index da84cf920b78..5ab4c9901ccc 100644
--- a/include/net/6lowpan.h
+++ b/include/net/6lowpan.h
@@ -141,6 +141,16 @@ struct lowpan_dev {
         u8 priv[0] __aligned(sizeof(void *));
 };
 
+struct lowpan_802154_neigh {
+        __le16 short_addr;
+};
+
+static inline
+struct lowpan_802154_neigh *lowpan_802154_neigh(void *neigh_priv)
+{
+        return neigh_priv;
+}
+
 static inline
 struct lowpan_dev *lowpan_dev(const struct net_device *dev)
 {
@@ -244,6 +254,12 @@ static inline bool lowpan_fetch_skb(struct sk_buff *skb, void *data,
         return false;
 }
 
+static inline bool lowpan_802154_is_valid_src_short_addr(__le16 addr)
+{
+        /* First bit of addr is multicast, reserved or 802.15.4 specific */
+        return !(addr & cpu_to_le16(0x8000));
+}
+
 static inline void lowpan_push_hc_data(u8 **hc_ptr, const void *data,
                                        const size_t len)
 {
diff --git a/include/net/act_api.h b/include/net/act_api.h
index 9a9a8edc138f..fb82b5b5d9e7 100644
--- a/include/net/act_api.h
+++ b/include/net/act_api.h
@@ -2,8 +2,8 @@
 #define __NET_ACT_API_H
 
 /*
- * Public police action API for classifiers/qdiscs
- */
+ * Public action API for classifiers/qdiscs
+*/
 
 #include <net/sch_generic.h>
 #include <net/pkt_sched.h>
@@ -76,6 +76,16 @@ static inline void tcf_lastuse_update(struct tcf_t *tm)
 
         if (tm->lastuse != now)
                 tm->lastuse = now;
+        if (unlikely(!tm->firstuse))
+                tm->firstuse = now;
+}
+
+static inline void tcf_tm_dump(struct tcf_t *dtm, const struct tcf_t *stm)
+{
+        dtm->install = jiffies_to_clock_t(jiffies - stm->install);
+        dtm->lastuse = jiffies_to_clock_t(jiffies - stm->lastuse);
+        dtm->firstuse = jiffies_to_clock_t(jiffies - stm->firstuse);
+        dtm->expires = jiffies_to_clock_t(stm->expires);
 }
 
 struct tc_action {
@@ -97,7 +107,8 @@ struct tc_action_ops {
         char    kind[IFNAMSIZ];
         __u32   type; /* TBD to match kind */
         struct module           *owner;
-        int     (*act)(struct sk_buff *, const struct tc_action *, struct tcf_result *);
+        int     (*act)(struct sk_buff *, const struct tc_action *,
+                       struct tcf_result *);
         int     (*dump)(struct sk_buff *, struct tc_action *, int, int);
         void    (*cleanup)(struct tc_action *, int bind);
         int     (*lookup)(struct net *, struct tc_action *, u32);
@@ -115,8 +126,8 @@ struct tc_action_net {
 };
 
 static inline
-int tc_action_net_init(struct tc_action_net *tn, const struct tc_action_ops *ops,
-                       unsigned int mask)
+int tc_action_net_init(struct tc_action_net *tn,
+                       const struct tc_action_ops *ops, unsigned int mask)
 {
         int err = 0;
 
@@ -144,8 +155,8 @@ int tcf_generic_walker(struct tc_action_net *tn, struct sk_buff *skb,
                        struct tc_action *a);
 int tcf_hash_search(struct tc_action_net *tn, struct tc_action *a, u32 index);
 u32 tcf_hash_new_index(struct tc_action_net *tn);
-int tcf_hash_check(struct tc_action_net *tn, u32 index, struct tc_action *a,
-                   int bind);
+bool tcf_hash_check(struct tc_action_net *tn, u32 index, struct tc_action *a,
+                    int bind);
 int tcf_hash_create(struct tc_action_net *tn, u32 index, struct nlattr *est,
                     struct tc_action *a, int size, int bind, bool cpustats);
 void tcf_hash_cleanup(struct tc_action *a, struct nlattr *est);
@@ -159,7 +170,8 @@ static inline int tcf_hash_release(struct tc_action *a, bool bind)
 }
 
 int tcf_register_action(struct tc_action_ops *a, struct pernet_operations *ops);
-int tcf_unregister_action(struct tc_action_ops *a, struct pernet_operations *ops);
+int tcf_unregister_action(struct tc_action_ops *a,
+                          struct pernet_operations *ops);
 int tcf_action_destroy(struct list_head *actions, int bind);
 int tcf_action_exec(struct sk_buff *skb, const struct list_head *actions,
                     struct tcf_result *res);
diff --git a/include/net/addrconf.h b/include/net/addrconf.h
index 730d856683e5..9826d3a9464c 100644
--- a/include/net/addrconf.h
+++ b/include/net/addrconf.h
@@ -94,6 +94,16 @@ int ipv6_rcv_saddr_equal(const struct sock *sk, const struct sock *sk2,
 void addrconf_join_solict(struct net_device *dev, const struct in6_addr *addr);
 void addrconf_leave_solict(struct inet6_dev *idev, const struct in6_addr *addr);
 
+void addrconf_add_linklocal(struct inet6_dev *idev,
+                            const struct in6_addr *addr, u32 flags);
+
+int addrconf_prefix_rcv_add_addr(struct net *net, struct net_device *dev,
+                                 const struct prefix_info *pinfo,
+                                 struct inet6_dev *in6_dev,
+                                 const struct in6_addr *addr, int addr_type,
+                                 u32 addr_flags, bool sllao, bool tokenized,
+                                 __u32 valid_lft, u32 prefered_lft);
+
 static inline int addrconf_ifid_eui48(u8 *eui, struct net_device *dev)
 {
         if (dev->addr_len != ETH_ALEN)
diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h
index 63921672bed0..7bbb00d8b2cd 100644
--- a/include/net/cfg80211.h
+++ b/include/net/cfg80211.h
@@ -2367,19 +2367,23 @@ struct cfg80211_qos_map {
  *      (invoked with the wireless_dev mutex held)
  *
  * @connect: Connect to the ESS with the specified parameters. When connected,
- *      call cfg80211_connect_result() with status code %WLAN_STATUS_SUCCESS.
- *      If the connection fails for some reason, call cfg80211_connect_result()
- *      with the status from the AP. The driver is allowed to roam to other
- *      BSSes within the ESS when the other BSS matches the connect parameters.
- *      When such roaming is initiated by the driver, the driver is expected to
- *      verify that the target matches the configured security parameters and
- *      to use Reassociation Request frame instead of Association Request frame.
- *      The connect function can also be used to request the driver to perform
- *      a specific roam when connected to an ESS. In that case, the prev_bssid
+ *      call cfg80211_connect_result()/cfg80211_connect_bss() with status code
+ *      %WLAN_STATUS_SUCCESS. If the connection fails for some reason, call
+ *      cfg80211_connect_result()/cfg80211_connect_bss() with the status code
+ *      from the AP or cfg80211_connect_timeout() if no frame with status code
+ *      was received.
+ *      The driver is allowed to roam to other BSSes within the ESS when the
+ *      other BSS matches the connect parameters. When such roaming is initiated
+ *      by the driver, the driver is expected to verify that the target matches
+ *      the configured security parameters and to use Reassociation Request
+ *      frame instead of Association Request frame.
+ *      The connect function can also be used to request the driver to perform a
+ *      specific roam when connected to an ESS. In that case, the prev_bssid
  *      parameter is set to the BSSID of the currently associated BSS as an
- *      indication of requesting reassociation. In both the driver-initiated and
- *      new connect() call initiated roaming cases, the result of roaming is
- *      indicated with a call to cfg80211_roamed() or cfg80211_roamed_bss().
+ *      indication of requesting reassociation.
+ *      In both the driver-initiated and new connect() call initiated roaming
+ *      cases, the result of roaming is indicated with a call to
+ *      cfg80211_roamed() or cfg80211_roamed_bss().
  *      (invoked with the wireless_dev mutex held)
  * @disconnect: Disconnect from the BSS/ESS.
  *      (invoked with the wireless_dev mutex held)
@@ -3080,6 +3084,24 @@ struct wiphy_vendor_command {
 };
 
 /**
+ * struct wiphy_iftype_ext_capab - extended capabilities per interface type
+ * @iftype: interface type
+ * @extended_capabilities: extended capabilities supported by the driver,
+ *      additional capabilities might be supported by userspace; these are the
+ *      802.11 extended capabilities ("Extended Capabilities element") and are
+ *      in the same format as in the information element. See IEEE Std
+ *      802.11-2012 8.4.2.29 for the defined fields.
+ * @extended_capabilities_mask: mask of the valid values
+ * @extended_capabilities_len: length of the extended capabilities
+ */
+struct wiphy_iftype_ext_capab {
+        enum nl80211_iftype iftype;
+        const u8 *extended_capabilities;
+        const u8 *extended_capabilities_mask;
+        u8 extended_capabilities_len;
+};
+
+/**
  * struct wiphy - wireless hardware description
  * @reg_notifier: the driver's regulatory notification callback,
  *      note that if your driver uses wiphy_apply_custom_regulatory()
@@ -3199,9 +3221,14 @@ struct wiphy_vendor_command {
  *      additional capabilities might be supported by userspace; these are
  *      the 802.11 extended capabilities ("Extended Capabilities element")
  *      and are in the same format as in the information element. See
- *      802.11-2012 8.4.2.29 for the defined fields.
+ *      802.11-2012 8.4.2.29 for the defined fields. These are the default
+ *      extended capabilities to be used if the capabilities are not specified
+ *      for a specific interface type in iftype_ext_capab.
  * @extended_capabilities_mask: mask of the valid values
  * @extended_capabilities_len: length of the extended capabilities
+ * @iftype_ext_capab: array of extended capabilities per interface type
+ * @num_iftype_ext_capab: number of interface types for which extended
+ *      capabilities are specified separately.
  * @coalesce: packet coalescing support information
  *
  * @vendor_commands: array of vendor commands supported by the hardware
@@ -3301,6 +3328,9 @@ struct wiphy {
         const u8 *extended_capabilities, *extended_capabilities_mask;
         u8 extended_capabilities_len;
 
+        const struct wiphy_iftype_ext_capab *iftype_ext_capab;
+        unsigned int num_iftype_ext_capab;
+
         /* If multiple wiphys are registered and you're handed e.g.
          * a regular netdev with assigned ieee80211_ptr, you won't
          * know whether it points to a wiphy your driver has registered
@@ -4680,7 +4710,7 @@ static inline void cfg80211_testmode_event(struct sk_buff *skb, gfp_t gfp)
 void cfg80211_connect_bss(struct net_device *dev, const u8 *bssid,
                           struct cfg80211_bss *bss, const u8 *req_ie,
                           size_t req_ie_len, const u8 *resp_ie,
-                          size_t resp_ie_len, u16 status, gfp_t gfp);
+                          size_t resp_ie_len, int status, gfp_t gfp);
 
 /**
  * cfg80211_connect_result - notify cfg80211 of connection result
@@ -4710,6 +4740,29 @@ cfg80211_connect_result(struct net_device *dev, const u8 *bssid,
 }
 
 /**
+ * cfg80211_connect_timeout - notify cfg80211 of connection timeout
+ *
+ * @dev: network device
+ * @bssid: the BSSID of the AP
+ * @req_ie: association request IEs (maybe be %NULL)
+ * @req_ie_len: association request IEs length
+ * @gfp: allocation flags
+ *
+ * It should be called by the underlying driver whenever connect() has failed
+ * in a sequence where no explicit authentication/association rejection was
+ * received from the AP. This could happen, e.g., due to not being able to send
+ * out the Authentication or Association Request frame or timing out while
+ * waiting for the response.
+ */
+static inline void
+cfg80211_connect_timeout(struct net_device *dev, const u8 *bssid,
+                         const u8 *req_ie, size_t req_ie_len, gfp_t gfp)
+{
+        cfg80211_connect_bss(dev, bssid, NULL, req_ie, req_ie_len, NULL, 0, -1,
+                             gfp);
+}
+
+/**
  * cfg80211_roamed - notify cfg80211 of roaming
  *
  * @dev: network device
diff --git a/include/net/codel_qdisc.h b/include/net/codel_qdisc.h
index 8144d9cd2908..098630f83a55 100644
--- a/include/net/codel_qdisc.h
+++ b/include/net/codel_qdisc.h
@@ -52,6 +52,7 @@
 /* Qdiscs using codel plugin must use codel_skb_cb in their own cb[] */
 struct codel_skb_cb {
         codel_time_t enqueue_time;
+        unsigned int mem_usage;
 };
 
 static struct codel_skb_cb *get_codel_cb(const struct sk_buff *skb)
diff --git a/include/net/dsa.h b/include/net/dsa.h
index 17c3d37b6779..20b3087ad193 100644
--- a/include/net/dsa.h
+++ b/include/net/dsa.h
@@ -26,6 +26,7 @@ enum dsa_tag_protocol {
         DSA_TAG_PROTO_TRAILER,
         DSA_TAG_PROTO_EDSA,
         DSA_TAG_PROTO_BRCM,
+        DSA_TAG_LAST,           /* MUST BE LAST */
 };
 
 #define DSA_MAX_SWITCHES        4
@@ -58,12 +59,11 @@ struct dsa_chip_data {
         struct device_node *port_dn[DSA_MAX_PORTS];
 
         /*
-         * An array (with nr_chips elements) of which element [a]
-         * indicates which port on this switch should be used to
-         * send packets to that are destined for switch a.  Can be
-         * NULL if there is only one switch chip.
+         * An array of which element [a] indicates which port on this
+         * switch should be used to send packets to that are destined
+         * for switch a. Can be NULL if there is only one switch chip.
          */
-        s8              *rtable;
+        s8              rtable[DSA_MAX_SWITCHES];
 };
 
 struct dsa_platform_data {
@@ -85,6 +85,17 @@ struct dsa_platform_data {
 struct packet_type;
 
 struct dsa_switch_tree {
+        struct list_head        list;
+
+        /* Tree identifier */
+        u32 tree;
+
+        /* Number of switches attached to this tree */
+        struct kref refcount;
+
+        /* Has this tree been applied to the hardware? */
+        bool applied;
+
         /*
          * Configuration data for the platform device that owns
          * this dsa switch tree instance.
@@ -100,12 +111,12 @@ struct dsa_switch_tree {
                                        struct net_device *dev,
                                        struct packet_type *pt,
                                        struct net_device *orig_dev);
-        enum dsa_tag_protocol   tag_protocol;
 
         /*
          * Original copy of the master netdev ethtool_ops
          */
         struct ethtool_ops      master_ethtool_ops;
+        const struct ethtool_ops *master_orig_ethtool_ops;
 
         /*
          * The switch and port to which the CPU is attached.
@@ -117,6 +128,17 @@ struct dsa_switch_tree {
          * Data for the individual switch chips.
          */
         struct dsa_switch       *ds[DSA_MAX_SWITCHES];
+
+        /*
+         * Tagging protocol operations for adding and removing an
+         * encapsulation tag.
+         */
+        const struct dsa_device_ops *tag_ops;
+};
+
+struct dsa_port {
+        struct net_device       *netdev;
+        struct device_node      *dn;
 };
 
 struct dsa_switch {
@@ -144,6 +166,13 @@ struct dsa_switch {
          */
         struct dsa_switch_driver        *drv;
 
+        /*
+         * An array of which element [a] indicates which port on this
+         * switch should be used to send packets to that are destined
+         * for switch a. Can be NULL if there is only one switch chip.
+         */
+        s8              rtable[DSA_MAX_SWITCHES];
+
 #ifdef CONFIG_NET_DSA_HWMON
         /*
          * Hardware monitoring information
@@ -153,13 +182,19 @@ struct dsa_switch {
 #endif
 
         /*
+         * The lower device this switch uses to talk to the host
+         */
+        struct net_device *master_netdev;
+
+        /*
          * Slave mii_bus and devices for the individual ports.
          */
         u32                     dsa_port_mask;
+        u32                     cpu_port_mask;
         u32                     enabled_port_mask;
         u32                     phys_mii_mask;
+        struct dsa_port         ports[DSA_MAX_PORTS];
         struct mii_bus          *slave_mii_bus;
-        struct net_device       *ports[DSA_MAX_PORTS];
 };
 
 static inline bool dsa_is_cpu_port(struct dsa_switch *ds, int p)
@@ -174,7 +209,7 @@ static inline bool dsa_is_dsa_port(struct dsa_switch *ds, int p)
 
 static inline bool dsa_is_port_initialized(struct dsa_switch *ds, int p)
 {
-        return ds->enabled_port_mask & (1 << p) && ds->ports[p];
+        return ds->enabled_port_mask & (1 << p) && ds->ports[p].netdev;
 }
 
 static inline u8 dsa_upstream_port(struct dsa_switch *ds)
@@ -190,7 +225,7 @@ static inline u8 dsa_upstream_port(struct dsa_switch *ds)
         if (dst->cpu_switch == ds->index)
                 return dst->cpu_port;
         else
-                return ds->cd->rtable[dst->cpu_switch];
+                return ds->rtable[dst->cpu_switch];
 }
 
 struct switchdev_trans;
@@ -344,4 +379,7 @@ static inline bool dsa_uses_tagged_protocol(struct dsa_switch_tree *dst)
 {
         return dst->rcv != NULL;
 }
+
+void dsa_unregister_switch(struct dsa_switch *ds);
+int dsa_register_switch(struct dsa_switch *ds, struct device_node *np);
 #endif
diff --git a/include/net/fib_rules.h b/include/net/fib_rules.h
index 59160de702b6..456e4a6006ab 100644
--- a/include/net/fib_rules.h
+++ b/include/net/fib_rules.h
@@ -17,7 +17,8 @@ struct fib_rule {
         u32                     flags;
         u32                     table;
         u8                      action;
-        /* 3 bytes hole, try to use */
+        u8                      l3mdev;
+        /* 2 bytes hole, try to use */
         u32                     target;
         __be64                  tun_id;
         struct fib_rule __rcu   *ctarget;
@@ -36,6 +37,7 @@ struct fib_lookup_arg {
         void                    *lookup_ptr;
         void                    *result;
         struct fib_rule         *rule;
+        u32                     table;
         int                     flags;
 #define FIB_LOOKUP_NOREF                1
 #define FIB_LOOKUP_IGNORE_LINKSTATE     2
@@ -89,7 +91,8 @@ struct fib_rules_ops {
         [FRA_TABLE]     = { .type = NLA_U32 }, \
         [FRA_SUPPRESS_PREFIXLEN] = { .type = NLA_U32 }, \
         [FRA_SUPPRESS_IFGROUP] = { .type = NLA_U32 }, \
-        [FRA_GOTO]      = { .type = NLA_U32 }
+        [FRA_GOTO]      = { .type = NLA_U32 }, \
+        [FRA_L3MDEV]    = { .type = NLA_U8 }
 
 static inline void fib_rule_get(struct fib_rule *rule)
 {
@@ -102,6 +105,20 @@ static inline void fib_rule_put(struct fib_rule *rule)
                 kfree_rcu(rule, rcu);
 }
 
+#ifdef CONFIG_NET_L3_MASTER_DEV
+static inline u32 fib_rule_get_table(struct fib_rule *rule,
+                                     struct fib_lookup_arg *arg)
+{
+        return rule->l3mdev ? arg->table : rule->table;
+}
+#else
+static inline u32 fib_rule_get_table(struct fib_rule *rule,
+                                     struct fib_lookup_arg *arg)
+{
+        return rule->table;
+}
+#endif
+
 static inline u32 frh_get_table(struct fib_rule_hdr *frh, struct nlattr **nla)
 {
         if (nla[FRA_TABLE])
@@ -117,4 +134,7 @@ int fib_rules_lookup(struct fib_rules_ops *, struct flowi *, int flags,
                      struct fib_lookup_arg *);
 int fib_default_rule_add(struct fib_rules_ops *, u32 pref, u32 table,
                          u32 flags);
+
+int fib_nl_newrule(struct sk_buff *skb, struct nlmsghdr *nlh);
+int fib_nl_delrule(struct sk_buff *skb, struct nlmsghdr *nlh);
 #endif
diff --git a/include/net/gen_stats.h b/include/net/gen_stats.h
index 610cd397890e..231e121cc7d9 100644
--- a/include/net/gen_stats.h
+++ b/include/net/gen_stats.h
@@ -33,10 +33,12 @@ int gnet_stats_start_copy_compat(struct sk_buff *skb, int type,
                                  spinlock_t *lock, struct gnet_dump *d,
                                  int padattr);
 
-int gnet_stats_copy_basic(struct gnet_dump *d,
+int gnet_stats_copy_basic(const seqcount_t *running,
+                          struct gnet_dump *d,
                           struct gnet_stats_basic_cpu __percpu *cpu,
                           struct gnet_stats_basic_packed *b);
-void __gnet_stats_copy_basic(struct gnet_stats_basic_packed *bstats,
+void __gnet_stats_copy_basic(const seqcount_t *running,
+                             struct gnet_stats_basic_packed *bstats,
                              struct gnet_stats_basic_cpu __percpu *cpu,
                              struct gnet_stats_basic_packed *b);
 int gnet_stats_copy_rate_est(struct gnet_dump *d,
@@ -52,13 +54,15 @@ int gnet_stats_finish_copy(struct gnet_dump *d);
 int gen_new_estimator(struct gnet_stats_basic_packed *bstats,
                       struct gnet_stats_basic_cpu __percpu *cpu_bstats,
                       struct gnet_stats_rate_est64 *rate_est,
-                      spinlock_t *stats_lock, struct nlattr *opt);
+                      spinlock_t *stats_lock,
+                      seqcount_t *running, struct nlattr *opt);
 void gen_kill_estimator(struct gnet_stats_basic_packed *bstats,
                         struct gnet_stats_rate_est64 *rate_est);
 int gen_replace_estimator(struct gnet_stats_basic_packed *bstats,
                           struct gnet_stats_basic_cpu __percpu *cpu_bstats,
                           struct gnet_stats_rate_est64 *rate_est,
-                          spinlock_t *stats_lock, struct nlattr *opt);
+                          spinlock_t *stats_lock,
+                          seqcount_t *running, struct nlattr *opt);
 bool gen_estimator_active(const struct gnet_stats_basic_packed *bstats,
                           const struct gnet_stats_rate_est64 *rate_est);
 #endif
diff --git a/include/net/geneve.h b/include/net/geneve.h
index cb544a530146..ec0327d4331b 100644
--- a/include/net/geneve.h
+++ b/include/net/geneve.h
@@ -1,10 +1,7 @@
 #ifndef __NET_GENEVE_H
 #define __NET_GENEVE_H  1
 
-#ifdef CONFIG_INET
 #include <net/udp_tunnel.h>
-#endif
-
 
 /* Geneve Header:
  *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -62,12 +59,6 @@ struct genevehdr {
         struct geneve_opt options[];
 };
 
-static inline void geneve_get_rx_port(struct net_device *netdev)
-{
-        ASSERT_RTNL();
-        call_netdevice_notifiers(NETDEV_OFFLOAD_PUSH_GENEVE, netdev);
-}
-
 #ifdef CONFIG_INET
 struct net_device *geneve_dev_create_fb(struct net *net, const char *name,
                                         u8 name_assign_type, u16 dst_port);
diff --git a/include/net/ip6_route.h b/include/net/ip6_route.h
index 54c779416eec..d97305d0e71f 100644
--- a/include/net/ip6_route.h
+++ b/include/net/ip6_route.h
@@ -18,6 +18,7 @@ struct route_info {
         __u8                    prefix[0];      /* 0,8 or 16 */
 };
 
+#include <net/addrconf.h>
 #include <net/flow.h>
 #include <net/ip6_fib.h>
 #include <net/sock.h>
@@ -76,6 +77,8 @@ static inline struct dst_entry *ip6_route_output(struct net *net,
 
 struct dst_entry *ip6_route_lookup(struct net *net, struct flowi6 *fl6,
                                    int flags);
+struct rt6_info *ip6_pol_route(struct net *net, struct fib6_table *table,
+                               int ifindex, struct flowi6 *fl6, int flags);
 
 int ip6_route_init(void);
 void ip6_route_cleanup(void);
@@ -86,9 +89,23 @@ int ip6_route_add(struct fib6_config *cfg);
 int ip6_ins_rt(struct rt6_info *);
 int ip6_del_rt(struct rt6_info *);
 
-int ip6_route_get_saddr(struct net *net, struct rt6_info *rt,
-                        const struct in6_addr *daddr, unsigned int prefs,
-                        struct in6_addr *saddr);
+static inline int ip6_route_get_saddr(struct net *net, struct rt6_info *rt,
+                                      const struct in6_addr *daddr,
+                                      unsigned int prefs,
+                                      struct in6_addr *saddr)
+{
+        struct inet6_dev *idev =
+                        rt ? ip6_dst_idev((struct dst_entry *)rt) : NULL;
+        int err = 0;
+
+        if (rt && rt->rt6i_prefsrc.plen)
+                *saddr = rt->rt6i_prefsrc.addr;
+        else
+                err = ipv6_dev_get_saddr(net, idev ? idev->dev : NULL,
+                                         daddr, prefs, saddr);
+
+        return err;
+}
 
 struct rt6_info *rt6_lookup(struct net *net, const struct in6_addr *daddr,
                             const struct in6_addr *saddr, int oif, int flags);
diff --git a/include/net/ip_tunnels.h b/include/net/ip_tunnels.h
index dbf444428437..a5e7035fb93f 100644
--- a/include/net/ip_tunnels.h
+++ b/include/net/ip_tunnels.h
@@ -132,6 +132,7 @@ struct ip_tunnel {
         int                     ip_tnl_net_id;
         struct gro_cells        gro_cells;
         bool                    collect_md;
+        bool                    ignore_df;
 };
 
 #define TUNNEL_CSUM             __cpu_to_be16(0x01)
@@ -156,6 +157,7 @@ struct tnl_ptk_info {
         __be16 proto;
         __be32 key;
         __be32 seq;
+        int hdr_len;
 };
 
 #define PACKET_RCVD     0
diff --git a/include/net/l3mdev.h b/include/net/l3mdev.h
index 374388dc01c8..e90095091aa0 100644
--- a/include/net/l3mdev.h
+++ b/include/net/l3mdev.h
@@ -11,6 +11,8 @@
 #ifndef _NET_L3MDEV_H_
 #define _NET_L3MDEV_H_
 
+#include <net/fib_rules.h>
+
 /**
  * struct l3mdev_ops - l3mdev operations
  *
@@ -36,11 +38,17 @@ struct l3mdev_ops {
 
         /* IPv6 ops */
         struct dst_entry * (*l3mdev_get_rt6_dst)(const struct net_device *dev,
-                                                 const struct flowi6 *fl6);
+                                                 struct flowi6 *fl6);
+        int                (*l3mdev_get_saddr6)(struct net_device *dev,
+                                                const struct sock *sk,
+                                                struct flowi6 *fl6);
 };
 
 #ifdef CONFIG_NET_L3_MASTER_DEV
 
+int l3mdev_fib_rule_match(struct net *net, struct flowi *fl,
+                          struct fib_lookup_arg *arg);
+
 int l3mdev_master_ifindex_rcu(const struct net_device *dev);
 static inline int l3mdev_master_ifindex(struct net_device *dev)
 {
@@ -71,6 +79,31 @@ static inline int l3mdev_master_ifindex_by_index(struct net *net, int ifindex)
         return rc;
 }
 
+static inline
+const struct net_device *l3mdev_master_dev_rcu(const struct net_device *_dev)
+{
+        /* netdev_master_upper_dev_get_rcu calls
+         * list_first_or_null_rcu to walk the upper dev list.
+         * list_first_or_null_rcu does not handle a const arg. We aren't
+         * making changes, just want the master device from that list so
+         * typecast to remove the const
+         */
+        struct net_device *dev = (struct net_device *)_dev;
+        const struct net_device *master;
+
+        if (!dev)
+                return NULL;
+
+        if (netif_is_l3_master(dev))
+                master = dev;
+        else if (netif_is_l3_slave(dev))
+                master = netdev_master_upper_dev_get_rcu(dev);
+        else
+                master = NULL;
+
+        return master;
+}
+
 /* get index of an interface to use for FIB lookups. For devices
  * enslaved to an L3 master device FIB lookups are based on the
  * master index
@@ -134,7 +167,9 @@ static inline bool netif_index_is_l3_master(struct net *net, int ifindex)
 
 int l3mdev_get_saddr(struct net *net, int ifindex, struct flowi4 *fl4);
 
-struct dst_entry *l3mdev_get_rt6_dst(struct net *net, const struct flowi6 *fl6);
+struct dst_entry *l3mdev_get_rt6_dst(struct net *net, struct flowi6 *fl6);
+int l3mdev_get_saddr6(struct net *net, const struct sock *sk,
+                      struct flowi6 *fl6);
 
 static inline
 struct sk_buff *l3mdev_l3_rcv(struct sk_buff *skb, u16 proto)
@@ -180,6 +215,12 @@ static inline int l3mdev_master_ifindex_by_index(struct net *net, int ifindex)
         return 0;
 }
 
+static inline
+const struct net_device *l3mdev_master_dev_rcu(const struct net_device *dev)
+{
+        return NULL;
+}
+
 static inline int l3mdev_fib_oif_rcu(struct net_device *dev)
 {
         return dev ? dev->ifindex : 0;
@@ -220,11 +261,17 @@ static inline int l3mdev_get_saddr(struct net *net, int ifindex,
 }
 
 static inline
-struct dst_entry *l3mdev_get_rt6_dst(struct net *net, const struct flowi6 *fl6)
+struct dst_entry *l3mdev_get_rt6_dst(struct net *net, struct flowi6 *fl6)
 {
         return NULL;
 }
 
+static inline int l3mdev_get_saddr6(struct net *net, const struct sock *sk,
+                                    struct flowi6 *fl6)
+{
+        return 0;
+}
+
 static inline
 struct sk_buff *l3mdev_ip_rcv(struct sk_buff *skb)
 {
@@ -236,6 +283,13 @@ struct sk_buff *l3mdev_ip6_rcv(struct sk_buff *skb)
 {
         return skb;
 }
+
+static inline
+int l3mdev_fib_rule_match(struct net *net, struct flowi *fl,
+                          struct fib_lookup_arg *arg)
+{
+        return 1;
+}
 #endif
 
 #endif /* _NET_L3MDEV_H_ */
diff --git a/include/net/mac80211.h b/include/net/mac80211.h
index be30b0549b88..a52009ffc19f 100644
--- a/include/net/mac80211.h
+++ b/include/net/mac80211.h
@@ -21,6 +21,7 @@
 #include <linux/skbuff.h>
 #include <linux/ieee80211.h>
 #include <net/cfg80211.h>
+#include <net/codel.h>
 #include <asm/unaligned.h>
 
 /**
@@ -895,7 +896,18 @@ struct ieee80211_tx_info {
                                 unsigned long jiffies;
                         };
                         /* NB: vif can be NULL for injected frames */
-                        struct ieee80211_vif *vif;
+                        union {
+                                /* NB: vif can be NULL for injected frames */
+                                struct ieee80211_vif *vif;
+
+                                /* When packets are enqueued on txq it's easy
+                                 * to re-construct the vif pointer. There's no
+                                 * more space in tx_info so it can be used to
+                                 * store the necessary enqueue time for packet
+                                 * sojourn time computation.
+                                 */
+                                codel_time_t enqueue_time;
+                        };
                         struct ieee80211_key_conf *hw_key;
                         u32 flags;
                         /* 4 bytes free */
@@ -2147,9 +2159,6 @@ enum ieee80211_hw_flags {
  * @n_cipher_schemes: a size of an array of cipher schemes definitions.
  * @cipher_schemes: a pointer to an array of cipher scheme definitions
  *      supported by HW.
- *
- * @txq_ac_max_pending: maximum number of frames per AC pending in all txq
- *      entries for a vif.
  */
 struct ieee80211_hw {
         struct ieee80211_conf conf;
@@ -2180,7 +2189,6 @@ struct ieee80211_hw {
         u8 uapsd_max_sp_len;
         u8 n_cipher_schemes;
         const struct ieee80211_cipher_scheme *cipher_schemes;
-        int txq_ac_max_pending;
 };
 
 static inline bool _ieee80211_hw_check(struct ieee80211_hw *hw,
diff --git a/include/net/ndisc.h b/include/net/ndisc.h
index 2d8edaad29cb..be1fe2283254 100644
--- a/include/net/ndisc.h
+++ b/include/net/ndisc.h
@@ -35,6 +35,7 @@ enum {
         ND_OPT_ROUTE_INFO = 24,         /* RFC4191 */
         ND_OPT_RDNSS = 25,              /* RFC5006 */
         ND_OPT_DNSSL = 31,              /* RFC6106 */
+        ND_OPT_6CO = 34,                /* RFC6775 */
         __ND_OPT_MAX
 };
 
@@ -53,11 +54,21 @@ enum {
 
 #include <net/neighbour.h>
 
+/* Set to 3 to get tracing... */
+#define ND_DEBUG 1
+
+#define ND_PRINTK(val, level, fmt, ...)                         \
+do {                                                            \
+        if (val <= ND_DEBUG)                                    \
+                net_##level##_ratelimited(fmt, ##__VA_ARGS__);  \
+} while (0)
+
 struct ctl_table;
 struct inet6_dev;
 struct net_device;
 struct net_proto_family;
 struct sk_buff;
+struct prefix_info;
 
 extern struct neigh_table nd_tbl;
 
@@ -99,20 +110,201 @@ struct ndisc_options {
 #endif
         struct nd_opt_hdr *nd_useropts;
         struct nd_opt_hdr *nd_useropts_end;
+#if IS_ENABLED(CONFIG_IEEE802154_6LOWPAN)
+        struct nd_opt_hdr *nd_802154_opt_array[ND_OPT_TARGET_LL_ADDR + 1];
+#endif
 };
 
-#define nd_opts_src_lladdr      nd_opt_array[ND_OPT_SOURCE_LL_ADDR]
-#define nd_opts_tgt_lladdr      nd_opt_array[ND_OPT_TARGET_LL_ADDR]
-#define nd_opts_pi              nd_opt_array[ND_OPT_PREFIX_INFO]
-#define nd_opts_pi_end          nd_opt_array[__ND_OPT_PREFIX_INFO_END]
-#define nd_opts_rh              nd_opt_array[ND_OPT_REDIRECT_HDR]
-#define nd_opts_mtu             nd_opt_array[ND_OPT_MTU]
+#define nd_opts_src_lladdr              nd_opt_array[ND_OPT_SOURCE_LL_ADDR]
+#define nd_opts_tgt_lladdr              nd_opt_array[ND_OPT_TARGET_LL_ADDR]
+#define nd_opts_pi                      nd_opt_array[ND_OPT_PREFIX_INFO]
+#define nd_opts_pi_end                  nd_opt_array[__ND_OPT_PREFIX_INFO_END]
+#define nd_opts_rh                      nd_opt_array[ND_OPT_REDIRECT_HDR]
+#define nd_opts_mtu                     nd_opt_array[ND_OPT_MTU]
+#define nd_802154_opts_src_lladdr       nd_802154_opt_array[ND_OPT_SOURCE_LL_ADDR]
+#define nd_802154_opts_tgt_lladdr       nd_802154_opt_array[ND_OPT_TARGET_LL_ADDR]
 
 #define NDISC_OPT_SPACE(len) (((len)+2+7)&~7)
 
-struct ndisc_options *ndisc_parse_options(u8 *opt, int opt_len,
+struct ndisc_options *ndisc_parse_options(const struct net_device *dev,
+                                          u8 *opt, int opt_len,
                                           struct ndisc_options *ndopts);
 
+void __ndisc_fill_addr_option(struct sk_buff *skb, int type, void *data,
+                              int data_len, int pad);
+
+#define NDISC_OPS_REDIRECT_DATA_SPACE   2
+
+/*
+ * This structure defines the hooks for IPv6 neighbour discovery.
+ * The following hooks can be defined; unless noted otherwise, they are
+ * optional and can be filled with a null pointer.
+ *
+ * int (*is_useropt)(u8 nd_opt_type):
+ *     This function is called when IPv6 decide RA userspace options. if
+ *     this function returns 1 then the option given by nd_opt_type will
+ *     be handled as userspace option additional to the IPv6 options.
+ *
+ * int (*parse_options)(const struct net_device *dev,
+ *                      struct nd_opt_hdr *nd_opt,
+ *                      struct ndisc_options *ndopts):
+ *     This function is called while parsing ndisc ops and put each position
+ *     as pointer into ndopts. If this function return unequal 0, then this
+ *     function took care about the ndisc option, if 0 then the IPv6 ndisc
+ *     option parser will take care about that option.
+ *
+ * void (*update)(const struct net_device *dev, struct neighbour *n,
+ *                u32 flags, u8 icmp6_type,
+ *                const struct ndisc_options *ndopts):
+ *     This function is called when IPv6 ndisc updates the neighbour cache
+ *     entry. Additional options which can be updated may be previously
+ *     parsed by parse_opts callback and accessible over ndopts parameter.
+ *
+ * int (*opt_addr_space)(const struct net_device *dev, u8 icmp6_type,
+ *                       struct neighbour *neigh, u8 *ha_buf,
+ *                       u8 **ha):
+ *     This function is called when the necessary option space will be
+ *     calculated before allocating a skb. The parameters neigh, ha_buf
+ *     abd ha are available on NDISC_REDIRECT messages only.
+ *
+ * void (*fill_addr_option)(const struct net_device *dev,
+ *                          struct sk_buff *skb, u8 icmp6_type,
+ *                          const u8 *ha):
+ *     This function is called when the skb will finally fill the option
+ *     fields inside skb. NOTE: this callback should fill the option
+ *     fields to the skb which are previously indicated by opt_space
+ *     parameter. That means the decision to add such option should
+ *     not lost between these two callbacks, e.g. protected by interface
+ *     up state.
+ *
+ * void (*prefix_rcv_add_addr)(struct net *net, struct net_device *dev,
+ *                             const struct prefix_info *pinfo,
+ *                             struct inet6_dev *in6_dev,
+ *                             struct in6_addr *addr,
+ *                             int addr_type, u32 addr_flags,
+ *                             bool sllao, bool tokenized,
+ *                             __u32 valid_lft, u32 prefered_lft,
+ *                             bool dev_addr_generated):
+ *     This function is called when a RA messages is received with valid
+ *     PIO option fields and an IPv6 address will be added to the interface
+ *     for autoconfiguration. The parameter dev_addr_generated reports about
+ *     if the address was based on dev->dev_addr or not. This can be used
+ *     to add a second address if link-layer operates with two link layer
+ *     addresses. E.g. 802.15.4 6LoWPAN.
+ */
+struct ndisc_ops {
+        int     (*is_useropt)(u8 nd_opt_type);
+        int     (*parse_options)(const struct net_device *dev,
+                                 struct nd_opt_hdr *nd_opt,
+                                 struct ndisc_options *ndopts);
+        void    (*update)(const struct net_device *dev, struct neighbour *n,
+                          u32 flags, u8 icmp6_type,
+                          const struct ndisc_options *ndopts);
+        int     (*opt_addr_space)(const struct net_device *dev, u8 icmp6_type,
+                                  struct neighbour *neigh, u8 *ha_buf,
+                                  u8 **ha);
+        void    (*fill_addr_option)(const struct net_device *dev,
+                                    struct sk_buff *skb, u8 icmp6_type,
+                                    const u8 *ha);
+        void    (*prefix_rcv_add_addr)(struct net *net, struct net_device *dev,
+                                       const struct prefix_info *pinfo,
+                                       struct inet6_dev *in6_dev,
+                                       struct in6_addr *addr,
+                                       int addr_type, u32 addr_flags,
+                                       bool sllao, bool tokenized,
+                                       __u32 valid_lft, u32 prefered_lft,
+                                       bool dev_addr_generated);
+};
+
+#if IS_ENABLED(CONFIG_IPV6)
+static inline int ndisc_ops_is_useropt(const struct net_device *dev,
+                                       u8 nd_opt_type)
+{
+        if (dev->ndisc_ops && dev->ndisc_ops->is_useropt)
+                return dev->ndisc_ops->is_useropt(nd_opt_type);
+        else
+                return 0;
+}
+
+static inline int ndisc_ops_parse_options(const struct net_device *dev,
+                                          struct nd_opt_hdr *nd_opt,
+                                          struct ndisc_options *ndopts)
+{
+        if (dev->ndisc_ops && dev->ndisc_ops->parse_options)
+                return dev->ndisc_ops->parse_options(dev, nd_opt, ndopts);
+        else
+                return 0;
+}
+
+static inline void ndisc_ops_update(const struct net_device *dev,
+                                          struct neighbour *n, u32 flags,
+                                          u8 icmp6_type,
+                                          const struct ndisc_options *ndopts)
+{
+        if (dev->ndisc_ops && dev->ndisc_ops->update)
+                dev->ndisc_ops->update(dev, n, flags, icmp6_type, ndopts);
+}
+
+static inline int ndisc_ops_opt_addr_space(const struct net_device *dev,
+                                           u8 icmp6_type)
+{
+        if (dev->ndisc_ops && dev->ndisc_ops->opt_addr_space &&
+            icmp6_type != NDISC_REDIRECT)
+                return dev->ndisc_ops->opt_addr_space(dev, icmp6_type, NULL,
+                                                      NULL, NULL);
+        else
+                return 0;
+}
+
+static inline int ndisc_ops_redirect_opt_addr_space(const struct net_device *dev,
+                                                    struct neighbour *neigh,
+                                                    u8 *ha_buf, u8 **ha)
+{
+        if (dev->ndisc_ops && dev->ndisc_ops->opt_addr_space)
+                return dev->ndisc_ops->opt_addr_space(dev, NDISC_REDIRECT,
+                                                      neigh, ha_buf, ha);
+        else
+                return 0;
+}
+
+static inline void ndisc_ops_fill_addr_option(const struct net_device *dev,
+                                              struct sk_buff *skb,
+                                              u8 icmp6_type)
+{
+        if (dev->ndisc_ops && dev->ndisc_ops->fill_addr_option &&
+            icmp6_type != NDISC_REDIRECT)
+                dev->ndisc_ops->fill_addr_option(dev, skb, icmp6_type, NULL);
+}
+
+static inline void ndisc_ops_fill_redirect_addr_option(const struct net_device *dev,
+                                                       struct sk_buff *skb,
+                                                       const u8 *ha)
+{
+        if (dev->ndisc_ops && dev->ndisc_ops->fill_addr_option)
+                dev->ndisc_ops->fill_addr_option(dev, skb, NDISC_REDIRECT, ha);
+}
+
+static inline void ndisc_ops_prefix_rcv_add_addr(struct net *net,
+                                                 struct net_device *dev,
+                                                 const struct prefix_info *pinfo,
+                                                 struct inet6_dev *in6_dev,
+                                                 struct in6_addr *addr,
+                                                 int addr_type, u32 addr_flags,
+                                                 bool sllao, bool tokenized,
+                                                 __u32 valid_lft,
+                                                 u32 prefered_lft,
+                                                 bool dev_addr_generated)
+{
+        if (dev->ndisc_ops && dev->ndisc_ops->prefix_rcv_add_addr)
+                dev->ndisc_ops->prefix_rcv_add_addr(net, dev, pinfo, in6_dev,
+                                                    addr, addr_type,
+                                                    addr_flags, sllao,
+                                                    tokenized, valid_lft,
+                                                    prefered_lft,
+                                                    dev_addr_generated);
+}
+#endif
+
 /*
  * Return the padding between the option length and the start of the
  * link addr.  Currently only IP-over-InfiniBand needs this, although
@@ -127,23 +319,48 @@ static inline int ndisc_addr_option_pad(unsigned short type)
         }
 }
 
-static inline int ndisc_opt_addr_space(struct net_device *dev)
+static inline int __ndisc_opt_addr_space(unsigned char addr_len, int pad)
 {
-        return NDISC_OPT_SPACE(dev->addr_len +
-                               ndisc_addr_option_pad(dev->type));
+        return NDISC_OPT_SPACE(addr_len + pad);
 }
 
-static inline u8 *ndisc_opt_addr_data(struct nd_opt_hdr *p,
-                                      struct net_device *dev)
+#if IS_ENABLED(CONFIG_IPV6)
+static inline int ndisc_opt_addr_space(struct net_device *dev, u8 icmp6_type)
+{
+        return __ndisc_opt_addr_space(dev->addr_len,
+                                      ndisc_addr_option_pad(dev->type)) +
+                ndisc_ops_opt_addr_space(dev, icmp6_type);
+}
+
+static inline int ndisc_redirect_opt_addr_space(struct net_device *dev,
+                                                struct neighbour *neigh,
+                                                u8 *ops_data_buf,
+                                                u8 **ops_data)
+{
+        return __ndisc_opt_addr_space(dev->addr_len,
+                                      ndisc_addr_option_pad(dev->type)) +
+                ndisc_ops_redirect_opt_addr_space(dev, neigh, ops_data_buf,
+                                                  ops_data);
+}
+#endif
+
+static inline u8 *__ndisc_opt_addr_data(struct nd_opt_hdr *p,
+                                        unsigned char addr_len, int prepad)
 {
         u8 *lladdr = (u8 *)(p + 1);
         int lladdrlen = p->nd_opt_len << 3;
-        int prepad = ndisc_addr_option_pad(dev->type);
-        if (lladdrlen != ndisc_opt_addr_space(dev))
+        if (lladdrlen != __ndisc_opt_addr_space(addr_len, prepad))
                 return NULL;
         return lladdr + prepad;
 }
 
+static inline u8 *ndisc_opt_addr_data(struct nd_opt_hdr *p,
+                                      struct net_device *dev)
+{
+        return __ndisc_opt_addr_data(p, dev->addr_len,
+                                     ndisc_addr_option_pad(dev->type));
+}
+
 static inline u32 ndisc_hashfn(const void *pkey, const struct net_device *dev, __u32 *hash_rnd)
 {
         const u32 *p32 = pkey;
@@ -194,6 +411,9 @@ void ndisc_send_redirect(struct sk_buff *skb, const struct in6_addr *target);
 int ndisc_mc_map(const struct in6_addr *addr, char *buf, struct net_device *dev,
                  int dir);
 
+void ndisc_update(const struct net_device *dev, struct neighbour *neigh,
+                  const u8 *lladdr, u8 new, u32 flags, u8 icmp6_type,
+                  struct ndisc_options *ndopts);
 
 /*
  *      IGMP
diff --git a/include/net/pkt_sched.h b/include/net/pkt_sched.h
index fea53f4d92ca..7caa99b482c6 100644
--- a/include/net/pkt_sched.h
+++ b/include/net/pkt_sched.h
@@ -67,12 +67,12 @@ struct qdisc_watchdog {
 };
 
 void qdisc_watchdog_init(struct qdisc_watchdog *wd, struct Qdisc *qdisc);
-void qdisc_watchdog_schedule_ns(struct qdisc_watchdog *wd, u64 expires, bool throttle);
+void qdisc_watchdog_schedule_ns(struct qdisc_watchdog *wd, u64 expires);
 
 static inline void qdisc_watchdog_schedule(struct qdisc_watchdog *wd,
                                            psched_time_t expires)
 {
-        qdisc_watchdog_schedule_ns(wd, PSCHED_TICKS2NS(expires), true);
+        qdisc_watchdog_schedule_ns(wd, PSCHED_TICKS2NS(expires));
 }
 
 void qdisc_watchdog_cancel(struct qdisc_watchdog *wd);
diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
index 62d553184e91..909aff2db2b3 100644
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
@@ -26,14 +26,6 @@ struct qdisc_rate_table {
 enum qdisc_state_t {
         __QDISC_STATE_SCHED,
         __QDISC_STATE_DEACTIVATED,
-        __QDISC_STATE_THROTTLED,
-};
-
-/*
- * following bits are only changed while qdisc lock is held
- */
-enum qdisc___state_t {
-        __QDISC___STATE_RUNNING = 1,
 };
 
 struct qdisc_size_table {
@@ -45,8 +37,10 @@ struct qdisc_size_table {
 };
 
 struct Qdisc {
-        int                     (*enqueue)(struct sk_buff *skb, struct Qdisc *dev);
-        struct sk_buff *        (*dequeue)(struct Qdisc *dev);
+        int                     (*enqueue)(struct sk_buff *skb,
+                                           struct Qdisc *sch,
+                                           struct sk_buff **to_free);
+        struct sk_buff *        (*dequeue)(struct Qdisc *sch);
         unsigned int            flags;
 #define TCQ_F_BUILTIN           1
 #define TCQ_F_INGRESS           2
@@ -70,31 +64,25 @@ struct Qdisc {
         struct list_head        list;
         u32                     handle;
         u32                     parent;
-        int                     (*reshape_fail)(struct sk_buff *skb,
-                                        struct Qdisc *q);
-
         void                    *u32_node;
 
-        /* This field is deprecated, but it is still used by CBQ
-         * and it will live until better solution will be invented.
-         */
-        struct Qdisc            *__parent;
         struct netdev_queue     *dev_queue;
 
         struct gnet_stats_rate_est64    rate_est;
         struct gnet_stats_basic_cpu __percpu *cpu_bstats;
         struct gnet_stats_queue __percpu *cpu_qstats;
 
-        struct Qdisc            *next_sched;
-        struct sk_buff          *gso_skb;
         /*
          * For performance sake on SMP, we put highly modified fields at the end
          */
-        unsigned long           state;
+        struct sk_buff          *gso_skb ____cacheline_aligned_in_smp;
         struct sk_buff_head     q;
         struct gnet_stats_basic_packed bstats;
-        unsigned int            __state;
+        seqcount_t              running;
         struct gnet_stats_queue qstats;
+        unsigned long           state;
+        struct Qdisc            *next_sched;
+        struct sk_buff          *skb_bad_txq;
         struct rcu_head         rcu_head;
         int                     padded;
         atomic_t                refcnt;
@@ -104,20 +92,24 @@ struct Qdisc {
 
 static inline bool qdisc_is_running(const struct Qdisc *qdisc)
 {
-        return (qdisc->__state & __QDISC___STATE_RUNNING) ? true : false;
+        return (raw_read_seqcount(&qdisc->running) & 1) ? true : false;
 }
 
 static inline bool qdisc_run_begin(struct Qdisc *qdisc)
 {
         if (qdisc_is_running(qdisc))
                 return false;
-        qdisc->__state |= __QDISC___STATE_RUNNING;
+        /* Variant of write_seqcount_begin() telling lockdep a trylock
+         * was attempted.
+         */
+        raw_write_seqcount_begin(&qdisc->running);
+        seqcount_acquire(&qdisc->running.dep_map, 0, 1, _RET_IP_);
         return true;
 }
 
 static inline void qdisc_run_end(struct Qdisc *qdisc)
 {
-        qdisc->__state &= ~__QDISC___STATE_RUNNING;
+        write_seqcount_end(&qdisc->running);
 }
 
 static inline bool qdisc_may_bulk(const struct Qdisc *qdisc)
@@ -135,21 +127,6 @@ static inline int qdisc_avail_bulklimit(const struct netdev_queue *txq)
 #endif
 }
 
-static inline bool qdisc_is_throttled(const struct Qdisc *qdisc)
-{
-        return test_bit(__QDISC_STATE_THROTTLED, &qdisc->state) ? true : false;
-}
-
-static inline void qdisc_throttled(struct Qdisc *qdisc)
-{
-        set_bit(__QDISC_STATE_THROTTLED, &qdisc->state);
-}
-
-static inline void qdisc_unthrottled(struct Qdisc *qdisc)
-{
-        clear_bit(__QDISC_STATE_THROTTLED, &qdisc->state);
-}
-
 struct Qdisc_class_ops {
         /* Child qdisc manipulation */
         struct netdev_queue *   (*select_queue)(struct Qdisc *, struct tcmsg *);
@@ -186,10 +163,11 @@ struct Qdisc_ops {
         char                    id[IFNAMSIZ];
         int                     priv_size;
 
-        int                     (*enqueue)(struct sk_buff *, struct Qdisc *);
+        int                     (*enqueue)(struct sk_buff *skb,
+                                           struct Qdisc *sch,
+                                           struct sk_buff **to_free);
         struct sk_buff *        (*dequeue)(struct Qdisc *);
         struct sk_buff *        (*peek)(struct Qdisc *);
-        unsigned int            (*drop)(struct Qdisc *);
 
         int                     (*init)(struct Qdisc *, struct nlattr *arg);
         void                    (*reset)(struct Qdisc *);
@@ -322,6 +300,14 @@ static inline spinlock_t *qdisc_root_sleeping_lock(const struct Qdisc *qdisc)
         return qdisc_lock(root);
 }
 
+static inline seqcount_t *qdisc_root_sleeping_running(const struct Qdisc *qdisc)
+{
+        struct Qdisc *root = qdisc_root_sleeping(qdisc);
+
+        ASSERT_RTNL();
+        return &root->running;
+}
+
 static inline struct net_device *qdisc_dev(const struct Qdisc *qdisc)
 {
         return qdisc->dev_queue->dev;
@@ -517,10 +503,11 @@ static inline void qdisc_calculate_pkt_len(struct sk_buff *skb,
 #endif
 }
 
-static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+                                struct sk_buff **to_free)
 {
         qdisc_calculate_pkt_len(skb, sch);
-        return sch->enqueue(skb, sch);
+        return sch->enqueue(skb, sch, to_free);
 }
 
 static inline bool qdisc_is_percpu_stats(const struct Qdisc *q)
@@ -645,40 +632,36 @@ static inline struct sk_buff *qdisc_dequeue_head(struct Qdisc *sch)
         return __qdisc_dequeue_head(sch, &sch->q);
 }
 
+/* Instead of calling kfree_skb() while root qdisc lock is held,
+ * queue the skb for future freeing at end of __dev_xmit_skb()
+ */
+static inline void __qdisc_drop(struct sk_buff *skb, struct sk_buff **to_free)
+{
+        skb->next = *to_free;
+        *to_free = skb;
+}
+
 static inline unsigned int __qdisc_queue_drop_head(struct Qdisc *sch,
-                                              struct sk_buff_head *list)
+                                                   struct sk_buff_head *list,
+                                                   struct sk_buff **to_free)
 {
         struct sk_buff *skb = __skb_dequeue(list);
 
         if (likely(skb != NULL)) {
                 unsigned int len = qdisc_pkt_len(skb);
+
                 qdisc_qstats_backlog_dec(sch, skb);
-                kfree_skb(skb);
+                __qdisc_drop(skb, to_free);
                 return len;
         }
 
         return 0;
 }
 
-static inline unsigned int qdisc_queue_drop_head(struct Qdisc *sch)
-{
-        return __qdisc_queue_drop_head(sch, &sch->q);
-}
-
-static inline struct sk_buff *__qdisc_dequeue_tail(struct Qdisc *sch,
-                                                   struct sk_buff_head *list)
-{
-        struct sk_buff *skb = __skb_dequeue_tail(list);
-
-        if (likely(skb != NULL))
-                qdisc_qstats_backlog_dec(sch, skb);
-
-        return skb;
-}
-
-static inline struct sk_buff *qdisc_dequeue_tail(struct Qdisc *sch)
+static inline unsigned int qdisc_queue_drop_head(struct Qdisc *sch,
+                                                 struct sk_buff **to_free)
 {
-        return __qdisc_dequeue_tail(sch, &sch->q);
+        return __qdisc_queue_drop_head(sch, &sch->q, to_free);
 }
 
 static inline struct sk_buff *qdisc_peek_head(struct Qdisc *sch)
@@ -718,19 +701,21 @@ static inline struct sk_buff *qdisc_dequeue_peeked(struct Qdisc *sch)
         return skb;
 }
 
-static inline void __qdisc_reset_queue(struct Qdisc *sch,
-                                       struct sk_buff_head *list)
+static inline void __qdisc_reset_queue(struct sk_buff_head *list)
 {
         /*
          * We do not know the backlog in bytes of this list, it
          * is up to the caller to correct it
          */
-        __skb_queue_purge(list);
+        if (!skb_queue_empty(list)) {
+                rtnl_kfree_skbs(list->next, list->prev);
+                __skb_queue_head_init(list);
+        }
 }
 
 static inline void qdisc_reset_queue(struct Qdisc *sch)
 {
-        __qdisc_reset_queue(sch, &sch->q);
+        __qdisc_reset_queue(&sch->q);
         sch->qstats.backlog = 0;
 }
 
@@ -751,46 +736,19 @@ static inline struct Qdisc *qdisc_replace(struct Qdisc *sch, struct Qdisc *new,
         return old;
 }
 
-static inline unsigned int __qdisc_queue_drop(struct Qdisc *sch,
-                                              struct sk_buff_head *list)
+static inline void rtnl_qdisc_drop(struct sk_buff *skb, struct Qdisc *sch)
 {
-        struct sk_buff *skb = __qdisc_dequeue_tail(sch, list);
-
-        if (likely(skb != NULL)) {
-                unsigned int len = qdisc_pkt_len(skb);
-                kfree_skb(skb);
-                return len;
-        }
-
-        return 0;
-}
-
-static inline unsigned int qdisc_queue_drop(struct Qdisc *sch)
-{
-        return __qdisc_queue_drop(sch, &sch->q);
-}
-
-static inline int qdisc_drop(struct sk_buff *skb, struct Qdisc *sch)
-{
-        kfree_skb(skb);
+        rtnl_kfree_skbs(skb, skb);
         qdisc_qstats_drop(sch);
-
-        return NET_XMIT_DROP;
 }
 
-static inline int qdisc_reshape_fail(struct sk_buff *skb, struct Qdisc *sch)
+
+static inline int qdisc_drop(struct sk_buff *skb, struct Qdisc *sch,
+                             struct sk_buff **to_free)
 {
+        __qdisc_drop(skb, to_free);
         qdisc_qstats_drop(sch);
 
-#ifdef CONFIG_NET_CLS_ACT
-        if (sch->reshape_fail == NULL || sch->reshape_fail(skb, sch))
-                goto drop;
-
-        return NET_XMIT_SUCCESS;
-
-drop:
-#endif
-        kfree_skb(skb);
         return NET_XMIT_DROP;
 }
 
diff --git a/include/net/sctp/sctp.h b/include/net/sctp/sctp.h
index b392ac8382f2..632e205ca54b 100644
--- a/include/net/sctp/sctp.h
+++ b/include/net/sctp/sctp.h
@@ -186,6 +186,10 @@ void sctp_assocs_proc_exit(struct net *net);
 int sctp_remaddr_proc_init(struct net *net);
 void sctp_remaddr_proc_exit(struct net *net);
 
+/*
+ * sctp/offload.c
+ */
+int sctp_offload_init(void);
 
 /*
  * Module global variables
diff --git a/include/net/sctp/structs.h b/include/net/sctp/structs.h
index 16b013a6191c..83c5ec58b93a 100644
--- a/include/net/sctp/structs.h
+++ b/include/net/sctp/structs.h
@@ -566,6 +566,9 @@ struct sctp_chunk {
         /* This points to the sk_buff containing the actual data.  */
         struct sk_buff *skb;
 
+        /* In case of GSO packets, this will store the head one */
+        struct sk_buff *head_skb;
+
         /* These are the SCTP headers by reverse order in a packet.
          * Note that some of these may happen more than once.  In that
          * case, we point at the "current" one, whatever that means
@@ -696,6 +699,8 @@ struct sctp_packet {
         size_t overhead;
         /* This is the total size of all chunks INCLUDING padding.  */
         size_t size;
+        /* This is the maximum size this packet may have */
+        size_t max_size;
 
         /* The packet is destined for this transport address.
          * The function we finally use to pass down to the next lower
diff --git a/include/net/tc_act/tc_defact.h b/include/net/tc_act/tc_defact.h
index 9763dcbb9bc3..ab9b5d6be67b 100644
--- a/include/net/tc_act/tc_defact.h
+++ b/include/net/tc_act/tc_defact.h
@@ -5,8 +5,8 @@
 
 struct tcf_defact {
         struct tcf_common       common;
-        u32                     tcfd_datalen;
-        void                    *tcfd_defdata;
+        u32             tcfd_datalen;
+        void            *tcfd_defdata;
 };
 #define to_defact(a) \
         container_of(a->priv, struct tcf_defact, common)
diff --git a/include/net/tcp.h b/include/net/tcp.h
index 0bcc70f4e1fb..a79894b66726 100644
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -767,6 +767,7 @@ struct tcp_skb_cb {
         union {
                 struct {
                         /* There is space for up to 20 bytes */
+                        __u32 in_flight;/* Bytes in flight when packet sent */
                 } tx;   /* only used for outgoing skbs */
                 union {
                         struct inet_skb_parm    h4;
@@ -859,6 +860,7 @@ union tcp_cc_info;
 struct ack_sample {
         u32 pkts_acked;
         s32 rtt_us;
+        u32 in_flight;
 };
 
 struct tcp_congestion_ops {
diff --git a/include/net/udp.h b/include/net/udp.h
index ae07f375370d..8894d7144189 100644
--- a/include/net/udp.h
+++ b/include/net/udp.h
@@ -160,8 +160,8 @@ void udp_set_csum(bool nocheck, struct sk_buff *skb,
 
 static inline void udp_csum_pull_header(struct sk_buff *skb)
 {
-        if (skb->ip_summed == CHECKSUM_NONE)
-                skb->csum = csum_partial(udp_hdr(skb), sizeof(struct udphdr),
+        if (!skb->csum_valid && skb->ip_summed == CHECKSUM_NONE)
+                skb->csum = csum_partial(skb->data, sizeof(struct udphdr),
                                          skb->csum);
         skb_pull_rcsum(skb, sizeof(struct udphdr));
         UDP_SKB_CB(skb)->cscov -= sizeof(struct udphdr);
diff --git a/include/net/udp_tunnel.h b/include/net/udp_tunnel.h
index 9d14f707e534..02c5be037451 100644
--- a/include/net/udp_tunnel.h
+++ b/include/net/udp_tunnel.h
@@ -84,6 +84,46 @@ struct udp_tunnel_sock_cfg {
 void setup_udp_tunnel_sock(struct net *net, struct socket *sock,
                            struct udp_tunnel_sock_cfg *sock_cfg);
 
+/* -- List of parsable UDP tunnel types --
+ *
+ * Adding to this list will result in serious debate.  The main issue is
+ * that this list is essentially a list of workarounds for either poorly
+ * designed tunnels, or poorly designed device offloads.
+ *
+ * The parsing supported via these types should really be used for Rx
+ * traffic only as the network stack will have already inserted offsets for
+ * the location of the headers in the skb.  In addition any ports that are
+ * pushed should be kept within the namespace without leaking to other
+ * devices such as VFs or other ports on the same device.
+ *
+ * It is strongly encouraged to use CHECKSUM_COMPLETE for Rx to avoid the
+ * need to use this for Rx checksum offload.  It should not be necessary to
+ * call this function to perform Tx offloads on outgoing traffic.
+ */
+enum udp_parsable_tunnel_type {
+        UDP_TUNNEL_TYPE_VXLAN,          /* RFC 7348 */
+        UDP_TUNNEL_TYPE_GENEVE,         /* draft-ietf-nvo3-geneve */
+        UDP_TUNNEL_TYPE_VXLAN_GPE,      /* draft-ietf-nvo3-vxlan-gpe */
+};
+
+struct udp_tunnel_info {
+        unsigned short type;
+        sa_family_t sa_family;
+        __be16 port;
+};
+
+/* Notify network devices of offloadable types */
+void udp_tunnel_push_rx_port(struct net_device *dev, struct socket *sock,
+                             unsigned short type);
+void udp_tunnel_notify_add_rx_port(struct socket *sock, unsigned short type);
+void udp_tunnel_notify_del_rx_port(struct socket *sock, unsigned short type);
+
+static inline void udp_tunnel_get_rx_info(struct net_device *dev)
+{
+        ASSERT_RTNL();
+        call_netdevice_notifiers(NETDEV_UDP_TUNNEL_PUSH_INFO, dev);
+}
+
 /* Transmit the skb using UDP encapsulation. */
 void udp_tunnel_xmit_skb(struct rtable *rt, struct sock *sk, struct sk_buff *skb,
                          __be32 src, __be32 dst, __u8 tos, __u8 ttl,
@@ -105,12 +145,14 @@ struct metadata_dst *udp_tun_rx_dst(struct sk_buff *skb, unsigned short family,
                                     __be16 flags, __be64 tunnel_id,
                                     int md_size);
 
+#ifdef CONFIG_INET
 static inline int udp_tunnel_handle_offloads(struct sk_buff *skb, bool udp_csum)
 {
         int type = udp_csum ? SKB_GSO_UDP_TUNNEL_CSUM : SKB_GSO_UDP_TUNNEL;
 
         return iptunnel_handle_offloads(skb, type);
 }
+#endif
 
 static inline void udp_tunnel_encap_enable(struct socket *sock)
 {
diff --git a/include/net/vxlan.h b/include/net/vxlan.h
index b8803165df91..b96d0360c095 100644
--- a/include/net/vxlan.h
+++ b/include/net/vxlan.h
@@ -1,13 +1,10 @@
 #ifndef __NET_VXLAN_H
 #define __NET_VXLAN_H 1
 
-#include <linux/ip.h>
-#include <linux/ipv6.h>
 #include <linux/if_vlan.h>
-#include <linux/skbuff.h>
-#include <linux/netdevice.h>
-#include <linux/udp.h>
+#include <net/udp_tunnel.h>
 #include <net/dst_metadata.h>
+#include <net/udp_tunnel.h>
 
 /* VXLAN protocol (RFC 7348) header:
  * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -392,12 +389,6 @@ static inline __be32 vxlan_compute_rco(unsigned int start, unsigned int offset)
         return vni_field;
 }
 
-static inline void vxlan_get_rx_port(struct net_device *netdev)
-{
-        ASSERT_RTNL();
-        call_netdevice_notifiers(NETDEV_OFFLOAD_PUSH_VXLAN, netdev);
-}
-
 static inline unsigned short vxlan_get_sk_family(struct vxlan_sock *vs)
 {
         return vs->sock->sk->sk_family;
diff --git a/include/soc/fsl/qe/immap_qe.h b/include/soc/fsl/qe/immap_qe.h
index bedbff891423..c76ef30b05ba 100644
--- a/include/soc/fsl/qe/immap_qe.h
+++ b/include/soc/fsl/qe/immap_qe.h
@@ -159,10 +159,7 @@ struct spi {
 
 /* SI */
 struct si1 {
-        __be16  siamr1;         /* SI1 TDMA mode register */
-        __be16  sibmr1;         /* SI1 TDMB mode register */
-        __be16  sicmr1;         /* SI1 TDMC mode register */
-        __be16  sidmr1;         /* SI1 TDMD mode register */
+        __be16  sixmr1[4];      /* SI1 TDMx (x = A B C D) mode register */
         u8      siglmr1_h;      /* SI1 global mode register high */
         u8      res0[0x1];
         u8      sicmdr1_h;      /* SI1 command register high */
diff --git a/include/soc/fsl/qe/qe.h b/include/soc/fsl/qe/qe.h
index 33b29ead3d55..70339d7958c0 100644
--- a/include/soc/fsl/qe/qe.h
+++ b/include/soc/fsl/qe/qe.h
@@ -80,6 +80,8 @@ enum qe_clock {
         QE_CLK22,               /* Clock 22 */
         QE_CLK23,               /* Clock 23 */
         QE_CLK24,               /* Clock 24 */
+        QE_RSYNC_PIN,           /* RSYNC from pin */
+        QE_TSYNC_PIN,           /* TSYNC from pin */
         QE_CLK_DUMMY
 };
 
@@ -242,6 +244,22 @@ static inline int qe_alive_during_sleep(void)
 #define qe_muram_addr cpm_muram_addr
 #define qe_muram_offset cpm_muram_offset
 
+#define qe_setbits32(_addr, _v) iowrite32be(ioread32be(_addr) |  (_v), (_addr))
+#define qe_clrbits32(_addr, _v) iowrite32be(ioread32be(_addr) & ~(_v), (_addr))
+
+#define qe_setbits16(_addr, _v) iowrite16be(ioread16be(_addr) |  (_v), (_addr))
+#define qe_clrbits16(_addr, _v) iowrite16be(ioread16be(_addr) & ~(_v), (_addr))
+
+#define qe_setbits8(_addr, _v) iowrite8(ioread8(_addr) |  (_v), (_addr))
+#define qe_clrbits8(_addr, _v) iowrite8(ioread8(_addr) & ~(_v), (_addr))
+
+#define qe_clrsetbits32(addr, clear, set) \
+        iowrite32be((ioread32be(addr) & ~(clear)) | (set), (addr))
+#define qe_clrsetbits16(addr, clear, set) \
+        iowrite16be((ioread16be(addr) & ~(clear)) | (set), (addr))
+#define qe_clrsetbits8(addr, clear, set) \
+        iowrite8((ioread8(addr) & ~(clear)) | (set), (addr))
+
 /* Structure that defines QE firmware binary files.
  *
  * See Documentation/powerpc/qe_firmware.txt for a description of these
@@ -639,6 +657,7 @@ struct ucc_slow_pram {
 #define UCC_SLOW_GUMR_L_MODE_QMC        0x00000002
 
 /* General UCC FAST Mode Register */
+#define UCC_FAST_GUMR_LOOPBACK  0x40000000
 #define UCC_FAST_GUMR_TCI       0x20000000
 #define UCC_FAST_GUMR_TRX       0x10000000
 #define UCC_FAST_GUMR_TTX       0x08000000
diff --git a/include/soc/fsl/qe/qe_tdm.h b/include/soc/fsl/qe/qe_tdm.h
new file mode 100644
index 000000000000..a1664b635f1a
--- /dev/null
+++ b/include/soc/fsl/qe/qe_tdm.h
@@ -0,0 +1,94 @@
+/*
+ * Internal header file for QE TDM mode routines.
+ *
+ * Copyright (C) 2016 Freescale Semiconductor, Inc. All rights reserved.
+ *
+ * Authors:     Zhao Qiang <qiang.zhao@nxp.com>
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version
+ */
+
+#ifndef _QE_TDM_H_
+#define _QE_TDM_H_
+
+#include <linux/kernel.h>
+#include <linux/list.h>
+
+#include <soc/fsl/qe/immap_qe.h>
+#include <soc/fsl/qe/qe.h>
+
+#include <soc/fsl/qe/ucc.h>
+#include <soc/fsl/qe/ucc_fast.h>
+
+/* SI RAM entries */
+#define SIR_LAST        0x0001
+#define SIR_BYTE        0x0002
+#define SIR_CNT(x)      ((x) << 2)
+#define SIR_CSEL(x)     ((x) << 5)
+#define SIR_SGS         0x0200
+#define SIR_SWTR        0x4000
+#define SIR_MCC         0x8000
+#define SIR_IDLE        0
+
+/* SIxMR fields */
+#define SIMR_SAD(x) ((x) << 12)
+#define SIMR_SDM_NORMAL 0x0000
+#define SIMR_SDM_INTERNAL_LOOPBACK      0x0800
+#define SIMR_SDM_MASK   0x0c00
+#define SIMR_CRT        0x0040
+#define SIMR_SL         0x0020
+#define SIMR_CE         0x0010
+#define SIMR_FE         0x0008
+#define SIMR_GM         0x0004
+#define SIMR_TFSD(n)    (n)
+#define SIMR_RFSD(n)    ((n) << 8)
+
+enum tdm_ts_t {
+        TDM_TX_TS,
+        TDM_RX_TS
+};
+
+enum tdm_framer_t {
+        TDM_FRAMER_T1,
+        TDM_FRAMER_E1
+};
+
+enum tdm_mode_t {
+        TDM_INTERNAL_LOOPBACK,
+        TDM_NORMAL
+};
+
+struct si_mode_info {
+        u8 simr_rfsd;
+        u8 simr_tfsd;
+        u8 simr_crt;
+        u8 simr_sl;
+        u8 simr_ce;
+        u8 simr_fe;
+        u8 simr_gm;
+};
+
+struct ucc_tdm_info {
+        struct ucc_fast_info uf_info;
+        struct si_mode_info si_info;
+};
+
+struct ucc_tdm {
+        u16 tdm_port;           /* port for this tdm:TDMA,TDMB */
+        u32 siram_entry_id;
+        u16 __iomem *siram;
+        struct si1 __iomem *si_regs;
+        enum tdm_framer_t tdm_framer_type;
+        enum tdm_mode_t tdm_mode;
+        u8 num_of_ts;           /* the number of timeslots in this tdm frame */
+        u32 tx_ts_mask;         /* tx time slot mask */
+        u32 rx_ts_mask;         /* rx time slot mask */
+};
+
+int ucc_of_parse_tdm(struct device_node *np, struct ucc_tdm *utdm,
+                     struct ucc_tdm_info *ut_info);
+void ucc_tdm_init(struct ucc_tdm *utdm, struct ucc_tdm_info *ut_info);
+#endif
diff --git a/include/soc/fsl/qe/ucc.h b/include/soc/fsl/qe/ucc.h
index 894f14cbb044..6bbbb597f2af 100644
--- a/include/soc/fsl/qe/ucc.h
+++ b/include/soc/fsl/qe/ucc.h
@@ -41,6 +41,10 @@ int ucc_set_qe_mux_mii_mng(unsigned int ucc_num);
 
 int ucc_set_qe_mux_rxtx(unsigned int ucc_num, enum qe_clock clock,
         enum comm_dir mode);
+int ucc_set_tdm_rxtx_clk(unsigned int tdm_num, enum qe_clock clock,
+                         enum comm_dir mode);
+int ucc_set_tdm_rxtx_sync(unsigned int tdm_num, enum qe_clock clock,
+                          enum comm_dir mode);
 
 int ucc_mux_set_grant_tsa_bkpt(unsigned int ucc_num, int set, u32 mask);
 
diff --git a/include/soc/fsl/qe/ucc_fast.h b/include/soc/fsl/qe/ucc_fast.h
index df8ea7958c63..3ee9e7c1a7d7 100644
--- a/include/soc/fsl/qe/ucc_fast.h
+++ b/include/soc/fsl/qe/ucc_fast.h
@@ -21,19 +21,37 @@
 
 #include <soc/fsl/qe/ucc.h>
 
-/* Receive BD's status */
+/* Receive BD's status and length*/
 #define R_E     0x80000000      /* buffer empty */
 #define R_W     0x20000000      /* wrap bit */
 #define R_I     0x10000000      /* interrupt on reception */
 #define R_L     0x08000000      /* last */
 #define R_F     0x04000000      /* first */
 
-/* transmit BD's status */
+/* transmit BD's status and length*/
 #define T_R     0x80000000      /* ready bit */
 #define T_W     0x20000000      /* wrap bit */
 #define T_I     0x10000000      /* interrupt on completion */
 #define T_L     0x08000000      /* last */
 
+/* Receive BD's status */
+#define R_E_S   0x8000  /* buffer empty */
+#define R_W_S   0x2000  /* wrap bit */
+#define R_I_S   0x1000  /* interrupt on reception */
+#define R_L_S   0x0800  /* last */
+#define R_F_S   0x0400  /* first */
+#define R_CM_S  0x0200  /* continuous mode */
+#define R_CR_S  0x0004  /* crc */
+#define R_OV_S  0x0002  /* crc */
+
+/* transmit BD's status */
+#define T_R_S   0x8000  /* ready bit */
+#define T_W_S   0x2000  /* wrap bit */
+#define T_I_S   0x1000  /* interrupt on completion */
+#define T_L_S   0x0800  /* last */
+#define T_TC_S  0x0400  /* crc */
+#define T_TM_S  0x0200  /* continuous mode */
+
 /* Rx Data buffer must be 4 bytes aligned in most cases */
 #define UCC_FAST_RX_ALIGN                       4
 #define UCC_FAST_MRBLR_ALIGNMENT                4
@@ -118,9 +136,12 @@ enum ucc_fast_transparent_tcrc {
 /* Fast UCC initialization structure */
 struct ucc_fast_info {
         int ucc_num;
+        int tdm_num;
         enum qe_clock rx_clock;
         enum qe_clock tx_clock;
-        u32 regs;
+        enum qe_clock rx_sync;
+        enum qe_clock tx_sync;
+        resource_size_t regs;
         int irq;
         u32 uccm_mask;
         int bd_mem_part;
diff --git a/include/uapi/linux/can/bcm.h b/include/uapi/linux/can/bcm.h
index 7a291dc1ff15..cefb304414ba 100644
--- a/include/uapi/linux/can/bcm.h
+++ b/include/uapi/linux/can/bcm.h
@@ -99,5 +99,6 @@ enum {
 #define RX_ANNOUNCE_RESUME  0x0100
 #define TX_RESET_MULTI_IDX  0x0200
 #define RX_RTR_FRAME        0x0400
+#define CAN_FD_FRAME        0x0800
 
 #endif /* !_UAPI_CAN_BCM_H */
diff --git a/include/uapi/linux/ethtool.h b/include/uapi/linux/ethtool.h
index 5f030b46cff4..b8f38e84d93a 100644
--- a/include/uapi/linux/ethtool.h
+++ b/include/uapi/linux/ethtool.h
@@ -1362,6 +1362,7 @@ enum ethtool_link_mode_bit_indices {
         ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT        = 37,
         ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT        = 38,
         ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT    = 39,
+        ETHTOOL_LINK_MODE_50000baseSR2_Full_BIT         = 40,
 
         /* Last allowed bit for __ETHTOOL_LINK_MODE_LEGACY_MASK is bit
          * 31. Please do NOT define any SUPPORTED_* or ADVERTISED_*
@@ -1370,7 +1371,7 @@ enum ethtool_link_mode_bit_indices {
          */
 
         __ETHTOOL_LINK_MODE_LAST
-          = ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT,
+          = ETHTOOL_LINK_MODE_50000baseSR2_Full_BIT,
 };
 
 #define __ETHTOOL_LINK_MODE_LEGACY_MASK(base_name)      \
diff --git a/include/uapi/linux/fib_rules.h b/include/uapi/linux/fib_rules.h
index 620c8a5ddc00..14404b3ebb89 100644
--- a/include/uapi/linux/fib_rules.h
+++ b/include/uapi/linux/fib_rules.h
@@ -50,6 +50,7 @@ enum {
         FRA_FWMASK,     /* mask for netfilter mark */
         FRA_OIFNAME,
         FRA_PAD,
+        FRA_L3MDEV,     /* iif or oif is l3mdev goto its table */
         __FRA_MAX
 };
 
diff --git a/include/uapi/linux/icmp.h b/include/uapi/linux/icmp.h
index 16fff055f734..fddd9d736284 100644
--- a/include/uapi/linux/icmp.h
+++ b/include/uapi/linux/icmp.h
@@ -79,6 +79,7 @@ struct icmphdr {
                 __be16  __unused;
                 __be16  mtu;
         } frag;
+        __u8    reserved[4];
   } un;
 };
 
diff --git a/include/uapi/linux/if_tunnel.h b/include/uapi/linux/if_tunnel.h
index af4de90ba27d..1046f5515174 100644
--- a/include/uapi/linux/if_tunnel.h
+++ b/include/uapi/linux/if_tunnel.h
@@ -113,6 +113,7 @@ enum {
         IFLA_GRE_ENCAP_SPORT,
         IFLA_GRE_ENCAP_DPORT,
         IFLA_GRE_COLLECT_METADATA,
+        IFLA_GRE_IGNORE_DF,
         __IFLA_GRE_MAX,
 };
 
diff --git a/include/uapi/linux/inet_diag.h b/include/uapi/linux/inet_diag.h
index a16643705669..abbd1dc5d683 100644
--- a/include/uapi/linux/inet_diag.h
+++ b/include/uapi/linux/inet_diag.h
@@ -72,6 +72,7 @@ enum {
         INET_DIAG_BC_AUTO,
         INET_DIAG_BC_S_COND,
         INET_DIAG_BC_D_COND,
+        INET_DIAG_BC_DEV_COND,   /* u32 ifindex */
 };
 
 struct inet_diag_hostcond {
diff --git a/include/uapi/linux/netlink_diag.h b/include/uapi/linux/netlink_diag.h
index d79399394b46..76b4d87c83a8 100644
--- a/include/uapi/linux/netlink_diag.h
+++ b/include/uapi/linux/netlink_diag.h
@@ -49,6 +49,7 @@ enum {
 #define NDIAG_SHOW_MEMINFO      0x00000001 /* show memory info of a socket */
 #define NDIAG_SHOW_GROUPS       0x00000002 /* show groups of a netlink socket */
 #ifndef __KERNEL__
+/* deprecated since 4.6 */
 #define NDIAG_SHOW_RING_CFG     0x00000004 /* show ring configuration */
 #endif
 
diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h
index e23d78685a01..53c8278827a0 100644
--- a/include/uapi/linux/nl80211.h
+++ b/include/uapi/linux/nl80211.h
@@ -493,7 +493,12 @@
  *      This attribute is ignored if driver does not support roam scan.
  *      It is also sent as an event, with the BSSID and response IEs when the
  *      connection is established or failed to be established. This can be
- *      determined by the STATUS_CODE attribute.
+ *      determined by the %NL80211_ATTR_STATUS_CODE attribute (0 = success,
+ *      non-zero = failure). If %NL80211_ATTR_TIMED_OUT is included in the
+ *      event, the connection attempt failed due to not being able to initiate
+ *      authentication/association or not receiving a response from the AP.
+ *      Non-zero %NL80211_ATTR_STATUS_CODE value is indicated in that case as
+ *      well to remain backwards compatible.
  * @NL80211_CMD_ROAM: request that the card roam (currently not implemented),
  *      sent as an event when the card/driver roamed by itself.
  * @NL80211_CMD_DISCONNECT: drop a given connection; also used to notify
@@ -1819,6 +1824,11 @@ enum nl80211_commands {
  *
  * @NL80211_ATTR_PAD: attribute used for padding for 64-bit alignment
  *
+ * @NL80211_ATTR_IFTYPE_EXT_CAPA: Nested attribute of the following attributes:
+ *      %NL80211_ATTR_IFTYPE, %NL80211_ATTR_EXT_CAPA,
+ *      %NL80211_ATTR_EXT_CAPA_MASK, to specify the extended capabilities per
+ *      interface type.
+ *
  * @NUM_NL80211_ATTR: total number of nl80211_attrs available
  * @NL80211_ATTR_MAX: highest attribute number currently defined
  * @__NL80211_ATTR_AFTER_LAST: internal use
@@ -2201,6 +2211,8 @@ enum nl80211_attrs {
 
         NL80211_ATTR_PAD,
 
+        NL80211_ATTR_IFTYPE_EXT_CAPA,
+
         /* add attributes here, update the policy in nl80211.c */
 
         __NL80211_ATTR_AFTER_LAST,
diff --git a/include/uapi/linux/openvswitch.h b/include/uapi/linux/openvswitch.h
index bb0d515b7654..d95a3018f6a1 100644
--- a/include/uapi/linux/openvswitch.h
+++ b/include/uapi/linux/openvswitch.h
@@ -166,6 +166,7 @@ enum ovs_packet_cmd {
  * output port is actually a tunnel port. Contains the output tunnel key
  * extracted from the packet as nested %OVS_TUNNEL_KEY_ATTR_* attributes.
  * @OVS_PACKET_ATTR_MRU: Present for an %OVS_PACKET_CMD_ACTION and
+ * @OVS_PACKET_ATTR_LEN: Packet size before truncation.
  * %OVS_PACKET_ATTR_USERSPACE action specify the Maximum received fragment
  * size.
  *
@@ -185,6 +186,7 @@ enum ovs_packet_attr {
         OVS_PACKET_ATTR_PROBE,      /* Packet operation is a feature probe,
                                        error logging should be suppressed. */
         OVS_PACKET_ATTR_MRU,        /* Maximum received IP fragment size. */
+        OVS_PACKET_ATTR_LEN,            /* Packet size before truncation. */
         __OVS_PACKET_ATTR_MAX
 };
 
@@ -580,6 +582,10 @@ enum ovs_userspace_attr {
 
 #define OVS_USERSPACE_ATTR_MAX (__OVS_USERSPACE_ATTR_MAX - 1)
 
+struct ovs_action_trunc {
+        uint32_t max_len; /* Max packet size in bytes. */
+};
+
 /**
  * struct ovs_action_push_mpls - %OVS_ACTION_ATTR_PUSH_MPLS action argument.
  * @mpls_lse: MPLS label stack entry to push.
@@ -703,6 +709,7 @@ enum ovs_nat_attr {
  * enum ovs_action_attr - Action types.
  *
  * @OVS_ACTION_ATTR_OUTPUT: Output packet to port.
+ * @OVS_ACTION_ATTR_TRUNC: Output packet to port with truncated packet size.
  * @OVS_ACTION_ATTR_USERSPACE: Send packet to userspace according to nested
  * %OVS_USERSPACE_ATTR_* attributes.
  * @OVS_ACTION_ATTR_SET: Replaces the contents of an existing header.  The
@@ -756,6 +763,7 @@ enum ovs_action_attr {
                                        * The data must be zero for the unmasked
                                        * bits. */
         OVS_ACTION_ATTR_CT,           /* Nested OVS_CT_ATTR_* . */
+        OVS_ACTION_ATTR_TRUNC,        /* u32 struct ovs_action_trunc. */
 
         __OVS_ACTION_ATTR_MAX,        /* Nothing past this will be accepted
                                        * from userspace. */
diff --git a/include/uapi/linux/pkt_cls.h b/include/uapi/linux/pkt_cls.h
index f4297c8a42fe..5702e933dc07 100644
--- a/include/uapi/linux/pkt_cls.h
+++ b/include/uapi/linux/pkt_cls.h
@@ -115,8 +115,8 @@ struct tc_police {
         __u32                   mtu;
         struct tc_ratespec      rate;
         struct tc_ratespec      peakrate;
-        int                     refcnt;
-        int                     bindcnt;
+        int                     refcnt;
+        int                     bindcnt;
         __u32                   capab;
 };
 
@@ -124,10 +124,11 @@ struct tcf_t {
         __u64   install;
         __u64   lastuse;
         __u64   expires;
+        __u64   firstuse;
 };
 
 struct tc_cnt {
-        int                   refcnt; 
+        int                   refcnt;
         int                   bindcnt;
 };
 
diff --git a/include/uapi/linux/virtio_net.h b/include/uapi/linux/virtio_net.h
index ec32293a00db..fc353b518288 100644
--- a/include/uapi/linux/virtio_net.h
+++ b/include/uapi/linux/virtio_net.h
@@ -1,5 +1,5 @@
-#ifndef _LINUX_VIRTIO_NET_H
-#define _LINUX_VIRTIO_NET_H
+#ifndef _UAPI_LINUX_VIRTIO_NET_H
+#define _UAPI_LINUX_VIRTIO_NET_H
 /* This header is BSD licensed so anyone can use the definitions to implement
  * compatible drivers/servers.
  *
@@ -35,6 +35,7 @@
 #define VIRTIO_NET_F_CSUM       0       /* Host handles pkts w/ partial csum */
 #define VIRTIO_NET_F_GUEST_CSUM 1       /* Guest handles pkts w/ partial csum */
 #define VIRTIO_NET_F_CTRL_GUEST_OFFLOADS 2 /* Dynamic offload configuration. */
+#define VIRTIO_NET_F_MTU        3       /* Initial MTU advice */
 #define VIRTIO_NET_F_MAC        5       /* Host has given MAC address. */
 #define VIRTIO_NET_F_GUEST_TSO4 7       /* Guest can handle TSOv4 in. */
 #define VIRTIO_NET_F_GUEST_TSO6 8       /* Guest can handle TSOv6 in. */
@@ -73,6 +74,8 @@ struct virtio_net_config {
          * Legal values are between 1 and 0x8000
          */
         __u16 max_virtqueue_pairs;
+        /* Default maximum transmit unit advice */
+        __u16 mtu;
 } __attribute__((packed));
 
 /*
@@ -242,4 +245,4 @@ struct virtio_net_ctrl_mq {
 #define VIRTIO_NET_CTRL_GUEST_OFFLOADS   5
 #define VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET        0
 
-#endif /* _LINUX_VIRTIO_NET_H */
+#endif /* _UAPI_LINUX_VIRTIO_NET_H */
diff --git a/include/uapi/linux/wireless.h b/include/uapi/linux/wireless.h
index c1592e3e4036..d9ecd7c6d691 100644
--- a/include/uapi/linux/wireless.h
+++ b/include/uapi/linux/wireless.h
@@ -670,8 +670,7 @@
 /*
  *      Generic format for most parameters that fit in an int
  */
-struct  iw_param
-{
+struct iw_param {
   __s32         value;          /* The value of the parameter itself */
   __u8          fixed;          /* Hardware should not use auto select */
   __u8          disabled;       /* Disable the feature */
@@ -682,8 +681,7 @@ struct	iw_param
  *      For all data larger than 16 octets, we need to use a
  *      pointer to memory allocated in user space.
  */
-struct  iw_point
-{
+struct iw_point {
   void __user   *pointer;       /* Pointer to the data  (in user space) */
   __u16         length;         /* number of fields or size in bytes */
   __u16         flags;          /* Optional params */
@@ -698,8 +696,7 @@ struct	iw_point
  *      of 10 to get 'm' lower than 10^9, with 'm'= f / (10^'e')...
  *      The power of 10 is in 'e', the result of the division is in 'm'.
  */
-struct  iw_freq
-{
+struct iw_freq {
         __s32           m;              /* Mantissa */
         __s16           e;              /* Exponent */
         __u8            i;              /* List index (when in range struct) */
@@ -709,8 +706,7 @@ struct	iw_freq
 /*
  *      Quality of the link
  */
-struct  iw_quality
-{
+struct iw_quality {
         __u8            qual;           /* link quality (%retries, SNR,
                                            %missed beacons or better...) */
         __u8            level;          /* signal level (dBm) */
@@ -725,8 +721,7 @@ struct	iw_quality
  *      is already pretty exhaustive, and you should use that first.
  *      This is only additional stats...
  */
-struct  iw_discarded
-{
+struct iw_discarded {
         __u32           nwid;           /* Rx : Wrong nwid/essid */
         __u32           code;           /* Rx : Unable to code/decode (WEP) */
         __u32           fragment;       /* Rx : Can't perform MAC reassembly */
@@ -738,16 +733,14 @@ struct	iw_discarded
  *      Packet/Time period missed in the wireless adapter due to
  *      "wireless" specific problems...
  */
-struct  iw_missed
-{
+struct iw_missed {
         __u32           beacon;         /* Missed beacons/superframe */
 };
 
 /*
  *      Quality range (for spy threshold)
  */
-struct  iw_thrspy
-{
+struct iw_thrspy {
         struct sockaddr         addr;           /* Source address (hw/mac) */
         struct iw_quality       qual;           /* Quality of the link */
         struct iw_quality       low;            /* Low threshold */
@@ -765,8 +758,7 @@ struct	iw_thrspy
  *      Especially, scan results are required to include an entry for the
  *      current BSS if the driver is in Managed mode and associated with an AP.
  */
-struct  iw_scan_req
-{
+struct iw_scan_req {
         __u8            scan_type; /* IW_SCAN_TYPE_{ACTIVE,PASSIVE} */
         __u8            essid_len;
         __u8            num_channels; /* num entries in channel_list;
@@ -827,8 +819,7 @@ struct	iw_scan_req
  *      RX_SEQ_VALID for SIOCGIWENCODEEXT are optional, but can be useful for
  *      debugging/testing.
  */
-struct  iw_encode_ext
-{
+struct iw_encode_ext {
         __u32           ext_flags; /* IW_ENCODE_EXT_* */
         __u8            tx_seq[IW_ENCODE_SEQ_MAX_SIZE]; /* LSB first */
         __u8            rx_seq[IW_ENCODE_SEQ_MAX_SIZE]; /* LSB first */
@@ -841,8 +832,7 @@ struct	iw_encode_ext
 };
 
 /* SIOCSIWMLME data */
-struct  iw_mlme
-{
+struct iw_mlme {
         __u16           cmd; /* IW_MLME_* */
         __u16           reason_code;
         struct sockaddr addr;
@@ -855,16 +845,14 @@ struct	iw_mlme
 
 #define IW_PMKID_LEN    16
 
-struct  iw_pmksa
-{
+struct iw_pmksa {
         __u32           cmd; /* IW_PMKSA_* */
         struct sockaddr bssid;
         __u8            pmkid[IW_PMKID_LEN];
 };
 
 /* IWEVMICHAELMICFAILURE data */
-struct  iw_michaelmicfailure
-{
+struct iw_michaelmicfailure {
         __u32           flags;
         struct sockaddr src_addr;
         __u8            tsc[IW_ENCODE_SEQ_MAX_SIZE]; /* LSB first */
@@ -872,8 +860,7 @@ struct	iw_michaelmicfailure
 
 /* IWEVPMKIDCAND data */
 #define IW_PMKID_CAND_PREAUTH   0x00000001 /* RNS pre-authentication enabled */
-struct  iw_pmkid_cand
-{
+struct iw_pmkid_cand {
         __u32           flags; /* IW_PMKID_CAND_* */
         __u32           index; /* the smaller the index, the higher the
                                 * priority */
@@ -884,8 +871,7 @@ struct	iw_pmkid_cand
 /*
  * Wireless statistics (used for /proc/net/wireless)
  */
-struct  iw_statistics
-{
+struct iw_statistics {
         __u16           status;         /* Status
                                          * - device dependent for now */
 
@@ -897,7 +883,7 @@ struct	iw_statistics
 
 /* ------------------------ IOCTL REQUEST ------------------------ */
 /*
- * This structure defines the payload of an ioctl, and is used 
+ * This structure defines the payload of an ioctl, and is used
  * below.
  *
  * Note that this structure should fit on the memory footprint
@@ -906,8 +892,7 @@ struct	iw_statistics
  * You should check this when increasing the structures defined
  * above in this file...
  */
-union   iwreq_data
-{
+union iwreq_data {
         /* Config - generic */
         char            name[IFNAMSIZ];
         /* Name : used to verify the presence of  wireless extensions.
@@ -944,15 +929,14 @@ union	iwreq_data
  * convenience...
  * Do I need to remind you about structure size (32 octets) ?
  */
-struct  iwreq 
-{
+struct iwreq {
         union
         {
                 char    ifrn_name[IFNAMSIZ];    /* if name, e.g. "eth0" */
         } ifr_ifrn;
 
         /* Data part (defined just above) */
-        union   iwreq_data      u;
+        union iwreq_data        u;
 };
 
 /* -------------------------- IOCTL DATA -------------------------- */
@@ -965,8 +949,7 @@ struct	iwreq
  *      Range of parameters
  */
 
-struct  iw_range
-{
+struct iw_range {
         /* Informative stuff (to choose between different interface) */
         __u32           throughput;     /* To give an idea... */
         /* In theory this value should be the maximum benchmarked
@@ -1069,9 +1052,8 @@ struct	iw_range
 /*
  * Private ioctl interface information
  */
- 
-struct  iw_priv_args
-{
+
+struct iw_priv_args {
         __u32           cmd;            /* Number of the ioctl to issue */
         __u16           set_args;       /* Type and number of args */
         __u16           get_args;       /* Type and number of args */
@@ -1088,8 +1070,7 @@ struct	iw_priv_args
 /*
  * A Wireless Event. Contains basically the same data as the ioctl...
  */
-struct iw_event
-{
+struct iw_event {
         __u16           len;                    /* Real length of this stuff */
         __u16           cmd;                    /* Wireless IOCTL */
         union iwreq_data        u;              /* IOCTL fixed payload */
diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c
index 76d5a794e426..5af30732697b 100644
--- a/kernel/bpf/arraymap.c
+++ b/kernel/bpf/arraymap.c
@@ -328,8 +328,8 @@ static void *fd_array_map_lookup_elem(struct bpf_map *map, void *key)
 }
 
 /* only called from syscall */
-static int fd_array_map_update_elem(struct bpf_map *map, void *key,
-                                    void *value, u64 map_flags)
+int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,
+                                 void *key, void *value, u64 map_flags)
 {
         struct bpf_array *array = container_of(map, struct bpf_array, map);
         void *new_ptr, *old_ptr;
@@ -342,7 +342,7 @@ static int fd_array_map_update_elem(struct bpf_map *map, void *key,
                 return -E2BIG;
 
         ufd = *(u32 *)value;
-        new_ptr = map->ops->map_fd_get_ptr(map, ufd);
+        new_ptr = map->ops->map_fd_get_ptr(map, map_file, ufd);
         if (IS_ERR(new_ptr))
                 return PTR_ERR(new_ptr);
 
@@ -371,10 +371,12 @@ static int fd_array_map_delete_elem(struct bpf_map *map, void *key)
         }
 }
 
-static void *prog_fd_array_get_ptr(struct bpf_map *map, int fd)
+static void *prog_fd_array_get_ptr(struct bpf_map *map,
+                                   struct file *map_file, int fd)
 {
         struct bpf_array *array = container_of(map, struct bpf_array, map);
         struct bpf_prog *prog = bpf_prog_get(fd);
+
         if (IS_ERR(prog))
                 return prog;
 
@@ -382,6 +384,7 @@ static void *prog_fd_array_get_ptr(struct bpf_map *map, int fd)
                 bpf_prog_put(prog);
                 return ERR_PTR(-EINVAL);
         }
+
         return prog;
 }
 
@@ -407,7 +410,6 @@ static const struct bpf_map_ops prog_array_ops = {
         .map_free = fd_array_map_free,
         .map_get_next_key = array_map_get_next_key,
         .map_lookup_elem = fd_array_map_lookup_elem,
-        .map_update_elem = fd_array_map_update_elem,
         .map_delete_elem = fd_array_map_delete_elem,
         .map_fd_get_ptr = prog_fd_array_get_ptr,
         .map_fd_put_ptr = prog_fd_array_put_ptr,
@@ -425,59 +427,105 @@ static int __init register_prog_array_map(void)
 }
 late_initcall(register_prog_array_map);
 
-static void perf_event_array_map_free(struct bpf_map *map)
+static struct bpf_event_entry *bpf_event_entry_gen(struct file *perf_file,
+                                                   struct file *map_file)
 {
-        bpf_fd_array_map_clear(map);
-        fd_array_map_free(map);
+        struct bpf_event_entry *ee;
+
+        ee = kzalloc(sizeof(*ee), GFP_KERNEL);
+        if (ee) {
+                ee->event = perf_file->private_data;
+                ee->perf_file = perf_file;
+                ee->map_file = map_file;
+        }
+
+        return ee;
 }
 
-static void *perf_event_fd_array_get_ptr(struct bpf_map *map, int fd)
+static void __bpf_event_entry_free(struct rcu_head *rcu)
 {
-        struct perf_event *event;
-        const struct perf_event_attr *attr;
-        struct file *file;
+        struct bpf_event_entry *ee;
 
-        file = perf_event_get(fd);
-        if (IS_ERR(file))
-                return file;
+        ee = container_of(rcu, struct bpf_event_entry, rcu);
+        fput(ee->perf_file);
+        kfree(ee);
+}
 
-        event = file->private_data;
+static void bpf_event_entry_free_rcu(struct bpf_event_entry *ee)
+{
+        call_rcu(&ee->rcu, __bpf_event_entry_free);
+}
 
-        attr = perf_event_attrs(event);
-        if (IS_ERR(attr))
-                goto err;
+static void *perf_event_fd_array_get_ptr(struct bpf_map *map,
+                                         struct file *map_file, int fd)
+{
+        const struct perf_event_attr *attr;
+        struct bpf_event_entry *ee;
+        struct perf_event *event;
+        struct file *perf_file;
 
-        if (attr->inherit)
-                goto err;
+        perf_file = perf_event_get(fd);
+        if (IS_ERR(perf_file))
+                return perf_file;
 
-        if (attr->type == PERF_TYPE_RAW)
-                return file;
+        event = perf_file->private_data;
+        ee = ERR_PTR(-EINVAL);
 
-        if (attr->type == PERF_TYPE_HARDWARE)
-                return file;
+        attr = perf_event_attrs(event);
+        if (IS_ERR(attr) || attr->inherit)
+                goto err_out;
+
+        switch (attr->type) {
+        case PERF_TYPE_SOFTWARE:
+                if (attr->config != PERF_COUNT_SW_BPF_OUTPUT)
+                        goto err_out;
+                /* fall-through */
+        case PERF_TYPE_RAW:
+        case PERF_TYPE_HARDWARE:
+                ee = bpf_event_entry_gen(perf_file, map_file);
+                if (ee)
+                        return ee;
+                ee = ERR_PTR(-ENOMEM);
+                /* fall-through */
+        default:
+                break;
+        }
 
-        if (attr->type == PERF_TYPE_SOFTWARE &&
-            attr->config == PERF_COUNT_SW_BPF_OUTPUT)
-                return file;
-err:
-        fput(file);
-        return ERR_PTR(-EINVAL);
+err_out:
+        fput(perf_file);
+        return ee;
 }
 
 static void perf_event_fd_array_put_ptr(void *ptr)
 {
-        fput((struct file *)ptr);
+        bpf_event_entry_free_rcu(ptr);
+}
+
+static void perf_event_fd_array_release(struct bpf_map *map,
+                                        struct file *map_file)
+{
+        struct bpf_array *array = container_of(map, struct bpf_array, map);
+        struct bpf_event_entry *ee;
+        int i;
+
+        rcu_read_lock();
+        for (i = 0; i < array->map.max_entries; i++) {
+                ee = READ_ONCE(array->ptrs[i]);
+                if (ee && ee->map_file == map_file)
+                        fd_array_map_delete_elem(map, &i);
+        }
+        rcu_read_unlock();
 }
 
 static const struct bpf_map_ops perf_event_array_ops = {
         .map_alloc = fd_array_map_alloc,
-        .map_free = perf_event_array_map_free,
+        .map_free = fd_array_map_free,
         .map_get_next_key = array_map_get_next_key,
         .map_lookup_elem = fd_array_map_lookup_elem,
-        .map_update_elem = fd_array_map_update_elem,
         .map_delete_elem = fd_array_map_delete_elem,
         .map_fd_get_ptr = perf_event_fd_array_get_ptr,
         .map_fd_put_ptr = perf_event_fd_array_put_ptr,
+        .map_release = perf_event_fd_array_release,
 };
 
 static struct bpf_map_type_list perf_event_array_type __read_mostly = {
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 46ecce4b79ed..c23a4e9311b3 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -124,7 +124,12 @@ void bpf_map_put_with_uref(struct bpf_map *map)
 
 static int bpf_map_release(struct inode *inode, struct file *filp)
 {
-        bpf_map_put_with_uref(filp->private_data);
+        struct bpf_map *map = filp->private_data;
+
+        if (map->ops->map_release)
+                map->ops->map_release(map, filp);
+
+        bpf_map_put_with_uref(map);
         return 0;
 }
 
@@ -387,6 +392,12 @@ static int map_update_elem(union bpf_attr *attr)
                 err = bpf_percpu_hash_update(map, key, value, attr->flags);
         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
                 err = bpf_percpu_array_update(map, key, value, attr->flags);
+        } else if (map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY ||
+                   map->map_type == BPF_MAP_TYPE_PROG_ARRAY) {
+                rcu_read_lock();
+                err = bpf_fd_array_map_update_elem(map, f.file, key, value,
+                                                   attr->flags);
+                rcu_read_unlock();
         } else {
                 rcu_read_lock();
                 err = map->ops->map_update_elem(map, key, value, attr->flags);
diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index 26f603da7e26..3de25fbed785 100644
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -192,18 +192,17 @@ static u64 bpf_perf_event_read(u64 r1, u64 index, u64 r3, u64 r4, u64 r5)
 {
         struct bpf_map *map = (struct bpf_map *) (unsigned long) r1;
         struct bpf_array *array = container_of(map, struct bpf_array, map);
+        struct bpf_event_entry *ee;
         struct perf_event *event;
-        struct file *file;
 
         if (unlikely(index >= array->map.max_entries))
                 return -E2BIG;
 
-        file = READ_ONCE(array->ptrs[index]);
-        if (unlikely(!file))
+        ee = READ_ONCE(array->ptrs[index]);
+        if (unlikely(!ee))
                 return -ENOENT;
 
-        event = file->private_data;
-
+        event = ee->event;
         /* make sure event is local and doesn't have pmu::count */
         if (event->oncpu != smp_processor_id() ||
             event->pmu->count)
@@ -237,8 +236,8 @@ static u64 bpf_perf_event_output(u64 r1, u64 r2, u64 flags, u64 r4, u64 size)
         u64 index = flags & BPF_F_INDEX_MASK;
         void *data = (void *) (long) r4;
         struct perf_sample_data sample_data;
+        struct bpf_event_entry *ee;
         struct perf_event *event;
-        struct file *file;
         struct perf_raw_record raw = {
                 .size = size,
                 .data = data,
@@ -251,12 +250,11 @@ static u64 bpf_perf_event_output(u64 r1, u64 r2, u64 flags, u64 r4, u64 size)
         if (unlikely(index >= array->map.max_entries))
                 return -E2BIG;
 
-        file = READ_ONCE(array->ptrs[index]);
-        if (unlikely(!file))
+        ee = READ_ONCE(array->ptrs[index]);
+        if (unlikely(!ee))
                 return -ENOENT;
 
-        event = file->private_data;
-
+        event = ee->event;
         if (unlikely(event->attr.type != PERF_TYPE_SOFTWARE ||
                      event->attr.config != PERF_COUNT_SW_BPF_OUTPUT))
                 return -EINVAL;
diff --git a/net/6lowpan/6lowpan_i.h b/net/6lowpan/6lowpan_i.h
index 97ecc27aeca6..a67caee11929 100644
--- a/net/6lowpan/6lowpan_i.h
+++ b/net/6lowpan/6lowpan_i.h
@@ -12,6 +12,10 @@ static inline bool lowpan_is_ll(const struct net_device *dev,
         return lowpan_dev(dev)->lltype == lltype;
 }
 
+extern const struct ndisc_ops lowpan_ndisc_ops;
+
+int addrconf_ifid_802154_6lowpan(u8 *eui, struct net_device *dev);
+
 #ifdef CONFIG_6LOWPAN_DEBUGFS
 int lowpan_dev_debugfs_init(struct net_device *dev);
 void lowpan_dev_debugfs_exit(struct net_device *dev);
diff --git a/net/6lowpan/Makefile b/net/6lowpan/Makefile
index e44f3bf2dd42..12d131ab2324 100644
--- a/net/6lowpan/Makefile
+++ b/net/6lowpan/Makefile
@@ -1,6 +1,6 @@
 obj-$(CONFIG_6LOWPAN) += 6lowpan.o
 
-6lowpan-y := core.o iphc.o nhc.o
+6lowpan-y := core.o iphc.o nhc.o ndisc.o
 6lowpan-$(CONFIG_6LOWPAN_DEBUGFS) += debugfs.o
 
 #rfc6282 nhcs
diff --git a/net/6lowpan/core.c b/net/6lowpan/core.c
index 7a240b3eaed1..5945f7e19c67 100644
--- a/net/6lowpan/core.c
+++ b/net/6lowpan/core.c
@@ -14,6 +14,7 @@
 #include <linux/module.h>
 
 #include <net/6lowpan.h>
+#include <net/addrconf.h>
 
 #include "6lowpan_i.h"
 
@@ -33,6 +34,8 @@ int lowpan_register_netdevice(struct net_device *dev,
         for (i = 0; i < LOWPAN_IPHC_CTX_TABLE_SIZE; i++)
                 lowpan_dev(dev)->ctx.table[i].id = i;
 
+        dev->ndisc_ops = &lowpan_ndisc_ops;
+
         ret = register_netdevice(dev);
         if (ret < 0)
                 return ret;
@@ -72,16 +75,61 @@ void lowpan_unregister_netdev(struct net_device *dev)
 }
 EXPORT_SYMBOL(lowpan_unregister_netdev);
 
+int addrconf_ifid_802154_6lowpan(u8 *eui, struct net_device *dev)
+{
+        struct wpan_dev *wpan_dev = lowpan_802154_dev(dev)->wdev->ieee802154_ptr;
+
+        /* Set short_addr autoconfiguration if short_addr is present only */
+        if (!lowpan_802154_is_valid_src_short_addr(wpan_dev->short_addr))
+                return -1;
+
+        /* For either address format, all zero addresses MUST NOT be used */
+        if (wpan_dev->pan_id == cpu_to_le16(0x0000) &&
+            wpan_dev->short_addr == cpu_to_le16(0x0000))
+                return -1;
+
+        /* Alternatively, if no PAN ID is known, 16 zero bits may be used */
+        if (wpan_dev->pan_id == cpu_to_le16(IEEE802154_PAN_ID_BROADCAST))
+                memset(eui, 0, 2);
+        else
+                ieee802154_le16_to_be16(eui, &wpan_dev->pan_id);
+
+        /* The "Universal/Local" (U/L) bit shall be set to zero */
+        eui[0] &= ~2;
+        eui[2] = 0;
+        eui[3] = 0xFF;
+        eui[4] = 0xFE;
+        eui[5] = 0;
+        ieee802154_le16_to_be16(&eui[6], &wpan_dev->short_addr);
+        return 0;
+}
+
 static int lowpan_event(struct notifier_block *unused,
                         unsigned long event, void *ptr)
 {
         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
+        struct inet6_dev *idev;
+        struct in6_addr addr;
         int i;
 
         if (dev->type != ARPHRD_6LOWPAN)
                 return NOTIFY_DONE;
 
+        idev = __in6_dev_get(dev);
+        if (!idev)
+                return NOTIFY_DONE;
+
         switch (event) {
+        case NETDEV_UP:
+        case NETDEV_CHANGE:
+                /* (802.15.4 6LoWPAN short address slaac handling */
+                if (lowpan_is_ll(dev, LOWPAN_LLTYPE_IEEE802154) &&
+                    addrconf_ifid_802154_6lowpan(addr.s6_addr + 8, dev) == 0) {
+                        __ipv6_addr_set_half(&addr.s6_addr32[0],
+                                             htonl(0xFE800000), 0);
+                        addrconf_add_linklocal(idev, &addr, 0);
+                }
+                break;
         case NETDEV_DOWN:
                 for (i = 0; i < LOWPAN_IPHC_CTX_TABLE_SIZE; i++)
                         clear_bit(LOWPAN_IPHC_CTX_FLAG_ACTIVE,
@@ -112,8 +160,6 @@ static int __init lowpan_module_init(void)
                 return ret;
         }
 
-        request_module_nowait("ipv6");
-
         request_module_nowait("nhc_dest");
         request_module_nowait("nhc_fragment");
         request_module_nowait("nhc_hop");
diff --git a/net/6lowpan/debugfs.c b/net/6lowpan/debugfs.c
index acbaa3db493b..24915e0bb9ea 100644
--- a/net/6lowpan/debugfs.c
+++ b/net/6lowpan/debugfs.c
@@ -245,6 +245,41 @@ static const struct file_operations lowpan_context_fops = {
         .release        = single_release,
 };
 
+static int lowpan_short_addr_get(void *data, u64 *val)
+{
+        struct wpan_dev *wdev = data;
+
+        rtnl_lock();
+        *val = le16_to_cpu(wdev->short_addr);
+        rtnl_unlock();
+
+        return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(lowpan_short_addr_fops, lowpan_short_addr_get,
+                        NULL, "0x%04llx\n");
+
+static int lowpan_dev_debugfs_802154_init(const struct net_device *dev,
+                                          struct lowpan_dev *ldev)
+{
+        struct dentry *dentry, *root;
+
+        if (!lowpan_is_ll(dev, LOWPAN_LLTYPE_IEEE802154))
+                return 0;
+
+        root = debugfs_create_dir("ieee802154", ldev->iface_debugfs);
+        if (!root)
+                return -EINVAL;
+
+        dentry = debugfs_create_file("short_addr", 0444, root,
+                                     lowpan_802154_dev(dev)->wdev->ieee802154_ptr,
+                                     &lowpan_short_addr_fops);
+        if (!dentry)
+                return -EINVAL;
+
+        return 0;
+}
+
 int lowpan_dev_debugfs_init(struct net_device *dev)
 {
         struct lowpan_dev *ldev = lowpan_dev(dev);
@@ -272,6 +307,10 @@ int lowpan_dev_debugfs_init(struct net_device *dev)
                         goto remove_root;
         }
 
+        ret = lowpan_dev_debugfs_802154_init(dev, ldev);
+        if (ret < 0)
+                goto remove_root;
+
         return 0;
 
 remove_root:
diff --git a/net/6lowpan/iphc.c b/net/6lowpan/iphc.c
index 8501dd532fe1..79f1fa22509a 100644
--- a/net/6lowpan/iphc.c
+++ b/net/6lowpan/iphc.c
@@ -761,22 +761,75 @@ static const u8 lowpan_iphc_dam_to_sam_value[] = {
         [LOWPAN_IPHC_DAM_11] = LOWPAN_IPHC_SAM_11,
 };
 
-static u8 lowpan_compress_ctx_addr(u8 **hc_ptr, const struct in6_addr *ipaddr,
+static inline bool
+lowpan_iphc_compress_ctx_802154_lladdr(const struct in6_addr *ipaddr,
+                                       const struct lowpan_iphc_ctx *ctx,
+                                       const void *lladdr)
+{
+        const struct ieee802154_addr *addr = lladdr;
+        unsigned char extended_addr[EUI64_ADDR_LEN];
+        bool lladdr_compress = false;
+        struct in6_addr tmp = {};
+
+        switch (addr->mode) {
+        case IEEE802154_ADDR_LONG:
+                ieee802154_le64_to_be64(&extended_addr, &addr->extended_addr);
+                /* check for SAM/DAM = 11 */
+                memcpy(&tmp.s6_addr[8], &extended_addr, EUI64_ADDR_LEN);
+                /* second bit-flip (Universe/Local) is done according RFC2464 */
+                tmp.s6_addr[8] ^= 0x02;
+                /* context information are always used */
+                ipv6_addr_prefix_copy(&tmp, &ctx->pfx, ctx->plen);
+                if (ipv6_addr_equal(&tmp, ipaddr))
+                        lladdr_compress = true;
+                break;
+        case IEEE802154_ADDR_SHORT:
+                tmp.s6_addr[11] = 0xFF;
+                tmp.s6_addr[12] = 0xFE;
+                ieee802154_le16_to_be16(&tmp.s6_addr16[7],
+                                        &addr->short_addr);
+                /* context information are always used */
+                ipv6_addr_prefix_copy(&tmp, &ctx->pfx, ctx->plen);
+                if (ipv6_addr_equal(&tmp, ipaddr))
+                        lladdr_compress = true;
+                break;
+        default:
+                /* should never handled and filtered by 802154 6lowpan */
+                WARN_ON_ONCE(1);
+                break;
+        }
+
+        return lladdr_compress;
+}
+
+static u8 lowpan_compress_ctx_addr(u8 **hc_ptr, const struct net_device *dev,
+                                   const struct in6_addr *ipaddr,
                                    const struct lowpan_iphc_ctx *ctx,
                                    const unsigned char *lladdr, bool sam)
 {
         struct in6_addr tmp = {};
         u8 dam;
 
-        /* check for SAM/DAM = 11 */
-        memcpy(&tmp.s6_addr[8], lladdr, 8);
-        /* second bit-flip (Universe/Local) is done according RFC2464 */
-        tmp.s6_addr[8] ^= 0x02;
-        /* context information are always used */
-        ipv6_addr_prefix_copy(&tmp, &ctx->pfx, ctx->plen);
-        if (ipv6_addr_equal(&tmp, ipaddr)) {
-                dam = LOWPAN_IPHC_DAM_11;
-                goto out;
+        switch (lowpan_dev(dev)->lltype) {
+        case LOWPAN_LLTYPE_IEEE802154:
+                if (lowpan_iphc_compress_ctx_802154_lladdr(ipaddr, ctx,
+                                                           lladdr)) {
+                        dam = LOWPAN_IPHC_DAM_11;
+                        goto out;
+                }
+                break;
+        default:
+                /* check for SAM/DAM = 11 */
+                memcpy(&tmp.s6_addr[8], lladdr, EUI64_ADDR_LEN);
+                /* second bit-flip (Universe/Local) is done according RFC2464 */
+                tmp.s6_addr[8] ^= 0x02;
+                /* context information are always used */
+                ipv6_addr_prefix_copy(&tmp, &ctx->pfx, ctx->plen);
+                if (ipv6_addr_equal(&tmp, ipaddr)) {
+                        dam = LOWPAN_IPHC_DAM_11;
+                        goto out;
+                }
+                break;
         }
 
         memset(&tmp, 0, sizeof(tmp));
@@ -813,28 +866,85 @@ out:
                 return dam;
 }
 
-static u8 lowpan_compress_addr_64(u8 **hc_ptr, const struct in6_addr *ipaddr,
+static inline bool
+lowpan_iphc_compress_802154_lladdr(const struct in6_addr *ipaddr,
+                                   const void *lladdr)
+{
+        const struct ieee802154_addr *addr = lladdr;
+        unsigned char extended_addr[EUI64_ADDR_LEN];
+        bool lladdr_compress = false;
+        struct in6_addr tmp = {};
+
+        switch (addr->mode) {
+        case IEEE802154_ADDR_LONG:
+                ieee802154_le64_to_be64(&extended_addr, &addr->extended_addr);
+                if (is_addr_mac_addr_based(ipaddr, extended_addr))
+                        lladdr_compress = true;
+                break;
+        case IEEE802154_ADDR_SHORT:
+                /* fe:80::ff:fe00:XXXX
+                 *                \__/
+                 *             short_addr
+                 *
+                 * Universe/Local bit is zero.
+                 */
+                tmp.s6_addr[0] = 0xFE;
+                tmp.s6_addr[1] = 0x80;
+                tmp.s6_addr[11] = 0xFF;
+                tmp.s6_addr[12] = 0xFE;
+                ieee802154_le16_to_be16(&tmp.s6_addr16[7],
+                                        &addr->short_addr);
+                if (ipv6_addr_equal(&tmp, ipaddr))
+                        lladdr_compress = true;
+                break;
+        default:
+                /* should never handled and filtered by 802154 6lowpan */
+                WARN_ON_ONCE(1);
+                break;
+        }
+
+        return lladdr_compress;
+}
+
+static u8 lowpan_compress_addr_64(u8 **hc_ptr, const struct net_device *dev,
+                                  const struct in6_addr *ipaddr,
                                   const unsigned char *lladdr, bool sam)
 {
-        u8 dam = LOWPAN_IPHC_DAM_00;
+        u8 dam = LOWPAN_IPHC_DAM_01;
 
-        if (is_addr_mac_addr_based(ipaddr, lladdr)) {
-                dam = LOWPAN_IPHC_DAM_11; /* 0-bits */
-                pr_debug("address compression 0 bits\n");
-        } else if (lowpan_is_iid_16_bit_compressable(ipaddr)) {
+        switch (lowpan_dev(dev)->lltype) {
+        case LOWPAN_LLTYPE_IEEE802154:
+                if (lowpan_iphc_compress_802154_lladdr(ipaddr, lladdr)) {
+                        dam = LOWPAN_IPHC_DAM_11; /* 0-bits */
+                        pr_debug("address compression 0 bits\n");
+                        goto out;
+                }
+                break;
+        default:
+                if (is_addr_mac_addr_based(ipaddr, lladdr)) {
+                        dam = LOWPAN_IPHC_DAM_11; /* 0-bits */
+                        pr_debug("address compression 0 bits\n");
+                        goto out;
+                }
+                break;
+        }
+
+        if (lowpan_is_iid_16_bit_compressable(ipaddr)) {
                 /* compress IID to 16 bits xxxx::XXXX */
                 lowpan_push_hc_data(hc_ptr, &ipaddr->s6_addr16[7], 2);
                 dam = LOWPAN_IPHC_DAM_10; /* 16-bits */
                 raw_dump_inline(NULL, "Compressed ipv6 addr is (16 bits)",
                                 *hc_ptr - 2, 2);
-        } else {
-                /* do not compress IID => xxxx::IID */
-                lowpan_push_hc_data(hc_ptr, &ipaddr->s6_addr16[4], 8);
-                dam = LOWPAN_IPHC_DAM_01; /* 64-bits */
-                raw_dump_inline(NULL, "Compressed ipv6 addr is (64 bits)",
-                                *hc_ptr - 8, 8);
+                goto out;
         }
 
+        /* do not compress IID => xxxx::IID */
+        lowpan_push_hc_data(hc_ptr, &ipaddr->s6_addr16[4], 8);
+        raw_dump_inline(NULL, "Compressed ipv6 addr is (64 bits)",
+                        *hc_ptr - 8, 8);
+
+out:
+
         if (sam)
                 return lowpan_iphc_dam_to_sam_value[dam];
         else
@@ -1013,9 +1123,6 @@ int lowpan_header_compress(struct sk_buff *skb, const struct net_device *dev,
         iphc0 = LOWPAN_DISPATCH_IPHC;
         iphc1 = 0;
 
-        raw_dump_inline(__func__, "saddr", saddr, EUI64_ADDR_LEN);
-        raw_dump_inline(__func__, "daddr", daddr, EUI64_ADDR_LEN);
-
         raw_dump_table(__func__, "sending raw skb network uncompressed packet",
                        skb->data, skb->len);
 
@@ -1088,14 +1195,15 @@ int lowpan_header_compress(struct sk_buff *skb, const struct net_device *dev,
                 iphc1 |= LOWPAN_IPHC_SAC;
         } else {
                 if (sci) {
-                        iphc1 |= lowpan_compress_ctx_addr(&hc_ptr, &hdr->saddr,
+                        iphc1 |= lowpan_compress_ctx_addr(&hc_ptr, dev,
+                                                          &hdr->saddr,
                                                           &sci_entry, saddr,
                                                           true);
                         iphc1 |= LOWPAN_IPHC_SAC;
                 } else {
                         if (ipv6_saddr_type & IPV6_ADDR_LINKLOCAL &&
                             lowpan_is_linklocal_zero_padded(hdr->saddr)) {
-                                iphc1 |= lowpan_compress_addr_64(&hc_ptr,
+                                iphc1 |= lowpan_compress_addr_64(&hc_ptr, dev,
                                                                  &hdr->saddr,
                                                                  saddr, true);
                                 pr_debug("source address unicast link-local %pI6c iphc1 0x%02x\n",
@@ -1123,14 +1231,15 @@ int lowpan_header_compress(struct sk_buff *skb, const struct net_device *dev,
                 }
         } else {
                 if (dci) {
-                        iphc1 |= lowpan_compress_ctx_addr(&hc_ptr, &hdr->daddr,
+                        iphc1 |= lowpan_compress_ctx_addr(&hc_ptr, dev,
+                                                          &hdr->daddr,
                                                           &dci_entry, daddr,
                                                           false);
                         iphc1 |= LOWPAN_IPHC_DAC;
                 } else {
                         if (ipv6_daddr_type & IPV6_ADDR_LINKLOCAL &&
                             lowpan_is_linklocal_zero_padded(hdr->daddr)) {
-                                iphc1 |= lowpan_compress_addr_64(&hc_ptr,
+                                iphc1 |= lowpan_compress_addr_64(&hc_ptr, dev,
                                                                  &hdr->daddr,
                                                                  daddr, false);
                                 pr_debug("dest address unicast link-local %pI6c iphc1 0x%02x\n",
diff --git a/net/6lowpan/ndisc.c b/net/6lowpan/ndisc.c
new file mode 100644
index 000000000000..ae1d4199aa4c
--- /dev/null
+++ b/net/6lowpan/ndisc.c
@@ -0,0 +1,234 @@
+/* This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * Authors:
+ * (C) 2016 Pengutronix, Alexander Aring <aar@pengutronix.de>
+ */
+
+#include <net/6lowpan.h>
+#include <net/addrconf.h>
+#include <net/ndisc.h>
+
+#include "6lowpan_i.h"
+
+static int lowpan_ndisc_is_useropt(u8 nd_opt_type)
+{
+        return nd_opt_type == ND_OPT_6CO;
+}
+
+#if IS_ENABLED(CONFIG_IEEE802154_6LOWPAN)
+#define NDISC_802154_SHORT_ADDR_LENGTH  1
+static int lowpan_ndisc_parse_802154_options(const struct net_device *dev,
+                                             struct nd_opt_hdr *nd_opt,
+                                             struct ndisc_options *ndopts)
+{
+        switch (nd_opt->nd_opt_len) {
+        case NDISC_802154_SHORT_ADDR_LENGTH:
+                if (ndopts->nd_802154_opt_array[nd_opt->nd_opt_type])
+                        ND_PRINTK(2, warn,
+                                  "%s: duplicated short addr ND6 option found: type=%d\n",
+                                  __func__, nd_opt->nd_opt_type);
+                else
+                        ndopts->nd_802154_opt_array[nd_opt->nd_opt_type] = nd_opt;
+                return 1;
+        default:
+                /* all others will be handled by ndisc IPv6 option parsing */
+                return 0;
+        }
+}
+
+static int lowpan_ndisc_parse_options(const struct net_device *dev,
+                                      struct nd_opt_hdr *nd_opt,
+                                      struct ndisc_options *ndopts)
+{
+        switch (nd_opt->nd_opt_type) {
+        case ND_OPT_SOURCE_LL_ADDR:
+        case ND_OPT_TARGET_LL_ADDR:
+                return lowpan_ndisc_parse_802154_options(dev, nd_opt, ndopts);
+        default:
+                return 0;
+        }
+}
+
+static void lowpan_ndisc_802154_update(struct neighbour *n, u32 flags,
+                                       u8 icmp6_type,
+                                       const struct ndisc_options *ndopts)
+{
+        struct lowpan_802154_neigh *neigh = lowpan_802154_neigh(neighbour_priv(n));
+        u8 *lladdr_short = NULL;
+
+        switch (icmp6_type) {
+        case NDISC_ROUTER_SOLICITATION:
+        case NDISC_ROUTER_ADVERTISEMENT:
+        case NDISC_NEIGHBOUR_SOLICITATION:
+                if (ndopts->nd_802154_opts_src_lladdr) {
+                        lladdr_short = __ndisc_opt_addr_data(ndopts->nd_802154_opts_src_lladdr,
+                                                             IEEE802154_SHORT_ADDR_LEN, 0);
+                        if (!lladdr_short) {
+                                ND_PRINTK(2, warn,
+                                          "NA: invalid short link-layer address length\n");
+                                return;
+                        }
+                }
+                break;
+        case NDISC_REDIRECT:
+        case NDISC_NEIGHBOUR_ADVERTISEMENT:
+                if (ndopts->nd_802154_opts_tgt_lladdr) {
+                        lladdr_short = __ndisc_opt_addr_data(ndopts->nd_802154_opts_tgt_lladdr,
+                                                             IEEE802154_SHORT_ADDR_LEN, 0);
+                        if (!lladdr_short) {
+                                ND_PRINTK(2, warn,
+                                          "NA: invalid short link-layer address length\n");
+                                return;
+                        }
+                }
+                break;
+        default:
+                break;
+        }
+
+        write_lock_bh(&n->lock);
+        if (lladdr_short)
+                ieee802154_be16_to_le16(&neigh->short_addr, lladdr_short);
+        else
+                neigh->short_addr = cpu_to_le16(IEEE802154_ADDR_SHORT_UNSPEC);
+        write_unlock_bh(&n->lock);
+}
+
+static void lowpan_ndisc_update(const struct net_device *dev,
+                                struct neighbour *n, u32 flags, u8 icmp6_type,
+                                const struct ndisc_options *ndopts)
+{
+        if (!lowpan_is_ll(dev, LOWPAN_LLTYPE_IEEE802154))
+                return;
+
+        /* react on overrides only. TODO check if this is really right. */
+        if (flags & NEIGH_UPDATE_F_OVERRIDE)
+                lowpan_ndisc_802154_update(n, flags, icmp6_type, ndopts);
+}
+
+static int lowpan_ndisc_opt_addr_space(const struct net_device *dev,
+                                       u8 icmp6_type, struct neighbour *neigh,
+                                       u8 *ha_buf, u8 **ha)
+{
+        struct lowpan_802154_neigh *n;
+        struct wpan_dev *wpan_dev;
+        int addr_space = 0;
+
+        if (!lowpan_is_ll(dev, LOWPAN_LLTYPE_IEEE802154))
+                return 0;
+
+        switch (icmp6_type) {
+        case NDISC_REDIRECT:
+                n = lowpan_802154_neigh(neighbour_priv(neigh));
+
+                read_lock_bh(&neigh->lock);
+                if (lowpan_802154_is_valid_src_short_addr(n->short_addr)) {
+                        memcpy(ha_buf, &n->short_addr,
+                               IEEE802154_SHORT_ADDR_LEN);
+                        read_unlock_bh(&neigh->lock);
+                        addr_space += __ndisc_opt_addr_space(IEEE802154_SHORT_ADDR_LEN, 0);
+                        *ha = ha_buf;
+                }
+                read_unlock_bh(&neigh->lock);
+                break;
+        case NDISC_NEIGHBOUR_ADVERTISEMENT:
+        case NDISC_NEIGHBOUR_SOLICITATION:
+        case NDISC_ROUTER_SOLICITATION:
+                wpan_dev = lowpan_802154_dev(dev)->wdev->ieee802154_ptr;
+
+                if (lowpan_802154_is_valid_src_short_addr(wpan_dev->short_addr))
+                        addr_space = __ndisc_opt_addr_space(IEEE802154_SHORT_ADDR_LEN, 0);
+                break;
+        default:
+                break;
+        }
+
+        return addr_space;
+}
+
+static void lowpan_ndisc_fill_addr_option(const struct net_device *dev,
+                                          struct sk_buff *skb, u8 icmp6_type,
+                                          const u8 *ha)
+{
+        struct wpan_dev *wpan_dev;
+        __be16 short_addr;
+        u8 opt_type;
+
+        if (!lowpan_is_ll(dev, LOWPAN_LLTYPE_IEEE802154))
+                return;
+
+        switch (icmp6_type) {
+        case NDISC_REDIRECT:
+                if (ha) {
+                        ieee802154_le16_to_be16(&short_addr, ha);
+                        __ndisc_fill_addr_option(skb, ND_OPT_TARGET_LL_ADDR,
+                                                 &short_addr,
+                                                 IEEE802154_SHORT_ADDR_LEN, 0);
+                }
+                return;
+        case NDISC_NEIGHBOUR_ADVERTISEMENT:
+                opt_type = ND_OPT_TARGET_LL_ADDR;
+                break;
+        case NDISC_ROUTER_SOLICITATION:
+        case NDISC_NEIGHBOUR_SOLICITATION:
+                opt_type = ND_OPT_SOURCE_LL_ADDR;
+                break;
+        default:
+                return;
+        }
+
+        wpan_dev = lowpan_802154_dev(dev)->wdev->ieee802154_ptr;
+
+        if (lowpan_802154_is_valid_src_short_addr(wpan_dev->short_addr)) {
+                ieee802154_le16_to_be16(&short_addr,
+                                        &wpan_dev->short_addr);
+                __ndisc_fill_addr_option(skb, opt_type, &short_addr,
+                                         IEEE802154_SHORT_ADDR_LEN, 0);
+        }
+}
+
+static void lowpan_ndisc_prefix_rcv_add_addr(struct net *net,
+                                             struct net_device *dev,
+                                             const struct prefix_info *pinfo,
+                                             struct inet6_dev *in6_dev,
+                                             struct in6_addr *addr,
+                                             int addr_type, u32 addr_flags,
+                                             bool sllao, bool tokenized,
+                                             __u32 valid_lft,
+                                             u32 prefered_lft,
+                                             bool dev_addr_generated)
+{
+        int err;
+
+        /* generates short based address for RA PIO's */
+        if (lowpan_is_ll(dev, LOWPAN_LLTYPE_IEEE802154) && dev_addr_generated &&
+            !addrconf_ifid_802154_6lowpan(addr->s6_addr + 8, dev)) {
+                err = addrconf_prefix_rcv_add_addr(net, dev, pinfo, in6_dev,
+                                                   addr, addr_type, addr_flags,
+                                                   sllao, tokenized, valid_lft,
+                                                   prefered_lft);
+                if (err)
+                        ND_PRINTK(2, warn,
+                                  "RA: could not add a short address based address for prefix: %pI6c\n",
+                                  &pinfo->prefix);
+        }
+}
+#endif
+
+const struct ndisc_ops lowpan_ndisc_ops = {
+        .is_useropt             = lowpan_ndisc_is_useropt,
+#if IS_ENABLED(CONFIG_IEEE802154_6LOWPAN)
+        .parse_options          = lowpan_ndisc_parse_options,
+        .update                 = lowpan_ndisc_update,
+        .opt_addr_space         = lowpan_ndisc_opt_addr_space,
+        .fill_addr_option       = lowpan_ndisc_fill_addr_option,
+        .prefix_rcv_add_addr    = lowpan_ndisc_prefix_rcv_add_addr,
+#endif
+};
diff --git a/net/batman-adv/routing.c b/net/batman-adv/routing.c
index 6c2901a86230..396c0134c5ab 100644
--- a/net/batman-adv/routing.c
+++ b/net/batman-adv/routing.c
@@ -654,7 +654,7 @@ static int batadv_route_unicast_packet(struct sk_buff *skb,
                                    len + ETH_HLEN);
 
                 ret = NET_RX_SUCCESS;
-        } else if (res == NET_XMIT_POLICED) {
+        } else if (res == -EINPROGRESS) {
                 /* skb was buffered and consumed */
                 ret = NET_RX_SUCCESS;
         }
diff --git a/net/batman-adv/send.c b/net/batman-adv/send.c
index f2f125684ed9..b1a4e8a811c8 100644
--- a/net/batman-adv/send.c
+++ b/net/batman-adv/send.c
@@ -156,7 +156,7 @@ int batadv_send_unicast_skb(struct sk_buff *skb,
  * attempted.
  *
  * Return: NET_XMIT_SUCCESS on success, NET_XMIT_DROP on failure, or
- * NET_XMIT_POLICED if the skb is buffered for later transmit.
+ * -EINPROGRESS if the skb is buffered for later transmit.
  */
 int batadv_send_skb_to_orig(struct sk_buff *skb,
                             struct batadv_orig_node *orig_node,
@@ -188,7 +188,7 @@ int batadv_send_skb_to_orig(struct sk_buff *skb,
          * network coding fails, then send the packet as usual.
          */
         if (recv_if && batadv_nc_skb_forward(skb, neigh_node)) {
-                ret = NET_XMIT_POLICED;
+                ret = -EINPROGRESS;
         } else {
                 batadv_send_unicast_skb(skb, neigh_node);
                 ret = NET_XMIT_SUCCESS;
diff --git a/net/bluetooth/6lowpan.c b/net/bluetooth/6lowpan.c
index 780089d75915..d020299baba4 100644
--- a/net/bluetooth/6lowpan.c
+++ b/net/bluetooth/6lowpan.c
@@ -627,20 +627,9 @@ static netdev_tx_t bt_xmit(struct sk_buff *skb, struct net_device *netdev)
         return err < 0 ? NET_XMIT_DROP : err;
 }
 
-static struct lock_class_key bt_tx_busylock;
-static struct lock_class_key bt_netdev_xmit_lock_key;
-
-static void bt_set_lockdep_class_one(struct net_device *dev,
-                                     struct netdev_queue *txq,
-                                     void *_unused)
-{
-        lockdep_set_class(&txq->_xmit_lock, &bt_netdev_xmit_lock_key);
-}
-
 static int bt_dev_init(struct net_device *dev)
 {
-        netdev_for_each_tx_queue(dev, bt_set_lockdep_class_one, NULL);
-        dev->qdisc_tx_busylock = &bt_tx_busylock;
+        netdev_lockdep_set_classes(dev);
 
         return 0;
 }
diff --git a/net/caif/chnl_net.c b/net/caif/chnl_net.c
index 67a4a36febd1..3408ed51b611 100644
--- a/net/caif/chnl_net.c
+++ b/net/caif/chnl_net.c
@@ -13,7 +13,6 @@
 #include <linux/module.h>
 #include <linux/netdevice.h>
 #include <linux/if_ether.h>
-#include <linux/moduleparam.h>
 #include <linux/ip.h>
 #include <linux/sched.h>
 #include <linux/sockios.h>
diff --git a/net/can/Makefile b/net/can/Makefile
index cef49eb1f5c7..10936754e3f2 100644
--- a/net/can/Makefile
+++ b/net/can/Makefile
@@ -3,7 +3,8 @@
 #
 
 obj-$(CONFIG_CAN)       += can.o
-can-y                   := af_can.o proc.o
+can-y                   := af_can.o
+can-$(CONFIG_PROC_FS)   += proc.o
 
 obj-$(CONFIG_CAN_RAW)   += can-raw.o
 can-raw-y               := raw.o
diff --git a/net/can/af_can.c b/net/can/af_can.c
index 166d436196c1..1108079d934f 100644
--- a/net/can/af_can.c
+++ b/net/can/af_can.c
@@ -911,14 +911,14 @@ static __init int can_init(void)
         if (!rcv_cache)
                 return -ENOMEM;
 
-        if (stats_timer) {
+        if (IS_ENABLED(CONFIG_PROC_FS)) {
+                if (stats_timer) {
                 /* the statistics are updated every second (timer triggered) */
-                setup_timer(&can_stattimer, can_stat_update, 0);
-                mod_timer(&can_stattimer, round_jiffies(jiffies + HZ));
-        } else
-                can_stattimer.function = NULL;
-
-        can_init_proc();
+                        setup_timer(&can_stattimer, can_stat_update, 0);
+                        mod_timer(&can_stattimer, round_jiffies(jiffies + HZ));
+                }
+                can_init_proc();
+        }
 
         /* protocol register */
         sock_register(&can_family_ops);
@@ -933,10 +933,12 @@ static __exit void can_exit(void)
 {
         struct net_device *dev;
 
-        if (stats_timer)
-                del_timer_sync(&can_stattimer);
+        if (IS_ENABLED(CONFIG_PROC_FS)) {
+                if (stats_timer)
+                        del_timer_sync(&can_stattimer);
 
-        can_remove_proc();
+                can_remove_proc();
+        }
 
         /* protocol unregister */
         dev_remove_pack(&canfd_packet);
diff --git a/net/can/bcm.c b/net/can/bcm.c
index 6863310d6973..8e999ffdf28b 100644
--- a/net/can/bcm.c
+++ b/net/can/bcm.c
@@ -1,7 +1,7 @@
 /*
  * bcm.c - Broadcast Manager to filter/send (cyclic) CAN content
  *
- * Copyright (c) 2002-2007 Volkswagen Group Electronic Research
+ * Copyright (c) 2002-2016 Volkswagen Group Electronic Research
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -67,27 +67,31 @@
  */
 #define MAX_NFRAMES 256
 
-/* use of last_frames[index].can_dlc */
+/* use of last_frames[index].flags */
 #define RX_RECV    0x40 /* received data for this element */
 #define RX_THR     0x80 /* element not been sent due to throttle feature */
-#define BCM_CAN_DLC_MASK 0x0F /* clean private flags in can_dlc by masking */
+#define BCM_CAN_FLAGS_MASK 0x3F /* to clean private flags after usage */
 
 /* get best masking value for can_rx_register() for a given single can_id */
 #define REGMASK(id) ((id & CAN_EFF_FLAG) ? \
                      (CAN_EFF_MASK | CAN_EFF_FLAG | CAN_RTR_FLAG) : \
                      (CAN_SFF_MASK | CAN_EFF_FLAG | CAN_RTR_FLAG))
 
-#define CAN_BCM_VERSION CAN_VERSION
+#define CAN_BCM_VERSION "20160617"
 
 MODULE_DESCRIPTION("PF_CAN broadcast manager protocol");
 MODULE_LICENSE("Dual BSD/GPL");
 MODULE_AUTHOR("Oliver Hartkopp <oliver.hartkopp@volkswagen.de>");
 MODULE_ALIAS("can-proto-2");
 
-/* easy access to can_frame payload */
-static inline u64 GET_U64(const struct can_frame *cp)
+/*
+ * easy access to the first 64 bit of can(fd)_frame payload. cp->data is
+ * 64 bit aligned so the offset has to be multiples of 8 which is ensured
+ * by the only callers in bcm_rx_cmp_to_index() bcm_rx_handler().
+ */
+static inline u64 get_u64(const struct canfd_frame *cp, int offset)
 {
-        return *(u64 *)cp->data;
+        return *(u64 *)(cp->data + offset);
 }
 
 struct bcm_op {
@@ -101,13 +105,14 @@ struct bcm_op {
         struct tasklet_struct tsklet, thrtsklet;
         ktime_t rx_stamp, kt_ival1, kt_ival2, kt_lastmsg;
         int rx_ifindex;
+        int cfsiz;
         u32 count;
         u32 nframes;
         u32 currframe;
-        struct can_frame *frames;
-        struct can_frame *last_frames;
-        struct can_frame sframe;
-        struct can_frame last_sframe;
+        struct canfd_frame *frames;
+        struct canfd_frame *last_frames;
+        struct canfd_frame sframe;
+        struct canfd_frame last_sframe;
         struct sock *sk;
         struct net_device *rx_reg_dev;
 };
@@ -136,7 +141,7 @@ static inline ktime_t bcm_timeval_to_ktime(struct bcm_timeval tv)
         return ktime_set(tv.tv_sec, tv.tv_usec * NSEC_PER_USEC);
 }
 
-#define CFSIZ sizeof(struct can_frame)
+#define CFSIZ(flags) ((flags & CAN_FD_FRAME) ? CANFD_MTU : CAN_MTU)
 #define OPSIZ sizeof(struct bcm_op)
 #define MHSIZ sizeof(struct bcm_msg_head)
 
@@ -183,43 +188,50 @@ static int bcm_proc_show(struct seq_file *m, void *v)
                 if (!op->frames_abs)
                         continue;
 
-                seq_printf(m, "rx_op: %03X %-5s ",
-                                op->can_id, bcm_proc_getifname(ifname, op->ifindex));
-                seq_printf(m, "[%u]%c ", op->nframes,
-                                (op->flags & RX_CHECK_DLC)?'d':' ');
+                seq_printf(m, "rx_op: %03X %-5s ", op->can_id,
+                           bcm_proc_getifname(ifname, op->ifindex));
+
+                if (op->flags & CAN_FD_FRAME)
+                        seq_printf(m, "(%u)", op->nframes);
+                else
+                        seq_printf(m, "[%u]", op->nframes);
+
+                seq_printf(m, "%c ", (op->flags & RX_CHECK_DLC) ? 'd' : ' ');
+
                 if (op->kt_ival1.tv64)
                         seq_printf(m, "timeo=%lld ",
-                                        (long long)
-                                        ktime_to_us(op->kt_ival1));
+                                   (long long)ktime_to_us(op->kt_ival1));
 
                 if (op->kt_ival2.tv64)
                         seq_printf(m, "thr=%lld ",
-                                        (long long)
-                                        ktime_to_us(op->kt_ival2));
+                                   (long long)ktime_to_us(op->kt_ival2));
 
                 seq_printf(m, "# recv %ld (%ld) => reduction: ",
-                                op->frames_filtered, op->frames_abs);
+                           op->frames_filtered, op->frames_abs);
 
                 reduction = 100 - (op->frames_filtered * 100) / op->frames_abs;
 
                 seq_printf(m, "%s%ld%%\n",
-                                (reduction == 100)?"near ":"", reduction);
+                           (reduction == 100) ? "near " : "", reduction);
         }
 
         list_for_each_entry(op, &bo->tx_ops, list) {
 
-                seq_printf(m, "tx_op: %03X %s [%u] ",
-                                op->can_id,
-                                bcm_proc_getifname(ifname, op->ifindex),
-                                op->nframes);
+                seq_printf(m, "tx_op: %03X %s ", op->can_id,
+                           bcm_proc_getifname(ifname, op->ifindex));
+
+                if (op->flags & CAN_FD_FRAME)
+                        seq_printf(m, "(%u) ", op->nframes);
+                else
+                        seq_printf(m, "[%u] ", op->nframes);
 
                 if (op->kt_ival1.tv64)
                         seq_printf(m, "t1=%lld ",
-                                        (long long) ktime_to_us(op->kt_ival1));
+                                   (long long)ktime_to_us(op->kt_ival1));
 
                 if (op->kt_ival2.tv64)
                         seq_printf(m, "t2=%lld ",
-                                        (long long) ktime_to_us(op->kt_ival2));
+                                   (long long)ktime_to_us(op->kt_ival2));
 
                 seq_printf(m, "# sent %ld\n", op->frames_abs);
         }
@@ -248,7 +260,7 @@ static void bcm_can_tx(struct bcm_op *op)
 {
         struct sk_buff *skb;
         struct net_device *dev;
-        struct can_frame *cf = &op->frames[op->currframe];
+        struct canfd_frame *cf = op->frames + op->cfsiz * op->currframe;
 
         /* no target device? => exit */
         if (!op->ifindex)
@@ -260,7 +272,7 @@ static void bcm_can_tx(struct bcm_op *op)
                 return;
         }
 
-        skb = alloc_skb(CFSIZ + sizeof(struct can_skb_priv), gfp_any());
+        skb = alloc_skb(op->cfsiz + sizeof(struct can_skb_priv), gfp_any());
         if (!skb)
                 goto out;
 
@@ -268,7 +280,7 @@ static void bcm_can_tx(struct bcm_op *op)
         can_skb_prv(skb)->ifindex = dev->ifindex;
         can_skb_prv(skb)->skbcnt = 0;
 
-        memcpy(skb_put(skb, CFSIZ), cf, CFSIZ);
+        memcpy(skb_put(skb, op->cfsiz), cf, op->cfsiz);
 
         /* send with loopback */
         skb->dev = dev;
@@ -282,7 +294,7 @@ static void bcm_can_tx(struct bcm_op *op)
         /* reached last frame? */
         if (op->currframe >= op->nframes)
                 op->currframe = 0;
- out:
+out:
         dev_put(dev);
 }
 
@@ -291,13 +303,13 @@ static void bcm_can_tx(struct bcm_op *op)
  *                    (consisting of bcm_msg_head + x CAN frames)
  */
 static void bcm_send_to_user(struct bcm_op *op, struct bcm_msg_head *head,
-                             struct can_frame *frames, int has_timestamp)
+                             struct canfd_frame *frames, int has_timestamp)
 {
         struct sk_buff *skb;
-        struct can_frame *firstframe;
+        struct canfd_frame *firstframe;
         struct sockaddr_can *addr;
         struct sock *sk = op->sk;
-        unsigned int datalen = head->nframes * CFSIZ;
+        unsigned int datalen = head->nframes * op->cfsiz;
         int err;
 
         skb = alloc_skb(sizeof(*head) + datalen, gfp_any());
@@ -307,19 +319,19 @@ static void bcm_send_to_user(struct bcm_op *op, struct bcm_msg_head *head,
         memcpy(skb_put(skb, sizeof(*head)), head, sizeof(*head));
 
         if (head->nframes) {
-                /* can_frames starting here */
-                firstframe = (struct can_frame *)skb_tail_pointer(skb);
+                /* CAN frames starting here */
+                firstframe = (struct canfd_frame *)skb_tail_pointer(skb);
 
                 memcpy(skb_put(skb, datalen), frames, datalen);
 
                 /*
-                 * the BCM uses the can_dlc-element of the can_frame
+                 * the BCM uses the flags-element of the canfd_frame
                  * structure for internal purposes. This is only
                  * relevant for updates that are generated by the
                  * BCM, where nframes is 1
                  */
                 if (head->nframes == 1)
-                        firstframe->can_dlc &= BCM_CAN_DLC_MASK;
+                        firstframe->flags &= BCM_CAN_FLAGS_MASK;
         }
 
         if (has_timestamp) {
@@ -406,7 +418,7 @@ static enum hrtimer_restart bcm_tx_timeout_handler(struct hrtimer *hrtimer)
 /*
  * bcm_rx_changed - create a RX_CHANGED notification due to changed content
  */
-static void bcm_rx_changed(struct bcm_op *op, struct can_frame *data)
+static void bcm_rx_changed(struct bcm_op *op, struct canfd_frame *data)
 {
         struct bcm_msg_head head;
 
@@ -418,7 +430,7 @@ static void bcm_rx_changed(struct bcm_op *op, struct can_frame *data)
                 op->frames_filtered = op->frames_abs = 0;
 
         /* this element is not throttled anymore */
-        data->can_dlc &= (BCM_CAN_DLC_MASK|RX_RECV);
+        data->flags &= (BCM_CAN_FLAGS_MASK|RX_RECV);
 
         head.opcode  = RX_CHANGED;
         head.flags   = op->flags;
@@ -437,13 +449,13 @@ static void bcm_rx_changed(struct bcm_op *op, struct can_frame *data)
  *                          2. send a notification to the user (if possible)
  */
 static void bcm_rx_update_and_send(struct bcm_op *op,
-                                   struct can_frame *lastdata,
-                                   const struct can_frame *rxdata)
+                                   struct canfd_frame *lastdata,
+                                   const struct canfd_frame *rxdata)
 {
-        memcpy(lastdata, rxdata, CFSIZ);
+        memcpy(lastdata, rxdata, op->cfsiz);
 
         /* mark as used and throttled by default */
-        lastdata->can_dlc |= (RX_RECV|RX_THR);
+        lastdata->flags |= (RX_RECV|RX_THR);
 
         /* throttling mode inactive ? */
         if (!op->kt_ival2.tv64) {
@@ -481,33 +493,36 @@ rx_changed_settime:
  *                       received data stored in op->last_frames[]
  */
 static void bcm_rx_cmp_to_index(struct bcm_op *op, unsigned int index,
-                                const struct can_frame *rxdata)
+                                const struct canfd_frame *rxdata)
 {
+        struct canfd_frame *cf = op->frames + op->cfsiz * index;
+        struct canfd_frame *lcf = op->last_frames + op->cfsiz * index;
+        int i;
+
         /*
-         * no one uses the MSBs of can_dlc for comparison,
+         * no one uses the MSBs of flags for comparison,
          * so we use it here to detect the first time of reception
          */
 
-        if (!(op->last_frames[index].can_dlc & RX_RECV)) {
+        if (!(lcf->flags & RX_RECV)) {
                 /* received data for the first time => send update to user */
-                bcm_rx_update_and_send(op, &op->last_frames[index], rxdata);
+                bcm_rx_update_and_send(op, lcf, rxdata);
                 return;
         }
 
-        /* do a real check in can_frame data section */
-
-        if ((GET_U64(&op->frames[index]) & GET_U64(rxdata)) !=
-            (GET_U64(&op->frames[index]) & GET_U64(&op->last_frames[index]))) {
-                bcm_rx_update_and_send(op, &op->last_frames[index], rxdata);
-                return;
+        /* do a real check in CAN frame data section */
+        for (i = 0; i < rxdata->len; i += 8) {
+                if ((get_u64(cf, i) & get_u64(rxdata, i)) !=
+                    (get_u64(cf, i) & get_u64(lcf, i))) {
+                        bcm_rx_update_and_send(op, lcf, rxdata);
+                        return;
+                }
         }
 
         if (op->flags & RX_CHECK_DLC) {
-                /* do a real check in can_frame dlc */
-                if (rxdata->can_dlc != (op->last_frames[index].can_dlc &
-                                        BCM_CAN_DLC_MASK)) {
-                        bcm_rx_update_and_send(op, &op->last_frames[index],
-                                               rxdata);
+                /* do a real check in CAN frame length */
+                if (rxdata->len != lcf->len) {
+                        bcm_rx_update_and_send(op, lcf, rxdata);
                         return;
                 }
         }
@@ -556,8 +571,8 @@ static enum hrtimer_restart bcm_rx_timeout_handler(struct hrtimer *hrtimer)
 
         /* if user wants to be informed, when cyclic CAN-Messages come back */
         if ((op->flags & RX_ANNOUNCE_RESUME) && op->last_frames) {
-                /* clear received can_frames to indicate 'nothing received' */
-                memset(op->last_frames, 0, op->nframes * CFSIZ);
+                /* clear received CAN frames to indicate 'nothing received' */
+                memset(op->last_frames, 0, op->nframes * op->cfsiz);
         }
 
         return HRTIMER_NORESTART;
@@ -569,9 +584,11 @@ static enum hrtimer_restart bcm_rx_timeout_handler(struct hrtimer *hrtimer)
 static inline int bcm_rx_do_flush(struct bcm_op *op, int update,
                                   unsigned int index)
 {
-        if ((op->last_frames) && (op->last_frames[index].can_dlc & RX_THR)) {
+        struct canfd_frame *lcf = op->last_frames + op->cfsiz * index;
+
+        if ((op->last_frames) && (lcf->flags & RX_THR)) {
                 if (update)
-                        bcm_rx_changed(op, &op->last_frames[index]);
+                        bcm_rx_changed(op, lcf);
                 return 1;
         }
         return 0;
@@ -636,15 +653,19 @@ static enum hrtimer_restart bcm_rx_thr_handler(struct hrtimer *hrtimer)
 static void bcm_rx_handler(struct sk_buff *skb, void *data)
 {
         struct bcm_op *op = (struct bcm_op *)data;
-        const struct can_frame *rxframe = (struct can_frame *)skb->data;
+        const struct canfd_frame *rxframe = (struct canfd_frame *)skb->data;
         unsigned int i;
 
-        /* disable timeout */
-        hrtimer_cancel(&op->timer);
-
         if (op->can_id != rxframe->can_id)
                 return;
 
+        /* make sure to handle the correct frame type (CAN / CAN FD) */
+        if (skb->len != op->cfsiz)
+                return;
+
+        /* disable timeout */
+        hrtimer_cancel(&op->timer);
+
         /* save rx timestamp */
         op->rx_stamp = skb->tstamp;
         /* save originator for recvfrom() */
@@ -675,13 +696,14 @@ static void bcm_rx_handler(struct sk_buff *skb, void *data)
                  * multiplex compare
                  *
                  * find the first multiplex mask that fits.
-                 * Remark: The MUX-mask is stored in index 0
+                 * Remark: The MUX-mask is stored in index 0 - but only the
+                 * first 64 bits of the frame data[] are relevant (CAN FD)
                  */
 
                 for (i = 1; i < op->nframes; i++) {
-                        if ((GET_U64(&op->frames[0]) & GET_U64(rxframe)) ==
-                            (GET_U64(&op->frames[0]) &
-                             GET_U64(&op->frames[i]))) {
+                        if ((get_u64(op->frames, 0) & get_u64(rxframe, 0)) ==
+                            (get_u64(op->frames, 0) &
+                             get_u64(op->frames + op->cfsiz * i, 0))) {
                                 bcm_rx_cmp_to_index(op, i, rxframe);
                                 break;
                         }
@@ -695,13 +717,14 @@ rx_starttimer:
 /*
  * helpers for bcm_op handling: find & delete bcm [rx|tx] op elements
  */
-static struct bcm_op *bcm_find_op(struct list_head *ops, canid_t can_id,
-                                  int ifindex)
+static struct bcm_op *bcm_find_op(struct list_head *ops,
+                                  struct bcm_msg_head *mh, int ifindex)
 {
         struct bcm_op *op;
 
         list_for_each_entry(op, ops, list) {
-                if ((op->can_id == can_id) && (op->ifindex == ifindex))
+                if ((op->can_id == mh->can_id) && (op->ifindex == ifindex) &&
+                    (op->flags & CAN_FD_FRAME) == (mh->flags & CAN_FD_FRAME))
                         return op;
         }
 
@@ -744,12 +767,14 @@ static void bcm_rx_unreg(struct net_device *dev, struct bcm_op *op)
 /*
  * bcm_delete_rx_op - find and remove a rx op (returns number of removed ops)
  */
-static int bcm_delete_rx_op(struct list_head *ops, canid_t can_id, int ifindex)
+static int bcm_delete_rx_op(struct list_head *ops, struct bcm_msg_head *mh,
+                            int ifindex)
 {
         struct bcm_op *op, *n;
 
         list_for_each_entry_safe(op, n, ops, list) {
-                if ((op->can_id == can_id) && (op->ifindex == ifindex)) {
+                if ((op->can_id == mh->can_id) && (op->ifindex == ifindex) &&
+                    (op->flags & CAN_FD_FRAME) == (mh->flags & CAN_FD_FRAME)) {
 
                         /*
                          * Don't care if we're bound or not (due to netdev
@@ -789,12 +814,14 @@ static int bcm_delete_rx_op(struct list_head *ops, canid_t can_id, int ifindex)
 /*
  * bcm_delete_tx_op - find and remove a tx op (returns number of removed ops)
  */
-static int bcm_delete_tx_op(struct list_head *ops, canid_t can_id, int ifindex)
+static int bcm_delete_tx_op(struct list_head *ops, struct bcm_msg_head *mh,
+                            int ifindex)
 {
         struct bcm_op *op, *n;
 
         list_for_each_entry_safe(op, n, ops, list) {
-                if ((op->can_id == can_id) && (op->ifindex == ifindex)) {
+                if ((op->can_id == mh->can_id) && (op->ifindex == ifindex) &&
+                    (op->flags & CAN_FD_FRAME) == (mh->flags & CAN_FD_FRAME)) {
                         list_del(&op->list);
                         bcm_remove_op(op);
                         return 1; /* done */
@@ -810,7 +837,7 @@ static int bcm_delete_tx_op(struct list_head *ops, canid_t can_id, int ifindex)
 static int bcm_read_op(struct list_head *ops, struct bcm_msg_head *msg_head,
                        int ifindex)
 {
-        struct bcm_op *op = bcm_find_op(ops, msg_head->can_id, ifindex);
+        struct bcm_op *op = bcm_find_op(ops, msg_head, ifindex);
 
         if (!op)
                 return -EINVAL;
@@ -835,6 +862,7 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
 {
         struct bcm_sock *bo = bcm_sk(sk);
         struct bcm_op *op;
+        struct canfd_frame *cf;
         unsigned int i;
         int err;
 
@@ -842,39 +870,46 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
         if (!ifindex)
                 return -ENODEV;
 
-        /* check nframes boundaries - we need at least one can_frame */
+        /* check nframes boundaries - we need at least one CAN frame */
         if (msg_head->nframes < 1 || msg_head->nframes > MAX_NFRAMES)
                 return -EINVAL;
 
         /* check the given can_id */
-        op = bcm_find_op(&bo->tx_ops, msg_head->can_id, ifindex);
-
+        op = bcm_find_op(&bo->tx_ops, msg_head, ifindex);
         if (op) {
                 /* update existing BCM operation */
 
                 /*
-                 * Do we need more space for the can_frames than currently
+                 * Do we need more space for the CAN frames than currently
                  * allocated? -> This is a _really_ unusual use-case and
                  * therefore (complexity / locking) it is not supported.
                  */
                 if (msg_head->nframes > op->nframes)
                         return -E2BIG;
 
-                /* update can_frames content */
+                /* update CAN frames content */
                 for (i = 0; i < msg_head->nframes; i++) {
-                        err = memcpy_from_msg((u8 *)&op->frames[i], msg, CFSIZ);
 
-                        if (op->frames[i].can_dlc > 8)
-                                err = -EINVAL;
+                        cf = op->frames + op->cfsiz * i;
+                        err = memcpy_from_msg((u8 *)cf, msg, op->cfsiz);
+
+                        if (op->flags & CAN_FD_FRAME) {
+                                if (cf->len > 64)
+                                        err = -EINVAL;
+                        } else {
+                                if (cf->len > 8)
+                                        err = -EINVAL;
+                        }
 
                         if (err < 0)
                                 return err;
 
                         if (msg_head->flags & TX_CP_CAN_ID) {
                                 /* copy can_id into frame */
-                                op->frames[i].can_id = msg_head->can_id;
+                                cf->can_id = msg_head->can_id;
                         }
                 }
+                op->flags = msg_head->flags;
 
         } else {
                 /* insert new BCM operation for the given can_id */
@@ -883,11 +918,13 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
                 if (!op)
                         return -ENOMEM;
 
-                op->can_id    = msg_head->can_id;
+                op->can_id = msg_head->can_id;
+                op->cfsiz = CFSIZ(msg_head->flags);
+                op->flags = msg_head->flags;
 
-                /* create array for can_frames and copy the data */
+                /* create array for CAN frames and copy the data */
                 if (msg_head->nframes > 1) {
-                        op->frames = kmalloc(msg_head->nframes * CFSIZ,
+                        op->frames = kmalloc(msg_head->nframes * op->cfsiz,
                                              GFP_KERNEL);
                         if (!op->frames) {
                                 kfree(op);
@@ -897,10 +934,17 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
                         op->frames = &op->sframe;
 
                 for (i = 0; i < msg_head->nframes; i++) {
-                        err = memcpy_from_msg((u8 *)&op->frames[i], msg, CFSIZ);
 
-                        if (op->frames[i].can_dlc > 8)
-                                err = -EINVAL;
+                        cf = op->frames + op->cfsiz * i;
+                        err = memcpy_from_msg((u8 *)cf, msg, op->cfsiz);
+
+                        if (op->flags & CAN_FD_FRAME) {
+                                if (cf->len > 64)
+                                        err = -EINVAL;
+                        } else {
+                                if (cf->len > 8)
+                                        err = -EINVAL;
+                        }
 
                         if (err < 0) {
                                 if (op->frames != &op->sframe)
@@ -911,7 +955,7 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
 
                         if (msg_head->flags & TX_CP_CAN_ID) {
                                 /* copy can_id into frame */
-                                op->frames[i].can_id = msg_head->can_id;
+                                cf->can_id = msg_head->can_id;
                         }
                 }
 
@@ -946,8 +990,6 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
 
         /* check flags */
 
-        op->flags = msg_head->flags;
-
         if (op->flags & TX_RESET_MULTI_IDX) {
                 /* start multiple frame transmission with index 0 */
                 op->currframe = 0;
@@ -968,7 +1010,7 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
 
         if (op->flags & STARTTIMER) {
                 hrtimer_cancel(&op->timer);
-                /* spec: send can_frame when starting timer */
+                /* spec: send CAN frame when starting timer */
                 op->flags |= TX_ANNOUNCE;
         }
 
@@ -981,7 +1023,7 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
         if (op->flags & STARTTIMER)
                 bcm_tx_start_timer(op);
 
-        return msg_head->nframes * CFSIZ + MHSIZ;
+        return msg_head->nframes * op->cfsiz + MHSIZ;
 }
 
 /*
@@ -1012,12 +1054,12 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
                 return -EINVAL;
 
         /* check the given can_id */
-        op = bcm_find_op(&bo->rx_ops, msg_head->can_id, ifindex);
+        op = bcm_find_op(&bo->rx_ops, msg_head, ifindex);
         if (op) {
                 /* update existing BCM operation */
 
                 /*
-                 * Do we need more space for the can_frames than currently
+                 * Do we need more space for the CAN frames than currently
                  * allocated? -> This is a _really_ unusual use-case and
                  * therefore (complexity / locking) it is not supported.
                  */
@@ -1025,17 +1067,18 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
                         return -E2BIG;
 
                 if (msg_head->nframes) {
-                        /* update can_frames content */
+                        /* update CAN frames content */
                         err = memcpy_from_msg((u8 *)op->frames, msg,
-                                              msg_head->nframes * CFSIZ);
+                                              msg_head->nframes * op->cfsiz);
                         if (err < 0)
                                 return err;
 
                         /* clear last_frames to indicate 'nothing received' */
-                        memset(op->last_frames, 0, msg_head->nframes * CFSIZ);
+                        memset(op->last_frames, 0, msg_head->nframes * op->cfsiz);
                 }
 
                 op->nframes = msg_head->nframes;
+                op->flags = msg_head->flags;
 
                 /* Only an update -> do not call can_rx_register() */
                 do_rx_register = 0;
@@ -1046,20 +1089,22 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
                 if (!op)
                         return -ENOMEM;
 
-                op->can_id    = msg_head->can_id;
-                op->nframes   = msg_head->nframes;
+                op->can_id = msg_head->can_id;
+                op->nframes = msg_head->nframes;
+                op->cfsiz = CFSIZ(msg_head->flags);
+                op->flags = msg_head->flags;
 
                 if (msg_head->nframes > 1) {
-                        /* create array for can_frames and copy the data */
-                        op->frames = kmalloc(msg_head->nframes * CFSIZ,
+                        /* create array for CAN frames and copy the data */
+                        op->frames = kmalloc(msg_head->nframes * op->cfsiz,
                                              GFP_KERNEL);
                         if (!op->frames) {
                                 kfree(op);
                                 return -ENOMEM;
                         }
 
-                        /* create and init array for received can_frames */
-                        op->last_frames = kzalloc(msg_head->nframes * CFSIZ,
+                        /* create and init array for received CAN frames */
+                        op->last_frames = kzalloc(msg_head->nframes * op->cfsiz,
                                                   GFP_KERNEL);
                         if (!op->last_frames) {
                                 kfree(op->frames);
@@ -1074,7 +1119,7 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
 
                 if (msg_head->nframes) {
                         err = memcpy_from_msg((u8 *)op->frames, msg,
-                                              msg_head->nframes * CFSIZ);
+                                              msg_head->nframes * op->cfsiz);
                         if (err < 0) {
                                 if (op->frames != &op->sframe)
                                         kfree(op->frames);
@@ -1116,7 +1161,6 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
         } /* if ((op = bcm_find_op(&bo->rx_ops, msg_head->can_id, ifindex))) */
 
         /* check flags */
-        op->flags = msg_head->flags;
 
         if (op->flags & RX_RTR_FRAME) {
 
@@ -1188,13 +1232,14 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
                 }
         }
 
-        return msg_head->nframes * CFSIZ + MHSIZ;
+        return msg_head->nframes * op->cfsiz + MHSIZ;
 }
 
 /*
  * bcm_tx_send - send a single CAN frame to the CAN interface (for bcm_sendmsg)
  */
-static int bcm_tx_send(struct msghdr *msg, int ifindex, struct sock *sk)
+static int bcm_tx_send(struct msghdr *msg, int ifindex, struct sock *sk,
+                       int cfsiz)
 {
         struct sk_buff *skb;
         struct net_device *dev;
@@ -1204,13 +1249,13 @@ static int bcm_tx_send(struct msghdr *msg, int ifindex, struct sock *sk)
         if (!ifindex)
                 return -ENODEV;
 
-        skb = alloc_skb(CFSIZ + sizeof(struct can_skb_priv), GFP_KERNEL);
+        skb = alloc_skb(cfsiz + sizeof(struct can_skb_priv), GFP_KERNEL);
         if (!skb)
                 return -ENOMEM;
 
         can_skb_reserve(skb);
 
-        err = memcpy_from_msg(skb_put(skb, CFSIZ), msg, CFSIZ);
+        err = memcpy_from_msg(skb_put(skb, cfsiz), msg, cfsiz);
         if (err < 0) {
                 kfree_skb(skb);
                 return err;
@@ -1232,7 +1277,7 @@ static int bcm_tx_send(struct msghdr *msg, int ifindex, struct sock *sk)
         if (err)
                 return err;
 
-        return CFSIZ + MHSIZ;
+        return cfsiz + MHSIZ;
 }
 
 /*
@@ -1244,13 +1289,23 @@ static int bcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
         struct bcm_sock *bo = bcm_sk(sk);
         int ifindex = bo->ifindex; /* default ifindex for this bcm_op */
         struct bcm_msg_head msg_head;
+        int cfsiz;
         int ret; /* read bytes or error codes as return value */
 
         if (!bo->bound)
                 return -ENOTCONN;
 
         /* check for valid message length from userspace */
-        if (size < MHSIZ || (size - MHSIZ) % CFSIZ)
+        if (size < MHSIZ)
+                return -EINVAL;
+
+        /* read message head information */
+        ret = memcpy_from_msg((u8 *)&msg_head, msg, MHSIZ);
+        if (ret < 0)
+                return ret;
+
+        cfsiz = CFSIZ(msg_head.flags);
+        if ((size - MHSIZ) % cfsiz)
                 return -EINVAL;
 
         /* check for alternative ifindex for this bcm_op */
@@ -1284,12 +1339,6 @@ static int bcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
                 }
         }
 
-        /* read message head information */
-
-        ret = memcpy_from_msg((u8 *)&msg_head, msg, MHSIZ);
-        if (ret < 0)
-                return ret;
-
         lock_sock(sk);
 
         switch (msg_head.opcode) {
@@ -1303,14 +1352,14 @@ static int bcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
                 break;
 
         case TX_DELETE:
-                if (bcm_delete_tx_op(&bo->tx_ops, msg_head.can_id, ifindex))
+                if (bcm_delete_tx_op(&bo->tx_ops, &msg_head, ifindex))
                         ret = MHSIZ;
                 else
                         ret = -EINVAL;
                 break;
 
         case RX_DELETE:
-                if (bcm_delete_rx_op(&bo->rx_ops, msg_head.can_id, ifindex))
+                if (bcm_delete_rx_op(&bo->rx_ops, &msg_head, ifindex))
                         ret = MHSIZ;
                 else
                         ret = -EINVAL;
@@ -1329,11 +1378,11 @@ static int bcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
                 break;
 
         case TX_SEND:
-                /* we need exactly one can_frame behind the msg head */
-                if ((msg_head.nframes != 1) || (size != CFSIZ + MHSIZ))
+                /* we need exactly one CAN frame behind the msg head */
+                if ((msg_head.nframes != 1) || (size != cfsiz + MHSIZ))
                         ret = -EINVAL;
                 else
-                        ret = bcm_tx_send(msg, ifindex, sk);
+                        ret = bcm_tx_send(msg, ifindex, sk, cfsiz);
                 break;
 
         default:
diff --git a/net/can/proc.c b/net/can/proc.c
index 1a19b985a868..85ef7bb0f176 100644
--- a/net/can/proc.c
+++ b/net/can/proc.c
@@ -517,8 +517,7 @@ void can_init_proc(void)
         can_dir = proc_mkdir("can", init_net.proc_net);
 
         if (!can_dir) {
-                printk(KERN_INFO "can: failed to create /proc/net/can . "
-                       "CONFIG_PROC_FS missing?\n");
+                pr_info("can: failed to create /proc/net/can.\n");
                 return;
         }
 
diff --git a/net/core/dev.c b/net/core/dev.c
index 904ff431d570..aba10d2a8bc3 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -139,6 +139,7 @@
 #include <linux/hrtimer.h>
 #include <linux/netfilter_ingress.h>
 #include <linux/sctp.h>
+#include <linux/crash_dump.h>
 
 #include "net-sysfs.h"
 
@@ -2249,11 +2250,12 @@ EXPORT_SYMBOL(netif_set_real_num_rx_queues);
  */
 int netif_get_num_default_rss_queues(void)
 {
-        return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
+        return is_kdump_kernel() ?
+                1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
 }
 EXPORT_SYMBOL(netif_get_num_default_rss_queues);
 
-static inline void __netif_reschedule(struct Qdisc *q)
+static void __netif_reschedule(struct Qdisc *q)
 {
         struct softnet_data *sd;
         unsigned long flags;
@@ -2420,7 +2422,7 @@ EXPORT_SYMBOL(__skb_tx_hash);
 
 static void skb_warn_bad_offload(const struct sk_buff *skb)
 {
-        static const netdev_features_t null_features = 0;
+        static const netdev_features_t null_features;
         struct net_device *dev = skb->dev;
         const char *name = "";
 
@@ -3068,6 +3070,7 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
                                  struct netdev_queue *txq)
 {
         spinlock_t *root_lock = qdisc_lock(q);
+        struct sk_buff *to_free = NULL;
         bool contended;
         int rc;
 
@@ -3075,7 +3078,7 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
         /*
          * Heuristic to force contended enqueues to serialize on a
          * separate lock before trying to get qdisc main lock.
-         * This permits __QDISC___STATE_RUNNING owner to get the lock more
+         * This permits qdisc->running owner to get the lock more
          * often and dequeue packets faster.
          */
         contended = qdisc_is_running(q);
@@ -3084,7 +3087,7 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
 
         spin_lock(root_lock);
         if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
-                kfree_skb(skb);
+                __qdisc_drop(skb, &to_free);
                 rc = NET_XMIT_DROP;
         } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
                    qdisc_run_begin(q)) {
@@ -3107,7 +3110,7 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
 
                 rc = NET_XMIT_SUCCESS;
         } else {
-                rc = q->enqueue(skb, q) & NET_XMIT_MASK;
+                rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
                 if (qdisc_run_begin(q)) {
                         if (unlikely(contended)) {
                                 spin_unlock(&q->busylock);
@@ -3117,6 +3120,8 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
                 }
         }
         spin_unlock(root_lock);
+        if (unlikely(to_free))
+                kfree_skb_list(to_free);
         if (unlikely(contended))
                 spin_unlock(&q->busylock);
         return rc;
@@ -3142,8 +3147,6 @@ static void skb_update_prio(struct sk_buff *skb)
 DEFINE_PER_CPU(int, xmit_recursion);
 EXPORT_SYMBOL(xmit_recursion);
 
-#define RECURSION_LIMIT 10
-
 /**
  *      dev_loopback_xmit - loop back @skb
  *      @net: network namespace this loopback is happening in
@@ -3386,8 +3389,8 @@ static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
                 int cpu = smp_processor_id(); /* ok because BHs are off */
 
                 if (txq->xmit_lock_owner != cpu) {
-
-                        if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
+                        if (unlikely(__this_cpu_read(xmit_recursion) >
+                                     XMIT_RECURSION_LIMIT))
                                 goto recursion_alert;
 
                         skb = validate_xmit_skb(skb, dev);
@@ -3898,22 +3901,14 @@ static void net_tx_action(struct softirq_action *h)
                         head = head->next_sched;
 
                         root_lock = qdisc_lock(q);
-                        if (spin_trylock(root_lock)) {
-                                smp_mb__before_atomic();
-                                clear_bit(__QDISC_STATE_SCHED,
-                                          &q->state);
-                                qdisc_run(q);
-                                spin_unlock(root_lock);
-                        } else {
-                                if (!test_bit(__QDISC_STATE_DEACTIVATED,
-                                              &q->state)) {
-                                        __netif_reschedule(q);
-                                } else {
-                                        smp_mb__before_atomic();
-                                        clear_bit(__QDISC_STATE_SCHED,
-                                                  &q->state);
-                                }
-                        }
+                        spin_lock(root_lock);
+                        /* We need to make sure head->next_sched is read
+                         * before clearing __QDISC_STATE_SCHED
+                         */
+                        smp_mb__before_atomic();
+                        clear_bit(__QDISC_STATE_SCHED, &q->state);
+                        qdisc_run(q);
+                        spin_unlock(root_lock);
                 }
         }
 }
@@ -5919,7 +5914,7 @@ static void netdev_adjacent_add_links(struct net_device *dev)
         struct net *net = dev_net(dev);
 
         list_for_each_entry(iter, &dev->adj_list.upper, list) {
-                if (!net_eq(net,dev_net(iter->dev)))
+                if (!net_eq(net, dev_net(iter->dev)))
                         continue;
                 netdev_adjacent_sysfs_add(iter->dev, dev,
                                           &iter->dev->adj_list.lower);
@@ -5928,7 +5923,7 @@ static void netdev_adjacent_add_links(struct net_device *dev)
         }
 
         list_for_each_entry(iter, &dev->adj_list.lower, list) {
-                if (!net_eq(net,dev_net(iter->dev)))
+                if (!net_eq(net, dev_net(iter->dev)))
                         continue;
                 netdev_adjacent_sysfs_add(iter->dev, dev,
                                           &iter->dev->adj_list.upper);
@@ -5944,7 +5939,7 @@ static void netdev_adjacent_del_links(struct net_device *dev)
         struct net *net = dev_net(dev);
 
         list_for_each_entry(iter, &dev->adj_list.upper, list) {
-                if (!net_eq(net,dev_net(iter->dev)))
+                if (!net_eq(net, dev_net(iter->dev)))
                         continue;
                 netdev_adjacent_sysfs_del(iter->dev, dev->name,
                                           &iter->dev->adj_list.lower);
@@ -5953,7 +5948,7 @@ static void netdev_adjacent_del_links(struct net_device *dev)
         }
 
         list_for_each_entry(iter, &dev->adj_list.lower, list) {
-                if (!net_eq(net,dev_net(iter->dev)))
+                if (!net_eq(net, dev_net(iter->dev)))
                         continue;
                 netdev_adjacent_sysfs_del(iter->dev, dev->name,
                                           &iter->dev->adj_list.upper);
@@ -5969,7 +5964,7 @@ void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
         struct net *net = dev_net(dev);
 
         list_for_each_entry(iter, &dev->adj_list.upper, list) {
-                if (!net_eq(net,dev_net(iter->dev)))
+                if (!net_eq(net, dev_net(iter->dev)))
                         continue;
                 netdev_adjacent_sysfs_del(iter->dev, oldname,
                                           &iter->dev->adj_list.lower);
@@ -5978,7 +5973,7 @@ void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
         }
 
         list_for_each_entry(iter, &dev->adj_list.lower, list) {
-                if (!net_eq(net,dev_net(iter->dev)))
+                if (!net_eq(net, dev_net(iter->dev)))
                         continue;
                 netdev_adjacent_sysfs_del(iter->dev, oldname,
                                           &iter->dev->adj_list.upper);
diff --git a/net/core/ethtool.c b/net/core/ethtool.c
index f4034817d255..977489820eb9 100644
--- a/net/core/ethtool.c
+++ b/net/core/ethtool.c
@@ -89,6 +89,7 @@ static const char netdev_features_strings[NETDEV_FEATURE_COUNT][ETH_GSTRING_LEN]
         [NETIF_F_GSO_UDP_TUNNEL_BIT] =   "tx-udp_tnl-segmentation",
         [NETIF_F_GSO_UDP_TUNNEL_CSUM_BIT] = "tx-udp_tnl-csum-segmentation",
         [NETIF_F_GSO_PARTIAL_BIT] =      "tx-gso-partial",
+        [NETIF_F_GSO_SCTP_BIT] =         "tx-sctp-segmentation",
 
         [NETIF_F_FCOE_CRC_BIT] =         "tx-checksum-fcoe-crc",
         [NETIF_F_SCTP_CRC_BIT] =        "tx-checksum-sctp",
diff --git a/net/core/fib_rules.c b/net/core/fib_rules.c
index 840acebbb80c..98298b11f534 100644
--- a/net/core/fib_rules.c
+++ b/net/core/fib_rules.c
@@ -173,7 +173,8 @@ void fib_rules_unregister(struct fib_rules_ops *ops)
 EXPORT_SYMBOL_GPL(fib_rules_unregister);
 
 static int fib_rule_match(struct fib_rule *rule, struct fib_rules_ops *ops,
-                          struct flowi *fl, int flags)
+                          struct flowi *fl, int flags,
+                          struct fib_lookup_arg *arg)
 {
         int ret = 0;
 
@@ -189,6 +190,9 @@ static int fib_rule_match(struct fib_rule *rule, struct fib_rules_ops *ops,
         if (rule->tun_id && (rule->tun_id != fl->flowi_tun_key.tun_id))
                 goto out;
 
+        if (rule->l3mdev && !l3mdev_fib_rule_match(rule->fr_net, fl, arg))
+                goto out;
+
         ret = ops->match(rule, fl, flags);
 out:
         return (rule->flags & FIB_RULE_INVERT) ? !ret : ret;
@@ -204,7 +208,7 @@ int fib_rules_lookup(struct fib_rules_ops *ops, struct flowi *fl,
 
         list_for_each_entry_rcu(rule, &ops->rules_list, list) {
 jumped:
-                if (!fib_rule_match(rule, ops, fl, flags))
+                if (!fib_rule_match(rule, ops, fl, flags, arg))
                         continue;
 
                 if (rule->action == FR_ACT_GOTO) {
@@ -265,7 +269,7 @@ errout:
         return err;
 }
 
-static int fib_nl_newrule(struct sk_buff *skb, struct nlmsghdr* nlh)
+int fib_nl_newrule(struct sk_buff *skb, struct nlmsghdr *nlh)
 {
         struct net *net = sock_net(skb->sk);
         struct fib_rule_hdr *frh = nlmsg_data(nlh);
@@ -336,6 +340,14 @@ static int fib_nl_newrule(struct sk_buff *skb, struct nlmsghdr* nlh)
         if (tb[FRA_TUN_ID])
                 rule->tun_id = nla_get_be64(tb[FRA_TUN_ID]);
 
+        if (tb[FRA_L3MDEV]) {
+#ifdef CONFIG_NET_L3_MASTER_DEV
+                rule->l3mdev = nla_get_u8(tb[FRA_L3MDEV]);
+                if (rule->l3mdev != 1)
+#endif
+                        goto errout_free;
+        }
+
         rule->action = frh->action;
         rule->flags = frh->flags;
         rule->table = frh_get_table(frh, tb);
@@ -371,6 +383,9 @@ static int fib_nl_newrule(struct sk_buff *skb, struct nlmsghdr* nlh)
         } else if (rule->action == FR_ACT_GOTO)
                 goto errout_free;
 
+        if (rule->l3mdev && rule->table)
+                goto errout_free;
+
         err = ops->configure(rule, skb, frh, tb);
         if (err < 0)
                 goto errout_free;
@@ -424,8 +439,9 @@ errout:
         rules_ops_put(ops);
         return err;
 }
+EXPORT_SYMBOL_GPL(fib_nl_newrule);
 
-static int fib_nl_delrule(struct sk_buff *skb, struct nlmsghdr* nlh)
+int fib_nl_delrule(struct sk_buff *skb, struct nlmsghdr *nlh)
 {
         struct net *net = sock_net(skb->sk);
         struct fib_rule_hdr *frh = nlmsg_data(nlh);
@@ -483,6 +499,10 @@ static int fib_nl_delrule(struct sk_buff *skb, struct nlmsghdr* nlh)
                     (rule->tun_id != nla_get_be64(tb[FRA_TUN_ID])))
                         continue;
 
+                if (tb[FRA_L3MDEV] &&
+                    (rule->l3mdev != nla_get_u8(tb[FRA_L3MDEV])))
+                        continue;
+
                 if (!ops->compare(rule, frh, tb))
                         continue;
 
@@ -536,6 +556,7 @@ errout:
         rules_ops_put(ops);
         return err;
 }
+EXPORT_SYMBOL_GPL(fib_nl_delrule);
 
 static inline size_t fib_rule_nlmsg_size(struct fib_rules_ops *ops,
                                          struct fib_rule *rule)
@@ -607,7 +628,9 @@ static int fib_nl_fill_rule(struct sk_buff *skb, struct fib_rule *rule,
             (rule->target &&
              nla_put_u32(skb, FRA_GOTO, rule->target)) ||
             (rule->tun_id &&
-             nla_put_be64(skb, FRA_TUN_ID, rule->tun_id, FRA_PAD)))
+             nla_put_be64(skb, FRA_TUN_ID, rule->tun_id, FRA_PAD)) ||
+            (rule->l3mdev &&
+             nla_put_u8(skb, FRA_L3MDEV, rule->l3mdev)))
                 goto nla_put_failure;
 
         if (rule->suppress_ifgroup != -1) {
diff --git a/net/core/filter.c b/net/core/filter.c
index c4b330c85c02..cb9fc16cac46 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -748,6 +748,17 @@ static bool chk_code_allowed(u16 code_to_probe)
         return codes[code_to_probe];
 }
 
+static bool bpf_check_basics_ok(const struct sock_filter *filter,
+                                unsigned int flen)
+{
+        if (filter == NULL)
+                return false;
+        if (flen == 0 || flen > BPF_MAXINSNS)
+                return false;
+
+        return true;
+}
+
 /**
  *      bpf_check_classic - verify socket filter code
  *      @filter: filter to verify
@@ -768,9 +779,6 @@ static int bpf_check_classic(const struct sock_filter *filter,
         bool anc_found;
         int pc;
 
-        if (flen == 0 || flen > BPF_MAXINSNS)
-                return -EINVAL;
-
         /* Check the filter code now */
         for (pc = 0; pc < flen; pc++) {
                 const struct sock_filter *ftest = &filter[pc];
@@ -1065,7 +1073,7 @@ int bpf_prog_create(struct bpf_prog **pfp, struct sock_fprog_kern *fprog)
         struct bpf_prog *fp;
 
         /* Make sure new filter is there and in the right amounts. */
-        if (fprog->filter == NULL)
+        if (!bpf_check_basics_ok(fprog->filter, fprog->len))
                 return -EINVAL;
 
         fp = bpf_prog_alloc(bpf_prog_size(fprog->len), 0);
@@ -1112,7 +1120,7 @@ int bpf_prog_create_from_user(struct bpf_prog **pfp, struct sock_fprog *fprog,
         int err;
 
         /* Make sure new filter is there and in the right amounts. */
-        if (fprog->filter == NULL)
+        if (!bpf_check_basics_ok(fprog->filter, fprog->len))
                 return -EINVAL;
 
         fp = bpf_prog_alloc(bpf_prog_size(fprog->len), 0);
@@ -1207,7 +1215,6 @@ static
 struct bpf_prog *__get_filter(struct sock_fprog *fprog, struct sock *sk)
 {
         unsigned int fsize = bpf_classic_proglen(fprog);
-        unsigned int bpf_fsize = bpf_prog_size(fprog->len);
         struct bpf_prog *prog;
         int err;
 
@@ -1215,10 +1222,10 @@ struct bpf_prog *__get_filter(struct sock_fprog *fprog, struct sock *sk)
                 return ERR_PTR(-EPERM);
 
         /* Make sure new filter is there and in the right amounts. */
-        if (fprog->filter == NULL)
+        if (!bpf_check_basics_ok(fprog->filter, fprog->len))
                 return ERR_PTR(-EINVAL);
 
-        prog = bpf_prog_alloc(bpf_fsize, 0);
+        prog = bpf_prog_alloc(bpf_prog_size(fprog->len), 0);
         if (!prog)
                 return ERR_PTR(-ENOMEM);
 
@@ -1603,9 +1610,36 @@ static const struct bpf_func_proto bpf_csum_diff_proto = {
         .arg5_type      = ARG_ANYTHING,
 };
 
+static inline int __bpf_rx_skb(struct net_device *dev, struct sk_buff *skb)
+{
+        if (skb_at_tc_ingress(skb))
+                skb_postpush_rcsum(skb, skb_mac_header(skb), skb->mac_len);
+
+        return dev_forward_skb(dev, skb);
+}
+
+static inline int __bpf_tx_skb(struct net_device *dev, struct sk_buff *skb)
+{
+        int ret;
+
+        if (unlikely(__this_cpu_read(xmit_recursion) > XMIT_RECURSION_LIMIT)) {
+                net_crit_ratelimited("bpf: recursion limit reached on datapath, buggy bpf program?\n");
+                kfree_skb(skb);
+                return -ENETDOWN;
+        }
+
+        skb->dev = dev;
+
+        __this_cpu_inc(xmit_recursion);
+        ret = dev_queue_xmit(skb);
+        __this_cpu_dec(xmit_recursion);
+
+        return ret;
+}
+
 static u64 bpf_clone_redirect(u64 r1, u64 ifindex, u64 flags, u64 r4, u64 r5)
 {
-        struct sk_buff *skb = (struct sk_buff *) (long) r1, *skb2;
+        struct sk_buff *skb = (struct sk_buff *) (long) r1;
         struct net_device *dev;
 
         if (unlikely(flags & ~(BPF_F_INGRESS)))
@@ -1615,19 +1649,12 @@ static u64 bpf_clone_redirect(u64 r1, u64 ifindex, u64 flags, u64 r4, u64 r5)
         if (unlikely(!dev))
                 return -EINVAL;
 
-        skb2 = skb_clone(skb, GFP_ATOMIC);
-        if (unlikely(!skb2))
+        skb = skb_clone(skb, GFP_ATOMIC);
+        if (unlikely(!skb))
                 return -ENOMEM;
 
-        if (flags & BPF_F_INGRESS) {
-                if (skb_at_tc_ingress(skb2))
-                        skb_postpush_rcsum(skb2, skb_mac_header(skb2),
-                                           skb2->mac_len);
-                return dev_forward_skb(dev, skb2);
-        }
-
-        skb2->dev = dev;
-        return dev_queue_xmit(skb2);
+        return flags & BPF_F_INGRESS ?
+               __bpf_rx_skb(dev, skb) : __bpf_tx_skb(dev, skb);
 }
 
 static const struct bpf_func_proto bpf_clone_redirect_proto = {
@@ -1671,15 +1698,8 @@ int skb_do_redirect(struct sk_buff *skb)
                 return -EINVAL;
         }
 
-        if (ri->flags & BPF_F_INGRESS) {
-                if (skb_at_tc_ingress(skb))
-                        skb_postpush_rcsum(skb, skb_mac_header(skb),
-                                           skb->mac_len);
-                return dev_forward_skb(dev, skb);
-        }
-
-        skb->dev = dev;
-        return dev_queue_xmit(skb);
+        return ri->flags & BPF_F_INGRESS ?
+               __bpf_rx_skb(dev, skb) : __bpf_tx_skb(dev, skb);
 }
 
 static const struct bpf_func_proto bpf_redirect_proto = {
diff --git a/net/core/gen_estimator.c b/net/core/gen_estimator.c
index 4573d81093fe..cad8e791f28e 100644
--- a/net/core/gen_estimator.c
+++ b/net/core/gen_estimator.c
@@ -84,6 +84,7 @@ struct gen_estimator
         struct gnet_stats_basic_packed  *bstats;
         struct gnet_stats_rate_est64    *rate_est;
         spinlock_t              *stats_lock;
+        seqcount_t              *running;
         int                     ewma_log;
         u32                     last_packets;
         unsigned long           avpps;
@@ -121,26 +122,28 @@ static void est_timer(unsigned long arg)
                 unsigned long rate;
                 u64 brate;
 
-                spin_lock(e->stats_lock);
+                if (e->stats_lock)
+                        spin_lock(e->stats_lock);
                 read_lock(&est_lock);
                 if (e->bstats == NULL)
                         goto skip;
 
-                __gnet_stats_copy_basic(&b, e->cpu_bstats, e->bstats);
+                __gnet_stats_copy_basic(e->running, &b, e->cpu_bstats, e->bstats);
 
                 brate = (b.bytes - e->last_bytes)<<(7 - idx);
                 e->last_bytes = b.bytes;
                 e->avbps += (brate >> e->ewma_log) - (e->avbps >> e->ewma_log);
-                e->rate_est->bps = (e->avbps+0xF)>>5;
+                WRITE_ONCE(e->rate_est->bps, (e->avbps + 0xF) >> 5);
 
                 rate = b.packets - e->last_packets;
                 rate <<= (7 - idx);
                 e->last_packets = b.packets;
                 e->avpps += (rate >> e->ewma_log) - (e->avpps >> e->ewma_log);
-                e->rate_est->pps = (e->avpps + 0xF) >> 5;
+                WRITE_ONCE(e->rate_est->pps, (e->avpps + 0xF) >> 5);
 skip:
                 read_unlock(&est_lock);
-                spin_unlock(e->stats_lock);
+                if (e->stats_lock)
+                        spin_unlock(e->stats_lock);
         }
 
         if (!list_empty(&elist[idx].list))
@@ -194,6 +197,7 @@ struct gen_estimator *gen_find_node(const struct gnet_stats_basic_packed *bstats
  * @cpu_bstats: bstats per cpu
  * @rate_est: rate estimator statistics
  * @stats_lock: statistics lock
+ * @running: qdisc running seqcount
  * @opt: rate estimator configuration TLV
  *
  * Creates a new rate estimator with &bstats as source and &rate_est
@@ -209,6 +213,7 @@ int gen_new_estimator(struct gnet_stats_basic_packed *bstats,
                       struct gnet_stats_basic_cpu __percpu *cpu_bstats,
                       struct gnet_stats_rate_est64 *rate_est,
                       spinlock_t *stats_lock,
+                      seqcount_t *running,
                       struct nlattr *opt)
 {
         struct gen_estimator *est;
@@ -226,12 +231,13 @@ int gen_new_estimator(struct gnet_stats_basic_packed *bstats,
         if (est == NULL)
                 return -ENOBUFS;
 
-        __gnet_stats_copy_basic(&b, cpu_bstats, bstats);
+        __gnet_stats_copy_basic(running, &b, cpu_bstats, bstats);
 
         idx = parm->interval + 2;
         est->bstats = bstats;
         est->rate_est = rate_est;
         est->stats_lock = stats_lock;
+        est->running  = running;
         est->ewma_log = parm->ewma_log;
         est->last_bytes = b.bytes;
         est->avbps = rate_est->bps<<5;
@@ -291,6 +297,7 @@ EXPORT_SYMBOL(gen_kill_estimator);
  * @cpu_bstats: bstats per cpu
  * @rate_est: rate estimator statistics
  * @stats_lock: statistics lock
+ * @running: qdisc running seqcount (might be NULL)
  * @opt: rate estimator configuration TLV
  *
  * Replaces the configuration of a rate estimator by calling
@@ -301,10 +308,11 @@ EXPORT_SYMBOL(gen_kill_estimator);
 int gen_replace_estimator(struct gnet_stats_basic_packed *bstats,
                           struct gnet_stats_basic_cpu __percpu *cpu_bstats,
                           struct gnet_stats_rate_est64 *rate_est,
-                          spinlock_t *stats_lock, struct nlattr *opt)
+                          spinlock_t *stats_lock,
+                          seqcount_t *running, struct nlattr *opt)
 {
         gen_kill_estimator(bstats, rate_est);
-        return gen_new_estimator(bstats, cpu_bstats, rate_est, stats_lock, opt);
+        return gen_new_estimator(bstats, cpu_bstats, rate_est, stats_lock, running, opt);
 }
 EXPORT_SYMBOL(gen_replace_estimator);
 
diff --git a/net/core/gen_stats.c b/net/core/gen_stats.c
index be873e4e3125..508e051304fb 100644
--- a/net/core/gen_stats.c
+++ b/net/core/gen_stats.c
@@ -32,10 +32,11 @@ gnet_stats_copy(struct gnet_dump *d, int type, void *buf, int size, int padattr)
         return 0;
 
 nla_put_failure:
+        if (d->lock)
+                spin_unlock_bh(d->lock);
         kfree(d->xstats);
         d->xstats = NULL;
         d->xstats_len = 0;
-        spin_unlock_bh(d->lock);
         return -1;
 }
 
@@ -66,15 +67,16 @@ gnet_stats_start_copy_compat(struct sk_buff *skb, int type, int tc_stats_type,
 {
         memset(d, 0, sizeof(*d));
 
-        spin_lock_bh(lock);
-        d->lock = lock;
         if (type)
                 d->tail = (struct nlattr *)skb_tail_pointer(skb);
         d->skb = skb;
         d->compat_tc_stats = tc_stats_type;
         d->compat_xstats = xstats_type;
         d->padattr = padattr;
-
+        if (lock) {
+                d->lock = lock;
+                spin_lock_bh(lock);
+        }
         if (d->tail)
                 return gnet_stats_copy(d, type, NULL, 0, padattr);
 
@@ -128,21 +130,29 @@ __gnet_stats_copy_basic_cpu(struct gnet_stats_basic_packed *bstats,
 }
 
 void
-__gnet_stats_copy_basic(struct gnet_stats_basic_packed *bstats,
+__gnet_stats_copy_basic(const seqcount_t *running,
+                        struct gnet_stats_basic_packed *bstats,
                         struct gnet_stats_basic_cpu __percpu *cpu,
                         struct gnet_stats_basic_packed *b)
 {
+        unsigned int seq;
+
         if (cpu) {
                 __gnet_stats_copy_basic_cpu(bstats, cpu);
-        } else {
+                return;
+        }
+        do {
+                if (running)
+                        seq = read_seqcount_begin(running);
                 bstats->bytes = b->bytes;
                 bstats->packets = b->packets;
-        }
+        } while (running && read_seqcount_retry(running, seq));
 }
 EXPORT_SYMBOL(__gnet_stats_copy_basic);
 
 /**
  * gnet_stats_copy_basic - copy basic statistics into statistic TLV
+ * @running: seqcount_t pointer
  * @d: dumping handle
  * @cpu: copy statistic per cpu
  * @b: basic statistics
@@ -154,13 +164,14 @@ EXPORT_SYMBOL(__gnet_stats_copy_basic);
  * if the room in the socket buffer was not sufficient.
  */
 int
-gnet_stats_copy_basic(struct gnet_dump *d,
+gnet_stats_copy_basic(const seqcount_t *running,
+                      struct gnet_dump *d,
                       struct gnet_stats_basic_cpu __percpu *cpu,
                       struct gnet_stats_basic_packed *b)
 {
         struct gnet_stats_basic_packed bstats = {0};
 
-        __gnet_stats_copy_basic(&bstats, cpu, b);
+        __gnet_stats_copy_basic(running, &bstats, cpu, b);
 
         if (d->compat_tc_stats) {
                 d->tc_stats.bytes = bstats.bytes;
@@ -330,8 +341,9 @@ gnet_stats_copy_app(struct gnet_dump *d, void *st, int len)
         return 0;
 
 err_out:
+        if (d->lock)
+                spin_unlock_bh(d->lock);
         d->xstats_len = 0;
-        spin_unlock_bh(d->lock);
         return -1;
 }
 EXPORT_SYMBOL(gnet_stats_copy_app);
@@ -365,10 +377,11 @@ gnet_stats_finish_copy(struct gnet_dump *d)
                         return -1;
         }
 
+        if (d->lock)
+                spin_unlock_bh(d->lock);
         kfree(d->xstats);
         d->xstats = NULL;
         d->xstats_len = 0;
-        spin_unlock_bh(d->lock);
         return 0;
 }
 EXPORT_SYMBOL(gnet_stats_finish_copy);
diff --git a/net/core/pktgen.c b/net/core/pktgen.c
index 8b02df0d354d..f74ab9c3b38f 100644
--- a/net/core/pktgen.c
+++ b/net/core/pktgen.c
@@ -3463,7 +3463,6 @@ xmit_more:
                 break;
         case NET_XMIT_DROP:
         case NET_XMIT_CN:
-        case NET_XMIT_POLICED:
                 /* skb has been consumed */
                 pkt_dev->errors++;
                 break;
diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c
index d69c4644f8f2..eb49ca24274a 100644
--- a/net/core/rtnetlink.c
+++ b/net/core/rtnetlink.c
@@ -71,9 +71,31 @@ void rtnl_lock(void)
 }
 EXPORT_SYMBOL(rtnl_lock);
 
+static struct sk_buff *defer_kfree_skb_list;
+void rtnl_kfree_skbs(struct sk_buff *head, struct sk_buff *tail)
+{
+        if (head && tail) {
+                tail->next = defer_kfree_skb_list;
+                defer_kfree_skb_list = head;
+        }
+}
+EXPORT_SYMBOL(rtnl_kfree_skbs);
+
 void __rtnl_unlock(void)
 {
+        struct sk_buff *head = defer_kfree_skb_list;
+
+        defer_kfree_skb_list = NULL;
+
         mutex_unlock(&rtnl_mutex);
+
+        while (head) {
+                struct sk_buff *next = head->next;
+
+                kfree_skb(head);
+                cond_resched();
+                head = next;
+        }
 }
 
 void rtnl_unlock(void)
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index f2b77e549c03..e7ec6d3ad5f0 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -49,6 +49,7 @@
 #include <linux/slab.h>
 #include <linux/tcp.h>
 #include <linux/udp.h>
+#include <linux/sctp.h>
 #include <linux/netdevice.h>
 #ifdef CONFIG_NET_CLS_ACT
 #include <net/pkt_sched.h>
@@ -3116,9 +3117,13 @@ struct sk_buff *skb_segment(struct sk_buff *head_skb,
                 int hsize;
                 int size;
 
-                len = head_skb->len - offset;
-                if (len > mss)
-                        len = mss;
+                if (unlikely(mss == GSO_BY_FRAGS)) {
+                        len = list_skb->len;
+                } else {
+                        len = head_skb->len - offset;
+                        if (len > mss)
+                                len = mss;
+                }
 
                 hsize = skb_headlen(head_skb) - offset;
                 if (hsize < 0)
@@ -3438,6 +3443,7 @@ done:
         NAPI_GRO_CB(skb)->same_flow = 1;
         return 0;
 }
+EXPORT_SYMBOL_GPL(skb_gro_receive);
 
 void __init skb_init(void)
 {
@@ -4378,6 +4384,8 @@ unsigned int skb_gso_transport_seglen(const struct sk_buff *skb)
                         thlen += inner_tcp_hdrlen(skb);
         } else if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
                 thlen = tcp_hdrlen(skb);
+        } else if (unlikely(shinfo->gso_type & SKB_GSO_SCTP)) {
+                thlen = sizeof(struct sctphdr);
         }
         /* UFO sets gso_size to the size of the fragmentation
          * payload, i.e. the size of the L4 (UDP) header is already
@@ -4387,6 +4395,38 @@ unsigned int skb_gso_transport_seglen(const struct sk_buff *skb)
 }
 EXPORT_SYMBOL_GPL(skb_gso_transport_seglen);
 
+/**
+ * skb_gso_validate_mtu - Return in case such skb fits a given MTU
+ *
+ * @skb: GSO skb
+ * @mtu: MTU to validate against
+ *
+ * skb_gso_validate_mtu validates if a given skb will fit a wanted MTU
+ * once split.
+ */
+bool skb_gso_validate_mtu(const struct sk_buff *skb, unsigned int mtu)
+{
+        const struct skb_shared_info *shinfo = skb_shinfo(skb);
+        const struct sk_buff *iter;
+        unsigned int hlen;
+
+        hlen = skb_gso_network_seglen(skb);
+
+        if (shinfo->gso_size != GSO_BY_FRAGS)
+                return hlen <= mtu;
+
+        /* Undo this so we can re-use header sizes */
+        hlen -= GSO_BY_FRAGS;
+
+        skb_walk_frags(skb, iter) {
+                if (hlen + skb_headlen(iter) > mtu)
+                        return false;
+        }
+
+        return true;
+}
+EXPORT_SYMBOL_GPL(skb_gso_validate_mtu);
+
 static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb)
 {
         if (skb_cow(skb, skb_headroom(skb)) < 0) {
diff --git a/net/core/utils.c b/net/core/utils.c
index 3d17ca8b4744..cf5622b9ccc4 100644
--- a/net/core/utils.c
+++ b/net/core/utils.c
@@ -133,7 +133,7 @@ int in4_pton(const char *src, int srclen,
         s = src;
         d = dbuf;
         i = 0;
-        while(1) {
+        while (1) {
                 int c;
                 c = xdigit2bin(srclen > 0 ? *s : '\0', delim);
                 if (!(c & (IN6PTON_DIGIT | IN6PTON_DOT | IN6PTON_DELIM | IN6PTON_COLON_MASK))) {
@@ -283,11 +283,11 @@ cont:
         i = 15; d--;
 
         if (dc) {
-                while(d >= dc)
+                while (d >= dc)
                         dst[i--] = *d--;
-                while(i >= dc - dbuf)
+                while (i >= dc - dbuf)
                         dst[i--] = 0;
-                while(i >= 0)
+                while (i >= 0)
                         dst[i--] = *d--;
         } else
                 memcpy(dst, dbuf, sizeof(dbuf));
diff --git a/net/dsa/Makefile b/net/dsa/Makefile
index da06ed1df620..8af4ded70f1c 100644
--- a/net/dsa/Makefile
+++ b/net/dsa/Makefile
@@ -1,6 +1,6 @@
 # the core
 obj-$(CONFIG_NET_DSA) += dsa_core.o
-dsa_core-y += dsa.o slave.o
+dsa_core-y += dsa.o slave.o dsa2.o
 
 # tagging formats
 dsa_core-$(CONFIG_NET_DSA_TAG_BRCM) += tag_brcm.o
diff --git a/net/dsa/dsa.c b/net/dsa/dsa.c
index eff5dfc2e33f..766d2a525ada 100644
--- a/net/dsa/dsa.c
+++ b/net/dsa/dsa.c
@@ -29,6 +29,33 @@
 
 char dsa_driver_version[] = "0.1";
 
+static struct sk_buff *dsa_slave_notag_xmit(struct sk_buff *skb,
+                                            struct net_device *dev)
+{
+        /* Just return the original SKB */
+        return skb;
+}
+
+static const struct dsa_device_ops none_ops = {
+        .xmit   = dsa_slave_notag_xmit,
+        .rcv    = NULL,
+};
+
+const struct dsa_device_ops *dsa_device_ops[DSA_TAG_LAST] = {
+#ifdef CONFIG_NET_DSA_TAG_DSA
+        [DSA_TAG_PROTO_DSA] = &dsa_netdev_ops,
+#endif
+#ifdef CONFIG_NET_DSA_TAG_EDSA
+        [DSA_TAG_PROTO_EDSA] = &edsa_netdev_ops,
+#endif
+#ifdef CONFIG_NET_DSA_TAG_TRAILER
+        [DSA_TAG_PROTO_TRAILER] = &trailer_netdev_ops,
+#endif
+#ifdef CONFIG_NET_DSA_TAG_BRCM
+        [DSA_TAG_PROTO_BRCM] = &brcm_netdev_ops,
+#endif
+        [DSA_TAG_PROTO_NONE] = &none_ops,
+};
 
 /* switch driver registration ***********************************************/
 static DEFINE_MUTEX(dsa_switch_drivers_mutex);
@@ -180,41 +207,100 @@ __ATTRIBUTE_GROUPS(dsa_hwmon);
 #endif /* CONFIG_NET_DSA_HWMON */
 
 /* basic switch operations **************************************************/
-static int dsa_cpu_dsa_setup(struct dsa_switch *ds, struct net_device *master)
+int dsa_cpu_dsa_setup(struct dsa_switch *ds, struct device *dev,
+                      struct device_node *port_dn, int port)
 {
-        struct dsa_chip_data *cd = ds->cd;
-        struct device_node *port_dn;
         struct phy_device *phydev;
-        int ret, port, mode;
+        int ret, mode;
+
+        if (of_phy_is_fixed_link(port_dn)) {
+                ret = of_phy_register_fixed_link(port_dn);
+                if (ret) {
+                        dev_err(dev, "failed to register fixed PHY\n");
+                        return ret;
+                }
+                phydev = of_phy_find_device(port_dn);
+
+                mode = of_get_phy_mode(port_dn);
+                if (mode < 0)
+                        mode = PHY_INTERFACE_MODE_NA;
+                phydev->interface = mode;
+
+                genphy_config_init(phydev);
+                genphy_read_status(phydev);
+                if (ds->drv->adjust_link)
+                        ds->drv->adjust_link(ds, port, phydev);
+        }
+
+        return 0;
+}
+
+static int dsa_cpu_dsa_setups(struct dsa_switch *ds, struct device *dev)
+{
+        struct device_node *port_dn;
+        int ret, port;
 
         for (port = 0; port < DSA_MAX_PORTS; port++) {
                 if (!(dsa_is_cpu_port(ds, port) || dsa_is_dsa_port(ds, port)))
                         continue;
 
-                port_dn = cd->port_dn[port];
-                if (of_phy_is_fixed_link(port_dn)) {
-                        ret = of_phy_register_fixed_link(port_dn);
-                        if (ret) {
-                                netdev_err(master,
-                                           "failed to register fixed PHY\n");
-                                return ret;
-                        }
-                        phydev = of_phy_find_device(port_dn);
+                port_dn = ds->ports[port].dn;
+                ret = dsa_cpu_dsa_setup(ds, dev, port_dn, port);
+                if (ret)
+                        return ret;
+        }
+        return 0;
+}
 
-                        mode = of_get_phy_mode(port_dn);
-                        if (mode < 0)
-                                mode = PHY_INTERFACE_MODE_NA;
-                        phydev->interface = mode;
+const struct dsa_device_ops *dsa_resolve_tag_protocol(int tag_protocol)
+{
+        const struct dsa_device_ops *ops;
+
+        if (tag_protocol >= DSA_TAG_LAST)
+                return ERR_PTR(-EINVAL);
+        ops = dsa_device_ops[tag_protocol];
+
+        if (!ops)
+                return ERR_PTR(-ENOPROTOOPT);
+
+        return ops;
+}
+
+int dsa_cpu_port_ethtool_setup(struct dsa_switch *ds)
+{
+        struct net_device *master;
+        struct ethtool_ops *cpu_ops;
+
+        master = ds->dst->master_netdev;
+        if (ds->master_netdev)
+                master = ds->master_netdev;
+
+        cpu_ops = devm_kzalloc(ds->dev, sizeof(*cpu_ops), GFP_KERNEL);
+        if (!cpu_ops)
+                return -ENOMEM;
+
+        memcpy(&ds->dst->master_ethtool_ops, master->ethtool_ops,
+               sizeof(struct ethtool_ops));
+        ds->dst->master_orig_ethtool_ops = master->ethtool_ops;
+        memcpy(cpu_ops, &ds->dst->master_ethtool_ops,
+               sizeof(struct ethtool_ops));
+        dsa_cpu_port_ethtool_init(cpu_ops);
+        master->ethtool_ops = cpu_ops;
 
-                        genphy_config_init(phydev);
-                        genphy_read_status(phydev);
-                        if (ds->drv->adjust_link)
-                                ds->drv->adjust_link(ds, port, phydev);
-                }
-        }
         return 0;
 }
 
+void dsa_cpu_port_ethtool_restore(struct dsa_switch *ds)
+{
+        struct net_device *master;
+
+        master = ds->dst->master_netdev;
+        if (ds->master_netdev)
+                master = ds->master_netdev;
+
+        master->ethtool_ops = ds->dst->master_orig_ethtool_ops;
+}
+
 static int dsa_switch_setup_one(struct dsa_switch *ds, struct device *parent)
 {
         struct dsa_switch_driver *drv = ds->drv;
@@ -243,6 +329,7 @@ static int dsa_switch_setup_one(struct dsa_switch *ds, struct device *parent)
                         }
                         dst->cpu_switch = index;
                         dst->cpu_port = i;
+                        ds->cpu_port_mask |= 1 << i;
                 } else if (!strcmp(name, "dsa")) {
                         ds->dsa_port_mask |= 1 << i;
                 } else {
@@ -267,37 +354,17 @@ static int dsa_switch_setup_one(struct dsa_switch *ds, struct device *parent)
          * switch.
          */
         if (dst->cpu_switch == index) {
-                switch (drv->tag_protocol) {
-#ifdef CONFIG_NET_DSA_TAG_DSA
-                case DSA_TAG_PROTO_DSA:
-                        dst->rcv = dsa_netdev_ops.rcv;
-                        break;
-#endif
-#ifdef CONFIG_NET_DSA_TAG_EDSA
-                case DSA_TAG_PROTO_EDSA:
-                        dst->rcv = edsa_netdev_ops.rcv;
-                        break;
-#endif
-#ifdef CONFIG_NET_DSA_TAG_TRAILER
-                case DSA_TAG_PROTO_TRAILER:
-                        dst->rcv = trailer_netdev_ops.rcv;
-                        break;
-#endif
-#ifdef CONFIG_NET_DSA_TAG_BRCM
-                case DSA_TAG_PROTO_BRCM:
-                        dst->rcv = brcm_netdev_ops.rcv;
-                        break;
-#endif
-                case DSA_TAG_PROTO_NONE:
-                        break;
-                default:
-                        ret = -ENOPROTOOPT;
+                dst->tag_ops = dsa_resolve_tag_protocol(drv->tag_protocol);
+                if (IS_ERR(dst->tag_ops)) {
+                        ret = PTR_ERR(dst->tag_ops);
                         goto out;
                 }
 
-                dst->tag_protocol = drv->tag_protocol;
+                dst->rcv = dst->tag_ops->rcv;
         }
 
+        memcpy(ds->rtable, cd->rtable, sizeof(ds->rtable));
+
         /*
          * Do basic register setup.
          */
@@ -309,22 +376,25 @@ static int dsa_switch_setup_one(struct dsa_switch *ds, struct device *parent)
         if (ret < 0)
                 goto out;
 
-        ds->slave_mii_bus = devm_mdiobus_alloc(parent);
-        if (ds->slave_mii_bus == NULL) {
-                ret = -ENOMEM;
-                goto out;
-        }
-        dsa_slave_mii_bus_init(ds);
-
-        ret = mdiobus_register(ds->slave_mii_bus);
-        if (ret < 0)
-                goto out;
+        if (!ds->slave_mii_bus && drv->phy_read) {
+                ds->slave_mii_bus = devm_mdiobus_alloc(parent);
+                if (!ds->slave_mii_bus) {
+                        ret = -ENOMEM;
+                        goto out;
+                }
+                dsa_slave_mii_bus_init(ds);
 
+                ret = mdiobus_register(ds->slave_mii_bus);
+                if (ret < 0)
+                        goto out;
+        }
 
         /*
          * Create network devices for physical switch ports.
          */
         for (i = 0; i < DSA_MAX_PORTS; i++) {
+                ds->ports[i].dn = cd->port_dn[i];
+
                 if (!(ds->enabled_port_mask & (1 << i)))
                         continue;
 
@@ -337,13 +407,17 @@ static int dsa_switch_setup_one(struct dsa_switch *ds, struct device *parent)
         }
 
         /* Perform configuration of the CPU and DSA ports */
-        ret = dsa_cpu_dsa_setup(ds, dst->master_netdev);
+        ret = dsa_cpu_dsa_setups(ds, parent);
         if (ret < 0) {
                 netdev_err(dst->master_netdev, "[%d] : can't configure CPU and DSA ports\n",
                            index);
                 ret = 0;
         }
 
+        ret = dsa_cpu_port_ethtool_setup(ds);
+        if (ret)
+                return ret;
+
 #ifdef CONFIG_NET_DSA_HWMON
         /* If the switch provides a temperature sensor,
          * register with hardware monitoring subsystem.
@@ -420,11 +494,21 @@ dsa_switch_setup(struct dsa_switch_tree *dst, int index,
         return ds;
 }
 
-static void dsa_switch_destroy(struct dsa_switch *ds)
+void dsa_cpu_dsa_destroy(struct device_node *port_dn)
 {
-        struct device_node *port_dn;
         struct phy_device *phydev;
-        struct dsa_chip_data *cd = ds->cd;
+
+        if (of_phy_is_fixed_link(port_dn)) {
+                phydev = of_phy_find_device(port_dn);
+                if (phydev) {
+                        phy_device_free(phydev);
+                        fixed_phy_unregister(phydev);
+                }
+        }
+}
+
+static void dsa_switch_destroy(struct dsa_switch *ds)
+{
         int port;
 
 #ifdef CONFIG_NET_DSA_HWMON
@@ -437,26 +521,25 @@ static void dsa_switch_destroy(struct dsa_switch *ds)
                 if (!(ds->enabled_port_mask & (1 << port)))
                         continue;
 
-                if (!ds->ports[port])
+                if (!ds->ports[port].netdev)
                         continue;
 
-                dsa_slave_destroy(ds->ports[port]);
+                dsa_slave_destroy(ds->ports[port].netdev);
         }
 
-        /* Remove any fixed link PHYs */
+        /* Disable configuration of the CPU and DSA ports */
         for (port = 0; port < DSA_MAX_PORTS; port++) {
-                port_dn = cd->port_dn[port];
-                if (of_phy_is_fixed_link(port_dn)) {
-                        phydev = of_phy_find_device(port_dn);
-                        if (phydev) {
-                                phy_device_free(phydev);
-                                of_node_put(port_dn);
-                                fixed_phy_unregister(phydev);
-                        }
-                }
+                if (!(dsa_is_cpu_port(ds, port) || dsa_is_dsa_port(ds, port)))
+                        continue;
+                dsa_cpu_dsa_destroy(ds->ports[port].dn);
+
+                /* Clearing a bit which is not set does no harm */
+                ds->cpu_port_mask |= ~(1 << port);
+                ds->dsa_port_mask |= ~(1 << port);
         }
 
-        mdiobus_unregister(ds->slave_mii_bus);
+        if (ds->slave_mii_bus && ds->drv->phy_read)
+                mdiobus_unregister(ds->slave_mii_bus);
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -469,7 +552,7 @@ static int dsa_switch_suspend(struct dsa_switch *ds)
                 if (!dsa_is_port_initialized(ds, i))
                         continue;
 
-                ret = dsa_slave_suspend(ds->ports[i]);
+                ret = dsa_slave_suspend(ds->ports[i].netdev);
                 if (ret)
                         return ret;
         }
@@ -495,7 +578,7 @@ static int dsa_switch_resume(struct dsa_switch *ds)
                 if (!dsa_is_port_initialized(ds, i))
                         continue;
 
-                ret = dsa_slave_resume(ds->ports[i]);
+                ret = dsa_slave_resume(ds->ports[i].netdev);
                 if (ret)
                         return ret;
         }
@@ -587,17 +670,6 @@ static int dsa_of_setup_routing_table(struct dsa_platform_data *pd,
         if (link_sw_addr >= pd->nr_chips)
                 return -EINVAL;
 
-        /* First time routing table allocation */
-        if (!cd->rtable) {
-                cd->rtable = kmalloc_array(pd->nr_chips, sizeof(s8),
-                                           GFP_KERNEL);
-                if (!cd->rtable)
-                        return -ENOMEM;
-
-                /* default to no valid uplink/downlink */
-                memset(cd->rtable, -1, pd->nr_chips * sizeof(s8));
-        }
-
         cd->rtable[link_sw_addr] = port_index;
 
         return 0;
@@ -639,7 +711,6 @@ static void dsa_of_free_platform_data(struct dsa_platform_data *pd)
                         kfree(pd->chip[i].port_names[port_index]);
                         port_index++;
                 }
-                kfree(pd->chip[i].rtable);
 
                 /* Drop our reference to the MDIO bus device */
                 if (pd->chip[i].host_dev)
@@ -931,6 +1002,8 @@ static void dsa_remove_dst(struct dsa_switch_tree *dst)
                         dsa_switch_destroy(ds);
         }
 
+        dsa_cpu_port_ethtool_restore(dst->ds[0]);
+
         dev_put(dst->master_netdev);
 }
 
diff --git a/net/dsa/dsa2.c b/net/dsa/dsa2.c
new file mode 100644
index 000000000000..83b95fc4cede
--- /dev/null
+++ b/net/dsa/dsa2.c
@@ -0,0 +1,690 @@
+/*
+ * net/dsa/dsa2.c - Hardware switch handling, binding version 2
+ * Copyright (c) 2008-2009 Marvell Semiconductor
+ * Copyright (c) 2013 Florian Fainelli <florian@openwrt.org>
+ * Copyright (c) 2016 Andrew Lunn <andrew@lunn.ch>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/rtnetlink.h>
+#include <net/dsa.h>
+#include <linux/of.h>
+#include <linux/of_net.h>
+#include "dsa_priv.h"
+
+static LIST_HEAD(dsa_switch_trees);
+static DEFINE_MUTEX(dsa2_mutex);
+
+static struct dsa_switch_tree *dsa_get_dst(u32 tree)
+{
+        struct dsa_switch_tree *dst;
+
+        list_for_each_entry(dst, &dsa_switch_trees, list)
+                if (dst->tree == tree)
+                        return dst;
+        return NULL;
+}
+
+static void dsa_free_dst(struct kref *ref)
+{
+        struct dsa_switch_tree *dst = container_of(ref, struct dsa_switch_tree,
+                                                   refcount);
+
+        list_del(&dst->list);
+        kfree(dst);
+}
+
+static void dsa_put_dst(struct dsa_switch_tree *dst)
+{
+        kref_put(&dst->refcount, dsa_free_dst);
+}
+
+static struct dsa_switch_tree *dsa_add_dst(u32 tree)
+{
+        struct dsa_switch_tree *dst;
+
+        dst = kzalloc(sizeof(*dst), GFP_KERNEL);
+        if (!dst)
+                return NULL;
+        dst->tree = tree;
+        dst->cpu_switch = -1;
+        INIT_LIST_HEAD(&dst->list);
+        list_add_tail(&dsa_switch_trees, &dst->list);
+        kref_init(&dst->refcount);
+
+        return dst;
+}
+
+static void dsa_dst_add_ds(struct dsa_switch_tree *dst,
+                           struct dsa_switch *ds, u32 index)
+{
+        kref_get(&dst->refcount);
+        dst->ds[index] = ds;
+}
+
+static void dsa_dst_del_ds(struct dsa_switch_tree *dst,
+                           struct dsa_switch *ds, u32 index)
+{
+        dst->ds[index] = NULL;
+        kref_put(&dst->refcount, dsa_free_dst);
+}
+
+static bool dsa_port_is_dsa(struct device_node *port)
+{
+        const char *name;
+
+        name = of_get_property(port, "label", NULL);
+        if (!name)
+                return false;
+
+        if (!strcmp(name, "dsa"))
+                return true;
+
+        return false;
+}
+
+static bool dsa_port_is_cpu(struct device_node *port)
+{
+        const char *name;
+
+        name = of_get_property(port, "label", NULL);
+        if (!name)
+                return false;
+
+        if (!strcmp(name, "cpu"))
+                return true;
+
+        return false;
+}
+
+static bool dsa_ds_find_port(struct dsa_switch *ds,
+                             struct device_node *port)
+{
+        u32 index;
+
+        for (index = 0; index < DSA_MAX_PORTS; index++)
+                if (ds->ports[index].dn == port)
+                        return true;
+        return false;
+}
+
+static struct dsa_switch *dsa_dst_find_port(struct dsa_switch_tree *dst,
+                                            struct device_node *port)
+{
+        struct dsa_switch *ds;
+        u32 index;
+
+        for (index = 0; index < DSA_MAX_SWITCHES; index++) {
+                ds = dst->ds[index];
+                if (!ds)
+                        continue;
+
+                if (dsa_ds_find_port(ds, port))
+                        return ds;
+        }
+
+        return NULL;
+}
+
+static int dsa_port_complete(struct dsa_switch_tree *dst,
+                             struct dsa_switch *src_ds,
+                             struct device_node *port,
+                             u32 src_port)
+{
+        struct device_node *link;
+        int index;
+        struct dsa_switch *dst_ds;
+
+        for (index = 0;; index++) {
+                link = of_parse_phandle(port, "link", index);
+                if (!link)
+                        break;
+
+                dst_ds = dsa_dst_find_port(dst, link);
+                of_node_put(link);
+
+                if (!dst_ds)
+                        return 1;
+
+                src_ds->rtable[dst_ds->index] = src_port;
+        }
+
+        return 0;
+}
+
+/* A switch is complete if all the DSA ports phandles point to ports
+ * known in the tree. A return value of 1 means the tree is not
+ * complete. This is not an error condition. A value of 0 is
+ * success.
+ */
+static int dsa_ds_complete(struct dsa_switch_tree *dst, struct dsa_switch *ds)
+{
+        struct device_node *port;
+        u32 index;
+        int err;
+
+        for (index = 0; index < DSA_MAX_PORTS; index++) {
+                port = ds->ports[index].dn;
+                if (!port)
+                        continue;
+
+                if (!dsa_port_is_dsa(port))
+                        continue;
+
+                err = dsa_port_complete(dst, ds, port, index);
+                if (err != 0)
+                        return err;
+
+                ds->dsa_port_mask |= BIT(index);
+        }
+
+        return 0;
+}
+
+/* A tree is complete if all the DSA ports phandles point to ports
+ * known in the tree. A return value of 1 means the tree is not
+ * complete. This is not an error condition. A value of 0 is
+ * success.
+ */
+static int dsa_dst_complete(struct dsa_switch_tree *dst)
+{
+        struct dsa_switch *ds;
+        u32 index;
+        int err;
+
+        for (index = 0; index < DSA_MAX_SWITCHES; index++) {
+                ds = dst->ds[index];
+                if (!ds)
+                        continue;
+
+                err = dsa_ds_complete(dst, ds);
+                if (err != 0)
+                        return err;
+        }
+
+        return 0;
+}
+
+static int dsa_dsa_port_apply(struct device_node *port, u32 index,
+                              struct dsa_switch *ds)
+{
+        int err;
+
+        err = dsa_cpu_dsa_setup(ds, ds->dev, port, index);
+        if (err) {
+                dev_warn(ds->dev, "Failed to setup dsa port %d: %d\n",
+                         index, err);
+                return err;
+        }
+
+        return 0;
+}
+
+static void dsa_dsa_port_unapply(struct device_node *port, u32 index,
+                                 struct dsa_switch *ds)
+{
+        dsa_cpu_dsa_destroy(port);
+}
+
+static int dsa_cpu_port_apply(struct device_node *port, u32 index,
+                              struct dsa_switch *ds)
+{
+        int err;
+
+        err = dsa_cpu_dsa_setup(ds, ds->dev, port, index);
+        if (err) {
+                dev_warn(ds->dev, "Failed to setup cpu port %d: %d\n",
+                         index, err);
+                return err;
+        }
+
+        ds->cpu_port_mask |= BIT(index);
+
+        return 0;
+}
+
+static void dsa_cpu_port_unapply(struct device_node *port, u32 index,
+                                 struct dsa_switch *ds)
+{
+        dsa_cpu_dsa_destroy(port);
+        ds->cpu_port_mask &= ~BIT(index);
+
+}
+
+static int dsa_user_port_apply(struct device_node *port, u32 index,
+                               struct dsa_switch *ds)
+{
+        const char *name;
+        int err;
+
+        name = of_get_property(port, "label", NULL);
+
+        err = dsa_slave_create(ds, ds->dev, index, name);
+        if (err) {
+                dev_warn(ds->dev, "Failed to create slave %d: %d\n",
+                         index, err);
+                return err;
+        }
+
+        return 0;
+}
+
+static void dsa_user_port_unapply(struct device_node *port, u32 index,
+                                  struct dsa_switch *ds)
+{
+        if (ds->ports[index].netdev) {
+                dsa_slave_destroy(ds->ports[index].netdev);
+                ds->ports[index].netdev = NULL;
+                ds->enabled_port_mask &= ~(1 << index);
+        }
+}
+
+static int dsa_ds_apply(struct dsa_switch_tree *dst, struct dsa_switch *ds)
+{
+        struct device_node *port;
+        u32 index;
+        int err;
+
+        /* Initialize ds->phys_mii_mask before registering the slave MDIO bus
+         * driver and before drv->setup() has run, since the switch drivers and
+         * the slave MDIO bus driver rely on these values for probing PHY
+         * devices or not
+         */
+        ds->phys_mii_mask = ds->enabled_port_mask;
+
+        err = ds->drv->setup(ds);
+        if (err < 0)
+                return err;
+
+        err = ds->drv->set_addr(ds, dst->master_netdev->dev_addr);
+        if (err < 0)
+                return err;
+
+        err = ds->drv->set_addr(ds, dst->master_netdev->dev_addr);
+        if (err < 0)
+                return err;
+
+        if (!ds->slave_mii_bus && ds->drv->phy_read) {
+                ds->slave_mii_bus = devm_mdiobus_alloc(ds->dev);
+                if (!ds->slave_mii_bus)
+                        return -ENOMEM;
+
+                dsa_slave_mii_bus_init(ds);
+
+                err = mdiobus_register(ds->slave_mii_bus);
+                if (err < 0)
+                        return err;
+        }
+
+        for (index = 0; index < DSA_MAX_PORTS; index++) {
+                port = ds->ports[index].dn;
+                if (!port)
+                        continue;
+
+                if (dsa_port_is_dsa(port)) {
+                        err = dsa_dsa_port_apply(port, index, ds);
+                        if (err)
+                                return err;
+                        continue;
+                }
+
+                if (dsa_port_is_cpu(port)) {
+                        err = dsa_cpu_port_apply(port, index, ds);
+                        if (err)
+                                return err;
+                        continue;
+                }
+
+                err = dsa_user_port_apply(port, index, ds);
+                if (err)
+                        continue;
+        }
+
+        return 0;
+}
+
+static void dsa_ds_unapply(struct dsa_switch_tree *dst, struct dsa_switch *ds)
+{
+        struct device_node *port;
+        u32 index;
+
+        for (index = 0; index < DSA_MAX_PORTS; index++) {
+                port = ds->ports[index].dn;
+                if (!port)
+                        continue;
+
+                if (dsa_port_is_dsa(port)) {
+                        dsa_dsa_port_unapply(port, index, ds);
+                        continue;
+                }
+
+                if (dsa_port_is_cpu(port)) {
+                        dsa_cpu_port_unapply(port, index, ds);
+                        continue;
+                }
+
+                dsa_user_port_unapply(port, index, ds);
+        }
+
+        if (ds->slave_mii_bus && ds->drv->phy_read)
+                mdiobus_unregister(ds->slave_mii_bus);
+}
+
+static int dsa_dst_apply(struct dsa_switch_tree *dst)
+{
+        struct dsa_switch *ds;
+        u32 index;
+        int err;
+
+        for (index = 0; index < DSA_MAX_SWITCHES; index++) {
+                ds = dst->ds[index];
+                if (!ds)
+                        continue;
+
+                err = dsa_ds_apply(dst, ds);
+                if (err)
+                        return err;
+        }
+
+        err = dsa_cpu_port_ethtool_setup(dst->ds[0]);
+        if (err)
+                return err;
+
+        /* If we use a tagging format that doesn't have an ethertype
+         * field, make sure that all packets from this point on get
+         * sent to the tag format's receive function.
+         */
+        wmb();
+        dst->master_netdev->dsa_ptr = (void *)dst;
+        dst->applied = true;
+
+        return 0;
+}
+
+static void dsa_dst_unapply(struct dsa_switch_tree *dst)
+{
+        struct dsa_switch *ds;
+        u32 index;
+
+        if (!dst->applied)
+                return;
+
+        dst->master_netdev->dsa_ptr = NULL;
+
+        /* If we used a tagging format that doesn't have an ethertype
+         * field, make sure that all packets from this point get sent
+         * without the tag and go through the regular receive path.
+         */
+        wmb();
+
+        for (index = 0; index < DSA_MAX_SWITCHES; index++) {
+                ds = dst->ds[index];
+                if (!ds)
+                        continue;
+
+                dsa_ds_unapply(dst, ds);
+        }
+
+        dsa_cpu_port_ethtool_restore(dst->ds[0]);
+
+        pr_info("DSA: tree %d unapplied\n", dst->tree);
+        dst->applied = false;
+}
+
+static int dsa_cpu_parse(struct device_node *port, u32 index,
+                         struct dsa_switch_tree *dst,
+                         struct dsa_switch *ds)
+{
+        struct net_device *ethernet_dev;
+        struct device_node *ethernet;
+
+        ethernet = of_parse_phandle(port, "ethernet", 0);
+        if (!ethernet)
+                return -EINVAL;
+
+        ethernet_dev = of_find_net_device_by_node(ethernet);
+        if (!ethernet_dev)
+                return -EPROBE_DEFER;
+
+        if (!ds->master_netdev)
+                ds->master_netdev = ethernet_dev;
+
+        if (!dst->master_netdev)
+                dst->master_netdev = ethernet_dev;
+
+        if (dst->cpu_switch == -1) {
+                dst->cpu_switch = ds->index;
+                dst->cpu_port = index;
+        }
+
+        dst->tag_ops = dsa_resolve_tag_protocol(ds->drv->tag_protocol);
+        if (IS_ERR(dst->tag_ops)) {
+                dev_warn(ds->dev, "No tagger for this switch\n");
+                return PTR_ERR(dst->tag_ops);
+        }
+
+        dst->rcv = dst->tag_ops->rcv;
+
+        return 0;
+}
+
+static int dsa_ds_parse(struct dsa_switch_tree *dst, struct dsa_switch *ds)
+{
+        struct device_node *port;
+        u32 index;
+        int err;
+
+        for (index = 0; index < DSA_MAX_PORTS; index++) {
+                port = ds->ports[index].dn;
+                if (!port)
+                        continue;
+
+                if (dsa_port_is_cpu(port)) {
+                        err = dsa_cpu_parse(port, index, dst, ds);
+                        if (err)
+                                return err;
+                }
+        }
+
+        pr_info("DSA: switch %d %d parsed\n", dst->tree, ds->index);
+
+        return 0;
+}
+
+static int dsa_dst_parse(struct dsa_switch_tree *dst)
+{
+        struct dsa_switch *ds;
+        u32 index;
+        int err;
+
+        for (index = 0; index < DSA_MAX_SWITCHES; index++) {
+                ds = dst->ds[index];
+                if (!ds)
+                        continue;
+
+                err = dsa_ds_parse(dst, ds);
+                if (err)
+                        return err;
+        }
+
+        if (!dst->master_netdev) {
+                pr_warn("Tree has no master device\n");
+                return -EINVAL;
+        }
+
+        pr_info("DSA: tree %d parsed\n", dst->tree);
+
+        return 0;
+}
+
+static int dsa_parse_ports_dn(struct device_node *ports, struct dsa_switch *ds)
+{
+        struct device_node *port;
+        int err;
+        u32 reg;
+
+        for_each_available_child_of_node(ports, port) {
+                err = of_property_read_u32(port, "reg", &reg);
+                if (err)
+                        return err;
+
+                if (reg >= DSA_MAX_PORTS)
+                        return -EINVAL;
+
+                ds->ports[reg].dn = port;
+
+                /* Initialize enabled_port_mask now for drv->setup()
+                 * to have access to a correct value, just like what
+                 * net/dsa/dsa.c::dsa_switch_setup_one does.
+                 */
+                if (!dsa_port_is_cpu(port))
+                        ds->enabled_port_mask |= 1 << reg;
+        }
+
+        return 0;
+}
+
+static int dsa_parse_member(struct device_node *np, u32 *tree, u32 *index)
+{
+        int err;
+
+        *tree = *index = 0;
+
+        err = of_property_read_u32_index(np, "dsa,member", 0, tree);
+        if (err) {
+                /* Does not exist, but it is optional */
+                if (err == -EINVAL)
+                        return 0;
+                return err;
+        }
+
+        err = of_property_read_u32_index(np, "dsa,member", 1, index);
+        if (err)
+                return err;
+
+        if (*index >= DSA_MAX_SWITCHES)
+                return -EINVAL;
+
+        return 0;
+}
+
+static struct device_node *dsa_get_ports(struct dsa_switch *ds,
+                                         struct device_node *np)
+{
+        struct device_node *ports;
+
+        ports = of_get_child_by_name(np, "ports");
+        if (!ports) {
+                dev_err(ds->dev, "no ports child node found\n");
+                return ERR_PTR(-EINVAL);
+        }
+
+        return ports;
+}
+
+static int _dsa_register_switch(struct dsa_switch *ds, struct device_node *np)
+{
+        struct device_node *ports = dsa_get_ports(ds, np);
+        struct dsa_switch_tree *dst;
+        u32 tree, index;
+        int err;
+
+        err = dsa_parse_member(np, &tree, &index);
+        if (err)
+                return err;
+
+        if (IS_ERR(ports))
+                return PTR_ERR(ports);
+
+        err = dsa_parse_ports_dn(ports, ds);
+        if (err)
+                return err;
+
+        dst = dsa_get_dst(tree);
+        if (!dst) {
+                dst = dsa_add_dst(tree);
+                if (!dst)
+                        return -ENOMEM;
+        }
+
+        if (dst->ds[index]) {
+                err = -EBUSY;
+                goto out;
+        }
+
+        ds->dst = dst;
+        ds->index = index;
+        dsa_dst_add_ds(dst, ds, index);
+
+        err = dsa_dst_complete(dst);
+        if (err < 0)
+                goto out_del_dst;
+
+        if (err == 1) {
+                /* Not all switches registered yet */
+                err = 0;
+                goto out;
+        }
+
+        if (dst->applied) {
+                pr_info("DSA: Disjoint trees?\n");
+                return -EINVAL;
+        }
+
+        err = dsa_dst_parse(dst);
+        if (err)
+                goto out_del_dst;
+
+        err = dsa_dst_apply(dst);
+        if (err) {
+                dsa_dst_unapply(dst);
+                goto out_del_dst;
+        }
+
+        dsa_put_dst(dst);
+        return 0;
+
+out_del_dst:
+        dsa_dst_del_ds(dst, ds, ds->index);
+out:
+        dsa_put_dst(dst);
+
+        return err;
+}
+
+int dsa_register_switch(struct dsa_switch *ds, struct device_node *np)
+{
+        int err;
+
+        mutex_lock(&dsa2_mutex);
+        err = _dsa_register_switch(ds, np);
+        mutex_unlock(&dsa2_mutex);
+
+        return err;
+}
+EXPORT_SYMBOL_GPL(dsa_register_switch);
+
+void _dsa_unregister_switch(struct dsa_switch *ds)
+{
+        struct dsa_switch_tree *dst = ds->dst;
+
+        dsa_dst_unapply(dst);
+
+        dsa_dst_del_ds(dst, ds, ds->index);
+}
+
+void dsa_unregister_switch(struct dsa_switch *ds)
+{
+        mutex_lock(&dsa2_mutex);
+        _dsa_unregister_switch(ds);
+        mutex_unlock(&dsa2_mutex);
+}
+EXPORT_SYMBOL_GPL(dsa_unregister_switch);
diff --git a/net/dsa/dsa_priv.h b/net/dsa/dsa_priv.h
index dfa33779d49c..00077a9c97f4 100644
--- a/net/dsa/dsa_priv.h
+++ b/net/dsa/dsa_priv.h
@@ -50,12 +50,19 @@ struct dsa_slave_priv {
 
 /* dsa.c */
 extern char dsa_driver_version[];
+int dsa_cpu_dsa_setup(struct dsa_switch *ds, struct device *dev,
+                      struct device_node *port_dn, int port);
+void dsa_cpu_dsa_destroy(struct device_node *port_dn);
+const struct dsa_device_ops *dsa_resolve_tag_protocol(int tag_protocol);
+int dsa_cpu_port_ethtool_setup(struct dsa_switch *ds);
+void dsa_cpu_port_ethtool_restore(struct dsa_switch *ds);
 
 /* slave.c */
 extern const struct dsa_device_ops notag_netdev_ops;
 void dsa_slave_mii_bus_init(struct dsa_switch *ds);
+void dsa_cpu_port_ethtool_init(struct ethtool_ops *ops);
 int dsa_slave_create(struct dsa_switch *ds, struct device *parent,
-                     int port, char *name);
+                     int port, const char *name);
 void dsa_slave_destroy(struct net_device *slave_dev);
 int dsa_slave_suspend(struct net_device *slave_dev);
 int dsa_slave_resume(struct net_device *slave_dev);
diff --git a/net/dsa/slave.c b/net/dsa/slave.c
index 152436cdab30..7236eb26dc97 100644
--- a/net/dsa/slave.c
+++ b/net/dsa/slave.c
@@ -49,8 +49,8 @@ void dsa_slave_mii_bus_init(struct dsa_switch *ds)
         ds->slave_mii_bus->name = "dsa slave smi";
         ds->slave_mii_bus->read = dsa_slave_phy_read;
         ds->slave_mii_bus->write = dsa_slave_phy_write;
-        snprintf(ds->slave_mii_bus->id, MII_BUS_ID_SIZE, "dsa-%d:%.2x",
-                        ds->index, ds->cd->sw_addr);
+        snprintf(ds->slave_mii_bus->id, MII_BUS_ID_SIZE, "dsa-%d.%d",
+                 ds->dst->tree, ds->index);
         ds->slave_mii_bus->parent = ds->dev;
         ds->slave_mii_bus->phy_mask = ~ds->phys_mii_mask;
 }
@@ -522,14 +522,6 @@ static netdev_tx_t dsa_slave_xmit(struct sk_buff *skb, struct net_device *dev)
         return NETDEV_TX_OK;
 }
 
-static struct sk_buff *dsa_slave_notag_xmit(struct sk_buff *skb,
-                                            struct net_device *dev)
-{
-        /* Just return the original SKB */
-        return skb;
-}
-
-
 /* ethtool operations *******************************************************/
 static int
 dsa_slave_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
@@ -615,7 +607,7 @@ static int dsa_slave_get_eeprom_len(struct net_device *dev)
         struct dsa_slave_priv *p = netdev_priv(dev);
         struct dsa_switch *ds = p->parent;
 
-        if (ds->cd->eeprom_len)
+        if (ds->cd && ds->cd->eeprom_len)
                 return ds->cd->eeprom_len;
 
         if (ds->drv->get_eeprom_len)
@@ -873,6 +865,13 @@ static void dsa_slave_poll_controller(struct net_device *dev)
 }
 #endif
 
+void dsa_cpu_port_ethtool_init(struct ethtool_ops *ops)
+{
+        ops->get_sset_count = dsa_cpu_port_get_sset_count;
+        ops->get_ethtool_stats = dsa_cpu_port_get_ethtool_stats;
+        ops->get_strings = dsa_cpu_port_get_strings;
+}
+
 static const struct ethtool_ops dsa_slave_ethtool_ops = {
         .get_settings           = dsa_slave_get_settings,
         .set_settings           = dsa_slave_set_settings,
@@ -893,8 +892,6 @@ static const struct ethtool_ops dsa_slave_ethtool_ops = {
         .get_eee                = dsa_slave_get_eee,
 };
 
-static struct ethtool_ops dsa_cpu_port_ethtool_ops;
-
 static const struct net_device_ops dsa_slave_netdev_ops = {
         .ndo_open               = dsa_slave_open,
         .ndo_stop               = dsa_slave_close,
@@ -999,13 +996,12 @@ static int dsa_slave_phy_setup(struct dsa_slave_priv *p,
                                 struct net_device *slave_dev)
 {
         struct dsa_switch *ds = p->parent;
-        struct dsa_chip_data *cd = ds->cd;
         struct device_node *phy_dn, *port_dn;
         bool phy_is_fixed = false;
         u32 phy_flags = 0;
         int mode, ret;
 
-        port_dn = cd->port_dn[p->port];
+        port_dn = ds->ports[p->port].dn;
         mode = of_get_phy_mode(port_dn);
         if (mode < 0)
                 mode = PHY_INTERFACE_MODE_NA;
@@ -1109,14 +1105,18 @@ int dsa_slave_resume(struct net_device *slave_dev)
 }
 
 int dsa_slave_create(struct dsa_switch *ds, struct device *parent,
-                     int port, char *name)
+                     int port, const char *name)
 {
-        struct net_device *master = ds->dst->master_netdev;
         struct dsa_switch_tree *dst = ds->dst;
+        struct net_device *master;
         struct net_device *slave_dev;
         struct dsa_slave_priv *p;
         int ret;
 
+        master = ds->dst->master_netdev;
+        if (ds->master_netdev)
+                master = ds->master_netdev;
+
         slave_dev = alloc_netdev(sizeof(struct dsa_slave_priv), name,
                                  NET_NAME_UNKNOWN, ether_setup);
         if (slave_dev == NULL)
@@ -1124,19 +1124,6 @@ int dsa_slave_create(struct dsa_switch *ds, struct device *parent,
 
         slave_dev->features = master->vlan_features;
         slave_dev->ethtool_ops = &dsa_slave_ethtool_ops;
-        if (master->ethtool_ops != &dsa_cpu_port_ethtool_ops) {
-                memcpy(&dst->master_ethtool_ops, master->ethtool_ops,
-                       sizeof(struct ethtool_ops));
-                memcpy(&dsa_cpu_port_ethtool_ops, &dst->master_ethtool_ops,
-                       sizeof(struct ethtool_ops));
-                dsa_cpu_port_ethtool_ops.get_sset_count =
-                                        dsa_cpu_port_get_sset_count;
-                dsa_cpu_port_ethtool_ops.get_ethtool_stats =
-                                        dsa_cpu_port_get_ethtool_stats;
-                dsa_cpu_port_ethtool_ops.get_strings =
-                                        dsa_cpu_port_get_strings;
-                master->ethtool_ops = &dsa_cpu_port_ethtool_ops;
-        }
         eth_hw_addr_inherit(slave_dev, master);
         slave_dev->priv_flags |= IFF_NO_QUEUE;
         slave_dev->netdev_ops = &dsa_slave_netdev_ops;
@@ -1147,49 +1134,24 @@ int dsa_slave_create(struct dsa_switch *ds, struct device *parent,
                                  NULL);
 
         SET_NETDEV_DEV(slave_dev, parent);
-        slave_dev->dev.of_node = ds->cd->port_dn[port];
+        slave_dev->dev.of_node = ds->ports[port].dn;
         slave_dev->vlan_features = master->vlan_features;
 
         p = netdev_priv(slave_dev);
         p->parent = ds;
         p->port = port;
-
-        switch (ds->dst->tag_protocol) {
-#ifdef CONFIG_NET_DSA_TAG_DSA
-        case DSA_TAG_PROTO_DSA:
-                p->xmit = dsa_netdev_ops.xmit;
-                break;
-#endif
-#ifdef CONFIG_NET_DSA_TAG_EDSA
-        case DSA_TAG_PROTO_EDSA:
-                p->xmit = edsa_netdev_ops.xmit;
-                break;
-#endif
-#ifdef CONFIG_NET_DSA_TAG_TRAILER
-        case DSA_TAG_PROTO_TRAILER:
-                p->xmit = trailer_netdev_ops.xmit;
-                break;
-#endif
-#ifdef CONFIG_NET_DSA_TAG_BRCM
-        case DSA_TAG_PROTO_BRCM:
-                p->xmit = brcm_netdev_ops.xmit;
-                break;
-#endif
-        default:
-                p->xmit = dsa_slave_notag_xmit;
-                break;
-        }
+        p->xmit = dst->tag_ops->xmit;
 
         p->old_pause = -1;
         p->old_link = -1;
         p->old_duplex = -1;
 
-        ds->ports[port] = slave_dev;
+        ds->ports[port].netdev = slave_dev;
         ret = register_netdev(slave_dev);
         if (ret) {
                 netdev_err(master, "error %d registering interface %s\n",
                            ret, slave_dev->name);
-                ds->ports[port] = NULL;
+                ds->ports[port].netdev = NULL;
                 free_netdev(slave_dev);
                 return ret;
         }
diff --git a/net/dsa/tag_brcm.c b/net/dsa/tag_brcm.c
index e2aadb73111d..21bffde6e4bf 100644
--- a/net/dsa/tag_brcm.c
+++ b/net/dsa/tag_brcm.c
@@ -127,7 +127,7 @@ static int brcm_tag_rcv(struct sk_buff *skb, struct net_device *dev,
         source_port = brcm_tag[3] & BRCM_EG_PID_MASK;
 
         /* Validate port against switch setup, either the port is totally */
-        if (source_port >= DSA_MAX_PORTS || ds->ports[source_port] == NULL)
+        if (source_port >= DSA_MAX_PORTS || !ds->ports[source_port].netdev)
                 goto out_drop;
 
         /* Remove Broadcom tag and update checksum */
@@ -140,7 +140,7 @@ static int brcm_tag_rcv(struct sk_buff *skb, struct net_device *dev,
 
         skb_push(skb, ETH_HLEN);
         skb->pkt_type = PACKET_HOST;
-        skb->dev = ds->ports[source_port];
+        skb->dev = ds->ports[source_port].netdev;
         skb->protocol = eth_type_trans(skb, skb->dev);
 
         skb->dev->stats.rx_packets++;
diff --git a/net/dsa/tag_dsa.c b/net/dsa/tag_dsa.c
index aa780e4ac0bd..bce79ffe342b 100644
--- a/net/dsa/tag_dsa.c
+++ b/net/dsa/tag_dsa.c
@@ -107,10 +107,14 @@ static int dsa_rcv(struct sk_buff *skb, struct net_device *dev,
          * Check that the source device exists and that the source
          * port is a registered DSA port.
          */
-        if (source_device >= dst->pd->nr_chips)
+        if (source_device >= DSA_MAX_SWITCHES)
                 goto out_drop;
+
         ds = dst->ds[source_device];
-        if (source_port >= DSA_MAX_PORTS || ds->ports[source_port] == NULL)
+        if (!ds)
+                goto out_drop;
+
+        if (source_port >= DSA_MAX_PORTS || !ds->ports[source_port].netdev)
                 goto out_drop;
 
         /*
@@ -159,7 +163,7 @@ static int dsa_rcv(struct sk_buff *skb, struct net_device *dev,
                         2 * ETH_ALEN);
         }
 
-        skb->dev = ds->ports[source_port];
+        skb->dev = ds->ports[source_port].netdev;
         skb_push(skb, ETH_HLEN);
         skb->pkt_type = PACKET_HOST;
         skb->protocol = eth_type_trans(skb, skb->dev);
diff --git a/net/dsa/tag_edsa.c b/net/dsa/tag_edsa.c
index 2288c8098c42..6c1720e88537 100644
--- a/net/dsa/tag_edsa.c
+++ b/net/dsa/tag_edsa.c
@@ -120,10 +120,14 @@ static int edsa_rcv(struct sk_buff *skb, struct net_device *dev,
          * Check that the source device exists and that the source
          * port is a registered DSA port.
          */
-        if (source_device >= dst->pd->nr_chips)
+        if (source_device >= DSA_MAX_SWITCHES)
                 goto out_drop;
+
         ds = dst->ds[source_device];
-        if (source_port >= DSA_MAX_PORTS || ds->ports[source_port] == NULL)
+        if (!ds)
+                goto out_drop;
+
+        if (source_port >= DSA_MAX_PORTS || !ds->ports[source_port].netdev)
                 goto out_drop;
 
         /*
@@ -178,7 +182,7 @@ static int edsa_rcv(struct sk_buff *skb, struct net_device *dev,
                         2 * ETH_ALEN);
         }
 
-        skb->dev = ds->ports[source_port];
+        skb->dev = ds->ports[source_port].netdev;
         skb_push(skb, ETH_HLEN);
         skb->pkt_type = PACKET_HOST;
         skb->protocol = eth_type_trans(skb, skb->dev);
diff --git a/net/dsa/tag_trailer.c b/net/dsa/tag_trailer.c
index b6ca0890d018..5e3903eb1afa 100644
--- a/net/dsa/tag_trailer.c
+++ b/net/dsa/tag_trailer.c
@@ -82,12 +82,12 @@ static int trailer_rcv(struct sk_buff *skb, struct net_device *dev,
                 goto out_drop;
 
         source_port = trailer[1] & 7;
-        if (source_port >= DSA_MAX_PORTS || ds->ports[source_port] == NULL)
+        if (source_port >= DSA_MAX_PORTS || !ds->ports[source_port].netdev)
                 goto out_drop;
 
         pskb_trim_rcsum(skb, skb->len - 4);
 
-        skb->dev = ds->ports[source_port];
+        skb->dev = ds->ports[source_port].netdev;
         skb_push(skb, ETH_HLEN);
         skb->pkt_type = PACKET_HOST;
         skb->protocol = eth_type_trans(skb, skb->dev);
diff --git a/net/ieee802154/6lowpan/core.c b/net/ieee802154/6lowpan/core.c
index dd085db8580e..8c004a0c8d64 100644
--- a/net/ieee802154/6lowpan/core.c
+++ b/net/ieee802154/6lowpan/core.c
@@ -58,21 +58,10 @@ static struct header_ops lowpan_header_ops = {
         .create = lowpan_header_create,
 };
 
-static struct lock_class_key lowpan_tx_busylock;
-static struct lock_class_key lowpan_netdev_xmit_lock_key;
-
-static void lowpan_set_lockdep_class_one(struct net_device *ldev,
-                                         struct netdev_queue *txq,
-                                         void *_unused)
-{
-        lockdep_set_class(&txq->_xmit_lock,
-                          &lowpan_netdev_xmit_lock_key);
-}
-
 static int lowpan_dev_init(struct net_device *ldev)
 {
-        netdev_for_each_tx_queue(ldev, lowpan_set_lockdep_class_one, NULL);
-        ldev->qdisc_tx_busylock = &lowpan_tx_busylock;
+        netdev_lockdep_set_classes(ldev);
+
         return 0;
 }
 
@@ -92,11 +81,21 @@ static int lowpan_stop(struct net_device *dev)
         return 0;
 }
 
+static int lowpan_neigh_construct(struct neighbour *n)
+{
+        struct lowpan_802154_neigh *neigh = lowpan_802154_neigh(neighbour_priv(n));
+
+        /* default no short_addr is available for a neighbour */
+        neigh->short_addr = cpu_to_le16(IEEE802154_ADDR_SHORT_UNSPEC);
+        return 0;
+}
+
 static const struct net_device_ops lowpan_netdev_ops = {
         .ndo_init               = lowpan_dev_init,
         .ndo_start_xmit         = lowpan_xmit,
         .ndo_open               = lowpan_open,
         .ndo_stop               = lowpan_stop,
+        .ndo_neigh_construct    = lowpan_neigh_construct,
 };
 
 static void lowpan_setup(struct net_device *ldev)
@@ -161,6 +160,8 @@ static int lowpan_newlink(struct net *src_net, struct net_device *ldev,
                                 wdev->needed_headroom;
         ldev->needed_tailroom = wdev->needed_tailroom;
 
+        ldev->neigh_priv_len = sizeof(struct lowpan_802154_neigh);
+
         ret = lowpan_register_netdevice(ldev, LOWPAN_LLTYPE_IEEE802154);
         if (ret < 0) {
                 dev_put(wdev);
diff --git a/net/ieee802154/6lowpan/tx.c b/net/ieee802154/6lowpan/tx.c
index e459afd16bb3..dbb476d7d38f 100644
--- a/net/ieee802154/6lowpan/tx.c
+++ b/net/ieee802154/6lowpan/tx.c
@@ -9,6 +9,7 @@
  */
 
 #include <net/6lowpan.h>
+#include <net/ndisc.h>
 #include <net/ieee802154_netdev.h>
 #include <net/mac802154.h>
 
@@ -17,19 +18,9 @@
 #define LOWPAN_FRAG1_HEAD_SIZE  0x4
 #define LOWPAN_FRAGN_HEAD_SIZE  0x5
 
-/* don't save pan id, it's intra pan */
-struct lowpan_addr {
-        u8 mode;
-        union {
-                /* IPv6 needs big endian here */
-                __be64 extended_addr;
-                __be16 short_addr;
-        } u;
-};
-
 struct lowpan_addr_info {
-        struct lowpan_addr daddr;
-        struct lowpan_addr saddr;
+        struct ieee802154_addr daddr;
+        struct ieee802154_addr saddr;
 };
 
 static inline struct
@@ -48,12 +39,14 @@ lowpan_addr_info *lowpan_skb_priv(const struct sk_buff *skb)
  * RAW/DGRAM sockets.
  */
 int lowpan_header_create(struct sk_buff *skb, struct net_device *ldev,
-                         unsigned short type, const void *_daddr,
-                         const void *_saddr, unsigned int len)
+                         unsigned short type, const void *daddr,
+                         const void *saddr, unsigned int len)
 {
-        const u8 *saddr = _saddr;
-        const u8 *daddr = _daddr;
-        struct lowpan_addr_info *info;
+        struct wpan_dev *wpan_dev = lowpan_802154_dev(ldev)->wdev->ieee802154_ptr;
+        struct lowpan_addr_info *info = lowpan_skb_priv(skb);
+        struct lowpan_802154_neigh *llneigh = NULL;
+        const struct ipv6hdr *hdr = ipv6_hdr(skb);
+        struct neighbour *n;
 
         /* TODO:
          * if this package isn't ipv6 one, where should it be routed?
@@ -61,21 +54,50 @@ int lowpan_header_create(struct sk_buff *skb, struct net_device *ldev,
         if (type != ETH_P_IPV6)
                 return 0;
 
-        if (!saddr)
-                saddr = ldev->dev_addr;
+        /* intra-pan communication */
+        info->saddr.pan_id = wpan_dev->pan_id;
+        info->daddr.pan_id = info->saddr.pan_id;
 
-        raw_dump_inline(__func__, "saddr", (unsigned char *)saddr, 8);
-        raw_dump_inline(__func__, "daddr", (unsigned char *)daddr, 8);
+        if (!memcmp(daddr, ldev->broadcast, EUI64_ADDR_LEN)) {
+                info->daddr.short_addr = cpu_to_le16(IEEE802154_ADDR_BROADCAST);
+                info->daddr.mode = IEEE802154_ADDR_SHORT;
+        } else {
+                __le16 short_addr = cpu_to_le16(IEEE802154_ADDR_SHORT_UNSPEC);
+
+                n = neigh_lookup(&nd_tbl, &hdr->daddr, ldev);
+                if (n) {
+                        llneigh = lowpan_802154_neigh(neighbour_priv(n));
+                        read_lock_bh(&n->lock);
+                        short_addr = llneigh->short_addr;
+                        read_unlock_bh(&n->lock);
+                }
 
-        info = lowpan_skb_priv(skb);
+                if (llneigh &&
+                    lowpan_802154_is_valid_src_short_addr(short_addr)) {
+                        info->daddr.short_addr = short_addr;
+                        info->daddr.mode = IEEE802154_ADDR_SHORT;
+                } else {
+                        info->daddr.mode = IEEE802154_ADDR_LONG;
+                        ieee802154_be64_to_le64(&info->daddr.extended_addr,
+                                                daddr);
+                }
 
-        /* TODO: Currently we only support extended_addr */
-        info->daddr.mode = IEEE802154_ADDR_LONG;
-        memcpy(&info->daddr.u.extended_addr, daddr,
-               sizeof(info->daddr.u.extended_addr));
-        info->saddr.mode = IEEE802154_ADDR_LONG;
-        memcpy(&info->saddr.u.extended_addr, saddr,
-               sizeof(info->daddr.u.extended_addr));
+                if (n)
+                        neigh_release(n);
+        }
+
+        if (!saddr) {
+                if (lowpan_802154_is_valid_src_short_addr(wpan_dev->short_addr)) {
+                        info->saddr.mode = IEEE802154_ADDR_SHORT;
+                        info->saddr.short_addr = wpan_dev->short_addr;
+                } else {
+                        info->saddr.mode = IEEE802154_ADDR_LONG;
+                        info->saddr.extended_addr = wpan_dev->extended_addr;
+                }
+        } else {
+                info->saddr.mode = IEEE802154_ADDR_LONG;
+                ieee802154_be64_to_le64(&info->saddr.extended_addr, saddr);
+        }
 
         return 0;
 }
@@ -209,47 +231,26 @@ static int lowpan_header(struct sk_buff *skb, struct net_device *ldev,
                          u16 *dgram_size, u16 *dgram_offset)
 {
         struct wpan_dev *wpan_dev = lowpan_802154_dev(ldev)->wdev->ieee802154_ptr;
-        struct ieee802154_addr sa, da;
         struct ieee802154_mac_cb *cb = mac_cb_init(skb);
         struct lowpan_addr_info info;
-        void *daddr, *saddr;
 
         memcpy(&info, lowpan_skb_priv(skb), sizeof(info));
 
-        /* TODO: Currently we only support extended_addr */
-        daddr = &info.daddr.u.extended_addr;
-        saddr = &info.saddr.u.extended_addr;
-
         *dgram_size = skb->len;
-        lowpan_header_compress(skb, ldev, daddr, saddr);
+        lowpan_header_compress(skb, ldev, &info.daddr, &info.saddr);
         /* dgram_offset = (saved bytes after compression) + lowpan header len */
         *dgram_offset = (*dgram_size - skb->len) + skb_network_header_len(skb);
 
         cb->type = IEEE802154_FC_TYPE_DATA;
 
-        /* prepare wpan address data */
-        sa.mode = IEEE802154_ADDR_LONG;
-        sa.pan_id = wpan_dev->pan_id;
-        sa.extended_addr = ieee802154_devaddr_from_raw(saddr);
-
-        /* intra-PAN communications */
-        da.pan_id = sa.pan_id;
-
-        /* if the destination address is the broadcast address, use the
-         * corresponding short address
-         */
-        if (!memcmp(daddr, ldev->broadcast, EUI64_ADDR_LEN)) {
-                da.mode = IEEE802154_ADDR_SHORT;
-                da.short_addr = cpu_to_le16(IEEE802154_ADDR_BROADCAST);
+        if (info.daddr.mode == IEEE802154_ADDR_SHORT &&
+            ieee802154_is_broadcast_short_addr(info.daddr.short_addr))
                 cb->ackreq = false;
-        } else {
-                da.mode = IEEE802154_ADDR_LONG;
-                da.extended_addr = ieee802154_devaddr_from_raw(daddr);
+        else
                 cb->ackreq = wpan_dev->ackreq;
-        }
 
-        return wpan_dev_hard_header(skb, lowpan_802154_dev(ldev)->wdev, &da,
-                                    &sa, 0);
+        return wpan_dev_hard_header(skb, lowpan_802154_dev(ldev)->wdev,
+                                    &info.daddr, &info.saddr, 0);
 }
 
 netdev_tx_t lowpan_xmit(struct sk_buff *skb, struct net_device *ldev)
diff --git a/net/ipv4/Kconfig b/net/ipv4/Kconfig
index 238225b0c970..50d6a9b49f6c 100644
--- a/net/ipv4/Kconfig
+++ b/net/ipv4/Kconfig
@@ -532,6 +532,22 @@ config TCP_CONG_VEGAS
         window. TCP Vegas should provide less packet loss, but it is
         not as aggressive as TCP Reno.
 
+config TCP_CONG_NV
+       tristate "TCP NV"
+       default n
+       ---help---
+       TCP NV is a follow up to TCP Vegas. It has been modified to deal with
+       10G networks, measurement noise introduced by LRO, GRO and interrupt
+       coalescence. In addition, it will decrease its cwnd multiplicatively
+       instead of linearly.
+
+       Note that in general congestion avoidance (cwnd decreased when # packets
+       queued grows) cannot coexist with congestion control (cwnd decreased only
+       when there is packet loss) due to fairness issues. One scenario when they
+       can coexist safely is when the CA flows have RTTs << CC flows RTTs.
+
+       For further details see http://www.brakmo.org/networking/tcp-nv/
+
 config TCP_CONG_SCALABLE
         tristate "Scalable TCP"
         default n
diff --git a/net/ipv4/Makefile b/net/ipv4/Makefile
index bfa133691cde..24629b6f57cc 100644
--- a/net/ipv4/Makefile
+++ b/net/ipv4/Makefile
@@ -50,6 +50,7 @@ obj-$(CONFIG_TCP_CONG_HSTCP) += tcp_highspeed.o
 obj-$(CONFIG_TCP_CONG_HYBLA) += tcp_hybla.o
 obj-$(CONFIG_TCP_CONG_HTCP) += tcp_htcp.o
 obj-$(CONFIG_TCP_CONG_VEGAS) += tcp_vegas.o
+obj-$(CONFIG_TCP_CONG_NV) += tcp_nv.o
 obj-$(CONFIG_TCP_CONG_VENO) += tcp_veno.o
 obj-$(CONFIG_TCP_CONG_SCALABLE) += tcp_scalable.o
 obj-$(CONFIG_TCP_CONG_LP) += tcp_lp.o
diff --git a/net/ipv4/fib_rules.c b/net/ipv4/fib_rules.c
index f2bda9e89c61..6e9ea69e5f75 100644
--- a/net/ipv4/fib_rules.c
+++ b/net/ipv4/fib_rules.c
@@ -76,6 +76,7 @@ static int fib4_rule_action(struct fib_rule *rule, struct flowi *flp,
 {
         int err = -EAGAIN;
         struct fib_table *tbl;
+        u32 tb_id;
 
         switch (rule->action) {
         case FR_ACT_TO_TBL:
@@ -94,7 +95,8 @@ static int fib4_rule_action(struct fib_rule *rule, struct flowi *flp,
 
         rcu_read_lock();
 
-        tbl = fib_get_table(rule->fr_net, rule->table);
+        tb_id = fib_rule_get_table(rule, arg);
+        tbl = fib_get_table(rule->fr_net, tb_id);
         if (tbl)
                 err = fib_table_lookup(tbl, &flp->u.ip4,
                                        (struct fib_result *)arg->result,
@@ -180,7 +182,7 @@ static int fib4_rule_configure(struct fib_rule *rule, struct sk_buff *skb,
         if (err)
                 goto errout;
 
-        if (rule->table == RT_TABLE_UNSPEC) {
+        if (rule->table == RT_TABLE_UNSPEC && !rule->l3mdev) {
                 if (rule->action == FR_ACT_TO_TBL) {
                         struct fib_table *table;
 
diff --git a/net/ipv4/fou.c b/net/ipv4/fou.c
index 5f9207c039e7..321d57f825ce 100644
--- a/net/ipv4/fou.c
+++ b/net/ipv4/fou.c
@@ -129,6 +129,36 @@ static int gue_udp_recv(struct sock *sk, struct sk_buff *skb)
 
         guehdr = (struct guehdr *)&udp_hdr(skb)[1];
 
+        switch (guehdr->version) {
+        case 0: /* Full GUE header present */
+                break;
+
+        case 1: {
+                /* Direct encasulation of IPv4 or IPv6 */
+
+                int prot;
+
+                switch (((struct iphdr *)guehdr)->version) {
+                case 4:
+                        prot = IPPROTO_IPIP;
+                        break;
+                case 6:
+                        prot = IPPROTO_IPV6;
+                        break;
+                default:
+                        goto drop;
+                }
+
+                if (fou_recv_pull(skb, fou, sizeof(struct udphdr)))
+                        goto drop;
+
+                return -prot;
+        }
+
+        default: /* Undefined version */
+                goto drop;
+        }
+
         optlen = guehdr->hlen << 2;
         len += optlen;
 
@@ -289,6 +319,7 @@ static struct sk_buff **gue_gro_receive(struct sock *sk,
         int flush = 1;
         struct fou *fou = fou_from_sock(sk);
         struct gro_remcsum grc;
+        u8 proto;
 
         skb_gro_remcsum_init(&grc);
 
@@ -302,6 +333,25 @@ static struct sk_buff **gue_gro_receive(struct sock *sk,
                         goto out;
         }
 
+        switch (guehdr->version) {
+        case 0:
+                break;
+        case 1:
+                switch (((struct iphdr *)guehdr)->version) {
+                case 4:
+                        proto = IPPROTO_IPIP;
+                        break;
+                case 6:
+                        proto = IPPROTO_IPV6;
+                        break;
+                default:
+                        goto out;
+                }
+                goto next_proto;
+        default:
+                goto out;
+        }
+
         optlen = guehdr->hlen << 2;
         len += optlen;
 
@@ -370,6 +420,10 @@ static struct sk_buff **gue_gro_receive(struct sock *sk,
                 }
         }
 
+        proto = guehdr->proto_ctype;
+
+next_proto:
+
         /* We can clear the encap_mark for GUE as we are essentially doing
          * one of two possible things.  We are either adding an L4 tunnel
          * header to the outer L3 tunnel header, or we are are simply
@@ -383,7 +437,7 @@ static struct sk_buff **gue_gro_receive(struct sock *sk,
 
         rcu_read_lock();
         offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads;
-        ops = rcu_dereference(offloads[guehdr->proto_ctype]);
+        ops = rcu_dereference(offloads[proto]);
         if (WARN_ON_ONCE(!ops || !ops->callbacks.gro_receive))
                 goto out_unlock;
 
@@ -404,13 +458,30 @@ static int gue_gro_complete(struct sock *sk, struct sk_buff *skb, int nhoff)
         const struct net_offload **offloads;
         struct guehdr *guehdr = (struct guehdr *)(skb->data + nhoff);
         const struct net_offload *ops;
-        unsigned int guehlen;
+        unsigned int guehlen = 0;
         u8 proto;
         int err = -ENOENT;
 
-        proto = guehdr->proto_ctype;
-
-        guehlen = sizeof(*guehdr) + (guehdr->hlen << 2);
+        switch (guehdr->version) {
+        case 0:
+                proto = guehdr->proto_ctype;
+                guehlen = sizeof(*guehdr) + (guehdr->hlen << 2);
+                break;
+        case 1:
+                switch (((struct iphdr *)guehdr)->version) {
+                case 4:
+                        proto = IPPROTO_IPIP;
+                        break;
+                case 6:
+                        proto = IPPROTO_IPV6;
+                        break;
+                default:
+                        return err;
+                }
+                break;
+        default:
+                return err;
+        }
 
         rcu_read_lock();
         offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads;
diff --git a/net/ipv4/gre_demux.c b/net/ipv4/gre_demux.c
index de1d119a4497..b798862b6be5 100644
--- a/net/ipv4/gre_demux.c
+++ b/net/ipv4/gre_demux.c
@@ -117,6 +117,7 @@ int gre_parse_header(struct sk_buff *skb, struct tnl_ptk_info *tpi,
                 if ((*(u8 *)options & 0xF0) != 0x40)
                         hdr_len += 4;
         }
+        tpi->hdr_len = hdr_len;
         return hdr_len;
 }
 EXPORT_SYMBOL(gre_parse_header);
diff --git a/net/ipv4/inet_diag.c b/net/ipv4/inet_diag.c
index 25af1243649b..38c2c47fe0e8 100644
--- a/net/ipv4/inet_diag.c
+++ b/net/ipv4/inet_diag.c
@@ -44,6 +44,7 @@ struct inet_diag_entry {
         u16 dport;
         u16 family;
         u16 userlocks;
+        u32 ifindex;
 };
 
 static DEFINE_MUTEX(inet_diag_table_mutex);
@@ -571,6 +572,14 @@ static int inet_diag_bc_run(const struct nlattr *_bc,
                         yes = 0;
                         break;
                 }
+                case INET_DIAG_BC_DEV_COND: {
+                        u32 ifindex;
+
+                        ifindex = *((const u32 *)(op + 1));
+                        if (ifindex != entry->ifindex)
+                                yes = 0;
+                        break;
+                }
                 }
 
                 if (yes) {
@@ -613,6 +622,7 @@ int inet_diag_bc_sk(const struct nlattr *bc, struct sock *sk)
         entry_fill_addrs(&entry, sk);
         entry.sport = inet->inet_num;
         entry.dport = ntohs(inet->inet_dport);
+        entry.ifindex = sk->sk_bound_dev_if;
         entry.userlocks = sk_fullsock(sk) ? sk->sk_userlocks : 0;
 
         return inet_diag_bc_run(bc, &entry);
@@ -636,6 +646,17 @@ static int valid_cc(const void *bc, int len, int cc)
         return 0;
 }
 
+/* data is u32 ifindex */
+static bool valid_devcond(const struct inet_diag_bc_op *op, int len,
+                          int *min_len)
+{
+        /* Check ifindex space. */
+        *min_len += sizeof(u32);
+        if (len < *min_len)
+                return false;
+
+        return true;
+}
 /* Validate an inet_diag_hostcond. */
 static bool valid_hostcond(const struct inet_diag_bc_op *op, int len,
                            int *min_len)
@@ -700,6 +721,10 @@ static int inet_diag_bc_audit(const void *bytecode, int bytecode_len)
                         if (!valid_hostcond(bc, len, &min_len))
                                 return -EINVAL;
                         break;
+                case INET_DIAG_BC_DEV_COND:
+                        if (!valid_devcond(bc, len, &min_len))
+                                return -EINVAL;
+                        break;
                 case INET_DIAG_BC_S_GE:
                 case INET_DIAG_BC_S_LE:
                 case INET_DIAG_BC_D_GE:
diff --git a/net/ipv4/inet_fragment.c b/net/ipv4/inet_fragment.c
index 3a88b0c73797..b5e9317eaf9e 100644
--- a/net/ipv4/inet_fragment.c
+++ b/net/ipv4/inet_fragment.c
@@ -355,7 +355,7 @@ static struct inet_frag_queue *inet_frag_alloc(struct netns_frags *nf,
 {
         struct inet_frag_queue *q;
 
-        if (frag_mem_limit(nf) > nf->high_thresh) {
+        if (!nf->high_thresh || frag_mem_limit(nf) > nf->high_thresh) {
                 inet_frag_schedule_worker(f);
                 return NULL;
         }
diff --git a/net/ipv4/ip_forward.c b/net/ipv4/ip_forward.c
index cbfb1808fcc4..9f0a7b96646f 100644
--- a/net/ipv4/ip_forward.c
+++ b/net/ipv4/ip_forward.c
@@ -54,7 +54,7 @@ static bool ip_exceeds_mtu(const struct sk_buff *skb, unsigned int mtu)
         if (skb->ignore_df)
                 return false;
 
-        if (skb_is_gso(skb) && skb_gso_network_seglen(skb) <= mtu)
+        if (skb_is_gso(skb) && skb_gso_validate_mtu(skb, mtu))
                 return false;
 
         return true;
diff --git a/net/ipv4/ip_gre.c b/net/ipv4/ip_gre.c
index 1d000af7f561..5b1481be0282 100644
--- a/net/ipv4/ip_gre.c
+++ b/net/ipv4/ip_gre.c
@@ -138,6 +138,7 @@ static void ipgre_err(struct sk_buff *skb, u32 info,
         const struct iphdr *iph;
         const int type = icmp_hdr(skb)->type;
         const int code = icmp_hdr(skb)->code;
+        unsigned int data_len = 0;
         struct ip_tunnel *t;
 
         switch (type) {
@@ -163,6 +164,7 @@ static void ipgre_err(struct sk_buff *skb, u32 info,
         case ICMP_TIME_EXCEEDED:
                 if (code != ICMP_EXC_TTL)
                         return;
+                data_len = icmp_hdr(skb)->un.reserved[1] * 4; /* RFC 4884 4.1 */
                 break;
 
         case ICMP_REDIRECT:
@@ -181,6 +183,13 @@ static void ipgre_err(struct sk_buff *skb, u32 info,
         if (!t)
                 return;
 
+#if IS_ENABLED(CONFIG_IPV6)
+       if (tpi->proto == htons(ETH_P_IPV6) &&
+           !ip6_err_gen_icmpv6_unreach(skb, iph->ihl * 4 + tpi->hdr_len,
+                                       type, data_len))
+               return;
+#endif
+
         if (t->parms.iph.daddr == 0 ||
             ipv4_is_multicast(t->parms.iph.daddr))
                 return;
@@ -837,17 +846,19 @@ out:
         return ipgre_tunnel_validate(tb, data);
 }
 
-static void ipgre_netlink_parms(struct net_device *dev,
+static int ipgre_netlink_parms(struct net_device *dev,
                                 struct nlattr *data[],
                                 struct nlattr *tb[],
                                 struct ip_tunnel_parm *parms)
 {
+        struct ip_tunnel *t = netdev_priv(dev);
+
         memset(parms, 0, sizeof(*parms));
 
         parms->iph.protocol = IPPROTO_GRE;
 
         if (!data)
-                return;
+                return 0;
 
         if (data[IFLA_GRE_LINK])
                 parms->link = nla_get_u32(data[IFLA_GRE_LINK]);
@@ -876,16 +887,26 @@ static void ipgre_netlink_parms(struct net_device *dev,
         if (data[IFLA_GRE_TOS])
                 parms->iph.tos = nla_get_u8(data[IFLA_GRE_TOS]);
 
-        if (!data[IFLA_GRE_PMTUDISC] || nla_get_u8(data[IFLA_GRE_PMTUDISC]))
+        if (!data[IFLA_GRE_PMTUDISC] || nla_get_u8(data[IFLA_GRE_PMTUDISC])) {
+                if (t->ignore_df)
+                        return -EINVAL;
                 parms->iph.frag_off = htons(IP_DF);
+        }
 
         if (data[IFLA_GRE_COLLECT_METADATA]) {
-                struct ip_tunnel *t = netdev_priv(dev);
-
                 t->collect_md = true;
                 if (dev->type == ARPHRD_IPGRE)
                         dev->type = ARPHRD_NONE;
         }
+
+        if (data[IFLA_GRE_IGNORE_DF]) {
+                if (nla_get_u8(data[IFLA_GRE_IGNORE_DF])
+                  && (parms->iph.frag_off & htons(IP_DF)))
+                        return -EINVAL;
+                t->ignore_df = !!nla_get_u8(data[IFLA_GRE_IGNORE_DF]);
+        }
+
+        return 0;
 }
 
 /* This function returns true when ENCAP attributes are present in the nl msg */
@@ -956,16 +977,19 @@ static int ipgre_newlink(struct net *src_net, struct net_device *dev,
 {
         struct ip_tunnel_parm p;
         struct ip_tunnel_encap ipencap;
+        int err;
 
         if (ipgre_netlink_encap_parms(data, &ipencap)) {
                 struct ip_tunnel *t = netdev_priv(dev);
-                int err = ip_tunnel_encap_setup(t, &ipencap);
+                err = ip_tunnel_encap_setup(t, &ipencap);
 
                 if (err < 0)
                         return err;
         }
 
-        ipgre_netlink_parms(dev, data, tb, &p);
+        err = ipgre_netlink_parms(dev, data, tb, &p);
+        if (err < 0)
+                return err;
         return ip_tunnel_newlink(dev, tb, &p);
 }
 
@@ -974,16 +998,19 @@ static int ipgre_changelink(struct net_device *dev, struct nlattr *tb[],
 {
         struct ip_tunnel_parm p;
         struct ip_tunnel_encap ipencap;
+        int err;
 
         if (ipgre_netlink_encap_parms(data, &ipencap)) {
                 struct ip_tunnel *t = netdev_priv(dev);
-                int err = ip_tunnel_encap_setup(t, &ipencap);
+                err = ip_tunnel_encap_setup(t, &ipencap);
 
                 if (err < 0)
                         return err;
         }
 
-        ipgre_netlink_parms(dev, data, tb, &p);
+        err = ipgre_netlink_parms(dev, data, tb, &p);
+        if (err < 0)
+                return err;
         return ip_tunnel_changelink(dev, tb, &p);
 }
 
@@ -1020,6 +1047,8 @@ static size_t ipgre_get_size(const struct net_device *dev)
                 nla_total_size(2) +
                 /* IFLA_GRE_COLLECT_METADATA */
                 nla_total_size(0) +
+                /* IFLA_GRE_IGNORE_DF */
+                nla_total_size(1) +
                 0;
 }
 
@@ -1053,6 +1082,9 @@ static int ipgre_fill_info(struct sk_buff *skb, const struct net_device *dev)
                         t->encap.flags))
                 goto nla_put_failure;
 
+        if (nla_put_u8(skb, IFLA_GRE_IGNORE_DF, t->ignore_df))
+                goto nla_put_failure;
+
         if (t->collect_md) {
                 if (nla_put_flag(skb, IFLA_GRE_COLLECT_METADATA))
                         goto nla_put_failure;
@@ -1080,6 +1112,7 @@ static const struct nla_policy ipgre_policy[IFLA_GRE_MAX + 1] = {
         [IFLA_GRE_ENCAP_SPORT]  = { .type = NLA_U16 },
         [IFLA_GRE_ENCAP_DPORT]  = { .type = NLA_U16 },
         [IFLA_GRE_COLLECT_METADATA]     = { .type = NLA_FLAG },
+        [IFLA_GRE_IGNORE_DF]    = { .type = NLA_U8 },
 };
 
 static struct rtnl_link_ops ipgre_link_ops __read_mostly = {
diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c
index 124bf0a66328..cbac493c913a 100644
--- a/net/ipv4/ip_output.c
+++ b/net/ipv4/ip_output.c
@@ -225,7 +225,7 @@ static int ip_finish_output_gso(struct net *net, struct sock *sk,
 
         /* common case: locally created skb or seglen is <= mtu */
         if (((IPCB(skb)->flags & IPSKB_FORWARDED) == 0) ||
-              skb_gso_network_seglen(skb) <= mtu)
+              skb_gso_validate_mtu(skb, mtu))
                 return ip_finish_output2(net, sk, skb);
 
         /* Slowpath -  GSO segment length is exceeding the dst MTU.
diff --git a/net/ipv4/ip_tunnel.c b/net/ipv4/ip_tunnel.c
index d8f5e0a269f5..95649ebd2874 100644
--- a/net/ipv4/ip_tunnel.c
+++ b/net/ipv4/ip_tunnel.c
@@ -682,7 +682,7 @@ void ip_tunnel_xmit(struct sk_buff *skb, struct net_device *dev,
         }
 
         df = tnl_params->frag_off;
-        if (skb->protocol == htons(ETH_P_IP))
+        if (skb->protocol == htons(ETH_P_IP) && !tunnel->ignore_df)
                 df |= (inner_iph->frag_off&htons(IP_DF));
 
         max_headroom = LL_RESERVED_SPACE(rt->dst.dev) + sizeof(struct iphdr)
diff --git a/net/ipv4/tcp_dctcp.c b/net/ipv4/tcp_dctcp.c
index 7e538f71f5fb..10d728b6804c 100644
--- a/net/ipv4/tcp_dctcp.c
+++ b/net/ipv4/tcp_dctcp.c
@@ -293,7 +293,7 @@ static size_t dctcp_get_info(struct sock *sk, u32 ext, int *attr,
          */
         if (ext & (1 << (INET_DIAG_DCTCPINFO - 1)) ||
             ext & (1 << (INET_DIAG_VEGASINFO - 1))) {
-                memset(info, 0, sizeof(struct tcp_dctcp_info));
+                memset(&info->dctcp, 0, sizeof(info->dctcp));
                 if (inet_csk(sk)->icsk_ca_ops != &dctcp_reno) {
                         info->dctcp.dctcp_enabled = 1;
                         info->dctcp.dctcp_ce_state = (u16) ca->ce_state;
@@ -303,7 +303,7 @@ static size_t dctcp_get_info(struct sock *sk, u32 ext, int *attr,
                 }
 
                 *attr = INET_DIAG_DCTCPINFO;
-                return sizeof(*info);
+                return sizeof(info->dctcp);
         }
         return 0;
 }
diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
index d6c8f4cd0800..94d4aff97523 100644
--- a/net/ipv4/tcp_input.c
+++ b/net/ipv4/tcp_input.c
@@ -3115,6 +3115,7 @@ static int tcp_clean_rtx_queue(struct sock *sk, int prior_fackets,
         long ca_rtt_us = -1L;
         struct sk_buff *skb;
         u32 pkts_acked = 0;
+        u32 last_in_flight = 0;
         bool rtt_update;
         int flag = 0;
 
@@ -3154,6 +3155,7 @@ static int tcp_clean_rtx_queue(struct sock *sk, int prior_fackets,
                         if (!first_ackt.v64)
                                 first_ackt = last_ackt;
 
+                        last_in_flight = TCP_SKB_CB(skb)->tx.in_flight;
                         reord = min(pkts_acked, reord);
                         if (!after(scb->end_seq, tp->high_seq))
                                 flag |= FLAG_ORIG_SACK_ACKED;
@@ -3250,7 +3252,8 @@ static int tcp_clean_rtx_queue(struct sock *sk, int prior_fackets,
 
         if (icsk->icsk_ca_ops->pkts_acked) {
                 struct ack_sample sample = { .pkts_acked = pkts_acked,
-                                             .rtt_us = ca_rtt_us };
+                                             .rtt_us = ca_rtt_us,
+                                             .in_flight = last_in_flight };
 
                 icsk->icsk_ca_ops->pkts_acked(sk, &sample);
         }
@@ -5159,6 +5162,7 @@ static bool tcp_validate_incoming(struct sock *sk, struct sk_buff *skb,
                                   const struct tcphdr *th, int syn_inerr)
 {
         struct tcp_sock *tp = tcp_sk(sk);
+        bool rst_seq_match = false;
 
         /* RFC1323: H1. Apply PAWS check first. */
         if (tcp_fast_parse_options(skb, th, tp) && tp->rx_opt.saw_tstamp &&
@@ -5195,13 +5199,32 @@ static bool tcp_validate_incoming(struct sock *sk, struct sk_buff *skb,
 
         /* Step 2: check RST bit */
         if (th->rst) {
-                /* RFC 5961 3.2 :
-                 * If sequence number exactly matches RCV.NXT, then
+                /* RFC 5961 3.2 (extend to match against SACK too if available):
+                 * If seq num matches RCV.NXT or the right-most SACK block,
+                 * then
                  *     RESET the connection
                  * else
                  *     Send a challenge ACK
                  */
-                if (TCP_SKB_CB(skb)->seq == tp->rcv_nxt)
+                if (TCP_SKB_CB(skb)->seq == tp->rcv_nxt) {
+                        rst_seq_match = true;
+                } else if (tcp_is_sack(tp) && tp->rx_opt.num_sacks > 0) {
+                        struct tcp_sack_block *sp = &tp->selective_acks[0];
+                        int max_sack = sp[0].end_seq;
+                        int this_sack;
+
+                        for (this_sack = 1; this_sack < tp->rx_opt.num_sacks;
+                             ++this_sack) {
+                                max_sack = after(sp[this_sack].end_seq,
+                                                 max_sack) ?
+                                        sp[this_sack].end_seq : max_sack;
+                        }
+
+                        if (TCP_SKB_CB(skb)->seq == max_sack)
+                                rst_seq_match = true;
+                }
+
+                if (rst_seq_match)
                         tcp_reset(sk);
                 else
                         tcp_send_challenge_ack(sk, skb);
diff --git a/net/ipv4/tcp_nv.c b/net/ipv4/tcp_nv.c
new file mode 100644
index 000000000000..5de82a8d4d87
--- /dev/null
+++ b/net/ipv4/tcp_nv.c
@@ -0,0 +1,476 @@
+/*
+ * TCP NV: TCP with Congestion Avoidance
+ *
+ * TCP-NV is a successor of TCP-Vegas that has been developed to
+ * deal with the issues that occur in modern networks.
+ * Like TCP-Vegas, TCP-NV supports true congestion avoidance,
+ * the ability to detect congestion before packet losses occur.
+ * When congestion (queue buildup) starts to occur, TCP-NV
+ * predicts what the cwnd size should be for the current
+ * throughput and it reduces the cwnd proportionally to
+ * the difference between the current cwnd and the predicted cwnd.
+ *
+ * NV is only recommeneded for traffic within a data center, and when
+ * all the flows are NV (at least those within the data center). This
+ * is due to the inherent unfairness between flows using losses to
+ * detect congestion (congestion control) and those that use queue
+ * buildup to detect congestion (congestion avoidance).
+ *
+ * Note: High NIC coalescence values may lower the performance of NV
+ * due to the increased noise in RTT values. In particular, we have
+ * seen issues with rx-frames values greater than 8.
+ *
+ * TODO:
+ * 1) Add mechanism to deal with reverse congestion.
+ */
+
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/math64.h>
+#include <net/tcp.h>
+#include <linux/inet_diag.h>
+
+/* TCP NV parameters
+ *
+ * nv_pad               Max number of queued packets allowed in network
+ * nv_pad_buffer        Do not grow cwnd if this closed to nv_pad
+ * nv_reset_period      How often (in) seconds)to reset min_rtt
+ * nv_min_cwnd          Don't decrease cwnd below this if there are no losses
+ * nv_cong_dec_mult     Decrease cwnd by X% (30%) of congestion when detected
+ * nv_ssthresh_factor   On congestion set ssthresh to this * <desired cwnd> / 8
+ * nv_rtt_factor        RTT averaging factor
+ * nv_loss_dec_factor   Decrease cwnd by this (50%) when losses occur
+ * nv_dec_eval_min_calls        Wait this many RTT measurements before dec cwnd
+ * nv_inc_eval_min_calls        Wait this many RTT measurements before inc cwnd
+ * nv_ssthresh_eval_min_calls   Wait this many RTT measurements before stopping
+ *                              slow-start due to congestion
+ * nv_stop_rtt_cnt      Only grow cwnd for this many RTTs after non-congestion
+ * nv_rtt_min_cnt       Wait these many RTTs before making congesion decision
+ * nv_cwnd_growth_rate_neg
+ * nv_cwnd_growth_rate_pos
+ *      How quickly to double growth rate (not rate) of cwnd when not
+ *      congested. One value (nv_cwnd_growth_rate_neg) for when
+ *      rate < 1 pkt/RTT (after losses). The other (nv_cwnd_growth_rate_pos)
+ *      otherwise.
+ */
+
+static int nv_pad __read_mostly = 10;
+static int nv_pad_buffer __read_mostly = 2;
+static int nv_reset_period __read_mostly = 5; /* in seconds */
+static int nv_min_cwnd __read_mostly = 2;
+static int nv_cong_dec_mult __read_mostly = 30 * 128 / 100; /* = 30% */
+static int nv_ssthresh_factor __read_mostly = 8; /* = 1 */
+static int nv_rtt_factor __read_mostly = 128; /* = 1/2*old + 1/2*new */
+static int nv_loss_dec_factor __read_mostly = 512; /* => 50% */
+static int nv_cwnd_growth_rate_neg __read_mostly = 8;
+static int nv_cwnd_growth_rate_pos __read_mostly; /* 0 => fixed like Reno */
+static int nv_dec_eval_min_calls __read_mostly = 60;
+static int nv_inc_eval_min_calls __read_mostly = 20;
+static int nv_ssthresh_eval_min_calls __read_mostly = 30;
+static int nv_stop_rtt_cnt __read_mostly = 10;
+static int nv_rtt_min_cnt __read_mostly = 2;
+
+module_param(nv_pad, int, 0644);
+MODULE_PARM_DESC(nv_pad, "max queued packets allowed in network");
+module_param(nv_reset_period, int, 0644);
+MODULE_PARM_DESC(nv_reset_period, "nv_min_rtt reset period (secs)");
+module_param(nv_min_cwnd, int, 0644);
+MODULE_PARM_DESC(nv_min_cwnd, "NV will not decrease cwnd below this value"
+                 " without losses");
+
+/* TCP NV Parameters */
+struct tcpnv {
+        unsigned long nv_min_rtt_reset_jiffies;  /* when to switch to
+                                                  * nv_min_rtt_new */
+        s8  cwnd_growth_factor; /* Current cwnd growth factor,
+                                 * < 0 => less than 1 packet/RTT */
+        u8  available8;
+        u16 available16;
+        u32 loss_cwnd;  /* cwnd at last loss */
+        u8  nv_allow_cwnd_growth:1, /* whether cwnd can grow */
+                nv_reset:1,         /* whether to reset values */
+                nv_catchup:1;       /* whether we are growing because
+                                     * of temporary cwnd decrease */
+        u8  nv_eval_call_cnt;   /* call count since last eval */
+        u8  nv_min_cwnd;        /* nv won't make a ca decision if cwnd is
+                                 * smaller than this. It may grow to handle
+                                 * TSO, LRO and interrupt coalescence because
+                                 * with these a small cwnd cannot saturate
+                                 * the link. Note that this is different from
+                                 * the file local nv_min_cwnd */
+        u8  nv_rtt_cnt;         /* RTTs without making ca decision */;
+        u32 nv_last_rtt;        /* last rtt */
+        u32 nv_min_rtt;         /* active min rtt. Used to determine slope */
+        u32 nv_min_rtt_new;     /* min rtt for future use */
+        u32 nv_rtt_max_rate;    /* max rate seen during current RTT */
+        u32 nv_rtt_start_seq;   /* current RTT ends when packet arrives
+                                 * acking beyond nv_rtt_start_seq */
+        u32 nv_last_snd_una;    /* Previous value of tp->snd_una. It is
+                                 * used to determine bytes acked since last
+                                 * call to bictcp_acked */
+        u32 nv_no_cong_cnt;     /* Consecutive no congestion decisions */
+};
+
+#define NV_INIT_RTT       U32_MAX
+#define NV_MIN_CWND       4
+#define NV_MIN_CWND_GROW  2
+#define NV_TSO_CWND_BOUND 80
+
+static inline void tcpnv_reset(struct tcpnv *ca, struct sock *sk)
+{
+        struct tcp_sock *tp = tcp_sk(sk);
+
+        ca->nv_reset = 0;
+        ca->loss_cwnd = 0;
+        ca->nv_no_cong_cnt = 0;
+        ca->nv_rtt_cnt = 0;
+        ca->nv_last_rtt = 0;
+        ca->nv_rtt_max_rate = 0;
+        ca->nv_rtt_start_seq = tp->snd_una;
+        ca->nv_eval_call_cnt = 0;
+        ca->nv_last_snd_una = tp->snd_una;
+}
+
+static void tcpnv_init(struct sock *sk)
+{
+        struct tcpnv *ca = inet_csk_ca(sk);
+
+        tcpnv_reset(ca, sk);
+
+        ca->nv_allow_cwnd_growth = 1;
+        ca->nv_min_rtt_reset_jiffies = jiffies + 2 * HZ;
+        ca->nv_min_rtt = NV_INIT_RTT;
+        ca->nv_min_rtt_new = NV_INIT_RTT;
+        ca->nv_min_cwnd = NV_MIN_CWND;
+        ca->nv_catchup = 0;
+        ca->cwnd_growth_factor = 0;
+}
+
+static void tcpnv_cong_avoid(struct sock *sk, u32 ack, u32 acked)
+{
+        struct tcp_sock *tp = tcp_sk(sk);
+        struct tcpnv *ca = inet_csk_ca(sk);
+        u32 cnt;
+
+        if (!tcp_is_cwnd_limited(sk))
+                return;
+
+        /* Only grow cwnd if NV has not detected congestion */
+        if (!ca->nv_allow_cwnd_growth)
+                return;
+
+        if (tcp_in_slow_start(tp)) {
+                acked = tcp_slow_start(tp, acked);
+                if (!acked)
+                        return;
+        }
+
+        if (ca->cwnd_growth_factor < 0) {
+                cnt = tp->snd_cwnd << -ca->cwnd_growth_factor;
+                tcp_cong_avoid_ai(tp, cnt, acked);
+        } else {
+                cnt = max(4U, tp->snd_cwnd >> ca->cwnd_growth_factor);
+                tcp_cong_avoid_ai(tp, cnt, acked);
+        }
+}
+
+static u32 tcpnv_recalc_ssthresh(struct sock *sk)
+{
+        const struct tcp_sock *tp = tcp_sk(sk);
+        struct tcpnv *ca = inet_csk_ca(sk);
+
+        ca->loss_cwnd = tp->snd_cwnd;
+        return max((tp->snd_cwnd * nv_loss_dec_factor) >> 10, 2U);
+}
+
+static u32 tcpnv_undo_cwnd(struct sock *sk)
+{
+        struct tcpnv *ca = inet_csk_ca(sk);
+
+        return max(tcp_sk(sk)->snd_cwnd, ca->loss_cwnd);
+}
+
+static void tcpnv_state(struct sock *sk, u8 new_state)
+{
+        struct tcpnv *ca = inet_csk_ca(sk);
+
+        if (new_state == TCP_CA_Open && ca->nv_reset) {
+                tcpnv_reset(ca, sk);
+        } else if (new_state == TCP_CA_Loss || new_state == TCP_CA_CWR ||
+                new_state == TCP_CA_Recovery) {
+                ca->nv_reset = 1;
+                ca->nv_allow_cwnd_growth = 0;
+                if (new_state == TCP_CA_Loss) {
+                        /* Reset cwnd growth factor to Reno value */
+                        if (ca->cwnd_growth_factor > 0)
+                                ca->cwnd_growth_factor = 0;
+                        /* Decrease growth rate if allowed */
+                        if (nv_cwnd_growth_rate_neg > 0 &&
+                            ca->cwnd_growth_factor > -8)
+                                ca->cwnd_growth_factor--;
+                }
+        }
+}
+
+/* Do congestion avoidance calculations for TCP-NV
+ */
+static void tcpnv_acked(struct sock *sk, const struct ack_sample *sample)
+{
+        const struct inet_connection_sock *icsk = inet_csk(sk);
+        struct tcp_sock *tp = tcp_sk(sk);
+        struct tcpnv *ca = inet_csk_ca(sk);
+        unsigned long now = jiffies;
+        s64 rate64 = 0;
+        u32 rate, max_win, cwnd_by_slope;
+        u32 avg_rtt;
+        u32 bytes_acked = 0;
+
+        /* Some calls are for duplicates without timetamps */
+        if (sample->rtt_us < 0)
+                return;
+
+        /* If not in TCP_CA_Open or TCP_CA_Disorder states, skip. */
+        if (icsk->icsk_ca_state != TCP_CA_Open &&
+            icsk->icsk_ca_state != TCP_CA_Disorder)
+                return;
+
+        /* Stop cwnd growth if we were in catch up mode */
+        if (ca->nv_catchup && tp->snd_cwnd >= nv_min_cwnd) {
+                ca->nv_catchup = 0;
+                ca->nv_allow_cwnd_growth = 0;
+        }
+
+        bytes_acked = tp->snd_una - ca->nv_last_snd_una;
+        ca->nv_last_snd_una = tp->snd_una;
+
+        if (sample->in_flight == 0)
+                return;
+
+        /* Calculate moving average of RTT */
+        if (nv_rtt_factor > 0) {
+                if (ca->nv_last_rtt > 0) {
+                        avg_rtt = (((u64)sample->rtt_us) * nv_rtt_factor +
+                                   ((u64)ca->nv_last_rtt)
+                                   * (256 - nv_rtt_factor)) >> 8;
+                } else {
+                        avg_rtt = sample->rtt_us;
+                        ca->nv_min_rtt = avg_rtt << 1;
+                }
+                ca->nv_last_rtt = avg_rtt;
+        } else {
+                avg_rtt = sample->rtt_us;
+        }
+
+        /* rate in 100's bits per second */
+        rate64 = ((u64)sample->in_flight) * 8000000;
+        rate = (u32)div64_u64(rate64, (u64)(avg_rtt * 100));
+
+        /* Remember the maximum rate seen during this RTT
+         * Note: It may be more than one RTT. This function should be
+         *       called at least nv_dec_eval_min_calls times.
+         */
+        if (ca->nv_rtt_max_rate < rate)
+                ca->nv_rtt_max_rate = rate;
+
+        /* We have valid information, increment counter */
+        if (ca->nv_eval_call_cnt < 255)
+                ca->nv_eval_call_cnt++;
+
+        /* update min rtt if necessary */
+        if (avg_rtt < ca->nv_min_rtt)
+                ca->nv_min_rtt = avg_rtt;
+
+        /* update future min_rtt if necessary */
+        if (avg_rtt < ca->nv_min_rtt_new)
+                ca->nv_min_rtt_new = avg_rtt;
+
+        /* nv_min_rtt is updated with the minimum (possibley averaged) rtt
+         * seen in the last sysctl_tcp_nv_reset_period seconds (i.e. a
+         * warm reset). This new nv_min_rtt will be continued to be updated
+         * and be used for another sysctl_tcp_nv_reset_period seconds,
+         * when it will be updated again.
+         * In practice we introduce some randomness, so the actual period used
+         * is chosen randomly from the range:
+         *   [sysctl_tcp_nv_reset_period*3/4, sysctl_tcp_nv_reset_period*5/4)
+         */
+        if (time_after_eq(now, ca->nv_min_rtt_reset_jiffies)) {
+                unsigned char rand;
+
+                ca->nv_min_rtt = ca->nv_min_rtt_new;
+                ca->nv_min_rtt_new = NV_INIT_RTT;
+                get_random_bytes(&rand, 1);
+                ca->nv_min_rtt_reset_jiffies =
+                        now + ((nv_reset_period * (384 + rand) * HZ) >> 9);
+                /* Every so often we decrease ca->nv_min_cwnd in case previous
+                 *  value is no longer accurate.
+                 */
+                ca->nv_min_cwnd = max(ca->nv_min_cwnd / 2, NV_MIN_CWND);
+        }
+
+        /* Once per RTT check if we need to do congestion avoidance */
+        if (before(ca->nv_rtt_start_seq, tp->snd_una)) {
+                ca->nv_rtt_start_seq = tp->snd_nxt;
+                if (ca->nv_rtt_cnt < 0xff)
+                        /* Increase counter for RTTs without CA decision */
+                        ca->nv_rtt_cnt++;
+
+                /* If this function is only called once within an RTT
+                 * the cwnd is probably too small (in some cases due to
+                 * tso, lro or interrupt coalescence), so we increase
+                 * ca->nv_min_cwnd.
+                 */
+                if (ca->nv_eval_call_cnt == 1 &&
+                    bytes_acked >= (ca->nv_min_cwnd - 1) * tp->mss_cache &&
+                    ca->nv_min_cwnd < (NV_TSO_CWND_BOUND + 1)) {
+                        ca->nv_min_cwnd = min(ca->nv_min_cwnd
+                                              + NV_MIN_CWND_GROW,
+                                              NV_TSO_CWND_BOUND + 1);
+                        ca->nv_rtt_start_seq = tp->snd_nxt +
+                                ca->nv_min_cwnd * tp->mss_cache;
+                        ca->nv_eval_call_cnt = 0;
+                        ca->nv_allow_cwnd_growth = 1;
+                        return;
+                }
+
+                /* Find the ideal cwnd for current rate from slope
+                 * slope = 80000.0 * mss / nv_min_rtt
+                 * cwnd_by_slope = nv_rtt_max_rate / slope
+                 */
+                cwnd_by_slope = (u32)
+                        div64_u64(((u64)ca->nv_rtt_max_rate) * ca->nv_min_rtt,
+                                  (u64)(80000 * tp->mss_cache));
+                max_win = cwnd_by_slope + nv_pad;
+
+                /* If cwnd > max_win, decrease cwnd
+                 * if cwnd < max_win, grow cwnd
+                 * else leave the same
+                 */
+                if (tp->snd_cwnd > max_win) {
+                        /* there is congestion, check that it is ok
+                         * to make a CA decision
+                         * 1. We should have at least nv_dec_eval_min_calls
+                         *    data points before making a CA  decision
+                         * 2. We only make a congesion decision after
+                         *    nv_rtt_min_cnt RTTs
+                         */
+                        if (ca->nv_rtt_cnt < nv_rtt_min_cnt) {
+                                return;
+                        } else if (tp->snd_ssthresh == TCP_INFINITE_SSTHRESH) {
+                                if (ca->nv_eval_call_cnt <
+                                    nv_ssthresh_eval_min_calls)
+                                        return;
+                                /* otherwise we will decrease cwnd */
+                        } else if (ca->nv_eval_call_cnt <
+                                   nv_dec_eval_min_calls) {
+                                if (ca->nv_allow_cwnd_growth &&
+                                    ca->nv_rtt_cnt > nv_stop_rtt_cnt)
+                                        ca->nv_allow_cwnd_growth = 0;
+                                return;
+                        }
+
+                        /* We have enough data to determine we are congested */
+                        ca->nv_allow_cwnd_growth = 0;
+                        tp->snd_ssthresh =
+                                (nv_ssthresh_factor * max_win) >> 3;
+                        if (tp->snd_cwnd - max_win > 2) {
+                                /* gap > 2, we do exponential cwnd decrease */
+                                int dec;
+
+                                dec = max(2U, ((tp->snd_cwnd - max_win) *
+                                               nv_cong_dec_mult) >> 7);
+                                tp->snd_cwnd -= dec;
+                        } else if (nv_cong_dec_mult > 0) {
+                                tp->snd_cwnd = max_win;
+                        }
+                        if (ca->cwnd_growth_factor > 0)
+                                ca->cwnd_growth_factor = 0;
+                        ca->nv_no_cong_cnt = 0;
+                } else if (tp->snd_cwnd <= max_win - nv_pad_buffer) {
+                        /* There is no congestion, grow cwnd if allowed*/
+                        if (ca->nv_eval_call_cnt < nv_inc_eval_min_calls)
+                                return;
+
+                        ca->nv_allow_cwnd_growth = 1;
+                        ca->nv_no_cong_cnt++;
+                        if (ca->cwnd_growth_factor < 0 &&
+                            nv_cwnd_growth_rate_neg > 0 &&
+                            ca->nv_no_cong_cnt > nv_cwnd_growth_rate_neg) {
+                                ca->cwnd_growth_factor++;
+                                ca->nv_no_cong_cnt = 0;
+                        } else if (ca->cwnd_growth_factor >= 0 &&
+                                   nv_cwnd_growth_rate_pos > 0 &&
+                                   ca->nv_no_cong_cnt >
+                                   nv_cwnd_growth_rate_pos) {
+                                ca->cwnd_growth_factor++;
+                                ca->nv_no_cong_cnt = 0;
+                        }
+                } else {
+                        /* cwnd is in-between, so do nothing */
+                        return;
+                }
+
+                /* update state */
+                ca->nv_eval_call_cnt = 0;
+                ca->nv_rtt_cnt = 0;
+                ca->nv_rtt_max_rate = 0;
+
+                /* Don't want to make cwnd < nv_min_cwnd
+                 * (it wasn't before, if it is now is because nv
+                 *  decreased it).
+                 */
+                if (tp->snd_cwnd < nv_min_cwnd)
+                        tp->snd_cwnd = nv_min_cwnd;
+        }
+}
+
+/* Extract info for Tcp socket info provided via netlink */
+size_t tcpnv_get_info(struct sock *sk, u32 ext, int *attr,
+                      union tcp_cc_info *info)
+{
+        const struct tcpnv *ca = inet_csk_ca(sk);
+
+        if (ext & (1 << (INET_DIAG_VEGASINFO - 1))) {
+                info->vegas.tcpv_enabled = 1;
+                info->vegas.tcpv_rttcnt = ca->nv_rtt_cnt;
+                info->vegas.tcpv_rtt = ca->nv_last_rtt;
+                info->vegas.tcpv_minrtt = ca->nv_min_rtt;
+
+                *attr = INET_DIAG_VEGASINFO;
+                return sizeof(struct tcpvegas_info);
+        }
+        return 0;
+}
+EXPORT_SYMBOL_GPL(tcpnv_get_info);
+
+static struct tcp_congestion_ops tcpnv __read_mostly = {
+        .init           = tcpnv_init,
+        .ssthresh       = tcpnv_recalc_ssthresh,
+        .cong_avoid     = tcpnv_cong_avoid,
+        .set_state      = tcpnv_state,
+        .undo_cwnd      = tcpnv_undo_cwnd,
+        .pkts_acked     = tcpnv_acked,
+        .get_info       = tcpnv_get_info,
+
+        .owner          = THIS_MODULE,
+        .name           = "nv",
+};
+
+static int __init tcpnv_register(void)
+{
+        BUILD_BUG_ON(sizeof(struct tcpnv) > ICSK_CA_PRIV_SIZE);
+
+        return tcp_register_congestion_control(&tcpnv);
+}
+
+static void __exit tcpnv_unregister(void)
+{
+        tcp_unregister_congestion_control(&tcpnv);
+}
+
+module_init(tcpnv_register);
+module_exit(tcpnv_unregister);
+
+MODULE_AUTHOR("Lawrence Brakmo");
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("TCP NV");
+MODULE_VERSION("1.0");
diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index e00e972c4e6a..b26aa870adc0 100644
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -911,9 +911,12 @@ static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it,
         int err;
 
         BUG_ON(!skb || !tcp_skb_pcount(skb));
+        tp = tcp_sk(sk);
 
         if (clone_it) {
                 skb_mstamp_get(&skb->skb_mstamp);
+                TCP_SKB_CB(skb)->tx.in_flight = TCP_SKB_CB(skb)->end_seq
+                        - tp->snd_una;
 
                 if (unlikely(skb_cloned(skb)))
                         skb = pskb_copy(skb, gfp_mask);
@@ -924,7 +927,6 @@ static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it,
         }
 
         inet = inet_sk(sk);
-        tp = tcp_sk(sk);
         tcb = TCP_SKB_CB(skb);
         memset(&opts, 0, sizeof(opts));
 
diff --git a/net/ipv4/udp_tunnel.c b/net/ipv4/udp_tunnel.c
index 47f12c73d959..58bd39fb14b4 100644
--- a/net/ipv4/udp_tunnel.c
+++ b/net/ipv4/udp_tunnel.c
@@ -76,6 +76,67 @@ void setup_udp_tunnel_sock(struct net *net, struct socket *sock,
 }
 EXPORT_SYMBOL_GPL(setup_udp_tunnel_sock);
 
+void udp_tunnel_push_rx_port(struct net_device *dev, struct socket *sock,
+                             unsigned short type)
+{
+        struct sock *sk = sock->sk;
+        struct udp_tunnel_info ti;
+
+        if (!dev->netdev_ops->ndo_udp_tunnel_add)
+                return;
+
+        ti.type = type;
+        ti.sa_family = sk->sk_family;
+        ti.port = inet_sk(sk)->inet_sport;
+
+        dev->netdev_ops->ndo_udp_tunnel_add(dev, &ti);
+}
+EXPORT_SYMBOL_GPL(udp_tunnel_push_rx_port);
+
+/* Notify netdevs that UDP port started listening */
+void udp_tunnel_notify_add_rx_port(struct socket *sock, unsigned short type)
+{
+        struct sock *sk = sock->sk;
+        struct net *net = sock_net(sk);
+        struct udp_tunnel_info ti;
+        struct net_device *dev;
+
+        ti.type = type;
+        ti.sa_family = sk->sk_family;
+        ti.port = inet_sk(sk)->inet_sport;
+
+        rcu_read_lock();
+        for_each_netdev_rcu(net, dev) {
+                if (!dev->netdev_ops->ndo_udp_tunnel_add)
+                        continue;
+                dev->netdev_ops->ndo_udp_tunnel_add(dev, &ti);
+        }
+        rcu_read_unlock();
+}
+EXPORT_SYMBOL_GPL(udp_tunnel_notify_add_rx_port);
+
+/* Notify netdevs that UDP port is no more listening */
+void udp_tunnel_notify_del_rx_port(struct socket *sock, unsigned short type)
+{
+        struct sock *sk = sock->sk;
+        struct net *net = sock_net(sk);
+        struct udp_tunnel_info ti;
+        struct net_device *dev;
+
+        ti.type = type;
+        ti.sa_family = sk->sk_family;
+        ti.port = inet_sk(sk)->inet_sport;
+
+        rcu_read_lock();
+        for_each_netdev_rcu(net, dev) {
+                if (!dev->netdev_ops->ndo_udp_tunnel_del)
+                        continue;
+                dev->netdev_ops->ndo_udp_tunnel_del(dev, &ti);
+        }
+        rcu_read_unlock();
+}
+EXPORT_SYMBOL_GPL(udp_tunnel_notify_del_rx_port);
+
 void udp_tunnel_xmit_skb(struct rtable *rt, struct sock *sk, struct sk_buff *skb,
                          __be32 src, __be32 dst, __u8 tos, __u8 ttl,
                          __be16 df, __be16 src_port, __be16 dst_port,
diff --git a/net/ipv4/xfrm4_policy.c b/net/ipv4/xfrm4_policy.c
index 7b0edb37a115..b644a23c3db0 100644
--- a/net/ipv4/xfrm4_policy.c
+++ b/net/ipv4/xfrm4_policy.c
@@ -295,7 +295,7 @@ static struct ctl_table xfrm4_policy_table[] = {
         { }
 };
 
-static int __net_init xfrm4_net_sysctl_init(struct net *net)
+static __net_init int xfrm4_net_sysctl_init(struct net *net)
 {
         struct ctl_table *table;
         struct ctl_table_header *hdr;
@@ -323,7 +323,7 @@ err_alloc:
         return -ENOMEM;
 }
 
-static void __net_exit xfrm4_net_sysctl_exit(struct net *net)
+static __net_exit void xfrm4_net_sysctl_exit(struct net *net)
 {
         struct ctl_table *table;
 
@@ -336,12 +336,12 @@ static void __net_exit xfrm4_net_sysctl_exit(struct net *net)
                 kfree(table);
 }
 #else /* CONFIG_SYSCTL */
-static int inline xfrm4_net_sysctl_init(struct net *net)
+static inline int xfrm4_net_sysctl_init(struct net *net)
 {
         return 0;
 }
 
-static void inline xfrm4_net_sysctl_exit(struct net *net)
+static inline void xfrm4_net_sysctl_exit(struct net *net)
 {
 }
 #endif
diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c
index 47f837a58e0a..a1f6b7b31531 100644
--- a/net/ipv6/addrconf.c
+++ b/net/ipv6/addrconf.c
@@ -1524,6 +1524,28 @@ out:
         return hiscore_idx;
 }
 
+static int ipv6_get_saddr_master(struct net *net,
+                                 const struct net_device *dst_dev,
+                                 const struct net_device *master,
+                                 struct ipv6_saddr_dst *dst,
+                                 struct ipv6_saddr_score *scores,
+                                 int hiscore_idx)
+{
+        struct inet6_dev *idev;
+
+        idev = __in6_dev_get(dst_dev);
+        if (idev)
+                hiscore_idx = __ipv6_dev_get_saddr(net, dst, idev,
+                                                   scores, hiscore_idx);
+
+        idev = __in6_dev_get(master);
+        if (idev)
+                hiscore_idx = __ipv6_dev_get_saddr(net, dst, idev,
+                                                   scores, hiscore_idx);
+
+        return hiscore_idx;
+}
+
 int ipv6_dev_get_saddr(struct net *net, const struct net_device *dst_dev,
                        const struct in6_addr *daddr, unsigned int prefs,
                        struct in6_addr *saddr)
@@ -1577,13 +1599,39 @@ int ipv6_dev_get_saddr(struct net *net, const struct net_device *dst_dev,
                 if (idev)
                         hiscore_idx = __ipv6_dev_get_saddr(net, &dst, idev, scores, hiscore_idx);
         } else {
+                const struct net_device *master;
+                int master_idx = 0;
+
+                /* if dst_dev exists and is enslaved to an L3 device, then
+                 * prefer addresses from dst_dev and then the master over
+                 * any other enslaved devices in the L3 domain.
+                 */
+                master = l3mdev_master_dev_rcu(dst_dev);
+                if (master) {
+                        master_idx = master->ifindex;
+
+                        hiscore_idx = ipv6_get_saddr_master(net, dst_dev,
+                                                            master, &dst,
+                                                            scores, hiscore_idx);
+
+                        if (scores[hiscore_idx].ifa)
+                                goto out;
+                }
+
                 for_each_netdev_rcu(net, dev) {
+                        /* only consider addresses on devices in the
+                         * same L3 domain
+                         */
+                        if (l3mdev_master_ifindex_rcu(dev) != master_idx)
+                                continue;
                         idev = __in6_dev_get(dev);
                         if (!idev)
                                 continue;
                         hiscore_idx = __ipv6_dev_get_saddr(net, &dst, idev, scores, hiscore_idx);
                 }
         }
+
+out:
         rcu_read_unlock();
 
         hiscore = &scores[hiscore_idx];
@@ -2254,7 +2302,7 @@ static struct inet6_dev *addrconf_add_dev(struct net_device *dev)
                 return ERR_PTR(-EACCES);
 
         /* Add default multicast route */
-        if (!(dev->flags & IFF_LOOPBACK))
+        if (!(dev->flags & IFF_LOOPBACK) && !netif_is_l3_master(dev))
                 addrconf_add_mroute(dev);
 
         return idev;
@@ -2333,12 +2381,109 @@ static bool is_addr_mode_generate_stable(struct inet6_dev *idev)
                idev->addr_gen_mode == IN6_ADDR_GEN_MODE_RANDOM;
 }
 
+int addrconf_prefix_rcv_add_addr(struct net *net, struct net_device *dev,
+                                 const struct prefix_info *pinfo,
+                                 struct inet6_dev *in6_dev,
+                                 const struct in6_addr *addr, int addr_type,
+                                 u32 addr_flags, bool sllao, bool tokenized,
+                                 __u32 valid_lft, u32 prefered_lft)
+{
+        struct inet6_ifaddr *ifp = ipv6_get_ifaddr(net, addr, dev, 1);
+        int create = 0, update_lft = 0;
+
+        if (!ifp && valid_lft) {
+                int max_addresses = in6_dev->cnf.max_addresses;
+
+#ifdef CONFIG_IPV6_OPTIMISTIC_DAD
+                if (in6_dev->cnf.optimistic_dad &&
+                    !net->ipv6.devconf_all->forwarding && sllao)
+                        addr_flags |= IFA_F_OPTIMISTIC;
+#endif
+
+                /* Do not allow to create too much of autoconfigured
+                 * addresses; this would be too easy way to crash kernel.
+                 */
+                if (!max_addresses ||
+                    ipv6_count_addresses(in6_dev) < max_addresses)
+                        ifp = ipv6_add_addr(in6_dev, addr, NULL,
+                                            pinfo->prefix_len,
+                                            addr_type&IPV6_ADDR_SCOPE_MASK,
+                                            addr_flags, valid_lft,
+                                            prefered_lft);
+
+                if (IS_ERR_OR_NULL(ifp))
+                        return -1;
+
+                update_lft = 0;
+                create = 1;
+                spin_lock_bh(&ifp->lock);
+                ifp->flags |= IFA_F_MANAGETEMPADDR;
+                ifp->cstamp = jiffies;
+                ifp->tokenized = tokenized;
+                spin_unlock_bh(&ifp->lock);
+                addrconf_dad_start(ifp);
+        }
+
+        if (ifp) {
+                u32 flags;
+                unsigned long now;
+                u32 stored_lft;
+
+                /* update lifetime (RFC2462 5.5.3 e) */
+                spin_lock_bh(&ifp->lock);
+                now = jiffies;
+                if (ifp->valid_lft > (now - ifp->tstamp) / HZ)
+                        stored_lft = ifp->valid_lft - (now - ifp->tstamp) / HZ;
+                else
+                        stored_lft = 0;
+                if (!update_lft && !create && stored_lft) {
+                        const u32 minimum_lft = min_t(u32,
+                                stored_lft, MIN_VALID_LIFETIME);
+                        valid_lft = max(valid_lft, minimum_lft);
+
+                        /* RFC4862 Section 5.5.3e:
+                         * "Note that the preferred lifetime of the
+                         *  corresponding address is always reset to
+                         *  the Preferred Lifetime in the received
+                         *  Prefix Information option, regardless of
+                         *  whether the valid lifetime is also reset or
+                         *  ignored."
+                         *
+                         * So we should always update prefered_lft here.
+                         */
+                        update_lft = 1;
+                }
+
+                if (update_lft) {
+                        ifp->valid_lft = valid_lft;
+                        ifp->prefered_lft = prefered_lft;
+                        ifp->tstamp = now;
+                        flags = ifp->flags;
+                        ifp->flags &= ~IFA_F_DEPRECATED;
+                        spin_unlock_bh(&ifp->lock);
+
+                        if (!(flags&IFA_F_TENTATIVE))
+                                ipv6_ifa_notify(0, ifp);
+                } else
+                        spin_unlock_bh(&ifp->lock);
+
+                manage_tempaddrs(in6_dev, ifp, valid_lft, prefered_lft,
+                                 create, now);
+
+                in6_ifa_put(ifp);
+                addrconf_verify();
+        }
+
+        return 0;
+}
+EXPORT_SYMBOL_GPL(addrconf_prefix_rcv_add_addr);
+
 void addrconf_prefix_rcv(struct net_device *dev, u8 *opt, int len, bool sllao)
 {
         struct prefix_info *pinfo;
         __u32 valid_lft;
         __u32 prefered_lft;
-        int addr_type;
+        int addr_type, err;
         u32 addr_flags = 0;
         struct inet6_dev *in6_dev;
         struct net *net = dev_net(dev);
@@ -2432,10 +2577,8 @@ void addrconf_prefix_rcv(struct net_device *dev, u8 *opt, int len, bool sllao)
         /* Try to figure out our local address for this prefix */
 
         if (pinfo->autoconf && in6_dev->cnf.autoconf) {
-                struct inet6_ifaddr *ifp;
                 struct in6_addr addr;
-                int create = 0, update_lft = 0;
-                bool tokenized = false;
+                bool tokenized = false, dev_addr_generated = false;
 
                 if (pinfo->prefix_len == 64) {
                         memcpy(&addr, &pinfo->prefix, 8);
@@ -2453,106 +2596,36 @@ void addrconf_prefix_rcv(struct net_device *dev, u8 *opt, int len, bool sllao)
                                 goto ok;
                         } else if (ipv6_generate_eui64(addr.s6_addr + 8, dev) &&
                                    ipv6_inherit_eui64(addr.s6_addr + 8, in6_dev)) {
-                                in6_dev_put(in6_dev);
-                                return;
+                                goto put;
+                        } else {
+                                dev_addr_generated = true;
                         }
                         goto ok;
                 }
                 net_dbg_ratelimited("IPv6 addrconf: prefix with wrong length %d\n",
                                     pinfo->prefix_len);
-                in6_dev_put(in6_dev);
-                return;
+                goto put;
 
 ok:
+                err = addrconf_prefix_rcv_add_addr(net, dev, pinfo, in6_dev,
+                                                   &addr, addr_type,
+                                                   addr_flags, sllao,
+                                                   tokenized, valid_lft,
+                                                   prefered_lft);
+                if (err)
+                        goto put;
 
-                ifp = ipv6_get_ifaddr(net, &addr, dev, 1);
-
-                if (!ifp && valid_lft) {
-                        int max_addresses = in6_dev->cnf.max_addresses;
-
-#ifdef CONFIG_IPV6_OPTIMISTIC_DAD
-                        if (in6_dev->cnf.optimistic_dad &&
-                            !net->ipv6.devconf_all->forwarding && sllao)
-                                addr_flags |= IFA_F_OPTIMISTIC;
-#endif
-
-                        /* Do not allow to create too much of autoconfigured
-                         * addresses; this would be too easy way to crash kernel.
-                         */
-                        if (!max_addresses ||
-                            ipv6_count_addresses(in6_dev) < max_addresses)
-                                ifp = ipv6_add_addr(in6_dev, &addr, NULL,
-                                                    pinfo->prefix_len,
-                                                    addr_type&IPV6_ADDR_SCOPE_MASK,
-                                                    addr_flags, valid_lft,
-                                                    prefered_lft);
-
-                        if (IS_ERR_OR_NULL(ifp)) {
-                                in6_dev_put(in6_dev);
-                                return;
-                        }
-
-                        update_lft = 0;
-                        create = 1;
-                        spin_lock_bh(&ifp->lock);
-                        ifp->flags |= IFA_F_MANAGETEMPADDR;
-                        ifp->cstamp = jiffies;
-                        ifp->tokenized = tokenized;
-                        spin_unlock_bh(&ifp->lock);
-                        addrconf_dad_start(ifp);
-                }
-
-                if (ifp) {
-                        u32 flags;
-                        unsigned long now;
-                        u32 stored_lft;
-
-                        /* update lifetime (RFC2462 5.5.3 e) */
-                        spin_lock_bh(&ifp->lock);
-                        now = jiffies;
-                        if (ifp->valid_lft > (now - ifp->tstamp) / HZ)
-                                stored_lft = ifp->valid_lft - (now - ifp->tstamp) / HZ;
-                        else
-                                stored_lft = 0;
-                        if (!update_lft && !create && stored_lft) {
-                                const u32 minimum_lft = min_t(u32,
-                                        stored_lft, MIN_VALID_LIFETIME);
-                                valid_lft = max(valid_lft, minimum_lft);
-
-                                /* RFC4862 Section 5.5.3e:
-                                 * "Note that the preferred lifetime of the
-                                 *  corresponding address is always reset to
-                                 *  the Preferred Lifetime in the received
-                                 *  Prefix Information option, regardless of
-                                 *  whether the valid lifetime is also reset or
-                                 *  ignored."
-                                 *
-                                 * So we should always update prefered_lft here.
-                                 */
-                                update_lft = 1;
-                        }
-
-                        if (update_lft) {
-                                ifp->valid_lft = valid_lft;
-                                ifp->prefered_lft = prefered_lft;
-                                ifp->tstamp = now;
-                                flags = ifp->flags;
-                                ifp->flags &= ~IFA_F_DEPRECATED;
-                                spin_unlock_bh(&ifp->lock);
-
-                                if (!(flags&IFA_F_TENTATIVE))
-                                        ipv6_ifa_notify(0, ifp);
-                        } else
-                                spin_unlock_bh(&ifp->lock);
-
-                        manage_tempaddrs(in6_dev, ifp, valid_lft, prefered_lft,
-                                         create, now);
-
-                        in6_ifa_put(ifp);
-                        addrconf_verify();
-                }
+                /* Ignore error case here because previous prefix add addr was
+                 * successful which will be notified.
+                 */
+                ndisc_ops_prefix_rcv_add_addr(net, dev, pinfo, in6_dev, &addr,
+                                              addr_type, addr_flags, sllao,
+                                              tokenized, valid_lft,
+                                              prefered_lft,
+                                              dev_addr_generated);
         }
         inet6_prefix_notify(RTM_NEWPREFIX, in6_dev, pinfo);
+put:
         in6_dev_put(in6_dev);
 }
 
@@ -2947,8 +3020,8 @@ static void init_loopback(struct net_device *dev)
         }
 }
 
-static void addrconf_add_linklocal(struct inet6_dev *idev,
-                                   const struct in6_addr *addr, u32 flags)
+void addrconf_add_linklocal(struct inet6_dev *idev,
+                            const struct in6_addr *addr, u32 flags)
 {
         struct inet6_ifaddr *ifp;
         u32 addr_flags = flags | IFA_F_PERMANENT;
@@ -2967,6 +3040,7 @@ static void addrconf_add_linklocal(struct inet6_dev *idev,
                 in6_ifa_put(ifp);
         }
 }
+EXPORT_SYMBOL_GPL(addrconf_add_linklocal);
 
 static bool ipv6_reserved_interfaceid(struct in6_addr address)
 {
diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c
index bfa86f040c16..2076c21107d0 100644
--- a/net/ipv6/af_inet6.c
+++ b/net/ipv6/af_inet6.c
@@ -92,6 +92,12 @@ MODULE_PARM_DESC(disable_ipv6, "Disable IPv6 on all interfaces");
 module_param_named(autoconf, ipv6_defaults.autoconf, int, 0444);
 MODULE_PARM_DESC(autoconf, "Enable IPv6 address autoconfiguration on all interfaces");
 
+bool ipv6_mod_enabled(void)
+{
+        return disable_ipv6_mod == 0;
+}
+EXPORT_SYMBOL_GPL(ipv6_mod_enabled);
+
 static __inline__ struct ipv6_pinfo *inet6_sk_generic(struct sock *sk)
 {
         const int offset = sk->sk_prot->obj_size - sizeof(struct ipv6_pinfo);
diff --git a/net/ipv6/fib6_rules.c b/net/ipv6/fib6_rules.c
index ed33abf57abd..5857c1fc8b67 100644
--- a/net/ipv6/fib6_rules.c
+++ b/net/ipv6/fib6_rules.c
@@ -67,6 +67,7 @@ static int fib6_rule_action(struct fib_rule *rule, struct flowi *flp,
         struct net *net = rule->fr_net;
         pol_lookup_t lookup = arg->lookup_ptr;
         int err = 0;
+        u32 tb_id;
 
         switch (rule->action) {
         case FR_ACT_TO_TBL:
@@ -86,7 +87,8 @@ static int fib6_rule_action(struct fib_rule *rule, struct flowi *flp,
                 goto discard_pkt;
         }
 
-        table = fib6_get_table(net, rule->table);
+        tb_id = fib_rule_get_table(rule, arg);
+        table = fib6_get_table(net, tb_id);
         if (!table) {
                 err = -EAGAIN;
                 goto out;
@@ -199,7 +201,7 @@ static int fib6_rule_configure(struct fib_rule *rule, struct sk_buff *skb,
         struct net *net = sock_net(skb->sk);
         struct fib6_rule *rule6 = (struct fib6_rule *) rule;
 
-        if (rule->action == FR_ACT_TO_TBL) {
+        if (rule->action == FR_ACT_TO_TBL && !rule->l3mdev) {
                 if (rule->table == RT6_TABLE_UNSPEC)
                         goto errout;
 
diff --git a/net/ipv6/icmp.c b/net/ipv6/icmp.c
index a4fa84076969..bd59c343d35f 100644
--- a/net/ipv6/icmp.c
+++ b/net/ipv6/icmp.c
@@ -388,7 +388,8 @@ relookup_failed:
 /*
  *      Send an ICMP message in response to a packet in error
  */
-static void icmp6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info)
+static void icmp6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info,
+                       const struct in6_addr *force_saddr)
 {
         struct net *net = dev_net(skb->dev);
         struct inet6_dev *idev = NULL;
@@ -475,6 +476,8 @@ static void icmp6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info)
         memset(&fl6, 0, sizeof(fl6));
         fl6.flowi6_proto = IPPROTO_ICMPV6;
         fl6.daddr = hdr->saddr;
+        if (force_saddr)
+                saddr = force_saddr;
         if (saddr)
                 fl6.saddr = *saddr;
         fl6.flowi6_mark = mark;
@@ -502,12 +505,14 @@ static void icmp6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info)
         else if (!fl6.flowi6_oif)
                 fl6.flowi6_oif = np->ucast_oif;
 
+        ipc6.tclass = np->tclass;
+        fl6.flowlabel = ip6_make_flowinfo(ipc6.tclass, fl6.flowlabel);
+
         dst = icmpv6_route_lookup(net, skb, sk, &fl6);
         if (IS_ERR(dst))
                 goto out;
 
         ipc6.hlimit = ip6_sk_dst_hoplimit(np, &fl6, dst);
-        ipc6.tclass = np->tclass;
         ipc6.dontfrag = np->dontfrag;
         ipc6.opt = NULL;
 
@@ -549,10 +554,75 @@ out:
  */
 void icmpv6_param_prob(struct sk_buff *skb, u8 code, int pos)
 {
-        icmp6_send(skb, ICMPV6_PARAMPROB, code, pos);
+        icmp6_send(skb, ICMPV6_PARAMPROB, code, pos, NULL);
         kfree_skb(skb);
 }
 
+/* Generate icmpv6 with type/code ICMPV6_DEST_UNREACH/ICMPV6_ADDR_UNREACH
+ * if sufficient data bytes are available
+ * @nhs is the size of the tunnel header(s) :
+ *  Either an IPv4 header for SIT encap
+ *         an IPv4 header + GRE header for GRE encap
+ */
+int ip6_err_gen_icmpv6_unreach(struct sk_buff *skb, int nhs, int type,
+                               unsigned int data_len)
+{
+        struct in6_addr temp_saddr;
+        struct rt6_info *rt;
+        struct sk_buff *skb2;
+        u32 info = 0;
+
+        if (!pskb_may_pull(skb, nhs + sizeof(struct ipv6hdr) + 8))
+                return 1;
+
+        /* RFC 4884 (partial) support for ICMP extensions */
+        if (data_len < 128 || (data_len & 7) || skb->len < data_len)
+                data_len = 0;
+
+        skb2 = data_len ? skb_copy(skb, GFP_ATOMIC) : skb_clone(skb, GFP_ATOMIC);
+
+        if (!skb2)
+                return 1;
+
+        skb_dst_drop(skb2);
+        skb_pull(skb2, nhs);
+        skb_reset_network_header(skb2);
+
+        rt = rt6_lookup(dev_net(skb->dev), &ipv6_hdr(skb2)->saddr, NULL, 0, 0);
+
+        if (rt && rt->dst.dev)
+                skb2->dev = rt->dst.dev;
+
+        ipv6_addr_set_v4mapped(ip_hdr(skb)->saddr, &temp_saddr);
+
+        if (data_len) {
+                /* RFC 4884 (partial) support :
+                 * insert 0 padding at the end, before the extensions
+                 */
+                __skb_push(skb2, nhs);
+                skb_reset_network_header(skb2);
+                memmove(skb2->data, skb2->data + nhs, data_len - nhs);
+                memset(skb2->data + data_len - nhs, 0, nhs);
+                /* RFC 4884 4.5 : Length is measured in 64-bit words,
+                 * and stored in reserved[0]
+                 */
+                info = (data_len/8) << 24;
+        }
+        if (type == ICMP_TIME_EXCEEDED)
+                icmp6_send(skb2, ICMPV6_TIME_EXCEED, ICMPV6_EXC_HOPLIMIT,
+                           info, &temp_saddr);
+        else
+                icmp6_send(skb2, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH,
+                           info, &temp_saddr);
+        if (rt)
+                ip6_rt_put(rt);
+
+        kfree_skb(skb2);
+
+        return 0;
+}
+EXPORT_SYMBOL(ip6_err_gen_icmpv6_unreach);
+
 static void icmpv6_echo_reply(struct sk_buff *skb)
 {
         struct net *net = dev_net(skb->dev);
@@ -585,7 +655,7 @@ static void icmpv6_echo_reply(struct sk_buff *skb)
         fl6.daddr = ipv6_hdr(skb)->saddr;
         if (saddr)
                 fl6.saddr = *saddr;
-        fl6.flowi6_oif = l3mdev_fib_oif(skb->dev);
+        fl6.flowi6_oif = skb->dev->ifindex;
         fl6.fl6_icmp_type = ICMPV6_ECHO_REPLY;
         fl6.flowi6_mark = mark;
         security_skb_classify_flow(skb, flowi6_to_flowi(&fl6));
diff --git a/net/ipv6/ila/ila.h b/net/ipv6/ila/ila.h
index d08fd2d48a78..e0170f62bc39 100644
--- a/net/ipv6/ila/ila.h
+++ b/net/ipv6/ila/ila.h
@@ -109,7 +109,8 @@ static inline bool ila_csum_neutral_set(struct ila_identifier ident)
         return !!(ident.csum_neutral);
 }
 
-void ila_update_ipv6_locator(struct sk_buff *skb, struct ila_params *p);
+void ila_update_ipv6_locator(struct sk_buff *skb, struct ila_params *p,
+                             bool set_csum_neutral);
 
 void ila_init_saved_csum(struct ila_params *p);
 
diff --git a/net/ipv6/ila/ila_common.c b/net/ipv6/ila/ila_common.c
index 0e94042d1289..ec9efbcdad35 100644
--- a/net/ipv6/ila/ila_common.c
+++ b/net/ipv6/ila/ila_common.c
@@ -34,12 +34,12 @@ static void ila_csum_do_neutral(struct ila_addr *iaddr,
         if (p->locator_match.v64) {
                 diff = p->csum_diff;
         } else {
-                diff = compute_csum_diff8((__be32 *)iaddr,
-                                          (__be32 *)&p->locator);
+                diff = compute_csum_diff8((__be32 *)&p->locator,
+                                          (__be32 *)iaddr);
         }
 
         fval = (__force __wsum)(ila_csum_neutral_set(iaddr->ident) ?
-                        ~CSUM_NEUTRAL_FLAG : CSUM_NEUTRAL_FLAG);
+                        CSUM_NEUTRAL_FLAG : ~CSUM_NEUTRAL_FLAG);
 
         diff = csum_add(diff, fval);
 
@@ -103,7 +103,8 @@ static void ila_csum_adjust_transport(struct sk_buff *skb,
         iaddr->loc = p->locator;
 }
 
-void ila_update_ipv6_locator(struct sk_buff *skb, struct ila_params *p)
+void ila_update_ipv6_locator(struct sk_buff *skb, struct ila_params *p,
+                             bool set_csum_neutral)
 {
         struct ipv6hdr *ip6h = ipv6_hdr(skb);
         struct ila_addr *iaddr = ila_a2i(&ip6h->daddr);
@@ -114,7 +115,8 @@ void ila_update_ipv6_locator(struct sk_buff *skb, struct ila_params *p)
                  * is a locator being translated to a SIR address.
                  * Perform (receiver) checksum-neutral translation.
                  */
-                ila_csum_do_neutral(iaddr, p);
+                if (!set_csum_neutral)
+                        ila_csum_do_neutral(iaddr, p);
         } else {
                 switch (p->csum_mode) {
                 case ILA_CSUM_ADJUST_TRANSPORT:
@@ -138,8 +140,8 @@ void ila_init_saved_csum(struct ila_params *p)
                 return;
 
         p->csum_diff = compute_csum_diff8(
-                                (__be32 *)&p->locator_match,
-                                (__be32 *)&p->locator);
+                                (__be32 *)&p->locator,
+                                (__be32 *)&p->locator_match);
 }
 
 static int __init ila_init(void)
diff --git a/net/ipv6/ila/ila_lwt.c b/net/ipv6/ila/ila_lwt.c
index 1dfb64166d7d..c8314c6b6154 100644
--- a/net/ipv6/ila/ila_lwt.c
+++ b/net/ipv6/ila/ila_lwt.c
@@ -26,7 +26,7 @@ static int ila_output(struct net *net, struct sock *sk, struct sk_buff *skb)
         if (skb->protocol != htons(ETH_P_IPV6))
                 goto drop;
 
-        ila_update_ipv6_locator(skb, ila_params_lwtunnel(dst->lwtstate));
+        ila_update_ipv6_locator(skb, ila_params_lwtunnel(dst->lwtstate), true);
 
         return dst->lwtstate->orig_output(net, sk, skb);
 
@@ -42,7 +42,7 @@ static int ila_input(struct sk_buff *skb)
         if (skb->protocol != htons(ETH_P_IPV6))
                 goto drop;
 
-        ila_update_ipv6_locator(skb, ila_params_lwtunnel(dst->lwtstate));
+        ila_update_ipv6_locator(skb, ila_params_lwtunnel(dst->lwtstate), false);
 
         return dst->lwtstate->orig_input(skb);
 
diff --git a/net/ipv6/ila/ila_xlat.c b/net/ipv6/ila/ila_xlat.c
index a90e57229c6c..e6eca5fdf4c9 100644
--- a/net/ipv6/ila/ila_xlat.c
+++ b/net/ipv6/ila/ila_xlat.c
@@ -210,14 +210,14 @@ static void ila_free_cb(void *ptr, void *arg)
         }
 }
 
-static int ila_xlat_addr(struct sk_buff *skb);
+static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral);
 
 static unsigned int
 ila_nf_input(void *priv,
              struct sk_buff *skb,
              const struct nf_hook_state *state)
 {
-        ila_xlat_addr(skb);
+        ila_xlat_addr(skb, false);
         return NF_ACCEPT;
 }
 
@@ -597,7 +597,7 @@ static struct pernet_operations ila_net_ops = {
         .size = sizeof(struct ila_net),
 };
 
-static int ila_xlat_addr(struct sk_buff *skb)
+static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral)
 {
         struct ila_map *ila;
         struct ipv6hdr *ip6h = ipv6_hdr(skb);
@@ -616,7 +616,7 @@ static int ila_xlat_addr(struct sk_buff *skb)
 
         ila = ila_lookup_wildcards(iaddr, skb->dev->ifindex, ilan);
         if (ila)
-                ila_update_ipv6_locator(skb, &ila->xp.ip);
+                ila_update_ipv6_locator(skb, &ila->xp.ip, set_csum_neutral);
 
         rcu_read_unlock();
 
diff --git a/net/ipv6/ip6_icmp.c b/net/ipv6/ip6_icmp.c
index 14dacc544c3e..713676f14a0e 100644
--- a/net/ipv6/ip6_icmp.c
+++ b/net/ipv6/ip6_icmp.c
@@ -39,7 +39,7 @@ void icmpv6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info)
 
         if (!send)
                 goto out;
-        send(skb, type, code, info);
+        send(skb, type, code, info, NULL);
 out:
         rcu_read_unlock();
 }
diff --git a/net/ipv6/ip6_input.c b/net/ipv6/ip6_input.c
index 94611e450ec9..aacfb4bce153 100644
--- a/net/ipv6/ip6_input.c
+++ b/net/ipv6/ip6_input.c
@@ -323,6 +323,7 @@ int ip6_input(struct sk_buff *skb)
                        dev_net(skb->dev), NULL, skb, skb->dev, NULL,
                        ip6_input_finish);
 }
+EXPORT_SYMBOL_GPL(ip6_input);
 
 int ip6_mc_input(struct sk_buff *skb)
 {
diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c
index 635b8d340cdb..1dfc402d9ad1 100644
--- a/net/ipv6/ip6_output.c
+++ b/net/ipv6/ip6_output.c
@@ -368,7 +368,7 @@ static bool ip6_pkt_too_big(const struct sk_buff *skb, unsigned int mtu)
         if (skb->ignore_df)
                 return false;
 
-        if (skb_is_gso(skb) && skb_gso_network_seglen(skb) <= mtu)
+        if (skb_is_gso(skb) && skb_gso_validate_mtu(skb, mtu))
                 return false;
 
         return true;
@@ -910,6 +910,13 @@ static int ip6_dst_lookup_tail(struct net *net, const struct sock *sk,
         int err;
         int flags = 0;
 
+        if (ipv6_addr_any(&fl6->saddr) && fl6->flowi6_oif &&
+            (!*dst || !(*dst)->error)) {
+                err = l3mdev_get_saddr6(net, sk, fl6);
+                if (err)
+                        goto out_err;
+        }
+
         /* The correct way to handle this would be to do
          * ip6_route_get_saddr, and then ip6_route_output; however,
          * the route-specific preferred source forces the
@@ -999,10 +1006,11 @@ static int ip6_dst_lookup_tail(struct net *net, const struct sock *sk,
         return 0;
 
 out_err_release:
-        if (err == -ENETUNREACH)
-                IP6_INC_STATS(net, NULL, IPSTATS_MIB_OUTNOROUTES);
         dst_release(*dst);
         *dst = NULL;
+out_err:
+        if (err == -ENETUNREACH)
+                IP6_INC_STATS(net, NULL, IPSTATS_MIB_OUTNOROUTES);
         return err;
 }
 
diff --git a/net/ipv6/ndisc.c b/net/ipv6/ndisc.c
index c245895a3d41..fe65cdc28a45 100644
--- a/net/ipv6/ndisc.c
+++ b/net/ipv6/ndisc.c
@@ -73,15 +73,6 @@
 #include <linux/netfilter.h>
 #include <linux/netfilter_ipv6.h>
 
-/* Set to 3 to get tracing... */
-#define ND_DEBUG 1
-
-#define ND_PRINTK(val, level, fmt, ...)                         \
-do {                                                            \
-        if (val <= ND_DEBUG)                                    \
-                net_##level##_ratelimited(fmt, ##__VA_ARGS__);  \
-} while (0)
-
 static u32 ndisc_hash(const void *pkey,
                       const struct net_device *dev,
                       __u32 *hash_rnd);
@@ -150,11 +141,10 @@ struct neigh_table nd_tbl = {
 };
 EXPORT_SYMBOL_GPL(nd_tbl);
 
-static void ndisc_fill_addr_option(struct sk_buff *skb, int type, void *data)
+void __ndisc_fill_addr_option(struct sk_buff *skb, int type, void *data,
+                              int data_len, int pad)
 {
-        int pad   = ndisc_addr_option_pad(skb->dev->type);
-        int data_len = skb->dev->addr_len;
-        int space = ndisc_opt_addr_space(skb->dev);
+        int space = __ndisc_opt_addr_space(data_len, pad);
         u8 *opt = skb_put(skb, space);
 
         opt[0] = type;
@@ -171,6 +161,23 @@ static void ndisc_fill_addr_option(struct sk_buff *skb, int type, void *data)
         if (space > 0)
                 memset(opt, 0, space);
 }
+EXPORT_SYMBOL_GPL(__ndisc_fill_addr_option);
+
+static inline void ndisc_fill_addr_option(struct sk_buff *skb, int type,
+                                          void *data, u8 icmp6_type)
+{
+        __ndisc_fill_addr_option(skb, type, data, skb->dev->addr_len,
+                                 ndisc_addr_option_pad(skb->dev->type));
+        ndisc_ops_fill_addr_option(skb->dev, skb, icmp6_type);
+}
+
+static inline void ndisc_fill_redirect_addr_option(struct sk_buff *skb,
+                                                   void *ha,
+                                                   const u8 *ops_data)
+{
+        ndisc_fill_addr_option(skb, ND_OPT_TARGET_LL_ADDR, ha, NDISC_REDIRECT);
+        ndisc_ops_fill_redirect_addr_option(skb->dev, skb, ops_data);
+}
 
 static struct nd_opt_hdr *ndisc_next_option(struct nd_opt_hdr *cur,
                                             struct nd_opt_hdr *end)
@@ -185,24 +192,28 @@ static struct nd_opt_hdr *ndisc_next_option(struct nd_opt_hdr *cur,
         return cur <= end && cur->nd_opt_type == type ? cur : NULL;
 }
 
-static inline int ndisc_is_useropt(struct nd_opt_hdr *opt)
+static inline int ndisc_is_useropt(const struct net_device *dev,
+                                   struct nd_opt_hdr *opt)
 {
         return opt->nd_opt_type == ND_OPT_RDNSS ||
-                opt->nd_opt_type == ND_OPT_DNSSL;
+                opt->nd_opt_type == ND_OPT_DNSSL ||
+                ndisc_ops_is_useropt(dev, opt->nd_opt_type);
 }
 
-static struct nd_opt_hdr *ndisc_next_useropt(struct nd_opt_hdr *cur,
+static struct nd_opt_hdr *ndisc_next_useropt(const struct net_device *dev,
+                                             struct nd_opt_hdr *cur,
                                              struct nd_opt_hdr *end)
 {
         if (!cur || !end || cur >= end)
                 return NULL;
         do {
                 cur = ((void *)cur) + (cur->nd_opt_len << 3);
-        } while (cur < end && !ndisc_is_useropt(cur));
-        return cur <= end && ndisc_is_useropt(cur) ? cur : NULL;
+        } while (cur < end && !ndisc_is_useropt(dev, cur));
+        return cur <= end && ndisc_is_useropt(dev, cur) ? cur : NULL;
 }
 
-struct ndisc_options *ndisc_parse_options(u8 *opt, int opt_len,
+struct ndisc_options *ndisc_parse_options(const struct net_device *dev,
+                                          u8 *opt, int opt_len,
                                           struct ndisc_options *ndopts)
 {
         struct nd_opt_hdr *nd_opt = (struct nd_opt_hdr *)opt;
@@ -217,6 +228,8 @@ struct ndisc_options *ndisc_parse_options(u8 *opt, int opt_len,
                 l = nd_opt->nd_opt_len << 3;
                 if (opt_len < l || l == 0)
                         return NULL;
+                if (ndisc_ops_parse_options(dev, nd_opt, ndopts))
+                        goto next_opt;
                 switch (nd_opt->nd_opt_type) {
                 case ND_OPT_SOURCE_LL_ADDR:
                 case ND_OPT_TARGET_LL_ADDR:
@@ -243,7 +256,7 @@ struct ndisc_options *ndisc_parse_options(u8 *opt, int opt_len,
                         break;
 #endif
                 default:
-                        if (ndisc_is_useropt(nd_opt)) {
+                        if (ndisc_is_useropt(dev, nd_opt)) {
                                 ndopts->nd_useropts_end = nd_opt;
                                 if (!ndopts->nd_useropts)
                                         ndopts->nd_useropts = nd_opt;
@@ -260,6 +273,7 @@ struct ndisc_options *ndisc_parse_options(u8 *opt, int opt_len,
                                           nd_opt->nd_opt_len);
                         }
                 }
+next_opt:
                 opt_len -= l;
                 nd_opt = ((void *)nd_opt) + l;
         }
@@ -509,7 +523,8 @@ void ndisc_send_na(struct net_device *dev, const struct in6_addr *daddr,
         if (!dev->addr_len)
                 inc_opt = 0;
         if (inc_opt)
-                optlen += ndisc_opt_addr_space(dev);
+                optlen += ndisc_opt_addr_space(dev,
+                                               NDISC_NEIGHBOUR_ADVERTISEMENT);
 
         skb = ndisc_alloc_skb(dev, sizeof(*msg) + optlen);
         if (!skb)
@@ -528,8 +543,8 @@ void ndisc_send_na(struct net_device *dev, const struct in6_addr *daddr,
 
         if (inc_opt)
                 ndisc_fill_addr_option(skb, ND_OPT_TARGET_LL_ADDR,
-                                       dev->dev_addr);
-
+                                       dev->dev_addr,
+                                       NDISC_NEIGHBOUR_ADVERTISEMENT);
 
         ndisc_send_skb(skb, daddr, src_addr);
 }
@@ -574,7 +589,8 @@ void ndisc_send_ns(struct net_device *dev, const struct in6_addr *solicit,
         if (ipv6_addr_any(saddr))
                 inc_opt = false;
         if (inc_opt)
-                optlen += ndisc_opt_addr_space(dev);
+                optlen += ndisc_opt_addr_space(dev,
+                                               NDISC_NEIGHBOUR_SOLICITATION);
 
         skb = ndisc_alloc_skb(dev, sizeof(*msg) + optlen);
         if (!skb)
@@ -590,7 +606,8 @@ void ndisc_send_ns(struct net_device *dev, const struct in6_addr *solicit,
 
         if (inc_opt)
                 ndisc_fill_addr_option(skb, ND_OPT_SOURCE_LL_ADDR,
-                                       dev->dev_addr);
+                                       dev->dev_addr,
+                                       NDISC_NEIGHBOUR_SOLICITATION);
 
         ndisc_send_skb(skb, daddr, saddr);
 }
@@ -626,7 +643,7 @@ void ndisc_send_rs(struct net_device *dev, const struct in6_addr *saddr,
         }
 #endif
         if (send_sllao)
-                optlen += ndisc_opt_addr_space(dev);
+                optlen += ndisc_opt_addr_space(dev, NDISC_ROUTER_SOLICITATION);
 
         skb = ndisc_alloc_skb(dev, sizeof(*msg) + optlen);
         if (!skb)
@@ -641,7 +658,8 @@ void ndisc_send_rs(struct net_device *dev, const struct in6_addr *saddr,
 
         if (send_sllao)
                 ndisc_fill_addr_option(skb, ND_OPT_SOURCE_LL_ADDR,
-                                       dev->dev_addr);
+                                       dev->dev_addr,
+                                       NDISC_ROUTER_SOLICITATION);
 
         ndisc_send_skb(skb, daddr, saddr);
 }
@@ -702,6 +720,15 @@ static int pndisc_is_router(const void *pkey,
         return ret;
 }
 
+void ndisc_update(const struct net_device *dev, struct neighbour *neigh,
+                  const u8 *lladdr, u8 new, u32 flags, u8 icmp6_type,
+                  struct ndisc_options *ndopts)
+{
+        neigh_update(neigh, lladdr, new, flags);
+        /* report ndisc ops about neighbour update */
+        ndisc_ops_update(dev, neigh, flags, icmp6_type, ndopts);
+}
+
 static void ndisc_recv_ns(struct sk_buff *skb)
 {
         struct nd_msg *msg = (struct nd_msg *)skb_transport_header(skb);
@@ -738,7 +765,7 @@ static void ndisc_recv_ns(struct sk_buff *skb)
                 return;
         }
 
-        if (!ndisc_parse_options(msg->opt, ndoptlen, &ndopts)) {
+        if (!ndisc_parse_options(dev, msg->opt, ndoptlen, &ndopts)) {
                 ND_PRINTK(2, warn, "NS: invalid ND options\n");
                 return;
         }
@@ -856,9 +883,10 @@ have_ifp:
         neigh = __neigh_lookup(&nd_tbl, saddr, dev,
                                !inc || lladdr || !dev->addr_len);
         if (neigh)
-                neigh_update(neigh, lladdr, NUD_STALE,
+                ndisc_update(dev, neigh, lladdr, NUD_STALE,
                              NEIGH_UPDATE_F_WEAK_OVERRIDE|
-                             NEIGH_UPDATE_F_OVERRIDE);
+                             NEIGH_UPDATE_F_OVERRIDE,
+                             NDISC_NEIGHBOUR_SOLICITATION, &ndopts);
         if (neigh || !dev->header_ops) {
                 ndisc_send_na(dev, saddr, &msg->target, !!is_router,
                               true, (ifp != NULL && inc), inc);
@@ -911,7 +939,7 @@ static void ndisc_recv_na(struct sk_buff *skb)
             idev->cnf.drop_unsolicited_na)
                 return;
 
-        if (!ndisc_parse_options(msg->opt, ndoptlen, &ndopts)) {
+        if (!ndisc_parse_options(dev, msg->opt, ndoptlen, &ndopts)) {
                 ND_PRINTK(2, warn, "NS: invalid ND option\n");
                 return;
         }
@@ -967,12 +995,13 @@ static void ndisc_recv_na(struct sk_buff *skb)
                         goto out;
                 }
 
-                neigh_update(neigh, lladdr,
+                ndisc_update(dev, neigh, lladdr,
                              msg->icmph.icmp6_solicited ? NUD_REACHABLE : NUD_STALE,
                              NEIGH_UPDATE_F_WEAK_OVERRIDE|
                              (msg->icmph.icmp6_override ? NEIGH_UPDATE_F_OVERRIDE : 0)|
                              NEIGH_UPDATE_F_OVERRIDE_ISROUTER|
-                             (msg->icmph.icmp6_router ? NEIGH_UPDATE_F_ISROUTER : 0));
+                             (msg->icmph.icmp6_router ? NEIGH_UPDATE_F_ISROUTER : 0),
+                             NDISC_NEIGHBOUR_ADVERTISEMENT, &ndopts);
 
                 if ((old_flags & ~neigh->flags) & NTF_ROUTER) {
                         /*
@@ -1017,7 +1046,7 @@ static void ndisc_recv_rs(struct sk_buff *skb)
                 goto out;
 
         /* Parse ND options */
-        if (!ndisc_parse_options(rs_msg->opt, ndoptlen, &ndopts)) {
+        if (!ndisc_parse_options(skb->dev, rs_msg->opt, ndoptlen, &ndopts)) {
                 ND_PRINTK(2, notice, "NS: invalid ND option, ignored\n");
                 goto out;
         }
@@ -1031,10 +1060,11 @@ static void ndisc_recv_rs(struct sk_buff *skb)
 
         neigh = __neigh_lookup(&nd_tbl, saddr, skb->dev, 1);
         if (neigh) {
-                neigh_update(neigh, lladdr, NUD_STALE,
+                ndisc_update(skb->dev, neigh, lladdr, NUD_STALE,
                              NEIGH_UPDATE_F_WEAK_OVERRIDE|
                              NEIGH_UPDATE_F_OVERRIDE|
-                             NEIGH_UPDATE_F_OVERRIDE_ISROUTER);
+                             NEIGH_UPDATE_F_OVERRIDE_ISROUTER,
+                             NDISC_ROUTER_SOLICITATION, &ndopts);
                 neigh_release(neigh);
         }
 out:
@@ -1135,7 +1165,7 @@ static void ndisc_router_discovery(struct sk_buff *skb)
                 return;
         }
 
-        if (!ndisc_parse_options(opt, optlen, &ndopts)) {
+        if (!ndisc_parse_options(skb->dev, opt, optlen, &ndopts)) {
                 ND_PRINTK(2, warn, "RA: invalid ND options\n");
                 return;
         }
@@ -1329,11 +1359,12 @@ skip_linkparms:
                                 goto out;
                         }
                 }
-                neigh_update(neigh, lladdr, NUD_STALE,
+                ndisc_update(skb->dev, neigh, lladdr, NUD_STALE,
                              NEIGH_UPDATE_F_WEAK_OVERRIDE|
                              NEIGH_UPDATE_F_OVERRIDE|
                              NEIGH_UPDATE_F_OVERRIDE_ISROUTER|
-                             NEIGH_UPDATE_F_ISROUTER);
+                             NEIGH_UPDATE_F_ISROUTER,
+                             NDISC_ROUTER_ADVERTISEMENT, &ndopts);
         }
 
         if (!ipv6_accept_ra(in6_dev)) {
@@ -1421,7 +1452,8 @@ skip_routeinfo:
                 struct nd_opt_hdr *p;
                 for (p = ndopts.nd_useropts;
                      p;
-                     p = ndisc_next_useropt(p, ndopts.nd_useropts_end)) {
+                     p = ndisc_next_useropt(skb->dev, p,
+                                            ndopts.nd_useropts_end)) {
                         ndisc_ra_useropt(skb, p);
                 }
         }
@@ -1459,7 +1491,7 @@ static void ndisc_redirect_rcv(struct sk_buff *skb)
                 return;
         }
 
-        if (!ndisc_parse_options(msg->opt, ndoptlen, &ndopts))
+        if (!ndisc_parse_options(skb->dev, msg->opt, ndoptlen, &ndopts))
                 return;
 
         if (!ndopts.nd_opts_rh) {
@@ -1504,7 +1536,8 @@ void ndisc_send_redirect(struct sk_buff *skb, const struct in6_addr *target)
         struct dst_entry *dst;
         struct flowi6 fl6;
         int rd_len;
-        u8 ha_buf[MAX_ADDR_LEN], *ha = NULL;
+        u8 ha_buf[MAX_ADDR_LEN], *ha = NULL,
+           ops_data_buf[NDISC_OPS_REDIRECT_DATA_SPACE], *ops_data = NULL;
         int oif = l3mdev_fib_oif(dev);
         bool ret;
 
@@ -1563,7 +1596,9 @@ void ndisc_send_redirect(struct sk_buff *skb, const struct in6_addr *target)
                         memcpy(ha_buf, neigh->ha, dev->addr_len);
                         read_unlock_bh(&neigh->lock);
                         ha = ha_buf;
-                        optlen += ndisc_opt_addr_space(dev);
+                        optlen += ndisc_redirect_opt_addr_space(dev, neigh,
+                                                                ops_data_buf,
+                                                                &ops_data);
                 } else
                         read_unlock_bh(&neigh->lock);
 
@@ -1594,7 +1629,7 @@ void ndisc_send_redirect(struct sk_buff *skb, const struct in6_addr *target)
          */
 
         if (ha)
-                ndisc_fill_addr_option(buff, ND_OPT_TARGET_LL_ADDR, ha);
+                ndisc_fill_redirect_addr_option(buff, ha, ops_data);
 
         /*
          *      build redirect option and copy skb over to the new packet.
diff --git a/net/ipv6/ping.c b/net/ipv6/ping.c
index 3ee3e444a66b..fed40d1ec29b 100644
--- a/net/ipv6/ping.c
+++ b/net/ipv6/ping.c
@@ -116,6 +116,9 @@ static int ping_v6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
         else if (!fl6.flowi6_oif)
                 fl6.flowi6_oif = np->ucast_oif;
 
+        ipc6.tclass = np->tclass;
+        fl6.flowlabel = ip6_make_flowinfo(ipc6.tclass, fl6.flowlabel);
+
         dst = ip6_sk_dst_lookup_flow(sk, &fl6,  daddr);
         if (IS_ERR(dst))
                 return PTR_ERR(dst);
@@ -140,7 +143,6 @@ static int ping_v6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
         pfh.family = AF_INET6;
 
         ipc6.hlimit = ip6_sk_dst_hoplimit(np, &fl6, dst);
-        ipc6.tclass = np->tclass;
         ipc6.dontfrag = np->dontfrag;
         ipc6.opt = NULL;
 
diff --git a/net/ipv6/raw.c b/net/ipv6/raw.c
index 896350df6423..590dd1f7746f 100644
--- a/net/ipv6/raw.c
+++ b/net/ipv6/raw.c
@@ -878,6 +878,11 @@ static int rawv6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
         if (inet->hdrincl)
                 fl6.flowi6_flags |= FLOWI_FLAG_KNOWN_NH;
 
+        if (ipc6.tclass < 0)
+                ipc6.tclass = np->tclass;
+
+        fl6.flowlabel = ip6_make_flowinfo(ipc6.tclass, fl6.flowlabel);
+
         dst = ip6_dst_lookup_flow(sk, &fl6, final_p);
         if (IS_ERR(dst)) {
                 err = PTR_ERR(dst);
@@ -886,9 +891,6 @@ static int rawv6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
         if (ipc6.hlimit < 0)
                 ipc6.hlimit = ip6_sk_dst_hoplimit(np, &fl6, dst);
 
-        if (ipc6.tclass < 0)
-                ipc6.tclass = np->tclass;
-
         if (ipc6.dontfrag < 0)
                 ipc6.dontfrag = np->dontfrag;
 
diff --git a/net/ipv6/route.c b/net/ipv6/route.c
index 520b7884d0c2..49817555449e 100644
--- a/net/ipv6/route.c
+++ b/net/ipv6/route.c
@@ -1042,8 +1042,8 @@ static struct rt6_info *rt6_make_pcpu_route(struct rt6_info *rt)
         return pcpu_rt;
 }
 
-static struct rt6_info *ip6_pol_route(struct net *net, struct fib6_table *table, int oif,
-                                      struct flowi6 *fl6, int flags)
+struct rt6_info *ip6_pol_route(struct net *net, struct fib6_table *table,
+                               int oif, struct flowi6 *fl6, int flags)
 {
         struct fib6_node *fn, *saved_fn;
         struct rt6_info *rt;
@@ -1139,6 +1139,7 @@ redo_rt6_select:
 
         }
 }
+EXPORT_SYMBOL_GPL(ip6_pol_route);
 
 static struct rt6_info *ip6_pol_route_input(struct net *net, struct fib6_table *table,
                                             struct flowi6 *fl6, int flags)
@@ -2200,7 +2201,7 @@ static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_bu
          *      first-hop router for the specified ICMP Destination Address.
          */
 
-        if (!ndisc_parse_options(msg->opt, optlen, &ndopts)) {
+        if (!ndisc_parse_options(skb->dev, msg->opt, optlen, &ndopts)) {
                 net_dbg_ratelimited("rt6_redirect: invalid ND options\n");
                 return;
         }
@@ -2235,12 +2236,12 @@ static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_bu
          *      We have finally decided to accept it.
          */
 
-        neigh_update(neigh, lladdr, NUD_STALE,
+        ndisc_update(skb->dev, neigh, lladdr, NUD_STALE,
                      NEIGH_UPDATE_F_WEAK_OVERRIDE|
                      NEIGH_UPDATE_F_OVERRIDE|
                      (on_link ? 0 : (NEIGH_UPDATE_F_OVERRIDE_ISROUTER|
-                                     NEIGH_UPDATE_F_ISROUTER))
-                     );
+                                     NEIGH_UPDATE_F_ISROUTER)),
+                     NDISC_REDIRECT, &ndopts);
 
         nrt = ip6_rt_cache_alloc(rt, &msg->dest, NULL);
         if (!nrt)
@@ -2585,23 +2586,6 @@ struct rt6_info *addrconf_dst_alloc(struct inet6_dev *idev,
         return rt;
 }
 
-int ip6_route_get_saddr(struct net *net,
-                        struct rt6_info *rt,
-                        const struct in6_addr *daddr,
-                        unsigned int prefs,
-                        struct in6_addr *saddr)
-{
-        struct inet6_dev *idev =
-                rt ? ip6_dst_idev((struct dst_entry *)rt) : NULL;
-        int err = 0;
-        if (rt && rt->rt6i_prefsrc.plen)
-                *saddr = rt->rt6i_prefsrc.addr;
-        else
-                err = ipv6_dev_get_saddr(net, idev ? idev->dev : NULL,
-                                         daddr, prefs, saddr);
-        return err;
-}
-
 /* remove deleted ip from prefsrc entries */
 struct arg_dev_net_ip {
         struct net_device *dev;
@@ -3306,6 +3290,8 @@ static int inet6_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh)
 
         err = -EINVAL;
         memset(&fl6, 0, sizeof(fl6));
+        rtm = nlmsg_data(nlh);
+        fl6.flowlabel = ip6_make_flowinfo(rtm->rtm_tos, 0);
 
         if (tb[RTA_SRC]) {
                 if (nla_len(tb[RTA_SRC]) < sizeof(struct in6_addr))
diff --git a/net/ipv6/sit.c b/net/ipv6/sit.c
index 0619ac70836d..917a5cd4b8fc 100644
--- a/net/ipv6/sit.c
+++ b/net/ipv6/sit.c
@@ -479,47 +479,12 @@ static void ipip6_tunnel_uninit(struct net_device *dev)
         dev_put(dev);
 }
 
-/* Generate icmpv6 with type/code ICMPV6_DEST_UNREACH/ICMPV6_ADDR_UNREACH
- * if sufficient data bytes are available
- */
-static int ipip6_err_gen_icmpv6_unreach(struct sk_buff *skb)
-{
-        int ihl = ((const struct iphdr *)skb->data)->ihl*4;
-        struct rt6_info *rt;
-        struct sk_buff *skb2;
-
-        if (!pskb_may_pull(skb, ihl + sizeof(struct ipv6hdr) + 8))
-                return 1;
-
-        skb2 = skb_clone(skb, GFP_ATOMIC);
-
-        if (!skb2)
-                return 1;
-
-        skb_dst_drop(skb2);
-        skb_pull(skb2, ihl);
-        skb_reset_network_header(skb2);
-
-        rt = rt6_lookup(dev_net(skb->dev), &ipv6_hdr(skb2)->saddr, NULL, 0, 0);
-
-        if (rt && rt->dst.dev)
-                skb2->dev = rt->dst.dev;
-
-        icmpv6_send(skb2, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0);
-
-        if (rt)
-                ip6_rt_put(rt);
-
-        kfree_skb(skb2);
-
-        return 0;
-}
-
 static int ipip6_err(struct sk_buff *skb, u32 info)
 {
         const struct iphdr *iph = (const struct iphdr *)skb->data;
         const int type = icmp_hdr(skb)->type;
         const int code = icmp_hdr(skb)->code;
+        unsigned int data_len = 0;
         struct ip_tunnel *t;
         int err;
 
@@ -544,6 +509,7 @@ static int ipip6_err(struct sk_buff *skb, u32 info)
         case ICMP_TIME_EXCEEDED:
                 if (code != ICMP_EXC_TTL)
                         return 0;
+                data_len = icmp_hdr(skb)->un.reserved[1] * 4; /* RFC 4884 4.1 */
                 break;
         case ICMP_REDIRECT:
                 break;
@@ -571,11 +537,11 @@ static int ipip6_err(struct sk_buff *skb, u32 info)
                 goto out;
         }
 
-        if (t->parms.iph.daddr == 0)
+        err = 0;
+        if (!ip6_err_gen_icmpv6_unreach(skb, iph->ihl * 4, type, data_len))
                 goto out;
 
-        err = 0;
-        if (!ipip6_err_gen_icmpv6_unreach(skb))
+        if (t->parms.iph.daddr == 0)
                 goto out;
 
         if (t->parms.iph.ttl == 0 && type == ICMP_TIME_EXCEEDED)
@@ -825,9 +791,6 @@ static netdev_tx_t ipip6_tunnel_xmit(struct sk_buff *skb,
         u8 protocol = IPPROTO_IPV6;
         int t_hlen = tunnel->hlen + sizeof(struct iphdr);
 
-        if (skb->protocol != htons(ETH_P_IPV6))
-                goto tx_error;
-
         if (tos == 1)
                 tos = ipv6_get_dsfield(iph6);
 
diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
index 005dc82c2138..0a71a312d0d8 100644
--- a/net/ipv6/udp.c
+++ b/net/ipv6/udp.c
@@ -1207,6 +1207,11 @@ do_udp_sendmsg:
 
         security_sk_classify_flow(sk, flowi6_to_flowi(&fl6));
 
+        if (ipc6.tclass < 0)
+                ipc6.tclass = np->tclass;
+
+        fl6.flowlabel = ip6_make_flowinfo(ipc6.tclass, fl6.flowlabel);
+
         dst = ip6_sk_dst_lookup_flow(sk, &fl6, final_p);
         if (IS_ERR(dst)) {
                 err = PTR_ERR(dst);
@@ -1217,9 +1222,6 @@ do_udp_sendmsg:
         if (ipc6.hlimit < 0)
                 ipc6.hlimit = ip6_sk_dst_hoplimit(np, &fl6, dst);
 
-        if (ipc6.tclass < 0)
-                ipc6.tclass = np->tclass;
-
         if (msg->msg_flags&MSG_CONFIRM)
                 goto do_confirm;
 back_from_confirm:
diff --git a/net/ipv6/xfrm6_policy.c b/net/ipv6/xfrm6_policy.c
index c074771a10f7..6cc97003e4a9 100644
--- a/net/ipv6/xfrm6_policy.c
+++ b/net/ipv6/xfrm6_policy.c
@@ -366,12 +366,12 @@ static void __net_exit xfrm6_net_sysctl_exit(struct net *net)
                 kfree(table);
 }
 #else /* CONFIG_SYSCTL */
-static int inline xfrm6_net_sysctl_init(struct net *net)
+static inline int xfrm6_net_sysctl_init(struct net *net)
 {
         return 0;
 }
 
-static void inline xfrm6_net_sysctl_exit(struct net *net)
+static inline void xfrm6_net_sysctl_exit(struct net *net)
 {
 }
 #endif
diff --git a/net/iucv/af_iucv.c b/net/iucv/af_iucv.c
index fc3598a922b0..37d674e6f8a9 100644
--- a/net/iucv/af_iucv.c
+++ b/net/iucv/af_iucv.c
@@ -1033,6 +1033,7 @@ static int iucv_sock_sendmsg(struct socket *sock, struct msghdr *msg,
 {
         struct sock *sk = sock->sk;
         struct iucv_sock *iucv = iucv_sk(sk);
+        size_t headroom, linear;
         struct sk_buff *skb;
         struct iucv_message txmsg = {0};
         struct cmsghdr *cmsg;
@@ -1110,20 +1111,31 @@ static int iucv_sock_sendmsg(struct socket *sock, struct msghdr *msg,
          * this is fine for SOCK_SEQPACKET (unless we want to support
          * segmented records using the MSG_EOR flag), but
          * for SOCK_STREAM we might want to improve it in future */
-        if (iucv->transport == AF_IUCV_TRANS_HIPER)
-                skb = sock_alloc_send_skb(sk,
-                        len + sizeof(struct af_iucv_trans_hdr) + ETH_HLEN,
-                        noblock, &err);
-        else
-                skb = sock_alloc_send_skb(sk, len, noblock, &err);
+        headroom = (iucv->transport == AF_IUCV_TRANS_HIPER)
+                   ? sizeof(struct af_iucv_trans_hdr) + ETH_HLEN : 0;
+        if (headroom + len < PAGE_SIZE) {
+                linear = len;
+        } else {
+                /* In nonlinear "classic" iucv skb,
+                 * reserve space for iucv_array
+                 */
+                if (iucv->transport != AF_IUCV_TRANS_HIPER)
+                        headroom += sizeof(struct iucv_array) *
+                                    (MAX_SKB_FRAGS + 1);
+                linear = PAGE_SIZE - headroom;
+        }
+        skb = sock_alloc_send_pskb(sk, headroom + linear, len - linear,
+                                   noblock, &err, 0);
         if (!skb)
                 goto out;
-        if (iucv->transport == AF_IUCV_TRANS_HIPER)
-                skb_reserve(skb, sizeof(struct af_iucv_trans_hdr) + ETH_HLEN);
-        if (memcpy_from_msg(skb_put(skb, len), msg, len)) {
-                err = -EFAULT;
+        if (headroom)
+                skb_reserve(skb, headroom);
+        skb_put(skb, linear);
+        skb->len = len;
+        skb->data_len = len - linear;
+        err = skb_copy_datagram_from_iter(skb, 0, &msg->msg_iter, len);
+        if (err)
                 goto fail;
-        }
 
         /* wait if outstanding messages for iucv path has reached */
         timeo = sock_sndtimeo(sk, noblock);
@@ -1148,49 +1160,67 @@ static int iucv_sock_sendmsg(struct socket *sock, struct msghdr *msg,
                         atomic_dec(&iucv->msg_sent);
                         goto fail;
                 }
-                goto release;
-        }
-        skb_queue_tail(&iucv->send_skb_q, skb);
-
-        if (((iucv->path->flags & IUCV_IPRMDATA) & iucv->flags)
-              && skb->len <= 7) {
-                err = iucv_send_iprm(iucv->path, &txmsg, skb);
+        } else { /* Classic VM IUCV transport */
+                skb_queue_tail(&iucv->send_skb_q, skb);
+
+                if (((iucv->path->flags & IUCV_IPRMDATA) & iucv->flags) &&
+                    skb->len <= 7) {
+                        err = iucv_send_iprm(iucv->path, &txmsg, skb);
+
+                        /* on success: there is no message_complete callback */
+                        /* for an IPRMDATA msg; remove skb from send queue   */
+                        if (err == 0) {
+                                skb_unlink(skb, &iucv->send_skb_q);
+                                kfree_skb(skb);
+                        }
 
-                /* on success: there is no message_complete callback
-                 * for an IPRMDATA msg; remove skb from send queue */
-                if (err == 0) {
-                        skb_unlink(skb, &iucv->send_skb_q);
-                        kfree_skb(skb);
+                        /* this error should never happen since the     */
+                        /* IUCV_IPRMDATA path flag is set... sever path */
+                        if (err == 0x15) {
+                                pr_iucv->path_sever(iucv->path, NULL);
+                                skb_unlink(skb, &iucv->send_skb_q);
+                                err = -EPIPE;
+                                goto fail;
+                        }
+                } else if (skb_is_nonlinear(skb)) {
+                        struct iucv_array *iba = (struct iucv_array *)skb->head;
+                        int i;
+
+                        /* skip iucv_array lying in the headroom */
+                        iba[0].address = (u32)(addr_t)skb->data;
+                        iba[0].length = (u32)skb_headlen(skb);
+                        for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
+                                skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
+
+                                iba[i + 1].address =
+                                        (u32)(addr_t)skb_frag_address(frag);
+                                iba[i + 1].length = (u32)skb_frag_size(frag);
+                        }
+                        err = pr_iucv->message_send(iucv->path, &txmsg,
+                                                    IUCV_IPBUFLST, 0,
+                                                    (void *)iba, skb->len);
+                } else { /* non-IPRM Linear skb */
+                        err = pr_iucv->message_send(iucv->path, &txmsg,
+                                        0, 0, (void *)skb->data, skb->len);
                 }
-
-                /* this error should never happen since the
-                 * IUCV_IPRMDATA path flag is set... sever path */
-                if (err == 0x15) {
-                        pr_iucv->path_sever(iucv->path, NULL);
+                if (err) {
+                        if (err == 3) {
+                                user_id[8] = 0;
+                                memcpy(user_id, iucv->dst_user_id, 8);
+                                appl_id[8] = 0;
+                                memcpy(appl_id, iucv->dst_name, 8);
+                                pr_err(
+                "Application %s on z/VM guest %s exceeds message limit\n",
+                                        appl_id, user_id);
+                                err = -EAGAIN;
+                        } else {
+                                err = -EPIPE;
+                        }
                         skb_unlink(skb, &iucv->send_skb_q);
-                        err = -EPIPE;
                         goto fail;
                 }
-        } else
-                err = pr_iucv->message_send(iucv->path, &txmsg, 0, 0,
-                                        (void *) skb->data, skb->len);
-        if (err) {
-                if (err == 3) {
-                        user_id[8] = 0;
-                        memcpy(user_id, iucv->dst_user_id, 8);
-                        appl_id[8] = 0;
-                        memcpy(appl_id, iucv->dst_name, 8);
-                        pr_err("Application %s on z/VM guest %s"
-                                " exceeds message limit\n",
-                                appl_id, user_id);
-                        err = -EAGAIN;
-                } else
-                        err = -EPIPE;
-                skb_unlink(skb, &iucv->send_skb_q);
-                goto fail;
         }
 
-release:
         release_sock(sk);
         return len;
 
@@ -1201,42 +1231,32 @@ out:
         return err;
 }
 
-/* iucv_fragment_skb() - Fragment a single IUCV message into multiple skb's
- *
- * Locking: must be called with message_q.lock held
- */
-static int iucv_fragment_skb(struct sock *sk, struct sk_buff *skb, int len)
+static struct sk_buff *alloc_iucv_recv_skb(unsigned long len)
 {
-        int dataleft, size, copied = 0;
-        struct sk_buff *nskb;
-
-        dataleft = len;
-        while (dataleft) {
-                if (dataleft >= sk->sk_rcvbuf / 4)
-                        size = sk->sk_rcvbuf / 4;
-                else
-                        size = dataleft;
-
-                nskb = alloc_skb(size, GFP_ATOMIC | GFP_DMA);
-                if (!nskb)
-                        return -ENOMEM;
-
-                /* copy target class to control buffer of new skb */
-                IUCV_SKB_CB(nskb)->class = IUCV_SKB_CB(skb)->class;
-
-                /* copy data fragment */
-                memcpy(nskb->data, skb->data + copied, size);
-                copied += size;
-                dataleft -= size;
-
-                skb_reset_transport_header(nskb);
-                skb_reset_network_header(nskb);
-                nskb->len = size;
+        size_t headroom, linear;
+        struct sk_buff *skb;
+        int err;
 
-                skb_queue_tail(&iucv_sk(sk)->backlog_skb_q, nskb);
+        if (len < PAGE_SIZE) {
+                headroom = 0;
+                linear = len;
+        } else {
+                headroom = sizeof(struct iucv_array) * (MAX_SKB_FRAGS + 1);
+                linear = PAGE_SIZE - headroom;
+        }
+        skb = alloc_skb_with_frags(headroom + linear, len - linear,
+                                   0, &err, GFP_ATOMIC | GFP_DMA);
+        WARN_ONCE(!skb,
+                  "alloc of recv iucv skb len=%lu failed with errcode=%d\n",
+                  len, err);
+        if (skb) {
+                if (headroom)
+                        skb_reserve(skb, headroom);
+                skb_put(skb, linear);
+                skb->len = len;
+                skb->data_len = len - linear;
         }
-
-        return 0;
+        return skb;
 }
 
 /* iucv_process_message() - Receive a single outstanding IUCV message
@@ -1263,31 +1283,32 @@ static void iucv_process_message(struct sock *sk, struct sk_buff *skb,
                         skb->len = 0;
                 }
         } else {
-                rc = pr_iucv->message_receive(path, msg,
+                if (skb_is_nonlinear(skb)) {
+                        struct iucv_array *iba = (struct iucv_array *)skb->head;
+                        int i;
+
+                        iba[0].address = (u32)(addr_t)skb->data;
+                        iba[0].length = (u32)skb_headlen(skb);
+                        for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
+                                skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
+
+                                iba[i + 1].address =
+                                        (u32)(addr_t)skb_frag_address(frag);
+                                iba[i + 1].length = (u32)skb_frag_size(frag);
+                        }
+                        rc = pr_iucv->message_receive(path, msg,
+                                              IUCV_IPBUFLST,
+                                              (void *)iba, len, NULL);
+                } else {
+                        rc = pr_iucv->message_receive(path, msg,
                                               msg->flags & IUCV_IPRMDATA,
                                               skb->data, len, NULL);
+                }
                 if (rc) {
                         kfree_skb(skb);
                         return;
                 }
-                /* we need to fragment iucv messages for SOCK_STREAM only;
-                 * for SOCK_SEQPACKET, it is only relevant if we support
-                 * record segmentation using MSG_EOR (see also recvmsg()) */
-                if (sk->sk_type == SOCK_STREAM &&
-                    skb->truesize >= sk->sk_rcvbuf / 4) {
-                        rc = iucv_fragment_skb(sk, skb, len);
-                        kfree_skb(skb);
-                        skb = NULL;
-                        if (rc) {
-                                pr_iucv->path_sever(path, NULL);
-                                return;
-                        }
-                        skb = skb_dequeue(&iucv_sk(sk)->backlog_skb_q);
-                } else {
-                        skb_reset_transport_header(skb);
-                        skb_reset_network_header(skb);
-                        skb->len = len;
-                }
+                WARN_ON_ONCE(skb->len != len);
         }
 
         IUCV_SKB_CB(skb)->offset = 0;
@@ -1306,7 +1327,7 @@ static void iucv_process_message_q(struct sock *sk)
         struct sock_msg_q *p, *n;
 
         list_for_each_entry_safe(p, n, &iucv->message_q.list, list) {
-                skb = alloc_skb(iucv_msg_length(&p->msg), GFP_ATOMIC | GFP_DMA);
+                skb = alloc_iucv_recv_skb(iucv_msg_length(&p->msg));
                 if (!skb)
                         break;
                 iucv_process_message(sk, skb, p->path, &p->msg);
@@ -1801,7 +1822,7 @@ static void iucv_callback_rx(struct iucv_path *path, struct iucv_message *msg)
         if (len > sk->sk_rcvbuf)
                 goto save_message;
 
-        skb = alloc_skb(iucv_msg_length(msg), GFP_ATOMIC | GFP_DMA);
+        skb = alloc_iucv_recv_skb(iucv_msg_length(msg));
         if (!skb)
                 goto save_message;
 
diff --git a/net/l2tp/l2tp_eth.c b/net/l2tp/l2tp_eth.c
index e253c26f31ac..57fc5a46ce06 100644
--- a/net/l2tp/l2tp_eth.c
+++ b/net/l2tp/l2tp_eth.c
@@ -67,7 +67,6 @@ static inline struct l2tp_eth_net *l2tp_eth_pernet(struct net *net)
         return net_generic(net, l2tp_eth_net_id);
 }
 
-static struct lock_class_key l2tp_eth_tx_busylock;
 static int l2tp_eth_dev_init(struct net_device *dev)
 {
         struct l2tp_eth *priv = netdev_priv(dev);
@@ -75,7 +74,8 @@ static int l2tp_eth_dev_init(struct net_device *dev)
         priv->dev = dev;
         eth_hw_addr_random(dev);
         eth_broadcast_addr(dev->broadcast);
-        dev->qdisc_tx_busylock = &l2tp_eth_tx_busylock;
+        netdev_lockdep_set_classes(dev);
+
         return 0;
 }
 
diff --git a/net/l2tp/l2tp_ip6.c b/net/l2tp/l2tp_ip6.c
index 6c54e03fe9c1..ea2ae6664cc8 100644
--- a/net/l2tp/l2tp_ip6.c
+++ b/net/l2tp/l2tp_ip6.c
@@ -611,6 +611,11 @@ static int l2tp_ip6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
 
         security_sk_classify_flow(sk, flowi6_to_flowi(&fl6));
 
+        if (ipc6.tclass < 0)
+                ipc6.tclass = np->tclass;
+
+        fl6.flowlabel = ip6_make_flowinfo(ipc6.tclass, fl6.flowlabel);
+
         dst = ip6_dst_lookup_flow(sk, &fl6, final_p);
         if (IS_ERR(dst)) {
                 err = PTR_ERR(dst);
@@ -620,9 +625,6 @@ static int l2tp_ip6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
         if (ipc6.hlimit < 0)
                 ipc6.hlimit = ip6_sk_dst_hoplimit(np, &fl6, dst);
 
-        if (ipc6.tclass < 0)
-                ipc6.tclass = np->tclass;
-
         if (ipc6.dontfrag < 0)
                 ipc6.dontfrag = np->dontfrag;
 
diff --git a/net/l3mdev/l3mdev.c b/net/l3mdev/l3mdev.c
index 6651a78e100c..c4a1c3e84e12 100644
--- a/net/l3mdev/l3mdev.c
+++ b/net/l3mdev/l3mdev.c
@@ -10,6 +10,7 @@
  */
 
 #include <linux/netdevice.h>
+#include <net/fib_rules.h>
 #include <net/l3mdev.h>
 
 /**
@@ -107,7 +108,7 @@ EXPORT_SYMBOL_GPL(l3mdev_fib_table_by_index);
  */
 
 struct dst_entry *l3mdev_get_rt6_dst(struct net *net,
-                                     const struct flowi6 *fl6)
+                                     struct flowi6 *fl6)
 {
         struct dst_entry *dst = NULL;
         struct net_device *dev;
@@ -160,3 +161,64 @@ int l3mdev_get_saddr(struct net *net, int ifindex, struct flowi4 *fl4)
         return rc;
 }
 EXPORT_SYMBOL_GPL(l3mdev_get_saddr);
+
+int l3mdev_get_saddr6(struct net *net, const struct sock *sk,
+                      struct flowi6 *fl6)
+{
+        struct net_device *dev;
+        int rc = 0;
+
+        if (fl6->flowi6_oif) {
+                rcu_read_lock();
+
+                dev = dev_get_by_index_rcu(net, fl6->flowi6_oif);
+                if (dev && netif_is_l3_slave(dev))
+                        dev = netdev_master_upper_dev_get_rcu(dev);
+
+                if (dev && netif_is_l3_master(dev) &&
+                    dev->l3mdev_ops->l3mdev_get_saddr6)
+                        rc = dev->l3mdev_ops->l3mdev_get_saddr6(dev, sk, fl6);
+
+                rcu_read_unlock();
+        }
+
+        return rc;
+}
+EXPORT_SYMBOL_GPL(l3mdev_get_saddr6);
+
+/**
+ *      l3mdev_fib_rule_match - Determine if flowi references an
+ *                              L3 master device
+ *      @net: network namespace for device index lookup
+ *      @fl:  flow struct
+ */
+
+int l3mdev_fib_rule_match(struct net *net, struct flowi *fl,
+                          struct fib_lookup_arg *arg)
+{
+        struct net_device *dev;
+        int rc = 0;
+
+        rcu_read_lock();
+
+        dev = dev_get_by_index_rcu(net, fl->flowi_oif);
+        if (dev && netif_is_l3_master(dev) &&
+            dev->l3mdev_ops->l3mdev_fib_table) {
+                arg->table = dev->l3mdev_ops->l3mdev_fib_table(dev);
+                rc = 1;
+                goto out;
+        }
+
+        dev = dev_get_by_index_rcu(net, fl->flowi_iif);
+        if (dev && netif_is_l3_master(dev) &&
+            dev->l3mdev_ops->l3mdev_fib_table) {
+                arg->table = dev->l3mdev_ops->l3mdev_fib_table(dev);
+                rc = 1;
+                goto out;
+        }
+
+out:
+        rcu_read_unlock();
+
+        return rc;
+}
diff --git a/net/mac80211/agg-tx.c b/net/mac80211/agg-tx.c
index 42fa81031dfa..5650c46bf91a 100644
--- a/net/mac80211/agg-tx.c
+++ b/net/mac80211/agg-tx.c
@@ -194,17 +194,21 @@ static void
 ieee80211_agg_stop_txq(struct sta_info *sta, int tid)
 {
         struct ieee80211_txq *txq = sta->sta.txq[tid];
+        struct ieee80211_sub_if_data *sdata;
+        struct fq *fq;
         struct txq_info *txqi;
 
         if (!txq)
                 return;
 
         txqi = to_txq_info(txq);
+        sdata = vif_to_sdata(txq->vif);
+        fq = &sdata->local->fq;
 
         /* Lock here to protect against further seqno updates on dequeue */
-        spin_lock_bh(&txqi->queue.lock);
+        spin_lock_bh(&fq->lock);
         set_bit(IEEE80211_TXQ_STOP, &txqi->flags);
-        spin_unlock_bh(&txqi->queue.lock);
+        spin_unlock_bh(&fq->lock);
 }
 
 static void
diff --git a/net/mac80211/debugfs.c b/net/mac80211/debugfs.c
index b251b2f7f8dd..2906c1004e1a 100644
--- a/net/mac80211/debugfs.c
+++ b/net/mac80211/debugfs.c
@@ -10,6 +10,7 @@
 
 #include <linux/debugfs.h>
 #include <linux/rtnetlink.h>
+#include <linux/vmalloc.h>
 #include "ieee80211_i.h"
 #include "driver-ops.h"
 #include "rate.h"
@@ -70,6 +71,177 @@ DEBUGFS_READONLY_FILE(wep_iv, "%#08x",
 DEBUGFS_READONLY_FILE(rate_ctrl_alg, "%s",
         local->rate_ctrl ? local->rate_ctrl->ops->name : "hw/driver");
 
+struct aqm_info {
+        struct ieee80211_local *local;
+        size_t size;
+        size_t len;
+        unsigned char buf[0];
+};
+
+#define AQM_HDR_LEN 200
+#define AQM_HW_ENTRY_LEN 40
+#define AQM_TXQ_ENTRY_LEN 110
+
+static int aqm_open(struct inode *inode, struct file *file)
+{
+        struct ieee80211_local *local = inode->i_private;
+        struct ieee80211_sub_if_data *sdata;
+        struct sta_info *sta;
+        struct txq_info *txqi;
+        struct fq *fq = &local->fq;
+        struct aqm_info *info = NULL;
+        int len = 0;
+        int i;
+
+        if (!local->ops->wake_tx_queue)
+                return -EOPNOTSUPP;
+
+        len += AQM_HDR_LEN;
+        len += 6 * AQM_HW_ENTRY_LEN;
+
+        rcu_read_lock();
+        list_for_each_entry_rcu(sdata, &local->interfaces, list)
+                len += AQM_TXQ_ENTRY_LEN;
+        list_for_each_entry_rcu(sta, &local->sta_list, list)
+                len += AQM_TXQ_ENTRY_LEN * ARRAY_SIZE(sta->sta.txq);
+        rcu_read_unlock();
+
+        info = vmalloc(len);
+        if (!info)
+                return -ENOMEM;
+
+        spin_lock_bh(&local->fq.lock);
+        rcu_read_lock();
+
+        file->private_data = info;
+        info->local = local;
+        info->size = len;
+        len = 0;
+
+        len += scnprintf(info->buf + len, info->size - len,
+                         "* hw\n"
+                         "access name value\n"
+                         "R fq_flows_cnt %u\n"
+                         "R fq_backlog %u\n"
+                         "R fq_overlimit %u\n"
+                         "R fq_collisions %u\n"
+                         "RW fq_limit %u\n"
+                         "RW fq_quantum %u\n",
+                         fq->flows_cnt,
+                         fq->backlog,
+                         fq->overlimit,
+                         fq->collisions,
+                         fq->limit,
+                         fq->quantum);
+
+        len += scnprintf(info->buf + len,
+                         info->size - len,
+                         "* vif\n"
+                         "ifname addr ac backlog-bytes backlog-packets flows overlimit collisions tx-bytes tx-packets\n");
+
+        list_for_each_entry_rcu(sdata, &local->interfaces, list) {
+                txqi = to_txq_info(sdata->vif.txq);
+                len += scnprintf(info->buf + len, info->size - len,
+                                 "%s %pM %u %u %u %u %u %u %u %u\n",
+                                 sdata->name,
+                                 sdata->vif.addr,
+                                 txqi->txq.ac,
+                                 txqi->tin.backlog_bytes,
+                                 txqi->tin.backlog_packets,
+                                 txqi->tin.flows,
+                                 txqi->tin.overlimit,
+                                 txqi->tin.collisions,
+                                 txqi->tin.tx_bytes,
+                                 txqi->tin.tx_packets);
+        }
+
+        len += scnprintf(info->buf + len,
+                         info->size - len,
+                         "* sta\n"
+                         "ifname addr tid ac backlog-bytes backlog-packets flows overlimit collisions tx-bytes tx-packets\n");
+
+        list_for_each_entry_rcu(sta, &local->sta_list, list) {
+                sdata = sta->sdata;
+                for (i = 0; i < ARRAY_SIZE(sta->sta.txq); i++) {
+                        txqi = to_txq_info(sta->sta.txq[i]);
+                        len += scnprintf(info->buf + len, info->size - len,
+                                         "%s %pM %d %d %u %u %u %u %u %u %u\n",
+                                         sdata->name,
+                                         sta->sta.addr,
+                                         txqi->txq.tid,
+                                         txqi->txq.ac,
+                                         txqi->tin.backlog_bytes,
+                                         txqi->tin.backlog_packets,
+                                         txqi->tin.flows,
+                                         txqi->tin.overlimit,
+                                         txqi->tin.collisions,
+                                         txqi->tin.tx_bytes,
+                                         txqi->tin.tx_packets);
+                }
+        }
+
+        info->len = len;
+
+        rcu_read_unlock();
+        spin_unlock_bh(&local->fq.lock);
+
+        return 0;
+}
+
+static int aqm_release(struct inode *inode, struct file *file)
+{
+        vfree(file->private_data);
+        return 0;
+}
+
+static ssize_t aqm_read(struct file *file,
+                        char __user *user_buf,
+                        size_t count,
+                        loff_t *ppos)
+{
+        struct aqm_info *info = file->private_data;
+
+        return simple_read_from_buffer(user_buf, count, ppos,
+                                       info->buf, info->len);
+}
+
+static ssize_t aqm_write(struct file *file,
+                         const char __user *user_buf,
+                         size_t count,
+                         loff_t *ppos)
+{
+        struct aqm_info *info = file->private_data;
+        struct ieee80211_local *local = info->local;
+        char buf[100];
+        size_t len;
+
+        if (count > sizeof(buf))
+                return -EINVAL;
+
+        if (copy_from_user(buf, user_buf, count))
+                return -EFAULT;
+
+        buf[sizeof(buf) - 1] = '\0';
+        len = strlen(buf);
+        if (len > 0 && buf[len-1] == '\n')
+                buf[len-1] = 0;
+
+        if (sscanf(buf, "fq_limit %u", &local->fq.limit) == 1)
+                return count;
+        else if (sscanf(buf, "fq_quantum %u", &local->fq.quantum) == 1)
+                return count;
+
+        return -EINVAL;
+}
+
+static const struct file_operations aqm_ops = {
+        .write = aqm_write,
+        .read = aqm_read,
+        .open = aqm_open,
+        .release = aqm_release,
+        .llseek = default_llseek,
+};
+
 #ifdef CONFIG_PM
 static ssize_t reset_write(struct file *file, const char __user *user_buf,
                            size_t count, loff_t *ppos)
@@ -256,6 +428,7 @@ void debugfs_hw_add(struct ieee80211_local *local)
         DEBUGFS_ADD(hwflags);
         DEBUGFS_ADD(user_power);
         DEBUGFS_ADD(power);
+        DEBUGFS_ADD_MODE(aqm, 0600);
 
         statsd = debugfs_create_dir("statistics", phyd);
 
diff --git a/net/mac80211/debugfs_sta.c b/net/mac80211/debugfs_sta.c
index 33dfcbc2bf9c..fd334133ff45 100644
--- a/net/mac80211/debugfs_sta.c
+++ b/net/mac80211/debugfs_sta.c
@@ -328,14 +328,88 @@ STA_OPS(ht_capa);
 static ssize_t sta_vht_capa_read(struct file *file, char __user *userbuf,
                                  size_t count, loff_t *ppos)
 {
-        char buf[128], *p = buf;
+        char buf[512], *p = buf;
         struct sta_info *sta = file->private_data;
         struct ieee80211_sta_vht_cap *vhtc = &sta->sta.vht_cap;
 
         p += scnprintf(p, sizeof(buf) + buf - p, "VHT %ssupported\n",
                         vhtc->vht_supported ? "" : "not ");
         if (vhtc->vht_supported) {
-                p += scnprintf(p, sizeof(buf)+buf-p, "cap: %#.8x\n", vhtc->cap);
+                p += scnprintf(p, sizeof(buf) + buf - p, "cap: %#.8x\n",
+                               vhtc->cap);
+#define PFLAG(a, b)                                                     \
+                do {                                                    \
+                        if (vhtc->cap & IEEE80211_VHT_CAP_ ## a)        \
+                                p += scnprintf(p, sizeof(buf) + buf - p, \
+                                               "\t\t%s\n", b);          \
+                } while (0)
+
+                switch (vhtc->cap & 0x3) {
+                case IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_3895:
+                        p += scnprintf(p, sizeof(buf) + buf - p,
+                                       "\t\tMAX-MPDU-3895\n");
+                        break;
+                case IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_7991:
+                        p += scnprintf(p, sizeof(buf) + buf - p,
+                                       "\t\tMAX-MPDU-7991\n");
+                        break;
+                case IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454:
+                        p += scnprintf(p, sizeof(buf) + buf - p,
+                                       "\t\tMAX-MPDU-11454\n");
+                        break;
+                default:
+                        p += scnprintf(p, sizeof(buf) + buf - p,
+                                       "\t\tMAX-MPDU-UNKNOWN\n");
+                };
+                switch (vhtc->cap & IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_MASK) {
+                case 0:
+                        p += scnprintf(p, sizeof(buf) + buf - p,
+                                       "\t\t80Mhz\n");
+                        break;
+                case IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160MHZ:
+                        p += scnprintf(p, sizeof(buf) + buf - p,
+                                       "\t\t160Mhz\n");
+                        break;
+                case IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160_80PLUS80MHZ:
+                        p += scnprintf(p, sizeof(buf) + buf - p,
+                                       "\t\t80+80Mhz\n");
+                        break;
+                default:
+                        p += scnprintf(p, sizeof(buf) + buf - p,
+                                       "\t\tUNKNOWN-MHZ: 0x%x\n",
+                                       (vhtc->cap >> 2) & 0x3);
+                };
+                PFLAG(RXLDPC, "RXLDPC");
+                PFLAG(SHORT_GI_80, "SHORT-GI-80");
+                PFLAG(SHORT_GI_160, "SHORT-GI-160");
+                PFLAG(TXSTBC, "TXSTBC");
+                p += scnprintf(p, sizeof(buf) + buf - p,
+                               "\t\tRXSTBC_%d\n", (vhtc->cap >> 8) & 0x7);
+                PFLAG(SU_BEAMFORMER_CAPABLE, "SU-BEAMFORMER-CAPABLE");
+                PFLAG(SU_BEAMFORMEE_CAPABLE, "SU-BEAMFORMEE-CAPABLE");
+                p += scnprintf(p, sizeof(buf) + buf - p,
+                        "\t\tBEAMFORMEE-STS: 0x%x\n",
+                        (vhtc->cap & IEEE80211_VHT_CAP_BEAMFORMEE_STS_MASK) >>
+                        IEEE80211_VHT_CAP_BEAMFORMEE_STS_SHIFT);
+                p += scnprintf(p, sizeof(buf) + buf - p,
+                        "\t\tSOUNDING-DIMENSIONS: 0x%x\n",
+                        (vhtc->cap & IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_MASK)
+                        >> IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_SHIFT);
+                PFLAG(MU_BEAMFORMER_CAPABLE, "MU-BEAMFORMER-CAPABLE");
+                PFLAG(MU_BEAMFORMEE_CAPABLE, "MU-BEAMFORMEE-CAPABLE");
+                PFLAG(VHT_TXOP_PS, "TXOP-PS");
+                PFLAG(HTC_VHT, "HTC-VHT");
+                p += scnprintf(p, sizeof(buf) + buf - p,
+                        "\t\tMPDU-LENGTH-EXPONENT: 0x%x\n",
+                        (vhtc->cap & IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK) >>
+                        IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_SHIFT);
+                PFLAG(VHT_LINK_ADAPTATION_VHT_UNSOL_MFB,
+                      "LINK-ADAPTATION-VHT-UNSOL-MFB");
+                p += scnprintf(p, sizeof(buf) + buf - p,
+                        "\t\tLINK-ADAPTATION-VHT-MRQ-MFB: 0x%x\n",
+                        (vhtc->cap & IEEE80211_VHT_CAP_VHT_LINK_ADAPTATION_VHT_MRQ_MFB) >> 26);
+                PFLAG(RX_ANTENNA_PATTERN, "RX-ANTENNA-PATTERN");
+                PFLAG(TX_ANTENNA_PATTERN, "TX-ANTENNA-PATTERN");
 
                 p += scnprintf(p, sizeof(buf)+buf-p, "RX MCS: %.4x\n",
                                le16_to_cpu(vhtc->vht_mcs.rx_mcs_map));
diff --git a/net/mac80211/ieee80211_i.h b/net/mac80211/ieee80211_i.h
index 9438c9406687..54edfb6fc1d1 100644
--- a/net/mac80211/ieee80211_i.h
+++ b/net/mac80211/ieee80211_i.h
@@ -30,6 +30,7 @@
 #include <net/ieee80211_radiotap.h>
 #include <net/cfg80211.h>
 #include <net/mac80211.h>
+#include <net/fq.h>
 #include "key.h"
 #include "sta_info.h"
 #include "debug.h"
@@ -805,10 +806,19 @@ enum txq_info_flags {
         IEEE80211_TXQ_NO_AMSDU,
 };
 
+/**
+ * struct txq_info - per tid queue
+ *
+ * @tin: contains packets split into multiple flows
+ * @def_flow: used as a fallback flow when a packet destined to @tin hashes to
+ *      a fq_flow which is already owned by a different tin
+ * @def_cvars: codel vars for @def_flow
+ */
 struct txq_info {
-        struct sk_buff_head queue;
+        struct fq_tin tin;
+        struct fq_flow def_flow;
+        struct codel_vars def_cvars;
         unsigned long flags;
-        unsigned long byte_cnt;
 
         /* keep last! */
         struct ieee80211_txq txq;
@@ -856,7 +866,7 @@ struct ieee80211_sub_if_data {
         bool control_port_no_encrypt;
         int encrypt_headroom;
 
-        atomic_t txqs_len[IEEE80211_NUM_ACS];
+        atomic_t num_tx_queued;
         struct ieee80211_tx_queue_params tx_conf[IEEE80211_NUM_ACS];
         struct mac80211_qos_map __rcu *qos_map;
 
@@ -1099,6 +1109,11 @@ struct ieee80211_local {
          * it first anyway so they become a no-op */
         struct ieee80211_hw hw;
 
+        struct fq fq;
+        struct codel_vars *cvars;
+        struct codel_params cparams;
+        struct codel_stats cstats;
+
         const struct ieee80211_ops *ops;
 
         /*
@@ -1931,9 +1946,13 @@ static inline bool ieee80211_can_run_worker(struct ieee80211_local *local)
         return true;
 }
 
-void ieee80211_init_tx_queue(struct ieee80211_sub_if_data *sdata,
-                             struct sta_info *sta,
-                             struct txq_info *txq, int tid);
+int ieee80211_txq_setup_flows(struct ieee80211_local *local);
+void ieee80211_txq_teardown_flows(struct ieee80211_local *local);
+void ieee80211_txq_init(struct ieee80211_sub_if_data *sdata,
+                        struct sta_info *sta,
+                        struct txq_info *txq, int tid);
+void ieee80211_txq_purge(struct ieee80211_local *local,
+                         struct txq_info *txqi);
 void ieee80211_send_auth(struct ieee80211_sub_if_data *sdata,
                          u16 transaction, u16 auth_alg, u16 status,
                          const u8 *extra, size_t extra_len, const u8 *bssid,
diff --git a/net/mac80211/iface.c b/net/mac80211/iface.c
index c59af3eb9fa4..b123a9e325b3 100644
--- a/net/mac80211/iface.c
+++ b/net/mac80211/iface.c
@@ -779,6 +779,7 @@ static void ieee80211_do_stop(struct ieee80211_sub_if_data *sdata,
                               bool going_down)
 {
         struct ieee80211_local *local = sdata->local;
+        struct fq *fq = &local->fq;
         unsigned long flags;
         struct sk_buff *skb, *tmp;
         u32 hw_reconf_flags = 0;
@@ -977,12 +978,9 @@ static void ieee80211_do_stop(struct ieee80211_sub_if_data *sdata,
         if (sdata->vif.txq) {
                 struct txq_info *txqi = to_txq_info(sdata->vif.txq);
 
-                spin_lock_bh(&txqi->queue.lock);
-                ieee80211_purge_tx_queue(&local->hw, &txqi->queue);
-                txqi->byte_cnt = 0;
-                spin_unlock_bh(&txqi->queue.lock);
-
-                atomic_set(&sdata->txqs_len[txqi->txq.ac], 0);
+                spin_lock_bh(&fq->lock);
+                ieee80211_txq_purge(local, txqi);
+                spin_unlock_bh(&fq->lock);
         }
 
         if (local->open_count == 0)
@@ -1198,6 +1196,12 @@ static void ieee80211_if_setup(struct net_device *dev)
         dev->destructor = ieee80211_if_free;
 }
 
+static void ieee80211_if_setup_no_queue(struct net_device *dev)
+{
+        ieee80211_if_setup(dev);
+        dev->priv_flags |= IFF_NO_QUEUE;
+}
+
 static void ieee80211_iface_work(struct work_struct *work)
 {
         struct ieee80211_sub_if_data *sdata =
@@ -1707,6 +1711,7 @@ int ieee80211_if_add(struct ieee80211_local *local, const char *name,
         struct net_device *ndev = NULL;
         struct ieee80211_sub_if_data *sdata = NULL;
         struct txq_info *txqi;
+        void (*if_setup)(struct net_device *dev);
         int ret, i;
         int txqs = 1;
 
@@ -1734,12 +1739,17 @@ int ieee80211_if_add(struct ieee80211_local *local, const char *name,
                         txq_size += sizeof(struct txq_info) +
                                     local->hw.txq_data_size;
 
+                if (local->ops->wake_tx_queue)
+                        if_setup = ieee80211_if_setup_no_queue;
+                else
+                        if_setup = ieee80211_if_setup;
+
                 if (local->hw.queues >= IEEE80211_NUM_ACS)
                         txqs = IEEE80211_NUM_ACS;
 
                 ndev = alloc_netdev_mqs(size + txq_size,
                                         name, name_assign_type,
-                                        ieee80211_if_setup, txqs, 1);
+                                        if_setup, txqs, 1);
                 if (!ndev)
                         return -ENOMEM;
                 dev_net_set(ndev, wiphy_net(local->hw.wiphy));
@@ -1780,7 +1790,7 @@ int ieee80211_if_add(struct ieee80211_local *local, const char *name,
 
                 if (txq_size) {
                         txqi = netdev_priv(ndev) + size;
-                        ieee80211_init_tx_queue(sdata, NULL, txqi, 0);
+                        ieee80211_txq_init(sdata, NULL, txqi, 0);
                 }
 
                 sdata->dev = ndev;
diff --git a/net/mac80211/main.c b/net/mac80211/main.c
index 7ee91d6151d1..d00ea9b13f49 100644
--- a/net/mac80211/main.c
+++ b/net/mac80211/main.c
@@ -1055,9 +1055,6 @@ int ieee80211_register_hw(struct ieee80211_hw *hw)
 
         local->dynamic_ps_forced_timeout = -1;
 
-        if (!local->hw.txq_ac_max_pending)
-                local->hw.txq_ac_max_pending = 64;
-
         result = ieee80211_wep_init(local);
         if (result < 0)
                 wiphy_debug(local->hw.wiphy, "Failed to initialize wep: %d\n",
@@ -1089,6 +1086,10 @@ int ieee80211_register_hw(struct ieee80211_hw *hw)
 
         rtnl_unlock();
 
+        result = ieee80211_txq_setup_flows(local);
+        if (result)
+                goto fail_flows;
+
 #ifdef CONFIG_INET
         local->ifa_notifier.notifier_call = ieee80211_ifa_changed;
         result = register_inetaddr_notifier(&local->ifa_notifier);
@@ -1114,6 +1115,8 @@ int ieee80211_register_hw(struct ieee80211_hw *hw)
 #if defined(CONFIG_INET) || defined(CONFIG_IPV6)
  fail_ifa:
 #endif
+        ieee80211_txq_teardown_flows(local);
+ fail_flows:
         rtnl_lock();
         rate_control_deinitialize(local);
         ieee80211_remove_interfaces(local);
@@ -1172,6 +1175,7 @@ void ieee80211_unregister_hw(struct ieee80211_hw *hw)
         skb_queue_purge(&local->skb_queue);
         skb_queue_purge(&local->skb_queue_unreliable);
         skb_queue_purge(&local->skb_queue_tdls_chsw);
+        ieee80211_txq_teardown_flows(local);
 
         destroy_workqueue(local->workqueue);
         wiphy_unregister(local->hw.wiphy);
diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c
index 5e65e838992a..9a1eb70cb120 100644
--- a/net/mac80211/rx.c
+++ b/net/mac80211/rx.c
@@ -1268,7 +1268,7 @@ static void sta_ps_start(struct sta_info *sta)
         for (tid = 0; tid < ARRAY_SIZE(sta->sta.txq); tid++) {
                 struct txq_info *txqi = to_txq_info(sta->sta.txq[tid]);
 
-                if (!skb_queue_len(&txqi->queue))
+                if (!txqi->tin.backlog_packets)
                         set_bit(tid, &sta->txq_buffered_tids);
                 else
                         clear_bit(tid, &sta->txq_buffered_tids);
diff --git a/net/mac80211/sta_info.c b/net/mac80211/sta_info.c
index 5ccfdbd406bd..76b737dcc36f 100644
--- a/net/mac80211/sta_info.c
+++ b/net/mac80211/sta_info.c
@@ -90,6 +90,7 @@ static void __cleanup_single_sta(struct sta_info *sta)
         struct tid_ampdu_tx *tid_tx;
         struct ieee80211_sub_if_data *sdata = sta->sdata;
         struct ieee80211_local *local = sdata->local;
+        struct fq *fq = &local->fq;
         struct ps_data *ps;
 
         if (test_sta_flag(sta, WLAN_STA_PS_STA) ||
@@ -113,11 +114,10 @@ static void __cleanup_single_sta(struct sta_info *sta)
         if (sta->sta.txq[0]) {
                 for (i = 0; i < ARRAY_SIZE(sta->sta.txq); i++) {
                         struct txq_info *txqi = to_txq_info(sta->sta.txq[i]);
-                        int n = skb_queue_len(&txqi->queue);
 
-                        ieee80211_purge_tx_queue(&local->hw, &txqi->queue);
-                        atomic_sub(n, &sdata->txqs_len[txqi->txq.ac]);
-                        txqi->byte_cnt = 0;
+                        spin_lock_bh(&fq->lock);
+                        ieee80211_txq_purge(local, txqi);
+                        spin_unlock_bh(&fq->lock);
                 }
         }
 
@@ -368,7 +368,7 @@ struct sta_info *sta_info_alloc(struct ieee80211_sub_if_data *sdata,
                 for (i = 0; i < ARRAY_SIZE(sta->sta.txq); i++) {
                         struct txq_info *txq = txq_data + i * size;
 
-                        ieee80211_init_tx_queue(sdata, sta, txq, i);
+                        ieee80211_txq_init(sdata, sta, txq, i);
                 }
         }
 
@@ -1211,7 +1211,7 @@ void ieee80211_sta_ps_deliver_wakeup(struct sta_info *sta)
                 for (i = 0; i < ARRAY_SIZE(sta->sta.txq); i++) {
                         struct txq_info *txqi = to_txq_info(sta->sta.txq[i]);
 
-                        if (!skb_queue_len(&txqi->queue))
+                        if (!txqi->tin.backlog_packets)
                                 continue;
 
                         drv_wake_tx_queue(local, txqi);
@@ -1648,7 +1648,7 @@ ieee80211_sta_ps_deliver_response(struct sta_info *sta,
                 for (tid = 0; tid < ARRAY_SIZE(sta->sta.txq); tid++) {
                         struct txq_info *txqi = to_txq_info(sta->sta.txq[tid]);
 
-                        if (!(tids & BIT(tid)) || skb_queue_len(&txqi->queue))
+                        if (!(tids & BIT(tid)) || txqi->tin.backlog_packets)
                                 continue;
 
                         sta_info_recalc_tim(sta);
diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c
index 203044379ce0..44ec605a5682 100644
--- a/net/mac80211/tx.c
+++ b/net/mac80211/tx.c
@@ -24,7 +24,10 @@
 #include <net/ieee80211_radiotap.h>
 #include <net/cfg80211.h>
 #include <net/mac80211.h>
+#include <net/codel.h>
+#include <net/codel_impl.h>
 #include <asm/unaligned.h>
+#include <net/fq_impl.h>
 
 #include "ieee80211_i.h"
 #include "driver-ops.h"
@@ -1236,27 +1239,21 @@ ieee80211_tx_prepare(struct ieee80211_sub_if_data *sdata,
         return TX_CONTINUE;
 }
 
-static void ieee80211_drv_tx(struct ieee80211_local *local,
-                             struct ieee80211_vif *vif,
-                             struct ieee80211_sta *pubsta,
-                             struct sk_buff *skb)
+static struct txq_info *ieee80211_get_txq(struct ieee80211_local *local,
+                                          struct ieee80211_vif *vif,
+                                          struct ieee80211_sta *pubsta,
+                                          struct sk_buff *skb)
 {
         struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
-        struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif);
         struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
-        struct ieee80211_tx_control control = {
-                .sta = pubsta,
-        };
         struct ieee80211_txq *txq = NULL;
-        struct txq_info *txqi;
-        u8 ac;
 
         if ((info->flags & IEEE80211_TX_CTL_SEND_AFTER_DTIM) ||
             (info->control.flags & IEEE80211_TX_CTRL_PS_RESPONSE))
-                goto tx_normal;
+                return NULL;
 
         if (!ieee80211_is_data(hdr->frame_control))
-                goto tx_normal;
+                return NULL;
 
         if (pubsta) {
                 u8 tid = skb->priority & IEEE80211_QOS_CTL_TID_MASK;
@@ -1267,51 +1264,230 @@ static void ieee80211_drv_tx(struct ieee80211_local *local,
         }
 
         if (!txq)
-                goto tx_normal;
+                return NULL;
 
-        ac = txq->ac;
-        txqi = to_txq_info(txq);
-        atomic_inc(&sdata->txqs_len[ac]);
-        if (atomic_read(&sdata->txqs_len[ac]) >= local->hw.txq_ac_max_pending)
-                netif_stop_subqueue(sdata->dev, ac);
+        return to_txq_info(txq);
+}
 
-        spin_lock_bh(&txqi->queue.lock);
-        txqi->byte_cnt += skb->len;
-        __skb_queue_tail(&txqi->queue, skb);
-        spin_unlock_bh(&txqi->queue.lock);
+static void ieee80211_set_skb_enqueue_time(struct sk_buff *skb)
+{
+        IEEE80211_SKB_CB(skb)->control.enqueue_time = codel_get_time();
+}
 
-        drv_wake_tx_queue(local, txqi);
+static void ieee80211_set_skb_vif(struct sk_buff *skb, struct txq_info *txqi)
+{
+        IEEE80211_SKB_CB(skb)->control.vif = txqi->txq.vif;
+}
 
-        return;
+static u32 codel_skb_len_func(const struct sk_buff *skb)
+{
+        return skb->len;
+}
+
+static codel_time_t codel_skb_time_func(const struct sk_buff *skb)
+{
+        const struct ieee80211_tx_info *info;
 
-tx_normal:
-        drv_tx(local, &control, skb);
+        info = (const struct ieee80211_tx_info *)skb->cb;
+        return info->control.enqueue_time;
+}
+
+static struct sk_buff *codel_dequeue_func(struct codel_vars *cvars,
+                                          void *ctx)
+{
+        struct ieee80211_local *local;
+        struct txq_info *txqi;
+        struct fq *fq;
+        struct fq_flow *flow;
+
+        txqi = ctx;
+        local = vif_to_sdata(txqi->txq.vif)->local;
+        fq = &local->fq;
+
+        if (cvars == &txqi->def_cvars)
+                flow = &txqi->def_flow;
+        else
+                flow = &fq->flows[cvars - local->cvars];
+
+        return fq_flow_dequeue(fq, flow);
+}
+
+static void codel_drop_func(struct sk_buff *skb,
+                            void *ctx)
+{
+        struct ieee80211_local *local;
+        struct ieee80211_hw *hw;
+        struct txq_info *txqi;
+
+        txqi = ctx;
+        local = vif_to_sdata(txqi->txq.vif)->local;
+        hw = &local->hw;
+
+        ieee80211_free_txskb(hw, skb);
+}
+
+static struct sk_buff *fq_tin_dequeue_func(struct fq *fq,
+                                           struct fq_tin *tin,
+                                           struct fq_flow *flow)
+{
+        struct ieee80211_local *local;
+        struct txq_info *txqi;
+        struct codel_vars *cvars;
+        struct codel_params *cparams;
+        struct codel_stats *cstats;
+
+        local = container_of(fq, struct ieee80211_local, fq);
+        txqi = container_of(tin, struct txq_info, tin);
+        cparams = &local->cparams;
+        cstats = &local->cstats;
+
+        if (flow == &txqi->def_flow)
+                cvars = &txqi->def_cvars;
+        else
+                cvars = &local->cvars[flow - fq->flows];
+
+        return codel_dequeue(txqi,
+                             &flow->backlog,
+                             cparams,
+                             cvars,
+                             cstats,
+                             codel_skb_len_func,
+                             codel_skb_time_func,
+                             codel_drop_func,
+                             codel_dequeue_func);
+}
+
+static void fq_skb_free_func(struct fq *fq,
+                             struct fq_tin *tin,
+                             struct fq_flow *flow,
+                             struct sk_buff *skb)
+{
+        struct ieee80211_local *local;
+
+        local = container_of(fq, struct ieee80211_local, fq);
+        ieee80211_free_txskb(&local->hw, skb);
+}
+
+static struct fq_flow *fq_flow_get_default_func(struct fq *fq,
+                                                struct fq_tin *tin,
+                                                int idx,
+                                                struct sk_buff *skb)
+{
+        struct txq_info *txqi;
+
+        txqi = container_of(tin, struct txq_info, tin);
+        return &txqi->def_flow;
+}
+
+static void ieee80211_txq_enqueue(struct ieee80211_local *local,
+                                  struct txq_info *txqi,
+                                  struct sk_buff *skb)
+{
+        struct fq *fq = &local->fq;
+        struct fq_tin *tin = &txqi->tin;
+
+        ieee80211_set_skb_enqueue_time(skb);
+        fq_tin_enqueue(fq, tin, skb,
+                       fq_skb_free_func,
+                       fq_flow_get_default_func);
+}
+
+void ieee80211_txq_init(struct ieee80211_sub_if_data *sdata,
+                        struct sta_info *sta,
+                        struct txq_info *txqi, int tid)
+{
+        fq_tin_init(&txqi->tin);
+        fq_flow_init(&txqi->def_flow);
+        codel_vars_init(&txqi->def_cvars);
+
+        txqi->txq.vif = &sdata->vif;
+
+        if (sta) {
+                txqi->txq.sta = &sta->sta;
+                sta->sta.txq[tid] = &txqi->txq;
+                txqi->txq.tid = tid;
+                txqi->txq.ac = ieee802_1d_to_ac[tid & 7];
+        } else {
+                sdata->vif.txq = &txqi->txq;
+                txqi->txq.tid = 0;
+                txqi->txq.ac = IEEE80211_AC_BE;
+        }
+}
+
+void ieee80211_txq_purge(struct ieee80211_local *local,
+                         struct txq_info *txqi)
+{
+        struct fq *fq = &local->fq;
+        struct fq_tin *tin = &txqi->tin;
+
+        fq_tin_reset(fq, tin, fq_skb_free_func);
+}
+
+int ieee80211_txq_setup_flows(struct ieee80211_local *local)
+{
+        struct fq *fq = &local->fq;
+        int ret;
+        int i;
+
+        if (!local->ops->wake_tx_queue)
+                return 0;
+
+        ret = fq_init(fq, 4096);
+        if (ret)
+                return ret;
+
+        codel_params_init(&local->cparams);
+        codel_stats_init(&local->cstats);
+        local->cparams.interval = MS2TIME(100);
+        local->cparams.target = MS2TIME(20);
+        local->cparams.ecn = true;
+
+        local->cvars = kcalloc(fq->flows_cnt, sizeof(local->cvars[0]),
+                               GFP_KERNEL);
+        if (!local->cvars) {
+                fq_reset(fq, fq_skb_free_func);
+                return -ENOMEM;
+        }
+
+        for (i = 0; i < fq->flows_cnt; i++)
+                codel_vars_init(&local->cvars[i]);
+
+        return 0;
+}
+
+void ieee80211_txq_teardown_flows(struct ieee80211_local *local)
+{
+        struct fq *fq = &local->fq;
+
+        if (!local->ops->wake_tx_queue)
+                return;
+
+        kfree(local->cvars);
+        local->cvars = NULL;
+
+        fq_reset(fq, fq_skb_free_func);
 }
 
 struct sk_buff *ieee80211_tx_dequeue(struct ieee80211_hw *hw,
                                      struct ieee80211_txq *txq)
 {
         struct ieee80211_local *local = hw_to_local(hw);
-        struct ieee80211_sub_if_data *sdata = vif_to_sdata(txq->vif);
         struct txq_info *txqi = container_of(txq, struct txq_info, txq);
         struct ieee80211_hdr *hdr;
         struct sk_buff *skb = NULL;
-        u8 ac = txq->ac;
+        struct fq *fq = &local->fq;
+        struct fq_tin *tin = &txqi->tin;
 
-        spin_lock_bh(&txqi->queue.lock);
+        spin_lock_bh(&fq->lock);
 
         if (test_bit(IEEE80211_TXQ_STOP, &txqi->flags))
                 goto out;
 
-        skb = __skb_dequeue(&txqi->queue);
+        skb = fq_tin_dequeue(fq, tin, fq_tin_dequeue_func);
         if (!skb)
                 goto out;
 
-        txqi->byte_cnt -= skb->len;
-
-        atomic_dec(&sdata->txqs_len[ac]);
-        if (__netif_subqueue_stopped(sdata->dev, ac))
-                ieee80211_propagate_queue_wake(local, sdata->vif.hw_queue[ac]);
+        ieee80211_set_skb_vif(skb, txqi);
 
         hdr = (struct ieee80211_hdr *)skb->data;
         if (txq->sta && ieee80211_is_data_qos(hdr->frame_control)) {
@@ -1327,7 +1503,7 @@ struct sk_buff *ieee80211_tx_dequeue(struct ieee80211_hw *hw,
         }
 
 out:
-        spin_unlock_bh(&txqi->queue.lock);
+        spin_unlock_bh(&fq->lock);
 
         if (skb && skb_has_frag_list(skb) &&
             !ieee80211_hw_check(&local->hw, TX_FRAG_LIST))
@@ -1343,7 +1519,10 @@ static bool ieee80211_tx_frags(struct ieee80211_local *local,
                                struct sk_buff_head *skbs,
                                bool txpending)
 {
+        struct ieee80211_tx_control control = {};
+        struct fq *fq = &local->fq;
         struct sk_buff *skb, *tmp;
+        struct txq_info *txqi;
         unsigned long flags;
 
         skb_queue_walk_safe(skbs, skb, tmp) {
@@ -1358,6 +1537,21 @@ static bool ieee80211_tx_frags(struct ieee80211_local *local,
                 }
 #endif
 
+                txqi = ieee80211_get_txq(local, vif, sta, skb);
+                if (txqi) {
+                        info->control.vif = vif;
+
+                        __skb_unlink(skb, skbs);
+
+                        spin_lock_bh(&fq->lock);
+                        ieee80211_txq_enqueue(local, txqi, skb);
+                        spin_unlock_bh(&fq->lock);
+
+                        drv_wake_tx_queue(local, txqi);
+
+                        continue;
+                }
+
                 spin_lock_irqsave(&local->queue_stop_reason_lock, flags);
                 if (local->queue_stop_reasons[q] ||
                     (!txpending && !skb_queue_empty(&local->pending[q]))) {
@@ -1400,9 +1594,10 @@ static bool ieee80211_tx_frags(struct ieee80211_local *local,
                 spin_unlock_irqrestore(&local->queue_stop_reason_lock, flags);
 
                 info->control.vif = vif;
+                control.sta = sta;
 
                 __skb_unlink(skb, skbs);
-                ieee80211_drv_tx(local, vif, sta, skb);
+                drv_tx(local, &control, skb);
         }
 
         return true;
@@ -2882,6 +3077,9 @@ static bool ieee80211_amsdu_aggregate(struct ieee80211_sub_if_data *sdata,
                                       struct sk_buff *skb)
 {
         struct ieee80211_local *local = sdata->local;
+        struct fq *fq = &local->fq;
+        struct fq_tin *tin;
+        struct fq_flow *flow;
         u8 tid = skb->priority & IEEE80211_QOS_CTL_TAG1D_MASK;
         struct ieee80211_txq *txq = sta->sta.txq[tid];
         struct txq_info *txqi;
@@ -2893,6 +3091,7 @@ static bool ieee80211_amsdu_aggregate(struct ieee80211_sub_if_data *sdata,
         __be16 len;
         void *data;
         bool ret = false;
+        unsigned int orig_len;
         int n = 1, nfrags;
 
         if (!ieee80211_hw_check(&local->hw, TX_AMSDU))
@@ -2909,12 +3108,20 @@ static bool ieee80211_amsdu_aggregate(struct ieee80211_sub_if_data *sdata,
                 max_amsdu_len = min_t(int, max_amsdu_len,
                                       sta->sta.max_rc_amsdu_len);
 
-        spin_lock_bh(&txqi->queue.lock);
+        spin_lock_bh(&fq->lock);
+
+        /* TODO: Ideally aggregation should be done on dequeue to remain
+         * responsive to environment changes.
+         */
 
-        head = skb_peek_tail(&txqi->queue);
+        tin = &txqi->tin;
+        flow = fq_flow_classify(fq, tin, skb, fq_flow_get_default_func);
+        head = skb_peek_tail(&flow->queue);
         if (!head)
                 goto out;
 
+        orig_len = head->len;
+
         if (skb->len + head->len > max_amsdu_len)
                 goto out;
 
@@ -2953,8 +3160,13 @@ static bool ieee80211_amsdu_aggregate(struct ieee80211_sub_if_data *sdata,
         head->data_len += skb->len;
         *frag_tail = skb;
 
+        flow->backlog += head->len - orig_len;
+        tin->backlog_bytes += head->len - orig_len;
+
+        fq_recalc_backlog(fq, tin, flow);
+
 out:
-        spin_unlock_bh(&txqi->queue.lock);
+        spin_unlock_bh(&fq->lock);
 
         return ret;
 }
diff --git a/net/mac80211/util.c b/net/mac80211/util.c
index 905003f75c4d..42bf0b6685e8 100644
--- a/net/mac80211/util.c
+++ b/net/mac80211/util.c
@@ -244,6 +244,9 @@ void ieee80211_propagate_queue_wake(struct ieee80211_local *local, int queue)
         struct ieee80211_sub_if_data *sdata;
         int n_acs = IEEE80211_NUM_ACS;
 
+        if (local->ops->wake_tx_queue)
+                return;
+
         if (local->hw.queues < IEEE80211_NUM_ACS)
                 n_acs = 1;
 
@@ -260,11 +263,6 @@ void ieee80211_propagate_queue_wake(struct ieee80211_local *local, int queue)
                 for (ac = 0; ac < n_acs; ac++) {
                         int ac_queue = sdata->vif.hw_queue[ac];
 
-                        if (local->ops->wake_tx_queue &&
-                            (atomic_read(&sdata->txqs_len[ac]) >
-                             local->hw.txq_ac_max_pending))
-                                continue;
-
                         if (ac_queue == queue ||
                             (sdata->vif.cab_queue == queue &&
                              local->queue_stop_reasons[ac_queue] == 0 &&
@@ -352,6 +350,9 @@ static void __ieee80211_stop_queue(struct ieee80211_hw *hw, int queue,
         if (__test_and_set_bit(reason, &local->queue_stop_reasons[queue]))
                 return;
 
+        if (local->ops->wake_tx_queue)
+                return;
+
         if (local->hw.queues < IEEE80211_NUM_ACS)
                 n_acs = 1;
 
@@ -3388,25 +3389,6 @@ u8 *ieee80211_add_wmm_info_ie(u8 *buf, u8 qosinfo)
         return buf;
 }
 
-void ieee80211_init_tx_queue(struct ieee80211_sub_if_data *sdata,
-                             struct sta_info *sta,
-                             struct txq_info *txqi, int tid)
-{
-        skb_queue_head_init(&txqi->queue);
-        txqi->txq.vif = &sdata->vif;
-
-        if (sta) {
-                txqi->txq.sta = &sta->sta;
-                sta->sta.txq[tid] = &txqi->txq;
-                txqi->txq.tid = tid;
-                txqi->txq.ac = ieee802_1d_to_ac[tid & 7];
-        } else {
-                sdata->vif.txq = &txqi->txq;
-                txqi->txq.tid = 0;
-                txqi->txq.ac = IEEE80211_AC_BE;
-        }
-}
-
 void ieee80211_txq_get_depth(struct ieee80211_txq *txq,
                              unsigned long *frame_cnt,
                              unsigned long *byte_cnt)
@@ -3414,9 +3396,9 @@ void ieee80211_txq_get_depth(struct ieee80211_txq *txq,
         struct txq_info *txqi = to_txq_info(txq);
 
         if (frame_cnt)
-                *frame_cnt = txqi->queue.qlen;
+                *frame_cnt = txqi->tin.backlog_packets;
 
         if (byte_cnt)
-                *byte_cnt = txqi->byte_cnt;
+                *byte_cnt = txqi->tin.backlog_bytes;
 }
 EXPORT_SYMBOL(ieee80211_txq_get_depth);
diff --git a/net/mpls/af_mpls.c b/net/mpls/af_mpls.c
index 0b80a7140cc4..e9beaa58573c 100644
--- a/net/mpls/af_mpls.c
+++ b/net/mpls/af_mpls.c
@@ -91,7 +91,7 @@ bool mpls_pkt_too_big(const struct sk_buff *skb, unsigned int mtu)
         if (skb->len <= mtu)
                 return false;
 
-        if (skb_is_gso(skb) && skb_gso_network_seglen(skb) <= mtu)
+        if (skb_is_gso(skb) && skb_gso_validate_mtu(skb, mtu))
                 return false;
 
         return true;
@@ -1009,9 +1009,10 @@ static int mpls_dev_notify(struct notifier_block *this, unsigned long event,
         unsigned int flags;
 
         if (event == NETDEV_REGISTER) {
-                /* For now just support ethernet devices */
-                if ((dev->type == ARPHRD_ETHER) ||
-                    (dev->type == ARPHRD_LOOPBACK)) {
+                /* For now just support Ethernet and IPGRE devices */
+                if (dev->type == ARPHRD_ETHER ||
+                    dev->type == ARPHRD_LOOPBACK ||
+                    dev->type == ARPHRD_IPGRE) {
                         mdev = mpls_add_dev(dev);
                         if (IS_ERR(mdev))
                                 return notifier_from_errno(PTR_ERR(mdev));
diff --git a/net/netfilter/xt_RATEEST.c b/net/netfilter/xt_RATEEST.c
index 604df6fae6fc..515131f9e021 100644
--- a/net/netfilter/xt_RATEEST.c
+++ b/net/netfilter/xt_RATEEST.c
@@ -137,7 +137,7 @@ static int xt_rateest_tg_checkentry(const struct xt_tgchk_param *par)
         cfg.est.ewma_log        = info->ewma_log;
 
         ret = gen_new_estimator(&est->bstats, NULL, &est->rstats,
-                                &est->lock, &cfg.opt);
+                                &est->lock, NULL, &cfg.opt);
         if (ret < 0)
                 goto err2;
 
diff --git a/net/netlink/af_netlink.h b/net/netlink/af_netlink.h
index e68ef9ccd703..3cfd6cc60504 100644
--- a/net/netlink/af_netlink.h
+++ b/net/netlink/af_netlink.h
@@ -8,20 +8,6 @@
 #define NLGRPSZ(x)      (ALIGN(x, sizeof(unsigned long) * 8) / 8)
 #define NLGRPLONGS(x)   (NLGRPSZ(x)/sizeof(unsigned long))
 
-struct netlink_ring {
-        void                    **pg_vec;
-        unsigned int            head;
-        unsigned int            frames_per_block;
-        unsigned int            frame_size;
-        unsigned int            frame_max;
-
-        unsigned int            pg_vec_order;
-        unsigned int            pg_vec_pages;
-        unsigned int            pg_vec_len;
-
-        atomic_t                pending;
-};
-
 struct netlink_sock {
         /* struct sock has to be the first member of netlink_sock */
         struct sock             sk;
diff --git a/net/openvswitch/actions.c b/net/openvswitch/actions.c
index 9a3eb7a0ebf4..1ecbd7715f6d 100644
--- a/net/openvswitch/actions.c
+++ b/net/openvswitch/actions.c
@@ -750,6 +750,14 @@ static void do_output(struct datapath *dp, struct sk_buff *skb, int out_port,
 
         if (likely(vport)) {
                 u16 mru = OVS_CB(skb)->mru;
+                u32 cutlen = OVS_CB(skb)->cutlen;
+
+                if (unlikely(cutlen > 0)) {
+                        if (skb->len - cutlen > ETH_HLEN)
+                                pskb_trim(skb, skb->len - cutlen);
+                        else
+                                pskb_trim(skb, ETH_HLEN);
+                }
 
                 if (likely(!mru || (skb->len <= mru + ETH_HLEN))) {
                         ovs_vport_send(vport, skb);
@@ -775,7 +783,8 @@ static void do_output(struct datapath *dp, struct sk_buff *skb, int out_port,
 
 static int output_userspace(struct datapath *dp, struct sk_buff *skb,
                             struct sw_flow_key *key, const struct nlattr *attr,
-                            const struct nlattr *actions, int actions_len)
+                            const struct nlattr *actions, int actions_len,
+                            uint32_t cutlen)
 {
         struct dp_upcall_info upcall;
         const struct nlattr *a;
@@ -822,7 +831,7 @@ static int output_userspace(struct datapath *dp, struct sk_buff *skb,
                 } /* End of switch. */
         }
 
-        return ovs_dp_upcall(dp, skb, key, &upcall);
+        return ovs_dp_upcall(dp, skb, key, &upcall, cutlen);
 }
 
 static int sample(struct datapath *dp, struct sk_buff *skb,
@@ -832,6 +841,7 @@ static int sample(struct datapath *dp, struct sk_buff *skb,
         const struct nlattr *acts_list = NULL;
         const struct nlattr *a;
         int rem;
+        u32 cutlen = 0;
 
         for (a = nla_data(attr), rem = nla_len(attr); rem > 0;
                  a = nla_next(a, &rem)) {
@@ -858,13 +868,24 @@ static int sample(struct datapath *dp, struct sk_buff *skb,
                 return 0;
 
         /* The only known usage of sample action is having a single user-space
+         * action, or having a truncate action followed by a single user-space
          * action. Treat this usage as a special case.
          * The output_userspace() should clone the skb to be sent to the
          * user space. This skb will be consumed by its caller.
          */
+        if (unlikely(nla_type(a) == OVS_ACTION_ATTR_TRUNC)) {
+                struct ovs_action_trunc *trunc = nla_data(a);
+
+                if (skb->len > trunc->max_len)
+                        cutlen = skb->len - trunc->max_len;
+
+                a = nla_next(a, &rem);
+        }
+
         if (likely(nla_type(a) == OVS_ACTION_ATTR_USERSPACE &&
                    nla_is_last(a, rem)))
-                return output_userspace(dp, skb, key, a, actions, actions_len);
+                return output_userspace(dp, skb, key, a, actions,
+                                        actions_len, cutlen);
 
         skb = skb_clone(skb, GFP_ATOMIC);
         if (!skb)
@@ -1051,6 +1072,7 @@ static int do_execute_actions(struct datapath *dp, struct sk_buff *skb,
                         if (out_skb)
                                 do_output(dp, out_skb, prev_port, key);
 
+                        OVS_CB(skb)->cutlen = 0;
                         prev_port = -1;
                 }
 
@@ -1059,8 +1081,18 @@ static int do_execute_actions(struct datapath *dp, struct sk_buff *skb,
                         prev_port = nla_get_u32(a);
                         break;
 
+                case OVS_ACTION_ATTR_TRUNC: {
+                        struct ovs_action_trunc *trunc = nla_data(a);
+
+                        if (skb->len > trunc->max_len)
+                                OVS_CB(skb)->cutlen = skb->len - trunc->max_len;
+                        break;
+                }
+
                 case OVS_ACTION_ATTR_USERSPACE:
-                        output_userspace(dp, skb, key, a, attr, len);
+                        output_userspace(dp, skb, key, a, attr,
+                                                     len, OVS_CB(skb)->cutlen);
+                        OVS_CB(skb)->cutlen = 0;
                         break;
 
                 case OVS_ACTION_ATTR_HASH:
diff --git a/net/openvswitch/conntrack.c b/net/openvswitch/conntrack.c
index d84312584ee4..b4069a90e375 100644
--- a/net/openvswitch/conntrack.c
+++ b/net/openvswitch/conntrack.c
@@ -834,6 +834,17 @@ static int ovs_ct_lookup(struct net *net, struct sw_flow_key *key,
         return 0;
 }
 
+static bool labels_nonzero(const struct ovs_key_ct_labels *labels)
+{
+        size_t i;
+
+        for (i = 0; i < sizeof(*labels); i++)
+                if (labels->ct_labels[i])
+                        return true;
+
+        return false;
+}
+
 /* Lookup connection and confirm if unconfirmed. */
 static int ovs_ct_commit(struct net *net, struct sw_flow_key *key,
                          const struct ovs_conntrack_info *info,
@@ -844,24 +855,32 @@ static int ovs_ct_commit(struct net *net, struct sw_flow_key *key,
         err = __ovs_ct_lookup(net, key, info, skb);
         if (err)
                 return err;
-        /* This is a no-op if the connection has already been confirmed. */
+
+        /* Apply changes before confirming the connection so that the initial
+         * conntrack NEW netlink event carries the values given in the CT
+         * action.
+         */
+        if (info->mark.mask) {
+                err = ovs_ct_set_mark(skb, key, info->mark.value,
+                                      info->mark.mask);
+                if (err)
+                        return err;
+        }
+        if (labels_nonzero(&info->labels.mask)) {
+                err = ovs_ct_set_labels(skb, key, &info->labels.value,
+                                        &info->labels.mask);
+                if (err)
+                        return err;
+        }
+        /* This will take care of sending queued events even if the connection
+         * is already confirmed.
+         */
         if (nf_conntrack_confirm(skb) != NF_ACCEPT)
                 return -EINVAL;
 
         return 0;
 }
 
-static bool labels_nonzero(const struct ovs_key_ct_labels *labels)
-{
-        size_t i;
-
-        for (i = 0; i < sizeof(*labels); i++)
-                if (labels->ct_labels[i])
-                        return true;
-
-        return false;
-}
-
 /* Returns 0 on success, -EINPROGRESS if 'skb' is stolen, or other nonzero
  * value if 'skb' is freed.
  */
@@ -886,19 +905,7 @@ int ovs_ct_execute(struct net *net, struct sk_buff *skb,
                 err = ovs_ct_commit(net, key, info, skb);
         else
                 err = ovs_ct_lookup(net, key, info, skb);
-        if (err)
-                goto err;
 
-        if (info->mark.mask) {
-                err = ovs_ct_set_mark(skb, key, info->mark.value,
-                                      info->mark.mask);
-                if (err)
-                        goto err;
-        }
-        if (labels_nonzero(&info->labels.mask))
-                err = ovs_ct_set_labels(skb, key, &info->labels.value,
-                                        &info->labels.mask);
-err:
         skb_push(skb, nh_ofs);
         if (err)
                 kfree_skb(skb);
@@ -1155,6 +1162,20 @@ static int parse_ct(const struct nlattr *attr, struct ovs_conntrack_info *info,
                 }
         }
 
+#ifdef CONFIG_NF_CONNTRACK_MARK
+        if (!info->commit && info->mark.mask) {
+                OVS_NLERR(log,
+                          "Setting conntrack mark requires 'commit' flag.");
+                return -EINVAL;
+        }
+#endif
+#ifdef CONFIG_NF_CONNTRACK_LABELS
+        if (!info->commit && labels_nonzero(&info->labels.mask)) {
+                OVS_NLERR(log,
+                          "Setting conntrack labels requires 'commit' flag.");
+                return -EINVAL;
+        }
+#endif
         if (rem > 0) {
                 OVS_NLERR(log, "Conntrack attr has %d unknown bytes", rem);
                 return -EINVAL;
diff --git a/net/openvswitch/datapath.c b/net/openvswitch/datapath.c
index 856bd8dba676..524c0fd3078e 100644
--- a/net/openvswitch/datapath.c
+++ b/net/openvswitch/datapath.c
@@ -137,10 +137,12 @@ EXPORT_SYMBOL_GPL(lockdep_ovsl_is_held);
 static struct vport *new_vport(const struct vport_parms *);
 static int queue_gso_packets(struct datapath *dp, struct sk_buff *,
                              const struct sw_flow_key *,
-                             const struct dp_upcall_info *);
+                             const struct dp_upcall_info *,
+                             uint32_t cutlen);
 static int queue_userspace_packet(struct datapath *dp, struct sk_buff *,
                                   const struct sw_flow_key *,
-                                  const struct dp_upcall_info *);
+                                  const struct dp_upcall_info *,
+                                  uint32_t cutlen);
 
 /* Must be called with rcu_read_lock. */
 static struct datapath *get_dp_rcu(struct net *net, int dp_ifindex)
@@ -275,7 +277,7 @@ void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key)
                 upcall.cmd = OVS_PACKET_CMD_MISS;
                 upcall.portid = ovs_vport_find_upcall_portid(p, skb);
                 upcall.mru = OVS_CB(skb)->mru;
-                error = ovs_dp_upcall(dp, skb, key, &upcall);
+                error = ovs_dp_upcall(dp, skb, key, &upcall, 0);
                 if (unlikely(error))
                         kfree_skb(skb);
                 else
@@ -300,7 +302,8 @@ out:
 
 int ovs_dp_upcall(struct datapath *dp, struct sk_buff *skb,
                   const struct sw_flow_key *key,
-                  const struct dp_upcall_info *upcall_info)
+                  const struct dp_upcall_info *upcall_info,
+                  uint32_t cutlen)
 {
         struct dp_stats_percpu *stats;
         int err;
@@ -311,9 +314,9 @@ int ovs_dp_upcall(struct datapath *dp, struct sk_buff *skb,
         }
 
         if (!skb_is_gso(skb))
-                err = queue_userspace_packet(dp, skb, key, upcall_info);
+                err = queue_userspace_packet(dp, skb, key, upcall_info, cutlen);
         else
-                err = queue_gso_packets(dp, skb, key, upcall_info);
+                err = queue_gso_packets(dp, skb, key, upcall_info, cutlen);
         if (err)
                 goto err;
 
@@ -331,7 +334,8 @@ err:
 
 static int queue_gso_packets(struct datapath *dp, struct sk_buff *skb,
                              const struct sw_flow_key *key,
-                             const struct dp_upcall_info *upcall_info)
+                             const struct dp_upcall_info *upcall_info,
+                                 uint32_t cutlen)
 {
         unsigned short gso_type = skb_shinfo(skb)->gso_type;
         struct sw_flow_key later_key;
@@ -360,7 +364,7 @@ static int queue_gso_packets(struct datapath *dp, struct sk_buff *skb,
                 if (gso_type & SKB_GSO_UDP && skb != segs)
                         key = &later_key;
 
-                err = queue_userspace_packet(dp, skb, key, upcall_info);
+                err = queue_userspace_packet(dp, skb, key, upcall_info, cutlen);
                 if (err)
                         break;
 
@@ -383,7 +387,8 @@ static size_t upcall_msg_size(const struct dp_upcall_info *upcall_info,
 {
         size_t size = NLMSG_ALIGN(sizeof(struct ovs_header))
                 + nla_total_size(hdrlen) /* OVS_PACKET_ATTR_PACKET */
-                + nla_total_size(ovs_key_attr_size()); /* OVS_PACKET_ATTR_KEY */
+                + nla_total_size(ovs_key_attr_size()) /* OVS_PACKET_ATTR_KEY */
+                + nla_total_size(sizeof(unsigned int)); /* OVS_PACKET_ATTR_LEN */
 
         /* OVS_PACKET_ATTR_USERDATA */
         if (upcall_info->userdata)
@@ -416,7 +421,8 @@ static void pad_packet(struct datapath *dp, struct sk_buff *skb)
 
 static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb,
                                   const struct sw_flow_key *key,
-                                  const struct dp_upcall_info *upcall_info)
+                                  const struct dp_upcall_info *upcall_info,
+                                  uint32_t cutlen)
 {
         struct ovs_header *upcall;
         struct sk_buff *nskb = NULL;
@@ -461,7 +467,7 @@ static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb,
         else
                 hlen = skb->len;
 
-        len = upcall_msg_size(upcall_info, hlen);
+        len = upcall_msg_size(upcall_info, hlen - cutlen);
         user_skb = genlmsg_new(len, GFP_ATOMIC);
         if (!user_skb) {
                 err = -ENOMEM;
@@ -509,15 +515,25 @@ static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb,
                 pad_packet(dp, user_skb);
         }
 
+        /* Add OVS_PACKET_ATTR_LEN when packet is truncated */
+        if (cutlen > 0) {
+                if (nla_put_u32(user_skb, OVS_PACKET_ATTR_LEN,
+                                skb->len)) {
+                        err = -ENOBUFS;
+                        goto out;
+                }
+                pad_packet(dp, user_skb);
+        }
+
         /* Only reserve room for attribute header, packet data is added
          * in skb_zerocopy() */
         if (!(nla = nla_reserve(user_skb, OVS_PACKET_ATTR_PACKET, 0))) {
                 err = -ENOBUFS;
                 goto out;
         }
-        nla->nla_len = nla_attr_size(skb->len);
+        nla->nla_len = nla_attr_size(skb->len - cutlen);
 
-        err = skb_zerocopy(user_skb, skb, skb->len, hlen);
+        err = skb_zerocopy(user_skb, skb, skb->len - cutlen, hlen);
         if (err)
                 goto out;
 
diff --git a/net/openvswitch/datapath.h b/net/openvswitch/datapath.h
index 427e39a045cf..ab85c1cae255 100644
--- a/net/openvswitch/datapath.h
+++ b/net/openvswitch/datapath.h
@@ -100,11 +100,13 @@ struct datapath {
  * @input_vport: The original vport packet came in on. This value is cached
  * when a packet is received by OVS.
  * @mru: The maximum received fragement size; 0 if the packet is not
+ * @cutlen: The number of bytes from the packet end to be removed.
  * fragmented.
  */
 struct ovs_skb_cb {
         struct vport            *input_vport;
         u16                     mru;
+        u32                     cutlen;
 };
 #define OVS_CB(skb) ((struct ovs_skb_cb *)(skb)->cb)
 
@@ -194,7 +196,8 @@ extern struct genl_family dp_vport_genl_family;
 void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key);
 void ovs_dp_detach_port(struct vport *);
 int ovs_dp_upcall(struct datapath *, struct sk_buff *,
-                  const struct sw_flow_key *, const struct dp_upcall_info *);
+                  const struct sw_flow_key *, const struct dp_upcall_info *,
+                  uint32_t cutlen);
 
 const char *ovs_dp_name(const struct datapath *dp);
 struct sk_buff *ovs_vport_cmd_build_info(struct vport *, u32 pid, u32 seq,
diff --git a/net/openvswitch/flow_netlink.c b/net/openvswitch/flow_netlink.c
index 0bb650f4f219..c78a6a1476fb 100644
--- a/net/openvswitch/flow_netlink.c
+++ b/net/openvswitch/flow_netlink.c
@@ -2229,6 +2229,7 @@ static int __ovs_nla_copy_actions(struct net *net, const struct nlattr *attr,
                         [OVS_ACTION_ATTR_SAMPLE] = (u32)-1,
                         [OVS_ACTION_ATTR_HASH] = sizeof(struct ovs_action_hash),
                         [OVS_ACTION_ATTR_CT] = (u32)-1,
+                        [OVS_ACTION_ATTR_TRUNC] = sizeof(struct ovs_action_trunc),
                 };
                 const struct ovs_action_push_vlan *vlan;
                 int type = nla_type(a);
@@ -2255,6 +2256,14 @@ static int __ovs_nla_copy_actions(struct net *net, const struct nlattr *attr,
                                 return -EINVAL;
                         break;
 
+                case OVS_ACTION_ATTR_TRUNC: {
+                        const struct ovs_action_trunc *trunc = nla_data(a);
+
+                        if (trunc->max_len < ETH_HLEN)
+                                return -EINVAL;
+                        break;
+                }
+
                 case OVS_ACTION_ATTR_HASH: {
                         const struct ovs_action_hash *act_hash = nla_data(a);
 
diff --git a/net/openvswitch/vport-internal_dev.c b/net/openvswitch/vport-internal_dev.c
index 2ee48e447b72..434e04c3a189 100644
--- a/net/openvswitch/vport-internal_dev.c
+++ b/net/openvswitch/vport-internal_dev.c
@@ -195,7 +195,7 @@ static struct vport *internal_dev_create(const struct vport_parms *parms)
         }
 
         vport->dev = alloc_netdev(sizeof(struct internal_dev),
-                                  parms->name, NET_NAME_UNKNOWN, do_setup);
+                                  parms->name, NET_NAME_USER, do_setup);
         if (!vport->dev) {
                 err = -ENOMEM;
                 goto error_free_vport;
diff --git a/net/openvswitch/vport.c b/net/openvswitch/vport.c
index 31cbc8c5c7db..6b21fd068d87 100644
--- a/net/openvswitch/vport.c
+++ b/net/openvswitch/vport.c
@@ -444,6 +444,7 @@ int ovs_vport_receive(struct vport *vport, struct sk_buff *skb,
 
         OVS_CB(skb)->input_vport = vport;
         OVS_CB(skb)->mru = 0;
+        OVS_CB(skb)->cutlen = 0;
         if (unlikely(dev_net(skb->dev) != ovs_dp_get_net(vport->dp))) {
                 u32 mark;
 
diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c
index 9bff6ef16fa7..d1f3b9e977e5 100644
--- a/net/packet/af_packet.c
+++ b/net/packet/af_packet.c
@@ -1979,40 +1979,8 @@ static int __packet_rcv_vnet(const struct sk_buff *skb,
 {
         *vnet_hdr = (const struct virtio_net_hdr) { 0 };
 
-        if (skb_is_gso(skb)) {
-                struct skb_shared_info *sinfo = skb_shinfo(skb);
-
-                /* This is a hint as to how much should be linear. */
-                vnet_hdr->hdr_len =
-                        __cpu_to_virtio16(vio_le(), skb_headlen(skb));
-                vnet_hdr->gso_size =
-                        __cpu_to_virtio16(vio_le(), sinfo->gso_size);
-
-                if (sinfo->gso_type & SKB_GSO_TCPV4)
-                        vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
-                else if (sinfo->gso_type & SKB_GSO_TCPV6)
-                        vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
-                else if (sinfo->gso_type & SKB_GSO_UDP)
-                        vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP;
-                else if (sinfo->gso_type & SKB_GSO_FCOE)
-                        return -EINVAL;
-                else
-                        BUG();
-
-                if (sinfo->gso_type & SKB_GSO_TCP_ECN)
-                        vnet_hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN;
-        } else
-                vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_NONE;
-
-        if (skb->ip_summed == CHECKSUM_PARTIAL) {
-                vnet_hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
-                vnet_hdr->csum_start = __cpu_to_virtio16(vio_le(),
-                                  skb_checksum_start_offset(skb));
-                vnet_hdr->csum_offset = __cpu_to_virtio16(vio_le(),
-                                                 skb->csum_offset);
-        } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
-                vnet_hdr->flags = VIRTIO_NET_HDR_F_DATA_VALID;
-        } /* else everything is zero */
+        if (virtio_net_hdr_from_skb(skb, vnet_hdr, vio_le()))
+                BUG();
 
         return 0;
 }
diff --git a/net/rds/cong.c b/net/rds/cong.c
index 6641bcf7c185..8398fee7c866 100644
--- a/net/rds/cong.c
+++ b/net/rds/cong.c
@@ -235,7 +235,8 @@ void rds_cong_queue_updates(struct rds_cong_map *map)
                          *    therefore trigger warnings.
                          * Defer the xmit to rds_send_worker() instead.
                          */
-                        queue_delayed_work(rds_wq, &conn->c_send_w, 0);
+                        queue_delayed_work(rds_wq,
+                                           &conn->c_path[0].cp_send_w, 0);
                 }
         }
 
diff --git a/net/rds/connection.c b/net/rds/connection.c
index e3b118cae81d..a4b07c899d89 100644
--- a/net/rds/connection.c
+++ b/net/rds/connection.c
@@ -95,14 +95,16 @@ static struct rds_connection *rds_conn_lookup(struct net *net,
  * and receiving over this connection again in the future.  It is up to
  * the transport to have serialized this call with its send and recv.
  */
-static void rds_conn_reset(struct rds_connection *conn)
+static void rds_conn_path_reset(struct rds_conn_path *cp)
 {
+        struct rds_connection *conn = cp->cp_conn;
+
         rdsdebug("connection %pI4 to %pI4 reset\n",
           &conn->c_laddr, &conn->c_faddr);
 
         rds_stats_inc(s_conn_reset);
-        rds_send_reset(conn);
-        conn->c_flags = 0;
+        rds_send_path_reset(cp);
+        cp->cp_flags = 0;
 
         /* Do not clear next_rx_seq here, else we cannot distinguish
          * retransmitted packets from new packets, and will hand all
@@ -110,6 +112,32 @@ static void rds_conn_reset(struct rds_connection *conn)
          * reliability guarantees of RDS. */
 }
 
+static void __rds_conn_path_init(struct rds_connection *conn,
+                                 struct rds_conn_path *cp, bool is_outgoing)
+{
+        spin_lock_init(&cp->cp_lock);
+        cp->cp_next_tx_seq = 1;
+        init_waitqueue_head(&cp->cp_waitq);
+        INIT_LIST_HEAD(&cp->cp_send_queue);
+        INIT_LIST_HEAD(&cp->cp_retrans);
+
+        cp->cp_conn = conn;
+        atomic_set(&cp->cp_state, RDS_CONN_DOWN);
+        cp->cp_send_gen = 0;
+        /* cp_outgoing is per-path. So we can only set it here
+         * for the single-path transports.
+         */
+        if (!conn->c_trans->t_mp_capable)
+                cp->cp_outgoing = (is_outgoing ? 1 : 0);
+        cp->cp_reconnect_jiffies = 0;
+        INIT_DELAYED_WORK(&cp->cp_send_w, rds_send_worker);
+        INIT_DELAYED_WORK(&cp->cp_recv_w, rds_recv_worker);
+        INIT_DELAYED_WORK(&cp->cp_conn_w, rds_connect_worker);
+        INIT_WORK(&cp->cp_down_w, rds_shutdown_worker);
+        mutex_init(&cp->cp_cm_lock);
+        cp->cp_flags = 0;
+}
+
 /*
  * There is only every one 'conn' for a given pair of addresses in the
  * system at a time.  They contain messages to be retransmitted and so
@@ -153,13 +181,8 @@ static struct rds_connection *__rds_conn_create(struct net *net,
         INIT_HLIST_NODE(&conn->c_hash_node);
         conn->c_laddr = laddr;
         conn->c_faddr = faddr;
-        spin_lock_init(&conn->c_lock);
-        conn->c_next_tx_seq = 1;
-        rds_conn_net_set(conn, net);
 
-        init_waitqueue_head(&conn->c_waitq);
-        INIT_LIST_HEAD(&conn->c_send_queue);
-        INIT_LIST_HEAD(&conn->c_retrans);
+        rds_conn_net_set(conn, net);
 
         ret = rds_cong_get_maps(conn);
         if (ret) {
@@ -195,17 +218,6 @@ static struct rds_connection *__rds_conn_create(struct net *net,
                 goto out;
         }
 
-        atomic_set(&conn->c_state, RDS_CONN_DOWN);
-        conn->c_send_gen = 0;
-        conn->c_outgoing = (is_outgoing ? 1 : 0);
-        conn->c_reconnect_jiffies = 0;
-        INIT_DELAYED_WORK(&conn->c_send_w, rds_send_worker);
-        INIT_DELAYED_WORK(&conn->c_recv_w, rds_recv_worker);
-        INIT_DELAYED_WORK(&conn->c_conn_w, rds_connect_worker);
-        INIT_WORK(&conn->c_down_w, rds_shutdown_worker);
-        mutex_init(&conn->c_cm_lock);
-        conn->c_flags = 0;
-
         rdsdebug("allocated conn %p for %pI4 -> %pI4 over %s %s\n",
           conn, &laddr, &faddr,
           trans->t_name ? trans->t_name : "[unknown]",
@@ -222,7 +234,7 @@ static struct rds_connection *__rds_conn_create(struct net *net,
         if (parent) {
                 /* Creating passive conn */
                 if (parent->c_passive) {
-                        trans->conn_free(conn->c_transport_data);
+                        trans->conn_free(conn->c_path[0].cp_transport_data);
                         kmem_cache_free(rds_conn_slab, conn);
                         conn = parent->c_passive;
                 } else {
@@ -236,10 +248,26 @@ static struct rds_connection *__rds_conn_create(struct net *net,
 
                 found = rds_conn_lookup(net, head, laddr, faddr, trans);
                 if (found) {
-                        trans->conn_free(conn->c_transport_data);
+                        struct rds_conn_path *cp;
+                        int i;
+
+                        for (i = 0; i < RDS_MPATH_WORKERS; i++) {
+                                cp = &conn->c_path[i];
+                                trans->conn_free(cp->cp_transport_data);
+                                if (!trans->t_mp_capable)
+                                        break;
+                        }
                         kmem_cache_free(rds_conn_slab, conn);
                         conn = found;
                 } else {
+                        int i;
+
+                        for (i = 0; i < RDS_MPATH_WORKERS; i++) {
+                                __rds_conn_path_init(conn, &conn->c_path[i],
+                                                     is_outgoing);
+                                conn->c_path[i].cp_index = i;
+                        }
+
                         hlist_add_head_rcu(&conn->c_hash_node, head);
                         rds_cong_add_conn(conn);
                         rds_conn_count++;
@@ -267,10 +295,12 @@ struct rds_connection *rds_conn_create_outgoing(struct net *net,
 }
 EXPORT_SYMBOL_GPL(rds_conn_create_outgoing);
 
-void rds_conn_shutdown(struct rds_connection *conn)
+void rds_conn_shutdown(struct rds_conn_path *cp)
 {
+        struct rds_connection *conn = cp->cp_conn;
+
         /* shut it down unless it's down already */
-        if (!rds_conn_transition(conn, RDS_CONN_DOWN, RDS_CONN_DOWN)) {
+        if (!rds_conn_path_transition(cp, RDS_CONN_DOWN, RDS_CONN_DOWN)) {
                 /*
                  * Quiesce the connection mgmt handlers before we start tearing
                  * things down. We don't hold the mutex for the entire
@@ -278,35 +308,41 @@ void rds_conn_shutdown(struct rds_connection *conn)
                  * deadlocking with the CM handler. Instead, the CM event
                  * handler is supposed to check for state DISCONNECTING
                  */
-                mutex_lock(&conn->c_cm_lock);
-                if (!rds_conn_transition(conn, RDS_CONN_UP, RDS_CONN_DISCONNECTING)
-                 && !rds_conn_transition(conn, RDS_CONN_ERROR, RDS_CONN_DISCONNECTING)) {
-                        rds_conn_error(conn, "shutdown called in state %d\n",
-                                        atomic_read(&conn->c_state));
-                        mutex_unlock(&conn->c_cm_lock);
+                mutex_lock(&cp->cp_cm_lock);
+                if (!rds_conn_path_transition(cp, RDS_CONN_UP,
+                                              RDS_CONN_DISCONNECTING) &&
+                    !rds_conn_path_transition(cp, RDS_CONN_ERROR,
+                                              RDS_CONN_DISCONNECTING)) {
+                        rds_conn_path_error(cp,
+                                            "shutdown called in state %d\n",
+                                            atomic_read(&cp->cp_state));
+                        mutex_unlock(&cp->cp_cm_lock);
                         return;
                 }
-                mutex_unlock(&conn->c_cm_lock);
+                mutex_unlock(&cp->cp_cm_lock);
 
-                wait_event(conn->c_waitq,
-                           !test_bit(RDS_IN_XMIT, &conn->c_flags));
-                wait_event(conn->c_waitq,
-                           !test_bit(RDS_RECV_REFILL, &conn->c_flags));
+                wait_event(cp->cp_waitq,
+                           !test_bit(RDS_IN_XMIT, &cp->cp_flags));
+                wait_event(cp->cp_waitq,
+                           !test_bit(RDS_RECV_REFILL, &cp->cp_flags));
 
-                conn->c_trans->conn_shutdown(conn);
-                rds_conn_reset(conn);
+                if (!conn->c_trans->t_mp_capable)
+                        conn->c_trans->conn_shutdown(conn);
+                else
+                        conn->c_trans->conn_path_shutdown(cp);
+                rds_conn_path_reset(cp);
 
-                if (!rds_conn_transition(conn, RDS_CONN_DISCONNECTING, RDS_CONN_DOWN)) {
+                if (!rds_conn_path_transition(cp, RDS_CONN_DISCONNECTING,
+                                              RDS_CONN_DOWN)) {
                         /* This can happen - eg when we're in the middle of tearing
                          * down the connection, and someone unloads the rds module.
                          * Quite reproduceable with loopback connections.
                          * Mostly harmless.
                          */
-                        rds_conn_error(conn,
-                                "%s: failed to transition to state DOWN, "
-                                "current state is %d\n",
-                                __func__,
-                                atomic_read(&conn->c_state));
+                        rds_conn_path_error(cp, "%s: failed to transition "
+                                            "to state DOWN, current state "
+                                            "is %d\n", __func__,
+                                            atomic_read(&cp->cp_state));
                         return;
                 }
         }
@@ -315,18 +351,46 @@ void rds_conn_shutdown(struct rds_connection *conn)
          * The passive side of an IB loopback connection is never added
          * to the conn hash, so we never trigger a reconnect on this
          * conn - the reconnect is always triggered by the active peer. */
-        cancel_delayed_work_sync(&conn->c_conn_w);
+        cancel_delayed_work_sync(&cp->cp_conn_w);
         rcu_read_lock();
         if (!hlist_unhashed(&conn->c_hash_node)) {
                 rcu_read_unlock();
                 if (conn->c_trans->t_type != RDS_TRANS_TCP ||
-                    conn->c_outgoing == 1)
-                        rds_queue_reconnect(conn);
+                    cp->cp_outgoing == 1)
+                        rds_queue_reconnect(cp);
         } else {
                 rcu_read_unlock();
         }
 }
 
+/* destroy a single rds_conn_path. rds_conn_destroy() iterates over
+ * all paths using rds_conn_path_destroy()
+ */
+static void rds_conn_path_destroy(struct rds_conn_path *cp)
+{
+        struct rds_message *rm, *rtmp;
+
+        rds_conn_path_drop(cp);
+        flush_work(&cp->cp_down_w);
+
+        /* make sure lingering queued work won't try to ref the conn */
+        cancel_delayed_work_sync(&cp->cp_send_w);
+        cancel_delayed_work_sync(&cp->cp_recv_w);
+
+        /* tear down queued messages */
+        list_for_each_entry_safe(rm, rtmp,
+                                 &cp->cp_send_queue,
+                                 m_conn_item) {
+                list_del_init(&rm->m_conn_item);
+                BUG_ON(!list_empty(&rm->m_sock_item));
+                rds_message_put(rm);
+        }
+        if (cp->cp_xmit_rm)
+                rds_message_put(cp->cp_xmit_rm);
+
+        cp->cp_conn->c_trans->conn_free(cp->cp_transport_data);
+}
+
 /*
  * Stop and free a connection.
  *
@@ -336,7 +400,6 @@ void rds_conn_shutdown(struct rds_connection *conn)
  */
 void rds_conn_destroy(struct rds_connection *conn)
 {
-        struct rds_message *rm, *rtmp;
         unsigned long flags;
 
         rdsdebug("freeing conn %p for %pI4 -> "
@@ -350,25 +413,19 @@ void rds_conn_destroy(struct rds_connection *conn)
         synchronize_rcu();
 
         /* shut the connection down */
-        rds_conn_drop(conn);
-        flush_work(&conn->c_down_w);
-
-        /* make sure lingering queued work won't try to ref the conn */
-        cancel_delayed_work_sync(&conn->c_send_w);
-        cancel_delayed_work_sync(&conn->c_recv_w);
+        if (!conn->c_trans->t_mp_capable) {
+                rds_conn_path_destroy(&conn->c_path[0]);
+                BUG_ON(!list_empty(&conn->c_path[0].cp_retrans));
+        } else {
+                int i;
+                struct rds_conn_path *cp;
 
-        /* tear down queued messages */
-        list_for_each_entry_safe(rm, rtmp,
-                                 &conn->c_send_queue,
-                                 m_conn_item) {
-                list_del_init(&rm->m_conn_item);
-                BUG_ON(!list_empty(&rm->m_sock_item));
-                rds_message_put(rm);
+                for (i = 0; i < RDS_MPATH_WORKERS; i++) {
+                        cp = &conn->c_path[i];
+                        rds_conn_path_destroy(cp);
+                        BUG_ON(!list_empty(&cp->cp_retrans));
+                }
         }
-        if (conn->c_xmit_rm)
-                rds_message_put(conn->c_xmit_rm);
-
-        conn->c_trans->conn_free(conn->c_transport_data);
 
         /*
          * The congestion maps aren't freed up here.  They're
@@ -377,7 +434,6 @@ void rds_conn_destroy(struct rds_connection *conn)
          */
         rds_cong_remove_conn(conn);
 
-        BUG_ON(!list_empty(&conn->c_retrans));
         kmem_cache_free(rds_conn_slab, conn);
 
         spin_lock_irqsave(&rds_conn_lock, flags);
@@ -398,6 +454,7 @@ static void rds_conn_message_info(struct socket *sock, unsigned int len,
         unsigned int total = 0;
         unsigned long flags;
         size_t i;
+        int j;
 
         len /= sizeof(struct rds_info_message);
 
@@ -406,23 +463,32 @@ static void rds_conn_message_info(struct socket *sock, unsigned int len,
         for (i = 0, head = rds_conn_hash; i < ARRAY_SIZE(rds_conn_hash);
              i++, head++) {
                 hlist_for_each_entry_rcu(conn, head, c_hash_node) {
-                        if (want_send)
-                                list = &conn->c_send_queue;
-                        else
-                                list = &conn->c_retrans;
-
-                        spin_lock_irqsave(&conn->c_lock, flags);
-
-                        /* XXX too lazy to maintain counts.. */
-                        list_for_each_entry(rm, list, m_conn_item) {
-                                total++;
-                                if (total <= len)
-                                        rds_inc_info_copy(&rm->m_inc, iter,
-                                                          conn->c_laddr,
-                                                          conn->c_faddr, 0);
+                        struct rds_conn_path *cp;
+
+                        for (j = 0; j < RDS_MPATH_WORKERS; j++) {
+                                cp = &conn->c_path[j];
+                                if (want_send)
+                                        list = &cp->cp_send_queue;
+                                else
+                                        list = &cp->cp_retrans;
+
+                                spin_lock_irqsave(&cp->cp_lock, flags);
+
+                                /* XXX too lazy to maintain counts.. */
+                                list_for_each_entry(rm, list, m_conn_item) {
+                                        total++;
+                                        if (total <= len)
+                                                rds_inc_info_copy(&rm->m_inc,
+                                                                  iter,
+                                                                  conn->c_laddr,
+                                                                  conn->c_faddr,
+                                                                  0);
+                                }
+
+                                spin_unlock_irqrestore(&cp->cp_lock, flags);
+                                if (!conn->c_trans->t_mp_capable)
+                                        break;
                         }
-
-                        spin_unlock_irqrestore(&conn->c_lock, flags);
                 }
         }
         rcu_read_unlock();
@@ -484,27 +550,72 @@ void rds_for_each_conn_info(struct socket *sock, unsigned int len,
 }
 EXPORT_SYMBOL_GPL(rds_for_each_conn_info);
 
-static int rds_conn_info_visitor(struct rds_connection *conn,
-                                  void *buffer)
+void rds_walk_conn_path_info(struct socket *sock, unsigned int len,
+                             struct rds_info_iterator *iter,
+                             struct rds_info_lengths *lens,
+                             int (*visitor)(struct rds_conn_path *, void *),
+                             size_t item_len)
+{
+        u64  buffer[(item_len + 7) / 8];
+        struct hlist_head *head;
+        struct rds_connection *conn;
+        size_t i;
+        int j;
+
+        rcu_read_lock();
+
+        lens->nr = 0;
+        lens->each = item_len;
+
+        for (i = 0, head = rds_conn_hash; i < ARRAY_SIZE(rds_conn_hash);
+             i++, head++) {
+                hlist_for_each_entry_rcu(conn, head, c_hash_node) {
+                        struct rds_conn_path *cp;
+
+                        for (j = 0; j < RDS_MPATH_WORKERS; j++) {
+                                cp = &conn->c_path[j];
+
+                                /* XXX no cp_lock usage.. */
+                                if (!visitor(cp, buffer))
+                                        continue;
+                                if (!conn->c_trans->t_mp_capable)
+                                        break;
+                        }
+
+                        /* We copy as much as we can fit in the buffer,
+                         * but we count all items so that the caller
+                         * can resize the buffer.
+                         */
+                        if (len >= item_len) {
+                                rds_info_copy(iter, buffer, item_len);
+                                len -= item_len;
+                        }
+                        lens->nr++;
+                }
+        }
+        rcu_read_unlock();
+}
+
+static int rds_conn_info_visitor(struct rds_conn_path *cp, void *buffer)
 {
         struct rds_info_connection *cinfo = buffer;
 
-        cinfo->next_tx_seq = conn->c_next_tx_seq;
-        cinfo->next_rx_seq = conn->c_next_rx_seq;
-        cinfo->laddr = conn->c_laddr;
-        cinfo->faddr = conn->c_faddr;
-        strncpy(cinfo->transport, conn->c_trans->t_name,
+        cinfo->next_tx_seq = cp->cp_next_tx_seq;
+        cinfo->next_rx_seq = cp->cp_next_rx_seq;
+        cinfo->laddr = cp->cp_conn->c_laddr;
+        cinfo->faddr = cp->cp_conn->c_faddr;
+        strncpy(cinfo->transport, cp->cp_conn->c_trans->t_name,
                 sizeof(cinfo->transport));
         cinfo->flags = 0;
 
-        rds_conn_info_set(cinfo->flags, test_bit(RDS_IN_XMIT, &conn->c_flags),
+        rds_conn_info_set(cinfo->flags, test_bit(RDS_IN_XMIT, &cp->cp_flags),
                           SENDING);
         /* XXX Future: return the state rather than these funky bits */
         rds_conn_info_set(cinfo->flags,
-                          atomic_read(&conn->c_state) == RDS_CONN_CONNECTING,
+                          atomic_read(&cp->cp_state) == RDS_CONN_CONNECTING,
                           CONNECTING);
         rds_conn_info_set(cinfo->flags,
-                          atomic_read(&conn->c_state) == RDS_CONN_UP,
+                          atomic_read(&cp->cp_state) == RDS_CONN_UP,
                           CONNECTED);
         return 1;
 }
@@ -513,7 +624,7 @@ static void rds_conn_info(struct socket *sock, unsigned int len,
                           struct rds_info_iterator *iter,
                           struct rds_info_lengths *lens)
 {
-        rds_for_each_conn_info(sock, len, iter, lens,
+        rds_walk_conn_path_info(sock, len, iter, lens,
                                 rds_conn_info_visitor,
                                 sizeof(struct rds_info_connection));
 }
@@ -553,10 +664,16 @@ void rds_conn_exit(void)
 /*
  * Force a disconnect
  */
+void rds_conn_path_drop(struct rds_conn_path *cp)
+{
+        atomic_set(&cp->cp_state, RDS_CONN_ERROR);
+        queue_work(rds_wq, &cp->cp_down_w);
+}
+EXPORT_SYMBOL_GPL(rds_conn_path_drop);
+
 void rds_conn_drop(struct rds_connection *conn)
 {
-        atomic_set(&conn->c_state, RDS_CONN_ERROR);
-        queue_work(rds_wq, &conn->c_down_w);
+        rds_conn_path_drop(&conn->c_path[0]);
 }
 EXPORT_SYMBOL_GPL(rds_conn_drop);
 
@@ -564,11 +681,17 @@ EXPORT_SYMBOL_GPL(rds_conn_drop);
  * If the connection is down, trigger a connect. We may have scheduled a
  * delayed reconnect however - in this case we should not interfere.
  */
+void rds_conn_path_connect_if_down(struct rds_conn_path *cp)
+{
+        if (rds_conn_path_state(cp) == RDS_CONN_DOWN &&
+            !test_and_set_bit(RDS_RECONNECT_PENDING, &cp->cp_flags))
+                queue_delayed_work(rds_wq, &cp->cp_conn_w, 0);
+}
+
 void rds_conn_connect_if_down(struct rds_connection *conn)
 {
-        if (rds_conn_state(conn) == RDS_CONN_DOWN &&
-            !test_and_set_bit(RDS_RECONNECT_PENDING, &conn->c_flags))
-                queue_delayed_work(rds_wq, &conn->c_conn_w, 0);
+        WARN_ON(conn->c_trans->t_mp_capable);
+        rds_conn_path_connect_if_down(&conn->c_path[0]);
 }
 EXPORT_SYMBOL_GPL(rds_conn_connect_if_down);
 
@@ -586,3 +709,15 @@ __rds_conn_error(struct rds_connection *conn, const char *fmt, ...)
 
         rds_conn_drop(conn);
 }
+
+void
+__rds_conn_path_error(struct rds_conn_path *cp, const char *fmt, ...)
+{
+        va_list ap;
+
+        va_start(ap, fmt);
+        vprintk(fmt, ap);
+        va_end(ap);
+
+        rds_conn_path_drop(cp);
+}
diff --git a/net/rds/ib.c b/net/rds/ib.c
index b5342fddaf98..44946a681a8c 100644
--- a/net/rds/ib.c
+++ b/net/rds/ib.c
@@ -40,6 +40,7 @@
 #include <linux/slab.h>
 #include <linux/module.h>
 
+#include "rds_single_path.h"
 #include "rds.h"
 #include "ib.h"
 #include "ib_mr.h"
diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c
index 7c2a65a6af5c..e48bb1ba3dfc 100644
--- a/net/rds/ib_cm.c
+++ b/net/rds/ib_cm.c
@@ -36,6 +36,7 @@
 #include <linux/vmalloc.h>
 #include <linux/ratelimit.h>
 
+#include "rds_single_path.h"
 #include "rds.h"
 #include "ib.h"
 
@@ -273,7 +274,7 @@ static void rds_ib_tasklet_fn_send(unsigned long data)
         if (rds_conn_up(conn) &&
             (!test_bit(RDS_LL_SEND_FULL, &conn->c_flags) ||
             test_bit(0, &conn->c_map_queued)))
-                rds_send_xmit(ic->conn);
+                rds_send_xmit(&ic->conn->c_path[0]);
 }
 
 static void poll_rcq(struct rds_ib_connection *ic, struct ib_cq *cq,
diff --git a/net/rds/ib_rdma.c b/net/rds/ib_rdma.c
index f7164ac1ffc1..977f69886c00 100644
--- a/net/rds/ib_rdma.c
+++ b/net/rds/ib_rdma.c
@@ -35,6 +35,7 @@
 #include <linux/rculist.h>
 #include <linux/llist.h>
 
+#include "rds_single_path.h"
 #include "ib_mr.h"
 
 struct workqueue_struct *rds_ib_mr_wq;
@@ -618,7 +619,7 @@ struct rds_ib_mr_pool *rds_ib_create_mr_pool(struct rds_ib_device *rds_ibdev,
 
 int rds_ib_mr_init(void)
 {
-        rds_ib_mr_wq = create_workqueue("rds_mr_flushd");
+        rds_ib_mr_wq = alloc_workqueue("rds_mr_flushd", WQ_MEM_RECLAIM, 0);
         if (!rds_ib_mr_wq)
                 return -ENOMEM;
         return 0;
diff --git a/net/rds/ib_recv.c b/net/rds/ib_recv.c
index abc8cc805e8d..4ea8cb17cc7a 100644
--- a/net/rds/ib_recv.c
+++ b/net/rds/ib_recv.c
@@ -36,6 +36,7 @@
 #include <linux/dma-mapping.h>
 #include <rdma/rdma_cm.h>
 
+#include "rds_single_path.h"
 #include "rds.h"
 #include "ib.h"
 
diff --git a/net/rds/ib_send.c b/net/rds/ib_send.c
index f27d2c82b036..6e4110aa5135 100644
--- a/net/rds/ib_send.c
+++ b/net/rds/ib_send.c
@@ -36,6 +36,7 @@
 #include <linux/dmapool.h>
 #include <linux/ratelimit.h>
 
+#include "rds_single_path.h"
 #include "rds.h"
 #include "ib.h"
 
diff --git a/net/rds/loop.c b/net/rds/loop.c
index 814173b466d9..15f83db78f0c 100644
--- a/net/rds/loop.c
+++ b/net/rds/loop.c
@@ -34,6 +34,7 @@
 #include <linux/slab.h>
 #include <linux/in.h>
 
+#include "rds_single_path.h"
 #include "rds.h"
 #include "loop.h"
 
diff --git a/net/rds/rdma_transport.c b/net/rds/rdma_transport.c
index 7220bebcf558..345f09059e9f 100644
--- a/net/rds/rdma_transport.c
+++ b/net/rds/rdma_transport.c
@@ -33,6 +33,7 @@
 #include <linux/module.h>
 #include <rdma/rdma_cm.h>
 
+#include "rds_single_path.h"
 #include "rdma_transport.h"
 #include "ib.h"
 
diff --git a/net/rds/rds.h b/net/rds/rds.h
index 387df5f32e49..2e35b738176f 100644
--- a/net/rds/rds.h
+++ b/net/rds/rds.h
@@ -84,56 +84,69 @@ enum {
 #define RDS_IN_XMIT             2
 #define RDS_RECV_REFILL         3
 
+/* Max number of multipaths per RDS connection. Must be a power of 2 */
+#define RDS_MPATH_WORKERS       1
+
+/* Per mpath connection state */
+struct rds_conn_path {
+        struct rds_connection   *cp_conn;
+        struct rds_message      *cp_xmit_rm;
+        unsigned long           cp_xmit_sg;
+        unsigned int            cp_xmit_hdr_off;
+        unsigned int            cp_xmit_data_off;
+        unsigned int            cp_xmit_atomic_sent;
+        unsigned int            cp_xmit_rdma_sent;
+        unsigned int            cp_xmit_data_sent;
+
+        spinlock_t              cp_lock;                /* protect msg queues */
+        u64                     cp_next_tx_seq;
+        struct list_head        cp_send_queue;
+        struct list_head        cp_retrans;
+
+        u64                     cp_next_rx_seq;
+
+        void                    *cp_transport_data;
+
+        atomic_t                cp_state;
+        unsigned long           cp_send_gen;
+        unsigned long           cp_flags;
+        unsigned long           cp_reconnect_jiffies;
+        struct delayed_work     cp_send_w;
+        struct delayed_work     cp_recv_w;
+        struct delayed_work     cp_conn_w;
+        struct work_struct      cp_down_w;
+        struct mutex            cp_cm_lock;     /* protect cp_state & cm */
+        wait_queue_head_t       cp_waitq;
+
+        unsigned int            cp_unacked_packets;
+        unsigned int            cp_unacked_bytes;
+        unsigned int            cp_outgoing:1,
+                                cp_pad_to_32:31;
+        unsigned int            cp_index;
+};
+
+/* One rds_connection per RDS address pair */
 struct rds_connection {
         struct hlist_node       c_hash_node;
         __be32                  c_laddr;
         __be32                  c_faddr;
         unsigned int            c_loopback:1,
-                                c_outgoing:1,
-                                c_pad_to_32:30;
+                                c_pad_to_32:31;
+        int                     c_npaths;
         struct rds_connection   *c_passive;
+        struct rds_transport    *c_trans;
 
         struct rds_cong_map     *c_lcong;
         struct rds_cong_map     *c_fcong;
 
-        struct rds_message      *c_xmit_rm;
-        unsigned long           c_xmit_sg;
-        unsigned int            c_xmit_hdr_off;
-        unsigned int            c_xmit_data_off;
-        unsigned int            c_xmit_atomic_sent;
-        unsigned int            c_xmit_rdma_sent;
-        unsigned int            c_xmit_data_sent;
-
-        spinlock_t              c_lock;         /* protect msg queues */
-        u64                     c_next_tx_seq;
-        struct list_head        c_send_queue;
-        struct list_head        c_retrans;
-
-        u64                     c_next_rx_seq;
-
-        struct rds_transport    *c_trans;
-        void                    *c_transport_data;
-
-        atomic_t                c_state;
-        unsigned long           c_send_gen;
-        unsigned long           c_flags;
-        unsigned long           c_reconnect_jiffies;
-        struct delayed_work     c_send_w;
-        struct delayed_work     c_recv_w;
-        struct delayed_work     c_conn_w;
-        struct work_struct      c_down_w;
-        struct mutex            c_cm_lock;      /* protect conn state & cm */
-        wait_queue_head_t       c_waitq;
+        /* Protocol version */
+        unsigned int            c_version;
+        possible_net_t          c_net;
 
         struct list_head        c_map_item;
         unsigned long           c_map_queued;
 
-        unsigned int            c_unacked_packets;
-        unsigned int            c_unacked_bytes;
-
-        /* Protocol version */
-        unsigned int            c_version;
-        possible_net_t          c_net;
+        struct rds_conn_path    c_path[RDS_MPATH_WORKERS];
 };
 
 static inline
@@ -218,6 +231,7 @@ struct rds_incoming {
         atomic_t                i_refcount;
         struct list_head        i_item;
         struct rds_connection   *i_conn;
+        struct rds_conn_path    *i_conn_path;
         struct rds_header       i_hdr;
         unsigned long           i_rx_jiffies;
         __be32                  i_saddr;
@@ -433,7 +447,8 @@ struct rds_transport {
         char                    t_name[TRANSNAMSIZ];
         struct list_head        t_item;
         struct module           *t_owner;
-        unsigned int            t_prefer_loopback:1;
+        unsigned int            t_prefer_loopback:1,
+                                t_mp_capable:1;
         unsigned int            t_type;
 
         int (*laddr_check)(struct net *net, __be32 addr);
@@ -441,8 +456,11 @@ struct rds_transport {
         void (*conn_free)(void *data);
         int (*conn_connect)(struct rds_connection *conn);
         void (*conn_shutdown)(struct rds_connection *conn);
+        void (*conn_path_shutdown)(struct rds_conn_path *conn);
         void (*xmit_prepare)(struct rds_connection *conn);
+        void (*xmit_path_prepare)(struct rds_conn_path *cp);
         void (*xmit_complete)(struct rds_connection *conn);
+        void (*xmit_path_complete)(struct rds_conn_path *cp);
         int (*xmit)(struct rds_connection *conn, struct rds_message *rm,
                     unsigned int hdr_off, unsigned int sg, unsigned int off);
         int (*xmit_rdma)(struct rds_connection *conn, struct rm_rdma_op *op);
@@ -636,10 +654,12 @@ struct rds_connection *rds_conn_create(struct net *net,
 struct rds_connection *rds_conn_create_outgoing(struct net *net,
                                                 __be32 laddr, __be32 faddr,
                                struct rds_transport *trans, gfp_t gfp);
-void rds_conn_shutdown(struct rds_connection *conn);
+void rds_conn_shutdown(struct rds_conn_path *cpath);
 void rds_conn_destroy(struct rds_connection *conn);
 void rds_conn_drop(struct rds_connection *conn);
+void rds_conn_path_drop(struct rds_conn_path *cpath);
 void rds_conn_connect_if_down(struct rds_connection *conn);
+void rds_conn_path_connect_if_down(struct rds_conn_path *cp);
 void rds_for_each_conn_info(struct socket *sock, unsigned int len,
                           struct rds_info_iterator *iter,
                           struct rds_info_lengths *lens,
@@ -650,28 +670,60 @@ void __rds_conn_error(struct rds_connection *conn, const char *, ...);
 #define rds_conn_error(conn, fmt...) \
         __rds_conn_error(conn, KERN_WARNING "RDS: " fmt)
 
+void __rds_conn_path_error(struct rds_conn_path *cp, const char *, ...);
+#define rds_conn_path_error(cp, fmt...) \
+        __rds_conn_path_error(cp, KERN_WARNING "RDS: " fmt)
+
+static inline int
+rds_conn_path_transition(struct rds_conn_path *cp, int old, int new)
+{
+        return atomic_cmpxchg(&cp->cp_state, old, new) == old;
+}
+
 static inline int
 rds_conn_transition(struct rds_connection *conn, int old, int new)
 {
-        return atomic_cmpxchg(&conn->c_state, old, new) == old;
+        WARN_ON(conn->c_trans->t_mp_capable);
+        return rds_conn_path_transition(&conn->c_path[0], old, new);
+}
+
+static inline int
+rds_conn_path_state(struct rds_conn_path *cp)
+{
+        return atomic_read(&cp->cp_state);
 }
 
 static inline int
 rds_conn_state(struct rds_connection *conn)
 {
-        return atomic_read(&conn->c_state);
+        WARN_ON(conn->c_trans->t_mp_capable);
+        return rds_conn_path_state(&conn->c_path[0]);
+}
+
+static inline int
+rds_conn_path_up(struct rds_conn_path *cp)
+{
+        return atomic_read(&cp->cp_state) == RDS_CONN_UP;
 }
 
 static inline int
 rds_conn_up(struct rds_connection *conn)
 {
-        return atomic_read(&conn->c_state) == RDS_CONN_UP;
+        WARN_ON(conn->c_trans->t_mp_capable);
+        return rds_conn_path_up(&conn->c_path[0]);
+}
+
+static inline int
+rds_conn_path_connecting(struct rds_conn_path *cp)
+{
+        return atomic_read(&cp->cp_state) == RDS_CONN_CONNECTING;
 }
 
 static inline int
 rds_conn_connecting(struct rds_connection *conn)
 {
-        return atomic_read(&conn->c_state) == RDS_CONN_CONNECTING;
+        WARN_ON(conn->c_trans->t_mp_capable);
+        return rds_conn_path_connecting(&conn->c_path[0]);
 }
 
 /* message.c */
@@ -720,6 +772,8 @@ void rds_page_exit(void);
 /* recv.c */
 void rds_inc_init(struct rds_incoming *inc, struct rds_connection *conn,
                   __be32 saddr);
+void rds_inc_path_init(struct rds_incoming *inc, struct rds_conn_path *conn,
+                       __be32 saddr);
 void rds_inc_put(struct rds_incoming *inc);
 void rds_recv_incoming(struct rds_connection *conn, __be32 saddr, __be32 daddr,
                        struct rds_incoming *inc, gfp_t gfp);
@@ -733,16 +787,16 @@ void rds_inc_info_copy(struct rds_incoming *inc,
 
 /* send.c */
 int rds_sendmsg(struct socket *sock, struct msghdr *msg, size_t payload_len);
-void rds_send_reset(struct rds_connection *conn);
-int rds_send_xmit(struct rds_connection *conn);
+void rds_send_path_reset(struct rds_conn_path *conn);
+int rds_send_xmit(struct rds_conn_path *cp);
 struct sockaddr_in;
 void rds_send_drop_to(struct rds_sock *rs, struct sockaddr_in *dest);
 typedef int (*is_acked_func)(struct rds_message *rm, uint64_t ack);
 void rds_send_drop_acked(struct rds_connection *conn, u64 ack,
                          is_acked_func is_acked);
-int rds_send_pong(struct rds_connection *conn, __be16 dport);
-struct rds_message *rds_send_get_message(struct rds_connection *,
-                                         struct rm_rdma_op *);
+void rds_send_path_drop_acked(struct rds_conn_path *cp, u64 ack,
+                              is_acked_func is_acked);
+int rds_send_pong(struct rds_conn_path *cp, __be16 dport);
 
 /* rdma.c */
 void rds_rdma_unuse(struct rds_sock *rs, u32 r_key, int force);
@@ -809,12 +863,12 @@ extern unsigned int  rds_sysctl_trace_level;
 int rds_threads_init(void);
 void rds_threads_exit(void);
 extern struct workqueue_struct *rds_wq;
-void rds_queue_reconnect(struct rds_connection *conn);
+void rds_queue_reconnect(struct rds_conn_path *cp);
 void rds_connect_worker(struct work_struct *);
 void rds_shutdown_worker(struct work_struct *);
 void rds_send_worker(struct work_struct *);
 void rds_recv_worker(struct work_struct *);
-void rds_connect_path_complete(struct rds_connection *conn, int curr);
+void rds_connect_path_complete(struct rds_conn_path *conn, int curr);
 void rds_connect_complete(struct rds_connection *conn);
 
 /* transport.c */
diff --git a/net/rds/rds_single_path.h b/net/rds/rds_single_path.h
new file mode 100644
index 000000000000..e1241af7c1ad
--- /dev/null
+++ b/net/rds/rds_single_path.h
@@ -0,0 +1,30 @@
+#ifndef _RDS_RDS_SINGLE_H
+#define _RDS_RDS_SINGLE_H
+
+#define c_xmit_rm               c_path[0].cp_xmit_rm
+#define c_xmit_sg               c_path[0].cp_xmit_sg
+#define c_xmit_hdr_off          c_path[0].cp_xmit_hdr_off
+#define c_xmit_data_off         c_path[0].cp_xmit_data_off
+#define c_xmit_atomic_sent      c_path[0].cp_xmit_atomic_sent
+#define c_xmit_rdma_sent        c_path[0].cp_xmit_rdma_sent
+#define c_xmit_data_sent        c_path[0].cp_xmit_data_sent
+#define c_lock                  c_path[0].cp_lock
+#define c_next_tx_seq           c_path[0].cp_next_tx_seq
+#define c_send_queue            c_path[0].cp_send_queue
+#define c_retrans               c_path[0].cp_retrans
+#define c_next_rx_seq           c_path[0].cp_next_rx_seq
+#define c_transport_data        c_path[0].cp_transport_data
+#define c_state                 c_path[0].cp_state
+#define c_send_gen              c_path[0].cp_send_gen
+#define c_flags                 c_path[0].cp_flags
+#define c_reconnect_jiffies     c_path[0].cp_reconnect_jiffies
+#define c_send_w                c_path[0].cp_send_w
+#define c_recv_w                c_path[0].cp_recv_w
+#define c_conn_w                c_path[0].cp_conn_w
+#define c_down_w                c_path[0].cp_down_w
+#define c_cm_lock               c_path[0].cp_cm_lock
+#define c_waitq                 c_path[0].cp_waitq
+#define c_unacked_packets       c_path[0].cp_unacked_packets
+#define c_unacked_bytes         c_path[0].cp_unacked_bytes
+
+#endif /* _RDS_RDS_SINGLE_H */
diff --git a/net/rds/recv.c b/net/rds/recv.c
index 8413f6c99e13..b58f50571782 100644
--- a/net/rds/recv.c
+++ b/net/rds/recv.c
@@ -53,6 +53,20 @@ void rds_inc_init(struct rds_incoming *inc, struct rds_connection *conn,
 }
 EXPORT_SYMBOL_GPL(rds_inc_init);
 
+void rds_inc_path_init(struct rds_incoming *inc, struct rds_conn_path *cp,
+                       __be32 saddr)
+{
+        atomic_set(&inc->i_refcount, 1);
+        INIT_LIST_HEAD(&inc->i_item);
+        inc->i_conn = cp->cp_conn;
+        inc->i_conn_path = cp;
+        inc->i_saddr = saddr;
+        inc->i_rdma_cookie = 0;
+        inc->i_rx_tstamp.tv_sec = 0;
+        inc->i_rx_tstamp.tv_usec = 0;
+}
+EXPORT_SYMBOL_GPL(rds_inc_path_init);
+
 static void rds_inc_addref(struct rds_incoming *inc)
 {
         rdsdebug("addref inc %p ref %d\n", inc, atomic_read(&inc->i_refcount));
@@ -164,13 +178,18 @@ void rds_recv_incoming(struct rds_connection *conn, __be32 saddr, __be32 daddr,
         struct rds_sock *rs = NULL;
         struct sock *sk;
         unsigned long flags;
+        struct rds_conn_path *cp;
 
         inc->i_conn = conn;
         inc->i_rx_jiffies = jiffies;
+        if (conn->c_trans->t_mp_capable)
+                cp = inc->i_conn_path;
+        else
+                cp = &conn->c_path[0];
 
         rdsdebug("conn %p next %llu inc %p seq %llu len %u sport %u dport %u "
                  "flags 0x%x rx_jiffies %lu\n", conn,
-                 (unsigned long long)conn->c_next_rx_seq,
+                 (unsigned long long)cp->cp_next_rx_seq,
                  inc,
                  (unsigned long long)be64_to_cpu(inc->i_hdr.h_sequence),
                  be32_to_cpu(inc->i_hdr.h_len),
@@ -199,16 +218,16 @@ void rds_recv_incoming(struct rds_connection *conn, __be32 saddr, __be32 daddr,
          * XXX we could spend more on the wire to get more robust failure
          * detection, arguably worth it to avoid data corruption.
          */
-        if (be64_to_cpu(inc->i_hdr.h_sequence) < conn->c_next_rx_seq &&
+        if (be64_to_cpu(inc->i_hdr.h_sequence) < cp->cp_next_rx_seq &&
             (inc->i_hdr.h_flags & RDS_FLAG_RETRANSMITTED)) {
                 rds_stats_inc(s_recv_drop_old_seq);
                 goto out;
         }
-        conn->c_next_rx_seq = be64_to_cpu(inc->i_hdr.h_sequence) + 1;
+        cp->cp_next_rx_seq = be64_to_cpu(inc->i_hdr.h_sequence) + 1;
 
         if (rds_sysctl_ping_enable && inc->i_hdr.h_dport == 0) {
                 rds_stats_inc(s_recv_ping);
-                rds_send_pong(conn, inc->i_hdr.h_sport);
+                rds_send_pong(cp, inc->i_hdr.h_sport);
                 goto out;
         }
 
diff --git a/net/rds/send.c b/net/rds/send.c
index b1962f8e30f7..ee43d6b2ea8f 100644
--- a/net/rds/send.c
+++ b/net/rds/send.c
@@ -62,14 +62,14 @@ static void rds_send_remove_from_sock(struct list_head *messages, int status);
  * Reset the send state.  Callers must ensure that this doesn't race with
  * rds_send_xmit().
  */
-void rds_send_reset(struct rds_connection *conn)
+void rds_send_path_reset(struct rds_conn_path *cp)
 {
         struct rds_message *rm, *tmp;
         unsigned long flags;
 
-        if (conn->c_xmit_rm) {
-                rm = conn->c_xmit_rm;
-                conn->c_xmit_rm = NULL;
+        if (cp->cp_xmit_rm) {
+                rm = cp->cp_xmit_rm;
+                cp->cp_xmit_rm = NULL;
                 /* Tell the user the RDMA op is no longer mapped by the
                  * transport. This isn't entirely true (it's flushed out
                  * independently) but as the connection is down, there's
@@ -78,37 +78,37 @@ void rds_send_reset(struct rds_connection *conn)
                 rds_message_put(rm);
         }
 
-        conn->c_xmit_sg = 0;
-        conn->c_xmit_hdr_off = 0;
-        conn->c_xmit_data_off = 0;
-        conn->c_xmit_atomic_sent = 0;
-        conn->c_xmit_rdma_sent = 0;
-        conn->c_xmit_data_sent = 0;
+        cp->cp_xmit_sg = 0;
+        cp->cp_xmit_hdr_off = 0;
+        cp->cp_xmit_data_off = 0;
+        cp->cp_xmit_atomic_sent = 0;
+        cp->cp_xmit_rdma_sent = 0;
+        cp->cp_xmit_data_sent = 0;
 
-        conn->c_map_queued = 0;
+        cp->cp_conn->c_map_queued = 0;
 
-        conn->c_unacked_packets = rds_sysctl_max_unacked_packets;
-        conn->c_unacked_bytes = rds_sysctl_max_unacked_bytes;
+        cp->cp_unacked_packets = rds_sysctl_max_unacked_packets;
+        cp->cp_unacked_bytes = rds_sysctl_max_unacked_bytes;
 
         /* Mark messages as retransmissions, and move them to the send q */
-        spin_lock_irqsave(&conn->c_lock, flags);
-        list_for_each_entry_safe(rm, tmp, &conn->c_retrans, m_conn_item) {
+        spin_lock_irqsave(&cp->cp_lock, flags);
+        list_for_each_entry_safe(rm, tmp, &cp->cp_retrans, m_conn_item) {
                 set_bit(RDS_MSG_ACK_REQUIRED, &rm->m_flags);
                 set_bit(RDS_MSG_RETRANSMITTED, &rm->m_flags);
         }
-        list_splice_init(&conn->c_retrans, &conn->c_send_queue);
-        spin_unlock_irqrestore(&conn->c_lock, flags);
+        list_splice_init(&cp->cp_retrans, &cp->cp_send_queue);
+        spin_unlock_irqrestore(&cp->cp_lock, flags);
 }
-EXPORT_SYMBOL_GPL(rds_send_reset);
+EXPORT_SYMBOL_GPL(rds_send_path_reset);
 
-static int acquire_in_xmit(struct rds_connection *conn)
+static int acquire_in_xmit(struct rds_conn_path *cp)
 {
-        return test_and_set_bit(RDS_IN_XMIT, &conn->c_flags) == 0;
+        return test_and_set_bit(RDS_IN_XMIT, &cp->cp_flags) == 0;
 }
 
-static void release_in_xmit(struct rds_connection *conn)
+static void release_in_xmit(struct rds_conn_path *cp)
 {
-        clear_bit(RDS_IN_XMIT, &conn->c_flags);
+        clear_bit(RDS_IN_XMIT, &cp->cp_flags);
         smp_mb__after_atomic();
         /*
          * We don't use wait_on_bit()/wake_up_bit() because our waking is in a
@@ -116,8 +116,8 @@ static void release_in_xmit(struct rds_connection *conn)
          * the system-wide hashed waitqueue buckets in the fast path only to
          * almost never find waiters.
          */
-        if (waitqueue_active(&conn->c_waitq))
-                wake_up_all(&conn->c_waitq);
+        if (waitqueue_active(&cp->cp_waitq))
+                wake_up_all(&cp->cp_waitq);
 }
 
 /*
@@ -134,8 +134,9 @@ static void release_in_xmit(struct rds_connection *conn)
  *      - small message latency is higher behind queued large messages
  *      - large message latency isn't starved by intervening small sends
  */
-int rds_send_xmit(struct rds_connection *conn)
+int rds_send_xmit(struct rds_conn_path *cp)
 {
+        struct rds_connection *conn = cp->cp_conn;
         struct rds_message *rm;
         unsigned long flags;
         unsigned int tmp;
@@ -155,7 +156,7 @@ restart:
          * avoids blocking the caller and trading per-connection data between
          * caches per message.
          */
-        if (!acquire_in_xmit(conn)) {
+        if (!acquire_in_xmit(cp)) {
                 rds_stats_inc(s_send_lock_contention);
                 ret = -ENOMEM;
                 goto out;
@@ -169,21 +170,25 @@ restart:
          * The acquire_in_xmit() check above ensures that only one
          * caller can increment c_send_gen at any time.
          */
-        conn->c_send_gen++;
-        send_gen = conn->c_send_gen;
+        cp->cp_send_gen++;
+        send_gen = cp->cp_send_gen;
 
         /*
          * rds_conn_shutdown() sets the conn state and then tests RDS_IN_XMIT,
          * we do the opposite to avoid races.
          */
-        if (!rds_conn_up(conn)) {
-                release_in_xmit(conn);
+        if (!rds_conn_path_up(cp)) {
+                release_in_xmit(cp);
                 ret = 0;
                 goto out;
         }
 
-        if (conn->c_trans->xmit_prepare)
+        if (conn->c_trans->t_mp_capable) {
+                if (conn->c_trans->xmit_path_prepare)
+                        conn->c_trans->xmit_path_prepare(cp);
+        } else if (conn->c_trans->xmit_prepare) {
                 conn->c_trans->xmit_prepare(conn);
+        }
 
         /*
          * spin trying to push headers and data down the connection until
@@ -191,7 +196,7 @@ restart:
          */
         while (1) {
 
-                rm = conn->c_xmit_rm;
+                rm = cp->cp_xmit_rm;
 
                 /*
                  * If between sending messages, we can send a pending congestion
@@ -204,14 +209,16 @@ restart:
                                 break;
                         }
                         rm->data.op_active = 1;
+                        rm->m_inc.i_conn_path = cp;
+                        rm->m_inc.i_conn = cp->cp_conn;
 
-                        conn->c_xmit_rm = rm;
+                        cp->cp_xmit_rm = rm;
                 }
 
                 /*
                  * If not already working on one, grab the next message.
                  *
-                 * c_xmit_rm holds a ref while we're sending this message down
+                 * cp_xmit_rm holds a ref while we're sending this message down
                  * the connction.  We can use this ref while holding the
                  * send_sem.. rds_send_reset() is serialized with it.
                  */
@@ -228,10 +235,10 @@ restart:
                         if (batch_count >= send_batch_count)
                                 goto over_batch;
 
-                        spin_lock_irqsave(&conn->c_lock, flags);
+                        spin_lock_irqsave(&cp->cp_lock, flags);
 
-                        if (!list_empty(&conn->c_send_queue)) {
-                                rm = list_entry(conn->c_send_queue.next,
+                        if (!list_empty(&cp->cp_send_queue)) {
+                                rm = list_entry(cp->cp_send_queue.next,
                                                 struct rds_message,
                                                 m_conn_item);
                                 rds_message_addref(rm);
@@ -240,10 +247,11 @@ restart:
                                  * Move the message from the send queue to the retransmit
                                  * list right away.
                                  */
-                                list_move_tail(&rm->m_conn_item, &conn->c_retrans);
+                                list_move_tail(&rm->m_conn_item,
+                                               &cp->cp_retrans);
                         }
 
-                        spin_unlock_irqrestore(&conn->c_lock, flags);
+                        spin_unlock_irqrestore(&cp->cp_lock, flags);
 
                         if (!rm)
                                 break;
@@ -257,32 +265,34 @@ restart:
                          */
                         if (rm->rdma.op_active &&
                             test_bit(RDS_MSG_RETRANSMITTED, &rm->m_flags)) {
-                                spin_lock_irqsave(&conn->c_lock, flags);
+                                spin_lock_irqsave(&cp->cp_lock, flags);
                                 if (test_and_clear_bit(RDS_MSG_ON_CONN, &rm->m_flags))
                                         list_move(&rm->m_conn_item, &to_be_dropped);
-                                spin_unlock_irqrestore(&conn->c_lock, flags);
+                                spin_unlock_irqrestore(&cp->cp_lock, flags);
                                 continue;
                         }
 
                         /* Require an ACK every once in a while */
                         len = ntohl(rm->m_inc.i_hdr.h_len);
-                        if (conn->c_unacked_packets == 0 ||
-                            conn->c_unacked_bytes < len) {
+                        if (cp->cp_unacked_packets == 0 ||
+                            cp->cp_unacked_bytes < len) {
                                 __set_bit(RDS_MSG_ACK_REQUIRED, &rm->m_flags);
 
-                                conn->c_unacked_packets = rds_sysctl_max_unacked_packets;
-                                conn->c_unacked_bytes = rds_sysctl_max_unacked_bytes;
+                                cp->cp_unacked_packets =
+                                        rds_sysctl_max_unacked_packets;
+                                cp->cp_unacked_bytes =
+                                        rds_sysctl_max_unacked_bytes;
                                 rds_stats_inc(s_send_ack_required);
                         } else {
-                                conn->c_unacked_bytes -= len;
-                                conn->c_unacked_packets--;
+                                cp->cp_unacked_bytes -= len;
+                                cp->cp_unacked_packets--;
                         }
 
-                        conn->c_xmit_rm = rm;
+                        cp->cp_xmit_rm = rm;
                 }
 
                 /* The transport either sends the whole rdma or none of it */
-                if (rm->rdma.op_active && !conn->c_xmit_rdma_sent) {
+                if (rm->rdma.op_active && !cp->cp_xmit_rdma_sent) {
                         rm->m_final_op = &rm->rdma;
                         /* The transport owns the mapped memory for now.
                          * You can't unmap it while it's on the send queue
@@ -294,11 +304,11 @@ restart:
                                 wake_up_interruptible(&rm->m_flush_wait);
                                 break;
                         }
-                        conn->c_xmit_rdma_sent = 1;
+                        cp->cp_xmit_rdma_sent = 1;
 
                 }
 
-                if (rm->atomic.op_active && !conn->c_xmit_atomic_sent) {
+                if (rm->atomic.op_active && !cp->cp_xmit_atomic_sent) {
                         rm->m_final_op = &rm->atomic;
                         /* The transport owns the mapped memory for now.
                          * You can't unmap it while it's on the send queue
@@ -310,7 +320,7 @@ restart:
                                 wake_up_interruptible(&rm->m_flush_wait);
                                 break;
                         }
-                        conn->c_xmit_atomic_sent = 1;
+                        cp->cp_xmit_atomic_sent = 1;
 
                 }
 
@@ -336,41 +346,42 @@ restart:
                                 rm->data.op_active = 0;
                 }
 
-                if (rm->data.op_active && !conn->c_xmit_data_sent) {
+                if (rm->data.op_active && !cp->cp_xmit_data_sent) {
                         rm->m_final_op = &rm->data;
+
                         ret = conn->c_trans->xmit(conn, rm,
-                                                  conn->c_xmit_hdr_off,
-                                                  conn->c_xmit_sg,
-                                                  conn->c_xmit_data_off);
+                                                  cp->cp_xmit_hdr_off,
+                                                  cp->cp_xmit_sg,
+                                                  cp->cp_xmit_data_off);
                         if (ret <= 0)
                                 break;
 
-                        if (conn->c_xmit_hdr_off < sizeof(struct rds_header)) {
+                        if (cp->cp_xmit_hdr_off < sizeof(struct rds_header)) {
                                 tmp = min_t(int, ret,
                                             sizeof(struct rds_header) -
-                                            conn->c_xmit_hdr_off);
-                                conn->c_xmit_hdr_off += tmp;
+                                            cp->cp_xmit_hdr_off);
+                                cp->cp_xmit_hdr_off += tmp;
                                 ret -= tmp;
                         }
 
-                        sg = &rm->data.op_sg[conn->c_xmit_sg];
+                        sg = &rm->data.op_sg[cp->cp_xmit_sg];
                         while (ret) {
                                 tmp = min_t(int, ret, sg->length -
-                                                      conn->c_xmit_data_off);
-                                conn->c_xmit_data_off += tmp;
+                                                      cp->cp_xmit_data_off);
+                                cp->cp_xmit_data_off += tmp;
                                 ret -= tmp;
-                                if (conn->c_xmit_data_off == sg->length) {
-                                        conn->c_xmit_data_off = 0;
+                                if (cp->cp_xmit_data_off == sg->length) {
+                                        cp->cp_xmit_data_off = 0;
                                         sg++;
-                                        conn->c_xmit_sg++;
-                                        BUG_ON(ret != 0 &&
-                                               conn->c_xmit_sg == rm->data.op_nents);
+                                        cp->cp_xmit_sg++;
+                                        BUG_ON(ret != 0 && cp->cp_xmit_sg ==
+                                               rm->data.op_nents);
                                 }
                         }
 
-                        if (conn->c_xmit_hdr_off == sizeof(struct rds_header) &&
-                            (conn->c_xmit_sg == rm->data.op_nents))
-                                conn->c_xmit_data_sent = 1;
+                        if (cp->cp_xmit_hdr_off == sizeof(struct rds_header) &&
+                            (cp->cp_xmit_sg == rm->data.op_nents))
+                                cp->cp_xmit_data_sent = 1;
                 }
 
                 /*
@@ -378,23 +389,27 @@ restart:
                  * if there is a data op. Thus, if the data is sent (or there was
                  * none), then we're done with the rm.
                  */
-                if (!rm->data.op_active || conn->c_xmit_data_sent) {
-                        conn->c_xmit_rm = NULL;
-                        conn->c_xmit_sg = 0;
-                        conn->c_xmit_hdr_off = 0;
-                        conn->c_xmit_data_off = 0;
-                        conn->c_xmit_rdma_sent = 0;
-                        conn->c_xmit_atomic_sent = 0;
-                        conn->c_xmit_data_sent = 0;
+                if (!rm->data.op_active || cp->cp_xmit_data_sent) {
+                        cp->cp_xmit_rm = NULL;
+                        cp->cp_xmit_sg = 0;
+                        cp->cp_xmit_hdr_off = 0;
+                        cp->cp_xmit_data_off = 0;
+                        cp->cp_xmit_rdma_sent = 0;
+                        cp->cp_xmit_atomic_sent = 0;
+                        cp->cp_xmit_data_sent = 0;
 
                         rds_message_put(rm);
                 }
         }
 
 over_batch:
-        if (conn->c_trans->xmit_complete)
+        if (conn->c_trans->t_mp_capable) {
+                if (conn->c_trans->xmit_path_complete)
+                        conn->c_trans->xmit_path_complete(cp);
+        } else if (conn->c_trans->xmit_complete) {
                 conn->c_trans->xmit_complete(conn);
-        release_in_xmit(conn);
+        }
+        release_in_xmit(cp);
 
         /* Nuke any messages we decided not to retransmit. */
         if (!list_empty(&to_be_dropped)) {
@@ -422,12 +437,12 @@ over_batch:
         if (ret == 0) {
                 smp_mb();
                 if ((test_bit(0, &conn->c_map_queued) ||
-                     !list_empty(&conn->c_send_queue)) &&
-                    send_gen == conn->c_send_gen) {
+                     !list_empty(&cp->cp_send_queue)) &&
+                    send_gen == cp->cp_send_gen) {
                         rds_stats_inc(s_send_lock_queue_raced);
                         if (batch_count < send_batch_count)
                                 goto restart;
-                        queue_delayed_work(rds_wq, &conn->c_send_w, 1);
+                        queue_delayed_work(rds_wq, &cp->cp_send_w, 1);
                 }
         }
 out:
@@ -560,42 +575,6 @@ __rds_send_complete(struct rds_sock *rs, struct rds_message *rm, int status)
 }
 
 /*
- * This is called from the IB send completion when we detect
- * a RDMA operation that failed with remote access error.
- * So speed is not an issue here.
- */
-struct rds_message *rds_send_get_message(struct rds_connection *conn,
-                                         struct rm_rdma_op *op)
-{
-        struct rds_message *rm, *tmp, *found = NULL;
-        unsigned long flags;
-
-        spin_lock_irqsave(&conn->c_lock, flags);
-
-        list_for_each_entry_safe(rm, tmp, &conn->c_retrans, m_conn_item) {
-                if (&rm->rdma == op) {
-                        atomic_inc(&rm->m_refcount);
-                        found = rm;
-                        goto out;
-                }
-        }
-
-        list_for_each_entry_safe(rm, tmp, &conn->c_send_queue, m_conn_item) {
-                if (&rm->rdma == op) {
-                        atomic_inc(&rm->m_refcount);
-                        found = rm;
-                        break;
-                }
-        }
-
-out:
-        spin_unlock_irqrestore(&conn->c_lock, flags);
-
-        return found;
-}
-EXPORT_SYMBOL_GPL(rds_send_get_message);
-
-/*
  * This removes messages from the socket's list if they're on it.  The list
  * argument must be private to the caller, we must be able to modify it
  * without locks.  The messages must have a reference held for their
@@ -685,16 +664,16 @@ unlock_and_drop:
  * assigned the m_ack_seq yet - but that's fine as long as tcp_is_acked
  * checks the RDS_MSG_HAS_ACK_SEQ bit.
  */
-void rds_send_drop_acked(struct rds_connection *conn, u64 ack,
-                         is_acked_func is_acked)
+void rds_send_path_drop_acked(struct rds_conn_path *cp, u64 ack,
+                              is_acked_func is_acked)
 {
         struct rds_message *rm, *tmp;
         unsigned long flags;
         LIST_HEAD(list);
 
-        spin_lock_irqsave(&conn->c_lock, flags);
+        spin_lock_irqsave(&cp->cp_lock, flags);
 
-        list_for_each_entry_safe(rm, tmp, &conn->c_retrans, m_conn_item) {
+        list_for_each_entry_safe(rm, tmp, &cp->cp_retrans, m_conn_item) {
                 if (!rds_send_is_acked(rm, ack, is_acked))
                         break;
 
@@ -706,17 +685,26 @@ void rds_send_drop_acked(struct rds_connection *conn, u64 ack,
         if (!list_empty(&list))
                 smp_mb__after_atomic();
 
-        spin_unlock_irqrestore(&conn->c_lock, flags);
+        spin_unlock_irqrestore(&cp->cp_lock, flags);
 
         /* now remove the messages from the sock list as needed */
         rds_send_remove_from_sock(&list, RDS_RDMA_SUCCESS);
 }
+EXPORT_SYMBOL_GPL(rds_send_path_drop_acked);
+
+void rds_send_drop_acked(struct rds_connection *conn, u64 ack,
+                         is_acked_func is_acked)
+{
+        WARN_ON(conn->c_trans->t_mp_capable);
+        rds_send_path_drop_acked(&conn->c_path[0], ack, is_acked);
+}
 EXPORT_SYMBOL_GPL(rds_send_drop_acked);
 
 void rds_send_drop_to(struct rds_sock *rs, struct sockaddr_in *dest)
 {
         struct rds_message *rm, *tmp;
         struct rds_connection *conn;
+        struct rds_conn_path *cp;
         unsigned long flags;
         LIST_HEAD(list);
 
@@ -745,22 +733,26 @@ void rds_send_drop_to(struct rds_sock *rs, struct sockaddr_in *dest)
         list_for_each_entry(rm, &list, m_sock_item) {
 
                 conn = rm->m_inc.i_conn;
+                if (conn->c_trans->t_mp_capable)
+                        cp = rm->m_inc.i_conn_path;
+                else
+                        cp = &conn->c_path[0];
 
-                spin_lock_irqsave(&conn->c_lock, flags);
+                spin_lock_irqsave(&cp->cp_lock, flags);
                 /*
                  * Maybe someone else beat us to removing rm from the conn.
                  * If we race with their flag update we'll get the lock and
                  * then really see that the flag has been cleared.
                  */
                 if (!test_and_clear_bit(RDS_MSG_ON_CONN, &rm->m_flags)) {
-                        spin_unlock_irqrestore(&conn->c_lock, flags);
+                        spin_unlock_irqrestore(&cp->cp_lock, flags);
                         spin_lock_irqsave(&rm->m_rs_lock, flags);
                         rm->m_rs = NULL;
                         spin_unlock_irqrestore(&rm->m_rs_lock, flags);
                         continue;
                 }
                 list_del_init(&rm->m_conn_item);
-                spin_unlock_irqrestore(&conn->c_lock, flags);
+                spin_unlock_irqrestore(&cp->cp_lock, flags);
 
                 /*
                  * Couldn't grab m_rs_lock in top loop (lock ordering),
@@ -809,6 +801,7 @@ void rds_send_drop_to(struct rds_sock *rs, struct sockaddr_in *dest)
  * message from the flow with RDS_CANCEL_SENT_TO.
  */
 static int rds_send_queue_rm(struct rds_sock *rs, struct rds_connection *conn,
+                             struct rds_conn_path *cp,
                              struct rds_message *rm, __be16 sport,
                              __be16 dport, int *queued)
 {
@@ -852,13 +845,14 @@ static int rds_send_queue_rm(struct rds_sock *rs, struct rds_connection *conn,
                    trying to minimize the time we hold c_lock */
                 rds_message_populate_header(&rm->m_inc.i_hdr, sport, dport, 0);
                 rm->m_inc.i_conn = conn;
+                rm->m_inc.i_conn_path = cp;
                 rds_message_addref(rm);
 
-                spin_lock(&conn->c_lock);
-                rm->m_inc.i_hdr.h_sequence = cpu_to_be64(conn->c_next_tx_seq++);
-                list_add_tail(&rm->m_conn_item, &conn->c_send_queue);
+                spin_lock(&cp->cp_lock);
+                rm->m_inc.i_hdr.h_sequence = cpu_to_be64(cp->cp_next_tx_seq++);
+                list_add_tail(&rm->m_conn_item, &cp->cp_send_queue);
                 set_bit(RDS_MSG_ON_CONN, &rm->m_flags);
-                spin_unlock(&conn->c_lock);
+                spin_unlock(&cp->cp_lock);
 
                 rdsdebug("queued msg %p len %d, rs %p bytes %d seq %llu\n",
                          rm, len, rs, rs->rs_snd_bytes,
@@ -990,6 +984,7 @@ int rds_sendmsg(struct socket *sock, struct msghdr *msg, size_t payload_len)
         int queued = 0, allocated_mr = 0;
         int nonblock = msg->msg_flags & MSG_DONTWAIT;
         long timeo = sock_sndtimeo(sk, nonblock);
+        struct rds_conn_path *cpath;
 
         /* Mirror Linux UDP mirror of BSD error message compatibility */
         /* XXX: Perhaps MSG_MORE someday */
@@ -1088,15 +1083,16 @@ int rds_sendmsg(struct socket *sock, struct msghdr *msg, size_t payload_len)
                 goto out;
         }
 
-        rds_conn_connect_if_down(conn);
+        cpath = &conn->c_path[0];
+
+        rds_conn_path_connect_if_down(cpath);
 
         ret = rds_cong_wait(conn->c_fcong, dport, nonblock, rs);
         if (ret) {
                 rs->rs_seen_congestion = 1;
                 goto out;
         }
-
-        while (!rds_send_queue_rm(rs, conn, rm, rs->rs_bound_port,
+        while (!rds_send_queue_rm(rs, conn, cpath, rm, rs->rs_bound_port,
                                   dport, &queued)) {
                 rds_stats_inc(s_send_queue_full);
 
@@ -1106,7 +1102,7 @@ int rds_sendmsg(struct socket *sock, struct msghdr *msg, size_t payload_len)
                 }
 
                 timeo = wait_event_interruptible_timeout(*sk_sleep(sk),
-                                        rds_send_queue_rm(rs, conn, rm,
+                                        rds_send_queue_rm(rs, conn, cpath, rm,
                                                           rs->rs_bound_port,
                                                           dport,
                                                           &queued),
@@ -1127,9 +1123,9 @@ int rds_sendmsg(struct socket *sock, struct msghdr *msg, size_t payload_len)
          */
         rds_stats_inc(s_send_queued);
 
-        ret = rds_send_xmit(conn);
+        ret = rds_send_xmit(cpath);
         if (ret == -ENOMEM || ret == -EAGAIN)
-                queue_delayed_work(rds_wq, &conn->c_send_w, 1);
+                queue_delayed_work(rds_wq, &cpath->cp_send_w, 1);
 
         rds_message_put(rm);
         return payload_len;
@@ -1150,7 +1146,7 @@ out:
  * Reply to a ping packet.
  */
 int
-rds_send_pong(struct rds_connection *conn, __be16 dport)
+rds_send_pong(struct rds_conn_path *cp, __be16 dport)
 {
         struct rds_message *rm;
         unsigned long flags;
@@ -1162,31 +1158,32 @@ rds_send_pong(struct rds_connection *conn, __be16 dport)
                 goto out;
         }
 
-        rm->m_daddr = conn->c_faddr;
+        rm->m_daddr = cp->cp_conn->c_faddr;
         rm->data.op_active = 1;
 
-        rds_conn_connect_if_down(conn);
+        rds_conn_path_connect_if_down(cp);
 
-        ret = rds_cong_wait(conn->c_fcong, dport, 1, NULL);
+        ret = rds_cong_wait(cp->cp_conn->c_fcong, dport, 1, NULL);
         if (ret)
                 goto out;
 
-        spin_lock_irqsave(&conn->c_lock, flags);
-        list_add_tail(&rm->m_conn_item, &conn->c_send_queue);
+        spin_lock_irqsave(&cp->cp_lock, flags);
+        list_add_tail(&rm->m_conn_item, &cp->cp_send_queue);
         set_bit(RDS_MSG_ON_CONN, &rm->m_flags);
         rds_message_addref(rm);
-        rm->m_inc.i_conn = conn;
+        rm->m_inc.i_conn = cp->cp_conn;
+        rm->m_inc.i_conn_path = cp;
 
         rds_message_populate_header(&rm->m_inc.i_hdr, 0, dport,
-                                    conn->c_next_tx_seq);
-        conn->c_next_tx_seq++;
-        spin_unlock_irqrestore(&conn->c_lock, flags);
+                                    cp->cp_next_tx_seq);
+        cp->cp_next_tx_seq++;
+        spin_unlock_irqrestore(&cp->cp_lock, flags);
 
         rds_stats_inc(s_send_queued);
         rds_stats_inc(s_send_pong);
 
         /* schedule the send work on rds_wq */
-        queue_delayed_work(rds_wq, &conn->c_send_w, 1);
+        queue_delayed_work(rds_wq, &cp->cp_send_w, 1);
 
         rds_message_put(rm);
         return 0;
diff --git a/net/rds/tcp.c b/net/rds/tcp.c
index 74ee126a6fe6..5217d49ce6d6 100644
--- a/net/rds/tcp.c
+++ b/net/rds/tcp.c
@@ -38,6 +38,7 @@
 #include <net/net_namespace.h>
 #include <net/netns/generic.h>
 
+#include "rds_single_path.h"
 #include "rds.h"
 #include "tcp.h"
 
@@ -56,8 +57,8 @@ static int rds_tcp_skbuf_handler(struct ctl_table *ctl, int write,
                                  void __user *buffer, size_t *lenp,
                                  loff_t *fpos);
 
-int rds_tcp_min_sndbuf = SOCK_MIN_SNDBUF;
-int rds_tcp_min_rcvbuf = SOCK_MIN_RCVBUF;
+static int rds_tcp_min_sndbuf = SOCK_MIN_SNDBUF;
+static int rds_tcp_min_rcvbuf = SOCK_MIN_RCVBUF;
 
 static struct ctl_table rds_tcp_sysctl_table[] = {
 #define RDS_TCP_SNDBUF  0
@@ -185,7 +186,7 @@ void rds_tcp_reset_callbacks(struct socket *sock,
         release_sock(osock->sk);
         sock_release(osock);
 newsock:
-        rds_send_reset(conn);
+        rds_send_path_reset(&conn->c_path[0]);
         lock_sock(sock->sk);
         write_lock_bh(&sock->sk->sk_callback_lock);
         tc->t_sock = sock;
diff --git a/net/rds/tcp_connect.c b/net/rds/tcp_connect.c
index f6e95d60db54..96c2c4d17909 100644
--- a/net/rds/tcp_connect.c
+++ b/net/rds/tcp_connect.c
@@ -34,6 +34,7 @@
 #include <linux/in.h>
 #include <net/tcp.h>
 
+#include "rds_single_path.h"
 #include "rds.h"
 #include "tcp.h"
 
@@ -60,7 +61,8 @@ void rds_tcp_state_change(struct sock *sk)
         case TCP_SYN_RECV:
                 break;
         case TCP_ESTABLISHED:
-                rds_connect_path_complete(conn, RDS_CONN_CONNECTING);
+                rds_connect_path_complete(&conn->c_path[0],
+                                          RDS_CONN_CONNECTING);
                 break;
         case TCP_CLOSE_WAIT:
         case TCP_CLOSE:
diff --git a/net/rds/tcp_listen.c b/net/rds/tcp_listen.c
index 245542ca4718..f9cc945a77b3 100644
--- a/net/rds/tcp_listen.c
+++ b/net/rds/tcp_listen.c
@@ -35,6 +35,7 @@
 #include <linux/in.h>
 #include <net/tcp.h>
 
+#include "rds_single_path.h"
 #include "rds.h"
 #include "tcp.h"
 
@@ -132,17 +133,19 @@ int rds_tcp_accept_one(struct socket *sock)
                  * c_transport_data.
                  */
                 if (ntohl(inet->inet_saddr) < ntohl(inet->inet_daddr) ||
-                    !conn->c_outgoing) {
+                    !conn->c_path[0].cp_outgoing) {
                         goto rst_nsk;
                 } else {
                         rds_tcp_reset_callbacks(new_sock, conn);
-                        conn->c_outgoing = 0;
+                        conn->c_path[0].cp_outgoing = 0;
                         /* rds_connect_path_complete() marks RDS_CONN_UP */
-                        rds_connect_path_complete(conn, RDS_CONN_RESETTING);
+                        rds_connect_path_complete(&conn->c_path[0],
+                                                  RDS_CONN_RESETTING);
                 }
         } else {
                 rds_tcp_set_callbacks(new_sock, conn);
-                rds_connect_path_complete(conn, RDS_CONN_CONNECTING);
+                rds_connect_path_complete(&conn->c_path[0],
+                                          RDS_CONN_CONNECTING);
         }
         new_sock = NULL;
         ret = 0;
diff --git a/net/rds/tcp_recv.c b/net/rds/tcp_recv.c
index 6e6a7111a034..4a87d9ef3084 100644
--- a/net/rds/tcp_recv.c
+++ b/net/rds/tcp_recv.c
@@ -34,6 +34,7 @@
 #include <linux/slab.h>
 #include <net/tcp.h>
 
+#include "rds_single_path.h"
 #include "rds.h"
 #include "tcp.h"
 
diff --git a/net/rds/tcp_send.c b/net/rds/tcp_send.c
index 618be69c9c3b..710f1aae97ad 100644
--- a/net/rds/tcp_send.c
+++ b/net/rds/tcp_send.c
@@ -34,6 +34,7 @@
 #include <linux/in.h>
 #include <net/tcp.h>
 
+#include "rds_single_path.h"
 #include "rds.h"
 #include "tcp.h"
 
diff --git a/net/rds/threads.c b/net/rds/threads.c
index 4a323045719b..9fbe95bb14a9 100644
--- a/net/rds/threads.c
+++ b/net/rds/threads.c
@@ -71,30 +71,30 @@
 struct workqueue_struct *rds_wq;
 EXPORT_SYMBOL_GPL(rds_wq);
 
-void rds_connect_path_complete(struct rds_connection *conn, int curr)
+void rds_connect_path_complete(struct rds_conn_path *cp, int curr)
 {
-        if (!rds_conn_transition(conn, curr, RDS_CONN_UP)) {
+        if (!rds_conn_path_transition(cp, curr, RDS_CONN_UP)) {
                 printk(KERN_WARNING "%s: Cannot transition to state UP, "
                                 "current state is %d\n",
                                 __func__,
-                                atomic_read(&conn->c_state));
-                rds_conn_drop(conn);
+                                atomic_read(&cp->cp_state));
+                rds_conn_path_drop(cp);
                 return;
         }
 
         rdsdebug("conn %p for %pI4 to %pI4 complete\n",
-          conn, &conn->c_laddr, &conn->c_faddr);
+          cp->cp_conn, &cp->cp_conn->c_laddr, &cp->cp_conn->c_faddr);
 
-        conn->c_reconnect_jiffies = 0;
-        set_bit(0, &conn->c_map_queued);
-        queue_delayed_work(rds_wq, &conn->c_send_w, 0);
-        queue_delayed_work(rds_wq, &conn->c_recv_w, 0);
+        cp->cp_reconnect_jiffies = 0;
+        set_bit(0, &cp->cp_conn->c_map_queued);
+        queue_delayed_work(rds_wq, &cp->cp_send_w, 0);
+        queue_delayed_work(rds_wq, &cp->cp_recv_w, 0);
 }
 EXPORT_SYMBOL_GPL(rds_connect_path_complete);
 
 void rds_connect_complete(struct rds_connection *conn)
 {
-        rds_connect_path_complete(conn, RDS_CONN_CONNECTING);
+        rds_connect_path_complete(&conn->c_path[0], RDS_CONN_CONNECTING);
 }
 EXPORT_SYMBOL_GPL(rds_connect_complete);
 
@@ -116,70 +116,79 @@ EXPORT_SYMBOL_GPL(rds_connect_complete);
  * We should *always* start with a random backoff; otherwise a broken connection
  * will always take several iterations to be re-established.
  */
-void rds_queue_reconnect(struct rds_connection *conn)
+void rds_queue_reconnect(struct rds_conn_path *cp)
 {
         unsigned long rand;
+        struct rds_connection *conn = cp->cp_conn;
 
         rdsdebug("conn %p for %pI4 to %pI4 reconnect jiffies %lu\n",
           conn, &conn->c_laddr, &conn->c_faddr,
-          conn->c_reconnect_jiffies);
+          cp->cp_reconnect_jiffies);
 
-        set_bit(RDS_RECONNECT_PENDING, &conn->c_flags);
-        if (conn->c_reconnect_jiffies == 0) {
-                conn->c_reconnect_jiffies = rds_sysctl_reconnect_min_jiffies;
-                queue_delayed_work(rds_wq, &conn->c_conn_w, 0);
+        set_bit(RDS_RECONNECT_PENDING, &cp->cp_flags);
+        if (cp->cp_reconnect_jiffies == 0) {
+                cp->cp_reconnect_jiffies = rds_sysctl_reconnect_min_jiffies;
+                queue_delayed_work(rds_wq, &cp->cp_conn_w, 0);
                 return;
         }
 
         get_random_bytes(&rand, sizeof(rand));
         rdsdebug("%lu delay %lu ceil conn %p for %pI4 -> %pI4\n",
-                 rand % conn->c_reconnect_jiffies, conn->c_reconnect_jiffies,
+                 rand % cp->cp_reconnect_jiffies, cp->cp_reconnect_jiffies,
                  conn, &conn->c_laddr, &conn->c_faddr);
-        queue_delayed_work(rds_wq, &conn->c_conn_w,
-                           rand % conn->c_reconnect_jiffies);
+        queue_delayed_work(rds_wq, &cp->cp_conn_w,
+                           rand % cp->cp_reconnect_jiffies);
 
-        conn->c_reconnect_jiffies = min(conn->c_reconnect_jiffies * 2,
+        cp->cp_reconnect_jiffies = min(cp->cp_reconnect_jiffies * 2,
                                         rds_sysctl_reconnect_max_jiffies);
 }
 
 void rds_connect_worker(struct work_struct *work)
 {
-        struct rds_connection *conn = container_of(work, struct rds_connection, c_conn_w.work);
+        struct rds_conn_path *cp = container_of(work,
+                                                struct rds_conn_path,
+                                                cp_conn_w.work);
+        struct rds_connection *conn = cp->cp_conn;
         int ret;
 
-        clear_bit(RDS_RECONNECT_PENDING, &conn->c_flags);
-        if (rds_conn_transition(conn, RDS_CONN_DOWN, RDS_CONN_CONNECTING)) {
+        clear_bit(RDS_RECONNECT_PENDING, &cp->cp_flags);
+        if (rds_conn_path_transition(cp, RDS_CONN_DOWN, RDS_CONN_CONNECTING)) {
                 ret = conn->c_trans->conn_connect(conn);
                 rdsdebug("conn %p for %pI4 to %pI4 dispatched, ret %d\n",
                         conn, &conn->c_laddr, &conn->c_faddr, ret);
 
                 if (ret) {
-                        if (rds_conn_transition(conn, RDS_CONN_CONNECTING, RDS_CONN_DOWN))
-                                rds_queue_reconnect(conn);
+                        if (rds_conn_path_transition(cp,
+                                                     RDS_CONN_CONNECTING,
+                                                     RDS_CONN_DOWN))
+                                rds_queue_reconnect(cp);
                         else
-                                rds_conn_error(conn, "RDS: connect failed\n");
+                                rds_conn_path_error(cp,
+                                                    "RDS: connect failed\n");
                 }
         }
 }
 
 void rds_send_worker(struct work_struct *work)
 {
-        struct rds_connection *conn = container_of(work, struct rds_connection, c_send_w.work);
+        struct rds_conn_path *cp = container_of(work,
+                                                struct rds_conn_path,
+                                                cp_send_w.work);
         int ret;
 
-        if (rds_conn_state(conn) == RDS_CONN_UP) {
-                clear_bit(RDS_LL_SEND_FULL, &conn->c_flags);
-                ret = rds_send_xmit(conn);
+        if (rds_conn_path_state(cp) == RDS_CONN_UP) {
+                clear_bit(RDS_LL_SEND_FULL, &cp->cp_flags);
+                ret = rds_send_xmit(cp);
                 cond_resched();
-                rdsdebug("conn %p ret %d\n", conn, ret);
+                rdsdebug("conn %p ret %d\n", cp->cp_conn, ret);
                 switch (ret) {
                 case -EAGAIN:
                         rds_stats_inc(s_send_immediate_retry);
-                        queue_delayed_work(rds_wq, &conn->c_send_w, 0);
+                        queue_delayed_work(rds_wq, &cp->cp_send_w, 0);
                         break;
                 case -ENOMEM:
                         rds_stats_inc(s_send_delayed_retry);
-                        queue_delayed_work(rds_wq, &conn->c_send_w, 2);
+                        queue_delayed_work(rds_wq, &cp->cp_send_w, 2);
                 default:
                         break;
                 }
@@ -188,20 +197,22 @@ void rds_send_worker(struct work_struct *work)
 
 void rds_recv_worker(struct work_struct *work)
 {
-        struct rds_connection *conn = container_of(work, struct rds_connection, c_recv_w.work);
+        struct rds_conn_path *cp = container_of(work,
+                                                struct rds_conn_path,
+                                                cp_recv_w.work);
         int ret;
 
-        if (rds_conn_state(conn) == RDS_CONN_UP) {
-                ret = conn->c_trans->recv(conn);
-                rdsdebug("conn %p ret %d\n", conn, ret);
+        if (rds_conn_path_state(cp) == RDS_CONN_UP) {
+                ret = cp->cp_conn->c_trans->recv(cp->cp_conn);
+                rdsdebug("conn %p ret %d\n", cp->cp_conn, ret);
                 switch (ret) {
                 case -EAGAIN:
                         rds_stats_inc(s_recv_immediate_retry);
-                        queue_delayed_work(rds_wq, &conn->c_recv_w, 0);
+                        queue_delayed_work(rds_wq, &cp->cp_recv_w, 0);
                         break;
                 case -ENOMEM:
                         rds_stats_inc(s_recv_delayed_retry);
-                        queue_delayed_work(rds_wq, &conn->c_recv_w, 2);
+                        queue_delayed_work(rds_wq, &cp->cp_recv_w, 2);
                 default:
                         break;
                 }
@@ -210,9 +221,11 @@ void rds_recv_worker(struct work_struct *work)
 
 void rds_shutdown_worker(struct work_struct *work)
 {
-        struct rds_connection *conn = container_of(work, struct rds_connection, c_down_w);
+        struct rds_conn_path *cp = container_of(work,
+                                                struct rds_conn_path,
+                                                cp_down_w);
 
-        rds_conn_shutdown(conn);
+        rds_conn_shutdown(cp);
 }
 
 void rds_threads_exit(void)
diff --git a/net/rxrpc/Makefile b/net/rxrpc/Makefile
index e05a06ef2254..6522e50fb750 100644
--- a/net/rxrpc/Makefile
+++ b/net/rxrpc/Makefile
@@ -4,25 +4,27 @@
 
 af-rxrpc-y := \
         af_rxrpc.o \
-        ar-accept.o \
-        ar-ack.o \
-        ar-call.o \
-        ar-connection.o \
-        ar-connevent.o \
-        ar-error.o \
-        ar-input.o \
-        ar-key.o \
-        ar-local.o \
-        ar-output.o \
-        ar-peer.o \
-        ar-recvmsg.o \
-        ar-security.o \
-        ar-skbuff.o \
-        ar-transport.o \
+        call_accept.o \
+        call_event.o \
+        call_object.o \
+        conn_client.o \
+        conn_event.o \
+        conn_object.o \
+        input.o \
         insecure.o \
-        misc.o
+        key.o \
+        local_event.o \
+        local_object.o \
+        misc.o \
+        output.o \
+        peer_event.o \
+        peer_object.o \
+        recvmsg.o \
+        security.o \
+        skbuff.o \
+        utils.o
 
-af-rxrpc-$(CONFIG_PROC_FS) += ar-proc.o
+af-rxrpc-$(CONFIG_PROC_FS) += proc.o
 af-rxrpc-$(CONFIG_RXKAD) += rxkad.o
 af-rxrpc-$(CONFIG_SYSCTL) += sysctl.o
 
diff --git a/net/rxrpc/af_rxrpc.c b/net/rxrpc/af_rxrpc.c
index e45e94ca030f..5d3e795a7c48 100644
--- a/net/rxrpc/af_rxrpc.c
+++ b/net/rxrpc/af_rxrpc.c
@@ -9,6 +9,8 @@
  * 2 of the License, or (at your option) any later version.
  */
 
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/net.h>
@@ -31,8 +33,6 @@ unsigned int rxrpc_debug; // = RXRPC_DEBUG_KPROTO;
 module_param_named(debug, rxrpc_debug, uint, S_IWUSR | S_IRUGO);
 MODULE_PARM_DESC(debug, "RxRPC debugging mask");
 
-static int sysctl_rxrpc_max_qlen __read_mostly = 10;
-
 static struct proto rxrpc_proto;
 static const struct proto_ops rxrpc_rpc_ops;
 
@@ -97,11 +97,13 @@ static int rxrpc_validate_address(struct rxrpc_sock *rx,
             srx->transport_len > len)
                 return -EINVAL;
 
-        if (srx->transport.family != rx->proto)
+        if (srx->transport.family != rx->family)
                 return -EAFNOSUPPORT;
 
         switch (srx->transport.family) {
         case AF_INET:
+                if (srx->transport_len < sizeof(struct sockaddr_in))
+                        return -EINVAL;
                 _debug("INET: %x @ %pI4",
                        ntohs(srx->transport.sin.sin_port),
                        &srx->transport.sin.sin_addr);
@@ -137,33 +139,33 @@ static int rxrpc_bind(struct socket *sock, struct sockaddr *saddr, int len)
 
         lock_sock(&rx->sk);
 
-        if (rx->sk.sk_state != RXRPC_UNCONNECTED) {
+        if (rx->sk.sk_state != RXRPC_UNBOUND) {
                 ret = -EINVAL;
                 goto error_unlock;
         }
 
         memcpy(&rx->srx, srx, sizeof(rx->srx));
 
-        /* Find or create a local transport endpoint to use */
         local = rxrpc_lookup_local(&rx->srx);
         if (IS_ERR(local)) {
                 ret = PTR_ERR(local);
                 goto error_unlock;
         }
 
-        rx->local = local;
-        if (srx->srx_service) {
+        if (rx->srx.srx_service) {
                 write_lock_bh(&local->services_lock);
                 list_for_each_entry(prx, &local->services, listen_link) {
-                        if (prx->srx.srx_service == srx->srx_service)
+                        if (prx->srx.srx_service == rx->srx.srx_service)
                                 goto service_in_use;
                 }
 
+                rx->local = local;
                 list_add_tail(&rx->listen_link, &local->services);
                 write_unlock_bh(&local->services_lock);
 
                 rx->sk.sk_state = RXRPC_SERVER_BOUND;
         } else {
+                rx->local = local;
                 rx->sk.sk_state = RXRPC_CLIENT_BOUND;
         }
 
@@ -172,8 +174,9 @@ static int rxrpc_bind(struct socket *sock, struct sockaddr *saddr, int len)
         return 0;
 
 service_in_use:
-        ret = -EADDRINUSE;
         write_unlock_bh(&local->services_lock);
+        rxrpc_put_local(local);
+        ret = -EADDRINUSE;
 error_unlock:
         release_sock(&rx->sk);
 error:
@@ -188,6 +191,7 @@ static int rxrpc_listen(struct socket *sock, int backlog)
 {
         struct sock *sk = sock->sk;
         struct rxrpc_sock *rx = rxrpc_sk(sk);
+        unsigned int max;
         int ret;
 
         _enter("%p,%d", rx, backlog);
@@ -195,20 +199,24 @@ static int rxrpc_listen(struct socket *sock, int backlog)
         lock_sock(&rx->sk);
 
         switch (rx->sk.sk_state) {
-        case RXRPC_UNCONNECTED:
+        case RXRPC_UNBOUND:
                 ret = -EADDRNOTAVAIL;
                 break;
-        case RXRPC_CLIENT_BOUND:
-        case RXRPC_CLIENT_CONNECTED:
-        default:
-                ret = -EBUSY;
-                break;
         case RXRPC_SERVER_BOUND:
                 ASSERT(rx->local != NULL);
+                max = READ_ONCE(rxrpc_max_backlog);
+                ret = -EINVAL;
+                if (backlog == INT_MAX)
+                        backlog = max;
+                else if (backlog < 0 || backlog > max)
+                        break;
                 sk->sk_max_ack_backlog = backlog;
                 rx->sk.sk_state = RXRPC_SERVER_LISTENING;
                 ret = 0;
                 break;
+        default:
+                ret = -EBUSY;
+                break;
         }
 
         release_sock(&rx->sk);
@@ -216,45 +224,10 @@ static int rxrpc_listen(struct socket *sock, int backlog)
         return ret;
 }
 
-/*
- * find a transport by address
- */
-static struct rxrpc_transport *rxrpc_name_to_transport(struct socket *sock,
-                                                       struct sockaddr *addr,
-                                                       int addr_len, int flags,
-                                                       gfp_t gfp)
-{
-        struct sockaddr_rxrpc *srx = (struct sockaddr_rxrpc *) addr;
-        struct rxrpc_transport *trans;
-        struct rxrpc_sock *rx = rxrpc_sk(sock->sk);
-        struct rxrpc_peer *peer;
-
-        _enter("%p,%p,%d,%d", rx, addr, addr_len, flags);
-
-        ASSERT(rx->local != NULL);
-        ASSERT(rx->sk.sk_state > RXRPC_UNCONNECTED);
-
-        if (rx->srx.transport_type != srx->transport_type)
-                return ERR_PTR(-ESOCKTNOSUPPORT);
-        if (rx->srx.transport.family != srx->transport.family)
-                return ERR_PTR(-EAFNOSUPPORT);
-
-        /* find a remote transport endpoint from the local one */
-        peer = rxrpc_get_peer(srx, gfp);
-        if (IS_ERR(peer))
-                return ERR_CAST(peer);
-
-        /* find a transport */
-        trans = rxrpc_get_transport(rx->local, peer, gfp);
-        rxrpc_put_peer(peer);
-        _leave(" = %p", trans);
-        return trans;
-}
-
 /**
  * rxrpc_kernel_begin_call - Allow a kernel service to begin a call
  * @sock: The socket on which to make the call
- * @srx: The address of the peer to contact (defaults to socket setting)
+ * @srx: The address of the peer to contact
  * @key: The security context to use (defaults to socket setting)
  * @user_call_ID: The ID to use
  *
@@ -271,51 +244,32 @@ struct rxrpc_call *rxrpc_kernel_begin_call(struct socket *sock,
                                            unsigned long user_call_ID,
                                            gfp_t gfp)
 {
-        struct rxrpc_conn_bundle *bundle;
-        struct rxrpc_transport *trans;
+        struct rxrpc_conn_parameters cp;
         struct rxrpc_call *call;
         struct rxrpc_sock *rx = rxrpc_sk(sock->sk);
+        int ret;
 
         _enter(",,%x,%lx", key_serial(key), user_call_ID);
 
-        lock_sock(&rx->sk);
+        ret = rxrpc_validate_address(rx, srx, sizeof(*srx));
+        if (ret < 0)
+                return ERR_PTR(ret);
 
-        if (srx) {
-                trans = rxrpc_name_to_transport(sock, (struct sockaddr *) srx,
-                                                sizeof(*srx), 0, gfp);
-                if (IS_ERR(trans)) {
-                        call = ERR_CAST(trans);
-                        trans = NULL;
-                        goto out_notrans;
-                }
-        } else {
-                trans = rx->trans;
-                if (!trans) {
-                        call = ERR_PTR(-ENOTCONN);
-                        goto out_notrans;
-                }
-                atomic_inc(&trans->usage);
-        }
+        lock_sock(&rx->sk);
 
-        if (!srx)
-                srx = &rx->srx;
         if (!key)
                 key = rx->key;
         if (key && !key->payload.data[0])
                 key = NULL; /* a no-security key */
 
-        bundle = rxrpc_get_bundle(rx, trans, key, srx->srx_service, gfp);
-        if (IS_ERR(bundle)) {
-                call = ERR_CAST(bundle);
-                goto out;
-        }
+        memset(&cp, 0, sizeof(cp));
+        cp.local                = rx->local;
+        cp.key                  = key;
+        cp.security_level       = 0;
+        cp.exclusive            = false;
+        cp.service_id           = srx->srx_service;
+        call = rxrpc_new_client_call(rx, &cp, srx, user_call_ID, gfp);
 
-        call = rxrpc_get_client_call(rx, trans, bundle, user_call_ID, true,
-                                     gfp);
-        rxrpc_put_bundle(trans, bundle);
-out:
-        rxrpc_put_transport(trans);
-out_notrans:
         release_sock(&rx->sk);
         _leave(" = %p", call);
         return call;
@@ -367,11 +321,8 @@ EXPORT_SYMBOL(rxrpc_kernel_intercept_rx_messages);
 static int rxrpc_connect(struct socket *sock, struct sockaddr *addr,
                          int addr_len, int flags)
 {
-        struct sockaddr_rxrpc *srx = (struct sockaddr_rxrpc *) addr;
-        struct sock *sk = sock->sk;
-        struct rxrpc_transport *trans;
-        struct rxrpc_local *local;
-        struct rxrpc_sock *rx = rxrpc_sk(sk);
+        struct sockaddr_rxrpc *srx = (struct sockaddr_rxrpc *)addr;
+        struct rxrpc_sock *rx = rxrpc_sk(sock->sk);
         int ret;
 
         _enter("%p,%p,%d,%d", rx, addr, addr_len, flags);
@@ -384,45 +335,28 @@ static int rxrpc_connect(struct socket *sock, struct sockaddr *addr,
 
         lock_sock(&rx->sk);
 
+        ret = -EISCONN;
+        if (test_bit(RXRPC_SOCK_CONNECTED, &rx->flags))
+                goto error;
+
         switch (rx->sk.sk_state) {
-        case RXRPC_UNCONNECTED:
-                /* find a local transport endpoint if we don't have one already */
-                ASSERTCMP(rx->local, ==, NULL);
-                rx->srx.srx_family = AF_RXRPC;
-                rx->srx.srx_service = 0;
-                rx->srx.transport_type = srx->transport_type;
-                rx->srx.transport_len = sizeof(sa_family_t);
-                rx->srx.transport.family = srx->transport.family;
-                local = rxrpc_lookup_local(&rx->srx);
-                if (IS_ERR(local)) {
-                        release_sock(&rx->sk);
-                        return PTR_ERR(local);
-                }
-                rx->local = local;
-                rx->sk.sk_state = RXRPC_CLIENT_BOUND;
+        case RXRPC_UNBOUND:
+                rx->sk.sk_state = RXRPC_CLIENT_UNBOUND;
+        case RXRPC_CLIENT_UNBOUND:
         case RXRPC_CLIENT_BOUND:
                 break;
-        case RXRPC_CLIENT_CONNECTED:
-                release_sock(&rx->sk);
-                return -EISCONN;
         default:
-                release_sock(&rx->sk);
-                return -EBUSY; /* server sockets can't connect as well */
-        }
-
-        trans = rxrpc_name_to_transport(sock, addr, addr_len, flags,
-                                        GFP_KERNEL);
-        if (IS_ERR(trans)) {
-                release_sock(&rx->sk);
-                _leave(" = %ld", PTR_ERR(trans));
-                return PTR_ERR(trans);
+                ret = -EBUSY;
+                goto error;
         }
 
-        rx->trans = trans;
-        rx->sk.sk_state = RXRPC_CLIENT_CONNECTED;
+        rx->connect_srx = *srx;
+        set_bit(RXRPC_SOCK_CONNECTED, &rx->flags);
+        ret = 0;
 
+error:
         release_sock(&rx->sk);
-        return 0;
+        return ret;
 }
 
 /*
@@ -436,7 +370,7 @@ static int rxrpc_connect(struct socket *sock, struct sockaddr *addr,
  */
 static int rxrpc_sendmsg(struct socket *sock, struct msghdr *m, size_t len)
 {
-        struct rxrpc_transport *trans;
+        struct rxrpc_local *local;
         struct rxrpc_sock *rx = rxrpc_sk(sock->sk);
         int ret;
 
@@ -453,48 +387,38 @@ static int rxrpc_sendmsg(struct socket *sock, struct msghdr *m, size_t len)
                 }
         }
 
-        trans = NULL;
         lock_sock(&rx->sk);
 
-        if (m->msg_name) {
-                ret = -EISCONN;
-                trans = rxrpc_name_to_transport(sock, m->msg_name,
-                                                m->msg_namelen, 0, GFP_KERNEL);
-                if (IS_ERR(trans)) {
-                        ret = PTR_ERR(trans);
-                        trans = NULL;
-                        goto out;
-                }
-        } else {
-                trans = rx->trans;
-                if (trans)
-                        atomic_inc(&trans->usage);
-        }
-
         switch (rx->sk.sk_state) {
-        case RXRPC_SERVER_LISTENING:
-                if (!m->msg_name) {
-                        ret = rxrpc_server_sendmsg(rx, m, len);
-                        break;
+        case RXRPC_UNBOUND:
+                local = rxrpc_lookup_local(&rx->srx);
+                if (IS_ERR(local)) {
+                        ret = PTR_ERR(local);
+                        goto error_unlock;
                 }
-        case RXRPC_SERVER_BOUND:
+
+                rx->local = local;
+                rx->sk.sk_state = RXRPC_CLIENT_UNBOUND;
+                /* Fall through */
+
+        case RXRPC_CLIENT_UNBOUND:
         case RXRPC_CLIENT_BOUND:
-                if (!m->msg_name) {
-                        ret = -ENOTCONN;
-                        break;
+                if (!m->msg_name &&
+                    test_bit(RXRPC_SOCK_CONNECTED, &rx->flags)) {
+                        m->msg_name = &rx->connect_srx;
+                        m->msg_namelen = sizeof(rx->connect_srx);
                 }
-        case RXRPC_CLIENT_CONNECTED:
-                ret = rxrpc_client_sendmsg(rx, trans, m, len);
+        case RXRPC_SERVER_BOUND:
+        case RXRPC_SERVER_LISTENING:
+                ret = rxrpc_do_sendmsg(rx, m, len);
                 break;
         default:
-                ret = -ENOTCONN;
+                ret = -EINVAL;
                 break;
         }
 
-out:
+error_unlock:
         release_sock(&rx->sk);
-        if (trans)
-                rxrpc_put_transport(trans);
         _leave(" = %d", ret);
         return ret;
 }
@@ -521,9 +445,9 @@ static int rxrpc_setsockopt(struct socket *sock, int level, int optname,
                         if (optlen != 0)
                                 goto error;
                         ret = -EISCONN;
-                        if (rx->sk.sk_state != RXRPC_UNCONNECTED)
+                        if (rx->sk.sk_state != RXRPC_UNBOUND)
                                 goto error;
-                        set_bit(RXRPC_SOCK_EXCLUSIVE_CONN, &rx->flags);
+                        rx->exclusive = true;
                         goto success;
 
                 case RXRPC_SECURITY_KEY:
@@ -531,7 +455,7 @@ static int rxrpc_setsockopt(struct socket *sock, int level, int optname,
                         if (rx->key)
                                 goto error;
                         ret = -EISCONN;
-                        if (rx->sk.sk_state != RXRPC_UNCONNECTED)
+                        if (rx->sk.sk_state != RXRPC_UNBOUND)
                                 goto error;
                         ret = rxrpc_request_key(rx, optval, optlen);
                         goto error;
@@ -541,7 +465,7 @@ static int rxrpc_setsockopt(struct socket *sock, int level, int optname,
                         if (rx->key)
                                 goto error;
                         ret = -EISCONN;
-                        if (rx->sk.sk_state != RXRPC_UNCONNECTED)
+                        if (rx->sk.sk_state != RXRPC_UNBOUND)
                                 goto error;
                         ret = rxrpc_server_keyring(rx, optval, optlen);
                         goto error;
@@ -551,7 +475,7 @@ static int rxrpc_setsockopt(struct socket *sock, int level, int optname,
                         if (optlen != sizeof(unsigned int))
                                 goto error;
                         ret = -EISCONN;
-                        if (rx->sk.sk_state != RXRPC_UNCONNECTED)
+                        if (rx->sk.sk_state != RXRPC_UNBOUND)
                                 goto error;
                         ret = get_user(min_sec_level,
                                        (unsigned int __user *) optval);
@@ -630,13 +554,13 @@ static int rxrpc_create(struct net *net, struct socket *sock, int protocol,
                 return -ENOMEM;
 
         sock_init_data(sock, sk);
-        sk->sk_state            = RXRPC_UNCONNECTED;
+        sk->sk_state            = RXRPC_UNBOUND;
         sk->sk_write_space      = rxrpc_write_space;
-        sk->sk_max_ack_backlog  = sysctl_rxrpc_max_qlen;
+        sk->sk_max_ack_backlog  = 0;
         sk->sk_destruct         = rxrpc_sock_destructor;
 
         rx = rxrpc_sk(sk);
-        rx->proto = protocol;
+        rx->family = protocol;
         rx->calls = RB_ROOT;
 
         INIT_LIST_HEAD(&rx->listen_link);
@@ -698,24 +622,8 @@ static int rxrpc_release_sock(struct sock *sk)
         flush_workqueue(rxrpc_workqueue);
         rxrpc_purge_queue(&sk->sk_receive_queue);
 
-        if (rx->conn) {
-                rxrpc_put_connection(rx->conn);
-                rx->conn = NULL;
-        }
-
-        if (rx->bundle) {
-                rxrpc_put_bundle(rx->trans, rx->bundle);
-                rx->bundle = NULL;
-        }
-        if (rx->trans) {
-                rxrpc_put_transport(rx->trans);
-                rx->trans = NULL;
-        }
-        if (rx->local) {
-                rxrpc_put_local(rx->local);
-                rx->local = NULL;
-        }
-
+        rxrpc_put_local(rx->local);
+        rx->local = NULL;
         key_put(rx->key);
         rx->key = NULL;
         key_put(rx->securities);
@@ -796,49 +704,49 @@ static int __init af_rxrpc_init(void)
                 "rxrpc_call_jar", sizeof(struct rxrpc_call), 0,
                 SLAB_HWCACHE_ALIGN, NULL);
         if (!rxrpc_call_jar) {
-                printk(KERN_NOTICE "RxRPC: Failed to allocate call jar\n");
+                pr_notice("Failed to allocate call jar\n");
                 goto error_call_jar;
         }
 
         rxrpc_workqueue = alloc_workqueue("krxrpcd", 0, 1);
         if (!rxrpc_workqueue) {
-                printk(KERN_NOTICE "RxRPC: Failed to allocate work queue\n");
+                pr_notice("Failed to allocate work queue\n");
                 goto error_work_queue;
         }
 
         ret = rxrpc_init_security();
         if (ret < 0) {
-                printk(KERN_CRIT "RxRPC: Cannot initialise security\n");
+                pr_crit("Cannot initialise security\n");
                 goto error_security;
         }
 
         ret = proto_register(&rxrpc_proto, 1);
         if (ret < 0) {
-                printk(KERN_CRIT "RxRPC: Cannot register protocol\n");
+                pr_crit("Cannot register protocol\n");
                 goto error_proto;
         }
 
         ret = sock_register(&rxrpc_family_ops);
         if (ret < 0) {
-                printk(KERN_CRIT "RxRPC: Cannot register socket family\n");
+                pr_crit("Cannot register socket family\n");
                 goto error_sock;
         }
 
         ret = register_key_type(&key_type_rxrpc);
         if (ret < 0) {
-                printk(KERN_CRIT "RxRPC: Cannot register client key type\n");
+                pr_crit("Cannot register client key type\n");
                 goto error_key_type;
         }
 
         ret = register_key_type(&key_type_rxrpc_s);
         if (ret < 0) {
-                printk(KERN_CRIT "RxRPC: Cannot register server key type\n");
+                pr_crit("Cannot register server key type\n");
                 goto error_key_type_s;
         }
 
         ret = rxrpc_sysctl_init();
         if (ret < 0) {
-                printk(KERN_CRIT "RxRPC: Cannot register sysctls\n");
+                pr_crit("Cannot register sysctls\n");
                 goto error_sysctls;
         }
 
@@ -880,14 +788,29 @@ static void __exit af_rxrpc_exit(void)
         proto_unregister(&rxrpc_proto);
         rxrpc_destroy_all_calls();
         rxrpc_destroy_all_connections();
-        rxrpc_destroy_all_transports();
-        rxrpc_destroy_all_peers();
-        rxrpc_destroy_all_locals();
 
         ASSERTCMP(atomic_read(&rxrpc_n_skbs), ==, 0);
 
+        /* We need to flush the scheduled work twice because the local endpoint
+         * records involve a work item in their destruction as they can only be
+         * destroyed from process context.  However, a connection may have a
+         * work item outstanding - and this will pin the local endpoint record
+         * until the connection goes away.
+         *
+         * Peers don't pin locals and calls pin sockets - which prevents the
+         * module from being unloaded - so we should only need two flushes.
+         */
         _debug("flush scheduled work");
         flush_workqueue(rxrpc_workqueue);
+        _debug("flush scheduled work 2");
+        flush_workqueue(rxrpc_workqueue);
+        _debug("synchronise RCU");
+        rcu_barrier();
+        _debug("destroy locals");
+        ASSERT(idr_is_empty(&rxrpc_client_conn_ids));
+        idr_destroy(&rxrpc_client_conn_ids);
+        rxrpc_destroy_all_locals();
+
         remove_proc_entry("rxrpc_conns", init_net.proc_net);
         remove_proc_entry("rxrpc_calls", init_net.proc_net);
         destroy_workqueue(rxrpc_workqueue);
diff --git a/net/rxrpc/ar-connection.c b/net/rxrpc/ar-connection.c
deleted file mode 100644
index 97f4fae74bca..000000000000
--- a/net/rxrpc/ar-connection.c
+++ /dev/null
@@ -1,927 +0,0 @@
-/* RxRPC virtual connection handler
- *
- * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
- * Written by David Howells (dhowells@redhat.com)
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
- */
-
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/net.h>
-#include <linux/skbuff.h>
-#include <linux/crypto.h>
-#include <net/sock.h>
-#include <net/af_rxrpc.h>
-#include "ar-internal.h"
-
-/*
- * Time till a connection expires after last use (in seconds).
- */
-unsigned int rxrpc_connection_expiry = 10 * 60;
-
-static void rxrpc_connection_reaper(struct work_struct *work);
-
-LIST_HEAD(rxrpc_connections);
-DEFINE_RWLOCK(rxrpc_connection_lock);
-static DECLARE_DELAYED_WORK(rxrpc_connection_reap, rxrpc_connection_reaper);
-
-/*
- * allocate a new client connection bundle
- */
-static struct rxrpc_conn_bundle *rxrpc_alloc_bundle(gfp_t gfp)
-{
-        struct rxrpc_conn_bundle *bundle;
-
-        _enter("");
-
-        bundle = kzalloc(sizeof(struct rxrpc_conn_bundle), gfp);
-        if (bundle) {
-                INIT_LIST_HEAD(&bundle->unused_conns);
-                INIT_LIST_HEAD(&bundle->avail_conns);
-                INIT_LIST_HEAD(&bundle->busy_conns);
-                init_waitqueue_head(&bundle->chanwait);
-                atomic_set(&bundle->usage, 1);
-        }
-
-        _leave(" = %p", bundle);
-        return bundle;
-}
-
-/*
- * compare bundle parameters with what we're looking for
- * - return -ve, 0 or +ve
- */
-static inline
-int rxrpc_cmp_bundle(const struct rxrpc_conn_bundle *bundle,
-                     struct key *key, u16 service_id)
-{
-        return (bundle->service_id - service_id) ?:
-                ((unsigned long)bundle->key - (unsigned long)key);
-}
-
-/*
- * get bundle of client connections that a client socket can make use of
- */
-struct rxrpc_conn_bundle *rxrpc_get_bundle(struct rxrpc_sock *rx,
-                                           struct rxrpc_transport *trans,
-                                           struct key *key,
-                                           u16 service_id,
-                                           gfp_t gfp)
-{
-        struct rxrpc_conn_bundle *bundle, *candidate;
-        struct rb_node *p, *parent, **pp;
-
-        _enter("%p{%x},%x,%hx,",
-               rx, key_serial(key), trans->debug_id, service_id);
-
-        if (rx->trans == trans && rx->bundle) {
-                atomic_inc(&rx->bundle->usage);
-                return rx->bundle;
-        }
-
-        /* search the extant bundles first for one that matches the specified
-         * user ID */
-        spin_lock(&trans->client_lock);
-
-        p = trans->bundles.rb_node;
-        while (p) {
-                bundle = rb_entry(p, struct rxrpc_conn_bundle, node);
-
-                if (rxrpc_cmp_bundle(bundle, key, service_id) < 0)
-                        p = p->rb_left;
-                else if (rxrpc_cmp_bundle(bundle, key, service_id) > 0)
-                        p = p->rb_right;
-                else
-                        goto found_extant_bundle;
-        }
-
-        spin_unlock(&trans->client_lock);
-
-        /* not yet present - create a candidate for a new record and then
-         * redo the search */
-        candidate = rxrpc_alloc_bundle(gfp);
-        if (!candidate) {
-                _leave(" = -ENOMEM");
-                return ERR_PTR(-ENOMEM);
-        }
-
-        candidate->key = key_get(key);
-        candidate->service_id = service_id;
-
-        spin_lock(&trans->client_lock);
-
-        pp = &trans->bundles.rb_node;
-        parent = NULL;
-        while (*pp) {
-                parent = *pp;
-                bundle = rb_entry(parent, struct rxrpc_conn_bundle, node);
-
-                if (rxrpc_cmp_bundle(bundle, key, service_id) < 0)
-                        pp = &(*pp)->rb_left;
-                else if (rxrpc_cmp_bundle(bundle, key, service_id) > 0)
-                        pp = &(*pp)->rb_right;
-                else
-                        goto found_extant_second;
-        }
-
-        /* second search also failed; add the new bundle */
-        bundle = candidate;
-        candidate = NULL;
-
-        rb_link_node(&bundle->node, parent, pp);
-        rb_insert_color(&bundle->node, &trans->bundles);
-        spin_unlock(&trans->client_lock);
-        _net("BUNDLE new on trans %d", trans->debug_id);
-        if (!rx->bundle && rx->sk.sk_state == RXRPC_CLIENT_CONNECTED) {
-                atomic_inc(&bundle->usage);
-                rx->bundle = bundle;
-        }
-        _leave(" = %p [new]", bundle);
-        return bundle;
-
-        /* we found the bundle in the list immediately */
-found_extant_bundle:
-        atomic_inc(&bundle->usage);
-        spin_unlock(&trans->client_lock);
-        _net("BUNDLE old on trans %d", trans->debug_id);
-        if (!rx->bundle && rx->sk.sk_state == RXRPC_CLIENT_CONNECTED) {
-                atomic_inc(&bundle->usage);
-                rx->bundle = bundle;
-        }
-        _leave(" = %p [extant %d]", bundle, atomic_read(&bundle->usage));
-        return bundle;
-
-        /* we found the bundle on the second time through the list */
-found_extant_second:
-        atomic_inc(&bundle->usage);
-        spin_unlock(&trans->client_lock);
-        kfree(candidate);
-        _net("BUNDLE old2 on trans %d", trans->debug_id);
-        if (!rx->bundle && rx->sk.sk_state == RXRPC_CLIENT_CONNECTED) {
-                atomic_inc(&bundle->usage);
-                rx->bundle = bundle;
-        }
-        _leave(" = %p [second %d]", bundle, atomic_read(&bundle->usage));
-        return bundle;
-}
-
-/*
- * release a bundle
- */
-void rxrpc_put_bundle(struct rxrpc_transport *trans,
-                      struct rxrpc_conn_bundle *bundle)
-{
-        _enter("%p,%p{%d}",trans, bundle, atomic_read(&bundle->usage));
-
-        if (atomic_dec_and_lock(&bundle->usage, &trans->client_lock)) {
-                _debug("Destroy bundle");
-                rb_erase(&bundle->node, &trans->bundles);
-                spin_unlock(&trans->client_lock);
-                ASSERT(list_empty(&bundle->unused_conns));
-                ASSERT(list_empty(&bundle->avail_conns));
-                ASSERT(list_empty(&bundle->busy_conns));
-                ASSERTCMP(bundle->num_conns, ==, 0);
-                key_put(bundle->key);
-                kfree(bundle);
-        }
-
-        _leave("");
-}
-
-/*
- * allocate a new connection
- */
-static struct rxrpc_connection *rxrpc_alloc_connection(gfp_t gfp)
-{
-        struct rxrpc_connection *conn;
-
-        _enter("");
-
-        conn = kzalloc(sizeof(struct rxrpc_connection), gfp);
-        if (conn) {
-                INIT_WORK(&conn->processor, &rxrpc_process_connection);
-                INIT_LIST_HEAD(&conn->bundle_link);
-                conn->calls = RB_ROOT;
-                skb_queue_head_init(&conn->rx_queue);
-                conn->security = &rxrpc_no_security;
-                rwlock_init(&conn->lock);
-                spin_lock_init(&conn->state_lock);
-                atomic_set(&conn->usage, 1);
-                conn->debug_id = atomic_inc_return(&rxrpc_debug_id);
-                conn->avail_calls = RXRPC_MAXCALLS;
-                conn->size_align = 4;
-                conn->header_size = sizeof(struct rxrpc_wire_header);
-        }
-
-        _leave(" = %p{%d}", conn, conn ? conn->debug_id : 0);
-        return conn;
-}
-
-/*
- * assign a connection ID to a connection and add it to the transport's
- * connection lookup tree
- * - called with transport client lock held
- */
-static void rxrpc_assign_connection_id(struct rxrpc_connection *conn)
-{
-        struct rxrpc_connection *xconn;
-        struct rb_node *parent, **p;
-        __be32 epoch;
-        u32 cid;
-
-        _enter("");
-
-        epoch = conn->epoch;
-
-        write_lock_bh(&conn->trans->conn_lock);
-
-        conn->trans->conn_idcounter += RXRPC_CID_INC;
-        if (conn->trans->conn_idcounter < RXRPC_CID_INC)
-                conn->trans->conn_idcounter = RXRPC_CID_INC;
-        cid = conn->trans->conn_idcounter;
-
-attempt_insertion:
-        parent = NULL;
-        p = &conn->trans->client_conns.rb_node;
-
-        while (*p) {
-                parent = *p;
-                xconn = rb_entry(parent, struct rxrpc_connection, node);
-
-                if (epoch < xconn->epoch)
-                        p = &(*p)->rb_left;
-                else if (epoch > xconn->epoch)
-                        p = &(*p)->rb_right;
-                else if (cid < xconn->cid)
-                        p = &(*p)->rb_left;
-                else if (cid > xconn->cid)
-                        p = &(*p)->rb_right;
-                else
-                        goto id_exists;
-        }
-
-        /* we've found a suitable hole - arrange for this connection to occupy
-         * it */
-        rb_link_node(&conn->node, parent, p);
-        rb_insert_color(&conn->node, &conn->trans->client_conns);
-
-        conn->cid = cid;
-        write_unlock_bh(&conn->trans->conn_lock);
-        _leave(" [CID %x]", cid);
-        return;
-
-        /* we found a connection with the proposed ID - walk the tree from that
-         * point looking for the next unused ID */
-id_exists:
-        for (;;) {
-                cid += RXRPC_CID_INC;
-                if (cid < RXRPC_CID_INC) {
-                        cid = RXRPC_CID_INC;
-                        conn->trans->conn_idcounter = cid;
-                        goto attempt_insertion;
-                }
-
-                parent = rb_next(parent);
-                if (!parent)
-                        goto attempt_insertion;
-
-                xconn = rb_entry(parent, struct rxrpc_connection, node);
-                if (epoch < xconn->epoch ||
-                    cid < xconn->cid)
-                        goto attempt_insertion;
-        }
-}
-
-/*
- * add a call to a connection's call-by-ID tree
- */
-static void rxrpc_add_call_ID_to_conn(struct rxrpc_connection *conn,
-                                      struct rxrpc_call *call)
-{
-        struct rxrpc_call *xcall;
-        struct rb_node *parent, **p;
-        __be32 call_id;
-
-        write_lock_bh(&conn->lock);
-
-        call_id = call->call_id;
-        p = &conn->calls.rb_node;
-        parent = NULL;
-        while (*p) {
-                parent = *p;
-                xcall = rb_entry(parent, struct rxrpc_call, conn_node);
-
-                if (call_id < xcall->call_id)
-                        p = &(*p)->rb_left;
-                else if (call_id > xcall->call_id)
-                        p = &(*p)->rb_right;
-                else
-                        BUG();
-        }
-
-        rb_link_node(&call->conn_node, parent, p);
-        rb_insert_color(&call->conn_node, &conn->calls);
-
-        write_unlock_bh(&conn->lock);
-}
-
-/*
- * connect a call on an exclusive connection
- */
-static int rxrpc_connect_exclusive(struct rxrpc_sock *rx,
-                                   struct rxrpc_transport *trans,
-                                   u16 service_id,
-                                   struct rxrpc_call *call,
-                                   gfp_t gfp)
-{
-        struct rxrpc_connection *conn;
-        int chan, ret;
-
-        _enter("");
-
-        conn = rx->conn;
-        if (!conn) {
-                /* not yet present - create a candidate for a new connection
-                 * and then redo the check */
-                conn = rxrpc_alloc_connection(gfp);
-                if (!conn) {
-                        _leave(" = -ENOMEM");
-                        return -ENOMEM;
-                }
-
-                conn->trans = trans;
-                conn->bundle = NULL;
-                conn->service_id = service_id;
-                conn->epoch = rxrpc_epoch;
-                conn->in_clientflag = 0;
-                conn->out_clientflag = RXRPC_CLIENT_INITIATED;
-                conn->cid = 0;
-                conn->state = RXRPC_CONN_CLIENT;
-                conn->avail_calls = RXRPC_MAXCALLS - 1;
-                conn->security_level = rx->min_sec_level;
-                conn->key = key_get(rx->key);
-
-                ret = rxrpc_init_client_conn_security(conn);
-                if (ret < 0) {
-                        key_put(conn->key);
-                        kfree(conn);
-                        _leave(" = %d [key]", ret);
-                        return ret;
-                }
-
-                write_lock_bh(&rxrpc_connection_lock);
-                list_add_tail(&conn->link, &rxrpc_connections);
-                write_unlock_bh(&rxrpc_connection_lock);
-
-                spin_lock(&trans->client_lock);
-                atomic_inc(&trans->usage);
-
-                _net("CONNECT EXCL new %d on TRANS %d",
-                     conn->debug_id, conn->trans->debug_id);
-
-                rxrpc_assign_connection_id(conn);
-                rx->conn = conn;
-        } else {
-                spin_lock(&trans->client_lock);
-        }
-
-        /* we've got a connection with a free channel and we can now attach the
-         * call to it
-         * - we're holding the transport's client lock
-         * - we're holding a reference on the connection
-         */
-        for (chan = 0; chan < RXRPC_MAXCALLS; chan++)
-                if (!conn->channels[chan])
-                        goto found_channel;
-        goto no_free_channels;
-
-found_channel:
-        atomic_inc(&conn->usage);
-        conn->channels[chan] = call;
-        call->conn = conn;
-        call->channel = chan;
-        call->cid = conn->cid | chan;
-        call->call_id = ++conn->call_counter;
-
-        _net("CONNECT client on conn %d chan %d as call %x",
-             conn->debug_id, chan, call->call_id);
-
-        spin_unlock(&trans->client_lock);
-
-        rxrpc_add_call_ID_to_conn(conn, call);
-        _leave(" = 0");
-        return 0;
-
-no_free_channels:
-        spin_unlock(&trans->client_lock);
-        _leave(" = -ENOSR");
-        return -ENOSR;
-}
-
-/*
- * find a connection for a call
- * - called in process context with IRQs enabled
- */
-int rxrpc_connect_call(struct rxrpc_sock *rx,
-                       struct rxrpc_transport *trans,
-                       struct rxrpc_conn_bundle *bundle,
-                       struct rxrpc_call *call,
-                       gfp_t gfp)
-{
-        struct rxrpc_connection *conn, *candidate;
-        int chan, ret;
-
-        DECLARE_WAITQUEUE(myself, current);
-
-        _enter("%p,%lx,", rx, call->user_call_ID);
-
-        if (test_bit(RXRPC_SOCK_EXCLUSIVE_CONN, &rx->flags))
-                return rxrpc_connect_exclusive(rx, trans, bundle->service_id,
-                                               call, gfp);
-
-        spin_lock(&trans->client_lock);
-        for (;;) {
-                /* see if the bundle has a call slot available */
-                if (!list_empty(&bundle->avail_conns)) {
-                        _debug("avail");
-                        conn = list_entry(bundle->avail_conns.next,
-                                          struct rxrpc_connection,
-                                          bundle_link);
-                        if (conn->state >= RXRPC_CONN_REMOTELY_ABORTED) {
-                                list_del_init(&conn->bundle_link);
-                                bundle->num_conns--;
-                                continue;
-                        }
-                        if (--conn->avail_calls == 0)
-                                list_move(&conn->bundle_link,
-                                          &bundle->busy_conns);
-                        ASSERTCMP(conn->avail_calls, <, RXRPC_MAXCALLS);
-                        ASSERT(conn->channels[0] == NULL ||
-                               conn->channels[1] == NULL ||
-                               conn->channels[2] == NULL ||
-                               conn->channels[3] == NULL);
-                        atomic_inc(&conn->usage);
-                        break;
-                }
-
-                if (!list_empty(&bundle->unused_conns)) {
-                        _debug("unused");
-                        conn = list_entry(bundle->unused_conns.next,
-                                          struct rxrpc_connection,
-                                          bundle_link);
-                        if (conn->state >= RXRPC_CONN_REMOTELY_ABORTED) {
-                                list_del_init(&conn->bundle_link);
-                                bundle->num_conns--;
-                                continue;
-                        }
-                        ASSERTCMP(conn->avail_calls, ==, RXRPC_MAXCALLS);
-                        conn->avail_calls = RXRPC_MAXCALLS - 1;
-                        ASSERT(conn->channels[0] == NULL &&
-                               conn->channels[1] == NULL &&
-                               conn->channels[2] == NULL &&
-                               conn->channels[3] == NULL);
-                        atomic_inc(&conn->usage);
-                        list_move(&conn->bundle_link, &bundle->avail_conns);
-                        break;
-                }
-
-                /* need to allocate a new connection */
-                _debug("get new conn [%d]", bundle->num_conns);
-
-                spin_unlock(&trans->client_lock);
-
-                if (signal_pending(current))
-                        goto interrupted;
-
-                if (bundle->num_conns >= 20) {
-                        _debug("too many conns");
-
-                        if (!gfpflags_allow_blocking(gfp)) {
-                                _leave(" = -EAGAIN");
-                                return -EAGAIN;
-                        }
-
-                        add_wait_queue(&bundle->chanwait, &myself);
-                        for (;;) {
-                                set_current_state(TASK_INTERRUPTIBLE);
-                                if (bundle->num_conns < 20 ||
-                                    !list_empty(&bundle->unused_conns) ||
-                                    !list_empty(&bundle->avail_conns))
-                                        break;
-                                if (signal_pending(current))
-                                        goto interrupted_dequeue;
-                                schedule();
-                        }
-                        remove_wait_queue(&bundle->chanwait, &myself);
-                        __set_current_state(TASK_RUNNING);
-                        spin_lock(&trans->client_lock);
-                        continue;
-                }
-
-                /* not yet present - create a candidate for a new connection and then
-                 * redo the check */
-                candidate = rxrpc_alloc_connection(gfp);
-                if (!candidate) {
-                        _leave(" = -ENOMEM");
-                        return -ENOMEM;
-                }
-
-                candidate->trans = trans;
-                candidate->bundle = bundle;
-                candidate->service_id = bundle->service_id;
-                candidate->epoch = rxrpc_epoch;
-                candidate->in_clientflag = 0;
-                candidate->out_clientflag = RXRPC_CLIENT_INITIATED;
-                candidate->cid = 0;
-                candidate->state = RXRPC_CONN_CLIENT;
-                candidate->avail_calls = RXRPC_MAXCALLS;
-                candidate->security_level = rx->min_sec_level;
-                candidate->key = key_get(bundle->key);
-
-                ret = rxrpc_init_client_conn_security(candidate);
-                if (ret < 0) {
-                        key_put(candidate->key);
-                        kfree(candidate);
-                        _leave(" = %d [key]", ret);
-                        return ret;
-                }
-
-                write_lock_bh(&rxrpc_connection_lock);
-                list_add_tail(&candidate->link, &rxrpc_connections);
-                write_unlock_bh(&rxrpc_connection_lock);
-
-                spin_lock(&trans->client_lock);
-
-                list_add(&candidate->bundle_link, &bundle->unused_conns);
-                bundle->num_conns++;
-                atomic_inc(&bundle->usage);
-                atomic_inc(&trans->usage);
-
-                _net("CONNECT new %d on TRANS %d",
-                     candidate->debug_id, candidate->trans->debug_id);
-
-                rxrpc_assign_connection_id(candidate);
-                candidate->security->prime_packet_security(candidate);
-
-                /* leave the candidate lurking in zombie mode attached to the
-                 * bundle until we're ready for it */
-                rxrpc_put_connection(candidate);
-                candidate = NULL;
-        }
-
-        /* we've got a connection with a free channel and we can now attach the
-         * call to it
-         * - we're holding the transport's client lock
-         * - we're holding a reference on the connection
-         * - we're holding a reference on the bundle
-         */
-        for (chan = 0; chan < RXRPC_MAXCALLS; chan++)
-                if (!conn->channels[chan])
-                        goto found_channel;
-        ASSERT(conn->channels[0] == NULL ||
-               conn->channels[1] == NULL ||
-               conn->channels[2] == NULL ||
-               conn->channels[3] == NULL);
-        BUG();
-
-found_channel:
-        conn->channels[chan] = call;
-        call->conn = conn;
-        call->channel = chan;
-        call->cid = conn->cid | chan;
-        call->call_id = ++conn->call_counter;
-
-        _net("CONNECT client on conn %d chan %d as call %x",
-             conn->debug_id, chan, call->call_id);
-
-        ASSERTCMP(conn->avail_calls, <, RXRPC_MAXCALLS);
-        spin_unlock(&trans->client_lock);
-
-        rxrpc_add_call_ID_to_conn(conn, call);
-
-        _leave(" = 0");
-        return 0;
-
-interrupted_dequeue:
-        remove_wait_queue(&bundle->chanwait, &myself);
-        __set_current_state(TASK_RUNNING);
-interrupted:
-        _leave(" = -ERESTARTSYS");
-        return -ERESTARTSYS;
-}
-
-/*
- * get a record of an incoming connection
- */
-struct rxrpc_connection *
-rxrpc_incoming_connection(struct rxrpc_transport *trans,
-                          struct rxrpc_host_header *hdr)
-{
-        struct rxrpc_connection *conn, *candidate = NULL;
-        struct rb_node *p, **pp;
-        const char *new = "old";
-        __be32 epoch;
-        u32 cid;
-
-        _enter("");
-
-        ASSERT(hdr->flags & RXRPC_CLIENT_INITIATED);
-
-        epoch = hdr->epoch;
-        cid = hdr->cid & RXRPC_CIDMASK;
-
-        /* search the connection list first */
-        read_lock_bh(&trans->conn_lock);
-
-        p = trans->server_conns.rb_node;
-        while (p) {
-                conn = rb_entry(p, struct rxrpc_connection, node);
-
-                _debug("maybe %x", conn->cid);
-
-                if (epoch < conn->epoch)
-                        p = p->rb_left;
-                else if (epoch > conn->epoch)
-                        p = p->rb_right;
-                else if (cid < conn->cid)
-                        p = p->rb_left;
-                else if (cid > conn->cid)
-                        p = p->rb_right;
-                else
-                        goto found_extant_connection;
-        }
-        read_unlock_bh(&trans->conn_lock);
-
-        /* not yet present - create a candidate for a new record and then
-         * redo the search */
-        candidate = rxrpc_alloc_connection(GFP_NOIO);
-        if (!candidate) {
-                _leave(" = -ENOMEM");
-                return ERR_PTR(-ENOMEM);
-        }
-
-        candidate->trans = trans;
-        candidate->epoch = hdr->epoch;
-        candidate->cid = hdr->cid & RXRPC_CIDMASK;
-        candidate->service_id = hdr->serviceId;
-        candidate->security_ix = hdr->securityIndex;
-        candidate->in_clientflag = RXRPC_CLIENT_INITIATED;
-        candidate->out_clientflag = 0;
-        candidate->state = RXRPC_CONN_SERVER;
-        if (candidate->service_id)
-                candidate->state = RXRPC_CONN_SERVER_UNSECURED;
-
-        write_lock_bh(&trans->conn_lock);
-
-        pp = &trans->server_conns.rb_node;
-        p = NULL;
-        while (*pp) {
-                p = *pp;
-                conn = rb_entry(p, struct rxrpc_connection, node);
-
-                if (epoch < conn->epoch)
-                        pp = &(*pp)->rb_left;
-                else if (epoch > conn->epoch)
-                        pp = &(*pp)->rb_right;
-                else if (cid < conn->cid)
-                        pp = &(*pp)->rb_left;
-                else if (cid > conn->cid)
-                        pp = &(*pp)->rb_right;
-                else
-                        goto found_extant_second;
-        }
-
-        /* we can now add the new candidate to the list */
-        conn = candidate;
-        candidate = NULL;
-        rb_link_node(&conn->node, p, pp);
-        rb_insert_color(&conn->node, &trans->server_conns);
-        atomic_inc(&conn->trans->usage);
-
-        write_unlock_bh(&trans->conn_lock);
-
-        write_lock_bh(&rxrpc_connection_lock);
-        list_add_tail(&conn->link, &rxrpc_connections);
-        write_unlock_bh(&rxrpc_connection_lock);
-
-        new = "new";
-
-success:
-        _net("CONNECTION %s %d {%x}", new, conn->debug_id, conn->cid);
-
-        _leave(" = %p {u=%d}", conn, atomic_read(&conn->usage));
-        return conn;
-
-        /* we found the connection in the list immediately */
-found_extant_connection:
-        if (hdr->securityIndex != conn->security_ix) {
-                read_unlock_bh(&trans->conn_lock);
-                goto security_mismatch;
-        }
-        atomic_inc(&conn->usage);
-        read_unlock_bh(&trans->conn_lock);
-        goto success;
-
-        /* we found the connection on the second time through the list */
-found_extant_second:
-        if (hdr->securityIndex != conn->security_ix) {
-                write_unlock_bh(&trans->conn_lock);
-                goto security_mismatch;
-        }
-        atomic_inc(&conn->usage);
-        write_unlock_bh(&trans->conn_lock);
-        kfree(candidate);
-        goto success;
-
-security_mismatch:
-        kfree(candidate);
-        _leave(" = -EKEYREJECTED");
-        return ERR_PTR(-EKEYREJECTED);
-}
-
-/*
- * find a connection based on transport and RxRPC connection ID for an incoming
- * packet
- */
-struct rxrpc_connection *rxrpc_find_connection(struct rxrpc_transport *trans,
-                                               struct rxrpc_host_header *hdr)
-{
-        struct rxrpc_connection *conn;
-        struct rb_node *p;
-        u32 epoch, cid;
-
-        _enter(",{%x,%x}", hdr->cid, hdr->flags);
-
-        read_lock_bh(&trans->conn_lock);
-
-        cid = hdr->cid & RXRPC_CIDMASK;
-        epoch = hdr->epoch;
-
-        if (hdr->flags & RXRPC_CLIENT_INITIATED)
-                p = trans->server_conns.rb_node;
-        else
-                p = trans->client_conns.rb_node;
-
-        while (p) {
-                conn = rb_entry(p, struct rxrpc_connection, node);
-
-                _debug("maybe %x", conn->cid);
-
-                if (epoch < conn->epoch)
-                        p = p->rb_left;
-                else if (epoch > conn->epoch)
-                        p = p->rb_right;
-                else if (cid < conn->cid)
-                        p = p->rb_left;
-                else if (cid > conn->cid)
-                        p = p->rb_right;
-                else
-                        goto found;
-        }
-
-        read_unlock_bh(&trans->conn_lock);
-        _leave(" = NULL");
-        return NULL;
-
-found:
-        atomic_inc(&conn->usage);
-        read_unlock_bh(&trans->conn_lock);
-        _leave(" = %p", conn);
-        return conn;
-}
-
-/*
- * release a virtual connection
- */
-void rxrpc_put_connection(struct rxrpc_connection *conn)
-{
-        _enter("%p{u=%d,d=%d}",
-               conn, atomic_read(&conn->usage), conn->debug_id);
-
-        ASSERTCMP(atomic_read(&conn->usage), >, 0);
-
-        conn->put_time = ktime_get_seconds();
-        if (atomic_dec_and_test(&conn->usage)) {
-                _debug("zombie");
-                rxrpc_queue_delayed_work(&rxrpc_connection_reap, 0);
-        }
-
-        _leave("");
-}
-
-/*
- * destroy a virtual connection
- */
-static void rxrpc_destroy_connection(struct rxrpc_connection *conn)
-{
-        _enter("%p{%d}", conn, atomic_read(&conn->usage));
-
-        ASSERTCMP(atomic_read(&conn->usage), ==, 0);
-
-        _net("DESTROY CONN %d", conn->debug_id);
-
-        if (conn->bundle)
-                rxrpc_put_bundle(conn->trans, conn->bundle);
-
-        ASSERT(RB_EMPTY_ROOT(&conn->calls));
-        rxrpc_purge_queue(&conn->rx_queue);
-
-        conn->security->clear(conn);
-        key_put(conn->key);
-        key_put(conn->server_key);
-
-        rxrpc_put_transport(conn->trans);
-        kfree(conn);
-        _leave("");
-}
-
-/*
- * reap dead connections
- */
-static void rxrpc_connection_reaper(struct work_struct *work)
-{
-        struct rxrpc_connection *conn, *_p;
-        unsigned long now, earliest, reap_time;
-
-        LIST_HEAD(graveyard);
-
-        _enter("");
-
-        now = ktime_get_seconds();
-        earliest = ULONG_MAX;
-
-        write_lock_bh(&rxrpc_connection_lock);
-        list_for_each_entry_safe(conn, _p, &rxrpc_connections, link) {
-                _debug("reap CONN %d { u=%d,t=%ld }",
-                       conn->debug_id, atomic_read(&conn->usage),
-                       (long) now - (long) conn->put_time);
-
-                if (likely(atomic_read(&conn->usage) > 0))
-                        continue;
-
-                spin_lock(&conn->trans->client_lock);
-                write_lock(&conn->trans->conn_lock);
-                reap_time = conn->put_time + rxrpc_connection_expiry;
-
-                if (atomic_read(&conn->usage) > 0) {
-                        ;
-                } else if (reap_time <= now) {
-                        list_move_tail(&conn->link, &graveyard);
-                        if (conn->out_clientflag)
-                                rb_erase(&conn->node,
-                                         &conn->trans->client_conns);
-                        else
-                                rb_erase(&conn->node,
-                                         &conn->trans->server_conns);
-                        if (conn->bundle) {
-                                list_del_init(&conn->bundle_link);
-                                conn->bundle->num_conns--;
-                        }
-
-                } else if (reap_time < earliest) {
-                        earliest = reap_time;
-                }
-
-                write_unlock(&conn->trans->conn_lock);
-                spin_unlock(&conn->trans->client_lock);
-        }
-        write_unlock_bh(&rxrpc_connection_lock);
-
-        if (earliest != ULONG_MAX) {
-                _debug("reschedule reaper %ld", (long) earliest - now);
-                ASSERTCMP(earliest, >, now);
-                rxrpc_queue_delayed_work(&rxrpc_connection_reap,
-                                         (earliest - now) * HZ);
-        }
-
-        /* then destroy all those pulled out */
-        while (!list_empty(&graveyard)) {
-                conn = list_entry(graveyard.next, struct rxrpc_connection,
-                                  link);
-                list_del_init(&conn->link);
-
-                ASSERTCMP(atomic_read(&conn->usage), ==, 0);
-                rxrpc_destroy_connection(conn);
-        }
-
-        _leave("");
-}
-
-/*
- * preemptively destroy all the connection records rather than waiting for them
- * to time out
- */
-void __exit rxrpc_destroy_all_connections(void)
-{
-        _enter("");
-
-        rxrpc_connection_expiry = 0;
-        cancel_delayed_work(&rxrpc_connection_reap);
-        rxrpc_queue_delayed_work(&rxrpc_connection_reap, 0);
-
-        _leave("");
-}
diff --git a/net/rxrpc/ar-error.c b/net/rxrpc/ar-error.c
deleted file mode 100644
index 3e82d6f0313c..000000000000
--- a/net/rxrpc/ar-error.c
+++ /dev/null
@@ -1,230 +0,0 @@
-/* Error message handling (ICMP)
- *
- * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
- * Written by David Howells (dhowells@redhat.com)
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
- */
-
-#include <linux/module.h>
-#include <linux/net.h>
-#include <linux/skbuff.h>
-#include <linux/errqueue.h>
-#include <linux/udp.h>
-#include <linux/in.h>
-#include <linux/in6.h>
-#include <linux/icmp.h>
-#include <net/sock.h>
-#include <net/af_rxrpc.h>
-#include <net/ip.h>
-#include "ar-internal.h"
-
-/*
- * handle an error received on the local endpoint
- */
-void rxrpc_UDP_error_report(struct sock *sk)
-{
-        struct sock_exterr_skb *serr;
-        struct rxrpc_transport *trans;
-        struct rxrpc_local *local = sk->sk_user_data;
-        struct rxrpc_peer *peer;
-        struct sk_buff *skb;
-        __be32 addr;
-        __be16 port;
-
-        _enter("%p{%d}", sk, local->debug_id);
-
-        skb = sock_dequeue_err_skb(sk);
-        if (!skb) {
-                _leave("UDP socket errqueue empty");
-                return;
-        }
-        serr = SKB_EXT_ERR(skb);
-        if (!skb->len && serr->ee.ee_origin == SO_EE_ORIGIN_TIMESTAMPING) {
-                _leave("UDP empty message");
-                kfree_skb(skb);
-                return;
-        }
-
-        rxrpc_new_skb(skb);
-
-        addr = *(__be32 *)(skb_network_header(skb) + serr->addr_offset);
-        port = serr->port;
-
-        _net("Rx UDP Error from %pI4:%hu", &addr, ntohs(port));
-        _debug("Msg l:%d d:%d", skb->len, skb->data_len);
-
-        peer = rxrpc_find_peer(local, addr, port);
-        if (IS_ERR(peer)) {
-                rxrpc_free_skb(skb);
-                _leave(" [no peer]");
-                return;
-        }
-
-        trans = rxrpc_find_transport(local, peer);
-        if (!trans) {
-                rxrpc_put_peer(peer);
-                rxrpc_free_skb(skb);
-                _leave(" [no trans]");
-                return;
-        }
-
-        if (serr->ee.ee_origin == SO_EE_ORIGIN_ICMP &&
-            serr->ee.ee_type == ICMP_DEST_UNREACH &&
-            serr->ee.ee_code == ICMP_FRAG_NEEDED
-            ) {
-                u32 mtu = serr->ee.ee_info;
-
-                _net("Rx Received ICMP Fragmentation Needed (%d)", mtu);
-
-                /* wind down the local interface MTU */
-                if (mtu > 0 && peer->if_mtu == 65535 && mtu < peer->if_mtu) {
-                        peer->if_mtu = mtu;
-                        _net("I/F MTU %u", mtu);
-                }
-
-                if (mtu == 0) {
-                        /* they didn't give us a size, estimate one */
-                        mtu = peer->if_mtu;
-                        if (mtu > 1500) {
-                                mtu >>= 1;
-                                if (mtu < 1500)
-                                        mtu = 1500;
-                        } else {
-                                mtu -= 100;
-                                if (mtu < peer->hdrsize)
-                                        mtu = peer->hdrsize + 4;
-                        }
-                }
-
-                if (mtu < peer->mtu) {
-                        spin_lock_bh(&peer->lock);
-                        peer->mtu = mtu;
-                        peer->maxdata = peer->mtu - peer->hdrsize;
-                        spin_unlock_bh(&peer->lock);
-                        _net("Net MTU %u (maxdata %u)",
-                             peer->mtu, peer->maxdata);
-                }
-        }
-
-        rxrpc_put_peer(peer);
-
-        /* pass the transport ref to error_handler to release */
-        skb_queue_tail(&trans->error_queue, skb);
-        rxrpc_queue_work(&trans->error_handler);
-        _leave("");
-}
-
-/*
- * deal with UDP error messages
- */
-void rxrpc_UDP_error_handler(struct work_struct *work)
-{
-        struct sock_extended_err *ee;
-        struct sock_exterr_skb *serr;
-        struct rxrpc_transport *trans =
-                container_of(work, struct rxrpc_transport, error_handler);
-        struct sk_buff *skb;
-        int err;
-
-        _enter("");
-
-        skb = skb_dequeue(&trans->error_queue);
-        if (!skb)
-                return;
-
-        serr = SKB_EXT_ERR(skb);
-        ee = &serr->ee;
-
-        _net("Rx Error o=%d t=%d c=%d e=%d",
-             ee->ee_origin, ee->ee_type, ee->ee_code, ee->ee_errno);
-
-        err = ee->ee_errno;
-
-        switch (ee->ee_origin) {
-        case SO_EE_ORIGIN_ICMP:
-                switch (ee->ee_type) {
-                case ICMP_DEST_UNREACH:
-                        switch (ee->ee_code) {
-                        case ICMP_NET_UNREACH:
-                                _net("Rx Received ICMP Network Unreachable");
-                                break;
-                        case ICMP_HOST_UNREACH:
-                                _net("Rx Received ICMP Host Unreachable");
-                                break;
-                        case ICMP_PORT_UNREACH:
-                                _net("Rx Received ICMP Port Unreachable");
-                                break;
-                        case ICMP_NET_UNKNOWN:
-                                _net("Rx Received ICMP Unknown Network");
-                                break;
-                        case ICMP_HOST_UNKNOWN:
-                                _net("Rx Received ICMP Unknown Host");
-                                break;
-                        default:
-                                _net("Rx Received ICMP DestUnreach code=%u",
-                                     ee->ee_code);
-                                break;
-                        }
-                        break;
-
-                case ICMP_TIME_EXCEEDED:
-                        _net("Rx Received ICMP TTL Exceeded");
-                        break;
-
-                default:
-                        _proto("Rx Received ICMP error { type=%u code=%u }",
-                               ee->ee_type, ee->ee_code);
-                        break;
-                }
-                break;
-
-        case SO_EE_ORIGIN_LOCAL:
-                _proto("Rx Received local error { error=%d }",
-                       ee->ee_errno);
-                break;
-
-        case SO_EE_ORIGIN_NONE:
-        case SO_EE_ORIGIN_ICMP6:
-        default:
-                _proto("Rx Received error report { orig=%u }",
-                       ee->ee_origin);
-                break;
-        }
-
-        /* terminate all the affected calls if there's an unrecoverable
-         * error */
-        if (err) {
-                struct rxrpc_call *call, *_n;
-
-                _debug("ISSUE ERROR %d", err);
-
-                spin_lock_bh(&trans->peer->lock);
-                trans->peer->net_error = err;
-
-                list_for_each_entry_safe(call, _n, &trans->peer->error_targets,
-                                         error_link) {
-                        write_lock(&call->state_lock);
-                        if (call->state != RXRPC_CALL_COMPLETE &&
-                            call->state < RXRPC_CALL_NETWORK_ERROR) {
-                                call->state = RXRPC_CALL_NETWORK_ERROR;
-                                set_bit(RXRPC_CALL_EV_RCVD_ERROR, &call->events);
-                                rxrpc_queue_call(call);
-                        }
-                        write_unlock(&call->state_lock);
-                        list_del_init(&call->error_link);
-                }
-
-                spin_unlock_bh(&trans->peer->lock);
-        }
-
-        if (!skb_queue_empty(&trans->error_queue))
-                rxrpc_queue_work(&trans->error_handler);
-
-        rxrpc_free_skb(skb);
-        rxrpc_put_transport(trans);
-        _leave("");
-}
diff --git a/net/rxrpc/ar-internal.h b/net/rxrpc/ar-internal.h
index f0b807a163fa..702db72196fb 100644
--- a/net/rxrpc/ar-internal.h
+++ b/net/rxrpc/ar-internal.h
@@ -9,7 +9,9 @@
  * 2 of the License, or (at your option) any later version.
  */
 
+#include <linux/atomic.h>
 #include <net/sock.h>
+#include <net/af_rxrpc.h>
 #include <rxrpc/packet.h>
 
 #if 0
@@ -35,13 +37,15 @@ struct rxrpc_crypt {
 #define rxrpc_queue_call(CALL)  rxrpc_queue_work(&(CALL)->processor)
 #define rxrpc_queue_conn(CONN)  rxrpc_queue_work(&(CONN)->processor)
 
+struct rxrpc_connection;
+
 /*
  * sk_state for RxRPC sockets
  */
 enum {
-        RXRPC_UNCONNECTED = 0,
+        RXRPC_UNBOUND = 0,
+        RXRPC_CLIENT_UNBOUND,           /* Unbound socket used as client */
         RXRPC_CLIENT_BOUND,             /* client local address bound */
-        RXRPC_CLIENT_CONNECTED,         /* client is connected */
         RXRPC_SERVER_BOUND,             /* server local address bound */
         RXRPC_SERVER_LISTENING,         /* server listening for connections */
         RXRPC_CLOSE,                    /* socket is being closed */
@@ -55,9 +59,6 @@ struct rxrpc_sock {
         struct sock             sk;
         rxrpc_interceptor_t     interceptor;    /* kernel service Rx interceptor function */
         struct rxrpc_local      *local;         /* local endpoint */
-        struct rxrpc_transport  *trans;         /* transport handler */
-        struct rxrpc_conn_bundle *bundle;       /* virtual connection bundle */
-        struct rxrpc_connection *conn;          /* exclusive virtual connection */
         struct list_head        listen_link;    /* link in the local endpoint's listen list */
         struct list_head        secureq;        /* calls awaiting connection security clearance */
         struct list_head        acceptq;        /* calls awaiting acceptance */
@@ -65,12 +66,14 @@ struct rxrpc_sock {
         struct key              *securities;    /* list of server security descriptors */
         struct rb_root          calls;          /* outstanding calls on this socket */
         unsigned long           flags;
-#define RXRPC_SOCK_EXCLUSIVE_CONN       1       /* exclusive connection for a client socket */
+#define RXRPC_SOCK_CONNECTED            0       /* connect_srx is set */
         rwlock_t                call_lock;      /* lock for calls */
         u32                     min_sec_level;  /* minimum security level */
 #define RXRPC_SECURITY_MAX      RXRPC_SECURITY_ENCRYPT
+        bool                    exclusive;      /* Exclusive connection for a client socket */
+        sa_family_t             family;         /* Protocol family created with */
         struct sockaddr_rxrpc   srx;            /* local address */
-        sa_family_t             proto;          /* protocol created with */
+        struct sockaddr_rxrpc   connect_srx;    /* Default client address from connect() */
 };
 
 #define rxrpc_sk(__sk) container_of((__sk), struct rxrpc_sock, sk)
@@ -168,46 +171,52 @@ struct rxrpc_security {
 };
 
 /*
- * RxRPC local transport endpoint definition
- * - matched by local port, address and protocol type
+ * RxRPC local transport endpoint description
+ * - owned by a single AF_RXRPC socket
+ * - pointed to by transport socket struct sk_user_data
  */
 struct rxrpc_local {
+        struct rcu_head         rcu;
+        atomic_t                usage;
+        struct list_head        link;
         struct socket           *socket;        /* my UDP socket */
-        struct work_struct      destroyer;      /* endpoint destroyer */
-        struct work_struct      acceptor;       /* incoming call processor */
-        struct work_struct      rejecter;       /* packet reject writer */
-        struct work_struct      event_processor; /* endpoint event processor */
+        struct work_struct      processor;
         struct list_head        services;       /* services listening on this endpoint */
-        struct list_head        link;           /* link in endpoint list */
         struct rw_semaphore     defrag_sem;     /* control re-enablement of IP DF bit */
         struct sk_buff_head     accept_queue;   /* incoming calls awaiting acceptance */
         struct sk_buff_head     reject_queue;   /* packets awaiting rejection */
         struct sk_buff_head     event_queue;    /* endpoint event packets awaiting processing */
+        struct rb_root          client_conns;   /* Client connections by socket params */
+        spinlock_t              client_conns_lock; /* Lock for client_conns */
         spinlock_t              lock;           /* access lock */
         rwlock_t                services_lock;  /* lock for services list */
-        atomic_t                usage;
         int                     debug_id;       /* debug ID for printks */
-        volatile char           error_rcvd;     /* T if received ICMP error outstanding */
+        bool                    dead;
         struct sockaddr_rxrpc   srx;            /* local address */
 };
 
 /*
  * RxRPC remote transport endpoint definition
- * - matched by remote port, address and protocol type
- * - holds the connection ID counter for connections between the two endpoints
+ * - matched by local endpoint, remote port, address and protocol type
  */
 struct rxrpc_peer {
-        struct work_struct      destroyer;      /* peer destroyer */
-        struct list_head        link;           /* link in master peer list */
-        struct list_head        error_targets;  /* targets for net error distribution */
-        spinlock_t              lock;           /* access lock */
+        struct rcu_head         rcu;            /* This must be first */
         atomic_t                usage;
+        unsigned long           hash_key;
+        struct hlist_node       hash_link;
+        struct rxrpc_local      *local;
+        struct hlist_head       error_targets;  /* targets for net error distribution */
+        struct work_struct      error_distributor;
+        struct rb_root          service_conns;  /* Service connections */
+        rwlock_t                conn_lock;
+        spinlock_t              lock;           /* access lock */
         unsigned int            if_mtu;         /* interface MTU for this peer */
         unsigned int            mtu;            /* network MTU for this peer */
         unsigned int            maxdata;        /* data size (MTU - hdrsize) */
         unsigned short          hdrsize;        /* header size (IP + UDP + RxRPC) */
         int                     debug_id;       /* debug ID for printks */
-        int                     net_error;      /* network error distributed */
+        int                     error_report;   /* Net (+0) or local (+1000000) to distribute */
+#define RXRPC_LOCAL_ERROR_OFFSET 1000000
         struct sockaddr_rxrpc   srx;            /* remote address */
 
         /* calculated RTT cache */
@@ -219,68 +228,63 @@ struct rxrpc_peer {
 };
 
 /*
- * RxRPC point-to-point transport / connection manager definition
- * - handles a bundle of connections between two endpoints
- * - matched by { local, peer }
- */
-struct rxrpc_transport {
-        struct rxrpc_local      *local;         /* local transport endpoint */
-        struct rxrpc_peer       *peer;          /* remote transport endpoint */
-        struct work_struct      error_handler;  /* network error distributor */
-        struct rb_root          bundles;        /* client connection bundles on this transport */
-        struct rb_root          client_conns;   /* client connections on this transport */
-        struct rb_root          server_conns;   /* server connections on this transport */
-        struct list_head        link;           /* link in master session list */
-        struct sk_buff_head     error_queue;    /* error packets awaiting processing */
-        unsigned long           put_time;       /* time at which to reap */
-        spinlock_t              client_lock;    /* client connection allocation lock */
-        rwlock_t                conn_lock;      /* lock for active/dead connections */
-        atomic_t                usage;
-        int                     debug_id;       /* debug ID for printks */
-        unsigned int            conn_idcounter; /* connection ID counter (client) */
+ * Keys for matching a connection.
+ */
+struct rxrpc_conn_proto {
+        unsigned long           hash_key;
+        struct rxrpc_local      *local;         /* Representation of local endpoint */
+        u32                     epoch;          /* epoch of this connection */
+        u32                     cid;            /* connection ID */
+        u8                      in_clientflag;  /* RXRPC_CLIENT_INITIATED if we are server */
+        u8                      addr_size;      /* Size of the address */
+        sa_family_t             family;         /* Transport protocol */
+        __be16                  port;           /* Peer UDP/UDP6 port */
+        union {                                 /* Peer address */
+                struct in_addr  ipv4_addr;
+                struct in6_addr ipv6_addr;
+                u32             raw_addr[0];
+        };
 };
 
-/*
- * RxRPC client connection bundle
- * - matched by { transport, service_id, key }
- */
-struct rxrpc_conn_bundle {
-        struct rb_node          node;           /* node in transport's lookup tree */
-        struct list_head        unused_conns;   /* unused connections in this bundle */
-        struct list_head        avail_conns;    /* available connections in this bundle */
-        struct list_head        busy_conns;     /* busy connections in this bundle */
-        struct key              *key;           /* security for this bundle */
-        wait_queue_head_t       chanwait;       /* wait for channel to become available */
-        atomic_t                usage;
-        int                     debug_id;       /* debug ID for printks */
-        unsigned short          num_conns;      /* number of connections in this bundle */
-        u16                     service_id;     /* Service ID for this bundle */
-        u8                      security_ix;    /* security type */
+struct rxrpc_conn_parameters {
+        struct rxrpc_local      *local;         /* Representation of local endpoint */
+        struct rxrpc_peer       *peer;          /* Remote endpoint */
+        struct key              *key;           /* Security details */
+        bool                    exclusive;      /* T if conn is exclusive */
+        u16                     service_id;     /* Service ID for this connection */
+        u32                     security_level; /* Security level selected */
 };
 
 /*
  * RxRPC connection definition
- * - matched by { transport, service_id, conn_id, direction, key }
+ * - matched by { local, peer, epoch, conn_id, direction }
  * - each connection can only handle four simultaneous calls
  */
 struct rxrpc_connection {
-        struct rxrpc_transport  *trans;         /* transport session */
-        struct rxrpc_conn_bundle *bundle;       /* connection bundle (client) */
+        struct rxrpc_conn_proto proto;
+        struct rxrpc_conn_parameters params;
+
+        spinlock_t              channel_lock;
+        struct rxrpc_call       *channels[RXRPC_MAXCALLS]; /* active calls */
+        wait_queue_head_t       channel_wq;     /* queue to wait for channel to become available */
+
         struct work_struct      processor;      /* connection event processor */
-        struct rb_node          node;           /* node in transport's lookup tree */
+        union {
+                struct rb_node  client_node;    /* Node in local->client_conns */
+                struct rb_node  service_node;   /* Node in peer->service_conns */
+        };
         struct list_head        link;           /* link in master connection list */
-        struct list_head        bundle_link;    /* link in bundle */
         struct rb_root          calls;          /* calls on this connection */
         struct sk_buff_head     rx_queue;       /* received conn-level packets */
-        struct rxrpc_call       *channels[RXRPC_MAXCALLS]; /* channels (active calls) */
         const struct rxrpc_security *security;  /* applied security module */
-        struct key              *key;           /* security for this connection (client) */
         struct key              *server_key;    /* security for this service */
         struct crypto_skcipher  *cipher;        /* encryption handle */
         struct rxrpc_crypt      csum_iv;        /* packet checksum base */
+        unsigned long           flags;
+#define RXRPC_CONN_HAS_IDR      0               /* - Has a client conn ID assigned */
         unsigned long           events;
 #define RXRPC_CONN_CHALLENGE    0               /* send challenge packet */
-        unsigned long           put_time;       /* time at which to reap */
+        unsigned long           put_time;       /* Time at which last put */
         rwlock_t                lock;           /* access lock */
         spinlock_t              state_lock;     /* state-change lock */
         atomic_t                usage;
@@ -301,17 +305,12 @@ struct rxrpc_connection {
         unsigned int            call_counter;   /* call ID counter */
         atomic_t                serial;         /* packet serial number counter */
         atomic_t                hi_serial;      /* highest serial number received */
-        u8                      avail_calls;    /* number of calls available */
+        atomic_t                avail_chans;    /* number of channels available */
         u8                      size_align;     /* data size alignment (for security) */
         u8                      header_size;    /* rxrpc + security header size */
         u8                      security_size;  /* security header size */
-        u32                     security_level; /* security level negotiated */
         u32                     security_nonce; /* response re-use preventer */
-        u32                     epoch;          /* epoch of this connection */
-        u32                     cid;            /* connection ID */
-        u16                     service_id;     /* service ID for this connection */
         u8                      security_ix;    /* security type */
-        u8                      in_clientflag;  /* RXRPC_CLIENT_INITIATED if we are server */
         u8                      out_clientflag; /* RXRPC_CLIENT_INITIATED if we are client */
 };
 
@@ -357,6 +356,8 @@ enum rxrpc_call_event {
  * The states that a call can be in.
  */
 enum rxrpc_call_state {
+        RXRPC_CALL_UNINITIALISED,
+        RXRPC_CALL_CLIENT_AWAIT_CONN,   /* - client waiting for connection to become available */
         RXRPC_CALL_CLIENT_SEND_REQUEST, /* - client sending request phase */
         RXRPC_CALL_CLIENT_AWAIT_REPLY,  /* - client awaiting reply */
         RXRPC_CALL_CLIENT_RECV_REPLY,   /* - client receiving reply phase */
@@ -390,7 +391,7 @@ struct rxrpc_call {
         struct work_struct      destroyer;      /* call destroyer */
         struct work_struct      processor;      /* packet processor and ACK generator */
         struct list_head        link;           /* link in master call list */
-        struct list_head        error_link;     /* link in error distribution list */
+        struct hlist_node       error_link;     /* link in error distribution list */
         struct list_head        accept_link;    /* calls awaiting acceptance */
         struct rb_node          sock_node;      /* node in socket call tree */
         struct rb_node          conn_node;      /* node in connection call tree */
@@ -408,7 +409,8 @@ struct rxrpc_call {
         atomic_t                sequence;       /* Tx data packet sequence counter */
         u32                     local_abort;    /* local abort code */
         u32                     remote_abort;   /* remote abort code */
-        int                     error;          /* local error incurred */
+        int                     error_report;   /* Network error (ICMP/local transport) */
+        int                     error;          /* Local error incurred */
         enum rxrpc_call_state   state : 8;      /* current state of call */
         int                     debug_id;       /* debug ID for printks */
         u8                      channel;        /* connection channel occupied by this call */
@@ -444,7 +446,7 @@ struct rxrpc_call {
         unsigned long           hash_key;       /* Full hash key */
         u8                      in_clientflag;  /* Copy of conn->in_clientflag for hashing */
         struct rxrpc_local      *local;         /* Local endpoint. Used for hashing. */
-        sa_family_t             proto;          /* Frame protocol */
+        sa_family_t             family;         /* Frame protocol */
         u32                     call_id;        /* call ID on connection  */
         u32                     cid;            /* connection ID plus channel index */
         u32                     epoch;          /* epoch of this connection */
@@ -478,21 +480,21 @@ extern atomic_t rxrpc_debug_id;
 extern struct workqueue_struct *rxrpc_workqueue;
 
 /*
- * ar-accept.c
+ * call_accept.c
  */
-void rxrpc_accept_incoming_calls(struct work_struct *);
+void rxrpc_accept_incoming_calls(struct rxrpc_local *);
 struct rxrpc_call *rxrpc_accept_call(struct rxrpc_sock *, unsigned long);
 int rxrpc_reject_call(struct rxrpc_sock *);
 
 /*
- * ar-ack.c
+ * call_event.c
  */
 void __rxrpc_propose_ACK(struct rxrpc_call *, u8, u32, bool);
 void rxrpc_propose_ACK(struct rxrpc_call *, u8, u32, bool);
 void rxrpc_process_call(struct work_struct *);
 
 /*
- * ar-call.c
+ * call_object.c
  */
 extern unsigned int rxrpc_max_call_lifetime;
 extern unsigned int rxrpc_dead_call_expiry;
@@ -502,70 +504,82 @@ extern rwlock_t rxrpc_call_lock;
 
 struct rxrpc_call *rxrpc_find_call_hash(struct rxrpc_host_header *,
                                         void *, sa_family_t, const void *);
-struct rxrpc_call *rxrpc_get_client_call(struct rxrpc_sock *,
-                                         struct rxrpc_transport *,
-                                         struct rxrpc_conn_bundle *,
-                                         unsigned long, int, gfp_t);
+struct rxrpc_call *rxrpc_find_call_by_user_ID(struct rxrpc_sock *, unsigned long);
+struct rxrpc_call *rxrpc_new_client_call(struct rxrpc_sock *,
+                                         struct rxrpc_conn_parameters *,
+                                         struct sockaddr_rxrpc *,
+                                         unsigned long, gfp_t);
 struct rxrpc_call *rxrpc_incoming_call(struct rxrpc_sock *,
                                        struct rxrpc_connection *,
-                                       struct rxrpc_host_header *);
-struct rxrpc_call *rxrpc_find_server_call(struct rxrpc_sock *, unsigned long);
+                                       struct sk_buff *);
 void rxrpc_release_call(struct rxrpc_call *);
 void rxrpc_release_calls_on_socket(struct rxrpc_sock *);
 void __rxrpc_put_call(struct rxrpc_call *);
 void __exit rxrpc_destroy_all_calls(void);
 
 /*
- * ar-connection.c
+ * conn_client.c
  */
-extern unsigned int rxrpc_connection_expiry;
-extern struct list_head rxrpc_connections;
-extern rwlock_t rxrpc_connection_lock;
+extern struct idr rxrpc_client_conn_ids;
 
-struct rxrpc_conn_bundle *rxrpc_get_bundle(struct rxrpc_sock *,
-                                           struct rxrpc_transport *,
-                                           struct key *, u16, gfp_t);
-void rxrpc_put_bundle(struct rxrpc_transport *, struct rxrpc_conn_bundle *);
-int rxrpc_connect_call(struct rxrpc_sock *, struct rxrpc_transport *,
-                       struct rxrpc_conn_bundle *, struct rxrpc_call *, gfp_t);
-void rxrpc_put_connection(struct rxrpc_connection *);
-void __exit rxrpc_destroy_all_connections(void);
-struct rxrpc_connection *rxrpc_find_connection(struct rxrpc_transport *,
-                                               struct rxrpc_host_header *);
-extern struct rxrpc_connection *
-rxrpc_incoming_connection(struct rxrpc_transport *, struct rxrpc_host_header *);
+int rxrpc_get_client_connection_id(struct rxrpc_connection *, gfp_t);
+void rxrpc_put_client_connection_id(struct rxrpc_connection *);
 
 /*
- * ar-connevent.c
+ * conn_event.c
  */
 void rxrpc_process_connection(struct work_struct *);
 void rxrpc_reject_packet(struct rxrpc_local *, struct sk_buff *);
-void rxrpc_reject_packets(struct work_struct *);
+void rxrpc_reject_packets(struct rxrpc_local *);
 
 /*
- * ar-error.c
+ * conn_object.c
  */
-void rxrpc_UDP_error_report(struct sock *);
-void rxrpc_UDP_error_handler(struct work_struct *);
+extern unsigned int rxrpc_connection_expiry;
+extern struct list_head rxrpc_connections;
+extern rwlock_t rxrpc_connection_lock;
+
+int rxrpc_connect_call(struct rxrpc_call *, struct rxrpc_conn_parameters *,
+                       struct sockaddr_rxrpc *, gfp_t);
+struct rxrpc_connection *rxrpc_find_connection(struct rxrpc_local *,
+                                               struct rxrpc_peer *,
+                                               struct sk_buff *);
+void rxrpc_disconnect_call(struct rxrpc_call *);
+void rxrpc_put_connection(struct rxrpc_connection *);
+void __exit rxrpc_destroy_all_connections(void);
+struct rxrpc_connection *rxrpc_incoming_connection(struct rxrpc_local *,
+                                                   struct rxrpc_peer *,
+                                                   struct sk_buff *);
+
+static inline bool rxrpc_conn_is_client(const struct rxrpc_connection *conn)
+{
+        return conn->out_clientflag;
+}
+
+static inline bool rxrpc_conn_is_service(const struct rxrpc_connection *conn)
+{
+        return conn->proto.in_clientflag;
+}
+
+static inline void rxrpc_get_connection(struct rxrpc_connection *conn)
+{
+        atomic_inc(&conn->usage);
+}
 
 /*
- * ar-input.c
+ * input.c
  */
 void rxrpc_data_ready(struct sock *);
 int rxrpc_queue_rcv_skb(struct rxrpc_call *, struct sk_buff *, bool, bool);
 void rxrpc_fast_process_packet(struct rxrpc_call *, struct sk_buff *);
 
 /*
- * ar-local.c
+ * insecure.c
  */
-extern rwlock_t rxrpc_local_lock;
-
-struct rxrpc_local *rxrpc_lookup_local(struct sockaddr_rxrpc *);
-void rxrpc_put_local(struct rxrpc_local *);
-void __exit rxrpc_destroy_all_locals(void);
+extern const struct rxrpc_security rxrpc_no_security;
 
 /*
- * ar-key.c
+ * key.c
  */
 extern struct key_type key_type_rxrpc;
 extern struct key_type key_type_rxrpc_s;
@@ -576,80 +590,103 @@ int rxrpc_get_server_data_key(struct rxrpc_connection *, const void *, time_t,
                               u32);
 
 /*
- * ar-output.c
+ * local_event.c
  */
-extern unsigned int rxrpc_resend_timeout;
-
-int rxrpc_send_packet(struct rxrpc_transport *, struct sk_buff *);
-int rxrpc_client_sendmsg(struct rxrpc_sock *, struct rxrpc_transport *,
-                         struct msghdr *, size_t);
-int rxrpc_server_sendmsg(struct rxrpc_sock *, struct msghdr *, size_t);
+extern void rxrpc_process_local_events(struct rxrpc_local *);
 
 /*
- * ar-peer.c
+ * local_object.c
  */
-struct rxrpc_peer *rxrpc_get_peer(struct sockaddr_rxrpc *, gfp_t);
-void rxrpc_put_peer(struct rxrpc_peer *);
-struct rxrpc_peer *rxrpc_find_peer(struct rxrpc_local *, __be32, __be16);
-void __exit rxrpc_destroy_all_peers(void);
+struct rxrpc_local *rxrpc_lookup_local(const struct sockaddr_rxrpc *);
+void __rxrpc_put_local(struct rxrpc_local *);
+void __exit rxrpc_destroy_all_locals(void);
 
-/*
- * ar-proc.c
- */
-extern const char *const rxrpc_call_states[];
-extern const struct file_operations rxrpc_call_seq_fops;
-extern const struct file_operations rxrpc_connection_seq_fops;
+static inline void rxrpc_get_local(struct rxrpc_local *local)
+{
+        atomic_inc(&local->usage);
+}
+
+static inline
+struct rxrpc_local *rxrpc_get_local_maybe(struct rxrpc_local *local)
+{
+        return atomic_inc_not_zero(&local->usage) ? local : NULL;
+}
+
+static inline void rxrpc_put_local(struct rxrpc_local *local)
+{
+        if (local && atomic_dec_and_test(&local->usage))
+                __rxrpc_put_local(local);
+}
 
 /*
- * ar-recvmsg.c
+ * misc.c
  */
-void rxrpc_remove_user_ID(struct rxrpc_sock *, struct rxrpc_call *);
-int rxrpc_recvmsg(struct socket *, struct msghdr *, size_t, int);
+extern unsigned int rxrpc_max_backlog __read_mostly;
+extern unsigned int rxrpc_requested_ack_delay;
+extern unsigned int rxrpc_soft_ack_delay;
+extern unsigned int rxrpc_idle_ack_delay;
+extern unsigned int rxrpc_rx_window_size;
+extern unsigned int rxrpc_rx_mtu;
+extern unsigned int rxrpc_rx_jumbo_max;
+
+extern const char *const rxrpc_pkts[];
+extern const s8 rxrpc_ack_priority[];
+
+extern const char *rxrpc_acks(u8 reason);
 
 /*
- * ar-security.c
+ * output.c
  */
-int __init rxrpc_init_security(void);
-void rxrpc_exit_security(void);
-int rxrpc_init_client_conn_security(struct rxrpc_connection *);
-int rxrpc_init_server_conn_security(struct rxrpc_connection *);
+extern unsigned int rxrpc_resend_timeout;
+
+int rxrpc_send_data_packet(struct rxrpc_connection *, struct sk_buff *);
+int rxrpc_do_sendmsg(struct rxrpc_sock *, struct msghdr *, size_t);
 
 /*
- * ar-skbuff.c
+ * peer_event.c
  */
-void rxrpc_packet_destructor(struct sk_buff *);
+void rxrpc_error_report(struct sock *);
+void rxrpc_peer_error_distributor(struct work_struct *);
 
 /*
- * ar-transport.c
+ * peer_object.c
  */
-extern unsigned int rxrpc_transport_expiry;
+struct rxrpc_peer *rxrpc_lookup_peer_rcu(struct rxrpc_local *,
+                                         const struct sockaddr_rxrpc *);
+struct rxrpc_peer *rxrpc_lookup_peer(struct rxrpc_local *,
+                                     struct sockaddr_rxrpc *, gfp_t);
+struct rxrpc_peer *rxrpc_alloc_peer(struct rxrpc_local *, gfp_t);
+
+static inline void rxrpc_get_peer(struct rxrpc_peer *peer)
+{
+        atomic_inc(&peer->usage);
+}
+
+static inline
+struct rxrpc_peer *rxrpc_get_peer_maybe(struct rxrpc_peer *peer)
+{
+        return atomic_inc_not_zero(&peer->usage) ? peer : NULL;
+}
 
-struct rxrpc_transport *rxrpc_get_transport(struct rxrpc_local *,
-                                            struct rxrpc_peer *, gfp_t);
-void rxrpc_put_transport(struct rxrpc_transport *);
-void __exit rxrpc_destroy_all_transports(void);
-struct rxrpc_transport *rxrpc_find_transport(struct rxrpc_local *,
-                                             struct rxrpc_peer *);
+extern void __rxrpc_put_peer(struct rxrpc_peer *peer);
+static inline void rxrpc_put_peer(struct rxrpc_peer *peer)
+{
+        if (peer && atomic_dec_and_test(&peer->usage))
+                __rxrpc_put_peer(peer);
+}
 
 /*
- * insecure.c
+ * proc.c
  */
-extern const struct rxrpc_security rxrpc_no_security;
+extern const char *const rxrpc_call_states[];
+extern const struct file_operations rxrpc_call_seq_fops;
+extern const struct file_operations rxrpc_connection_seq_fops;
 
 /*
- * misc.c
+ * recvmsg.c
  */
-extern unsigned int rxrpc_requested_ack_delay;
-extern unsigned int rxrpc_soft_ack_delay;
-extern unsigned int rxrpc_idle_ack_delay;
-extern unsigned int rxrpc_rx_window_size;
-extern unsigned int rxrpc_rx_mtu;
-extern unsigned int rxrpc_rx_jumbo_max;
-
-extern const char *const rxrpc_pkts[];
-extern const s8 rxrpc_ack_priority[];
-
-extern const char *rxrpc_acks(u8 reason);
+void rxrpc_remove_user_ID(struct rxrpc_sock *, struct rxrpc_call *);
+int rxrpc_recvmsg(struct socket *, struct msghdr *, size_t, int);
 
 /*
  * rxkad.c
@@ -659,6 +696,19 @@ extern const struct rxrpc_security rxkad;
 #endif
 
 /*
+ * security.c
+ */
+int __init rxrpc_init_security(void);
+void rxrpc_exit_security(void);
+int rxrpc_init_client_conn_security(struct rxrpc_connection *);
+int rxrpc_init_server_conn_security(struct rxrpc_connection *);
+
+/*
+ * skbuff.c
+ */
+void rxrpc_packet_destructor(struct sk_buff *);
+
+/*
  * sysctl.c
  */
 #ifdef CONFIG_SYSCTL
@@ -670,6 +720,12 @@ static inline void rxrpc_sysctl_exit(void) {}
 #endif
 
 /*
+ * utils.c
+ */
+void rxrpc_get_addr_from_skb(struct rxrpc_local *, const struct sk_buff *,
+                             struct sockaddr_rxrpc *);
+
+/*
  * debug tracing
  */
 extern unsigned int rxrpc_debug;
@@ -744,21 +800,18 @@ do {							\
 #define ASSERT(X)                                               \
 do {                                                            \
         if (unlikely(!(X))) {                                   \
-                printk(KERN_ERR "\n");                          \
-                printk(KERN_ERR "RxRPC: Assertion failed\n");   \
+                pr_err("Assertion failed\n");                   \
                 BUG();                                          \
         }                                                       \
 } while (0)
 
 #define ASSERTCMP(X, OP, Y)                                             \
 do {                                                                    \
-        if (unlikely(!((X) OP (Y)))) {                                  \
-                printk(KERN_ERR "\n");                                  \
-                printk(KERN_ERR "RxRPC: Assertion failed\n");           \
-                printk(KERN_ERR "%lu " #OP " %lu is false\n",           \
-                       (unsigned long)(X), (unsigned long)(Y));         \
-                printk(KERN_ERR "0x%lx " #OP " 0x%lx is false\n",       \
-                       (unsigned long)(X), (unsigned long)(Y));         \
+        unsigned long _x = (unsigned long)(X);                          \
+        unsigned long _y = (unsigned long)(Y);                          \
+        if (unlikely(!(_x OP _y))) {                                    \
+                pr_err("Assertion failed - %lu(0x%lx) %s %lu(0x%lx) is false\n",                        \
+                       _x, _x, #OP, _y, _y);                            \
                 BUG();                                                  \
         }                                                               \
 } while (0)
@@ -766,21 +819,18 @@ do {									\
 #define ASSERTIF(C, X)                                          \
 do {                                                            \
         if (unlikely((C) && !(X))) {                            \
-                printk(KERN_ERR "\n");                          \
-                printk(KERN_ERR "RxRPC: Assertion failed\n");   \
+                pr_err("Assertion failed\n");                   \
                 BUG();                                          \
         }                                                       \
 } while (0)
 
 #define ASSERTIFCMP(C, X, OP, Y)                                        \
 do {                                                                    \
-        if (unlikely((C) && !((X) OP (Y)))) {                           \
-                printk(KERN_ERR "\n");                                  \
-                printk(KERN_ERR "RxRPC: Assertion failed\n");           \
-                printk(KERN_ERR "%lu " #OP " %lu is false\n",           \
-                       (unsigned long)(X), (unsigned long)(Y));         \
-                printk(KERN_ERR "0x%lx " #OP " 0x%lx is false\n",       \
-                       (unsigned long)(X), (unsigned long)(Y));         \
+        unsigned long _x = (unsigned long)(X);                          \
+        unsigned long _y = (unsigned long)(Y);                          \
+        if (unlikely((C) && !(_x OP _y))) {                             \
+                pr_err("Assertion failed - %lu(0x%lx) %s %lu(0x%lx) is false\n", \
+                       _x, _x, #OP, _y, _y);                            \
                 BUG();                                                  \
         }                                                               \
 } while (0)
@@ -844,15 +894,6 @@ static inline void rxrpc_purge_queue(struct sk_buff_head *list)
                 rxrpc_free_skb(skb);
 }
 
-static inline void __rxrpc_get_local(struct rxrpc_local *local, const char *f)
-{
-        CHECK_SLAB_OKAY(&local->usage);
-        if (atomic_inc_return(&local->usage) == 1)
-                printk("resurrected (%s)\n", f);
-}
-
-#define rxrpc_get_local(LOCAL) __rxrpc_get_local((LOCAL), __func__)
-
 #define rxrpc_get_call(CALL)                            \
 do {                                                    \
         CHECK_SLAB_OKAY(&(CALL)->usage);                \
diff --git a/net/rxrpc/ar-local.c b/net/rxrpc/ar-local.c
deleted file mode 100644
index 4e1e6db0050b..000000000000
--- a/net/rxrpc/ar-local.c
+++ /dev/null
@@ -1,415 +0,0 @@
-/* AF_RXRPC local endpoint management
- *
- * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
- * Written by David Howells (dhowells@redhat.com)
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
- */
-
-#include <linux/module.h>
-#include <linux/net.h>
-#include <linux/skbuff.h>
-#include <linux/slab.h>
-#include <linux/udp.h>
-#include <linux/ip.h>
-#include <net/sock.h>
-#include <net/af_rxrpc.h>
-#include <generated/utsrelease.h>
-#include "ar-internal.h"
-
-static const char rxrpc_version_string[65] = "linux-" UTS_RELEASE " AF_RXRPC";
-
-static LIST_HEAD(rxrpc_locals);
-DEFINE_RWLOCK(rxrpc_local_lock);
-static DECLARE_RWSEM(rxrpc_local_sem);
-static DECLARE_WAIT_QUEUE_HEAD(rxrpc_local_wq);
-
-static void rxrpc_destroy_local(struct work_struct *work);
-static void rxrpc_process_local_events(struct work_struct *work);
-
-/*
- * allocate a new local
- */
-static
-struct rxrpc_local *rxrpc_alloc_local(struct sockaddr_rxrpc *srx)
-{
-        struct rxrpc_local *local;
-
-        local = kzalloc(sizeof(struct rxrpc_local), GFP_KERNEL);
-        if (local) {
-                INIT_WORK(&local->destroyer, &rxrpc_destroy_local);
-                INIT_WORK(&local->acceptor, &rxrpc_accept_incoming_calls);
-                INIT_WORK(&local->rejecter, &rxrpc_reject_packets);
-                INIT_WORK(&local->event_processor, &rxrpc_process_local_events);
-                INIT_LIST_HEAD(&local->services);
-                INIT_LIST_HEAD(&local->link);
-                init_rwsem(&local->defrag_sem);
-                skb_queue_head_init(&local->accept_queue);
-                skb_queue_head_init(&local->reject_queue);
-                skb_queue_head_init(&local->event_queue);
-                spin_lock_init(&local->lock);
-                rwlock_init(&local->services_lock);
-                atomic_set(&local->usage, 1);
-                local->debug_id = atomic_inc_return(&rxrpc_debug_id);
-                memcpy(&local->srx, srx, sizeof(*srx));
-        }
-
-        _leave(" = %p", local);
-        return local;
-}
-
-/*
- * create the local socket
- * - must be called with rxrpc_local_sem writelocked
- */
-static int rxrpc_create_local(struct rxrpc_local *local)
-{
-        struct sock *sock;
-        int ret, opt;
-
-        _enter("%p{%d}", local, local->srx.transport_type);
-
-        /* create a socket to represent the local endpoint */
-        ret = sock_create_kern(&init_net, PF_INET, local->srx.transport_type,
-                               IPPROTO_UDP, &local->socket);
-        if (ret < 0) {
-                _leave(" = %d [socket]", ret);
-                return ret;
-        }
-
-        /* if a local address was supplied then bind it */
-        if (local->srx.transport_len > sizeof(sa_family_t)) {
-                _debug("bind");
-                ret = kernel_bind(local->socket,
-                                  (struct sockaddr *) &local->srx.transport,
-                                  local->srx.transport_len);
-                if (ret < 0) {
-                        _debug("bind failed");
-                        goto error;
-                }
-        }
-
-        /* we want to receive ICMP errors */
-        opt = 1;
-        ret = kernel_setsockopt(local->socket, SOL_IP, IP_RECVERR,
-                                (char *) &opt, sizeof(opt));
-        if (ret < 0) {
-                _debug("setsockopt failed");
-                goto error;
-        }
-
-        /* we want to set the don't fragment bit */
-        opt = IP_PMTUDISC_DO;
-        ret = kernel_setsockopt(local->socket, SOL_IP, IP_MTU_DISCOVER,
-                                (char *) &opt, sizeof(opt));
-        if (ret < 0) {
-                _debug("setsockopt failed");
-                goto error;
-        }
-
-        write_lock_bh(&rxrpc_local_lock);
-        list_add(&local->link, &rxrpc_locals);
-        write_unlock_bh(&rxrpc_local_lock);
-
-        /* set the socket up */
-        sock = local->socket->sk;
-        sock->sk_user_data      = local;
-        sock->sk_data_ready     = rxrpc_data_ready;
-        sock->sk_error_report   = rxrpc_UDP_error_report;
-        _leave(" = 0");
-        return 0;
-
-error:
-        kernel_sock_shutdown(local->socket, SHUT_RDWR);
-        local->socket->sk->sk_user_data = NULL;
-        sock_release(local->socket);
-        local->socket = NULL;
-
-        _leave(" = %d", ret);
-        return ret;
-}
-
-/*
- * create a new local endpoint using the specified UDP address
- */
-struct rxrpc_local *rxrpc_lookup_local(struct sockaddr_rxrpc *srx)
-{
-        struct rxrpc_local *local;
-        int ret;
-
-        _enter("{%d,%u,%pI4+%hu}",
-               srx->transport_type,
-               srx->transport.family,
-               &srx->transport.sin.sin_addr,
-               ntohs(srx->transport.sin.sin_port));
-
-        down_write(&rxrpc_local_sem);
-
-        /* see if we have a suitable local local endpoint already */
-        read_lock_bh(&rxrpc_local_lock);
-
-        list_for_each_entry(local, &rxrpc_locals, link) {
-                _debug("CMP {%d,%u,%pI4+%hu}",
-                       local->srx.transport_type,
-                       local->srx.transport.family,
-                       &local->srx.transport.sin.sin_addr,
-                       ntohs(local->srx.transport.sin.sin_port));
-
-                if (local->srx.transport_type != srx->transport_type ||
-                    local->srx.transport.family != srx->transport.family)
-                        continue;
-
-                switch (srx->transport.family) {
-                case AF_INET:
-                        if (local->srx.transport.sin.sin_port !=
-                            srx->transport.sin.sin_port)
-                                continue;
-                        if (memcmp(&local->srx.transport.sin.sin_addr,
-                                   &srx->transport.sin.sin_addr,
-                                   sizeof(struct in_addr)) != 0)
-                                continue;
-                        goto found_local;
-
-                default:
-                        BUG();
-                }
-        }
-
-        read_unlock_bh(&rxrpc_local_lock);
-
-        /* we didn't find one, so we need to create one */
-        local = rxrpc_alloc_local(srx);
-        if (!local) {
-                up_write(&rxrpc_local_sem);
-                return ERR_PTR(-ENOMEM);
-        }
-
-        ret = rxrpc_create_local(local);
-        if (ret < 0) {
-                up_write(&rxrpc_local_sem);
-                kfree(local);
-                _leave(" = %d", ret);
-                return ERR_PTR(ret);
-        }
-
-        up_write(&rxrpc_local_sem);
-
-        _net("LOCAL new %d {%d,%u,%pI4+%hu}",
-             local->debug_id,
-             local->srx.transport_type,
-             local->srx.transport.family,
-             &local->srx.transport.sin.sin_addr,
-             ntohs(local->srx.transport.sin.sin_port));
-
-        _leave(" = %p [new]", local);
-        return local;
-
-found_local:
-        rxrpc_get_local(local);
-        read_unlock_bh(&rxrpc_local_lock);
-        up_write(&rxrpc_local_sem);
-
-        _net("LOCAL old %d {%d,%u,%pI4+%hu}",
-             local->debug_id,
-             local->srx.transport_type,
-             local->srx.transport.family,
-             &local->srx.transport.sin.sin_addr,
-             ntohs(local->srx.transport.sin.sin_port));
-
-        _leave(" = %p [reuse]", local);
-        return local;
-}
-
-/*
- * release a local endpoint
- */
-void rxrpc_put_local(struct rxrpc_local *local)
-{
-        _enter("%p{u=%d}", local, atomic_read(&local->usage));
-
-        ASSERTCMP(atomic_read(&local->usage), >, 0);
-
-        /* to prevent a race, the decrement and the dequeue must be effectively
-         * atomic */
-        write_lock_bh(&rxrpc_local_lock);
-        if (unlikely(atomic_dec_and_test(&local->usage))) {
-                _debug("destroy local");
-                rxrpc_queue_work(&local->destroyer);
-        }
-        write_unlock_bh(&rxrpc_local_lock);
-        _leave("");
-}
-
-/*
- * destroy a local endpoint
- */
-static void rxrpc_destroy_local(struct work_struct *work)
-{
-        struct rxrpc_local *local =
-                container_of(work, struct rxrpc_local, destroyer);
-
-        _enter("%p{%d}", local, atomic_read(&local->usage));
-
-        down_write(&rxrpc_local_sem);
-
-        write_lock_bh(&rxrpc_local_lock);
-        if (atomic_read(&local->usage) > 0) {
-                write_unlock_bh(&rxrpc_local_lock);
-                up_read(&rxrpc_local_sem);
-                _leave(" [resurrected]");
-                return;
-        }
-
-        list_del(&local->link);
-        local->socket->sk->sk_user_data = NULL;
-        write_unlock_bh(&rxrpc_local_lock);
-
-        downgrade_write(&rxrpc_local_sem);
-
-        ASSERT(list_empty(&local->services));
-        ASSERT(!work_pending(&local->acceptor));
-        ASSERT(!work_pending(&local->rejecter));
-        ASSERT(!work_pending(&local->event_processor));
-
-        /* finish cleaning up the local descriptor */
-        rxrpc_purge_queue(&local->accept_queue);
-        rxrpc_purge_queue(&local->reject_queue);
-        rxrpc_purge_queue(&local->event_queue);
-        kernel_sock_shutdown(local->socket, SHUT_RDWR);
-        sock_release(local->socket);
-
-        up_read(&rxrpc_local_sem);
-
-        _net("DESTROY LOCAL %d", local->debug_id);
-        kfree(local);
-
-        if (list_empty(&rxrpc_locals))
-                wake_up_all(&rxrpc_local_wq);
-
-        _leave("");
-}
-
-/*
- * preemptively destroy all local local endpoint rather than waiting for
- * them to be destroyed
- */
-void __exit rxrpc_destroy_all_locals(void)
-{
-        DECLARE_WAITQUEUE(myself,current);
-
-        _enter("");
-
-        /* we simply have to wait for them to go away */
-        if (!list_empty(&rxrpc_locals)) {
-                set_current_state(TASK_UNINTERRUPTIBLE);
-                add_wait_queue(&rxrpc_local_wq, &myself);
-
-                while (!list_empty(&rxrpc_locals)) {
-                        schedule();
-                        set_current_state(TASK_UNINTERRUPTIBLE);
-                }
-
-                remove_wait_queue(&rxrpc_local_wq, &myself);
-                set_current_state(TASK_RUNNING);
-        }
-
-        _leave("");
-}
-
-/*
- * Reply to a version request
- */
-static void rxrpc_send_version_request(struct rxrpc_local *local,
-                                       struct rxrpc_host_header *hdr,
-                                       struct sk_buff *skb)
-{
-        struct rxrpc_wire_header whdr;
-        struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
-        struct sockaddr_in sin;
-        struct msghdr msg;
-        struct kvec iov[2];
-        size_t len;
-        int ret;
-
-        _enter("");
-
-        sin.sin_family = AF_INET;
-        sin.sin_port = udp_hdr(skb)->source;
-        sin.sin_addr.s_addr = ip_hdr(skb)->saddr;
-
-        msg.msg_name    = &sin;
-        msg.msg_namelen = sizeof(sin);
-        msg.msg_control = NULL;
-        msg.msg_controllen = 0;
-        msg.msg_flags   = 0;
-
-        whdr.epoch      = htonl(sp->hdr.epoch);
-        whdr.cid        = htonl(sp->hdr.cid);
-        whdr.callNumber = htonl(sp->hdr.callNumber);
-        whdr.seq        = 0;
-        whdr.serial     = 0;
-        whdr.type       = RXRPC_PACKET_TYPE_VERSION;
-        whdr.flags      = RXRPC_LAST_PACKET | (~hdr->flags & RXRPC_CLIENT_INITIATED);
-        whdr.userStatus = 0;
-        whdr.securityIndex = 0;
-        whdr._rsvd      = 0;
-        whdr.serviceId  = htons(sp->hdr.serviceId);
-
-        iov[0].iov_base = &whdr;
-        iov[0].iov_len  = sizeof(whdr);
-        iov[1].iov_base = (char *)rxrpc_version_string;
-        iov[1].iov_len  = sizeof(rxrpc_version_string);
-
-        len = iov[0].iov_len + iov[1].iov_len;
-
-        _proto("Tx VERSION (reply)");
-
-        ret = kernel_sendmsg(local->socket, &msg, iov, 2, len);
-        if (ret < 0)
-                _debug("sendmsg failed: %d", ret);
-
-        _leave("");
-}
-
-/*
- * Process event packets targetted at a local endpoint.
- */
-static void rxrpc_process_local_events(struct work_struct *work)
-{
-        struct rxrpc_local *local = container_of(work, struct rxrpc_local, event_processor);
-        struct sk_buff *skb;
-        char v;
-
-        _enter("");
-
-        atomic_inc(&local->usage);
-        
-        while ((skb = skb_dequeue(&local->event_queue))) {
-                struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
-
-                _debug("{%d},{%u}", local->debug_id, sp->hdr.type);
-
-                switch (sp->hdr.type) {
-                case RXRPC_PACKET_TYPE_VERSION:
-                        if (skb_copy_bits(skb, 0, &v, 1) < 0)
-                                return;
-                        _proto("Rx VERSION { %02x }", v);
-                        if (v == 0)
-                                rxrpc_send_version_request(local, &sp->hdr, skb);
-                        break;
-
-                default:
-                        /* Just ignore anything we don't understand */
-                        break;
-                }
-
-                rxrpc_put_local(local);
-                rxrpc_free_skb(skb);
-        }
-
-        rxrpc_put_local(local);
-        _leave("");
-}
diff --git a/net/rxrpc/ar-peer.c b/net/rxrpc/ar-peer.c
deleted file mode 100644
index dc089b1976aa..000000000000
--- a/net/rxrpc/ar-peer.c
+++ /dev/null
@@ -1,303 +0,0 @@
-/* RxRPC remote transport endpoint management
- *
- * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
- * Written by David Howells (dhowells@redhat.com)
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
- */
-
-#include <linux/module.h>
-#include <linux/net.h>
-#include <linux/skbuff.h>
-#include <linux/udp.h>
-#include <linux/in.h>
-#include <linux/in6.h>
-#include <linux/icmp.h>
-#include <linux/slab.h>
-#include <net/sock.h>
-#include <net/af_rxrpc.h>
-#include <net/ip.h>
-#include <net/route.h>
-#include "ar-internal.h"
-
-static LIST_HEAD(rxrpc_peers);
-static DEFINE_RWLOCK(rxrpc_peer_lock);
-static DECLARE_WAIT_QUEUE_HEAD(rxrpc_peer_wq);
-
-static void rxrpc_destroy_peer(struct work_struct *work);
-
-/*
- * assess the MTU size for the network interface through which this peer is
- * reached
- */
-static void rxrpc_assess_MTU_size(struct rxrpc_peer *peer)
-{
-        struct rtable *rt;
-        struct flowi4 fl4;
-
-        peer->if_mtu = 1500;
-
-        rt = ip_route_output_ports(&init_net, &fl4, NULL,
-                                   peer->srx.transport.sin.sin_addr.s_addr, 0,
-                                   htons(7000), htons(7001),
-                                   IPPROTO_UDP, 0, 0);
-        if (IS_ERR(rt)) {
-                _leave(" [route err %ld]", PTR_ERR(rt));
-                return;
-        }
-
-        peer->if_mtu = dst_mtu(&rt->dst);
-        dst_release(&rt->dst);
-
-        _leave(" [if_mtu %u]", peer->if_mtu);
-}
-
-/*
- * allocate a new peer
- */
-static struct rxrpc_peer *rxrpc_alloc_peer(struct sockaddr_rxrpc *srx,
-                                           gfp_t gfp)
-{
-        struct rxrpc_peer *peer;
-
-        _enter("");
-
-        peer = kzalloc(sizeof(struct rxrpc_peer), gfp);
-        if (peer) {
-                INIT_WORK(&peer->destroyer, &rxrpc_destroy_peer);
-                INIT_LIST_HEAD(&peer->link);
-                INIT_LIST_HEAD(&peer->error_targets);
-                spin_lock_init(&peer->lock);
-                atomic_set(&peer->usage, 1);
-                peer->debug_id = atomic_inc_return(&rxrpc_debug_id);
-                memcpy(&peer->srx, srx, sizeof(*srx));
-
-                rxrpc_assess_MTU_size(peer);
-                peer->mtu = peer->if_mtu;
-
-                if (srx->transport.family == AF_INET) {
-                        peer->hdrsize = sizeof(struct iphdr);
-                        switch (srx->transport_type) {
-                        case SOCK_DGRAM:
-                                peer->hdrsize += sizeof(struct udphdr);
-                                break;
-                        default:
-                                BUG();
-                                break;
-                        }
-                } else {
-                        BUG();
-                }
-
-                peer->hdrsize += sizeof(struct rxrpc_wire_header);
-                peer->maxdata = peer->mtu - peer->hdrsize;
-        }
-
-        _leave(" = %p", peer);
-        return peer;
-}
-
-/*
- * obtain a remote transport endpoint for the specified address
- */
-struct rxrpc_peer *rxrpc_get_peer(struct sockaddr_rxrpc *srx, gfp_t gfp)
-{
-        struct rxrpc_peer *peer, *candidate;
-        const char *new = "old";
-        int usage;
-
-        _enter("{%d,%d,%pI4+%hu}",
-               srx->transport_type,
-               srx->transport_len,
-               &srx->transport.sin.sin_addr,
-               ntohs(srx->transport.sin.sin_port));
-
-        /* search the peer list first */
-        read_lock_bh(&rxrpc_peer_lock);
-        list_for_each_entry(peer, &rxrpc_peers, link) {
-                _debug("check PEER %d { u=%d t=%d l=%d }",
-                       peer->debug_id,
-                       atomic_read(&peer->usage),
-                       peer->srx.transport_type,
-                       peer->srx.transport_len);
-
-                if (atomic_read(&peer->usage) > 0 &&
-                    peer->srx.transport_type == srx->transport_type &&
-                    peer->srx.transport_len == srx->transport_len &&
-                    memcmp(&peer->srx.transport,
-                           &srx->transport,
-                           srx->transport_len) == 0)
-                        goto found_extant_peer;
-        }
-        read_unlock_bh(&rxrpc_peer_lock);
-
-        /* not yet present - create a candidate for a new record and then
-         * redo the search */
-        candidate = rxrpc_alloc_peer(srx, gfp);
-        if (!candidate) {
-                _leave(" = -ENOMEM");
-                return ERR_PTR(-ENOMEM);
-        }
-
-        write_lock_bh(&rxrpc_peer_lock);
-
-        list_for_each_entry(peer, &rxrpc_peers, link) {
-                if (atomic_read(&peer->usage) > 0 &&
-                    peer->srx.transport_type == srx->transport_type &&
-                    peer->srx.transport_len == srx->transport_len &&
-                    memcmp(&peer->srx.transport,
-                           &srx->transport,
-                           srx->transport_len) == 0)
-                        goto found_extant_second;
-        }
-
-        /* we can now add the new candidate to the list */
-        peer = candidate;
-        candidate = NULL;
-        usage = atomic_read(&peer->usage);
-
-        list_add_tail(&peer->link, &rxrpc_peers);
-        write_unlock_bh(&rxrpc_peer_lock);
-        new = "new";
-
-success:
-        _net("PEER %s %d {%d,%u,%pI4+%hu}",
-             new,
-             peer->debug_id,
-             peer->srx.transport_type,
-             peer->srx.transport.family,
-             &peer->srx.transport.sin.sin_addr,
-             ntohs(peer->srx.transport.sin.sin_port));
-
-        _leave(" = %p {u=%d}", peer, usage);
-        return peer;
-
-        /* we found the peer in the list immediately */
-found_extant_peer:
-        usage = atomic_inc_return(&peer->usage);
-        read_unlock_bh(&rxrpc_peer_lock);
-        goto success;
-
-        /* we found the peer on the second time through the list */
-found_extant_second:
-        usage = atomic_inc_return(&peer->usage);
-        write_unlock_bh(&rxrpc_peer_lock);
-        kfree(candidate);
-        goto success;
-}
-
-/*
- * find the peer associated with a packet
- */
-struct rxrpc_peer *rxrpc_find_peer(struct rxrpc_local *local,
-                                   __be32 addr, __be16 port)
-{
-        struct rxrpc_peer *peer;
-
-        _enter("");
-
-        /* search the peer list */
-        read_lock_bh(&rxrpc_peer_lock);
-
-        if (local->srx.transport.family == AF_INET &&
-            local->srx.transport_type == SOCK_DGRAM
-            ) {
-                list_for_each_entry(peer, &rxrpc_peers, link) {
-                        if (atomic_read(&peer->usage) > 0 &&
-                            peer->srx.transport_type == SOCK_DGRAM &&
-                            peer->srx.transport.family == AF_INET &&
-                            peer->srx.transport.sin.sin_port == port &&
-                            peer->srx.transport.sin.sin_addr.s_addr == addr)
-                                goto found_UDP_peer;
-                }
-
-                goto new_UDP_peer;
-        }
-
-        read_unlock_bh(&rxrpc_peer_lock);
-        _leave(" = -EAFNOSUPPORT");
-        return ERR_PTR(-EAFNOSUPPORT);
-
-found_UDP_peer:
-        _net("Rx UDP DGRAM from peer %d", peer->debug_id);
-        atomic_inc(&peer->usage);
-        read_unlock_bh(&rxrpc_peer_lock);
-        _leave(" = %p", peer);
-        return peer;
-
-new_UDP_peer:
-        _net("Rx UDP DGRAM from NEW peer");
-        read_unlock_bh(&rxrpc_peer_lock);
-        _leave(" = -EBUSY [new]");
-        return ERR_PTR(-EBUSY);
-}
-
-/*
- * release a remote transport endpoint
- */
-void rxrpc_put_peer(struct rxrpc_peer *peer)
-{
-        _enter("%p{u=%d}", peer, atomic_read(&peer->usage));
-
-        ASSERTCMP(atomic_read(&peer->usage), >, 0);
-
-        if (likely(!atomic_dec_and_test(&peer->usage))) {
-                _leave(" [in use]");
-                return;
-        }
-
-        rxrpc_queue_work(&peer->destroyer);
-        _leave("");
-}
-
-/*
- * destroy a remote transport endpoint
- */
-static void rxrpc_destroy_peer(struct work_struct *work)
-{
-        struct rxrpc_peer *peer =
-                container_of(work, struct rxrpc_peer, destroyer);
-
-        _enter("%p{%d}", peer, atomic_read(&peer->usage));
-
-        write_lock_bh(&rxrpc_peer_lock);
-        list_del(&peer->link);
-        write_unlock_bh(&rxrpc_peer_lock);
-
-        _net("DESTROY PEER %d", peer->debug_id);
-        kfree(peer);
-
-        if (list_empty(&rxrpc_peers))
-                wake_up_all(&rxrpc_peer_wq);
-        _leave("");
-}
-
-/*
- * preemptively destroy all the peer records from a transport endpoint rather
- * than waiting for them to time out
- */
-void __exit rxrpc_destroy_all_peers(void)
-{
-        DECLARE_WAITQUEUE(myself,current);
-
-        _enter("");
-
-        /* we simply have to wait for them to go away */
-        if (!list_empty(&rxrpc_peers)) {
-                set_current_state(TASK_UNINTERRUPTIBLE);
-                add_wait_queue(&rxrpc_peer_wq, &myself);
-
-                while (!list_empty(&rxrpc_peers)) {
-                        schedule();
-                        set_current_state(TASK_UNINTERRUPTIBLE);
-                }
-
-                remove_wait_queue(&rxrpc_peer_wq, &myself);
-                set_current_state(TASK_RUNNING);
-        }
-
-        _leave("");
-}
diff --git a/net/rxrpc/ar-transport.c b/net/rxrpc/ar-transport.c
deleted file mode 100644
index 66a1a5676446..000000000000
--- a/net/rxrpc/ar-transport.c
+++ /dev/null
@@ -1,284 +0,0 @@
-/* RxRPC point-to-point transport session management
- *
- * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
- * Written by David Howells (dhowells@redhat.com)
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
- */
-
-#include <linux/module.h>
-#include <linux/net.h>
-#include <linux/skbuff.h>
-#include <linux/slab.h>
-#include <net/sock.h>
-#include <net/af_rxrpc.h>
-#include "ar-internal.h"
-
-/*
- * Time after last use at which transport record is cleaned up.
- */
-unsigned int rxrpc_transport_expiry = 3600 * 24;
-
-static void rxrpc_transport_reaper(struct work_struct *work);
-
-static LIST_HEAD(rxrpc_transports);
-static DEFINE_RWLOCK(rxrpc_transport_lock);
-static DECLARE_DELAYED_WORK(rxrpc_transport_reap, rxrpc_transport_reaper);
-
-/*
- * allocate a new transport session manager
- */
-static struct rxrpc_transport *rxrpc_alloc_transport(struct rxrpc_local *local,
-                                                     struct rxrpc_peer *peer,
-                                                     gfp_t gfp)
-{
-        struct rxrpc_transport *trans;
-
-        _enter("");
-
-        trans = kzalloc(sizeof(struct rxrpc_transport), gfp);
-        if (trans) {
-                trans->local = local;
-                trans->peer = peer;
-                INIT_LIST_HEAD(&trans->link);
-                trans->bundles = RB_ROOT;
-                trans->client_conns = RB_ROOT;
-                trans->server_conns = RB_ROOT;
-                skb_queue_head_init(&trans->error_queue);
-                spin_lock_init(&trans->client_lock);
-                rwlock_init(&trans->conn_lock);
-                atomic_set(&trans->usage, 1);
-                trans->conn_idcounter = peer->srx.srx_service << 16;
-                trans->debug_id = atomic_inc_return(&rxrpc_debug_id);
-
-                if (peer->srx.transport.family == AF_INET) {
-                        switch (peer->srx.transport_type) {
-                        case SOCK_DGRAM:
-                                INIT_WORK(&trans->error_handler,
-                                          rxrpc_UDP_error_handler);
-                                break;
-                        default:
-                                BUG();
-                                break;
-                        }
-                } else {
-                        BUG();
-                }
-        }
-
-        _leave(" = %p", trans);
-        return trans;
-}
-
-/*
- * obtain a transport session for the nominated endpoints
- */
-struct rxrpc_transport *rxrpc_get_transport(struct rxrpc_local *local,
-                                            struct rxrpc_peer *peer,
-                                            gfp_t gfp)
-{
-        struct rxrpc_transport *trans, *candidate;
-        const char *new = "old";
-        int usage;
-
-        _enter("{%pI4+%hu},{%pI4+%hu},",
-               &local->srx.transport.sin.sin_addr,
-               ntohs(local->srx.transport.sin.sin_port),
-               &peer->srx.transport.sin.sin_addr,
-               ntohs(peer->srx.transport.sin.sin_port));
-
-        /* search the transport list first */
-        read_lock_bh(&rxrpc_transport_lock);
-        list_for_each_entry(trans, &rxrpc_transports, link) {
-                if (trans->local == local && trans->peer == peer)
-                        goto found_extant_transport;
-        }
-        read_unlock_bh(&rxrpc_transport_lock);
-
-        /* not yet present - create a candidate for a new record and then
-         * redo the search */
-        candidate = rxrpc_alloc_transport(local, peer, gfp);
-        if (!candidate) {
-                _leave(" = -ENOMEM");
-                return ERR_PTR(-ENOMEM);
-        }
-
-        write_lock_bh(&rxrpc_transport_lock);
-
-        list_for_each_entry(trans, &rxrpc_transports, link) {
-                if (trans->local == local && trans->peer == peer)
-                        goto found_extant_second;
-        }
-
-        /* we can now add the new candidate to the list */
-        trans = candidate;
-        candidate = NULL;
-        usage = atomic_read(&trans->usage);
-
-        rxrpc_get_local(trans->local);
-        atomic_inc(&trans->peer->usage);
-        list_add_tail(&trans->link, &rxrpc_transports);
-        write_unlock_bh(&rxrpc_transport_lock);
-        new = "new";
-
-success:
-        _net("TRANSPORT %s %d local %d -> peer %d",
-             new,
-             trans->debug_id,
-             trans->local->debug_id,
-             trans->peer->debug_id);
-
-        _leave(" = %p {u=%d}", trans, usage);
-        return trans;
-
-        /* we found the transport in the list immediately */
-found_extant_transport:
-        usage = atomic_inc_return(&trans->usage);
-        read_unlock_bh(&rxrpc_transport_lock);
-        goto success;
-
-        /* we found the transport on the second time through the list */
-found_extant_second:
-        usage = atomic_inc_return(&trans->usage);
-        write_unlock_bh(&rxrpc_transport_lock);
-        kfree(candidate);
-        goto success;
-}
-
-/*
- * find the transport connecting two endpoints
- */
-struct rxrpc_transport *rxrpc_find_transport(struct rxrpc_local *local,
-                                             struct rxrpc_peer *peer)
-{
-        struct rxrpc_transport *trans;
-
-        _enter("{%pI4+%hu},{%pI4+%hu},",
-               &local->srx.transport.sin.sin_addr,
-               ntohs(local->srx.transport.sin.sin_port),
-               &peer->srx.transport.sin.sin_addr,
-               ntohs(peer->srx.transport.sin.sin_port));
-
-        /* search the transport list */
-        read_lock_bh(&rxrpc_transport_lock);
-
-        list_for_each_entry(trans, &rxrpc_transports, link) {
-                if (trans->local == local && trans->peer == peer)
-                        goto found_extant_transport;
-        }
-
-        read_unlock_bh(&rxrpc_transport_lock);
-        _leave(" = NULL");
-        return NULL;
-
-found_extant_transport:
-        atomic_inc(&trans->usage);
-        read_unlock_bh(&rxrpc_transport_lock);
-        _leave(" = %p", trans);
-        return trans;
-}
-
-/*
- * release a transport session
- */
-void rxrpc_put_transport(struct rxrpc_transport *trans)
-{
-        _enter("%p{u=%d}", trans, atomic_read(&trans->usage));
-
-        ASSERTCMP(atomic_read(&trans->usage), >, 0);
-
-        trans->put_time = ktime_get_seconds();
-        if (unlikely(atomic_dec_and_test(&trans->usage))) {
-                _debug("zombie");
-                /* let the reaper determine the timeout to avoid a race with
-                 * overextending the timeout if the reaper is running at the
-                 * same time */
-                rxrpc_queue_delayed_work(&rxrpc_transport_reap, 0);
-        }
-        _leave("");
-}
-
-/*
- * clean up a transport session
- */
-static void rxrpc_cleanup_transport(struct rxrpc_transport *trans)
-{
-        _net("DESTROY TRANS %d", trans->debug_id);
-
-        rxrpc_purge_queue(&trans->error_queue);
-
-        rxrpc_put_local(trans->local);
-        rxrpc_put_peer(trans->peer);
-        kfree(trans);
-}
-
-/*
- * reap dead transports that have passed their expiry date
- */
-static void rxrpc_transport_reaper(struct work_struct *work)
-{
-        struct rxrpc_transport *trans, *_p;
-        unsigned long now, earliest, reap_time;
-
-        LIST_HEAD(graveyard);
-
-        _enter("");
-
-        now = ktime_get_seconds();
-        earliest = ULONG_MAX;
-
-        /* extract all the transports that have been dead too long */
-        write_lock_bh(&rxrpc_transport_lock);
-        list_for_each_entry_safe(trans, _p, &rxrpc_transports, link) {
-                _debug("reap TRANS %d { u=%d t=%ld }",
-                       trans->debug_id, atomic_read(&trans->usage),
-                       (long) now - (long) trans->put_time);
-
-                if (likely(atomic_read(&trans->usage) > 0))
-                        continue;
-
-                reap_time = trans->put_time + rxrpc_transport_expiry;
-                if (reap_time <= now)
-                        list_move_tail(&trans->link, &graveyard);
-                else if (reap_time < earliest)
-                        earliest = reap_time;
-        }
-        write_unlock_bh(&rxrpc_transport_lock);
-
-        if (earliest != ULONG_MAX) {
-                _debug("reschedule reaper %ld", (long) earliest - now);
-                ASSERTCMP(earliest, >, now);
-                rxrpc_queue_delayed_work(&rxrpc_transport_reap,
-                                         (earliest - now) * HZ);
-        }
-
-        /* then destroy all those pulled out */
-        while (!list_empty(&graveyard)) {
-                trans = list_entry(graveyard.next, struct rxrpc_transport,
-                                   link);
-                list_del_init(&trans->link);
-
-                ASSERTCMP(atomic_read(&trans->usage), ==, 0);
-                rxrpc_cleanup_transport(trans);
-        }
-
-        _leave("");
-}
-
-/*
- * preemptively destroy all the transport session records rather than waiting
- * for them to time out
- */
-void __exit rxrpc_destroy_all_transports(void)
-{
-        _enter("");
-
-        rxrpc_transport_expiry = 0;
-        cancel_delayed_work(&rxrpc_transport_reap);
-        rxrpc_queue_delayed_work(&rxrpc_transport_reap, 0);
-
-        _leave("");
-}
diff --git a/net/rxrpc/ar-accept.c b/net/rxrpc/call_accept.c
index e7a7f05f13e2..202e053a3c6d 100644
--- a/net/rxrpc/ar-accept.c
+++ b/net/rxrpc/call_accept.c
@@ -9,6 +9,8 @@
  * 2 of the License, or (at your option) any later version.
  */
 
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
 #include <linux/module.h>
 #include <linux/net.h>
 #include <linux/skbuff.h>
@@ -72,7 +74,6 @@ static int rxrpc_accept_incoming_call(struct rxrpc_local *local,
                                       struct sockaddr_rxrpc *srx)
 {
         struct rxrpc_connection *conn;
-        struct rxrpc_transport *trans;
         struct rxrpc_skb_priv *sp, *nsp;
         struct rxrpc_peer *peer;
         struct rxrpc_call *call;
@@ -93,30 +94,22 @@ static int rxrpc_accept_incoming_call(struct rxrpc_local *local,
         rxrpc_new_skb(notification);
         notification->mark = RXRPC_SKB_MARK_NEW_CALL;
 
-        peer = rxrpc_get_peer(srx, GFP_NOIO);
-        if (IS_ERR(peer)) {
+        peer = rxrpc_lookup_peer(local, srx, GFP_NOIO);
+        if (!peer) {
                 _debug("no peer");
                 ret = -EBUSY;
                 goto error;
         }
 
-        trans = rxrpc_get_transport(local, peer, GFP_NOIO);
+        conn = rxrpc_incoming_connection(local, peer, skb);
         rxrpc_put_peer(peer);
-        if (IS_ERR(trans)) {
-                _debug("no trans");
-                ret = -EBUSY;
-                goto error;
-        }
-
-        conn = rxrpc_incoming_connection(trans, &sp->hdr);
-        rxrpc_put_transport(trans);
         if (IS_ERR(conn)) {
                 _debug("no conn");
                 ret = PTR_ERR(conn);
                 goto error;
         }
 
-        call = rxrpc_incoming_call(rx, conn, &sp->hdr);
+        call = rxrpc_incoming_call(rx, conn, skb);
         rxrpc_put_connection(conn);
         if (IS_ERR(call)) {
                 _debug("no call");
@@ -139,7 +132,7 @@ static int rxrpc_accept_incoming_call(struct rxrpc_local *local,
                         _debug("await conn sec");
                         list_add_tail(&call->accept_link, &rx->secureq);
                         call->conn->state = RXRPC_CONN_SERVER_CHALLENGING;
-                        atomic_inc(&call->conn->usage);
+                        rxrpc_get_connection(call->conn);
                         set_bit(RXRPC_CONN_CHALLENGE, &call->conn->events);
                         rxrpc_queue_conn(call->conn);
                 } else {
@@ -200,10 +193,8 @@ error_nofree:
  * accept incoming calls that need peer, transport and/or connection setting up
  * - the packets we get are all incoming client DATA packets that have seq == 1
  */
-void rxrpc_accept_incoming_calls(struct work_struct *work)
+void rxrpc_accept_incoming_calls(struct rxrpc_local *local)
 {
-        struct rxrpc_local *local =
-                container_of(work, struct rxrpc_local, acceptor);
         struct rxrpc_skb_priv *sp;
         struct sockaddr_rxrpc srx;
         struct rxrpc_sock *rx;
@@ -213,21 +204,8 @@ void rxrpc_accept_incoming_calls(struct work_struct *work)
 
         _enter("%d", local->debug_id);
 
-        read_lock_bh(&rxrpc_local_lock);
-        if (atomic_read(&local->usage) > 0)
-                rxrpc_get_local(local);
-        else
-                local = NULL;
-        read_unlock_bh(&rxrpc_local_lock);
-        if (!local) {
-                _leave(" [local dead]");
-                return;
-        }
-
-process_next_packet:
         skb = skb_dequeue(&local->accept_queue);
         if (!skb) {
-                rxrpc_put_local(local);
                 _leave("\n");
                 return;
         }
@@ -290,7 +268,7 @@ found_service:
         case -ECONNRESET: /* old calls are ignored */
         case -ECONNABORTED: /* aborted calls are reaborted or ignored */
         case 0:
-                goto process_next_packet;
+                return;
         case -ECONNREFUSED:
                 goto invalid_service;
         case -EBUSY:
@@ -306,18 +284,18 @@ backlog_full:
 busy:
         rxrpc_busy(local, &srx, &whdr);
         rxrpc_free_skb(skb);
-        goto process_next_packet;
+        return;
 
 invalid_service:
         skb->priority = RX_INVALID_OPERATION;
         rxrpc_reject_packet(local, skb);
-        goto process_next_packet;
+        return;
 
         /* can't change connection security type mid-flow */
 security_mismatch:
         skb->priority = RX_PROTOCOL_ERROR;
         rxrpc_reject_packet(local, skb);
-        goto process_next_packet;
+        return;
 }
 
 /*
diff --git a/net/rxrpc/ar-ack.c b/net/rxrpc/call_event.c
index 374478e006e7..0ba84295f913 100644
--- a/net/rxrpc/ar-ack.c
+++ b/net/rxrpc/call_event.c
@@ -9,6 +9,8 @@
  * 2 of the License, or (at your option) any later version.
  */
 
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
 #include <linux/module.h>
 #include <linux/circ_buf.h>
 #include <linux/net.h>
@@ -185,7 +187,7 @@ static void rxrpc_resend(struct rxrpc_call *call)
 
                         _proto("Tx DATA %%%u { #%d }",
                                sp->hdr.serial, sp->hdr.seq);
-                        if (rxrpc_send_packet(call->conn->trans, txb) < 0) {
+                        if (rxrpc_send_data_packet(call->conn, txb) < 0) {
                                 stop = true;
                                 sp->resend_at = jiffies + 3;
                         } else {
@@ -543,7 +545,7 @@ static void rxrpc_extract_ackinfo(struct rxrpc_call *call, struct sk_buff *skb,
 
         mtu = min(ntohl(ackinfo.rxMTU), ntohl(ackinfo.maxMTU));
 
-        peer = call->conn->trans->peer;
+        peer = call->conn->params.peer;
         if (mtu < peer->maxdata) {
                 spin_lock_bh(&peer->lock);
                 peer->maxdata = mtu;
@@ -834,13 +836,13 @@ void rxrpc_process_call(struct work_struct *work)
 
         /* there's a good chance we're going to have to send a message, so set
          * one up in advance */
-        msg.msg_name    = &call->conn->trans->peer->srx.transport;
-        msg.msg_namelen = call->conn->trans->peer->srx.transport_len;
+        msg.msg_name    = &call->conn->params.peer->srx.transport;
+        msg.msg_namelen = call->conn->params.peer->srx.transport_len;
         msg.msg_control = NULL;
         msg.msg_controllen = 0;
         msg.msg_flags   = 0;
 
-        whdr.epoch      = htonl(call->conn->epoch);
+        whdr.epoch      = htonl(call->conn->proto.epoch);
         whdr.cid        = htonl(call->cid);
         whdr.callNumber = htonl(call->call_id);
         whdr.seq        = 0;
@@ -862,17 +864,24 @@ void rxrpc_process_call(struct work_struct *work)
         }
 
         if (test_bit(RXRPC_CALL_EV_RCVD_ERROR, &call->events)) {
+                enum rxrpc_skb_mark mark;
                 int error;
 
                 clear_bit(RXRPC_CALL_EV_CONN_ABORT, &call->events);
                 clear_bit(RXRPC_CALL_EV_REJECT_BUSY, &call->events);
                 clear_bit(RXRPC_CALL_EV_ABORT, &call->events);
 
-                error = call->conn->trans->peer->net_error;
-                _debug("post net error %d", error);
+                error = call->error_report;
+                if (error < RXRPC_LOCAL_ERROR_OFFSET) {
+                        mark = RXRPC_SKB_MARK_NET_ERROR;
+                        _debug("post net error %d", error);
+                } else {
+                        mark = RXRPC_SKB_MARK_LOCAL_ERROR;
+                        error -= RXRPC_LOCAL_ERROR_OFFSET;
+                        _debug("post net local error %d", error);
+                }
 
-                if (rxrpc_post_message(call, RXRPC_SKB_MARK_NET_ERROR,
-                                       error, true) < 0)
+                if (rxrpc_post_message(call, mark, error, true) < 0)
                         goto no_mem;
                 clear_bit(RXRPC_CALL_EV_RCVD_ERROR, &call->events);
                 goto kill_ACKs;
@@ -1142,8 +1151,8 @@ send_ACK_with_skew:
         ack.maxSkew = htons(atomic_read(&call->conn->hi_serial) -
                             ntohl(ack.serial));
 send_ACK:
-        mtu = call->conn->trans->peer->if_mtu;
-        mtu -= call->conn->trans->peer->hdrsize;
+        mtu = call->conn->params.peer->if_mtu;
+        mtu -= call->conn->params.peer->hdrsize;
         ackinfo.maxMTU  = htonl(mtu);
         ackinfo.rwind   = htonl(rxrpc_rx_window_size);
 
@@ -1197,7 +1206,7 @@ send_message_2:
                 len += iov[1].iov_len;
         }
 
-        ret = kernel_sendmsg(call->conn->trans->local->socket,
+        ret = kernel_sendmsg(call->conn->params.local->socket,
                              &msg, iov, ioc, len);
         if (ret < 0) {
                 _debug("sendmsg failed: %d", ret);
@@ -1255,7 +1264,7 @@ maybe_reschedule:
         if (call->state >= RXRPC_CALL_COMPLETE &&
             !list_empty(&call->accept_link)) {
                 _debug("X unlinking once-pending call %p { e=%lx f=%lx c=%x }",
-                       call, call->events, call->flags, call->conn->cid);
+                       call, call->events, call->flags, call->conn->proto.cid);
 
                 read_lock_bh(&call->state_lock);
                 if (!test_bit(RXRPC_CALL_RELEASED, &call->flags) &&
@@ -1273,7 +1282,7 @@ error:
          * this means there's a race between clearing the flag and setting the
          * work pending bit and the work item being processed again */
         if (call->events && !work_pending(&call->processor)) {
-                _debug("jumpstart %x", call->conn->cid);
+                _debug("jumpstart %x", call->conn->proto.cid);
                 rxrpc_queue_call(call);
         }
 
diff --git a/net/rxrpc/ar-call.c b/net/rxrpc/call_object.c
index 571a41fd5a32..ad933daae13b 100644
--- a/net/rxrpc/ar-call.c
+++ b/net/rxrpc/call_object.c
@@ -9,6 +9,8 @@
  * 2 of the License, or (at your option) any later version.
  */
 
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
 #include <linux/slab.h>
 #include <linux/module.h>
 #include <linux/circ_buf.h>
@@ -29,6 +31,8 @@ unsigned int rxrpc_max_call_lifetime = 60 * HZ;
 unsigned int rxrpc_dead_call_expiry = 2 * HZ;
 
 const char *const rxrpc_call_states[NR__RXRPC_CALL_STATES] = {
+        [RXRPC_CALL_UNINITIALISED]              = "Uninit",
+        [RXRPC_CALL_CLIENT_AWAIT_CONN]          = "ClWtConn",
         [RXRPC_CALL_CLIENT_SEND_REQUEST]        = "ClSndReq",
         [RXRPC_CALL_CLIENT_AWAIT_REPLY]         = "ClAwtRpl",
         [RXRPC_CALL_CLIENT_RECV_REPLY]          = "ClRcvRpl",
@@ -69,7 +73,7 @@ static unsigned long rxrpc_call_hashfunc(
         u32             call_id,
         u32             epoch,
         u16             service_id,
-        sa_family_t     proto,
+        sa_family_t     family,
         void            *localptr,
         unsigned int    addr_size,
         const u8        *peer_addr)
@@ -90,7 +94,7 @@ static unsigned long rxrpc_call_hashfunc(
         key += (cid & RXRPC_CIDMASK) >> RXRPC_CIDSHIFT;
         key += cid & RXRPC_CHANNELMASK;
         key += in_clientflag;
-        key += proto;
+        key += family;
         /* Step through the peer address in 16-bit portions for speed */
         for (i = 0, p = (const u16 *)peer_addr; i < addr_size >> 1; i++, p++)
                 key += *p;
@@ -107,7 +111,7 @@ static void rxrpc_call_hash_add(struct rxrpc_call *call)
         unsigned int addr_size = 0;
 
         _enter("");
-        switch (call->proto) {
+        switch (call->family) {
         case AF_INET:
                 addr_size = sizeof(call->peer_ip.ipv4_addr);
                 break;
@@ -119,8 +123,8 @@ static void rxrpc_call_hash_add(struct rxrpc_call *call)
         }
         key = rxrpc_call_hashfunc(call->in_clientflag, call->cid,
                                   call->call_id, call->epoch,
-                                  call->service_id, call->proto,
-                                  call->conn->trans->local, addr_size,
+                                  call->service_id, call->family,
+                                  call->conn->params.local, addr_size,
                                   call->peer_ip.ipv6_addr);
         /* Store the full key in the call */
         call->hash_key = key;
@@ -149,7 +153,7 @@ static void rxrpc_call_hash_del(struct rxrpc_call *call)
 struct rxrpc_call *rxrpc_find_call_hash(
         struct rxrpc_host_header *hdr,
         void            *localptr,
-        sa_family_t     proto,
+        sa_family_t     family,
         const void      *peer_addr)
 {
         unsigned long key;
@@ -159,7 +163,7 @@ struct rxrpc_call *rxrpc_find_call_hash(
         u8 in_clientflag = hdr->flags & RXRPC_CLIENT_INITIATED;
 
         _enter("");
-        switch (proto) {
+        switch (family) {
         case AF_INET:
                 addr_size = sizeof(call->peer_ip.ipv4_addr);
                 break;
@@ -172,7 +176,7 @@ struct rxrpc_call *rxrpc_find_call_hash(
 
         key = rxrpc_call_hashfunc(in_clientflag, hdr->cid, hdr->callNumber,
                                   hdr->epoch, hdr->serviceId,
-                                  proto, localptr, addr_size,
+                                  family, localptr, addr_size,
                                   peer_addr);
         hash_for_each_possible_rcu(rxrpc_call_hash, call, hash_node, key) {
                 if (call->hash_key == key &&
@@ -180,7 +184,7 @@ struct rxrpc_call *rxrpc_find_call_hash(
                     call->cid == hdr->cid &&
                     call->in_clientflag == in_clientflag &&
                     call->service_id == hdr->serviceId &&
-                    call->proto == proto &&
+                    call->family == family &&
                     call->local == localptr &&
                     memcmp(call->peer_ip.ipv6_addr, peer_addr,
                            addr_size) == 0 &&
@@ -194,6 +198,43 @@ struct rxrpc_call *rxrpc_find_call_hash(
 }
 
 /*
+ * find an extant server call
+ * - called in process context with IRQs enabled
+ */
+struct rxrpc_call *rxrpc_find_call_by_user_ID(struct rxrpc_sock *rx,
+                                              unsigned long user_call_ID)
+{
+        struct rxrpc_call *call;
+        struct rb_node *p;
+
+        _enter("%p,%lx", rx, user_call_ID);
+
+        read_lock(&rx->call_lock);
+
+        p = rx->calls.rb_node;
+        while (p) {
+                call = rb_entry(p, struct rxrpc_call, sock_node);
+
+                if (user_call_ID < call->user_call_ID)
+                        p = p->rb_left;
+                else if (user_call_ID > call->user_call_ID)
+                        p = p->rb_right;
+                else
+                        goto found_extant_call;
+        }
+
+        read_unlock(&rx->call_lock);
+        _leave(" = NULL");
+        return NULL;
+
+found_extant_call:
+        rxrpc_get_call(call);
+        read_unlock(&rx->call_lock);
+        _leave(" = %p [%d]", call, atomic_read(&call->usage));
+        return call;
+}
+
+/*
  * allocate a new call
  */
 static struct rxrpc_call *rxrpc_alloc_call(gfp_t gfp)
@@ -222,6 +263,7 @@ static struct rxrpc_call *rxrpc_alloc_call(gfp_t gfp)
                     (unsigned long) call);
         INIT_WORK(&call->destroyer, &rxrpc_destroy_call);
         INIT_WORK(&call->processor, &rxrpc_process_call);
+        INIT_LIST_HEAD(&call->link);
         INIT_LIST_HEAD(&call->accept_link);
         skb_queue_head_init(&call->rx_queue);
         skb_queue_head_init(&call->rx_oos_queue);
@@ -230,7 +272,6 @@ static struct rxrpc_call *rxrpc_alloc_call(gfp_t gfp)
         rwlock_init(&call->state_lock);
         atomic_set(&call->usage, 1);
         call->debug_id = atomic_inc_return(&rxrpc_debug_id);
-        call->state = RXRPC_CALL_CLIENT_SEND_REQUEST;
 
         memset(&call->sock_node, 0xed, sizeof(call->sock_node));
 
@@ -243,117 +284,104 @@ static struct rxrpc_call *rxrpc_alloc_call(gfp_t gfp)
 }
 
 /*
- * allocate a new client call and attempt to get a connection slot for it
+ * Allocate a new client call.
  */
-static struct rxrpc_call *rxrpc_alloc_client_call(
-        struct rxrpc_sock *rx,
-        struct rxrpc_transport *trans,
-        struct rxrpc_conn_bundle *bundle,
-        gfp_t gfp)
+static struct rxrpc_call *rxrpc_alloc_client_call(struct rxrpc_sock *rx,
+                                                  struct sockaddr_rxrpc *srx,
+                                                  gfp_t gfp)
 {
         struct rxrpc_call *call;
-        int ret;
 
         _enter("");
 
-        ASSERT(rx != NULL);
-        ASSERT(trans != NULL);
-        ASSERT(bundle != NULL);
+        ASSERT(rx->local != NULL);
 
         call = rxrpc_alloc_call(gfp);
         if (!call)
                 return ERR_PTR(-ENOMEM);
+        call->state = RXRPC_CALL_CLIENT_AWAIT_CONN;
 
         sock_hold(&rx->sk);
         call->socket = rx;
         call->rx_data_post = 1;
 
-        ret = rxrpc_connect_call(rx, trans, bundle, call, gfp);
-        if (ret < 0) {
-                kmem_cache_free(rxrpc_call_jar, call);
-                return ERR_PTR(ret);
-        }
-
         /* Record copies of information for hashtable lookup */
-        call->proto = rx->proto;
-        call->local = trans->local;
-        switch (call->proto) {
+        call->family = rx->family;
+        call->local = rx->local;
+        switch (call->family) {
         case AF_INET:
-                call->peer_ip.ipv4_addr =
-                        trans->peer->srx.transport.sin.sin_addr.s_addr;
+                call->peer_ip.ipv4_addr = srx->transport.sin.sin_addr.s_addr;
                 break;
         case AF_INET6:
                 memcpy(call->peer_ip.ipv6_addr,
-                       trans->peer->srx.transport.sin6.sin6_addr.in6_u.u6_addr8,
+                       srx->transport.sin6.sin6_addr.in6_u.u6_addr8,
                        sizeof(call->peer_ip.ipv6_addr));
                 break;
         }
-        call->epoch = call->conn->epoch;
-        call->service_id = call->conn->service_id;
-        call->in_clientflag = call->conn->in_clientflag;
+
+        call->service_id = srx->srx_service;
+        call->in_clientflag = 0;
+
+        _leave(" = %p", call);
+        return call;
+}
+
+/*
+ * Begin client call.
+ */
+static int rxrpc_begin_client_call(struct rxrpc_call *call,
+                                   struct rxrpc_conn_parameters *cp,
+                                   struct sockaddr_rxrpc *srx,
+                                   gfp_t gfp)
+{
+        int ret;
+
+        /* Set up or get a connection record and set the protocol parameters,
+         * including channel number and call ID.
+         */
+        ret = rxrpc_connect_call(call, cp, srx, gfp);
+        if (ret < 0)
+                return ret;
+
+        call->state = RXRPC_CALL_CLIENT_SEND_REQUEST;
+
         /* Add the new call to the hashtable */
         rxrpc_call_hash_add(call);
 
-        spin_lock(&call->conn->trans->peer->lock);
-        list_add(&call->error_link, &call->conn->trans->peer->error_targets);
-        spin_unlock(&call->conn->trans->peer->lock);
+        spin_lock(&call->conn->params.peer->lock);
+        hlist_add_head(&call->error_link, &call->conn->params.peer->error_targets);
+        spin_unlock(&call->conn->params.peer->lock);
 
         call->lifetimer.expires = jiffies + rxrpc_max_call_lifetime;
         add_timer(&call->lifetimer);
-
-        _leave(" = %p", call);
-        return call;
+        return 0;
 }
 
 /*
  * set up a call for the given data
  * - called in process context with IRQs enabled
  */
-struct rxrpc_call *rxrpc_get_client_call(struct rxrpc_sock *rx,
-                                         struct rxrpc_transport *trans,
-                                         struct rxrpc_conn_bundle *bundle,
+struct rxrpc_call *rxrpc_new_client_call(struct rxrpc_sock *rx,
+                                         struct rxrpc_conn_parameters *cp,
+                                         struct sockaddr_rxrpc *srx,
                                          unsigned long user_call_ID,
-                                         int create,
                                          gfp_t gfp)
 {
-        struct rxrpc_call *call, *candidate;
-        struct rb_node *p, *parent, **pp;
-
-        _enter("%p,%d,%d,%lx,%d",
-               rx, trans ? trans->debug_id : -1, bundle ? bundle->debug_id : -1,
-               user_call_ID, create);
-
-        /* search the extant calls first for one that matches the specified
-         * user ID */
-        read_lock(&rx->call_lock);
-
-        p = rx->calls.rb_node;
-        while (p) {
-                call = rb_entry(p, struct rxrpc_call, sock_node);
-
-                if (user_call_ID < call->user_call_ID)
-                        p = p->rb_left;
-                else if (user_call_ID > call->user_call_ID)
-                        p = p->rb_right;
-                else
-                        goto found_extant_call;
-        }
-
-        read_unlock(&rx->call_lock);
+        struct rxrpc_call *call, *xcall;
+        struct rb_node *parent, **pp;
+        int ret;
 
-        if (!create || !trans)
-                return ERR_PTR(-EBADSLT);
+        _enter("%p,%lx", rx, user_call_ID);
 
-        /* not yet present - create a candidate for a new record and then
-         * redo the search */
-        candidate = rxrpc_alloc_client_call(rx, trans, bundle, gfp);
-        if (IS_ERR(candidate)) {
-                _leave(" = %ld", PTR_ERR(candidate));
-                return candidate;
+        call = rxrpc_alloc_client_call(rx, srx, gfp);
+        if (IS_ERR(call)) {
+                _leave(" = %ld", PTR_ERR(call));
+                return call;
         }
 
-        candidate->user_call_ID = user_call_ID;
-        __set_bit(RXRPC_CALL_HAS_USERID, &candidate->flags);
+        /* Publish the call, even though it is incompletely set up as yet */
+        call->user_call_ID = user_call_ID;
+        __set_bit(RXRPC_CALL_HAS_USERID, &call->flags);
 
         write_lock(&rx->call_lock);
 
@@ -361,19 +389,16 @@ struct rxrpc_call *rxrpc_get_client_call(struct rxrpc_sock *rx,
         parent = NULL;
         while (*pp) {
                 parent = *pp;
-                call = rb_entry(parent, struct rxrpc_call, sock_node);
+                xcall = rb_entry(parent, struct rxrpc_call, sock_node);
 
-                if (user_call_ID < call->user_call_ID)
+                if (user_call_ID < xcall->user_call_ID)
                         pp = &(*pp)->rb_left;
-                else if (user_call_ID > call->user_call_ID)
+                else if (user_call_ID > xcall->user_call_ID)
                         pp = &(*pp)->rb_right;
                 else
-                        goto found_extant_second;
+                        goto found_user_ID_now_present;
         }
 
-        /* second search also failed; add the new call */
-        call = candidate;
-        candidate = NULL;
         rxrpc_get_call(call);
 
         rb_link_node(&call->sock_node, parent, pp);
@@ -384,25 +409,39 @@ struct rxrpc_call *rxrpc_get_client_call(struct rxrpc_sock *rx,
         list_add_tail(&call->link, &rxrpc_calls);
         write_unlock_bh(&rxrpc_call_lock);
 
+        ret = rxrpc_begin_client_call(call, cp, srx, gfp);
+        if (ret < 0)
+                goto error;
+
         _net("CALL new %d on CONN %d", call->debug_id, call->conn->debug_id);
 
         _leave(" = %p [new]", call);
         return call;
 
-        /* we found the call in the list immediately */
-found_extant_call:
-        rxrpc_get_call(call);
-        read_unlock(&rx->call_lock);
-        _leave(" = %p [extant %d]", call, atomic_read(&call->usage));
-        return call;
+error:
+        write_lock(&rx->call_lock);
+        rb_erase(&call->sock_node, &rx->calls);
+        write_unlock(&rx->call_lock);
+        rxrpc_put_call(call);
 
-        /* we found the call on the second time through the list */
-found_extant_second:
-        rxrpc_get_call(call);
+        write_lock_bh(&rxrpc_call_lock);
+        list_del(&call->link);
+        write_unlock_bh(&rxrpc_call_lock);
+
+        rxrpc_put_call(call);
+        _leave(" = %d", ret);
+        return ERR_PTR(ret);
+
+        /* We unexpectedly found the user ID in the list after taking
+         * the call_lock.  This shouldn't happen unless the user races
+         * with itself and tries to add the same user ID twice at the
+         * same time in different threads.
+         */
+found_user_ID_now_present:
         write_unlock(&rx->call_lock);
-        rxrpc_put_call(candidate);
-        _leave(" = %p [second %d]", call, atomic_read(&call->usage));
-        return call;
+        rxrpc_put_call(call);
+        _leave(" = -EEXIST [%p]", call);
+        return ERR_PTR(-EEXIST);
 }
 
 /*
@@ -411,8 +450,9 @@ found_extant_second:
  */
 struct rxrpc_call *rxrpc_incoming_call(struct rxrpc_sock *rx,
                                        struct rxrpc_connection *conn,
-                                       struct rxrpc_host_header *hdr)
+                                       struct sk_buff *skb)
 {
+        struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
         struct rxrpc_call *call, *candidate;
         struct rb_node **p, *parent;
         u32 call_id;
@@ -425,13 +465,13 @@ struct rxrpc_call *rxrpc_incoming_call(struct rxrpc_sock *rx,
         if (!candidate)
                 return ERR_PTR(-EBUSY);
 
-        candidate->socket = rx;
-        candidate->conn = conn;
-        candidate->cid = hdr->cid;
-        candidate->call_id = hdr->callNumber;
-        candidate->channel = hdr->cid & RXRPC_CHANNELMASK;
-        candidate->rx_data_post = 0;
-        candidate->state = RXRPC_CALL_SERVER_ACCEPTING;
+        candidate->socket       = rx;
+        candidate->conn         = conn;
+        candidate->cid          = sp->hdr.cid;
+        candidate->call_id      = sp->hdr.callNumber;
+        candidate->channel      = sp->hdr.cid & RXRPC_CHANNELMASK;
+        candidate->rx_data_post = 0;
+        candidate->state        = RXRPC_CALL_SERVER_ACCEPTING;
         if (conn->security_ix > 0)
                 candidate->state = RXRPC_CALL_SERVER_SECURING;
 
@@ -440,7 +480,7 @@ struct rxrpc_call *rxrpc_incoming_call(struct rxrpc_sock *rx,
         /* set the channel for this call */
         call = conn->channels[candidate->channel];
         _debug("channel[%u] is %p", candidate->channel, call);
-        if (call && call->call_id == hdr->callNumber) {
+        if (call && call->call_id == sp->hdr.callNumber) {
                 /* already set; must've been a duplicate packet */
                 _debug("extant call [%d]", call->state);
                 ASSERTCMP(call->conn, ==, conn);
@@ -478,7 +518,7 @@ struct rxrpc_call *rxrpc_incoming_call(struct rxrpc_sock *rx,
 
         /* check the call number isn't duplicate */
         _debug("check dup");
-        call_id = hdr->callNumber;
+        call_id = sp->hdr.callNumber;
         p = &conn->calls.rb_node;
         parent = NULL;
         while (*p) {
@@ -504,36 +544,36 @@ struct rxrpc_call *rxrpc_incoming_call(struct rxrpc_sock *rx,
         rb_insert_color(&call->conn_node, &conn->calls);
         conn->channels[call->channel] = call;
         sock_hold(&rx->sk);
-        atomic_inc(&conn->usage);
+        rxrpc_get_connection(conn);
         write_unlock_bh(&conn->lock);
 
-        spin_lock(&conn->trans->peer->lock);
-        list_add(&call->error_link, &conn->trans->peer->error_targets);
-        spin_unlock(&conn->trans->peer->lock);
+        spin_lock(&conn->params.peer->lock);
+        hlist_add_head(&call->error_link, &conn->params.peer->error_targets);
+        spin_unlock(&conn->params.peer->lock);
 
         write_lock_bh(&rxrpc_call_lock);
         list_add_tail(&call->link, &rxrpc_calls);
         write_unlock_bh(&rxrpc_call_lock);
 
         /* Record copies of information for hashtable lookup */
-        call->proto = rx->proto;
-        call->local = conn->trans->local;
-        switch (call->proto) {
+        call->family = rx->family;
+        call->local = conn->params.local;
+        switch (call->family) {
         case AF_INET:
                 call->peer_ip.ipv4_addr =
-                        conn->trans->peer->srx.transport.sin.sin_addr.s_addr;
+                        conn->params.peer->srx.transport.sin.sin_addr.s_addr;
                 break;
         case AF_INET6:
                 memcpy(call->peer_ip.ipv6_addr,
-                       conn->trans->peer->srx.transport.sin6.sin6_addr.in6_u.u6_addr8,
+                       conn->params.peer->srx.transport.sin6.sin6_addr.in6_u.u6_addr8,
                        sizeof(call->peer_ip.ipv6_addr));
                 break;
         default:
                 break;
         }
-        call->epoch = conn->epoch;
-        call->service_id = conn->service_id;
-        call->in_clientflag = conn->in_clientflag;
+        call->epoch = conn->proto.epoch;
+        call->service_id = conn->params.service_id;
+        call->in_clientflag = conn->proto.in_clientflag;
         /* Add the new call to the hashtable */
         rxrpc_call_hash_add(call);
 
@@ -564,46 +604,6 @@ old_call:
 }
 
 /*
- * find an extant server call
- * - called in process context with IRQs enabled
- */
-struct rxrpc_call *rxrpc_find_server_call(struct rxrpc_sock *rx,
-                                          unsigned long user_call_ID)
-{
-        struct rxrpc_call *call;
-        struct rb_node *p;
-
-        _enter("%p,%lx", rx, user_call_ID);
-
-        /* search the extant calls for one that matches the specified user
-         * ID */
-        read_lock(&rx->call_lock);
-
-        p = rx->calls.rb_node;
-        while (p) {
-                call = rb_entry(p, struct rxrpc_call, sock_node);
-
-                if (user_call_ID < call->user_call_ID)
-                        p = p->rb_left;
-                else if (user_call_ID > call->user_call_ID)
-                        p = p->rb_right;
-                else
-                        goto found_extant_call;
-        }
-
-        read_unlock(&rx->call_lock);
-        _leave(" = NULL");
-        return NULL;
-
-        /* we found the call in the list immediately */
-found_extant_call:
-        rxrpc_get_call(call);
-        read_unlock(&rx->call_lock);
-        _leave(" = %p [%d]", call, atomic_read(&call->usage));
-        return call;
-}
-
-/*
  * detach a call from a socket and set up for release
  */
 void rxrpc_release_call(struct rxrpc_call *call)
@@ -641,41 +641,13 @@ void rxrpc_release_call(struct rxrpc_call *call)
         write_unlock_bh(&rx->call_lock);
 
         /* free up the channel for reuse */
-        spin_lock(&conn->trans->client_lock);
+        spin_lock(&conn->channel_lock);
         write_lock_bh(&conn->lock);
         write_lock(&call->state_lock);
 
-        if (conn->channels[call->channel] == call)
-                conn->channels[call->channel] = NULL;
-
-        if (conn->out_clientflag && conn->bundle) {
-                conn->avail_calls++;
-                switch (conn->avail_calls) {
-                case 1:
-                        list_move_tail(&conn->bundle_link,
-                                       &conn->bundle->avail_conns);
-                case 2 ... RXRPC_MAXCALLS - 1:
-                        ASSERT(conn->channels[0] == NULL ||
-                               conn->channels[1] == NULL ||
-                               conn->channels[2] == NULL ||
-                               conn->channels[3] == NULL);
-                        break;
-                case RXRPC_MAXCALLS:
-                        list_move_tail(&conn->bundle_link,
-                                       &conn->bundle->unused_conns);
-                        ASSERT(conn->channels[0] == NULL &&
-                               conn->channels[1] == NULL &&
-                               conn->channels[2] == NULL &&
-                               conn->channels[3] == NULL);
-                        break;
-                default:
-                        printk(KERN_ERR "RxRPC: conn->avail_calls=%d\n",
-                               conn->avail_calls);
-                        BUG();
-                }
-        }
+        rxrpc_disconnect_call(call);
 
-        spin_unlock(&conn->trans->client_lock);
+        spin_unlock(&conn->channel_lock);
 
         if (call->state < RXRPC_CALL_COMPLETE &&
             call->state != RXRPC_CALL_CLIENT_FINAL_ACK) {
@@ -844,9 +816,9 @@ static void rxrpc_cleanup_call(struct rxrpc_call *call)
         }
 
         if (call->conn) {
-                spin_lock(&call->conn->trans->peer->lock);
-                list_del(&call->error_link);
-                spin_unlock(&call->conn->trans->peer->lock);
+                spin_lock(&call->conn->params.peer->lock);
+                hlist_del_init(&call->error_link);
+                spin_unlock(&call->conn->params.peer->lock);
 
                 write_lock_bh(&call->conn->lock);
                 rb_erase(&call->conn_node, &call->conn->calls);
@@ -935,16 +907,15 @@ void __exit rxrpc_destroy_all_calls(void)
                         if (call->state != RXRPC_CALL_DEAD)
                                 break;
                 default:
-                        printk(KERN_ERR "RXRPC:"
-                               " Call %p still in use (%d,%d,%s,%lx,%lx)!\n",
+                        pr_err("Call %p still in use (%d,%d,%s,%lx,%lx)!\n",
                                call, atomic_read(&call->usage),
                                atomic_read(&call->ackr_not_idle),
                                rxrpc_call_states[call->state],
                                call->flags, call->events);
                         if (!skb_queue_empty(&call->rx_queue))
-                                printk(KERN_ERR"RXRPC: Rx queue occupied\n");
+                                pr_err("Rx queue occupied\n");
                         if (!skb_queue_empty(&call->rx_oos_queue))
-                                printk(KERN_ERR"RXRPC: OOS queue occupied\n");
+                                pr_err("OOS queue occupied\n");
                         break;
                 }
 
diff --git a/net/rxrpc/conn_client.c b/net/rxrpc/conn_client.c
new file mode 100644
index 000000000000..82488d6adb83
--- /dev/null
+++ b/net/rxrpc/conn_client.c
@@ -0,0 +1,94 @@
+/* Client connection-specific management code.
+ *
+ * Copyright (C) 2016 Red Hat, Inc. All Rights Reserved.
+ * Written by David Howells (dhowells@redhat.com)
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public Licence
+ * as published by the Free Software Foundation; either version
+ * 2 of the Licence, or (at your option) any later version.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/slab.h>
+#include <linux/idr.h>
+#include <linux/timer.h>
+#include "ar-internal.h"
+
+/*
+ * We use machine-unique IDs for our client connections.
+ */
+DEFINE_IDR(rxrpc_client_conn_ids);
+static DEFINE_SPINLOCK(rxrpc_conn_id_lock);
+
+/*
+ * Get a connection ID and epoch for a client connection from the global pool.
+ * The connection struct pointer is then recorded in the idr radix tree.  The
+ * epoch is changed if this wraps.
+ *
+ * TODO: The IDR tree gets very expensive on memory if the connection IDs are
+ * widely scattered throughout the number space, so we shall need to retire
+ * connections that have, say, an ID more than four times the maximum number of
+ * client conns away from the current allocation point to try and keep the IDs
+ * concentrated.  We will also need to retire connections from an old epoch.
+ */
+int rxrpc_get_client_connection_id(struct rxrpc_connection *conn, gfp_t gfp)
+{
+        u32 epoch;
+        int id;
+
+        _enter("");
+
+        idr_preload(gfp);
+        spin_lock(&rxrpc_conn_id_lock);
+
+        epoch = rxrpc_epoch;
+
+        /* We could use idr_alloc_cyclic() here, but we really need to know
+         * when the thing wraps so that we can advance the epoch.
+         */
+        if (rxrpc_client_conn_ids.cur == 0)
+                rxrpc_client_conn_ids.cur = 1;
+        id = idr_alloc(&rxrpc_client_conn_ids, conn,
+                       rxrpc_client_conn_ids.cur, 0x40000000, GFP_NOWAIT);
+        if (id < 0) {
+                if (id != -ENOSPC)
+                        goto error;
+                id = idr_alloc(&rxrpc_client_conn_ids, conn,
+                               1, 0x40000000, GFP_NOWAIT);
+                if (id < 0)
+                        goto error;
+                epoch++;
+                rxrpc_epoch = epoch;
+        }
+        rxrpc_client_conn_ids.cur = id + 1;
+
+        spin_unlock(&rxrpc_conn_id_lock);
+        idr_preload_end();
+
+        conn->proto.epoch = epoch;
+        conn->proto.cid = id << RXRPC_CIDSHIFT;
+        set_bit(RXRPC_CONN_HAS_IDR, &conn->flags);
+        _leave(" [CID %x:%x]", epoch, conn->proto.cid);
+        return 0;
+
+error:
+        spin_unlock(&rxrpc_conn_id_lock);
+        idr_preload_end();
+        _leave(" = %d", id);
+        return id;
+}
+
+/*
+ * Release a connection ID for a client connection from the global pool.
+ */
+void rxrpc_put_client_connection_id(struct rxrpc_connection *conn)
+{
+        if (test_bit(RXRPC_CONN_HAS_IDR, &conn->flags)) {
+                spin_lock(&rxrpc_conn_id_lock);
+                idr_remove(&rxrpc_client_conn_ids,
+                           conn->proto.cid >> RXRPC_CIDSHIFT);
+                spin_unlock(&rxrpc_conn_id_lock);
+        }
+}
diff --git a/net/rxrpc/ar-connevent.c b/net/rxrpc/conn_event.c
index 5f9563968a5b..bf6971555eac 100644
--- a/net/rxrpc/ar-connevent.c
+++ b/net/rxrpc/conn_event.c
@@ -9,6 +9,8 @@
  * 2 of the License, or (at your option) any later version.
  */
 
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
 #include <linux/module.h>
 #include <linux/net.h>
 #include <linux/skbuff.h>
@@ -86,14 +88,14 @@ static int rxrpc_abort_connection(struct rxrpc_connection *conn,
 
         rxrpc_abort_calls(conn, RXRPC_CALL_LOCALLY_ABORTED, abort_code);
 
-        msg.msg_name    = &conn->trans->peer->srx.transport;
-        msg.msg_namelen = conn->trans->peer->srx.transport_len;
+        msg.msg_name    = &conn->params.peer->srx.transport;
+        msg.msg_namelen = conn->params.peer->srx.transport_len;
         msg.msg_control = NULL;
         msg.msg_controllen = 0;
         msg.msg_flags   = 0;
 
-        whdr.epoch      = htonl(conn->epoch);
-        whdr.cid        = htonl(conn->cid);
+        whdr.epoch      = htonl(conn->proto.epoch);
+        whdr.cid        = htonl(conn->proto.cid);
         whdr.callNumber = 0;
         whdr.seq        = 0;
         whdr.type       = RXRPC_PACKET_TYPE_ABORT;
@@ -101,7 +103,7 @@ static int rxrpc_abort_connection(struct rxrpc_connection *conn,
         whdr.userStatus = 0;
         whdr.securityIndex = conn->security_ix;
         whdr._rsvd      = 0;
-        whdr.serviceId  = htons(conn->service_id);
+        whdr.serviceId  = htons(conn->params.service_id);
 
         word            = htonl(conn->local_abort);
 
@@ -116,7 +118,7 @@ static int rxrpc_abort_connection(struct rxrpc_connection *conn,
         whdr.serial = htonl(serial);
         _proto("Tx CONN ABORT %%%u { %d }", serial, conn->local_abort);
 
-        ret = kernel_sendmsg(conn->trans->local->socket, &msg, iov, 2, len);
+        ret = kernel_sendmsg(conn->params.local->socket, &msg, iov, 2, len);
         if (ret < 0) {
                 _debug("sendmsg failed: %d", ret);
                 return -EAGAIN;
@@ -218,7 +220,7 @@ static void rxrpc_secure_connection(struct rxrpc_connection *conn)
 
         ASSERT(conn->security_ix != 0);
 
-        if (!conn->key) {
+        if (!conn->params.key) {
                 _debug("set up security");
                 ret = rxrpc_init_server_conn_security(conn);
                 switch (ret) {
@@ -261,7 +263,7 @@ void rxrpc_process_connection(struct work_struct *work)
 
         _enter("{%d}", conn->debug_id);
 
-        atomic_inc(&conn->usage);
+        rxrpc_get_connection(conn);
 
         if (test_and_clear_bit(RXRPC_CONN_CHALLENGE, &conn->events)) {
                 rxrpc_secure_connection(conn);
@@ -312,19 +314,14 @@ void rxrpc_reject_packet(struct rxrpc_local *local, struct sk_buff *skb)
 {
         CHECK_SLAB_OKAY(&local->usage);
 
-        if (!atomic_inc_not_zero(&local->usage)) {
-                printk("resurrected on reject\n");
-                BUG();
-        }
-
         skb_queue_tail(&local->reject_queue, skb);
-        rxrpc_queue_work(&local->rejecter);
+        rxrpc_queue_work(&local->processor);
 }
 
 /*
  * reject packets through the local endpoint
  */
-void rxrpc_reject_packets(struct work_struct *work)
+void rxrpc_reject_packets(struct rxrpc_local *local)
 {
         union {
                 struct sockaddr sa;
@@ -332,16 +329,12 @@ void rxrpc_reject_packets(struct work_struct *work)
         } sa;
         struct rxrpc_skb_priv *sp;
         struct rxrpc_wire_header whdr;
-        struct rxrpc_local *local;
         struct sk_buff *skb;
         struct msghdr msg;
         struct kvec iov[2];
         size_t size;
         __be32 code;
 
-        local = container_of(work, struct rxrpc_local, rejecter);
-        rxrpc_get_local(local);
-
         _enter("%d", local->debug_id);
 
         iov[0].iov_base = &whdr;
@@ -393,9 +386,7 @@ void rxrpc_reject_packets(struct work_struct *work)
                 }
 
                 rxrpc_free_skb(skb);
-                rxrpc_put_local(local);
         }
 
-        rxrpc_put_local(local);
         _leave("");
 }
diff --git a/net/rxrpc/conn_object.c b/net/rxrpc/conn_object.c
new file mode 100644
index 000000000000..4bfad7cf96cb
--- /dev/null
+++ b/net/rxrpc/conn_object.c
@@ -0,0 +1,686 @@
+/* RxRPC virtual connection handler
+ *
+ * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
+ * Written by David Howells (dhowells@redhat.com)
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/net.h>
+#include <linux/skbuff.h>
+#include <linux/crypto.h>
+#include <net/sock.h>
+#include <net/af_rxrpc.h>
+#include "ar-internal.h"
+
+/*
+ * Time till a connection expires after last use (in seconds).
+ */
+unsigned int rxrpc_connection_expiry = 10 * 60;
+
+static void rxrpc_connection_reaper(struct work_struct *work);
+
+LIST_HEAD(rxrpc_connections);
+DEFINE_RWLOCK(rxrpc_connection_lock);
+static DECLARE_DELAYED_WORK(rxrpc_connection_reap, rxrpc_connection_reaper);
+
+/*
+ * allocate a new connection
+ */
+static struct rxrpc_connection *rxrpc_alloc_connection(gfp_t gfp)
+{
+        struct rxrpc_connection *conn;
+
+        _enter("");
+
+        conn = kzalloc(sizeof(struct rxrpc_connection), gfp);
+        if (conn) {
+                spin_lock_init(&conn->channel_lock);
+                init_waitqueue_head(&conn->channel_wq);
+                INIT_WORK(&conn->processor, &rxrpc_process_connection);
+                INIT_LIST_HEAD(&conn->link);
+                conn->calls = RB_ROOT;
+                skb_queue_head_init(&conn->rx_queue);
+                conn->security = &rxrpc_no_security;
+                rwlock_init(&conn->lock);
+                spin_lock_init(&conn->state_lock);
+                atomic_set(&conn->usage, 1);
+                conn->debug_id = atomic_inc_return(&rxrpc_debug_id);
+                atomic_set(&conn->avail_chans, RXRPC_MAXCALLS);
+                conn->size_align = 4;
+                conn->header_size = sizeof(struct rxrpc_wire_header);
+        }
+
+        _leave(" = %p{%d}", conn, conn ? conn->debug_id : 0);
+        return conn;
+}
+
+/*
+ * add a call to a connection's call-by-ID tree
+ */
+static void rxrpc_add_call_ID_to_conn(struct rxrpc_connection *conn,
+                                      struct rxrpc_call *call)
+{
+        struct rxrpc_call *xcall;
+        struct rb_node *parent, **p;
+        __be32 call_id;
+
+        write_lock_bh(&conn->lock);
+
+        call_id = call->call_id;
+        p = &conn->calls.rb_node;
+        parent = NULL;
+        while (*p) {
+                parent = *p;
+                xcall = rb_entry(parent, struct rxrpc_call, conn_node);
+
+                if (call_id < xcall->call_id)
+                        p = &(*p)->rb_left;
+                else if (call_id > xcall->call_id)
+                        p = &(*p)->rb_right;
+                else
+                        BUG();
+        }
+
+        rb_link_node(&call->conn_node, parent, p);
+        rb_insert_color(&call->conn_node, &conn->calls);
+
+        write_unlock_bh(&conn->lock);
+}
+
+/*
+ * Allocate a client connection.  The caller must take care to clear any
+ * padding bytes in *cp.
+ */
+static struct rxrpc_connection *
+rxrpc_alloc_client_connection(struct rxrpc_conn_parameters *cp, gfp_t gfp)
+{
+        struct rxrpc_connection *conn;
+        int ret;
+
+        _enter("");
+
+        conn = rxrpc_alloc_connection(gfp);
+        if (!conn) {
+                _leave(" = -ENOMEM");
+                return ERR_PTR(-ENOMEM);
+        }
+
+        conn->params            = *cp;
+        conn->proto.local       = cp->local;
+        conn->proto.epoch       = rxrpc_epoch;
+        conn->proto.cid         = 0;
+        conn->proto.in_clientflag = 0;
+        conn->proto.family      = cp->peer->srx.transport.family;
+        conn->out_clientflag    = RXRPC_CLIENT_INITIATED;
+        conn->state             = RXRPC_CONN_CLIENT;
+
+        switch (conn->proto.family) {
+        case AF_INET:
+                conn->proto.addr_size = sizeof(conn->proto.ipv4_addr);
+                conn->proto.ipv4_addr = cp->peer->srx.transport.sin.sin_addr;
+                conn->proto.port = cp->peer->srx.transport.sin.sin_port;
+                break;
+        }
+
+        ret = rxrpc_get_client_connection_id(conn, gfp);
+        if (ret < 0)
+                goto error_0;
+
+        ret = rxrpc_init_client_conn_security(conn);
+        if (ret < 0)
+                goto error_1;
+
+        conn->security->prime_packet_security(conn);
+
+        write_lock(&rxrpc_connection_lock);
+        list_add_tail(&conn->link, &rxrpc_connections);
+        write_unlock(&rxrpc_connection_lock);
+
+        /* We steal the caller's peer ref. */
+        cp->peer = NULL;
+        rxrpc_get_local(conn->params.local);
+        key_get(conn->params.key);
+
+        _leave(" = %p", conn);
+        return conn;
+
+error_1:
+        rxrpc_put_client_connection_id(conn);
+error_0:
+        kfree(conn);
+        _leave(" = %d", ret);
+        return ERR_PTR(ret);
+}
+
+/*
+ * find a connection for a call
+ * - called in process context with IRQs enabled
+ */
+int rxrpc_connect_call(struct rxrpc_call *call,
+                       struct rxrpc_conn_parameters *cp,
+                       struct sockaddr_rxrpc *srx,
+                       gfp_t gfp)
+{
+        struct rxrpc_connection *conn, *candidate = NULL;
+        struct rxrpc_local *local = cp->local;
+        struct rb_node *p, **pp, *parent;
+        long diff;
+        int chan;
+
+        DECLARE_WAITQUEUE(myself, current);
+
+        _enter("{%d,%lx},", call->debug_id, call->user_call_ID);
+
+        cp->peer = rxrpc_lookup_peer(cp->local, srx, gfp);
+        if (!cp->peer)
+                return -ENOMEM;
+
+        if (!cp->exclusive) {
+                /* Search for a existing client connection unless this is going
+                 * to be a connection that's used exclusively for a single call.
+                 */
+                _debug("search 1");
+                spin_lock(&local->client_conns_lock);
+                p = local->client_conns.rb_node;
+                while (p) {
+                        conn = rb_entry(p, struct rxrpc_connection, client_node);
+
+#define cmp(X) ((long)conn->params.X - (long)cp->X)
+                        diff = (cmp(peer) ?:
+                                cmp(key) ?:
+                                cmp(security_level));
+                        if (diff < 0)
+                                p = p->rb_left;
+                        else if (diff > 0)
+                                p = p->rb_right;
+                        else
+                                goto found_extant_conn;
+                }
+                spin_unlock(&local->client_conns_lock);
+        }
+
+        /* We didn't find a connection or we want an exclusive one. */
+        _debug("get new conn");
+        candidate = rxrpc_alloc_client_connection(cp, gfp);
+        if (!candidate) {
+                _leave(" = -ENOMEM");
+                return -ENOMEM;
+        }
+
+        if (cp->exclusive) {
+                /* Assign the call on an exclusive connection to channel 0 and
+                 * don't add the connection to the endpoint's shareable conn
+                 * lookup tree.
+                 */
+                _debug("exclusive chan 0");
+                conn = candidate;
+                atomic_set(&conn->avail_chans, RXRPC_MAXCALLS - 1);
+                spin_lock(&conn->channel_lock);
+                chan = 0;
+                goto found_channel;
+        }
+
+        /* We need to redo the search before attempting to add a new connection
+         * lest we race with someone else adding a conflicting instance.
+         */
+        _debug("search 2");
+        spin_lock(&local->client_conns_lock);
+
+        pp = &local->client_conns.rb_node;
+        parent = NULL;
+        while (*pp) {
+                parent = *pp;
+                conn = rb_entry(parent, struct rxrpc_connection, client_node);
+
+                diff = (cmp(peer) ?:
+                        cmp(key) ?:
+                        cmp(security_level));
+                if (diff < 0)
+                        pp = &(*pp)->rb_left;
+                else if (diff > 0)
+                        pp = &(*pp)->rb_right;
+                else
+                        goto found_extant_conn;
+        }
+
+        /* The second search also failed; simply add the new connection with
+         * the new call in channel 0.  Note that we need to take the channel
+         * lock before dropping the client conn lock.
+         */
+        _debug("new conn");
+        conn = candidate;
+        candidate = NULL;
+
+        rb_link_node(&conn->client_node, parent, pp);
+        rb_insert_color(&conn->client_node, &local->client_conns);
+
+        atomic_set(&conn->avail_chans, RXRPC_MAXCALLS - 1);
+        spin_lock(&conn->channel_lock);
+        spin_unlock(&local->client_conns_lock);
+        chan = 0;
+
+found_channel:
+        _debug("found chan");
+        call->conn      = conn;
+        call->channel   = chan;
+        call->epoch     = conn->proto.epoch;
+        call->cid       = conn->proto.cid | chan;
+        call->call_id   = ++conn->call_counter;
+        rcu_assign_pointer(conn->channels[chan], call);
+
+        _net("CONNECT call %d on conn %d", call->debug_id, conn->debug_id);
+
+        rxrpc_add_call_ID_to_conn(conn, call);
+        spin_unlock(&conn->channel_lock);
+        rxrpc_put_peer(cp->peer);
+        cp->peer = NULL;
+        _leave(" = %p {u=%d}", conn, atomic_read(&conn->usage));
+        return 0;
+
+        /* We found a suitable connection already in existence.  Discard any
+         * candidate we may have allocated, and try to get a channel on this
+         * one.
+         */
+found_extant_conn:
+        _debug("found conn");
+        rxrpc_get_connection(conn);
+        spin_unlock(&local->client_conns_lock);
+
+        rxrpc_put_connection(candidate);
+
+        if (!atomic_add_unless(&conn->avail_chans, -1, 0)) {
+                if (!gfpflags_allow_blocking(gfp)) {
+                        rxrpc_put_connection(conn);
+                        _leave(" = -EAGAIN");
+                        return -EAGAIN;
+                }
+
+                add_wait_queue(&conn->channel_wq, &myself);
+                for (;;) {
+                        set_current_state(TASK_INTERRUPTIBLE);
+                        if (atomic_add_unless(&conn->avail_chans, -1, 0))
+                                break;
+                        if (signal_pending(current))
+                                goto interrupted;
+                        schedule();
+                }
+                remove_wait_queue(&conn->channel_wq, &myself);
+                __set_current_state(TASK_RUNNING);
+        }
+
+        /* The connection allegedly now has a free channel and we can now
+         * attach the call to it.
+         */
+        spin_lock(&conn->channel_lock);
+
+        for (chan = 0; chan < RXRPC_MAXCALLS; chan++)
+                if (!conn->channels[chan])
+                        goto found_channel;
+        BUG();
+
+interrupted:
+        remove_wait_queue(&conn->channel_wq, &myself);
+        __set_current_state(TASK_RUNNING);
+        rxrpc_put_connection(conn);
+        rxrpc_put_peer(cp->peer);
+        cp->peer = NULL;
+        _leave(" = -ERESTARTSYS");
+        return -ERESTARTSYS;
+}
+
+/*
+ * get a record of an incoming connection
+ */
+struct rxrpc_connection *rxrpc_incoming_connection(struct rxrpc_local *local,
+                                                   struct rxrpc_peer *peer,
+                                                   struct sk_buff *skb)
+{
+        struct rxrpc_connection *conn, *candidate = NULL;
+        struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
+        struct rb_node *p, **pp;
+        const char *new = "old";
+        __be32 epoch;
+        u32 cid;
+
+        _enter("");
+
+        ASSERT(sp->hdr.flags & RXRPC_CLIENT_INITIATED);
+
+        epoch = sp->hdr.epoch;
+        cid = sp->hdr.cid & RXRPC_CIDMASK;
+
+        /* search the connection list first */
+        read_lock_bh(&peer->conn_lock);
+
+        p = peer->service_conns.rb_node;
+        while (p) {
+                conn = rb_entry(p, struct rxrpc_connection, service_node);
+
+                _debug("maybe %x", conn->proto.cid);
+
+                if (epoch < conn->proto.epoch)
+                        p = p->rb_left;
+                else if (epoch > conn->proto.epoch)
+                        p = p->rb_right;
+                else if (cid < conn->proto.cid)
+                        p = p->rb_left;
+                else if (cid > conn->proto.cid)
+                        p = p->rb_right;
+                else
+                        goto found_extant_connection;
+        }
+        read_unlock_bh(&peer->conn_lock);
+
+        /* not yet present - create a candidate for a new record and then
+         * redo the search */
+        candidate = rxrpc_alloc_connection(GFP_NOIO);
+        if (!candidate) {
+                _leave(" = -ENOMEM");
+                return ERR_PTR(-ENOMEM);
+        }
+
+        candidate->proto.local          = local;
+        candidate->proto.epoch          = sp->hdr.epoch;
+        candidate->proto.cid            = sp->hdr.cid & RXRPC_CIDMASK;
+        candidate->proto.in_clientflag  = RXRPC_CLIENT_INITIATED;
+        candidate->params.local         = local;
+        candidate->params.peer          = peer;
+        candidate->params.service_id    = sp->hdr.serviceId;
+        candidate->security_ix          = sp->hdr.securityIndex;
+        candidate->out_clientflag       = 0;
+        candidate->state                = RXRPC_CONN_SERVER;
+        if (candidate->params.service_id)
+                candidate->state        = RXRPC_CONN_SERVER_UNSECURED;
+
+        write_lock_bh(&peer->conn_lock);
+
+        pp = &peer->service_conns.rb_node;
+        p = NULL;
+        while (*pp) {
+                p = *pp;
+                conn = rb_entry(p, struct rxrpc_connection, service_node);
+
+                if (epoch < conn->proto.epoch)
+                        pp = &(*pp)->rb_left;
+                else if (epoch > conn->proto.epoch)
+                        pp = &(*pp)->rb_right;
+                else if (cid < conn->proto.cid)
+                        pp = &(*pp)->rb_left;
+                else if (cid > conn->proto.cid)
+                        pp = &(*pp)->rb_right;
+                else
+                        goto found_extant_second;
+        }
+
+        /* we can now add the new candidate to the list */
+        conn = candidate;
+        candidate = NULL;
+        rb_link_node(&conn->service_node, p, pp);
+        rb_insert_color(&conn->service_node, &peer->service_conns);
+        rxrpc_get_peer(peer);
+        rxrpc_get_local(local);
+
+        write_unlock_bh(&peer->conn_lock);
+
+        write_lock(&rxrpc_connection_lock);
+        list_add_tail(&conn->link, &rxrpc_connections);
+        write_unlock(&rxrpc_connection_lock);
+
+        new = "new";
+
+success:
+        _net("CONNECTION %s %d {%x}", new, conn->debug_id, conn->proto.cid);
+
+        _leave(" = %p {u=%d}", conn, atomic_read(&conn->usage));
+        return conn;
+
+        /* we found the connection in the list immediately */
+found_extant_connection:
+        if (sp->hdr.securityIndex != conn->security_ix) {
+                read_unlock_bh(&peer->conn_lock);
+                goto security_mismatch;
+        }
+        rxrpc_get_connection(conn);
+        read_unlock_bh(&peer->conn_lock);
+        goto success;
+
+        /* we found the connection on the second time through the list */
+found_extant_second:
+        if (sp->hdr.securityIndex != conn->security_ix) {
+                write_unlock_bh(&peer->conn_lock);
+                goto security_mismatch;
+        }
+        rxrpc_get_connection(conn);
+        write_unlock_bh(&peer->conn_lock);
+        kfree(candidate);
+        goto success;
+
+security_mismatch:
+        kfree(candidate);
+        _leave(" = -EKEYREJECTED");
+        return ERR_PTR(-EKEYREJECTED);
+}
+
+/*
+ * find a connection based on transport and RxRPC connection ID for an incoming
+ * packet
+ */
+struct rxrpc_connection *rxrpc_find_connection(struct rxrpc_local *local,
+                                               struct rxrpc_peer *peer,
+                                               struct sk_buff *skb)
+{
+        struct rxrpc_connection *conn;
+        struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
+        struct rb_node *p;
+        u32 epoch, cid;
+
+        _enter(",{%x,%x}", sp->hdr.cid, sp->hdr.flags);
+
+        read_lock_bh(&peer->conn_lock);
+
+        cid     = sp->hdr.cid & RXRPC_CIDMASK;
+        epoch   = sp->hdr.epoch;
+
+        if (sp->hdr.flags & RXRPC_CLIENT_INITIATED) {
+                p = peer->service_conns.rb_node;
+                while (p) {
+                        conn = rb_entry(p, struct rxrpc_connection, service_node);
+
+                        _debug("maybe %x", conn->proto.cid);
+
+                        if (epoch < conn->proto.epoch)
+                                p = p->rb_left;
+                        else if (epoch > conn->proto.epoch)
+                                p = p->rb_right;
+                        else if (cid < conn->proto.cid)
+                                p = p->rb_left;
+                        else if (cid > conn->proto.cid)
+                                p = p->rb_right;
+                        else
+                                goto found;
+                }
+        } else {
+                conn = idr_find(&rxrpc_client_conn_ids, cid >> RXRPC_CIDSHIFT);
+                if (conn && conn->proto.epoch == epoch)
+                        goto found;
+        }
+
+        read_unlock_bh(&peer->conn_lock);
+        _leave(" = NULL");
+        return NULL;
+
+found:
+        rxrpc_get_connection(conn);
+        read_unlock_bh(&peer->conn_lock);
+        _leave(" = %p", conn);
+        return conn;
+}
+
+/*
+ * Disconnect a call and clear any channel it occupies when that call
+ * terminates.
+ */
+void rxrpc_disconnect_call(struct rxrpc_call *call)
+{
+        struct rxrpc_connection *conn = call->conn;
+        unsigned chan = call->channel;
+
+        _enter("%d,%d", conn->debug_id, call->channel);
+
+        if (conn->channels[chan] == call) {
+                rcu_assign_pointer(conn->channels[chan], NULL);
+                atomic_inc(&conn->avail_chans);
+                wake_up(&conn->channel_wq);
+        }
+}
+
+/*
+ * release a virtual connection
+ */
+void rxrpc_put_connection(struct rxrpc_connection *conn)
+{
+        if (!conn)
+                return;
+
+        _enter("%p{u=%d,d=%d}",
+               conn, atomic_read(&conn->usage), conn->debug_id);
+
+        ASSERTCMP(atomic_read(&conn->usage), >, 0);
+
+        conn->put_time = ktime_get_seconds();
+        if (atomic_dec_and_test(&conn->usage)) {
+                _debug("zombie");
+                rxrpc_queue_delayed_work(&rxrpc_connection_reap, 0);
+        }
+
+        _leave("");
+}
+
+/*
+ * destroy a virtual connection
+ */
+static void rxrpc_destroy_connection(struct rxrpc_connection *conn)
+{
+        _enter("%p{%d}", conn, atomic_read(&conn->usage));
+
+        ASSERTCMP(atomic_read(&conn->usage), ==, 0);
+
+        _net("DESTROY CONN %d", conn->debug_id);
+
+        ASSERT(RB_EMPTY_ROOT(&conn->calls));
+        rxrpc_purge_queue(&conn->rx_queue);
+
+        conn->security->clear(conn);
+        key_put(conn->params.key);
+        key_put(conn->server_key);
+        rxrpc_put_peer(conn->params.peer);
+        rxrpc_put_local(conn->params.local);
+
+        kfree(conn);
+        _leave("");
+}
+
+/*
+ * reap dead connections
+ */
+static void rxrpc_connection_reaper(struct work_struct *work)
+{
+        struct rxrpc_connection *conn, *_p;
+        struct rxrpc_peer *peer;
+        unsigned long now, earliest, reap_time;
+
+        LIST_HEAD(graveyard);
+
+        _enter("");
+
+        now = ktime_get_seconds();
+        earliest = ULONG_MAX;
+
+        write_lock(&rxrpc_connection_lock);
+        list_for_each_entry_safe(conn, _p, &rxrpc_connections, link) {
+                _debug("reap CONN %d { u=%d,t=%ld }",
+                       conn->debug_id, atomic_read(&conn->usage),
+                       (long) now - (long) conn->put_time);
+
+                if (likely(atomic_read(&conn->usage) > 0))
+                        continue;
+
+                if (rxrpc_conn_is_client(conn)) {
+                        struct rxrpc_local *local = conn->params.local;
+                        spin_lock(&local->client_conns_lock);
+                        reap_time = conn->put_time + rxrpc_connection_expiry;
+
+                        if (atomic_read(&conn->usage) > 0) {
+                                ;
+                        } else if (reap_time <= now) {
+                                list_move_tail(&conn->link, &graveyard);
+                                rxrpc_put_client_connection_id(conn);
+                                rb_erase(&conn->client_node,
+                                         &local->client_conns);
+                        } else if (reap_time < earliest) {
+                                earliest = reap_time;
+                        }
+
+                        spin_unlock(&local->client_conns_lock);
+                } else {
+                        peer = conn->params.peer;
+                        write_lock_bh(&peer->conn_lock);
+                        reap_time = conn->put_time + rxrpc_connection_expiry;
+
+                        if (atomic_read(&conn->usage) > 0) {
+                                ;
+                        } else if (reap_time <= now) {
+                                list_move_tail(&conn->link, &graveyard);
+                                rb_erase(&conn->service_node,
+                                         &peer->service_conns);
+                        } else if (reap_time < earliest) {
+                                earliest = reap_time;
+                        }
+
+                        write_unlock_bh(&peer->conn_lock);
+                }
+        }
+        write_unlock(&rxrpc_connection_lock);
+
+        if (earliest != ULONG_MAX) {
+                _debug("reschedule reaper %ld", (long) earliest - now);
+                ASSERTCMP(earliest, >, now);
+                rxrpc_queue_delayed_work(&rxrpc_connection_reap,
+                                         (earliest - now) * HZ);
+        }
+
+        /* then destroy all those pulled out */
+        while (!list_empty(&graveyard)) {
+                conn = list_entry(graveyard.next, struct rxrpc_connection,
+                                  link);
+                list_del_init(&conn->link);
+
+                ASSERTCMP(atomic_read(&conn->usage), ==, 0);
+                rxrpc_destroy_connection(conn);
+        }
+
+        _leave("");
+}
+
+/*
+ * preemptively destroy all the connection records rather than waiting for them
+ * to time out
+ */
+void __exit rxrpc_destroy_all_connections(void)
+{
+        _enter("");
+
+        rxrpc_connection_expiry = 0;
+        cancel_delayed_work(&rxrpc_connection_reap);
+        rxrpc_queue_delayed_work(&rxrpc_connection_reap, 0);
+
+        _leave("");
+}
diff --git a/net/rxrpc/ar-input.c b/net/rxrpc/input.c
index 6ff97412a0bb..f4bd57b77b93 100644
--- a/net/rxrpc/ar-input.c
+++ b/net/rxrpc/input.c
@@ -9,6 +9,8 @@
  * 2 of the License, or (at your option) any later version.
  */
 
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
 #include <linux/module.h>
 #include <linux/net.h>
 #include <linux/skbuff.h>
@@ -358,7 +360,7 @@ void rxrpc_fast_process_packet(struct rxrpc_call *call, struct sk_buff *skb)
         case RXRPC_PACKET_TYPE_BUSY:
                 _proto("Rx BUSY %%%u", sp->hdr.serial);
 
-                if (call->conn->out_clientflag)
+                if (rxrpc_conn_is_service(call->conn))
                         goto protocol_error;
 
                 write_lock_bh(&call->state_lock);
@@ -531,7 +533,7 @@ static void rxrpc_post_packet_to_call(struct rxrpc_call *call,
         case RXRPC_CALL_COMPLETE:
         case RXRPC_CALL_CLIENT_FINAL_ACK:
                 /* complete server call */
-                if (call->conn->in_clientflag)
+                if (rxrpc_conn_is_service(call->conn))
                         goto dead_call;
                 /* resend last packet of a completed call */
                 _debug("final ack again");
@@ -558,7 +560,7 @@ static void rxrpc_post_packet_to_call(struct rxrpc_call *call,
 dead_call:
         if (sp->hdr.type != RXRPC_PACKET_TYPE_ABORT) {
                 skb->priority = RX_CALL_DEAD;
-                rxrpc_reject_packet(call->conn->trans->local, skb);
+                rxrpc_reject_packet(call->conn->params.local, skb);
                 goto unlock;
         }
 free_unlock:
@@ -578,7 +580,7 @@ static void rxrpc_post_packet_to_conn(struct rxrpc_connection *conn,
 {
         _enter("%p,%p", conn, skb);
 
-        atomic_inc(&conn->usage);
+        rxrpc_get_connection(conn);
         skb_queue_tail(&conn->rx_queue, skb);
         rxrpc_queue_conn(conn);
 }
@@ -592,9 +594,8 @@ static void rxrpc_post_packet_to_local(struct rxrpc_local *local,
 {
         _enter("%p,%p", local, skb);
 
-        atomic_inc(&local->usage);
         skb_queue_tail(&local->event_queue, skb);
-        rxrpc_queue_work(&local->event_processor);
+        rxrpc_queue_work(&local->processor);
 }
 
 /*
@@ -627,29 +628,27 @@ int rxrpc_extract_header(struct rxrpc_skb_priv *sp, struct sk_buff *skb)
 }
 
 static struct rxrpc_connection *rxrpc_conn_from_local(struct rxrpc_local *local,
-                                               struct sk_buff *skb,
-                                               struct rxrpc_skb_priv *sp)
+                                                      struct sk_buff *skb)
 {
         struct rxrpc_peer *peer;
-        struct rxrpc_transport *trans;
         struct rxrpc_connection *conn;
+        struct sockaddr_rxrpc srx;
 
-        peer = rxrpc_find_peer(local, ip_hdr(skb)->saddr,
-                                udp_hdr(skb)->source);
-        if (IS_ERR(peer))
-                goto cant_find_conn;
-
-        trans = rxrpc_find_transport(local, peer);
-        rxrpc_put_peer(peer);
-        if (!trans)
-                goto cant_find_conn;
+        rxrpc_get_addr_from_skb(local, skb, &srx);
+        rcu_read_lock();
+        peer = rxrpc_lookup_peer_rcu(local, &srx);
+        if (!peer)
+                goto cant_find_peer;
 
-        conn = rxrpc_find_connection(trans, &sp->hdr);
-        rxrpc_put_transport(trans);
+        conn = rxrpc_find_connection(local, peer, skb);
+        rcu_read_unlock();
         if (!conn)
                 goto cant_find_conn;
 
         return conn;
+
+cant_find_peer:
+        rcu_read_unlock();
 cant_find_conn:
         return NULL;
 }
@@ -657,11 +656,15 @@ cant_find_conn:
 /*
  * handle data received on the local endpoint
  * - may be called in interrupt context
+ *
+ * The socket is locked by the caller and this prevents the socket from being
+ * shut down and the local endpoint from going away, thus sk_user_data will not
+ * be cleared until this function returns.
  */
 void rxrpc_data_ready(struct sock *sk)
 {
         struct rxrpc_skb_priv *sp;
-        struct rxrpc_local *local;
+        struct rxrpc_local *local = sk->sk_user_data;
         struct sk_buff *skb;
         int ret;
 
@@ -669,21 +672,8 @@ void rxrpc_data_ready(struct sock *sk)
 
         ASSERT(!irqs_disabled());
 
-        read_lock_bh(&rxrpc_local_lock);
-        local = sk->sk_user_data;
-        if (local && atomic_read(&local->usage) > 0)
-                rxrpc_get_local(local);
-        else
-                local = NULL;
-        read_unlock_bh(&rxrpc_local_lock);
-        if (!local) {
-                _leave(" [local dead]");
-                return;
-        }
-
         skb = skb_recv_datagram(sk, 0, 1, &ret);
         if (!skb) {
-                rxrpc_put_local(local);
                 if (ret == -EAGAIN)
                         return;
                 _debug("UDP socket error %d", ret);
@@ -697,7 +687,6 @@ void rxrpc_data_ready(struct sock *sk)
         /* we'll probably need to checksum it (didn't call sock_recvmsg) */
         if (skb_checksum_complete(skb)) {
                 rxrpc_free_skb(skb);
-                rxrpc_put_local(local);
                 __UDP_INC_STATS(&init_net, UDP_MIB_INERRORS, 0);
                 _leave(" [CSUM failed]");
                 return;
@@ -732,7 +721,7 @@ void rxrpc_data_ready(struct sock *sk)
                 rxrpc_post_packet_to_local(local, skb);
                 goto out;
         }
-        
+
         if (sp->hdr.type == RXRPC_PACKET_TYPE_DATA &&
             (sp->hdr.callNumber == 0 || sp->hdr.seq == 0))
                 goto bad_message;
@@ -743,7 +732,7 @@ void rxrpc_data_ready(struct sock *sk)
                  * old-fashioned way doesn't really hurt */
                 struct rxrpc_connection *conn;
 
-                conn = rxrpc_conn_from_local(local, skb, sp);
+                conn = rxrpc_conn_from_local(local, skb);
                 if (!conn)
                         goto cant_route_call;
 
@@ -762,7 +751,6 @@ void rxrpc_data_ready(struct sock *sk)
         }
 
 out:
-        rxrpc_put_local(local);
         return;
 
 cant_route_call:
@@ -772,8 +760,7 @@ cant_route_call:
                 if (sp->hdr.seq == 1) {
                         _debug("first packet");
                         skb_queue_tail(&local->accept_queue, skb);
-                        rxrpc_queue_work(&local->acceptor);
-                        rxrpc_put_local(local);
+                        rxrpc_queue_work(&local->processor);
                         _leave(" [incoming]");
                         return;
                 }
@@ -786,13 +773,11 @@ cant_route_call:
                 _debug("reject type %d",sp->hdr.type);
                 rxrpc_reject_packet(local, skb);
         }
-        rxrpc_put_local(local);
         _leave(" [no call]");
         return;
 
 bad_message:
         skb->priority = RX_PROTOCOL_ERROR;
         rxrpc_reject_packet(local, skb);
-        rxrpc_put_local(local);
         _leave(" [badmsg]");
 }
diff --git a/net/rxrpc/ar-key.c b/net/rxrpc/key.c
index 1021b4c0bdd2..18c737a61d80 100644
--- a/net/rxrpc/ar-key.c
+++ b/net/rxrpc/key.c
@@ -12,6 +12,8 @@
  *      "afs@CAMBRIDGE.REDHAT.COM>
  */
 
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
 #include <crypto/skcipher.h>
 #include <linux/module.h>
 #include <linux/net.h>
@@ -800,7 +802,7 @@ static void rxrpc_free_token_list(struct rxrpc_key_token *token)
                                 rxrpc_rxk5_free(token->k5);
                         break;
                 default:
-                        printk(KERN_ERR "Unknown token type %x on rxrpc key\n",
+                        pr_err("Unknown token type %x on rxrpc key\n",
                                token->security_index);
                         BUG();
                 }
@@ -985,7 +987,7 @@ int rxrpc_get_server_data_key(struct rxrpc_connection *conn,
         if (ret < 0)
                 goto error;
 
-        conn->key = key;
+        conn->params.key = key;
         _leave(" = 0 [%d]", key_serial(key));
         return 0;
 
diff --git a/net/rxrpc/local_event.c b/net/rxrpc/local_event.c
new file mode 100644
index 000000000000..31a3f86ef2f6
--- /dev/null
+++ b/net/rxrpc/local_event.c
@@ -0,0 +1,116 @@
+/* AF_RXRPC local endpoint management
+ *
+ * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
+ * Written by David Howells (dhowells@redhat.com)
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/module.h>
+#include <linux/net.h>
+#include <linux/skbuff.h>
+#include <linux/slab.h>
+#include <linux/udp.h>
+#include <linux/ip.h>
+#include <net/sock.h>
+#include <net/af_rxrpc.h>
+#include <generated/utsrelease.h>
+#include "ar-internal.h"
+
+static const char rxrpc_version_string[65] = "linux-" UTS_RELEASE " AF_RXRPC";
+
+/*
+ * Reply to a version request
+ */
+static void rxrpc_send_version_request(struct rxrpc_local *local,
+                                       struct rxrpc_host_header *hdr,
+                                       struct sk_buff *skb)
+{
+        struct rxrpc_wire_header whdr;
+        struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
+        struct sockaddr_in sin;
+        struct msghdr msg;
+        struct kvec iov[2];
+        size_t len;
+        int ret;
+
+        _enter("");
+
+        sin.sin_family = AF_INET;
+        sin.sin_port = udp_hdr(skb)->source;
+        sin.sin_addr.s_addr = ip_hdr(skb)->saddr;
+
+        msg.msg_name    = &sin;
+        msg.msg_namelen = sizeof(sin);
+        msg.msg_control = NULL;
+        msg.msg_controllen = 0;
+        msg.msg_flags   = 0;
+
+        whdr.epoch      = htonl(sp->hdr.epoch);
+        whdr.cid        = htonl(sp->hdr.cid);
+        whdr.callNumber = htonl(sp->hdr.callNumber);
+        whdr.seq        = 0;
+        whdr.serial     = 0;
+        whdr.type       = RXRPC_PACKET_TYPE_VERSION;
+        whdr.flags      = RXRPC_LAST_PACKET | (~hdr->flags & RXRPC_CLIENT_INITIATED);
+        whdr.userStatus = 0;
+        whdr.securityIndex = 0;
+        whdr._rsvd      = 0;
+        whdr.serviceId  = htons(sp->hdr.serviceId);
+
+        iov[0].iov_base = &whdr;
+        iov[0].iov_len  = sizeof(whdr);
+        iov[1].iov_base = (char *)rxrpc_version_string;
+        iov[1].iov_len  = sizeof(rxrpc_version_string);
+
+        len = iov[0].iov_len + iov[1].iov_len;
+
+        _proto("Tx VERSION (reply)");
+
+        ret = kernel_sendmsg(local->socket, &msg, iov, 2, len);
+        if (ret < 0)
+                _debug("sendmsg failed: %d", ret);
+
+        _leave("");
+}
+
+/*
+ * Process event packets targetted at a local endpoint.
+ */
+void rxrpc_process_local_events(struct rxrpc_local *local)
+{
+        struct sk_buff *skb;
+        char v;
+
+        _enter("");
+
+        skb = skb_dequeue(&local->event_queue);
+        if (skb) {
+                struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
+
+                _debug("{%d},{%u}", local->debug_id, sp->hdr.type);
+
+                switch (sp->hdr.type) {
+                case RXRPC_PACKET_TYPE_VERSION:
+                        if (skb_copy_bits(skb, 0, &v, 1) < 0)
+                                return;
+                        _proto("Rx VERSION { %02x }", v);
+                        if (v == 0)
+                                rxrpc_send_version_request(local, &sp->hdr, skb);
+                        break;
+
+                default:
+                        /* Just ignore anything we don't understand */
+                        break;
+                }
+
+                rxrpc_free_skb(skb);
+        }
+
+        _leave("");
+}
diff --git a/net/rxrpc/local_object.c b/net/rxrpc/local_object.c
new file mode 100644
index 000000000000..3ab7764f7cd8
--- /dev/null
+++ b/net/rxrpc/local_object.c
@@ -0,0 +1,387 @@
+/* Local endpoint object management
+ *
+ * Copyright (C) 2016 Red Hat, Inc. All Rights Reserved.
+ * Written by David Howells (dhowells@redhat.com)
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public Licence
+ * as published by the Free Software Foundation; either version
+ * 2 of the Licence, or (at your option) any later version.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/module.h>
+#include <linux/net.h>
+#include <linux/skbuff.h>
+#include <linux/slab.h>
+#include <linux/udp.h>
+#include <linux/ip.h>
+#include <linux/hashtable.h>
+#include <net/sock.h>
+#include <net/af_rxrpc.h>
+#include "ar-internal.h"
+
+static void rxrpc_local_processor(struct work_struct *);
+static void rxrpc_local_rcu(struct rcu_head *);
+
+static DEFINE_MUTEX(rxrpc_local_mutex);
+static LIST_HEAD(rxrpc_local_endpoints);
+
+/*
+ * Compare a local to an address.  Return -ve, 0 or +ve to indicate less than,
+ * same or greater than.
+ *
+ * We explicitly don't compare the RxRPC service ID as we want to reject
+ * conflicting uses by differing services.  Further, we don't want to share
+ * addresses with different options (IPv6), so we don't compare those bits
+ * either.
+ */
+static long rxrpc_local_cmp_key(const struct rxrpc_local *local,
+                                const struct sockaddr_rxrpc *srx)
+{
+        long diff;
+
+        diff = ((local->srx.transport_type - srx->transport_type) ?:
+                (local->srx.transport_len - srx->transport_len) ?:
+                (local->srx.transport.family - srx->transport.family));
+        if (diff != 0)
+                return diff;
+
+        switch (srx->transport.family) {
+        case AF_INET:
+                /* If the choice of UDP port is left up to the transport, then
+                 * the endpoint record doesn't match.
+                 */
+                return ((u16 __force)local->srx.transport.sin.sin_port -
+                        (u16 __force)srx->transport.sin.sin_port) ?:
+                        memcmp(&local->srx.transport.sin.sin_addr,
+                               &srx->transport.sin.sin_addr,
+                               sizeof(struct in_addr));
+        default:
+                BUG();
+        }
+}
+
+/*
+ * Allocate a new local endpoint.
+ */
+static struct rxrpc_local *rxrpc_alloc_local(const struct sockaddr_rxrpc *srx)
+{
+        struct rxrpc_local *local;
+
+        local = kzalloc(sizeof(struct rxrpc_local), GFP_KERNEL);
+        if (local) {
+                atomic_set(&local->usage, 1);
+                INIT_LIST_HEAD(&local->link);
+                INIT_WORK(&local->processor, rxrpc_local_processor);
+                INIT_LIST_HEAD(&local->services);
+                init_rwsem(&local->defrag_sem);
+                skb_queue_head_init(&local->accept_queue);
+                skb_queue_head_init(&local->reject_queue);
+                skb_queue_head_init(&local->event_queue);
+                local->client_conns = RB_ROOT;
+                spin_lock_init(&local->client_conns_lock);
+                spin_lock_init(&local->lock);
+                rwlock_init(&local->services_lock);
+                local->debug_id = atomic_inc_return(&rxrpc_debug_id);
+                memcpy(&local->srx, srx, sizeof(*srx));
+        }
+
+        _leave(" = %p", local);
+        return local;
+}
+
+/*
+ * create the local socket
+ * - must be called with rxrpc_local_mutex locked
+ */
+static int rxrpc_open_socket(struct rxrpc_local *local)
+{
+        struct sock *sock;
+        int ret, opt;
+
+        _enter("%p{%d}", local, local->srx.transport_type);
+
+        /* create a socket to represent the local endpoint */
+        ret = sock_create_kern(&init_net, PF_INET, local->srx.transport_type,
+                               IPPROTO_UDP, &local->socket);
+        if (ret < 0) {
+                _leave(" = %d [socket]", ret);
+                return ret;
+        }
+
+        /* if a local address was supplied then bind it */
+        if (local->srx.transport_len > sizeof(sa_family_t)) {
+                _debug("bind");
+                ret = kernel_bind(local->socket,
+                                  (struct sockaddr *)&local->srx.transport,
+                                  local->srx.transport_len);
+                if (ret < 0) {
+                        _debug("bind failed %d", ret);
+                        goto error;
+                }
+        }
+
+        /* we want to receive ICMP errors */
+        opt = 1;
+        ret = kernel_setsockopt(local->socket, SOL_IP, IP_RECVERR,
+                                (char *) &opt, sizeof(opt));
+        if (ret < 0) {
+                _debug("setsockopt failed");
+                goto error;
+        }
+
+        /* we want to set the don't fragment bit */
+        opt = IP_PMTUDISC_DO;
+        ret = kernel_setsockopt(local->socket, SOL_IP, IP_MTU_DISCOVER,
+                                (char *) &opt, sizeof(opt));
+        if (ret < 0) {
+                _debug("setsockopt failed");
+                goto error;
+        }
+
+        /* set the socket up */
+        sock = local->socket->sk;
+        sock->sk_user_data      = local;
+        sock->sk_data_ready     = rxrpc_data_ready;
+        sock->sk_error_report   = rxrpc_error_report;
+        _leave(" = 0");
+        return 0;
+
+error:
+        kernel_sock_shutdown(local->socket, SHUT_RDWR);
+        local->socket->sk->sk_user_data = NULL;
+        sock_release(local->socket);
+        local->socket = NULL;
+
+        _leave(" = %d", ret);
+        return ret;
+}
+
+/*
+ * Look up or create a new local endpoint using the specified local address.
+ */
+struct rxrpc_local *rxrpc_lookup_local(const struct sockaddr_rxrpc *srx)
+{
+        struct rxrpc_local *local;
+        struct list_head *cursor;
+        const char *age;
+        long diff;
+        int ret;
+
+        if (srx->transport.family == AF_INET) {
+                _enter("{%d,%u,%pI4+%hu}",
+                       srx->transport_type,
+                       srx->transport.family,
+                       &srx->transport.sin.sin_addr,
+                       ntohs(srx->transport.sin.sin_port));
+        } else {
+                _enter("{%d,%u}",
+                       srx->transport_type,
+                       srx->transport.family);
+                return ERR_PTR(-EAFNOSUPPORT);
+        }
+
+        mutex_lock(&rxrpc_local_mutex);
+
+        for (cursor = rxrpc_local_endpoints.next;
+             cursor != &rxrpc_local_endpoints;
+             cursor = cursor->next) {
+                local = list_entry(cursor, struct rxrpc_local, link);
+
+                diff = rxrpc_local_cmp_key(local, srx);
+                if (diff < 0)
+                        continue;
+                if (diff > 0)
+                        break;
+
+                /* Services aren't allowed to share transport sockets, so
+                 * reject that here.  It is possible that the object is dying -
+                 * but it may also still have the local transport address that
+                 * we want bound.
+                 */
+                if (srx->srx_service) {
+                        local = NULL;
+                        goto addr_in_use;
+                }
+
+                /* Found a match.  We replace a dying object.  Attempting to
+                 * bind the transport socket may still fail if we're attempting
+                 * to use a local address that the dying object is still using.
+                 */
+                if (!rxrpc_get_local_maybe(local)) {
+                        cursor = cursor->next;
+                        list_del_init(&local->link);
+                        break;
+                }
+
+                age = "old";
+                goto found;
+        }
+
+        local = rxrpc_alloc_local(srx);
+        if (!local)
+                goto nomem;
+
+        ret = rxrpc_open_socket(local);
+        if (ret < 0)
+                goto sock_error;
+
+        list_add_tail(&local->link, cursor);
+        age = "new";
+
+found:
+        mutex_unlock(&rxrpc_local_mutex);
+
+        _net("LOCAL %s %d {%d,%u,%pI4+%hu}",
+             age,
+             local->debug_id,
+             local->srx.transport_type,
+             local->srx.transport.family,
+             &local->srx.transport.sin.sin_addr,
+             ntohs(local->srx.transport.sin.sin_port));
+
+        _leave(" = %p", local);
+        return local;
+
+nomem:
+        ret = -ENOMEM;
+sock_error:
+        mutex_unlock(&rxrpc_local_mutex);
+        kfree(local);
+        _leave(" = %d", ret);
+        return ERR_PTR(ret);
+
+addr_in_use:
+        mutex_unlock(&rxrpc_local_mutex);
+        _leave(" = -EADDRINUSE");
+        return ERR_PTR(-EADDRINUSE);
+}
+
+/*
+ * A local endpoint reached its end of life.
+ */
+void __rxrpc_put_local(struct rxrpc_local *local)
+{
+        _enter("%d", local->debug_id);
+        rxrpc_queue_work(&local->processor);
+}
+
+/*
+ * Destroy a local endpoint's socket and then hand the record to RCU to dispose
+ * of.
+ *
+ * Closing the socket cannot be done from bottom half context or RCU callback
+ * context because it might sleep.
+ */
+static void rxrpc_local_destroyer(struct rxrpc_local *local)
+{
+        struct socket *socket = local->socket;
+
+        _enter("%d", local->debug_id);
+
+        /* We can get a race between an incoming call packet queueing the
+         * processor again and the work processor starting the destruction
+         * process which will shut down the UDP socket.
+         */
+        if (local->dead) {
+                _leave(" [already dead]");
+                return;
+        }
+        local->dead = true;
+
+        mutex_lock(&rxrpc_local_mutex);
+        list_del_init(&local->link);
+        mutex_unlock(&rxrpc_local_mutex);
+
+        ASSERT(RB_EMPTY_ROOT(&local->client_conns));
+        ASSERT(list_empty(&local->services));
+
+        if (socket) {
+                local->socket = NULL;
+                kernel_sock_shutdown(socket, SHUT_RDWR);
+                socket->sk->sk_user_data = NULL;
+                sock_release(socket);
+        }
+
+        /* At this point, there should be no more packets coming in to the
+         * local endpoint.
+         */
+        rxrpc_purge_queue(&local->accept_queue);
+        rxrpc_purge_queue(&local->reject_queue);
+        rxrpc_purge_queue(&local->event_queue);
+
+        _debug("rcu local %d", local->debug_id);
+        call_rcu(&local->rcu, rxrpc_local_rcu);
+}
+
+/*
+ * Process events on an endpoint
+ */
+static void rxrpc_local_processor(struct work_struct *work)
+{
+        struct rxrpc_local *local =
+                container_of(work, struct rxrpc_local, processor);
+        bool again;
+
+        _enter("%d", local->debug_id);
+
+        do {
+                again = false;
+                if (atomic_read(&local->usage) == 0)
+                        return rxrpc_local_destroyer(local);
+
+                if (!skb_queue_empty(&local->accept_queue)) {
+                        rxrpc_accept_incoming_calls(local);
+                        again = true;
+                }
+
+                if (!skb_queue_empty(&local->reject_queue)) {
+                        rxrpc_reject_packets(local);
+                        again = true;
+                }
+
+                if (!skb_queue_empty(&local->event_queue)) {
+                        rxrpc_process_local_events(local);
+                        again = true;
+                }
+        } while (again);
+}
+
+/*
+ * Destroy a local endpoint after the RCU grace period expires.
+ */
+static void rxrpc_local_rcu(struct rcu_head *rcu)
+{
+        struct rxrpc_local *local = container_of(rcu, struct rxrpc_local, rcu);
+
+        _enter("%d", local->debug_id);
+
+        ASSERT(!work_pending(&local->processor));
+
+        _net("DESTROY LOCAL %d", local->debug_id);
+        kfree(local);
+        _leave("");
+}
+
+/*
+ * Verify the local endpoint list is empty by this point.
+ */
+void __exit rxrpc_destroy_all_locals(void)
+{
+        struct rxrpc_local *local;
+
+        _enter("");
+
+        if (list_empty(&rxrpc_local_endpoints))
+                return;
+
+        mutex_lock(&rxrpc_local_mutex);
+        list_for_each_entry(local, &rxrpc_local_endpoints, link) {
+                pr_err("AF_RXRPC: Leaked local %p {%d}\n",
+                       local, atomic_read(&local->usage));
+        }
+        mutex_unlock(&rxrpc_local_mutex);
+        BUG();
+}
diff --git a/net/rxrpc/misc.c b/net/rxrpc/misc.c
index 1afe9876e79f..bdc5e42fe600 100644
--- a/net/rxrpc/misc.c
+++ b/net/rxrpc/misc.c
@@ -15,6 +15,12 @@
 #include "ar-internal.h"
 
 /*
+ * The maximum listening backlog queue size that may be set on a socket by
+ * listen().
+ */
+unsigned int rxrpc_max_backlog __read_mostly = 10;
+
+/*
  * How long to wait before scheduling ACK generation after seeing a
  * packet with RXRPC_REQUEST_ACK set (in jiffies).
  */
diff --git a/net/rxrpc/ar-output.c b/net/rxrpc/output.c
index 51cb10062a8d..f4bda06b7d2d 100644
--- a/net/rxrpc/ar-output.c
+++ b/net/rxrpc/output.c
@@ -9,6 +9,8 @@
  * 2 of the License, or (at your option) any later version.
  */
 
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
 #include <linux/net.h>
 #include <linux/gfp.h>
 #include <linux/skbuff.h>
@@ -30,13 +32,14 @@ static int rxrpc_send_data(struct rxrpc_sock *rx,
 /*
  * extract control messages from the sendmsg() control buffer
  */
-static int rxrpc_sendmsg_cmsg(struct rxrpc_sock *rx, struct msghdr *msg,
+static int rxrpc_sendmsg_cmsg(struct msghdr *msg,
                               unsigned long *user_call_ID,
                               enum rxrpc_command *command,
                               u32 *abort_code,
-                              bool server)
+                              bool *_exclusive)
 {
         struct cmsghdr *cmsg;
+        bool got_user_ID = false;
         int len;
 
         *command = RXRPC_CMD_SEND_DATA;
@@ -68,6 +71,7 @@ static int rxrpc_sendmsg_cmsg(struct rxrpc_sock *rx, struct msghdr *msg,
                                         CMSG_DATA(cmsg);
                         }
                         _debug("User Call ID %lx", *user_call_ID);
+                        got_user_ID = true;
                         break;
 
                 case RXRPC_ABORT:
@@ -88,15 +92,20 @@ static int rxrpc_sendmsg_cmsg(struct rxrpc_sock *rx, struct msghdr *msg,
                         *command = RXRPC_CMD_ACCEPT;
                         if (len != 0)
                                 return -EINVAL;
-                        if (!server)
-                                return -EISCONN;
                         break;
 
+                case RXRPC_EXCLUSIVE_CALL:
+                        *_exclusive = true;
+                        if (len != 0)
+                                return -EINVAL;
+                        break;
                 default:
                         return -EINVAL;
                 }
         }
 
+        if (!got_user_ID)
+                return -EINVAL;
         _leave(" = 0");
         return 0;
 }
@@ -124,55 +133,78 @@ static void rxrpc_send_abort(struct rxrpc_call *call, u32 abort_code)
 }
 
 /*
+ * Create a new client call for sendmsg().
+ */
+static struct rxrpc_call *
+rxrpc_new_client_call_for_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg,
+                                  unsigned long user_call_ID, bool exclusive)
+{
+        struct rxrpc_conn_parameters cp;
+        struct rxrpc_call *call;
+        struct key *key;
+
+        DECLARE_SOCKADDR(struct sockaddr_rxrpc *, srx, msg->msg_name);
+
+        _enter("");
+
+        if (!msg->msg_name)
+                return ERR_PTR(-EDESTADDRREQ);
+
+        key = rx->key;
+        if (key && !rx->key->payload.data[0])
+                key = NULL;
+
+        memset(&cp, 0, sizeof(cp));
+        cp.local                = rx->local;
+        cp.key                  = rx->key;
+        cp.security_level       = rx->min_sec_level;
+        cp.exclusive            = rx->exclusive | exclusive;
+        cp.service_id           = srx->srx_service;
+        call = rxrpc_new_client_call(rx, &cp, srx, user_call_ID, GFP_KERNEL);
+
+        _leave(" = %p\n", call);
+        return call;
+}
+
+/*
  * send a message forming part of a client call through an RxRPC socket
  * - caller holds the socket locked
  * - the socket may be either a client socket or a server socket
  */
-int rxrpc_client_sendmsg(struct rxrpc_sock *rx, struct rxrpc_transport *trans,
-                         struct msghdr *msg, size_t len)
+int rxrpc_do_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg, size_t len)
 {
-        struct rxrpc_conn_bundle *bundle;
         enum rxrpc_command cmd;
         struct rxrpc_call *call;
         unsigned long user_call_ID = 0;
-        struct key *key;
-        u16 service_id;
+        bool exclusive = false;
         u32 abort_code = 0;
         int ret;
 
         _enter("");
 
-        ASSERT(trans != NULL);
-
-        ret = rxrpc_sendmsg_cmsg(rx, msg, &user_call_ID, &cmd, &abort_code,
-                                 false);
+        ret = rxrpc_sendmsg_cmsg(msg, &user_call_ID, &cmd, &abort_code,
+                                 &exclusive);
         if (ret < 0)
                 return ret;
 
-        bundle = NULL;
-        if (trans) {
-                service_id = rx->srx.srx_service;
-                if (msg->msg_name) {
-                        DECLARE_SOCKADDR(struct sockaddr_rxrpc *, srx,
-                                         msg->msg_name);
-                        service_id = srx->srx_service;
-                }
-                key = rx->key;
-                if (key && !rx->key->payload.data[0])
-                        key = NULL;
-                bundle = rxrpc_get_bundle(rx, trans, key, service_id,
-                                          GFP_KERNEL);
-                if (IS_ERR(bundle))
-                        return PTR_ERR(bundle);
+        if (cmd == RXRPC_CMD_ACCEPT) {
+                if (rx->sk.sk_state != RXRPC_SERVER_LISTENING)
+                        return -EINVAL;
+                call = rxrpc_accept_call(rx, user_call_ID);
+                if (IS_ERR(call))
+                        return PTR_ERR(call);
+                rxrpc_put_call(call);
+                return 0;
         }
 
-        call = rxrpc_get_client_call(rx, trans, bundle, user_call_ID,
-                                     abort_code == 0, GFP_KERNEL);
-        if (trans)
-                rxrpc_put_bundle(trans, bundle);
-        if (IS_ERR(call)) {
-                _leave(" = %ld", PTR_ERR(call));
-                return PTR_ERR(call);
+        call = rxrpc_find_call_by_user_ID(rx, user_call_ID);
+        if (!call) {
+                if (cmd != RXRPC_CMD_SEND_DATA)
+                        return -EBADSLT;
+                call = rxrpc_new_client_call_for_sendmsg(rx, msg, user_call_ID,
+                                                         exclusive);
+                if (IS_ERR(call))
+                        return PTR_ERR(call);
         }
 
         _debug("CALL %d USR %lx ST %d on CONN %p",
@@ -180,14 +212,21 @@ int rxrpc_client_sendmsg(struct rxrpc_sock *rx, struct rxrpc_transport *trans,
 
         if (call->state >= RXRPC_CALL_COMPLETE) {
                 /* it's too late for this call */
-                ret = -ESHUTDOWN;
+                ret = -ECONNRESET;
         } else if (cmd == RXRPC_CMD_SEND_ABORT) {
                 rxrpc_send_abort(call, abort_code);
+                ret = 0;
         } else if (cmd != RXRPC_CMD_SEND_DATA) {
                 ret = -EINVAL;
-        } else if (call->state != RXRPC_CALL_CLIENT_SEND_REQUEST) {
+        } else if (!call->in_clientflag &&
+                   call->state != RXRPC_CALL_CLIENT_SEND_REQUEST) {
                 /* request phase complete for this client call */
                 ret = -EPROTO;
+        } else if (call->in_clientflag &&
+                   call->state != RXRPC_CALL_SERVER_ACK_REQUEST &&
+                   call->state != RXRPC_CALL_SERVER_SEND_REPLY) {
+                /* Reply phase not begun or not complete for service call. */
+                ret = -EPROTO;
         } else {
                 ret = rxrpc_send_data(rx, call, msg, len);
         }
@@ -266,70 +305,9 @@ void rxrpc_kernel_abort_call(struct rxrpc_call *call, u32 abort_code)
 EXPORT_SYMBOL(rxrpc_kernel_abort_call);
 
 /*
- * send a message through a server socket
- * - caller holds the socket locked
- */
-int rxrpc_server_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg, size_t len)
-{
-        enum rxrpc_command cmd;
-        struct rxrpc_call *call;
-        unsigned long user_call_ID = 0;
-        u32 abort_code = 0;
-        int ret;
-
-        _enter("");
-
-        ret = rxrpc_sendmsg_cmsg(rx, msg, &user_call_ID, &cmd, &abort_code,
-                                 true);
-        if (ret < 0)
-                return ret;
-
-        if (cmd == RXRPC_CMD_ACCEPT) {
-                call = rxrpc_accept_call(rx, user_call_ID);
-                if (IS_ERR(call))
-                        return PTR_ERR(call);
-                rxrpc_put_call(call);
-                return 0;
-        }
-
-        call = rxrpc_find_server_call(rx, user_call_ID);
-        if (!call)
-                return -EBADSLT;
-        if (call->state >= RXRPC_CALL_COMPLETE) {
-                ret = -ESHUTDOWN;
-                goto out;
-        }
-
-        switch (cmd) {
-        case RXRPC_CMD_SEND_DATA:
-                if (call->state != RXRPC_CALL_CLIENT_SEND_REQUEST &&
-                    call->state != RXRPC_CALL_SERVER_ACK_REQUEST &&
-                    call->state != RXRPC_CALL_SERVER_SEND_REPLY) {
-                        /* Tx phase not yet begun for this call */
-                        ret = -EPROTO;
-                        break;
-                }
-
-                ret = rxrpc_send_data(rx, call, msg, len);
-                break;
-
-        case RXRPC_CMD_SEND_ABORT:
-                rxrpc_send_abort(call, abort_code);
-                break;
-        default:
-                BUG();
-        }
-
-        out:
-        rxrpc_put_call(call);
-        _leave(" = %d", ret);
-        return ret;
-}
-
-/*
  * send a packet through the transport endpoint
  */
-int rxrpc_send_packet(struct rxrpc_transport *trans, struct sk_buff *skb)
+int rxrpc_send_data_packet(struct rxrpc_connection *conn, struct sk_buff *skb)
 {
         struct kvec iov[1];
         struct msghdr msg;
@@ -340,30 +318,30 @@ int rxrpc_send_packet(struct rxrpc_transport *trans, struct sk_buff *skb)
         iov[0].iov_base = skb->head;
         iov[0].iov_len = skb->len;
 
-        msg.msg_name = &trans->peer->srx.transport.sin;
-        msg.msg_namelen = sizeof(trans->peer->srx.transport.sin);
+        msg.msg_name = &conn->params.peer->srx.transport;
+        msg.msg_namelen = conn->params.peer->srx.transport_len;
         msg.msg_control = NULL;
         msg.msg_controllen = 0;
         msg.msg_flags = 0;
 
         /* send the packet with the don't fragment bit set if we currently
          * think it's small enough */
-        if (skb->len - sizeof(struct rxrpc_wire_header) < trans->peer->maxdata) {
-                down_read(&trans->local->defrag_sem);
+        if (skb->len - sizeof(struct rxrpc_wire_header) < conn->params.peer->maxdata) {
+                down_read(&conn->params.local->defrag_sem);
                 /* send the packet by UDP
                  * - returns -EMSGSIZE if UDP would have to fragment the packet
                  *   to go out of the interface
                  *   - in which case, we'll have processed the ICMP error
                  *     message and update the peer record
                  */
-                ret = kernel_sendmsg(trans->local->socket, &msg, iov, 1,
+                ret = kernel_sendmsg(conn->params.local->socket, &msg, iov, 1,
                                      iov[0].iov_len);
 
-                up_read(&trans->local->defrag_sem);
+                up_read(&conn->params.local->defrag_sem);
                 if (ret == -EMSGSIZE)
                         goto send_fragmentable;
 
-                _leave(" = %d [%u]", ret, trans->peer->maxdata);
+                _leave(" = %d [%u]", ret, conn->params.peer->maxdata);
                 return ret;
         }
 
@@ -371,21 +349,28 @@ send_fragmentable:
         /* attempt to send this message with fragmentation enabled */
         _debug("send fragment");
 
-        down_write(&trans->local->defrag_sem);
-        opt = IP_PMTUDISC_DONT;
-        ret = kernel_setsockopt(trans->local->socket, SOL_IP, IP_MTU_DISCOVER,
-                                (char *) &opt, sizeof(opt));
-        if (ret == 0) {
-                ret = kernel_sendmsg(trans->local->socket, &msg, iov, 1,
-                                     iov[0].iov_len);
-
-                opt = IP_PMTUDISC_DO;
-                kernel_setsockopt(trans->local->socket, SOL_IP,
-                                  IP_MTU_DISCOVER, (char *) &opt, sizeof(opt));
+        down_write(&conn->params.local->defrag_sem);
+
+        switch (conn->params.local->srx.transport.family) {
+        case AF_INET:
+                opt = IP_PMTUDISC_DONT;
+                ret = kernel_setsockopt(conn->params.local->socket,
+                                        SOL_IP, IP_MTU_DISCOVER,
+                                        (char *)&opt, sizeof(opt));
+                if (ret == 0) {
+                        ret = kernel_sendmsg(conn->params.local->socket, &msg, iov, 1,
+                                             iov[0].iov_len);
+
+                        opt = IP_PMTUDISC_DO;
+                        kernel_setsockopt(conn->params.local->socket, SOL_IP,
+                                          IP_MTU_DISCOVER,
+                                          (char *)&opt, sizeof(opt));
+                }
+                break;
         }
 
-        up_write(&trans->local->defrag_sem);
-        _leave(" = %d [frag %u]", ret, trans->peer->maxdata);
+        up_write(&conn->params.local->defrag_sem);
+        _leave(" = %d [frag %u]", ret, conn->params.peer->maxdata);
         return ret;
 }
 
@@ -497,7 +482,7 @@ static void rxrpc_queue_packet(struct rxrpc_call *call, struct sk_buff *skb,
         if (try_to_del_timer_sync(&call->ack_timer) >= 0) {
                 /* the packet may be freed by rxrpc_process_call() before this
                  * returns */
-                ret = rxrpc_send_packet(call->conn->trans, skb);
+                ret = rxrpc_send_data_packet(call->conn, skb);
                 _net("sent skb %p", skb);
         } else {
                 _debug("failed to delete ACK timer");
@@ -583,7 +568,7 @@ static int rxrpc_send_data(struct rxrpc_sock *rx,
                                         goto maybe_error;
                         }
 
-                        max = call->conn->trans->peer->maxdata;
+                        max = call->conn->params.peer->maxdata;
                         max -= call->conn->security_size;
                         max &= ~(call->conn->size_align - 1UL);
 
@@ -674,7 +659,7 @@ static int rxrpc_send_data(struct rxrpc_sock *rx,
 
                         seq = atomic_inc_return(&call->sequence);
 
-                        sp->hdr.epoch   = conn->epoch;
+                        sp->hdr.epoch   = conn->proto.epoch;
                         sp->hdr.cid     = call->cid;
                         sp->hdr.callNumber = call->call_id;
                         sp->hdr.seq     = seq;
@@ -717,7 +702,9 @@ out:
 call_aborted:
         rxrpc_free_skb(skb);
         if (call->state == RXRPC_CALL_NETWORK_ERROR)
-                ret = call->conn->trans->peer->net_error;
+                ret = call->error_report < RXRPC_LOCAL_ERROR_OFFSET ?
+                        call->error_report :
+                        call->error_report - RXRPC_LOCAL_ERROR_OFFSET;
         else
                 ret = -ECONNABORTED;
         _leave(" = %d", ret);
diff --git a/net/rxrpc/peer_event.c b/net/rxrpc/peer_event.c
new file mode 100644
index 000000000000..8940674b5e08
--- /dev/null
+++ b/net/rxrpc/peer_event.c
@@ -0,0 +1,281 @@
+/* Peer event handling, typically ICMP messages.
+ *
+ * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
+ * Written by David Howells (dhowells@redhat.com)
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#include <linux/module.h>
+#include <linux/net.h>
+#include <linux/skbuff.h>
+#include <linux/errqueue.h>
+#include <linux/udp.h>
+#include <linux/in.h>
+#include <linux/in6.h>
+#include <linux/icmp.h>
+#include <net/sock.h>
+#include <net/af_rxrpc.h>
+#include <net/ip.h>
+#include "ar-internal.h"
+
+static void rxrpc_store_error(struct rxrpc_peer *, struct sock_exterr_skb *);
+
+/*
+ * Find the peer associated with an ICMP packet.
+ */
+static struct rxrpc_peer *rxrpc_lookup_peer_icmp_rcu(struct rxrpc_local *local,
+                                                     const struct sk_buff *skb)
+{
+        struct sock_exterr_skb *serr = SKB_EXT_ERR(skb);
+        struct sockaddr_rxrpc srx;
+
+        _enter("");
+
+        memset(&srx, 0, sizeof(srx));
+        srx.transport_type = local->srx.transport_type;
+        srx.transport.family = local->srx.transport.family;
+
+        /* Can we see an ICMP4 packet on an ICMP6 listening socket?  and vice
+         * versa?
+         */
+        switch (srx.transport.family) {
+        case AF_INET:
+                srx.transport.sin.sin_port = serr->port;
+                srx.transport_len = sizeof(struct sockaddr_in);
+                switch (serr->ee.ee_origin) {
+                case SO_EE_ORIGIN_ICMP:
+                        _net("Rx ICMP");
+                        memcpy(&srx.transport.sin.sin_addr,
+                               skb_network_header(skb) + serr->addr_offset,
+                               sizeof(struct in_addr));
+                        break;
+                case SO_EE_ORIGIN_ICMP6:
+                        _net("Rx ICMP6 on v4 sock");
+                        memcpy(&srx.transport.sin.sin_addr,
+                               skb_network_header(skb) + serr->addr_offset + 12,
+                               sizeof(struct in_addr));
+                        break;
+                default:
+                        memcpy(&srx.transport.sin.sin_addr, &ip_hdr(skb)->saddr,
+                               sizeof(struct in_addr));
+                        break;
+                }
+                break;
+
+        default:
+                BUG();
+        }
+
+        return rxrpc_lookup_peer_rcu(local, &srx);
+}
+
+/*
+ * Handle an MTU/fragmentation problem.
+ */
+static void rxrpc_adjust_mtu(struct rxrpc_peer *peer, struct sock_exterr_skb *serr)
+{
+        u32 mtu = serr->ee.ee_info;
+
+        _net("Rx ICMP Fragmentation Needed (%d)", mtu);
+
+        /* wind down the local interface MTU */
+        if (mtu > 0 && peer->if_mtu == 65535 && mtu < peer->if_mtu) {
+                peer->if_mtu = mtu;
+                _net("I/F MTU %u", mtu);
+        }
+
+        if (mtu == 0) {
+                /* they didn't give us a size, estimate one */
+                mtu = peer->if_mtu;
+                if (mtu > 1500) {
+                        mtu >>= 1;
+                        if (mtu < 1500)
+                                mtu = 1500;
+                } else {
+                        mtu -= 100;
+                        if (mtu < peer->hdrsize)
+                                mtu = peer->hdrsize + 4;
+                }
+        }
+
+        if (mtu < peer->mtu) {
+                spin_lock_bh(&peer->lock);
+                peer->mtu = mtu;
+                peer->maxdata = peer->mtu - peer->hdrsize;
+                spin_unlock_bh(&peer->lock);
+                _net("Net MTU %u (maxdata %u)",
+                     peer->mtu, peer->maxdata);
+        }
+}
+
+/*
+ * Handle an error received on the local endpoint.
+ */
+void rxrpc_error_report(struct sock *sk)
+{
+        struct sock_exterr_skb *serr;
+        struct rxrpc_local *local = sk->sk_user_data;
+        struct rxrpc_peer *peer;
+        struct sk_buff *skb;
+
+        _enter("%p{%d}", sk, local->debug_id);
+
+        skb = sock_dequeue_err_skb(sk);
+        if (!skb) {
+                _leave("UDP socket errqueue empty");
+                return;
+        }
+        serr = SKB_EXT_ERR(skb);
+        if (!skb->len && serr->ee.ee_origin == SO_EE_ORIGIN_TIMESTAMPING) {
+                _leave("UDP empty message");
+                kfree_skb(skb);
+                return;
+        }
+
+        rxrpc_new_skb(skb);
+
+        rcu_read_lock();
+        peer = rxrpc_lookup_peer_icmp_rcu(local, skb);
+        if (peer && !rxrpc_get_peer_maybe(peer))
+                peer = NULL;
+        if (!peer) {
+                rcu_read_unlock();
+                rxrpc_free_skb(skb);
+                _leave(" [no peer]");
+                return;
+        }
+
+        if ((serr->ee.ee_origin == SO_EE_ORIGIN_ICMP &&
+             serr->ee.ee_type == ICMP_DEST_UNREACH &&
+             serr->ee.ee_code == ICMP_FRAG_NEEDED)) {
+                rxrpc_adjust_mtu(peer, serr);
+                rcu_read_unlock();
+                rxrpc_free_skb(skb);
+                rxrpc_put_peer(peer);
+                _leave(" [MTU update]");
+                return;
+        }
+
+        rxrpc_store_error(peer, serr);
+        rcu_read_unlock();
+        rxrpc_free_skb(skb);
+
+        /* The ref we obtained is passed off to the work item */
+        rxrpc_queue_work(&peer->error_distributor);
+        _leave("");
+}
+
+/*
+ * Map an error report to error codes on the peer record.
+ */
+static void rxrpc_store_error(struct rxrpc_peer *peer,
+                              struct sock_exterr_skb *serr)
+{
+        struct sock_extended_err *ee;
+        int err;
+
+        _enter("");
+
+        ee = &serr->ee;
+
+        _net("Rx Error o=%d t=%d c=%d e=%d",
+             ee->ee_origin, ee->ee_type, ee->ee_code, ee->ee_errno);
+
+        err = ee->ee_errno;
+
+        switch (ee->ee_origin) {
+        case SO_EE_ORIGIN_ICMP:
+                switch (ee->ee_type) {
+                case ICMP_DEST_UNREACH:
+                        switch (ee->ee_code) {
+                        case ICMP_NET_UNREACH:
+                                _net("Rx Received ICMP Network Unreachable");
+                                break;
+                        case ICMP_HOST_UNREACH:
+                                _net("Rx Received ICMP Host Unreachable");
+                                break;
+                        case ICMP_PORT_UNREACH:
+                                _net("Rx Received ICMP Port Unreachable");
+                                break;
+                        case ICMP_NET_UNKNOWN:
+                                _net("Rx Received ICMP Unknown Network");
+                                break;
+                        case ICMP_HOST_UNKNOWN:
+                                _net("Rx Received ICMP Unknown Host");
+                                break;
+                        default:
+                                _net("Rx Received ICMP DestUnreach code=%u",
+                                     ee->ee_code);
+                                break;
+                        }
+                        break;
+
+                case ICMP_TIME_EXCEEDED:
+                        _net("Rx Received ICMP TTL Exceeded");
+                        break;
+
+                default:
+                        _proto("Rx Received ICMP error { type=%u code=%u }",
+                               ee->ee_type, ee->ee_code);
+                        break;
+                }
+                break;
+
+        case SO_EE_ORIGIN_NONE:
+        case SO_EE_ORIGIN_LOCAL:
+                _proto("Rx Received local error { error=%d }", err);
+                err += RXRPC_LOCAL_ERROR_OFFSET;
+                break;
+
+        case SO_EE_ORIGIN_ICMP6:
+        default:
+                _proto("Rx Received error report { orig=%u }", ee->ee_origin);
+                break;
+        }
+
+        peer->error_report = err;
+}
+
+/*
+ * Distribute an error that occurred on a peer
+ */
+void rxrpc_peer_error_distributor(struct work_struct *work)
+{
+        struct rxrpc_peer *peer =
+                container_of(work, struct rxrpc_peer, error_distributor);
+        struct rxrpc_call *call;
+        int error_report;
+
+        _enter("");
+
+        error_report = READ_ONCE(peer->error_report);
+
+        _debug("ISSUE ERROR %d", error_report);
+
+        spin_lock_bh(&peer->lock);
+
+        while (!hlist_empty(&peer->error_targets)) {
+                call = hlist_entry(peer->error_targets.first,
+                                   struct rxrpc_call, error_link);
+                hlist_del_init(&call->error_link);
+
+                write_lock(&call->state_lock);
+                if (call->state != RXRPC_CALL_COMPLETE &&
+                    call->state < RXRPC_CALL_NETWORK_ERROR) {
+                        call->error_report = error_report;
+                        call->state = RXRPC_CALL_NETWORK_ERROR;
+                        set_bit(RXRPC_CALL_EV_RCVD_ERROR, &call->events);
+                        rxrpc_queue_call(call);
+                }
+                write_unlock(&call->state_lock);
+        }
+
+        spin_unlock_bh(&peer->lock);
+
+        rxrpc_put_peer(peer);
+        _leave("");
+}
diff --git a/net/rxrpc/peer_object.c b/net/rxrpc/peer_object.c
new file mode 100644
index 000000000000..01d4930a11f7
--- /dev/null
+++ b/net/rxrpc/peer_object.c
@@ -0,0 +1,315 @@
+/* RxRPC remote transport endpoint record management
+ *
+ * Copyright (C) 2007, 2016 Red Hat, Inc. All Rights Reserved.
+ * Written by David Howells (dhowells@redhat.com)
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/module.h>
+#include <linux/net.h>
+#include <linux/skbuff.h>
+#include <linux/udp.h>
+#include <linux/in.h>
+#include <linux/slab.h>
+#include <linux/hashtable.h>
+#include <net/sock.h>
+#include <net/af_rxrpc.h>
+#include <net/ip.h>
+#include <net/route.h>
+#include "ar-internal.h"
+
+static DEFINE_HASHTABLE(rxrpc_peer_hash, 10);
+static DEFINE_SPINLOCK(rxrpc_peer_hash_lock);
+
+/*
+ * Hash a peer key.
+ */
+static unsigned long rxrpc_peer_hash_key(struct rxrpc_local *local,
+                                         const struct sockaddr_rxrpc *srx)
+{
+        const u16 *p;
+        unsigned int i, size;
+        unsigned long hash_key;
+
+        _enter("");
+
+        hash_key = (unsigned long)local / __alignof__(*local);
+        hash_key += srx->transport_type;
+        hash_key += srx->transport_len;
+        hash_key += srx->transport.family;
+
+        switch (srx->transport.family) {
+        case AF_INET:
+                hash_key += (u16 __force)srx->transport.sin.sin_port;
+                size = sizeof(srx->transport.sin.sin_addr);
+                p = (u16 *)&srx->transport.sin.sin_addr;
+                break;
+        default:
+                WARN(1, "AF_RXRPC: Unsupported transport address family\n");
+                return 0;
+        }
+
+        /* Step through the peer address in 16-bit portions for speed */
+        for (i = 0; i < size; i += sizeof(*p), p++)
+                hash_key += *p;
+
+        _leave(" 0x%lx", hash_key);
+        return hash_key;
+}
+
+/*
+ * Compare a peer to a key.  Return -ve, 0 or +ve to indicate less than, same
+ * or greater than.
+ *
+ * Unfortunately, the primitives in linux/hashtable.h don't allow for sorted
+ * buckets and mid-bucket insertion, so we don't make full use of this
+ * information at this point.
+ */
+static long rxrpc_peer_cmp_key(const struct rxrpc_peer *peer,
+                               struct rxrpc_local *local,
+                               const struct sockaddr_rxrpc *srx,
+                               unsigned long hash_key)
+{
+        long diff;
+
+        diff = ((peer->hash_key - hash_key) ?:
+                ((unsigned long)peer->local - (unsigned long)local) ?:
+                (peer->srx.transport_type - srx->transport_type) ?:
+                (peer->srx.transport_len - srx->transport_len) ?:
+                (peer->srx.transport.family - srx->transport.family));
+        if (diff != 0)
+                return diff;
+
+        switch (srx->transport.family) {
+        case AF_INET:
+                return ((u16 __force)peer->srx.transport.sin.sin_port -
+                        (u16 __force)srx->transport.sin.sin_port) ?:
+                        memcmp(&peer->srx.transport.sin.sin_addr,
+                               &srx->transport.sin.sin_addr,
+                               sizeof(struct in_addr));
+        default:
+                BUG();
+        }
+}
+
+/*
+ * Look up a remote transport endpoint for the specified address using RCU.
+ */
+static struct rxrpc_peer *__rxrpc_lookup_peer_rcu(
+        struct rxrpc_local *local,
+        const struct sockaddr_rxrpc *srx,
+        unsigned long hash_key)
+{
+        struct rxrpc_peer *peer;
+
+        hash_for_each_possible_rcu(rxrpc_peer_hash, peer, hash_link, hash_key) {
+                if (rxrpc_peer_cmp_key(peer, local, srx, hash_key) == 0) {
+                        if (atomic_read(&peer->usage) == 0)
+                                return NULL;
+                        return peer;
+                }
+        }
+
+        return NULL;
+}
+
+/*
+ * Look up a remote transport endpoint for the specified address using RCU.
+ */
+struct rxrpc_peer *rxrpc_lookup_peer_rcu(struct rxrpc_local *local,
+                                         const struct sockaddr_rxrpc *srx)
+{
+        struct rxrpc_peer *peer;
+        unsigned long hash_key = rxrpc_peer_hash_key(local, srx);
+
+        peer = __rxrpc_lookup_peer_rcu(local, srx, hash_key);
+        if (peer) {
+                switch (srx->transport.family) {
+                case AF_INET:
+                        _net("PEER %d {%d,%u,%pI4+%hu}",
+                             peer->debug_id,
+                             peer->srx.transport_type,
+                             peer->srx.transport.family,
+                             &peer->srx.transport.sin.sin_addr,
+                             ntohs(peer->srx.transport.sin.sin_port));
+                        break;
+                }
+
+                _leave(" = %p {u=%d}", peer, atomic_read(&peer->usage));
+        }
+        return peer;
+}
+
+/*
+ * assess the MTU size for the network interface through which this peer is
+ * reached
+ */
+static void rxrpc_assess_MTU_size(struct rxrpc_peer *peer)
+{
+        struct rtable *rt;
+        struct flowi4 fl4;
+
+        peer->if_mtu = 1500;
+
+        rt = ip_route_output_ports(&init_net, &fl4, NULL,
+                                   peer->srx.transport.sin.sin_addr.s_addr, 0,
+                                   htons(7000), htons(7001),
+                                   IPPROTO_UDP, 0, 0);
+        if (IS_ERR(rt)) {
+                _leave(" [route err %ld]", PTR_ERR(rt));
+                return;
+        }
+
+        peer->if_mtu = dst_mtu(&rt->dst);
+        dst_release(&rt->dst);
+
+        _leave(" [if_mtu %u]", peer->if_mtu);
+}
+
+/*
+ * Allocate a peer.
+ */
+struct rxrpc_peer *rxrpc_alloc_peer(struct rxrpc_local *local, gfp_t gfp)
+{
+        struct rxrpc_peer *peer;
+
+        _enter("");
+
+        peer = kzalloc(sizeof(struct rxrpc_peer), gfp);
+        if (peer) {
+                atomic_set(&peer->usage, 1);
+                peer->local = local;
+                INIT_HLIST_HEAD(&peer->error_targets);
+                INIT_WORK(&peer->error_distributor,
+                          &rxrpc_peer_error_distributor);
+                peer->service_conns = RB_ROOT;
+                rwlock_init(&peer->conn_lock);
+                spin_lock_init(&peer->lock);
+                peer->debug_id = atomic_inc_return(&rxrpc_debug_id);
+        }
+
+        _leave(" = %p", peer);
+        return peer;
+}
+
+/*
+ * Set up a new peer.
+ */
+static struct rxrpc_peer *rxrpc_create_peer(struct rxrpc_local *local,
+                                            struct sockaddr_rxrpc *srx,
+                                            unsigned long hash_key,
+                                            gfp_t gfp)
+{
+        struct rxrpc_peer *peer;
+
+        _enter("");
+
+        peer = rxrpc_alloc_peer(local, gfp);
+        if (peer) {
+                peer->hash_key = hash_key;
+                memcpy(&peer->srx, srx, sizeof(*srx));
+
+                rxrpc_assess_MTU_size(peer);
+                peer->mtu = peer->if_mtu;
+
+                if (srx->transport.family == AF_INET) {
+                        peer->hdrsize = sizeof(struct iphdr);
+                        switch (srx->transport_type) {
+                        case SOCK_DGRAM:
+                                peer->hdrsize += sizeof(struct udphdr);
+                                break;
+                        default:
+                                BUG();
+                                break;
+                        }
+                } else {
+                        BUG();
+                }
+
+                peer->hdrsize += sizeof(struct rxrpc_wire_header);
+                peer->maxdata = peer->mtu - peer->hdrsize;
+        }
+
+        _leave(" = %p", peer);
+        return peer;
+}
+
+/*
+ * obtain a remote transport endpoint for the specified address
+ */
+struct rxrpc_peer *rxrpc_lookup_peer(struct rxrpc_local *local,
+                                     struct sockaddr_rxrpc *srx, gfp_t gfp)
+{
+        struct rxrpc_peer *peer, *candidate;
+        unsigned long hash_key = rxrpc_peer_hash_key(local, srx);
+
+        _enter("{%d,%d,%pI4+%hu}",
+               srx->transport_type,
+               srx->transport_len,
+               &srx->transport.sin.sin_addr,
+               ntohs(srx->transport.sin.sin_port));
+
+        /* search the peer list first */
+        rcu_read_lock();
+        peer = __rxrpc_lookup_peer_rcu(local, srx, hash_key);
+        if (peer && !rxrpc_get_peer_maybe(peer))
+                peer = NULL;
+        rcu_read_unlock();
+
+        if (!peer) {
+                /* The peer is not yet present in hash - create a candidate
+                 * for a new record and then redo the search.
+                 */
+                candidate = rxrpc_create_peer(local, srx, hash_key, gfp);
+                if (!candidate) {
+                        _leave(" = NULL [nomem]");
+                        return NULL;
+                }
+
+                spin_lock(&rxrpc_peer_hash_lock);
+
+                /* Need to check that we aren't racing with someone else */
+                peer = __rxrpc_lookup_peer_rcu(local, srx, hash_key);
+                if (peer && !rxrpc_get_peer_maybe(peer))
+                        peer = NULL;
+                if (!peer)
+                        hash_add_rcu(rxrpc_peer_hash,
+                                     &candidate->hash_link, hash_key);
+
+                spin_unlock(&rxrpc_peer_hash_lock);
+
+                if (peer)
+                        kfree(candidate);
+                else
+                        peer = candidate;
+        }
+
+        _net("PEER %d {%d,%pI4+%hu}",
+             peer->debug_id,
+             peer->srx.transport_type,
+             &peer->srx.transport.sin.sin_addr,
+             ntohs(peer->srx.transport.sin.sin_port));
+
+        _leave(" = %p {u=%d}", peer, atomic_read(&peer->usage));
+        return peer;
+}
+
+/*
+ * Discard a ref on a remote peer record.
+ */
+void __rxrpc_put_peer(struct rxrpc_peer *peer)
+{
+        ASSERT(hlist_empty(&peer->error_targets));
+
+        spin_lock(&rxrpc_peer_hash_lock);
+        hash_del_rcu(&peer->hash_link);
+        spin_unlock(&rxrpc_peer_hash_lock);
+
+        kfree_rcu(peer, rcu);
+}
diff --git a/net/rxrpc/ar-proc.c b/net/rxrpc/proc.c
index 225163bc658d..500cdcdc843c 100644
--- a/net/rxrpc/ar-proc.c
+++ b/net/rxrpc/proc.c
@@ -46,7 +46,7 @@ static void rxrpc_call_seq_stop(struct seq_file *seq, void *v)
 
 static int rxrpc_call_seq_show(struct seq_file *seq, void *v)
 {
-        struct rxrpc_transport *trans;
+        struct rxrpc_connection *conn;
         struct rxrpc_call *call;
         char lbuff[4 + 4 + 4 + 4 + 5 + 1], rbuff[4 + 4 + 4 + 4 + 5 + 1];
 
@@ -59,25 +59,28 @@ static int rxrpc_call_seq_show(struct seq_file *seq, void *v)
         }
 
         call = list_entry(v, struct rxrpc_call, link);
-        trans = call->conn->trans;
 
         sprintf(lbuff, "%pI4:%u",
-                &trans->local->srx.transport.sin.sin_addr,
-                ntohs(trans->local->srx.transport.sin.sin_port));
+                &call->local->srx.transport.sin.sin_addr,
+                ntohs(call->local->srx.transport.sin.sin_port));
 
-        sprintf(rbuff, "%pI4:%u",
-                &trans->peer->srx.transport.sin.sin_addr,
-                ntohs(trans->peer->srx.transport.sin.sin_port));
+        conn = call->conn;
+        if (conn)
+                sprintf(rbuff, "%pI4:%u",
+                        &conn->params.peer->srx.transport.sin.sin_addr,
+                        ntohs(conn->params.peer->srx.transport.sin.sin_port));
+        else
+                strcpy(rbuff, "no_connection");
 
         seq_printf(seq,
                    "UDP   %-22.22s %-22.22s %4x %08x %08x %s %3u"
                    " %-8.8s %08x %lx\n",
                    lbuff,
                    rbuff,
-                   call->conn->service_id,
+                   call->service_id,
                    call->cid,
                    call->call_id,
-                   call->conn->in_clientflag ? "Svc" : "Clt",
+                   call->in_clientflag ? "Svc" : "Clt",
                    atomic_read(&call->usage),
                    rxrpc_call_states[call->state],
                    call->remote_abort ?: call->local_abort,
@@ -129,7 +132,6 @@ static void rxrpc_connection_seq_stop(struct seq_file *seq, void *v)
 static int rxrpc_connection_seq_show(struct seq_file *seq, void *v)
 {
         struct rxrpc_connection *conn;
-        struct rxrpc_transport *trans;
         char lbuff[4 + 4 + 4 + 4 + 5 + 1], rbuff[4 + 4 + 4 + 4 + 5 + 1];
 
         if (v == &rxrpc_connections) {
@@ -142,28 +144,27 @@ static int rxrpc_connection_seq_show(struct seq_file *seq, void *v)
         }
 
         conn = list_entry(v, struct rxrpc_connection, link);
-        trans = conn->trans;
 
         sprintf(lbuff, "%pI4:%u",
-                &trans->local->srx.transport.sin.sin_addr,
-                ntohs(trans->local->srx.transport.sin.sin_port));
+                &conn->params.local->srx.transport.sin.sin_addr,
+                ntohs(conn->params.local->srx.transport.sin.sin_port));
 
         sprintf(rbuff, "%pI4:%u",
-                &trans->peer->srx.transport.sin.sin_addr,
-                ntohs(trans->peer->srx.transport.sin.sin_port));
+                &conn->params.peer->srx.transport.sin.sin_addr,
+                ntohs(conn->params.peer->srx.transport.sin.sin_port));
 
         seq_printf(seq,
                    "UDP   %-22.22s %-22.22s %4x %08x %08x %s %3u"
                    " %s %08x %08x %08x\n",
                    lbuff,
                    rbuff,
-                   conn->service_id,
-                   conn->cid,
+                   conn->params.service_id,
+                   conn->proto.cid,
                    conn->call_counter,
-                   conn->in_clientflag ? "Svc" : "Clt",
+                   rxrpc_conn_is_service(conn) ? "Svc" : "Clt",
                    atomic_read(&conn->usage),
                    rxrpc_conn_states[conn->state],
-                   key_serial(conn->key),
+                   key_serial(conn->params.key),
                    atomic_read(&conn->serial),
                    atomic_read(&conn->hi_serial));
 
diff --git a/net/rxrpc/ar-recvmsg.c b/net/rxrpc/recvmsg.c
index 160f0927aa3e..a3fa2ed85d63 100644
--- a/net/rxrpc/ar-recvmsg.c
+++ b/net/rxrpc/recvmsg.c
@@ -9,6 +9,8 @@
  * 2 of the License, or (at your option) any later version.
  */
 
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
 #include <linux/net.h>
 #include <linux/skbuff.h>
 #include <linux/export.h>
@@ -145,9 +147,9 @@ int rxrpc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
                 if (!continue_call) {
                         if (msg->msg_name) {
                                 size_t len =
-                                        sizeof(call->conn->trans->peer->srx);
+                                        sizeof(call->conn->params.peer->srx);
                                 memcpy(msg->msg_name,
-                                       &call->conn->trans->peer->srx, len);
+                                       &call->conn->params.peer->srx, len);
                                 msg->msg_namelen = len;
                         }
                         sock_recv_timestamp(msg, &rx->sk, skb);
@@ -203,7 +205,7 @@ int rxrpc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
                 /* we transferred the whole data packet */
                 if (sp->hdr.flags & RXRPC_LAST_PACKET) {
                         _debug("last");
-                        if (call->conn->out_clientflag) {
+                        if (rxrpc_conn_is_client(call->conn)) {
                                  /* last byte of reply received */
                                 ret = copied;
                                 goto terminal_message;
@@ -307,7 +309,7 @@ receive_non_data_message:
                                &abort_code);
                 break;
         default:
-                pr_err("RxRPC: Unknown packet mark %u\n", skb->mark);
+                pr_err("Unknown packet mark %u\n", skb->mark);
                 BUG();
                 break;
         }
diff --git a/net/rxrpc/rxkad.c b/net/rxrpc/rxkad.c
index bab56ed649ba..23c05ec6fa28 100644
--- a/net/rxrpc/rxkad.c
+++ b/net/rxrpc/rxkad.c
@@ -9,6 +9,8 @@
  * 2 of the License, or (at your option) any later version.
  */
 
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
 #include <crypto/skcipher.h>
 #include <linux/module.h>
 #include <linux/net.h>
@@ -56,9 +58,9 @@ static int rxkad_init_connection_security(struct rxrpc_connection *conn)
         struct rxrpc_key_token *token;
         int ret;
 
-        _enter("{%d},{%x}", conn->debug_id, key_serial(conn->key));
+        _enter("{%d},{%x}", conn->debug_id, key_serial(conn->params.key));
 
-        token = conn->key->payload.data[0];
+        token = conn->params.key->payload.data[0];
         conn->security_ix = token->security_index;
 
         ci = crypto_alloc_skcipher("pcbc(fcrypt)", 0, CRYPTO_ALG_ASYNC);
@@ -72,7 +74,7 @@ static int rxkad_init_connection_security(struct rxrpc_connection *conn)
                                    sizeof(token->kad->session_key)) < 0)
                 BUG();
 
-        switch (conn->security_level) {
+        switch (conn->params.security_level) {
         case RXRPC_SECURITY_PLAIN:
                 break;
         case RXRPC_SECURITY_AUTH:
@@ -113,14 +115,14 @@ static void rxkad_prime_packet_security(struct rxrpc_connection *conn)
 
         _enter("");
 
-        if (!conn->key)
+        if (!conn->params.key)
                 return;
 
-        token = conn->key->payload.data[0];
+        token = conn->params.key->payload.data[0];
         memcpy(&iv, token->kad->session_key, sizeof(iv));
 
-        tmpbuf.x[0] = htonl(conn->epoch);
-        tmpbuf.x[1] = htonl(conn->cid);
+        tmpbuf.x[0] = htonl(conn->proto.epoch);
+        tmpbuf.x[1] = htonl(conn->proto.cid);
         tmpbuf.x[2] = 0;
         tmpbuf.x[3] = htonl(conn->security_ix);
 
@@ -218,7 +220,7 @@ static int rxkad_secure_packet_encrypt(const struct rxrpc_call *call,
         rxkhdr.checksum = 0;
 
         /* encrypt from the session key */
-        token = call->conn->key->payload.data[0];
+        token = call->conn->params.key->payload.data[0];
         memcpy(&iv, token->kad->session_key, sizeof(iv));
 
         sg_init_one(&sg[0], sechdr, sizeof(rxkhdr));
@@ -275,13 +277,13 @@ static int rxkad_secure_packet(const struct rxrpc_call *call,
         sp = rxrpc_skb(skb);
 
         _enter("{%d{%x}},{#%u},%zu,",
-               call->debug_id, key_serial(call->conn->key), sp->hdr.seq,
-               data_size);
+               call->debug_id, key_serial(call->conn->params.key),
+               sp->hdr.seq, data_size);
 
         if (!call->conn->cipher)
                 return 0;
 
-        ret = key_validate(call->conn->key);
+        ret = key_validate(call->conn->params.key);
         if (ret < 0)
                 return ret;
 
@@ -310,7 +312,7 @@ static int rxkad_secure_packet(const struct rxrpc_call *call,
                 y = 1; /* zero checksums are not permitted */
         sp->hdr.cksum = y;
 
-        switch (call->conn->security_level) {
+        switch (call->conn->params.security_level) {
         case RXRPC_SECURITY_PLAIN:
                 ret = 0;
                 break;
@@ -444,7 +446,7 @@ static int rxkad_verify_packet_encrypt(const struct rxrpc_call *call,
         skb_to_sgvec(skb, sg, 0, skb->len);
 
         /* decrypt from the session key */
-        token = call->conn->key->payload.data[0];
+        token = call->conn->params.key->payload.data[0];
         memcpy(&iv, token->kad->session_key, sizeof(iv));
 
         skcipher_request_set_tfm(req, call->conn->cipher);
@@ -514,7 +516,7 @@ static int rxkad_verify_packet(const struct rxrpc_call *call,
         sp = rxrpc_skb(skb);
 
         _enter("{%d{%x}},{#%u}",
-               call->debug_id, key_serial(call->conn->key), sp->hdr.seq);
+               call->debug_id, key_serial(call->conn->params.key), sp->hdr.seq);
 
         if (!call->conn->cipher)
                 return 0;
@@ -555,7 +557,7 @@ static int rxkad_verify_packet(const struct rxrpc_call *call,
                 return -EPROTO;
         }
 
-        switch (call->conn->security_level) {
+        switch (call->conn->params.security_level) {
         case RXRPC_SECURITY_PLAIN:
                 ret = 0;
                 break;
@@ -587,9 +589,9 @@ static int rxkad_issue_challenge(struct rxrpc_connection *conn)
         u32 serial;
         int ret;
 
-        _enter("{%d,%x}", conn->debug_id, key_serial(conn->key));
+        _enter("{%d,%x}", conn->debug_id, key_serial(conn->params.key));
 
-        ret = key_validate(conn->key);
+        ret = key_validate(conn->params.key);
         if (ret < 0)
                 return ret;
 
@@ -600,14 +602,14 @@ static int rxkad_issue_challenge(struct rxrpc_connection *conn)
         challenge.min_level     = htonl(0);
         challenge.__padding     = 0;
 
-        msg.msg_name    = &conn->trans->peer->srx.transport.sin;
-        msg.msg_namelen = sizeof(conn->trans->peer->srx.transport.sin);
+        msg.msg_name    = &conn->params.peer->srx.transport.sin;
+        msg.msg_namelen = sizeof(conn->params.peer->srx.transport.sin);
         msg.msg_control = NULL;
         msg.msg_controllen = 0;
         msg.msg_flags   = 0;
 
-        whdr.epoch      = htonl(conn->epoch);
-        whdr.cid        = htonl(conn->cid);
+        whdr.epoch      = htonl(conn->proto.epoch);
+        whdr.cid        = htonl(conn->proto.cid);
         whdr.callNumber = 0;
         whdr.seq        = 0;
         whdr.type       = RXRPC_PACKET_TYPE_CHALLENGE;
@@ -615,7 +617,7 @@ static int rxkad_issue_challenge(struct rxrpc_connection *conn)
         whdr.userStatus = 0;
         whdr.securityIndex = conn->security_ix;
         whdr._rsvd      = 0;
-        whdr.serviceId  = htons(conn->service_id);
+        whdr.serviceId  = htons(conn->params.service_id);
 
         iov[0].iov_base = &whdr;
         iov[0].iov_len  = sizeof(whdr);
@@ -628,7 +630,7 @@ static int rxkad_issue_challenge(struct rxrpc_connection *conn)
         whdr.serial = htonl(serial);
         _proto("Tx CHALLENGE %%%u", serial);
 
-        ret = kernel_sendmsg(conn->trans->local->socket, &msg, iov, 2, len);
+        ret = kernel_sendmsg(conn->params.local->socket, &msg, iov, 2, len);
         if (ret < 0) {
                 _debug("sendmsg failed: %d", ret);
                 return -EAGAIN;
@@ -655,8 +657,8 @@ static int rxkad_send_response(struct rxrpc_connection *conn,
 
         _enter("");
 
-        msg.msg_name    = &conn->trans->peer->srx.transport.sin;
-        msg.msg_namelen = sizeof(conn->trans->peer->srx.transport.sin);
+        msg.msg_name    = &conn->params.peer->srx.transport.sin;
+        msg.msg_namelen = sizeof(conn->params.peer->srx.transport.sin);
         msg.msg_control = NULL;
         msg.msg_controllen = 0;
         msg.msg_flags   = 0;
@@ -682,7 +684,7 @@ static int rxkad_send_response(struct rxrpc_connection *conn,
         whdr.serial = htonl(serial);
         _proto("Tx RESPONSE %%%u", serial);
 
-        ret = kernel_sendmsg(conn->trans->local->socket, &msg, iov, 3, len);
+        ret = kernel_sendmsg(conn->params.local->socket, &msg, iov, 3, len);
         if (ret < 0) {
                 _debug("sendmsg failed: %d", ret);
                 return -EAGAIN;
@@ -769,14 +771,14 @@ static int rxkad_respond_to_challenge(struct rxrpc_connection *conn,
         u32 version, nonce, min_level, abort_code;
         int ret;
 
-        _enter("{%d,%x}", conn->debug_id, key_serial(conn->key));
+        _enter("{%d,%x}", conn->debug_id, key_serial(conn->params.key));
 
-        if (!conn->key) {
+        if (!conn->params.key) {
                 _leave(" = -EPROTO [no key]");
                 return -EPROTO;
         }
 
-        ret = key_validate(conn->key);
+        ret = key_validate(conn->params.key);
         if (ret < 0) {
                 *_abort_code = RXKADEXPIRED;
                 return ret;
@@ -799,20 +801,20 @@ static int rxkad_respond_to_challenge(struct rxrpc_connection *conn,
                 goto protocol_error;
 
         abort_code = RXKADLEVELFAIL;
-        if (conn->security_level < min_level)
+        if (conn->params.security_level < min_level)
                 goto protocol_error;
 
-        token = conn->key->payload.data[0];
+        token = conn->params.key->payload.data[0];
 
         /* build the response packet */
         memset(&resp, 0, sizeof(resp));
 
         resp.version                    = htonl(RXKAD_VERSION);
-        resp.encrypted.epoch            = htonl(conn->epoch);
-        resp.encrypted.cid              = htonl(conn->cid);
+        resp.encrypted.epoch            = htonl(conn->proto.epoch);
+        resp.encrypted.cid              = htonl(conn->proto.cid);
         resp.encrypted.securityIndex    = htonl(conn->security_ix);
         resp.encrypted.inc_nonce        = htonl(nonce + 1);
-        resp.encrypted.level            = htonl(conn->security_level);
+        resp.encrypted.level            = htonl(conn->params.security_level);
         resp.kvno                       = htonl(token->kad->kvno);
         resp.ticket_len                 = htonl(token->kad->ticket_len);
 
@@ -1094,9 +1096,9 @@ static int rxkad_verify_response(struct rxrpc_connection *conn,
         rxkad_decrypt_response(conn, &response, &session_key);
 
         abort_code = RXKADSEALEDINCON;
-        if (ntohl(response.encrypted.epoch) != conn->epoch)
+        if (ntohl(response.encrypted.epoch) != conn->proto.epoch)
                 goto protocol_error_free;
-        if (ntohl(response.encrypted.cid) != conn->cid)
+        if (ntohl(response.encrypted.cid) != conn->proto.cid)
                 goto protocol_error_free;
         if (ntohl(response.encrypted.securityIndex) != conn->security_ix)
                 goto protocol_error_free;
@@ -1120,7 +1122,7 @@ static int rxkad_verify_response(struct rxrpc_connection *conn,
         level = ntohl(response.encrypted.level);
         if (level > RXRPC_SECURITY_ENCRYPT)
                 goto protocol_error_free;
-        conn->security_level = level;
+        conn->params.security_level = level;
 
         /* create a key to hold the security data and expiration time - after
          * this the connection security can be handled in exactly the same way
diff --git a/net/rxrpc/ar-security.c b/net/rxrpc/security.c
index d223253b22fa..814d285ff802 100644
--- a/net/rxrpc/ar-security.c
+++ b/net/rxrpc/security.c
@@ -76,7 +76,7 @@ int rxrpc_init_client_conn_security(struct rxrpc_connection *conn)
 {
         const struct rxrpc_security *sec;
         struct rxrpc_key_token *token;
-        struct key *key = conn->key;
+        struct key *key = conn->params.key;
         int ret;
 
         _enter("{%d},{%x}", conn->debug_id, key_serial(key));
@@ -113,7 +113,7 @@ int rxrpc_init_client_conn_security(struct rxrpc_connection *conn)
 int rxrpc_init_server_conn_security(struct rxrpc_connection *conn)
 {
         const struct rxrpc_security *sec;
-        struct rxrpc_local *local = conn->trans->local;
+        struct rxrpc_local *local = conn->params.local;
         struct rxrpc_sock *rx;
         struct key *key;
         key_ref_t kref;
@@ -121,7 +121,7 @@ int rxrpc_init_server_conn_security(struct rxrpc_connection *conn)
 
         _enter("");
 
-        sprintf(kdesc, "%u:%u", conn->service_id, conn->security_ix);
+        sprintf(kdesc, "%u:%u", conn->params.service_id, conn->security_ix);
 
         sec = rxrpc_security_lookup(conn->security_ix);
         if (!sec) {
@@ -132,7 +132,7 @@ int rxrpc_init_server_conn_security(struct rxrpc_connection *conn)
         /* find the service */
         read_lock_bh(&local->services_lock);
         list_for_each_entry(rx, &local->services, listen_link) {
-                if (rx->srx.srx_service == conn->service_id)
+                if (rx->srx.srx_service == conn->params.service_id)
                         goto found_service;
         }
 
diff --git a/net/rxrpc/ar-skbuff.c b/net/rxrpc/skbuff.c
index 62a267472fce..eee0cfd9ac8c 100644
--- a/net/rxrpc/ar-skbuff.c
+++ b/net/rxrpc/skbuff.c
@@ -9,6 +9,8 @@
  * 2 of the License, or (at your option) any later version.
  */
 
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
 #include <linux/module.h>
 #include <linux/net.h>
 #include <linux/skbuff.h>
diff --git a/net/rxrpc/sysctl.c b/net/rxrpc/sysctl.c
index d20ed575acf4..03ad08774d4e 100644
--- a/net/rxrpc/sysctl.c
+++ b/net/rxrpc/sysctl.c
@@ -18,6 +18,7 @@ static struct ctl_table_header *rxrpc_sysctl_reg_table;
 static const unsigned int zero = 0;
 static const unsigned int one = 1;
 static const unsigned int four = 4;
+static const unsigned int thirtytwo = 32;
 static const unsigned int n_65535 = 65535;
 static const unsigned int n_max_acks = RXRPC_MAXACKS;
 
@@ -89,16 +90,17 @@ static struct ctl_table rxrpc_sysctl_table[] = {
                 .proc_handler   = proc_dointvec_minmax,
                 .extra1         = (void *)&one,
         },
+
+        /* Non-time values */
         {
-                .procname       = "transport_expiry",
-                .data           = &rxrpc_transport_expiry,
+                .procname       = "max_backlog",
+                .data           = &rxrpc_max_backlog,
                 .maxlen         = sizeof(unsigned int),
                 .mode           = 0644,
                 .proc_handler   = proc_dointvec_minmax,
-                .extra1         = (void *)&one,
+                .extra1         = (void *)&four,
+                .extra2         = (void *)&thirtytwo,
         },
-
-        /* Non-time values */
         {
                 .procname       = "rx_window_size",
                 .data           = &rxrpc_rx_window_size,
diff --git a/net/rxrpc/utils.c b/net/rxrpc/utils.c
new file mode 100644
index 000000000000..f28122a15a24
--- /dev/null
+++ b/net/rxrpc/utils.c
@@ -0,0 +1,41 @@
+/* Utility routines
+ *
+ * Copyright (C) 2015 Red Hat, Inc. All Rights Reserved.
+ * Written by David Howells (dhowells@redhat.com)
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public Licence
+ * as published by the Free Software Foundation; either version
+ * 2 of the Licence, or (at your option) any later version.
+ */
+
+#include <linux/ip.h>
+#include <linux/udp.h>
+#include "ar-internal.h"
+
+/*
+ * Set up an RxRPC address from a socket buffer.
+ */
+void rxrpc_get_addr_from_skb(struct rxrpc_local *local,
+                             const struct sk_buff *skb,
+                             struct sockaddr_rxrpc *srx)
+{
+        memset(srx, 0, sizeof(*srx));
+        srx->transport_type = local->srx.transport_type;
+        srx->transport.family = local->srx.transport.family;
+
+        /* Can we see an ipv4 UDP packet on an ipv6 UDP socket?  and vice
+         * versa?
+         */
+        switch (srx->transport.family) {
+        case AF_INET:
+                srx->transport.sin.sin_port = udp_hdr(skb)->source;
+                srx->transport_len = sizeof(struct sockaddr_in);
+                memcpy(&srx->transport.sin.sin_addr, &ip_hdr(skb)->saddr,
+                       sizeof(struct in_addr));
+                break;
+
+        default:
+                BUG();
+        }
+}
diff --git a/net/sched/act_api.c b/net/sched/act_api.c
index c7a0b0d481c0..47ec2305f920 100644
--- a/net/sched/act_api.c
+++ b/net/sched/act_api.c
@@ -224,8 +224,8 @@ int tcf_hash_search(struct tc_action_net *tn, struct tc_action *a, u32 index)
 }
 EXPORT_SYMBOL(tcf_hash_search);
 
-int tcf_hash_check(struct tc_action_net *tn, u32 index, struct tc_action *a,
-                   int bind)
+bool tcf_hash_check(struct tc_action_net *tn, u32 index, struct tc_action *a,
+                    int bind)
 {
         struct tcf_hashinfo *hinfo = tn->hinfo;
         struct tcf_common *p = NULL;
@@ -235,9 +235,9 @@ int tcf_hash_check(struct tc_action_net *tn, u32 index, struct tc_action *a,
                 p->tcfc_refcnt++;
                 a->priv = p;
                 a->hinfo = hinfo;
-                return 1;
+                return true;
         }
-        return 0;
+        return false;
 }
 EXPORT_SYMBOL(tcf_hash_check);
 
@@ -283,10 +283,11 @@ err2:
         p->tcfc_index = index ? index : tcf_hash_new_index(tn);
         p->tcfc_tm.install = jiffies;
         p->tcfc_tm.lastuse = jiffies;
+        p->tcfc_tm.firstuse = 0;
         if (est) {
                 err = gen_new_estimator(&p->tcfc_bstats, p->cpu_bstats,
                                         &p->tcfc_rate_est,
-                                        &p->tcfc_lock, est);
+                                        &p->tcfc_lock, NULL, est);
                 if (err) {
                         free_percpu(p->cpu_qstats);
                         goto err2;
@@ -503,8 +504,8 @@ nla_put_failure:
 }
 EXPORT_SYMBOL(tcf_action_dump_1);
 
-int
-tcf_action_dump(struct sk_buff *skb, struct list_head *actions, int bind, int ref)
+int tcf_action_dump(struct sk_buff *skb, struct list_head *actions,
+                    int bind, int ref)
 {
         struct tc_action *a;
         int err = -EINVAL;
@@ -670,7 +671,7 @@ int tcf_action_copy_stats(struct sk_buff *skb, struct tc_action *a,
         if (err < 0)
                 goto errout;
 
-        if (gnet_stats_copy_basic(&d, p->cpu_bstats, &p->tcfc_bstats) < 0 ||
+        if (gnet_stats_copy_basic(NULL, &d, p->cpu_bstats, &p->tcfc_bstats) < 0 ||
             gnet_stats_copy_rate_est(&d, &p->tcfc_bstats,
                                      &p->tcfc_rate_est) < 0 ||
             gnet_stats_copy_queue(&d, p->cpu_qstats,
@@ -687,9 +688,9 @@ errout:
         return -1;
 }
 
-static int
-tca_get_fill(struct sk_buff *skb, struct list_head *actions, u32 portid, u32 seq,
-             u16 flags, int event, int bind, int ref)
+static int tca_get_fill(struct sk_buff *skb, struct list_head *actions,
+                        u32 portid, u32 seq, u16 flags, int event, int bind,
+                        int ref)
 {
         struct tcamsg *t;
         struct nlmsghdr *nlh;
@@ -730,7 +731,8 @@ act_get_notify(struct net *net, u32 portid, struct nlmsghdr *n,
         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
         if (!skb)
                 return -ENOBUFS;
-        if (tca_get_fill(skb, actions, portid, n->nlmsg_seq, 0, event, 0, 0) <= 0) {
+        if (tca_get_fill(skb, actions, portid, n->nlmsg_seq, 0, event,
+                         0, 0) <= 0) {
                 kfree_skb(skb);
                 return -EINVAL;
         }
@@ -838,7 +840,8 @@ static int tca_action_flush(struct net *net, struct nlattr *nla,
         if (a.ops == NULL) /*some idjot trying to flush unknown action */
                 goto err_out;
 
-        nlh = nlmsg_put(skb, portid, n->nlmsg_seq, RTM_DELACTION, sizeof(*t), 0);
+        nlh = nlmsg_put(skb, portid, n->nlmsg_seq, RTM_DELACTION,
+                        sizeof(*t), 0);
         if (!nlh)
                 goto out_module_put;
         t = nlmsg_data(nlh);
@@ -1001,7 +1004,8 @@ static int tc_ctl_action(struct sk_buff *skb, struct nlmsghdr *n)
         u32 portid = skb ? NETLINK_CB(skb).portid : 0;
         int ret = 0, ovr = 0;
 
-        if ((n->nlmsg_type != RTM_GETACTION) && !netlink_capable(skb, CAP_NET_ADMIN))
+        if ((n->nlmsg_type != RTM_GETACTION) &&
+            !netlink_capable(skb, CAP_NET_ADMIN))
                 return -EPERM;
 
         ret = nlmsg_parse(n, sizeof(struct tcamsg), tca, TCA_ACT_MAX, NULL);
diff --git a/net/sched/act_bpf.c b/net/sched/act_bpf.c
index c7123e01c2ca..f7b6cf49ea6f 100644
--- a/net/sched/act_bpf.c
+++ b/net/sched/act_bpf.c
@@ -154,10 +154,7 @@ static int tcf_bpf_dump(struct sk_buff *skb, struct tc_action *act,
         if (ret)
                 goto nla_put_failure;
 
-        tm.install = jiffies_to_clock_t(jiffies - prog->tcf_tm.install);
-        tm.lastuse = jiffies_to_clock_t(jiffies - prog->tcf_tm.lastuse);
-        tm.expires = jiffies_to_clock_t(prog->tcf_tm.expires);
-
+        tcf_tm_dump(&tm, &prog->tcf_tm);
         if (nla_put_64bit(skb, TCA_ACT_BPF_TM, sizeof(tm), &tm,
                           TCA_ACT_BPF_PAD))
                 goto nla_put_failure;
@@ -172,7 +169,8 @@ nla_put_failure:
 static const struct nla_policy act_bpf_policy[TCA_ACT_BPF_MAX + 1] = {
         [TCA_ACT_BPF_PARMS]     = { .len = sizeof(struct tc_act_bpf) },
         [TCA_ACT_BPF_FD]        = { .type = NLA_U32 },
-        [TCA_ACT_BPF_NAME]      = { .type = NLA_NUL_STRING, .len = ACT_BPF_NAME_LEN },
+        [TCA_ACT_BPF_NAME]      = { .type = NLA_NUL_STRING,
+                                    .len = ACT_BPF_NAME_LEN },
         [TCA_ACT_BPF_OPS_LEN]   = { .type = NLA_U16 },
         [TCA_ACT_BPF_OPS]       = { .type = NLA_BINARY,
                                     .len = sizeof(struct sock_filter) * BPF_MAXINSNS },
diff --git a/net/sched/act_connmark.c b/net/sched/act_connmark.c
index 2ba700c765e0..35a5270f289d 100644
--- a/net/sched/act_connmark.c
+++ b/net/sched/act_connmark.c
@@ -44,7 +44,7 @@ static int tcf_connmark(struct sk_buff *skb, const struct tc_action *a,
         int proto;
 
         spin_lock(&ca->tcf_lock);
-        ca->tcf_tm.lastuse = jiffies;
+        tcf_lastuse_update(&ca->tcf_tm);
         bstats_update(&ca->tcf_bstats, skb);
 
         if (skb->protocol == htons(ETH_P_IP)) {
@@ -160,9 +160,7 @@ static inline int tcf_connmark_dump(struct sk_buff *skb, struct tc_action *a,
         if (nla_put(skb, TCA_CONNMARK_PARMS, sizeof(opt), &opt))
                 goto nla_put_failure;
 
-        t.install = jiffies_to_clock_t(jiffies - ci->tcf_tm.install);
-        t.lastuse = jiffies_to_clock_t(jiffies - ci->tcf_tm.lastuse);
-        t.expires = jiffies_to_clock_t(ci->tcf_tm.expires);
+        tcf_tm_dump(&t, &ci->tcf_tm);
         if (nla_put_64bit(skb, TCA_CONNMARK_TM, sizeof(t), &t,
                           TCA_CONNMARK_PAD))
                 goto nla_put_failure;
diff --git a/net/sched/act_csum.c b/net/sched/act_csum.c
index 28e934ed038a..dcd9ababd351 100644
--- a/net/sched/act_csum.c
+++ b/net/sched/act_csum.c
@@ -501,7 +501,7 @@ static int tcf_csum(struct sk_buff *skb,
         u32 update_flags;
 
         spin_lock(&p->tcf_lock);
-        p->tcf_tm.lastuse = jiffies;
+        tcf_lastuse_update(&p->tcf_tm);
         bstats_update(&p->tcf_bstats, skb);
         action = p->tcf_action;
         update_flags = p->update_flags;
@@ -546,9 +546,8 @@ static int tcf_csum_dump(struct sk_buff *skb,
 
         if (nla_put(skb, TCA_CSUM_PARMS, sizeof(opt), &opt))
                 goto nla_put_failure;
-        t.install = jiffies_to_clock_t(jiffies - p->tcf_tm.install);
-        t.lastuse = jiffies_to_clock_t(jiffies - p->tcf_tm.lastuse);
-        t.expires = jiffies_to_clock_t(p->tcf_tm.expires);
+
+        tcf_tm_dump(&t, &p->tcf_tm);
         if (nla_put_64bit(skb, TCA_CSUM_TM, sizeof(t), &t, TCA_CSUM_PAD))
                 goto nla_put_failure;
 
diff --git a/net/sched/act_gact.c b/net/sched/act_gact.c
index ec5cc8435238..19058a7f3e5c 100644
--- a/net/sched/act_gact.c
+++ b/net/sched/act_gact.c
@@ -162,7 +162,8 @@ static void tcf_gact_stats_update(struct tc_action *a, u64 bytes, u32 packets,
         tm->lastuse = lastuse;
 }
 
-static int tcf_gact_dump(struct sk_buff *skb, struct tc_action *a, int bind, int ref)
+static int tcf_gact_dump(struct sk_buff *skb, struct tc_action *a,
+                         int bind, int ref)
 {
         unsigned char *b = skb_tail_pointer(skb);
         struct tcf_gact *gact = a->priv;
@@ -188,9 +189,7 @@ static int tcf_gact_dump(struct sk_buff *skb, struct tc_action *a, int bind, int
                         goto nla_put_failure;
         }
 #endif
-        t.install = jiffies_to_clock_t(jiffies - gact->tcf_tm.install);
-        t.lastuse = jiffies_to_clock_t(jiffies - gact->tcf_tm.lastuse);
-        t.expires = jiffies_to_clock_t(gact->tcf_tm.expires);
+        tcf_tm_dump(&t, &gact->tcf_tm);
         if (nla_put_64bit(skb, TCA_GACT_TM, sizeof(t), &t, TCA_GACT_PAD))
                 goto nla_put_failure;
         return skb->len;
diff --git a/net/sched/act_ife.c b/net/sched/act_ife.c
index ea4a2fef1b71..845ab5119c05 100644
--- a/net/sched/act_ife.c
+++ b/net/sched/act_ife.c
@@ -428,7 +428,8 @@ static int tcf_ife_init(struct net *net, struct nlattr *nla,
         u16 ife_type = 0;
         u8 *daddr = NULL;
         u8 *saddr = NULL;
-        int ret = 0, exists = 0;
+        bool exists = false;
+        int ret = 0;
         int err;
 
         err = nla_parse_nested(tb, TCA_IFE_MAX, nla, ife_policy);
@@ -562,9 +563,7 @@ static int tcf_ife_dump(struct sk_buff *skb, struct tc_action *a, int bind,
         if (nla_put(skb, TCA_IFE_PARMS, sizeof(opt), &opt))
                 goto nla_put_failure;
 
-        t.install = jiffies_to_clock_t(jiffies - ife->tcf_tm.install);
-        t.lastuse = jiffies_to_clock_t(jiffies - ife->tcf_tm.lastuse);
-        t.expires = jiffies_to_clock_t(ife->tcf_tm.expires);
+        tcf_tm_dump(&t, &ife->tcf_tm);
         if (nla_put_64bit(skb, TCA_IFE_TM, sizeof(t), &t, TCA_IFE_PAD))
                 goto nla_put_failure;
 
@@ -632,7 +631,7 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a,
 
         spin_lock(&ife->tcf_lock);
         bstats_update(&ife->tcf_bstats, skb);
-        ife->tcf_tm.lastuse = jiffies;
+        tcf_lastuse_update(&ife->tcf_tm);
         spin_unlock(&ife->tcf_lock);
 
         ifehdrln = ntohs(ifehdrln);
@@ -720,7 +719,7 @@ static int tcf_ife_encode(struct sk_buff *skb, const struct tc_action *a,
 
         spin_lock(&ife->tcf_lock);
         bstats_update(&ife->tcf_bstats, skb);
-        ife->tcf_tm.lastuse = jiffies;
+        tcf_lastuse_update(&ife->tcf_tm);
 
         if (!metalen) {         /* no metadata to send */
                 /* abuse overlimits to count when we allow packet
@@ -811,7 +810,7 @@ static int tcf_ife_act(struct sk_buff *skb, const struct tc_action *a,
         pr_info_ratelimited("unknown failure(policy neither de/encode\n");
         spin_lock(&ife->tcf_lock);
         bstats_update(&ife->tcf_bstats, skb);
-        ife->tcf_tm.lastuse = jiffies;
+        tcf_lastuse_update(&ife->tcf_tm);
         ife->tcf_qstats.drops++;
         spin_unlock(&ife->tcf_lock);
 
diff --git a/net/sched/act_ipt.c b/net/sched/act_ipt.c
index d4bd19ee5822..b8c50600697a 100644
--- a/net/sched/act_ipt.c
+++ b/net/sched/act_ipt.c
@@ -34,7 +34,8 @@ static int ipt_net_id;
 
 static int xt_net_id;
 
-static int ipt_init_target(struct xt_entry_target *t, char *table, unsigned int hook)
+static int ipt_init_target(struct xt_entry_target *t, char *table,
+                           unsigned int hook)
 {
         struct xt_tgchk_param par;
         struct xt_target *target;
@@ -96,7 +97,8 @@ static int __tcf_ipt_init(struct tc_action_net *tn, struct nlattr *nla,
         struct tcf_ipt *ipt;
         struct xt_entry_target *td, *t;
         char *tname;
-        int ret = 0, err, exists = 0;
+        bool exists = false;
+        int ret = 0, err;
         u32 hook = 0;
         u32 index = 0;
 
@@ -215,7 +217,7 @@ static int tcf_ipt(struct sk_buff *skb, const struct tc_action *a,
 
         spin_lock(&ipt->tcf_lock);
 
-        ipt->tcf_tm.lastuse = jiffies;
+        tcf_lastuse_update(&ipt->tcf_tm);
         bstats_update(&ipt->tcf_bstats, skb);
 
         /* yes, we have to worry about both in and out dev
@@ -245,7 +247,7 @@ static int tcf_ipt(struct sk_buff *skb, const struct tc_action *a,
         default:
                 net_notice_ratelimited("tc filter: Bogus netfilter code %d assume ACCEPT\n",
                                        ret);
-                result = TC_POLICE_OK;
+                result = TC_ACT_OK;
                 break;
         }
         spin_unlock(&ipt->tcf_lock);
@@ -253,7 +255,8 @@ static int tcf_ipt(struct sk_buff *skb, const struct tc_action *a,
 
 }
 
-static int tcf_ipt_dump(struct sk_buff *skb, struct tc_action *a, int bind, int ref)
+static int tcf_ipt_dump(struct sk_buff *skb, struct tc_action *a, int bind,
+                        int ref)
 {
         unsigned char *b = skb_tail_pointer(skb);
         struct tcf_ipt *ipt = a->priv;
@@ -280,11 +283,11 @@ static int tcf_ipt_dump(struct sk_buff *skb, struct tc_action *a, int bind, int
             nla_put(skb, TCA_IPT_CNT, sizeof(struct tc_cnt), &c) ||
             nla_put_string(skb, TCA_IPT_TABLE, ipt->tcfi_tname))
                 goto nla_put_failure;
-        tm.install = jiffies_to_clock_t(jiffies - ipt->tcf_tm.install);
-        tm.lastuse = jiffies_to_clock_t(jiffies - ipt->tcf_tm.lastuse);
-        tm.expires = jiffies_to_clock_t(ipt->tcf_tm.expires);
+
+        tcf_tm_dump(&tm, &ipt->tcf_tm);
         if (nla_put_64bit(skb, TCA_IPT_TM, sizeof(tm), &tm, TCA_IPT_PAD))
                 goto nla_put_failure;
+
         kfree(t);
         return skb->len;
 
diff --git a/net/sched/act_mirred.c b/net/sched/act_mirred.c
index 128942bc9e42..5b135d357e1e 100644
--- a/net/sched/act_mirred.c
+++ b/net/sched/act_mirred.c
@@ -62,7 +62,8 @@ static int tcf_mirred_init(struct net *net, struct nlattr *nla,
         struct tc_mirred *parm;
         struct tcf_mirred *m;
         struct net_device *dev;
-        int ret, ok_push = 0, exists = 0;
+        int ret, ok_push = 0;
+        bool exists = false;
 
         if (nla == NULL)
                 return -EINVAL;
@@ -157,7 +158,6 @@ static int tcf_mirred(struct sk_buff *skb, const struct tc_action *a,
         u32 at;
 
         tcf_lastuse_update(&m->tcf_tm);
-
         bstats_cpu_update(this_cpu_ptr(m->common.cpu_bstats), skb);
 
         rcu_read_lock();
@@ -219,9 +219,8 @@ static int tcf_mirred_dump(struct sk_buff *skb, struct tc_action *a, int bind, i
 
         if (nla_put(skb, TCA_MIRRED_PARMS, sizeof(opt), &opt))
                 goto nla_put_failure;
-        t.install = jiffies_to_clock_t(jiffies - m->tcf_tm.install);
-        t.lastuse = jiffies_to_clock_t(jiffies - m->tcf_tm.lastuse);
-        t.expires = jiffies_to_clock_t(m->tcf_tm.expires);
+
+        tcf_tm_dump(&t, &m->tcf_tm);
         if (nla_put_64bit(skb, TCA_MIRRED_TM, sizeof(t), &t, TCA_MIRRED_PAD))
                 goto nla_put_failure;
         return skb->len;
diff --git a/net/sched/act_nat.c b/net/sched/act_nat.c
index c0a879f940de..06ccb03f25da 100644
--- a/net/sched/act_nat.c
+++ b/net/sched/act_nat.c
@@ -103,7 +103,7 @@ static int tcf_nat(struct sk_buff *skb, const struct tc_action *a,
 
         spin_lock(&p->tcf_lock);
 
-        p->tcf_tm.lastuse = jiffies;
+        tcf_lastuse_update(&p->tcf_tm);
         old_addr = p->old_addr;
         new_addr = p->new_addr;
         mask = p->mask;
@@ -264,9 +264,8 @@ static int tcf_nat_dump(struct sk_buff *skb, struct tc_action *a,
 
         if (nla_put(skb, TCA_NAT_PARMS, sizeof(opt), &opt))
                 goto nla_put_failure;
-        t.install = jiffies_to_clock_t(jiffies - p->tcf_tm.install);
-        t.lastuse = jiffies_to_clock_t(jiffies - p->tcf_tm.lastuse);
-        t.expires = jiffies_to_clock_t(p->tcf_tm.expires);
+
+        tcf_tm_dump(&t, &p->tcf_tm);
         if (nla_put_64bit(skb, TCA_NAT_TM, sizeof(t), &t, TCA_NAT_PAD))
                 goto nla_put_failure;
 
diff --git a/net/sched/act_pedit.c b/net/sched/act_pedit.c
index c6e18f230af6..82d3c1479029 100644
--- a/net/sched/act_pedit.c
+++ b/net/sched/act_pedit.c
@@ -121,7 +121,7 @@ static int tcf_pedit(struct sk_buff *skb, const struct tc_action *a,
 
         spin_lock(&p->tcf_lock);
 
-        p->tcf_tm.lastuse = jiffies;
+        tcf_lastuse_update(&p->tcf_tm);
 
         if (p->tcfp_nkeys > 0) {
                 struct tc_pedit_key *tkey = p->tcfp_keys;
@@ -200,11 +200,11 @@ static int tcf_pedit_dump(struct sk_buff *skb, struct tc_action *a,
 
         if (nla_put(skb, TCA_PEDIT_PARMS, s, opt))
                 goto nla_put_failure;
-        t.install = jiffies_to_clock_t(jiffies - p->tcf_tm.install);
-        t.lastuse = jiffies_to_clock_t(jiffies - p->tcf_tm.lastuse);
-        t.expires = jiffies_to_clock_t(p->tcf_tm.expires);
+
+        tcf_tm_dump(&t, &p->tcf_tm);
         if (nla_put_64bit(skb, TCA_PEDIT_TM, sizeof(t), &t, TCA_PEDIT_PAD))
                 goto nla_put_failure;
+
         kfree(opt);
         return skb->len;
 
diff --git a/net/sched/act_police.c b/net/sched/act_police.c
index c557789765dc..1e8ede3955f4 100644
--- a/net/sched/act_police.c
+++ b/net/sched/act_police.c
@@ -115,9 +115,9 @@ static const struct nla_policy police_policy[TCA_POLICE_MAX + 1] = {
         [TCA_POLICE_RESULT]     = { .type = NLA_U32 },
 };
 
-static int tcf_act_police_locate(struct net *net, struct nlattr *nla,
-                                 struct nlattr *est, struct tc_action *a,
-                                 int ovr, int bind)
+static int tcf_act_police_init(struct net *net, struct nlattr *nla,
+                               struct nlattr *est, struct tc_action *a,
+                               int ovr, int bind)
 {
         int ret = 0, err;
         struct nlattr *tb[TCA_POLICE_MAX + 1];
@@ -182,7 +182,8 @@ override:
         if (est) {
                 err = gen_replace_estimator(&police->tcf_bstats, NULL,
                                             &police->tcf_rate_est,
-                                            &police->tcf_lock, est);
+                                            &police->tcf_lock,
+                                            NULL, est);
                 if (err)
                         goto failure_unlock;
         } else if (tb[TCA_POLICE_AVRATE] &&
@@ -336,6 +337,7 @@ tcf_act_police_dump(struct sk_buff *skb, struct tc_action *a, int bind, int ref)
 
         t.install = jiffies_to_clock_t(jiffies - police->tcf_tm.install);
         t.lastuse = jiffies_to_clock_t(jiffies - police->tcf_tm.lastuse);
+        t.firstuse = jiffies_to_clock_t(jiffies - police->tcf_tm.firstuse);
         t.expires = jiffies_to_clock_t(police->tcf_tm.expires);
         if (nla_put_64bit(skb, TCA_POLICE_TM, sizeof(t), &t, TCA_POLICE_PAD))
                 goto nla_put_failure;
@@ -364,7 +366,7 @@ static struct tc_action_ops act_police_ops = {
         .owner          =       THIS_MODULE,
         .act            =       tcf_act_police,
         .dump           =       tcf_act_police_dump,
-        .init           =       tcf_act_police_locate,
+        .init           =       tcf_act_police_init,
         .walk           =       tcf_act_police_walker,
         .lookup         =       tcf_police_search,
 };
diff --git a/net/sched/act_simple.c b/net/sched/act_simple.c
index e42f8daca147..318328d34d12 100644
--- a/net/sched/act_simple.c
+++ b/net/sched/act_simple.c
@@ -35,7 +35,7 @@ static int tcf_simp(struct sk_buff *skb, const struct tc_action *a,
         struct tcf_defact *d = a->priv;
 
         spin_lock(&d->tcf_lock);
-        d->tcf_tm.lastuse = jiffies;
+        tcf_lastuse_update(&d->tcf_tm);
         bstats_update(&d->tcf_bstats, skb);
 
         /* print policy string followed by _ then packet count
@@ -86,8 +86,9 @@ static int tcf_simp_init(struct net *net, struct nlattr *nla,
         struct nlattr *tb[TCA_DEF_MAX + 1];
         struct tc_defact *parm;
         struct tcf_defact *d;
+        bool exists = false;
+        int ret = 0, err;
         char *defdata;
-        int ret = 0, err, exists = 0;
 
         if (nla == NULL)
                 return -EINVAL;
@@ -158,9 +159,8 @@ static int tcf_simp_dump(struct sk_buff *skb, struct tc_action *a,
         if (nla_put(skb, TCA_DEF_PARMS, sizeof(opt), &opt) ||
             nla_put_string(skb, TCA_DEF_DATA, d->tcfd_defdata))
                 goto nla_put_failure;
-        t.install = jiffies_to_clock_t(jiffies - d->tcf_tm.install);
-        t.lastuse = jiffies_to_clock_t(jiffies - d->tcf_tm.lastuse);
-        t.expires = jiffies_to_clock_t(d->tcf_tm.expires);
+
+        tcf_tm_dump(&t, &d->tcf_tm);
         if (nla_put_64bit(skb, TCA_DEF_TM, sizeof(t), &t, TCA_DEF_PAD))
                 goto nla_put_failure;
         return skb->len;
diff --git a/net/sched/act_skbedit.c b/net/sched/act_skbedit.c
index e928802966bc..53d1486cddf7 100644
--- a/net/sched/act_skbedit.c
+++ b/net/sched/act_skbedit.c
@@ -37,7 +37,7 @@ static int tcf_skbedit(struct sk_buff *skb, const struct tc_action *a,
         struct tcf_skbedit *d = a->priv;
 
         spin_lock(&d->tcf_lock);
-        d->tcf_tm.lastuse = jiffies;
+        tcf_lastuse_update(&d->tcf_tm);
         bstats_update(&d->tcf_bstats, skb);
 
         if (d->flags & SKBEDIT_F_PRIORITY)
@@ -69,7 +69,8 @@ static int tcf_skbedit_init(struct net *net, struct nlattr *nla,
         struct tcf_skbedit *d;
         u32 flags = 0, *priority = NULL, *mark = NULL;
         u16 *queue_mapping = NULL;
-        int ret = 0, err, exists = 0;
+        bool exists = false;
+        int ret = 0, err;
 
         if (nla == NULL)
                 return -EINVAL;
@@ -168,9 +169,8 @@ static int tcf_skbedit_dump(struct sk_buff *skb, struct tc_action *a,
             nla_put(skb, TCA_SKBEDIT_MARK, sizeof(d->mark),
                     &d->mark))
                 goto nla_put_failure;
-        t.install = jiffies_to_clock_t(jiffies - d->tcf_tm.install);
-        t.lastuse = jiffies_to_clock_t(jiffies - d->tcf_tm.lastuse);
-        t.expires = jiffies_to_clock_t(d->tcf_tm.expires);
+
+        tcf_tm_dump(&t, &d->tcf_tm);
         if (nla_put_64bit(skb, TCA_SKBEDIT_TM, sizeof(t), &t, TCA_SKBEDIT_PAD))
                 goto nla_put_failure;
         return skb->len;
diff --git a/net/sched/act_vlan.c b/net/sched/act_vlan.c
index ac4adc812c12..db9b7ed570ba 100644
--- a/net/sched/act_vlan.c
+++ b/net/sched/act_vlan.c
@@ -31,7 +31,7 @@ static int tcf_vlan(struct sk_buff *skb, const struct tc_action *a,
         int err;
 
         spin_lock(&v->tcf_lock);
-        v->tcf_tm.lastuse = jiffies;
+        tcf_lastuse_update(&v->tcf_tm);
         bstats_update(&v->tcf_bstats, skb);
         action = v->tcf_action;
 
@@ -77,8 +77,8 @@ static int tcf_vlan_init(struct net *net, struct nlattr *nla,
         int action;
         __be16 push_vid = 0;
         __be16 push_proto = 0;
-        int ret = 0, exists = 0;
-        int err;
+        bool exists = false;
+        int ret = 0, err;
 
         if (!nla)
                 return -EINVAL;
@@ -179,12 +179,11 @@ static int tcf_vlan_dump(struct sk_buff *skb, struct tc_action *a,
 
         if (v->tcfv_action == TCA_VLAN_ACT_PUSH &&
             (nla_put_u16(skb, TCA_VLAN_PUSH_VLAN_ID, v->tcfv_push_vid) ||
-             nla_put_be16(skb, TCA_VLAN_PUSH_VLAN_PROTOCOL, v->tcfv_push_proto)))
+             nla_put_be16(skb, TCA_VLAN_PUSH_VLAN_PROTOCOL,
+                          v->tcfv_push_proto)))
                 goto nla_put_failure;
 
-        t.install = jiffies_to_clock_t(jiffies - v->tcf_tm.install);
-        t.lastuse = jiffies_to_clock_t(jiffies - v->tcf_tm.lastuse);
-        t.expires = jiffies_to_clock_t(v->tcf_tm.expires);
+        tcf_tm_dump(&t, &v->tcf_tm);
         if (nla_put_64bit(skb, TCA_VLAN_TM, sizeof(t), &t, TCA_VLAN_PAD))
                 goto nla_put_failure;
         return skb->len;
diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c
index a75864d93142..843a716a4303 100644
--- a/net/sched/cls_api.c
+++ b/net/sched/cls_api.c
@@ -103,6 +103,17 @@ static int tfilter_notify(struct net *net, struct sk_buff *oskb,
                           struct nlmsghdr *n, struct tcf_proto *tp,
                           unsigned long fh, int event);
 
+static void tfilter_notify_chain(struct net *net, struct sk_buff *oskb,
+                                 struct nlmsghdr *n,
+                                 struct tcf_proto __rcu **chain, int event)
+{
+        struct tcf_proto __rcu **it_chain;
+        struct tcf_proto *tp;
+
+        for (it_chain = chain; (tp = rtnl_dereference(*it_chain)) != NULL;
+             it_chain = &tp->next)
+                tfilter_notify(net, oskb, n, tp, 0, event);
+}
 
 /* Select new prio value from the range, managed by kernel. */
 
@@ -156,11 +167,23 @@ replay:
         cl = 0;
 
         if (prio == 0) {
-                /* If no priority is given, user wants we allocated it. */
-                if (n->nlmsg_type != RTM_NEWTFILTER ||
-                    !(n->nlmsg_flags & NLM_F_CREATE))
+                switch (n->nlmsg_type) {
+                case RTM_DELTFILTER:
+                        if (protocol || t->tcm_handle || tca[TCA_KIND])
+                                return -ENOENT;
+                        break;
+                case RTM_NEWTFILTER:
+                        /* If no priority is provided by the user,
+                         * we allocate one.
+                         */
+                        if (n->nlmsg_flags & NLM_F_CREATE) {
+                                prio = TC_H_MAKE(0x80000000U, 0U);
+                                break;
+                        }
+                        /* fall-through */
+                default:
                         return -ENOENT;
-                prio = TC_H_MAKE(0x80000000U, 0U);
+                }
         }
 
         /* Find head of filter chain. */
@@ -200,6 +223,12 @@ replay:
         err = -EINVAL;
         if (chain == NULL)
                 goto errout;
+        if (n->nlmsg_type == RTM_DELTFILTER && prio == 0) {
+                tfilter_notify_chain(net, skb, n, chain, RTM_DELTFILTER);
+                tcf_destroy_chain(chain);
+                err = 0;
+                goto errout;
+        }
 
         /* Check the chain for existence of proto-tcf with this priority */
         for (back = chain;
@@ -351,8 +380,9 @@ errout:
         return err;
 }
 
-static int tcf_fill_node(struct net *net, struct sk_buff *skb, struct tcf_proto *tp,
-                         unsigned long fh, u32 portid, u32 seq, u16 flags, int event)
+static int tcf_fill_node(struct net *net, struct sk_buff *skb,
+                         struct tcf_proto *tp, unsigned long fh, u32 portid,
+                         u32 seq, u16 flags, int event)
 {
         struct tcmsg *tcm;
         struct nlmsghdr  *nlh;
@@ -474,9 +504,11 @@ static int tc_dump_tfilter(struct sk_buff *skb, struct netlink_callback *cb)
                     TC_H_MIN(tcm->tcm_info) != tp->protocol)
                         continue;
                 if (t > s_t)
-                        memset(&cb->args[1], 0, sizeof(cb->args)-sizeof(cb->args[0]));
+                        memset(&cb->args[1], 0,
+                               sizeof(cb->args)-sizeof(cb->args[0]));
                 if (cb->args[1] == 0) {
-                        if (tcf_fill_node(net, skb, tp, 0, NETLINK_CB(cb->skb).portid,
+                        if (tcf_fill_node(net, skb, tp, 0,
+                                          NETLINK_CB(cb->skb).portid,
                                           cb->nlh->nlmsg_seq, NLM_F_MULTI,
                                           RTM_NEWTFILTER) <= 0)
                                 break;
diff --git a/net/sched/cls_flower.c b/net/sched/cls_flower.c
index b3b7978f4182..5060801a2f6d 100644
--- a/net/sched/cls_flower.c
+++ b/net/sched/cls_flower.c
@@ -66,6 +66,7 @@ struct cls_fl_filter {
         struct fl_flow_key key;
         struct list_head list;
         u32 handle;
+        u32 flags;
         struct rcu_head rcu;
 };
 
@@ -123,6 +124,9 @@ static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp,
         struct fl_flow_key skb_key;
         struct fl_flow_key skb_mkey;
 
+        if (!atomic_read(&head->ht.nelems))
+                return -1;
+
         fl_clear_masked_range(&skb_key, &head->mask);
         skb_key.indev_ifindex = skb->skb_iif;
         /* skb_flow_dissect() does not set n_proto in case an unknown protocol,
@@ -136,7 +140,7 @@ static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp,
         f = rhashtable_lookup_fast(&head->ht,
                                    fl_key_get_start(&skb_mkey, &head->mask),
                                    head->ht_params);
-        if (f) {
+        if (f && !tc_skip_sw(f->flags)) {
                 *res = f->res;
                 return tcf_exts_exec(skb, &f->exts, res);
         }
@@ -183,19 +187,20 @@ static void fl_hw_destroy_filter(struct tcf_proto *tp, unsigned long cookie)
         dev->netdev_ops->ndo_setup_tc(dev, tp->q->handle, tp->protocol, &tc);
 }
 
-static void fl_hw_replace_filter(struct tcf_proto *tp,
-                                 struct flow_dissector *dissector,
-                                 struct fl_flow_key *mask,
-                                 struct fl_flow_key *key,
-                                 struct tcf_exts *actions,
-                                 unsigned long cookie, u32 flags)
+static int fl_hw_replace_filter(struct tcf_proto *tp,
+                                struct flow_dissector *dissector,
+                                struct fl_flow_key *mask,
+                                struct fl_flow_key *key,
+                                struct tcf_exts *actions,
+                                unsigned long cookie, u32 flags)
 {
         struct net_device *dev = tp->q->dev_queue->dev;
         struct tc_cls_flower_offload offload = {0};
         struct tc_to_netdev tc;
+        int err;
 
         if (!tc_should_offload(dev, tp, flags))
-                return;
+                return tc_skip_sw(flags) ? -EINVAL : 0;
 
         offload.command = TC_CLSFLOWER_REPLACE;
         offload.cookie = cookie;
@@ -207,7 +212,12 @@ static void fl_hw_replace_filter(struct tcf_proto *tp,
         tc.type = TC_SETUP_CLSFLOWER;
         tc.cls_flower = &offload;
 
-        dev->netdev_ops->ndo_setup_tc(dev, tp->q->handle, tp->protocol, &tc);
+        err = dev->netdev_ops->ndo_setup_tc(dev, tp->q->handle, tp->protocol, &tc);
+
+        if (tc_skip_sw(flags))
+                return err;
+
+        return 0;
 }
 
 static void fl_hw_update_stats(struct tcf_proto *tp, struct cls_fl_filter *f)
@@ -524,7 +534,6 @@ static int fl_change(struct net *net, struct sk_buff *in_skb,
         struct cls_fl_filter *fnew;
         struct nlattr *tb[TCA_FLOWER_MAX + 1];
         struct fl_flow_mask mask = {};
-        u32 flags = 0;
         int err;
 
         if (!tca[TCA_OPTIONS])
@@ -552,8 +561,14 @@ static int fl_change(struct net *net, struct sk_buff *in_skb,
         }
         fnew->handle = handle;
 
-        if (tb[TCA_FLOWER_FLAGS])
-                flags = nla_get_u32(tb[TCA_FLOWER_FLAGS]);
+        if (tb[TCA_FLOWER_FLAGS]) {
+                fnew->flags = nla_get_u32(tb[TCA_FLOWER_FLAGS]);
+
+                if (!tc_flags_valid(fnew->flags)) {
+                        err = -EINVAL;
+                        goto errout;
+                }
+        }
 
         err = fl_set_parms(net, tp, fnew, &mask, base, tb, tca[TCA_RATE], ovr);
         if (err)
@@ -563,19 +578,23 @@ static int fl_change(struct net *net, struct sk_buff *in_skb,
         if (err)
                 goto errout;
 
-        err = rhashtable_insert_fast(&head->ht, &fnew->ht_node,
-                                     head->ht_params);
+        if (!tc_skip_sw(fnew->flags)) {
+                err = rhashtable_insert_fast(&head->ht, &fnew->ht_node,
+                                             head->ht_params);
+                if (err)
+                        goto errout;
+        }
+
+        err = fl_hw_replace_filter(tp,
+                                   &head->dissector,
+                                   &mask.key,
+                                   &fnew->key,
+                                   &fnew->exts,
+                                   (unsigned long)fnew,
+                                   fnew->flags);
         if (err)
                 goto errout;
 
-        fl_hw_replace_filter(tp,
-                             &head->dissector,
-                             &mask.key,
-                             &fnew->key,
-                             &fnew->exts,
-                             (unsigned long)fnew,
-                             flags);
-
         if (fold) {
                 rhashtable_remove_fast(&head->ht, &fold->ht_node,
                                        head->ht_params);
@@ -734,6 +753,8 @@ static int fl_dump(struct net *net, struct tcf_proto *tp, unsigned long fh,
                                   sizeof(key->tp.dst))))
                 goto nla_put_failure;
 
+        nla_put_u32(skb, TCA_FLOWER_FLAGS, f->flags);
+
         if (tcf_exts_dump(skb, &f->exts))
                 goto nla_put_failure;
 
diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c
index ddf047df5361..12ebde845523 100644
--- a/net/sched/sch_api.c
+++ b/net/sched/sch_api.c
@@ -95,8 +95,6 @@ static int tclass_notify(struct net *net, struct sk_buff *oskb,
      Expected action: do not backoff, but wait until queue will clear.
    NET_XMIT_CN          - probably this packet enqueued, but another one dropped.
      Expected action: backoff or ignore
-   NET_XMIT_POLICED     - dropped by police.
-     Expected action: backoff or error to real-time apps.
 
    Auxiliary routines:
 
@@ -583,7 +581,6 @@ static enum hrtimer_restart qdisc_watchdog(struct hrtimer *timer)
                                                  timer);
 
         rcu_read_lock();
-        qdisc_unthrottled(wd->qdisc);
         __netif_schedule(qdisc_root(wd->qdisc));
         rcu_read_unlock();
 
@@ -598,15 +595,12 @@ void qdisc_watchdog_init(struct qdisc_watchdog *wd, struct Qdisc *qdisc)
 }
 EXPORT_SYMBOL(qdisc_watchdog_init);
 
-void qdisc_watchdog_schedule_ns(struct qdisc_watchdog *wd, u64 expires, bool throttle)
+void qdisc_watchdog_schedule_ns(struct qdisc_watchdog *wd, u64 expires)
 {
         if (test_bit(__QDISC_STATE_DEACTIVATED,
                      &qdisc_root_sleeping(wd->qdisc)->state))
                 return;
 
-        if (throttle)
-                qdisc_throttled(wd->qdisc);
-
         if (wd->last_expires == expires)
                 return;
 
@@ -620,7 +614,6 @@ EXPORT_SYMBOL(qdisc_watchdog_schedule_ns);
 void qdisc_watchdog_cancel(struct qdisc_watchdog *wd)
 {
         hrtimer_cancel(&wd->timer);
-        qdisc_unthrottled(wd->qdisc);
 }
 EXPORT_SYMBOL(qdisc_watchdog_cancel);
 
@@ -982,7 +975,7 @@ qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue,
                         rcu_assign_pointer(sch->stab, stab);
                 }
                 if (tca[TCA_RATE]) {
-                        spinlock_t *root_lock;
+                        seqcount_t *running;
 
                         err = -EOPNOTSUPP;
                         if (sch->flags & TCQ_F_MQROOT)
@@ -991,14 +984,15 @@ qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue,
                         if ((sch->parent != TC_H_ROOT) &&
                             !(sch->flags & TCQ_F_INGRESS) &&
                             (!p || !(p->flags & TCQ_F_MQROOT)))
-                                root_lock = qdisc_root_sleeping_lock(sch);
+                                running = qdisc_root_sleeping_running(sch);
                         else
-                                root_lock = qdisc_lock(sch);
+                                running = &sch->running;
 
                         err = gen_new_estimator(&sch->bstats,
                                                 sch->cpu_bstats,
                                                 &sch->rate_est,
-                                                root_lock,
+                                                NULL,
+                                                running,
                                                 tca[TCA_RATE]);
                         if (err)
                                 goto err_out4;
@@ -1061,7 +1055,8 @@ static int qdisc_change(struct Qdisc *sch, struct nlattr **tca)
                 gen_replace_estimator(&sch->bstats,
                                       sch->cpu_bstats,
                                       &sch->rate_est,
-                                      qdisc_root_sleeping_lock(sch),
+                                      NULL,
+                                      qdisc_root_sleeping_running(sch),
                                       tca[TCA_RATE]);
         }
 out:
@@ -1369,8 +1364,7 @@ static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,
                 goto nla_put_failure;
 
         if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS,
-                                         qdisc_root_sleeping_lock(q), &d,
-                                         TCA_PAD) < 0)
+                                         NULL, &d, TCA_PAD) < 0)
                 goto nla_put_failure;
 
         if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0)
@@ -1381,7 +1375,8 @@ static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,
                 cpu_qstats = q->cpu_qstats;
         }
 
-        if (gnet_stats_copy_basic(&d, cpu_bstats, &q->bstats) < 0 ||
+        if (gnet_stats_copy_basic(qdisc_root_sleeping_running(q),
+                                  &d, cpu_bstats, &q->bstats) < 0 ||
             gnet_stats_copy_rate_est(&d, &q->bstats, &q->rate_est) < 0 ||
             gnet_stats_copy_queue(&d, cpu_qstats, &q->qstats, qlen) < 0)
                 goto nla_put_failure;
@@ -1684,8 +1679,7 @@ static int tc_fill_tclass(struct sk_buff *skb, struct Qdisc *q,
                 goto nla_put_failure;
 
         if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS,
-                                         qdisc_root_sleeping_lock(q), &d,
-                                         TCA_PAD) < 0)
+                                         NULL, &d, TCA_PAD) < 0)
                 goto nla_put_failure;
 
         if (cl_ops->dump_stats && cl_ops->dump_stats(q, cl, &d) < 0)
diff --git a/net/sched/sch_atm.c b/net/sched/sch_atm.c
index 1911af3ca7c0..481e4f12aeb4 100644
--- a/net/sched/sch_atm.c
+++ b/net/sched/sch_atm.c
@@ -357,16 +357,17 @@ static struct tcf_proto __rcu **atm_tc_find_tcf(struct Qdisc *sch,
 
 /* --------------------------- Qdisc operations ---------------------------- */
 
-static int atm_tc_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+static int atm_tc_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+                          struct sk_buff **to_free)
 {
         struct atm_qdisc_data *p = qdisc_priv(sch);
         struct atm_flow_data *flow;
         struct tcf_result res;
         int result;
-        int ret = NET_XMIT_POLICED;
+        int ret = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
 
         pr_debug("atm_tc_enqueue(skb %p,sch %p,[qdisc %p])\n", skb, sch, p);
-        result = TC_POLICE_OK;  /* be nice to gcc */
+        result = TC_ACT_OK;     /* be nice to gcc */
         flow = NULL;
         if (TC_H_MAJ(skb->priority) != sch->handle ||
             !(flow = (struct atm_flow_data *)atm_tc_get(sch, skb->priority))) {
@@ -398,12 +399,12 @@ done:
                 switch (result) {
                 case TC_ACT_QUEUED:
                 case TC_ACT_STOLEN:
-                        kfree_skb(skb);
+                        __qdisc_drop(skb, to_free);
                         return NET_XMIT_SUCCESS | __NET_XMIT_STOLEN;
                 case TC_ACT_SHOT:
-                        kfree_skb(skb);
+                        __qdisc_drop(skb, to_free);
                         goto drop;
-                case TC_POLICE_RECLASSIFY:
+                case TC_ACT_RECLASSIFY:
                         if (flow->excess)
                                 flow = flow->excess;
                         else
@@ -413,7 +414,7 @@ done:
 #endif
         }
 
-        ret = qdisc_enqueue(skb, flow->q);
+        ret = qdisc_enqueue(skb, flow->q, to_free);
         if (ret != NET_XMIT_SUCCESS) {
 drop: __maybe_unused
                 if (net_xmit_drop_count(ret)) {
@@ -519,20 +520,6 @@ static struct sk_buff *atm_tc_peek(struct Qdisc *sch)
         return p->link.q->ops->peek(p->link.q);
 }
 
-static unsigned int atm_tc_drop(struct Qdisc *sch)
-{
-        struct atm_qdisc_data *p = qdisc_priv(sch);
-        struct atm_flow_data *flow;
-        unsigned int len;
-
-        pr_debug("atm_tc_drop(sch %p,[qdisc %p])\n", sch, p);
-        list_for_each_entry(flow, &p->flows, list) {
-                if (flow->q->ops->drop && (len = flow->q->ops->drop(flow->q)))
-                        return len;
-        }
-        return 0;
-}
-
 static int atm_tc_init(struct Qdisc *sch, struct nlattr *opt)
 {
         struct atm_qdisc_data *p = qdisc_priv(sch);
@@ -637,7 +624,8 @@ atm_tc_dump_class_stats(struct Qdisc *sch, unsigned long arg,
 {
         struct atm_flow_data *flow = (struct atm_flow_data *)arg;
 
-        if (gnet_stats_copy_basic(d, NULL, &flow->bstats) < 0 ||
+        if (gnet_stats_copy_basic(qdisc_root_sleeping_running(sch),
+                                  d, NULL, &flow->bstats) < 0 ||
             gnet_stats_copy_queue(d, NULL, &flow->qstats, flow->q->q.qlen) < 0)
                 return -1;
 
@@ -671,7 +659,6 @@ static struct Qdisc_ops atm_qdisc_ops __read_mostly = {
         .enqueue        = atm_tc_enqueue,
         .dequeue        = atm_tc_dequeue,
         .peek           = atm_tc_peek,
-        .drop           = atm_tc_drop,
         .init           = atm_tc_init,
         .reset          = atm_tc_reset,
         .destroy        = atm_tc_destroy,
diff --git a/net/sched/sch_blackhole.c b/net/sched/sch_blackhole.c
index 3fee70d9814f..c98a61e980ba 100644
--- a/net/sched/sch_blackhole.c
+++ b/net/sched/sch_blackhole.c
@@ -17,9 +17,10 @@
 #include <linux/skbuff.h>
 #include <net/pkt_sched.h>
 
-static int blackhole_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+static int blackhole_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+                             struct sk_buff **to_free)
 {
-        qdisc_drop(skb, sch);
+        qdisc_drop(skb, sch, to_free);
         return NET_XMIT_SUCCESS;
 }
 
diff --git a/net/sched/sch_cbq.c b/net/sched/sch_cbq.c
index baafddf229ce..beb554aa8cfb 100644
--- a/net/sched/sch_cbq.c
+++ b/net/sched/sch_cbq.c
@@ -80,10 +80,6 @@ struct cbq_class {
         unsigned char           priority;       /* class priority */
         unsigned char           priority2;      /* priority to be used after overlimit */
         unsigned char           ewma_log;       /* time constant for idle time calculation */
-        unsigned char           ovl_strategy;
-#ifdef CONFIG_NET_CLS_ACT
-        unsigned char           police;
-#endif
 
         u32                     defmap;
 
@@ -94,10 +90,6 @@ struct cbq_class {
         u32                     avpkt;
         struct qdisc_rate_table *R_tab;
 
-        /* Overlimit strategy parameters */
-        void                    (*overlimit)(struct cbq_class *cl);
-        psched_tdiff_t          penalty;
-
         /* General scheduler (WRR) parameters */
         long                    allot;
         long                    quantum;        /* Allotment per WRR round */
@@ -353,7 +345,7 @@ cbq_mark_toplevel(struct cbq_sched_data *q, struct cbq_class *cl)
 {
         int toplevel = q->toplevel;
 
-        if (toplevel > cl->level && !(qdisc_is_throttled(cl->q))) {
+        if (toplevel > cl->level) {
                 psched_time_t now = psched_get_time();
 
                 do {
@@ -366,7 +358,8 @@ cbq_mark_toplevel(struct cbq_sched_data *q, struct cbq_class *cl)
 }
 
 static int
-cbq_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+cbq_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+            struct sk_buff **to_free)
 {
         struct cbq_sched_data *q = qdisc_priv(sch);
         int uninitialized_var(ret);
@@ -378,14 +371,11 @@ cbq_enqueue(struct sk_buff *skb, struct Qdisc *sch)
         if (cl == NULL) {
                 if (ret & __NET_XMIT_BYPASS)
                         qdisc_qstats_drop(sch);
-                kfree_skb(skb);
+                __qdisc_drop(skb, to_free);
                 return ret;
         }
 
-#ifdef CONFIG_NET_CLS_ACT
-        cl->q->__parent = sch;
-#endif
-        ret = qdisc_enqueue(skb, cl->q);
+        ret = qdisc_enqueue(skb, cl->q, to_free);
         if (ret == NET_XMIT_SUCCESS) {
                 sch->q.qlen++;
                 cbq_mark_toplevel(q, cl);
@@ -402,11 +392,8 @@ cbq_enqueue(struct sk_buff *skb, struct Qdisc *sch)
         return ret;
 }
 
-/* Overlimit actions */
-
-/* TC_CBQ_OVL_CLASSIC: (default) penalize leaf class by adding offtime */
-
-static void cbq_ovl_classic(struct cbq_class *cl)
+/* Overlimit action: penalize leaf class by adding offtime */
+static void cbq_overlimit(struct cbq_class *cl)
 {
         struct cbq_sched_data *q = qdisc_priv(cl->qdisc);
         psched_tdiff_t delay = cl->undertime - q->now;
@@ -456,99 +443,6 @@ static void cbq_ovl_classic(struct cbq_class *cl)
         }
 }
 
-/* TC_CBQ_OVL_RCLASSIC: penalize by offtime classes in hierarchy, when
- * they go overlimit
- */
-
-static void cbq_ovl_rclassic(struct cbq_class *cl)
-{
-        struct cbq_sched_data *q = qdisc_priv(cl->qdisc);
-        struct cbq_class *this = cl;
-
-        do {
-                if (cl->level > q->toplevel) {
-                        cl = NULL;
-                        break;
-                }
-        } while ((cl = cl->borrow) != NULL);
-
-        if (cl == NULL)
-                cl = this;
-        cbq_ovl_classic(cl);
-}
-
-/* TC_CBQ_OVL_DELAY: delay until it will go to underlimit */
-
-static void cbq_ovl_delay(struct cbq_class *cl)
-{
-        struct cbq_sched_data *q = qdisc_priv(cl->qdisc);
-        psched_tdiff_t delay = cl->undertime - q->now;
-
-        if (test_bit(__QDISC_STATE_DEACTIVATED,
-                     &qdisc_root_sleeping(cl->qdisc)->state))
-                return;
-
-        if (!cl->delayed) {
-                psched_time_t sched = q->now;
-                ktime_t expires;
-
-                delay += cl->offtime;
-                if (cl->avgidle < 0)
-                        delay -= (-cl->avgidle) - ((-cl->avgidle) >> cl->ewma_log);
-                if (cl->avgidle < cl->minidle)
-                        cl->avgidle = cl->minidle;
-                cl->undertime = q->now + delay;
-
-                if (delay > 0) {
-                        sched += delay + cl->penalty;
-                        cl->penalized = sched;
-                        cl->cpriority = TC_CBQ_MAXPRIO;
-                        q->pmask |= (1<<TC_CBQ_MAXPRIO);
-
-                        expires = ns_to_ktime(PSCHED_TICKS2NS(sched));
-                        if (hrtimer_try_to_cancel(&q->delay_timer) &&
-                            ktime_to_ns(ktime_sub(
-                                        hrtimer_get_expires(&q->delay_timer),
-                                        expires)) > 0)
-                                hrtimer_set_expires(&q->delay_timer, expires);
-                        hrtimer_restart(&q->delay_timer);
-                        cl->delayed = 1;
-                        cl->xstats.overactions++;
-                        return;
-                }
-                delay = 1;
-        }
-        if (q->wd_expires == 0 || q->wd_expires > delay)
-                q->wd_expires = delay;
-}
-
-/* TC_CBQ_OVL_LOWPRIO: penalize class by lowering its priority band */
-
-static void cbq_ovl_lowprio(struct cbq_class *cl)
-{
-        struct cbq_sched_data *q = qdisc_priv(cl->qdisc);
-
-        cl->penalized = q->now + cl->penalty;
-
-        if (cl->cpriority != cl->priority2) {
-                cl->cpriority = cl->priority2;
-                q->pmask |= (1<<cl->cpriority);
-                cl->xstats.overactions++;
-        }
-        cbq_ovl_classic(cl);
-}
-
-/* TC_CBQ_OVL_DROP: penalize class by dropping */
-
-static void cbq_ovl_drop(struct cbq_class *cl)
-{
-        if (cl->q->ops->drop)
-                if (cl->q->ops->drop(cl->q))
-                        cl->qdisc->q.qlen--;
-        cl->xstats.overactions++;
-        cbq_ovl_classic(cl);
-}
-
 static psched_tdiff_t cbq_undelay_prio(struct cbq_sched_data *q, int prio,
                                        psched_time_t now)
 {
@@ -620,45 +514,10 @@ static enum hrtimer_restart cbq_undelay(struct hrtimer *timer)
                 hrtimer_start(&q->delay_timer, time, HRTIMER_MODE_ABS_PINNED);
         }
 
-        qdisc_unthrottled(sch);
         __netif_schedule(qdisc_root(sch));
         return HRTIMER_NORESTART;
 }
 
-#ifdef CONFIG_NET_CLS_ACT
-static int cbq_reshape_fail(struct sk_buff *skb, struct Qdisc *child)
-{
-        struct Qdisc *sch = child->__parent;
-        struct cbq_sched_data *q = qdisc_priv(sch);
-        struct cbq_class *cl = q->rx_class;
-
-        q->rx_class = NULL;
-
-        if (cl && (cl = cbq_reclassify(skb, cl)) != NULL) {
-                int ret;
-
-                cbq_mark_toplevel(q, cl);
-
-                q->rx_class = cl;
-                cl->q->__parent = sch;
-
-                ret = qdisc_enqueue(skb, cl->q);
-                if (ret == NET_XMIT_SUCCESS) {
-                        sch->q.qlen++;
-                        if (!cl->next_alive)
-                                cbq_activate_class(cl);
-                        return 0;
-                }
-                if (net_xmit_drop_count(ret))
-                        qdisc_qstats_drop(sch);
-                return 0;
-        }
-
-        qdisc_qstats_drop(sch);
-        return -1;
-}
-#endif
-
 /*
  * It is mission critical procedure.
  *
@@ -807,7 +666,7 @@ cbq_under_limit(struct cbq_class *cl)
                 cl = cl->borrow;
                 if (!cl) {
                         this_cl->qstats.overlimits++;
-                        this_cl->overlimit(this_cl);
+                        cbq_overlimit(this_cl);
                         return NULL;
                 }
                 if (cl->level > q->toplevel)
@@ -960,7 +819,6 @@ cbq_dequeue(struct Qdisc *sch)
                 if (skb) {
                         qdisc_bstats_update(sch, skb);
                         sch->q.qlen--;
-                        qdisc_unthrottled(sch);
                         return skb;
                 }
 
@@ -1166,31 +1024,6 @@ static void cbq_link_class(struct cbq_class *this)
         }
 }
 
-static unsigned int cbq_drop(struct Qdisc *sch)
-{
-        struct cbq_sched_data *q = qdisc_priv(sch);
-        struct cbq_class *cl, *cl_head;
-        int prio;
-        unsigned int len;
-
-        for (prio = TC_CBQ_MAXPRIO; prio >= 0; prio--) {
-                cl_head = q->active[prio];
-                if (!cl_head)
-                        continue;
-
-                cl = cl_head;
-                do {
-                        if (cl->q->ops->drop && (len = cl->q->ops->drop(cl->q))) {
-                                sch->q.qlen--;
-                                if (!cl->q->q.qlen)
-                                        cbq_deactivate_class(cl);
-                                return len;
-                        }
-                } while ((cl = cl->next_alive) != cl_head);
-        }
-        return 0;
-}
-
 static void
 cbq_reset(struct Qdisc *sch)
 {
@@ -1280,50 +1113,6 @@ static int cbq_set_wrr(struct cbq_class *cl, struct tc_cbq_wrropt *wrr)
         return 0;
 }
 
-static int cbq_set_overlimit(struct cbq_class *cl, struct tc_cbq_ovl *ovl)
-{
-        switch (ovl->strategy) {
-        case TC_CBQ_OVL_CLASSIC:
-                cl->overlimit = cbq_ovl_classic;
-                break;
-        case TC_CBQ_OVL_DELAY:
-                cl->overlimit = cbq_ovl_delay;
-                break;
-        case TC_CBQ_OVL_LOWPRIO:
-                if (ovl->priority2 - 1 >= TC_CBQ_MAXPRIO ||
-                    ovl->priority2 - 1 <= cl->priority)
-                        return -EINVAL;
-                cl->priority2 = ovl->priority2 - 1;
-                cl->overlimit = cbq_ovl_lowprio;
-                break;
-        case TC_CBQ_OVL_DROP:
-                cl->overlimit = cbq_ovl_drop;
-                break;
-        case TC_CBQ_OVL_RCLASSIC:
-                cl->overlimit = cbq_ovl_rclassic;
-                break;
-        default:
-                return -EINVAL;
-        }
-        cl->penalty = ovl->penalty;
-        return 0;
-}
-
-#ifdef CONFIG_NET_CLS_ACT
-static int cbq_set_police(struct cbq_class *cl, struct tc_cbq_police *p)
-{
-        cl->police = p->police;
-
-        if (cl->q->handle) {
-                if (p->police == TC_POLICE_RECLASSIFY)
-                        cl->q->reshape_fail = cbq_reshape_fail;
-                else
-                        cl->q->reshape_fail = NULL;
-        }
-        return 0;
-}
-#endif
-
 static int cbq_set_fopt(struct cbq_class *cl, struct tc_cbq_fopt *fopt)
 {
         cbq_change_defmap(cl, fopt->split, fopt->defmap, fopt->defchange);
@@ -1375,8 +1164,6 @@ static int cbq_init(struct Qdisc *sch, struct nlattr *opt)
         q->link.priority = TC_CBQ_MAXPRIO - 1;
         q->link.priority2 = TC_CBQ_MAXPRIO - 1;
         q->link.cpriority = TC_CBQ_MAXPRIO - 1;
-        q->link.ovl_strategy = TC_CBQ_OVL_CLASSIC;
-        q->link.overlimit = cbq_ovl_classic;
         q->link.allot = psched_mtu(qdisc_dev(sch));
         q->link.quantum = q->link.allot;
         q->link.weight = q->link.R_tab->rate.rate;
@@ -1463,24 +1250,6 @@ nla_put_failure:
         return -1;
 }
 
-static int cbq_dump_ovl(struct sk_buff *skb, struct cbq_class *cl)
-{
-        unsigned char *b = skb_tail_pointer(skb);
-        struct tc_cbq_ovl opt;
-
-        opt.strategy = cl->ovl_strategy;
-        opt.priority2 = cl->priority2 + 1;
-        opt.pad = 0;
-        opt.penalty = cl->penalty;
-        if (nla_put(skb, TCA_CBQ_OVL_STRATEGY, sizeof(opt), &opt))
-                goto nla_put_failure;
-        return skb->len;
-
-nla_put_failure:
-        nlmsg_trim(skb, b);
-        return -1;
-}
-
 static int cbq_dump_fopt(struct sk_buff *skb, struct cbq_class *cl)
 {
         unsigned char *b = skb_tail_pointer(skb);
@@ -1500,36 +1269,11 @@ nla_put_failure:
         return -1;
 }
 
-#ifdef CONFIG_NET_CLS_ACT
-static int cbq_dump_police(struct sk_buff *skb, struct cbq_class *cl)
-{
-        unsigned char *b = skb_tail_pointer(skb);
-        struct tc_cbq_police opt;
-
-        if (cl->police) {
-                opt.police = cl->police;
-                opt.__res1 = 0;
-                opt.__res2 = 0;
-                if (nla_put(skb, TCA_CBQ_POLICE, sizeof(opt), &opt))
-                        goto nla_put_failure;
-        }
-        return skb->len;
-
-nla_put_failure:
-        nlmsg_trim(skb, b);
-        return -1;
-}
-#endif
-
 static int cbq_dump_attr(struct sk_buff *skb, struct cbq_class *cl)
 {
         if (cbq_dump_lss(skb, cl) < 0 ||
             cbq_dump_rate(skb, cl) < 0 ||
             cbq_dump_wrr(skb, cl) < 0 ||
-            cbq_dump_ovl(skb, cl) < 0 ||
-#ifdef CONFIG_NET_CLS_ACT
-            cbq_dump_police(skb, cl) < 0 ||
-#endif
             cbq_dump_fopt(skb, cl) < 0)
                 return -1;
         return 0;
@@ -1600,7 +1344,8 @@ cbq_dump_class_stats(struct Qdisc *sch, unsigned long arg,
         if (cl->undertime != PSCHED_PASTPERFECT)
                 cl->xstats.undertime = cl->undertime - q->now;
 
-        if (gnet_stats_copy_basic(d, NULL, &cl->bstats) < 0 ||
+        if (gnet_stats_copy_basic(qdisc_root_sleeping_running(sch),
+                                  d, NULL, &cl->bstats) < 0 ||
             gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 ||
             gnet_stats_copy_queue(d, NULL, &cl->qstats, cl->q->q.qlen) < 0)
                 return -1;
@@ -1618,11 +1363,6 @@ static int cbq_graft(struct Qdisc *sch, unsigned long arg, struct Qdisc *new,
                                         &pfifo_qdisc_ops, cl->common.classid);
                 if (new == NULL)
                         return -ENOBUFS;
-        } else {
-#ifdef CONFIG_NET_CLS_ACT
-                if (cl->police == TC_POLICE_RECLASSIFY)
-                        new->reshape_fail = cbq_reshape_fail;
-#endif
         }
 
         *old = qdisc_replace(sch, new, &cl->q);
@@ -1735,6 +1475,9 @@ cbq_change_class(struct Qdisc *sch, u32 classid, u32 parentid, struct nlattr **t
         if (err < 0)
                 return err;
 
+        if (tb[TCA_CBQ_OVL_STRATEGY] || tb[TCA_CBQ_POLICE])
+                return -EOPNOTSUPP;
+
         if (cl) {
                 /* Check parent */
                 if (parentid) {
@@ -1755,7 +1498,8 @@ cbq_change_class(struct Qdisc *sch, u32 classid, u32 parentid, struct nlattr **t
                 if (tca[TCA_RATE]) {
                         err = gen_replace_estimator(&cl->bstats, NULL,
                                                     &cl->rate_est,
-                                                    qdisc_root_sleeping_lock(sch),
+                                                    NULL,
+                                                    qdisc_root_sleeping_running(sch),
                                                     tca[TCA_RATE]);
                         if (err) {
                                 qdisc_put_rtab(rtab);
@@ -1782,14 +1526,6 @@ cbq_change_class(struct Qdisc *sch, u32 classid, u32 parentid, struct nlattr **t
                         cbq_set_wrr(cl, nla_data(tb[TCA_CBQ_WRROPT]));
                 }
 
-                if (tb[TCA_CBQ_OVL_STRATEGY])
-                        cbq_set_overlimit(cl, nla_data(tb[TCA_CBQ_OVL_STRATEGY]));
-
-#ifdef CONFIG_NET_CLS_ACT
-                if (tb[TCA_CBQ_POLICE])
-                        cbq_set_police(cl, nla_data(tb[TCA_CBQ_POLICE]));
-#endif
-
                 if (tb[TCA_CBQ_FOPT])
                         cbq_set_fopt(cl, nla_data(tb[TCA_CBQ_FOPT]));
 
@@ -1848,7 +1584,8 @@ cbq_change_class(struct Qdisc *sch, u32 classid, u32 parentid, struct nlattr **t
 
         if (tca[TCA_RATE]) {
                 err = gen_new_estimator(&cl->bstats, NULL, &cl->rate_est,
-                                        qdisc_root_sleeping_lock(sch),
+                                        NULL,
+                                        qdisc_root_sleeping_running(sch),
                                         tca[TCA_RATE]);
                 if (err) {
                         kfree(cl);
@@ -1884,13 +1621,6 @@ cbq_change_class(struct Qdisc *sch, u32 classid, u32 parentid, struct nlattr **t
                 cl->maxidle = q->link.maxidle;
         if (cl->avpkt == 0)
                 cl->avpkt = q->link.avpkt;
-        cl->overlimit = cbq_ovl_classic;
-        if (tb[TCA_CBQ_OVL_STRATEGY])
-                cbq_set_overlimit(cl, nla_data(tb[TCA_CBQ_OVL_STRATEGY]));
-#ifdef CONFIG_NET_CLS_ACT
-        if (tb[TCA_CBQ_POLICE])
-                cbq_set_police(cl, nla_data(tb[TCA_CBQ_POLICE]));
-#endif
         if (tb[TCA_CBQ_FOPT])
                 cbq_set_fopt(cl, nla_data(tb[TCA_CBQ_FOPT]));
         sch_tree_unlock(sch);
@@ -2035,7 +1765,6 @@ static struct Qdisc_ops cbq_qdisc_ops __read_mostly = {
         .enqueue        =       cbq_enqueue,
         .dequeue        =       cbq_dequeue,
         .peek           =       qdisc_peek_dequeued,
-        .drop           =       cbq_drop,
         .init           =       cbq_init,
         .reset          =       cbq_reset,
         .destroy        =       cbq_destroy,
diff --git a/net/sched/sch_choke.c b/net/sched/sch_choke.c
index 0a08c860eee4..3b6d5bd69101 100644
--- a/net/sched/sch_choke.c
+++ b/net/sched/sch_choke.c
@@ -115,7 +115,8 @@ static void choke_zap_tail_holes(struct choke_sched_data *q)
 }
 
 /* Drop packet from queue array by creating a "hole" */
-static void choke_drop_by_idx(struct Qdisc *sch, unsigned int idx)
+static void choke_drop_by_idx(struct Qdisc *sch, unsigned int idx,
+                              struct sk_buff **to_free)
 {
         struct choke_sched_data *q = qdisc_priv(sch);
         struct sk_buff *skb = q->tab[idx];
@@ -129,7 +130,7 @@ static void choke_drop_by_idx(struct Qdisc *sch, unsigned int idx)
 
         qdisc_qstats_backlog_dec(sch, skb);
         qdisc_tree_reduce_backlog(sch, 1, qdisc_pkt_len(skb));
-        qdisc_drop(skb, sch);
+        qdisc_drop(skb, sch, to_free);
         --sch->q.qlen;
 }
 
@@ -261,7 +262,8 @@ static bool choke_match_random(const struct choke_sched_data *q,
         return choke_match_flow(oskb, nskb);
 }
 
-static int choke_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+static int choke_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+                         struct sk_buff **to_free)
 {
         int ret = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
         struct choke_sched_data *q = qdisc_priv(sch);
@@ -288,7 +290,7 @@ static int choke_enqueue(struct sk_buff *skb, struct Qdisc *sch)
                 /* Draw a packet at random from queue and compare flow */
                 if (choke_match_random(q, skb, &idx)) {
                         q->stats.matched++;
-                        choke_drop_by_idx(sch, idx);
+                        choke_drop_by_idx(sch, idx, to_free);
                         goto congestion_drop;
                 }
 
@@ -331,16 +333,16 @@ static int choke_enqueue(struct sk_buff *skb, struct Qdisc *sch)
         }
 
         q->stats.pdrop++;
-        return qdisc_drop(skb, sch);
+        return qdisc_drop(skb, sch, to_free);
 
 congestion_drop:
-        qdisc_drop(skb, sch);
+        qdisc_drop(skb, sch, to_free);
         return NET_XMIT_CN;
 
 other_drop:
         if (ret & __NET_XMIT_BYPASS)
                 qdisc_qstats_drop(sch);
-        kfree_skb(skb);
+        __qdisc_drop(skb, to_free);
         return ret;
 }
 
@@ -365,22 +367,6 @@ static struct sk_buff *choke_dequeue(struct Qdisc *sch)
         return skb;
 }
 
-static unsigned int choke_drop(struct Qdisc *sch)
-{
-        struct choke_sched_data *q = qdisc_priv(sch);
-        unsigned int len;
-
-        len = qdisc_queue_drop(sch);
-        if (len > 0)
-                q->stats.other++;
-        else {
-                if (!red_is_idling(&q->vars))
-                        red_start_of_idle_period(&q->vars);
-        }
-
-        return len;
-}
-
 static void choke_reset(struct Qdisc *sch)
 {
         struct choke_sched_data *q = qdisc_priv(sch);
@@ -391,11 +377,11 @@ static void choke_reset(struct Qdisc *sch)
                 q->head = (q->head + 1) & q->tab_mask;
                 if (!skb)
                         continue;
-                qdisc_qstats_backlog_dec(sch, skb);
-                --sch->q.qlen;
-                qdisc_drop(skb, sch);
+                rtnl_qdisc_drop(skb, sch);
         }
 
+        sch->q.qlen = 0;
+        sch->qstats.backlog = 0;
         memset(q->tab, 0, (q->tab_mask + 1) * sizeof(struct sk_buff *));
         q->head = q->tail = 0;
         red_restart(&q->vars);
@@ -471,7 +457,7 @@ static int choke_change(struct Qdisc *sch, struct nlattr *opt)
                                 dropped += qdisc_pkt_len(skb);
                                 qdisc_qstats_backlog_dec(sch, skb);
                                 --sch->q.qlen;
-                                qdisc_drop(skb, sch);
+                                rtnl_qdisc_drop(skb, sch);
                         }
                         qdisc_tree_reduce_backlog(sch, oqlen - sch->q.qlen, dropped);
                         q->head = 0;
@@ -569,7 +555,6 @@ static struct Qdisc_ops choke_qdisc_ops __read_mostly = {
         .enqueue        =       choke_enqueue,
         .dequeue        =       choke_dequeue,
         .peek           =       choke_peek_head,
-        .drop           =       choke_drop,
         .init           =       choke_init,
         .destroy        =       choke_destroy,
         .reset          =       choke_reset,
diff --git a/net/sched/sch_codel.c b/net/sched/sch_codel.c
index dddf3bb65a32..4002df3c7d9f 100644
--- a/net/sched/sch_codel.c
+++ b/net/sched/sch_codel.c
@@ -82,7 +82,8 @@ static void drop_func(struct sk_buff *skb, void *ctx)
 {
         struct Qdisc *sch = ctx;
 
-        qdisc_drop(skb, sch);
+        kfree_skb(skb);
+        qdisc_qstats_drop(sch);
 }
 
 static struct sk_buff *codel_qdisc_dequeue(struct Qdisc *sch)
@@ -107,7 +108,8 @@ static struct sk_buff *codel_qdisc_dequeue(struct Qdisc *sch)
         return skb;
 }
 
-static int codel_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+static int codel_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+                               struct sk_buff **to_free)
 {
         struct codel_sched_data *q;
 
@@ -117,7 +119,7 @@ static int codel_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch)
         }
         q = qdisc_priv(sch);
         q->drop_overlimit++;
-        return qdisc_drop(skb, sch);
+        return qdisc_drop(skb, sch, to_free);
 }
 
 static const struct nla_policy codel_policy[TCA_CODEL_MAX + 1] = {
@@ -174,7 +176,7 @@ static int codel_change(struct Qdisc *sch, struct nlattr *opt)
 
                 dropped += qdisc_pkt_len(skb);
                 qdisc_qstats_backlog_dec(sch, skb);
-                qdisc_drop(skb, sch);
+                rtnl_qdisc_drop(skb, sch);
         }
         qdisc_tree_reduce_backlog(sch, qlen - sch->q.qlen, dropped);
 
diff --git a/net/sched/sch_drr.c b/net/sched/sch_drr.c
index bf8af2c43c2c..8af5c59eef84 100644
--- a/net/sched/sch_drr.c
+++ b/net/sched/sch_drr.c
@@ -91,7 +91,8 @@ static int drr_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
                 if (tca[TCA_RATE]) {
                         err = gen_replace_estimator(&cl->bstats, NULL,
                                                     &cl->rate_est,
-                                                    qdisc_root_sleeping_lock(sch),
+                                                    NULL,
+                                                    qdisc_root_sleeping_running(sch),
                                                     tca[TCA_RATE]);
                         if (err)
                                 return err;
@@ -119,7 +120,8 @@ static int drr_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
 
         if (tca[TCA_RATE]) {
                 err = gen_replace_estimator(&cl->bstats, NULL, &cl->rate_est,
-                                            qdisc_root_sleeping_lock(sch),
+                                            NULL,
+                                            qdisc_root_sleeping_running(sch),
                                             tca[TCA_RATE]);
                 if (err) {
                         qdisc_destroy(cl->qdisc);
@@ -279,7 +281,8 @@ static int drr_dump_class_stats(struct Qdisc *sch, unsigned long arg,
         if (qlen)
                 xstats.deficit = cl->deficit;
 
-        if (gnet_stats_copy_basic(d, NULL, &cl->bstats) < 0 ||
+        if (gnet_stats_copy_basic(qdisc_root_sleeping_running(sch),
+                                  d, NULL, &cl->bstats) < 0 ||
             gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 ||
             gnet_stats_copy_queue(d, NULL, &cl->qdisc->qstats, qlen) < 0)
                 return -1;
@@ -347,7 +350,8 @@ static struct drr_class *drr_classify(struct sk_buff *skb, struct Qdisc *sch,
         return NULL;
 }
 
-static int drr_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+static int drr_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+                       struct sk_buff **to_free)
 {
         struct drr_sched *q = qdisc_priv(sch);
         struct drr_class *cl;
@@ -357,11 +361,11 @@ static int drr_enqueue(struct sk_buff *skb, struct Qdisc *sch)
         if (cl == NULL) {
                 if (err & __NET_XMIT_BYPASS)
                         qdisc_qstats_drop(sch);
-                kfree_skb(skb);
+                __qdisc_drop(skb, to_free);
                 return err;
         }
 
-        err = qdisc_enqueue(skb, cl->qdisc);
+        err = qdisc_enqueue(skb, cl->qdisc, to_free);
         if (unlikely(err != NET_XMIT_SUCCESS)) {
                 if (net_xmit_drop_count(err)) {
                         cl->qstats.drops++;
@@ -420,27 +424,6 @@ out:
         return NULL;
 }
 
-static unsigned int drr_drop(struct Qdisc *sch)
-{
-        struct drr_sched *q = qdisc_priv(sch);
-        struct drr_class *cl;
-        unsigned int len;
-
-        list_for_each_entry(cl, &q->active, alist) {
-                if (cl->qdisc->ops->drop) {
-                        len = cl->qdisc->ops->drop(cl->qdisc);
-                        if (len > 0) {
-                                sch->qstats.backlog -= len;
-                                sch->q.qlen--;
-                                if (cl->qdisc->q.qlen == 0)
-                                        list_del(&cl->alist);
-                                return len;
-                        }
-                }
-        }
-        return 0;
-}
-
 static int drr_init_qdisc(struct Qdisc *sch, struct nlattr *opt)
 {
         struct drr_sched *q = qdisc_priv(sch);
@@ -510,7 +493,6 @@ static struct Qdisc_ops drr_qdisc_ops __read_mostly = {
         .enqueue        = drr_enqueue,
         .dequeue        = drr_dequeue,
         .peek           = qdisc_peek_dequeued,
-        .drop           = drr_drop,
         .init           = drr_init_qdisc,
         .reset          = drr_reset_qdisc,
         .destroy        = drr_destroy_qdisc,
diff --git a/net/sched/sch_dsmark.c b/net/sched/sch_dsmark.c
index 34b4ddaca27c..1308bbf460f7 100644
--- a/net/sched/sch_dsmark.c
+++ b/net/sched/sch_dsmark.c
@@ -191,7 +191,8 @@ static inline struct tcf_proto __rcu **dsmark_find_tcf(struct Qdisc *sch,
 
 /* --------------------------- Qdisc operations ---------------------------- */
 
-static int dsmark_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+static int dsmark_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+                          struct sk_buff **to_free)
 {
         struct dsmark_qdisc_data *p = qdisc_priv(sch);
         int err;
@@ -234,7 +235,7 @@ static int dsmark_enqueue(struct sk_buff *skb, struct Qdisc *sch)
 #ifdef CONFIG_NET_CLS_ACT
                 case TC_ACT_QUEUED:
                 case TC_ACT_STOLEN:
-                        kfree_skb(skb);
+                        __qdisc_drop(skb, to_free);
                         return NET_XMIT_SUCCESS | __NET_XMIT_STOLEN;
 
                 case TC_ACT_SHOT:
@@ -251,7 +252,7 @@ static int dsmark_enqueue(struct sk_buff *skb, struct Qdisc *sch)
                 }
         }
 
-        err = qdisc_enqueue(skb, p->q);
+        err = qdisc_enqueue(skb, p->q, to_free);
         if (err != NET_XMIT_SUCCESS) {
                 if (net_xmit_drop_count(err))
                         qdisc_qstats_drop(sch);
@@ -264,7 +265,7 @@ static int dsmark_enqueue(struct sk_buff *skb, struct Qdisc *sch)
         return NET_XMIT_SUCCESS;
 
 drop:
-        qdisc_drop(skb, sch);
+        qdisc_drop(skb, sch, to_free);
         return NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
 }
 
@@ -320,23 +321,6 @@ static struct sk_buff *dsmark_peek(struct Qdisc *sch)
         return p->q->ops->peek(p->q);
 }
 
-static unsigned int dsmark_drop(struct Qdisc *sch)
-{
-        struct dsmark_qdisc_data *p = qdisc_priv(sch);
-        unsigned int len;
-
-        pr_debug("%s(sch %p,[qdisc %p])\n", __func__, sch, p);
-
-        if (p->q->ops->drop == NULL)
-                return 0;
-
-        len = p->q->ops->drop(p->q);
-        if (len)
-                sch->q.qlen--;
-
-        return len;
-}
-
 static int dsmark_init(struct Qdisc *sch, struct nlattr *opt)
 {
         struct dsmark_qdisc_data *p = qdisc_priv(sch);
@@ -489,7 +473,6 @@ static struct Qdisc_ops dsmark_qdisc_ops __read_mostly = {
         .enqueue        =       dsmark_enqueue,
         .dequeue        =       dsmark_dequeue,
         .peek           =       dsmark_peek,
-        .drop           =       dsmark_drop,
         .init           =       dsmark_init,
         .reset          =       dsmark_reset,
         .destroy        =       dsmark_destroy,
diff --git a/net/sched/sch_fifo.c b/net/sched/sch_fifo.c
index 2e4bd2c0a50c..baeed6a78d28 100644
--- a/net/sched/sch_fifo.c
+++ b/net/sched/sch_fifo.c
@@ -19,23 +19,26 @@
 
 /* 1 band FIFO pseudo-"scheduler" */
 
-static int bfifo_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+static int bfifo_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+                         struct sk_buff **to_free)
 {
         if (likely(sch->qstats.backlog + qdisc_pkt_len(skb) <= sch->limit))
                 return qdisc_enqueue_tail(skb, sch);
 
-        return qdisc_reshape_fail(skb, sch);
+        return qdisc_drop(skb, sch, to_free);
 }
 
-static int pfifo_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+static int pfifo_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+                         struct sk_buff **to_free)
 {
         if (likely(skb_queue_len(&sch->q) < sch->limit))
                 return qdisc_enqueue_tail(skb, sch);
 
-        return qdisc_reshape_fail(skb, sch);
+        return qdisc_drop(skb, sch, to_free);
 }
 
-static int pfifo_tail_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+static int pfifo_tail_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+                              struct sk_buff **to_free)
 {
         unsigned int prev_backlog;
 
@@ -44,7 +47,7 @@ static int pfifo_tail_enqueue(struct sk_buff *skb, struct Qdisc *sch)
 
         prev_backlog = sch->qstats.backlog;
         /* queue full, remove one skb to fulfill the limit */
-        __qdisc_queue_drop_head(sch, &sch->q);
+        __qdisc_queue_drop_head(sch, &sch->q, to_free);
         qdisc_qstats_drop(sch);
         qdisc_enqueue_tail(skb, sch);
 
@@ -103,7 +106,6 @@ struct Qdisc_ops pfifo_qdisc_ops __read_mostly = {
         .enqueue        =       pfifo_enqueue,
         .dequeue        =       qdisc_dequeue_head,
         .peek           =       qdisc_peek_head,
-        .drop           =       qdisc_queue_drop,
         .init           =       fifo_init,
         .reset          =       qdisc_reset_queue,
         .change         =       fifo_init,
@@ -118,7 +120,6 @@ struct Qdisc_ops bfifo_qdisc_ops __read_mostly = {
         .enqueue        =       bfifo_enqueue,
         .dequeue        =       qdisc_dequeue_head,
         .peek           =       qdisc_peek_head,
-        .drop           =       qdisc_queue_drop,
         .init           =       fifo_init,
         .reset          =       qdisc_reset_queue,
         .change         =       fifo_init,
@@ -133,7 +134,6 @@ struct Qdisc_ops pfifo_head_drop_qdisc_ops __read_mostly = {
         .enqueue        =       pfifo_tail_enqueue,
         .dequeue        =       qdisc_dequeue_head,
         .peek           =       qdisc_peek_head,
-        .drop           =       qdisc_queue_drop_head,
         .init           =       fifo_init,
         .reset          =       qdisc_reset_queue,
         .change         =       fifo_init,
diff --git a/net/sched/sch_fq.c b/net/sched/sch_fq.c
index 3c6a47d66a04..e5458b99e09c 100644
--- a/net/sched/sch_fq.c
+++ b/net/sched/sch_fq.c
@@ -368,18 +368,19 @@ static void flow_queue_add(struct fq_flow *flow, struct sk_buff *skb)
         }
 }
 
-static int fq_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+static int fq_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+                      struct sk_buff **to_free)
 {
         struct fq_sched_data *q = qdisc_priv(sch);
         struct fq_flow *f;
 
         if (unlikely(sch->q.qlen >= sch->limit))
-                return qdisc_drop(skb, sch);
+                return qdisc_drop(skb, sch, to_free);
 
         f = fq_classify(skb, q);
         if (unlikely(f->qlen >= q->flow_plimit && f != &q->internal)) {
                 q->stat_flows_plimit++;
-                return qdisc_drop(skb, sch);
+                return qdisc_drop(skb, sch, to_free);
         }
 
         f->qlen++;
@@ -445,8 +446,7 @@ begin:
                 if (!head->first) {
                         if (q->time_next_delayed_flow != ~0ULL)
                                 qdisc_watchdog_schedule_ns(&q->watchdog,
-                                                           q->time_next_delayed_flow,
-                                                           false);
+                                                           q->time_next_delayed_flow);
                         return NULL;
                 }
         }
@@ -515,17 +515,25 @@ out:
         return skb;
 }
 
+static void fq_flow_purge(struct fq_flow *flow)
+{
+        rtnl_kfree_skbs(flow->head, flow->tail);
+        flow->head = NULL;
+        flow->qlen = 0;
+}
+
 static void fq_reset(struct Qdisc *sch)
 {
         struct fq_sched_data *q = qdisc_priv(sch);
         struct rb_root *root;
-        struct sk_buff *skb;
         struct rb_node *p;
         struct fq_flow *f;
         unsigned int idx;
 
-        while ((skb = fq_dequeue_head(sch, &q->internal)) != NULL)
-                kfree_skb(skb);
+        sch->q.qlen = 0;
+        sch->qstats.backlog = 0;
+
+        fq_flow_purge(&q->internal);
 
         if (!q->fq_root)
                 return;
@@ -536,8 +544,7 @@ static void fq_reset(struct Qdisc *sch)
                         f = container_of(p, struct fq_flow, fq_node);
                         rb_erase(p, root);
 
-                        while ((skb = fq_dequeue_head(sch, f)) != NULL)
-                                kfree_skb(skb);
+                        fq_flow_purge(f);
 
                         kmem_cache_free(fq_flow_cachep, f);
                 }
@@ -738,7 +745,7 @@ static int fq_change(struct Qdisc *sch, struct nlattr *opt)
                 if (!skb)
                         break;
                 drop_len += qdisc_pkt_len(skb);
-                kfree_skb(skb);
+                rtnl_kfree_skbs(skb, skb);
                 drop_count++;
         }
         qdisc_tree_reduce_backlog(sch, drop_count, drop_len);
diff --git a/net/sched/sch_fq_codel.c b/net/sched/sch_fq_codel.c
index da250b2e06ae..a5ea0e9b6be4 100644
--- a/net/sched/sch_fq_codel.c
+++ b/net/sched/sch_fq_codel.c
@@ -139,7 +139,8 @@ static inline void flow_queue_add(struct fq_codel_flow *flow,
         skb->next = NULL;
 }
 
-static unsigned int fq_codel_drop(struct Qdisc *sch, unsigned int max_packets)
+static unsigned int fq_codel_drop(struct Qdisc *sch, unsigned int max_packets,
+                                  struct sk_buff **to_free)
 {
         struct fq_codel_sched_data *q = qdisc_priv(sch);
         struct sk_buff *skb;
@@ -171,8 +172,8 @@ static unsigned int fq_codel_drop(struct Qdisc *sch, unsigned int max_packets)
         do {
                 skb = dequeue_head(flow);
                 len += qdisc_pkt_len(skb);
-                mem += skb->truesize;
-                kfree_skb(skb);
+                mem += get_codel_cb(skb)->mem_usage;
+                __qdisc_drop(skb, to_free);
         } while (++i < max_packets && len < threshold);
 
         flow->dropped += i;
@@ -184,16 +185,8 @@ static unsigned int fq_codel_drop(struct Qdisc *sch, unsigned int max_packets)
         return idx;
 }
 
-static unsigned int fq_codel_qdisc_drop(struct Qdisc *sch)
-{
-        unsigned int prev_backlog;
-
-        prev_backlog = sch->qstats.backlog;
-        fq_codel_drop(sch, 1U);
-        return prev_backlog - sch->qstats.backlog;
-}
-
-static int fq_codel_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+static int fq_codel_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+                            struct sk_buff **to_free)
 {
         struct fq_codel_sched_data *q = qdisc_priv(sch);
         unsigned int idx, prev_backlog, prev_qlen;
@@ -206,7 +199,7 @@ static int fq_codel_enqueue(struct sk_buff *skb, struct Qdisc *sch)
         if (idx == 0) {
                 if (ret & __NET_XMIT_BYPASS)
                         qdisc_qstats_drop(sch);
-                kfree_skb(skb);
+                __qdisc_drop(skb, to_free);
                 return ret;
         }
         idx--;
@@ -223,7 +216,8 @@ static int fq_codel_enqueue(struct sk_buff *skb, struct Qdisc *sch)
                 flow->deficit = q->quantum;
                 flow->dropped = 0;
         }
-        q->memory_usage += skb->truesize;
+        get_codel_cb(skb)->mem_usage = skb->truesize;
+        q->memory_usage += get_codel_cb(skb)->mem_usage;
         memory_limited = q->memory_usage > q->memory_limit;
         if (++sch->q.qlen <= sch->limit && !memory_limited)
                 return NET_XMIT_SUCCESS;
@@ -238,7 +232,7 @@ static int fq_codel_enqueue(struct sk_buff *skb, struct Qdisc *sch)
          * So instead of dropping a single packet, drop half of its backlog
          * with a 64 packets limit to not add a too big cpu spike here.
          */
-        ret = fq_codel_drop(sch, q->drop_batch_size);
+        ret = fq_codel_drop(sch, q->drop_batch_size, to_free);
 
         prev_qlen -= sch->q.qlen;
         prev_backlog -= sch->qstats.backlog;
@@ -274,7 +268,7 @@ static struct sk_buff *dequeue_func(struct codel_vars *vars, void *ctx)
         if (flow->head) {
                 skb = dequeue_head(flow);
                 q->backlogs[flow - q->flows] -= qdisc_pkt_len(skb);
-                q->memory_usage -= skb->truesize;
+                q->memory_usage -= get_codel_cb(skb)->mem_usage;
                 sch->q.qlen--;
                 sch->qstats.backlog -= qdisc_pkt_len(skb);
         }
@@ -285,7 +279,8 @@ static void drop_func(struct sk_buff *skb, void *ctx)
 {
         struct Qdisc *sch = ctx;
 
-        qdisc_drop(skb, sch);
+        kfree_skb(skb);
+        qdisc_qstats_drop(sch);
 }
 
 static struct sk_buff *fq_codel_dequeue(struct Qdisc *sch)
@@ -345,6 +340,12 @@ begin:
         return skb;
 }
 
+static void fq_codel_flow_purge(struct fq_codel_flow *flow)
+{
+        rtnl_kfree_skbs(flow->head, flow->tail);
+        flow->head = NULL;
+}
+
 static void fq_codel_reset(struct Qdisc *sch)
 {
         struct fq_codel_sched_data *q = qdisc_priv(sch);
@@ -355,18 +356,13 @@ static void fq_codel_reset(struct Qdisc *sch)
         for (i = 0; i < q->flows_cnt; i++) {
                 struct fq_codel_flow *flow = q->flows + i;
 
-                while (flow->head) {
-                        struct sk_buff *skb = dequeue_head(flow);
-
-                        qdisc_qstats_backlog_dec(sch, skb);
-                        kfree_skb(skb);
-                }
-
+                fq_codel_flow_purge(flow);
                 INIT_LIST_HEAD(&flow->flowchain);
                 codel_vars_init(&flow->cvars);
         }
         memset(q->backlogs, 0, q->flows_cnt * sizeof(u32));
         sch->q.qlen = 0;
+        sch->qstats.backlog = 0;
         q->memory_usage = 0;
 }
 
@@ -442,7 +438,7 @@ static int fq_codel_change(struct Qdisc *sch, struct nlattr *opt)
                 struct sk_buff *skb = fq_codel_dequeue(sch);
 
                 q->cstats.drop_len += qdisc_pkt_len(skb);
-                kfree_skb(skb);
+                rtnl_kfree_skbs(skb, skb);
                 q->cstats.drop_count++;
         }
         qdisc_tree_reduce_backlog(sch, q->cstats.drop_count, q->cstats.drop_len);
@@ -578,11 +574,13 @@ static int fq_codel_dump_stats(struct Qdisc *sch, struct gnet_dump *d)
         st.qdisc_stats.memory_usage  = q->memory_usage;
         st.qdisc_stats.drop_overmemory = q->drop_overmemory;
 
+        sch_tree_lock(sch);
         list_for_each(pos, &q->new_flows)
                 st.qdisc_stats.new_flows_len++;
 
         list_for_each(pos, &q->old_flows)
                 st.qdisc_stats.old_flows_len++;
+        sch_tree_unlock(sch);
 
         return gnet_stats_copy_app(d, &st, sizeof(st));
 }
@@ -636,7 +634,7 @@ static int fq_codel_dump_class_stats(struct Qdisc *sch, unsigned long cl,
 
         if (idx < q->flows_cnt) {
                 const struct fq_codel_flow *flow = &q->flows[idx];
-                const struct sk_buff *skb = flow->head;
+                const struct sk_buff *skb;
 
                 memset(&xstats, 0, sizeof(xstats));
                 xstats.type = TCA_FQ_CODEL_XSTATS_CLASS;
@@ -654,9 +652,14 @@ static int fq_codel_dump_class_stats(struct Qdisc *sch, unsigned long cl,
                                 codel_time_to_us(delta) :
                                 -codel_time_to_us(-delta);
                 }
-                while (skb) {
-                        qs.qlen++;
-                        skb = skb->next;
+                if (flow->head) {
+                        sch_tree_lock(sch);
+                        skb = flow->head;
+                        while (skb) {
+                                qs.qlen++;
+                                skb = skb->next;
+                        }
+                        sch_tree_unlock(sch);
                 }
                 qs.backlog = q->backlogs[idx];
                 qs.drops = flow->dropped;
@@ -709,7 +712,6 @@ static struct Qdisc_ops fq_codel_qdisc_ops __read_mostly = {
         .enqueue        =       fq_codel_enqueue,
         .dequeue        =       fq_codel_dequeue,
         .peek           =       qdisc_peek_dequeued,
-        .drop           =       fq_codel_qdisc_drop,
         .init           =       fq_codel_init,
         .reset          =       fq_codel_reset,
         .destroy        =       fq_codel_destroy,
diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c
index f9e0e9c03d0a..e95b67cd5718 100644
--- a/net/sched/sch_generic.c
+++ b/net/sched/sch_generic.c
@@ -77,6 +77,34 @@ static void try_bulk_dequeue_skb(struct Qdisc *q,
         skb->next = NULL;
 }
 
+/* This variant of try_bulk_dequeue_skb() makes sure
+ * all skbs in the chain are for the same txq
+ */
+static void try_bulk_dequeue_skb_slow(struct Qdisc *q,
+                                      struct sk_buff *skb,
+                                      int *packets)
+{
+        int mapping = skb_get_queue_mapping(skb);
+        struct sk_buff *nskb;
+        int cnt = 0;
+
+        do {
+                nskb = q->dequeue(q);
+                if (!nskb)
+                        break;
+                if (unlikely(skb_get_queue_mapping(nskb) != mapping)) {
+                        q->skb_bad_txq = nskb;
+                        qdisc_qstats_backlog_inc(q, nskb);
+                        q->q.qlen++;
+                        break;
+                }
+                skb->next = nskb;
+                skb = nskb;
+        } while (++cnt < 8);
+        (*packets) += cnt;
+        skb->next = NULL;
+}
+
 /* Note that dequeue_skb can possibly return a SKB list (via skb->next).
  * A requeued skb (via q->gso_skb) can also be a SKB list.
  */
@@ -87,8 +115,9 @@ static struct sk_buff *dequeue_skb(struct Qdisc *q, bool *validate,
         const struct netdev_queue *txq = q->dev_queue;
 
         *packets = 1;
-        *validate = true;
         if (unlikely(skb)) {
+                /* skb in gso_skb were already validated */
+                *validate = false;
                 /* check the reason of requeuing without tx lock first */
                 txq = skb_get_tx_queue(txq->dev, skb);
                 if (!netif_xmit_frozen_or_stopped(txq)) {
@@ -97,22 +126,37 @@ static struct sk_buff *dequeue_skb(struct Qdisc *q, bool *validate,
                         q->q.qlen--;
                 } else
                         skb = NULL;
-                /* skb in gso_skb were already validated */
-                *validate = false;
-        } else {
-                if (!(q->flags & TCQ_F_ONETXQUEUE) ||
-                    !netif_xmit_frozen_or_stopped(txq)) {
-                        skb = q->dequeue(q);
-                        if (skb && qdisc_may_bulk(q))
-                                try_bulk_dequeue_skb(q, skb, txq, packets);
+                return skb;
+        }
+        *validate = true;
+        skb = q->skb_bad_txq;
+        if (unlikely(skb)) {
+                /* check the reason of requeuing without tx lock first */
+                txq = skb_get_tx_queue(txq->dev, skb);
+                if (!netif_xmit_frozen_or_stopped(txq)) {
+                        q->skb_bad_txq = NULL;
+                        qdisc_qstats_backlog_dec(q, skb);
+                        q->q.qlen--;
+                        goto bulk;
                 }
+                return NULL;
+        }
+        if (!(q->flags & TCQ_F_ONETXQUEUE) ||
+            !netif_xmit_frozen_or_stopped(txq))
+                skb = q->dequeue(q);
+        if (skb) {
+bulk:
+                if (qdisc_may_bulk(q))
+                        try_bulk_dequeue_skb(q, skb, txq, packets);
+                else
+                        try_bulk_dequeue_skb_slow(q, skb, packets);
         }
         return skb;
 }
 
 /*
  * Transmit possibly several skbs, and handle the return status as
- * required. Holding the __QDISC___STATE_RUNNING bit guarantees that
+ * required. Owning running seqcount bit guarantees that
  * only one CPU can execute this function.
  *
  * Returns to the caller:
@@ -165,7 +209,7 @@ int sch_direct_xmit(struct sk_buff *skb, struct Qdisc *q,
 /*
  * NOTE: Called under qdisc_lock(q) with locally disabled BH.
  *
- * __QDISC___STATE_RUNNING guarantees only one CPU can process
+ * running seqcount guarantees only one CPU can process
  * this qdisc at a time. qdisc_lock(q) serializes queue accesses for
  * this queue.
  *
@@ -348,9 +392,10 @@ EXPORT_SYMBOL(netif_carrier_off);
    cheaper.
  */
 
-static int noop_enqueue(struct sk_buff *skb, struct Qdisc *qdisc)
+static int noop_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
+                        struct sk_buff **to_free)
 {
-        kfree_skb(skb);
+        __qdisc_drop(skb, to_free);
         return NET_XMIT_CN;
 }
 
@@ -381,6 +426,7 @@ struct Qdisc noop_qdisc = {
         .list           =       LIST_HEAD_INIT(noop_qdisc.list),
         .q.lock         =       __SPIN_LOCK_UNLOCKED(noop_qdisc.q.lock),
         .dev_queue      =       &noop_netdev_queue,
+        .running        =       SEQCNT_ZERO(noop_qdisc.running),
         .busylock       =       __SPIN_LOCK_UNLOCKED(noop_qdisc.busylock),
 };
 EXPORT_SYMBOL(noop_qdisc);
@@ -438,7 +484,8 @@ static inline struct sk_buff_head *band2list(struct pfifo_fast_priv *priv,
         return priv->q + band;
 }
 
-static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc)
+static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
+                              struct sk_buff **to_free)
 {
         if (skb_queue_len(&qdisc->q) < qdisc_dev(qdisc)->tx_queue_len) {
                 int band = prio2band[skb->priority & TC_PRIO_MAX];
@@ -450,7 +497,7 @@ static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc)
                 return __qdisc_enqueue_tail(skb, qdisc, list);
         }
 
-        return qdisc_drop(skb, qdisc);
+        return qdisc_drop(skb, qdisc, to_free);
 }
 
 static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc)
@@ -492,7 +539,7 @@ static void pfifo_fast_reset(struct Qdisc *qdisc)
         struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
 
         for (prio = 0; prio < PFIFO_FAST_BANDS; prio++)
-                __qdisc_reset_queue(qdisc, band2list(priv, prio));
+                __qdisc_reset_queue(band2list(priv, prio));
 
         priv->bitmap = 0;
         qdisc->qstats.backlog = 0;
@@ -539,6 +586,7 @@ struct Qdisc_ops pfifo_fast_ops __read_mostly = {
 EXPORT_SYMBOL(pfifo_fast_ops);
 
 static struct lock_class_key qdisc_tx_busylock;
+static struct lock_class_key qdisc_running_key;
 
 struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
                           const struct Qdisc_ops *ops)
@@ -572,6 +620,10 @@ struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
         lockdep_set_class(&sch->busylock,
                           dev->qdisc_tx_busylock ?: &qdisc_tx_busylock);
 
+        seqcount_init(&sch->running);
+        lockdep_set_class(&sch->running,
+                          dev->qdisc_running_key ?: &qdisc_running_key);
+
         sch->ops = ops;
         sch->enqueue = ops->enqueue;
         sch->dequeue = ops->dequeue;
@@ -616,11 +668,14 @@ void qdisc_reset(struct Qdisc *qdisc)
         if (ops->reset)
                 ops->reset(qdisc);
 
+        kfree_skb(qdisc->skb_bad_txq);
+        qdisc->skb_bad_txq = NULL;
+
         if (qdisc->gso_skb) {
                 kfree_skb_list(qdisc->gso_skb);
                 qdisc->gso_skb = NULL;
-                qdisc->q.qlen = 0;
         }
+        qdisc->q.qlen = 0;
 }
 EXPORT_SYMBOL(qdisc_reset);
 
@@ -659,6 +714,7 @@ void qdisc_destroy(struct Qdisc *qdisc)
         dev_put(qdisc_dev(qdisc));
 
         kfree_skb_list(qdisc->gso_skb);
+        kfree_skb(qdisc->skb_bad_txq);
         /*
          * gen_estimator est_timer() might access qdisc->q.lock,
          * wait a RCU grace period before freeing qdisc.
diff --git a/net/sched/sch_gred.c b/net/sched/sch_gred.c
index 80105109f756..c78a093c551a 100644
--- a/net/sched/sch_gred.c
+++ b/net/sched/sch_gred.c
@@ -149,7 +149,8 @@ static inline int gred_use_harddrop(struct gred_sched *t)
         return t->red_flags & TC_RED_HARDDROP;
 }
 
-static int gred_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+static int gred_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+                        struct sk_buff **to_free)
 {
         struct gred_sched_data *q = NULL;
         struct gred_sched *t = qdisc_priv(sch);
@@ -237,10 +238,10 @@ static int gred_enqueue(struct sk_buff *skb, struct Qdisc *sch)
 
         q->stats.pdrop++;
 drop:
-        return qdisc_drop(skb, sch);
+        return qdisc_drop(skb, sch, to_free);
 
 congestion_drop:
-        qdisc_drop(skb, sch);
+        qdisc_drop(skb, sch, to_free);
         return NET_XMIT_CN;
 }
 
@@ -276,40 +277,6 @@ static struct sk_buff *gred_dequeue(struct Qdisc *sch)
         return NULL;
 }
 
-static unsigned int gred_drop(struct Qdisc *sch)
-{
-        struct sk_buff *skb;
-        struct gred_sched *t = qdisc_priv(sch);
-
-        skb = qdisc_dequeue_tail(sch);
-        if (skb) {
-                unsigned int len = qdisc_pkt_len(skb);
-                struct gred_sched_data *q;
-                u16 dp = tc_index_to_dp(skb);
-
-                if (dp >= t->DPs || (q = t->tab[dp]) == NULL) {
-                        net_warn_ratelimited("GRED: Unable to relocate VQ 0x%x while dropping, screwing up backlog\n",
-                                             tc_index_to_dp(skb));
-                } else {
-                        q->backlog -= len;
-                        q->stats.other++;
-
-                        if (gred_wred_mode(t)) {
-                                if (!sch->qstats.backlog)
-                                        red_start_of_idle_period(&t->wred_set);
-                        } else {
-                                if (!q->backlog)
-                                        red_start_of_idle_period(&q->vars);
-                        }
-                }
-
-                qdisc_drop(skb, sch);
-                return len;
-        }
-
-        return 0;
-}
-
 static void gred_reset(struct Qdisc *sch)
 {
         int i;
@@ -623,7 +590,6 @@ static struct Qdisc_ops gred_qdisc_ops __read_mostly = {
         .enqueue        =       gred_enqueue,
         .dequeue        =       gred_dequeue,
         .peek           =       qdisc_peek_head,
-        .drop           =       gred_drop,
         .init           =       gred_init,
         .reset          =       gred_reset,
         .destroy        =       gred_destroy,
diff --git a/net/sched/sch_hfsc.c b/net/sched/sch_hfsc.c
index 1ac9f9f03fe3..8cb5eff7b79c 100644
--- a/net/sched/sch_hfsc.c
+++ b/net/sched/sch_hfsc.c
@@ -1015,11 +1015,10 @@ hfsc_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
                 cur_time = psched_get_time();
 
                 if (tca[TCA_RATE]) {
-                        spinlock_t *lock = qdisc_root_sleeping_lock(sch);
-
                         err = gen_replace_estimator(&cl->bstats, NULL,
                                                     &cl->rate_est,
-                                                    lock,
+                                                    NULL,
+                                                    qdisc_root_sleeping_running(sch),
                                                     tca[TCA_RATE]);
                         if (err)
                                 return err;
@@ -1068,7 +1067,8 @@ hfsc_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
 
         if (tca[TCA_RATE]) {
                 err = gen_new_estimator(&cl->bstats, NULL, &cl->rate_est,
-                                        qdisc_root_sleeping_lock(sch),
+                                        NULL,
+                                        qdisc_root_sleeping_running(sch),
                                         tca[TCA_RATE]);
                 if (err) {
                         kfree(cl);
@@ -1373,7 +1373,7 @@ hfsc_dump_class_stats(struct Qdisc *sch, unsigned long arg,
         xstats.work    = cl->cl_total;
         xstats.rtwork  = cl->cl_cumul;
 
-        if (gnet_stats_copy_basic(d, NULL, &cl->bstats) < 0 ||
+        if (gnet_stats_copy_basic(qdisc_root_sleeping_running(sch), d, NULL, &cl->bstats) < 0 ||
             gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 ||
             gnet_stats_copy_queue(d, NULL, &cl->qstats, cl->qdisc->q.qlen) < 0)
                 return -1;
@@ -1572,7 +1572,7 @@ hfsc_dump_qdisc(struct Qdisc *sch, struct sk_buff *skb)
 }
 
 static int
-hfsc_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+hfsc_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)
 {
         struct hfsc_class *cl;
         int uninitialized_var(err);
@@ -1581,11 +1581,11 @@ hfsc_enqueue(struct sk_buff *skb, struct Qdisc *sch)
         if (cl == NULL) {
                 if (err & __NET_XMIT_BYPASS)
                         qdisc_qstats_drop(sch);
-                kfree_skb(skb);
+                __qdisc_drop(skb, to_free);
                 return err;
         }
 
-        err = qdisc_enqueue(skb, cl->qdisc);
+        err = qdisc_enqueue(skb, cl->qdisc, to_free);
         if (unlikely(err != NET_XMIT_SUCCESS)) {
                 if (net_xmit_drop_count(err)) {
                         cl->qstats.drops++;
@@ -1664,7 +1664,6 @@ hfsc_dequeue(struct Qdisc *sch)
                 set_passive(cl);
         }
 
-        qdisc_unthrottled(sch);
         qdisc_bstats_update(sch, skb);
         qdisc_qstats_backlog_dec(sch, skb);
         sch->q.qlen--;
@@ -1672,32 +1671,6 @@ hfsc_dequeue(struct Qdisc *sch)
         return skb;
 }
 
-static unsigned int
-hfsc_drop(struct Qdisc *sch)
-{
-        struct hfsc_sched *q = qdisc_priv(sch);
-        struct hfsc_class *cl;
-        unsigned int len;
-
-        list_for_each_entry(cl, &q->droplist, dlist) {
-                if (cl->qdisc->ops->drop != NULL &&
-                    (len = cl->qdisc->ops->drop(cl->qdisc)) > 0) {
-                        if (cl->qdisc->q.qlen == 0) {
-                                update_vf(cl, 0, 0);
-                                set_passive(cl);
-                        } else {
-                                list_move_tail(&cl->dlist, &q->droplist);
-                        }
-                        cl->qstats.drops++;
-                        qdisc_qstats_drop(sch);
-                        sch->qstats.backlog -= len;
-                        sch->q.qlen--;
-                        return len;
-                }
-        }
-        return 0;
-}
-
 static const struct Qdisc_class_ops hfsc_class_ops = {
         .change         = hfsc_change_class,
         .delete         = hfsc_delete_class,
@@ -1724,7 +1697,6 @@ static struct Qdisc_ops hfsc_qdisc_ops __read_mostly = {
         .enqueue        = hfsc_enqueue,
         .dequeue        = hfsc_dequeue,
         .peek           = qdisc_peek_dequeued,
-        .drop           = hfsc_drop,
         .cl_ops         = &hfsc_class_ops,
         .priv_size      = sizeof(struct hfsc_sched),
         .owner          = THIS_MODULE
diff --git a/net/sched/sch_hhf.c b/net/sched/sch_hhf.c
index 13d6f83ec491..e3d0458af17b 100644
--- a/net/sched/sch_hhf.c
+++ b/net/sched/sch_hhf.c
@@ -345,7 +345,7 @@ static void bucket_add(struct wdrr_bucket *bucket, struct sk_buff *skb)
         skb->next = NULL;
 }
 
-static unsigned int hhf_drop(struct Qdisc *sch)
+static unsigned int hhf_drop(struct Qdisc *sch, struct sk_buff **to_free)
 {
         struct hhf_sched_data *q = qdisc_priv(sch);
         struct wdrr_bucket *bucket;
@@ -359,25 +359,16 @@ static unsigned int hhf_drop(struct Qdisc *sch)
                 struct sk_buff *skb = dequeue_head(bucket);
 
                 sch->q.qlen--;
-                qdisc_qstats_drop(sch);
                 qdisc_qstats_backlog_dec(sch, skb);
-                kfree_skb(skb);
+                qdisc_drop(skb, sch, to_free);
         }
 
         /* Return id of the bucket from which the packet was dropped. */
         return bucket - q->buckets;
 }
 
-static unsigned int hhf_qdisc_drop(struct Qdisc *sch)
-{
-        unsigned int prev_backlog;
-
-        prev_backlog = sch->qstats.backlog;
-        hhf_drop(sch);
-        return prev_backlog - sch->qstats.backlog;
-}
-
-static int hhf_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+static int hhf_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+                       struct sk_buff **to_free)
 {
         struct hhf_sched_data *q = qdisc_priv(sch);
         enum wdrr_bucket_idx idx;
@@ -415,7 +406,7 @@ static int hhf_enqueue(struct sk_buff *skb, struct Qdisc *sch)
         /* Return Congestion Notification only if we dropped a packet from this
          * bucket.
          */
-        if (hhf_drop(sch) == idx)
+        if (hhf_drop(sch, to_free) == idx)
                 return NET_XMIT_CN;
 
         /* As we dropped a packet, better let upper stack know this. */
@@ -473,7 +464,7 @@ static void hhf_reset(struct Qdisc *sch)
         struct sk_buff *skb;
 
         while ((skb = hhf_dequeue(sch)) != NULL)
-                kfree_skb(skb);
+                rtnl_kfree_skbs(skb, skb);
 }
 
 static void *hhf_zalloc(size_t sz)
@@ -583,7 +574,7 @@ static int hhf_change(struct Qdisc *sch, struct nlattr *opt)
         while (sch->q.qlen > sch->limit) {
                 struct sk_buff *skb = hhf_dequeue(sch);
 
-                kfree_skb(skb);
+                rtnl_kfree_skbs(skb, skb);
         }
         qdisc_tree_reduce_backlog(sch, qlen - sch->q.qlen,
                                   prev_backlog - sch->qstats.backlog);
@@ -709,7 +700,6 @@ static struct Qdisc_ops hhf_qdisc_ops __read_mostly = {
         .enqueue        =       hhf_enqueue,
         .dequeue        =       hhf_dequeue,
         .peek           =       qdisc_peek_dequeued,
-        .drop           =       hhf_qdisc_drop,
         .init           =       hhf_init,
         .reset          =       hhf_reset,
         .destroy        =       hhf_destroy,
diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c
index 62f9d8100c6e..91982d9784b3 100644
--- a/net/sched/sch_htb.c
+++ b/net/sched/sch_htb.c
@@ -117,7 +117,6 @@ struct htb_class {
          * Written often fields
          */
         struct gnet_stats_basic_packed bstats;
-        struct gnet_stats_queue qstats;
         struct tc_htb_xstats    xstats; /* our special stats */
 
         /* token bucket parameters */
@@ -140,6 +139,8 @@ struct htb_class {
         enum htb_cmode          cmode;          /* current mode of the class */
         struct rb_node          pq_node;        /* node for event queue */
         struct rb_node          node[TC_HTB_NUMPRIO];   /* node for self or feed tree */
+
+        unsigned int drops ____cacheline_aligned_in_smp;
 };
 
 struct htb_level {
@@ -569,7 +570,8 @@ static inline void htb_deactivate(struct htb_sched *q, struct htb_class *cl)
         list_del_init(&cl->un.leaf.drop_list);
 }
 
-static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+                       struct sk_buff **to_free)
 {
         int uninitialized_var(ret);
         struct htb_sched *q = qdisc_priv(sch);
@@ -581,19 +583,20 @@ static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch)
                         __skb_queue_tail(&q->direct_queue, skb);
                         q->direct_pkts++;
                 } else {
-                        return qdisc_drop(skb, sch);
+                        return qdisc_drop(skb, sch, to_free);
                 }
 #ifdef CONFIG_NET_CLS_ACT
         } else if (!cl) {
                 if (ret & __NET_XMIT_BYPASS)
                         qdisc_qstats_drop(sch);
-                kfree_skb(skb);
+                __qdisc_drop(skb, to_free);
                 return ret;
 #endif
-        } else if ((ret = qdisc_enqueue(skb, cl->un.leaf.q)) != NET_XMIT_SUCCESS) {
+        } else if ((ret = qdisc_enqueue(skb, cl->un.leaf.q,
+                                        to_free)) != NET_XMIT_SUCCESS) {
                 if (net_xmit_drop_count(ret)) {
                         qdisc_qstats_drop(sch);
-                        cl->qstats.drops++;
+                        cl->drops++;
                 }
                 return ret;
         } else {
@@ -889,7 +892,6 @@ static struct sk_buff *htb_dequeue(struct Qdisc *sch)
         if (skb != NULL) {
 ok:
                 qdisc_bstats_update(sch, skb);
-                qdisc_unthrottled(sch);
                 qdisc_qstats_backlog_dec(sch, skb);
                 sch->q.qlen--;
                 return skb;
@@ -929,38 +931,13 @@ ok:
         }
         qdisc_qstats_overlimit(sch);
         if (likely(next_event > q->now))
-                qdisc_watchdog_schedule_ns(&q->watchdog, next_event, true);
+                qdisc_watchdog_schedule_ns(&q->watchdog, next_event);
         else
                 schedule_work(&q->work);
 fin:
         return skb;
 }
 
-/* try to drop from each class (by prio) until one succeed */
-static unsigned int htb_drop(struct Qdisc *sch)
-{
-        struct htb_sched *q = qdisc_priv(sch);
-        int prio;
-
-        for (prio = TC_HTB_NUMPRIO - 1; prio >= 0; prio--) {
-                struct list_head *p;
-                list_for_each(p, q->drops + prio) {
-                        struct htb_class *cl = list_entry(p, struct htb_class,
-                                                          un.leaf.drop_list);
-                        unsigned int len;
-                        if (cl->un.leaf.q->ops->drop &&
-                            (len = cl->un.leaf.q->ops->drop(cl->un.leaf.q))) {
-                                sch->qstats.backlog -= len;
-                                sch->q.qlen--;
-                                if (!cl->un.leaf.q->q.qlen)
-                                        htb_deactivate(q, cl);
-                                return len;
-                        }
-                }
-        }
-        return 0;
-}
-
 /* reset all classes */
 /* always caled under BH & queue lock */
 static void htb_reset(struct Qdisc *sch)
@@ -983,7 +960,7 @@ static void htb_reset(struct Qdisc *sch)
                 }
         }
         qdisc_watchdog_cancel(&q->watchdog);
-        __skb_queue_purge(&q->direct_queue);
+        __qdisc_reset_queue(&q->direct_queue);
         sch->q.qlen = 0;
         sch->qstats.backlog = 0;
         memset(q->hlevel, 0, sizeof(q->hlevel));
@@ -1136,16 +1113,22 @@ static int
 htb_dump_class_stats(struct Qdisc *sch, unsigned long arg, struct gnet_dump *d)
 {
         struct htb_class *cl = (struct htb_class *)arg;
+        struct gnet_stats_queue qs = {
+                .drops = cl->drops,
+        };
         __u32 qlen = 0;
 
-        if (!cl->level && cl->un.leaf.q)
+        if (!cl->level && cl->un.leaf.q) {
                 qlen = cl->un.leaf.q->q.qlen;
+                qs.backlog = cl->un.leaf.q->qstats.backlog;
+        }
         cl->xstats.tokens = PSCHED_NS2TICKS(cl->tokens);
         cl->xstats.ctokens = PSCHED_NS2TICKS(cl->ctokens);
 
-        if (gnet_stats_copy_basic(d, NULL, &cl->bstats) < 0 ||
+        if (gnet_stats_copy_basic(qdisc_root_sleeping_running(sch),
+                                  d, NULL, &cl->bstats) < 0 ||
             gnet_stats_copy_rate_est(d, NULL, &cl->rate_est) < 0 ||
-            gnet_stats_copy_queue(d, NULL, &cl->qstats, qlen) < 0)
+            gnet_stats_copy_queue(d, NULL, &qs, qlen) < 0)
                 return -1;
 
         return gnet_stats_copy_app(d, &cl->xstats, sizeof(cl->xstats));
@@ -1258,7 +1241,7 @@ static void htb_destroy(struct Qdisc *sch)
                         htb_destroy_class(sch, cl);
         }
         qdisc_class_hash_destroy(&q->clhash);
-        __skb_queue_purge(&q->direct_queue);
+        __qdisc_reset_queue(&q->direct_queue);
 }
 
 static int htb_delete(struct Qdisc *sch, unsigned long arg)
@@ -1397,7 +1380,8 @@ static int htb_change_class(struct Qdisc *sch, u32 classid,
                 if (htb_rate_est || tca[TCA_RATE]) {
                         err = gen_new_estimator(&cl->bstats, NULL,
                                                 &cl->rate_est,
-                                                qdisc_root_sleeping_lock(sch),
+                                                NULL,
+                                                qdisc_root_sleeping_running(sch),
                                                 tca[TCA_RATE] ? : &est.nla);
                         if (err) {
                                 kfree(cl);
@@ -1459,11 +1443,10 @@ static int htb_change_class(struct Qdisc *sch, u32 classid,
                         parent->children++;
         } else {
                 if (tca[TCA_RATE]) {
-                        spinlock_t *lock = qdisc_root_sleeping_lock(sch);
-
                         err = gen_replace_estimator(&cl->bstats, NULL,
                                                     &cl->rate_est,
-                                                    lock,
+                                                    NULL,
+                                                    qdisc_root_sleeping_running(sch),
                                                     tca[TCA_RATE]);
                         if (err)
                                 return err;
@@ -1601,7 +1584,6 @@ static struct Qdisc_ops htb_qdisc_ops __read_mostly = {
         .enqueue        =       htb_enqueue,
         .dequeue        =       htb_dequeue,
         .peek           =       qdisc_peek_dequeued,
-        .drop           =       htb_drop,
         .init           =       htb_init,
         .reset          =       htb_reset,
         .destroy        =       htb_destroy,
diff --git a/net/sched/sch_mq.c b/net/sched/sch_mq.c
index 56a77b878eb3..b9439827c172 100644
--- a/net/sched/sch_mq.c
+++ b/net/sched/sch_mq.c
@@ -199,7 +199,7 @@ static int mq_dump_class_stats(struct Qdisc *sch, unsigned long cl,
         struct netdev_queue *dev_queue = mq_queue_get(sch, cl);
 
         sch = dev_queue->qdisc_sleeping;
-        if (gnet_stats_copy_basic(d, NULL, &sch->bstats) < 0 ||
+        if (gnet_stats_copy_basic(&sch->running, d, NULL, &sch->bstats) < 0 ||
             gnet_stats_copy_queue(d, NULL, &sch->qstats, sch->q.qlen) < 0)
                 return -1;
         return 0;
diff --git a/net/sched/sch_mqprio.c b/net/sched/sch_mqprio.c
index b8002ce3d010..549c66359924 100644
--- a/net/sched/sch_mqprio.c
+++ b/net/sched/sch_mqprio.c
@@ -342,7 +342,8 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl,
                  * hold here is the look on dev_queue->qdisc_sleeping
                  * also acquired below.
                  */
-                spin_unlock_bh(d->lock);
+                if (d->lock)
+                        spin_unlock_bh(d->lock);
 
                 for (i = tc.offset; i < tc.offset + tc.count; i++) {
                         struct netdev_queue *q = netdev_get_tx_queue(dev, i);
@@ -359,15 +360,17 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl,
                         spin_unlock_bh(qdisc_lock(qdisc));
                 }
                 /* Reclaim root sleeping lock before completing stats */
-                spin_lock_bh(d->lock);
-                if (gnet_stats_copy_basic(d, NULL, &bstats) < 0 ||
+                if (d->lock)
+                        spin_lock_bh(d->lock);
+                if (gnet_stats_copy_basic(NULL, d, NULL, &bstats) < 0 ||
                     gnet_stats_copy_queue(d, NULL, &qstats, qlen) < 0)
                         return -1;
         } else {
                 struct netdev_queue *dev_queue = mqprio_queue_get(sch, cl);
 
                 sch = dev_queue->qdisc_sleeping;
-                if (gnet_stats_copy_basic(d, NULL, &sch->bstats) < 0 ||
+                if (gnet_stats_copy_basic(qdisc_root_sleeping_running(sch),
+                                          d, NULL, &sch->bstats) < 0 ||
                     gnet_stats_copy_queue(d, NULL,
                                           &sch->qstats, sch->q.qlen) < 0)
                         return -1;
diff --git a/net/sched/sch_multiq.c b/net/sched/sch_multiq.c
index bcdd54bb101c..9ffbb025b37e 100644
--- a/net/sched/sch_multiq.c
+++ b/net/sched/sch_multiq.c
@@ -65,7 +65,8 @@ multiq_classify(struct sk_buff *skb, struct Qdisc *sch, int *qerr)
 }
 
 static int
-multiq_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+multiq_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+               struct sk_buff **to_free)
 {
         struct Qdisc *qdisc;
         int ret;
@@ -76,12 +77,12 @@ multiq_enqueue(struct sk_buff *skb, struct Qdisc *sch)
 
                 if (ret & __NET_XMIT_BYPASS)
                         qdisc_qstats_drop(sch);
-                kfree_skb(skb);
+                __qdisc_drop(skb, to_free);
                 return ret;
         }
 #endif
 
-        ret = qdisc_enqueue(skb, qdisc);
+        ret = qdisc_enqueue(skb, qdisc, to_free);
         if (ret == NET_XMIT_SUCCESS) {
                 sch->q.qlen++;
                 return NET_XMIT_SUCCESS;
@@ -151,27 +152,6 @@ static struct sk_buff *multiq_peek(struct Qdisc *sch)
 
 }
 
-static unsigned int multiq_drop(struct Qdisc *sch)
-{
-        struct multiq_sched_data *q = qdisc_priv(sch);
-        int band;
-        unsigned int len;
-        struct Qdisc *qdisc;
-
-        for (band = q->bands - 1; band >= 0; band--) {
-                qdisc = q->queues[band];
-                if (qdisc->ops->drop) {
-                        len = qdisc->ops->drop(qdisc);
-                        if (len != 0) {
-                                sch->q.qlen--;
-                                return len;
-                        }
-                }
-        }
-        return 0;
-}
-
-
 static void
 multiq_reset(struct Qdisc *sch)
 {
@@ -356,7 +336,8 @@ static int multiq_dump_class_stats(struct Qdisc *sch, unsigned long cl,
         struct Qdisc *cl_q;
 
         cl_q = q->queues[cl - 1];
-        if (gnet_stats_copy_basic(d, NULL, &cl_q->bstats) < 0 ||
+        if (gnet_stats_copy_basic(qdisc_root_sleeping_running(sch),
+                                  d, NULL, &cl_q->bstats) < 0 ||
             gnet_stats_copy_queue(d, NULL, &cl_q->qstats, cl_q->q.qlen) < 0)
                 return -1;
 
@@ -415,7 +396,6 @@ static struct Qdisc_ops multiq_qdisc_ops __read_mostly = {
         .enqueue        =       multiq_enqueue,
         .dequeue        =       multiq_dequeue,
         .peek           =       multiq_peek,
-        .drop           =       multiq_drop,
         .init           =       multiq_init,
         .reset          =       multiq_reset,
         .destroy        =       multiq_destroy,
diff --git a/net/sched/sch_netem.c b/net/sched/sch_netem.c
index 178f1630a036..aaaf02175338 100644
--- a/net/sched/sch_netem.c
+++ b/net/sched/sch_netem.c
@@ -368,9 +368,7 @@ static void tfifo_reset(struct Qdisc *sch)
                 struct sk_buff *skb = netem_rb_to_skb(p);
 
                 rb_erase(p, &q->t_root);
-                skb->next = NULL;
-                skb->prev = NULL;
-                kfree_skb(skb);
+                rtnl_kfree_skbs(skb, skb);
         }
 }
 
@@ -399,7 +397,8 @@ static void tfifo_enqueue(struct sk_buff *nskb, struct Qdisc *sch)
  * when we statistically choose to corrupt one, we instead segment it, returning
  * the first packet to be corrupted, and re-enqueue the remaining frames
  */
-static struct sk_buff *netem_segment(struct sk_buff *skb, struct Qdisc *sch)
+static struct sk_buff *netem_segment(struct sk_buff *skb, struct Qdisc *sch,
+                                     struct sk_buff **to_free)
 {
         struct sk_buff *segs;
         netdev_features_t features = netif_skb_features(skb);
@@ -407,7 +406,7 @@ static struct sk_buff *netem_segment(struct sk_buff *skb, struct Qdisc *sch)
         segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK);
 
         if (IS_ERR_OR_NULL(segs)) {
-                qdisc_reshape_fail(skb, sch);
+                qdisc_drop(skb, sch, to_free);
                 return NULL;
         }
         consume_skb(skb);
@@ -420,7 +419,8 @@ static struct sk_buff *netem_segment(struct sk_buff *skb, struct Qdisc *sch)
  *      NET_XMIT_DROP: queue length didn't change.
  *      NET_XMIT_SUCCESS: one skb was queued.
  */
-static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+                         struct sk_buff **to_free)
 {
         struct netem_sched_data *q = qdisc_priv(sch);
         /* We don't fill cb now as skb_unshare() may invalidate it */
@@ -445,7 +445,7 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch)
         }
         if (count == 0) {
                 qdisc_qstats_drop(sch);
-                kfree_skb(skb);
+                __qdisc_drop(skb, to_free);
                 return NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
         }
 
@@ -465,7 +465,7 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch)
                 u32 dupsave = q->duplicate; /* prevent duplicating a dup... */
 
                 q->duplicate = 0;
-                rootq->enqueue(skb2, rootq);
+                rootq->enqueue(skb2, rootq, to_free);
                 q->duplicate = dupsave;
         }
 
@@ -477,7 +477,7 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch)
          */
         if (q->corrupt && q->corrupt >= get_crandom(&q->corrupt_cor)) {
                 if (skb_is_gso(skb)) {
-                        segs = netem_segment(skb, sch);
+                        segs = netem_segment(skb, sch, to_free);
                         if (!segs)
                                 return NET_XMIT_DROP;
                 } else {
@@ -487,10 +487,14 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch)
                 skb = segs;
                 segs = segs->next;
 
-                if (!(skb = skb_unshare(skb, GFP_ATOMIC)) ||
-                    (skb->ip_summed == CHECKSUM_PARTIAL &&
-                     skb_checksum_help(skb))) {
-                        rc = qdisc_drop(skb, sch);
+                skb = skb_unshare(skb, GFP_ATOMIC);
+                if (unlikely(!skb)) {
+                        qdisc_qstats_drop(sch);
+                        goto finish_segs;
+                }
+                if (skb->ip_summed == CHECKSUM_PARTIAL &&
+                    skb_checksum_help(skb)) {
+                        qdisc_drop(skb, sch, to_free);
                         goto finish_segs;
                 }
 
@@ -499,7 +503,7 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch)
         }
 
         if (unlikely(skb_queue_len(&sch->q) >= sch->limit))
-                return qdisc_reshape_fail(skb, sch);
+                return qdisc_drop(skb, sch, to_free);
 
         qdisc_qstats_backlog_inc(sch, skb);
 
@@ -559,7 +563,7 @@ finish_segs:
                         segs->next = NULL;
                         qdisc_skb_cb(segs)->pkt_len = segs->len;
                         last_len = segs->len;
-                        rc = qdisc_enqueue(segs, sch);
+                        rc = qdisc_enqueue(segs, sch, to_free);
                         if (rc != NET_XMIT_SUCCESS) {
                                 if (net_xmit_drop_count(rc))
                                         qdisc_qstats_drop(sch);
@@ -576,50 +580,17 @@ finish_segs:
         return NET_XMIT_SUCCESS;
 }
 
-static unsigned int netem_drop(struct Qdisc *sch)
-{
-        struct netem_sched_data *q = qdisc_priv(sch);
-        unsigned int len;
-
-        len = qdisc_queue_drop(sch);
-
-        if (!len) {
-                struct rb_node *p = rb_first(&q->t_root);
-
-                if (p) {
-                        struct sk_buff *skb = netem_rb_to_skb(p);
-
-                        rb_erase(p, &q->t_root);
-                        sch->q.qlen--;
-                        skb->next = NULL;
-                        skb->prev = NULL;
-                        qdisc_qstats_backlog_dec(sch, skb);
-                        kfree_skb(skb);
-                }
-        }
-        if (!len && q->qdisc && q->qdisc->ops->drop)
-            len = q->qdisc->ops->drop(q->qdisc);
-        if (len)
-                qdisc_qstats_drop(sch);
-
-        return len;
-}
-
 static struct sk_buff *netem_dequeue(struct Qdisc *sch)
 {
         struct netem_sched_data *q = qdisc_priv(sch);
         struct sk_buff *skb;
         struct rb_node *p;
 
-        if (qdisc_is_throttled(sch))
-                return NULL;
-
 tfifo_dequeue:
         skb = __skb_dequeue(&sch->q);
         if (skb) {
                 qdisc_qstats_backlog_dec(sch, skb);
 deliver:
-                qdisc_unthrottled(sch);
                 qdisc_bstats_update(sch, skb);
                 return skb;
         }
@@ -651,8 +622,11 @@ deliver:
 
                         if (q->qdisc) {
                                 unsigned int pkt_len = qdisc_pkt_len(skb);
-                                int err = qdisc_enqueue(skb, q->qdisc);
+                                struct sk_buff *to_free = NULL;
+                                int err;
 
+                                err = qdisc_enqueue(skb, q->qdisc, &to_free);
+                                kfree_skb_list(to_free);
                                 if (err != NET_XMIT_SUCCESS &&
                                     net_xmit_drop_count(err)) {
                                         qdisc_qstats_drop(sch);
@@ -1143,7 +1117,6 @@ static struct Qdisc_ops netem_qdisc_ops __read_mostly = {
         .enqueue        =       netem_enqueue,
         .dequeue        =       netem_dequeue,
         .peek           =       qdisc_peek_dequeued,
-        .drop           =       netem_drop,
         .init           =       netem_init,
         .reset          =       netem_reset,
         .destroy        =       netem_destroy,
diff --git a/net/sched/sch_pie.c b/net/sched/sch_pie.c
index 71ae3b9629f9..a570b0bb254c 100644
--- a/net/sched/sch_pie.c
+++ b/net/sched/sch_pie.c
@@ -134,7 +134,8 @@ static bool drop_early(struct Qdisc *sch, u32 packet_size)
         return false;
 }
 
-static int pie_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+static int pie_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+                             struct sk_buff **to_free)
 {
         struct pie_sched_data *q = qdisc_priv(sch);
         bool enqueue = false;
@@ -166,7 +167,7 @@ static int pie_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch)
 
 out:
         q->stats.dropped++;
-        return qdisc_drop(skb, sch);
+        return qdisc_drop(skb, sch, to_free);
 }
 
 static const struct nla_policy pie_policy[TCA_PIE_MAX + 1] = {
@@ -234,7 +235,7 @@ static int pie_change(struct Qdisc *sch, struct nlattr *opt)
 
                 dropped += qdisc_pkt_len(skb);
                 qdisc_qstats_backlog_dec(sch, skb);
-                qdisc_drop(skb, sch);
+                rtnl_qdisc_drop(skb, sch);
         }
         qdisc_tree_reduce_backlog(sch, qlen - sch->q.qlen, dropped);
 
diff --git a/net/sched/sch_plug.c b/net/sched/sch_plug.c
index 5abfe44678d4..1c6cbab3e7b9 100644
--- a/net/sched/sch_plug.c
+++ b/net/sched/sch_plug.c
@@ -64,6 +64,8 @@ struct plug_sched_data {
          */
         bool unplug_indefinite;
 
+        bool throttled;
+
         /* Queue Limit in bytes */
         u32 limit;
 
@@ -86,7 +88,8 @@ struct plug_sched_data {
         u32 pkts_to_release;
 };
 
-static int plug_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+static int plug_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+                        struct sk_buff **to_free)
 {
         struct plug_sched_data *q = qdisc_priv(sch);
 
@@ -96,14 +99,14 @@ static int plug_enqueue(struct sk_buff *skb, struct Qdisc *sch)
                 return qdisc_enqueue_tail(skb, sch);
         }
 
-        return qdisc_reshape_fail(skb, sch);
+        return qdisc_drop(skb, sch, to_free);
 }
 
 static struct sk_buff *plug_dequeue(struct Qdisc *sch)
 {
         struct plug_sched_data *q = qdisc_priv(sch);
 
-        if (qdisc_is_throttled(sch))
+        if (q->throttled)
                 return NULL;
 
         if (!q->unplug_indefinite) {
@@ -111,7 +114,7 @@ static struct sk_buff *plug_dequeue(struct Qdisc *sch)
                         /* No more packets to dequeue. Block the queue
                          * and wait for the next release command.
                          */
-                        qdisc_throttled(sch);
+                        q->throttled = true;
                         return NULL;
                 }
                 q->pkts_to_release--;
@@ -141,7 +144,7 @@ static int plug_init(struct Qdisc *sch, struct nlattr *opt)
                 q->limit = ctl->limit;
         }
 
-        qdisc_throttled(sch);
+        q->throttled = true;
         return 0;
 }
 
@@ -173,7 +176,7 @@ static int plug_change(struct Qdisc *sch, struct nlattr *opt)
                 q->pkts_last_epoch = q->pkts_current_epoch;
                 q->pkts_current_epoch = 0;
                 if (q->unplug_indefinite)
-                        qdisc_throttled(sch);
+                        q->throttled = true;
                 q->unplug_indefinite = false;
                 break;
         case TCQ_PLUG_RELEASE_ONE:
@@ -182,7 +185,7 @@ static int plug_change(struct Qdisc *sch, struct nlattr *opt)
                  */
                 q->pkts_to_release += q->pkts_last_epoch;
                 q->pkts_last_epoch = 0;
-                qdisc_unthrottled(sch);
+                q->throttled = false;
                 netif_schedule_queue(sch->dev_queue);
                 break;
         case TCQ_PLUG_RELEASE_INDEFINITE:
@@ -190,7 +193,7 @@ static int plug_change(struct Qdisc *sch, struct nlattr *opt)
                 q->pkts_to_release = 0;
                 q->pkts_last_epoch = 0;
                 q->pkts_current_epoch = 0;
-                qdisc_unthrottled(sch);
+                q->throttled = false;
                 netif_schedule_queue(sch->dev_queue);
                 break;
         case TCQ_PLUG_LIMIT:
diff --git a/net/sched/sch_prio.c b/net/sched/sch_prio.c
index a356450b747b..8f575899adfa 100644
--- a/net/sched/sch_prio.c
+++ b/net/sched/sch_prio.c
@@ -67,7 +67,7 @@ prio_classify(struct sk_buff *skb, struct Qdisc *sch, int *qerr)
 }
 
 static int
-prio_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+prio_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)
 {
         struct Qdisc *qdisc;
         int ret;
@@ -83,7 +83,7 @@ prio_enqueue(struct sk_buff *skb, struct Qdisc *sch)
         }
 #endif
 
-        ret = qdisc_enqueue(skb, qdisc);
+        ret = qdisc_enqueue(skb, qdisc, to_free);
         if (ret == NET_XMIT_SUCCESS) {
                 qdisc_qstats_backlog_inc(sch, skb);
                 sch->q.qlen++;
@@ -127,25 +127,6 @@ static struct sk_buff *prio_dequeue(struct Qdisc *sch)
 
 }
 
-static unsigned int prio_drop(struct Qdisc *sch)
-{
-        struct prio_sched_data *q = qdisc_priv(sch);
-        int prio;
-        unsigned int len;
-        struct Qdisc *qdisc;
-
-        for (prio = q->bands-1; prio >= 0; prio--) {
-                qdisc = q->queues[prio];
-                if (qdisc->ops->drop && (len = qdisc->ops->drop(qdisc)) != 0) {
-                        sch->qstats.backlog -= len;
-                        sch->q.qlen--;
-                        return len;
-                }
-        }
-        return 0;
-}
-
-
 static void
 prio_reset(struct Qdisc *sch)
 {
@@ -304,7 +285,8 @@ static int prio_dump_class_stats(struct Qdisc *sch, unsigned long cl,
         struct Qdisc *cl_q;
 
         cl_q = q->queues[cl - 1];
-        if (gnet_stats_copy_basic(d, NULL, &cl_q->bstats) < 0 ||
+        if (gnet_stats_copy_basic(qdisc_root_sleeping_running(sch),
+                                  d, NULL, &cl_q->bstats) < 0 ||
             gnet_stats_copy_queue(d, NULL, &cl_q->qstats, cl_q->q.qlen) < 0)
                 return -1;
 
@@ -363,7 +345,6 @@ static struct Qdisc_ops prio_qdisc_ops __read_mostly = {
         .enqueue        =       prio_enqueue,
         .dequeue        =       prio_dequeue,
         .peek           =       prio_peek,
-        .drop           =       prio_drop,
         .init           =       prio_init,
         .reset          =       prio_reset,
         .destroy        =       prio_destroy,
diff --git a/net/sched/sch_qfq.c b/net/sched/sch_qfq.c
index f18857febdad..f27ffee106f6 100644
--- a/net/sched/sch_qfq.c
+++ b/net/sched/sch_qfq.c
@@ -460,7 +460,8 @@ static int qfq_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
                 if (tca[TCA_RATE]) {
                         err = gen_replace_estimator(&cl->bstats, NULL,
                                                     &cl->rate_est,
-                                                    qdisc_root_sleeping_lock(sch),
+                                                    NULL,
+                                                    qdisc_root_sleeping_running(sch),
                                                     tca[TCA_RATE]);
                         if (err)
                                 return err;
@@ -486,7 +487,8 @@ static int qfq_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
         if (tca[TCA_RATE]) {
                 err = gen_new_estimator(&cl->bstats, NULL,
                                         &cl->rate_est,
-                                        qdisc_root_sleeping_lock(sch),
+                                        NULL,
+                                        qdisc_root_sleeping_running(sch),
                                         tca[TCA_RATE]);
                 if (err)
                         goto destroy_class;
@@ -663,7 +665,8 @@ static int qfq_dump_class_stats(struct Qdisc *sch, unsigned long arg,
         xstats.weight = cl->agg->class_weight;
         xstats.lmax = cl->agg->lmax;
 
-        if (gnet_stats_copy_basic(d, NULL, &cl->bstats) < 0 ||
+        if (gnet_stats_copy_basic(qdisc_root_sleeping_running(sch),
+                                  d, NULL, &cl->bstats) < 0 ||
             gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 ||
             gnet_stats_copy_queue(d, NULL,
                                   &cl->qdisc->qstats, cl->qdisc->q.qlen) < 0)
@@ -1214,7 +1217,8 @@ static struct qfq_aggregate *qfq_choose_next_agg(struct qfq_sched *q)
         return agg;
 }
 
-static int qfq_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+static int qfq_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+                       struct sk_buff **to_free)
 {
         struct qfq_sched *q = qdisc_priv(sch);
         struct qfq_class *cl;
@@ -1237,11 +1241,11 @@ static int qfq_enqueue(struct sk_buff *skb, struct Qdisc *sch)
                                      qdisc_pkt_len(skb));
                 if (err) {
                         cl->qstats.drops++;
-                        return qdisc_drop(skb, sch);
+                        return qdisc_drop(skb, sch, to_free);
                 }
         }
 
-        err = qdisc_enqueue(skb, cl->qdisc);
+        err = qdisc_enqueue(skb, cl->qdisc, to_free);
         if (unlikely(err != NET_XMIT_SUCCESS)) {
                 pr_debug("qfq_enqueue: enqueue failed %d\n", err);
                 if (net_xmit_drop_count(err)) {
@@ -1422,52 +1426,6 @@ static void qfq_qlen_notify(struct Qdisc *sch, unsigned long arg)
                 qfq_deactivate_class(q, cl);
 }
 
-static unsigned int qfq_drop_from_slot(struct qfq_sched *q,
-                                       struct hlist_head *slot)
-{
-        struct qfq_aggregate *agg;
-        struct qfq_class *cl;
-        unsigned int len;
-
-        hlist_for_each_entry(agg, slot, next) {
-                list_for_each_entry(cl, &agg->active, alist) {
-
-                        if (!cl->qdisc->ops->drop)
-                                continue;
-
-                        len = cl->qdisc->ops->drop(cl->qdisc);
-                        if (len > 0) {
-                                if (cl->qdisc->q.qlen == 0)
-                                        qfq_deactivate_class(q, cl);
-
-                                return len;
-                        }
-                }
-        }
-        return 0;
-}
-
-static unsigned int qfq_drop(struct Qdisc *sch)
-{
-        struct qfq_sched *q = qdisc_priv(sch);
-        struct qfq_group *grp;
-        unsigned int i, j, len;
-
-        for (i = 0; i <= QFQ_MAX_INDEX; i++) {
-                grp = &q->groups[i];
-                for (j = 0; j < QFQ_MAX_SLOTS; j++) {
-                        len = qfq_drop_from_slot(q, &grp->slots[j]);
-                        if (len > 0) {
-                                sch->q.qlen--;
-                                return len;
-                        }
-                }
-
-        }
-
-        return 0;
-}
-
 static int qfq_init_qdisc(struct Qdisc *sch, struct nlattr *opt)
 {
         struct qfq_sched *q = qdisc_priv(sch);
@@ -1562,7 +1520,6 @@ static struct Qdisc_ops qfq_qdisc_ops __read_mostly = {
         .enqueue        = qfq_enqueue,
         .dequeue        = qfq_dequeue,
         .peek           = qdisc_peek_dequeued,
-        .drop           = qfq_drop,
         .init           = qfq_init_qdisc,
         .reset          = qfq_reset_qdisc,
         .destroy        = qfq_destroy_qdisc,
diff --git a/net/sched/sch_red.c b/net/sched/sch_red.c
index 91578bdd378c..249b2a18acbd 100644
--- a/net/sched/sch_red.c
+++ b/net/sched/sch_red.c
@@ -56,7 +56,8 @@ static inline int red_use_harddrop(struct red_sched_data *q)
         return q->flags & TC_RED_HARDDROP;
 }
 
-static int red_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+static int red_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+                       struct sk_buff **to_free)
 {
         struct red_sched_data *q = qdisc_priv(sch);
         struct Qdisc *child = q->qdisc;
@@ -95,7 +96,7 @@ static int red_enqueue(struct sk_buff *skb, struct Qdisc *sch)
                 break;
         }
 
-        ret = qdisc_enqueue(skb, child);
+        ret = qdisc_enqueue(skb, child, to_free);
         if (likely(ret == NET_XMIT_SUCCESS)) {
                 qdisc_qstats_backlog_inc(sch, skb);
                 sch->q.qlen++;
@@ -106,7 +107,7 @@ static int red_enqueue(struct sk_buff *skb, struct Qdisc *sch)
         return ret;
 
 congestion_drop:
-        qdisc_drop(skb, sch);
+        qdisc_drop(skb, sch, to_free);
         return NET_XMIT_CN;
 }
 
@@ -136,26 +137,6 @@ static struct sk_buff *red_peek(struct Qdisc *sch)
         return child->ops->peek(child);
 }
 
-static unsigned int red_drop(struct Qdisc *sch)
-{
-        struct red_sched_data *q = qdisc_priv(sch);
-        struct Qdisc *child = q->qdisc;
-        unsigned int len;
-
-        if (child->ops->drop && (len = child->ops->drop(child)) > 0) {
-                q->stats.other++;
-                qdisc_qstats_drop(sch);
-                sch->qstats.backlog -= len;
-                sch->q.qlen--;
-                return len;
-        }
-
-        if (!red_is_idling(&q->vars))
-                red_start_of_idle_period(&q->vars);
-
-        return 0;
-}
-
 static void red_reset(struct Qdisc *sch)
 {
         struct red_sched_data *q = qdisc_priv(sch);
@@ -365,7 +346,6 @@ static struct Qdisc_ops red_qdisc_ops __read_mostly = {
         .enqueue        =       red_enqueue,
         .dequeue        =       red_dequeue,
         .peek           =       red_peek,
-        .drop           =       red_drop,
         .init           =       red_init,
         .reset          =       red_reset,
         .destroy        =       red_destroy,
diff --git a/net/sched/sch_sfb.c b/net/sched/sch_sfb.c
index c69611640fa5..add3cc7d37ec 100644
--- a/net/sched/sch_sfb.c
+++ b/net/sched/sch_sfb.c
@@ -275,7 +275,8 @@ static bool sfb_classify(struct sk_buff *skb, struct tcf_proto *fl,
         return false;
 }
 
-static int sfb_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+static int sfb_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+                       struct sk_buff **to_free)
 {
 
         struct sfb_sched_data *q = qdisc_priv(sch);
@@ -397,7 +398,7 @@ static int sfb_enqueue(struct sk_buff *skb, struct Qdisc *sch)
         }
 
 enqueue:
-        ret = qdisc_enqueue(skb, child);
+        ret = qdisc_enqueue(skb, child, to_free);
         if (likely(ret == NET_XMIT_SUCCESS)) {
                 sch->q.qlen++;
                 increment_qlen(skb, q);
@@ -408,7 +409,7 @@ enqueue:
         return ret;
 
 drop:
-        qdisc_drop(skb, sch);
+        qdisc_drop(skb, sch, to_free);
         return NET_XMIT_CN;
 other_drop:
         if (ret & __NET_XMIT_BYPASS)
diff --git a/net/sched/sch_sfq.c b/net/sched/sch_sfq.c
index 498f0a2cb47f..7f195ed4d568 100644
--- a/net/sched/sch_sfq.c
+++ b/net/sched/sch_sfq.c
@@ -343,7 +343,7 @@ static int sfq_headdrop(const struct sfq_sched_data *q)
 }
 
 static int
-sfq_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+sfq_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)
 {
         struct sfq_sched_data *q = qdisc_priv(sch);
         unsigned int hash, dropped;
@@ -367,7 +367,7 @@ sfq_enqueue(struct sk_buff *skb, struct Qdisc *sch)
         if (x == SFQ_EMPTY_SLOT) {
                 x = q->dep[0].next; /* get a free slot */
                 if (x >= SFQ_MAX_FLOWS)
-                        return qdisc_drop(skb, sch);
+                        return qdisc_drop(skb, sch, to_free);
                 q->ht[hash] = x;
                 slot = &q->slots[x];
                 slot->hash = hash;
@@ -424,14 +424,14 @@ sfq_enqueue(struct sk_buff *skb, struct Qdisc *sch)
         if (slot->qlen >= q->maxdepth) {
 congestion_drop:
                 if (!sfq_headdrop(q))
-                        return qdisc_drop(skb, sch);
+                        return qdisc_drop(skb, sch, to_free);
 
                 /* We know we have at least one packet in queue */
                 head = slot_dequeue_head(slot);
                 delta = qdisc_pkt_len(head) - qdisc_pkt_len(skb);
                 sch->qstats.backlog -= delta;
                 slot->backlog -= delta;
-                qdisc_drop(head, sch);
+                qdisc_drop(head, sch, to_free);
 
                 slot_queue_add(slot, skb);
                 return NET_XMIT_CN;
@@ -520,7 +520,7 @@ sfq_reset(struct Qdisc *sch)
         struct sk_buff *skb;
 
         while ((skb = sfq_dequeue(sch)) != NULL)
-                kfree_skb(skb);
+                rtnl_kfree_skbs(skb, skb);
 }
 
 /*
@@ -896,7 +896,6 @@ static struct Qdisc_ops sfq_qdisc_ops __read_mostly = {
         .enqueue        =       sfq_enqueue,
         .dequeue        =       sfq_dequeue,
         .peek           =       qdisc_peek_dequeued,
-        .drop           =       sfq_drop,
         .init           =       sfq_init,
         .reset          =       sfq_reset,
         .destroy        =       sfq_destroy,
diff --git a/net/sched/sch_tbf.c b/net/sched/sch_tbf.c
index 3161e491990b..303355c449ab 100644
--- a/net/sched/sch_tbf.c
+++ b/net/sched/sch_tbf.c
@@ -155,7 +155,8 @@ static unsigned int skb_gso_mac_seglen(const struct sk_buff *skb)
 /* GSO packet is too big, segment it so that tbf can transmit
  * each segment in time
  */
-static int tbf_segment(struct sk_buff *skb, struct Qdisc *sch)
+static int tbf_segment(struct sk_buff *skb, struct Qdisc *sch,
+                       struct sk_buff **to_free)
 {
         struct tbf_sched_data *q = qdisc_priv(sch);
         struct sk_buff *segs, *nskb;
@@ -166,7 +167,7 @@ static int tbf_segment(struct sk_buff *skb, struct Qdisc *sch)
         segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK);
 
         if (IS_ERR_OR_NULL(segs))
-                return qdisc_reshape_fail(skb, sch);
+                return qdisc_drop(skb, sch, to_free);
 
         nb = 0;
         while (segs) {
@@ -174,7 +175,7 @@ static int tbf_segment(struct sk_buff *skb, struct Qdisc *sch)
                 segs->next = NULL;
                 qdisc_skb_cb(segs)->pkt_len = segs->len;
                 len += segs->len;
-                ret = qdisc_enqueue(segs, q->qdisc);
+                ret = qdisc_enqueue(segs, q->qdisc, to_free);
                 if (ret != NET_XMIT_SUCCESS) {
                         if (net_xmit_drop_count(ret))
                                 qdisc_qstats_drop(sch);
@@ -190,17 +191,18 @@ static int tbf_segment(struct sk_buff *skb, struct Qdisc *sch)
         return nb > 0 ? NET_XMIT_SUCCESS : NET_XMIT_DROP;
 }
 
-static int tbf_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+static int tbf_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+                       struct sk_buff **to_free)
 {
         struct tbf_sched_data *q = qdisc_priv(sch);
         int ret;
 
         if (qdisc_pkt_len(skb) > q->max_size) {
                 if (skb_is_gso(skb) && skb_gso_mac_seglen(skb) <= q->max_size)
-                        return tbf_segment(skb, sch);
-                return qdisc_reshape_fail(skb, sch);
+                        return tbf_segment(skb, sch, to_free);
+                return qdisc_drop(skb, sch, to_free);
         }
-        ret = qdisc_enqueue(skb, q->qdisc);
+        ret = qdisc_enqueue(skb, q->qdisc, to_free);
         if (ret != NET_XMIT_SUCCESS) {
                 if (net_xmit_drop_count(ret))
                         qdisc_qstats_drop(sch);
@@ -212,19 +214,6 @@ static int tbf_enqueue(struct sk_buff *skb, struct Qdisc *sch)
         return NET_XMIT_SUCCESS;
 }
 
-static unsigned int tbf_drop(struct Qdisc *sch)
-{
-        struct tbf_sched_data *q = qdisc_priv(sch);
-        unsigned int len = 0;
-
-        if (q->qdisc->ops->drop && (len = q->qdisc->ops->drop(q->qdisc)) != 0) {
-                sch->qstats.backlog -= len;
-                sch->q.qlen--;
-                qdisc_qstats_drop(sch);
-        }
-        return len;
-}
-
 static bool tbf_peak_present(const struct tbf_sched_data *q)
 {
         return q->peak.rate_bytes_ps;
@@ -267,14 +256,12 @@ static struct sk_buff *tbf_dequeue(struct Qdisc *sch)
                         q->ptokens = ptoks;
                         qdisc_qstats_backlog_dec(sch, skb);
                         sch->q.qlen--;
-                        qdisc_unthrottled(sch);
                         qdisc_bstats_update(sch, skb);
                         return skb;
                 }
 
                 qdisc_watchdog_schedule_ns(&q->watchdog,
-                                           now + max_t(long, -toks, -ptoks),
-                                           true);
+                                           now + max_t(long, -toks, -ptoks));
 
                 /* Maybe we have a shorter packet in the queue,
                    which can be sent now. It sounds cool,
@@ -559,7 +546,6 @@ static struct Qdisc_ops tbf_qdisc_ops __read_mostly = {
         .enqueue        =       tbf_enqueue,
         .dequeue        =       tbf_dequeue,
         .peek           =       qdisc_peek_dequeued,
-        .drop           =       tbf_drop,
         .init           =       tbf_init,
         .reset          =       tbf_reset,
         .destroy        =       tbf_destroy,
diff --git a/net/sched/sch_teql.c b/net/sched/sch_teql.c
index e02687185a59..2cd9b4478b92 100644
--- a/net/sched/sch_teql.c
+++ b/net/sched/sch_teql.c
@@ -77,7 +77,7 @@ struct teql_sched_data {
 /* "teql*" qdisc routines */
 
 static int
-teql_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+teql_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)
 {
         struct net_device *dev = qdisc_dev(sch);
         struct teql_sched_data *q = qdisc_priv(sch);
@@ -87,7 +87,7 @@ teql_enqueue(struct sk_buff *skb, struct Qdisc *sch)
                 return NET_XMIT_SUCCESS;
         }
 
-        return qdisc_drop(skb, sch);
+        return qdisc_drop(skb, sch, to_free);
 }
 
 static struct sk_buff *
diff --git a/net/sctp/Makefile b/net/sctp/Makefile
index 0fca5824ad0e..6c4f7496cec6 100644
--- a/net/sctp/Makefile
+++ b/net/sctp/Makefile
@@ -11,7 +11,8 @@ sctp-y := sm_statetable.o sm_statefuns.o sm_sideeffect.o \
           transport.o chunk.o sm_make_chunk.o ulpevent.o \
           inqueue.o outqueue.o ulpqueue.o \
           tsnmap.o bind_addr.o socket.o primitive.o \
-          output.o input.o debug.o ssnmap.o auth.o
+          output.o input.o debug.o ssnmap.o auth.o \
+          offload.o
 
 sctp_probe-y := probe.o
 
diff --git a/net/sctp/input.c b/net/sctp/input.c
index a701527a9480..6f8e676d285e 100644
--- a/net/sctp/input.c
+++ b/net/sctp/input.c
@@ -112,7 +112,6 @@ int sctp_rcv(struct sk_buff *skb)
         struct sctp_ep_common *rcvr;
         struct sctp_transport *transport = NULL;
         struct sctp_chunk *chunk;
-        struct sctphdr *sh;
         union sctp_addr src;
         union sctp_addr dest;
         int family;
@@ -124,28 +123,29 @@ int sctp_rcv(struct sk_buff *skb)
 
         __SCTP_INC_STATS(net, SCTP_MIB_INSCTPPACKS);
 
-        if (skb_linearize(skb))
+        /* If packet is too small to contain a single chunk, let's not
+         * waste time on it anymore.
+         */
+        if (skb->len < sizeof(struct sctphdr) + sizeof(struct sctp_chunkhdr) +
+                       skb_transport_offset(skb))
                 goto discard_it;
 
-        sh = sctp_hdr(skb);
+        if (!pskb_may_pull(skb, sizeof(struct sctphdr)))
+                goto discard_it;
 
-        /* Pull up the IP and SCTP headers. */
+        /* Pull up the IP header. */
         __skb_pull(skb, skb_transport_offset(skb));
-        if (skb->len < sizeof(struct sctphdr))
-                goto discard_it;
 
         skb->csum_valid = 0; /* Previous value not applicable */
         if (skb_csum_unnecessary(skb))
                 __skb_decr_checksum_unnecessary(skb);
-        else if (!sctp_checksum_disable && sctp_rcv_checksum(net, skb) < 0)
+        else if (!sctp_checksum_disable &&
+                 !(skb_shinfo(skb)->gso_type & SKB_GSO_SCTP) &&
+                 sctp_rcv_checksum(net, skb) < 0)
                 goto discard_it;
         skb->csum_valid = 1;
 
-        skb_pull(skb, sizeof(struct sctphdr));
-
-        /* Make sure we at least have chunk headers worth of data left. */
-        if (skb->len < sizeof(struct sctp_chunkhdr))
-                goto discard_it;
+        __skb_pull(skb, sizeof(struct sctphdr));
 
         family = ipver2af(ip_hdr(skb)->version);
         af = sctp_get_af_specific(family);
@@ -230,7 +230,7 @@ int sctp_rcv(struct sk_buff *skb)
         chunk->rcvr = rcvr;
 
         /* Remember the SCTP header. */
-        chunk->sctp_hdr = sh;
+        chunk->sctp_hdr = sctp_hdr(skb);
 
         /* Set the source and destination addresses of the incoming chunk.  */
         sctp_init_addrs(chunk, &src, &dest);
@@ -660,19 +660,23 @@ out_unlock:
  */
 static int sctp_rcv_ootb(struct sk_buff *skb)
 {
-        sctp_chunkhdr_t *ch;
-        __u8 *ch_end;
-
-        ch = (sctp_chunkhdr_t *) skb->data;
+        sctp_chunkhdr_t *ch, _ch;
+        int ch_end, offset = 0;
 
         /* Scan through all the chunks in the packet.  */
         do {
+                /* Make sure we have at least the header there */
+                if (offset + sizeof(sctp_chunkhdr_t) > skb->len)
+                        break;
+
+                ch = skb_header_pointer(skb, offset, sizeof(*ch), &_ch);
+
                 /* Break out if chunk length is less then minimal. */
                 if (ntohs(ch->length) < sizeof(sctp_chunkhdr_t))
                         break;
 
-                ch_end = ((__u8 *)ch) + WORD_ROUND(ntohs(ch->length));
-                if (ch_end > skb_tail_pointer(skb))
+                ch_end = offset + WORD_ROUND(ntohs(ch->length));
+                if (ch_end > skb->len)
                         break;
 
                 /* RFC 8.4, 2) If the OOTB packet contains an ABORT chunk, the
@@ -697,8 +701,8 @@ static int sctp_rcv_ootb(struct sk_buff *skb)
                 if (SCTP_CID_INIT == ch->type && (void *)ch != skb->data)
                         goto discard;
 
-                ch = (sctp_chunkhdr_t *) ch_end;
-        } while (ch_end < skb_tail_pointer(skb));
+                offset = ch_end;
+        } while (ch_end < skb->len);
 
         return 0;
 
@@ -1173,6 +1177,17 @@ static struct sctp_association *__sctp_rcv_lookup_harder(struct net *net,
 {
         sctp_chunkhdr_t *ch;
 
+        /* We do not allow GSO frames here as we need to linearize and
+         * then cannot guarantee frame boundaries. This shouldn't be an
+         * issue as packets hitting this are mostly INIT or INIT-ACK and
+         * those cannot be on GSO-style anyway.
+         */
+        if ((skb_shinfo(skb)->gso_type & SKB_GSO_SCTP) == SKB_GSO_SCTP)
+                return NULL;
+
+        if (skb_linearize(skb))
+                return NULL;
+
         ch = (sctp_chunkhdr_t *) skb->data;
 
         /* The code below will attempt to walk the chunk and extract
diff --git a/net/sctp/inqueue.c b/net/sctp/inqueue.c
index 9d87bba0ff1d..edabbbdfca54 100644
--- a/net/sctp/inqueue.c
+++ b/net/sctp/inqueue.c
@@ -130,13 +130,25 @@ struct sctp_chunk *sctp_inq_pop(struct sctp_inq *queue)
          * at this time.
          */
 
-        if ((chunk = queue->in_progress)) {
+        chunk = queue->in_progress;
+        if (chunk) {
                 /* There is a packet that we have been working on.
                  * Any post processing work to do before we move on?
                  */
                 if (chunk->singleton ||
                     chunk->end_of_packet ||
                     chunk->pdiscard) {
+                        if (chunk->head_skb == chunk->skb) {
+                                chunk->skb = skb_shinfo(chunk->skb)->frag_list;
+                                goto new_skb;
+                        }
+                        if (chunk->skb->next) {
+                                chunk->skb = chunk->skb->next;
+                                goto new_skb;
+                        }
+
+                        if (chunk->head_skb)
+                                chunk->skb = chunk->head_skb;
                         sctp_chunk_free(chunk);
                         chunk = queue->in_progress = NULL;
                 } else {
@@ -152,34 +164,64 @@ struct sctp_chunk *sctp_inq_pop(struct sctp_inq *queue)
         if (!chunk) {
                 struct list_head *entry;
 
+next_chunk:
                 /* Is the queue empty?  */
-                if (list_empty(&queue->in_chunk_list))
+                entry = sctp_list_dequeue(&queue->in_chunk_list);
+                if (!entry)
                         return NULL;
 
-                entry = queue->in_chunk_list.next;
-                chunk = queue->in_progress =
-                        list_entry(entry, struct sctp_chunk, list);
-                list_del_init(entry);
+                chunk = list_entry(entry, struct sctp_chunk, list);
 
-                /* This is the first chunk in the packet.  */
-                chunk->singleton = 1;
-                ch = (sctp_chunkhdr_t *) chunk->skb->data;
-                chunk->data_accepted = 0;
+                /* Linearize if it's not GSO */
+                if ((skb_shinfo(chunk->skb)->gso_type & SKB_GSO_SCTP) != SKB_GSO_SCTP &&
+                    skb_is_nonlinear(chunk->skb)) {
+                        if (skb_linearize(chunk->skb)) {
+                                __SCTP_INC_STATS(dev_net(chunk->skb->dev), SCTP_MIB_IN_PKT_DISCARDS);
+                                sctp_chunk_free(chunk);
+                                goto next_chunk;
+                        }
+
+                        /* Update sctp_hdr as it probably changed */
+                        chunk->sctp_hdr = sctp_hdr(chunk->skb);
+                }
+
+                if ((skb_shinfo(chunk->skb)->gso_type & SKB_GSO_SCTP) == SKB_GSO_SCTP) {
+                        /* GSO-marked skbs but without frags, handle
+                         * them normally
+                         */
+                        if (skb_shinfo(chunk->skb)->frag_list)
+                                chunk->head_skb = chunk->skb;
+
+                        /* skbs with "cover letter" */
+                        if (chunk->head_skb && chunk->skb->data_len == chunk->skb->len)
+                                chunk->skb = skb_shinfo(chunk->skb)->frag_list;
+
+                        if (WARN_ON(!chunk->skb)) {
+                                __SCTP_INC_STATS(dev_net(chunk->skb->dev), SCTP_MIB_IN_PKT_DISCARDS);
+                                sctp_chunk_free(chunk);
+                                goto next_chunk;
+                        }
+                }
 
                 if (chunk->asoc)
                         sock_rps_save_rxhash(chunk->asoc->base.sk, chunk->skb);
+
+                queue->in_progress = chunk;
+
+new_skb:
+                /* This is the first chunk in the packet.  */
+                ch = (sctp_chunkhdr_t *) chunk->skb->data;
+                chunk->singleton = 1;
+                chunk->data_accepted = 0;
+                chunk->pdiscard = 0;
+                chunk->auth = 0;
+                chunk->has_asconf = 0;
+                chunk->end_of_packet = 0;
+                chunk->ecn_ce_done = 0;
         }
 
         chunk->chunk_hdr = ch;
         chunk->chunk_end = ((__u8 *)ch) + WORD_ROUND(ntohs(ch->length));
-        /* In the unlikely case of an IP reassembly, the skb could be
-         * non-linear. If so, update chunk_end so that it doesn't go past
-         * the skb->tail.
-         */
-        if (unlikely(skb_is_nonlinear(chunk->skb))) {
-                if (chunk->chunk_end > skb_tail_pointer(chunk->skb))
-                        chunk->chunk_end = skb_tail_pointer(chunk->skb);
-        }
         skb_pull(chunk->skb, sizeof(sctp_chunkhdr_t));
         chunk->subh.v = NULL; /* Subheader is no longer valid.  */
 
diff --git a/net/sctp/offload.c b/net/sctp/offload.c
new file mode 100644
index 000000000000..a37887b373a7
--- /dev/null
+++ b/net/sctp/offload.c
@@ -0,0 +1,98 @@
+/*
+ * sctp_offload - GRO/GSO Offloading for SCTP
+ *
+ * Copyright (C) 2015, Marcelo Ricardo Leitner <marcelo.leitner@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/kprobes.h>
+#include <linux/socket.h>
+#include <linux/sctp.h>
+#include <linux/proc_fs.h>
+#include <linux/vmalloc.h>
+#include <linux/module.h>
+#include <linux/kfifo.h>
+#include <linux/time.h>
+#include <net/net_namespace.h>
+
+#include <linux/skbuff.h>
+#include <net/sctp/sctp.h>
+#include <net/sctp/checksum.h>
+#include <net/protocol.h>
+
+static __le32 sctp_gso_make_checksum(struct sk_buff *skb)
+{
+        skb->ip_summed = CHECKSUM_NONE;
+        return sctp_compute_cksum(skb, skb_transport_offset(skb));
+}
+
+static struct sk_buff *sctp_gso_segment(struct sk_buff *skb,
+                                        netdev_features_t features)
+{
+        struct sk_buff *segs = ERR_PTR(-EINVAL);
+        struct sctphdr *sh;
+
+        sh = sctp_hdr(skb);
+        if (!pskb_may_pull(skb, sizeof(*sh)))
+                goto out;
+
+        __skb_pull(skb, sizeof(*sh));
+
+        if (skb_gso_ok(skb, features | NETIF_F_GSO_ROBUST)) {
+                /* Packet is from an untrusted source, reset gso_segs. */
+                struct skb_shared_info *pinfo = skb_shinfo(skb);
+                struct sk_buff *frag_iter;
+
+                pinfo->gso_segs = 0;
+                if (skb->len != skb->data_len) {
+                        /* Means we have chunks in here too */
+                        pinfo->gso_segs++;
+                }
+
+                skb_walk_frags(skb, frag_iter)
+                        pinfo->gso_segs++;
+
+                segs = NULL;
+                goto out;
+        }
+
+        segs = skb_segment(skb, features | NETIF_F_HW_CSUM);
+        if (IS_ERR(segs))
+                goto out;
+
+        /* All that is left is update SCTP CRC if necessary */
+        if (!(features & NETIF_F_SCTP_CRC)) {
+                for (skb = segs; skb; skb = skb->next) {
+                        if (skb->ip_summed == CHECKSUM_PARTIAL) {
+                                sh = sctp_hdr(skb);
+                                sh->checksum = sctp_gso_make_checksum(skb);
+                        }
+                }
+        }
+
+out:
+        return segs;
+}
+
+static const struct net_offload sctp_offload = {
+        .callbacks = {
+                .gso_segment = sctp_gso_segment,
+        },
+};
+
+int __init sctp_offload_init(void)
+{
+        return inet_add_offload(&sctp_offload, IPPROTO_SCTP);
+}
diff --git a/net/sctp/output.c b/net/sctp/output.c
index 9844fe573029..1541a91d6d9d 100644
--- a/net/sctp/output.c
+++ b/net/sctp/output.c
@@ -84,18 +84,42 @@ static void sctp_packet_reset(struct sctp_packet *packet)
 struct sctp_packet *sctp_packet_config(struct sctp_packet *packet,
                                        __u32 vtag, int ecn_capable)
 {
-        struct sctp_chunk *chunk = NULL;
+        struct sctp_transport *tp = packet->transport;
+        struct sctp_association *asoc = tp->asoc;
 
         pr_debug("%s: packet:%p vtag:0x%x\n", __func__, packet, vtag);
 
         packet->vtag = vtag;
 
+        if (asoc && tp->dst) {
+                struct sock *sk = asoc->base.sk;
+
+                rcu_read_lock();
+                if (__sk_dst_get(sk) != tp->dst) {
+                        dst_hold(tp->dst);
+                        sk_setup_caps(sk, tp->dst);
+                }
+
+                if (sk_can_gso(sk)) {
+                        struct net_device *dev = tp->dst->dev;
+
+                        packet->max_size = dev->gso_max_size;
+                } else {
+                        packet->max_size = asoc->pathmtu;
+                }
+                rcu_read_unlock();
+
+        } else {
+                packet->max_size = tp->pathmtu;
+        }
+
         if (ecn_capable && sctp_packet_empty(packet)) {
-                chunk = sctp_get_ecne_prepend(packet->transport->asoc);
+                struct sctp_chunk *chunk;
 
                 /* If there a is a prepend chunk stick it on the list before
                  * any other chunks get appended.
                  */
+                chunk = sctp_get_ecne_prepend(asoc);
                 if (chunk)
                         sctp_packet_append_chunk(packet, chunk);
         }
@@ -158,7 +182,8 @@ sctp_xmit_t sctp_packet_transmit_chunk(struct sctp_packet *packet,
         sctp_xmit_t retval;
         int error = 0;
 
-        pr_debug("%s: packet:%p chunk:%p\n", __func__, packet, chunk);
+        pr_debug("%s: packet:%p size:%Zu chunk:%p size:%d\n", __func__,
+                 packet, packet->size, chunk, chunk->skb ? chunk->skb->len : -1);
 
         switch ((retval = (sctp_packet_append_chunk(packet, chunk)))) {
         case SCTP_XMIT_PMTU_FULL:
@@ -381,12 +406,15 @@ int sctp_packet_transmit(struct sctp_packet *packet, gfp_t gfp)
         struct sctp_transport *tp = packet->transport;
         struct sctp_association *asoc = tp->asoc;
         struct sctphdr *sh;
-        struct sk_buff *nskb;
+        struct sk_buff *nskb = NULL, *head = NULL;
         struct sctp_chunk *chunk, *tmp;
         struct sock *sk;
         int err = 0;
         int padding;            /* How much padding do we need?  */
+        int pkt_size;
         __u8 has_data = 0;
+        int gso = 0;
+        int pktcount = 0;
         struct dst_entry *dst;
         unsigned char *auth = NULL;     /* pointer to auth in skb data */
 
@@ -400,18 +428,37 @@ int sctp_packet_transmit(struct sctp_packet *packet, gfp_t gfp)
         chunk = list_entry(packet->chunk_list.next, struct sctp_chunk, list);
         sk = chunk->skb->sk;
 
-        /* Allocate the new skb.  */
-        nskb = alloc_skb(packet->size + MAX_HEADER, gfp);
-        if (!nskb)
+        /* Allocate the head skb, or main one if not in GSO */
+        if (packet->size > tp->pathmtu && !packet->ipfragok) {
+                if (sk_can_gso(sk)) {
+                        gso = 1;
+                        pkt_size = packet->overhead;
+                } else {
+                        /* If this happens, we trash this packet and try
+                         * to build a new one, hopefully correct this
+                         * time. Application may notice this error.
+                         */
+                        pr_err_once("Trying to GSO but underlying device doesn't support it.");
+                        goto nomem;
+                }
+        } else {
+                pkt_size = packet->size;
+        }
+        head = alloc_skb(pkt_size + MAX_HEADER, gfp);
+        if (!head)
                 goto nomem;
+        if (gso) {
+                NAPI_GRO_CB(head)->last = head;
+                skb_shinfo(head)->gso_type = sk->sk_gso_type;
+        }
 
         /* Make sure the outbound skb has enough header room reserved. */
-        skb_reserve(nskb, packet->overhead + MAX_HEADER);
+        skb_reserve(head, packet->overhead + MAX_HEADER);
 
         /* Set the owning socket so that we know where to get the
          * destination IP address.
          */
-        sctp_packet_set_owner_w(nskb, sk);
+        sctp_packet_set_owner_w(head, sk);
 
         if (!sctp_transport_dst_check(tp)) {
                 sctp_transport_route(tp, NULL, sctp_sk(sk));
@@ -422,11 +469,11 @@ int sctp_packet_transmit(struct sctp_packet *packet, gfp_t gfp)
         dst = dst_clone(tp->dst);
         if (!dst)
                 goto no_route;
-        skb_dst_set(nskb, dst);
+        skb_dst_set(head, dst);
 
         /* Build the SCTP header.  */
-        sh = (struct sctphdr *)skb_push(nskb, sizeof(struct sctphdr));
-        skb_reset_transport_header(nskb);
+        sh = (struct sctphdr *)skb_push(head, sizeof(struct sctphdr));
+        skb_reset_transport_header(head);
         sh->source = htons(packet->source_port);
         sh->dest   = htons(packet->destination_port);
 
@@ -441,90 +488,133 @@ int sctp_packet_transmit(struct sctp_packet *packet, gfp_t gfp)
         sh->vtag     = htonl(packet->vtag);
         sh->checksum = 0;
 
-        /**
-         * 6.10 Bundling
-         *
-         *    An endpoint bundles chunks by simply including multiple
-         *    chunks in one outbound SCTP packet.  ...
-         */
-
-        /**
-         * 3.2  Chunk Field Descriptions
-         *
-         * The total length of a chunk (including Type, Length and
-         * Value fields) MUST be a multiple of 4 bytes.  If the length
-         * of the chunk is not a multiple of 4 bytes, the sender MUST
-         * pad the chunk with all zero bytes and this padding is not
-         * included in the chunk length field.  The sender should
-         * never pad with more than 3 bytes.
-         *
-         * [This whole comment explains WORD_ROUND() below.]
-         */
-
         pr_debug("***sctp_transmit_packet***\n");
 
-        list_for_each_entry_safe(chunk, tmp, &packet->chunk_list, list) {
-                list_del_init(&chunk->list);
-                if (sctp_chunk_is_data(chunk)) {
-                        /* 6.3.1 C4) When data is in flight and when allowed
-                         * by rule C5, a new RTT measurement MUST be made each
-                         * round trip.  Furthermore, new RTT measurements
-                         * SHOULD be made no more than once per round-trip
-                         * for a given destination transport address.
-                         */
+        do {
+                /* Set up convenience variables... */
+                chunk = list_entry(packet->chunk_list.next, struct sctp_chunk, list);
+                pktcount++;
 
-                        if (!chunk->resent && !tp->rto_pending) {
-                                chunk->rtt_in_progress = 1;
-                                tp->rto_pending = 1;
+                /* Calculate packet size, so it fits in PMTU. Leave
+                 * other chunks for the next packets.
+                 */
+                if (gso) {
+                        pkt_size = packet->overhead;
+                        list_for_each_entry(chunk, &packet->chunk_list, list) {
+                                int padded = WORD_ROUND(chunk->skb->len);
+
+                                if (pkt_size + padded > tp->pathmtu)
+                                        break;
+                                pkt_size += padded;
                         }
 
-                        has_data = 1;
-                }
+                        /* Allocate a new skb. */
+                        nskb = alloc_skb(pkt_size + MAX_HEADER, gfp);
+                        if (!nskb)
+                                goto nomem;
 
-                padding = WORD_ROUND(chunk->skb->len) - chunk->skb->len;
-                if (padding)
-                        memset(skb_put(chunk->skb, padding), 0, padding);
+                        /* Make sure the outbound skb has enough header
+                         * room reserved.
+                         */
+                        skb_reserve(nskb, packet->overhead + MAX_HEADER);
+                } else {
+                        nskb = head;
+                }
 
-                /* if this is the auth chunk that we are adding,
-                 * store pointer where it will be added and put
-                 * the auth into the packet.
+                /**
+                 * 3.2  Chunk Field Descriptions
+                 *
+                 * The total length of a chunk (including Type, Length and
+                 * Value fields) MUST be a multiple of 4 bytes.  If the length
+                 * of the chunk is not a multiple of 4 bytes, the sender MUST
+                 * pad the chunk with all zero bytes and this padding is not
+                 * included in the chunk length field.  The sender should
+                 * never pad with more than 3 bytes.
+                 *
+                 * [This whole comment explains WORD_ROUND() below.]
                  */
-                if (chunk == packet->auth)
-                        auth = skb_tail_pointer(nskb);
 
-                memcpy(skb_put(nskb, chunk->skb->len),
+                pkt_size -= packet->overhead;
+                list_for_each_entry_safe(chunk, tmp, &packet->chunk_list, list) {
+                        list_del_init(&chunk->list);
+                        if (sctp_chunk_is_data(chunk)) {
+                                /* 6.3.1 C4) When data is in flight and when allowed
+                                 * by rule C5, a new RTT measurement MUST be made each
+                                 * round trip.  Furthermore, new RTT measurements
+                                 * SHOULD be made no more than once per round-trip
+                                 * for a given destination transport address.
+                                 */
+
+                                if (!chunk->resent && !tp->rto_pending) {
+                                        chunk->rtt_in_progress = 1;
+                                        tp->rto_pending = 1;
+                                }
+
+                                has_data = 1;
+                        }
+
+                        padding = WORD_ROUND(chunk->skb->len) - chunk->skb->len;
+                        if (padding)
+                                memset(skb_put(chunk->skb, padding), 0, padding);
+
+                        /* if this is the auth chunk that we are adding,
+                         * store pointer where it will be added and put
+                         * the auth into the packet.
+                         */
+                        if (chunk == packet->auth)
+                                auth = skb_tail_pointer(nskb);
+
+                        memcpy(skb_put(nskb, chunk->skb->len),
                                chunk->skb->data, chunk->skb->len);
 
-                pr_debug("*** Chunk:%p[%s] %s 0x%x, length:%d, chunk->skb->len:%d, "
-                         "rtt_in_progress:%d\n", chunk,
-                         sctp_cname(SCTP_ST_CHUNK(chunk->chunk_hdr->type)),
-                         chunk->has_tsn ? "TSN" : "No TSN",
-                         chunk->has_tsn ? ntohl(chunk->subh.data_hdr->tsn) : 0,
-                         ntohs(chunk->chunk_hdr->length), chunk->skb->len,
-                         chunk->rtt_in_progress);
-
-                /*
-                 * If this is a control chunk, this is our last
-                 * reference. Free data chunks after they've been
-                 * acknowledged or have failed.
-                 */
-                if (!sctp_chunk_is_data(chunk))
-                        sctp_chunk_free(chunk);
-        }
+                        pr_debug("*** Chunk:%p[%s] %s 0x%x, length:%d, chunk->skb->len:%d, rtt_in_progress:%d\n",
+                                 chunk,
+                                 sctp_cname(SCTP_ST_CHUNK(chunk->chunk_hdr->type)),
+                                 chunk->has_tsn ? "TSN" : "No TSN",
+                                 chunk->has_tsn ? ntohl(chunk->subh.data_hdr->tsn) : 0,
+                                 ntohs(chunk->chunk_hdr->length), chunk->skb->len,
+                                 chunk->rtt_in_progress);
+
+                        /* If this is a control chunk, this is our last
+                         * reference. Free data chunks after they've been
+                         * acknowledged or have failed.
+                         * Re-queue auth chunks if needed.
+                         */
+                        pkt_size -= WORD_ROUND(chunk->skb->len);
 
-        /* SCTP-AUTH, Section 6.2
-         *    The sender MUST calculate the MAC as described in RFC2104 [2]
-         *    using the hash function H as described by the MAC Identifier and
-         *    the shared association key K based on the endpoint pair shared key
-         *    described by the shared key identifier.  The 'data' used for the
-         *    computation of the AUTH-chunk is given by the AUTH chunk with its
-         *    HMAC field set to zero (as shown in Figure 6) followed by all
-         *    chunks that are placed after the AUTH chunk in the SCTP packet.
-         */
-        if (auth)
-                sctp_auth_calculate_hmac(asoc, nskb,
-                                         (struct sctp_auth_chunk *)auth,
-                                         gfp);
+                        if (chunk == packet->auth && !list_empty(&packet->chunk_list))
+                                list_add(&chunk->list, &packet->chunk_list);
+                        else if (!sctp_chunk_is_data(chunk))
+                                sctp_chunk_free(chunk);
+
+                        if (!pkt_size)
+                                break;
+                }
+
+                /* SCTP-AUTH, Section 6.2
+                 *    The sender MUST calculate the MAC as described in RFC2104 [2]
+                 *    using the hash function H as described by the MAC Identifier and
+                 *    the shared association key K based on the endpoint pair shared key
+                 *    described by the shared key identifier.  The 'data' used for the
+                 *    computation of the AUTH-chunk is given by the AUTH chunk with its
+                 *    HMAC field set to zero (as shown in Figure 6) followed by all
+                 *    chunks that are placed after the AUTH chunk in the SCTP packet.
+                 */
+                if (auth)
+                        sctp_auth_calculate_hmac(asoc, nskb,
+                                                 (struct sctp_auth_chunk *)auth,
+                                                 gfp);
+
+                if (!gso)
+                        break;
+
+                if (skb_gro_receive(&head, nskb))
+                        goto nomem;
+                nskb = NULL;
+                if (WARN_ON_ONCE(skb_shinfo(head)->gso_segs >=
+                                 sk->sk_gso_max_segs))
+                        goto nomem;
+        } while (!list_empty(&packet->chunk_list));
 
         /* 2) Calculate the Adler-32 checksum of the whole packet,
          *    including the SCTP common header and all the
@@ -532,16 +622,18 @@ int sctp_packet_transmit(struct sctp_packet *packet, gfp_t gfp)
          *
          * Note: Adler-32 is no longer applicable, as has been replaced
          * by CRC32-C as described in <draft-ietf-tsvwg-sctpcsum-02.txt>.
+         *
+         * If it's a GSO packet, it's postponed to sctp_skb_segment.
          */
-        if (!sctp_checksum_disable) {
-                if (!(dst->dev->features & NETIF_F_SCTP_CRC) ||
-                    (dst_xfrm(dst) != NULL) || packet->ipfragok) {
-                        sh->checksum = sctp_compute_cksum(nskb, 0);
+        if (!sctp_checksum_disable || gso) {
+                if (!gso && (!(dst->dev->features & NETIF_F_SCTP_CRC) ||
+                             dst_xfrm(dst) || packet->ipfragok)) {
+                        sh->checksum = sctp_compute_cksum(head, 0);
                 } else {
                         /* no need to seed pseudo checksum for SCTP */
-                        nskb->ip_summed = CHECKSUM_PARTIAL;
-                        nskb->csum_start = skb_transport_header(nskb) - nskb->head;
-                        nskb->csum_offset = offsetof(struct sctphdr, checksum);
+                        head->ip_summed = CHECKSUM_PARTIAL;
+                        head->csum_start = skb_transport_header(head) - head->head;
+                        head->csum_offset = offsetof(struct sctphdr, checksum);
                 }
         }
 
@@ -557,7 +649,7 @@ int sctp_packet_transmit(struct sctp_packet *packet, gfp_t gfp)
          * Note: The works for IPv6 layer checks this bit too later
          * in transmission.  See IP6_ECN_flow_xmit().
          */
-        tp->af_specific->ecn_capable(nskb->sk);
+        tp->af_specific->ecn_capable(sk);
 
         /* Set up the IP options.  */
         /* BUG: not implemented
@@ -566,7 +658,7 @@ int sctp_packet_transmit(struct sctp_packet *packet, gfp_t gfp)
 
         /* Dump that on IP!  */
         if (asoc) {
-                asoc->stats.opackets++;
+                asoc->stats.opackets += pktcount;
                 if (asoc->peer.last_sent_to != tp)
                         /* Considering the multiple CPU scenario, this is a
                          * "correcter" place for last_sent_to.  --xguo
@@ -589,16 +681,36 @@ int sctp_packet_transmit(struct sctp_packet *packet, gfp_t gfp)
                 }
         }
 
-        pr_debug("***sctp_transmit_packet*** skb->len:%d\n", nskb->len);
+        pr_debug("***sctp_transmit_packet*** skb->len:%d\n", head->len);
+
+        if (gso) {
+                /* Cleanup our debris for IP stacks */
+                memset(head->cb, 0, max(sizeof(struct inet_skb_parm),
+                                        sizeof(struct inet6_skb_parm)));
 
-        nskb->ignore_df = packet->ipfragok;
-        tp->af_specific->sctp_xmit(nskb, tp);
+                skb_shinfo(head)->gso_segs = pktcount;
+                skb_shinfo(head)->gso_size = GSO_BY_FRAGS;
+
+                /* We have to refresh this in case we are xmiting to
+                 * more than one transport at a time
+                 */
+                rcu_read_lock();
+                if (__sk_dst_get(sk) != tp->dst) {
+                        dst_hold(tp->dst);
+                        sk_setup_caps(sk, tp->dst);
+                }
+                rcu_read_unlock();
+        }
+        head->ignore_df = packet->ipfragok;
+        tp->af_specific->sctp_xmit(head, tp);
 
 out:
         sctp_packet_reset(packet);
         return err;
 no_route:
-        kfree_skb(nskb);
+        kfree_skb(head);
+        if (nskb != head)
+                kfree_skb(nskb);
 
         if (asoc)
                 IP_INC_STATS(sock_net(asoc->base.sk), IPSTATS_MIB_OUTNOROUTES);
@@ -751,39 +863,63 @@ static sctp_xmit_t sctp_packet_will_fit(struct sctp_packet *packet,
                                         struct sctp_chunk *chunk,
                                         u16 chunk_len)
 {
-        size_t psize;
-        size_t pmtu;
-        int too_big;
+        size_t psize, pmtu;
         sctp_xmit_t retval = SCTP_XMIT_OK;
 
         psize = packet->size;
-        pmtu  = ((packet->transport->asoc) ?
-                (packet->transport->asoc->pathmtu) :
-                (packet->transport->pathmtu));
-
-        too_big = (psize + chunk_len > pmtu);
+        if (packet->transport->asoc)
+                pmtu = packet->transport->asoc->pathmtu;
+        else
+                pmtu = packet->transport->pathmtu;
 
         /* Decide if we need to fragment or resubmit later. */
-        if (too_big) {
-                /* It's OK to fragmet at IP level if any one of the following
+        if (psize + chunk_len > pmtu) {
+                /* It's OK to fragment at IP level if any one of the following
                  * is true:
-                 *      1. The packet is empty (meaning this chunk is greater
-                 *         the MTU)
-                 *      2. The chunk we are adding is a control chunk
-                 *      3. The packet doesn't have any data in it yet and data
-                 *      requires authentication.
+                 *      1. The packet is empty (meaning this chunk is greater
+                 *         the MTU)
+                 *      2. The packet doesn't have any data in it yet and data
+                 *         requires authentication.
                  */
-                if (sctp_packet_empty(packet) || !sctp_chunk_is_data(chunk) ||
+                if (sctp_packet_empty(packet) ||
                     (!packet->has_data && chunk->auth)) {
                         /* We no longer do re-fragmentation.
                          * Just fragment at the IP layer, if we
                          * actually hit this condition
                          */
                         packet->ipfragok = 1;
-                } else {
-                        retval = SCTP_XMIT_PMTU_FULL;
+                        goto out;
                 }
+
+                /* It is also okay to fragment if the chunk we are
+                 * adding is a control chunk, but only if current packet
+                 * is not a GSO one otherwise it causes fragmentation of
+                 * a large frame. So in this case we allow the
+                 * fragmentation by forcing it to be in a new packet.
+                 */
+                if (!sctp_chunk_is_data(chunk) && packet->has_data)
+                        retval = SCTP_XMIT_PMTU_FULL;
+
+                if (psize + chunk_len > packet->max_size)
+                        /* Hit GSO/PMTU limit, gotta flush */
+                        retval = SCTP_XMIT_PMTU_FULL;
+
+                if (!packet->transport->burst_limited &&
+                    psize + chunk_len > (packet->transport->cwnd >> 1))
+                        /* Do not allow a single GSO packet to use more
+                         * than half of cwnd.
+                         */
+                        retval = SCTP_XMIT_PMTU_FULL;
+
+                if (packet->transport->burst_limited &&
+                    psize + chunk_len > (packet->transport->burst_limited >> 1))
+                        /* Do not allow a single GSO packet to use more
+                         * than half of original cwnd.
+                         */
+                        retval = SCTP_XMIT_PMTU_FULL;
+                /* Otherwise it will fit in the GSO packet */
         }
 
+out:
         return retval;
 }
diff --git a/net/sctp/protocol.c b/net/sctp/protocol.c
index d3d50daa248b..3b56ae55aba3 100644
--- a/net/sctp/protocol.c
+++ b/net/sctp/protocol.c
@@ -1479,7 +1479,8 @@ static __init int sctp_init(void)
                 INIT_HLIST_HEAD(&sctp_port_hashtable[i].chain);
         }
 
-        if (sctp_transport_hashtable_init())
+        status = sctp_transport_hashtable_init();
+        if (status)
                 goto err_thash_alloc;
 
         pr_info("Hash tables configured (bind %d/%d)\n", sctp_port_hashsize,
@@ -1516,6 +1517,9 @@ static __init int sctp_init(void)
         if (status)
                 goto err_v6_add_protocol;
 
+        if (sctp_offload_init() < 0)
+                pr_crit("%s: Cannot add SCTP protocol offload\n", __func__);
+
 out:
         return status;
 err_v6_add_protocol:
diff --git a/net/sctp/sm_sideeffect.c b/net/sctp/sm_sideeffect.c
index aa3712259368..12d45193357c 100644
--- a/net/sctp/sm_sideeffect.c
+++ b/net/sctp/sm_sideeffect.c
@@ -806,8 +806,10 @@ static void sctp_cmd_new_state(sctp_cmd_seq_t *cmds,
 
                 /* Set the RCV_SHUTDOWN flag when a SHUTDOWN is received. */
                 if (sctp_state(asoc, SHUTDOWN_RECEIVED) &&
-                    sctp_sstate(sk, ESTABLISHED))
+                    sctp_sstate(sk, ESTABLISHED)) {
+                        sk->sk_state = SCTP_SS_CLOSING;
                         sk->sk_shutdown |= RCV_SHUTDOWN;
+                }
         }
 
         if (sctp_state(asoc, COOKIE_WAIT)) {
diff --git a/net/sctp/socket.c b/net/sctp/socket.c
index 67154b848aa9..cdabbd8219b1 100644
--- a/net/sctp/socket.c
+++ b/net/sctp/socket.c
@@ -4003,6 +4003,8 @@ static int sctp_init_sock(struct sock *sk)
                 return -ESOCKTNOSUPPORT;
         }
 
+        sk->sk_gso_type = SKB_GSO_SCTP;
+
         /* Initialize default send parameters. These parameters can be
          * modified with the SCTP_DEFAULT_SEND_PARAM socket option.
          */
@@ -4193,6 +4195,7 @@ static void sctp_shutdown(struct sock *sk, int how)
                 return;
 
         if (how & SEND_SHUTDOWN) {
+                sk->sk_state = SCTP_SS_CLOSING;
                 ep = sctp_sk(sk)->ep;
                 if (!list_empty(&ep->asocs)) {
                         asoc = list_entry(ep->asocs.next,
@@ -7564,10 +7567,13 @@ static void sctp_sock_migrate(struct sock *oldsk, struct sock *newsk,
         /* If the association on the newsk is already closed before accept()
          * is called, set RCV_SHUTDOWN flag.
          */
-        if (sctp_state(assoc, CLOSED) && sctp_style(newsk, TCP))
+        if (sctp_state(assoc, CLOSED) && sctp_style(newsk, TCP)) {
+                newsk->sk_state = SCTP_SS_CLOSED;
                 newsk->sk_shutdown |= RCV_SHUTDOWN;
+        } else {
+                newsk->sk_state = SCTP_SS_ESTABLISHED;
+        }
 
-        newsk->sk_state = SCTP_SS_ESTABLISHED;
         release_sock(newsk);
 }
 
diff --git a/net/tipc/Makefile b/net/tipc/Makefile
index 57e460be4692..31b9f9c52974 100644
--- a/net/tipc/Makefile
+++ b/net/tipc/Makefile
@@ -6,7 +6,7 @@ obj-$(CONFIG_TIPC) := tipc.o
 
 tipc-y  += addr.o bcast.o bearer.o \
            core.o link.o discover.o msg.o  \
-           name_distr.o  subscr.o name_table.o net.o  \
+           name_distr.o  subscr.o monitor.o name_table.o net.o  \
            netlink.o netlink_compat.o node.o socket.o eth_media.o \
            server.o socket.o
 
diff --git a/net/tipc/addr.h b/net/tipc/addr.h
index 93f7c983be33..64f4004a6fac 100644
--- a/net/tipc/addr.h
+++ b/net/tipc/addr.h
@@ -73,4 +73,5 @@ int tipc_addr_node_valid(u32 addr);
 int tipc_in_scope(u32 domain, u32 addr);
 int tipc_addr_scope(u32 domain);
 char *tipc_addr_string_fill(char *string, u32 addr);
+
 #endif
diff --git a/net/tipc/bearer.c b/net/tipc/bearer.c
index bf8f05c3eb82..8584cc48654c 100644
--- a/net/tipc/bearer.c
+++ b/net/tipc/bearer.c
@@ -1,7 +1,7 @@
 /*
  * net/tipc/bearer.c: TIPC bearer code
  *
- * Copyright (c) 1996-2006, 2013-2014, Ericsson AB
+ * Copyright (c) 1996-2006, 2013-2016, Ericsson AB
  * Copyright (c) 2004-2006, 2010-2013, Wind River Systems
  * All rights reserved.
  *
@@ -39,6 +39,7 @@
 #include "bearer.h"
 #include "link.h"
 #include "discover.h"
+#include "monitor.h"
 #include "bcast.h"
 #include "netlink.h"
 
@@ -313,6 +314,10 @@ restart:
         rcu_assign_pointer(tn->bearer_list[bearer_id], b);
         if (skb)
                 tipc_bearer_xmit_skb(net, bearer_id, skb, &b->bcast_addr);
+
+        if (tipc_mon_create(net, bearer_id))
+                return -ENOMEM;
+
         pr_info("Enabled bearer <%s>, discovery domain %s, priority %u\n",
                 name,
                 tipc_addr_string_fill(addr_string, disc_domain), priority);
@@ -348,6 +353,7 @@ static void bearer_disable(struct net *net, struct tipc_bearer *b)
                 tipc_disc_delete(b->link_req);
         RCU_INIT_POINTER(tn->bearer_list[bearer_id], NULL);
         kfree_rcu(b, rcu);
+        tipc_mon_delete(net, bearer_id);
 }
 
 int tipc_enable_l2_media(struct net *net, struct tipc_bearer *b,
diff --git a/net/tipc/bearer.h b/net/tipc/bearer.h
index f686e41b5abb..0d337c7b6fad 100644
--- a/net/tipc/bearer.h
+++ b/net/tipc/bearer.h
@@ -1,7 +1,7 @@
 /*
  * net/tipc/bearer.h: Include file for TIPC bearer code
  *
- * Copyright (c) 1996-2006, 2013-2014, Ericsson AB
+ * Copyright (c) 1996-2006, 2013-2016, Ericsson AB
  * Copyright (c) 2005, 2010-2011, Wind River Systems
  * All rights reserved.
  *
diff --git a/net/tipc/core.c b/net/tipc/core.c
index fe1b062c4f18..236b043a4156 100644
--- a/net/tipc/core.c
+++ b/net/tipc/core.c
@@ -57,6 +57,7 @@ static int __net_init tipc_init_net(struct net *net)
 
         tn->net_id = 4711;
         tn->own_addr = 0;
+        tn->mon_threshold = TIPC_DEF_MON_THRESHOLD;
         get_random_bytes(&tn->random, sizeof(int));
         INIT_LIST_HEAD(&tn->node_list);
         spin_lock_init(&tn->node_list_lock);
diff --git a/net/tipc/core.h b/net/tipc/core.h
index eff58dc53aa1..a1845fb27d80 100644
--- a/net/tipc/core.h
+++ b/net/tipc/core.h
@@ -66,11 +66,13 @@ struct tipc_bc_base;
 struct tipc_link;
 struct tipc_name_table;
 struct tipc_server;
+struct tipc_monitor;
 
 #define TIPC_MOD_VER "2.0.0"
 
-#define NODE_HTABLE_SIZE   512
-#define MAX_BEARERS        3
+#define NODE_HTABLE_SIZE       512
+#define MAX_BEARERS              3
+#define TIPC_DEF_MON_THRESHOLD  32
 
 extern int tipc_net_id __read_mostly;
 extern int sysctl_tipc_rmem[3] __read_mostly;
@@ -88,6 +90,10 @@ struct tipc_net {
         u32 num_nodes;
         u32 num_links;
 
+        /* Neighbor monitoring list */
+        struct tipc_monitor *monitors[MAX_BEARERS];
+        int mon_threshold;
+
         /* Bearer list */
         struct tipc_bearer __rcu *bearer_list[MAX_BEARERS + 1];
 
@@ -126,6 +132,11 @@ static inline struct list_head *tipc_nodes(struct net *net)
         return &tipc_net(net)->node_list;
 }
 
+static inline unsigned int tipc_hashfn(u32 addr)
+{
+        return addr & (NODE_HTABLE_SIZE - 1);
+}
+
 static inline u16 mod(u16 x)
 {
         return x & 0xffffu;
diff --git a/net/tipc/discover.c b/net/tipc/discover.c
index ad9d477cc242..6b109a808d4c 100644
--- a/net/tipc/discover.c
+++ b/net/tipc/discover.c
@@ -135,9 +135,12 @@ void tipc_disc_rcv(struct net *net, struct sk_buff *skb,
         u16 caps = msg_node_capabilities(hdr);
         bool respond = false;
         bool dupl_addr = false;
+        int err;
 
-        bearer->media->msg2addr(bearer, &maddr, msg_media_addr(hdr));
+        err = bearer->media->msg2addr(bearer, &maddr, msg_media_addr(hdr));
         kfree_skb(skb);
+        if (err)
+                return;
 
         /* Ensure message from node is valid and communication is permitted */
         if (net_id != tn->net_id)
diff --git a/net/tipc/link.c b/net/tipc/link.c
index 67b6ab9f4c8d..c1df33f878b2 100644
--- a/net/tipc/link.c
+++ b/net/tipc/link.c
@@ -42,6 +42,7 @@
 #include "name_distr.h"
 #include "discover.h"
 #include "netlink.h"
+#include "monitor.h"
 
 #include <linux/pkt_sched.h>
 
@@ -87,7 +88,6 @@ struct tipc_stats {
  * @peer_bearer_id: bearer id used by link's peer endpoint
  * @bearer_id: local bearer id used by link
  * @tolerance: minimum link continuity loss needed to reset link [in ms]
- * @keepalive_intv: link keepalive timer interval
  * @abort_limit: # of unacknowledged continuity probes needed to reset link
  * @state: current state of link FSM
  * @peer_caps: bitmap describing capabilities of peer node
@@ -96,6 +96,7 @@ struct tipc_stats {
  * @pmsg: convenience pointer to "proto_msg" field
  * @priority: current link priority
  * @net_plane: current link network plane ('A' through 'H')
+ * @mon_state: cookie with information needed by link monitor
  * @backlog_limit: backlog queue congestion thresholds (indexed by importance)
  * @exp_msg_count: # of tunnelled messages expected during link changeover
  * @reset_rcv_checkpt: seq # of last acknowledged message at time of link reset
@@ -131,7 +132,6 @@ struct tipc_link {
         u32 peer_bearer_id;
         u32 bearer_id;
         u32 tolerance;
-        unsigned long keepalive_intv;
         u32 abort_limit;
         u32 state;
         u16 peer_caps;
@@ -140,6 +140,7 @@ struct tipc_link {
         char if_name[TIPC_MAX_IF_NAME];
         u32 priority;
         char net_plane;
+        struct tipc_mon_state mon_state;
         u16 rst_cnt;
 
         /* Failover/synch */
@@ -711,18 +712,25 @@ int tipc_link_timeout(struct tipc_link *l, struct sk_buff_head *xmitq)
         bool setup = false;
         u16 bc_snt = l->bc_sndlink->snd_nxt - 1;
         u16 bc_acked = l->bc_rcvlink->acked;
-
-        link_profile_stats(l);
+        struct tipc_mon_state *mstate = &l->mon_state;
 
         switch (l->state) {
         case LINK_ESTABLISHED:
         case LINK_SYNCHING:
-                if (l->silent_intv_cnt > l->abort_limit)
-                        return tipc_link_fsm_evt(l, LINK_FAILURE_EVT);
                 mtyp = STATE_MSG;
+                link_profile_stats(l);
+                tipc_mon_get_state(l->net, l->addr, mstate, l->bearer_id);
+                if (mstate->reset || (l->silent_intv_cnt > l->abort_limit))
+                        return tipc_link_fsm_evt(l, LINK_FAILURE_EVT);
                 state = bc_acked != bc_snt;
-                probe = l->silent_intv_cnt;
-                l->silent_intv_cnt++;
+                state |= l->bc_rcvlink->rcv_unacked;
+                state |= l->rcv_unacked;
+                state |= !skb_queue_empty(&l->transmq);
+                state |= !skb_queue_empty(&l->deferdq);
+                probe = mstate->probing;
+                probe |= l->silent_intv_cnt;
+                if (probe || mstate->monitoring)
+                        l->silent_intv_cnt++;
                 break;
         case LINK_RESET:
                 setup = l->rst_cnt++ <= 4;
@@ -833,6 +841,7 @@ void tipc_link_reset(struct tipc_link *l)
         l->stats.recv_info = 0;
         l->stale_count = 0;
         l->bc_peer_is_up = false;
+        memset(&l->mon_state, 0, sizeof(l->mon_state));
         tipc_link_reset_stats(l);
 }
 
@@ -1241,6 +1250,9 @@ static void tipc_link_build_proto_msg(struct tipc_link *l, int mtyp, bool probe,
         struct tipc_msg *hdr;
         struct sk_buff_head *dfq = &l->deferdq;
         bool node_up = link_is_up(l->bc_rcvlink);
+        struct tipc_mon_state *mstate = &l->mon_state;
+        int dlen = 0;
+        void *data;
 
         /* Don't send protocol message during reset or link failover */
         if (tipc_link_is_blocked(l))
@@ -1253,12 +1265,13 @@ static void tipc_link_build_proto_msg(struct tipc_link *l, int mtyp, bool probe,
                 rcvgap = buf_seqno(skb_peek(dfq)) - l->rcv_nxt;
 
         skb = tipc_msg_create(LINK_PROTOCOL, mtyp, INT_H_SIZE,
-                              TIPC_MAX_IF_NAME, l->addr,
+                              tipc_max_domain_size, l->addr,
                               tipc_own_addr(l->net), 0, 0, 0);
         if (!skb)
                 return;
 
         hdr = buf_msg(skb);
+        data = msg_data(hdr);
         msg_set_session(hdr, l->session);
         msg_set_bearer_id(hdr, l->bearer_id);
         msg_set_net_plane(hdr, l->net_plane);
@@ -1274,14 +1287,18 @@ static void tipc_link_build_proto_msg(struct tipc_link *l, int mtyp, bool probe,
 
         if (mtyp == STATE_MSG) {
                 msg_set_seq_gap(hdr, rcvgap);
-                msg_set_size(hdr, INT_H_SIZE);
                 msg_set_probe(hdr, probe);
+                tipc_mon_prep(l->net, data, &dlen, mstate, l->bearer_id);
+                msg_set_size(hdr, INT_H_SIZE + dlen);
+                skb_trim(skb, INT_H_SIZE + dlen);
                 l->stats.sent_states++;
                 l->rcv_unacked = 0;
         } else {
                 /* RESET_MSG or ACTIVATE_MSG */
                 msg_set_max_pkt(hdr, l->advertised_mtu);
-                strcpy(msg_data(hdr), l->if_name);
+                strcpy(data, l->if_name);
+                msg_set_size(hdr, INT_H_SIZE + TIPC_MAX_IF_NAME);
+                skb_trim(skb, INT_H_SIZE + TIPC_MAX_IF_NAME);
         }
         if (probe)
                 l->stats.sent_probes++;
@@ -1374,7 +1391,9 @@ static int tipc_link_proto_rcv(struct tipc_link *l, struct sk_buff *skb,
         u16 peers_tol = msg_link_tolerance(hdr);
         u16 peers_prio = msg_linkprio(hdr);
         u16 rcv_nxt = l->rcv_nxt;
+        u16 dlen = msg_data_sz(hdr);
         int mtyp = msg_type(hdr);
+        void *data;
         char *if_name;
         int rc = 0;
 
@@ -1384,6 +1403,10 @@ static int tipc_link_proto_rcv(struct tipc_link *l, struct sk_buff *skb,
         if (tipc_own_addr(l->net) > msg_prevnode(hdr))
                 l->net_plane = msg_net_plane(hdr);
 
+        skb_linearize(skb);
+        hdr = buf_msg(skb);
+        data = msg_data(hdr);
+
         switch (mtyp) {
         case RESET_MSG:
 
@@ -1394,8 +1417,6 @@ static int tipc_link_proto_rcv(struct tipc_link *l, struct sk_buff *skb,
                 /* fall thru' */
 
         case ACTIVATE_MSG:
-                skb_linearize(skb);
-                hdr = buf_msg(skb);
 
                 /* Complete own link name with peer's interface name */
                 if_name =  strrchr(l->name, ':') + 1;
@@ -1403,7 +1424,7 @@ static int tipc_link_proto_rcv(struct tipc_link *l, struct sk_buff *skb,
                         break;
                 if (msg_data_sz(hdr) < TIPC_MAX_IF_NAME)
                         break;
-                strncpy(if_name, msg_data(hdr), TIPC_MAX_IF_NAME);
+                strncpy(if_name, data, TIPC_MAX_IF_NAME);
 
                 /* Update own tolerance if peer indicates a non-zero value */
                 if (in_range(peers_tol, TIPC_MIN_LINK_TOL, TIPC_MAX_LINK_TOL))
@@ -1451,6 +1472,8 @@ static int tipc_link_proto_rcv(struct tipc_link *l, struct sk_buff *skb,
                                 rc = TIPC_LINK_UP_EVT;
                         break;
                 }
+                tipc_mon_rcv(l->net, data, dlen, l->addr,
+                             &l->mon_state, l->bearer_id);
 
                 /* Send NACK if peer has sent pkts we haven't received yet */
                 if (more(peers_snd_nxt, rcv_nxt) && !tipc_link_is_synching(l))
diff --git a/net/tipc/monitor.c b/net/tipc/monitor.c
new file mode 100644
index 000000000000..0d489e81fcca
--- /dev/null
+++ b/net/tipc/monitor.c
@@ -0,0 +1,651 @@
+/*
+ * net/tipc/monitor.c
+ *
+ * Copyright (c) 2016, Ericsson AB
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the names of the copyright holders nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "core.h"
+#include "addr.h"
+#include "monitor.h"
+
+#define MAX_MON_DOMAIN       64
+#define MON_TIMEOUT          120000
+#define MAX_PEER_DOWN_EVENTS 4
+
+/* struct tipc_mon_domain: domain record to be transferred between peers
+ * @len: actual size of domain record
+ * @gen: current generation of sender's domain
+ * @ack_gen: most recent generation of self's domain acked by peer
+ * @member_cnt: number of domain member nodes described in this record
+ * @up_map: bit map indicating which of the members the sender considers up
+ * @members: identity of the domain members
+ */
+struct tipc_mon_domain {
+        u16 len;
+        u16 gen;
+        u16 ack_gen;
+        u16 member_cnt;
+        u64 up_map;
+        u32 members[MAX_MON_DOMAIN];
+};
+
+/* struct tipc_peer: state of a peer node and its domain
+ * @addr: tipc node identity of peer
+ * @head_map: shows which other nodes currently consider peer 'up'
+ * @domain: most recent domain record from peer
+ * @hash: position in hashed lookup list
+ * @list: position in linked list, in circular ascending order by 'addr'
+ * @applied: number of reported domain members applied on this monitor list
+ * @is_up: peer is up as seen from this node
+ * @is_head: peer is assigned domain head as seen from this node
+ * @is_local: peer is in local domain and should be continuously monitored
+ * @down_cnt: - numbers of other peers which have reported this on lost
+ */
+struct tipc_peer {
+        u32 addr;
+        struct tipc_mon_domain *domain;
+        struct hlist_node hash;
+        struct list_head list;
+        u8 applied;
+        u8 down_cnt;
+        bool is_up;
+        bool is_head;
+        bool is_local;
+};
+
+struct tipc_monitor {
+        struct hlist_head peers[NODE_HTABLE_SIZE];
+        int peer_cnt;
+        struct tipc_peer *self;
+        rwlock_t lock;
+        struct tipc_mon_domain cache;
+        u16 list_gen;
+        u16 dom_gen;
+        struct net *net;
+        struct timer_list timer;
+        unsigned long timer_intv;
+};
+
+static struct tipc_monitor *tipc_monitor(struct net *net, int bearer_id)
+{
+        return tipc_net(net)->monitors[bearer_id];
+}
+
+const int tipc_max_domain_size = sizeof(struct tipc_mon_domain);
+
+/* dom_rec_len(): actual length of domain record for transport
+ */
+static int dom_rec_len(struct tipc_mon_domain *dom, u16 mcnt)
+{
+        return ((void *)&dom->members - (void *)dom) + (mcnt * sizeof(u32));
+}
+
+/* dom_size() : calculate size of own domain based on number of peers
+ */
+static int dom_size(int peers)
+{
+        int i = 0;
+
+        while ((i * i) < peers)
+                i++;
+        return i < MAX_MON_DOMAIN ? i : MAX_MON_DOMAIN;
+}
+
+static void map_set(u64 *up_map, int i, unsigned int v)
+{
+        *up_map &= ~(1ULL << i);
+        *up_map |= ((u64)v << i);
+}
+
+static int map_get(u64 up_map, int i)
+{
+        return (up_map & (1 << i)) >> i;
+}
+
+static struct tipc_peer *peer_prev(struct tipc_peer *peer)
+{
+        return list_last_entry(&peer->list, struct tipc_peer, list);
+}
+
+static struct tipc_peer *peer_nxt(struct tipc_peer *peer)
+{
+        return list_first_entry(&peer->list, struct tipc_peer, list);
+}
+
+static struct tipc_peer *peer_head(struct tipc_peer *peer)
+{
+        while (!peer->is_head)
+                peer = peer_prev(peer);
+        return peer;
+}
+
+static struct tipc_peer *get_peer(struct tipc_monitor *mon, u32 addr)
+{
+        struct tipc_peer *peer;
+        unsigned int thash = tipc_hashfn(addr);
+
+        hlist_for_each_entry(peer, &mon->peers[thash], hash) {
+                if (peer->addr == addr)
+                        return peer;
+        }
+        return NULL;
+}
+
+static struct tipc_peer *get_self(struct net *net, int bearer_id)
+{
+        struct tipc_monitor *mon = tipc_monitor(net, bearer_id);
+
+        return mon->self;
+}
+
+static inline bool tipc_mon_is_active(struct net *net, struct tipc_monitor *mon)
+{
+        struct tipc_net *tn = tipc_net(net);
+
+        return mon->peer_cnt > tn->mon_threshold;
+}
+
+/* mon_identify_lost_members() : - identify amd mark potentially lost members
+ */
+static void mon_identify_lost_members(struct tipc_peer *peer,
+                                      struct tipc_mon_domain *dom_bef,
+                                      int applied_bef)
+{
+        struct tipc_peer *member = peer;
+        struct tipc_mon_domain *dom_aft = peer->domain;
+        int applied_aft = peer->applied;
+        int i;
+
+        for (i = 0; i < applied_bef; i++) {
+                member = peer_nxt(member);
+
+                /* Do nothing if self or peer already see member as down */
+                if (!member->is_up || !map_get(dom_bef->up_map, i))
+                        continue;
+
+                /* Loss of local node must be detected by active probing */
+                if (member->is_local)
+                        continue;
+
+                /* Start probing if member was removed from applied domain */
+                if (!applied_aft || (applied_aft < i)) {
+                        member->down_cnt = 1;
+                        continue;
+                }
+
+                /* Member loss is confirmed if it is still in applied domain */
+                if (!map_get(dom_aft->up_map, i))
+                        member->down_cnt++;
+        }
+}
+
+/* mon_apply_domain() : match a peer's domain record against monitor list
+ */
+static void mon_apply_domain(struct tipc_monitor *mon,
+                             struct tipc_peer *peer)
+{
+        struct tipc_mon_domain *dom = peer->domain;
+        struct tipc_peer *member;
+        u32 addr;
+        int i;
+
+        if (!dom || !peer->is_up)
+                return;
+
+        /* Scan across domain members and match against monitor list */
+        peer->applied = 0;
+        member = peer_nxt(peer);
+        for (i = 0; i < dom->member_cnt; i++) {
+                addr = dom->members[i];
+                if (addr != member->addr)
+                        return;
+                peer->applied++;
+                member = peer_nxt(member);
+        }
+}
+
+/* mon_update_local_domain() : update after peer addition/removal/up/down
+ */
+static void mon_update_local_domain(struct tipc_monitor *mon)
+{
+        struct tipc_peer *self = mon->self;
+        struct tipc_mon_domain *cache = &mon->cache;
+        struct tipc_mon_domain *dom = self->domain;
+        struct tipc_peer *peer = self;
+        u64 prev_up_map = dom->up_map;
+        u16 member_cnt, i;
+        bool diff;
+
+        /* Update local domain size based on current size of cluster */
+        member_cnt = dom_size(mon->peer_cnt) - 1;
+        self->applied = member_cnt;
+
+        /* Update native and cached outgoing local domain records */
+        dom->len = dom_rec_len(dom, member_cnt);
+        diff = dom->member_cnt != member_cnt;
+        dom->member_cnt = member_cnt;
+        for (i = 0; i < member_cnt; i++) {
+                peer = peer_nxt(peer);
+                diff |= dom->members[i] != peer->addr;
+                dom->members[i] = peer->addr;
+                map_set(&dom->up_map, i, peer->is_up);
+                cache->members[i] = htonl(peer->addr);
+        }
+        diff |= dom->up_map != prev_up_map;
+        if (!diff)
+                return;
+        dom->gen = ++mon->dom_gen;
+        cache->len = htons(dom->len);
+        cache->gen = htons(dom->gen);
+        cache->member_cnt = htons(member_cnt);
+        cache->up_map = cpu_to_be64(dom->up_map);
+        mon_apply_domain(mon, self);
+}
+
+/* mon_update_neighbors() : update preceding neighbors of added/removed peer
+ */
+static void mon_update_neighbors(struct tipc_monitor *mon,
+                                 struct tipc_peer *peer)
+{
+        int dz, i;
+
+        dz = dom_size(mon->peer_cnt);
+        for (i = 0; i < dz; i++) {
+                mon_apply_domain(mon, peer);
+                peer = peer_prev(peer);
+        }
+}
+
+/* mon_assign_roles() : reassign peer roles after a network change
+ * The monitor list is consistent at this stage; i.e., each peer is monitoring
+ * a set of domain members as matched between domain record and the monitor list
+ */
+static void mon_assign_roles(struct tipc_monitor *mon, struct tipc_peer *head)
+{
+        struct tipc_peer *peer = peer_nxt(head);
+        struct tipc_peer *self = mon->self;
+        int i = 0;
+
+        for (; peer != self; peer = peer_nxt(peer)) {
+                peer->is_local = false;
+
+                /* Update domain member */
+                if (i++ < head->applied) {
+                        peer->is_head = false;
+                        if (head == self)
+                                peer->is_local = true;
+                        continue;
+                }
+                /* Assign next domain head */
+                if (!peer->is_up)
+                        continue;
+                if (peer->is_head)
+                        break;
+                head = peer;
+                head->is_head = true;
+                i = 0;
+        }
+        mon->list_gen++;
+}
+
+void tipc_mon_remove_peer(struct net *net, u32 addr, int bearer_id)
+{
+        struct tipc_monitor *mon = tipc_monitor(net, bearer_id);
+        struct tipc_peer *self = get_self(net, bearer_id);
+        struct tipc_peer *peer, *prev, *head;
+
+        write_lock_bh(&mon->lock);
+        peer = get_peer(mon, addr);
+        if (!peer)
+                goto exit;
+        prev = peer_prev(peer);
+        list_del(&peer->list);
+        hlist_del(&peer->hash);
+        kfree(peer->domain);
+        kfree(peer);
+        mon->peer_cnt--;
+        head = peer_head(prev);
+        if (head == self)
+                mon_update_local_domain(mon);
+        mon_update_neighbors(mon, prev);
+
+        /* Revert to full-mesh monitoring if we reach threshold */
+        if (!tipc_mon_is_active(net, mon)) {
+                list_for_each_entry(peer, &self->list, list) {
+                        kfree(peer->domain);
+                        peer->domain = NULL;
+                        peer->applied = 0;
+                }
+        }
+        mon_assign_roles(mon, head);
+exit:
+        write_unlock_bh(&mon->lock);
+}
+
+static bool tipc_mon_add_peer(struct tipc_monitor *mon, u32 addr,
+                              struct tipc_peer **peer)
+{
+        struct tipc_peer *self = mon->self;
+        struct tipc_peer *cur, *prev, *p;
+
+        p = kzalloc(sizeof(*p), GFP_ATOMIC);
+        *peer = p;
+        if (!p)
+                return false;
+        p->addr = addr;
+
+        /* Add new peer to lookup list */
+        INIT_LIST_HEAD(&p->list);
+        hlist_add_head(&p->hash, &mon->peers[tipc_hashfn(addr)]);
+
+        /* Sort new peer into iterator list, in ascending circular order */
+        prev = self;
+        list_for_each_entry(cur, &self->list, list) {
+                if ((addr > prev->addr) && (addr < cur->addr))
+                        break;
+                if (((addr < cur->addr) || (addr > prev->addr)) &&
+                    (prev->addr > cur->addr))
+                        break;
+                prev = cur;
+        }
+        list_add_tail(&p->list, &cur->list);
+        mon->peer_cnt++;
+        mon_update_neighbors(mon, p);
+        return true;
+}
+
+void tipc_mon_peer_up(struct net *net, u32 addr, int bearer_id)
+{
+        struct tipc_monitor *mon = tipc_monitor(net, bearer_id);
+        struct tipc_peer *self = get_self(net, bearer_id);
+        struct tipc_peer *peer, *head;
+
+        write_lock_bh(&mon->lock);
+        peer = get_peer(mon, addr);
+        if (!peer && !tipc_mon_add_peer(mon, addr, &peer))
+                goto exit;
+        peer->is_up = true;
+        head = peer_head(peer);
+        if (head == self)
+                mon_update_local_domain(mon);
+        mon_assign_roles(mon, head);
+exit:
+        write_unlock_bh(&mon->lock);
+}
+
+void tipc_mon_peer_down(struct net *net, u32 addr, int bearer_id)
+{
+        struct tipc_monitor *mon = tipc_monitor(net, bearer_id);
+        struct tipc_peer *self = get_self(net, bearer_id);
+        struct tipc_peer *peer, *head;
+        struct tipc_mon_domain *dom;
+        int applied;
+
+        write_lock_bh(&mon->lock);
+        peer = get_peer(mon, addr);
+        if (!peer) {
+                pr_warn("Mon: unknown link %x/%u DOWN\n", addr, bearer_id);
+                goto exit;
+        }
+        applied = peer->applied;
+        peer->applied = 0;
+        dom = peer->domain;
+        peer->domain = NULL;
+        if (peer->is_head)
+                mon_identify_lost_members(peer, dom, applied);
+        kfree(dom);
+        peer->is_up = false;
+        peer->is_head = false;
+        peer->is_local = false;
+        peer->down_cnt = 0;
+        head = peer_head(peer);
+        if (head == self)
+                mon_update_local_domain(mon);
+        mon_assign_roles(mon, head);
+exit:
+        write_unlock_bh(&mon->lock);
+}
+
+/* tipc_mon_rcv - process monitor domain event message
+ */
+void tipc_mon_rcv(struct net *net, void *data, u16 dlen, u32 addr,
+                  struct tipc_mon_state *state, int bearer_id)
+{
+        struct tipc_monitor *mon = tipc_monitor(net, bearer_id);
+        struct tipc_mon_domain *arrv_dom = data;
+        struct tipc_mon_domain dom_bef;
+        struct tipc_mon_domain *dom;
+        struct tipc_peer *peer;
+        u16 new_member_cnt = ntohs(arrv_dom->member_cnt);
+        int new_dlen = dom_rec_len(arrv_dom, new_member_cnt);
+        u16 new_gen = ntohs(arrv_dom->gen);
+        u16 acked_gen = ntohs(arrv_dom->ack_gen);
+        bool probing = state->probing;
+        int i, applied_bef;
+
+        state->probing = false;
+        if (!dlen)
+                return;
+
+        /* Sanity check received domain record */
+        if ((dlen < new_dlen) || ntohs(arrv_dom->len) != new_dlen) {
+                pr_warn_ratelimited("Received illegal domain record\n");
+                return;
+        }
+
+        /* Synch generation numbers with peer if link just came up */
+        if (!state->synched) {
+                state->peer_gen = new_gen - 1;
+                state->acked_gen = acked_gen;
+                state->synched = true;
+        }
+
+        if (more(acked_gen, state->acked_gen))
+                state->acked_gen = acked_gen;
+
+        /* Drop duplicate unless we are waiting for a probe response */
+        if (!more(new_gen, state->peer_gen) && !probing)
+                return;
+
+        write_lock_bh(&mon->lock);
+        peer = get_peer(mon, addr);
+        if (!peer || !peer->is_up)
+                goto exit;
+
+        /* Peer is confirmed, stop any ongoing probing */
+        peer->down_cnt = 0;
+
+        /* Task is done for duplicate record */
+        if (!more(new_gen, state->peer_gen))
+                goto exit;
+
+        state->peer_gen = new_gen;
+
+        /* Cache current domain record for later use */
+        dom_bef.member_cnt = 0;
+        dom = peer->domain;
+        if (dom)
+                memcpy(&dom_bef, dom, dom->len);
+
+        /* Transform and store received domain record */
+        if (!dom || (dom->len < new_dlen)) {
+                kfree(dom);
+                dom = kmalloc(new_dlen, GFP_ATOMIC);
+                peer->domain = dom;
+                if (!dom)
+                        goto exit;
+        }
+        dom->len = new_dlen;
+        dom->gen = new_gen;
+        dom->member_cnt = new_member_cnt;
+        dom->up_map = be64_to_cpu(arrv_dom->up_map);
+        for (i = 0; i < new_member_cnt; i++)
+                dom->members[i] = ntohl(arrv_dom->members[i]);
+
+        /* Update peers affected by this domain record */
+        applied_bef = peer->applied;
+        mon_apply_domain(mon, peer);
+        mon_identify_lost_members(peer, &dom_bef, applied_bef);
+        mon_assign_roles(mon, peer_head(peer));
+exit:
+        write_unlock_bh(&mon->lock);
+}
+
+void tipc_mon_prep(struct net *net, void *data, int *dlen,
+                   struct tipc_mon_state *state, int bearer_id)
+{
+        struct tipc_monitor *mon = tipc_monitor(net, bearer_id);
+        struct tipc_mon_domain *dom = data;
+        u16 gen = mon->dom_gen;
+        u16 len;
+
+        if (!tipc_mon_is_active(net, mon))
+                return;
+
+        /* Send only a dummy record with ack if peer has acked our last sent */
+        if (likely(state->acked_gen == gen)) {
+                len = dom_rec_len(dom, 0);
+                *dlen = len;
+                dom->len = htons(len);
+                dom->gen = htons(gen);
+                dom->ack_gen = htons(state->peer_gen);
+                dom->member_cnt = 0;
+                return;
+        }
+        /* Send the full record */
+        read_lock_bh(&mon->lock);
+        len = ntohs(mon->cache.len);
+        *dlen = len;
+        memcpy(data, &mon->cache, len);
+        read_unlock_bh(&mon->lock);
+        dom->ack_gen = htons(state->peer_gen);
+}
+
+void tipc_mon_get_state(struct net *net, u32 addr,
+                        struct tipc_mon_state *state,
+                        int bearer_id)
+{
+        struct tipc_monitor *mon = tipc_monitor(net, bearer_id);
+        struct tipc_peer *peer;
+
+        /* Used cached state if table has not changed */
+        if (!state->probing &&
+            (state->list_gen == mon->list_gen) &&
+            (state->acked_gen == mon->dom_gen))
+                return;
+
+        read_lock_bh(&mon->lock);
+        peer = get_peer(mon, addr);
+        if (peer) {
+                state->probing = state->acked_gen != mon->dom_gen;
+                state->probing |= peer->down_cnt;
+                state->reset |= peer->down_cnt >= MAX_PEER_DOWN_EVENTS;
+                state->monitoring = peer->is_local;
+                state->monitoring |= peer->is_head;
+                state->list_gen = mon->list_gen;
+        }
+        read_unlock_bh(&mon->lock);
+}
+
+static void mon_timeout(unsigned long m)
+{
+        struct tipc_monitor *mon = (void *)m;
+        struct tipc_peer *self;
+        int best_member_cnt = dom_size(mon->peer_cnt) - 1;
+
+        write_lock_bh(&mon->lock);
+        self = mon->self;
+        if (self && (best_member_cnt != self->applied)) {
+                mon_update_local_domain(mon);
+                mon_assign_roles(mon, self);
+        }
+        write_unlock_bh(&mon->lock);
+        mod_timer(&mon->timer, jiffies + mon->timer_intv);
+}
+
+int tipc_mon_create(struct net *net, int bearer_id)
+{
+        struct tipc_net *tn = tipc_net(net);
+        struct tipc_monitor *mon;
+        struct tipc_peer *self;
+        struct tipc_mon_domain *dom;
+
+        if (tn->monitors[bearer_id])
+                return 0;
+
+        mon = kzalloc(sizeof(*mon), GFP_ATOMIC);
+        self = kzalloc(sizeof(*self), GFP_ATOMIC);
+        dom = kzalloc(sizeof(*dom), GFP_ATOMIC);
+        if (!mon || !self || !dom) {
+                kfree(mon);
+                kfree(self);
+                kfree(dom);
+                return -ENOMEM;
+        }
+        tn->monitors[bearer_id] = mon;
+        rwlock_init(&mon->lock);
+        mon->net = net;
+        mon->peer_cnt = 1;
+        mon->self = self;
+        self->domain = dom;
+        self->addr = tipc_own_addr(net);
+        self->is_up = true;
+        self->is_head = true;
+        INIT_LIST_HEAD(&self->list);
+        setup_timer(&mon->timer, mon_timeout, (unsigned long)mon);
+        mon->timer_intv = msecs_to_jiffies(MON_TIMEOUT + (tn->random & 0xffff));
+        mod_timer(&mon->timer, jiffies + mon->timer_intv);
+        return 0;
+}
+
+void tipc_mon_delete(struct net *net, int bearer_id)
+{
+        struct tipc_net *tn = tipc_net(net);
+        struct tipc_monitor *mon = tipc_monitor(net, bearer_id);
+        struct tipc_peer *self = get_self(net, bearer_id);
+        struct tipc_peer *peer, *tmp;
+
+        write_lock_bh(&mon->lock);
+        tn->monitors[bearer_id] = NULL;
+        list_for_each_entry_safe(peer, tmp, &self->list, list) {
+                list_del(&peer->list);
+                hlist_del(&peer->hash);
+                kfree(peer->domain);
+                kfree(peer);
+        }
+        mon->self = NULL;
+        write_unlock_bh(&mon->lock);
+        del_timer_sync(&mon->timer);
+        kfree(self->domain);
+        kfree(self);
+        kfree(mon);
+}
diff --git a/net/tipc/monitor.h b/net/tipc/monitor.h
new file mode 100644
index 000000000000..598459cbed5d
--- /dev/null
+++ b/net/tipc/monitor.h
@@ -0,0 +1,73 @@
+/*
+ * net/tipc/monitor.h
+ *
+ * Copyright (c) 2015, Ericsson AB
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the names of the copyright holders nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _TIPC_MONITOR_H
+#define _TIPC_MONITOR_H
+
+/* struct tipc_mon_state: link instance's cache of monitor list and domain state
+ * @list_gen: current generation of this node's monitor list
+ * @gen: current generation of this node's local domain
+ * @peer_gen: most recent domain generation received from peer
+ * @acked_gen: most recent generation of self's domain acked by peer
+ * @monitoring: this peer endpoint should continuously monitored
+ * @probing: peer endpoint should be temporarily probed for potential loss
+ * @synched: domain record's generation has been synched with peer after reset
+ */
+struct tipc_mon_state {
+        u16 list_gen;
+        u16 peer_gen;
+        u16 acked_gen;
+        bool monitoring :1;
+        bool probing    :1;
+        bool reset      :1;
+        bool synched    :1;
+};
+
+int tipc_mon_create(struct net *net, int bearer_id);
+void tipc_mon_delete(struct net *net, int bearer_id);
+
+void tipc_mon_peer_up(struct net *net, u32 addr, int bearer_id);
+void tipc_mon_peer_down(struct net *net, u32 addr, int bearer_id);
+void tipc_mon_prep(struct net *net, void *data, int *dlen,
+                   struct tipc_mon_state *state, int bearer_id);
+void tipc_mon_rcv(struct net *net, void *data, u16 dlen, u32 addr,
+                  struct tipc_mon_state *state, int bearer_id);
+void tipc_mon_get_state(struct net *net, u32 addr,
+                        struct tipc_mon_state *state,
+                        int bearer_id);
+void tipc_mon_remove_peer(struct net *net, u32 addr, int bearer_id);
+
+extern const int tipc_max_domain_size;
+#endif
diff --git a/net/tipc/node.c b/net/tipc/node.c
index e01e2c71b5a1..a3fc0a3f4077 100644
--- a/net/tipc/node.c
+++ b/net/tipc/node.c
@@ -40,6 +40,7 @@
 #include "name_distr.h"
 #include "socket.h"
 #include "bcast.h"
+#include "monitor.h"
 #include "discover.h"
 #include "netlink.h"
 
@@ -205,17 +206,6 @@ u16 tipc_node_get_capabilities(struct net *net, u32 addr)
         return caps;
 }
 
-/*
- * A trivial power-of-two bitmask technique is used for speed, since this
- * operation is done for every incoming TIPC packet. The number of hash table
- * entries has been chosen so that no hash chain exceeds 8 nodes and will
- * usually be much smaller (typically only a single node).
- */
-static unsigned int tipc_hashfn(u32 addr)
-{
-        return addr & (NODE_HTABLE_SIZE - 1);
-}
-
 static void tipc_node_kref_release(struct kref *kref)
 {
         struct tipc_node *n = container_of(kref, struct tipc_node, kref);
@@ -279,6 +269,7 @@ static void tipc_node_write_unlock(struct tipc_node *n)
         u32 addr = 0;
         u32 flags = n->action_flags;
         u32 link_id = 0;
+        u32 bearer_id;
         struct list_head *publ_list;
 
         if (likely(!flags)) {
@@ -288,6 +279,7 @@ static void tipc_node_write_unlock(struct tipc_node *n)
 
         addr = n->addr;
         link_id = n->link_id;
+        bearer_id = link_id & 0xffff;
         publ_list = &n->publ_list;
 
         n->action_flags &= ~(TIPC_NOTIFY_NODE_DOWN | TIPC_NOTIFY_NODE_UP |
@@ -301,13 +293,16 @@ static void tipc_node_write_unlock(struct tipc_node *n)
         if (flags & TIPC_NOTIFY_NODE_UP)
                 tipc_named_node_up(net, addr);
 
-        if (flags & TIPC_NOTIFY_LINK_UP)
+        if (flags & TIPC_NOTIFY_LINK_UP) {
+                tipc_mon_peer_up(net, addr, bearer_id);
                 tipc_nametbl_publish(net, TIPC_LINK_STATE, addr, addr,
                                      TIPC_NODE_SCOPE, link_id, addr);
-
-        if (flags & TIPC_NOTIFY_LINK_DOWN)
+        }
+        if (flags & TIPC_NOTIFY_LINK_DOWN) {
+                tipc_mon_peer_down(net, addr, bearer_id);
                 tipc_nametbl_withdraw(net, TIPC_LINK_STATE, addr,
                                       link_id, addr);
+        }
 }
 
 struct tipc_node *tipc_node_create(struct net *net, u32 addr, u16 capabilities)
@@ -378,14 +373,13 @@ static void tipc_node_calculate_timer(struct tipc_node *n, struct tipc_link *l)
 {
         unsigned long tol = tipc_link_tolerance(l);
         unsigned long intv = ((tol / 4) > 500) ? 500 : tol / 4;
-        unsigned long keepalive_intv = msecs_to_jiffies(intv);
 
         /* Link with lowest tolerance determines timer interval */
-        if (keepalive_intv < n->keepalive_intv)
-                n->keepalive_intv = keepalive_intv;
+        if (intv < n->keepalive_intv)
+                n->keepalive_intv = intv;
 
-        /* Ensure link's abort limit corresponds to current interval */
-        tipc_link_set_abort_limit(l, tol / jiffies_to_msecs(n->keepalive_intv));
+        /* Ensure link's abort limit corresponds to current tolerance */
+        tipc_link_set_abort_limit(l, tol / n->keepalive_intv);
 }
 
 static void tipc_node_delete(struct tipc_node *node)
@@ -526,7 +520,7 @@ static void tipc_node_timeout(unsigned long data)
                 if (rc & TIPC_LINK_DOWN_EVT)
                         tipc_node_link_down(n, bearer_id, false);
         }
-        mod_timer(&n->timer, jiffies + n->keepalive_intv);
+        mod_timer(&n->timer, jiffies + msecs_to_jiffies(n->keepalive_intv));
 }
 
 /**
@@ -692,6 +686,7 @@ static void tipc_node_link_down(struct tipc_node *n, int bearer_id, bool delete)
         struct tipc_link *l = le->link;
         struct tipc_media_addr *maddr;
         struct sk_buff_head xmitq;
+        int old_bearer_id = bearer_id;
 
         if (!l)
                 return;
@@ -711,6 +706,8 @@ static void tipc_node_link_down(struct tipc_node *n, int bearer_id, bool delete)
                 tipc_link_fsm_evt(l, LINK_RESET_EVT);
         }
         tipc_node_write_unlock(n);
+        if (delete)
+                tipc_mon_remove_peer(n->net, n->addr, old_bearer_id);
         tipc_bearer_xmit(n->net, bearer_id, &xmitq, maddr);
         tipc_sk_rcv(n->net, &le->inputq);
 }
@@ -735,6 +732,7 @@ void tipc_node_check_dest(struct net *net, u32 onode,
         bool accept_addr = false;
         bool reset = true;
         char *if_name;
+        unsigned long intv;
 
         *dupl_addr = false;
         *respond = false;
@@ -840,9 +838,11 @@ void tipc_node_check_dest(struct net *net, u32 onode,
                 le->link = l;
                 n->link_cnt++;
                 tipc_node_calculate_timer(n, l);
-                if (n->link_cnt == 1)
-                        if (!mod_timer(&n->timer, jiffies + n->keepalive_intv))
+                if (n->link_cnt == 1) {
+                        intv = jiffies + msecs_to_jiffies(n->keepalive_intv);
+                        if (!mod_timer(&n->timer, intv))
                                 tipc_node_get(n);
+                }
         }
         memcpy(&le->maddr, maddr, sizeof(*maddr));
 exit:
@@ -950,7 +950,7 @@ static void tipc_node_fsm_evt(struct tipc_node *n, int evt)
                         state = SELF_UP_PEER_UP;
                         break;
                 case SELF_LOST_CONTACT_EVT:
-                        state = SELF_DOWN_PEER_LEAVING;
+                        state = SELF_DOWN_PEER_DOWN;
                         break;
                 case SELF_ESTABL_CONTACT_EVT:
                 case PEER_LOST_CONTACT_EVT:
@@ -969,7 +969,7 @@ static void tipc_node_fsm_evt(struct tipc_node *n, int evt)
                         state = SELF_UP_PEER_UP;
                         break;
                 case PEER_LOST_CONTACT_EVT:
-                        state = SELF_LEAVING_PEER_DOWN;
+                        state = SELF_DOWN_PEER_DOWN;
                         break;
                 case SELF_LOST_CONTACT_EVT:
                 case PEER_ESTABL_CONTACT_EVT:
diff --git a/net/tipc/server.c b/net/tipc/server.c
index 272d20a795d5..215849ce453d 100644
--- a/net/tipc/server.c
+++ b/net/tipc/server.c
@@ -418,13 +418,12 @@ static struct outqueue_entry *tipc_alloc_entry(void *data, int len)
         if (!entry)
                 return NULL;
 
-        buf = kmalloc(len, GFP_ATOMIC);
+        buf = kmemdup(data, len, GFP_ATOMIC);
         if (!buf) {
                 kfree(entry);
                 return NULL;
         }
 
-        memcpy(buf, data, len);
         entry->iov.iov_base = buf;
         entry->iov.iov_len = len;
 
diff --git a/net/tipc/udp_media.c b/net/tipc/udp_media.c
index c9cf2be3674a..b016c011970b 100644
--- a/net/tipc/udp_media.c
+++ b/net/tipc/udp_media.c
@@ -63,7 +63,7 @@
  */
 struct udp_media_addr {
         __be16  proto;
-        __be16  udp_port;
+        __be16  port;
         union {
                 struct in_addr ipv4;
                 struct in6_addr ipv6;
@@ -108,9 +108,9 @@ static int tipc_udp_addr2str(struct tipc_media_addr *a, char *buf, int size)
         struct udp_media_addr *ua = (struct udp_media_addr *)&a->value;
 
         if (ntohs(ua->proto) == ETH_P_IP)
-                snprintf(buf, size, "%pI4:%u", &ua->ipv4, ntohs(ua->udp_port));
+                snprintf(buf, size, "%pI4:%u", &ua->ipv4, ntohs(ua->port));
         else if (ntohs(ua->proto) == ETH_P_IPV6)
-                snprintf(buf, size, "%pI6:%u", &ua->ipv6, ntohs(ua->udp_port));
+                snprintf(buf, size, "%pI6:%u", &ua->ipv6, ntohs(ua->port));
         else
                 pr_err("Invalid UDP media address\n");
         return 0;
@@ -178,8 +178,8 @@ static int tipc_udp_send_msg(struct net *net, struct sk_buff *skb,
                 skb->dev = rt->dst.dev;
                 ttl = ip4_dst_hoplimit(&rt->dst);
                 udp_tunnel_xmit_skb(rt, ub->ubsock->sk, skb, src->ipv4.s_addr,
-                                    dst->ipv4.s_addr, 0, ttl, 0, src->udp_port,
-                                    dst->udp_port, false, true);
+                                    dst->ipv4.s_addr, 0, ttl, 0, src->port,
+                                    dst->port, false, true);
 #if IS_ENABLED(CONFIG_IPV6)
         } else {
                 struct dst_entry *ndst;
@@ -196,8 +196,8 @@ static int tipc_udp_send_msg(struct net *net, struct sk_buff *skb,
                 ttl = ip6_dst_hoplimit(ndst);
                 err = udp_tunnel6_xmit_skb(ndst, ub->ubsock->sk, skb,
                                            ndst->dev, &src->ipv6,
-                                           &dst->ipv6, 0, ttl, 0, src->udp_port,
-                                           dst->udp_port, false);
+                                           &dst->ipv6, 0, ttl, 0, src->port,
+                                           dst->port, false);
 #endif
         }
         return err;
@@ -292,12 +292,12 @@ err:
 
                 ip4 = (struct sockaddr_in *)&sa_local;
                 local->proto = htons(ETH_P_IP);
-                local->udp_port = ip4->sin_port;
+                local->port = ip4->sin_port;
                 local->ipv4.s_addr = ip4->sin_addr.s_addr;
 
                 ip4 = (struct sockaddr_in *)&sa_remote;
                 remote->proto = htons(ETH_P_IP);
-                remote->udp_port = ip4->sin_port;
+                remote->port = ip4->sin_port;
                 remote->ipv4.s_addr = ip4->sin_addr.s_addr;
                 return 0;
 
@@ -312,13 +312,13 @@ err:
                         return -EINVAL;
 
                 local->proto = htons(ETH_P_IPV6);
-                local->udp_port = ip6->sin6_port;
+                local->port = ip6->sin6_port;
                 memcpy(&local->ipv6, &ip6->sin6_addr, sizeof(struct in6_addr));
                 ub->ifindex = ip6->sin6_scope_id;
 
                 ip6 = (struct sockaddr_in6 *)&sa_remote;
                 remote->proto = htons(ETH_P_IPV6);
-                remote->udp_port = ip6->sin6_port;
+                remote->port = ip6->sin6_port;
                 memcpy(&remote->ipv6, &ip6->sin6_addr, sizeof(struct in6_addr));
                 return 0;
 #endif
@@ -386,7 +386,7 @@ static int tipc_udp_enable(struct net *net, struct tipc_bearer *b,
                 err = -EAFNOSUPPORT;
                 goto err;
         }
-        udp_conf.local_udp_port = local.udp_port;
+        udp_conf.local_udp_port = local.port;
         err = udp_sock_create(net, &udp_conf, &ub->ubsock);
         if (err)
                 goto err;
diff --git a/net/wireless/core.c b/net/wireless/core.c
index ecca3896b9f7..39d9abd309ea 100644
--- a/net/wireless/core.c
+++ b/net/wireless/core.c
@@ -748,6 +748,36 @@ int wiphy_register(struct wiphy *wiphy)
                 nl80211_send_reg_change_event(&request);
         }
 
+        /* Check that nobody globally advertises any capabilities they do not
+         * advertise on all possible interface types.
+         */
+        if (wiphy->extended_capabilities_len &&
+            wiphy->num_iftype_ext_capab &&
+            wiphy->iftype_ext_capab) {
+                u8 supported_on_all, j;
+                const struct wiphy_iftype_ext_capab *capab;
+
+                capab = wiphy->iftype_ext_capab;
+                for (j = 0; j < wiphy->extended_capabilities_len; j++) {
+                        if (capab[0].extended_capabilities_len > j)
+                                supported_on_all =
+                                        capab[0].extended_capabilities[j];
+                        else
+                                supported_on_all = 0x00;
+                        for (i = 1; i < wiphy->num_iftype_ext_capab; i++) {
+                                if (j >= capab[i].extended_capabilities_len) {
+                                        supported_on_all = 0x00;
+                                        break;
+                                }
+                                supported_on_all &=
+                                        capab[i].extended_capabilities[j];
+                        }
+                        if (WARN_ON(wiphy->extended_capabilities[j] &
+                                    ~supported_on_all))
+                                break;
+                }
+        }
+
         rdev->wiphy.registered = true;
         rtnl_unlock();
 
diff --git a/net/wireless/core.h b/net/wireless/core.h
index 025b7a5d508b..a4d547f99f8d 100644
--- a/net/wireless/core.h
+++ b/net/wireless/core.h
@@ -214,7 +214,7 @@ struct cfg80211_event {
                         size_t req_ie_len;
                         size_t resp_ie_len;
                         struct cfg80211_bss *bss;
-                        u16 status;
+                        int status; /* -1 = failed; 0..65535 = status code */
                 } cr;
                 struct {
                         const u8 *req_ie;
@@ -374,7 +374,7 @@ int cfg80211_connect(struct cfg80211_registered_device *rdev,
 void __cfg80211_connect_result(struct net_device *dev, const u8 *bssid,
                                const u8 *req_ie, size_t req_ie_len,
                                const u8 *resp_ie, size_t resp_ie_len,
-                               u16 status, bool wextev,
+                               int status, bool wextev,
                                struct cfg80211_bss *bss);
 void __cfg80211_disconnected(struct net_device *dev, const u8 *ie,
                              size_t ie_len, u16 reason, bool from_ap);
diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c
index d7599014055d..c503e96bfd5a 100644
--- a/net/wireless/nl80211.c
+++ b/net/wireless/nl80211.c
@@ -167,6 +167,7 @@ __cfg80211_rdev_from_attrs(struct net *netns, struct nlattr **attrs)
 
         if (attrs[NL80211_ATTR_IFINDEX]) {
                 int ifindex = nla_get_u32(attrs[NL80211_ATTR_IFINDEX]);
+
                 netdev = __dev_get_by_index(netns, ifindex);
                 if (netdev) {
                         if (netdev->ieee80211_ptr)
@@ -731,6 +732,7 @@ static int nl80211_parse_key_new(struct nlattr *key, struct key_parse *k)
 
         if (tb[NL80211_KEY_DEFAULT_TYPES]) {
                 struct nlattr *kdt[NUM_NL80211_KEY_DEFAULT_TYPES];
+
                 err = nla_parse_nested(kdt, NUM_NL80211_KEY_DEFAULT_TYPES - 1,
                                        tb[NL80211_KEY_DEFAULT_TYPES],
                                        nl80211_key_default_policy);
@@ -1264,7 +1266,7 @@ nl80211_send_mgmt_stypes(struct sk_buff *msg,
 struct nl80211_dump_wiphy_state {
         s64 filter_wiphy;
         long start;
-        long split_start, band_start, chan_start;
+        long split_start, band_start, chan_start, capa_start;
         bool split;
 };
 
@@ -1382,6 +1384,7 @@ static int nl80211_send_wiphy(struct cfg80211_registered_device *rdev,
                     rdev->ops->get_antenna) {
                         u32 tx_ant = 0, rx_ant = 0;
                         int res;
+
                         res = rdev_get_antenna(rdev, &tx_ant, &rx_ant);
                         if (!res) {
                                 if (nla_put_u32(msg,
@@ -1761,6 +1764,47 @@ static int nl80211_send_wiphy(struct cfg80211_registered_device *rdev,
                         nla_nest_end(msg, nested);
                 }
 
+                state->split_start++;
+                break;
+        case 13:
+                if (rdev->wiphy.num_iftype_ext_capab &&
+                    rdev->wiphy.iftype_ext_capab) {
+                        struct nlattr *nested_ext_capab, *nested;
+
+                        nested = nla_nest_start(msg,
+                                                NL80211_ATTR_IFTYPE_EXT_CAPA);
+                        if (!nested)
+                                goto nla_put_failure;
+
+                        for (i = state->capa_start;
+                             i < rdev->wiphy.num_iftype_ext_capab; i++) {
+                                const struct wiphy_iftype_ext_capab *capab;
+
+                                capab = &rdev->wiphy.iftype_ext_capab[i];
+
+                                nested_ext_capab = nla_nest_start(msg, i);
+                                if (!nested_ext_capab ||
+                                    nla_put_u32(msg, NL80211_ATTR_IFTYPE,
+                                                capab->iftype) ||
+                                    nla_put(msg, NL80211_ATTR_EXT_CAPA,
+                                            capab->extended_capabilities_len,
+                                            capab->extended_capabilities) ||
+                                    nla_put(msg, NL80211_ATTR_EXT_CAPA_MASK,
+                                            capab->extended_capabilities_len,
+                                            capab->extended_capabilities_mask))
+                                        goto nla_put_failure;
+
+                                nla_nest_end(msg, nested_ext_capab);
+                                if (state->split)
+                                        break;
+                        }
+                        nla_nest_end(msg, nested);
+                        if (i < rdev->wiphy.num_iftype_ext_capab) {
+                                state->capa_start = i + 1;
+                                break;
+                        }
+                }
+
                 /* done */
                 state->split_start = 0;
                 break;
@@ -2116,7 +2160,6 @@ static int nl80211_set_wds_peer(struct sk_buff *skb, struct genl_info *info)
         return rdev_set_wds_peer(rdev, dev, bssid);
 }
 
-
 static int nl80211_set_wiphy(struct sk_buff *skb, struct genl_info *info)
 {
         struct cfg80211_registered_device *rdev;
@@ -2251,6 +2294,7 @@ static int nl80211_set_wiphy(struct sk_buff *skb, struct genl_info *info)
         if (info->attrs[NL80211_ATTR_WIPHY_ANTENNA_TX] &&
             info->attrs[NL80211_ATTR_WIPHY_ANTENNA_RX]) {
                 u32 tx_ant, rx_ant;
+
                 if ((!rdev->wiphy.available_antennas_tx &&
                      !rdev->wiphy.available_antennas_rx) ||
                     !rdev->ops->set_antenna)
@@ -2919,6 +2963,7 @@ static int nl80211_get_key(struct sk_buff *skb, struct genl_info *info)
         pairwise = !!mac_addr;
         if (info->attrs[NL80211_ATTR_KEY_TYPE]) {
                 u32 kt = nla_get_u32(info->attrs[NL80211_ATTR_KEY_TYPE]);
+
                 if (kt >= NUM_NL80211_KEYTYPES)
                         return -EINVAL;
                 if (kt != NL80211_KEYTYPE_GROUP &&
@@ -3962,7 +4007,6 @@ static int nl80211_dump_station(struct sk_buff *skb,
                 sta_idx++;
         }
 
-
  out:
         cb->args[2] = sta_idx;
         err = skb->len;
@@ -4763,7 +4807,6 @@ static int nl80211_dump_mpath(struct sk_buff *skb,
                 path_idx++;
         }
 
-
  out:
         cb->args[2] = path_idx;
         err = skb->len;
@@ -5053,7 +5096,6 @@ static int nl80211_req_set_reg(struct sk_buff *skb, struct genl_info *info)
         enum nl80211_user_reg_hint_type user_reg_hint_type;
         u32 owner_nlportid;
 
-
         /*
          * You should only get this when cfg80211 hasn't yet initialized
          * completely when built-in to the kernel right between the time
@@ -5262,7 +5304,6 @@ do {									    \
         }                                                                   \
 } while (0)
 
-
         if (!info->attrs[NL80211_ATTR_MESH_CONFIG])
                 return -EINVAL;
         if (nla_parse_nested(tb, NL80211_MESHCONF_ATTR_MAX,
@@ -5409,7 +5450,6 @@ static int nl80211_parse_mesh_setup(struct genl_info *info,
                  IEEE80211_PATH_METRIC_VENDOR :
                  IEEE80211_PATH_METRIC_AIRTIME;
 
-
         if (tb[NL80211_MESH_SETUP_IE]) {
                 struct nlattr *ieattr =
                         tb[NL80211_MESH_SETUP_IE];
@@ -5796,10 +5836,8 @@ static int nl80211_set_reg(struct sk_buff *skb, struct genl_info *info)
                 }
         }
 
-        r = set_regdom(rd, REGD_SOURCE_CRDA);
-        /* set_regdom took ownership */
-        rd = NULL;
-
+        /* set_regdom takes ownership of rd */
+        return set_regdom(rd, REGD_SOURCE_CRDA);
  bad_reg:
         kfree(rd);
         return r;
@@ -6033,6 +6071,7 @@ static int nl80211_trigger_scan(struct sk_buff *skb, struct genl_info *info)
                 /* all channels */
                 for (band = 0; band < NUM_NL80211_BANDS; band++) {
                         int j;
+
                         if (!wiphy->bands[band])
                                 continue;
                         for (j = 0; j < wiphy->bands[band]->n_channels; j++) {
@@ -6442,6 +6481,7 @@ nl80211_parse_sched_scan(struct wiphy *wiphy, struct wireless_dev *wdev,
                 /* all channels */
                 for (band = 0; band < NUM_NL80211_BANDS; band++) {
                         int j;
+
                         if (!wiphy->bands[band])
                                 continue;
                         for (j = 0; j < wiphy->bands[band]->n_channels; j++) {
@@ -6511,7 +6551,7 @@ nl80211_parse_sched_scan(struct wiphy *wiphy, struct wireless_dev *wdev,
                                        nla_data(ssid), nla_len(ssid));
                                 request->match_sets[i].ssid.ssid_len =
                                         nla_len(ssid);
-                                /* special attribute - old implemenation w/a */
+                                /* special attribute - old implementation w/a */
                                 request->match_sets[i].rssi_thold =
                                         default_match_rssi;
                                 rssi = tb[NL80211_SCHED_SCAN_MATCH_ATTR_RSSI];
@@ -7204,6 +7244,7 @@ static int nl80211_authenticate(struct sk_buff *skb, struct genl_info *info)
         if (key.idx >= 0) {
                 int i;
                 bool ok = false;
+
                 for (i = 0; i < rdev->wiphy.n_cipher_suites; i++) {
                         if (key.p.cipher == rdev->wiphy.cipher_suites[i]) {
                                 ok = true;
@@ -7282,6 +7323,7 @@ static int nl80211_crypto_settings(struct cfg80211_registered_device *rdev,
 
         if (info->attrs[NL80211_ATTR_CONTROL_PORT_ETHERTYPE]) {
                 u16 proto;
+
                 proto = nla_get_u16(
                         info->attrs[NL80211_ATTR_CONTROL_PORT_ETHERTYPE]);
                 settings->control_port_ethertype = cpu_to_be16(proto);
@@ -8435,6 +8477,7 @@ static u32 rateset_to_mask(struct ieee80211_supported_band *sband,
         for (i = 0; i < rates_len; i++) {
                 int rate = (rates[i] & 0x7f) * 5;
                 int ridx;
+
                 for (ridx = 0; ridx < sband->n_bitrates; ridx++) {
                         struct ieee80211_rate *srate =
                                 &sband->bitrates[ridx];
@@ -8743,7 +8786,6 @@ static int nl80211_tx_mgmt(struct sk_buff *skb, struct genl_info *info)
                 if (params.wait < NL80211_MIN_REMAIN_ON_CHANNEL_TIME ||
                     params.wait > rdev->wiphy.max_remain_on_channel_duration)
                         return -EINVAL;
-
         }
 
         params.offchan = info->attrs[NL80211_ATTR_OFFCHANNEL_TX_OK];
@@ -10590,7 +10632,6 @@ int cfg80211_vendor_cmd_reply(struct sk_buff *skb)
 }
 EXPORT_SYMBOL_GPL(cfg80211_vendor_cmd_reply);
 
-
 static int nl80211_set_qos_map(struct sk_buff *skb,
                                struct genl_info *info)
 {
@@ -10945,7 +10986,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_SET_WIPHY,
                 .doit = nl80211_set_wiphy,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_RTNL,
         },
         {
@@ -10961,7 +11002,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_SET_INTERFACE,
                 .doit = nl80211_set_interface,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -10969,7 +11010,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_NEW_INTERFACE,
                 .doit = nl80211_new_interface,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_WIPHY |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -10977,7 +11018,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_DEL_INTERFACE,
                 .doit = nl80211_del_interface,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_WDEV |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -10985,7 +11026,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_GET_KEY,
                 .doit = nl80211_get_key,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -10993,7 +11034,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_SET_KEY,
                 .doit = nl80211_set_key,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL |
                                   NL80211_FLAG_CLEAR_SKB,
@@ -11002,7 +11043,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_NEW_KEY,
                 .doit = nl80211_new_key,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL |
                                   NL80211_FLAG_CLEAR_SKB,
@@ -11011,14 +11052,14 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_DEL_KEY,
                 .doit = nl80211_del_key,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
         {
                 .cmd = NL80211_CMD_SET_BEACON,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .doit = nl80211_set_beacon,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
@@ -11026,7 +11067,7 @@ static const struct genl_ops nl80211_ops[] = {
         {
                 .cmd = NL80211_CMD_START_AP,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .doit = nl80211_start_ap,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
@@ -11034,7 +11075,7 @@ static const struct genl_ops nl80211_ops[] = {
         {
                 .cmd = NL80211_CMD_STOP_AP,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .doit = nl80211_stop_ap,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
@@ -11051,7 +11092,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_SET_STATION,
                 .doit = nl80211_set_station,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11059,7 +11100,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_NEW_STATION,
                 .doit = nl80211_new_station,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11067,7 +11108,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_DEL_STATION,
                 .doit = nl80211_del_station,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11076,7 +11117,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .doit = nl80211_get_mpath,
                 .dumpit = nl80211_dump_mpath,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11085,7 +11126,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .doit = nl80211_get_mpp,
                 .dumpit = nl80211_dump_mpp,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11093,7 +11134,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_SET_MPATH,
                 .doit = nl80211_set_mpath,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11101,7 +11142,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_NEW_MPATH,
                 .doit = nl80211_new_mpath,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11109,7 +11150,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_DEL_MPATH,
                 .doit = nl80211_del_mpath,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11117,7 +11158,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_SET_BSS,
                 .doit = nl80211_set_bss,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11156,7 +11197,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_SET_MESH_CONFIG,
                 .doit = nl80211_update_mesh_config,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11164,7 +11205,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_TRIGGER_SCAN,
                 .doit = nl80211_trigger_scan,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_WDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11172,7 +11213,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_ABORT_SCAN,
                 .doit = nl80211_abort_scan,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_WDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11185,7 +11226,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_START_SCHED_SCAN,
                 .doit = nl80211_start_sched_scan,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11193,7 +11234,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_STOP_SCHED_SCAN,
                 .doit = nl80211_stop_sched_scan,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11201,7 +11242,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_AUTHENTICATE,
                 .doit = nl80211_authenticate,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL |
                                   NL80211_FLAG_CLEAR_SKB,
@@ -11210,7 +11251,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_ASSOCIATE,
                 .doit = nl80211_associate,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11218,7 +11259,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_DEAUTHENTICATE,
                 .doit = nl80211_deauthenticate,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11226,7 +11267,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_DISASSOCIATE,
                 .doit = nl80211_disassociate,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11234,7 +11275,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_JOIN_IBSS,
                 .doit = nl80211_join_ibss,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11242,7 +11283,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_LEAVE_IBSS,
                 .doit = nl80211_leave_ibss,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11252,7 +11293,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .doit = nl80211_testmode_do,
                 .dumpit = nl80211_testmode_dump,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_WIPHY |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11261,7 +11302,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_CONNECT,
                 .doit = nl80211_connect,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11269,7 +11310,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_DISCONNECT,
                 .doit = nl80211_disconnect,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11277,7 +11318,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_SET_WIPHY_NETNS,
                 .doit = nl80211_wiphy_netns,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_WIPHY |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11290,7 +11331,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_SET_PMKSA,
                 .doit = nl80211_setdel_pmksa,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11298,7 +11339,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_DEL_PMKSA,
                 .doit = nl80211_setdel_pmksa,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11306,7 +11347,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_FLUSH_PMKSA,
                 .doit = nl80211_flush_pmksa,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11314,7 +11355,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_REMAIN_ON_CHANNEL,
                 .doit = nl80211_remain_on_channel,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_WDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11322,7 +11363,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_CANCEL_REMAIN_ON_CHANNEL,
                 .doit = nl80211_cancel_remain_on_channel,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_WDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11330,7 +11371,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_SET_TX_BITRATE_MASK,
                 .doit = nl80211_set_tx_bitrate_mask,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11338,7 +11379,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_REGISTER_FRAME,
                 .doit = nl80211_register_mgmt,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_WDEV |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11346,7 +11387,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_FRAME,
                 .doit = nl80211_tx_mgmt,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_WDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11354,7 +11395,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_FRAME_WAIT_CANCEL,
                 .doit = nl80211_tx_mgmt_cancel_wait,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_WDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11362,7 +11403,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_SET_POWER_SAVE,
                 .doit = nl80211_set_power_save,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11378,7 +11419,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_SET_CQM,
                 .doit = nl80211_set_cqm,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11386,7 +11427,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_SET_CHANNEL,
                 .doit = nl80211_set_channel,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11394,7 +11435,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_SET_WDS_PEER,
                 .doit = nl80211_set_wds_peer,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11402,7 +11443,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_JOIN_MESH,
                 .doit = nl80211_join_mesh,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11410,7 +11451,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_LEAVE_MESH,
                 .doit = nl80211_leave_mesh,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11418,7 +11459,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_JOIN_OCB,
                 .doit = nl80211_join_ocb,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11426,7 +11467,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_LEAVE_OCB,
                 .doit = nl80211_leave_ocb,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11443,7 +11484,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_SET_WOWLAN,
                 .doit = nl80211_set_wowlan,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_WIPHY |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11452,7 +11493,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_SET_REKEY_OFFLOAD,
                 .doit = nl80211_set_rekey_data,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL |
                                   NL80211_FLAG_CLEAR_SKB,
@@ -11461,7 +11502,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_TDLS_MGMT,
                 .doit = nl80211_tdls_mgmt,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11469,7 +11510,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_TDLS_OPER,
                 .doit = nl80211_tdls_oper,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11477,7 +11518,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_UNEXPECTED_FRAME,
                 .doit = nl80211_register_unexpected_frame,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11485,7 +11526,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_PROBE_CLIENT,
                 .doit = nl80211_probe_client,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11493,7 +11534,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_REGISTER_BEACONS,
                 .doit = nl80211_register_beacons,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_WIPHY |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11501,7 +11542,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_SET_NOACK_MAP,
                 .doit = nl80211_set_noack_map,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11509,7 +11550,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_START_P2P_DEVICE,
                 .doit = nl80211_start_p2p_device,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_WDEV |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11517,7 +11558,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_STOP_P2P_DEVICE,
                 .doit = nl80211_stop_p2p_device,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_WDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11525,7 +11566,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_SET_MCAST_RATE,
                 .doit = nl80211_set_mcast_rate,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11533,7 +11574,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_SET_MAC_ACL,
                 .doit = nl80211_set_mac_acl,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11541,7 +11582,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_RADAR_DETECT,
                 .doit = nl80211_start_radar_detection,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11554,7 +11595,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_UPDATE_FT_IES,
                 .doit = nl80211_update_ft_ies,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11562,7 +11603,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_CRIT_PROTOCOL_START,
                 .doit = nl80211_crit_protocol_start,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_WDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11570,7 +11611,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_CRIT_PROTOCOL_STOP,
                 .doit = nl80211_crit_protocol_stop,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_WDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11585,7 +11626,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_SET_COALESCE,
                 .doit = nl80211_set_coalesce,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_WIPHY |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11593,7 +11634,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_CHANNEL_SWITCH,
                 .doit = nl80211_channel_switch,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11602,7 +11643,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .doit = nl80211_vendor_cmd,
                 .dumpit = nl80211_vendor_cmd_dump,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_WIPHY |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11610,7 +11651,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_SET_QOS_MAP,
                 .doit = nl80211_set_qos_map,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11618,7 +11659,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_ADD_TX_TS,
                 .doit = nl80211_add_tx_ts,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11626,7 +11667,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_DEL_TX_TS,
                 .doit = nl80211_del_tx_ts,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11634,7 +11675,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_TDLS_CHANNEL_SWITCH,
                 .doit = nl80211_tdls_channel_switch,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -11642,7 +11683,7 @@ static const struct genl_ops nl80211_ops[] = {
                 .cmd = NL80211_CMD_TDLS_CANCEL_CHANNEL_SWITCH,
                 .doit = nl80211_tdls_cancel_channel_switch,
                 .policy = nl80211_policy,
-                .flags = GENL_ADMIN_PERM,
+                .flags = GENL_UNS_ADMIN_PERM,
                 .internal_flags = NL80211_FLAG_NEED_NETDEV_UP |
                                   NL80211_FLAG_NEED_RTNL,
         },
@@ -12092,7 +12133,7 @@ void nl80211_send_connect_result(struct cfg80211_registered_device *rdev,
                                  struct net_device *netdev, const u8 *bssid,
                                  const u8 *req_ie, size_t req_ie_len,
                                  const u8 *resp_ie, size_t resp_ie_len,
-                                 u16 status, gfp_t gfp)
+                                 int status, gfp_t gfp)
 {
         struct sk_buff *msg;
         void *hdr;
@@ -12110,7 +12151,10 @@ void nl80211_send_connect_result(struct cfg80211_registered_device *rdev,
         if (nla_put_u32(msg, NL80211_ATTR_WIPHY, rdev->wiphy_idx) ||
             nla_put_u32(msg, NL80211_ATTR_IFINDEX, netdev->ifindex) ||
             (bssid && nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, bssid)) ||
-            nla_put_u16(msg, NL80211_ATTR_STATUS_CODE, status) ||
+            nla_put_u16(msg, NL80211_ATTR_STATUS_CODE,
+                        status < 0 ? WLAN_STATUS_UNSPECIFIED_FAILURE :
+                        status) ||
+            (status < 0 && nla_put_flag(msg, NL80211_ATTR_TIMED_OUT)) ||
             (req_ie &&
              nla_put(msg, NL80211_ATTR_REQ_IE, req_ie_len, req_ie)) ||
             (resp_ie &&
@@ -12126,7 +12170,6 @@ void nl80211_send_connect_result(struct cfg80211_registered_device *rdev,
  nla_put_failure:
         genlmsg_cancel(msg, hdr);
         nlmsg_free(msg);
-
 }
 
 void nl80211_send_roamed(struct cfg80211_registered_device *rdev,
@@ -12165,7 +12208,6 @@ void nl80211_send_roamed(struct cfg80211_registered_device *rdev,
  nla_put_failure:
         genlmsg_cancel(msg, hdr);
         nlmsg_free(msg);
-
 }
 
 void nl80211_send_disconnected(struct cfg80211_registered_device *rdev,
@@ -12203,7 +12245,6 @@ void nl80211_send_disconnected(struct cfg80211_registered_device *rdev,
  nla_put_failure:
         genlmsg_cancel(msg, hdr);
         nlmsg_free(msg);
-
 }
 
 void nl80211_send_ibss_bssid(struct cfg80211_registered_device *rdev,
@@ -13545,7 +13586,6 @@ void cfg80211_crit_proto_stopped(struct wireless_dev *wdev, gfp_t gfp)
         if (hdr)
                 genlmsg_cancel(msg, hdr);
         nlmsg_free(msg);
-
 }
 EXPORT_SYMBOL(cfg80211_crit_proto_stopped);
 
diff --git a/net/wireless/nl80211.h b/net/wireless/nl80211.h
index 84d4edf1d545..a63f402b10b7 100644
--- a/net/wireless/nl80211.h
+++ b/net/wireless/nl80211.h
@@ -55,7 +55,7 @@ void nl80211_send_connect_result(struct cfg80211_registered_device *rdev,
                                  struct net_device *netdev, const u8 *bssid,
                                  const u8 *req_ie, size_t req_ie_len,
                                  const u8 *resp_ie, size_t resp_ie_len,
-                                 u16 status, gfp_t gfp);
+                                 int status, gfp_t gfp);
 void nl80211_send_roamed(struct cfg80211_registered_device *rdev,
                          struct net_device *netdev, const u8 *bssid,
                          const u8 *req_ie, size_t req_ie_len,
diff --git a/net/wireless/sme.c b/net/wireless/sme.c
index 584fdc347221..add6824c44fd 100644
--- a/net/wireless/sme.c
+++ b/net/wireless/sme.c
@@ -244,9 +244,7 @@ void cfg80211_conn_work(struct work_struct *work)
                 if (cfg80211_conn_do_work(wdev)) {
                         __cfg80211_connect_result(
                                         wdev->netdev, bssid,
-                                        NULL, 0, NULL, 0,
-                                        WLAN_STATUS_UNSPECIFIED_FAILURE,
-                                        false, NULL);
+                                        NULL, 0, NULL, 0, -1, false, NULL);
                 }
                 wdev_unlock(wdev);
         }
@@ -648,7 +646,7 @@ static DECLARE_WORK(cfg80211_disconnect_work, disconnect_work);
 void __cfg80211_connect_result(struct net_device *dev, const u8 *bssid,
                                const u8 *req_ie, size_t req_ie_len,
                                const u8 *resp_ie, size_t resp_ie_len,
-                               u16 status, bool wextev,
+                               int status, bool wextev,
                                struct cfg80211_bss *bss)
 {
         struct wireless_dev *wdev = dev->ieee80211_ptr;
@@ -757,7 +755,7 @@ void __cfg80211_connect_result(struct net_device *dev, const u8 *bssid,
 void cfg80211_connect_bss(struct net_device *dev, const u8 *bssid,
                           struct cfg80211_bss *bss, const u8 *req_ie,
                           size_t req_ie_len, const u8 *resp_ie,
-                          size_t resp_ie_len, u16 status, gfp_t gfp)
+                          size_t resp_ie_len, int status, gfp_t gfp)
 {
         struct wireless_dev *wdev = dev->ieee80211_ptr;
         struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
diff --git a/samples/bpf/sockex2_user.c b/samples/bpf/sockex2_user.c
index 29a276d766fc..8a4085c2d117 100644
--- a/samples/bpf/sockex2_user.c
+++ b/samples/bpf/sockex2_user.c
@@ -5,6 +5,7 @@
 #include "bpf_load.h"
 #include <unistd.h>
 #include <arpa/inet.h>
+#include <sys/resource.h>
 
 struct pair {
         __u64 packets;
@@ -13,11 +14,13 @@ struct pair {
 
 int main(int ac, char **argv)
 {
+        struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY};
         char filename[256];
         FILE *f;
         int i, sock;
 
         snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]);
+        setrlimit(RLIMIT_MEMLOCK, &r);
 
         if (load_bpf_file(filename)) {
                 printf("%s", bpf_log_buf);
diff --git a/samples/bpf/sockex3_user.c b/samples/bpf/sockex3_user.c
index 2617772d060d..d4184ab5f3ac 100644
--- a/samples/bpf/sockex3_user.c
+++ b/samples/bpf/sockex3_user.c
@@ -5,6 +5,7 @@
 #include "bpf_load.h"
 #include <unistd.h>
 #include <arpa/inet.h>
+#include <sys/resource.h>
 
 struct flow_keys {
         __be32 src;
@@ -23,11 +24,13 @@ struct pair {
 
 int main(int argc, char **argv)
 {
+        struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY};
         char filename[256];
         FILE *f;
         int i, sock;
 
         snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]);
+        setrlimit(RLIMIT_MEMLOCK, &r);
 
         if (load_bpf_file(filename)) {
                 printf("%s", bpf_log_buf);
diff --git a/tools/virtio/ringtest/Makefile b/tools/virtio/ringtest/Makefile
index 6173adae9f08..877a8a4721b6 100644
--- a/tools/virtio/ringtest/Makefile
+++ b/tools/virtio/ringtest/Makefile
@@ -1,6 +1,6 @@
 all:
 
-all: ring virtio_ring_0_9 virtio_ring_poll virtio_ring_inorder noring
+all: ring virtio_ring_0_9 virtio_ring_poll virtio_ring_inorder ptr_ring noring
 
 CFLAGS += -Wall
 CFLAGS += -pthread -O2 -ggdb
@@ -8,6 +8,7 @@ LDFLAGS += -pthread -O2 -ggdb
 
 main.o: main.c main.h
 ring.o: ring.c main.h
+ptr_ring.o: ptr_ring.c main.h ../../../include/linux/ptr_ring.h
 virtio_ring_0_9.o: virtio_ring_0_9.c main.h
 virtio_ring_poll.o: virtio_ring_poll.c virtio_ring_0_9.c main.h
 virtio_ring_inorder.o: virtio_ring_inorder.c virtio_ring_0_9.c main.h
@@ -15,6 +16,7 @@ ring: ring.o main.o
 virtio_ring_0_9: virtio_ring_0_9.o main.o
 virtio_ring_poll: virtio_ring_poll.o main.o
 virtio_ring_inorder: virtio_ring_inorder.o main.o
+ptr_ring: ptr_ring.o main.o
 noring: noring.o main.o
 clean:
         -rm main.o
@@ -22,6 +24,7 @@ clean:
         -rm virtio_ring_0_9.o virtio_ring_0_9
         -rm virtio_ring_poll.o virtio_ring_poll
         -rm virtio_ring_inorder.o virtio_ring_inorder
+        -rm ptr_ring.o ptr_ring
         -rm noring.o noring
 
 .PHONY: all clean
diff --git a/tools/virtio/ringtest/ptr_ring.c b/tools/virtio/ringtest/ptr_ring.c
new file mode 100644
index 000000000000..74abd746ae91
--- /dev/null
+++ b/tools/virtio/ringtest/ptr_ring.c
@@ -0,0 +1,192 @@
+#define _GNU_SOURCE
+#include "main.h"
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <pthread.h>
+#include <malloc.h>
+#include <assert.h>
+#include <errno.h>
+#include <limits.h>
+
+#define SMP_CACHE_BYTES 64
+#define cache_line_size() SMP_CACHE_BYTES
+#define ____cacheline_aligned_in_smp __attribute__ ((aligned (SMP_CACHE_BYTES)))
+#define unlikely(x)    (__builtin_expect(!!(x), 0))
+#define ALIGN(x, a) (((x) + (a) - 1) / (a) * (a))
+typedef pthread_spinlock_t  spinlock_t;
+
+typedef int gfp_t;
+static void *kzalloc(unsigned size, gfp_t gfp)
+{
+        void *p = memalign(64, size);
+        if (!p)
+                return p;
+        memset(p, 0, size);
+
+        return p;
+}
+
+static void kfree(void *p)
+{
+        if (p)
+                free(p);
+}
+
+static void spin_lock_init(spinlock_t *lock)
+{
+        int r = pthread_spin_init(lock, 0);
+        assert(!r);
+}
+
+static void spin_lock(spinlock_t *lock)
+{
+        int ret = pthread_spin_lock(lock);
+        assert(!ret);
+}
+
+static void spin_unlock(spinlock_t *lock)
+{
+        int ret = pthread_spin_unlock(lock);
+        assert(!ret);
+}
+
+static void spin_lock_bh(spinlock_t *lock)
+{
+        spin_lock(lock);
+}
+
+static void spin_unlock_bh(spinlock_t *lock)
+{
+        spin_unlock(lock);
+}
+
+static void spin_lock_irq(spinlock_t *lock)
+{
+        spin_lock(lock);
+}
+
+static void spin_unlock_irq(spinlock_t *lock)
+{
+        spin_unlock(lock);
+}
+
+static void spin_lock_irqsave(spinlock_t *lock, unsigned long f)
+{
+        spin_lock(lock);
+}
+
+static void spin_unlock_irqrestore(spinlock_t *lock, unsigned long f)
+{
+        spin_unlock(lock);
+}
+
+#include "../../../include/linux/ptr_ring.h"
+
+static unsigned long long headcnt, tailcnt;
+static struct ptr_ring array ____cacheline_aligned_in_smp;
+
+/* implemented by ring */
+void alloc_ring(void)
+{
+        int ret = ptr_ring_init(&array, ring_size, 0);
+        assert(!ret);
+}
+
+/* guest side */
+int add_inbuf(unsigned len, void *buf, void *datap)
+{
+        int ret;
+
+        ret = __ptr_ring_produce(&array, buf);
+        if (ret >= 0) {
+                ret = 0;
+                headcnt++;
+        }
+
+        return ret;
+}
+
+/*
+ * ptr_ring API provides no way for producer to find out whether a given
+ * buffer was consumed.  Our tests merely require that a successful get_buf
+ * implies that add_inbuf succeed in the past, and that add_inbuf will succeed,
+ * fake it accordingly.
+ */
+void *get_buf(unsigned *lenp, void **bufp)
+{
+        void *datap;
+
+        if (tailcnt == headcnt || __ptr_ring_full(&array))
+                datap = NULL;
+        else {
+                datap = "Buffer\n";
+                ++tailcnt;
+        }
+
+        return datap;
+}
+
+void poll_used(void)
+{
+        void *b;
+
+        do {
+                if (tailcnt == headcnt || __ptr_ring_full(&array)) {
+                        b = NULL;
+                        barrier();
+                } else {
+                        b = "Buffer\n";
+                }
+        } while (!b);
+}
+
+void disable_call()
+{
+        assert(0);
+}
+
+bool enable_call()
+{
+        assert(0);
+}
+
+void kick_available(void)
+{
+        assert(0);
+}
+
+/* host side */
+void disable_kick()
+{
+        assert(0);
+}
+
+bool enable_kick()
+{
+        assert(0);
+}
+
+void poll_avail(void)
+{
+        void *b;
+
+        do {
+                barrier();
+                b = __ptr_ring_peek(&array);
+        } while (!b);
+}
+
+bool use_buf(unsigned *lenp, void **bufp)
+{
+        void *ptr;
+
+        ptr = __ptr_ring_consume(&array);
+
+        return ptr;
+}
+
+void call_used(void)
+{
+        assert(0);
+}