IB/ipath: Header file changes to support IBA7220

This is part of a patch series to add support for a new HCA.  This patch
adds new fields to the header files.

Signed-off-by: Ralph Campbell <ralph.campbell@qlogic.com>
Signed-off-by: Roland Dreier <rolandd@cisco.com>

4 files changed, 282 insertions, 54 deletions

diff --git a/drivers/infiniband/hw/ipath/ipath_common.h b/drivers/infiniband/hw/ipath/ipath_common.h
index 3c05d4be34fd..02fd310d1ef4 100644
--- a/drivers/infiniband/hw/ipath/ipath_common.h
+++ b/drivers/infiniband/hw/ipath/ipath_common.h
@@ -201,6 +201,7 @@ typedef enum _ipath_ureg {
 #define IPATH_RUNTIME_RCVHDR_COPY       0x8
 #define IPATH_RUNTIME_MASTER    0x10
 #define IPATH_RUNTIME_NODMA_RTAIL 0x80
+#define IPATH_RUNTIME_SDMA            0x200
 #define IPATH_RUNTIME_FORCE_PIOAVAIL 0x400
 #define IPATH_RUNTIME_PIO_REGSWAPPED 0x800
 
@@ -539,7 +540,7 @@ struct ipath_diag_pkt {
 
 /* The second diag_pkt struct is the expanded version that allows
  * more control over the packet, specifically, by allowing a custom
- * pbc (+ extra) qword, so that special modes and deliberate
+ * pbc (+ static rate) qword, so that special modes and deliberate
  * changes to CRCs can be used. The elements were also re-ordered
  * for better alignment and to avoid padding issues.
  */
diff --git a/drivers/infiniband/hw/ipath/ipath_kernel.h b/drivers/infiniband/hw/ipath/ipath_kernel.h
index 890599621098..feb4f9dbac77 100644
--- a/drivers/infiniband/hw/ipath/ipath_kernel.h
+++ b/drivers/infiniband/hw/ipath/ipath_kernel.h
@@ -42,6 +42,8 @@
 #include <linux/pci.h>
 #include <linux/dma-mapping.h>
 #include <linux/mutex.h>
+#include <linux/list.h>
+#include <linux/scatterlist.h>
 #include <asm/io.h>
 #include <rdma/ib_verbs.h>
 
@@ -180,6 +182,8 @@ struct ipath_portdata {
 };
 
 struct sk_buff;
+struct ipath_sge_state;
+struct ipath_verbs_txreq;
 
 /*
  * control information for layered drivers
@@ -193,6 +197,37 @@ struct ipath_skbinfo {
         dma_addr_t phys;
 };
 
+struct ipath_sdma_txreq {
+        int                 flags;
+        int                 sg_count;
+        union {
+                struct scatterlist *sg;
+                void *map_addr;
+        };
+        void              (*callback)(void *, int);
+        void               *callback_cookie;
+        int                 callback_status;
+        u16                 start_idx;  /* sdma private */
+        u16                 next_descq_idx;  /* sdma private */
+        struct list_head    list;       /* sdma private */
+};
+
+struct ipath_sdma_desc {
+        __le64 qw[2];
+};
+
+#define IPATH_SDMA_TXREQ_F_USELARGEBUF  0x1
+#define IPATH_SDMA_TXREQ_F_HEADTOHOST   0x2
+#define IPATH_SDMA_TXREQ_F_INTREQ       0x4
+#define IPATH_SDMA_TXREQ_F_FREEBUF      0x8
+#define IPATH_SDMA_TXREQ_F_FREEDESC     0x10
+#define IPATH_SDMA_TXREQ_F_VL15         0x20
+
+#define IPATH_SDMA_TXREQ_S_OK        0
+#define IPATH_SDMA_TXREQ_S_SENDERROR 1
+#define IPATH_SDMA_TXREQ_S_ABORTED   2
+#define IPATH_SDMA_TXREQ_S_SHUTDOWN  3
+
 /* max dwords in small buffer packet */
 #define IPATH_SMALLBUF_DWORDS (dd->ipath_piosize2k >> 2)
 
@@ -385,6 +420,15 @@ struct ipath_devdata {
         u32 ipath_pcibar0;
         /* so we can rewrite it after a chip reset */
         u32 ipath_pcibar1;
+        u32 ipath_x1_fix_tries;
+        u32 ipath_autoneg_tries;
+        u32 serdes_first_init_done;
+
+        struct ipath_relock {
+                atomic_t ipath_relock_timer_active;
+                struct timer_list ipath_relock_timer;
+                unsigned int ipath_relock_interval; /* in jiffies */
+        } ipath_relock_singleton;
 
         /* interrupt number */
         int ipath_irq;
@@ -431,8 +475,38 @@ struct ipath_devdata {
         void *ipath_dummy_hdrq; /* used after port close */
         dma_addr_t ipath_dummy_hdrq_phys;
 
+        /* SendDMA related entries */
+        spinlock_t            ipath_sdma_lock;
+        u64                   ipath_sdma_status;
+        unsigned long         ipath_sdma_abort_jiffies;
+        unsigned long         ipath_sdma_abort_intr_timeout;
+        unsigned long         ipath_sdma_buf_jiffies;
+        struct ipath_sdma_desc *ipath_sdma_descq;
+        u64                   ipath_sdma_descq_added;
+        u64                   ipath_sdma_descq_removed;
+        int                   ipath_sdma_desc_nreserved;
+        u16                   ipath_sdma_descq_cnt;
+        u16                   ipath_sdma_descq_tail;
+        u16                   ipath_sdma_descq_head;
+        u16                   ipath_sdma_next_intr;
+        u16                   ipath_sdma_reset_wait;
+        u8                    ipath_sdma_generation;
+        struct tasklet_struct ipath_sdma_abort_task;
+        struct tasklet_struct ipath_sdma_notify_task;
+        struct list_head      ipath_sdma_activelist;
+        struct list_head      ipath_sdma_notifylist;
+        atomic_t              ipath_sdma_vl15_count;
+        struct timer_list     ipath_sdma_vl15_timer;
+
+        dma_addr_t       ipath_sdma_descq_phys;
+        volatile __le64 *ipath_sdma_head_dma;
+        dma_addr_t       ipath_sdma_head_phys;
+
         unsigned long ipath_ureg_align; /* user register alignment */
 
+        struct delayed_work ipath_autoneg_work;
+        wait_queue_head_t ipath_autoneg_wait;
+
         /* HoL blocking / user app forward-progress state */
         unsigned          ipath_hol_state;
         unsigned          ipath_hol_next;
@@ -485,6 +559,8 @@ struct ipath_devdata {
         u64 ipath_intconfig;
         /* kr_sendpiobufbase value */
         u64 ipath_piobufbase;
+        /* kr_ibcddrctrl shadow */
+        u64 ipath_ibcddrctrl;
 
         /* these are the "32 bit" regs */
 
@@ -501,7 +577,10 @@ struct ipath_devdata {
         unsigned long ipath_rcvctrl;
         /* shadow kr_sendctrl */
         unsigned long ipath_sendctrl;
-        unsigned long ipath_lastcancel; /* to not count armlaunch after cancel */
+        /* to not count armlaunch after cancel */
+        unsigned long ipath_lastcancel;
+        /* count cases where special trigger was needed (double write) */
+        unsigned long ipath_spectriggerhit;
 
         /* value we put in kr_rcvhdrcnt */
         u32 ipath_rcvhdrcnt;
@@ -523,6 +602,7 @@ struct ipath_devdata {
         u32 ipath_piobcnt4k;
         /* size in bytes of "4KB" PIO buffers */
         u32 ipath_piosize4k;
+        u32 ipath_pioreserved; /* reserved special-inkernel; */
         /* kr_rcvegrbase value */
         u32 ipath_rcvegrbase;
         /* kr_rcvegrcnt value */
@@ -586,7 +666,7 @@ struct ipath_devdata {
          */
         u8 ipath_serial[16];
         /* human readable board version */
-        u8 ipath_boardversion[80];
+        u8 ipath_boardversion[96];
         u8 ipath_lbus_info[32]; /* human readable localbus info */
         /* chip major rev, from ipath_revision */
         u8 ipath_majrev;
@@ -715,6 +795,13 @@ struct ipath_devdata {
         /* interrupt mitigation reload register info */
         u16 ipath_jint_idle_ticks;      /* idle clock ticks */
         u16 ipath_jint_max_packets;     /* max packets across all ports */
+
+        /*
+         * lock for access to SerDes, and flags to sequence preset
+         * versus steady-state. 7220-only at the moment.
+         */
+        spinlock_t ipath_sdepb_lock;
+        u8 ipath_presets_needed; /* Set if presets to be restored next DOWN */
 };
 
 /* ipath_hol_state values (stopping/starting user proc, send flushing) */
@@ -724,11 +811,35 @@ struct ipath_devdata {
 #define IPATH_HOL_DOWNSTOP 0
 #define IPATH_HOL_DOWNCONT 1
 
+/* bit positions for sdma_status */
+#define IPATH_SDMA_ABORTING  0
+#define IPATH_SDMA_DISARMED  1
+#define IPATH_SDMA_DISABLED  2
+#define IPATH_SDMA_LAYERBUF  3
+#define IPATH_SDMA_RUNNING  62
+#define IPATH_SDMA_SHUTDOWN 63
+
+/* bit combinations that correspond to abort states */
+#define IPATH_SDMA_ABORT_NONE 0
+#define IPATH_SDMA_ABORT_ABORTING (1UL << IPATH_SDMA_ABORTING)
+#define IPATH_SDMA_ABORT_DISARMED ((1UL << IPATH_SDMA_ABORTING) | \
+        (1UL << IPATH_SDMA_DISARMED))
+#define IPATH_SDMA_ABORT_DISABLED ((1UL << IPATH_SDMA_ABORTING) | \
+        (1UL << IPATH_SDMA_DISABLED))
+#define IPATH_SDMA_ABORT_ABORTED ((1UL << IPATH_SDMA_ABORTING) | \
+        (1UL << IPATH_SDMA_DISARMED) | (1UL << IPATH_SDMA_DISABLED))
+#define IPATH_SDMA_ABORT_MASK ((1UL<<IPATH_SDMA_ABORTING) | \
+        (1UL << IPATH_SDMA_DISARMED) | (1UL << IPATH_SDMA_DISABLED))
+
+#define IPATH_SDMA_BUF_NONE 0
+#define IPATH_SDMA_BUF_MASK (1UL<<IPATH_SDMA_LAYERBUF)
+
 /* Private data for file operations */
 struct ipath_filedata {
         struct ipath_portdata *pd;
         unsigned subport;
         unsigned tidcursor;
+        struct ipath_user_sdma_queue *pq;
 };
 extern struct list_head ipath_dev_list;
 extern spinlock_t ipath_devs_lock;
@@ -796,6 +907,10 @@ void ipath_disable_armlaunch(struct ipath_devdata *);
 void ipath_hol_down(struct ipath_devdata *);
 void ipath_hol_up(struct ipath_devdata *);
 void ipath_hol_event(unsigned long);
+void ipath_toggle_rclkrls(struct ipath_devdata *);
+void ipath_sd7220_clr_ibpar(struct ipath_devdata *);
+void ipath_set_relock_poll(struct ipath_devdata *, int);
+void ipath_shutdown_relock_poll(struct ipath_devdata *);
 
 /* for use in system calls, where we want to know device type, etc. */
 #define port_fp(fp) ((struct ipath_filedata *)(fp)->private_data)->pd
@@ -803,6 +918,8 @@ void ipath_hol_event(unsigned long);
         ((struct ipath_filedata *)(fp)->private_data)->subport
 #define tidcursor_fp(fp) \
         ((struct ipath_filedata *)(fp)->private_data)->tidcursor
+#define user_sdma_queue_fp(fp) \
+        ((struct ipath_filedata *)(fp)->private_data)->pq
 
 /*
  * values for ipath_flags
@@ -852,9 +969,16 @@ void ipath_hol_event(unsigned long);
                 /* Use GPIO interrupts for new counters */
 #define IPATH_GPIO_ERRINTRS 0x100000
 #define IPATH_SWAP_PIOBUFS  0x200000
+                /* Supports Send DMA */
+#define IPATH_HAS_SEND_DMA  0x400000
+                /* Supports Send Count (not just word count) in PBC */
+#define IPATH_HAS_PBC_CNT   0x800000
                 /* Suppress heartbeat, even if turning off loopback */
 #define IPATH_NO_HRTBT      0x1000000
+#define IPATH_HAS_THRESH_UPDATE 0x4000000
 #define IPATH_HAS_MULT_IB_SPEED 0x8000000
+#define IPATH_IB_AUTONEG_INPROG 0x10000000
+#define IPATH_IB_AUTONEG_FAILED 0x20000000
                 /* Linkdown-disable intentionally, Do not attempt to bring up */
 #define IPATH_IB_LINK_DISABLED 0x40000000
 #define IPATH_IB_FORCE_NOTIFY 0x80000000 /* force notify on next ib change */
@@ -879,6 +1003,7 @@ void ipath_free_data(struct ipath_portdata *dd);
 u32 __iomem *ipath_getpiobuf(struct ipath_devdata *, u32, u32 *);
 void ipath_chg_pioavailkernel(struct ipath_devdata *dd, unsigned start,
                                 unsigned len, int avail);
+void ipath_init_iba7220_funcs(struct ipath_devdata *);
 void ipath_init_iba6120_funcs(struct ipath_devdata *);
 void ipath_init_iba6110_funcs(struct ipath_devdata *);
 void ipath_get_eeprom_info(struct ipath_devdata *);
@@ -897,6 +1022,33 @@ void signal_ib_event(struct ipath_devdata *dd, enum ib_event_type ev);
 #define IPATH_LED_LOG 2  /* Logical (link) YELLOW LED */
 void ipath_set_led_override(struct ipath_devdata *dd, unsigned int val);
 
+/* send dma routines */
+int setup_sdma(struct ipath_devdata *);
+void teardown_sdma(struct ipath_devdata *);
+void ipath_sdma_intr(struct ipath_devdata *);
+int ipath_sdma_verbs_send(struct ipath_devdata *, struct ipath_sge_state *,
+                          u32, struct ipath_verbs_txreq *);
+/* ipath_sdma_lock should be locked before calling this. */
+int ipath_sdma_make_progress(struct ipath_devdata *dd);
+
+/* must be called under ipath_sdma_lock */
+static inline u16 ipath_sdma_descq_freecnt(const struct ipath_devdata *dd)
+{
+        return dd->ipath_sdma_descq_cnt -
+                (dd->ipath_sdma_descq_added - dd->ipath_sdma_descq_removed) -
+                1 - dd->ipath_sdma_desc_nreserved;
+}
+
+static inline void ipath_sdma_desc_reserve(struct ipath_devdata *dd, u16 cnt)
+{
+        dd->ipath_sdma_desc_nreserved += cnt;
+}
+
+static inline void ipath_sdma_desc_unreserve(struct ipath_devdata *dd, u16 cnt)
+{
+        dd->ipath_sdma_desc_nreserved -= cnt;
+}
+
 /*
  * number of words used for protocol header if not set by ipath_userinit();
  */
@@ -925,8 +1077,7 @@ void ipath_write_kreg_port(const struct ipath_devdata *, ipath_kreg,
 
 /*
  * At the moment, none of the s-registers are writable, so no
- * ipath_write_sreg(), and none of the c-registers are writable, so no
- * ipath_write_creg().
+ * ipath_write_sreg().
  */
 
 /**
@@ -1123,6 +1274,7 @@ int ipathfs_remove_device(struct ipath_devdata *);
 dma_addr_t ipath_map_page(struct pci_dev *, struct page *, unsigned long,
                           size_t, int);
 dma_addr_t ipath_map_single(struct pci_dev *, void *, size_t, int);
+const char *ipath_get_unit_name(int unit);
 
 /*
  * Flush write combining store buffers (if present) and perform a write
@@ -1137,11 +1289,6 @@ dma_addr_t ipath_map_single(struct pci_dev *, void *, size_t, int);
 extern unsigned ipath_debug; /* debugging bit mask */
 extern unsigned ipath_linkrecovery;
 extern unsigned ipath_mtu4096;
-
-#define IPATH_MAX_PARITY_ATTEMPTS 10000 /* max times to try recovery */
-
-const char *ipath_get_unit_name(int unit);
-
 extern struct mutex ipath_mutex;
 
 #define IPATH_DRV_NAME          "ib_ipath"
diff --git a/drivers/infiniband/hw/ipath/ipath_registers.h b/drivers/infiniband/hw/ipath/ipath_registers.h
index b7d87d3252e2..8f44d0cf3833 100644
--- a/drivers/infiniband/hw/ipath/ipath_registers.h
+++ b/drivers/infiniband/hw/ipath/ipath_registers.h
@@ -73,56 +73,82 @@
 #define IPATH_S_PIOINTBUFAVAIL  1
 #define IPATH_S_PIOBUFAVAILUPD  2
 #define IPATH_S_PIOENABLE       3
+#define IPATH_S_SDMAINTENABLE   9
+#define IPATH_S_SDMASINGLEDESCRIPTOR    10
+#define IPATH_S_SDMAENABLE      11
+#define IPATH_S_SDMAHALT        12
 #define IPATH_S_DISARM          31
 
 #define INFINIPATH_S_ABORT              (1U << IPATH_S_ABORT)
 #define INFINIPATH_S_PIOINTBUFAVAIL     (1U << IPATH_S_PIOINTBUFAVAIL)
 #define INFINIPATH_S_PIOBUFAVAILUPD     (1U << IPATH_S_PIOBUFAVAILUPD)
 #define INFINIPATH_S_PIOENABLE          (1U << IPATH_S_PIOENABLE)
+#define INFINIPATH_S_SDMAINTENABLE      (1U << IPATH_S_SDMAINTENABLE)
+#define INFINIPATH_S_SDMASINGLEDESCRIPTOR \
+                                        (1U << IPATH_S_SDMASINGLEDESCRIPTOR)
+#define INFINIPATH_S_SDMAENABLE         (1U << IPATH_S_SDMAENABLE)
+#define INFINIPATH_S_SDMAHALT           (1U << IPATH_S_SDMAHALT)
 #define INFINIPATH_S_DISARM             (1U << IPATH_S_DISARM)
 
-/* kr_rcvctrl bits */
+/* kr_rcvctrl bits that are the same on multiple chips */
 #define INFINIPATH_R_PORTENABLE_SHIFT 0
 #define INFINIPATH_R_QPMAP_ENABLE (1ULL << 38)
 
 /* kr_intstatus, kr_intclear, kr_intmask bits */
-#define INFINIPATH_I_ERROR        0x80000000
-#define INFINIPATH_I_SPIOSENT     0x40000000
-#define INFINIPATH_I_SPIOBUFAVAIL 0x20000000
-#define INFINIPATH_I_GPIO         0x10000000
+#define INFINIPATH_I_SDMAINT            0x8000000000000000ULL
+#define INFINIPATH_I_SDMADISABLED       0x4000000000000000ULL
+#define INFINIPATH_I_ERROR              0x0000000080000000ULL
+#define INFINIPATH_I_SPIOSENT           0x0000000040000000ULL
+#define INFINIPATH_I_SPIOBUFAVAIL       0x0000000020000000ULL
+#define INFINIPATH_I_GPIO               0x0000000010000000ULL
+#define INFINIPATH_I_JINT               0x0000000004000000ULL
 
 /* kr_errorstatus, kr_errorclear, kr_errormask bits */
-#define INFINIPATH_E_RFORMATERR      0x0000000000000001ULL
-#define INFINIPATH_E_RVCRC           0x0000000000000002ULL
-#define INFINIPATH_E_RICRC           0x0000000000000004ULL
-#define INFINIPATH_E_RMINPKTLEN      0x0000000000000008ULL
-#define INFINIPATH_E_RMAXPKTLEN      0x0000000000000010ULL
-#define INFINIPATH_E_RLONGPKTLEN     0x0000000000000020ULL
-#define INFINIPATH_E_RSHORTPKTLEN    0x0000000000000040ULL
-#define INFINIPATH_E_RUNEXPCHAR      0x0000000000000080ULL
-#define INFINIPATH_E_RUNSUPVL        0x0000000000000100ULL
-#define INFINIPATH_E_REBP            0x0000000000000200ULL
-#define INFINIPATH_E_RIBFLOW         0x0000000000000400ULL
-#define INFINIPATH_E_RBADVERSION     0x0000000000000800ULL
-#define INFINIPATH_E_RRCVEGRFULL     0x0000000000001000ULL
-#define INFINIPATH_E_RRCVHDRFULL     0x0000000000002000ULL
-#define INFINIPATH_E_RBADTID         0x0000000000004000ULL
-#define INFINIPATH_E_RHDRLEN         0x0000000000008000ULL
-#define INFINIPATH_E_RHDR            0x0000000000010000ULL
-#define INFINIPATH_E_RIBLOSTLINK     0x0000000000020000ULL
-#define INFINIPATH_E_SMINPKTLEN      0x0000000020000000ULL
-#define INFINIPATH_E_SMAXPKTLEN      0x0000000040000000ULL
-#define INFINIPATH_E_SUNDERRUN       0x0000000080000000ULL
-#define INFINIPATH_E_SPKTLEN         0x0000000100000000ULL
-#define INFINIPATH_E_SDROPPEDSMPPKT  0x0000000200000000ULL
-#define INFINIPATH_E_SDROPPEDDATAPKT 0x0000000400000000ULL
-#define INFINIPATH_E_SPIOARMLAUNCH   0x0000000800000000ULL
-#define INFINIPATH_E_SUNEXPERRPKTNUM 0x0000001000000000ULL
-#define INFINIPATH_E_SUNSUPVL        0x0000002000000000ULL
-#define INFINIPATH_E_IBSTATUSCHANGED 0x0001000000000000ULL
-#define INFINIPATH_E_INVALIDADDR     0x0002000000000000ULL
-#define INFINIPATH_E_RESET           0x0004000000000000ULL
-#define INFINIPATH_E_HARDWARE        0x0008000000000000ULL
+#define INFINIPATH_E_RFORMATERR                 0x0000000000000001ULL
+#define INFINIPATH_E_RVCRC                      0x0000000000000002ULL
+#define INFINIPATH_E_RICRC                      0x0000000000000004ULL
+#define INFINIPATH_E_RMINPKTLEN                 0x0000000000000008ULL
+#define INFINIPATH_E_RMAXPKTLEN                 0x0000000000000010ULL
+#define INFINIPATH_E_RLONGPKTLEN                0x0000000000000020ULL
+#define INFINIPATH_E_RSHORTPKTLEN               0x0000000000000040ULL
+#define INFINIPATH_E_RUNEXPCHAR                 0x0000000000000080ULL
+#define INFINIPATH_E_RUNSUPVL                   0x0000000000000100ULL
+#define INFINIPATH_E_REBP                       0x0000000000000200ULL
+#define INFINIPATH_E_RIBFLOW                    0x0000000000000400ULL
+#define INFINIPATH_E_RBADVERSION                0x0000000000000800ULL
+#define INFINIPATH_E_RRCVEGRFULL                0x0000000000001000ULL
+#define INFINIPATH_E_RRCVHDRFULL                0x0000000000002000ULL
+#define INFINIPATH_E_RBADTID                    0x0000000000004000ULL
+#define INFINIPATH_E_RHDRLEN                    0x0000000000008000ULL
+#define INFINIPATH_E_RHDR                       0x0000000000010000ULL
+#define INFINIPATH_E_RIBLOSTLINK                0x0000000000020000ULL
+#define INFINIPATH_E_SENDSPECIALTRIGGER         0x0000000008000000ULL
+#define INFINIPATH_E_SDMADISABLED               0x0000000010000000ULL
+#define INFINIPATH_E_SMINPKTLEN                 0x0000000020000000ULL
+#define INFINIPATH_E_SMAXPKTLEN                 0x0000000040000000ULL
+#define INFINIPATH_E_SUNDERRUN                  0x0000000080000000ULL
+#define INFINIPATH_E_SPKTLEN                    0x0000000100000000ULL
+#define INFINIPATH_E_SDROPPEDSMPPKT             0x0000000200000000ULL
+#define INFINIPATH_E_SDROPPEDDATAPKT            0x0000000400000000ULL
+#define INFINIPATH_E_SPIOARMLAUNCH              0x0000000800000000ULL
+#define INFINIPATH_E_SUNEXPERRPKTNUM            0x0000001000000000ULL
+#define INFINIPATH_E_SUNSUPVL                   0x0000002000000000ULL
+#define INFINIPATH_E_SENDBUFMISUSE              0x0000004000000000ULL
+#define INFINIPATH_E_SDMAGENMISMATCH            0x0000008000000000ULL
+#define INFINIPATH_E_SDMAOUTOFBOUND             0x0000010000000000ULL
+#define INFINIPATH_E_SDMATAILOUTOFBOUND         0x0000020000000000ULL
+#define INFINIPATH_E_SDMABASE                   0x0000040000000000ULL
+#define INFINIPATH_E_SDMA1STDESC                0x0000080000000000ULL
+#define INFINIPATH_E_SDMARPYTAG                 0x0000100000000000ULL
+#define INFINIPATH_E_SDMADWEN                   0x0000200000000000ULL
+#define INFINIPATH_E_SDMAMISSINGDW              0x0000400000000000ULL
+#define INFINIPATH_E_SDMAUNEXPDATA              0x0000800000000000ULL
+#define INFINIPATH_E_IBSTATUSCHANGED            0x0001000000000000ULL
+#define INFINIPATH_E_INVALIDADDR                0x0002000000000000ULL
+#define INFINIPATH_E_RESET                      0x0004000000000000ULL
+#define INFINIPATH_E_HARDWARE                   0x0008000000000000ULL
+#define INFINIPATH_E_SDMADESCADDRMISALIGN       0x0010000000000000ULL
+#define INFINIPATH_E_INVALIDEEPCMD              0x0020000000000000ULL
 
 /*
  * this is used to print "common" packet errors only when the
@@ -133,6 +159,17 @@
                 | INFINIPATH_E_RICRC | INFINIPATH_E_RSHORTPKTLEN \
                 | INFINIPATH_E_REBP )
 
+/* Convenience for decoding Send DMA errors */
+#define INFINIPATH_E_SDMAERRS ( \
+        INFINIPATH_E_SDMAGENMISMATCH | INFINIPATH_E_SDMAOUTOFBOUND | \
+        INFINIPATH_E_SDMATAILOUTOFBOUND | INFINIPATH_E_SDMABASE | \
+        INFINIPATH_E_SDMA1STDESC | INFINIPATH_E_SDMARPYTAG | \
+        INFINIPATH_E_SDMADWEN | INFINIPATH_E_SDMAMISSINGDW | \
+        INFINIPATH_E_SDMAUNEXPDATA | \
+        INFINIPATH_E_SDMADESCADDRMISALIGN | \
+        INFINIPATH_E_SDMADISABLED | \
+        INFINIPATH_E_SENDBUFMISUSE)
+
 /* kr_hwerrclear, kr_hwerrmask, kr_hwerrstatus, bits */
 /* TXEMEMPARITYERR bit 0: PIObuf, 1: PIOpbc, 2: launchfifo
  * RXEMEMPARITYERR bit 0: rcvbuf, 1: lookupq, 2:  expTID, 3: eagerTID
@@ -157,7 +194,7 @@
 #define INFINIPATH_HWE_RXEMEMPARITYERR_HDRINFO  0x40ULL
 /* waldo specific -- find the rest in ipath_6110.c */
 #define INFINIPATH_HWE_RXDSYNCMEMPARITYERR  0x0000000400000000ULL
-/* monty specific -- find the rest in ipath_6120.c */
+/* 6120/7220 specific -- find the rest in ipath_6120.c and ipath_7220.c */
 #define INFINIPATH_HWE_MEMBISTFAILED    0x0040000000000000ULL
 
 /* kr_hwdiagctrl bits */
@@ -202,7 +239,7 @@
 /* kr_ibcstatus bits */
 #define INFINIPATH_IBCS_LINKTRAININGSTATE_SHIFT 0
 #define INFINIPATH_IBCS_LINKSTATE_MASK 0x7
-#define INFINIPATH_IBCS_LINKSTATE_SHIFT 4
+
 #define INFINIPATH_IBCS_TXREADY       0x40000000
 #define INFINIPATH_IBCS_TXCREDITOK    0x80000000
 /* link training states (shift by
@@ -267,7 +304,7 @@
 /* L1 Power down; use with RXDETECT, Otherwise not used on IB side */
 #define INFINIPATH_SERDC0_L1PWR_DN       0xF0ULL
 
-/* kr_xgxsconfig bits */
+/* common kr_xgxsconfig bits (or safe in all, even if not implemented) */
 #define INFINIPATH_XGXS_RX_POL_SHIFT 19
 #define INFINIPATH_XGXS_RX_POL_MASK 0xfULL
 
@@ -397,6 +434,29 @@ struct ipath_kregs {
         ipath_kreg kr_pcieq1serdesconfig0;
         ipath_kreg kr_pcieq1serdesconfig1;
         ipath_kreg kr_pcieq1serdesstatus;
+        ipath_kreg kr_hrtbt_guid;
+        ipath_kreg kr_ibcddrctrl;
+        ipath_kreg kr_ibcddrstatus;
+        ipath_kreg kr_jintreload;
+
+        /* send dma related regs */
+        ipath_kreg kr_senddmabase;
+        ipath_kreg kr_senddmalengen;
+        ipath_kreg kr_senddmatail;
+        ipath_kreg kr_senddmahead;
+        ipath_kreg kr_senddmaheadaddr;
+        ipath_kreg kr_senddmabufmask0;
+        ipath_kreg kr_senddmabufmask1;
+        ipath_kreg kr_senddmabufmask2;
+        ipath_kreg kr_senddmastatus;
+
+        /* SerDes related regs (IBA7220-only) */
+        ipath_kreg kr_ibserdesctrl;
+        ipath_kreg kr_ib_epbacc;
+        ipath_kreg kr_ib_epbtrans;
+        ipath_kreg kr_pcie_epbacc;
+        ipath_kreg kr_pcie_epbtrans;
+        ipath_kreg kr_ib_ddsrxeq;
 };
 
 struct ipath_cregs {
diff --git a/drivers/infiniband/hw/ipath/ipath_verbs.h b/drivers/infiniband/hw/ipath/ipath_verbs.h
index 3d59736b49b2..5ed00e6d9270 100644
--- a/drivers/infiniband/hw/ipath/ipath_verbs.h
+++ b/drivers/infiniband/hw/ipath/ipath_verbs.h
@@ -138,6 +138,11 @@ struct ipath_ib_header {
         } u;
 } __attribute__ ((packed));
 
+struct ipath_pio_header {
+        __le32 pbc[2];
+        struct ipath_ib_header hdr;
+} __attribute__ ((packed));
+
 /*
  * There is one struct ipath_mcast for each multicast GID.
  * All attached QPs are then stored as a list of
@@ -319,6 +324,7 @@ struct ipath_sge_state {
         struct ipath_sge *sg_list;      /* next SGE to be used if any */
         struct ipath_sge sge;   /* progress state for the current SGE */
         u8 num_sge;
+        u8 static_rate;
 };
 
 /*
@@ -356,6 +362,7 @@ struct ipath_qp {
         struct tasklet_struct s_task;
         struct ipath_mmap_info *ip;
         struct ipath_sge_state *s_cur_sge;
+        struct ipath_verbs_txreq *s_tx;
         struct ipath_sge_state s_sge;   /* current send request data */
         struct ipath_ack_entry s_ack_queue[IPATH_MAX_RDMA_ATOMIC + 1];
         struct ipath_sge_state s_ack_rdma_sge;
@@ -363,7 +370,8 @@ struct ipath_qp {
         struct ipath_sge_state r_sge;   /* current receive data */
         spinlock_t s_lock;
         unsigned long s_busy;
-        u32 s_hdrwords;         /* size of s_hdr in 32 bit words */
+        u16 s_pkt_delay;
+        u16 s_hdrwords;         /* size of s_hdr in 32 bit words */
         u32 s_cur_size;         /* size of send packet in bytes */
         u32 s_len;              /* total length of s_sge */
         u32 s_rdma_read_len;    /* total length of s_rdma_read_sge */
@@ -387,7 +395,6 @@ struct ipath_qp {
         u8 r_nak_state;         /* non-zero if NAK is pending */
         u8 r_min_rnr_timer;     /* retry timeout value for RNR NAKs */
         u8 r_reuse_sge;         /* for UC receive errors */
-        u8 r_sge_inx;           /* current index into sg_list */
         u8 r_wrid_valid;        /* r_wrid set but CQ entry not yet made */
         u8 r_max_rd_atomic;     /* max number of RDMA read/atomic to receive */
         u8 r_head_ack_queue;    /* index into s_ack_queue[] */
@@ -403,6 +410,7 @@ struct ipath_qp {
         u8 s_num_rd_atomic;     /* number of RDMA read/atomic pending */
         u8 s_tail_ack_queue;    /* index into s_ack_queue[] */
         u8 s_flags;
+        u8 s_dmult;
         u8 timeout;             /* Timeout for this QP */
         enum ib_mtu path_mtu;
         u32 remote_qpn;
@@ -510,6 +518,8 @@ struct ipath_ibdev {
         struct ipath_lkey_table lk_table;
         struct list_head pending[3];    /* FIFO of QPs waiting for ACKs */
         struct list_head piowait;       /* list for wait PIO buf */
+        struct list_head txreq_free;
+        void *txreq_bufs;
         /* list of QPs waiting for RNR timer */
         struct list_head rnrwait;
         spinlock_t pending_lock;
@@ -570,6 +580,7 @@ struct ipath_ibdev {
         u32 n_rdma_dup_busy;
         u32 n_piowait;
         u32 n_no_piobuf;
+        u32 n_unaligned;
         u32 port_cap_flags;
         u32 pma_sample_start;
         u32 pma_sample_interval;
@@ -581,7 +592,6 @@ struct ipath_ibdev {
         u16 pending_index;      /* which pending queue is active */
         u8 pma_sample_status;
         u8 subnet_timeout;
-        u8 link_width_enabled;
         u8 vl_high_limit;
         struct ipath_opcode_stats opstats[128];
 };
@@ -602,6 +612,16 @@ struct ipath_verbs_counters {
         u32 vl15_dropped;
 };
 
+struct ipath_verbs_txreq {
+        struct ipath_qp         *qp;
+        struct ipath_swqe       *wqe;
+        u32                      map_len;
+        u32                      len;
+        struct ipath_sge_state  *ss;
+        struct ipath_pio_header  hdr;
+        struct ipath_sdma_txreq  txreq;
+};
+
 static inline struct ipath_mr *to_imr(struct ib_mr *ibmr)
 {
         return container_of(ibmr, struct ipath_mr, ibmr);
@@ -694,11 +714,11 @@ void ipath_sqerror_qp(struct ipath_qp *qp, struct ib_wc *wc);
 
 void ipath_get_credit(struct ipath_qp *qp, u32 aeth);
 
+unsigned ipath_ib_rate_to_mult(enum ib_rate rate);
+
 int ipath_verbs_send(struct ipath_qp *qp, struct ipath_ib_header *hdr,
                      u32 hdrwords, struct ipath_sge_state *ss, u32 len);
 
-void ipath_cq_enter(struct ipath_cq *cq, struct ib_wc *entry, int sig);
-
 void ipath_copy_sge(struct ipath_sge_state *ss, void *data, u32 length);
 
 void ipath_skip_sge(struct ipath_sge_state *ss, u32 length);