1 files changed, 263 insertions, 41 deletions

diff --git a/drivers/infiniband/hw/ipath/ipath_kernel.h b/drivers/infiniband/hw/ipath/ipath_kernel.h
index ecf3f7ff7717..5863cbe99303 100644
--- a/drivers/infiniband/hw/ipath/ipath_kernel.h
+++ b/drivers/infiniband/hw/ipath/ipath_kernel.h
@@ -1,7 +1,7 @@
 #ifndef _IPATH_KERNEL_H
 #define _IPATH_KERNEL_H
 /*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
  * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
  *
  * This software is available to you under a choice of one of two
@@ -42,6 +42,8 @@
 #include <linux/pci.h>
 #include <linux/dma-mapping.h>
 #include <linux/mutex.h>
+#include <linux/list.h>
+#include <linux/scatterlist.h>
 #include <asm/io.h>
 #include <rdma/ib_verbs.h>
 
@@ -175,9 +177,13 @@ struct ipath_portdata {
         u16 poll_type;
         /* port rcvhdrq head offset */
         u32 port_head;
+        /* receive packet sequence counter */
+        u32 port_seq_cnt;
 };
 
 struct sk_buff;
+struct ipath_sge_state;
+struct ipath_verbs_txreq;
 
 /*
  * control information for layered drivers
@@ -191,6 +197,40 @@ struct ipath_skbinfo {
         dma_addr_t phys;
 };
 
+struct ipath_sdma_txreq {
+        int                 flags;
+        int                 sg_count;
+        union {
+                struct scatterlist *sg;
+                void *map_addr;
+        };
+        void              (*callback)(void *, int);
+        void               *callback_cookie;
+        int                 callback_status;
+        u16                 start_idx;  /* sdma private */
+        u16                 next_descq_idx;  /* sdma private */
+        struct list_head    list;       /* sdma private */
+};
+
+struct ipath_sdma_desc {
+        __le64 qw[2];
+};
+
+#define IPATH_SDMA_TXREQ_F_USELARGEBUF  0x1
+#define IPATH_SDMA_TXREQ_F_HEADTOHOST   0x2
+#define IPATH_SDMA_TXREQ_F_INTREQ       0x4
+#define IPATH_SDMA_TXREQ_F_FREEBUF      0x8
+#define IPATH_SDMA_TXREQ_F_FREEDESC     0x10
+#define IPATH_SDMA_TXREQ_F_VL15         0x20
+
+#define IPATH_SDMA_TXREQ_S_OK        0
+#define IPATH_SDMA_TXREQ_S_SENDERROR 1
+#define IPATH_SDMA_TXREQ_S_ABORTED   2
+#define IPATH_SDMA_TXREQ_S_SHUTDOWN  3
+
+/* max dwords in small buffer packet */
+#define IPATH_SMALLBUF_DWORDS (dd->ipath_piosize2k >> 2)
+
 /*
  * Possible IB config parameters for ipath_f_get/set_ib_cfg()
  */
@@ -221,11 +261,6 @@ struct ipath_devdata {
         unsigned long ipath_physaddr;
         /* base of memory alloced for ipath_kregbase, for free */
         u64 *ipath_kregalloc;
-        /*
-         * virtual address where port0 rcvhdrqtail updated for this unit.
-         * only written to by the chip, not the driver.
-         */
-        volatile __le64 *ipath_hdrqtailptr;
         /* ipath_cfgports pointers */
         struct ipath_portdata **ipath_pd;
         /* sk_buffs used by port 0 eager receive queue */
@@ -283,6 +318,7 @@ struct ipath_devdata {
         /* per chip actions needed for IB Link up/down changes */
         int (*ipath_f_ib_updown)(struct ipath_devdata *, int, u64);
 
+        unsigned ipath_lastegr_idx;
         struct ipath_ibdev *verbs_dev;
         struct timer_list verbs_timer;
         /* total dwords sent (summed from counter) */
@@ -309,6 +345,7 @@ struct ipath_devdata {
         ipath_err_t ipath_lasthwerror;
         /* errors masked because they occur too fast */
         ipath_err_t ipath_maskederrs;
+        u64 ipath_lastlinkrecov; /* link recoveries at last ACTIVE */
         /* time in jiffies at which to re-enable maskederrs */
         unsigned long ipath_unmasktime;
         /* count of egrfull errors, combined for all ports */
@@ -347,6 +384,7 @@ struct ipath_devdata {
         u32 ipath_lastrpkts;
         /* pio bufs allocated per port */
         u32 ipath_pbufsport;
+        u32 ipath_pioupd_thresh; /* update threshold, some chips */
         /*
          * number of ports configured as max; zero is set to number chip
          * supports, less gives more pio bufs/port, etc.
@@ -365,6 +403,7 @@ struct ipath_devdata {
          * get to multiple devices
          */
         u32 ipath_lastpioindex;
+        u32 ipath_lastpioindexl;
         /* max length of freezemsg */
         u32 ipath_freezelen;
         /*
@@ -381,6 +420,15 @@ struct ipath_devdata {
         u32 ipath_pcibar0;
         /* so we can rewrite it after a chip reset */
         u32 ipath_pcibar1;
+        u32 ipath_x1_fix_tries;
+        u32 ipath_autoneg_tries;
+        u32 serdes_first_init_done;
+
+        struct ipath_relock {
+                atomic_t ipath_relock_timer_active;
+                struct timer_list ipath_relock_timer;
+                unsigned int ipath_relock_interval; /* in jiffies */
+        } ipath_relock_singleton;
 
         /* interrupt number */
         int ipath_irq;
@@ -403,7 +451,7 @@ struct ipath_devdata {
         u64 __iomem *ipath_egrtidbase;
         /* lock to workaround chip bug 9437 and others */
         spinlock_t ipath_kernel_tid_lock;
-        spinlock_t ipath_tid_lock;
+        spinlock_t ipath_user_tid_lock;
         spinlock_t ipath_sendctrl_lock;
 
         /*
@@ -422,11 +470,48 @@ struct ipath_devdata {
         struct class_device *diag_class_dev;
         /* timer used to prevent stats overflow, error throttling, etc. */
         struct timer_list ipath_stats_timer;
+        /* timer to verify interrupts work, and fallback if possible */
+        struct timer_list ipath_intrchk_timer;
         void *ipath_dummy_hdrq; /* used after port close */
         dma_addr_t ipath_dummy_hdrq_phys;
 
+        /* SendDMA related entries */
+        spinlock_t            ipath_sdma_lock;
+        u64                   ipath_sdma_status;
+        unsigned long         ipath_sdma_abort_jiffies;
+        unsigned long         ipath_sdma_abort_intr_timeout;
+        unsigned long         ipath_sdma_buf_jiffies;
+        struct ipath_sdma_desc *ipath_sdma_descq;
+        u64                   ipath_sdma_descq_added;
+        u64                   ipath_sdma_descq_removed;
+        int                   ipath_sdma_desc_nreserved;
+        u16                   ipath_sdma_descq_cnt;
+        u16                   ipath_sdma_descq_tail;
+        u16                   ipath_sdma_descq_head;
+        u16                   ipath_sdma_next_intr;
+        u16                   ipath_sdma_reset_wait;
+        u8                    ipath_sdma_generation;
+        struct tasklet_struct ipath_sdma_abort_task;
+        struct tasklet_struct ipath_sdma_notify_task;
+        struct list_head      ipath_sdma_activelist;
+        struct list_head      ipath_sdma_notifylist;
+        atomic_t              ipath_sdma_vl15_count;
+        struct timer_list     ipath_sdma_vl15_timer;
+
+        dma_addr_t       ipath_sdma_descq_phys;
+        volatile __le64 *ipath_sdma_head_dma;
+        dma_addr_t       ipath_sdma_head_phys;
+
         unsigned long ipath_ureg_align; /* user register alignment */
 
+        struct delayed_work ipath_autoneg_work;
+        wait_queue_head_t ipath_autoneg_wait;
+
+        /* HoL blocking / user app forward-progress state */
+        unsigned          ipath_hol_state;
+        unsigned          ipath_hol_next;
+        struct timer_list ipath_hol_timer;
+
         /*
          * Shadow copies of registers; size indicates read access size.
          * Most of them are readonly, but some are write-only register,
@@ -447,6 +532,8 @@ struct ipath_devdata {
          * init time.
          */
         unsigned long ipath_pioavailshadow[8];
+        /* bitmap of send buffers available for the kernel to use with PIO. */
+        unsigned long ipath_pioavailkernel[8];
         /* shadow of kr_gpio_out, for rmw ops */
         u64 ipath_gpio_out;
         /* shadow the gpio mask register */
@@ -472,6 +559,8 @@ struct ipath_devdata {
         u64 ipath_intconfig;
         /* kr_sendpiobufbase value */
         u64 ipath_piobufbase;
+        /* kr_ibcddrctrl shadow */
+        u64 ipath_ibcddrctrl;
 
         /* these are the "32 bit" regs */
 
@@ -488,7 +577,10 @@ struct ipath_devdata {
         unsigned long ipath_rcvctrl;
         /* shadow kr_sendctrl */
         unsigned long ipath_sendctrl;
-        unsigned long ipath_lastcancel; /* to not count armlaunch after cancel */
+        /* to not count armlaunch after cancel */
+        unsigned long ipath_lastcancel;
+        /* count cases where special trigger was needed (double write) */
+        unsigned long ipath_spectriggerhit;
 
         /* value we put in kr_rcvhdrcnt */
         u32 ipath_rcvhdrcnt;
@@ -510,6 +602,7 @@ struct ipath_devdata {
         u32 ipath_piobcnt4k;
         /* size in bytes of "4KB" PIO buffers */
         u32 ipath_piosize4k;
+        u32 ipath_pioreserved; /* reserved special-inkernel; */
         /* kr_rcvegrbase value */
         u32 ipath_rcvegrbase;
         /* kr_rcvegrcnt value */
@@ -546,10 +639,10 @@ struct ipath_devdata {
         u32 ipath_init_ibmaxlen;
         /* size of each rcvegrbuffer */
         u32 ipath_rcvegrbufsize;
-        /* width (2,4,8,16,32) from HT config reg */
-        u32 ipath_htwidth;
-        /* HT speed (200,400,800,1000) from HT config */
-        u32 ipath_htspeed;
+        /* localbus width (1, 2,4,8,16,32) from config space  */
+        u32 ipath_lbus_width;
+        /* localbus speed (HT: 200,400,800,1000; PCIe 2500) */
+        u32 ipath_lbus_speed;
         /*
          * number of sequential ibcstatus change for polling active/quiet
          * (i.e., link not coming up).
@@ -573,21 +666,14 @@ struct ipath_devdata {
          */
         u8 ipath_serial[16];
         /* human readable board version */
-        u8 ipath_boardversion[80];
+        u8 ipath_boardversion[96];
+        u8 ipath_lbus_info[32]; /* human readable localbus info */
         /* chip major rev, from ipath_revision */
         u8 ipath_majrev;
         /* chip minor rev, from ipath_revision */
         u8 ipath_minrev;
         /* board rev, from ipath_revision */
         u8 ipath_boardrev;
-
-        u8 ipath_r_portenable_shift;
-        u8 ipath_r_intravail_shift;
-        u8 ipath_r_tailupd_shift;
-        u8 ipath_r_portcfg_shift;
-
-        /* unit # of this chip, if present */
-        int ipath_unit;
         /* saved for restore after reset */
         u8 ipath_pci_cacheline;
         /* LID mask control */
@@ -603,6 +689,14 @@ struct ipath_devdata {
         /* Rx Polarity inversion (compensate for ~tx on partner) */
         u8 ipath_rx_pol_inv;
 
+        u8 ipath_r_portenable_shift;
+        u8 ipath_r_intravail_shift;
+        u8 ipath_r_tailupd_shift;
+        u8 ipath_r_portcfg_shift;
+
+        /* unit # of this chip, if present */
+        int ipath_unit;
+
         /* local link integrity counter */
         u32 ipath_lli_counter;
         /* local link integrity errors */
@@ -617,9 +711,6 @@ struct ipath_devdata {
         u32 ipath_overrun_thresh_errs;
         u32 ipath_lli_errs;
 
-        /* status check work */
-        struct delayed_work status_work;
-
         /*
          * Not all devices managed by a driver instance are the same
          * type, so these fields must be per-device.
@@ -632,8 +723,8 @@ struct ipath_devdata {
          * Below should be computable from number of ports,
          * since they are never modified.
          */
-        u32 ipath_i_rcvavail_mask;
-        u32 ipath_i_rcvurg_mask;
+        u64 ipath_i_rcvavail_mask;
+        u64 ipath_i_rcvurg_mask;
         u16 ipath_i_rcvurg_shift;
         u16 ipath_i_rcvavail_shift;
 
@@ -641,8 +732,9 @@ struct ipath_devdata {
          * Register bits for selecting i2c direction and values, used for
          * I2C serial flash.
          */
-        u16 ipath_gpio_sda_num;
-        u16 ipath_gpio_scl_num;
+        u8 ipath_gpio_sda_num;
+        u8 ipath_gpio_scl_num;
+        u8 ipath_i2c_chain_type;
         u64 ipath_gpio_sda;
         u64 ipath_gpio_scl;
 
@@ -703,13 +795,51 @@ struct ipath_devdata {
         /* interrupt mitigation reload register info */
         u16 ipath_jint_idle_ticks;      /* idle clock ticks */
         u16 ipath_jint_max_packets;     /* max packets across all ports */
+
+        /*
+         * lock for access to SerDes, and flags to sequence preset
+         * versus steady-state. 7220-only at the moment.
+         */
+        spinlock_t ipath_sdepb_lock;
+        u8 ipath_presets_needed; /* Set if presets to be restored next DOWN */
 };
 
+/* ipath_hol_state values (stopping/starting user proc, send flushing) */
+#define IPATH_HOL_UP       0
+#define IPATH_HOL_DOWN     1
+/* ipath_hol_next toggle values, used when hol_state IPATH_HOL_DOWN */
+#define IPATH_HOL_DOWNSTOP 0
+#define IPATH_HOL_DOWNCONT 1
+
+/* bit positions for sdma_status */
+#define IPATH_SDMA_ABORTING  0
+#define IPATH_SDMA_DISARMED  1
+#define IPATH_SDMA_DISABLED  2
+#define IPATH_SDMA_LAYERBUF  3
+#define IPATH_SDMA_RUNNING  62
+#define IPATH_SDMA_SHUTDOWN 63
+
+/* bit combinations that correspond to abort states */
+#define IPATH_SDMA_ABORT_NONE 0
+#define IPATH_SDMA_ABORT_ABORTING (1UL << IPATH_SDMA_ABORTING)
+#define IPATH_SDMA_ABORT_DISARMED ((1UL << IPATH_SDMA_ABORTING) | \
+        (1UL << IPATH_SDMA_DISARMED))
+#define IPATH_SDMA_ABORT_DISABLED ((1UL << IPATH_SDMA_ABORTING) | \
+        (1UL << IPATH_SDMA_DISABLED))
+#define IPATH_SDMA_ABORT_ABORTED ((1UL << IPATH_SDMA_ABORTING) | \
+        (1UL << IPATH_SDMA_DISARMED) | (1UL << IPATH_SDMA_DISABLED))
+#define IPATH_SDMA_ABORT_MASK ((1UL<<IPATH_SDMA_ABORTING) | \
+        (1UL << IPATH_SDMA_DISARMED) | (1UL << IPATH_SDMA_DISABLED))
+
+#define IPATH_SDMA_BUF_NONE 0
+#define IPATH_SDMA_BUF_MASK (1UL<<IPATH_SDMA_LAYERBUF)
+
 /* Private data for file operations */
 struct ipath_filedata {
         struct ipath_portdata *pd;
         unsigned subport;
         unsigned tidcursor;
+        struct ipath_user_sdma_queue *pq;
 };
 extern struct list_head ipath_dev_list;
 extern spinlock_t ipath_devs_lock;
@@ -718,7 +848,7 @@ extern struct ipath_devdata *ipath_lookup(int unit);
 int ipath_init_chip(struct ipath_devdata *, int);
 int ipath_enable_wc(struct ipath_devdata *dd);
 void ipath_disable_wc(struct ipath_devdata *dd);
-int ipath_count_units(int *npresentp, int *nupp, u32 *maxportsp);
+int ipath_count_units(int *npresentp, int *nupp, int *maxportsp);
 void ipath_shutdown_device(struct ipath_devdata *);
 void ipath_clear_freeze(struct ipath_devdata *);
 
@@ -741,7 +871,8 @@ struct sk_buff *ipath_alloc_skb(struct ipath_devdata *dd, gfp_t);
 extern int ipath_diag_inuse;
 
 irqreturn_t ipath_intr(int irq, void *devid);
-int ipath_decode_err(char *buf, size_t blen, ipath_err_t err);
+int ipath_decode_err(struct ipath_devdata *dd, char *buf, size_t blen,
+                     ipath_err_t err);
 #if __IPATH_INFO || __IPATH_DBG
 extern const char *ipath_ibcstatus_str[];
 #endif
@@ -774,6 +905,13 @@ int ipath_set_lid(struct ipath_devdata *, u32, u8);
 int ipath_set_rx_pol_inv(struct ipath_devdata *dd, u8 new_pol_inv);
 void ipath_enable_armlaunch(struct ipath_devdata *);
 void ipath_disable_armlaunch(struct ipath_devdata *);
+void ipath_hol_down(struct ipath_devdata *);
+void ipath_hol_up(struct ipath_devdata *);
+void ipath_hol_event(unsigned long);
+void ipath_toggle_rclkrls(struct ipath_devdata *);
+void ipath_sd7220_clr_ibpar(struct ipath_devdata *);
+void ipath_set_relock_poll(struct ipath_devdata *, int);
+void ipath_shutdown_relock_poll(struct ipath_devdata *);
 
 /* for use in system calls, where we want to know device type, etc. */
 #define port_fp(fp) ((struct ipath_filedata *)(fp)->private_data)->pd
@@ -781,11 +919,15 @@ void ipath_disable_armlaunch(struct ipath_devdata *);
         ((struct ipath_filedata *)(fp)->private_data)->subport
 #define tidcursor_fp(fp) \
         ((struct ipath_filedata *)(fp)->private_data)->tidcursor
+#define user_sdma_queue_fp(fp) \
+        ((struct ipath_filedata *)(fp)->private_data)->pq
 
 /*
  * values for ipath_flags
  */
-/* The chip is up and initted */
+                /* chip can report link latency (IB 1.2) */
+#define IPATH_HAS_LINK_LATENCY 0x1
+                /* The chip is up and initted */
 #define IPATH_INITTED       0x2
                 /* set if any user code has set kr_rcvhdrsize */
 #define IPATH_RCVHDRSZ_SET  0x4
@@ -809,6 +951,8 @@ void ipath_disable_armlaunch(struct ipath_devdata *);
 #define IPATH_LINKUNK       0x400
                 /* Write combining flush needed for PIO */
 #define IPATH_PIO_FLUSH_WC  0x1000
+                /* DMA Receive tail pointer */
+#define IPATH_NODMA_RTAIL   0x2000
                 /* no IB cable, or no device on IB cable */
 #define IPATH_NOCABLE       0x4000
                 /* Supports port zero per packet receive interrupts via
@@ -819,16 +963,26 @@ void ipath_disable_armlaunch(struct ipath_devdata *);
                 /* packet/word counters are 32 bit, else those 4 counters
                  * are 64bit */
 #define IPATH_32BITCOUNTERS 0x20000
-                /* can miss port0 rx interrupts */
                 /* Interrupt register is 64 bits */
 #define IPATH_INTREG_64     0x40000
+                /* can miss port0 rx interrupts */
 #define IPATH_DISABLED      0x80000 /* administratively disabled */
                 /* Use GPIO interrupts for new counters */
 #define IPATH_GPIO_ERRINTRS 0x100000
 #define IPATH_SWAP_PIOBUFS  0x200000
+                /* Supports Send DMA */
+#define IPATH_HAS_SEND_DMA  0x400000
+                /* Supports Send Count (not just word count) in PBC */
+#define IPATH_HAS_PBC_CNT   0x800000
                 /* Suppress heartbeat, even if turning off loopback */
 #define IPATH_NO_HRTBT      0x1000000
+#define IPATH_HAS_THRESH_UPDATE 0x4000000
 #define IPATH_HAS_MULT_IB_SPEED 0x8000000
+#define IPATH_IB_AUTONEG_INPROG 0x10000000
+#define IPATH_IB_AUTONEG_FAILED 0x20000000
+                /* Linkdown-disable intentionally, Do not attempt to bring up */
+#define IPATH_IB_LINK_DISABLED 0x40000000
+#define IPATH_IB_FORCE_NOTIFY 0x80000000 /* force notify on next ib change */
 
 /* Bits in GPIO for the added interrupts */
 #define IPATH_GPIO_PORT0_BIT 2
@@ -847,13 +1001,18 @@ void ipath_disable_armlaunch(struct ipath_devdata *);
 
 /* free up any allocated data at closes */
 void ipath_free_data(struct ipath_portdata *dd);
-u32 __iomem *ipath_getpiobuf(struct ipath_devdata *, u32 *);
+u32 __iomem *ipath_getpiobuf(struct ipath_devdata *, u32, u32 *);
+void ipath_chg_pioavailkernel(struct ipath_devdata *dd, unsigned start,
+                                unsigned len, int avail);
+void ipath_init_iba7220_funcs(struct ipath_devdata *);
 void ipath_init_iba6120_funcs(struct ipath_devdata *);
 void ipath_init_iba6110_funcs(struct ipath_devdata *);
 void ipath_get_eeprom_info(struct ipath_devdata *);
 int ipath_update_eeprom_log(struct ipath_devdata *dd);
 void ipath_inc_eeprom_err(struct ipath_devdata *dd, u32 eidx, u32 incr);
 u64 ipath_snap_cntr(struct ipath_devdata *, ipath_creg);
+void ipath_disarm_senderrbufs(struct ipath_devdata *, int);
+void ipath_force_pio_avail_update(struct ipath_devdata *);
 void signal_ib_event(struct ipath_devdata *dd, enum ib_event_type ev);
 
 /*
@@ -865,6 +1024,34 @@ void signal_ib_event(struct ipath_devdata *dd, enum ib_event_type ev);
 #define IPATH_LED_LOG 2  /* Logical (link) YELLOW LED */
 void ipath_set_led_override(struct ipath_devdata *dd, unsigned int val);
 
+/* send dma routines */
+int setup_sdma(struct ipath_devdata *);
+void teardown_sdma(struct ipath_devdata *);
+void ipath_restart_sdma(struct ipath_devdata *);
+void ipath_sdma_intr(struct ipath_devdata *);
+int ipath_sdma_verbs_send(struct ipath_devdata *, struct ipath_sge_state *,
+                          u32, struct ipath_verbs_txreq *);
+/* ipath_sdma_lock should be locked before calling this. */
+int ipath_sdma_make_progress(struct ipath_devdata *dd);
+
+/* must be called under ipath_sdma_lock */
+static inline u16 ipath_sdma_descq_freecnt(const struct ipath_devdata *dd)
+{
+        return dd->ipath_sdma_descq_cnt -
+                (dd->ipath_sdma_descq_added - dd->ipath_sdma_descq_removed) -
+                1 - dd->ipath_sdma_desc_nreserved;
+}
+
+static inline void ipath_sdma_desc_reserve(struct ipath_devdata *dd, u16 cnt)
+{
+        dd->ipath_sdma_desc_nreserved += cnt;
+}
+
+static inline void ipath_sdma_desc_unreserve(struct ipath_devdata *dd, u16 cnt)
+{
+        dd->ipath_sdma_desc_nreserved -= cnt;
+}
+
 /*
  * number of words used for protocol header if not set by ipath_userinit();
  */
@@ -875,6 +1062,8 @@ void ipath_release_user_pages(struct page **, size_t);
 void ipath_release_user_pages_on_close(struct page **, size_t);
 int ipath_eeprom_read(struct ipath_devdata *, u8, void *, int);
 int ipath_eeprom_write(struct ipath_devdata *, u8, const void *, int);
+int ipath_tempsense_read(struct ipath_devdata *, u8 regnum);
+int ipath_tempsense_write(struct ipath_devdata *, u8 regnum, u8 data);
 
 /* these are used for the registers that vary with port */
 void ipath_write_kreg_port(const struct ipath_devdata *, ipath_kreg,
@@ -891,8 +1080,7 @@ void ipath_write_kreg_port(const struct ipath_devdata *, ipath_kreg,
 
 /*
  * At the moment, none of the s-registers are writable, so no
- * ipath_write_sreg(), and none of the c-registers are writable, so no
- * ipath_write_creg().
+ * ipath_write_sreg().
  */
 
 /**
@@ -1001,6 +1189,27 @@ static inline u32 ipath_get_rcvhdrtail(const struct ipath_portdata *pd)
                                 pd->port_rcvhdrtail_kvaddr));
 }
 
+static inline u32 ipath_get_hdrqtail(const struct ipath_portdata *pd)
+{
+        const struct ipath_devdata *dd = pd->port_dd;
+        u32 hdrqtail;
+
+        if (dd->ipath_flags & IPATH_NODMA_RTAIL) {
+                __le32 *rhf_addr;
+                u32 seq;
+
+                rhf_addr = (__le32 *) pd->port_rcvhdrq +
+                        pd->port_head + dd->ipath_rhf_offset;
+                seq = ipath_hdrget_seq(rhf_addr);
+                hdrqtail = pd->port_head;
+                if (seq == pd->port_seq_cnt)
+                        hdrqtail++;
+        } else
+                hdrqtail = ipath_get_rcvhdrtail(pd);
+
+        return hdrqtail;
+}
+
 static inline u64 ipath_read_ireg(const struct ipath_devdata *dd, ipath_kreg r)
 {
         return (dd->ipath_flags & IPATH_INTREG_64) ?
@@ -1029,6 +1238,21 @@ static inline u32 ipath_ib_linktrstate(struct ipath_devdata *dd, u64 ibcs)
 }
 
 /*
+ * from contents of IBCStatus (or a saved copy), return logical link state
+ * combination of link state and linktraining state (down, active, init,
+ * arm, etc.
+ */
+static inline u32 ipath_ib_state(struct ipath_devdata *dd, u64 ibcs)
+{
+        u32 ibs;
+        ibs = (u32)(ibcs >> INFINIPATH_IBCS_LINKTRAININGSTATE_SHIFT) &
+                dd->ibcs_lts_mask;
+        ibs |= (u32)(ibcs &
+                (INFINIPATH_IBCS_LINKSTATE_MASK << dd->ibcs_ls_shift));
+        return ibs;
+}
+
+/*
  * sysfs interface.
  */
 
@@ -1053,6 +1277,7 @@ int ipathfs_remove_device(struct ipath_devdata *);
 dma_addr_t ipath_map_page(struct pci_dev *, struct page *, unsigned long,
                           size_t, int);
 dma_addr_t ipath_map_single(struct pci_dev *, void *, size_t, int);
+const char *ipath_get_unit_name(int unit);
 
 /*
  * Flush write combining store buffers (if present) and perform a write
@@ -1065,11 +1290,8 @@ dma_addr_t ipath_map_single(struct pci_dev *, void *, size_t, int);
 #endif
 
 extern unsigned ipath_debug; /* debugging bit mask */
-
-#define IPATH_MAX_PARITY_ATTEMPTS 10000 /* max times to try recovery */
-
-const char *ipath_get_unit_name(int unit);
-
+extern unsigned ipath_linkrecovery;
+extern unsigned ipath_mtu4096;
 extern struct mutex ipath_mutex;
 
 #define IPATH_DRV_NAME          "ib_ipath"
@@ -1096,7 +1318,7 @@ extern struct mutex ipath_mutex;
 
 # define __IPATH_DBG_WHICH(which,fmt,...) \
         do { \
-                if(unlikely(ipath_debug&(which))) \
+                if (unlikely(ipath_debug & (which))) \
                         printk(KERN_DEBUG IPATH_DRV_NAME ": %s: " fmt, \
                                __func__,##__VA_ARGS__); \
         } while(0)