1 files changed, 167 insertions, 85 deletions

diff --git a/drivers/infiniband/hw/ipath/ipath_verbs.h b/drivers/infiniband/hw/ipath/ipath_verbs.h
index 2df684727dc1..09bbb3f9a217 100644
--- a/drivers/infiniband/hw/ipath/ipath_verbs.h
+++ b/drivers/infiniband/hw/ipath/ipath_verbs.h
@@ -38,10 +38,10 @@
 #include <linux/spinlock.h>
 #include <linux/kernel.h>
 #include <linux/interrupt.h>
+#include <linux/kref.h>
 #include <rdma/ib_pack.h>
 
 #include "ipath_layer.h"
-#include "verbs_debug.h"
 
 #define QPN_MAX                 (1 << 24)
 #define QPNMAP_ENTRIES          (QPN_MAX / PAGE_SIZE / BITS_PER_BYTE)
@@ -50,7 +50,7 @@
  * Increment this value if any changes that break userspace ABI
  * compatibility are made.
  */
-#define IPATH_UVERBS_ABI_VERSION       1
+#define IPATH_UVERBS_ABI_VERSION       2
 
 /*
  * Define an ib_cq_notify value that is not valid so we know when CQ
@@ -152,19 +152,6 @@ struct ipath_mcast {
         int n_attached;
 };
 
-/* Memory region */
-struct ipath_mr {
-        struct ib_mr ibmr;
-        struct ipath_mregion mr;        /* must be last */
-};
-
-/* Fast memory region */
-struct ipath_fmr {
-        struct ib_fmr ibfmr;
-        u8 page_shift;
-        struct ipath_mregion mr;        /* must be last */
-};
-
 /* Protection domain */
 struct ipath_pd {
         struct ib_pd ibpd;
@@ -178,58 +165,89 @@ struct ipath_ah {
 };
 
 /*
- * Quick description of our CQ/QP locking scheme:
- *
- * We have one global lock that protects dev->cq/qp_table.  Each
- * struct ipath_cq/qp also has its own lock.  An individual qp lock
- * may be taken inside of an individual cq lock.  Both cqs attached to
- * a qp may be locked, with the send cq locked first.  No other
- * nesting should be done.
- *
- * Each struct ipath_cq/qp also has an atomic_t ref count.  The
- * pointer from the cq/qp_table to the struct counts as one reference.
- * This reference also is good for access through the consumer API, so
- * modifying the CQ/QP etc doesn't need to take another reference.
- * Access because of a completion being polled does need a reference.
- *
- * Finally, each struct ipath_cq/qp has a wait_queue_head_t for the
- * destroy function to sleep on.
- *
- * This means that access from the consumer API requires nothing but
- * taking the struct's lock.
- *
- * Access because of a completion event should go as follows:
- * - lock cq/qp_table and look up struct
- * - increment ref count in struct
- * - drop cq/qp_table lock
- * - lock struct, do your thing, and unlock struct
- * - decrement ref count; if zero, wake up waiters
- *
- * To destroy a CQ/QP, we can do the following:
- * - lock cq/qp_table, remove pointer, unlock cq/qp_table lock
- * - decrement ref count
- * - wait_event until ref count is zero
- *
- * It is the consumer's responsibilty to make sure that no QP
- * operations (WQE posting or state modification) are pending when the
- * QP is destroyed.  Also, the consumer must make sure that calls to
- * qp_modify are serialized.
- *
- * Possible optimizations (wait for profile data to see if/where we
- * have locks bouncing between CPUs):
- * - split cq/qp table lock into n separate (cache-aligned) locks,
- *   indexed (say) by the page in the table
+ * This structure is used by ipath_mmap() to validate an offset
+ * when an mmap() request is made.  The vm_area_struct then uses
+ * this as its vm_private_data.
+ */
+struct ipath_mmap_info {
+        struct ipath_mmap_info *next;
+        struct ib_ucontext *context;
+        void *obj;
+        struct kref ref;
+        unsigned size;
+        unsigned mmap_cnt;
+};
+
+/*
+ * This structure is used to contain the head pointer, tail pointer,
+ * and completion queue entries as a single memory allocation so
+ * it can be mmap'ed into user space.
  */
+struct ipath_cq_wc {
+        u32 head;               /* index of next entry to fill */
+        u32 tail;               /* index of next ib_poll_cq() entry */
+        struct ib_wc queue[1];  /* this is actually size ibcq.cqe + 1 */
+};
 
+/*
+ * The completion queue structure.
+ */
 struct ipath_cq {
         struct ib_cq ibcq;
         struct tasklet_struct comptask;
         spinlock_t lock;
         u8 notify;
         u8 triggered;
-        u32 head;               /* new records added to the head */
-        u32 tail;               /* poll_cq() reads from here. */
-        struct ib_wc *queue;    /* this is actually ibcq.cqe + 1 */
+        struct ipath_cq_wc *queue;
+        struct ipath_mmap_info *ip;
+};
+
+/*
+ * A segment is a linear region of low physical memory.
+ * XXX Maybe we should use phys addr here and kmap()/kunmap().
+ * Used by the verbs layer.
+ */
+struct ipath_seg {
+        void *vaddr;
+        size_t length;
+};
+
+/* The number of ipath_segs that fit in a page. */
+#define IPATH_SEGSZ     (PAGE_SIZE / sizeof (struct ipath_seg))
+
+struct ipath_segarray {
+        struct ipath_seg segs[IPATH_SEGSZ];
+};
+
+struct ipath_mregion {
+        u64 user_base;          /* User's address for this region */
+        u64 iova;               /* IB start address of this region */
+        size_t length;
+        u32 lkey;
+        u32 offset;             /* offset (bytes) to start of region */
+        int access_flags;
+        u32 max_segs;           /* number of ipath_segs in all the arrays */
+        u32 mapsz;              /* size of the map array */
+        struct ipath_segarray *map[0];  /* the segments */
+};
+
+/*
+ * These keep track of the copy progress within a memory region.
+ * Used by the verbs layer.
+ */
+struct ipath_sge {
+        struct ipath_mregion *mr;
+        void *vaddr;            /* current pointer into the segment */
+        u32 sge_length;         /* length of the SGE */
+        u32 length;             /* remaining length of the segment */
+        u16 m;                  /* current index: mr->map[m] */
+        u16 n;                  /* current index: mr->map[m]->segs[n] */
+};
+
+/* Memory region */
+struct ipath_mr {
+        struct ib_mr ibmr;
+        struct ipath_mregion mr;        /* must be last */
 };
 
 /*
@@ -248,32 +266,50 @@ struct ipath_swqe {
 
 /*
  * Receive work request queue entry.
- * The size of the sg_list is determined when the QP is created and stored
- * in qp->r_max_sge.
+ * The size of the sg_list is determined when the QP (or SRQ) is created
+ * and stored in qp->r_rq.max_sge (or srq->rq.max_sge).
  */
 struct ipath_rwqe {
         u64 wr_id;
-        u32 length;             /* total length of data in sg_list */
         u8 num_sge;
-        struct ipath_sge sg_list[0];
+        struct ib_sge sg_list[0];
 };
 
-struct ipath_rq {
-        spinlock_t lock;
+/*
+ * This structure is used to contain the head pointer, tail pointer,
+ * and receive work queue entries as a single memory allocation so
+ * it can be mmap'ed into user space.
+ * Note that the wq array elements are variable size so you can't
+ * just index into the array to get the N'th element;
+ * use get_rwqe_ptr() instead.
+ */
+struct ipath_rwq {
         u32 head;               /* new work requests posted to the head */
         u32 tail;               /* receives pull requests from here. */
+        struct ipath_rwqe wq[0];
+};
+
+struct ipath_rq {
+        struct ipath_rwq *wq;
+        spinlock_t lock;
         u32 size;               /* size of RWQE array */
         u8 max_sge;
-        struct ipath_rwqe *wq;  /* RWQE array */
 };
 
 struct ipath_srq {
         struct ib_srq ibsrq;
         struct ipath_rq rq;
+        struct ipath_mmap_info *ip;
         /* send signal when number of RWQEs < limit */
         u32 limit;
 };
 
+struct ipath_sge_state {
+        struct ipath_sge *sg_list;      /* next SGE to be used if any */
+        struct ipath_sge sge;   /* progress state for the current SGE */
+        u8 num_sge;
+};
+
 /*
  * Variables prefixed with s_ are for the requester (sender).
  * Variables prefixed with r_ are for the responder (receiver).
@@ -293,6 +329,7 @@ struct ipath_qp {
         atomic_t refcount;
         wait_queue_head_t wait;
         struct tasklet_struct s_task;
+        struct ipath_mmap_info *ip;
         struct ipath_sge_state *s_cur_sge;
         struct ipath_sge_state s_sge;   /* current send request data */
         /* current RDMA read send data */
@@ -334,6 +371,7 @@ struct ipath_qp {
         u8 s_retry;             /* requester retry counter */
         u8 s_rnr_retry;         /* requester RNR retry counter */
         u8 s_pkey_index;        /* PKEY index to use */
+        u8 timeout;             /* Timeout for this QP */
         enum ib_mtu path_mtu;
         u32 remote_qpn;
         u32 qkey;               /* QKEY for this QP (for UD or RD) */
@@ -345,7 +383,8 @@ struct ipath_qp {
         u32 s_ssn;              /* SSN of tail entry */
         u32 s_lsn;              /* limit sequence number (credit) */
         struct ipath_swqe *s_wq;        /* send work queue */
-        struct ipath_rq r_rq;   /* receive work queue */
+        struct ipath_rq r_rq;           /* receive work queue */
+        struct ipath_sge r_sg_list[0];  /* verified SGEs */
 };
 
 /*
@@ -369,15 +408,15 @@ static inline struct ipath_swqe *get_swqe_ptr(struct ipath_qp *qp,
 
 /*
  * Since struct ipath_rwqe is not a fixed size, we can't simply index into
- * struct ipath_rq.wq.  This function does the array index computation.
+ * struct ipath_rwq.wq.  This function does the array index computation.
  */
 static inline struct ipath_rwqe *get_rwqe_ptr(struct ipath_rq *rq,
                                               unsigned n)
 {
         return (struct ipath_rwqe *)
-                ((char *) rq->wq +
+                ((char *) rq->wq->wq +
                  (sizeof(struct ipath_rwqe) +
-                  rq->max_sge * sizeof(struct ipath_sge)) * n);
+                  rq->max_sge * sizeof(struct ib_sge)) * n);
 }
 
 /*
@@ -417,6 +456,7 @@ struct ipath_ibdev {
         struct ib_device ibdev;
         struct list_head dev_list;
         struct ipath_devdata *dd;
+        struct ipath_mmap_info *pending_mmaps;
         int ib_unit;            /* This is the device number */
         u16 sm_lid;             /* in host order */
         u8 sm_sl;
@@ -435,11 +475,20 @@ struct ipath_ibdev {
         __be64 sys_image_guid;  /* in network order */
         __be64 gid_prefix;      /* in network order */
         __be64 mkey;
+
         u32 n_pds_allocated;    /* number of PDs allocated for device */
+        spinlock_t n_pds_lock;
         u32 n_ahs_allocated;    /* number of AHs allocated for device */
+        spinlock_t n_ahs_lock;
         u32 n_cqs_allocated;    /* number of CQs allocated for device */
+        spinlock_t n_cqs_lock;
+        u32 n_qps_allocated;    /* number of QPs allocated for device */
+        spinlock_t n_qps_lock;
         u32 n_srqs_allocated;   /* number of SRQs allocated for device */
+        spinlock_t n_srqs_lock;
         u32 n_mcast_grps_allocated; /* number of mcast groups allocated */
+        spinlock_t n_mcast_grps_lock;
+
         u64 ipath_sword;        /* total dwords sent (sample result) */
         u64 ipath_rword;        /* total dwords received (sample result) */
         u64 ipath_spkts;        /* total packets sent (sample result) */
@@ -494,8 +543,19 @@ struct ipath_ibdev {
         struct ipath_opcode_stats opstats[128];
 };
 
-struct ipath_ucontext {
-        struct ib_ucontext ibucontext;
+struct ipath_verbs_counters {
+        u64 symbol_error_counter;
+        u64 link_error_recovery_counter;
+        u64 link_downed_counter;
+        u64 port_rcv_errors;
+        u64 port_rcv_remphys_errors;
+        u64 port_xmit_discards;
+        u64 port_xmit_data;
+        u64 port_rcv_data;
+        u64 port_xmit_packets;
+        u64 port_rcv_packets;
+        u32 local_link_integrity_errors;
+        u32 excessive_buffer_overrun_errors;
 };
 
 static inline struct ipath_mr *to_imr(struct ib_mr *ibmr)
@@ -503,11 +563,6 @@ static inline struct ipath_mr *to_imr(struct ib_mr *ibmr)
         return container_of(ibmr, struct ipath_mr, ibmr);
 }
 
-static inline struct ipath_fmr *to_ifmr(struct ib_fmr *ibfmr)
-{
-        return container_of(ibfmr, struct ipath_fmr, ibfmr);
-}
-
 static inline struct ipath_pd *to_ipd(struct ib_pd *ibpd)
 {
         return container_of(ibpd, struct ipath_pd, ibpd);
@@ -545,12 +600,6 @@ int ipath_process_mad(struct ib_device *ibdev,
                       struct ib_grh *in_grh,
                       struct ib_mad *in_mad, struct ib_mad *out_mad);
 
-static inline struct ipath_ucontext *to_iucontext(struct ib_ucontext
-                                                  *ibucontext)
-{
-        return container_of(ibucontext, struct ipath_ucontext, ibucontext);
-}
-
 /*
  * Compare the lower 24 bits of the two values.
  * Returns an integer <, ==, or > than zero.
@@ -562,6 +611,13 @@ static inline int ipath_cmp24(u32 a, u32 b)
 
 struct ipath_mcast *ipath_mcast_find(union ib_gid *mgid);
 
+int ipath_snapshot_counters(struct ipath_devdata *dd, u64 *swords,
+                            u64 *rwords, u64 *spkts, u64 *rpkts,
+                            u64 *xmit_wait);
+
+int ipath_get_counters(struct ipath_devdata *dd,
+                       struct ipath_verbs_counters *cntrs);
+
 int ipath_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid);
 
 int ipath_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid);
@@ -579,7 +635,7 @@ struct ib_qp *ipath_create_qp(struct ib_pd *ibpd,
 int ipath_destroy_qp(struct ib_qp *ibqp);
 
 int ipath_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
-                    int attr_mask);
+                    int attr_mask, struct ib_udata *udata);
 
 int ipath_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
                    int attr_mask, struct ib_qp_init_attr *init_attr);
@@ -592,6 +648,9 @@ void ipath_sqerror_qp(struct ipath_qp *qp, struct ib_wc *wc);
 
 void ipath_get_credit(struct ipath_qp *qp, u32 aeth);
 
+int ipath_verbs_send(struct ipath_devdata *dd, u32 hdrwords,
+                     u32 *hdr, u32 len, struct ipath_sge_state *ss);
+
 void ipath_cq_enter(struct ipath_cq *cq, struct ib_wc *entry, int sig);
 
 int ipath_rkey_ok(struct ipath_ibdev *dev, struct ipath_sge_state *ss,
@@ -638,7 +697,8 @@ struct ib_srq *ipath_create_srq(struct ib_pd *ibpd,
                                 struct ib_udata *udata);
 
 int ipath_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
-                     enum ib_srq_attr_mask attr_mask);
+                     enum ib_srq_attr_mask attr_mask,
+                     struct ib_udata *udata);
 
 int ipath_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr);
 
@@ -680,6 +740,10 @@ int ipath_unmap_fmr(struct list_head *fmr_list);
 
 int ipath_dealloc_fmr(struct ib_fmr *ibfmr);
 
+void ipath_release_mmap_info(struct kref *ref);
+
+int ipath_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
+
 void ipath_no_bufs_available(struct ipath_qp *qp, struct ipath_ibdev *dev);
 
 void ipath_insert_rnr_queue(struct ipath_qp *qp);
@@ -700,6 +764,22 @@ int ipath_make_rc_req(struct ipath_qp *qp, struct ipath_other_headers *ohdr,
 int ipath_make_uc_req(struct ipath_qp *qp, struct ipath_other_headers *ohdr,
                       u32 pmtu, u32 *bth0p, u32 *bth2p);
 
+int ipath_register_ib_device(struct ipath_devdata *);
+
+void ipath_unregister_ib_device(struct ipath_ibdev *);
+
+void ipath_ib_rcv(struct ipath_ibdev *, void *, void *, u32);
+
+int ipath_ib_piobufavail(struct ipath_ibdev *);
+
+void ipath_ib_timer(struct ipath_ibdev *);
+
+unsigned ipath_get_npkeys(struct ipath_devdata *);
+
+u32 ipath_get_cr_errpkey(struct ipath_devdata *);
+
+unsigned ipath_get_pkey(struct ipath_devdata *, unsigned);
+
 extern const enum ib_wc_opcode ib_ipath_wc_opcode[];
 
 extern const u8 ipath_cvt_physportstate[];
@@ -714,6 +794,8 @@ extern unsigned int ib_ipath_max_cqs;
 
 extern unsigned int ib_ipath_max_qp_wrs;
 
+extern unsigned int ib_ipath_max_qps;
+
 extern unsigned int ib_ipath_max_sges;
 
 extern unsigned int ib_ipath_max_mcast_grps;