summaryrefslogtreecommitdiffstats
path: root/include
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2019-07-15 23:38:15 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2019-07-15 23:38:15 -0400
commit2a3c389a0fde49b241430df806a34276568cfb29 (patch)
tree9cf35829317e8cc2aaffc4341fb824dad63fce02 /include
parent8de262531f5fbb7458463224a7587429800c24bf (diff)
parent0b043644c0ca601cb19943a81aa1f1455dbe9461 (diff)
Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma
Pull rdma updates from Jason Gunthorpe: "A smaller cycle this time. Notably we see another new driver, 'Soft iWarp', and the deletion of an ancient unused driver for nes. - Revise and simplify the signature offload RDMA MR APIs - More progress on hoisting object allocation boiler plate code out of the drivers - Driver bug fixes and revisions for hns, hfi1, efa, cxgb4, qib, i40iw - Tree wide cleanups: struct_size, put_user_page, xarray, rst doc conversion - Removal of obsolete ib_ucm chardev and nes driver - netlink based discovery of chardevs and autoloading of the modules providing them - Move more of the rdamvt/hfi1 uapi to include/uapi/rdma - New driver 'siw' for software based iWarp running on top of netdev, much like rxe's software RoCE. - mlx5 feature to report events in their raw devx format to userspace - Expose per-object counters through rdma tool - Adaptive interrupt moderation for RDMA (DIM), sharing the DIM core from netdev" * tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma: (194 commits) RMDA/siw: Require a 64 bit arch RDMA/siw: Mark expected switch fall-throughs RDMA/core: Fix -Wunused-const-variable warnings rdma/siw: Remove set but not used variable 's' rdma/siw: Add missing dependencies on LIBCRC32C and DMA_VIRT_OPS RDMA/siw: Add missing rtnl_lock around access to ifa rdma/siw: Use proper enumerated type in map_cqe_status RDMA/siw: Remove unnecessary kthread create/destroy printouts IB/rdmavt: Fix variable shadowing issue in rvt_create_cq RDMA/core: Fix race when resolving IP address RDMA/core: Make rdma_counter.h compile stand alone IB/core: Work on the caller socket net namespace in nldev_newlink() RDMA/rxe: Fill in wc byte_len with IB_WC_RECV_RDMA_WITH_IMM RDMA/mlx5: Set RDMA DIM to be enabled by default RDMA/nldev: Added configuration of RDMA dynamic interrupt moderation to netlink RDMA/core: Provide RDMA DIM support for ULPs linux/dim: Implement RDMA adaptive moderation (DIM) IB/mlx5: Report correctly tag matching rendezvous capability docs: infiniband: add it to the driver-api bookset IB/mlx5: Implement VHCA tunnel mechanism in DEVX ...
Diffstat (limited to 'include')
-rw-r--r--include/linux/dim.h23
-rw-r--r--include/linux/mlx5/mlx5_ifc.h6
-rw-r--r--include/linux/mlx5/qp.h4
-rw-r--r--include/rdma/ib_umem.h19
-rw-r--r--include/rdma/ib_umem_odp.h20
-rw-r--r--include/rdma/ib_verbs.h247
-rw-r--r--include/rdma/mr_pool.h2
-rw-r--r--include/rdma/rdma_counter.h65
-rw-r--r--include/rdma/rdma_netlink.h8
-rw-r--r--include/rdma/rdma_vt.h5
-rw-r--r--include/rdma/rdmavt_cq.h25
-rw-r--r--include/rdma/rdmavt_qp.h312
-rw-r--r--include/rdma/restrack.h9
-rw-r--r--include/rdma/rw.h9
-rw-r--r--include/rdma/signature.h122
-rw-r--r--include/uapi/rdma/ib_user_cm.h326
-rw-r--r--include/uapi/rdma/mlx5_user_ioctl_cmds.h19
-rw-r--r--include/uapi/rdma/mlx5_user_ioctl_verbs.h9
-rw-r--r--include/uapi/rdma/rdma_netlink.h86
-rw-r--r--include/uapi/rdma/rdma_user_ioctl_cmds.h1
-rw-r--r--include/uapi/rdma/rvt-abi.h66
-rw-r--r--include/uapi/rdma/siw-abi.h185
22 files changed, 980 insertions, 588 deletions
diff --git a/include/linux/dim.h b/include/linux/dim.h
index aa9bdd47a648..d3a0fbfff2bb 100644
--- a/include/linux/dim.h
+++ b/include/linux/dim.h
@@ -82,6 +82,7 @@ struct dim_stats {
82 * @prev_stats: Measured rates from previous iteration (for comparison) 82 * @prev_stats: Measured rates from previous iteration (for comparison)
83 * @start_sample: Sampled data at start of current iteration 83 * @start_sample: Sampled data at start of current iteration
84 * @work: Work to perform on action required 84 * @work: Work to perform on action required
85 * @priv: A pointer to the struct that points to dim
85 * @profile_ix: Current moderation profile 86 * @profile_ix: Current moderation profile
86 * @mode: CQ period count mode 87 * @mode: CQ period count mode
87 * @tune_state: Algorithm tuning state (see below) 88 * @tune_state: Algorithm tuning state (see below)
@@ -95,6 +96,7 @@ struct dim {
95 struct dim_sample start_sample; 96 struct dim_sample start_sample;
96 struct dim_sample measuring_sample; 97 struct dim_sample measuring_sample;
97 struct work_struct work; 98 struct work_struct work;
99 void *priv;
98 u8 profile_ix; 100 u8 profile_ix;
99 u8 mode; 101 u8 mode;
100 u8 tune_state; 102 u8 tune_state;
@@ -363,4 +365,25 @@ struct dim_cq_moder net_dim_get_def_tx_moderation(u8 cq_period_mode);
363 */ 365 */
364void net_dim(struct dim *dim, struct dim_sample end_sample); 366void net_dim(struct dim *dim, struct dim_sample end_sample);
365 367
368/* RDMA DIM */
369
370/*
371 * RDMA DIM profile:
372 * profile size must be of RDMA_DIM_PARAMS_NUM_PROFILES.
373 */
374#define RDMA_DIM_PARAMS_NUM_PROFILES 9
375#define RDMA_DIM_START_PROFILE 0
376
377/**
378 * rdma_dim - Runs the adaptive moderation.
379 * @dim: The moderation struct.
380 * @completions: The number of completions collected in this round.
381 *
382 * Each call to rdma_dim takes the latest amount of completions that
383 * have been collected and counts them as a new event.
384 * Once enough events have been collected the algorithm decides a new
385 * moderation level.
386 */
387void rdma_dim(struct dim *dim, u64 completions);
388
366#endif /* DIM_H */ 389#endif /* DIM_H */
diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h
index 515624c66ce1..b3d5752657d9 100644
--- a/include/linux/mlx5/mlx5_ifc.h
+++ b/include/linux/mlx5/mlx5_ifc.h
@@ -1391,7 +1391,9 @@ struct mlx5_ifc_cmd_hca_cap_bits {
1391 u8 reserved_at_6c8[0x28]; 1391 u8 reserved_at_6c8[0x28];
1392 u8 sf_base_id[0x10]; 1392 u8 sf_base_id[0x10];
1393 1393
1394 u8 reserved_at_700[0x100]; 1394 u8 reserved_at_700[0x80];
1395 u8 vhca_tunnel_commands[0x40];
1396 u8 reserved_at_7c0[0x40];
1395}; 1397};
1396 1398
1397enum mlx5_flow_destination_type { 1399enum mlx5_flow_destination_type {
@@ -9695,7 +9697,7 @@ struct mlx5_ifc_general_obj_in_cmd_hdr_bits {
9695 u8 opcode[0x10]; 9697 u8 opcode[0x10];
9696 u8 uid[0x10]; 9698 u8 uid[0x10];
9697 9699
9698 u8 reserved_at_20[0x10]; 9700 u8 vhca_tunnel_id[0x10];
9699 u8 obj_type[0x10]; 9701 u8 obj_type[0x10];
9700 9702
9701 u8 obj_id[0x20]; 9703 u8 obj_id[0x20];
diff --git a/include/linux/mlx5/qp.h b/include/linux/mlx5/qp.h
index 127d224443e3..ae63b1ae9004 100644
--- a/include/linux/mlx5/qp.h
+++ b/include/linux/mlx5/qp.h
@@ -37,7 +37,8 @@
37#include <linux/mlx5/driver.h> 37#include <linux/mlx5/driver.h>
38 38
39#define MLX5_INVALID_LKEY 0x100 39#define MLX5_INVALID_LKEY 0x100
40#define MLX5_SIG_WQE_SIZE (MLX5_SEND_WQE_BB * 5) 40/* UMR (3 WQE_BB's) + SIG (3 WQE_BB's) + PSV (mem) + PSV (wire) */
41#define MLX5_SIG_WQE_SIZE (MLX5_SEND_WQE_BB * 8)
41#define MLX5_DIF_SIZE 8 42#define MLX5_DIF_SIZE 8
42#define MLX5_STRIDE_BLOCK_OP 0x400 43#define MLX5_STRIDE_BLOCK_OP 0x400
43#define MLX5_CPY_GRD_MASK 0xc0 44#define MLX5_CPY_GRD_MASK 0xc0
@@ -70,6 +71,7 @@ enum mlx5_qp_optpar {
70 MLX5_QP_OPTPAR_CQN_RCV = 1 << 19, 71 MLX5_QP_OPTPAR_CQN_RCV = 1 << 19,
71 MLX5_QP_OPTPAR_DC_HS = 1 << 20, 72 MLX5_QP_OPTPAR_DC_HS = 1 << 20,
72 MLX5_QP_OPTPAR_DC_KEY = 1 << 21, 73 MLX5_QP_OPTPAR_DC_KEY = 1 << 21,
74 MLX5_QP_OPTPAR_COUNTER_SET_ID = 1 << 25,
73}; 75};
74 76
75enum mlx5_qp_state { 77enum mlx5_qp_state {
diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h
index 040d853077c6..1052d0d62be7 100644
--- a/include/rdma/ib_umem.h
+++ b/include/rdma/ib_umem.h
@@ -46,7 +46,6 @@ struct ib_umem {
46 struct mm_struct *owning_mm; 46 struct mm_struct *owning_mm;
47 size_t length; 47 size_t length;
48 unsigned long address; 48 unsigned long address;
49 int page_shift;
50 u32 writable : 1; 49 u32 writable : 1;
51 u32 is_odp : 1; 50 u32 is_odp : 1;
52 struct work_struct work; 51 struct work_struct work;
@@ -58,24 +57,14 @@ struct ib_umem {
58/* Returns the offset of the umem start relative to the first page. */ 57/* Returns the offset of the umem start relative to the first page. */
59static inline int ib_umem_offset(struct ib_umem *umem) 58static inline int ib_umem_offset(struct ib_umem *umem)
60{ 59{
61 return umem->address & (BIT(umem->page_shift) - 1); 60 return umem->address & ~PAGE_MASK;
62}
63
64/* Returns the first page of an ODP umem. */
65static inline unsigned long ib_umem_start(struct ib_umem *umem)
66{
67 return umem->address - ib_umem_offset(umem);
68}
69
70/* Returns the address of the page after the last one of an ODP umem. */
71static inline unsigned long ib_umem_end(struct ib_umem *umem)
72{
73 return ALIGN(umem->address + umem->length, BIT(umem->page_shift));
74} 61}
75 62
76static inline size_t ib_umem_num_pages(struct ib_umem *umem) 63static inline size_t ib_umem_num_pages(struct ib_umem *umem)
77{ 64{
78 return (ib_umem_end(umem) - ib_umem_start(umem)) >> umem->page_shift; 65 return (ALIGN(umem->address + umem->length, PAGE_SIZE) -
66 ALIGN_DOWN(umem->address, PAGE_SIZE)) >>
67 PAGE_SHIFT;
79} 68}
80 69
81#ifdef CONFIG_INFINIBAND_USER_MEM 70#ifdef CONFIG_INFINIBAND_USER_MEM
diff --git a/include/rdma/ib_umem_odp.h b/include/rdma/ib_umem_odp.h
index eeec4e53c448..479db5c98ff6 100644
--- a/include/rdma/ib_umem_odp.h
+++ b/include/rdma/ib_umem_odp.h
@@ -76,6 +76,7 @@ struct ib_umem_odp {
76 76
77 struct completion notifier_completion; 77 struct completion notifier_completion;
78 int dying; 78 int dying;
79 unsigned int page_shift;
79 struct work_struct work; 80 struct work_struct work;
80}; 81};
81 82
@@ -84,6 +85,25 @@ static inline struct ib_umem_odp *to_ib_umem_odp(struct ib_umem *umem)
84 return container_of(umem, struct ib_umem_odp, umem); 85 return container_of(umem, struct ib_umem_odp, umem);
85} 86}
86 87
88/* Returns the first page of an ODP umem. */
89static inline unsigned long ib_umem_start(struct ib_umem_odp *umem_odp)
90{
91 return ALIGN_DOWN(umem_odp->umem.address, 1UL << umem_odp->page_shift);
92}
93
94/* Returns the address of the page after the last one of an ODP umem. */
95static inline unsigned long ib_umem_end(struct ib_umem_odp *umem_odp)
96{
97 return ALIGN(umem_odp->umem.address + umem_odp->umem.length,
98 1UL << umem_odp->page_shift);
99}
100
101static inline size_t ib_umem_odp_num_pages(struct ib_umem_odp *umem_odp)
102{
103 return (ib_umem_end(umem_odp) - ib_umem_start(umem_odp)) >>
104 umem_odp->page_shift;
105}
106
87/* 107/*
88 * The lower 2 bits of the DMA address signal the R/W permissions for 108 * The lower 2 bits of the DMA address signal the R/W permissions for
89 * the entry. To upgrade the permissions, provide the appropriate 109 * the entry. To upgrade the permissions, provide the appropriate
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 54873085f2da..c5f8a9f17063 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -61,8 +61,11 @@
61#include <linux/cgroup_rdma.h> 61#include <linux/cgroup_rdma.h>
62#include <linux/irqflags.h> 62#include <linux/irqflags.h>
63#include <linux/preempt.h> 63#include <linux/preempt.h>
64#include <linux/dim.h>
64#include <uapi/rdma/ib_user_verbs.h> 65#include <uapi/rdma/ib_user_verbs.h>
66#include <rdma/rdma_counter.h>
65#include <rdma/restrack.h> 67#include <rdma/restrack.h>
68#include <rdma/signature.h>
66#include <uapi/rdma/rdma_user_ioctl.h> 69#include <uapi/rdma/rdma_user_ioctl.h>
67#include <uapi/rdma/ib_user_ioctl_verbs.h> 70#include <uapi/rdma/ib_user_ioctl_verbs.h>
68 71
@@ -132,17 +135,6 @@ struct ib_gid_attr {
132 u8 port_num; 135 u8 port_num;
133}; 136};
134 137
135enum rdma_node_type {
136 /* IB values map to NodeInfo:NodeType. */
137 RDMA_NODE_IB_CA = 1,
138 RDMA_NODE_IB_SWITCH,
139 RDMA_NODE_IB_ROUTER,
140 RDMA_NODE_RNIC,
141 RDMA_NODE_USNIC,
142 RDMA_NODE_USNIC_UDP,
143 RDMA_NODE_UNSPECIFIED,
144};
145
146enum { 138enum {
147 /* set the local administered indication */ 139 /* set the local administered indication */
148 IB_SA_WELL_KNOWN_GUID = BIT_ULL(57) | 2, 140 IB_SA_WELL_KNOWN_GUID = BIT_ULL(57) | 2,
@@ -164,7 +156,7 @@ enum rdma_protocol_type {
164}; 156};
165 157
166__attribute_const__ enum rdma_transport_type 158__attribute_const__ enum rdma_transport_type
167rdma_node_get_transport(enum rdma_node_type node_type); 159rdma_node_get_transport(unsigned int node_type);
168 160
169enum rdma_network_type { 161enum rdma_network_type {
170 RDMA_NETWORK_IB, 162 RDMA_NETWORK_IB,
@@ -263,7 +255,7 @@ enum ib_device_cap_flags {
263 */ 255 */
264 IB_DEVICE_CROSS_CHANNEL = (1 << 27), 256 IB_DEVICE_CROSS_CHANNEL = (1 << 27),
265 IB_DEVICE_MANAGED_FLOW_STEERING = (1 << 29), 257 IB_DEVICE_MANAGED_FLOW_STEERING = (1 << 29),
266 IB_DEVICE_SIGNATURE_HANDOVER = (1 << 30), 258 IB_DEVICE_INTEGRITY_HANDOVER = (1 << 30),
267 IB_DEVICE_ON_DEMAND_PAGING = (1ULL << 31), 259 IB_DEVICE_ON_DEMAND_PAGING = (1ULL << 31),
268 IB_DEVICE_SG_GAPS_REG = (1ULL << 32), 260 IB_DEVICE_SG_GAPS_REG = (1ULL << 32),
269 IB_DEVICE_VIRTUAL_FUNCTION = (1ULL << 33), 261 IB_DEVICE_VIRTUAL_FUNCTION = (1ULL << 33),
@@ -275,17 +267,6 @@ enum ib_device_cap_flags {
275 IB_DEVICE_ALLOW_USER_UNREG = (1ULL << 37), 267 IB_DEVICE_ALLOW_USER_UNREG = (1ULL << 37),
276}; 268};
277 269
278enum ib_signature_prot_cap {
279 IB_PROT_T10DIF_TYPE_1 = 1,
280 IB_PROT_T10DIF_TYPE_2 = 1 << 1,
281 IB_PROT_T10DIF_TYPE_3 = 1 << 2,
282};
283
284enum ib_signature_guard_cap {
285 IB_GUARD_T10DIF_CRC = 1,
286 IB_GUARD_T10DIF_CSUM = 1 << 1,
287};
288
289enum ib_atomic_cap { 270enum ib_atomic_cap {
290 IB_ATOMIC_NONE, 271 IB_ATOMIC_NONE,
291 IB_ATOMIC_HCA, 272 IB_ATOMIC_HCA,
@@ -327,8 +308,8 @@ struct ib_rss_caps {
327}; 308};
328 309
329enum ib_tm_cap_flags { 310enum ib_tm_cap_flags {
330 /* Support tag matching on RC transport */ 311 /* Support tag matching with rendezvous offload for RC transport */
331 IB_TM_CAP_RC = 1 << 0, 312 IB_TM_CAP_RNDV_RC = 1 << 0,
332}; 313};
333 314
334struct ib_tm_caps { 315struct ib_tm_caps {
@@ -411,6 +392,7 @@ struct ib_device_attr {
411 int max_srq_wr; 392 int max_srq_wr;
412 int max_srq_sge; 393 int max_srq_sge;
413 unsigned int max_fast_reg_page_list_len; 394 unsigned int max_fast_reg_page_list_len;
395 unsigned int max_pi_fast_reg_page_list_len;
414 u16 max_pkeys; 396 u16 max_pkeys;
415 u8 local_ca_ack_delay; 397 u8 local_ca_ack_delay;
416 int sig_prot_cap; 398 int sig_prot_cap;
@@ -796,118 +778,26 @@ __attribute_const__ int ib_rate_to_mbps(enum ib_rate rate);
796 * enum ib_mr_type - memory region type 778 * enum ib_mr_type - memory region type
797 * @IB_MR_TYPE_MEM_REG: memory region that is used for 779 * @IB_MR_TYPE_MEM_REG: memory region that is used for
798 * normal registration 780 * normal registration
799 * @IB_MR_TYPE_SIGNATURE: memory region that is used for
800 * signature operations (data-integrity
801 * capable regions)
802 * @IB_MR_TYPE_SG_GAPS: memory region that is capable to 781 * @IB_MR_TYPE_SG_GAPS: memory region that is capable to
803 * register any arbitrary sg lists (without 782 * register any arbitrary sg lists (without
804 * the normal mr constraints - see 783 * the normal mr constraints - see
805 * ib_map_mr_sg) 784 * ib_map_mr_sg)
785 * @IB_MR_TYPE_DM: memory region that is used for device
786 * memory registration
787 * @IB_MR_TYPE_USER: memory region that is used for the user-space
788 * application
789 * @IB_MR_TYPE_DMA: memory region that is used for DMA operations
790 * without address translations (VA=PA)
791 * @IB_MR_TYPE_INTEGRITY: memory region that is used for
792 * data integrity operations
806 */ 793 */
807enum ib_mr_type { 794enum ib_mr_type {
808 IB_MR_TYPE_MEM_REG, 795 IB_MR_TYPE_MEM_REG,
809 IB_MR_TYPE_SIGNATURE,
810 IB_MR_TYPE_SG_GAPS, 796 IB_MR_TYPE_SG_GAPS,
811}; 797 IB_MR_TYPE_DM,
812 798 IB_MR_TYPE_USER,
813/** 799 IB_MR_TYPE_DMA,
814 * Signature types 800 IB_MR_TYPE_INTEGRITY,
815 * IB_SIG_TYPE_NONE: Unprotected.
816 * IB_SIG_TYPE_T10_DIF: Type T10-DIF
817 */
818enum ib_signature_type {
819 IB_SIG_TYPE_NONE,
820 IB_SIG_TYPE_T10_DIF,
821};
822
823/**
824 * Signature T10-DIF block-guard types
825 * IB_T10DIF_CRC: Corresponds to T10-PI mandated CRC checksum rules.
826 * IB_T10DIF_CSUM: Corresponds to IP checksum rules.
827 */
828enum ib_t10_dif_bg_type {
829 IB_T10DIF_CRC,
830 IB_T10DIF_CSUM
831};
832
833/**
834 * struct ib_t10_dif_domain - Parameters specific for T10-DIF
835 * domain.
836 * @bg_type: T10-DIF block guard type (CRC|CSUM)
837 * @pi_interval: protection information interval.
838 * @bg: seed of guard computation.
839 * @app_tag: application tag of guard block
840 * @ref_tag: initial guard block reference tag.
841 * @ref_remap: Indicate wethear the reftag increments each block
842 * @app_escape: Indicate to skip block check if apptag=0xffff
843 * @ref_escape: Indicate to skip block check if reftag=0xffffffff
844 * @apptag_check_mask: check bitmask of application tag.
845 */
846struct ib_t10_dif_domain {
847 enum ib_t10_dif_bg_type bg_type;
848 u16 pi_interval;
849 u16 bg;
850 u16 app_tag;
851 u32 ref_tag;
852 bool ref_remap;
853 bool app_escape;
854 bool ref_escape;
855 u16 apptag_check_mask;
856};
857
858/**
859 * struct ib_sig_domain - Parameters for signature domain
860 * @sig_type: specific signauture type
861 * @sig: union of all signature domain attributes that may
862 * be used to set domain layout.
863 */
864struct ib_sig_domain {
865 enum ib_signature_type sig_type;
866 union {
867 struct ib_t10_dif_domain dif;
868 } sig;
869};
870
871/**
872 * struct ib_sig_attrs - Parameters for signature handover operation
873 * @check_mask: bitmask for signature byte check (8 bytes)
874 * @mem: memory domain layout desciptor.
875 * @wire: wire domain layout desciptor.
876 */
877struct ib_sig_attrs {
878 u8 check_mask;
879 struct ib_sig_domain mem;
880 struct ib_sig_domain wire;
881};
882
883enum ib_sig_err_type {
884 IB_SIG_BAD_GUARD,
885 IB_SIG_BAD_REFTAG,
886 IB_SIG_BAD_APPTAG,
887};
888
889/**
890 * Signature check masks (8 bytes in total) according to the T10-PI standard:
891 * -------- -------- ------------
892 * | GUARD | APPTAG | REFTAG |
893 * | 2B | 2B | 4B |
894 * -------- -------- ------------
895 */
896enum {
897 IB_SIG_CHECK_GUARD = 0xc0,
898 IB_SIG_CHECK_APPTAG = 0x30,
899 IB_SIG_CHECK_REFTAG = 0x0f,
900};
901
902/**
903 * struct ib_sig_err - signature error descriptor
904 */
905struct ib_sig_err {
906 enum ib_sig_err_type err_type;
907 u32 expected;
908 u32 actual;
909 u64 sig_err_offset;
910 u32 key;
911}; 801};
912 802
913enum ib_mr_status_check { 803enum ib_mr_status_check {
@@ -1164,7 +1054,7 @@ enum ib_qp_create_flags {
1164 IB_QP_CREATE_MANAGED_SEND = 1 << 3, 1054 IB_QP_CREATE_MANAGED_SEND = 1 << 3,
1165 IB_QP_CREATE_MANAGED_RECV = 1 << 4, 1055 IB_QP_CREATE_MANAGED_RECV = 1 << 4,
1166 IB_QP_CREATE_NETIF_QP = 1 << 5, 1056 IB_QP_CREATE_NETIF_QP = 1 << 5,
1167 IB_QP_CREATE_SIGNATURE_EN = 1 << 6, 1057 IB_QP_CREATE_INTEGRITY_EN = 1 << 6,
1168 /* FREE = 1 << 7, */ 1058 /* FREE = 1 << 7, */
1169 IB_QP_CREATE_SCATTER_FCS = 1 << 8, 1059 IB_QP_CREATE_SCATTER_FCS = 1 << 8,
1170 IB_QP_CREATE_CVLAN_STRIPPING = 1 << 9, 1060 IB_QP_CREATE_CVLAN_STRIPPING = 1 << 9,
@@ -1343,7 +1233,7 @@ enum ib_wr_opcode {
1343 1233
1344 /* These are kernel only and can not be issued by userspace */ 1234 /* These are kernel only and can not be issued by userspace */
1345 IB_WR_REG_MR = 0x20, 1235 IB_WR_REG_MR = 0x20,
1346 IB_WR_REG_SIG_MR, 1236 IB_WR_REG_MR_INTEGRITY,
1347 1237
1348 /* reserve values for low level drivers' internal use. 1238 /* reserve values for low level drivers' internal use.
1349 * These values will not be used at all in the ib core layer. 1239 * These values will not be used at all in the ib core layer.
@@ -1453,20 +1343,6 @@ static inline const struct ib_reg_wr *reg_wr(const struct ib_send_wr *wr)
1453 return container_of(wr, struct ib_reg_wr, wr); 1343 return container_of(wr, struct ib_reg_wr, wr);
1454} 1344}
1455 1345
1456struct ib_sig_handover_wr {
1457 struct ib_send_wr wr;
1458 struct ib_sig_attrs *sig_attrs;
1459 struct ib_mr *sig_mr;
1460 int access_flags;
1461 struct ib_sge *prot;
1462};
1463
1464static inline const struct ib_sig_handover_wr *
1465sig_handover_wr(const struct ib_send_wr *wr)
1466{
1467 return container_of(wr, struct ib_sig_handover_wr, wr);
1468}
1469
1470struct ib_recv_wr { 1346struct ib_recv_wr {
1471 struct ib_recv_wr *next; 1347 struct ib_recv_wr *next;
1472 union { 1348 union {
@@ -1634,6 +1510,7 @@ struct ib_cq {
1634 struct work_struct work; 1510 struct work_struct work;
1635 }; 1511 };
1636 struct workqueue_struct *comp_wq; 1512 struct workqueue_struct *comp_wq;
1513 struct dim *dim;
1637 /* 1514 /*
1638 * Implementation details of the RDMA core, don't use in drivers: 1515 * Implementation details of the RDMA core, don't use in drivers:
1639 */ 1516 */
@@ -1818,10 +1695,14 @@ struct ib_qp {
1818 struct ib_qp_security *qp_sec; 1695 struct ib_qp_security *qp_sec;
1819 u8 port; 1696 u8 port;
1820 1697
1698 bool integrity_en;
1821 /* 1699 /*
1822 * Implementation details of the RDMA core, don't use in drivers: 1700 * Implementation details of the RDMA core, don't use in drivers:
1823 */ 1701 */
1824 struct rdma_restrack_entry res; 1702 struct rdma_restrack_entry res;
1703
1704 /* The counter the qp is bind to */
1705 struct rdma_counter *counter;
1825}; 1706};
1826 1707
1827struct ib_dm { 1708struct ib_dm {
@@ -1840,6 +1721,7 @@ struct ib_mr {
1840 u64 iova; 1721 u64 iova;
1841 u64 length; 1722 u64 length;
1842 unsigned int page_size; 1723 unsigned int page_size;
1724 enum ib_mr_type type;
1843 bool need_inval; 1725 bool need_inval;
1844 union { 1726 union {
1845 struct ib_uobject *uobject; /* user */ 1727 struct ib_uobject *uobject; /* user */
@@ -1847,7 +1729,7 @@ struct ib_mr {
1847 }; 1729 };
1848 1730
1849 struct ib_dm *dm; 1731 struct ib_dm *dm;
1850 1732 struct ib_sig_attrs *sig_attrs; /* only for IB_MR_TYPE_INTEGRITY MRs */
1851 /* 1733 /*
1852 * Implementation details of the RDMA core, don't use in drivers: 1734 * Implementation details of the RDMA core, don't use in drivers:
1853 */ 1735 */
@@ -2243,6 +2125,8 @@ struct ib_port_data {
2243 spinlock_t netdev_lock; 2125 spinlock_t netdev_lock;
2244 struct net_device __rcu *netdev; 2126 struct net_device __rcu *netdev;
2245 struct hlist_node ndev_hash_link; 2127 struct hlist_node ndev_hash_link;
2128 struct rdma_port_counter port_counter;
2129 struct rdma_hw_stats *hw_stats;
2246}; 2130};
2247 2131
2248/* rdma netdev type - specifies protocol type */ 2132/* rdma netdev type - specifies protocol type */
@@ -2329,6 +2213,11 @@ struct iw_cm_conn_param;
2329 * need to define the supported operations, otherwise they will be set to null. 2213 * need to define the supported operations, otherwise they will be set to null.
2330 */ 2214 */
2331struct ib_device_ops { 2215struct ib_device_ops {
2216 struct module *owner;
2217 enum rdma_driver_id driver_id;
2218 u32 uverbs_abi_ver;
2219 unsigned int uverbs_no_driver_id_binding:1;
2220
2332 int (*post_send)(struct ib_qp *qp, const struct ib_send_wr *send_wr, 2221 int (*post_send)(struct ib_qp *qp, const struct ib_send_wr *send_wr,
2333 const struct ib_send_wr **bad_send_wr); 2222 const struct ib_send_wr **bad_send_wr);
2334 int (*post_recv)(struct ib_qp *qp, const struct ib_recv_wr *recv_wr, 2223 int (*post_recv)(struct ib_qp *qp, const struct ib_recv_wr *recv_wr,
@@ -2454,11 +2343,10 @@ struct ib_device_ops {
2454 int (*query_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr, 2343 int (*query_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
2455 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr); 2344 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);
2456 int (*destroy_qp)(struct ib_qp *qp, struct ib_udata *udata); 2345 int (*destroy_qp)(struct ib_qp *qp, struct ib_udata *udata);
2457 struct ib_cq *(*create_cq)(struct ib_device *device, 2346 int (*create_cq)(struct ib_cq *cq, const struct ib_cq_init_attr *attr,
2458 const struct ib_cq_init_attr *attr, 2347 struct ib_udata *udata);
2459 struct ib_udata *udata);
2460 int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period); 2348 int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period);
2461 int (*destroy_cq)(struct ib_cq *cq, struct ib_udata *udata); 2349 void (*destroy_cq)(struct ib_cq *cq, struct ib_udata *udata);
2462 int (*resize_cq)(struct ib_cq *cq, int cqe, struct ib_udata *udata); 2350 int (*resize_cq)(struct ib_cq *cq, int cqe, struct ib_udata *udata);
2463 struct ib_mr *(*get_dma_mr)(struct ib_pd *pd, int mr_access_flags); 2351 struct ib_mr *(*get_dma_mr)(struct ib_pd *pd, int mr_access_flags);
2464 struct ib_mr *(*reg_user_mr)(struct ib_pd *pd, u64 start, u64 length, 2352 struct ib_mr *(*reg_user_mr)(struct ib_pd *pd, u64 start, u64 length,
@@ -2470,6 +2358,9 @@ struct ib_device_ops {
2470 int (*dereg_mr)(struct ib_mr *mr, struct ib_udata *udata); 2358 int (*dereg_mr)(struct ib_mr *mr, struct ib_udata *udata);
2471 struct ib_mr *(*alloc_mr)(struct ib_pd *pd, enum ib_mr_type mr_type, 2359 struct ib_mr *(*alloc_mr)(struct ib_pd *pd, enum ib_mr_type mr_type,
2472 u32 max_num_sg, struct ib_udata *udata); 2360 u32 max_num_sg, struct ib_udata *udata);
2361 struct ib_mr *(*alloc_mr_integrity)(struct ib_pd *pd,
2362 u32 max_num_data_sg,
2363 u32 max_num_meta_sg);
2473 int (*advise_mr)(struct ib_pd *pd, 2364 int (*advise_mr)(struct ib_pd *pd,
2474 enum ib_uverbs_advise_mr_advice advice, u32 flags, 2365 enum ib_uverbs_advise_mr_advice advice, u32 flags,
2475 struct ib_sge *sg_list, u32 num_sge, 2366 struct ib_sge *sg_list, u32 num_sge,
@@ -2516,7 +2407,7 @@ struct ib_device_ops {
2516 struct ib_wq *(*create_wq)(struct ib_pd *pd, 2407 struct ib_wq *(*create_wq)(struct ib_pd *pd,
2517 struct ib_wq_init_attr *init_attr, 2408 struct ib_wq_init_attr *init_attr,
2518 struct ib_udata *udata); 2409 struct ib_udata *udata);
2519 int (*destroy_wq)(struct ib_wq *wq, struct ib_udata *udata); 2410 void (*destroy_wq)(struct ib_wq *wq, struct ib_udata *udata);
2520 int (*modify_wq)(struct ib_wq *wq, struct ib_wq_attr *attr, 2411 int (*modify_wq)(struct ib_wq *wq, struct ib_wq_attr *attr,
2521 u32 wq_attr_mask, struct ib_udata *udata); 2412 u32 wq_attr_mask, struct ib_udata *udata);
2522 struct ib_rwq_ind_table *(*create_rwq_ind_table)( 2413 struct ib_rwq_ind_table *(*create_rwq_ind_table)(
@@ -2538,6 +2429,11 @@ struct ib_device_ops {
2538 int (*read_counters)(struct ib_counters *counters, 2429 int (*read_counters)(struct ib_counters *counters,
2539 struct ib_counters_read_attr *counters_read_attr, 2430 struct ib_counters_read_attr *counters_read_attr,
2540 struct uverbs_attr_bundle *attrs); 2431 struct uverbs_attr_bundle *attrs);
2432 int (*map_mr_sg_pi)(struct ib_mr *mr, struct scatterlist *data_sg,
2433 int data_sg_nents, unsigned int *data_sg_offset,
2434 struct scatterlist *meta_sg, int meta_sg_nents,
2435 unsigned int *meta_sg_offset);
2436
2541 /** 2437 /**
2542 * alloc_hw_stats - Allocate a struct rdma_hw_stats and fill in the 2438 * alloc_hw_stats - Allocate a struct rdma_hw_stats and fill in the
2543 * driver initialized data. The struct is kfree()'ed by the sysfs 2439 * driver initialized data. The struct is kfree()'ed by the sysfs
@@ -2595,8 +2491,34 @@ struct ib_device_ops {
2595 u8 pdata_len); 2491 u8 pdata_len);
2596 int (*iw_create_listen)(struct iw_cm_id *cm_id, int backlog); 2492 int (*iw_create_listen)(struct iw_cm_id *cm_id, int backlog);
2597 int (*iw_destroy_listen)(struct iw_cm_id *cm_id); 2493 int (*iw_destroy_listen)(struct iw_cm_id *cm_id);
2494 /**
2495 * counter_bind_qp - Bind a QP to a counter.
2496 * @counter - The counter to be bound. If counter->id is zero then
2497 * the driver needs to allocate a new counter and set counter->id
2498 */
2499 int (*counter_bind_qp)(struct rdma_counter *counter, struct ib_qp *qp);
2500 /**
2501 * counter_unbind_qp - Unbind the qp from the dynamically-allocated
2502 * counter and bind it onto the default one
2503 */
2504 int (*counter_unbind_qp)(struct ib_qp *qp);
2505 /**
2506 * counter_dealloc -De-allocate the hw counter
2507 */
2508 int (*counter_dealloc)(struct rdma_counter *counter);
2509 /**
2510 * counter_alloc_stats - Allocate a struct rdma_hw_stats and fill in
2511 * the driver initialized data.
2512 */
2513 struct rdma_hw_stats *(*counter_alloc_stats)(
2514 struct rdma_counter *counter);
2515 /**
2516 * counter_update_stats - Query the stats value of this counter
2517 */
2518 int (*counter_update_stats)(struct rdma_counter *counter);
2598 2519
2599 DECLARE_RDMA_OBJ_SIZE(ib_ah); 2520 DECLARE_RDMA_OBJ_SIZE(ib_ah);
2521 DECLARE_RDMA_OBJ_SIZE(ib_cq);
2600 DECLARE_RDMA_OBJ_SIZE(ib_pd); 2522 DECLARE_RDMA_OBJ_SIZE(ib_pd);
2601 DECLARE_RDMA_OBJ_SIZE(ib_srq); 2523 DECLARE_RDMA_OBJ_SIZE(ib_srq);
2602 DECLARE_RDMA_OBJ_SIZE(ib_ucontext); 2524 DECLARE_RDMA_OBJ_SIZE(ib_ucontext);
@@ -2636,7 +2558,6 @@ struct ib_device {
2636 2558
2637 int num_comp_vectors; 2559 int num_comp_vectors;
2638 2560
2639 struct module *owner;
2640 union { 2561 union {
2641 struct device dev; 2562 struct device dev;
2642 struct ib_core_device coredev; 2563 struct ib_core_device coredev;
@@ -2648,7 +2569,6 @@ struct ib_device {
2648 */ 2569 */
2649 const struct attribute_group *groups[3]; 2570 const struct attribute_group *groups[3];
2650 2571
2651 int uverbs_abi_ver;
2652 u64 uverbs_cmd_mask; 2572 u64 uverbs_cmd_mask;
2653 u64 uverbs_ex_cmd_mask; 2573 u64 uverbs_ex_cmd_mask;
2654 2574
@@ -2658,6 +2578,8 @@ struct ib_device {
2658 u16 is_switch:1; 2578 u16 is_switch:1;
2659 /* Indicates kernel verbs support, should not be used in drivers */ 2579 /* Indicates kernel verbs support, should not be used in drivers */
2660 u16 kverbs_provider:1; 2580 u16 kverbs_provider:1;
2581 /* CQ adaptive moderation (RDMA DIM) */
2582 u16 use_cq_dim:1;
2661 u8 node_type; 2583 u8 node_type;
2662 u8 phys_port_cnt; 2584 u8 phys_port_cnt;
2663 struct ib_device_attr attrs; 2585 struct ib_device_attr attrs;
@@ -2672,7 +2594,6 @@ struct ib_device {
2672 struct rdma_restrack_root *res; 2594 struct rdma_restrack_root *res;
2673 2595
2674 const struct uapi_definition *driver_def; 2596 const struct uapi_definition *driver_def;
2675 enum rdma_driver_id driver_id;
2676 2597
2677 /* 2598 /*
2678 * Positive refcount indicates that the device is currently 2599 * Positive refcount indicates that the device is currently
@@ -2694,11 +2615,15 @@ struct ib_device {
2694 u32 iw_driver_flags; 2615 u32 iw_driver_flags;
2695}; 2616};
2696 2617
2618struct ib_client_nl_info;
2697struct ib_client { 2619struct ib_client {
2698 const char *name; 2620 const char *name;
2699 void (*add) (struct ib_device *); 2621 void (*add) (struct ib_device *);
2700 void (*remove)(struct ib_device *, void *client_data); 2622 void (*remove)(struct ib_device *, void *client_data);
2701 void (*rename)(struct ib_device *dev, void *client_data); 2623 void (*rename)(struct ib_device *dev, void *client_data);
2624 int (*get_nl_info)(struct ib_device *ibdev, void *client_data,
2625 struct ib_client_nl_info *res);
2626 int (*get_global_nl_info)(struct ib_client_nl_info *res);
2702 2627
2703 /* Returns the net_dev belonging to this ib_client and matching the 2628 /* Returns the net_dev belonging to this ib_client and matching the
2704 * given parameters. 2629 * given parameters.
@@ -3859,9 +3784,9 @@ int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata);
3859 * 3784 *
3860 * NOTE: for user cq use ib_destroy_cq_user with valid udata! 3785 * NOTE: for user cq use ib_destroy_cq_user with valid udata!
3861 */ 3786 */
3862static inline int ib_destroy_cq(struct ib_cq *cq) 3787static inline void ib_destroy_cq(struct ib_cq *cq)
3863{ 3788{
3864 return ib_destroy_cq_user(cq, NULL); 3789 ib_destroy_cq_user(cq, NULL);
3865} 3790}
3866 3791
3867/** 3792/**
@@ -4148,6 +4073,10 @@ static inline struct ib_mr *ib_alloc_mr(struct ib_pd *pd,
4148 return ib_alloc_mr_user(pd, mr_type, max_num_sg, NULL); 4073 return ib_alloc_mr_user(pd, mr_type, max_num_sg, NULL);
4149} 4074}
4150 4075
4076struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd,
4077 u32 max_num_data_sg,
4078 u32 max_num_meta_sg);
4079
4151/** 4080/**
4152 * ib_update_fast_reg_key - updates the key portion of the fast_reg MR 4081 * ib_update_fast_reg_key - updates the key portion of the fast_reg MR
4153 * R_Key and L_Key. 4082 * R_Key and L_Key.
@@ -4332,6 +4261,10 @@ int ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table);
4332 4261
4333int ib_map_mr_sg(struct ib_mr *mr, struct scatterlist *sg, int sg_nents, 4262int ib_map_mr_sg(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
4334 unsigned int *sg_offset, unsigned int page_size); 4263 unsigned int *sg_offset, unsigned int page_size);
4264int ib_map_mr_sg_pi(struct ib_mr *mr, struct scatterlist *data_sg,
4265 int data_sg_nents, unsigned int *data_sg_offset,
4266 struct scatterlist *meta_sg, int meta_sg_nents,
4267 unsigned int *meta_sg_offset, unsigned int page_size);
4335 4268
4336static inline int 4269static inline int
4337ib_map_mr_sg_zbva(struct ib_mr *mr, struct scatterlist *sg, int sg_nents, 4270ib_map_mr_sg_zbva(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
diff --git a/include/rdma/mr_pool.h b/include/rdma/mr_pool.h
index 83763ef82354..e77123bcb43b 100644
--- a/include/rdma/mr_pool.h
+++ b/include/rdma/mr_pool.h
@@ -11,7 +11,7 @@ struct ib_mr *ib_mr_pool_get(struct ib_qp *qp, struct list_head *list);
11void ib_mr_pool_put(struct ib_qp *qp, struct list_head *list, struct ib_mr *mr); 11void ib_mr_pool_put(struct ib_qp *qp, struct list_head *list, struct ib_mr *mr);
12 12
13int ib_mr_pool_init(struct ib_qp *qp, struct list_head *list, int nr, 13int ib_mr_pool_init(struct ib_qp *qp, struct list_head *list, int nr,
14 enum ib_mr_type type, u32 max_num_sg); 14 enum ib_mr_type type, u32 max_num_sg, u32 max_num_meta_sg);
15void ib_mr_pool_destroy(struct ib_qp *qp, struct list_head *list); 15void ib_mr_pool_destroy(struct ib_qp *qp, struct list_head *list);
16 16
17#endif /* _RDMA_MR_POOL_H */ 17#endif /* _RDMA_MR_POOL_H */
diff --git a/include/rdma/rdma_counter.h b/include/rdma/rdma_counter.h
new file mode 100644
index 000000000000..eb99856e8b30
--- /dev/null
+++ b/include/rdma/rdma_counter.h
@@ -0,0 +1,65 @@
1/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2/*
3 * Copyright (c) 2019 Mellanox Technologies. All rights reserved.
4 */
5
6#ifndef _RDMA_COUNTER_H_
7#define _RDMA_COUNTER_H_
8
9#include <linux/mutex.h>
10#include <linux/pid_namespace.h>
11
12#include <rdma/restrack.h>
13#include <rdma/rdma_netlink.h>
14
15struct ib_device;
16struct ib_qp;
17
18struct auto_mode_param {
19 int qp_type;
20};
21
22struct rdma_counter_mode {
23 enum rdma_nl_counter_mode mode;
24 enum rdma_nl_counter_mask mask;
25 struct auto_mode_param param;
26};
27
28struct rdma_port_counter {
29 struct rdma_counter_mode mode;
30 struct rdma_hw_stats *hstats;
31 unsigned int num_counters;
32 struct mutex lock;
33};
34
35struct rdma_counter {
36 struct rdma_restrack_entry res;
37 struct ib_device *device;
38 uint32_t id;
39 struct kref kref;
40 struct rdma_counter_mode mode;
41 struct mutex lock;
42 struct rdma_hw_stats *stats;
43 u8 port;
44};
45
46void rdma_counter_init(struct ib_device *dev);
47void rdma_counter_release(struct ib_device *dev);
48int rdma_counter_set_auto_mode(struct ib_device *dev, u8 port,
49 bool on, enum rdma_nl_counter_mask mask);
50int rdma_counter_bind_qp_auto(struct ib_qp *qp, u8 port);
51int rdma_counter_unbind_qp(struct ib_qp *qp, bool force);
52
53int rdma_counter_query_stats(struct rdma_counter *counter);
54u64 rdma_counter_get_hwstat_value(struct ib_device *dev, u8 port, u32 index);
55int rdma_counter_bind_qpn(struct ib_device *dev, u8 port,
56 u32 qp_num, u32 counter_id);
57int rdma_counter_bind_qpn_alloc(struct ib_device *dev, u8 port,
58 u32 qp_num, u32 *counter_id);
59int rdma_counter_unbind_qpn(struct ib_device *dev, u8 port,
60 u32 qp_num, u32 counter_id);
61int rdma_counter_get_mode(struct ib_device *dev, u8 port,
62 enum rdma_nl_counter_mode *mode,
63 enum rdma_nl_counter_mask *mask);
64
65#endif /* _RDMA_COUNTER_H_ */
diff --git a/include/rdma/rdma_netlink.h b/include/rdma/rdma_netlink.h
index 10732ab31ba2..6631624e4d7c 100644
--- a/include/rdma/rdma_netlink.h
+++ b/include/rdma/rdma_netlink.h
@@ -6,6 +6,12 @@
6#include <linux/netlink.h> 6#include <linux/netlink.h>
7#include <uapi/rdma/rdma_netlink.h> 7#include <uapi/rdma/rdma_netlink.h>
8 8
9enum {
10 RDMA_NLDEV_ATTR_EMPTY_STRING = 1,
11 RDMA_NLDEV_ATTR_ENTRY_STRLEN = 16,
12 RDMA_NLDEV_ATTR_CHARDEV_TYPE_SIZE = 32,
13};
14
9struct rdma_nl_cbs { 15struct rdma_nl_cbs {
10 int (*doit)(struct sk_buff *skb, struct nlmsghdr *nlh, 16 int (*doit)(struct sk_buff *skb, struct nlmsghdr *nlh,
11 struct netlink_ext_ack *extack); 17 struct netlink_ext_ack *extack);
@@ -110,4 +116,6 @@ void rdma_link_register(struct rdma_link_ops *ops);
110void rdma_link_unregister(struct rdma_link_ops *ops); 116void rdma_link_unregister(struct rdma_link_ops *ops);
111 117
112#define MODULE_ALIAS_RDMA_LINK(type) MODULE_ALIAS("rdma-link-" type) 118#define MODULE_ALIAS_RDMA_LINK(type) MODULE_ALIAS("rdma-link-" type)
119#define MODULE_ALIAS_RDMA_CLIENT(type) MODULE_ALIAS("rdma-client-" type)
120
113#endif /* _RDMA_NETLINK_H */ 121#endif /* _RDMA_NETLINK_H */
diff --git a/include/rdma/rdma_vt.h b/include/rdma/rdma_vt.h
index b9cd06db1a71..525848e227dc 100644
--- a/include/rdma/rdma_vt.h
+++ b/include/rdma/rdma_vt.h
@@ -2,7 +2,7 @@
2#define DEF_RDMA_VT_H 2#define DEF_RDMA_VT_H
3 3
4/* 4/*
5 * Copyright(c) 2016 - 2018 Intel Corporation. 5 * Copyright(c) 2016 - 2019 Intel Corporation.
6 * 6 *
7 * This file is provided under a dual BSD/GPLv2 license. When using or 7 * This file is provided under a dual BSD/GPLv2 license. When using or
8 * redistributing this file, you may do so under either license. 8 * redistributing this file, you may do so under either license.
@@ -202,7 +202,6 @@ struct rvt_pd {
202struct rvt_ah { 202struct rvt_ah {
203 struct ib_ah ibah; 203 struct ib_ah ibah;
204 struct rdma_ah_attr attr; 204 struct rdma_ah_attr attr;
205 atomic_t refcount;
206 u8 vl; 205 u8 vl;
207 u8 log_pmtu; 206 u8 log_pmtu;
208}; 207};
@@ -555,7 +554,7 @@ static inline u16 rvt_get_pkey(struct rvt_dev_info *rdi,
555 554
556struct rvt_dev_info *rvt_alloc_device(size_t size, int nports); 555struct rvt_dev_info *rvt_alloc_device(size_t size, int nports);
557void rvt_dealloc_device(struct rvt_dev_info *rdi); 556void rvt_dealloc_device(struct rvt_dev_info *rdi);
558int rvt_register_device(struct rvt_dev_info *rvd, u32 driver_id); 557int rvt_register_device(struct rvt_dev_info *rvd);
559void rvt_unregister_device(struct rvt_dev_info *rvd); 558void rvt_unregister_device(struct rvt_dev_info *rvd);
560int rvt_check_ah(struct ib_device *ibdev, struct rdma_ah_attr *ah_attr); 559int rvt_check_ah(struct ib_device *ibdev, struct rdma_ah_attr *ah_attr);
561int rvt_init_port(struct rvt_dev_info *rdi, struct rvt_ibport *port, 560int rvt_init_port(struct rvt_dev_info *rdi, struct rvt_ibport *port,
diff --git a/include/rdma/rdmavt_cq.h b/include/rdma/rdmavt_cq.h
index 75dc65c0bfb8..04c519ef6d71 100644
--- a/include/rdma/rdmavt_cq.h
+++ b/include/rdma/rdmavt_cq.h
@@ -61,18 +61,27 @@
61#define RVT_CQ_NONE (IB_CQ_NEXT_COMP + 1) 61#define RVT_CQ_NONE (IB_CQ_NEXT_COMP + 1)
62 62
63/* 63/*
64 * Define read macro that apply smp_load_acquire memory barrier
65 * when reading indice of circular buffer that mmaped to user space.
66 */
67#define RDMA_READ_UAPI_ATOMIC(member) smp_load_acquire(&(member).val)
68
69/*
70 * Define write macro that uses smp_store_release memory barrier
71 * when writing indice of circular buffer that mmaped to user space.
72 */
73#define RDMA_WRITE_UAPI_ATOMIC(member, x) smp_store_release(&(member).val, x)
74#include <rdma/rvt-abi.h>
75
76/*
64 * This structure is used to contain the head pointer, tail pointer, 77 * This structure is used to contain the head pointer, tail pointer,
65 * and completion queue entries as a single memory allocation so 78 * and completion queue entries as a single memory allocation so
66 * it can be mmap'ed into user space. 79 * it can be mmap'ed into user space.
67 */ 80 */
68struct rvt_cq_wc { 81struct rvt_k_cq_wc {
69 u32 head; /* index of next entry to fill */ 82 u32 head; /* index of next entry to fill */
70 u32 tail; /* index of next ib_poll_cq() entry */ 83 u32 tail; /* index of next ib_poll_cq() entry */
71 union { 84 struct ib_wc kqueue[];
72 /* these are actually size ibcq.cqe + 1 */
73 struct ib_uverbs_wc uqueue[0];
74 struct ib_wc kqueue[0];
75 };
76}; 85};
77 86
78/* 87/*
@@ -84,10 +93,12 @@ struct rvt_cq {
84 spinlock_t lock; /* protect changes in this struct */ 93 spinlock_t lock; /* protect changes in this struct */
85 u8 notify; 94 u8 notify;
86 u8 triggered; 95 u8 triggered;
96 u8 cq_full;
87 int comp_vector_cpu; 97 int comp_vector_cpu;
88 struct rvt_dev_info *rdi; 98 struct rvt_dev_info *rdi;
89 struct rvt_cq_wc *queue; 99 struct rvt_cq_wc *queue;
90 struct rvt_mmap_info *ip; 100 struct rvt_mmap_info *ip;
101 struct rvt_k_cq_wc *kqueue;
91}; 102};
92 103
93static inline struct rvt_cq *ibcq_to_rvtcq(struct ib_cq *ibcq) 104static inline struct rvt_cq *ibcq_to_rvtcq(struct ib_cq *ibcq)
@@ -95,6 +106,6 @@ static inline struct rvt_cq *ibcq_to_rvtcq(struct ib_cq *ibcq)
95 return container_of(ibcq, struct rvt_cq, ibcq); 106 return container_of(ibcq, struct rvt_cq, ibcq);
96} 107}
97 108
98void rvt_cq_enter(struct rvt_cq *cq, struct ib_wc *entry, bool solicited); 109bool rvt_cq_enter(struct rvt_cq *cq, struct ib_wc *entry, bool solicited);
99 110
100#endif /* DEF_RDMAVT_INCCQH */ 111#endif /* DEF_RDMAVT_INCCQH */
diff --git a/include/rdma/rdmavt_qp.h b/include/rdma/rdmavt_qp.h
index 68e38c20afc0..0eeea520a853 100644
--- a/include/rdma/rdmavt_qp.h
+++ b/include/rdma/rdmavt_qp.h
@@ -2,7 +2,7 @@
2#define DEF_RDMAVT_INCQP_H 2#define DEF_RDMAVT_INCQP_H
3 3
4/* 4/*
5 * Copyright(c) 2016 - 2018 Intel Corporation. 5 * Copyright(c) 2016 - 2019 Intel Corporation.
6 * 6 *
7 * This file is provided under a dual BSD/GPLv2 license. When using or 7 * This file is provided under a dual BSD/GPLv2 license. When using or
8 * redistributing this file, you may do so under either license. 8 * redistributing this file, you may do so under either license.
@@ -52,6 +52,7 @@
52#include <rdma/ib_pack.h> 52#include <rdma/ib_pack.h>
53#include <rdma/ib_verbs.h> 53#include <rdma/ib_verbs.h>
54#include <rdma/rdmavt_cq.h> 54#include <rdma/rdmavt_cq.h>
55#include <rdma/rvt-abi.h>
55/* 56/*
56 * Atomic bit definitions for r_aflags. 57 * Atomic bit definitions for r_aflags.
57 */ 58 */
@@ -156,6 +157,22 @@
156#define RVT_SEND_RESERVE_USED IB_SEND_RESERVED_START 157#define RVT_SEND_RESERVE_USED IB_SEND_RESERVED_START
157#define RVT_SEND_COMPLETION_ONLY (IB_SEND_RESERVED_START << 1) 158#define RVT_SEND_COMPLETION_ONLY (IB_SEND_RESERVED_START << 1)
158 159
160/**
161 * rvt_ud_wr - IB UD work plus AH cache
162 * @wr: valid IB work request
163 * @attr: pointer to an allocated AH attribute
164 *
165 * Special case the UD WR so we can keep track of the AH attributes.
166 *
167 * NOTE: This data structure is stricly ordered wr then attr. I.e the attr
168 * MUST come after wr. The ib_ud_wr is sized and copied in rvt_post_one_wr.
169 * The copy assumes that wr is first.
170 */
171struct rvt_ud_wr {
172 struct ib_ud_wr wr;
173 struct rdma_ah_attr *attr;
174};
175
159/* 176/*
160 * Send work request queue entry. 177 * Send work request queue entry.
161 * The size of the sg_list is determined when the QP is created and stored 178 * The size of the sg_list is determined when the QP is created and stored
@@ -164,7 +181,7 @@
164struct rvt_swqe { 181struct rvt_swqe {
165 union { 182 union {
166 struct ib_send_wr wr; /* don't use wr.sg_list */ 183 struct ib_send_wr wr; /* don't use wr.sg_list */
167 struct ib_ud_wr ud_wr; 184 struct rvt_ud_wr ud_wr;
168 struct ib_reg_wr reg_wr; 185 struct ib_reg_wr reg_wr;
169 struct ib_rdma_wr rdma_wr; 186 struct ib_rdma_wr rdma_wr;
170 struct ib_atomic_wr atomic_wr; 187 struct ib_atomic_wr atomic_wr;
@@ -177,33 +194,84 @@ struct rvt_swqe {
177 struct rvt_sge sg_list[0]; 194 struct rvt_sge sg_list[0];
178}; 195};
179 196
180/* 197/**
181 * Receive work request queue entry. 198 * struct rvt_krwq - kernel struct receive work request
182 * The size of the sg_list is determined when the QP (or SRQ) is created 199 * @p_lock: lock to protect producer of the kernel buffer
183 * and stored in qp->r_rq.max_sge (or srq->rq.max_sge). 200 * @head: index of next entry to fill
201 * @c_lock:lock to protect consumer of the kernel buffer
202 * @tail: index of next entry to pull
203 * @count: count is aproximate of total receive enteries posted
204 * @rvt_rwqe: struct of receive work request queue entry
205 *
206 * This structure is used to contain the head pointer,
207 * tail pointer and receive work queue entries for kernel
208 * mode user.
184 */ 209 */
185struct rvt_rwqe { 210struct rvt_krwq {
186 u64 wr_id; 211 spinlock_t p_lock; /* protect producer */
187 u8 num_sge;
188 struct ib_sge sg_list[0];
189};
190
191/*
192 * This structure is used to contain the head pointer, tail pointer,
193 * and receive work queue entries as a single memory allocation so
194 * it can be mmap'ed into user space.
195 * Note that the wq array elements are variable size so you can't
196 * just index into the array to get the N'th element;
197 * use get_rwqe_ptr() instead.
198 */
199struct rvt_rwq {
200 u32 head; /* new work requests posted to the head */ 212 u32 head; /* new work requests posted to the head */
213
214 /* protect consumer */
215 spinlock_t c_lock ____cacheline_aligned_in_smp;
201 u32 tail; /* receives pull requests from here. */ 216 u32 tail; /* receives pull requests from here. */
202 struct rvt_rwqe wq[0]; 217 u32 count; /* approx count of receive entries posted */
218 struct rvt_rwqe *curr_wq;
219 struct rvt_rwqe wq[];
203}; 220};
204 221
222/*
223 * rvt_get_swqe_ah - Return the pointer to the struct rvt_ah
224 * @swqe: valid Send WQE
225 *
226 */
227static inline struct rvt_ah *rvt_get_swqe_ah(struct rvt_swqe *swqe)
228{
229 return ibah_to_rvtah(swqe->ud_wr.wr.ah);
230}
231
232/**
233 * rvt_get_swqe_ah_attr - Return the cached ah attribute information
234 * @swqe: valid Send WQE
235 *
236 */
237static inline struct rdma_ah_attr *rvt_get_swqe_ah_attr(struct rvt_swqe *swqe)
238{
239 return swqe->ud_wr.attr;
240}
241
242/**
243 * rvt_get_swqe_remote_qpn - Access the remote QPN value
244 * @swqe: valid Send WQE
245 *
246 */
247static inline u32 rvt_get_swqe_remote_qpn(struct rvt_swqe *swqe)
248{
249 return swqe->ud_wr.wr.remote_qpn;
250}
251
252/**
253 * rvt_get_swqe_remote_qkey - Acces the remote qkey value
254 * @swqe: valid Send WQE
255 *
256 */
257static inline u32 rvt_get_swqe_remote_qkey(struct rvt_swqe *swqe)
258{
259 return swqe->ud_wr.wr.remote_qkey;
260}
261
262/**
263 * rvt_get_swqe_pkey_index - Access the pkey index
264 * @swqe: valid Send WQE
265 *
266 */
267static inline u16 rvt_get_swqe_pkey_index(struct rvt_swqe *swqe)
268{
269 return swqe->ud_wr.wr.pkey_index;
270}
271
205struct rvt_rq { 272struct rvt_rq {
206 struct rvt_rwq *wq; 273 struct rvt_rwq *wq;
274 struct rvt_krwq *kwq;
207 u32 size; /* size of RWQE array */ 275 u32 size; /* size of RWQE array */
208 u8 max_sge; 276 u8 max_sge;
209 /* protect changes in this struct */ 277 /* protect changes in this struct */
@@ -472,7 +540,7 @@ static inline struct rvt_swqe *rvt_get_swqe_ptr(struct rvt_qp *qp,
472static inline struct rvt_rwqe *rvt_get_rwqe_ptr(struct rvt_rq *rq, unsigned n) 540static inline struct rvt_rwqe *rvt_get_rwqe_ptr(struct rvt_rq *rq, unsigned n)
473{ 541{
474 return (struct rvt_rwqe *) 542 return (struct rvt_rwqe *)
475 ((char *)rq->wq->wq + 543 ((char *)rq->kwq->curr_wq +
476 (sizeof(struct rvt_rwqe) + 544 (sizeof(struct rvt_rwqe) +
477 rq->max_sge * sizeof(struct ib_sge)) * n); 545 rq->max_sge * sizeof(struct ib_sge)) * n);
478} 546}
@@ -565,42 +633,6 @@ static inline void rvt_qp_wqe_unreserve(
565 633
566extern const enum ib_wc_opcode ib_rvt_wc_opcode[]; 634extern const enum ib_wc_opcode ib_rvt_wc_opcode[];
567 635
568/**
569 * rvt_qp_swqe_complete() - insert send completion
570 * @qp - the qp
571 * @wqe - the send wqe
572 * @status - completion status
573 *
574 * Insert a send completion into the completion
575 * queue if the qp indicates it should be done.
576 *
577 * See IBTA 10.7.3.1 for info on completion
578 * control.
579 */
580static inline void rvt_qp_swqe_complete(
581 struct rvt_qp *qp,
582 struct rvt_swqe *wqe,
583 enum ib_wc_opcode opcode,
584 enum ib_wc_status status)
585{
586 if (unlikely(wqe->wr.send_flags & RVT_SEND_RESERVE_USED))
587 return;
588 if (!(qp->s_flags & RVT_S_SIGNAL_REQ_WR) ||
589 (wqe->wr.send_flags & IB_SEND_SIGNALED) ||
590 status != IB_WC_SUCCESS) {
591 struct ib_wc wc;
592
593 memset(&wc, 0, sizeof(wc));
594 wc.wr_id = wqe->wr.wr_id;
595 wc.status = status;
596 wc.opcode = opcode;
597 wc.qp = &qp->ibqp;
598 wc.byte_len = wqe->length;
599 rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.send_cq), &wc,
600 status != IB_WC_SUCCESS);
601 }
602}
603
604/* 636/*
605 * Compare the lower 24 bits of the msn values. 637 * Compare the lower 24 bits of the msn values.
606 * Returns an integer <, ==, or > than zero. 638 * Returns an integer <, ==, or > than zero.
@@ -734,7 +766,119 @@ static inline void rvt_put_qp_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe)
734{ 766{
735 rvt_put_swqe(wqe); 767 rvt_put_swqe(wqe);
736 if (qp->allowed_ops == IB_OPCODE_UD) 768 if (qp->allowed_ops == IB_OPCODE_UD)
737 atomic_dec(&ibah_to_rvtah(wqe->ud_wr.ah)->refcount); 769 rdma_destroy_ah_attr(wqe->ud_wr.attr);
770}
771
772/**
773 * rvt_qp_sqwe_incr - increment ring index
774 * @qp: the qp
775 * @val: the starting value
776 *
777 * Return: the new value wrapping as appropriate
778 */
779static inline u32
780rvt_qp_swqe_incr(struct rvt_qp *qp, u32 val)
781{
782 if (++val >= qp->s_size)
783 val = 0;
784 return val;
785}
786
787int rvt_error_qp(struct rvt_qp *qp, enum ib_wc_status err);
788
789/**
790 * rvt_recv_cq - add a new entry to completion queue
791 * by receive queue
792 * @qp: receive queue
793 * @wc: work completion entry to add
794 * @solicited: true if @entry is solicited
795 *
796 * This is wrapper function for rvt_enter_cq function call by
797 * receive queue. If rvt_cq_enter return false, it means cq is
798 * full and the qp is put into error state.
799 */
800static inline void rvt_recv_cq(struct rvt_qp *qp, struct ib_wc *wc,
801 bool solicited)
802{
803 struct rvt_cq *cq = ibcq_to_rvtcq(qp->ibqp.recv_cq);
804
805 if (unlikely(!rvt_cq_enter(cq, wc, solicited)))
806 rvt_error_qp(qp, IB_WC_LOC_QP_OP_ERR);
807}
808
809/**
810 * rvt_send_cq - add a new entry to completion queue
811 * by send queue
812 * @qp: send queue
813 * @wc: work completion entry to add
814 * @solicited: true if @entry is solicited
815 *
816 * This is wrapper function for rvt_enter_cq function call by
817 * send queue. If rvt_cq_enter return false, it means cq is
818 * full and the qp is put into error state.
819 */
820static inline void rvt_send_cq(struct rvt_qp *qp, struct ib_wc *wc,
821 bool solicited)
822{
823 struct rvt_cq *cq = ibcq_to_rvtcq(qp->ibqp.send_cq);
824
825 if (unlikely(!rvt_cq_enter(cq, wc, solicited)))
826 rvt_error_qp(qp, IB_WC_LOC_QP_OP_ERR);
827}
828
829/**
830 * rvt_qp_complete_swqe - insert send completion
831 * @qp - the qp
832 * @wqe - the send wqe
833 * @opcode - wc operation (driver dependent)
834 * @status - completion status
835 *
836 * Update the s_last information, and then insert a send
837 * completion into the completion
838 * queue if the qp indicates it should be done.
839 *
840 * See IBTA 10.7.3.1 for info on completion
841 * control.
842 *
843 * Return: new last
844 */
845static inline u32
846rvt_qp_complete_swqe(struct rvt_qp *qp,
847 struct rvt_swqe *wqe,
848 enum ib_wc_opcode opcode,
849 enum ib_wc_status status)
850{
851 bool need_completion;
852 u64 wr_id;
853 u32 byte_len, last;
854 int flags = wqe->wr.send_flags;
855
856 rvt_put_qp_swqe(qp, wqe);
857
858 need_completion =
859 !(flags & RVT_SEND_RESERVE_USED) &&
860 (!(qp->s_flags & RVT_S_SIGNAL_REQ_WR) ||
861 (flags & IB_SEND_SIGNALED) ||
862 status != IB_WC_SUCCESS);
863 if (need_completion) {
864 wr_id = wqe->wr.wr_id;
865 byte_len = wqe->length;
866 /* above fields required before writing s_last */
867 }
868 last = rvt_qp_swqe_incr(qp, qp->s_last);
869 /* see rvt_qp_is_avail() */
870 smp_store_release(&qp->s_last, last);
871 if (need_completion) {
872 struct ib_wc w = {
873 .wr_id = wr_id,
874 .status = status,
875 .opcode = opcode,
876 .qp = &qp->ibqp,
877 .byte_len = byte_len,
878 };
879 rvt_send_cq(qp, &w, status != IB_WC_SUCCESS);
880 }
881 return last;
738} 882}
739 883
740extern const int ib_rvt_state_ops[]; 884extern const int ib_rvt_state_ops[];
@@ -742,7 +886,6 @@ extern const int ib_rvt_state_ops[];
742struct rvt_dev_info; 886struct rvt_dev_info;
743int rvt_get_rwqe(struct rvt_qp *qp, bool wr_id_only); 887int rvt_get_rwqe(struct rvt_qp *qp, bool wr_id_only);
744void rvt_comm_est(struct rvt_qp *qp); 888void rvt_comm_est(struct rvt_qp *qp);
745int rvt_error_qp(struct rvt_qp *qp, enum ib_wc_status err);
746void rvt_rc_error(struct rvt_qp *qp, enum ib_wc_status err); 889void rvt_rc_error(struct rvt_qp *qp, enum ib_wc_status err);
747unsigned long rvt_rnr_tbl_to_usec(u32 index); 890unsigned long rvt_rnr_tbl_to_usec(u32 index);
748enum hrtimer_restart rvt_rc_rnr_retry(struct hrtimer *t); 891enum hrtimer_restart rvt_rc_rnr_retry(struct hrtimer *t);
@@ -784,6 +927,53 @@ struct rvt_qp_iter {
784 int n; 927 int n;
785}; 928};
786 929
930/**
931 * ib_cq_tail - Return tail index of cq buffer
932 * @send_cq - The cq for send
933 *
934 * This is called in qp_iter_print to get tail
935 * of cq buffer.
936 */
937static inline u32 ib_cq_tail(struct ib_cq *send_cq)
938{
939 struct rvt_cq *cq = ibcq_to_rvtcq(send_cq);
940
941 return ibcq_to_rvtcq(send_cq)->ip ?
942 RDMA_READ_UAPI_ATOMIC(cq->queue->tail) :
943 ibcq_to_rvtcq(send_cq)->kqueue->tail;
944}
945
946/**
947 * ib_cq_head - Return head index of cq buffer
948 * @send_cq - The cq for send
949 *
950 * This is called in qp_iter_print to get head
951 * of cq buffer.
952 */
953static inline u32 ib_cq_head(struct ib_cq *send_cq)
954{
955 struct rvt_cq *cq = ibcq_to_rvtcq(send_cq);
956
957 return ibcq_to_rvtcq(send_cq)->ip ?
958 RDMA_READ_UAPI_ATOMIC(cq->queue->head) :
959 ibcq_to_rvtcq(send_cq)->kqueue->head;
960}
961
962/**
963 * rvt_free_rq - free memory allocated for rvt_rq struct
964 * @rvt_rq: request queue data structure
965 *
966 * This function should only be called if the rvt_mmap_info()
967 * has not succeeded.
968 */
969static inline void rvt_free_rq(struct rvt_rq *rq)
970{
971 kvfree(rq->kwq);
972 rq->kwq = NULL;
973 vfree(rq->wq);
974 rq->wq = NULL;
975}
976
787struct rvt_qp_iter *rvt_qp_iter_init(struct rvt_dev_info *rdi, 977struct rvt_qp_iter *rvt_qp_iter_init(struct rvt_dev_info *rdi,
788 u64 v, 978 u64 v,
789 void (*cb)(struct rvt_qp *qp, u64 v)); 979 void (*cb)(struct rvt_qp *qp, u64 v));
diff --git a/include/rdma/restrack.h b/include/rdma/restrack.h
index ecf3c7702a4f..b0fc6b26bdf5 100644
--- a/include/rdma/restrack.h
+++ b/include/rdma/restrack.h
@@ -14,6 +14,9 @@
14#include <uapi/rdma/rdma_netlink.h> 14#include <uapi/rdma/rdma_netlink.h>
15#include <linux/xarray.h> 15#include <linux/xarray.h>
16 16
17struct ib_device;
18struct sk_buff;
19
17/** 20/**
18 * enum rdma_restrack_type - HW objects to track 21 * enum rdma_restrack_type - HW objects to track
19 */ 22 */
@@ -43,13 +46,15 @@ enum rdma_restrack_type {
43 */ 46 */
44 RDMA_RESTRACK_CTX, 47 RDMA_RESTRACK_CTX,
45 /** 48 /**
49 * @RDMA_RESTRACK_COUNTER: Statistic Counter
50 */
51 RDMA_RESTRACK_COUNTER,
52 /**
46 * @RDMA_RESTRACK_MAX: Last entry, used for array dclarations 53 * @RDMA_RESTRACK_MAX: Last entry, used for array dclarations
47 */ 54 */
48 RDMA_RESTRACK_MAX 55 RDMA_RESTRACK_MAX
49}; 56};
50 57
51struct ib_device;
52
53/** 58/**
54 * struct rdma_restrack_entry - metadata per-entry 59 * struct rdma_restrack_entry - metadata per-entry
55 */ 60 */
diff --git a/include/rdma/rw.h b/include/rdma/rw.h
index 494f79ca3e62..6ad9dc836c10 100644
--- a/include/rdma/rw.h
+++ b/include/rdma/rw.h
@@ -39,15 +39,6 @@ struct rdma_rw_ctx {
39 struct ib_send_wr inv_wr; 39 struct ib_send_wr inv_wr;
40 struct ib_mr *mr; 40 struct ib_mr *mr;
41 } *reg; 41 } *reg;
42
43 struct {
44 struct rdma_rw_reg_ctx data;
45 struct rdma_rw_reg_ctx prot;
46 struct ib_send_wr sig_inv_wr;
47 struct ib_mr *sig_mr;
48 struct ib_sge sig_sge;
49 struct ib_sig_handover_wr sig_wr;
50 } *sig;
51 }; 42 };
52}; 43};
53 44
diff --git a/include/rdma/signature.h b/include/rdma/signature.h
new file mode 100644
index 000000000000..f24cc2a1d3c5
--- /dev/null
+++ b/include/rdma/signature.h
@@ -0,0 +1,122 @@
1/* SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) */
2/*
3 * Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved.
4 */
5
6#ifndef _RDMA_SIGNATURE_H_
7#define _RDMA_SIGNATURE_H_
8
9enum ib_signature_prot_cap {
10 IB_PROT_T10DIF_TYPE_1 = 1,
11 IB_PROT_T10DIF_TYPE_2 = 1 << 1,
12 IB_PROT_T10DIF_TYPE_3 = 1 << 2,
13};
14
15enum ib_signature_guard_cap {
16 IB_GUARD_T10DIF_CRC = 1,
17 IB_GUARD_T10DIF_CSUM = 1 << 1,
18};
19
20/**
21 * enum ib_signature_type - Signature types
22 * @IB_SIG_TYPE_NONE: Unprotected.
23 * @IB_SIG_TYPE_T10_DIF: Type T10-DIF
24 */
25enum ib_signature_type {
26 IB_SIG_TYPE_NONE,
27 IB_SIG_TYPE_T10_DIF,
28};
29
30/**
31 * enum ib_t10_dif_bg_type - Signature T10-DIF block-guard types
32 * @IB_T10DIF_CRC: Corresponds to T10-PI mandated CRC checksum rules.
33 * @IB_T10DIF_CSUM: Corresponds to IP checksum rules.
34 */
35enum ib_t10_dif_bg_type {
36 IB_T10DIF_CRC,
37 IB_T10DIF_CSUM,
38};
39
40/**
41 * struct ib_t10_dif_domain - Parameters specific for T10-DIF
42 * domain.
43 * @bg_type: T10-DIF block guard type (CRC|CSUM)
44 * @pi_interval: protection information interval.
45 * @bg: seed of guard computation.
46 * @app_tag: application tag of guard block
47 * @ref_tag: initial guard block reference tag.
48 * @ref_remap: Indicate wethear the reftag increments each block
49 * @app_escape: Indicate to skip block check if apptag=0xffff
50 * @ref_escape: Indicate to skip block check if reftag=0xffffffff
51 * @apptag_check_mask: check bitmask of application tag.
52 */
53struct ib_t10_dif_domain {
54 enum ib_t10_dif_bg_type bg_type;
55 u16 pi_interval;
56 u16 bg;
57 u16 app_tag;
58 u32 ref_tag;
59 bool ref_remap;
60 bool app_escape;
61 bool ref_escape;
62 u16 apptag_check_mask;
63};
64
65/**
66 * struct ib_sig_domain - Parameters for signature domain
67 * @sig_type: specific signauture type
68 * @sig: union of all signature domain attributes that may
69 * be used to set domain layout.
70 */
71struct ib_sig_domain {
72 enum ib_signature_type sig_type;
73 union {
74 struct ib_t10_dif_domain dif;
75 } sig;
76};
77
78/**
79 * struct ib_sig_attrs - Parameters for signature handover operation
80 * @check_mask: bitmask for signature byte check (8 bytes)
81 * @mem: memory domain layout descriptor.
82 * @wire: wire domain layout descriptor.
83 * @meta_length: metadata length
84 */
85struct ib_sig_attrs {
86 u8 check_mask;
87 struct ib_sig_domain mem;
88 struct ib_sig_domain wire;
89 int meta_length;
90};
91
92enum ib_sig_err_type {
93 IB_SIG_BAD_GUARD,
94 IB_SIG_BAD_REFTAG,
95 IB_SIG_BAD_APPTAG,
96};
97
98/*
99 * Signature check masks (8 bytes in total) according to the T10-PI standard:
100 * -------- -------- ------------
101 * | GUARD | APPTAG | REFTAG |
102 * | 2B | 2B | 4B |
103 * -------- -------- ------------
104 */
105enum {
106 IB_SIG_CHECK_GUARD = 0xc0,
107 IB_SIG_CHECK_APPTAG = 0x30,
108 IB_SIG_CHECK_REFTAG = 0x0f,
109};
110
111/*
112 * struct ib_sig_err - signature error descriptor
113 */
114struct ib_sig_err {
115 enum ib_sig_err_type err_type;
116 u32 expected;
117 u32 actual;
118 u64 sig_err_offset;
119 u32 key;
120};
121
122#endif /* _RDMA_SIGNATURE_H_ */
diff --git a/include/uapi/rdma/ib_user_cm.h b/include/uapi/rdma/ib_user_cm.h
deleted file mode 100644
index e2709bb8cb18..000000000000
--- a/include/uapi/rdma/ib_user_cm.h
+++ /dev/null
@@ -1,326 +0,0 @@
1/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR Linux-OpenIB) */
2/*
3 * Copyright (c) 2005 Topspin Communications. All rights reserved.
4 * Copyright (c) 2005 Intel Corporation. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35#ifndef IB_USER_CM_H
36#define IB_USER_CM_H
37
38#include <linux/types.h>
39#include <rdma/ib_user_sa.h>
40
41#define IB_USER_CM_ABI_VERSION 5
42
43enum {
44 IB_USER_CM_CMD_CREATE_ID,
45 IB_USER_CM_CMD_DESTROY_ID,
46 IB_USER_CM_CMD_ATTR_ID,
47
48 IB_USER_CM_CMD_LISTEN,
49 IB_USER_CM_CMD_NOTIFY,
50
51 IB_USER_CM_CMD_SEND_REQ,
52 IB_USER_CM_CMD_SEND_REP,
53 IB_USER_CM_CMD_SEND_RTU,
54 IB_USER_CM_CMD_SEND_DREQ,
55 IB_USER_CM_CMD_SEND_DREP,
56 IB_USER_CM_CMD_SEND_REJ,
57 IB_USER_CM_CMD_SEND_MRA,
58 IB_USER_CM_CMD_SEND_LAP,
59 IB_USER_CM_CMD_SEND_APR,
60 IB_USER_CM_CMD_SEND_SIDR_REQ,
61 IB_USER_CM_CMD_SEND_SIDR_REP,
62
63 IB_USER_CM_CMD_EVENT,
64 IB_USER_CM_CMD_INIT_QP_ATTR,
65};
66/*
67 * command ABI structures.
68 */
69struct ib_ucm_cmd_hdr {
70 __u32 cmd;
71 __u16 in;
72 __u16 out;
73};
74
75struct ib_ucm_create_id {
76 __aligned_u64 uid;
77 __aligned_u64 response;
78};
79
80struct ib_ucm_create_id_resp {
81 __u32 id;
82};
83
84struct ib_ucm_destroy_id {
85 __aligned_u64 response;
86 __u32 id;
87 __u32 reserved;
88};
89
90struct ib_ucm_destroy_id_resp {
91 __u32 events_reported;
92};
93
94struct ib_ucm_attr_id {
95 __aligned_u64 response;
96 __u32 id;
97 __u32 reserved;
98};
99
100struct ib_ucm_attr_id_resp {
101 __be64 service_id;
102 __be64 service_mask;
103 __be32 local_id;
104 __be32 remote_id;
105};
106
107struct ib_ucm_init_qp_attr {
108 __aligned_u64 response;
109 __u32 id;
110 __u32 qp_state;
111};
112
113struct ib_ucm_listen {
114 __be64 service_id;
115 __be64 service_mask;
116 __u32 id;
117 __u32 reserved;
118};
119
120struct ib_ucm_notify {
121 __u32 id;
122 __u32 event;
123};
124
125struct ib_ucm_private_data {
126 __aligned_u64 data;
127 __u32 id;
128 __u8 len;
129 __u8 reserved[3];
130};
131
132struct ib_ucm_req {
133 __u32 id;
134 __u32 qpn;
135 __u32 qp_type;
136 __u32 psn;
137 __be64 sid;
138 __aligned_u64 data;
139 __aligned_u64 primary_path;
140 __aligned_u64 alternate_path;
141 __u8 len;
142 __u8 peer_to_peer;
143 __u8 responder_resources;
144 __u8 initiator_depth;
145 __u8 remote_cm_response_timeout;
146 __u8 flow_control;
147 __u8 local_cm_response_timeout;
148 __u8 retry_count;
149 __u8 rnr_retry_count;
150 __u8 max_cm_retries;
151 __u8 srq;
152 __u8 reserved[5];
153};
154
155struct ib_ucm_rep {
156 __aligned_u64 uid;
157 __aligned_u64 data;
158 __u32 id;
159 __u32 qpn;
160 __u32 psn;
161 __u8 len;
162 __u8 responder_resources;
163 __u8 initiator_depth;
164 __u8 target_ack_delay;
165 __u8 failover_accepted;
166 __u8 flow_control;
167 __u8 rnr_retry_count;
168 __u8 srq;
169 __u8 reserved[4];
170};
171
172struct ib_ucm_info {
173 __u32 id;
174 __u32 status;
175 __aligned_u64 info;
176 __aligned_u64 data;
177 __u8 info_len;
178 __u8 data_len;
179 __u8 reserved[6];
180};
181
182struct ib_ucm_mra {
183 __aligned_u64 data;
184 __u32 id;
185 __u8 len;
186 __u8 timeout;
187 __u8 reserved[2];
188};
189
190struct ib_ucm_lap {
191 __aligned_u64 path;
192 __aligned_u64 data;
193 __u32 id;
194 __u8 len;
195 __u8 reserved[3];
196};
197
198struct ib_ucm_sidr_req {
199 __u32 id;
200 __u32 timeout;
201 __be64 sid;
202 __aligned_u64 data;
203 __aligned_u64 path;
204 __u16 reserved_pkey;
205 __u8 len;
206 __u8 max_cm_retries;
207 __u8 reserved[4];
208};
209
210struct ib_ucm_sidr_rep {
211 __u32 id;
212 __u32 qpn;
213 __u32 qkey;
214 __u32 status;
215 __aligned_u64 info;
216 __aligned_u64 data;
217 __u8 info_len;
218 __u8 data_len;
219 __u8 reserved[6];
220};
221/*
222 * event notification ABI structures.
223 */
224struct ib_ucm_event_get {
225 __aligned_u64 response;
226 __aligned_u64 data;
227 __aligned_u64 info;
228 __u8 data_len;
229 __u8 info_len;
230 __u8 reserved[6];
231};
232
233struct ib_ucm_req_event_resp {
234 struct ib_user_path_rec primary_path;
235 struct ib_user_path_rec alternate_path;
236 __be64 remote_ca_guid;
237 __u32 remote_qkey;
238 __u32 remote_qpn;
239 __u32 qp_type;
240 __u32 starting_psn;
241 __u8 responder_resources;
242 __u8 initiator_depth;
243 __u8 local_cm_response_timeout;
244 __u8 flow_control;
245 __u8 remote_cm_response_timeout;
246 __u8 retry_count;
247 __u8 rnr_retry_count;
248 __u8 srq;
249 __u8 port;
250 __u8 reserved[7];
251};
252
253struct ib_ucm_rep_event_resp {
254 __be64 remote_ca_guid;
255 __u32 remote_qkey;
256 __u32 remote_qpn;
257 __u32 starting_psn;
258 __u8 responder_resources;
259 __u8 initiator_depth;
260 __u8 target_ack_delay;
261 __u8 failover_accepted;
262 __u8 flow_control;
263 __u8 rnr_retry_count;
264 __u8 srq;
265 __u8 reserved[5];
266};
267
268struct ib_ucm_rej_event_resp {
269 __u32 reason;
270 /* ari in ib_ucm_event_get info field. */
271};
272
273struct ib_ucm_mra_event_resp {
274 __u8 timeout;
275 __u8 reserved[3];
276};
277
278struct ib_ucm_lap_event_resp {
279 struct ib_user_path_rec path;
280};
281
282struct ib_ucm_apr_event_resp {
283 __u32 status;
284 /* apr info in ib_ucm_event_get info field. */
285};
286
287struct ib_ucm_sidr_req_event_resp {
288 __u16 pkey;
289 __u8 port;
290 __u8 reserved;
291};
292
293struct ib_ucm_sidr_rep_event_resp {
294 __u32 status;
295 __u32 qkey;
296 __u32 qpn;
297 /* info in ib_ucm_event_get info field. */
298};
299
300#define IB_UCM_PRES_DATA 0x01
301#define IB_UCM_PRES_INFO 0x02
302#define IB_UCM_PRES_PRIMARY 0x04
303#define IB_UCM_PRES_ALTERNATE 0x08
304
305struct ib_ucm_event_resp {
306 __aligned_u64 uid;
307 __u32 id;
308 __u32 event;
309 __u32 present;
310 __u32 reserved;
311 union {
312 struct ib_ucm_req_event_resp req_resp;
313 struct ib_ucm_rep_event_resp rep_resp;
314 struct ib_ucm_rej_event_resp rej_resp;
315 struct ib_ucm_mra_event_resp mra_resp;
316 struct ib_ucm_lap_event_resp lap_resp;
317 struct ib_ucm_apr_event_resp apr_resp;
318
319 struct ib_ucm_sidr_req_event_resp sidr_req_resp;
320 struct ib_ucm_sidr_rep_event_resp sidr_rep_resp;
321
322 __u32 send_status;
323 } u;
324};
325
326#endif /* IB_USER_CM_H */
diff --git a/include/uapi/rdma/mlx5_user_ioctl_cmds.h b/include/uapi/rdma/mlx5_user_ioctl_cmds.h
index d404c951954c..d0da070cf0ab 100644
--- a/include/uapi/rdma/mlx5_user_ioctl_cmds.h
+++ b/include/uapi/rdma/mlx5_user_ioctl_cmds.h
@@ -51,6 +51,7 @@ enum mlx5_ib_devx_methods {
51 MLX5_IB_METHOD_DEVX_OTHER = (1U << UVERBS_ID_NS_SHIFT), 51 MLX5_IB_METHOD_DEVX_OTHER = (1U << UVERBS_ID_NS_SHIFT),
52 MLX5_IB_METHOD_DEVX_QUERY_UAR, 52 MLX5_IB_METHOD_DEVX_QUERY_UAR,
53 MLX5_IB_METHOD_DEVX_QUERY_EQN, 53 MLX5_IB_METHOD_DEVX_QUERY_EQN,
54 MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT,
54}; 55};
55 56
56enum mlx5_ib_devx_other_attrs { 57enum mlx5_ib_devx_other_attrs {
@@ -93,6 +94,14 @@ enum mlx5_ib_devx_obj_query_async_attrs {
93 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN, 94 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN,
94}; 95};
95 96
97enum mlx5_ib_devx_subscribe_event_attrs {
98 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
99 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE,
100 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,
101 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM,
102 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE,
103};
104
96enum mlx5_ib_devx_query_eqn_attrs { 105enum mlx5_ib_devx_query_eqn_attrs {
97 MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC = (1U << UVERBS_ID_NS_SHIFT), 106 MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC = (1U << UVERBS_ID_NS_SHIFT),
98 MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN, 107 MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
@@ -127,16 +136,26 @@ enum mlx5_ib_devx_async_cmd_fd_alloc_attrs {
127 MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE = (1U << UVERBS_ID_NS_SHIFT), 136 MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
128}; 137};
129 138
139enum mlx5_ib_devx_async_event_fd_alloc_attrs {
140 MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
141 MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS,
142};
143
130enum mlx5_ib_devx_async_cmd_fd_methods { 144enum mlx5_ib_devx_async_cmd_fd_methods {
131 MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC = (1U << UVERBS_ID_NS_SHIFT), 145 MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC = (1U << UVERBS_ID_NS_SHIFT),
132}; 146};
133 147
148enum mlx5_ib_devx_async_event_fd_methods {
149 MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC = (1U << UVERBS_ID_NS_SHIFT),
150};
151
134enum mlx5_ib_objects { 152enum mlx5_ib_objects {
135 MLX5_IB_OBJECT_DEVX = (1U << UVERBS_ID_NS_SHIFT), 153 MLX5_IB_OBJECT_DEVX = (1U << UVERBS_ID_NS_SHIFT),
136 MLX5_IB_OBJECT_DEVX_OBJ, 154 MLX5_IB_OBJECT_DEVX_OBJ,
137 MLX5_IB_OBJECT_DEVX_UMEM, 155 MLX5_IB_OBJECT_DEVX_UMEM,
138 MLX5_IB_OBJECT_FLOW_MATCHER, 156 MLX5_IB_OBJECT_FLOW_MATCHER,
139 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD, 157 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
158 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
140}; 159};
141 160
142enum mlx5_ib_flow_matcher_create_attrs { 161enum mlx5_ib_flow_matcher_create_attrs {
diff --git a/include/uapi/rdma/mlx5_user_ioctl_verbs.h b/include/uapi/rdma/mlx5_user_ioctl_verbs.h
index a8f34c237458..7e9900b0e746 100644
--- a/include/uapi/rdma/mlx5_user_ioctl_verbs.h
+++ b/include/uapi/rdma/mlx5_user_ioctl_verbs.h
@@ -63,5 +63,14 @@ enum mlx5_ib_uapi_dm_type {
63 MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM, 63 MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM,
64}; 64};
65 65
66enum mlx5_ib_uapi_devx_create_event_channel_flags {
67 MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA = 1 << 0,
68};
69
70struct mlx5_ib_uapi_devx_async_event_hdr {
71 __aligned_u64 cookie;
72 __u8 out_data[];
73};
74
66#endif 75#endif
67 76
diff --git a/include/uapi/rdma/rdma_netlink.h b/include/uapi/rdma/rdma_netlink.h
index 41db51367efa..8e277783fa96 100644
--- a/include/uapi/rdma/rdma_netlink.h
+++ b/include/uapi/rdma/rdma_netlink.h
@@ -147,6 +147,18 @@ enum {
147 IWPM_NLA_HELLO_MAX 147 IWPM_NLA_HELLO_MAX
148}; 148};
149 149
150/* For RDMA_NLDEV_ATTR_DEV_NODE_TYPE */
151enum {
152 /* IB values map to NodeInfo:NodeType. */
153 RDMA_NODE_IB_CA = 1,
154 RDMA_NODE_IB_SWITCH,
155 RDMA_NODE_IB_ROUTER,
156 RDMA_NODE_RNIC,
157 RDMA_NODE_USNIC,
158 RDMA_NODE_USNIC_UDP,
159 RDMA_NODE_UNSPECIFIED,
160};
161
150/* 162/*
151 * Local service operations: 163 * Local service operations:
152 * RESOLVE - The client requests the local service to resolve a path. 164 * RESOLVE - The client requests the local service to resolve a path.
@@ -267,11 +279,15 @@ enum rdma_nldev_command {
267 279
268 RDMA_NLDEV_CMD_RES_PD_GET, /* can dump */ 280 RDMA_NLDEV_CMD_RES_PD_GET, /* can dump */
269 281
270 RDMA_NLDEV_NUM_OPS 282 RDMA_NLDEV_CMD_GET_CHARDEV,
271};
272 283
273enum { 284 RDMA_NLDEV_CMD_STAT_SET,
274 RDMA_NLDEV_ATTR_ENTRY_STRLEN = 16, 285
286 RDMA_NLDEV_CMD_STAT_GET, /* can dump */
287
288 RDMA_NLDEV_CMD_STAT_DEL,
289
290 RDMA_NLDEV_NUM_OPS
275}; 291};
276 292
277enum rdma_nldev_print_type { 293enum rdma_nldev_print_type {
@@ -478,10 +494,72 @@ enum rdma_nldev_attr {
478 * File descriptor handle of the net namespace object 494 * File descriptor handle of the net namespace object
479 */ 495 */
480 RDMA_NLDEV_NET_NS_FD, /* u32 */ 496 RDMA_NLDEV_NET_NS_FD, /* u32 */
497 /*
498 * Information about a chardev.
499 * CHARDEV_TYPE is the name of the chardev ABI (ie uverbs, umad, etc)
500 * CHARDEV_ABI signals the ABI revision (historical)
501 * CHARDEV_NAME is the kernel name for the /dev/ file (no directory)
502 * CHARDEV is the 64 bit dev_t for the inode
503 */
504 RDMA_NLDEV_ATTR_CHARDEV_TYPE, /* string */
505 RDMA_NLDEV_ATTR_CHARDEV_NAME, /* string */
506 RDMA_NLDEV_ATTR_CHARDEV_ABI, /* u64 */
507 RDMA_NLDEV_ATTR_CHARDEV, /* u64 */
508 RDMA_NLDEV_ATTR_UVERBS_DRIVER_ID, /* u64 */
509 /*
510 * Counter-specific attributes.
511 */
512 RDMA_NLDEV_ATTR_STAT_MODE, /* u32 */
513 RDMA_NLDEV_ATTR_STAT_RES, /* u32 */
514 RDMA_NLDEV_ATTR_STAT_AUTO_MODE_MASK, /* u32 */
515 RDMA_NLDEV_ATTR_STAT_COUNTER, /* nested table */
516 RDMA_NLDEV_ATTR_STAT_COUNTER_ENTRY, /* nested table */
517 RDMA_NLDEV_ATTR_STAT_COUNTER_ID, /* u32 */
518 RDMA_NLDEV_ATTR_STAT_HWCOUNTERS, /* nested table */
519 RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY, /* nested table */
520 RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME, /* string */
521 RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_VALUE, /* u64 */
522
523 /*
524 * CQ adaptive moderatio (DIM)
525 */
526 RDMA_NLDEV_ATTR_DEV_DIM, /* u8 */
481 527
482 /* 528 /*
483 * Always the end 529 * Always the end
484 */ 530 */
485 RDMA_NLDEV_ATTR_MAX 531 RDMA_NLDEV_ATTR_MAX
486}; 532};
533
534/*
535 * Supported counter bind modes. All modes are mutual-exclusive.
536 */
537enum rdma_nl_counter_mode {
538 RDMA_COUNTER_MODE_NONE,
539
540 /*
541 * A qp is bound with a counter automatically during initialization
542 * based on the auto mode (e.g., qp type, ...)
543 */
544 RDMA_COUNTER_MODE_AUTO,
545
546 /*
547 * Which qp are bound with which counter is explicitly specified
548 * by the user
549 */
550 RDMA_COUNTER_MODE_MANUAL,
551
552 /*
553 * Always the end
554 */
555 RDMA_COUNTER_MODE_MAX,
556};
557
558/*
559 * Supported criteria in counter auto mode.
560 * Currently only "qp type" is supported
561 */
562enum rdma_nl_counter_mask {
563 RDMA_COUNTER_MASK_QP_TYPE = 1,
564};
487#endif /* _UAPI_RDMA_NETLINK_H */ 565#endif /* _UAPI_RDMA_NETLINK_H */
diff --git a/include/uapi/rdma/rdma_user_ioctl_cmds.h b/include/uapi/rdma/rdma_user_ioctl_cmds.h
index 26213f49f5c8..64c14cb0022f 100644
--- a/include/uapi/rdma/rdma_user_ioctl_cmds.h
+++ b/include/uapi/rdma/rdma_user_ioctl_cmds.h
@@ -103,6 +103,7 @@ enum rdma_driver_id {
103 RDMA_DRIVER_HFI1, 103 RDMA_DRIVER_HFI1,
104 RDMA_DRIVER_QIB, 104 RDMA_DRIVER_QIB,
105 RDMA_DRIVER_EFA, 105 RDMA_DRIVER_EFA,
106 RDMA_DRIVER_SIW,
106}; 107};
107 108
108#endif 109#endif
diff --git a/include/uapi/rdma/rvt-abi.h b/include/uapi/rdma/rvt-abi.h
new file mode 100644
index 000000000000..7328293c715c
--- /dev/null
+++ b/include/uapi/rdma/rvt-abi.h
@@ -0,0 +1,66 @@
1/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */
2
3/*
4 * This file contains defines, structures, etc. that are used
5 * to communicate between kernel and user code.
6 */
7
8#ifndef RVT_ABI_USER_H
9#define RVT_ABI_USER_H
10
11#include <linux/types.h>
12#include <rdma/ib_user_verbs.h>
13#ifndef RDMA_ATOMIC_UAPI
14#define RDMA_ATOMIC_UAPI(_type, _name) struct{ _type val; } _name
15#endif
16
17struct rvt_wqe_sge {
18 __aligned_u64 addr;
19 __u32 length;
20 __u32 lkey;
21};
22
23/*
24 * This structure is used to contain the head pointer, tail pointer,
25 * and completion queue entries as a single memory allocation so
26 * it can be mmap'ed into user space.
27 */
28struct rvt_cq_wc {
29 /* index of next entry to fill */
30 RDMA_ATOMIC_UAPI(__u32, head);
31 /* index of next ib_poll_cq() entry */
32 RDMA_ATOMIC_UAPI(__u32, tail);
33
34 /* these are actually size ibcq.cqe + 1 */
35 struct ib_uverbs_wc uqueue[];
36};
37
38/*
39 * Receive work request queue entry.
40 * The size of the sg_list is determined when the QP (or SRQ) is created
41 * and stored in qp->r_rq.max_sge (or srq->rq.max_sge).
42 */
43struct rvt_rwqe {
44 __u64 wr_id;
45 __u8 num_sge;
46 __u8 padding[7];
47 struct rvt_wqe_sge sg_list[];
48};
49
50/*
51 * This structure is used to contain the head pointer, tail pointer,
52 * and receive work queue entries as a single memory allocation so
53 * it can be mmap'ed into user space.
54 * Note that the wq array elements are variable size so you can't
55 * just index into the array to get the N'th element;
56 * use get_rwqe_ptr() for user space and rvt_get_rwqe_ptr()
57 * for kernel space.
58 */
59struct rvt_rwq {
60 /* new work requests posted to the head */
61 RDMA_ATOMIC_UAPI(__u32, head);
62 /* receives pull requests from here. */
63 RDMA_ATOMIC_UAPI(__u32, tail);
64 struct rvt_rwqe wq[];
65};
66#endif /* RVT_ABI_USER_H */
diff --git a/include/uapi/rdma/siw-abi.h b/include/uapi/rdma/siw-abi.h
new file mode 100644
index 000000000000..3dd8071ace7b
--- /dev/null
+++ b/include/uapi/rdma/siw-abi.h
@@ -0,0 +1,185 @@
1/* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */
2
3/* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
4/* Copyright (c) 2008-2019, IBM Corporation */
5
6#ifndef _SIW_USER_H
7#define _SIW_USER_H
8
9#include <linux/types.h>
10
11#define SIW_NODE_DESC_COMMON "Software iWARP stack"
12#define SIW_ABI_VERSION 1
13#define SIW_MAX_SGE 6
14#define SIW_UOBJ_MAX_KEY 0x08FFFF
15#define SIW_INVAL_UOBJ_KEY (SIW_UOBJ_MAX_KEY + 1)
16
17struct siw_uresp_create_cq {
18 __u32 cq_id;
19 __u32 num_cqe;
20 __aligned_u64 cq_key;
21};
22
23struct siw_uresp_create_qp {
24 __u32 qp_id;
25 __u32 num_sqe;
26 __u32 num_rqe;
27 __u32 pad;
28 __aligned_u64 sq_key;
29 __aligned_u64 rq_key;
30};
31
32struct siw_ureq_reg_mr {
33 __u8 stag_key;
34 __u8 reserved[3];
35 __u32 pad;
36};
37
38struct siw_uresp_reg_mr {
39 __u32 stag;
40 __u32 pad;
41};
42
43struct siw_uresp_create_srq {
44 __u32 num_rqe;
45 __u32 pad;
46 __aligned_u64 srq_key;
47};
48
49struct siw_uresp_alloc_ctx {
50 __u32 dev_id;
51 __u32 pad;
52};
53
54enum siw_opcode {
55 SIW_OP_WRITE,
56 SIW_OP_READ,
57 SIW_OP_READ_LOCAL_INV,
58 SIW_OP_SEND,
59 SIW_OP_SEND_WITH_IMM,
60 SIW_OP_SEND_REMOTE_INV,
61
62 /* Unsupported */
63 SIW_OP_FETCH_AND_ADD,
64 SIW_OP_COMP_AND_SWAP,
65
66 SIW_OP_RECEIVE,
67 /* provider internal SQE */
68 SIW_OP_READ_RESPONSE,
69 /*
70 * below opcodes valid for
71 * in-kernel clients only
72 */
73 SIW_OP_INVAL_STAG,
74 SIW_OP_REG_MR,
75 SIW_NUM_OPCODES
76};
77
78/* Keep it same as ibv_sge to allow for memcpy */
79struct siw_sge {
80 __aligned_u64 laddr;
81 __u32 length;
82 __u32 lkey;
83};
84
85/*
86 * Inline data are kept within the work request itself occupying
87 * the space of sge[1] .. sge[n]. Therefore, inline data cannot be
88 * supported if SIW_MAX_SGE is below 2 elements.
89 */
90#define SIW_MAX_INLINE (sizeof(struct siw_sge) * (SIW_MAX_SGE - 1))
91
92#if SIW_MAX_SGE < 2
93#error "SIW_MAX_SGE must be at least 2"
94#endif
95
96enum siw_wqe_flags {
97 SIW_WQE_VALID = 1,
98 SIW_WQE_INLINE = (1 << 1),
99 SIW_WQE_SIGNALLED = (1 << 2),
100 SIW_WQE_SOLICITED = (1 << 3),
101 SIW_WQE_READ_FENCE = (1 << 4),
102 SIW_WQE_REM_INVAL = (1 << 5),
103 SIW_WQE_COMPLETED = (1 << 6)
104};
105
106/* Send Queue Element */
107struct siw_sqe {
108 __aligned_u64 id;
109 __u16 flags;
110 __u8 num_sge;
111 /* Contains enum siw_opcode values */
112 __u8 opcode;
113 __u32 rkey;
114 union {
115 __aligned_u64 raddr;
116 __aligned_u64 base_mr;
117 };
118 union {
119 struct siw_sge sge[SIW_MAX_SGE];
120 __aligned_u64 access;
121 };
122};
123
124/* Receive Queue Element */
125struct siw_rqe {
126 __aligned_u64 id;
127 __u16 flags;
128 __u8 num_sge;
129 /*
130 * only used by kernel driver,
131 * ignored if set by user
132 */
133 __u8 opcode;
134 __u32 unused;
135 struct siw_sge sge[SIW_MAX_SGE];
136};
137
138enum siw_notify_flags {
139 SIW_NOTIFY_NOT = (0),
140 SIW_NOTIFY_SOLICITED = (1 << 0),
141 SIW_NOTIFY_NEXT_COMPLETION = (1 << 1),
142 SIW_NOTIFY_MISSED_EVENTS = (1 << 2),
143 SIW_NOTIFY_ALL = SIW_NOTIFY_SOLICITED | SIW_NOTIFY_NEXT_COMPLETION |
144 SIW_NOTIFY_MISSED_EVENTS
145};
146
147enum siw_wc_status {
148 SIW_WC_SUCCESS,
149 SIW_WC_LOC_LEN_ERR,
150 SIW_WC_LOC_PROT_ERR,
151 SIW_WC_LOC_QP_OP_ERR,
152 SIW_WC_WR_FLUSH_ERR,
153 SIW_WC_BAD_RESP_ERR,
154 SIW_WC_LOC_ACCESS_ERR,
155 SIW_WC_REM_ACCESS_ERR,
156 SIW_WC_REM_INV_REQ_ERR,
157 SIW_WC_GENERAL_ERR,
158 SIW_NUM_WC_STATUS
159};
160
161struct siw_cqe {
162 __aligned_u64 id;
163 __u8 flags;
164 __u8 opcode;
165 __u16 status;
166 __u32 bytes;
167 union {
168 __aligned_u64 imm_data;
169 __u32 inval_stag;
170 };
171 /* QP number or QP pointer */
172 union {
173 struct ib_qp *base_qp;
174 __aligned_u64 qp_id;
175 };
176};
177
178/*
179 * Shared structure between user and kernel
180 * to control CQ arming.
181 */
182struct siw_cq_ctrl {
183 __aligned_u64 notify;
184};
185#endif
generated by cgit v1.2.2 (git 2.25.0) at 2025-10-15 03:40:54 -0400