aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--include/linux/rds.h12
-rw-r--r--net/rds/ib_send.c14
-rw-r--r--net/rds/rdma.c33
-rw-r--r--net/rds/rds.h14
-rw-r--r--net/rds/send.c4
5 files changed, 64 insertions, 13 deletions
diff --git a/include/linux/rds.h b/include/linux/rds.h
index 109f1d34331..a2a5edb4a27 100644
--- a/include/linux/rds.h
+++ b/include/linux/rds.h
@@ -75,6 +75,8 @@
75#define RDS_CMSG_CONG_UPDATE 5 75#define RDS_CMSG_CONG_UPDATE 5
76#define RDS_CMSG_ATOMIC_FADD 6 76#define RDS_CMSG_ATOMIC_FADD 6
77#define RDS_CMSG_ATOMIC_CSWP 7 77#define RDS_CMSG_ATOMIC_CSWP 7
78#define RDS_CMSG_MASKED_ATOMIC_FADD 8
79#define RDS_CMSG_MASKED_ATOMIC_CSWP 9
78 80
79#define RDS_INFO_FIRST 10000 81#define RDS_INFO_FIRST 10000
80#define RDS_INFO_COUNTERS 10000 82#define RDS_INFO_COUNTERS 10000
@@ -251,6 +253,16 @@ struct rds_atomic_args {
251 struct { 253 struct {
252 uint64_t add; 254 uint64_t add;
253 } fadd; 255 } fadd;
256 struct {
257 uint64_t compare;
258 uint64_t swap;
259 uint64_t compare_mask;
260 uint64_t swap_mask;
261 } m_cswp;
262 struct {
263 uint64_t add;
264 uint64_t nocarry_mask;
265 } m_fadd;
254 }; 266 };
255 uint64_t flags; 267 uint64_t flags;
256 uint64_t user_token; 268 uint64_t user_token;
diff --git a/net/rds/ib_send.c b/net/rds/ib_send.c
index 808544aebb7..71f373c421b 100644
--- a/net/rds/ib_send.c
+++ b/net/rds/ib_send.c
@@ -807,13 +807,17 @@ int rds_ib_xmit_atomic(struct rds_connection *conn, struct rm_atomic_op *op)
807 send->s_queued = jiffies; 807 send->s_queued = jiffies;
808 808
809 if (op->op_type == RDS_ATOMIC_TYPE_CSWP) { 809 if (op->op_type == RDS_ATOMIC_TYPE_CSWP) {
810 send->s_wr.opcode = IB_WR_ATOMIC_CMP_AND_SWP; 810 send->s_wr.opcode = IB_WR_MASKED_ATOMIC_CMP_AND_SWP;
811 send->s_wr.wr.atomic.compare_add = op->op_compare; 811 send->s_wr.wr.atomic.compare_add = op->op_m_cswp.compare;
812 send->s_wr.wr.atomic.swap = op->op_swap_add; 812 send->s_wr.wr.atomic.swap = op->op_m_cswp.swap;
813 send->s_wr.wr.atomic.compare_add_mask = op->op_m_cswp.compare_mask;
814 send->s_wr.wr.atomic.swap_mask = op->op_m_cswp.swap_mask;
813 } else { /* FADD */ 815 } else { /* FADD */
814 send->s_wr.opcode = IB_WR_ATOMIC_FETCH_AND_ADD; 816 send->s_wr.opcode = IB_WR_MASKED_ATOMIC_FETCH_AND_ADD;
815 send->s_wr.wr.atomic.compare_add = op->op_swap_add; 817 send->s_wr.wr.atomic.compare_add = op->op_m_fadd.add;
816 send->s_wr.wr.atomic.swap = 0; 818 send->s_wr.wr.atomic.swap = 0;
819 send->s_wr.wr.atomic.compare_add_mask = op->op_m_fadd.nocarry_mask;
820 send->s_wr.wr.atomic.swap_mask = 0;
817 } 821 }
818 nr_sig = rds_ib_set_wr_signal_state(ic, send, op->op_notify); 822 nr_sig = rds_ib_set_wr_signal_state(ic, send, op->op_notify);
819 send->s_wr.num_sge = 1; 823 send->s_wr.num_sge = 1;
diff --git a/net/rds/rdma.c b/net/rds/rdma.c
index 48781fe4431..48064673fc7 100644
--- a/net/rds/rdma.c
+++ b/net/rds/rdma.c
@@ -738,13 +738,34 @@ int rds_cmsg_atomic(struct rds_sock *rs, struct rds_message *rm,
738 738
739 args = CMSG_DATA(cmsg); 739 args = CMSG_DATA(cmsg);
740 740
741 if (cmsg->cmsg_type == RDS_CMSG_ATOMIC_CSWP) { 741 /* Nonmasked & masked cmsg ops converted to masked hw ops */
742 rm->atomic.op_type = RDS_ATOMIC_TYPE_CSWP; 742 switch (cmsg->cmsg_type) {
743 rm->atomic.op_swap_add = args->cswp.swap; 743 case RDS_CMSG_ATOMIC_FADD:
744 rm->atomic.op_compare = args->cswp.compare; 744 rm->atomic.op_type = RDS_ATOMIC_TYPE_FADD;
745 } else { 745 rm->atomic.op_m_fadd.add = args->fadd.add;
746 rm->atomic.op_m_fadd.nocarry_mask = 0;
747 break;
748 case RDS_CMSG_MASKED_ATOMIC_FADD:
746 rm->atomic.op_type = RDS_ATOMIC_TYPE_FADD; 749 rm->atomic.op_type = RDS_ATOMIC_TYPE_FADD;
747 rm->atomic.op_swap_add = args->fadd.add; 750 rm->atomic.op_m_fadd.add = args->m_fadd.add;
751 rm->atomic.op_m_fadd.nocarry_mask = args->m_fadd.nocarry_mask;
752 break;
753 case RDS_CMSG_ATOMIC_CSWP:
754 rm->atomic.op_type = RDS_ATOMIC_TYPE_CSWP;
755 rm->atomic.op_m_cswp.compare = args->cswp.compare;
756 rm->atomic.op_m_cswp.swap = args->cswp.swap;
757 rm->atomic.op_m_cswp.compare_mask = ~0;
758 rm->atomic.op_m_cswp.swap_mask = ~0;
759 break;
760 case RDS_CMSG_MASKED_ATOMIC_CSWP:
761 rm->atomic.op_type = RDS_ATOMIC_TYPE_CSWP;
762 rm->atomic.op_m_cswp.compare = args->m_cswp.compare;
763 rm->atomic.op_m_cswp.swap = args->m_cswp.swap;
764 rm->atomic.op_m_cswp.compare_mask = args->m_cswp.compare_mask;
765 rm->atomic.op_m_cswp.swap_mask = args->m_cswp.swap_mask;
766 break;
767 default:
768 BUG(); /* should never happen */
748 } 769 }
749 770
750 rm->atomic.op_notify = !!(args->flags & RDS_RDMA_NOTIFY_ME); 771 rm->atomic.op_notify = !!(args->flags & RDS_RDMA_NOTIFY_ME);
diff --git a/net/rds/rds.h b/net/rds/rds.h
index aadaddba88a..8103dcf8b97 100644
--- a/net/rds/rds.h
+++ b/net/rds/rds.h
@@ -316,8 +316,18 @@ struct rds_message {
316 struct { 316 struct {
317 struct rm_atomic_op { 317 struct rm_atomic_op {
318 int op_type; 318 int op_type;
319 uint64_t op_swap_add; 319 union {
320 uint64_t op_compare; 320 struct {
321 uint64_t compare;
322 uint64_t swap;
323 uint64_t compare_mask;
324 uint64_t swap_mask;
325 } op_m_cswp;
326 struct {
327 uint64_t add;
328 uint64_t nocarry_mask;
329 } op_m_fadd;
330 };
321 331
322 u32 op_rkey; 332 u32 op_rkey;
323 u64 op_remote_addr; 333 u64 op_remote_addr;
diff --git a/net/rds/send.c b/net/rds/send.c
index 81471b25373..9b951a0ab6b 100644
--- a/net/rds/send.c
+++ b/net/rds/send.c
@@ -843,6 +843,8 @@ static int rds_rm_size(struct msghdr *msg, int data_len)
843 843
844 case RDS_CMSG_ATOMIC_CSWP: 844 case RDS_CMSG_ATOMIC_CSWP:
845 case RDS_CMSG_ATOMIC_FADD: 845 case RDS_CMSG_ATOMIC_FADD:
846 case RDS_CMSG_MASKED_ATOMIC_CSWP:
847 case RDS_CMSG_MASKED_ATOMIC_FADD:
846 cmsg_groups |= 1; 848 cmsg_groups |= 1;
847 size += sizeof(struct scatterlist); 849 size += sizeof(struct scatterlist);
848 break; 850 break;
@@ -894,6 +896,8 @@ static int rds_cmsg_send(struct rds_sock *rs, struct rds_message *rm,
894 break; 896 break;
895 case RDS_CMSG_ATOMIC_CSWP: 897 case RDS_CMSG_ATOMIC_CSWP:
896 case RDS_CMSG_ATOMIC_FADD: 898 case RDS_CMSG_ATOMIC_FADD:
899 case RDS_CMSG_MASKED_ATOMIC_CSWP:
900 case RDS_CMSG_MASKED_ATOMIC_FADD:
897 ret = rds_cmsg_atomic(rs, rm, cmsg); 901 ret = rds_cmsg_atomic(rs, rm, cmsg);
898 break; 902 break;
899 903
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-11 09:27:41 -0500