aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDavid S. Miller <davem@davemloft.net>2017-01-09 17:09:31 -0500
committerDavid S. Miller <davem@davemloft.net>2017-01-09 17:09:31 -0500
commitbda65b4255ac983ce36a6c0ea6a7794f8e8fcc86 (patch)
treead280b2ecc1266329416f043dd20cea515b3dc48
parentb369e7fd41f7cdbe2488cb736ef4f958bb94b5e2 (diff)
parentf502d834950a28e02651bb7e2cc7111ddd352644 (diff)
Merge tag 'mlx5-4kuar-for-4.11' of git://git.kernel.org/pub/scm/linux/kernel/git/mellanox/linux
Saeed Mahameed says: ==================== mlx5 4K UAR The following series of patches optimizes the usage of the UAR area which is contained within the BAR 0-1. Previous versions of the firmware and the driver assumed each system page contains a single UAR. This patch set will query the firmware for a new capability that if published, means that the firmware can support UARs of fixed 4K regardless of system page size. In the case of powerpc, where page size equals 64KB, this means we can utilize 16 UARs per system page. Since user space processes by default consume eight UARs per context this means that with this change a process will need a single system page to fulfill that requirement and in fact make use of more UARs which is better in terms of performance. In addition to optimizing user-space processes, we introduce an allocator that can be used by kernel consumers to allocate blue flame registers (which are areas within a UAR that are used to write doorbells). This provides further optimization on using the UAR area since the Ethernet driver makes use of a single blue flame register per system page and now it will use two blue flame registers per 4K. The series also makes changes to naming conventions and now the terms used in the driver code match the terms used in the PRM (programmers reference manual). Thus, what used to be called UUAR (micro UAR) is now called BFREG (blue flame register). In order to support compatibility between different versions of library/driver/firmware, the library has now means to notify the kernel driver that it supports the new scheme and the kernel can notify the library if it supports this extension. So mixed versions of libraries can run concurrently without any issues. ==================== Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat
-rw-r--r--drivers/infiniband/hw/mlx5/cq.c10
-rw-r--r--drivers/infiniband/hw/mlx5/main.c278
-rw-r--r--drivers/infiniband/hw/mlx5/mlx5_ib.h32
-rw-r--r--drivers/infiniband/hw/mlx5/qp.c290
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/cq.c2
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en.h11
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_common.c12
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_main.c21
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/eq.c14
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/main.c26
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/uar.c351
-rw-r--r--include/linux/mlx5/cq.h5
-rw-r--r--include/linux/mlx5/device.h23
-rw-r--r--include/linux/mlx5/doorbell.h6
-rw-r--r--include/linux/mlx5/driver.h81
-rw-r--r--include/linux/mlx5/mlx5_ifc.h7
-rw-r--r--include/uapi/rdma/mlx5-abi.h19
17 files changed, 672 insertions, 516 deletions
diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c
index b3ef47c3ab73..31803b367104 100644
--- a/drivers/infiniband/hw/mlx5/cq.c
+++ b/drivers/infiniband/hw/mlx5/cq.c
@@ -689,7 +689,7 @@ int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
689{ 689{
690 struct mlx5_core_dev *mdev = to_mdev(ibcq->device)->mdev; 690 struct mlx5_core_dev *mdev = to_mdev(ibcq->device)->mdev;
691 struct mlx5_ib_cq *cq = to_mcq(ibcq); 691 struct mlx5_ib_cq *cq = to_mcq(ibcq);
692 void __iomem *uar_page = mdev->priv.uuari.uars[0].map; 692 void __iomem *uar_page = mdev->priv.uar->map;
693 unsigned long irq_flags; 693 unsigned long irq_flags;
694 int ret = 0; 694 int ret = 0;
695 695
@@ -704,9 +704,7 @@ int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
704 mlx5_cq_arm(&cq->mcq, 704 mlx5_cq_arm(&cq->mcq,
705 (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ? 705 (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
706 MLX5_CQ_DB_REQ_NOT_SOL : MLX5_CQ_DB_REQ_NOT, 706 MLX5_CQ_DB_REQ_NOT_SOL : MLX5_CQ_DB_REQ_NOT,
707 uar_page, 707 uar_page, to_mcq(ibcq)->mcq.cons_index);
708 MLX5_GET_DOORBELL_LOCK(&mdev->priv.cq_uar_lock),
709 to_mcq(ibcq)->mcq.cons_index);
710 708
711 return ret; 709 return ret;
712} 710}
@@ -790,7 +788,7 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
790 MLX5_SET(cqc, cqc, log_page_size, 788 MLX5_SET(cqc, cqc, log_page_size,
791 page_shift - MLX5_ADAPTER_PAGE_SHIFT); 789 page_shift - MLX5_ADAPTER_PAGE_SHIFT);
792 790
793 *index = to_mucontext(context)->uuari.uars[0].index; 791 *index = to_mucontext(context)->bfregi.sys_pages[0];
794 792
795 if (ucmd.cqe_comp_en == 1) { 793 if (ucmd.cqe_comp_en == 1) {
796 if (unlikely((*cqe_size != 64) || 794 if (unlikely((*cqe_size != 64) ||
@@ -886,7 +884,7 @@ static int create_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
886 MLX5_SET(cqc, cqc, log_page_size, 884 MLX5_SET(cqc, cqc, log_page_size,
887 cq->buf.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 885 cq->buf.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
888 886
889 *index = dev->mdev->priv.uuari.uars[0].index; 887 *index = dev->mdev->priv.uar->index;
890 888
891 return 0; 889 return 0;
892 890
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 86c61e73780e..a191b9327b0c 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -992,6 +992,86 @@ out:
992 return err; 992 return err;
993} 993}
994 994
995static void print_lib_caps(struct mlx5_ib_dev *dev, u64 caps)
996{
997 mlx5_ib_dbg(dev, "MLX5_LIB_CAP_4K_UAR = %s\n",
998 caps & MLX5_LIB_CAP_4K_UAR ? "y" : "n");
999}
1000
1001static int calc_total_bfregs(struct mlx5_ib_dev *dev, bool lib_uar_4k,
1002 struct mlx5_ib_alloc_ucontext_req_v2 *req,
1003 u32 *num_sys_pages)
1004{
1005 int uars_per_sys_page;
1006 int bfregs_per_sys_page;
1007 int ref_bfregs = req->total_num_bfregs;
1008
1009 if (req->total_num_bfregs == 0)
1010 return -EINVAL;
1011
1012 BUILD_BUG_ON(MLX5_MAX_BFREGS % MLX5_NON_FP_BFREGS_IN_PAGE);
1013 BUILD_BUG_ON(MLX5_MAX_BFREGS < MLX5_NON_FP_BFREGS_IN_PAGE);
1014
1015 if (req->total_num_bfregs > MLX5_MAX_BFREGS)
1016 return -ENOMEM;
1017
1018 uars_per_sys_page = get_uars_per_sys_page(dev, lib_uar_4k);
1019 bfregs_per_sys_page = uars_per_sys_page * MLX5_NON_FP_BFREGS_PER_UAR;
1020 req->total_num_bfregs = ALIGN(req->total_num_bfregs, bfregs_per_sys_page);
1021 *num_sys_pages = req->total_num_bfregs / bfregs_per_sys_page;
1022
1023 if (req->num_low_latency_bfregs > req->total_num_bfregs - 1)
1024 return -EINVAL;
1025
1026 mlx5_ib_dbg(dev, "uar_4k: fw support %s, lib support %s, user requested %d bfregs, alloated %d, using %d sys pages\n",
1027 MLX5_CAP_GEN(dev->mdev, uar_4k) ? "yes" : "no",
1028 lib_uar_4k ? "yes" : "no", ref_bfregs,
1029 req->total_num_bfregs, *num_sys_pages);
1030
1031 return 0;
1032}
1033
1034static int allocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *context)
1035{
1036 struct mlx5_bfreg_info *bfregi;
1037 int err;
1038 int i;
1039
1040 bfregi = &context->bfregi;
1041 for (i = 0; i < bfregi->num_sys_pages; i++) {
1042 err = mlx5_cmd_alloc_uar(dev->mdev, &bfregi->sys_pages[i]);
1043 if (err)
1044 goto error;
1045
1046 mlx5_ib_dbg(dev, "allocated uar %d\n", bfregi->sys_pages[i]);
1047 }
1048 return 0;
1049
1050error:
1051 for (--i; i >= 0; i--)
1052 if (mlx5_cmd_free_uar(dev->mdev, bfregi->sys_pages[i]))
1053 mlx5_ib_warn(dev, "failed to free uar %d\n", i);
1054
1055 return err;
1056}
1057
1058static int deallocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *context)
1059{
1060 struct mlx5_bfreg_info *bfregi;
1061 int err;
1062 int i;
1063
1064 bfregi = &context->bfregi;
1065 for (i = 0; i < bfregi->num_sys_pages; i++) {
1066 err = mlx5_cmd_free_uar(dev->mdev, bfregi->sys_pages[i]);
1067 if (err) {
1068 mlx5_ib_warn(dev, "failed to free uar %d\n", i);
1069 return err;
1070 }
1071 }
1072 return 0;
1073}
1074
995static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev, 1075static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
996 struct ib_udata *udata) 1076 struct ib_udata *udata)
997{ 1077{
@@ -999,17 +1079,13 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
999 struct mlx5_ib_alloc_ucontext_req_v2 req = {}; 1079 struct mlx5_ib_alloc_ucontext_req_v2 req = {};
1000 struct mlx5_ib_alloc_ucontext_resp resp = {}; 1080 struct mlx5_ib_alloc_ucontext_resp resp = {};
1001 struct mlx5_ib_ucontext *context; 1081 struct mlx5_ib_ucontext *context;
1002 struct mlx5_uuar_info *uuari; 1082 struct mlx5_bfreg_info *bfregi;
1003 struct mlx5_uar *uars;
1004 int gross_uuars;
1005 int num_uars;
1006 int ver; 1083 int ver;
1007 int uuarn;
1008 int err; 1084 int err;
1009 int i;
1010 size_t reqlen; 1085 size_t reqlen;
1011 size_t min_req_v2 = offsetof(struct mlx5_ib_alloc_ucontext_req_v2, 1086 size_t min_req_v2 = offsetof(struct mlx5_ib_alloc_ucontext_req_v2,
1012 max_cqe_version); 1087 max_cqe_version);
1088 bool lib_uar_4k;
1013 1089
1014 if (!dev->ib_active) 1090 if (!dev->ib_active)
1015 return ERR_PTR(-EAGAIN); 1091 return ERR_PTR(-EAGAIN);
@@ -1032,27 +1108,14 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
1032 if (req.flags) 1108 if (req.flags)
1033 return ERR_PTR(-EINVAL); 1109 return ERR_PTR(-EINVAL);
1034 1110
1035 if (req.total_num_uuars > MLX5_MAX_UUARS)
1036 return ERR_PTR(-ENOMEM);
1037
1038 if (req.total_num_uuars == 0)
1039 return ERR_PTR(-EINVAL);
1040
1041 if (req.comp_mask || req.reserved0 || req.reserved1 || req.reserved2) 1111 if (req.comp_mask || req.reserved0 || req.reserved1 || req.reserved2)
1042 return ERR_PTR(-EOPNOTSUPP); 1112 return ERR_PTR(-EOPNOTSUPP);
1043 1113
1044 if (reqlen > sizeof(req) && 1114 req.total_num_bfregs = ALIGN(req.total_num_bfregs,
1045 !ib_is_udata_cleared(udata, sizeof(req), 1115 MLX5_NON_FP_BFREGS_PER_UAR);
1046 reqlen - sizeof(req))) 1116 if (req.num_low_latency_bfregs > req.total_num_bfregs - 1)
1047 return ERR_PTR(-EOPNOTSUPP);
1048
1049 req.total_num_uuars = ALIGN(req.total_num_uuars,
1050 MLX5_NON_FP_BF_REGS_PER_PAGE);
1051 if (req.num_low_latency_uuars > req.total_num_uuars - 1)
1052 return ERR_PTR(-EINVAL); 1117 return ERR_PTR(-EINVAL);
1053 1118
1054 num_uars = req.total_num_uuars / MLX5_NON_FP_BF_REGS_PER_PAGE;
1055 gross_uuars = num_uars * MLX5_BF_REGS_PER_PAGE;
1056 resp.qp_tab_size = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp); 1119 resp.qp_tab_size = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp);
1057 if (mlx5_core_is_pf(dev->mdev) && MLX5_CAP_GEN(dev->mdev, bf)) 1120 if (mlx5_core_is_pf(dev->mdev) && MLX5_CAP_GEN(dev->mdev, bf))
1058 resp.bf_reg_size = 1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size); 1121 resp.bf_reg_size = 1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size);
@@ -1065,6 +1128,10 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
1065 resp.cqe_version = min_t(__u8, 1128 resp.cqe_version = min_t(__u8,
1066 (__u8)MLX5_CAP_GEN(dev->mdev, cqe_version), 1129 (__u8)MLX5_CAP_GEN(dev->mdev, cqe_version),
1067 req.max_cqe_version); 1130 req.max_cqe_version);
1131 resp.log_uar_size = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1132 MLX5_ADAPTER_PAGE_SHIFT : PAGE_SHIFT;
1133 resp.num_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1134 MLX5_CAP_GEN(dev->mdev, num_of_uars_per_page) : 1;
1068 resp.response_length = min(offsetof(typeof(resp), response_length) + 1135 resp.response_length = min(offsetof(typeof(resp), response_length) +
1069 sizeof(resp.response_length), udata->outlen); 1136 sizeof(resp.response_length), udata->outlen);
1070 1137
@@ -1072,41 +1139,34 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
1072 if (!context) 1139 if (!context)
1073 return ERR_PTR(-ENOMEM); 1140 return ERR_PTR(-ENOMEM);
1074 1141
1075 uuari = &context->uuari; 1142 lib_uar_4k = req.lib_caps & MLX5_LIB_CAP_4K_UAR;
1076 mutex_init(&uuari->lock); 1143 bfregi = &context->bfregi;
1077 uars = kcalloc(num_uars, sizeof(*uars), GFP_KERNEL); 1144
1078 if (!uars) { 1145 /* updates req->total_num_bfregs */
1079 err = -ENOMEM; 1146 err = calc_total_bfregs(dev, lib_uar_4k, &req, &bfregi->num_sys_pages);
1147 if (err)
1080 goto out_ctx; 1148 goto out_ctx;
1081 }
1082 1149
1083 uuari->bitmap = kcalloc(BITS_TO_LONGS(gross_uuars), 1150 mutex_init(&bfregi->lock);
1084 sizeof(*uuari->bitmap), 1151 bfregi->lib_uar_4k = lib_uar_4k;
1152 bfregi->count = kcalloc(req.total_num_bfregs, sizeof(*bfregi->count),
1085 GFP_KERNEL); 1153 GFP_KERNEL);
1086 if (!uuari->bitmap) { 1154 if (!bfregi->count) {
1087 err = -ENOMEM; 1155 err = -ENOMEM;
1088 goto out_uar_ctx; 1156 goto out_ctx;
1089 }
1090 /*
1091 * clear all fast path uuars
1092 */
1093 for (i = 0; i < gross_uuars; i++) {
1094 uuarn = i & 3;
1095 if (uuarn == 2 || uuarn == 3)
1096 set_bit(i, uuari->bitmap);
1097 } 1157 }
1098 1158
1099 uuari->count = kcalloc(gross_uuars, sizeof(*uuari->count), GFP_KERNEL); 1159 bfregi->sys_pages = kcalloc(bfregi->num_sys_pages,
1100 if (!uuari->count) { 1160 sizeof(*bfregi->sys_pages),
1161 GFP_KERNEL);
1162 if (!bfregi->sys_pages) {
1101 err = -ENOMEM; 1163 err = -ENOMEM;
1102 goto out_bitmap; 1164 goto out_count;
1103 } 1165 }
1104 1166
1105 for (i = 0; i < num_uars; i++) { 1167 err = allocate_uars(dev, context);
1106 err = mlx5_cmd_alloc_uar(dev->mdev, &uars[i].index); 1168 if (err)
1107 if (err) 1169 goto out_sys_pages;
1108 goto out_count;
1109 }
1110 1170
1111#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 1171#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1112 context->ibucontext.invalidate_range = &mlx5_ib_invalidate_range; 1172 context->ibucontext.invalidate_range = &mlx5_ib_invalidate_range;
@@ -1130,7 +1190,7 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
1130 INIT_LIST_HEAD(&context->db_page_list); 1190 INIT_LIST_HEAD(&context->db_page_list);
1131 mutex_init(&context->db_page_mutex); 1191 mutex_init(&context->db_page_mutex);
1132 1192
1133 resp.tot_uuars = req.total_num_uuars; 1193 resp.tot_bfregs = req.total_num_bfregs;
1134 resp.num_ports = MLX5_CAP_GEN(dev->mdev, num_ports); 1194 resp.num_ports = MLX5_CAP_GEN(dev->mdev, num_ports);
1135 1195
1136 if (field_avail(typeof(resp), cqe_version, udata->outlen)) 1196 if (field_avail(typeof(resp), cqe_version, udata->outlen))
@@ -1148,26 +1208,32 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
1148 * pretend we don't support reading the HCA's core clock. This is also 1208 * pretend we don't support reading the HCA's core clock. This is also
1149 * forced by mmap function. 1209 * forced by mmap function.
1150 */ 1210 */
1151 if (PAGE_SIZE <= 4096 && 1211 if (field_avail(typeof(resp), hca_core_clock_offset, udata->outlen)) {
1152 field_avail(typeof(resp), hca_core_clock_offset, udata->outlen)) { 1212 if (PAGE_SIZE <= 4096) {
1153 resp.comp_mask |= 1213 resp.comp_mask |=
1154 MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET; 1214 MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET;
1155 resp.hca_core_clock_offset = 1215 resp.hca_core_clock_offset =
1156 offsetof(struct mlx5_init_seg, internal_timer_h) % 1216 offsetof(struct mlx5_init_seg, internal_timer_h) % PAGE_SIZE;
1157 PAGE_SIZE; 1217 }
1158 resp.response_length += sizeof(resp.hca_core_clock_offset) + 1218 resp.response_length += sizeof(resp.hca_core_clock_offset) +
1159 sizeof(resp.reserved2); 1219 sizeof(resp.reserved2);
1160 } 1220 }
1161 1221
1222 if (field_avail(typeof(resp), log_uar_size, udata->outlen))
1223 resp.response_length += sizeof(resp.log_uar_size);
1224
1225 if (field_avail(typeof(resp), num_uars_per_page, udata->outlen))
1226 resp.response_length += sizeof(resp.num_uars_per_page);
1227
1162 err = ib_copy_to_udata(udata, &resp, resp.response_length); 1228 err = ib_copy_to_udata(udata, &resp, resp.response_length);
1163 if (err) 1229 if (err)
1164 goto out_td; 1230 goto out_td;
1165 1231
1166 uuari->ver = ver; 1232 bfregi->ver = ver;
1167 uuari->num_low_latency_uuars = req.num_low_latency_uuars; 1233 bfregi->num_low_latency_bfregs = req.num_low_latency_bfregs;
1168 uuari->uars = uars;
1169 uuari->num_uars = num_uars;
1170 context->cqe_version = resp.cqe_version; 1234 context->cqe_version = resp.cqe_version;
1235 context->lib_caps = req.lib_caps;
1236 print_lib_caps(dev, context->lib_caps);
1171 1237
1172 return &context->ibucontext; 1238 return &context->ibucontext;
1173 1239
@@ -1179,19 +1245,17 @@ out_page:
1179 free_page(context->upd_xlt_page); 1245 free_page(context->upd_xlt_page);
1180 1246
1181out_uars: 1247out_uars:
1182 for (i--; i >= 0; i--) 1248 deallocate_uars(dev, context);
1183 mlx5_cmd_free_uar(dev->mdev, uars[i].index);
1184out_count:
1185 kfree(uuari->count);
1186 1249
1187out_bitmap: 1250out_sys_pages:
1188 kfree(uuari->bitmap); 1251 kfree(bfregi->sys_pages);
1189 1252
1190out_uar_ctx: 1253out_count:
1191 kfree(uars); 1254 kfree(bfregi->count);
1192 1255
1193out_ctx: 1256out_ctx:
1194 kfree(context); 1257 kfree(context);
1258
1195 return ERR_PTR(err); 1259 return ERR_PTR(err);
1196} 1260}
1197 1261
@@ -1199,30 +1263,31 @@ static int mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
1199{ 1263{
1200 struct mlx5_ib_ucontext *context = to_mucontext(ibcontext); 1264 struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
1201 struct mlx5_ib_dev *dev = to_mdev(ibcontext->device); 1265 struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
1202 struct mlx5_uuar_info *uuari = &context->uuari; 1266 struct mlx5_bfreg_info *bfregi;
1203 int i;
1204 1267
1268 bfregi = &context->bfregi;
1205 if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain)) 1269 if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
1206 mlx5_core_dealloc_transport_domain(dev->mdev, context->tdn); 1270 mlx5_core_dealloc_transport_domain(dev->mdev, context->tdn);
1207 1271
1208 free_page(context->upd_xlt_page); 1272 free_page(context->upd_xlt_page);
1209 1273 deallocate_uars(dev, context);
1210 for (i = 0; i < uuari->num_uars; i++) { 1274 kfree(bfregi->sys_pages);
1211 if (mlx5_cmd_free_uar(dev->mdev, uuari->uars[i].index)) 1275 kfree(bfregi->count);
1212 mlx5_ib_warn(dev, "failed to free UAR 0x%x\n", uuari->uars[i].index);
1213 }
1214
1215 kfree(uuari->count);
1216 kfree(uuari->bitmap);
1217 kfree(uuari->uars);
1218 kfree(context); 1276 kfree(context);
1219 1277
1220 return 0; 1278 return 0;
1221} 1279}
1222 1280
1223static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev, int index) 1281static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev,
1282 struct mlx5_bfreg_info *bfregi,
1283 int idx)
1224{ 1284{
1225 return (pci_resource_start(dev->mdev->pdev, 0) >> PAGE_SHIFT) + index; 1285 int fw_uars_per_page;
1286
1287 fw_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ? MLX5_UARS_IN_PAGE : 1;
1288
1289 return (pci_resource_start(dev->mdev->pdev, 0) >> PAGE_SHIFT) +
1290 bfregi->sys_pages[idx] / fw_uars_per_page;
1226} 1291}
1227 1292
1228static int get_command(unsigned long offset) 1293static int get_command(unsigned long offset)
@@ -1377,11 +1442,23 @@ static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
1377 struct vm_area_struct *vma, 1442 struct vm_area_struct *vma,
1378 struct mlx5_ib_ucontext *context) 1443 struct mlx5_ib_ucontext *context)
1379{ 1444{
1380 struct mlx5_uuar_info *uuari = &context->uuari; 1445 struct mlx5_bfreg_info *bfregi = &context->bfregi;
1381 int err; 1446 int err;
1382 unsigned long idx; 1447 unsigned long idx;
1383 phys_addr_t pfn, pa; 1448 phys_addr_t pfn, pa;
1384 pgprot_t prot; 1449 pgprot_t prot;
1450 int uars_per_page;
1451
1452 if (vma->vm_end - vma->vm_start != PAGE_SIZE)
1453 return -EINVAL;
1454
1455 uars_per_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k);
1456 idx = get_index(vma->vm_pgoff);
1457 if (idx % uars_per_page ||
1458 idx * uars_per_page >= bfregi->num_sys_pages) {
1459 mlx5_ib_warn(dev, "invalid uar index %lu\n", idx);
1460 return -EINVAL;
1461 }
1385 1462
1386 switch (cmd) { 1463 switch (cmd) {
1387 case MLX5_IB_MMAP_WC_PAGE: 1464 case MLX5_IB_MMAP_WC_PAGE:
@@ -1404,14 +1481,7 @@ static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
1404 return -EINVAL; 1481 return -EINVAL;
1405 } 1482 }
1406 1483
1407 if (vma->vm_end - vma->vm_start != PAGE_SIZE) 1484 pfn = uar_index2pfn(dev, bfregi, idx);
1408 return -EINVAL;
1409
1410 idx = get_index(vma->vm_pgoff);
1411 if (idx >= uuari->num_uars)
1412 return -EINVAL;
1413
1414 pfn = uar_index2pfn(dev, uuari->uars[idx].index);
1415 mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn %pa\n", idx, &pfn); 1485 mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn %pa\n", idx, &pfn);
1416 1486
1417 vma->vm_page_prot = prot; 1487 vma->vm_page_prot = prot;
@@ -3072,8 +3142,6 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
3072 if (mlx5_use_mad_ifc(dev)) 3142 if (mlx5_use_mad_ifc(dev))
3073 get_ext_port_caps(dev); 3143 get_ext_port_caps(dev);
3074 3144
3075 MLX5_INIT_DOORBELL_LOCK(&dev->uar_lock);
3076
3077 if (!mlx5_lag_is_active(mdev)) 3145 if (!mlx5_lag_is_active(mdev))
3078 name = "mlx5_%d"; 3146 name = "mlx5_%d";
3079 else 3147 else
@@ -3249,9 +3317,21 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
3249 if (err) 3317 if (err)
3250 goto err_odp; 3318 goto err_odp;
3251 3319
3320 dev->mdev->priv.uar = mlx5_get_uars_page(dev->mdev);
3321 if (!dev->mdev->priv.uar)
3322 goto err_q_cnt;
3323
3324 err = mlx5_alloc_bfreg(dev->mdev, &dev->bfreg, false, false);
3325 if (err)
3326 goto err_uar_page;
3327
3328 err = mlx5_alloc_bfreg(dev->mdev, &dev->fp_bfreg, false, true);
3329 if (err)
3330 goto err_bfreg;
3331
3252 err = ib_register_device(&dev->ib_dev, NULL); 3332 err = ib_register_device(&dev->ib_dev, NULL);
3253 if (err) 3333 if (err)
3254 goto err_q_cnt; 3334 goto err_fp_bfreg;
3255 3335
3256 err = create_umr_res(dev); 3336 err = create_umr_res(dev);
3257 if (err) 3337 if (err)
@@ -3274,6 +3354,15 @@ err_umrc:
3274err_dev: 3354err_dev:
3275 ib_unregister_device(&dev->ib_dev); 3355 ib_unregister_device(&dev->ib_dev);
3276 3356
3357err_fp_bfreg:
3358 mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
3359
3360err_bfreg:
3361 mlx5_free_bfreg(dev->mdev, &dev->bfreg);
3362
3363err_uar_page:
3364 mlx5_put_uars_page(dev->mdev, dev->mdev->priv.uar);
3365
3277err_q_cnt: 3366err_q_cnt:
3278 mlx5_ib_dealloc_q_counters(dev); 3367 mlx5_ib_dealloc_q_counters(dev);
3279 3368
@@ -3305,6 +3394,9 @@ static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context)
3305 3394
3306 mlx5_remove_netdev_notifier(dev); 3395 mlx5_remove_netdev_notifier(dev);
3307 ib_unregister_device(&dev->ib_dev); 3396 ib_unregister_device(&dev->ib_dev);
3397 mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
3398 mlx5_free_bfreg(dev->mdev, &dev->bfreg);
3399 mlx5_put_uars_page(dev->mdev, mdev->priv.uar);
3308 mlx5_ib_dealloc_q_counters(dev); 3400 mlx5_ib_dealloc_q_counters(dev);
3309 destroy_umrc_res(dev); 3401 destroy_umrc_res(dev);
3310 mlx5_ib_odp_remove_one(dev); 3402 mlx5_ib_odp_remove_one(dev);
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index a51c8051aeb2..e1a4b93dce6b 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -90,7 +90,6 @@ enum mlx5_ib_latency_class {
90 MLX5_IB_LATENCY_CLASS_LOW, 90 MLX5_IB_LATENCY_CLASS_LOW,
91 MLX5_IB_LATENCY_CLASS_MEDIUM, 91 MLX5_IB_LATENCY_CLASS_MEDIUM,
92 MLX5_IB_LATENCY_CLASS_HIGH, 92 MLX5_IB_LATENCY_CLASS_HIGH,
93 MLX5_IB_LATENCY_CLASS_FAST_PATH
94}; 93};
95 94
96enum mlx5_ib_mad_ifc_flags { 95enum mlx5_ib_mad_ifc_flags {
@@ -100,7 +99,7 @@ enum mlx5_ib_mad_ifc_flags {
100}; 99};
101 100
102enum { 101enum {
103 MLX5_CROSS_CHANNEL_UUAR = 0, 102 MLX5_CROSS_CHANNEL_BFREG = 0,
104}; 103};
105 104
106enum { 105enum {
@@ -120,7 +119,7 @@ struct mlx5_ib_ucontext {
120 /* protect doorbell record alloc/free 119 /* protect doorbell record alloc/free
121 */ 120 */
122 struct mutex db_page_mutex; 121 struct mutex db_page_mutex;
123 struct mlx5_uuar_info uuari; 122 struct mlx5_bfreg_info bfregi;
124 u8 cqe_version; 123 u8 cqe_version;
125 /* Transport Domain number */ 124 /* Transport Domain number */
126 u32 tdn; 125 u32 tdn;
@@ -129,6 +128,7 @@ struct mlx5_ib_ucontext {
129 unsigned long upd_xlt_page; 128 unsigned long upd_xlt_page;
130 /* protect ODP/KSM */ 129 /* protect ODP/KSM */
131 struct mutex upd_xlt_page_mutex; 130 struct mutex upd_xlt_page_mutex;
131 u64 lib_caps;
132}; 132};
133 133
134static inline struct mlx5_ib_ucontext *to_mucontext(struct ib_ucontext *ibucontext) 134static inline struct mlx5_ib_ucontext *to_mucontext(struct ib_ucontext *ibucontext)
@@ -324,6 +324,12 @@ struct mlx5_ib_raw_packet_qp {
324 struct mlx5_ib_rq rq; 324 struct mlx5_ib_rq rq;
325}; 325};
326 326
327struct mlx5_bf {
328 int buf_size;
329 unsigned long offset;
330 struct mlx5_sq_bfreg *bfreg;
331};
332
327struct mlx5_ib_qp { 333struct mlx5_ib_qp {
328 struct ib_qp ibqp; 334 struct ib_qp ibqp;
329 union { 335 union {
@@ -349,13 +355,13 @@ struct mlx5_ib_qp {
349 int wq_sig; 355 int wq_sig;
350 int scat_cqe; 356 int scat_cqe;
351 int max_inline_data; 357 int max_inline_data;
352 struct mlx5_bf *bf; 358 struct mlx5_bf bf;
353 int has_rq; 359 int has_rq;
354 360
355 /* only for user space QPs. For kernel 361 /* only for user space QPs. For kernel
356 * we have it from the bf object 362 * we have it from the bf object
357 */ 363 */
358 int uuarn; 364 int bfregn;
359 365
360 int create_type; 366 int create_type;
361 367
@@ -591,7 +597,6 @@ struct mlx5_ib_dev {
591 struct ib_device ib_dev; 597 struct ib_device ib_dev;
592 struct mlx5_core_dev *mdev; 598 struct mlx5_core_dev *mdev;
593 struct mlx5_roce roce; 599 struct mlx5_roce roce;
594 MLX5_DECLARE_DOORBELL_LOCK(uar_lock);
595 int num_ports; 600 int num_ports;
596 /* serialize update of capability mask 601 /* serialize update of capability mask
597 */ 602 */
@@ -621,6 +626,8 @@ struct mlx5_ib_dev {
621 struct list_head qp_list; 626 struct list_head qp_list;
622 /* Array with num_ports elements */ 627 /* Array with num_ports elements */
623 struct mlx5_ib_port *port; 628 struct mlx5_ib_port *port;
629 struct mlx5_sq_bfreg bfreg;
630 struct mlx5_sq_bfreg fp_bfreg;
624}; 631};
625 632
626static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq) 633static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)
@@ -968,4 +975,17 @@ static inline int get_srq_user_index(struct mlx5_ib_ucontext *ucontext,
968 975
969 return verify_assign_uidx(cqe_version, ucmd->uidx, user_index); 976 return verify_assign_uidx(cqe_version, ucmd->uidx, user_index);
970} 977}
978
979static inline int get_uars_per_sys_page(struct mlx5_ib_dev *dev, bool lib_support)
980{
981 return lib_support && MLX5_CAP_GEN(dev->mdev, uar_4k) ?
982 MLX5_UARS_IN_PAGE : 1;
983}
984
985static inline int get_num_uars(struct mlx5_ib_dev *dev,
986 struct mlx5_bfreg_info *bfregi)
987{
988 return get_uars_per_sys_page(dev, bfregi->lib_uar_4k) * bfregi->num_sys_pages;
989}
990
971#endif /* MLX5_IB_H */ 991#endif /* MLX5_IB_H */
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index 53f4dd32f956..6a83fb32599d 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -475,60 +475,53 @@ static int qp_has_rq(struct ib_qp_init_attr *attr)
475 return 1; 475 return 1;
476} 476}
477 477
478static int first_med_uuar(void) 478static int first_med_bfreg(void)
479{ 479{
480 return 1; 480 return 1;
481} 481}
482 482
483static int next_uuar(int n) 483enum {
484{ 484 /* this is the first blue flame register in the array of bfregs assigned
485 n++; 485 * to a processes. Since we do not use it for blue flame but rather
486 486 * regular 64 bit doorbells, we do not need a lock for maintaiing
487 while (((n % 4) & 2)) 487 * "odd/even" order
488 n++; 488 */
489 NUM_NON_BLUE_FLAME_BFREGS = 1,
490};
489 491
490 return n; 492static int max_bfregs(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi)
493{
494 return get_num_uars(dev, bfregi) * MLX5_NON_FP_BFREGS_PER_UAR;
491} 495}
492 496
493static int num_med_uuar(struct mlx5_uuar_info *uuari) 497static int num_med_bfreg(struct mlx5_ib_dev *dev,
498 struct mlx5_bfreg_info *bfregi)
494{ 499{
495 int n; 500 int n;
496 501
497 n = uuari->num_uars * MLX5_NON_FP_BF_REGS_PER_PAGE - 502 n = max_bfregs(dev, bfregi) - bfregi->num_low_latency_bfregs -
498 uuari->num_low_latency_uuars - 1; 503 NUM_NON_BLUE_FLAME_BFREGS;
499 504
500 return n >= 0 ? n : 0; 505 return n >= 0 ? n : 0;
501} 506}
502 507
503static int max_uuari(struct mlx5_uuar_info *uuari) 508static int first_hi_bfreg(struct mlx5_ib_dev *dev,
504{ 509 struct mlx5_bfreg_info *bfregi)
505 return uuari->num_uars * 4;
506}
507
508static int first_hi_uuar(struct mlx5_uuar_info *uuari)
509{ 510{
510 int med; 511 int med;
511 int i;
512 int t;
513
514 med = num_med_uuar(uuari);
515 for (t = 0, i = first_med_uuar();; i = next_uuar(i)) {
516 t++;
517 if (t == med)
518 return next_uuar(i);
519 }
520 512
521 return 0; 513 med = num_med_bfreg(dev, bfregi);
514 return ++med;
522} 515}
523 516
524static int alloc_high_class_uuar(struct mlx5_uuar_info *uuari) 517static int alloc_high_class_bfreg(struct mlx5_ib_dev *dev,
518 struct mlx5_bfreg_info *bfregi)
525{ 519{
526 int i; 520 int i;
527 521
528 for (i = first_hi_uuar(uuari); i < max_uuari(uuari); i = next_uuar(i)) { 522 for (i = first_hi_bfreg(dev, bfregi); i < max_bfregs(dev, bfregi); i++) {
529 if (!test_bit(i, uuari->bitmap)) { 523 if (!bfregi->count[i]) {
530 set_bit(i, uuari->bitmap); 524 bfregi->count[i]++;
531 uuari->count[i]++;
532 return i; 525 return i;
533 } 526 }
534 } 527 }
@@ -536,87 +529,61 @@ static int alloc_high_class_uuar(struct mlx5_uuar_info *uuari)
536 return -ENOMEM; 529 return -ENOMEM;
537} 530}
538 531
539static int alloc_med_class_uuar(struct mlx5_uuar_info *uuari) 532static int alloc_med_class_bfreg(struct mlx5_ib_dev *dev,
533 struct mlx5_bfreg_info *bfregi)
540{ 534{
541 int minidx = first_med_uuar(); 535 int minidx = first_med_bfreg();
542 int i; 536 int i;
543 537
544 for (i = first_med_uuar(); i < first_hi_uuar(uuari); i = next_uuar(i)) { 538 for (i = first_med_bfreg(); i < first_hi_bfreg(dev, bfregi); i++) {
545 if (uuari->count[i] < uuari->count[minidx]) 539 if (bfregi->count[i] < bfregi->count[minidx])
546 minidx = i; 540 minidx = i;
541 if (!bfregi->count[minidx])
542 break;
547 } 543 }
548 544
549 uuari->count[minidx]++; 545 bfregi->count[minidx]++;
550 return minidx; 546 return minidx;
551} 547}
552 548
553static int alloc_uuar(struct mlx5_uuar_info *uuari, 549static int alloc_bfreg(struct mlx5_ib_dev *dev,
554 enum mlx5_ib_latency_class lat) 550 struct mlx5_bfreg_info *bfregi,
551 enum mlx5_ib_latency_class lat)
555{ 552{
556 int uuarn = -EINVAL; 553 int bfregn = -EINVAL;
557 554
558 mutex_lock(&uuari->lock); 555 mutex_lock(&bfregi->lock);
559 switch (lat) { 556 switch (lat) {
560 case MLX5_IB_LATENCY_CLASS_LOW: 557 case MLX5_IB_LATENCY_CLASS_LOW:
561 uuarn = 0; 558 BUILD_BUG_ON(NUM_NON_BLUE_FLAME_BFREGS != 1);
562 uuari->count[uuarn]++; 559 bfregn = 0;
560 bfregi->count[bfregn]++;
563 break; 561 break;
564 562
565 case MLX5_IB_LATENCY_CLASS_MEDIUM: 563 case MLX5_IB_LATENCY_CLASS_MEDIUM:
566 if (uuari->ver < 2) 564 if (bfregi->ver < 2)
567 uuarn = -ENOMEM; 565 bfregn = -ENOMEM;
568 else 566 else
569 uuarn = alloc_med_class_uuar(uuari); 567 bfregn = alloc_med_class_bfreg(dev, bfregi);
570 break; 568 break;
571 569
572 case MLX5_IB_LATENCY_CLASS_HIGH: 570 case MLX5_IB_LATENCY_CLASS_HIGH:
573 if (uuari->ver < 2) 571 if (bfregi->ver < 2)
574 uuarn = -ENOMEM; 572 bfregn = -ENOMEM;
575 else 573 else
576 uuarn = alloc_high_class_uuar(uuari); 574 bfregn = alloc_high_class_bfreg(dev, bfregi);
577 break;
578
579 case MLX5_IB_LATENCY_CLASS_FAST_PATH:
580 uuarn = 2;
581 break; 575 break;
582 } 576 }
583 mutex_unlock(&uuari->lock); 577 mutex_unlock(&bfregi->lock);
584
585 return uuarn;
586}
587 578
588static void free_med_class_uuar(struct mlx5_uuar_info *uuari, int uuarn) 579 return bfregn;
589{
590 clear_bit(uuarn, uuari->bitmap);
591 --uuari->count[uuarn];
592} 580}
593 581
594static void free_high_class_uuar(struct mlx5_uuar_info *uuari, int uuarn) 582static void free_bfreg(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi, int bfregn)
595{ 583{
596 clear_bit(uuarn, uuari->bitmap); 584 mutex_lock(&bfregi->lock);
597 --uuari->count[uuarn]; 585 bfregi->count[bfregn]--;
598} 586 mutex_unlock(&bfregi->lock);
599
600static void free_uuar(struct mlx5_uuar_info *uuari, int uuarn)
601{
602 int nuuars = uuari->num_uars * MLX5_BF_REGS_PER_PAGE;
603 int high_uuar = nuuars - uuari->num_low_latency_uuars;
604
605 mutex_lock(&uuari->lock);
606 if (uuarn == 0) {
607 --uuari->count[uuarn];
608 goto out;
609 }
610
611 if (uuarn < high_uuar) {
612 free_med_class_uuar(uuari, uuarn);
613 goto out;
614 }
615
616 free_high_class_uuar(uuari, uuarn);
617
618out:
619 mutex_unlock(&uuari->lock);
620} 587}
621 588
622static enum mlx5_qp_state to_mlx5_state(enum ib_qp_state state) 589static enum mlx5_qp_state to_mlx5_state(enum ib_qp_state state)
@@ -657,9 +624,20 @@ static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq,
657static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq, 624static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq,
658 struct mlx5_ib_cq *recv_cq); 625 struct mlx5_ib_cq *recv_cq);
659 626
660static int uuarn_to_uar_index(struct mlx5_uuar_info *uuari, int uuarn) 627static int bfregn_to_uar_index(struct mlx5_ib_dev *dev,
628 struct mlx5_bfreg_info *bfregi, int bfregn)
661{ 629{
662 return uuari->uars[uuarn / MLX5_BF_REGS_PER_PAGE].index; 630 int bfregs_per_sys_page;
631 int index_of_sys_page;
632 int offset;
633
634 bfregs_per_sys_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k) *
635 MLX5_NON_FP_BFREGS_PER_UAR;
636 index_of_sys_page = bfregn / bfregs_per_sys_page;
637
638 offset = bfregn % bfregs_per_sys_page / MLX5_NON_FP_BFREGS_PER_UAR;
639
640 return bfregi->sys_pages[index_of_sys_page] + offset;
663} 641}
664 642
665static int mlx5_ib_umem_get(struct mlx5_ib_dev *dev, 643static int mlx5_ib_umem_get(struct mlx5_ib_dev *dev,
@@ -762,6 +740,13 @@ err_umem:
762 return err; 740 return err;
763} 741}
764 742
743static int adjust_bfregn(struct mlx5_ib_dev *dev,
744 struct mlx5_bfreg_info *bfregi, int bfregn)
745{
746 return bfregn / MLX5_NON_FP_BFREGS_PER_UAR * MLX5_BFREGS_PER_UAR +
747 bfregn % MLX5_NON_FP_BFREGS_PER_UAR;
748}
749
765static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, 750static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
766 struct mlx5_ib_qp *qp, struct ib_udata *udata, 751 struct mlx5_ib_qp *qp, struct ib_udata *udata,
767 struct ib_qp_init_attr *attr, 752 struct ib_qp_init_attr *attr,
@@ -776,7 +761,7 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
776 int uar_index; 761 int uar_index;
777 int npages; 762 int npages;
778 u32 offset = 0; 763 u32 offset = 0;
779 int uuarn; 764 int bfregn;
780 int ncont = 0; 765 int ncont = 0;
781 __be64 *pas; 766 __be64 *pas;
782 void *qpc; 767 void *qpc;
@@ -794,27 +779,27 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
794 */ 779 */
795 if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL) 780 if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL)
796 /* In CROSS_CHANNEL CQ and QP must use the same UAR */ 781 /* In CROSS_CHANNEL CQ and QP must use the same UAR */
797 uuarn = MLX5_CROSS_CHANNEL_UUAR; 782 bfregn = MLX5_CROSS_CHANNEL_BFREG;
798 else { 783 else {
799 uuarn = alloc_uuar(&context->uuari, MLX5_IB_LATENCY_CLASS_HIGH); 784 bfregn = alloc_bfreg(dev, &context->bfregi, MLX5_IB_LATENCY_CLASS_HIGH);
800 if (uuarn < 0) { 785 if (bfregn < 0) {
801 mlx5_ib_dbg(dev, "failed to allocate low latency UUAR\n"); 786 mlx5_ib_dbg(dev, "failed to allocate low latency BFREG\n");
802 mlx5_ib_dbg(dev, "reverting to medium latency\n"); 787 mlx5_ib_dbg(dev, "reverting to medium latency\n");
803 uuarn = alloc_uuar(&context->uuari, MLX5_IB_LATENCY_CLASS_MEDIUM); 788 bfregn = alloc_bfreg(dev, &context->bfregi, MLX5_IB_LATENCY_CLASS_MEDIUM);
804 if (uuarn < 0) { 789 if (bfregn < 0) {
805 mlx5_ib_dbg(dev, "failed to allocate medium latency UUAR\n"); 790 mlx5_ib_dbg(dev, "failed to allocate medium latency BFREG\n");
806 mlx5_ib_dbg(dev, "reverting to high latency\n"); 791 mlx5_ib_dbg(dev, "reverting to high latency\n");
807 uuarn = alloc_uuar(&context->uuari, MLX5_IB_LATENCY_CLASS_LOW); 792 bfregn = alloc_bfreg(dev, &context->bfregi, MLX5_IB_LATENCY_CLASS_LOW);
808 if (uuarn < 0) { 793 if (bfregn < 0) {
809 mlx5_ib_warn(dev, "uuar allocation failed\n"); 794 mlx5_ib_warn(dev, "bfreg allocation failed\n");
810 return uuarn; 795 return bfregn;
811 } 796 }
812 } 797 }
813 } 798 }
814 } 799 }
815 800
816 uar_index = uuarn_to_uar_index(&context->uuari, uuarn); 801 uar_index = bfregn_to_uar_index(dev, &context->bfregi, bfregn);
817 mlx5_ib_dbg(dev, "uuarn 0x%x, uar_index 0x%x\n", uuarn, uar_index); 802 mlx5_ib_dbg(dev, "bfregn 0x%x, uar_index 0x%x\n", bfregn, uar_index);
818 803
819 qp->rq.offset = 0; 804 qp->rq.offset = 0;
820 qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB); 805 qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
@@ -822,7 +807,7 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
822 807
823 err = set_user_buf_size(dev, qp, &ucmd, base, attr); 808 err = set_user_buf_size(dev, qp, &ucmd, base, attr);
824 if (err) 809 if (err)
825 goto err_uuar; 810 goto err_bfreg;
826 811
827 if (ucmd.buf_addr && ubuffer->buf_size) { 812 if (ucmd.buf_addr && ubuffer->buf_size) {
828 ubuffer->buf_addr = ucmd.buf_addr; 813 ubuffer->buf_addr = ucmd.buf_addr;
@@ -831,7 +816,7 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
831 &ubuffer->umem, &npages, &page_shift, 816 &ubuffer->umem, &npages, &page_shift,
832 &ncont, &offset); 817 &ncont, &offset);
833 if (err) 818 if (err)
834 goto err_uuar; 819 goto err_bfreg;
835 } else { 820 } else {
836 ubuffer->umem = NULL; 821 ubuffer->umem = NULL;
837 } 822 }
@@ -854,8 +839,8 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
854 MLX5_SET(qpc, qpc, page_offset, offset); 839 MLX5_SET(qpc, qpc, page_offset, offset);
855 840
856 MLX5_SET(qpc, qpc, uar_page, uar_index); 841 MLX5_SET(qpc, qpc, uar_page, uar_index);
857 resp->uuar_index = uuarn; 842 resp->bfreg_index = adjust_bfregn(dev, &context->bfregi, bfregn);
858 qp->uuarn = uuarn; 843 qp->bfregn = bfregn;
859 844
860 err = mlx5_ib_db_map_user(context, ucmd.db_addr, &qp->db); 845 err = mlx5_ib_db_map_user(context, ucmd.db_addr, &qp->db);
861 if (err) { 846 if (err) {
@@ -882,13 +867,13 @@ err_umem:
882 if (ubuffer->umem) 867 if (ubuffer->umem)
883 ib_umem_release(ubuffer->umem); 868 ib_umem_release(ubuffer->umem);
884 869
885err_uuar: 870err_bfreg:
886 free_uuar(&context->uuari, uuarn); 871 free_bfreg(dev, &context->bfregi, bfregn);
887 return err; 872 return err;
888} 873}
889 874
890static void destroy_qp_user(struct ib_pd *pd, struct mlx5_ib_qp *qp, 875static void destroy_qp_user(struct mlx5_ib_dev *dev, struct ib_pd *pd,
891 struct mlx5_ib_qp_base *base) 876 struct mlx5_ib_qp *qp, struct mlx5_ib_qp_base *base)
892{ 877{
893 struct mlx5_ib_ucontext *context; 878 struct mlx5_ib_ucontext *context;
894 879
@@ -896,7 +881,7 @@ static void destroy_qp_user(struct ib_pd *pd, struct mlx5_ib_qp *qp,
896 mlx5_ib_db_unmap_user(context, &qp->db); 881 mlx5_ib_db_unmap_user(context, &qp->db);
897 if (base->ubuffer.umem) 882 if (base->ubuffer.umem)
898 ib_umem_release(base->ubuffer.umem); 883 ib_umem_release(base->ubuffer.umem);
899 free_uuar(&context->uuari, qp->uuarn); 884 free_bfreg(dev, &context->bfregi, qp->bfregn);
900} 885}
901 886
902static int create_kernel_qp(struct mlx5_ib_dev *dev, 887static int create_kernel_qp(struct mlx5_ib_dev *dev,
@@ -905,14 +890,10 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev,
905 u32 **in, int *inlen, 890 u32 **in, int *inlen,
906 struct mlx5_ib_qp_base *base) 891 struct mlx5_ib_qp_base *base)
907{ 892{
908 enum mlx5_ib_latency_class lc = MLX5_IB_LATENCY_CLASS_LOW;
909 struct mlx5_uuar_info *uuari;
910 int uar_index; 893 int uar_index;
911 void *qpc; 894 void *qpc;
912 int uuarn;
913 int err; 895 int err;
914 896
915 uuari = &dev->mdev->priv.uuari;
916 if (init_attr->create_flags & ~(IB_QP_CREATE_SIGNATURE_EN | 897 if (init_attr->create_flags & ~(IB_QP_CREATE_SIGNATURE_EN |
917 IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK | 898 IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK |
918 IB_QP_CREATE_IPOIB_UD_LSO | 899 IB_QP_CREATE_IPOIB_UD_LSO |
@@ -920,21 +901,17 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev,
920 return -EINVAL; 901 return -EINVAL;
921 902
922 if (init_attr->qp_type == MLX5_IB_QPT_REG_UMR) 903 if (init_attr->qp_type == MLX5_IB_QPT_REG_UMR)
923 lc = MLX5_IB_LATENCY_CLASS_FAST_PATH; 904 qp->bf.bfreg = &dev->fp_bfreg;
924 905 else
925 uuarn = alloc_uuar(uuari, lc); 906 qp->bf.bfreg = &dev->bfreg;
926 if (uuarn < 0) {
927 mlx5_ib_dbg(dev, "\n");
928 return -ENOMEM;
929 }
930 907
931 qp->bf = &uuari->bfs[uuarn]; 908 qp->bf.buf_size = 1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size);
932 uar_index = qp->bf->uar->index; 909 uar_index = qp->bf.bfreg->index;
933 910
934 err = calc_sq_size(dev, init_attr, qp); 911 err = calc_sq_size(dev, init_attr, qp);
935 if (err < 0) { 912 if (err < 0) {
936 mlx5_ib_dbg(dev, "err %d\n", err); 913 mlx5_ib_dbg(dev, "err %d\n", err);
937 goto err_uuar; 914 return err;
938 } 915 }
939 916
940 qp->rq.offset = 0; 917 qp->rq.offset = 0;
@@ -944,7 +921,7 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev,
944 err = mlx5_buf_alloc(dev->mdev, base->ubuffer.buf_size, &qp->buf); 921 err = mlx5_buf_alloc(dev->mdev, base->ubuffer.buf_size, &qp->buf);
945 if (err) { 922 if (err) {
946 mlx5_ib_dbg(dev, "err %d\n", err); 923 mlx5_ib_dbg(dev, "err %d\n", err);
947 goto err_uuar; 924 return err;
948 } 925 }
949 926
950 qp->sq.qend = mlx5_get_send_wqe(qp, qp->sq.wqe_cnt); 927 qp->sq.qend = mlx5_get_send_wqe(qp, qp->sq.wqe_cnt);
@@ -994,34 +971,30 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev,
994 return 0; 971 return 0;
995 972
996err_wrid: 973err_wrid:
997 mlx5_db_free(dev->mdev, &qp->db);
998 kfree(qp->sq.wqe_head); 974 kfree(qp->sq.wqe_head);
999 kfree(qp->sq.w_list); 975 kfree(qp->sq.w_list);
1000 kfree(qp->sq.wrid); 976 kfree(qp->sq.wrid);
1001 kfree(qp->sq.wr_data); 977 kfree(qp->sq.wr_data);
1002 kfree(qp->rq.wrid); 978 kfree(qp->rq.wrid);
979 mlx5_db_free(dev->mdev, &qp->db);
1003 980
1004err_free: 981err_free:
1005 kvfree(*in); 982 kvfree(*in);
1006 983
1007err_buf: 984err_buf:
1008 mlx5_buf_free(dev->mdev, &qp->buf); 985 mlx5_buf_free(dev->mdev, &qp->buf);
1009
1010err_uuar:
1011 free_uuar(&dev->mdev->priv.uuari, uuarn);
1012 return err; 986 return err;
1013} 987}
1014 988
1015static void destroy_qp_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp) 989static void destroy_qp_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
1016{ 990{
1017 mlx5_db_free(dev->mdev, &qp->db);
1018 kfree(qp->sq.wqe_head); 991 kfree(qp->sq.wqe_head);
1019 kfree(qp->sq.w_list); 992 kfree(qp->sq.w_list);
1020 kfree(qp->sq.wrid); 993 kfree(qp->sq.wrid);
1021 kfree(qp->sq.wr_data); 994 kfree(qp->sq.wr_data);
1022 kfree(qp->rq.wrid); 995 kfree(qp->rq.wrid);
996 mlx5_db_free(dev->mdev, &qp->db);
1023 mlx5_buf_free(dev->mdev, &qp->buf); 997 mlx5_buf_free(dev->mdev, &qp->buf);
1024 free_uuar(&dev->mdev->priv.uuari, qp->bf->uuarn);
1025} 998}
1026 999
1027static u32 get_rx_type(struct mlx5_ib_qp *qp, struct ib_qp_init_attr *attr) 1000static u32 get_rx_type(struct mlx5_ib_qp *qp, struct ib_qp_init_attr *attr)
@@ -1353,7 +1326,7 @@ static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1353 if (init_attr->create_flags || init_attr->send_cq) 1326 if (init_attr->create_flags || init_attr->send_cq)
1354 return -EINVAL; 1327 return -EINVAL;
1355 1328
1356 min_resp_len = offsetof(typeof(resp), uuar_index) + sizeof(resp.uuar_index); 1329 min_resp_len = offsetof(typeof(resp), bfreg_index) + sizeof(resp.bfreg_index);
1357 if (udata->outlen < min_resp_len) 1330 if (udata->outlen < min_resp_len)
1358 return -EINVAL; 1331 return -EINVAL;
1359 1332
@@ -1792,7 +1765,7 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
1792 1765
1793err_create: 1766err_create:
1794 if (qp->create_type == MLX5_QP_USER) 1767 if (qp->create_type == MLX5_QP_USER)
1795 destroy_qp_user(pd, qp, base); 1768 destroy_qp_user(dev, pd, qp, base);
1796 else if (qp->create_type == MLX5_QP_KERNEL) 1769 else if (qp->create_type == MLX5_QP_KERNEL)
1797 destroy_qp_kernel(dev, qp); 1770 destroy_qp_kernel(dev, qp);
1798 1771
@@ -1970,7 +1943,7 @@ static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
1970 if (qp->create_type == MLX5_QP_KERNEL) 1943 if (qp->create_type == MLX5_QP_KERNEL)
1971 destroy_qp_kernel(dev, qp); 1944 destroy_qp_kernel(dev, qp);
1972 else if (qp->create_type == MLX5_QP_USER) 1945 else if (qp->create_type == MLX5_QP_USER)
1973 destroy_qp_user(&get_pd(qp)->ibpd, qp, base); 1946 destroy_qp_user(dev, &get_pd(qp)->ibpd, qp, base);
1974} 1947}
1975 1948
1976static const char *ib_qp_type_str(enum ib_qp_type type) 1949static const char *ib_qp_type_str(enum ib_qp_type type)
@@ -3740,24 +3713,6 @@ static void dump_wqe(struct mlx5_ib_qp *qp, int idx, int size_16)
3740 } 3713 }
3741} 3714}
3742 3715
3743static void mlx5_bf_copy(u64 __iomem *dst, u64 *src,
3744 unsigned bytecnt, struct mlx5_ib_qp *qp)
3745{
3746 while (bytecnt > 0) {
3747 __iowrite64_copy(dst++, src++, 8);
3748 __iowrite64_copy(dst++, src++, 8);
3749 __iowrite64_copy(dst++, src++, 8);
3750 __iowrite64_copy(dst++, src++, 8);
3751 __iowrite64_copy(dst++, src++, 8);
3752 __iowrite64_copy(dst++, src++, 8);
3753 __iowrite64_copy(dst++, src++, 8);
3754 __iowrite64_copy(dst++, src++, 8);
3755 bytecnt -= 64;
3756 if (unlikely(src == qp->sq.qend))
3757 src = mlx5_get_send_wqe(qp, 0);
3758 }
3759}
3760
3761static u8 get_fence(u8 fence, struct ib_send_wr *wr) 3716static u8 get_fence(u8 fence, struct ib_send_wr *wr)
3762{ 3717{
3763 if (unlikely(wr->opcode == IB_WR_LOCAL_INV && 3718 if (unlikely(wr->opcode == IB_WR_LOCAL_INV &&
@@ -3853,7 +3808,7 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
3853 return mlx5_ib_gsi_post_send(ibqp, wr, bad_wr); 3808 return mlx5_ib_gsi_post_send(ibqp, wr, bad_wr);
3854 3809
3855 qp = to_mqp(ibqp); 3810 qp = to_mqp(ibqp);
3856 bf = qp->bf; 3811 bf = &qp->bf;
3857 qend = qp->sq.qend; 3812 qend = qp->sq.qend;
3858 3813
3859 spin_lock_irqsave(&qp->sq.lock, flags); 3814 spin_lock_irqsave(&qp->sq.lock, flags);
@@ -4126,28 +4081,13 @@ out:
4126 * we hit doorbell */ 4081 * we hit doorbell */
4127 wmb(); 4082 wmb();
4128 4083
4129 if (bf->need_lock) 4084 /* currently we support only regular doorbells */
4130 spin_lock(&bf->lock); 4085 mlx5_write64((__be32 *)ctrl, bf->bfreg->map + bf->offset, NULL);
4131 else 4086 /* Make sure doorbells don't leak out of SQ spinlock
4132 __acquire(&bf->lock); 4087 * and reach the HCA out of order.
4133 4088 */
4134 /* TBD enable WC */ 4089 mmiowb();
4135 if (0 && nreq == 1 && bf->uuarn && inl && size > 1 && size <= bf->buf_size / 16) {
4136 mlx5_bf_copy(bf->reg + bf->offset, (u64 *)ctrl, ALIGN(size * 16, 64), qp);
4137 /* wc_wmb(); */
4138 } else {
4139 mlx5_write64((__be32 *)ctrl, bf->regreg + bf->offset,
4140 MLX5_GET_DOORBELL_LOCK(&bf->lock32));
4141 /* Make sure doorbells don't leak out of SQ spinlock
4142 * and reach the HCA out of order.
4143 */
4144 mmiowb();
4145 }
4146 bf->offset ^= bf->buf_size; 4090 bf->offset ^= bf->buf_size;
4147 if (bf->need_lock)
4148 spin_unlock(&bf->lock);
4149 else
4150 __release(&bf->lock);
4151 } 4091 }
4152 4092
4153 spin_unlock_irqrestore(&qp->sq.lock, flags); 4093 spin_unlock_irqrestore(&qp->sq.lock, flags);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cq.c b/drivers/net/ethernet/mellanox/mlx5/core/cq.c
index 32d4af9b594d..336d4738b807 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/cq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/cq.c
@@ -179,6 +179,8 @@ int mlx5_core_create_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,
179 mlx5_core_dbg(dev, "failed adding CP 0x%x to debug file system\n", 179 mlx5_core_dbg(dev, "failed adding CP 0x%x to debug file system\n",
180 cq->cqn); 180 cq->cqn);
181 181
182 cq->uar = dev->priv.uar;
183
182 return 0; 184 return 0;
183 185
184err_cmd: 186err_cmd:
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
index 951dbd58594d..a473cea10c16 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -465,7 +465,6 @@ struct mlx5e_sq {
465 /* read only */ 465 /* read only */
466 struct mlx5_wq_cyc wq; 466 struct mlx5_wq_cyc wq;
467 u32 dma_fifo_mask; 467 u32 dma_fifo_mask;
468 void __iomem *uar_map;
469 struct netdev_queue *txq; 468 struct netdev_queue *txq;
470 u32 sqn; 469 u32 sqn;
471 u16 bf_buf_size; 470 u16 bf_buf_size;
@@ -479,7 +478,7 @@ struct mlx5e_sq {
479 478
480 /* control path */ 479 /* control path */
481 struct mlx5_wq_ctrl wq_ctrl; 480 struct mlx5_wq_ctrl wq_ctrl;
482 struct mlx5_uar uar; 481 struct mlx5_sq_bfreg bfreg;
483 struct mlx5e_channel *channel; 482 struct mlx5e_channel *channel;
484 int tc; 483 int tc;
485 u32 rate_limit; 484 u32 rate_limit;
@@ -806,7 +805,7 @@ void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params,
806static inline void mlx5e_tx_notify_hw(struct mlx5e_sq *sq, 805static inline void mlx5e_tx_notify_hw(struct mlx5e_sq *sq,
807 struct mlx5_wqe_ctrl_seg *ctrl, int bf_sz) 806 struct mlx5_wqe_ctrl_seg *ctrl, int bf_sz)
808{ 807{
809 u16 ofst = MLX5_BF_OFFSET + sq->bf_offset; 808 u16 ofst = sq->bf_offset;
810 809
811 /* ensure wqe is visible to device before updating doorbell record */ 810 /* ensure wqe is visible to device before updating doorbell record */
812 dma_wmb(); 811 dma_wmb();
@@ -818,9 +817,9 @@ static inline void mlx5e_tx_notify_hw(struct mlx5e_sq *sq,
818 */ 817 */
819 wmb(); 818 wmb();
820 if (bf_sz) 819 if (bf_sz)
821 __iowrite64_copy(sq->uar_map + ofst, ctrl, bf_sz); 820 __iowrite64_copy(sq->bfreg.map + ofst, ctrl, bf_sz);
822 else 821 else
823 mlx5_write64((__be32 *)ctrl, sq->uar_map + ofst, NULL); 822 mlx5_write64((__be32 *)ctrl, sq->bfreg.map + ofst, NULL);
824 /* flush the write-combining mapped buffer */ 823 /* flush the write-combining mapped buffer */
825 wmb(); 824 wmb();
826 825
@@ -832,7 +831,7 @@ static inline void mlx5e_cq_arm(struct mlx5e_cq *cq)
832 struct mlx5_core_cq *mcq; 831 struct mlx5_core_cq *mcq;
833 832
834 mcq = &cq->mcq; 833 mcq = &cq->mcq;
835 mlx5_cq_arm(mcq, MLX5_CQ_DB_REQ_NOT, mcq->uar->map, NULL, cq->wq.cc); 834 mlx5_cq_arm(mcq, MLX5_CQ_DB_REQ_NOT, mcq->uar->map, cq->wq.cc);
836} 835}
837 836
838static inline u32 mlx5e_get_wqe_mtt_offset(struct mlx5e_rq *rq, u16 wqe_ix) 837static inline u32 mlx5e_get_wqe_mtt_offset(struct mlx5e_rq *rq, u16 wqe_ix)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c
index f175518ff07a..bd898d8deda0 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c
@@ -89,16 +89,10 @@ int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev)
89 struct mlx5e_resources *res = &mdev->mlx5e_res; 89 struct mlx5e_resources *res = &mdev->mlx5e_res;
90 int err; 90 int err;
91 91
92 err = mlx5_alloc_map_uar(mdev, &res->cq_uar, false);
93 if (err) {
94 mlx5_core_err(mdev, "alloc_map uar failed, %d\n", err);
95 return err;
96 }
97
98 err = mlx5_core_alloc_pd(mdev, &res->pdn); 92 err = mlx5_core_alloc_pd(mdev, &res->pdn);
99 if (err) { 93 if (err) {
100 mlx5_core_err(mdev, "alloc pd failed, %d\n", err); 94 mlx5_core_err(mdev, "alloc pd failed, %d\n", err);
101 goto err_unmap_free_uar; 95 return err;
102 } 96 }
103 97
104 err = mlx5_core_alloc_transport_domain(mdev, &res->td.tdn); 98 err = mlx5_core_alloc_transport_domain(mdev, &res->td.tdn);
@@ -121,9 +115,6 @@ err_dealloc_transport_domain:
121 mlx5_core_dealloc_transport_domain(mdev, res->td.tdn); 115 mlx5_core_dealloc_transport_domain(mdev, res->td.tdn);
122err_dealloc_pd: 116err_dealloc_pd:
123 mlx5_core_dealloc_pd(mdev, res->pdn); 117 mlx5_core_dealloc_pd(mdev, res->pdn);
124err_unmap_free_uar:
125 mlx5_unmap_free_uar(mdev, &res->cq_uar);
126
127 return err; 118 return err;
128} 119}
129 120
@@ -134,7 +125,6 @@ void mlx5e_destroy_mdev_resources(struct mlx5_core_dev *mdev)
134 mlx5_core_destroy_mkey(mdev, &res->mkey); 125 mlx5_core_destroy_mkey(mdev, &res->mkey);
135 mlx5_core_dealloc_transport_domain(mdev, res->td.tdn); 126 mlx5_core_dealloc_transport_domain(mdev, res->td.tdn);
136 mlx5_core_dealloc_pd(mdev, res->pdn); 127 mlx5_core_dealloc_pd(mdev, res->pdn);
137 mlx5_unmap_free_uar(mdev, &res->cq_uar);
138} 128}
139 129
140int mlx5e_refresh_tirs_self_loopback(struct mlx5_core_dev *mdev, 130int mlx5e_refresh_tirs_self_loopback(struct mlx5_core_dev *mdev,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index 60e5670452a1..2cc774260903 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -967,7 +967,7 @@ static int mlx5e_create_sq(struct mlx5e_channel *c,
967 sq->channel = c; 967 sq->channel = c;
968 sq->tc = tc; 968 sq->tc = tc;
969 969
970 err = mlx5_alloc_map_uar(mdev, &sq->uar, !!MLX5_CAP_GEN(mdev, bf)); 970 err = mlx5_alloc_bfreg(mdev, &sq->bfreg, MLX5_CAP_GEN(mdev, bf), false);
971 if (err) 971 if (err)
972 return err; 972 return err;
973 973
@@ -979,12 +979,9 @@ static int mlx5e_create_sq(struct mlx5e_channel *c,
979 goto err_unmap_free_uar; 979 goto err_unmap_free_uar;
980 980
981 sq->wq.db = &sq->wq.db[MLX5_SND_DBR]; 981 sq->wq.db = &sq->wq.db[MLX5_SND_DBR];
982 if (sq->uar.bf_map) { 982 if (sq->bfreg.wc)
983 set_bit(MLX5E_SQ_STATE_BF_ENABLE, &sq->state); 983 set_bit(MLX5E_SQ_STATE_BF_ENABLE, &sq->state);
984 sq->uar_map = sq->uar.bf_map; 984
985 } else {
986 sq->uar_map = sq->uar.map;
987 }
988 sq->bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2; 985 sq->bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2;
989 sq->max_inline = param->max_inline; 986 sq->max_inline = param->max_inline;
990 sq->min_inline_mode = 987 sq->min_inline_mode =
@@ -1012,7 +1009,7 @@ err_sq_wq_destroy:
1012 mlx5_wq_destroy(&sq->wq_ctrl); 1009 mlx5_wq_destroy(&sq->wq_ctrl);
1013 1010
1014err_unmap_free_uar: 1011err_unmap_free_uar:
1015 mlx5_unmap_free_uar(mdev, &sq->uar); 1012 mlx5_free_bfreg(mdev, &sq->bfreg);
1016 1013
1017 return err; 1014 return err;
1018} 1015}
@@ -1024,7 +1021,7 @@ static void mlx5e_destroy_sq(struct mlx5e_sq *sq)
1024 1021
1025 mlx5e_free_sq_db(sq); 1022 mlx5e_free_sq_db(sq);
1026 mlx5_wq_destroy(&sq->wq_ctrl); 1023 mlx5_wq_destroy(&sq->wq_ctrl);
1027 mlx5_unmap_free_uar(priv->mdev, &sq->uar); 1024 mlx5_free_bfreg(priv->mdev, &sq->bfreg);
1028} 1025}
1029 1026
1030static int mlx5e_enable_sq(struct mlx5e_sq *sq, struct mlx5e_sq_param *param) 1027static int mlx5e_enable_sq(struct mlx5e_sq *sq, struct mlx5e_sq_param *param)
@@ -1058,7 +1055,7 @@ static int mlx5e_enable_sq(struct mlx5e_sq *sq, struct mlx5e_sq_param *param)
1058 MLX5_SET(sqc, sqc, tis_lst_sz, param->type == MLX5E_SQ_ICO ? 0 : 1); 1055 MLX5_SET(sqc, sqc, tis_lst_sz, param->type == MLX5E_SQ_ICO ? 0 : 1);
1059 1056
1060 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); 1057 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
1061 MLX5_SET(wq, wq, uar_page, sq->uar.index); 1058 MLX5_SET(wq, wq, uar_page, sq->bfreg.index);
1062 MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift - 1059 MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift -
1063 MLX5_ADAPTER_PAGE_SHIFT); 1060 MLX5_ADAPTER_PAGE_SHIFT);
1064 MLX5_SET64(wq, wq, dbr_addr, sq->wq_ctrl.db.dma); 1061 MLX5_SET64(wq, wq, dbr_addr, sq->wq_ctrl.db.dma);
@@ -1216,7 +1213,6 @@ static int mlx5e_create_cq(struct mlx5e_channel *c,
1216 mcq->comp = mlx5e_completion_event; 1213 mcq->comp = mlx5e_completion_event;
1217 mcq->event = mlx5e_cq_error_event; 1214 mcq->event = mlx5e_cq_error_event;
1218 mcq->irqn = irqn; 1215 mcq->irqn = irqn;
1219 mcq->uar = &mdev->mlx5e_res.cq_uar;
1220 1216
1221 for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) { 1217 for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
1222 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i); 1218 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
@@ -1265,7 +1261,7 @@ static int mlx5e_enable_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
1265 1261
1266 MLX5_SET(cqc, cqc, cq_period_mode, param->cq_period_mode); 1262 MLX5_SET(cqc, cqc, cq_period_mode, param->cq_period_mode);
1267 MLX5_SET(cqc, cqc, c_eqn, eqn); 1263 MLX5_SET(cqc, cqc, c_eqn, eqn);
1268 MLX5_SET(cqc, cqc, uar_page, mcq->uar->index); 1264 MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index);
1269 MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.frag_buf.page_shift - 1265 MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.frag_buf.page_shift -
1270 MLX5_ADAPTER_PAGE_SHIFT); 1266 MLX5_ADAPTER_PAGE_SHIFT);
1271 MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma); 1267 MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma);
@@ -1677,7 +1673,7 @@ static void mlx5e_build_common_cq_param(struct mlx5e_priv *priv,
1677{ 1673{
1678 void *cqc = param->cqc; 1674 void *cqc = param->cqc;
1679 1675
1680 MLX5_SET(cqc, cqc, uar_page, priv->mdev->mlx5e_res.cq_uar.index); 1676 MLX5_SET(cqc, cqc, uar_page, priv->mdev->priv.uar->index);
1681} 1677}
1682 1678
1683static void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv, 1679static void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
@@ -2296,7 +2292,6 @@ static int mlx5e_create_drop_cq(struct mlx5e_priv *priv,
2296 mcq->comp = mlx5e_completion_event; 2292 mcq->comp = mlx5e_completion_event;
2297 mcq->event = mlx5e_cq_error_event; 2293 mcq->event = mlx5e_cq_error_event;
2298 mcq->irqn = irqn; 2294 mcq->irqn = irqn;
2299 mcq->uar = &mdev->mlx5e_res.cq_uar;
2300 2295
2301 cq->priv = priv; 2296 cq->priv = priv;
2302 2297
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
index 4aff8ac68e14..5130d65dd41a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
@@ -512,7 +512,7 @@ static void init_eq_buf(struct mlx5_eq *eq)
512 512
513int mlx5_create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, u8 vecidx, 513int mlx5_create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, u8 vecidx,
514 int nent, u64 mask, const char *name, 514 int nent, u64 mask, const char *name,
515 struct mlx5_uar *uar, enum mlx5_eq_type type) 515 enum mlx5_eq_type type)
516{ 516{
517 u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {0}; 517 u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {0};
518 struct mlx5_priv *priv = &dev->priv; 518 struct mlx5_priv *priv = &dev->priv;
@@ -556,7 +556,7 @@ int mlx5_create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, u8 vecidx,
556 556
557 eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry); 557 eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry);
558 MLX5_SET(eqc, eqc, log_eq_size, ilog2(eq->nent)); 558 MLX5_SET(eqc, eqc, log_eq_size, ilog2(eq->nent));
559 MLX5_SET(eqc, eqc, uar_page, uar->index); 559 MLX5_SET(eqc, eqc, uar_page, priv->uar->index);
560 MLX5_SET(eqc, eqc, intr, vecidx); 560 MLX5_SET(eqc, eqc, intr, vecidx);
561 MLX5_SET(eqc, eqc, log_page_size, 561 MLX5_SET(eqc, eqc, log_page_size,
562 eq->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 562 eq->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
@@ -571,7 +571,7 @@ int mlx5_create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, u8 vecidx,
571 eq->eqn = MLX5_GET(create_eq_out, out, eq_number); 571 eq->eqn = MLX5_GET(create_eq_out, out, eq_number);
572 eq->irqn = priv->msix_arr[vecidx].vector; 572 eq->irqn = priv->msix_arr[vecidx].vector;
573 eq->dev = dev; 573 eq->dev = dev;
574 eq->doorbell = uar->map + MLX5_EQ_DOORBEL_OFFSET; 574 eq->doorbell = priv->uar->map + MLX5_EQ_DOORBEL_OFFSET;
575 err = request_irq(eq->irqn, handler, 0, 575 err = request_irq(eq->irqn, handler, 0,
576 priv->irq_info[vecidx].name, eq); 576 priv->irq_info[vecidx].name, eq);
577 if (err) 577 if (err)
@@ -686,8 +686,7 @@ int mlx5_start_eqs(struct mlx5_core_dev *dev)
686 686
687 err = mlx5_create_map_eq(dev, &table->cmd_eq, MLX5_EQ_VEC_CMD, 687 err = mlx5_create_map_eq(dev, &table->cmd_eq, MLX5_EQ_VEC_CMD,
688 MLX5_NUM_CMD_EQE, 1ull << MLX5_EVENT_TYPE_CMD, 688 MLX5_NUM_CMD_EQE, 1ull << MLX5_EVENT_TYPE_CMD,
689 "mlx5_cmd_eq", &dev->priv.uuari.uars[0], 689 "mlx5_cmd_eq", MLX5_EQ_TYPE_ASYNC);
690 MLX5_EQ_TYPE_ASYNC);
691 if (err) { 690 if (err) {
692 mlx5_core_warn(dev, "failed to create cmd EQ %d\n", err); 691 mlx5_core_warn(dev, "failed to create cmd EQ %d\n", err);
693 return err; 692 return err;
@@ -697,8 +696,7 @@ int mlx5_start_eqs(struct mlx5_core_dev *dev)
697 696
698 err = mlx5_create_map_eq(dev, &table->async_eq, MLX5_EQ_VEC_ASYNC, 697 err = mlx5_create_map_eq(dev, &table->async_eq, MLX5_EQ_VEC_ASYNC,
699 MLX5_NUM_ASYNC_EQE, async_event_mask, 698 MLX5_NUM_ASYNC_EQE, async_event_mask,
700 "mlx5_async_eq", &dev->priv.uuari.uars[0], 699 "mlx5_async_eq", MLX5_EQ_TYPE_ASYNC);
701 MLX5_EQ_TYPE_ASYNC);
702 if (err) { 700 if (err) {
703 mlx5_core_warn(dev, "failed to create async EQ %d\n", err); 701 mlx5_core_warn(dev, "failed to create async EQ %d\n", err);
704 goto err1; 702 goto err1;
@@ -708,7 +706,6 @@ int mlx5_start_eqs(struct mlx5_core_dev *dev)
708 MLX5_EQ_VEC_PAGES, 706 MLX5_EQ_VEC_PAGES,
709 /* TODO: sriov max_vf + */ 1, 707 /* TODO: sriov max_vf + */ 1,
710 1 << MLX5_EVENT_TYPE_PAGE_REQUEST, "mlx5_pages_eq", 708 1 << MLX5_EVENT_TYPE_PAGE_REQUEST, "mlx5_pages_eq",
711 &dev->priv.uuari.uars[0],
712 MLX5_EQ_TYPE_ASYNC); 709 MLX5_EQ_TYPE_ASYNC);
713 if (err) { 710 if (err) {
714 mlx5_core_warn(dev, "failed to create pages EQ %d\n", err); 711 mlx5_core_warn(dev, "failed to create pages EQ %d\n", err);
@@ -722,7 +719,6 @@ int mlx5_start_eqs(struct mlx5_core_dev *dev)
722 MLX5_NUM_ASYNC_EQE, 719 MLX5_NUM_ASYNC_EQE,
723 1 << MLX5_EVENT_TYPE_PAGE_FAULT, 720 1 << MLX5_EVENT_TYPE_PAGE_FAULT,
724 "mlx5_page_fault_eq", 721 "mlx5_page_fault_eq",
725 &dev->priv.uuari.uars[0],
726 MLX5_EQ_TYPE_PF); 722 MLX5_EQ_TYPE_PF);
727 if (err) { 723 if (err) {
728 mlx5_core_warn(dev, "failed to create page fault EQ %d\n", 724 mlx5_core_warn(dev, "failed to create page fault EQ %d\n",
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index f1ed009b871c..cb7708b45bb1 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -537,6 +537,10 @@ static int handle_hca_cap(struct mlx5_core_dev *dev)
537 /* disable cmdif checksum */ 537 /* disable cmdif checksum */
538 MLX5_SET(cmd_hca_cap, set_hca_cap, cmdif_checksum, 0); 538 MLX5_SET(cmd_hca_cap, set_hca_cap, cmdif_checksum, 0);
539 539
540 /* If the HCA supports 4K UARs use it */
541 if (MLX5_CAP_GEN_MAX(dev, uar_4k))
542 MLX5_SET(cmd_hca_cap, set_hca_cap, uar_4k, 1);
543
540 MLX5_SET(cmd_hca_cap, set_hca_cap, log_uar_page_sz, PAGE_SHIFT - 12); 544 MLX5_SET(cmd_hca_cap, set_hca_cap, log_uar_page_sz, PAGE_SHIFT - 12);
541 545
542 err = set_caps(dev, set_ctx, set_sz, 546 err = set_caps(dev, set_ctx, set_sz,
@@ -759,8 +763,7 @@ static int alloc_comp_eqs(struct mlx5_core_dev *dev)
759 snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_comp%d", i); 763 snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_comp%d", i);
760 err = mlx5_create_map_eq(dev, eq, 764 err = mlx5_create_map_eq(dev, eq,
761 i + MLX5_EQ_VEC_COMP_BASE, nent, 0, 765 i + MLX5_EQ_VEC_COMP_BASE, nent, 0,
762 name, &dev->priv.uuari.uars[0], 766 name, MLX5_EQ_TYPE_COMP);
763 MLX5_EQ_TYPE_COMP);
764 if (err) { 767 if (err) {
765 kfree(eq); 768 kfree(eq);
766 goto clean; 769 goto clean;
@@ -920,8 +923,6 @@ static int mlx5_init_once(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
920 goto out; 923 goto out;
921 } 924 }
922 925
923 MLX5_INIT_DOORBELL_LOCK(&priv->cq_uar_lock);
924
925 err = mlx5_init_cq_table(dev); 926 err = mlx5_init_cq_table(dev);
926 if (err) { 927 if (err) {
927 dev_err(&pdev->dev, "failed to initialize cq table\n"); 928 dev_err(&pdev->dev, "failed to initialize cq table\n");
@@ -1100,8 +1101,8 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
1100 goto err_cleanup_once; 1101 goto err_cleanup_once;
1101 } 1102 }
1102 1103
1103 err = mlx5_alloc_uuars(dev, &priv->uuari); 1104 dev->priv.uar = mlx5_get_uars_page(dev);
1104 if (err) { 1105 if (!dev->priv.uar) {
1105 dev_err(&pdev->dev, "Failed allocating uar, aborting\n"); 1106 dev_err(&pdev->dev, "Failed allocating uar, aborting\n");
1106 goto err_disable_msix; 1107 goto err_disable_msix;
1107 } 1108 }
@@ -1109,7 +1110,7 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
1109 err = mlx5_start_eqs(dev); 1110 err = mlx5_start_eqs(dev);
1110 if (err) { 1111 if (err) {
1111 dev_err(&pdev->dev, "Failed to start pages and async EQs\n"); 1112 dev_err(&pdev->dev, "Failed to start pages and async EQs\n");
1112 goto err_free_uar; 1113 goto err_put_uars;
1113 } 1114 }
1114 1115
1115 err = alloc_comp_eqs(dev); 1116 err = alloc_comp_eqs(dev);
@@ -1175,8 +1176,8 @@ err_affinity_hints:
1175err_stop_eqs: 1176err_stop_eqs:
1176 mlx5_stop_eqs(dev); 1177 mlx5_stop_eqs(dev);
1177 1178
1178err_free_uar: 1179err_put_uars:
1179 mlx5_free_uuars(dev, &priv->uuari); 1180 mlx5_put_uars_page(dev, priv->uar);
1180 1181
1181err_disable_msix: 1182err_disable_msix:
1182 mlx5_disable_msix(dev); 1183 mlx5_disable_msix(dev);
@@ -1238,7 +1239,7 @@ static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
1238 mlx5_irq_clear_affinity_hints(dev); 1239 mlx5_irq_clear_affinity_hints(dev);
1239 free_comp_eqs(dev); 1240 free_comp_eqs(dev);
1240 mlx5_stop_eqs(dev); 1241 mlx5_stop_eqs(dev);
1241 mlx5_free_uuars(dev, &priv->uuari); 1242 mlx5_put_uars_page(dev, priv->uar);
1242 mlx5_disable_msix(dev); 1243 mlx5_disable_msix(dev);
1243 if (cleanup) 1244 if (cleanup)
1244 mlx5_cleanup_once(dev); 1245 mlx5_cleanup_once(dev);
@@ -1313,6 +1314,11 @@ static int init_one(struct pci_dev *pdev,
1313 goto clean_dev; 1314 goto clean_dev;
1314 } 1315 }
1315#endif 1316#endif
1317 mutex_init(&priv->bfregs.reg_head.lock);
1318 mutex_init(&priv->bfregs.wc_head.lock);
1319 INIT_LIST_HEAD(&priv->bfregs.reg_head.list);
1320 INIT_LIST_HEAD(&priv->bfregs.wc_head.list);
1321
1316 err = mlx5_pci_init(dev, priv); 1322 err = mlx5_pci_init(dev, priv);
1317 if (err) { 1323 if (err) {
1318 dev_err(&pdev->dev, "mlx5_pci_init failed with error code %d\n", err); 1324 dev_err(&pdev->dev, "mlx5_pci_init failed with error code %d\n", err);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/uar.c b/drivers/net/ethernet/mellanox/mlx5/core/uar.c
index ab0b896621a0..2e6b0f290ddc 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/uar.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/uar.c
@@ -37,11 +37,6 @@
37#include <linux/mlx5/cmd.h> 37#include <linux/mlx5/cmd.h>
38#include "mlx5_core.h" 38#include "mlx5_core.h"
39 39
40enum {
41 NUM_DRIVER_UARS = 4,
42 NUM_LOW_LAT_UUARS = 4,
43};
44
45int mlx5_cmd_alloc_uar(struct mlx5_core_dev *dev, u32 *uarn) 40int mlx5_cmd_alloc_uar(struct mlx5_core_dev *dev, u32 *uarn)
46{ 41{
47 u32 out[MLX5_ST_SZ_DW(alloc_uar_out)] = {0}; 42 u32 out[MLX5_ST_SZ_DW(alloc_uar_out)] = {0};
@@ -67,167 +62,269 @@ int mlx5_cmd_free_uar(struct mlx5_core_dev *dev, u32 uarn)
67} 62}
68EXPORT_SYMBOL(mlx5_cmd_free_uar); 63EXPORT_SYMBOL(mlx5_cmd_free_uar);
69 64
70static int need_uuar_lock(int uuarn) 65static int uars_per_sys_page(struct mlx5_core_dev *mdev)
71{ 66{
72 int tot_uuars = NUM_DRIVER_UARS * MLX5_BF_REGS_PER_PAGE; 67 if (MLX5_CAP_GEN(mdev, uar_4k))
73 68 return MLX5_CAP_GEN(mdev, num_of_uars_per_page);
74 if (uuarn == 0 || tot_uuars - NUM_LOW_LAT_UUARS)
75 return 0;
76 69
77 return 1; 70 return 1;
78} 71}
79 72
80int mlx5_alloc_uuars(struct mlx5_core_dev *dev, struct mlx5_uuar_info *uuari) 73static u64 uar2pfn(struct mlx5_core_dev *mdev, u32 index)
81{ 74{
82 int tot_uuars = NUM_DRIVER_UARS * MLX5_BF_REGS_PER_PAGE; 75 u32 system_page_index;
83 struct mlx5_bf *bf; 76
84 phys_addr_t addr; 77 if (MLX5_CAP_GEN(mdev, uar_4k))
85 int err; 78 system_page_index = index >> (PAGE_SHIFT - MLX5_ADAPTER_PAGE_SHIFT);
79 else
80 system_page_index = index;
81
82 return (pci_resource_start(mdev->pdev, 0) >> PAGE_SHIFT) + system_page_index;
83}
84
85static void up_rel_func(struct kref *kref)
86{
87 struct mlx5_uars_page *up = container_of(kref, struct mlx5_uars_page, ref_count);
88
89 list_del(&up->list);
90 if (mlx5_cmd_free_uar(up->mdev, up->index))
91 mlx5_core_warn(up->mdev, "failed to free uar index %d\n", up->index);
92 kfree(up->reg_bitmap);
93 kfree(up->fp_bitmap);
94 kfree(up);
95}
96
97static struct mlx5_uars_page *alloc_uars_page(struct mlx5_core_dev *mdev,
98 bool map_wc)
99{
100 struct mlx5_uars_page *up;
101 int err = -ENOMEM;
102 phys_addr_t pfn;
103 int bfregs;
86 int i; 104 int i;
87 105
88 uuari->num_uars = NUM_DRIVER_UARS; 106 bfregs = uars_per_sys_page(mdev) * MLX5_BFREGS_PER_UAR;
89 uuari->num_low_latency_uuars = NUM_LOW_LAT_UUARS; 107 up = kzalloc(sizeof(*up), GFP_KERNEL);
108 if (!up)
109 return ERR_PTR(err);
90 110
91 mutex_init(&uuari->lock); 111 up->mdev = mdev;
92 uuari->uars = kcalloc(uuari->num_uars, sizeof(*uuari->uars), GFP_KERNEL); 112 up->reg_bitmap = kcalloc(BITS_TO_LONGS(bfregs), sizeof(unsigned long), GFP_KERNEL);
93 if (!uuari->uars) 113 if (!up->reg_bitmap)
94 return -ENOMEM; 114 goto error1;
95 115
96 uuari->bfs = kcalloc(tot_uuars, sizeof(*uuari->bfs), GFP_KERNEL); 116 up->fp_bitmap = kcalloc(BITS_TO_LONGS(bfregs), sizeof(unsigned long), GFP_KERNEL);
97 if (!uuari->bfs) { 117 if (!up->fp_bitmap)
98 err = -ENOMEM; 118 goto error1;
99 goto out_uars;
100 }
101 119
102 uuari->bitmap = kcalloc(BITS_TO_LONGS(tot_uuars), sizeof(*uuari->bitmap), 120 for (i = 0; i < bfregs; i++)
103 GFP_KERNEL); 121 if ((i % MLX5_BFREGS_PER_UAR) < MLX5_NON_FP_BFREGS_PER_UAR)
104 if (!uuari->bitmap) { 122 set_bit(i, up->reg_bitmap);
105 err = -ENOMEM; 123 else
106 goto out_bfs; 124 set_bit(i, up->fp_bitmap);
107 }
108 125
109 uuari->count = kcalloc(tot_uuars, sizeof(*uuari->count), GFP_KERNEL); 126 up->bfregs = bfregs;
110 if (!uuari->count) { 127 up->fp_avail = bfregs * MLX5_FP_BFREGS_PER_UAR / MLX5_BFREGS_PER_UAR;
111 err = -ENOMEM; 128 up->reg_avail = bfregs * MLX5_NON_FP_BFREGS_PER_UAR / MLX5_BFREGS_PER_UAR;
112 goto out_bitmap;
113 }
114 129
115 for (i = 0; i < uuari->num_uars; i++) { 130 err = mlx5_cmd_alloc_uar(mdev, &up->index);
116 err = mlx5_cmd_alloc_uar(dev, &uuari->uars[i].index); 131 if (err) {
117 if (err) 132 mlx5_core_warn(mdev, "mlx5_cmd_alloc_uar() failed, %d\n", err);
118 goto out_count; 133 goto error1;
134 }
119 135
120 addr = dev->iseg_base + ((phys_addr_t)(uuari->uars[i].index) << PAGE_SHIFT); 136 pfn = uar2pfn(mdev, up->index);
121 uuari->uars[i].map = ioremap(addr, PAGE_SIZE); 137 if (map_wc) {
122 if (!uuari->uars[i].map) { 138 up->map = ioremap_wc(pfn << PAGE_SHIFT, PAGE_SIZE);
123 mlx5_cmd_free_uar(dev, uuari->uars[i].index); 139 if (!up->map) {
140 err = -EAGAIN;
141 goto error2;
142 }
143 } else {
144 up->map = ioremap(pfn << PAGE_SHIFT, PAGE_SIZE);
145 if (!up->map) {
124 err = -ENOMEM; 146 err = -ENOMEM;
125 goto out_count; 147 goto error2;
126 } 148 }
127 mlx5_core_dbg(dev, "allocated uar index 0x%x, mmaped at %p\n",
128 uuari->uars[i].index, uuari->uars[i].map);
129 }
130
131 for (i = 0; i < tot_uuars; i++) {
132 bf = &uuari->bfs[i];
133
134 bf->buf_size = (1 << MLX5_CAP_GEN(dev, log_bf_reg_size)) / 2;
135 bf->uar = &uuari->uars[i / MLX5_BF_REGS_PER_PAGE];
136 bf->regreg = uuari->uars[i / MLX5_BF_REGS_PER_PAGE].map;
137 bf->reg = NULL; /* Add WC support */
138 bf->offset = (i % MLX5_BF_REGS_PER_PAGE) *
139 (1 << MLX5_CAP_GEN(dev, log_bf_reg_size)) +
140 MLX5_BF_OFFSET;
141 bf->need_lock = need_uuar_lock(i);
142 spin_lock_init(&bf->lock);
143 spin_lock_init(&bf->lock32);
144 bf->uuarn = i;
145 } 149 }
150 kref_init(&up->ref_count);
151 mlx5_core_dbg(mdev, "allocated UAR page: index %d, total bfregs %d\n",
152 up->index, up->bfregs);
153 return up;
154
155error2:
156 if (mlx5_cmd_free_uar(mdev, up->index))
157 mlx5_core_warn(mdev, "failed to free uar index %d\n", up->index);
158error1:
159 kfree(up->fp_bitmap);
160 kfree(up->reg_bitmap);
161 kfree(up);
162 return ERR_PTR(err);
163}
146 164
147 return 0; 165struct mlx5_uars_page *mlx5_get_uars_page(struct mlx5_core_dev *mdev)
148 166{
149out_count: 167 struct mlx5_uars_page *ret;
150 for (i--; i >= 0; i--) { 168
151 iounmap(uuari->uars[i].map); 169 mutex_lock(&mdev->priv.bfregs.reg_head.lock);
152 mlx5_cmd_free_uar(dev, uuari->uars[i].index); 170 if (list_empty(&mdev->priv.bfregs.reg_head.list)) {
171 ret = alloc_uars_page(mdev, false);
172 if (IS_ERR(ret)) {
173 ret = NULL;
174 goto out;
175 }
176 list_add(&ret->list, &mdev->priv.bfregs.reg_head.list);
177 } else {
178 ret = list_first_entry(&mdev->priv.bfregs.reg_head.list,
179 struct mlx5_uars_page, list);
180 kref_get(&ret->ref_count);
153 } 181 }
154 kfree(uuari->count); 182out:
183 mutex_unlock(&mdev->priv.bfregs.reg_head.lock);
155 184
156out_bitmap: 185 return ret;
157 kfree(uuari->bitmap); 186}
158 187EXPORT_SYMBOL(mlx5_get_uars_page);
159out_bfs:
160 kfree(uuari->bfs);
161 188
162out_uars: 189void mlx5_put_uars_page(struct mlx5_core_dev *mdev, struct mlx5_uars_page *up)
163 kfree(uuari->uars); 190{
164 return err; 191 mutex_lock(&mdev->priv.bfregs.reg_head.lock);
192 kref_put(&up->ref_count, up_rel_func);
193 mutex_unlock(&mdev->priv.bfregs.reg_head.lock);
165} 194}
195EXPORT_SYMBOL(mlx5_put_uars_page);
166 196
167int mlx5_free_uuars(struct mlx5_core_dev *dev, struct mlx5_uuar_info *uuari) 197static unsigned long map_offset(struct mlx5_core_dev *mdev, int dbi)
168{ 198{
169 int i = uuari->num_uars; 199 /* return the offset in bytes from the start of the page to the
200 * blue flame area of the UAR
201 */
202 return dbi / MLX5_BFREGS_PER_UAR * MLX5_ADAPTER_PAGE_SIZE +
203 (dbi % MLX5_BFREGS_PER_UAR) *
204 (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) + MLX5_BF_OFFSET;
205}
170 206
171 for (i--; i >= 0; i--) { 207static int alloc_bfreg(struct mlx5_core_dev *mdev, struct mlx5_sq_bfreg *bfreg,
172 iounmap(uuari->uars[i].map); 208 bool map_wc, bool fast_path)
173 mlx5_cmd_free_uar(dev, uuari->uars[i].index); 209{
210 struct mlx5_bfreg_data *bfregs;
211 struct mlx5_uars_page *up;
212 struct list_head *head;
213 unsigned long *bitmap;
214 unsigned int *avail;
215 struct mutex *lock; /* pointer to right mutex */
216 int dbi;
217
218 bfregs = &mdev->priv.bfregs;
219 if (map_wc) {
220 head = &bfregs->wc_head.list;
221 lock = &bfregs->wc_head.lock;
222 } else {
223 head = &bfregs->reg_head.list;
224 lock = &bfregs->reg_head.lock;
174 } 225 }
175 226 mutex_lock(lock);
176 kfree(uuari->count); 227 if (list_empty(head)) {
177 kfree(uuari->bitmap); 228 up = alloc_uars_page(mdev, map_wc);
178 kfree(uuari->bfs); 229 if (IS_ERR(up)) {
179 kfree(uuari->uars); 230 mutex_unlock(lock);
231 return PTR_ERR(up);
232 }
233 list_add(&up->list, head);
234 } else {
235 up = list_entry(head->next, struct mlx5_uars_page, list);
236 kref_get(&up->ref_count);
237 }
238 if (fast_path) {
239 bitmap = up->fp_bitmap;
240 avail = &up->fp_avail;
241 } else {
242 bitmap = up->reg_bitmap;
243 avail = &up->reg_avail;
244 }
245 dbi = find_first_bit(bitmap, up->bfregs);
246 clear_bit(dbi, bitmap);
247 (*avail)--;
248 if (!(*avail))
249 list_del(&up->list);
250
251 bfreg->map = up->map + map_offset(mdev, dbi);
252 bfreg->up = up;
253 bfreg->wc = map_wc;
254 bfreg->index = up->index + dbi / MLX5_BFREGS_PER_UAR;
255 mutex_unlock(lock);
180 256
181 return 0; 257 return 0;
182} 258}
183 259
184int mlx5_alloc_map_uar(struct mlx5_core_dev *mdev, struct mlx5_uar *uar, 260int mlx5_alloc_bfreg(struct mlx5_core_dev *mdev, struct mlx5_sq_bfreg *bfreg,
185 bool map_wc) 261 bool map_wc, bool fast_path)
186{ 262{
187 phys_addr_t pfn;
188 phys_addr_t uar_bar_start;
189 int err; 263 int err;
190 264
191 err = mlx5_cmd_alloc_uar(mdev, &uar->index); 265 err = alloc_bfreg(mdev, bfreg, map_wc, fast_path);
192 if (err) { 266 if (!err)
193 mlx5_core_warn(mdev, "mlx5_cmd_alloc_uar() failed, %d\n", err); 267 return 0;
194 return err;
195 }
196 268
197 uar_bar_start = pci_resource_start(mdev->pdev, 0); 269 if (err == -EAGAIN && map_wc)
198 pfn = (uar_bar_start >> PAGE_SHIFT) + uar->index; 270 return alloc_bfreg(mdev, bfreg, false, fast_path);
199 271
200 if (map_wc) { 272 return err;
201 uar->bf_map = ioremap_wc(pfn << PAGE_SHIFT, PAGE_SIZE); 273}
202 if (!uar->bf_map) { 274EXPORT_SYMBOL(mlx5_alloc_bfreg);
203 mlx5_core_warn(mdev, "ioremap_wc() failed\n");
204 uar->map = ioremap(pfn << PAGE_SHIFT, PAGE_SIZE);
205 if (!uar->map)
206 goto err_free_uar;
207 }
208 } else {
209 uar->map = ioremap(pfn << PAGE_SHIFT, PAGE_SIZE);
210 if (!uar->map)
211 goto err_free_uar;
212 }
213 275
214 return 0; 276static unsigned int addr_to_dbi_in_syspage(struct mlx5_core_dev *dev,
277 struct mlx5_uars_page *up,
278 struct mlx5_sq_bfreg *bfreg)
279{
280 unsigned int uar_idx;
281 unsigned int bfreg_idx;
282 unsigned int bf_reg_size;
215 283
216err_free_uar: 284 bf_reg_size = 1 << MLX5_CAP_GEN(dev, log_bf_reg_size);
217 mlx5_core_warn(mdev, "ioremap() failed\n");
218 err = -ENOMEM;
219 mlx5_cmd_free_uar(mdev, uar->index);
220 285
221 return err; 286 uar_idx = (bfreg->map - up->map) >> MLX5_ADAPTER_PAGE_SHIFT;
287 bfreg_idx = (((uintptr_t)bfreg->map % MLX5_ADAPTER_PAGE_SIZE) - MLX5_BF_OFFSET) / bf_reg_size;
288
289 return uar_idx * MLX5_BFREGS_PER_UAR + bfreg_idx;
222} 290}
223EXPORT_SYMBOL(mlx5_alloc_map_uar);
224 291
225void mlx5_unmap_free_uar(struct mlx5_core_dev *mdev, struct mlx5_uar *uar) 292void mlx5_free_bfreg(struct mlx5_core_dev *mdev, struct mlx5_sq_bfreg *bfreg)
226{ 293{
227 if (uar->map) 294 struct mlx5_bfreg_data *bfregs;
228 iounmap(uar->map); 295 struct mlx5_uars_page *up;
229 else 296 struct mutex *lock; /* pointer to right mutex */
230 iounmap(uar->bf_map); 297 unsigned int dbi;
231 mlx5_cmd_free_uar(mdev, uar->index); 298 bool fp;
299 unsigned int *avail;
300 unsigned long *bitmap;
301 struct list_head *head;
302
303 bfregs = &mdev->priv.bfregs;
304 if (bfreg->wc) {
305 head = &bfregs->wc_head.list;
306 lock = &bfregs->wc_head.lock;
307 } else {
308 head = &bfregs->reg_head.list;
309 lock = &bfregs->reg_head.lock;
310 }
311 up = bfreg->up;
312 dbi = addr_to_dbi_in_syspage(mdev, up, bfreg);
313 fp = (dbi % MLX5_BFREGS_PER_UAR) >= MLX5_NON_FP_BFREGS_PER_UAR;
314 if (fp) {
315 avail = &up->fp_avail;
316 bitmap = up->fp_bitmap;
317 } else {
318 avail = &up->reg_avail;
319 bitmap = up->reg_bitmap;
320 }
321 mutex_lock(lock);
322 (*avail)++;
323 set_bit(dbi, bitmap);
324 if (*avail == 1)
325 list_add_tail(&up->list, head);
326
327 kref_put(&up->ref_count, up_rel_func);
328 mutex_unlock(lock);
232} 329}
233EXPORT_SYMBOL(mlx5_unmap_free_uar); 330EXPORT_SYMBOL(mlx5_free_bfreg);
diff --git a/include/linux/mlx5/cq.h b/include/linux/mlx5/cq.h
index 7c3c0d3aca37..95898847c7d4 100644
--- a/include/linux/mlx5/cq.h
+++ b/include/linux/mlx5/cq.h
@@ -42,13 +42,13 @@ struct mlx5_core_cq {
42 int cqe_sz; 42 int cqe_sz;
43 __be32 *set_ci_db; 43 __be32 *set_ci_db;
44 __be32 *arm_db; 44 __be32 *arm_db;
45 struct mlx5_uars_page *uar;
45 atomic_t refcount; 46 atomic_t refcount;
46 struct completion free; 47 struct completion free;
47 unsigned vector; 48 unsigned vector;
48 unsigned int irqn; 49 unsigned int irqn;
49 void (*comp) (struct mlx5_core_cq *); 50 void (*comp) (struct mlx5_core_cq *);
50 void (*event) (struct mlx5_core_cq *, enum mlx5_event); 51 void (*event) (struct mlx5_core_cq *, enum mlx5_event);
51 struct mlx5_uar *uar;
52 u32 cons_index; 52 u32 cons_index;
53 unsigned arm_sn; 53 unsigned arm_sn;
54 struct mlx5_rsc_debug *dbg; 54 struct mlx5_rsc_debug *dbg;
@@ -144,7 +144,6 @@ enum {
144 144
145static inline void mlx5_cq_arm(struct mlx5_core_cq *cq, u32 cmd, 145static inline void mlx5_cq_arm(struct mlx5_core_cq *cq, u32 cmd,
146 void __iomem *uar_page, 146 void __iomem *uar_page,
147 spinlock_t *doorbell_lock,
148 u32 cons_index) 147 u32 cons_index)
149{ 148{
150 __be32 doorbell[2]; 149 __be32 doorbell[2];
@@ -164,7 +163,7 @@ static inline void mlx5_cq_arm(struct mlx5_core_cq *cq, u32 cmd,
164 doorbell[0] = cpu_to_be32(sn << 28 | cmd | ci); 163 doorbell[0] = cpu_to_be32(sn << 28 | cmd | ci);
165 doorbell[1] = cpu_to_be32(cq->cqn); 164 doorbell[1] = cpu_to_be32(cq->cqn);
166 165
167 mlx5_write64(doorbell, uar_page + MLX5_CQ_DOORBELL, doorbell_lock); 166 mlx5_write64(doorbell, uar_page + MLX5_CQ_DOORBELL, NULL);
168} 167}
169 168
170int mlx5_init_cq_table(struct mlx5_core_dev *dev); 169int mlx5_init_cq_table(struct mlx5_core_dev *dev);
diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h
index 7a79a20c54d2..7c5265db02a9 100644
--- a/include/linux/mlx5/device.h
+++ b/include/linux/mlx5/device.h
@@ -212,10 +212,20 @@ enum {
212}; 212};
213 213
214enum { 214enum {
215 MLX5_BF_REGS_PER_PAGE = 4, 215 MLX5_ADAPTER_PAGE_SHIFT = 12,
216 MLX5_MAX_UAR_PAGES = 1 << 8, 216 MLX5_ADAPTER_PAGE_SIZE = 1 << MLX5_ADAPTER_PAGE_SHIFT,
217 MLX5_NON_FP_BF_REGS_PER_PAGE = 2, 217};
218 MLX5_MAX_UUARS = MLX5_MAX_UAR_PAGES * MLX5_NON_FP_BF_REGS_PER_PAGE, 218
219enum {
220 MLX5_BFREGS_PER_UAR = 4,
221 MLX5_MAX_UARS = 1 << 8,
222 MLX5_NON_FP_BFREGS_PER_UAR = 2,
223 MLX5_FP_BFREGS_PER_UAR = MLX5_BFREGS_PER_UAR -
224 MLX5_NON_FP_BFREGS_PER_UAR,
225 MLX5_MAX_BFREGS = MLX5_MAX_UARS *
226 MLX5_NON_FP_BFREGS_PER_UAR,
227 MLX5_UARS_IN_PAGE = PAGE_SIZE / MLX5_ADAPTER_PAGE_SIZE,
228 MLX5_NON_FP_BFREGS_IN_PAGE = MLX5_NON_FP_BFREGS_PER_UAR * MLX5_UARS_IN_PAGE,
219}; 229};
220 230
221enum { 231enum {
@@ -389,11 +399,6 @@ enum {
389}; 399};
390 400
391enum { 401enum {
392 MLX5_ADAPTER_PAGE_SHIFT = 12,
393 MLX5_ADAPTER_PAGE_SIZE = 1 << MLX5_ADAPTER_PAGE_SHIFT,
394};
395
396enum {
397 MLX5_CAP_OFF_CMDIF_CSUM = 46, 402 MLX5_CAP_OFF_CMDIF_CSUM = 46,
398}; 403};
399 404
diff --git a/include/linux/mlx5/doorbell.h b/include/linux/mlx5/doorbell.h
index afc78a3f4462..0787de28f2fc 100644
--- a/include/linux/mlx5/doorbell.h
+++ b/include/linux/mlx5/doorbell.h
@@ -68,10 +68,12 @@ static inline void mlx5_write64(__be32 val[2], void __iomem *dest,
68{ 68{
69 unsigned long flags; 69 unsigned long flags;
70 70
71 spin_lock_irqsave(doorbell_lock, flags); 71 if (doorbell_lock)
72 spin_lock_irqsave(doorbell_lock, flags);
72 __raw_writel((__force u32) val[0], dest); 73 __raw_writel((__force u32) val[0], dest);
73 __raw_writel((__force u32) val[1], dest + 4); 74 __raw_writel((__force u32) val[1], dest + 4);
74 spin_unlock_irqrestore(doorbell_lock, flags); 75 if (doorbell_lock)
76 spin_unlock_irqrestore(doorbell_lock, flags);
75} 77}
76 78
77#endif 79#endif
diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index cdd2bd62f86d..3a309f6a4a15 100644
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -187,36 +187,18 @@ enum mlx5_eq_type {
187#endif 187#endif
188}; 188};
189 189
190struct mlx5_uuar_info { 190struct mlx5_bfreg_info {
191 struct mlx5_uar *uars; 191 u32 *sys_pages;
192 int num_uars; 192 int num_low_latency_bfregs;
193 int num_low_latency_uuars;
194 unsigned long *bitmap;
195 unsigned int *count; 193 unsigned int *count;
196 struct mlx5_bf *bfs;
197 194
198 /* 195 /*
199 * protect uuar allocation data structs 196 * protect bfreg allocation data structs
200 */ 197 */
201 struct mutex lock; 198 struct mutex lock;
202 u32 ver; 199 u32 ver;
203}; 200 bool lib_uar_4k;
204 201 u32 num_sys_pages;
205struct mlx5_bf {
206 void __iomem *reg;
207 void __iomem *regreg;
208 int buf_size;
209 struct mlx5_uar *uar;
210 unsigned long offset;
211 int need_lock;
212 /* protect blue flame buffer selection when needed
213 */
214 spinlock_t lock;
215
216 /* serialize 64 bit writes when done as two 32 bit accesses
217 */
218 spinlock_t lock32;
219 int uuarn;
220}; 202};
221 203
222struct mlx5_cmd_first { 204struct mlx5_cmd_first {
@@ -451,14 +433,38 @@ struct mlx5_eq_table {
451 spinlock_t lock; 433 spinlock_t lock;
452}; 434};
453 435
454struct mlx5_uar { 436struct mlx5_uars_page {
455 u32 index;
456 struct list_head bf_list;
457 unsigned free_bf_bmap;
458 void __iomem *bf_map;
459 void __iomem *map; 437 void __iomem *map;
438 bool wc;
439 u32 index;
440 struct list_head list;
441 unsigned int bfregs;
442 unsigned long *reg_bitmap; /* for non fast path bf regs */
443 unsigned long *fp_bitmap;
444 unsigned int reg_avail;
445 unsigned int fp_avail;
446 struct kref ref_count;
447 struct mlx5_core_dev *mdev;
460}; 448};
461 449
450struct mlx5_bfreg_head {
451 /* protect blue flame registers allocations */
452 struct mutex lock;
453 struct list_head list;
454};
455
456struct mlx5_bfreg_data {
457 struct mlx5_bfreg_head reg_head;
458 struct mlx5_bfreg_head wc_head;
459};
460
461struct mlx5_sq_bfreg {
462 void __iomem *map;
463 struct mlx5_uars_page *up;
464 bool wc;
465 u32 index;
466 unsigned int offset;
467};
462 468
463struct mlx5_core_health { 469struct mlx5_core_health {
464 struct health_buffer __iomem *health; 470 struct health_buffer __iomem *health;
@@ -578,8 +584,6 @@ struct mlx5_priv {
578 struct mlx5_eq_table eq_table; 584 struct mlx5_eq_table eq_table;
579 struct msix_entry *msix_arr; 585 struct msix_entry *msix_arr;
580 struct mlx5_irq_info *irq_info; 586 struct mlx5_irq_info *irq_info;
581 struct mlx5_uuar_info uuari;
582 MLX5_DECLARE_DOORBELL_LOCK(cq_uar_lock);
583 587
584 /* pages stuff */ 588 /* pages stuff */
585 struct workqueue_struct *pg_wq; 589 struct workqueue_struct *pg_wq;
@@ -644,6 +648,8 @@ struct mlx5_priv {
644 void *pfault_ctx; 648 void *pfault_ctx;
645 struct srcu_struct pfault_srcu; 649 struct srcu_struct pfault_srcu;
646#endif 650#endif
651 struct mlx5_bfreg_data bfregs;
652 struct mlx5_uars_page *uar;
647}; 653};
648 654
649enum mlx5_device_state { 655enum mlx5_device_state {
@@ -712,7 +718,6 @@ struct mlx5_td {
712}; 718};
713 719
714struct mlx5e_resources { 720struct mlx5e_resources {
715 struct mlx5_uar cq_uar;
716 u32 pdn; 721 u32 pdn;
717 struct mlx5_td td; 722 struct mlx5_td td;
718 struct mlx5_core_mkey mkey; 723 struct mlx5_core_mkey mkey;
@@ -902,11 +907,6 @@ void mlx5_cmd_mbox_status(void *out, u8 *status, u32 *syndrome);
902int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type); 907int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type);
903int mlx5_cmd_alloc_uar(struct mlx5_core_dev *dev, u32 *uarn); 908int mlx5_cmd_alloc_uar(struct mlx5_core_dev *dev, u32 *uarn);
904int mlx5_cmd_free_uar(struct mlx5_core_dev *dev, u32 uarn); 909int mlx5_cmd_free_uar(struct mlx5_core_dev *dev, u32 uarn);
905int mlx5_alloc_uuars(struct mlx5_core_dev *dev, struct mlx5_uuar_info *uuari);
906int mlx5_free_uuars(struct mlx5_core_dev *dev, struct mlx5_uuar_info *uuari);
907int mlx5_alloc_map_uar(struct mlx5_core_dev *mdev, struct mlx5_uar *uar,
908 bool map_wc);
909void mlx5_unmap_free_uar(struct mlx5_core_dev *mdev, struct mlx5_uar *uar);
910void mlx5_health_cleanup(struct mlx5_core_dev *dev); 910void mlx5_health_cleanup(struct mlx5_core_dev *dev);
911int mlx5_health_init(struct mlx5_core_dev *dev); 911int mlx5_health_init(struct mlx5_core_dev *dev);
912void mlx5_start_health_poll(struct mlx5_core_dev *dev); 912void mlx5_start_health_poll(struct mlx5_core_dev *dev);
@@ -972,7 +972,7 @@ void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec);
972void mlx5_cq_event(struct mlx5_core_dev *dev, u32 cqn, int event_type); 972void mlx5_cq_event(struct mlx5_core_dev *dev, u32 cqn, int event_type);
973int mlx5_create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, u8 vecidx, 973int mlx5_create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, u8 vecidx,
974 int nent, u64 mask, const char *name, 974 int nent, u64 mask, const char *name,
975 struct mlx5_uar *uar, enum mlx5_eq_type type); 975 enum mlx5_eq_type type);
976int mlx5_destroy_unmap_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq); 976int mlx5_destroy_unmap_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq);
977int mlx5_start_eqs(struct mlx5_core_dev *dev); 977int mlx5_start_eqs(struct mlx5_core_dev *dev);
978int mlx5_stop_eqs(struct mlx5_core_dev *dev); 978int mlx5_stop_eqs(struct mlx5_core_dev *dev);
@@ -1021,6 +1021,9 @@ void mlx5_cleanup_rl_table(struct mlx5_core_dev *dev);
1021int mlx5_rl_add_rate(struct mlx5_core_dev *dev, u32 rate, u16 *index); 1021int mlx5_rl_add_rate(struct mlx5_core_dev *dev, u32 rate, u16 *index);
1022void mlx5_rl_remove_rate(struct mlx5_core_dev *dev, u32 rate); 1022void mlx5_rl_remove_rate(struct mlx5_core_dev *dev, u32 rate);
1023bool mlx5_rl_is_in_range(struct mlx5_core_dev *dev, u32 rate); 1023bool mlx5_rl_is_in_range(struct mlx5_core_dev *dev, u32 rate);
1024int mlx5_alloc_bfreg(struct mlx5_core_dev *mdev, struct mlx5_sq_bfreg *bfreg,
1025 bool map_wc, bool fast_path);
1026void mlx5_free_bfreg(struct mlx5_core_dev *mdev, struct mlx5_sq_bfreg *bfreg);
1024 1027
1025static inline int fw_initializing(struct mlx5_core_dev *dev) 1028static inline int fw_initializing(struct mlx5_core_dev *dev)
1026{ 1029{
@@ -1080,6 +1083,8 @@ int mlx5_cmd_create_vport_lag(struct mlx5_core_dev *dev);
1080int mlx5_cmd_destroy_vport_lag(struct mlx5_core_dev *dev); 1083int mlx5_cmd_destroy_vport_lag(struct mlx5_core_dev *dev);
1081bool mlx5_lag_is_active(struct mlx5_core_dev *dev); 1084bool mlx5_lag_is_active(struct mlx5_core_dev *dev);
1082struct net_device *mlx5_lag_get_roce_netdev(struct mlx5_core_dev *dev); 1085struct net_device *mlx5_lag_get_roce_netdev(struct mlx5_core_dev *dev);
1086struct mlx5_uars_page *mlx5_get_uars_page(struct mlx5_core_dev *mdev);
1087void mlx5_put_uars_page(struct mlx5_core_dev *mdev, struct mlx5_uars_page *up);
1083 1088
1084struct mlx5_profile { 1089struct mlx5_profile {
1085 u64 mask; 1090 u64 mask;
diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h
index d0f090884617..37327f6ba9cb 100644
--- a/include/linux/mlx5/mlx5_ifc.h
+++ b/include/linux/mlx5/mlx5_ifc.h
@@ -905,7 +905,8 @@ struct mlx5_ifc_cmd_hca_cap_bits {
905 u8 uc[0x1]; 905 u8 uc[0x1];
906 u8 rc[0x1]; 906 u8 rc[0x1];
907 907
908 u8 reserved_at_240[0xa]; 908 u8 uar_4k[0x1];
909 u8 reserved_at_241[0x9];
909 u8 uar_sz[0x6]; 910 u8 uar_sz[0x6];
910 u8 reserved_at_250[0x8]; 911 u8 reserved_at_250[0x8];
911 u8 log_pg_sz[0x8]; 912 u8 log_pg_sz[0x8];
@@ -997,7 +998,9 @@ struct mlx5_ifc_cmd_hca_cap_bits {
997 u8 device_frequency_mhz[0x20]; 998 u8 device_frequency_mhz[0x20];
998 u8 device_frequency_khz[0x20]; 999 u8 device_frequency_khz[0x20];
999 1000
1000 u8 reserved_at_500[0x80]; 1001 u8 reserved_at_500[0x20];
1002 u8 num_of_uars_per_page[0x20];
1003 u8 reserved_at_540[0x40];
1001 1004
1002 u8 reserved_at_580[0x3f]; 1005 u8 reserved_at_580[0x3f];
1003 u8 cqe_compression[0x1]; 1006 u8 cqe_compression[0x1];
diff --git a/include/uapi/rdma/mlx5-abi.h b/include/uapi/rdma/mlx5-abi.h
index fae6cdaeb56d..85dc966ea70b 100644
--- a/include/uapi/rdma/mlx5-abi.h
+++ b/include/uapi/rdma/mlx5-abi.h
@@ -61,19 +61,24 @@ enum {
61 */ 61 */
62 62
63struct mlx5_ib_alloc_ucontext_req { 63struct mlx5_ib_alloc_ucontext_req {
64 __u32 total_num_uuars; 64 __u32 total_num_bfregs;
65 __u32 num_low_latency_uuars; 65 __u32 num_low_latency_bfregs;
66};
67
68enum mlx5_lib_caps {
69 MLX5_LIB_CAP_4K_UAR = (u64)1 << 0,
66}; 70};
67 71
68struct mlx5_ib_alloc_ucontext_req_v2 { 72struct mlx5_ib_alloc_ucontext_req_v2 {
69 __u32 total_num_uuars; 73 __u32 total_num_bfregs;
70 __u32 num_low_latency_uuars; 74 __u32 num_low_latency_bfregs;
71 __u32 flags; 75 __u32 flags;
72 __u32 comp_mask; 76 __u32 comp_mask;
73 __u8 max_cqe_version; 77 __u8 max_cqe_version;
74 __u8 reserved0; 78 __u8 reserved0;
75 __u16 reserved1; 79 __u16 reserved1;
76 __u32 reserved2; 80 __u32 reserved2;
81 __u64 lib_caps;
77}; 82};
78 83
79enum mlx5_ib_alloc_ucontext_resp_mask { 84enum mlx5_ib_alloc_ucontext_resp_mask {
@@ -88,7 +93,7 @@ enum mlx5_user_cmds_supp_uhw {
88struct mlx5_ib_alloc_ucontext_resp { 93struct mlx5_ib_alloc_ucontext_resp {
89 __u32 qp_tab_size; 94 __u32 qp_tab_size;
90 __u32 bf_reg_size; 95 __u32 bf_reg_size;
91 __u32 tot_uuars; 96 __u32 tot_bfregs;
92 __u32 cache_line_size; 97 __u32 cache_line_size;
93 __u16 max_sq_desc_sz; 98 __u16 max_sq_desc_sz;
94 __u16 max_rq_desc_sz; 99 __u16 max_rq_desc_sz;
@@ -103,6 +108,8 @@ struct mlx5_ib_alloc_ucontext_resp {
103 __u8 cmds_supp_uhw; 108 __u8 cmds_supp_uhw;
104 __u16 reserved2; 109 __u16 reserved2;
105 __u64 hca_core_clock_offset; 110 __u64 hca_core_clock_offset;
111 __u32 log_uar_size;
112 __u32 num_uars_per_page;
106}; 113};
107 114
108struct mlx5_ib_alloc_pd_resp { 115struct mlx5_ib_alloc_pd_resp {
@@ -241,7 +248,7 @@ struct mlx5_ib_create_qp_rss {
241}; 248};
242 249
243struct mlx5_ib_create_qp_resp { 250struct mlx5_ib_create_qp_resp {
244 __u32 uuar_index; 251 __u32 bfreg_index;
245}; 252};
246 253
247struct mlx5_ib_alloc_mw { 254struct mlx5_ib_alloc_mw {
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-16 10:33:23 -0400