RDMA/nldev: Provide global resource utilization

Expose through the netlink interface the global per-device utilization of
the supported object types.

Provide both dumpit and doit callbacks.

As an example of possible output from rdmatool for system with 5
mlx5 cards:

$ rdma res
1: mlx5_0: qp 4 cq 5 pd 3
2: mlx5_1: qp 4 cq 5 pd 3
3: mlx5_2: qp 4 cq 5 pd 3
4: mlx5_3: qp 2 cq 3 pd 2
5: mlx5_4: qp 4 cq 5 pd 3

Reviewed-by: Mark Bloch <markb@mellanox.com>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
Reviewed-by: Steve Wise <swise@opengridcomputing.com>
Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>

2 files changed, 154 insertions, 0 deletions

diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c
index 5d790c507c7e..c37bb041f647 100644
--- a/drivers/infiniband/core/nldev.c
+++ b/drivers/infiniband/core/nldev.c
@@ -31,6 +31,8 @@
  */
 
 #include <linux/module.h>
+#include <linux/pid.h>
+#include <linux/pid_namespace.h>
 #include <net/netlink.h>
 #include <rdma/rdma_netlink.h>
 
@@ -52,6 +54,11 @@ static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = {
         [RDMA_NLDEV_ATTR_PORT_STATE]    = { .type = NLA_U8 },
         [RDMA_NLDEV_ATTR_PORT_PHYS_STATE] = { .type = NLA_U8 },
         [RDMA_NLDEV_ATTR_DEV_NODE_TYPE] = { .type = NLA_U8 },
+        [RDMA_NLDEV_ATTR_RES_SUMMARY]   = { .type = NLA_NESTED },
+        [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY]     = { .type = NLA_NESTED },
+        [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME] = { .type = NLA_NUL_STRING,
+                                             .len = 16 },
+        [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR] = { .type = NLA_U64 },
 };
 
 static int fill_nldev_handle(struct sk_buff *msg, struct ib_device *device)
@@ -134,6 +141,65 @@ static int fill_port_info(struct sk_buff *msg,
         return 0;
 }
 
+static int fill_res_info_entry(struct sk_buff *msg,
+                               const char *name, u64 curr)
+{
+        struct nlattr *entry_attr;
+
+        entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY);
+        if (!entry_attr)
+                return -EMSGSIZE;
+
+        if (nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME, name))
+                goto err;
+        if (nla_put_u64_64bit(msg,
+                              RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR, curr, 0))
+                goto err;
+
+        nla_nest_end(msg, entry_attr);
+        return 0;
+
+err:
+        nla_nest_cancel(msg, entry_attr);
+        return -EMSGSIZE;
+}
+
+static int fill_res_info(struct sk_buff *msg, struct ib_device *device)
+{
+        static const char * const names[RDMA_RESTRACK_MAX] = {
+                [RDMA_RESTRACK_PD] = "pd",
+                [RDMA_RESTRACK_CQ] = "cq",
+                [RDMA_RESTRACK_QP] = "qp",
+        };
+
+        struct rdma_restrack_root *res = &device->res;
+        struct nlattr *table_attr;
+        int ret, i, curr;
+
+        if (fill_nldev_handle(msg, device))
+                return -EMSGSIZE;
+
+        table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_SUMMARY);
+        if (!table_attr)
+                return -EMSGSIZE;
+
+        for (i = 0; i < RDMA_RESTRACK_MAX; i++) {
+                if (!names[i])
+                        continue;
+                curr = rdma_restrack_count(res, i, task_active_pid_ns(current));
+                ret = fill_res_info_entry(msg, names[i], curr);
+                if (ret)
+                        goto err;
+        }
+
+        nla_nest_end(msg, table_attr);
+        return 0;
+
+err:
+        nla_nest_cancel(msg, table_attr);
+        return ret;
+}
+
 static int nldev_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
                           struct netlink_ext_ack *extack)
 {
@@ -329,6 +395,83 @@ out:
         return skb->len;
 }
 
+static int nldev_res_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
+                              struct netlink_ext_ack *extack)
+{
+        struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
+        struct ib_device *device;
+        struct sk_buff *msg;
+        u32 index;
+        int ret;
+
+        ret = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
+                          nldev_policy, extack);
+        if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
+                return -EINVAL;
+
+        index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
+        device = ib_device_get_by_index(index);
+        if (!device)
+                return -EINVAL;
+
+        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+        if (!msg)
+                goto err;
+
+        nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
+                        RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET),
+                        0, 0);
+
+        ret = fill_res_info(msg, device);
+        if (ret)
+                goto err_free;
+
+        nlmsg_end(msg, nlh);
+        put_device(&device->dev);
+        return rdma_nl_unicast(msg, NETLINK_CB(skb).portid);
+
+err_free:
+        nlmsg_free(msg);
+err:
+        put_device(&device->dev);
+        return ret;
+}
+
+static int _nldev_res_get_dumpit(struct ib_device *device,
+                                 struct sk_buff *skb,
+                                 struct netlink_callback *cb,
+                                 unsigned int idx)
+{
+        int start = cb->args[0];
+        struct nlmsghdr *nlh;
+
+        if (idx < start)
+                return 0;
+
+        nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
+                        RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET),
+                        0, NLM_F_MULTI);
+
+        if (fill_res_info(skb, device)) {
+                nlmsg_cancel(skb, nlh);
+                goto out;
+        }
+
+        nlmsg_end(skb, nlh);
+
+        idx++;
+
+out:
+        cb->args[0] = idx;
+        return skb->len;
+}
+
+static int nldev_res_get_dumpit(struct sk_buff *skb,
+                                struct netlink_callback *cb)
+{
+        return ib_enum_all_devs(_nldev_res_get_dumpit, skb, cb);
+}
+
 static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {
         [RDMA_NLDEV_CMD_GET] = {
                 .doit = nldev_get_doit,
@@ -338,6 +481,10 @@ static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {
                 .doit = nldev_port_get_doit,
                 .dump = nldev_port_get_dumpit,
         },
+        [RDMA_NLDEV_CMD_RES_GET] = {
+                .doit = nldev_res_get_doit,
+                .dump = nldev_res_get_dumpit,
+        },
 };
 
 void __init nldev_init(void)
diff --git a/include/uapi/rdma/rdma_netlink.h b/include/uapi/rdma/rdma_netlink.h
index cc002e316d09..22c39532c411 100644
--- a/include/uapi/rdma/rdma_netlink.h
+++ b/include/uapi/rdma/rdma_netlink.h
@@ -236,6 +236,8 @@ enum rdma_nldev_command {
         RDMA_NLDEV_CMD_PORT_NEW,
         RDMA_NLDEV_CMD_PORT_DEL,
 
+        RDMA_NLDEV_CMD_RES_GET, /* can dump */
+
         RDMA_NLDEV_NUM_OPS
 };
 
@@ -303,6 +305,11 @@ enum rdma_nldev_attr {
 
         RDMA_NLDEV_ATTR_DEV_NODE_TYPE,          /* u8 */
 
+        RDMA_NLDEV_ATTR_RES_SUMMARY,            /* nested table */
+        RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY,      /* nested table */
+        RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME, /* string */
+        RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR, /* u64 */
+
         RDMA_NLDEV_ATTR_MAX
 };
 #endif /* _UAPI_RDMA_NETLINK_H */