aboutsummaryrefslogtreecommitdiffstats
path: root/net/core
diff options
context:
space:
mode:
Diffstat (limited to 'net/core')
-rw-r--r--net/core/devlink.c1080
-rw-r--r--net/core/drop_monitor.c724
2 files changed, 1776 insertions, 28 deletions
diff --git a/net/core/devlink.c b/net/core/devlink.c
index d3dbb904bf3b..650f36379203 100644
--- a/net/core/devlink.c
+++ b/net/core/devlink.c
@@ -18,6 +18,8 @@
18#include <linux/spinlock.h> 18#include <linux/spinlock.h>
19#include <linux/refcount.h> 19#include <linux/refcount.h>
20#include <linux/workqueue.h> 20#include <linux/workqueue.h>
21#include <linux/u64_stats_sync.h>
22#include <linux/timekeeping.h>
21#include <rdma/ib_verbs.h> 23#include <rdma/ib_verbs.h>
22#include <net/netlink.h> 24#include <net/netlink.h>
23#include <net/genetlink.h> 25#include <net/genetlink.h>
@@ -25,6 +27,7 @@
25#include <net/net_namespace.h> 27#include <net/net_namespace.h>
26#include <net/sock.h> 28#include <net/sock.h>
27#include <net/devlink.h> 29#include <net/devlink.h>
30#include <net/drop_monitor.h>
28#define CREATE_TRACE_POINTS 31#define CREATE_TRACE_POINTS
29#include <trace/events/devlink.h> 32#include <trace/events/devlink.h>
30 33
@@ -551,7 +554,7 @@ static int devlink_nl_port_fill(struct sk_buff *msg, struct devlink *devlink,
551 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index)) 554 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
552 goto nla_put_failure; 555 goto nla_put_failure;
553 556
554 spin_lock(&devlink_port->type_lock); 557 spin_lock_bh(&devlink_port->type_lock);
555 if (nla_put_u16(msg, DEVLINK_ATTR_PORT_TYPE, devlink_port->type)) 558 if (nla_put_u16(msg, DEVLINK_ATTR_PORT_TYPE, devlink_port->type))
556 goto nla_put_failure_type_locked; 559 goto nla_put_failure_type_locked;
557 if (devlink_port->desired_type != DEVLINK_PORT_TYPE_NOTSET && 560 if (devlink_port->desired_type != DEVLINK_PORT_TYPE_NOTSET &&
@@ -576,7 +579,7 @@ static int devlink_nl_port_fill(struct sk_buff *msg, struct devlink *devlink,
576 ibdev->name)) 579 ibdev->name))
577 goto nla_put_failure_type_locked; 580 goto nla_put_failure_type_locked;
578 } 581 }
579 spin_unlock(&devlink_port->type_lock); 582 spin_unlock_bh(&devlink_port->type_lock);
580 if (devlink_nl_port_attrs_put(msg, devlink_port)) 583 if (devlink_nl_port_attrs_put(msg, devlink_port))
581 goto nla_put_failure; 584 goto nla_put_failure;
582 585
@@ -584,7 +587,7 @@ static int devlink_nl_port_fill(struct sk_buff *msg, struct devlink *devlink,
584 return 0; 587 return 0;
585 588
586nla_put_failure_type_locked: 589nla_put_failure_type_locked:
587 spin_unlock(&devlink_port->type_lock); 590 spin_unlock_bh(&devlink_port->type_lock);
588nla_put_failure: 591nla_put_failure:
589 genlmsg_cancel(msg, hdr); 592 genlmsg_cancel(msg, hdr);
590 return -EMSGSIZE; 593 return -EMSGSIZE;
@@ -5154,6 +5157,571 @@ devlink_nl_cmd_health_reporter_dump_clear_doit(struct sk_buff *skb,
5154 return 0; 5157 return 0;
5155} 5158}
5156 5159
5160struct devlink_stats {
5161 u64 rx_bytes;
5162 u64 rx_packets;
5163 struct u64_stats_sync syncp;
5164};
5165
5166/**
5167 * struct devlink_trap_group_item - Packet trap group attributes.
5168 * @group: Immutable packet trap group attributes.
5169 * @refcount: Number of trap items using the group.
5170 * @list: trap_group_list member.
5171 * @stats: Trap group statistics.
5172 *
5173 * Describes packet trap group attributes. Created by devlink during trap
5174 * registration.
5175 */
5176struct devlink_trap_group_item {
5177 const struct devlink_trap_group *group;
5178 refcount_t refcount;
5179 struct list_head list;
5180 struct devlink_stats __percpu *stats;
5181};
5182
5183/**
5184 * struct devlink_trap_item - Packet trap attributes.
5185 * @trap: Immutable packet trap attributes.
5186 * @group_item: Associated group item.
5187 * @list: trap_list member.
5188 * @action: Trap action.
5189 * @stats: Trap statistics.
5190 * @priv: Driver private information.
5191 *
5192 * Describes both mutable and immutable packet trap attributes. Created by
5193 * devlink during trap registration and used for all trap related operations.
5194 */
5195struct devlink_trap_item {
5196 const struct devlink_trap *trap;
5197 struct devlink_trap_group_item *group_item;
5198 struct list_head list;
5199 enum devlink_trap_action action;
5200 struct devlink_stats __percpu *stats;
5201 void *priv;
5202};
5203
5204static struct devlink_trap_item *
5205devlink_trap_item_lookup(struct devlink *devlink, const char *name)
5206{
5207 struct devlink_trap_item *trap_item;
5208
5209 list_for_each_entry(trap_item, &devlink->trap_list, list) {
5210 if (!strcmp(trap_item->trap->name, name))
5211 return trap_item;
5212 }
5213
5214 return NULL;
5215}
5216
5217static struct devlink_trap_item *
5218devlink_trap_item_get_from_info(struct devlink *devlink,
5219 struct genl_info *info)
5220{
5221 struct nlattr *attr;
5222
5223 if (!info->attrs[DEVLINK_ATTR_TRAP_NAME])
5224 return NULL;
5225 attr = info->attrs[DEVLINK_ATTR_TRAP_NAME];
5226
5227 return devlink_trap_item_lookup(devlink, nla_data(attr));
5228}
5229
5230static int
5231devlink_trap_action_get_from_info(struct genl_info *info,
5232 enum devlink_trap_action *p_trap_action)
5233{
5234 u8 val;
5235
5236 val = nla_get_u8(info->attrs[DEVLINK_ATTR_TRAP_ACTION]);
5237 switch (val) {
5238 case DEVLINK_TRAP_ACTION_DROP: /* fall-through */
5239 case DEVLINK_TRAP_ACTION_TRAP:
5240 *p_trap_action = val;
5241 break;
5242 default:
5243 return -EINVAL;
5244 }
5245
5246 return 0;
5247}
5248
5249static int devlink_trap_metadata_put(struct sk_buff *msg,
5250 const struct devlink_trap *trap)
5251{
5252 struct nlattr *attr;
5253
5254 attr = nla_nest_start(msg, DEVLINK_ATTR_TRAP_METADATA);
5255 if (!attr)
5256 return -EMSGSIZE;
5257
5258 if ((trap->metadata_cap & DEVLINK_TRAP_METADATA_TYPE_F_IN_PORT) &&
5259 nla_put_flag(msg, DEVLINK_ATTR_TRAP_METADATA_TYPE_IN_PORT))
5260 goto nla_put_failure;
5261
5262 nla_nest_end(msg, attr);
5263
5264 return 0;
5265
5266nla_put_failure:
5267 nla_nest_cancel(msg, attr);
5268 return -EMSGSIZE;
5269}
5270
5271static void devlink_trap_stats_read(struct devlink_stats __percpu *trap_stats,
5272 struct devlink_stats *stats)
5273{
5274 int i;
5275
5276 memset(stats, 0, sizeof(*stats));
5277 for_each_possible_cpu(i) {
5278 struct devlink_stats *cpu_stats;
5279 u64 rx_packets, rx_bytes;
5280 unsigned int start;
5281
5282 cpu_stats = per_cpu_ptr(trap_stats, i);
5283 do {
5284 start = u64_stats_fetch_begin_irq(&cpu_stats->syncp);
5285 rx_packets = cpu_stats->rx_packets;
5286 rx_bytes = cpu_stats->rx_bytes;
5287 } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start));
5288
5289 stats->rx_packets += rx_packets;
5290 stats->rx_bytes += rx_bytes;
5291 }
5292}
5293
5294static int devlink_trap_stats_put(struct sk_buff *msg,
5295 struct devlink_stats __percpu *trap_stats)
5296{
5297 struct devlink_stats stats;
5298 struct nlattr *attr;
5299
5300 devlink_trap_stats_read(trap_stats, &stats);
5301
5302 attr = nla_nest_start(msg, DEVLINK_ATTR_STATS);
5303 if (!attr)
5304 return -EMSGSIZE;
5305
5306 if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_PACKETS,
5307 stats.rx_packets, DEVLINK_ATTR_PAD))
5308 goto nla_put_failure;
5309
5310 if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_BYTES,
5311 stats.rx_bytes, DEVLINK_ATTR_PAD))
5312 goto nla_put_failure;
5313
5314 nla_nest_end(msg, attr);
5315
5316 return 0;
5317
5318nla_put_failure:
5319 nla_nest_cancel(msg, attr);
5320 return -EMSGSIZE;
5321}
5322
5323static int devlink_nl_trap_fill(struct sk_buff *msg, struct devlink *devlink,
5324 const struct devlink_trap_item *trap_item,
5325 enum devlink_command cmd, u32 portid, u32 seq,
5326 int flags)
5327{
5328 struct devlink_trap_group_item *group_item = trap_item->group_item;
5329 void *hdr;
5330 int err;
5331
5332 hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
5333 if (!hdr)
5334 return -EMSGSIZE;
5335
5336 if (devlink_nl_put_handle(msg, devlink))
5337 goto nla_put_failure;
5338
5339 if (nla_put_string(msg, DEVLINK_ATTR_TRAP_GROUP_NAME,
5340 group_item->group->name))
5341 goto nla_put_failure;
5342
5343 if (nla_put_string(msg, DEVLINK_ATTR_TRAP_NAME, trap_item->trap->name))
5344 goto nla_put_failure;
5345
5346 if (nla_put_u8(msg, DEVLINK_ATTR_TRAP_TYPE, trap_item->trap->type))
5347 goto nla_put_failure;
5348
5349 if (trap_item->trap->generic &&
5350 nla_put_flag(msg, DEVLINK_ATTR_TRAP_GENERIC))
5351 goto nla_put_failure;
5352
5353 if (nla_put_u8(msg, DEVLINK_ATTR_TRAP_ACTION, trap_item->action))
5354 goto nla_put_failure;
5355
5356 err = devlink_trap_metadata_put(msg, trap_item->trap);
5357 if (err)
5358 goto nla_put_failure;
5359
5360 err = devlink_trap_stats_put(msg, trap_item->stats);
5361 if (err)
5362 goto nla_put_failure;
5363
5364 genlmsg_end(msg, hdr);
5365
5366 return 0;
5367
5368nla_put_failure:
5369 genlmsg_cancel(msg, hdr);
5370 return -EMSGSIZE;
5371}
5372
5373static int devlink_nl_cmd_trap_get_doit(struct sk_buff *skb,
5374 struct genl_info *info)
5375{
5376 struct netlink_ext_ack *extack = info->extack;
5377 struct devlink *devlink = info->user_ptr[0];
5378 struct devlink_trap_item *trap_item;
5379 struct sk_buff *msg;
5380 int err;
5381
5382 if (list_empty(&devlink->trap_list))
5383 return -EOPNOTSUPP;
5384
5385 trap_item = devlink_trap_item_get_from_info(devlink, info);
5386 if (!trap_item) {
5387 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap");
5388 return -ENOENT;
5389 }
5390
5391 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
5392 if (!msg)
5393 return -ENOMEM;
5394
5395 err = devlink_nl_trap_fill(msg, devlink, trap_item,
5396 DEVLINK_CMD_TRAP_NEW, info->snd_portid,
5397 info->snd_seq, 0);
5398 if (err)
5399 goto err_trap_fill;
5400
5401 return genlmsg_reply(msg, info);
5402
5403err_trap_fill:
5404 nlmsg_free(msg);
5405 return err;
5406}
5407
5408static int devlink_nl_cmd_trap_get_dumpit(struct sk_buff *msg,
5409 struct netlink_callback *cb)
5410{
5411 struct devlink_trap_item *trap_item;
5412 struct devlink *devlink;
5413 int start = cb->args[0];
5414 int idx = 0;
5415 int err;
5416
5417 mutex_lock(&devlink_mutex);
5418 list_for_each_entry(devlink, &devlink_list, list) {
5419 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
5420 continue;
5421 mutex_lock(&devlink->lock);
5422 list_for_each_entry(trap_item, &devlink->trap_list, list) {
5423 if (idx < start) {
5424 idx++;
5425 continue;
5426 }
5427 err = devlink_nl_trap_fill(msg, devlink, trap_item,
5428 DEVLINK_CMD_TRAP_NEW,
5429 NETLINK_CB(cb->skb).portid,
5430 cb->nlh->nlmsg_seq,
5431 NLM_F_MULTI);
5432 if (err) {
5433 mutex_unlock(&devlink->lock);
5434 goto out;
5435 }
5436 idx++;
5437 }
5438 mutex_unlock(&devlink->lock);
5439 }
5440out:
5441 mutex_unlock(&devlink_mutex);
5442
5443 cb->args[0] = idx;
5444 return msg->len;
5445}
5446
5447static int __devlink_trap_action_set(struct devlink *devlink,
5448 struct devlink_trap_item *trap_item,
5449 enum devlink_trap_action trap_action,
5450 struct netlink_ext_ack *extack)
5451{
5452 int err;
5453
5454 if (trap_item->action != trap_action &&
5455 trap_item->trap->type != DEVLINK_TRAP_TYPE_DROP) {
5456 NL_SET_ERR_MSG_MOD(extack, "Cannot change action of non-drop traps. Skipping");
5457 return 0;
5458 }
5459
5460 err = devlink->ops->trap_action_set(devlink, trap_item->trap,
5461 trap_action);
5462 if (err)
5463 return err;
5464
5465 trap_item->action = trap_action;
5466
5467 return 0;
5468}
5469
5470static int devlink_trap_action_set(struct devlink *devlink,
5471 struct devlink_trap_item *trap_item,
5472 struct genl_info *info)
5473{
5474 enum devlink_trap_action trap_action;
5475 int err;
5476
5477 if (!info->attrs[DEVLINK_ATTR_TRAP_ACTION])
5478 return 0;
5479
5480 err = devlink_trap_action_get_from_info(info, &trap_action);
5481 if (err) {
5482 NL_SET_ERR_MSG_MOD(info->extack, "Invalid trap action");
5483 return -EINVAL;
5484 }
5485
5486 return __devlink_trap_action_set(devlink, trap_item, trap_action,
5487 info->extack);
5488}
5489
5490static int devlink_nl_cmd_trap_set_doit(struct sk_buff *skb,
5491 struct genl_info *info)
5492{
5493 struct netlink_ext_ack *extack = info->extack;
5494 struct devlink *devlink = info->user_ptr[0];
5495 struct devlink_trap_item *trap_item;
5496 int err;
5497
5498 if (list_empty(&devlink->trap_list))
5499 return -EOPNOTSUPP;
5500
5501 trap_item = devlink_trap_item_get_from_info(devlink, info);
5502 if (!trap_item) {
5503 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap");
5504 return -ENOENT;
5505 }
5506
5507 err = devlink_trap_action_set(devlink, trap_item, info);
5508 if (err)
5509 return err;
5510
5511 return 0;
5512}
5513
5514static struct devlink_trap_group_item *
5515devlink_trap_group_item_lookup(struct devlink *devlink, const char *name)
5516{
5517 struct devlink_trap_group_item *group_item;
5518
5519 list_for_each_entry(group_item, &devlink->trap_group_list, list) {
5520 if (!strcmp(group_item->group->name, name))
5521 return group_item;
5522 }
5523
5524 return NULL;
5525}
5526
5527static struct devlink_trap_group_item *
5528devlink_trap_group_item_get_from_info(struct devlink *devlink,
5529 struct genl_info *info)
5530{
5531 char *name;
5532
5533 if (!info->attrs[DEVLINK_ATTR_TRAP_GROUP_NAME])
5534 return NULL;
5535 name = nla_data(info->attrs[DEVLINK_ATTR_TRAP_GROUP_NAME]);
5536
5537 return devlink_trap_group_item_lookup(devlink, name);
5538}
5539
5540static int
5541devlink_nl_trap_group_fill(struct sk_buff *msg, struct devlink *devlink,
5542 const struct devlink_trap_group_item *group_item,
5543 enum devlink_command cmd, u32 portid, u32 seq,
5544 int flags)
5545{
5546 void *hdr;
5547 int err;
5548
5549 hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
5550 if (!hdr)
5551 return -EMSGSIZE;
5552
5553 if (devlink_nl_put_handle(msg, devlink))
5554 goto nla_put_failure;
5555
5556 if (nla_put_string(msg, DEVLINK_ATTR_TRAP_GROUP_NAME,
5557 group_item->group->name))
5558 goto nla_put_failure;
5559
5560 if (group_item->group->generic &&
5561 nla_put_flag(msg, DEVLINK_ATTR_TRAP_GENERIC))
5562 goto nla_put_failure;
5563
5564 err = devlink_trap_stats_put(msg, group_item->stats);
5565 if (err)
5566 goto nla_put_failure;
5567
5568 genlmsg_end(msg, hdr);
5569
5570 return 0;
5571
5572nla_put_failure:
5573 genlmsg_cancel(msg, hdr);
5574 return -EMSGSIZE;
5575}
5576
5577static int devlink_nl_cmd_trap_group_get_doit(struct sk_buff *skb,
5578 struct genl_info *info)
5579{
5580 struct netlink_ext_ack *extack = info->extack;
5581 struct devlink *devlink = info->user_ptr[0];
5582 struct devlink_trap_group_item *group_item;
5583 struct sk_buff *msg;
5584 int err;
5585
5586 if (list_empty(&devlink->trap_group_list))
5587 return -EOPNOTSUPP;
5588
5589 group_item = devlink_trap_group_item_get_from_info(devlink, info);
5590 if (!group_item) {
5591 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap group");
5592 return -ENOENT;
5593 }
5594
5595 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
5596 if (!msg)
5597 return -ENOMEM;
5598
5599 err = devlink_nl_trap_group_fill(msg, devlink, group_item,
5600 DEVLINK_CMD_TRAP_GROUP_NEW,
5601 info->snd_portid, info->snd_seq, 0);
5602 if (err)
5603 goto err_trap_group_fill;
5604
5605 return genlmsg_reply(msg, info);
5606
5607err_trap_group_fill:
5608 nlmsg_free(msg);
5609 return err;
5610}
5611
5612static int devlink_nl_cmd_trap_group_get_dumpit(struct sk_buff *msg,
5613 struct netlink_callback *cb)
5614{
5615 enum devlink_command cmd = DEVLINK_CMD_TRAP_GROUP_NEW;
5616 struct devlink_trap_group_item *group_item;
5617 u32 portid = NETLINK_CB(cb->skb).portid;
5618 struct devlink *devlink;
5619 int start = cb->args[0];
5620 int idx = 0;
5621 int err;
5622
5623 mutex_lock(&devlink_mutex);
5624 list_for_each_entry(devlink, &devlink_list, list) {
5625 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
5626 continue;
5627 mutex_lock(&devlink->lock);
5628 list_for_each_entry(group_item, &devlink->trap_group_list,
5629 list) {
5630 if (idx < start) {
5631 idx++;
5632 continue;
5633 }
5634 err = devlink_nl_trap_group_fill(msg, devlink,
5635 group_item, cmd,
5636 portid,
5637 cb->nlh->nlmsg_seq,
5638 NLM_F_MULTI);
5639 if (err) {
5640 mutex_unlock(&devlink->lock);
5641 goto out;
5642 }
5643 idx++;
5644 }
5645 mutex_unlock(&devlink->lock);
5646 }
5647out:
5648 mutex_unlock(&devlink_mutex);
5649
5650 cb->args[0] = idx;
5651 return msg->len;
5652}
5653
5654static int
5655__devlink_trap_group_action_set(struct devlink *devlink,
5656 struct devlink_trap_group_item *group_item,
5657 enum devlink_trap_action trap_action,
5658 struct netlink_ext_ack *extack)
5659{
5660 const char *group_name = group_item->group->name;
5661 struct devlink_trap_item *trap_item;
5662 int err;
5663
5664 list_for_each_entry(trap_item, &devlink->trap_list, list) {
5665 if (strcmp(trap_item->trap->group.name, group_name))
5666 continue;
5667 err = __devlink_trap_action_set(devlink, trap_item,
5668 trap_action, extack);
5669 if (err)
5670 return err;
5671 }
5672
5673 return 0;
5674}
5675
5676static int
5677devlink_trap_group_action_set(struct devlink *devlink,
5678 struct devlink_trap_group_item *group_item,
5679 struct genl_info *info)
5680{
5681 enum devlink_trap_action trap_action;
5682 int err;
5683
5684 if (!info->attrs[DEVLINK_ATTR_TRAP_ACTION])
5685 return 0;
5686
5687 err = devlink_trap_action_get_from_info(info, &trap_action);
5688 if (err) {
5689 NL_SET_ERR_MSG_MOD(info->extack, "Invalid trap action");
5690 return -EINVAL;
5691 }
5692
5693 err = __devlink_trap_group_action_set(devlink, group_item, trap_action,
5694 info->extack);
5695 if (err)
5696 return err;
5697
5698 return 0;
5699}
5700
5701static int devlink_nl_cmd_trap_group_set_doit(struct sk_buff *skb,
5702 struct genl_info *info)
5703{
5704 struct netlink_ext_ack *extack = info->extack;
5705 struct devlink *devlink = info->user_ptr[0];
5706 struct devlink_trap_group_item *group_item;
5707 int err;
5708
5709 if (list_empty(&devlink->trap_group_list))
5710 return -EOPNOTSUPP;
5711
5712 group_item = devlink_trap_group_item_get_from_info(devlink, info);
5713 if (!group_item) {
5714 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap group");
5715 return -ENOENT;
5716 }
5717
5718 err = devlink_trap_group_action_set(devlink, group_item, info);
5719 if (err)
5720 return err;
5721
5722 return 0;
5723}
5724
5157static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = { 5725static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = {
5158 [DEVLINK_ATTR_BUS_NAME] = { .type = NLA_NUL_STRING }, 5726 [DEVLINK_ATTR_BUS_NAME] = { .type = NLA_NUL_STRING },
5159 [DEVLINK_ATTR_DEV_NAME] = { .type = NLA_NUL_STRING }, 5727 [DEVLINK_ATTR_DEV_NAME] = { .type = NLA_NUL_STRING },
@@ -5184,6 +5752,9 @@ static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = {
5184 [DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER] = { .type = NLA_U8 }, 5752 [DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER] = { .type = NLA_U8 },
5185 [DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME] = { .type = NLA_NUL_STRING }, 5753 [DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME] = { .type = NLA_NUL_STRING },
5186 [DEVLINK_ATTR_FLASH_UPDATE_COMPONENT] = { .type = NLA_NUL_STRING }, 5754 [DEVLINK_ATTR_FLASH_UPDATE_COMPONENT] = { .type = NLA_NUL_STRING },
5755 [DEVLINK_ATTR_TRAP_NAME] = { .type = NLA_NUL_STRING },
5756 [DEVLINK_ATTR_TRAP_ACTION] = { .type = NLA_U8 },
5757 [DEVLINK_ATTR_TRAP_GROUP_NAME] = { .type = NLA_NUL_STRING },
5187}; 5758};
5188 5759
5189static const struct genl_ops devlink_nl_ops[] = { 5760static const struct genl_ops devlink_nl_ops[] = {
@@ -5483,6 +6054,32 @@ static const struct genl_ops devlink_nl_ops[] = {
5483 .flags = GENL_ADMIN_PERM, 6054 .flags = GENL_ADMIN_PERM,
5484 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK, 6055 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK,
5485 }, 6056 },
6057 {
6058 .cmd = DEVLINK_CMD_TRAP_GET,
6059 .doit = devlink_nl_cmd_trap_get_doit,
6060 .dumpit = devlink_nl_cmd_trap_get_dumpit,
6061 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK,
6062 /* can be retrieved by unprivileged users */
6063 },
6064 {
6065 .cmd = DEVLINK_CMD_TRAP_SET,
6066 .doit = devlink_nl_cmd_trap_set_doit,
6067 .flags = GENL_ADMIN_PERM,
6068 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK,
6069 },
6070 {
6071 .cmd = DEVLINK_CMD_TRAP_GROUP_GET,
6072 .doit = devlink_nl_cmd_trap_group_get_doit,
6073 .dumpit = devlink_nl_cmd_trap_group_get_dumpit,
6074 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK,
6075 /* can be retrieved by unprivileged users */
6076 },
6077 {
6078 .cmd = DEVLINK_CMD_TRAP_GROUP_SET,
6079 .doit = devlink_nl_cmd_trap_group_set_doit,
6080 .flags = GENL_ADMIN_PERM,
6081 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK,
6082 },
5486}; 6083};
5487 6084
5488static struct genl_family devlink_nl_family __ro_after_init = { 6085static struct genl_family devlink_nl_family __ro_after_init = {
@@ -5528,6 +6125,8 @@ struct devlink *devlink_alloc(const struct devlink_ops *ops, size_t priv_size)
5528 INIT_LIST_HEAD(&devlink->param_list); 6125 INIT_LIST_HEAD(&devlink->param_list);
5529 INIT_LIST_HEAD(&devlink->region_list); 6126 INIT_LIST_HEAD(&devlink->region_list);
5530 INIT_LIST_HEAD(&devlink->reporter_list); 6127 INIT_LIST_HEAD(&devlink->reporter_list);
6128 INIT_LIST_HEAD(&devlink->trap_list);
6129 INIT_LIST_HEAD(&devlink->trap_group_list);
5531 mutex_init(&devlink->lock); 6130 mutex_init(&devlink->lock);
5532 mutex_init(&devlink->reporters_lock); 6131 mutex_init(&devlink->reporters_lock);
5533 return devlink; 6132 return devlink;
@@ -5574,6 +6173,8 @@ void devlink_free(struct devlink *devlink)
5574{ 6173{
5575 mutex_destroy(&devlink->reporters_lock); 6174 mutex_destroy(&devlink->reporters_lock);
5576 mutex_destroy(&devlink->lock); 6175 mutex_destroy(&devlink->lock);
6176 WARN_ON(!list_empty(&devlink->trap_group_list));
6177 WARN_ON(!list_empty(&devlink->trap_list));
5577 WARN_ON(!list_empty(&devlink->reporter_list)); 6178 WARN_ON(!list_empty(&devlink->reporter_list));
5578 WARN_ON(!list_empty(&devlink->region_list)); 6179 WARN_ON(!list_empty(&devlink->region_list));
5579 WARN_ON(!list_empty(&devlink->param_list)); 6180 WARN_ON(!list_empty(&devlink->param_list));
@@ -5678,10 +6279,10 @@ static void __devlink_port_type_set(struct devlink_port *devlink_port,
5678 if (WARN_ON(!devlink_port->registered)) 6279 if (WARN_ON(!devlink_port->registered))
5679 return; 6280 return;
5680 devlink_port_type_warn_cancel(devlink_port); 6281 devlink_port_type_warn_cancel(devlink_port);
5681 spin_lock(&devlink_port->type_lock); 6282 spin_lock_bh(&devlink_port->type_lock);
5682 devlink_port->type = type; 6283 devlink_port->type = type;
5683 devlink_port->type_dev = type_dev; 6284 devlink_port->type_dev = type_dev;
5684 spin_unlock(&devlink_port->type_lock); 6285 spin_unlock_bh(&devlink_port->type_lock);
5685 devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW); 6286 devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
5686} 6287}
5687 6288
@@ -6834,6 +7435,475 @@ unlock:
6834} 7435}
6835EXPORT_SYMBOL_GPL(devlink_region_snapshot_create); 7436EXPORT_SYMBOL_GPL(devlink_region_snapshot_create);
6836 7437
7438#define DEVLINK_TRAP(_id, _type) \
7439 { \
7440 .type = DEVLINK_TRAP_TYPE_##_type, \
7441 .id = DEVLINK_TRAP_GENERIC_ID_##_id, \
7442 .name = DEVLINK_TRAP_GENERIC_NAME_##_id, \
7443 }
7444
7445static const struct devlink_trap devlink_trap_generic[] = {
7446 DEVLINK_TRAP(SMAC_MC, DROP),
7447 DEVLINK_TRAP(VLAN_TAG_MISMATCH, DROP),
7448 DEVLINK_TRAP(INGRESS_VLAN_FILTER, DROP),
7449 DEVLINK_TRAP(INGRESS_STP_FILTER, DROP),
7450 DEVLINK_TRAP(EMPTY_TX_LIST, DROP),
7451 DEVLINK_TRAP(PORT_LOOPBACK_FILTER, DROP),
7452 DEVLINK_TRAP(BLACKHOLE_ROUTE, DROP),
7453 DEVLINK_TRAP(TTL_ERROR, EXCEPTION),
7454 DEVLINK_TRAP(TAIL_DROP, DROP),
7455};
7456
7457#define DEVLINK_TRAP_GROUP(_id) \
7458 { \
7459 .id = DEVLINK_TRAP_GROUP_GENERIC_ID_##_id, \
7460 .name = DEVLINK_TRAP_GROUP_GENERIC_NAME_##_id, \
7461 }
7462
7463static const struct devlink_trap_group devlink_trap_group_generic[] = {
7464 DEVLINK_TRAP_GROUP(L2_DROPS),
7465 DEVLINK_TRAP_GROUP(L3_DROPS),
7466 DEVLINK_TRAP_GROUP(BUFFER_DROPS),
7467};
7468
7469static int devlink_trap_generic_verify(const struct devlink_trap *trap)
7470{
7471 if (trap->id > DEVLINK_TRAP_GENERIC_ID_MAX)
7472 return -EINVAL;
7473
7474 if (strcmp(trap->name, devlink_trap_generic[trap->id].name))
7475 return -EINVAL;
7476
7477 if (trap->type != devlink_trap_generic[trap->id].type)
7478 return -EINVAL;
7479
7480 return 0;
7481}
7482
7483static int devlink_trap_driver_verify(const struct devlink_trap *trap)
7484{
7485 int i;
7486
7487 if (trap->id <= DEVLINK_TRAP_GENERIC_ID_MAX)
7488 return -EINVAL;
7489
7490 for (i = 0; i < ARRAY_SIZE(devlink_trap_generic); i++) {
7491 if (!strcmp(trap->name, devlink_trap_generic[i].name))
7492 return -EEXIST;
7493 }
7494
7495 return 0;
7496}
7497
7498static int devlink_trap_verify(const struct devlink_trap *trap)
7499{
7500 if (!trap || !trap->name || !trap->group.name)
7501 return -EINVAL;
7502
7503 if (trap->generic)
7504 return devlink_trap_generic_verify(trap);
7505 else
7506 return devlink_trap_driver_verify(trap);
7507}
7508
7509static int
7510devlink_trap_group_generic_verify(const struct devlink_trap_group *group)
7511{
7512 if (group->id > DEVLINK_TRAP_GROUP_GENERIC_ID_MAX)
7513 return -EINVAL;
7514
7515 if (strcmp(group->name, devlink_trap_group_generic[group->id].name))
7516 return -EINVAL;
7517
7518 return 0;
7519}
7520
7521static int
7522devlink_trap_group_driver_verify(const struct devlink_trap_group *group)
7523{
7524 int i;
7525
7526 if (group->id <= DEVLINK_TRAP_GROUP_GENERIC_ID_MAX)
7527 return -EINVAL;
7528
7529 for (i = 0; i < ARRAY_SIZE(devlink_trap_group_generic); i++) {
7530 if (!strcmp(group->name, devlink_trap_group_generic[i].name))
7531 return -EEXIST;
7532 }
7533
7534 return 0;
7535}
7536
7537static int devlink_trap_group_verify(const struct devlink_trap_group *group)
7538{
7539 if (group->generic)
7540 return devlink_trap_group_generic_verify(group);
7541 else
7542 return devlink_trap_group_driver_verify(group);
7543}
7544
7545static void
7546devlink_trap_group_notify(struct devlink *devlink,
7547 const struct devlink_trap_group_item *group_item,
7548 enum devlink_command cmd)
7549{
7550 struct sk_buff *msg;
7551 int err;
7552
7553 WARN_ON_ONCE(cmd != DEVLINK_CMD_TRAP_GROUP_NEW &&
7554 cmd != DEVLINK_CMD_TRAP_GROUP_DEL);
7555
7556 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
7557 if (!msg)
7558 return;
7559
7560 err = devlink_nl_trap_group_fill(msg, devlink, group_item, cmd, 0, 0,
7561 0);
7562 if (err) {
7563 nlmsg_free(msg);
7564 return;
7565 }
7566
7567 genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
7568 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
7569}
7570
7571static struct devlink_trap_group_item *
7572devlink_trap_group_item_create(struct devlink *devlink,
7573 const struct devlink_trap_group *group)
7574{
7575 struct devlink_trap_group_item *group_item;
7576 int err;
7577
7578 err = devlink_trap_group_verify(group);
7579 if (err)
7580 return ERR_PTR(err);
7581
7582 group_item = kzalloc(sizeof(*group_item), GFP_KERNEL);
7583 if (!group_item)
7584 return ERR_PTR(-ENOMEM);
7585
7586 group_item->stats = netdev_alloc_pcpu_stats(struct devlink_stats);
7587 if (!group_item->stats) {
7588 err = -ENOMEM;
7589 goto err_stats_alloc;
7590 }
7591
7592 group_item->group = group;
7593 refcount_set(&group_item->refcount, 1);
7594
7595 if (devlink->ops->trap_group_init) {
7596 err = devlink->ops->trap_group_init(devlink, group);
7597 if (err)
7598 goto err_group_init;
7599 }
7600
7601 list_add_tail(&group_item->list, &devlink->trap_group_list);
7602 devlink_trap_group_notify(devlink, group_item,
7603 DEVLINK_CMD_TRAP_GROUP_NEW);
7604
7605 return group_item;
7606
7607err_group_init:
7608 free_percpu(group_item->stats);
7609err_stats_alloc:
7610 kfree(group_item);
7611 return ERR_PTR(err);
7612}
7613
7614static void
7615devlink_trap_group_item_destroy(struct devlink *devlink,
7616 struct devlink_trap_group_item *group_item)
7617{
7618 devlink_trap_group_notify(devlink, group_item,
7619 DEVLINK_CMD_TRAP_GROUP_DEL);
7620 list_del(&group_item->list);
7621 free_percpu(group_item->stats);
7622 kfree(group_item);
7623}
7624
7625static struct devlink_trap_group_item *
7626devlink_trap_group_item_get(struct devlink *devlink,
7627 const struct devlink_trap_group *group)
7628{
7629 struct devlink_trap_group_item *group_item;
7630
7631 group_item = devlink_trap_group_item_lookup(devlink, group->name);
7632 if (group_item) {
7633 refcount_inc(&group_item->refcount);
7634 return group_item;
7635 }
7636
7637 return devlink_trap_group_item_create(devlink, group);
7638}
7639
7640static void
7641devlink_trap_group_item_put(struct devlink *devlink,
7642 struct devlink_trap_group_item *group_item)
7643{
7644 if (!refcount_dec_and_test(&group_item->refcount))
7645 return;
7646
7647 devlink_trap_group_item_destroy(devlink, group_item);
7648}
7649
7650static int
7651devlink_trap_item_group_link(struct devlink *devlink,
7652 struct devlink_trap_item *trap_item)
7653{
7654 struct devlink_trap_group_item *group_item;
7655
7656 group_item = devlink_trap_group_item_get(devlink,
7657 &trap_item->trap->group);
7658 if (IS_ERR(group_item))
7659 return PTR_ERR(group_item);
7660
7661 trap_item->group_item = group_item;
7662
7663 return 0;
7664}
7665
7666static void
7667devlink_trap_item_group_unlink(struct devlink *devlink,
7668 struct devlink_trap_item *trap_item)
7669{
7670 devlink_trap_group_item_put(devlink, trap_item->group_item);
7671}
7672
7673static void devlink_trap_notify(struct devlink *devlink,
7674 const struct devlink_trap_item *trap_item,
7675 enum devlink_command cmd)
7676{
7677 struct sk_buff *msg;
7678 int err;
7679
7680 WARN_ON_ONCE(cmd != DEVLINK_CMD_TRAP_NEW &&
7681 cmd != DEVLINK_CMD_TRAP_DEL);
7682
7683 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
7684 if (!msg)
7685 return;
7686
7687 err = devlink_nl_trap_fill(msg, devlink, trap_item, cmd, 0, 0, 0);
7688 if (err) {
7689 nlmsg_free(msg);
7690 return;
7691 }
7692
7693 genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
7694 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
7695}
7696
7697static int
7698devlink_trap_register(struct devlink *devlink,
7699 const struct devlink_trap *trap, void *priv)
7700{
7701 struct devlink_trap_item *trap_item;
7702 int err;
7703
7704 if (devlink_trap_item_lookup(devlink, trap->name))
7705 return -EEXIST;
7706
7707 trap_item = kzalloc(sizeof(*trap_item), GFP_KERNEL);
7708 if (!trap_item)
7709 return -ENOMEM;
7710
7711 trap_item->stats = netdev_alloc_pcpu_stats(struct devlink_stats);
7712 if (!trap_item->stats) {
7713 err = -ENOMEM;
7714 goto err_stats_alloc;
7715 }
7716
7717 trap_item->trap = trap;
7718 trap_item->action = trap->init_action;
7719 trap_item->priv = priv;
7720
7721 err = devlink_trap_item_group_link(devlink, trap_item);
7722 if (err)
7723 goto err_group_link;
7724
7725 err = devlink->ops->trap_init(devlink, trap, trap_item);
7726 if (err)
7727 goto err_trap_init;
7728
7729 list_add_tail(&trap_item->list, &devlink->trap_list);
7730 devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_NEW);
7731
7732 return 0;
7733
7734err_trap_init:
7735 devlink_trap_item_group_unlink(devlink, trap_item);
7736err_group_link:
7737 free_percpu(trap_item->stats);
7738err_stats_alloc:
7739 kfree(trap_item);
7740 return err;
7741}
7742
7743static void devlink_trap_unregister(struct devlink *devlink,
7744 const struct devlink_trap *trap)
7745{
7746 struct devlink_trap_item *trap_item;
7747
7748 trap_item = devlink_trap_item_lookup(devlink, trap->name);
7749 if (WARN_ON_ONCE(!trap_item))
7750 return;
7751
7752 devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_DEL);
7753 list_del(&trap_item->list);
7754 if (devlink->ops->trap_fini)
7755 devlink->ops->trap_fini(devlink, trap, trap_item);
7756 devlink_trap_item_group_unlink(devlink, trap_item);
7757 free_percpu(trap_item->stats);
7758 kfree(trap_item);
7759}
7760
7761static void devlink_trap_disable(struct devlink *devlink,
7762 const struct devlink_trap *trap)
7763{
7764 struct devlink_trap_item *trap_item;
7765
7766 trap_item = devlink_trap_item_lookup(devlink, trap->name);
7767 if (WARN_ON_ONCE(!trap_item))
7768 return;
7769
7770 devlink->ops->trap_action_set(devlink, trap, DEVLINK_TRAP_ACTION_DROP);
7771 trap_item->action = DEVLINK_TRAP_ACTION_DROP;
7772}
7773
7774/**
7775 * devlink_traps_register - Register packet traps with devlink.
7776 * @devlink: devlink.
7777 * @traps: Packet traps.
7778 * @traps_count: Count of provided packet traps.
7779 * @priv: Driver private information.
7780 *
7781 * Return: Non-zero value on failure.
7782 */
7783int devlink_traps_register(struct devlink *devlink,
7784 const struct devlink_trap *traps,
7785 size_t traps_count, void *priv)
7786{
7787 int i, err;
7788
7789 if (!devlink->ops->trap_init || !devlink->ops->trap_action_set)
7790 return -EINVAL;
7791
7792 mutex_lock(&devlink->lock);
7793 for (i = 0; i < traps_count; i++) {
7794 const struct devlink_trap *trap = &traps[i];
7795
7796 err = devlink_trap_verify(trap);
7797 if (err)
7798 goto err_trap_verify;
7799
7800 err = devlink_trap_register(devlink, trap, priv);
7801 if (err)
7802 goto err_trap_register;
7803 }
7804 mutex_unlock(&devlink->lock);
7805
7806 return 0;
7807
7808err_trap_register:
7809err_trap_verify:
7810 for (i--; i >= 0; i--)
7811 devlink_trap_unregister(devlink, &traps[i]);
7812 mutex_unlock(&devlink->lock);
7813 return err;
7814}
7815EXPORT_SYMBOL_GPL(devlink_traps_register);
7816
7817/**
7818 * devlink_traps_unregister - Unregister packet traps from devlink.
7819 * @devlink: devlink.
7820 * @traps: Packet traps.
7821 * @traps_count: Count of provided packet traps.
7822 */
7823void devlink_traps_unregister(struct devlink *devlink,
7824 const struct devlink_trap *traps,
7825 size_t traps_count)
7826{
7827 int i;
7828
7829 mutex_lock(&devlink->lock);
7830 /* Make sure we do not have any packets in-flight while unregistering
7831 * traps by disabling all of them and waiting for a grace period.
7832 */
7833 for (i = traps_count - 1; i >= 0; i--)
7834 devlink_trap_disable(devlink, &traps[i]);
7835 synchronize_rcu();
7836 for (i = traps_count - 1; i >= 0; i--)
7837 devlink_trap_unregister(devlink, &traps[i]);
7838 mutex_unlock(&devlink->lock);
7839}
7840EXPORT_SYMBOL_GPL(devlink_traps_unregister);
7841
7842static void
7843devlink_trap_stats_update(struct devlink_stats __percpu *trap_stats,
7844 size_t skb_len)
7845{
7846 struct devlink_stats *stats;
7847
7848 stats = this_cpu_ptr(trap_stats);
7849 u64_stats_update_begin(&stats->syncp);
7850 stats->rx_bytes += skb_len;
7851 stats->rx_packets++;
7852 u64_stats_update_end(&stats->syncp);
7853}
7854
7855static void
7856devlink_trap_report_metadata_fill(struct net_dm_hw_metadata *hw_metadata,
7857 const struct devlink_trap_item *trap_item,
7858 struct devlink_port *in_devlink_port)
7859{
7860 struct devlink_trap_group_item *group_item = trap_item->group_item;
7861
7862 hw_metadata->trap_group_name = group_item->group->name;
7863 hw_metadata->trap_name = trap_item->trap->name;
7864
7865 spin_lock(&in_devlink_port->type_lock);
7866 if (in_devlink_port->type == DEVLINK_PORT_TYPE_ETH)
7867 hw_metadata->input_dev = in_devlink_port->type_dev;
7868 spin_unlock(&in_devlink_port->type_lock);
7869}
7870
7871/**
7872 * devlink_trap_report - Report trapped packet to drop monitor.
7873 * @devlink: devlink.
7874 * @skb: Trapped packet.
7875 * @trap_ctx: Trap context.
7876 * @in_devlink_port: Input devlink port.
7877 */
7878void devlink_trap_report(struct devlink *devlink, struct sk_buff *skb,
7879 void *trap_ctx, struct devlink_port *in_devlink_port)
7880{
7881 struct devlink_trap_item *trap_item = trap_ctx;
7882 struct net_dm_hw_metadata hw_metadata = {};
7883
7884 devlink_trap_stats_update(trap_item->stats, skb->len);
7885 devlink_trap_stats_update(trap_item->group_item->stats, skb->len);
7886
7887 devlink_trap_report_metadata_fill(&hw_metadata, trap_item,
7888 in_devlink_port);
7889 net_dm_hw_report(skb, &hw_metadata);
7890}
7891EXPORT_SYMBOL_GPL(devlink_trap_report);
7892
7893/**
7894 * devlink_trap_ctx_priv - Trap context to driver private information.
7895 * @trap_ctx: Trap context.
7896 *
7897 * Return: Driver private information passed during registration.
7898 */
7899void *devlink_trap_ctx_priv(void *trap_ctx)
7900{
7901 struct devlink_trap_item *trap_item = trap_ctx;
7902
7903 return trap_item->priv;
7904}
7905EXPORT_SYMBOL_GPL(devlink_trap_ctx_priv);
7906
6837static void __devlink_compat_running_version(struct devlink *devlink, 7907static void __devlink_compat_running_version(struct devlink *devlink,
6838 char *buf, size_t len) 7908 char *buf, size_t len)
6839{ 7909{
diff --git a/net/core/drop_monitor.c b/net/core/drop_monitor.c
index 39e094907391..bfc024024aa3 100644
--- a/net/core/drop_monitor.c
+++ b/net/core/drop_monitor.c
@@ -26,6 +26,7 @@
26#include <linux/bitops.h> 26#include <linux/bitops.h>
27#include <linux/slab.h> 27#include <linux/slab.h>
28#include <linux/module.h> 28#include <linux/module.h>
29#include <net/drop_monitor.h>
29#include <net/genetlink.h> 30#include <net/genetlink.h>
30#include <net/netevent.h> 31#include <net/netevent.h>
31 32
@@ -43,6 +44,7 @@
43 * netlink alerts 44 * netlink alerts
44 */ 45 */
45static int trace_state = TRACE_OFF; 46static int trace_state = TRACE_OFF;
47static bool monitor_hw;
46 48
47/* net_dm_mutex 49/* net_dm_mutex
48 * 50 *
@@ -56,9 +58,26 @@ struct net_dm_stats {
56 struct u64_stats_sync syncp; 58 struct u64_stats_sync syncp;
57}; 59};
58 60
61#define NET_DM_MAX_HW_TRAP_NAME_LEN 40
62
63struct net_dm_hw_entry {
64 char trap_name[NET_DM_MAX_HW_TRAP_NAME_LEN];
65 u32 count;
66};
67
68struct net_dm_hw_entries {
69 u32 num_entries;
70 struct net_dm_hw_entry entries[0];
71};
72
59struct per_cpu_dm_data { 73struct per_cpu_dm_data {
60 spinlock_t lock; /* Protects 'skb' and 'send_timer' */ 74 spinlock_t lock; /* Protects 'skb', 'hw_entries' and
61 struct sk_buff *skb; 75 * 'send_timer'
76 */
77 union {
78 struct sk_buff *skb;
79 struct net_dm_hw_entries *hw_entries;
80 };
62 struct sk_buff_head drop_queue; 81 struct sk_buff_head drop_queue;
63 struct work_struct dm_alert_work; 82 struct work_struct dm_alert_work;
64 struct timer_list send_timer; 83 struct timer_list send_timer;
@@ -76,6 +95,7 @@ struct dm_hw_stat_delta {
76static struct genl_family net_drop_monitor_family; 95static struct genl_family net_drop_monitor_family;
77 96
78static DEFINE_PER_CPU(struct per_cpu_dm_data, dm_cpu_data); 97static DEFINE_PER_CPU(struct per_cpu_dm_data, dm_cpu_data);
98static DEFINE_PER_CPU(struct per_cpu_dm_data, dm_hw_cpu_data);
79 99
80static int dm_hit_limit = 64; 100static int dm_hit_limit = 64;
81static int dm_delay = 1; 101static int dm_delay = 1;
@@ -92,10 +112,16 @@ struct net_dm_alert_ops {
92 void (*napi_poll_probe)(void *ignore, struct napi_struct *napi, 112 void (*napi_poll_probe)(void *ignore, struct napi_struct *napi,
93 int work, int budget); 113 int work, int budget);
94 void (*work_item_func)(struct work_struct *work); 114 void (*work_item_func)(struct work_struct *work);
115 void (*hw_work_item_func)(struct work_struct *work);
116 void (*hw_probe)(struct sk_buff *skb,
117 const struct net_dm_hw_metadata *hw_metadata);
95}; 118};
96 119
97struct net_dm_skb_cb { 120struct net_dm_skb_cb {
98 void *pc; 121 union {
122 struct net_dm_hw_metadata *hw_metadata;
123 void *pc;
124 };
99}; 125};
100 126
101#define NET_DM_SKB_CB(__skb) ((struct net_dm_skb_cb *)&((__skb)->cb[0])) 127#define NET_DM_SKB_CB(__skb) ((struct net_dm_skb_cb *)&((__skb)->cb[0]))
@@ -266,10 +292,190 @@ static void trace_napi_poll_hit(void *ignore, struct napi_struct *napi,
266 rcu_read_unlock(); 292 rcu_read_unlock();
267} 293}
268 294
295static struct net_dm_hw_entries *
296net_dm_hw_reset_per_cpu_data(struct per_cpu_dm_data *hw_data)
297{
298 struct net_dm_hw_entries *hw_entries;
299 unsigned long flags;
300
301 hw_entries = kzalloc(struct_size(hw_entries, entries, dm_hit_limit),
302 GFP_KERNEL);
303 if (!hw_entries) {
304 /* If the memory allocation failed, we try to perform another
305 * allocation in 1/10 second. Otherwise, the probe function
306 * will constantly bail out.
307 */
308 mod_timer(&hw_data->send_timer, jiffies + HZ / 10);
309 }
310
311 spin_lock_irqsave(&hw_data->lock, flags);
312 swap(hw_data->hw_entries, hw_entries);
313 spin_unlock_irqrestore(&hw_data->lock, flags);
314
315 return hw_entries;
316}
317
318static int net_dm_hw_entry_put(struct sk_buff *msg,
319 const struct net_dm_hw_entry *hw_entry)
320{
321 struct nlattr *attr;
322
323 attr = nla_nest_start(msg, NET_DM_ATTR_HW_ENTRY);
324 if (!attr)
325 return -EMSGSIZE;
326
327 if (nla_put_string(msg, NET_DM_ATTR_HW_TRAP_NAME, hw_entry->trap_name))
328 goto nla_put_failure;
329
330 if (nla_put_u32(msg, NET_DM_ATTR_HW_TRAP_COUNT, hw_entry->count))
331 goto nla_put_failure;
332
333 nla_nest_end(msg, attr);
334
335 return 0;
336
337nla_put_failure:
338 nla_nest_cancel(msg, attr);
339 return -EMSGSIZE;
340}
341
342static int net_dm_hw_entries_put(struct sk_buff *msg,
343 const struct net_dm_hw_entries *hw_entries)
344{
345 struct nlattr *attr;
346 int i;
347
348 attr = nla_nest_start(msg, NET_DM_ATTR_HW_ENTRIES);
349 if (!attr)
350 return -EMSGSIZE;
351
352 for (i = 0; i < hw_entries->num_entries; i++) {
353 int rc;
354
355 rc = net_dm_hw_entry_put(msg, &hw_entries->entries[i]);
356 if (rc)
357 goto nla_put_failure;
358 }
359
360 nla_nest_end(msg, attr);
361
362 return 0;
363
364nla_put_failure:
365 nla_nest_cancel(msg, attr);
366 return -EMSGSIZE;
367}
368
369static int
370net_dm_hw_summary_report_fill(struct sk_buff *msg,
371 const struct net_dm_hw_entries *hw_entries)
372{
373 struct net_dm_alert_msg anc_hdr = { 0 };
374 void *hdr;
375 int rc;
376
377 hdr = genlmsg_put(msg, 0, 0, &net_drop_monitor_family, 0,
378 NET_DM_CMD_ALERT);
379 if (!hdr)
380 return -EMSGSIZE;
381
382 /* We need to put the ancillary header in order not to break user
383 * space.
384 */
385 if (nla_put(msg, NLA_UNSPEC, sizeof(anc_hdr), &anc_hdr))
386 goto nla_put_failure;
387
388 rc = net_dm_hw_entries_put(msg, hw_entries);
389 if (rc)
390 goto nla_put_failure;
391
392 genlmsg_end(msg, hdr);
393
394 return 0;
395
396nla_put_failure:
397 genlmsg_cancel(msg, hdr);
398 return -EMSGSIZE;
399}
400
401static void net_dm_hw_summary_work(struct work_struct *work)
402{
403 struct net_dm_hw_entries *hw_entries;
404 struct per_cpu_dm_data *hw_data;
405 struct sk_buff *msg;
406 int rc;
407
408 hw_data = container_of(work, struct per_cpu_dm_data, dm_alert_work);
409
410 hw_entries = net_dm_hw_reset_per_cpu_data(hw_data);
411 if (!hw_entries)
412 return;
413
414 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
415 if (!msg)
416 goto out;
417
418 rc = net_dm_hw_summary_report_fill(msg, hw_entries);
419 if (rc) {
420 nlmsg_free(msg);
421 goto out;
422 }
423
424 genlmsg_multicast(&net_drop_monitor_family, msg, 0, 0, GFP_KERNEL);
425
426out:
427 kfree(hw_entries);
428}
429
430static void
431net_dm_hw_summary_probe(struct sk_buff *skb,
432 const struct net_dm_hw_metadata *hw_metadata)
433{
434 struct net_dm_hw_entries *hw_entries;
435 struct net_dm_hw_entry *hw_entry;
436 struct per_cpu_dm_data *hw_data;
437 unsigned long flags;
438 int i;
439
440 hw_data = this_cpu_ptr(&dm_hw_cpu_data);
441 spin_lock_irqsave(&hw_data->lock, flags);
442 hw_entries = hw_data->hw_entries;
443
444 if (!hw_entries)
445 goto out;
446
447 for (i = 0; i < hw_entries->num_entries; i++) {
448 hw_entry = &hw_entries->entries[i];
449 if (!strncmp(hw_entry->trap_name, hw_metadata->trap_name,
450 NET_DM_MAX_HW_TRAP_NAME_LEN - 1)) {
451 hw_entry->count++;
452 goto out;
453 }
454 }
455 if (WARN_ON_ONCE(hw_entries->num_entries == dm_hit_limit))
456 goto out;
457
458 hw_entry = &hw_entries->entries[hw_entries->num_entries];
459 strlcpy(hw_entry->trap_name, hw_metadata->trap_name,
460 NET_DM_MAX_HW_TRAP_NAME_LEN - 1);
461 hw_entry->count = 1;
462 hw_entries->num_entries++;
463
464 if (!timer_pending(&hw_data->send_timer)) {
465 hw_data->send_timer.expires = jiffies + dm_delay * HZ;
466 add_timer(&hw_data->send_timer);
467 }
468
469out:
470 spin_unlock_irqrestore(&hw_data->lock, flags);
471}
472
269static const struct net_dm_alert_ops net_dm_alert_summary_ops = { 473static const struct net_dm_alert_ops net_dm_alert_summary_ops = {
270 .kfree_skb_probe = trace_kfree_skb_hit, 474 .kfree_skb_probe = trace_kfree_skb_hit,
271 .napi_poll_probe = trace_napi_poll_hit, 475 .napi_poll_probe = trace_napi_poll_hit,
272 .work_item_func = send_dm_alert, 476 .work_item_func = send_dm_alert,
477 .hw_work_item_func = net_dm_hw_summary_work,
478 .hw_probe = net_dm_hw_summary_probe,
273}; 479};
274 480
275static void net_dm_packet_trace_kfree_skb_hit(void *ignore, 481static void net_dm_packet_trace_kfree_skb_hit(void *ignore,
@@ -323,7 +529,9 @@ static size_t net_dm_in_port_size(void)
323 /* NET_DM_ATTR_IN_PORT nest */ 529 /* NET_DM_ATTR_IN_PORT nest */
324 return nla_total_size(0) + 530 return nla_total_size(0) +
325 /* NET_DM_ATTR_PORT_NETDEV_IFINDEX */ 531 /* NET_DM_ATTR_PORT_NETDEV_IFINDEX */
326 nla_total_size(sizeof(u32)); 532 nla_total_size(sizeof(u32)) +
533 /* NET_DM_ATTR_PORT_NETDEV_NAME */
534 nla_total_size(IFNAMSIZ + 1);
327} 535}
328 536
329#define NET_DM_MAX_SYMBOL_LEN 40 537#define NET_DM_MAX_SYMBOL_LEN 40
@@ -335,6 +543,8 @@ static size_t net_dm_packet_report_size(size_t payload_len)
335 size = nlmsg_msg_size(GENL_HDRLEN + net_drop_monitor_family.hdrsize); 543 size = nlmsg_msg_size(GENL_HDRLEN + net_drop_monitor_family.hdrsize);
336 544
337 return NLMSG_ALIGN(size) + 545 return NLMSG_ALIGN(size) +
546 /* NET_DM_ATTR_ORIGIN */
547 nla_total_size(sizeof(u16)) +
338 /* NET_DM_ATTR_PC */ 548 /* NET_DM_ATTR_PC */
339 nla_total_size(sizeof(u64)) + 549 nla_total_size(sizeof(u64)) +
340 /* NET_DM_ATTR_SYMBOL */ 550 /* NET_DM_ATTR_SYMBOL */
@@ -351,7 +561,8 @@ static size_t net_dm_packet_report_size(size_t payload_len)
351 nla_total_size(payload_len); 561 nla_total_size(payload_len);
352} 562}
353 563
354static int net_dm_packet_report_in_port_put(struct sk_buff *msg, int ifindex) 564static int net_dm_packet_report_in_port_put(struct sk_buff *msg, int ifindex,
565 const char *name)
355{ 566{
356 struct nlattr *attr; 567 struct nlattr *attr;
357 568
@@ -363,6 +574,9 @@ static int net_dm_packet_report_in_port_put(struct sk_buff *msg, int ifindex)
363 nla_put_u32(msg, NET_DM_ATTR_PORT_NETDEV_IFINDEX, ifindex)) 574 nla_put_u32(msg, NET_DM_ATTR_PORT_NETDEV_IFINDEX, ifindex))
364 goto nla_put_failure; 575 goto nla_put_failure;
365 576
577 if (name && nla_put_string(msg, NET_DM_ATTR_PORT_NETDEV_NAME, name))
578 goto nla_put_failure;
579
366 nla_nest_end(msg, attr); 580 nla_nest_end(msg, attr);
367 581
368 return 0; 582 return 0;
@@ -387,6 +601,9 @@ static int net_dm_packet_report_fill(struct sk_buff *msg, struct sk_buff *skb,
387 if (!hdr) 601 if (!hdr)
388 return -EMSGSIZE; 602 return -EMSGSIZE;
389 603
604 if (nla_put_u16(msg, NET_DM_ATTR_ORIGIN, NET_DM_ORIGIN_SW))
605 goto nla_put_failure;
606
390 if (nla_put_u64_64bit(msg, NET_DM_ATTR_PC, pc, NET_DM_ATTR_PAD)) 607 if (nla_put_u64_64bit(msg, NET_DM_ATTR_PC, pc, NET_DM_ATTR_PAD))
391 goto nla_put_failure; 608 goto nla_put_failure;
392 609
@@ -394,7 +611,7 @@ static int net_dm_packet_report_fill(struct sk_buff *msg, struct sk_buff *skb,
394 if (nla_put_string(msg, NET_DM_ATTR_SYMBOL, buf)) 611 if (nla_put_string(msg, NET_DM_ATTR_SYMBOL, buf))
395 goto nla_put_failure; 612 goto nla_put_failure;
396 613
397 rc = net_dm_packet_report_in_port_put(msg, skb->skb_iif); 614 rc = net_dm_packet_report_in_port_put(msg, skb->skb_iif, NULL);
398 if (rc) 615 if (rc)
399 goto nla_put_failure; 616 goto nla_put_failure;
400 617
@@ -481,10 +698,250 @@ static void net_dm_packet_work(struct work_struct *work)
481 net_dm_packet_report(skb); 698 net_dm_packet_report(skb);
482} 699}
483 700
701static size_t
702net_dm_hw_packet_report_size(size_t payload_len,
703 const struct net_dm_hw_metadata *hw_metadata)
704{
705 size_t size;
706
707 size = nlmsg_msg_size(GENL_HDRLEN + net_drop_monitor_family.hdrsize);
708
709 return NLMSG_ALIGN(size) +
710 /* NET_DM_ATTR_ORIGIN */
711 nla_total_size(sizeof(u16)) +
712 /* NET_DM_ATTR_HW_TRAP_GROUP_NAME */
713 nla_total_size(strlen(hw_metadata->trap_group_name) + 1) +
714 /* NET_DM_ATTR_HW_TRAP_NAME */
715 nla_total_size(strlen(hw_metadata->trap_name) + 1) +
716 /* NET_DM_ATTR_IN_PORT */
717 net_dm_in_port_size() +
718 /* NET_DM_ATTR_TIMESTAMP */
719 nla_total_size(sizeof(struct timespec)) +
720 /* NET_DM_ATTR_ORIG_LEN */
721 nla_total_size(sizeof(u32)) +
722 /* NET_DM_ATTR_PROTO */
723 nla_total_size(sizeof(u16)) +
724 /* NET_DM_ATTR_PAYLOAD */
725 nla_total_size(payload_len);
726}
727
728static int net_dm_hw_packet_report_fill(struct sk_buff *msg,
729 struct sk_buff *skb, size_t payload_len)
730{
731 struct net_dm_hw_metadata *hw_metadata;
732 struct nlattr *attr;
733 struct timespec ts;
734 void *hdr;
735
736 hw_metadata = NET_DM_SKB_CB(skb)->hw_metadata;
737
738 hdr = genlmsg_put(msg, 0, 0, &net_drop_monitor_family, 0,
739 NET_DM_CMD_PACKET_ALERT);
740 if (!hdr)
741 return -EMSGSIZE;
742
743 if (nla_put_u16(msg, NET_DM_ATTR_ORIGIN, NET_DM_ORIGIN_HW))
744 goto nla_put_failure;
745
746 if (nla_put_string(msg, NET_DM_ATTR_HW_TRAP_GROUP_NAME,
747 hw_metadata->trap_group_name))
748 goto nla_put_failure;
749
750 if (nla_put_string(msg, NET_DM_ATTR_HW_TRAP_NAME,
751 hw_metadata->trap_name))
752 goto nla_put_failure;
753
754 if (hw_metadata->input_dev) {
755 struct net_device *dev = hw_metadata->input_dev;
756 int rc;
757
758 rc = net_dm_packet_report_in_port_put(msg, dev->ifindex,
759 dev->name);
760 if (rc)
761 goto nla_put_failure;
762 }
763
764 if (ktime_to_timespec_cond(skb->tstamp, &ts) &&
765 nla_put(msg, NET_DM_ATTR_TIMESTAMP, sizeof(ts), &ts))
766 goto nla_put_failure;
767
768 if (nla_put_u32(msg, NET_DM_ATTR_ORIG_LEN, skb->len))
769 goto nla_put_failure;
770
771 if (!payload_len)
772 goto out;
773
774 if (nla_put_u16(msg, NET_DM_ATTR_PROTO, be16_to_cpu(skb->protocol)))
775 goto nla_put_failure;
776
777 attr = skb_put(msg, nla_total_size(payload_len));
778 attr->nla_type = NET_DM_ATTR_PAYLOAD;
779 attr->nla_len = nla_attr_size(payload_len);
780 if (skb_copy_bits(skb, 0, nla_data(attr), payload_len))
781 goto nla_put_failure;
782
783out:
784 genlmsg_end(msg, hdr);
785
786 return 0;
787
788nla_put_failure:
789 genlmsg_cancel(msg, hdr);
790 return -EMSGSIZE;
791}
792
793static struct net_dm_hw_metadata *
794net_dm_hw_metadata_clone(const struct net_dm_hw_metadata *hw_metadata)
795{
796 struct net_dm_hw_metadata *n_hw_metadata;
797 const char *trap_group_name;
798 const char *trap_name;
799
800 n_hw_metadata = kmalloc(sizeof(*hw_metadata), GFP_ATOMIC);
801 if (!n_hw_metadata)
802 return NULL;
803
804 trap_group_name = kmemdup(hw_metadata->trap_group_name,
805 strlen(hw_metadata->trap_group_name) + 1,
806 GFP_ATOMIC | __GFP_ZERO);
807 if (!trap_group_name)
808 goto free_hw_metadata;
809 n_hw_metadata->trap_group_name = trap_group_name;
810
811 trap_name = kmemdup(hw_metadata->trap_name,
812 strlen(hw_metadata->trap_name) + 1,
813 GFP_ATOMIC | __GFP_ZERO);
814 if (!trap_name)
815 goto free_trap_group;
816 n_hw_metadata->trap_name = trap_name;
817
818 n_hw_metadata->input_dev = hw_metadata->input_dev;
819 if (n_hw_metadata->input_dev)
820 dev_hold(n_hw_metadata->input_dev);
821
822 return n_hw_metadata;
823
824free_trap_group:
825 kfree(trap_group_name);
826free_hw_metadata:
827 kfree(n_hw_metadata);
828 return NULL;
829}
830
831static void
832net_dm_hw_metadata_free(const struct net_dm_hw_metadata *hw_metadata)
833{
834 if (hw_metadata->input_dev)
835 dev_put(hw_metadata->input_dev);
836 kfree(hw_metadata->trap_name);
837 kfree(hw_metadata->trap_group_name);
838 kfree(hw_metadata);
839}
840
841static void net_dm_hw_packet_report(struct sk_buff *skb)
842{
843 struct net_dm_hw_metadata *hw_metadata;
844 struct sk_buff *msg;
845 size_t payload_len;
846 int rc;
847
848 if (skb->data > skb_mac_header(skb))
849 skb_push(skb, skb->data - skb_mac_header(skb));
850 else
851 skb_pull(skb, skb_mac_header(skb) - skb->data);
852
853 payload_len = min_t(size_t, skb->len, NET_DM_MAX_PACKET_SIZE);
854 if (net_dm_trunc_len)
855 payload_len = min_t(size_t, net_dm_trunc_len, payload_len);
856
857 hw_metadata = NET_DM_SKB_CB(skb)->hw_metadata;
858 msg = nlmsg_new(net_dm_hw_packet_report_size(payload_len, hw_metadata),
859 GFP_KERNEL);
860 if (!msg)
861 goto out;
862
863 rc = net_dm_hw_packet_report_fill(msg, skb, payload_len);
864 if (rc) {
865 nlmsg_free(msg);
866 goto out;
867 }
868
869 genlmsg_multicast(&net_drop_monitor_family, msg, 0, 0, GFP_KERNEL);
870
871out:
872 net_dm_hw_metadata_free(NET_DM_SKB_CB(skb)->hw_metadata);
873 consume_skb(skb);
874}
875
876static void net_dm_hw_packet_work(struct work_struct *work)
877{
878 struct per_cpu_dm_data *hw_data;
879 struct sk_buff_head list;
880 struct sk_buff *skb;
881 unsigned long flags;
882
883 hw_data = container_of(work, struct per_cpu_dm_data, dm_alert_work);
884
885 __skb_queue_head_init(&list);
886
887 spin_lock_irqsave(&hw_data->drop_queue.lock, flags);
888 skb_queue_splice_tail_init(&hw_data->drop_queue, &list);
889 spin_unlock_irqrestore(&hw_data->drop_queue.lock, flags);
890
891 while ((skb = __skb_dequeue(&list)))
892 net_dm_hw_packet_report(skb);
893}
894
895static void
896net_dm_hw_packet_probe(struct sk_buff *skb,
897 const struct net_dm_hw_metadata *hw_metadata)
898{
899 struct net_dm_hw_metadata *n_hw_metadata;
900 ktime_t tstamp = ktime_get_real();
901 struct per_cpu_dm_data *hw_data;
902 struct sk_buff *nskb;
903 unsigned long flags;
904
905 nskb = skb_clone(skb, GFP_ATOMIC);
906 if (!nskb)
907 return;
908
909 n_hw_metadata = net_dm_hw_metadata_clone(hw_metadata);
910 if (!n_hw_metadata)
911 goto free;
912
913 NET_DM_SKB_CB(nskb)->hw_metadata = n_hw_metadata;
914 nskb->tstamp = tstamp;
915
916 hw_data = this_cpu_ptr(&dm_hw_cpu_data);
917
918 spin_lock_irqsave(&hw_data->drop_queue.lock, flags);
919 if (skb_queue_len(&hw_data->drop_queue) < net_dm_queue_len)
920 __skb_queue_tail(&hw_data->drop_queue, nskb);
921 else
922 goto unlock_free;
923 spin_unlock_irqrestore(&hw_data->drop_queue.lock, flags);
924
925 schedule_work(&hw_data->dm_alert_work);
926
927 return;
928
929unlock_free:
930 spin_unlock_irqrestore(&hw_data->drop_queue.lock, flags);
931 u64_stats_update_begin(&hw_data->stats.syncp);
932 hw_data->stats.dropped++;
933 u64_stats_update_end(&hw_data->stats.syncp);
934 net_dm_hw_metadata_free(n_hw_metadata);
935free:
936 consume_skb(nskb);
937}
938
484static const struct net_dm_alert_ops net_dm_alert_packet_ops = { 939static const struct net_dm_alert_ops net_dm_alert_packet_ops = {
485 .kfree_skb_probe = net_dm_packet_trace_kfree_skb_hit, 940 .kfree_skb_probe = net_dm_packet_trace_kfree_skb_hit,
486 .napi_poll_probe = net_dm_packet_trace_napi_poll_hit, 941 .napi_poll_probe = net_dm_packet_trace_napi_poll_hit,
487 .work_item_func = net_dm_packet_work, 942 .work_item_func = net_dm_packet_work,
943 .hw_work_item_func = net_dm_hw_packet_work,
944 .hw_probe = net_dm_hw_packet_probe,
488}; 945};
489 946
490static const struct net_dm_alert_ops *net_dm_alert_ops_arr[] = { 947static const struct net_dm_alert_ops *net_dm_alert_ops_arr[] = {
@@ -492,6 +949,85 @@ static const struct net_dm_alert_ops *net_dm_alert_ops_arr[] = {
492 [NET_DM_ALERT_MODE_PACKET] = &net_dm_alert_packet_ops, 949 [NET_DM_ALERT_MODE_PACKET] = &net_dm_alert_packet_ops,
493}; 950};
494 951
952void net_dm_hw_report(struct sk_buff *skb,
953 const struct net_dm_hw_metadata *hw_metadata)
954{
955 rcu_read_lock();
956
957 if (!monitor_hw)
958 goto out;
959
960 net_dm_alert_ops_arr[net_dm_alert_mode]->hw_probe(skb, hw_metadata);
961
962out:
963 rcu_read_unlock();
964}
965EXPORT_SYMBOL_GPL(net_dm_hw_report);
966
967static int net_dm_hw_monitor_start(struct netlink_ext_ack *extack)
968{
969 const struct net_dm_alert_ops *ops;
970 int cpu;
971
972 if (monitor_hw) {
973 NL_SET_ERR_MSG_MOD(extack, "Hardware monitoring already enabled");
974 return -EAGAIN;
975 }
976
977 ops = net_dm_alert_ops_arr[net_dm_alert_mode];
978
979 if (!try_module_get(THIS_MODULE)) {
980 NL_SET_ERR_MSG_MOD(extack, "Failed to take reference on module");
981 return -ENODEV;
982 }
983
984 for_each_possible_cpu(cpu) {
985 struct per_cpu_dm_data *hw_data = &per_cpu(dm_hw_cpu_data, cpu);
986 struct net_dm_hw_entries *hw_entries;
987
988 INIT_WORK(&hw_data->dm_alert_work, ops->hw_work_item_func);
989 timer_setup(&hw_data->send_timer, sched_send_work, 0);
990 hw_entries = net_dm_hw_reset_per_cpu_data(hw_data);
991 kfree(hw_entries);
992 }
993
994 monitor_hw = true;
995
996 return 0;
997}
998
999static void net_dm_hw_monitor_stop(struct netlink_ext_ack *extack)
1000{
1001 int cpu;
1002
1003 if (!monitor_hw)
1004 NL_SET_ERR_MSG_MOD(extack, "Hardware monitoring already disabled");
1005
1006 monitor_hw = false;
1007
1008 /* After this call returns we are guaranteed that no CPU is processing
1009 * any hardware drops.
1010 */
1011 synchronize_rcu();
1012
1013 for_each_possible_cpu(cpu) {
1014 struct per_cpu_dm_data *hw_data = &per_cpu(dm_hw_cpu_data, cpu);
1015 struct sk_buff *skb;
1016
1017 del_timer_sync(&hw_data->send_timer);
1018 cancel_work_sync(&hw_data->dm_alert_work);
1019 while ((skb = __skb_dequeue(&hw_data->drop_queue))) {
1020 struct net_dm_hw_metadata *hw_metadata;
1021
1022 hw_metadata = NET_DM_SKB_CB(skb)->hw_metadata;
1023 net_dm_hw_metadata_free(hw_metadata);
1024 consume_skb(skb);
1025 }
1026 }
1027
1028 module_put(THIS_MODULE);
1029}
1030
495static int net_dm_trace_on_set(struct netlink_ext_ack *extack) 1031static int net_dm_trace_on_set(struct netlink_ext_ack *extack)
496{ 1032{
497 const struct net_dm_alert_ops *ops; 1033 const struct net_dm_alert_ops *ops;
@@ -604,6 +1140,11 @@ static int set_all_monitor_traces(int state, struct netlink_ext_ack *extack)
604 return rc; 1140 return rc;
605} 1141}
606 1142
1143static bool net_dm_is_monitoring(void)
1144{
1145 return trace_state == TRACE_ON || monitor_hw;
1146}
1147
607static int net_dm_alert_mode_get_from_info(struct genl_info *info, 1148static int net_dm_alert_mode_get_from_info(struct genl_info *info,
608 enum net_dm_alert_mode *p_alert_mode) 1149 enum net_dm_alert_mode *p_alert_mode)
609{ 1150{
@@ -665,8 +1206,8 @@ static int net_dm_cmd_config(struct sk_buff *skb,
665 struct netlink_ext_ack *extack = info->extack; 1206 struct netlink_ext_ack *extack = info->extack;
666 int rc; 1207 int rc;
667 1208
668 if (trace_state == TRACE_ON) { 1209 if (net_dm_is_monitoring()) {
669 NL_SET_ERR_MSG_MOD(extack, "Cannot configure drop monitor while tracing is on"); 1210 NL_SET_ERR_MSG_MOD(extack, "Cannot configure drop monitor during monitoring");
670 return -EBUSY; 1211 return -EBUSY;
671 } 1212 }
672 1213
@@ -681,14 +1222,61 @@ static int net_dm_cmd_config(struct sk_buff *skb,
681 return 0; 1222 return 0;
682} 1223}
683 1224
1225static int net_dm_monitor_start(bool set_sw, bool set_hw,
1226 struct netlink_ext_ack *extack)
1227{
1228 bool sw_set = false;
1229 int rc;
1230
1231 if (set_sw) {
1232 rc = set_all_monitor_traces(TRACE_ON, extack);
1233 if (rc)
1234 return rc;
1235 sw_set = true;
1236 }
1237
1238 if (set_hw) {
1239 rc = net_dm_hw_monitor_start(extack);
1240 if (rc)
1241 goto err_monitor_hw;
1242 }
1243
1244 return 0;
1245
1246err_monitor_hw:
1247 if (sw_set)
1248 set_all_monitor_traces(TRACE_OFF, extack);
1249 return rc;
1250}
1251
1252static void net_dm_monitor_stop(bool set_sw, bool set_hw,
1253 struct netlink_ext_ack *extack)
1254{
1255 if (set_hw)
1256 net_dm_hw_monitor_stop(extack);
1257 if (set_sw)
1258 set_all_monitor_traces(TRACE_OFF, extack);
1259}
1260
684static int net_dm_cmd_trace(struct sk_buff *skb, 1261static int net_dm_cmd_trace(struct sk_buff *skb,
685 struct genl_info *info) 1262 struct genl_info *info)
686{ 1263{
1264 bool set_sw = !!info->attrs[NET_DM_ATTR_SW_DROPS];
1265 bool set_hw = !!info->attrs[NET_DM_ATTR_HW_DROPS];
1266 struct netlink_ext_ack *extack = info->extack;
1267
1268 /* To maintain backward compatibility, we start / stop monitoring of
1269 * software drops if no flag is specified.
1270 */
1271 if (!set_sw && !set_hw)
1272 set_sw = true;
1273
687 switch (info->genlhdr->cmd) { 1274 switch (info->genlhdr->cmd) {
688 case NET_DM_CMD_START: 1275 case NET_DM_CMD_START:
689 return set_all_monitor_traces(TRACE_ON, info->extack); 1276 return net_dm_monitor_start(set_sw, set_hw, extack);
690 case NET_DM_CMD_STOP: 1277 case NET_DM_CMD_STOP:
691 return set_all_monitor_traces(TRACE_OFF, info->extack); 1278 net_dm_monitor_stop(set_sw, set_hw, extack);
1279 return 0;
692 } 1280 }
693 1281
694 return -EOPNOTSUPP; 1282 return -EOPNOTSUPP;
@@ -785,6 +1373,50 @@ nla_put_failure:
785 return -EMSGSIZE; 1373 return -EMSGSIZE;
786} 1374}
787 1375
1376static void net_dm_hw_stats_read(struct net_dm_stats *stats)
1377{
1378 int cpu;
1379
1380 memset(stats, 0, sizeof(*stats));
1381 for_each_possible_cpu(cpu) {
1382 struct per_cpu_dm_data *hw_data = &per_cpu(dm_hw_cpu_data, cpu);
1383 struct net_dm_stats *cpu_stats = &hw_data->stats;
1384 unsigned int start;
1385 u64 dropped;
1386
1387 do {
1388 start = u64_stats_fetch_begin_irq(&cpu_stats->syncp);
1389 dropped = cpu_stats->dropped;
1390 } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start));
1391
1392 stats->dropped += dropped;
1393 }
1394}
1395
1396static int net_dm_hw_stats_put(struct sk_buff *msg)
1397{
1398 struct net_dm_stats stats;
1399 struct nlattr *attr;
1400
1401 net_dm_hw_stats_read(&stats);
1402
1403 attr = nla_nest_start(msg, NET_DM_ATTR_HW_STATS);
1404 if (!attr)
1405 return -EMSGSIZE;
1406
1407 if (nla_put_u64_64bit(msg, NET_DM_ATTR_STATS_DROPPED,
1408 stats.dropped, NET_DM_ATTR_PAD))
1409 goto nla_put_failure;
1410
1411 nla_nest_end(msg, attr);
1412
1413 return 0;
1414
1415nla_put_failure:
1416 nla_nest_cancel(msg, attr);
1417 return -EMSGSIZE;
1418}
1419
788static int net_dm_stats_fill(struct sk_buff *msg, struct genl_info *info) 1420static int net_dm_stats_fill(struct sk_buff *msg, struct genl_info *info)
789{ 1421{
790 void *hdr; 1422 void *hdr;
@@ -799,6 +1431,10 @@ static int net_dm_stats_fill(struct sk_buff *msg, struct genl_info *info)
799 if (rc) 1431 if (rc)
800 goto nla_put_failure; 1432 goto nla_put_failure;
801 1433
1434 rc = net_dm_hw_stats_put(msg);
1435 if (rc)
1436 goto nla_put_failure;
1437
802 genlmsg_end(msg, hdr); 1438 genlmsg_end(msg, hdr);
803 1439
804 return 0; 1440 return 0;
@@ -872,6 +1508,8 @@ static const struct nla_policy net_dm_nl_policy[NET_DM_ATTR_MAX + 1] = {
872 [NET_DM_ATTR_ALERT_MODE] = { .type = NLA_U8 }, 1508 [NET_DM_ATTR_ALERT_MODE] = { .type = NLA_U8 },
873 [NET_DM_ATTR_TRUNC_LEN] = { .type = NLA_U32 }, 1509 [NET_DM_ATTR_TRUNC_LEN] = { .type = NLA_U32 },
874 [NET_DM_ATTR_QUEUE_LEN] = { .type = NLA_U32 }, 1510 [NET_DM_ATTR_QUEUE_LEN] = { .type = NLA_U32 },
1511 [NET_DM_ATTR_SW_DROPS] = {. type = NLA_FLAG },
1512 [NET_DM_ATTR_HW_DROPS] = {. type = NLA_FLAG },
875}; 1513};
876 1514
877static const struct genl_ops dropmon_ops[] = { 1515static const struct genl_ops dropmon_ops[] = {
@@ -934,9 +1572,57 @@ static struct notifier_block dropmon_net_notifier = {
934 .notifier_call = dropmon_net_event 1572 .notifier_call = dropmon_net_event
935}; 1573};
936 1574
937static int __init init_net_drop_monitor(void) 1575static void __net_dm_cpu_data_init(struct per_cpu_dm_data *data)
1576{
1577 spin_lock_init(&data->lock);
1578 skb_queue_head_init(&data->drop_queue);
1579 u64_stats_init(&data->stats.syncp);
1580}
1581
1582static void __net_dm_cpu_data_fini(struct per_cpu_dm_data *data)
1583{
1584 WARN_ON(!skb_queue_empty(&data->drop_queue));
1585}
1586
1587static void net_dm_cpu_data_init(int cpu)
938{ 1588{
939 struct per_cpu_dm_data *data; 1589 struct per_cpu_dm_data *data;
1590
1591 data = &per_cpu(dm_cpu_data, cpu);
1592 __net_dm_cpu_data_init(data);
1593}
1594
1595static void net_dm_cpu_data_fini(int cpu)
1596{
1597 struct per_cpu_dm_data *data;
1598
1599 data = &per_cpu(dm_cpu_data, cpu);
1600 /* At this point, we should have exclusive access
1601 * to this struct and can free the skb inside it.
1602 */
1603 consume_skb(data->skb);
1604 __net_dm_cpu_data_fini(data);
1605}
1606
1607static void net_dm_hw_cpu_data_init(int cpu)
1608{
1609 struct per_cpu_dm_data *hw_data;
1610
1611 hw_data = &per_cpu(dm_hw_cpu_data, cpu);
1612 __net_dm_cpu_data_init(hw_data);
1613}
1614
1615static void net_dm_hw_cpu_data_fini(int cpu)
1616{
1617 struct per_cpu_dm_data *hw_data;
1618
1619 hw_data = &per_cpu(dm_hw_cpu_data, cpu);
1620 kfree(hw_data->hw_entries);
1621 __net_dm_cpu_data_fini(hw_data);
1622}
1623
1624static int __init init_net_drop_monitor(void)
1625{
940 int cpu, rc; 1626 int cpu, rc;
941 1627
942 pr_info("Initializing network drop monitor service\n"); 1628 pr_info("Initializing network drop monitor service\n");
@@ -962,10 +1648,8 @@ static int __init init_net_drop_monitor(void)
962 rc = 0; 1648 rc = 0;
963 1649
964 for_each_possible_cpu(cpu) { 1650 for_each_possible_cpu(cpu) {
965 data = &per_cpu(dm_cpu_data, cpu); 1651 net_dm_cpu_data_init(cpu);
966 spin_lock_init(&data->lock); 1652 net_dm_hw_cpu_data_init(cpu);
967 skb_queue_head_init(&data->drop_queue);
968 u64_stats_init(&data->stats.syncp);
969 } 1653 }
970 1654
971 goto out; 1655 goto out;
@@ -978,7 +1662,6 @@ out:
978 1662
979static void exit_net_drop_monitor(void) 1663static void exit_net_drop_monitor(void)
980{ 1664{
981 struct per_cpu_dm_data *data;
982 int cpu; 1665 int cpu;
983 1666
984 BUG_ON(unregister_netdevice_notifier(&dropmon_net_notifier)); 1667 BUG_ON(unregister_netdevice_notifier(&dropmon_net_notifier));
@@ -989,13 +1672,8 @@ static void exit_net_drop_monitor(void)
989 */ 1672 */
990 1673
991 for_each_possible_cpu(cpu) { 1674 for_each_possible_cpu(cpu) {
992 data = &per_cpu(dm_cpu_data, cpu); 1675 net_dm_hw_cpu_data_fini(cpu);
993 /* 1676 net_dm_cpu_data_fini(cpu);
994 * At this point, we should have exclusive access
995 * to this struct and can free the skb inside it
996 */
997 kfree_skb(data->skb);
998 WARN_ON(!skb_queue_empty(&data->drop_queue));
999 } 1677 }
1000 1678
1001 BUG_ON(genl_unregister_family(&net_drop_monitor_family)); 1679 BUG_ON(genl_unregister_family(&net_drop_monitor_family));
generated by cgit v1.2.2 (git 2.25.0) at 2025-10-02 13:16:23 -0400