diff options
author | Roopa Prabhu <roopa@cumulusnetworks.com> | 2015-07-21 04:43:47 -0400 |
---|---|---|
committer | David S. Miller <davem@davemloft.net> | 2015-07-21 13:39:03 -0400 |
commit | 571e722676fe386bb66f72a75b64a6ebf535c077 (patch) | |
tree | 80e374cf7f848408bad3be37237e4f43c9a9701b | |
parent | 499a24256862714539e902c0499b67da2bb3ab72 (diff) |
ipv4: support for fib route lwtunnel encap attributes
This patch adds support in ipv4 fib functions to parse user
provided encap attributes and attach encap state data to fib_nh
and rtable.
Signed-off-by: Roopa Prabhu <roopa@cumulusnetworks.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
-rw-r--r-- | include/net/ip_fib.h | 5 | ||||
-rw-r--r-- | include/net/route.h | 1 | ||||
-rw-r--r-- | net/ipv4/fib_frontend.c | 8 | ||||
-rw-r--r-- | net/ipv4/fib_semantics.c | 96 | ||||
-rw-r--r-- | net/ipv4/route.c | 16 |
5 files changed, 122 insertions, 4 deletions
diff --git a/include/net/ip_fib.h b/include/net/ip_fib.h index 49c142bdf01e..5e0196084f1e 100644 --- a/include/net/ip_fib.h +++ b/include/net/ip_fib.h | |||
@@ -44,7 +44,9 @@ struct fib_config { | |||
44 | u32 fc_flow; | 44 | u32 fc_flow; |
45 | u32 fc_nlflags; | 45 | u32 fc_nlflags; |
46 | struct nl_info fc_nlinfo; | 46 | struct nl_info fc_nlinfo; |
47 | }; | 47 | struct nlattr *fc_encap; |
48 | u16 fc_encap_type; | ||
49 | }; | ||
48 | 50 | ||
49 | struct fib_info; | 51 | struct fib_info; |
50 | struct rtable; | 52 | struct rtable; |
@@ -89,6 +91,7 @@ struct fib_nh { | |||
89 | struct rtable __rcu * __percpu *nh_pcpu_rth_output; | 91 | struct rtable __rcu * __percpu *nh_pcpu_rth_output; |
90 | struct rtable __rcu *nh_rth_input; | 92 | struct rtable __rcu *nh_rth_input; |
91 | struct fnhe_hash_bucket __rcu *nh_exceptions; | 93 | struct fnhe_hash_bucket __rcu *nh_exceptions; |
94 | struct lwtunnel_state *nh_lwtstate; | ||
92 | }; | 95 | }; |
93 | 96 | ||
94 | /* | 97 | /* |
diff --git a/include/net/route.h b/include/net/route.h index fe22d03afb6a..2d45f419477f 100644 --- a/include/net/route.h +++ b/include/net/route.h | |||
@@ -66,6 +66,7 @@ struct rtable { | |||
66 | 66 | ||
67 | struct list_head rt_uncached; | 67 | struct list_head rt_uncached; |
68 | struct uncached_list *rt_uncached_list; | 68 | struct uncached_list *rt_uncached_list; |
69 | struct lwtunnel_state *rt_lwtstate; | ||
69 | }; | 70 | }; |
70 | 71 | ||
71 | static inline bool rt_is_input_route(const struct rtable *rt) | 72 | static inline bool rt_is_input_route(const struct rtable *rt) |
diff --git a/net/ipv4/fib_frontend.c b/net/ipv4/fib_frontend.c index 6bbc54940eb4..9b2019cc3586 100644 --- a/net/ipv4/fib_frontend.c +++ b/net/ipv4/fib_frontend.c | |||
@@ -591,6 +591,8 @@ const struct nla_policy rtm_ipv4_policy[RTA_MAX + 1] = { | |||
591 | [RTA_METRICS] = { .type = NLA_NESTED }, | 591 | [RTA_METRICS] = { .type = NLA_NESTED }, |
592 | [RTA_MULTIPATH] = { .len = sizeof(struct rtnexthop) }, | 592 | [RTA_MULTIPATH] = { .len = sizeof(struct rtnexthop) }, |
593 | [RTA_FLOW] = { .type = NLA_U32 }, | 593 | [RTA_FLOW] = { .type = NLA_U32 }, |
594 | [RTA_ENCAP_TYPE] = { .type = NLA_U16 }, | ||
595 | [RTA_ENCAP] = { .type = NLA_NESTED }, | ||
594 | }; | 596 | }; |
595 | 597 | ||
596 | static int rtm_to_fib_config(struct net *net, struct sk_buff *skb, | 598 | static int rtm_to_fib_config(struct net *net, struct sk_buff *skb, |
@@ -656,6 +658,12 @@ static int rtm_to_fib_config(struct net *net, struct sk_buff *skb, | |||
656 | case RTA_TABLE: | 658 | case RTA_TABLE: |
657 | cfg->fc_table = nla_get_u32(attr); | 659 | cfg->fc_table = nla_get_u32(attr); |
658 | break; | 660 | break; |
661 | case RTA_ENCAP: | ||
662 | cfg->fc_encap = attr; | ||
663 | break; | ||
664 | case RTA_ENCAP_TYPE: | ||
665 | cfg->fc_encap_type = nla_get_u16(attr); | ||
666 | break; | ||
659 | } | 667 | } |
660 | } | 668 | } |
661 | 669 | ||
diff --git a/net/ipv4/fib_semantics.c b/net/ipv4/fib_semantics.c index c7358ea4ae93..6754c64b2fe0 100644 --- a/net/ipv4/fib_semantics.c +++ b/net/ipv4/fib_semantics.c | |||
@@ -42,6 +42,7 @@ | |||
42 | #include <net/ip_fib.h> | 42 | #include <net/ip_fib.h> |
43 | #include <net/netlink.h> | 43 | #include <net/netlink.h> |
44 | #include <net/nexthop.h> | 44 | #include <net/nexthop.h> |
45 | #include <net/lwtunnel.h> | ||
45 | 46 | ||
46 | #include "fib_lookup.h" | 47 | #include "fib_lookup.h" |
47 | 48 | ||
@@ -208,6 +209,7 @@ static void free_fib_info_rcu(struct rcu_head *head) | |||
208 | change_nexthops(fi) { | 209 | change_nexthops(fi) { |
209 | if (nexthop_nh->nh_dev) | 210 | if (nexthop_nh->nh_dev) |
210 | dev_put(nexthop_nh->nh_dev); | 211 | dev_put(nexthop_nh->nh_dev); |
212 | lwtunnel_state_put(nexthop_nh->nh_lwtstate); | ||
211 | free_nh_exceptions(nexthop_nh); | 213 | free_nh_exceptions(nexthop_nh); |
212 | rt_fibinfo_free_cpus(nexthop_nh->nh_pcpu_rth_output); | 214 | rt_fibinfo_free_cpus(nexthop_nh->nh_pcpu_rth_output); |
213 | rt_fibinfo_free(&nexthop_nh->nh_rth_input); | 215 | rt_fibinfo_free(&nexthop_nh->nh_rth_input); |
@@ -266,6 +268,7 @@ static inline int nh_comp(const struct fib_info *fi, const struct fib_info *ofi) | |||
266 | #ifdef CONFIG_IP_ROUTE_CLASSID | 268 | #ifdef CONFIG_IP_ROUTE_CLASSID |
267 | nh->nh_tclassid != onh->nh_tclassid || | 269 | nh->nh_tclassid != onh->nh_tclassid || |
268 | #endif | 270 | #endif |
271 | lwtunnel_cmp_encap(nh->nh_lwtstate, onh->nh_lwtstate) || | ||
269 | ((nh->nh_flags ^ onh->nh_flags) & ~RTNH_COMPARE_MASK)) | 272 | ((nh->nh_flags ^ onh->nh_flags) & ~RTNH_COMPARE_MASK)) |
270 | return -1; | 273 | return -1; |
271 | onh++; | 274 | onh++; |
@@ -366,6 +369,7 @@ static inline size_t fib_nlmsg_size(struct fib_info *fi) | |||
366 | payload += nla_total_size((RTAX_MAX * nla_total_size(4))); | 369 | payload += nla_total_size((RTAX_MAX * nla_total_size(4))); |
367 | 370 | ||
368 | if (fi->fib_nhs) { | 371 | if (fi->fib_nhs) { |
372 | size_t nh_encapsize = 0; | ||
369 | /* Also handles the special case fib_nhs == 1 */ | 373 | /* Also handles the special case fib_nhs == 1 */ |
370 | 374 | ||
371 | /* each nexthop is packed in an attribute */ | 375 | /* each nexthop is packed in an attribute */ |
@@ -374,8 +378,21 @@ static inline size_t fib_nlmsg_size(struct fib_info *fi) | |||
374 | /* may contain flow and gateway attribute */ | 378 | /* may contain flow and gateway attribute */ |
375 | nhsize += 2 * nla_total_size(4); | 379 | nhsize += 2 * nla_total_size(4); |
376 | 380 | ||
381 | /* grab encap info */ | ||
382 | for_nexthops(fi) { | ||
383 | if (nh->nh_lwtstate) { | ||
384 | /* RTA_ENCAP_TYPE */ | ||
385 | nh_encapsize += lwtunnel_get_encap_size( | ||
386 | nh->nh_lwtstate); | ||
387 | /* RTA_ENCAP */ | ||
388 | nh_encapsize += nla_total_size(2); | ||
389 | } | ||
390 | } endfor_nexthops(fi); | ||
391 | |||
377 | /* all nexthops are packed in a nested attribute */ | 392 | /* all nexthops are packed in a nested attribute */ |
378 | payload += nla_total_size(fi->fib_nhs * nhsize); | 393 | payload += nla_total_size((fi->fib_nhs * nhsize) + |
394 | nh_encapsize); | ||
395 | |||
379 | } | 396 | } |
380 | 397 | ||
381 | return payload; | 398 | return payload; |
@@ -452,6 +469,9 @@ static int fib_count_nexthops(struct rtnexthop *rtnh, int remaining) | |||
452 | static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh, | 469 | static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh, |
453 | int remaining, struct fib_config *cfg) | 470 | int remaining, struct fib_config *cfg) |
454 | { | 471 | { |
472 | struct net *net = cfg->fc_nlinfo.nl_net; | ||
473 | int ret; | ||
474 | |||
455 | change_nexthops(fi) { | 475 | change_nexthops(fi) { |
456 | int attrlen; | 476 | int attrlen; |
457 | 477 | ||
@@ -475,18 +495,66 @@ static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh, | |||
475 | if (nexthop_nh->nh_tclassid) | 495 | if (nexthop_nh->nh_tclassid) |
476 | fi->fib_net->ipv4.fib_num_tclassid_users++; | 496 | fi->fib_net->ipv4.fib_num_tclassid_users++; |
477 | #endif | 497 | #endif |
498 | nla = nla_find(attrs, attrlen, RTA_ENCAP); | ||
499 | if (nla) { | ||
500 | struct lwtunnel_state *lwtstate; | ||
501 | struct net_device *dev = NULL; | ||
502 | struct nlattr *nla_entype; | ||
503 | |||
504 | nla_entype = nla_find(attrs, attrlen, | ||
505 | RTA_ENCAP_TYPE); | ||
506 | if (!nla_entype) | ||
507 | goto err_inval; | ||
508 | if (cfg->fc_oif) | ||
509 | dev = __dev_get_by_index(net, cfg->fc_oif); | ||
510 | ret = lwtunnel_build_state(dev, nla_get_u16( | ||
511 | nla_entype), | ||
512 | nla, &lwtstate); | ||
513 | if (ret) | ||
514 | goto errout; | ||
515 | lwtunnel_state_get(lwtstate); | ||
516 | nexthop_nh->nh_lwtstate = lwtstate; | ||
517 | } | ||
478 | } | 518 | } |
479 | 519 | ||
480 | rtnh = rtnh_next(rtnh, &remaining); | 520 | rtnh = rtnh_next(rtnh, &remaining); |
481 | } endfor_nexthops(fi); | 521 | } endfor_nexthops(fi); |
482 | 522 | ||
483 | return 0; | 523 | return 0; |
524 | |||
525 | err_inval: | ||
526 | ret = -EINVAL; | ||
527 | |||
528 | errout: | ||
529 | return ret; | ||
484 | } | 530 | } |
485 | 531 | ||
486 | #endif | 532 | #endif |
487 | 533 | ||
534 | int fib_encap_match(struct net *net, u16 encap_type, | ||
535 | struct nlattr *encap, | ||
536 | int oif, const struct fib_nh *nh) | ||
537 | { | ||
538 | struct lwtunnel_state *lwtstate; | ||
539 | struct net_device *dev = NULL; | ||
540 | int ret; | ||
541 | |||
542 | if (encap_type == LWTUNNEL_ENCAP_NONE) | ||
543 | return 0; | ||
544 | |||
545 | if (oif) | ||
546 | dev = __dev_get_by_index(net, oif); | ||
547 | ret = lwtunnel_build_state(dev, encap_type, | ||
548 | encap, &lwtstate); | ||
549 | if (!ret) | ||
550 | return lwtunnel_cmp_encap(lwtstate, nh->nh_lwtstate); | ||
551 | |||
552 | return 0; | ||
553 | } | ||
554 | |||
488 | int fib_nh_match(struct fib_config *cfg, struct fib_info *fi) | 555 | int fib_nh_match(struct fib_config *cfg, struct fib_info *fi) |
489 | { | 556 | { |
557 | struct net *net = cfg->fc_nlinfo.nl_net; | ||
490 | #ifdef CONFIG_IP_ROUTE_MULTIPATH | 558 | #ifdef CONFIG_IP_ROUTE_MULTIPATH |
491 | struct rtnexthop *rtnh; | 559 | struct rtnexthop *rtnh; |
492 | int remaining; | 560 | int remaining; |
@@ -496,6 +564,12 @@ int fib_nh_match(struct fib_config *cfg, struct fib_info *fi) | |||
496 | return 1; | 564 | return 1; |
497 | 565 | ||
498 | if (cfg->fc_oif || cfg->fc_gw) { | 566 | if (cfg->fc_oif || cfg->fc_gw) { |
567 | if (cfg->fc_encap) { | ||
568 | if (fib_encap_match(net, cfg->fc_encap_type, | ||
569 | cfg->fc_encap, cfg->fc_oif, | ||
570 | fi->fib_nh)) | ||
571 | return 1; | ||
572 | } | ||
499 | if ((!cfg->fc_oif || cfg->fc_oif == fi->fib_nh->nh_oif) && | 573 | if ((!cfg->fc_oif || cfg->fc_oif == fi->fib_nh->nh_oif) && |
500 | (!cfg->fc_gw || cfg->fc_gw == fi->fib_nh->nh_gw)) | 574 | (!cfg->fc_gw || cfg->fc_gw == fi->fib_nh->nh_gw)) |
501 | return 0; | 575 | return 0; |
@@ -882,6 +956,22 @@ struct fib_info *fib_create_info(struct fib_config *cfg) | |||
882 | } else { | 956 | } else { |
883 | struct fib_nh *nh = fi->fib_nh; | 957 | struct fib_nh *nh = fi->fib_nh; |
884 | 958 | ||
959 | if (cfg->fc_encap) { | ||
960 | struct lwtunnel_state *lwtstate; | ||
961 | struct net_device *dev = NULL; | ||
962 | |||
963 | if (cfg->fc_encap_type == LWTUNNEL_ENCAP_NONE) | ||
964 | goto err_inval; | ||
965 | if (cfg->fc_oif) | ||
966 | dev = __dev_get_by_index(net, cfg->fc_oif); | ||
967 | err = lwtunnel_build_state(dev, cfg->fc_encap_type, | ||
968 | cfg->fc_encap, &lwtstate); | ||
969 | if (err) | ||
970 | goto failure; | ||
971 | |||
972 | lwtunnel_state_get(lwtstate); | ||
973 | nh->nh_lwtstate = lwtstate; | ||
974 | } | ||
885 | nh->nh_oif = cfg->fc_oif; | 975 | nh->nh_oif = cfg->fc_oif; |
886 | nh->nh_gw = cfg->fc_gw; | 976 | nh->nh_gw = cfg->fc_gw; |
887 | nh->nh_flags = cfg->fc_flags; | 977 | nh->nh_flags = cfg->fc_flags; |
@@ -1055,6 +1145,8 @@ int fib_dump_info(struct sk_buff *skb, u32 portid, u32 seq, int event, | |||
1055 | nla_put_u32(skb, RTA_FLOW, fi->fib_nh[0].nh_tclassid)) | 1145 | nla_put_u32(skb, RTA_FLOW, fi->fib_nh[0].nh_tclassid)) |
1056 | goto nla_put_failure; | 1146 | goto nla_put_failure; |
1057 | #endif | 1147 | #endif |
1148 | if (fi->fib_nh->nh_lwtstate) | ||
1149 | lwtunnel_fill_encap(skb, fi->fib_nh->nh_lwtstate); | ||
1058 | } | 1150 | } |
1059 | #ifdef CONFIG_IP_ROUTE_MULTIPATH | 1151 | #ifdef CONFIG_IP_ROUTE_MULTIPATH |
1060 | if (fi->fib_nhs > 1) { | 1152 | if (fi->fib_nhs > 1) { |
@@ -1090,6 +1182,8 @@ int fib_dump_info(struct sk_buff *skb, u32 portid, u32 seq, int event, | |||
1090 | nla_put_u32(skb, RTA_FLOW, nh->nh_tclassid)) | 1182 | nla_put_u32(skb, RTA_FLOW, nh->nh_tclassid)) |
1091 | goto nla_put_failure; | 1183 | goto nla_put_failure; |
1092 | #endif | 1184 | #endif |
1185 | if (nh->nh_lwtstate) | ||
1186 | lwtunnel_fill_encap(skb, nh->nh_lwtstate); | ||
1093 | /* length of rtnetlink header + attributes */ | 1187 | /* length of rtnetlink header + attributes */ |
1094 | rtnh->rtnh_len = nlmsg_get_pos(skb) - (void *) rtnh; | 1188 | rtnh->rtnh_len = nlmsg_get_pos(skb) - (void *) rtnh; |
1095 | } endfor_nexthops(fi); | 1189 | } endfor_nexthops(fi); |
diff --git a/net/ipv4/route.c b/net/ipv4/route.c index 04c83de4f79e..226570ba1ced 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c | |||
@@ -102,6 +102,7 @@ | |||
102 | #include <net/tcp.h> | 102 | #include <net/tcp.h> |
103 | #include <net/icmp.h> | 103 | #include <net/icmp.h> |
104 | #include <net/xfrm.h> | 104 | #include <net/xfrm.h> |
105 | #include <net/lwtunnel.h> | ||
105 | #include <net/netevent.h> | 106 | #include <net/netevent.h> |
106 | #include <net/rtnetlink.h> | 107 | #include <net/rtnetlink.h> |
107 | #ifdef CONFIG_SYSCTL | 108 | #ifdef CONFIG_SYSCTL |
@@ -1355,6 +1356,7 @@ static void ipv4_dst_destroy(struct dst_entry *dst) | |||
1355 | list_del(&rt->rt_uncached); | 1356 | list_del(&rt->rt_uncached); |
1356 | spin_unlock_bh(&ul->lock); | 1357 | spin_unlock_bh(&ul->lock); |
1357 | } | 1358 | } |
1359 | lwtunnel_state_put(rt->rt_lwtstate); | ||
1358 | } | 1360 | } |
1359 | 1361 | ||
1360 | void rt_flush_dev(struct net_device *dev) | 1362 | void rt_flush_dev(struct net_device *dev) |
@@ -1403,6 +1405,12 @@ static void rt_set_nexthop(struct rtable *rt, __be32 daddr, | |||
1403 | #ifdef CONFIG_IP_ROUTE_CLASSID | 1405 | #ifdef CONFIG_IP_ROUTE_CLASSID |
1404 | rt->dst.tclassid = nh->nh_tclassid; | 1406 | rt->dst.tclassid = nh->nh_tclassid; |
1405 | #endif | 1407 | #endif |
1408 | if (nh->nh_lwtstate) { | ||
1409 | lwtunnel_state_get(nh->nh_lwtstate); | ||
1410 | rt->rt_lwtstate = nh->nh_lwtstate; | ||
1411 | } else { | ||
1412 | rt->rt_lwtstate = NULL; | ||
1413 | } | ||
1406 | if (unlikely(fnhe)) | 1414 | if (unlikely(fnhe)) |
1407 | cached = rt_bind_exception(rt, fnhe, daddr); | 1415 | cached = rt_bind_exception(rt, fnhe, daddr); |
1408 | else if (!(rt->dst.flags & DST_NOCACHE)) | 1416 | else if (!(rt->dst.flags & DST_NOCACHE)) |
@@ -1488,6 +1496,7 @@ static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr, | |||
1488 | rth->rt_gateway = 0; | 1496 | rth->rt_gateway = 0; |
1489 | rth->rt_uses_gateway = 0; | 1497 | rth->rt_uses_gateway = 0; |
1490 | INIT_LIST_HEAD(&rth->rt_uncached); | 1498 | INIT_LIST_HEAD(&rth->rt_uncached); |
1499 | rth->rt_lwtstate = NULL; | ||
1491 | if (our) { | 1500 | if (our) { |
1492 | rth->dst.input= ip_local_deliver; | 1501 | rth->dst.input= ip_local_deliver; |
1493 | rth->rt_flags |= RTCF_LOCAL; | 1502 | rth->rt_flags |= RTCF_LOCAL; |
@@ -1617,6 +1626,7 @@ static int __mkroute_input(struct sk_buff *skb, | |||
1617 | rth->rt_gateway = 0; | 1626 | rth->rt_gateway = 0; |
1618 | rth->rt_uses_gateway = 0; | 1627 | rth->rt_uses_gateway = 0; |
1619 | INIT_LIST_HEAD(&rth->rt_uncached); | 1628 | INIT_LIST_HEAD(&rth->rt_uncached); |
1629 | rth->rt_lwtstate = NULL; | ||
1620 | RT_CACHE_STAT_INC(in_slow_tot); | 1630 | RT_CACHE_STAT_INC(in_slow_tot); |
1621 | 1631 | ||
1622 | rth->dst.input = ip_forward; | 1632 | rth->dst.input = ip_forward; |
@@ -1791,6 +1801,8 @@ local_input: | |||
1791 | rth->rt_gateway = 0; | 1801 | rth->rt_gateway = 0; |
1792 | rth->rt_uses_gateway = 0; | 1802 | rth->rt_uses_gateway = 0; |
1793 | INIT_LIST_HEAD(&rth->rt_uncached); | 1803 | INIT_LIST_HEAD(&rth->rt_uncached); |
1804 | rth->rt_lwtstate = NULL; | ||
1805 | |||
1794 | RT_CACHE_STAT_INC(in_slow_tot); | 1806 | RT_CACHE_STAT_INC(in_slow_tot); |
1795 | if (res.type == RTN_UNREACHABLE) { | 1807 | if (res.type == RTN_UNREACHABLE) { |
1796 | rth->dst.input= ip_error; | 1808 | rth->dst.input= ip_error; |
@@ -1980,7 +1992,7 @@ add: | |||
1980 | rth->rt_gateway = 0; | 1992 | rth->rt_gateway = 0; |
1981 | rth->rt_uses_gateway = 0; | 1993 | rth->rt_uses_gateway = 0; |
1982 | INIT_LIST_HEAD(&rth->rt_uncached); | 1994 | INIT_LIST_HEAD(&rth->rt_uncached); |
1983 | 1995 | rth->rt_lwtstate = NULL; | |
1984 | RT_CACHE_STAT_INC(out_slow_tot); | 1996 | RT_CACHE_STAT_INC(out_slow_tot); |
1985 | 1997 | ||
1986 | if (flags & RTCF_LOCAL) | 1998 | if (flags & RTCF_LOCAL) |
@@ -2260,7 +2272,7 @@ struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_or | |||
2260 | rt->rt_uses_gateway = ort->rt_uses_gateway; | 2272 | rt->rt_uses_gateway = ort->rt_uses_gateway; |
2261 | 2273 | ||
2262 | INIT_LIST_HEAD(&rt->rt_uncached); | 2274 | INIT_LIST_HEAD(&rt->rt_uncached); |
2263 | 2275 | rt->rt_lwtstate = NULL; | |
2264 | dst_free(new); | 2276 | dst_free(new); |
2265 | } | 2277 | } |
2266 | 2278 | ||