diff options
Diffstat (limited to 'net/core/net-sysfs.c')
-rw-r--r-- | net/core/net-sysfs.c | 430 |
1 files changed, 412 insertions, 18 deletions
diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c index 7f902cad10f..e23c01be5a5 100644 --- a/net/core/net-sysfs.c +++ b/net/core/net-sysfs.c | |||
@@ -706,7 +706,6 @@ static struct attribute *rx_queue_default_attrs[] = { | |||
706 | static void rx_queue_release(struct kobject *kobj) | 706 | static void rx_queue_release(struct kobject *kobj) |
707 | { | 707 | { |
708 | struct netdev_rx_queue *queue = to_rx_queue(kobj); | 708 | struct netdev_rx_queue *queue = to_rx_queue(kobj); |
709 | struct netdev_rx_queue *first = queue->first; | ||
710 | struct rps_map *map; | 709 | struct rps_map *map; |
711 | struct rps_dev_flow_table *flow_table; | 710 | struct rps_dev_flow_table *flow_table; |
712 | 711 | ||
@@ -723,10 +722,8 @@ static void rx_queue_release(struct kobject *kobj) | |||
723 | call_rcu(&flow_table->rcu, rps_dev_flow_table_release); | 722 | call_rcu(&flow_table->rcu, rps_dev_flow_table_release); |
724 | } | 723 | } |
725 | 724 | ||
726 | if (atomic_dec_and_test(&first->count)) | 725 | memset(kobj, 0, sizeof(*kobj)); |
727 | kfree(first); | 726 | dev_put(queue->dev); |
728 | else | ||
729 | memset(kobj, 0, sizeof(*kobj)); | ||
730 | } | 727 | } |
731 | 728 | ||
732 | static struct kobj_type rx_queue_ktype = { | 729 | static struct kobj_type rx_queue_ktype = { |
@@ -738,7 +735,6 @@ static struct kobj_type rx_queue_ktype = { | |||
738 | static int rx_queue_add_kobject(struct net_device *net, int index) | 735 | static int rx_queue_add_kobject(struct net_device *net, int index) |
739 | { | 736 | { |
740 | struct netdev_rx_queue *queue = net->_rx + index; | 737 | struct netdev_rx_queue *queue = net->_rx + index; |
741 | struct netdev_rx_queue *first = queue->first; | ||
742 | struct kobject *kobj = &queue->kobj; | 738 | struct kobject *kobj = &queue->kobj; |
743 | int error = 0; | 739 | int error = 0; |
744 | 740 | ||
@@ -751,14 +747,16 @@ static int rx_queue_add_kobject(struct net_device *net, int index) | |||
751 | } | 747 | } |
752 | 748 | ||
753 | kobject_uevent(kobj, KOBJ_ADD); | 749 | kobject_uevent(kobj, KOBJ_ADD); |
754 | atomic_inc(&first->count); | 750 | dev_hold(queue->dev); |
755 | 751 | ||
756 | return error; | 752 | return error; |
757 | } | 753 | } |
754 | #endif /* CONFIG_RPS */ | ||
758 | 755 | ||
759 | int | 756 | int |
760 | net_rx_queue_update_kobjects(struct net_device *net, int old_num, int new_num) | 757 | net_rx_queue_update_kobjects(struct net_device *net, int old_num, int new_num) |
761 | { | 758 | { |
759 | #ifdef CONFIG_RPS | ||
762 | int i; | 760 | int i; |
763 | int error = 0; | 761 | int error = 0; |
764 | 762 | ||
@@ -774,23 +772,423 @@ net_rx_queue_update_kobjects(struct net_device *net, int old_num, int new_num) | |||
774 | kobject_put(&net->_rx[i].kobj); | 772 | kobject_put(&net->_rx[i].kobj); |
775 | 773 | ||
776 | return error; | 774 | return error; |
775 | #else | ||
776 | return 0; | ||
777 | #endif | ||
778 | } | ||
779 | |||
780 | #ifdef CONFIG_XPS | ||
781 | /* | ||
782 | * netdev_queue sysfs structures and functions. | ||
783 | */ | ||
784 | struct netdev_queue_attribute { | ||
785 | struct attribute attr; | ||
786 | ssize_t (*show)(struct netdev_queue *queue, | ||
787 | struct netdev_queue_attribute *attr, char *buf); | ||
788 | ssize_t (*store)(struct netdev_queue *queue, | ||
789 | struct netdev_queue_attribute *attr, const char *buf, size_t len); | ||
790 | }; | ||
791 | #define to_netdev_queue_attr(_attr) container_of(_attr, \ | ||
792 | struct netdev_queue_attribute, attr) | ||
793 | |||
794 | #define to_netdev_queue(obj) container_of(obj, struct netdev_queue, kobj) | ||
795 | |||
796 | static ssize_t netdev_queue_attr_show(struct kobject *kobj, | ||
797 | struct attribute *attr, char *buf) | ||
798 | { | ||
799 | struct netdev_queue_attribute *attribute = to_netdev_queue_attr(attr); | ||
800 | struct netdev_queue *queue = to_netdev_queue(kobj); | ||
801 | |||
802 | if (!attribute->show) | ||
803 | return -EIO; | ||
804 | |||
805 | return attribute->show(queue, attribute, buf); | ||
777 | } | 806 | } |
778 | 807 | ||
779 | static int rx_queue_register_kobjects(struct net_device *net) | 808 | static ssize_t netdev_queue_attr_store(struct kobject *kobj, |
809 | struct attribute *attr, | ||
810 | const char *buf, size_t count) | ||
780 | { | 811 | { |
812 | struct netdev_queue_attribute *attribute = to_netdev_queue_attr(attr); | ||
813 | struct netdev_queue *queue = to_netdev_queue(kobj); | ||
814 | |||
815 | if (!attribute->store) | ||
816 | return -EIO; | ||
817 | |||
818 | return attribute->store(queue, attribute, buf, count); | ||
819 | } | ||
820 | |||
821 | static const struct sysfs_ops netdev_queue_sysfs_ops = { | ||
822 | .show = netdev_queue_attr_show, | ||
823 | .store = netdev_queue_attr_store, | ||
824 | }; | ||
825 | |||
826 | static inline unsigned int get_netdev_queue_index(struct netdev_queue *queue) | ||
827 | { | ||
828 | struct net_device *dev = queue->dev; | ||
829 | int i; | ||
830 | |||
831 | for (i = 0; i < dev->num_tx_queues; i++) | ||
832 | if (queue == &dev->_tx[i]) | ||
833 | break; | ||
834 | |||
835 | BUG_ON(i >= dev->num_tx_queues); | ||
836 | |||
837 | return i; | ||
838 | } | ||
839 | |||
840 | |||
841 | static ssize_t show_xps_map(struct netdev_queue *queue, | ||
842 | struct netdev_queue_attribute *attribute, char *buf) | ||
843 | { | ||
844 | struct net_device *dev = queue->dev; | ||
845 | struct xps_dev_maps *dev_maps; | ||
846 | cpumask_var_t mask; | ||
847 | unsigned long index; | ||
848 | size_t len = 0; | ||
849 | int i; | ||
850 | |||
851 | if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) | ||
852 | return -ENOMEM; | ||
853 | |||
854 | index = get_netdev_queue_index(queue); | ||
855 | |||
856 | rcu_read_lock(); | ||
857 | dev_maps = rcu_dereference(dev->xps_maps); | ||
858 | if (dev_maps) { | ||
859 | for_each_possible_cpu(i) { | ||
860 | struct xps_map *map = | ||
861 | rcu_dereference(dev_maps->cpu_map[i]); | ||
862 | if (map) { | ||
863 | int j; | ||
864 | for (j = 0; j < map->len; j++) { | ||
865 | if (map->queues[j] == index) { | ||
866 | cpumask_set_cpu(i, mask); | ||
867 | break; | ||
868 | } | ||
869 | } | ||
870 | } | ||
871 | } | ||
872 | } | ||
873 | rcu_read_unlock(); | ||
874 | |||
875 | len += cpumask_scnprintf(buf + len, PAGE_SIZE, mask); | ||
876 | if (PAGE_SIZE - len < 3) { | ||
877 | free_cpumask_var(mask); | ||
878 | return -EINVAL; | ||
879 | } | ||
880 | |||
881 | free_cpumask_var(mask); | ||
882 | len += sprintf(buf + len, "\n"); | ||
883 | return len; | ||
884 | } | ||
885 | |||
886 | static void xps_map_release(struct rcu_head *rcu) | ||
887 | { | ||
888 | struct xps_map *map = container_of(rcu, struct xps_map, rcu); | ||
889 | |||
890 | kfree(map); | ||
891 | } | ||
892 | |||
893 | static void xps_dev_maps_release(struct rcu_head *rcu) | ||
894 | { | ||
895 | struct xps_dev_maps *dev_maps = | ||
896 | container_of(rcu, struct xps_dev_maps, rcu); | ||
897 | |||
898 | kfree(dev_maps); | ||
899 | } | ||
900 | |||
901 | static DEFINE_MUTEX(xps_map_mutex); | ||
902 | #define xmap_dereference(P) \ | ||
903 | rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex)) | ||
904 | |||
905 | static ssize_t store_xps_map(struct netdev_queue *queue, | ||
906 | struct netdev_queue_attribute *attribute, | ||
907 | const char *buf, size_t len) | ||
908 | { | ||
909 | struct net_device *dev = queue->dev; | ||
910 | cpumask_var_t mask; | ||
911 | int err, i, cpu, pos, map_len, alloc_len, need_set; | ||
912 | unsigned long index; | ||
913 | struct xps_map *map, *new_map; | ||
914 | struct xps_dev_maps *dev_maps, *new_dev_maps; | ||
915 | int nonempty = 0; | ||
916 | int numa_node = -2; | ||
917 | |||
918 | if (!capable(CAP_NET_ADMIN)) | ||
919 | return -EPERM; | ||
920 | |||
921 | if (!alloc_cpumask_var(&mask, GFP_KERNEL)) | ||
922 | return -ENOMEM; | ||
923 | |||
924 | index = get_netdev_queue_index(queue); | ||
925 | |||
926 | err = bitmap_parse(buf, len, cpumask_bits(mask), nr_cpumask_bits); | ||
927 | if (err) { | ||
928 | free_cpumask_var(mask); | ||
929 | return err; | ||
930 | } | ||
931 | |||
932 | new_dev_maps = kzalloc(max_t(unsigned, | ||
933 | XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES), GFP_KERNEL); | ||
934 | if (!new_dev_maps) { | ||
935 | free_cpumask_var(mask); | ||
936 | return -ENOMEM; | ||
937 | } | ||
938 | |||
939 | mutex_lock(&xps_map_mutex); | ||
940 | |||
941 | dev_maps = xmap_dereference(dev->xps_maps); | ||
942 | |||
943 | for_each_possible_cpu(cpu) { | ||
944 | map = dev_maps ? | ||
945 | xmap_dereference(dev_maps->cpu_map[cpu]) : NULL; | ||
946 | new_map = map; | ||
947 | if (map) { | ||
948 | for (pos = 0; pos < map->len; pos++) | ||
949 | if (map->queues[pos] == index) | ||
950 | break; | ||
951 | map_len = map->len; | ||
952 | alloc_len = map->alloc_len; | ||
953 | } else | ||
954 | pos = map_len = alloc_len = 0; | ||
955 | |||
956 | need_set = cpu_isset(cpu, *mask) && cpu_online(cpu); | ||
957 | #ifdef CONFIG_NUMA | ||
958 | if (need_set) { | ||
959 | if (numa_node == -2) | ||
960 | numa_node = cpu_to_node(cpu); | ||
961 | else if (numa_node != cpu_to_node(cpu)) | ||
962 | numa_node = -1; | ||
963 | } | ||
964 | #endif | ||
965 | if (need_set && pos >= map_len) { | ||
966 | /* Need to add queue to this CPU's map */ | ||
967 | if (map_len >= alloc_len) { | ||
968 | alloc_len = alloc_len ? | ||
969 | 2 * alloc_len : XPS_MIN_MAP_ALLOC; | ||
970 | new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), | ||
971 | GFP_KERNEL, | ||
972 | cpu_to_node(cpu)); | ||
973 | if (!new_map) | ||
974 | goto error; | ||
975 | new_map->alloc_len = alloc_len; | ||
976 | for (i = 0; i < map_len; i++) | ||
977 | new_map->queues[i] = map->queues[i]; | ||
978 | new_map->len = map_len; | ||
979 | } | ||
980 | new_map->queues[new_map->len++] = index; | ||
981 | } else if (!need_set && pos < map_len) { | ||
982 | /* Need to remove queue from this CPU's map */ | ||
983 | if (map_len > 1) | ||
984 | new_map->queues[pos] = | ||
985 | new_map->queues[--new_map->len]; | ||
986 | else | ||
987 | new_map = NULL; | ||
988 | } | ||
989 | RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], new_map); | ||
990 | } | ||
991 | |||
992 | /* Cleanup old maps */ | ||
993 | for_each_possible_cpu(cpu) { | ||
994 | map = dev_maps ? | ||
995 | xmap_dereference(dev_maps->cpu_map[cpu]) : NULL; | ||
996 | if (map && xmap_dereference(new_dev_maps->cpu_map[cpu]) != map) | ||
997 | call_rcu(&map->rcu, xps_map_release); | ||
998 | if (new_dev_maps->cpu_map[cpu]) | ||
999 | nonempty = 1; | ||
1000 | } | ||
1001 | |||
1002 | if (nonempty) | ||
1003 | rcu_assign_pointer(dev->xps_maps, new_dev_maps); | ||
1004 | else { | ||
1005 | kfree(new_dev_maps); | ||
1006 | rcu_assign_pointer(dev->xps_maps, NULL); | ||
1007 | } | ||
1008 | |||
1009 | if (dev_maps) | ||
1010 | call_rcu(&dev_maps->rcu, xps_dev_maps_release); | ||
1011 | |||
1012 | netdev_queue_numa_node_write(queue, (numa_node >= 0) ? numa_node : | ||
1013 | NUMA_NO_NODE); | ||
1014 | |||
1015 | mutex_unlock(&xps_map_mutex); | ||
1016 | |||
1017 | free_cpumask_var(mask); | ||
1018 | return len; | ||
1019 | |||
1020 | error: | ||
1021 | mutex_unlock(&xps_map_mutex); | ||
1022 | |||
1023 | if (new_dev_maps) | ||
1024 | for_each_possible_cpu(i) | ||
1025 | kfree(rcu_dereference_protected( | ||
1026 | new_dev_maps->cpu_map[i], | ||
1027 | 1)); | ||
1028 | kfree(new_dev_maps); | ||
1029 | free_cpumask_var(mask); | ||
1030 | return -ENOMEM; | ||
1031 | } | ||
1032 | |||
1033 | static struct netdev_queue_attribute xps_cpus_attribute = | ||
1034 | __ATTR(xps_cpus, S_IRUGO | S_IWUSR, show_xps_map, store_xps_map); | ||
1035 | |||
1036 | static struct attribute *netdev_queue_default_attrs[] = { | ||
1037 | &xps_cpus_attribute.attr, | ||
1038 | NULL | ||
1039 | }; | ||
1040 | |||
1041 | static void netdev_queue_release(struct kobject *kobj) | ||
1042 | { | ||
1043 | struct netdev_queue *queue = to_netdev_queue(kobj); | ||
1044 | struct net_device *dev = queue->dev; | ||
1045 | struct xps_dev_maps *dev_maps; | ||
1046 | struct xps_map *map; | ||
1047 | unsigned long index; | ||
1048 | int i, pos, nonempty = 0; | ||
1049 | |||
1050 | index = get_netdev_queue_index(queue); | ||
1051 | |||
1052 | mutex_lock(&xps_map_mutex); | ||
1053 | dev_maps = xmap_dereference(dev->xps_maps); | ||
1054 | |||
1055 | if (dev_maps) { | ||
1056 | for_each_possible_cpu(i) { | ||
1057 | map = xmap_dereference(dev_maps->cpu_map[i]); | ||
1058 | if (!map) | ||
1059 | continue; | ||
1060 | |||
1061 | for (pos = 0; pos < map->len; pos++) | ||
1062 | if (map->queues[pos] == index) | ||
1063 | break; | ||
1064 | |||
1065 | if (pos < map->len) { | ||
1066 | if (map->len > 1) | ||
1067 | map->queues[pos] = | ||
1068 | map->queues[--map->len]; | ||
1069 | else { | ||
1070 | RCU_INIT_POINTER(dev_maps->cpu_map[i], | ||
1071 | NULL); | ||
1072 | call_rcu(&map->rcu, xps_map_release); | ||
1073 | map = NULL; | ||
1074 | } | ||
1075 | } | ||
1076 | if (map) | ||
1077 | nonempty = 1; | ||
1078 | } | ||
1079 | |||
1080 | if (!nonempty) { | ||
1081 | RCU_INIT_POINTER(dev->xps_maps, NULL); | ||
1082 | call_rcu(&dev_maps->rcu, xps_dev_maps_release); | ||
1083 | } | ||
1084 | } | ||
1085 | |||
1086 | mutex_unlock(&xps_map_mutex); | ||
1087 | |||
1088 | memset(kobj, 0, sizeof(*kobj)); | ||
1089 | dev_put(queue->dev); | ||
1090 | } | ||
1091 | |||
1092 | static struct kobj_type netdev_queue_ktype = { | ||
1093 | .sysfs_ops = &netdev_queue_sysfs_ops, | ||
1094 | .release = netdev_queue_release, | ||
1095 | .default_attrs = netdev_queue_default_attrs, | ||
1096 | }; | ||
1097 | |||
1098 | static int netdev_queue_add_kobject(struct net_device *net, int index) | ||
1099 | { | ||
1100 | struct netdev_queue *queue = net->_tx + index; | ||
1101 | struct kobject *kobj = &queue->kobj; | ||
1102 | int error = 0; | ||
1103 | |||
1104 | kobj->kset = net->queues_kset; | ||
1105 | error = kobject_init_and_add(kobj, &netdev_queue_ktype, NULL, | ||
1106 | "tx-%u", index); | ||
1107 | if (error) { | ||
1108 | kobject_put(kobj); | ||
1109 | return error; | ||
1110 | } | ||
1111 | |||
1112 | kobject_uevent(kobj, KOBJ_ADD); | ||
1113 | dev_hold(queue->dev); | ||
1114 | |||
1115 | return error; | ||
1116 | } | ||
1117 | #endif /* CONFIG_XPS */ | ||
1118 | |||
1119 | int | ||
1120 | netdev_queue_update_kobjects(struct net_device *net, int old_num, int new_num) | ||
1121 | { | ||
1122 | #ifdef CONFIG_XPS | ||
1123 | int i; | ||
1124 | int error = 0; | ||
1125 | |||
1126 | for (i = old_num; i < new_num; i++) { | ||
1127 | error = netdev_queue_add_kobject(net, i); | ||
1128 | if (error) { | ||
1129 | new_num = old_num; | ||
1130 | break; | ||
1131 | } | ||
1132 | } | ||
1133 | |||
1134 | while (--i >= new_num) | ||
1135 | kobject_put(&net->_tx[i].kobj); | ||
1136 | |||
1137 | return error; | ||
1138 | #else | ||
1139 | return 0; | ||
1140 | #endif | ||
1141 | } | ||
1142 | |||
1143 | static int register_queue_kobjects(struct net_device *net) | ||
1144 | { | ||
1145 | int error = 0, txq = 0, rxq = 0, real_rx = 0, real_tx = 0; | ||
1146 | |||
1147 | #if defined(CONFIG_RPS) || defined(CONFIG_XPS) | ||
781 | net->queues_kset = kset_create_and_add("queues", | 1148 | net->queues_kset = kset_create_and_add("queues", |
782 | NULL, &net->dev.kobj); | 1149 | NULL, &net->dev.kobj); |
783 | if (!net->queues_kset) | 1150 | if (!net->queues_kset) |
784 | return -ENOMEM; | 1151 | return -ENOMEM; |
785 | return net_rx_queue_update_kobjects(net, 0, net->real_num_rx_queues); | 1152 | #endif |
1153 | |||
1154 | #ifdef CONFIG_RPS | ||
1155 | real_rx = net->real_num_rx_queues; | ||
1156 | #endif | ||
1157 | real_tx = net->real_num_tx_queues; | ||
1158 | |||
1159 | error = net_rx_queue_update_kobjects(net, 0, real_rx); | ||
1160 | if (error) | ||
1161 | goto error; | ||
1162 | rxq = real_rx; | ||
1163 | |||
1164 | error = netdev_queue_update_kobjects(net, 0, real_tx); | ||
1165 | if (error) | ||
1166 | goto error; | ||
1167 | txq = real_tx; | ||
1168 | |||
1169 | return 0; | ||
1170 | |||
1171 | error: | ||
1172 | netdev_queue_update_kobjects(net, txq, 0); | ||
1173 | net_rx_queue_update_kobjects(net, rxq, 0); | ||
1174 | return error; | ||
786 | } | 1175 | } |
787 | 1176 | ||
788 | static void rx_queue_remove_kobjects(struct net_device *net) | 1177 | static void remove_queue_kobjects(struct net_device *net) |
789 | { | 1178 | { |
790 | net_rx_queue_update_kobjects(net, net->real_num_rx_queues, 0); | 1179 | int real_rx = 0, real_tx = 0; |
1180 | |||
1181 | #ifdef CONFIG_RPS | ||
1182 | real_rx = net->real_num_rx_queues; | ||
1183 | #endif | ||
1184 | real_tx = net->real_num_tx_queues; | ||
1185 | |||
1186 | net_rx_queue_update_kobjects(net, real_rx, 0); | ||
1187 | netdev_queue_update_kobjects(net, real_tx, 0); | ||
1188 | #if defined(CONFIG_RPS) || defined(CONFIG_XPS) | ||
791 | kset_unregister(net->queues_kset); | 1189 | kset_unregister(net->queues_kset); |
1190 | #endif | ||
792 | } | 1191 | } |
793 | #endif /* CONFIG_RPS */ | ||
794 | 1192 | ||
795 | static const void *net_current_ns(void) | 1193 | static const void *net_current_ns(void) |
796 | { | 1194 | { |
@@ -889,9 +1287,7 @@ void netdev_unregister_kobject(struct net_device * net) | |||
889 | 1287 | ||
890 | kobject_get(&dev->kobj); | 1288 | kobject_get(&dev->kobj); |
891 | 1289 | ||
892 | #ifdef CONFIG_RPS | 1290 | remove_queue_kobjects(net); |
893 | rx_queue_remove_kobjects(net); | ||
894 | #endif | ||
895 | 1291 | ||
896 | device_del(dev); | 1292 | device_del(dev); |
897 | } | 1293 | } |
@@ -930,13 +1326,11 @@ int netdev_register_kobject(struct net_device *net) | |||
930 | if (error) | 1326 | if (error) |
931 | return error; | 1327 | return error; |
932 | 1328 | ||
933 | #ifdef CONFIG_RPS | 1329 | error = register_queue_kobjects(net); |
934 | error = rx_queue_register_kobjects(net); | ||
935 | if (error) { | 1330 | if (error) { |
936 | device_del(dev); | 1331 | device_del(dev); |
937 | return error; | 1332 | return error; |
938 | } | 1333 | } |
939 | #endif | ||
940 | 1334 | ||
941 | return error; | 1335 | return error; |
942 | } | 1336 | } |