aboutsummaryrefslogtreecommitdiffstats
path: root/lib/radix-tree.c
diff options
context:
space:
mode:
authorKonstantin Khlebnikov <khlebnikov@openvz.org>2012-03-28 17:42:53 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2012-03-28 20:14:37 -0400
commitcebbd29e1c2f7a969919f19f74583070840163d7 (patch)
treef9ec01b2cb5c3affe64587b83d4877429f0edc0c /lib/radix-tree.c
parent78c1d78488a3c45685d993130c9f17102dc79a54 (diff)
radix-tree: rewrite gang lookup using iterator
Rewrite radix_tree_gang_lookup_* functions using the new radix-tree iterator. Signed-off-by: Konstantin Khlebnikov <khlebnikov@openvz.org> Tested-by: Hugh Dickins <hughd@google.com> Cc: Christoph Hellwig <hch@lst.de> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Diffstat (limited to 'lib/radix-tree.c')
-rw-r--r--lib/radix-tree.c291
1 files changed, 33 insertions, 258 deletions
diff --git a/lib/radix-tree.c b/lib/radix-tree.c
index fefa76e6ff96..86516f5588e3 100644
--- a/lib/radix-tree.c
+++ b/lib/radix-tree.c
@@ -968,57 +968,6 @@ unsigned long radix_tree_prev_hole(struct radix_tree_root *root,
968} 968}
969EXPORT_SYMBOL(radix_tree_prev_hole); 969EXPORT_SYMBOL(radix_tree_prev_hole);
970 970
971static unsigned int
972__lookup(struct radix_tree_node *slot, void ***results, unsigned long *indices,
973 unsigned long index, unsigned int max_items, unsigned long *next_index)
974{
975 unsigned int nr_found = 0;
976 unsigned int shift, height;
977 unsigned long i;
978
979 height = slot->height;
980 if (height == 0)
981 goto out;
982 shift = (height-1) * RADIX_TREE_MAP_SHIFT;
983
984 for ( ; height > 1; height--) {
985 i = (index >> shift) & RADIX_TREE_MAP_MASK;
986 for (;;) {
987 if (slot->slots[i] != NULL)
988 break;
989 index &= ~((1UL << shift) - 1);
990 index += 1UL << shift;
991 if (index == 0)
992 goto out; /* 32-bit wraparound */
993 i++;
994 if (i == RADIX_TREE_MAP_SIZE)
995 goto out;
996 }
997
998 shift -= RADIX_TREE_MAP_SHIFT;
999 slot = rcu_dereference_raw(slot->slots[i]);
1000 if (slot == NULL)
1001 goto out;
1002 }
1003
1004 /* Bottom level: grab some items */
1005 for (i = index & RADIX_TREE_MAP_MASK; i < RADIX_TREE_MAP_SIZE; i++) {
1006 if (slot->slots[i]) {
1007 results[nr_found] = &(slot->slots[i]);
1008 if (indices)
1009 indices[nr_found] = index;
1010 if (++nr_found == max_items) {
1011 index++;
1012 goto out;
1013 }
1014 }
1015 index++;
1016 }
1017out:
1018 *next_index = index;
1019 return nr_found;
1020}
1021
1022/** 971/**
1023 * radix_tree_gang_lookup - perform multiple lookup on a radix tree 972 * radix_tree_gang_lookup - perform multiple lookup on a radix tree
1024 * @root: radix tree root 973 * @root: radix tree root
@@ -1042,48 +991,19 @@ unsigned int
1042radix_tree_gang_lookup(struct radix_tree_root *root, void **results, 991radix_tree_gang_lookup(struct radix_tree_root *root, void **results,
1043 unsigned long first_index, unsigned int max_items) 992 unsigned long first_index, unsigned int max_items)
1044{ 993{
1045 unsigned long max_index; 994 struct radix_tree_iter iter;
1046 struct radix_tree_node *node; 995 void **slot;
1047 unsigned long cur_index = first_index; 996 unsigned int ret = 0;
1048 unsigned int ret;
1049 997
1050 node = rcu_dereference_raw(root->rnode); 998 if (unlikely(!max_items))
1051 if (!node)
1052 return 0; 999 return 0;
1053 1000
1054 if (!radix_tree_is_indirect_ptr(node)) { 1001 radix_tree_for_each_slot(slot, root, &iter, first_index) {
1055 if (first_index > 0) 1002 results[ret] = indirect_to_ptr(rcu_dereference_raw(*slot));
1056 return 0; 1003 if (!results[ret])
1057 results[0] = node; 1004 continue;
1058 return 1; 1005 if (++ret == max_items)
1059 }
1060 node = indirect_to_ptr(node);
1061
1062 max_index = radix_tree_maxindex(node->height);
1063
1064 ret = 0;
1065 while (ret < max_items) {
1066 unsigned int nr_found, slots_found, i;
1067 unsigned long next_index; /* Index of next search */
1068
1069 if (cur_index > max_index)
1070 break;
1071 slots_found = __lookup(node, (void ***)results + ret, NULL,
1072 cur_index, max_items - ret, &next_index);
1073 nr_found = 0;
1074 for (i = 0; i < slots_found; i++) {
1075 struct radix_tree_node *slot;
1076 slot = *(((void ***)results)[ret + i]);
1077 if (!slot)
1078 continue;
1079 results[ret + nr_found] =
1080 indirect_to_ptr(rcu_dereference_raw(slot));
1081 nr_found++;
1082 }
1083 ret += nr_found;
1084 if (next_index == 0)
1085 break; 1006 break;
1086 cur_index = next_index;
1087 } 1007 }
1088 1008
1089 return ret; 1009 return ret;
@@ -1113,112 +1033,25 @@ radix_tree_gang_lookup_slot(struct radix_tree_root *root,
1113 void ***results, unsigned long *indices, 1033 void ***results, unsigned long *indices,
1114 unsigned long first_index, unsigned int max_items) 1034 unsigned long first_index, unsigned int max_items)
1115{ 1035{
1116 unsigned long max_index; 1036 struct radix_tree_iter iter;
1117 struct radix_tree_node *node; 1037 void **slot;
1118 unsigned long cur_index = first_index; 1038 unsigned int ret = 0;
1119 unsigned int ret;
1120 1039
1121 node = rcu_dereference_raw(root->rnode); 1040 if (unlikely(!max_items))
1122 if (!node)
1123 return 0; 1041 return 0;
1124 1042
1125 if (!radix_tree_is_indirect_ptr(node)) { 1043 radix_tree_for_each_slot(slot, root, &iter, first_index) {
1126 if (first_index > 0) 1044 results[ret] = slot;
1127 return 0;
1128 results[0] = (void **)&root->rnode;
1129 if (indices) 1045 if (indices)
1130 indices[0] = 0; 1046 indices[ret] = iter.index;
1131 return 1; 1047 if (++ret == max_items)
1132 }
1133 node = indirect_to_ptr(node);
1134
1135 max_index = radix_tree_maxindex(node->height);
1136
1137 ret = 0;
1138 while (ret < max_items) {
1139 unsigned int slots_found;
1140 unsigned long next_index; /* Index of next search */
1141
1142 if (cur_index > max_index)
1143 break; 1048 break;
1144 slots_found = __lookup(node, results + ret,
1145 indices ? indices + ret : NULL,
1146 cur_index, max_items - ret, &next_index);
1147 ret += slots_found;
1148 if (next_index == 0)
1149 break;
1150 cur_index = next_index;
1151 } 1049 }
1152 1050
1153 return ret; 1051 return ret;
1154} 1052}
1155EXPORT_SYMBOL(radix_tree_gang_lookup_slot); 1053EXPORT_SYMBOL(radix_tree_gang_lookup_slot);
1156 1054
1157/*
1158 * FIXME: the two tag_get()s here should use find_next_bit() instead of
1159 * open-coding the search.
1160 */
1161static unsigned int
1162__lookup_tag(struct radix_tree_node *slot, void ***results, unsigned long index,
1163 unsigned int max_items, unsigned long *next_index, unsigned int tag)
1164{
1165 unsigned int nr_found = 0;
1166 unsigned int shift, height;
1167
1168 height = slot->height;
1169 if (height == 0)
1170 goto out;
1171 shift = (height-1) * RADIX_TREE_MAP_SHIFT;
1172
1173 while (height > 0) {
1174 unsigned long i = (index >> shift) & RADIX_TREE_MAP_MASK ;
1175
1176 for (;;) {
1177 if (tag_get(slot, tag, i))
1178 break;
1179 index &= ~((1UL << shift) - 1);
1180 index += 1UL << shift;
1181 if (index == 0)
1182 goto out; /* 32-bit wraparound */
1183 i++;
1184 if (i == RADIX_TREE_MAP_SIZE)
1185 goto out;
1186 }
1187 height--;
1188 if (height == 0) { /* Bottom level: grab some items */
1189 unsigned long j = index & RADIX_TREE_MAP_MASK;
1190
1191 for ( ; j < RADIX_TREE_MAP_SIZE; j++) {
1192 index++;
1193 if (!tag_get(slot, tag, j))
1194 continue;
1195 /*
1196 * Even though the tag was found set, we need to
1197 * recheck that we have a non-NULL node, because
1198 * if this lookup is lockless, it may have been
1199 * subsequently deleted.
1200 *
1201 * Similar care must be taken in any place that
1202 * lookup ->slots[x] without a lock (ie. can't
1203 * rely on its value remaining the same).
1204 */
1205 if (slot->slots[j]) {
1206 results[nr_found++] = &(slot->slots[j]);
1207 if (nr_found == max_items)
1208 goto out;
1209 }
1210 }
1211 }
1212 shift -= RADIX_TREE_MAP_SHIFT;
1213 slot = rcu_dereference_raw(slot->slots[i]);
1214 if (slot == NULL)
1215 break;
1216 }
1217out:
1218 *next_index = index;
1219 return nr_found;
1220}
1221
1222/** 1055/**
1223 * radix_tree_gang_lookup_tag - perform multiple lookup on a radix tree 1056 * radix_tree_gang_lookup_tag - perform multiple lookup on a radix tree
1224 * based on a tag 1057 * based on a tag
@@ -1237,52 +1070,19 @@ radix_tree_gang_lookup_tag(struct radix_tree_root *root, void **results,
1237 unsigned long first_index, unsigned int max_items, 1070 unsigned long first_index, unsigned int max_items,
1238 unsigned int tag) 1071 unsigned int tag)
1239{ 1072{
1240 struct radix_tree_node *node; 1073 struct radix_tree_iter iter;
1241 unsigned long max_index; 1074 void **slot;
1242 unsigned long cur_index = first_index; 1075 unsigned int ret = 0;
1243 unsigned int ret;
1244 1076
1245 /* check the root's tag bit */ 1077 if (unlikely(!max_items))
1246 if (!root_tag_get(root, tag))
1247 return 0; 1078 return 0;
1248 1079
1249 node = rcu_dereference_raw(root->rnode); 1080 radix_tree_for_each_tagged(slot, root, &iter, first_index, tag) {
1250 if (!node) 1081 results[ret] = indirect_to_ptr(rcu_dereference_raw(*slot));
1251 return 0; 1082 if (!results[ret])
1252 1083 continue;
1253 if (!radix_tree_is_indirect_ptr(node)) { 1084 if (++ret == max_items)
1254 if (first_index > 0)
1255 return 0;
1256 results[0] = node;
1257 return 1;
1258 }
1259 node = indirect_to_ptr(node);
1260
1261 max_index = radix_tree_maxindex(node->height);
1262
1263 ret = 0;
1264 while (ret < max_items) {
1265 unsigned int nr_found, slots_found, i;
1266 unsigned long next_index; /* Index of next search */
1267
1268 if (cur_index > max_index)
1269 break;
1270 slots_found = __lookup_tag(node, (void ***)results + ret,
1271 cur_index, max_items - ret, &next_index, tag);
1272 nr_found = 0;
1273 for (i = 0; i < slots_found; i++) {
1274 struct radix_tree_node *slot;
1275 slot = *(((void ***)results)[ret + i]);
1276 if (!slot)
1277 continue;
1278 results[ret + nr_found] =
1279 indirect_to_ptr(rcu_dereference_raw(slot));
1280 nr_found++;
1281 }
1282 ret += nr_found;
1283 if (next_index == 0)
1284 break; 1085 break;
1285 cur_index = next_index;
1286 } 1086 }
1287 1087
1288 return ret; 1088 return ret;
@@ -1307,42 +1107,17 @@ radix_tree_gang_lookup_tag_slot(struct radix_tree_root *root, void ***results,
1307 unsigned long first_index, unsigned int max_items, 1107 unsigned long first_index, unsigned int max_items,
1308 unsigned int tag) 1108 unsigned int tag)
1309{ 1109{
1310 struct radix_tree_node *node; 1110 struct radix_tree_iter iter;
1311 unsigned long max_index; 1111 void **slot;
1312 unsigned long cur_index = first_index; 1112 unsigned int ret = 0;
1313 unsigned int ret;
1314 1113
1315 /* check the root's tag bit */ 1114 if (unlikely(!max_items))
1316 if (!root_tag_get(root, tag))
1317 return 0;
1318
1319 node = rcu_dereference_raw(root->rnode);
1320 if (!node)
1321 return 0; 1115 return 0;
1322 1116
1323 if (!radix_tree_is_indirect_ptr(node)) { 1117 radix_tree_for_each_tagged(slot, root, &iter, first_index, tag) {
1324 if (first_index > 0) 1118 results[ret] = slot;
1325 return 0; 1119 if (++ret == max_items)
1326 results[0] = (void **)&root->rnode;
1327 return 1;
1328 }
1329 node = indirect_to_ptr(node);
1330
1331 max_index = radix_tree_maxindex(node->height);
1332
1333 ret = 0;
1334 while (ret < max_items) {
1335 unsigned int slots_found;
1336 unsigned long next_index; /* Index of next search */
1337
1338 if (cur_index > max_index)
1339 break;
1340 slots_found = __lookup_tag(node, results + ret,
1341 cur_index, max_items - ret, &next_index, tag);
1342 ret += slots_found;
1343 if (next_index == 0)
1344 break; 1120 break;
1345 cur_index = next_index;
1346 } 1121 }
1347 1122
1348 return ret; 1123 return ret;
generated by cgit v1.2.2 (git 2.25.0) at 2024-09-21 06:33:54 -0400