bpf: introduce BPF_F_LOCK flag

Introduce BPF_F_LOCK flag for map_lookup and map_update syscall commands
and for map_update() helper function.
In all these cases take a lock of existing element (which was provided
in BTF description) before copying (in or out) the rest of map value.

Implementation details that are part of uapi:

Array:
The array map takes the element lock for lookup/update.

Hash:
hash map also takes the lock for lookup/update and tries to avoid the bucket lock.
If old element exists it takes the element lock and updates the element in place.
If element doesn't exist it allocates new one and inserts into hash table
while holding the bucket lock.
In rare case the hashmap has to take both the bucket lock and the element lock
to update old value in place.

Cgroup local storage:
It is similar to array. update in place and lookup are done with lock taken.

Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>

1 files changed, 23 insertions, 2 deletions

diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index b29e6dc44650..0834958f1dc4 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -682,7 +682,7 @@ static void *__bpf_copy_key(void __user *ukey, u64 key_size)
 }
 
 /* last field in 'union bpf_attr' used by this command */
-#define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value
+#define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags
 
 static int map_lookup_elem(union bpf_attr *attr)
 {
@@ -698,6 +698,9 @@ static int map_lookup_elem(union bpf_attr *attr)
         if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
                 return -EINVAL;
 
+        if (attr->flags & ~BPF_F_LOCK)
+                return -EINVAL;
+
         f = fdget(ufd);
         map = __bpf_map_get(f);
         if (IS_ERR(map))
@@ -708,6 +711,12 @@ static int map_lookup_elem(union bpf_attr *attr)
                 goto err_put;
         }
 
+        if ((attr->flags & BPF_F_LOCK) &&
+            !map_value_has_spin_lock(map)) {
+                err = -EINVAL;
+                goto err_put;
+        }
+
         key = __bpf_copy_key(ukey, map->key_size);
         if (IS_ERR(key)) {
                 err = PTR_ERR(key);
@@ -758,7 +767,13 @@ static int map_lookup_elem(union bpf_attr *attr)
                         err = -ENOENT;
                 } else {
                         err = 0;
-                        copy_map_value(map, value, ptr);
+                        if (attr->flags & BPF_F_LOCK)
+                                /* lock 'ptr' and copy everything but lock */
+                                copy_map_value_locked(map, value, ptr, true);
+                        else
+                                copy_map_value(map, value, ptr);
+                        /* mask lock, since value wasn't zero inited */
+                        check_and_init_map_lock(map, value);
                 }
                 rcu_read_unlock();
         }
@@ -818,6 +833,12 @@ static int map_update_elem(union bpf_attr *attr)
                 goto err_put;
         }
 
+        if ((attr->flags & BPF_F_LOCK) &&
+            !map_value_has_spin_lock(map)) {
+                err = -EINVAL;
+                goto err_put;
+        }
+
         key = __bpf_copy_key(ukey, map->key_size);
         if (IS_ERR(key)) {
                 err = PTR_ERR(key);