Merge branch 'access-creds'

The access() (and faccessat()) credentials change can cause an
unnecessary load on the RCU machinery because every access() call ends
up freeing the temporary access credential using RCU.

This isn't really noticeable on small machines, but if you have hundreds
of cores you can cause huge slowdowns due to RCU storms.

It's easy to avoid: the temporary access crededntials aren't actually
normally accessed using RCU at all, so we can avoid the whole issue by
just marking them as such.

* access-creds:
  access: avoid the RCU grace period for the temporary subjective credentials

3 files changed, 45 insertions, 3 deletions

diff --git a/fs/open.c b/fs/open.c
index b5b80469b93d..a59abe3c669a 100644
--- a/fs/open.c
+++ b/fs/open.c
@@ -374,6 +374,25 @@ long do_faccessat(int dfd, const char __user *filename, int mode)
                                 override_cred->cap_permitted;
         }
 
+        /*
+         * The new set of credentials can *only* be used in
+         * task-synchronous circumstances, and does not need
+         * RCU freeing, unless somebody then takes a separate
+         * reference to it.
+         *
+         * NOTE! This is _only_ true because this credential
+         * is used purely for override_creds() that installs
+         * it as the subjective cred. Other threads will be
+         * accessing ->real_cred, not the subjective cred.
+         *
+         * If somebody _does_ make a copy of this (using the
+         * 'get_current_cred()' function), that will clear the
+         * non_rcu field, because now that other user may be
+         * expecting RCU freeing. But normal thread-synchronous
+         * cred accesses will keep things non-RCY.
+         */
+        override_cred->non_rcu = 1;
+
         old_cred = override_creds(override_cred);
 retry:
         res = user_path_at(dfd, filename, lookup_flags, &path);
diff --git a/include/linux/cred.h b/include/linux/cred.h
index 7eb43a038330..f7a30e0099be 100644
--- a/include/linux/cred.h
+++ b/include/linux/cred.h
@@ -145,7 +145,11 @@ struct cred {
         struct user_struct *user;       /* real user ID subscription */
         struct user_namespace *user_ns; /* user_ns the caps and keyrings are relative to. */
         struct group_info *group_info;  /* supplementary groups for euid/fsgid */
-        struct rcu_head rcu;            /* RCU deletion hook */
+        /* RCU deletion */
+        union {
+                int non_rcu;                    /* Can we skip RCU deletion? */
+                struct rcu_head rcu;            /* RCU deletion hook */
+        };
 } __randomize_layout;
 
 extern void __put_cred(struct cred *);
@@ -246,6 +250,7 @@ static inline const struct cred *get_cred(const struct cred *cred)
         if (!cred)
                 return cred;
         validate_creds(cred);
+        nonconst_cred->non_rcu = 0;
         return get_new_cred(nonconst_cred);
 }
 
@@ -257,6 +262,7 @@ static inline const struct cred *get_cred_rcu(const struct cred *cred)
         if (!atomic_inc_not_zero(&nonconst_cred->usage))
                 return NULL;
         validate_creds(cred);
+        nonconst_cred->non_rcu = 0;
         return cred;
 }
 
diff --git a/kernel/cred.c b/kernel/cred.c
index f9a0ce66c9c3..c0a4c12d38b2 100644
--- a/kernel/cred.c
+++ b/kernel/cred.c
@@ -144,7 +144,10 @@ void __put_cred(struct cred *cred)
         BUG_ON(cred == current->cred);
         BUG_ON(cred == current->real_cred);
 
-        call_rcu(&cred->rcu, put_cred_rcu);
+        if (cred->non_rcu)
+                put_cred_rcu(&cred->rcu);
+        else
+                call_rcu(&cred->rcu, put_cred_rcu);
 }
 EXPORT_SYMBOL(__put_cred);
 
@@ -261,6 +264,7 @@ struct cred *prepare_creds(void)
         old = task->cred;
         memcpy(new, old, sizeof(struct cred));
 
+        new->non_rcu = 0;
         atomic_set(&new->usage, 1);
         set_cred_subscribers(new, 0);
         get_group_info(new->group_info);
@@ -544,7 +548,19 @@ const struct cred *override_creds(const struct cred *new)
 
         validate_creds(old);
         validate_creds(new);
-        get_cred(new);
+
+        /*
+         * NOTE! This uses 'get_new_cred()' rather than 'get_cred()'.
+         *
+         * That means that we do not clear the 'non_rcu' flag, since
+         * we are only installing the cred into the thread-synchronous
+         * '->cred' pointer, not the '->real_cred' pointer that is
+         * visible to other threads under RCU.
+         *
+         * Also note that we did validate_creds() manually, not depending
+         * on the validation in 'get_cred()'.
+         */
+        get_new_cred((struct cred *)new);
         alter_cred_subscribers(new, 1);
         rcu_assign_pointer(current->cred, new);
         alter_cred_subscribers(old, -1);
@@ -681,6 +697,7 @@ struct cred *prepare_kernel_cred(struct task_struct *daemon)
         validate_creds(old);
 
         *new = *old;
+        new->non_rcu = 0;
         atomic_set(&new->usage, 1);
         set_cred_subscribers(new, 0);
         get_uid(new->user);