3 files changed, 130 insertions, 0 deletions

diff --git a/include/linux/psi.h b/include/linux/psi.h
new file mode 100644
index 000000000000..b0daf050de58
--- /dev/null
+++ b/include/linux/psi.h
@@ -0,0 +1,28 @@
+#ifndef _LINUX_PSI_H
+#define _LINUX_PSI_H
+
+#include <linux/psi_types.h>
+#include <linux/sched.h>
+
+#ifdef CONFIG_PSI
+
+extern bool psi_disabled;
+
+void psi_init(void);
+
+void psi_task_change(struct task_struct *task, int clear, int set);
+
+void psi_memstall_tick(struct task_struct *task, int cpu);
+void psi_memstall_enter(unsigned long *flags);
+void psi_memstall_leave(unsigned long *flags);
+
+#else /* CONFIG_PSI */
+
+static inline void psi_init(void) {}
+
+static inline void psi_memstall_enter(unsigned long *flags) {}
+static inline void psi_memstall_leave(unsigned long *flags) {}
+
+#endif /* CONFIG_PSI */
+
+#endif /* _LINUX_PSI_H */
diff --git a/include/linux/psi_types.h b/include/linux/psi_types.h
new file mode 100644
index 000000000000..2cf422db5d18
--- /dev/null
+++ b/include/linux/psi_types.h
@@ -0,0 +1,92 @@
+#ifndef _LINUX_PSI_TYPES_H
+#define _LINUX_PSI_TYPES_H
+
+#include <linux/seqlock.h>
+#include <linux/types.h>
+
+#ifdef CONFIG_PSI
+
+/* Tracked task states */
+enum psi_task_count {
+        NR_IOWAIT,
+        NR_MEMSTALL,
+        NR_RUNNING,
+        NR_PSI_TASK_COUNTS,
+};
+
+/* Task state bitmasks */
+#define TSK_IOWAIT      (1 << NR_IOWAIT)
+#define TSK_MEMSTALL    (1 << NR_MEMSTALL)
+#define TSK_RUNNING     (1 << NR_RUNNING)
+
+/* Resources that workloads could be stalled on */
+enum psi_res {
+        PSI_IO,
+        PSI_MEM,
+        PSI_CPU,
+        NR_PSI_RESOURCES,
+};
+
+/*
+ * Pressure states for each resource:
+ *
+ * SOME: Stalled tasks & working tasks
+ * FULL: Stalled tasks & no working tasks
+ */
+enum psi_states {
+        PSI_IO_SOME,
+        PSI_IO_FULL,
+        PSI_MEM_SOME,
+        PSI_MEM_FULL,
+        PSI_CPU_SOME,
+        /* Only per-CPU, to weigh the CPU in the global average: */
+        PSI_NONIDLE,
+        NR_PSI_STATES,
+};
+
+struct psi_group_cpu {
+        /* 1st cacheline updated by the scheduler */
+
+        /* Aggregator needs to know of concurrent changes */
+        seqcount_t seq ____cacheline_aligned_in_smp;
+
+        /* States of the tasks belonging to this group */
+        unsigned int tasks[NR_PSI_TASK_COUNTS];
+
+        /* Period time sampling buckets for each state of interest (ns) */
+        u32 times[NR_PSI_STATES];
+
+        /* Time of last task change in this group (rq_clock) */
+        u64 state_start;
+
+        /* 2nd cacheline updated by the aggregator */
+
+        /* Delta detection against the sampling buckets */
+        u32 times_prev[NR_PSI_STATES] ____cacheline_aligned_in_smp;
+};
+
+struct psi_group {
+        /* Protects data updated during an aggregation */
+        struct mutex stat_lock;
+
+        /* Per-cpu task state & time tracking */
+        struct psi_group_cpu __percpu *pcpu;
+
+        /* Periodic aggregation state */
+        u64 total_prev[NR_PSI_STATES - 1];
+        u64 last_update;
+        u64 next_update;
+        struct delayed_work clock_work;
+
+        /* Total stall times and sampled pressure averages */
+        u64 total[NR_PSI_STATES - 1];
+        unsigned long avg[NR_PSI_STATES - 1][3];
+};
+
+#else /* CONFIG_PSI */
+
+struct psi_group { };
+
+#endif /* CONFIG_PSI */
+
+#endif /* _LINUX_PSI_TYPES_H */
diff --git a/include/linux/sched.h b/include/linux/sched.h
index adfb3f9a7597..b8fcc6b3080c 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -25,6 +25,7 @@
 #include <linux/latencytop.h>
 #include <linux/sched/prio.h>
 #include <linux/signal_types.h>
+#include <linux/psi_types.h>
 #include <linux/mm_types_task.h>
 #include <linux/task_io_accounting.h>
 #include <linux/rseq.h>
@@ -706,6 +707,10 @@ struct task_struct {
         unsigned                        sched_contributes_to_load:1;
         unsigned                        sched_migrated:1;
         unsigned                        sched_remote_wakeup:1;
+#ifdef CONFIG_PSI
+        unsigned                        sched_psi_wake_requeue:1;
+#endif
+
         /* Force alignment to the next boundary: */
         unsigned                        :0;
 
@@ -965,6 +970,10 @@ struct task_struct {
         kernel_siginfo_t                *last_siginfo;
 
         struct task_io_accounting       ioac;
+#ifdef CONFIG_PSI
+        /* Pressure stall state */
+        unsigned int                    psi_flags;
+#endif
 #ifdef CONFIG_TASK_XACCT
         /* Accumulated RSS usage: */
         u64                             acct_rss_mem1;
@@ -1391,6 +1400,7 @@ extern struct pid *cad_pid;
 #define PF_KTHREAD              0x00200000      /* I am a kernel thread */
 #define PF_RANDOMIZE            0x00400000      /* Randomize virtual address space */
 #define PF_SWAPWRITE            0x00800000      /* Allowed to write to swap */
+#define PF_MEMSTALL             0x01000000      /* Stalled due to lack of memory */
 #define PF_NO_SETAFFINITY       0x04000000      /* Userland is not allowed to meddle with cpus_allowed */
 #define PF_MCE_EARLY            0x08000000      /* Early kill for mce process policy */
 #define PF_MUTEX_TESTER         0x20000000      /* Thread belongs to the rt mutex tester */