mm/page_owner: keep track of page owners

This is the page owner tracking code which is introduced so far ago.  It
is resident on Andrew's tree, though, nobody tried to upstream so it
remain as is.  Our company uses this feature actively to debug memory leak
or to find a memory hogger so I decide to upstream this feature.

This functionality help us to know who allocates the page.  When
allocating a page, we store some information about allocation in extra
memory.  Later, if we need to know status of all pages, we can get and
analyze it from this stored information.

In previous version of this feature, extra memory is statically defined in
struct page, but, in this version, extra memory is allocated outside of
struct page.  It enables us to turn on/off this feature at boottime
without considerable memory waste.

Although we already have tracepoint for tracing page allocation/free,
using it to analyze page owner is rather complex.  We need to enlarge the
trace buffer for preventing overlapping until userspace program launched.
And, launched program continually dump out the trace buffer for later
analysis and it would change system behaviour with more possibility rather
than just keeping it in memory, so bad for debug.

Moreover, we can use page_owner feature further for various purposes.  For
example, we can use it for fragmentation statistics implemented in this
patch.  And, I also plan to implement some CMA failure debugging feature
using this interface.

I'd like to give the credit for all developers contributed this feature,
but, it's not easy because I don't know exact history.  Sorry about that.
Below is people who has "Signed-off-by" in the patches in Andrew's tree.

Contributor:
Alexander Nyberg <alexn@dsv.su.se>
Mel Gorman <mgorman@suse.de>
Dave Hansen <dave@linux.vnet.ibm.com>
Minchan Kim <minchan@kernel.org>
Michal Nazarewicz <mina86@mina86.com>
Andrew Morton <akpm@linux-foundation.org>
Jungsoo Son <jungsoo.son@lge.com>

Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Dave Hansen <dave@sr71.net>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Jungsoo Son <jungsoo.son@lge.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 222 insertions, 0 deletions

diff --git a/mm/page_owner.c b/mm/page_owner.c
new file mode 100644
index 000000000000..85eec7ea6735
--- /dev/null
+++ b/mm/page_owner.c
@@ -0,0 +1,222 @@
+#include <linux/debugfs.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/bootmem.h>
+#include <linux/stacktrace.h>
+#include <linux/page_owner.h>
+#include "internal.h"
+
+static bool page_owner_disabled = true;
+bool page_owner_inited __read_mostly;
+
+static int early_page_owner_param(char *buf)
+{
+        if (!buf)
+                return -EINVAL;
+
+        if (strcmp(buf, "on") == 0)
+                page_owner_disabled = false;
+
+        return 0;
+}
+early_param("page_owner", early_page_owner_param);
+
+static bool need_page_owner(void)
+{
+        if (page_owner_disabled)
+                return false;
+
+        return true;
+}
+
+static void init_page_owner(void)
+{
+        if (page_owner_disabled)
+                return;
+
+        page_owner_inited = true;
+}
+
+struct page_ext_operations page_owner_ops = {
+        .need = need_page_owner,
+        .init = init_page_owner,
+};
+
+void __reset_page_owner(struct page *page, unsigned int order)
+{
+        int i;
+        struct page_ext *page_ext;
+
+        for (i = 0; i < (1 << order); i++) {
+                page_ext = lookup_page_ext(page + i);
+                __clear_bit(PAGE_EXT_OWNER, &page_ext->flags);
+        }
+}
+
+void __set_page_owner(struct page *page, unsigned int order, gfp_t gfp_mask)
+{
+        struct page_ext *page_ext;
+        struct stack_trace *trace;
+
+        page_ext = lookup_page_ext(page);
+
+        trace = &page_ext->trace;
+        trace->nr_entries = 0;
+        trace->max_entries = ARRAY_SIZE(page_ext->trace_entries);
+        trace->entries = &page_ext->trace_entries[0];
+        trace->skip = 3;
+        save_stack_trace(&page_ext->trace);
+
+        page_ext->order = order;
+        page_ext->gfp_mask = gfp_mask;
+
+        __set_bit(PAGE_EXT_OWNER, &page_ext->flags);
+}
+
+static ssize_t
+print_page_owner(char __user *buf, size_t count, unsigned long pfn,
+                struct page *page, struct page_ext *page_ext)
+{
+        int ret;
+        int pageblock_mt, page_mt;
+        char *kbuf;
+
+        kbuf = kmalloc(count, GFP_KERNEL);
+        if (!kbuf)
+                return -ENOMEM;
+
+        ret = snprintf(kbuf, count,
+                        "Page allocated via order %u, mask 0x%x\n",
+                        page_ext->order, page_ext->gfp_mask);
+
+        if (ret >= count)
+                goto err;
+
+        /* Print information relevant to grouping pages by mobility */
+        pageblock_mt = get_pfnblock_migratetype(page, pfn);
+        page_mt  = gfpflags_to_migratetype(page_ext->gfp_mask);
+        ret += snprintf(kbuf + ret, count - ret,
+                        "PFN %lu Block %lu type %d %s Flags %s%s%s%s%s%s%s%s%s%s%s%s\n",
+                        pfn,
+                        pfn >> pageblock_order,
+                        pageblock_mt,
+                        pageblock_mt != page_mt ? "Fallback" : "        ",
+                        PageLocked(page)        ? "K" : " ",
+                        PageError(page)         ? "E" : " ",
+                        PageReferenced(page)    ? "R" : " ",
+                        PageUptodate(page)      ? "U" : " ",
+                        PageDirty(page)         ? "D" : " ",
+                        PageLRU(page)           ? "L" : " ",
+                        PageActive(page)        ? "A" : " ",
+                        PageSlab(page)          ? "S" : " ",
+                        PageWriteback(page)     ? "W" : " ",
+                        PageCompound(page)      ? "C" : " ",
+                        PageSwapCache(page)     ? "B" : " ",
+                        PageMappedToDisk(page)  ? "M" : " ");
+
+        if (ret >= count)
+                goto err;
+
+        ret += snprint_stack_trace(kbuf + ret, count - ret,
+                                        &page_ext->trace, 0);
+        if (ret >= count)
+                goto err;
+
+        ret += snprintf(kbuf + ret, count - ret, "\n");
+        if (ret >= count)
+                goto err;
+
+        if (copy_to_user(buf, kbuf, ret))
+                ret = -EFAULT;
+
+        kfree(kbuf);
+        return ret;
+
+err:
+        kfree(kbuf);
+        return -ENOMEM;
+}
+
+static ssize_t
+read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos)
+{
+        unsigned long pfn;
+        struct page *page;
+        struct page_ext *page_ext;
+
+        if (!page_owner_inited)
+                return -EINVAL;
+
+        page = NULL;
+        pfn = min_low_pfn + *ppos;
+
+        /* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */
+        while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0)
+                pfn++;
+
+        drain_all_pages(NULL);
+
+        /* Find an allocated page */
+        for (; pfn < max_pfn; pfn++) {
+                /*
+                 * If the new page is in a new MAX_ORDER_NR_PAGES area,
+                 * validate the area as existing, skip it if not
+                 */
+                if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0 && !pfn_valid(pfn)) {
+                        pfn += MAX_ORDER_NR_PAGES - 1;
+                        continue;
+                }
+
+                /* Check for holes within a MAX_ORDER area */
+                if (!pfn_valid_within(pfn))
+                        continue;
+
+                page = pfn_to_page(pfn);
+                if (PageBuddy(page)) {
+                        unsigned long freepage_order = page_order_unsafe(page);
+
+                        if (freepage_order < MAX_ORDER)
+                                pfn += (1UL << freepage_order) - 1;
+                        continue;
+                }
+
+                page_ext = lookup_page_ext(page);
+
+                /*
+                 * Pages allocated before initialization of page_owner are
+                 * non-buddy and have no page_owner info.
+                 */
+                if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags))
+                        continue;
+
+                /* Record the next PFN to read in the file offset */
+                *ppos = (pfn - min_low_pfn) + 1;
+
+                return print_page_owner(buf, count, pfn, page, page_ext);
+        }
+
+        return 0;
+}
+
+static const struct file_operations proc_page_owner_operations = {
+        .read           = read_page_owner,
+};
+
+static int __init pageowner_init(void)
+{
+        struct dentry *dentry;
+
+        if (!page_owner_inited) {
+                pr_info("page_owner is disabled\n");
+                return 0;
+        }
+
+        dentry = debugfs_create_file("page_owner", S_IRUSR, NULL,
+                        NULL, &proc_page_owner_operations);
+        if (IS_ERR(dentry))
+                return PTR_ERR(dentry);
+
+        return 0;
+}
+module_init(pageowner_init)