Merge branch 'akpm' (Andrew's patch-bomb)

Merge Andrew's second set of patches:
 - MM
 - a few random fixes
 - a couple of RTC leftovers

* emailed patches from Andrew Morton <akpm@linux-foundation.org>: (120 commits)
  rtc/rtc-88pm80x: remove unneed devm_kfree
  rtc/rtc-88pm80x: assign ret only when rtc_register_driver fails
  mm: hugetlbfs: close race during teardown of hugetlbfs shared page tables
  tmpfs: distribute interleave better across nodes
  mm: remove redundant initialization
  mm: warn if pg_data_t isn't initialized with zero
  mips: zero out pg_data_t when it's allocated
  memcg: gix memory accounting scalability in shrink_page_list
  mm/sparse: remove index_init_lock
  mm/sparse: more checks on mem_section number
  mm/sparse: optimize sparse_index_alloc
  memcg: add mem_cgroup_from_css() helper
  memcg: further prevent OOM with too many dirty pages
  memcg: prevent OOM with too many dirty pages
  mm: mmu_notifier: fix freed page still mapped in secondary MMU
  mm: memcg: only check anon swapin page charges for swap cache
  mm: memcg: only check swap cache pages for repeated charging
  mm: memcg: split swapin charge function into private and public part
  mm: memcg: remove needless !mm fixup to init_mm when charging
  mm: memcg: remove unneeded shmem charge type
  ...

4 files changed, 211 insertions, 33 deletions

diff --git a/net/core/dev.c b/net/core/dev.c
index c8569f826b71..0cb3fe8d8e72 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -3156,6 +3156,23 @@ void netdev_rx_handler_unregister(struct net_device *dev)
 }
 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
 
+/*
+ * Limit the use of PFMEMALLOC reserves to those protocols that implement
+ * the special handling of PFMEMALLOC skbs.
+ */
+static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
+{
+        switch (skb->protocol) {
+        case __constant_htons(ETH_P_ARP):
+        case __constant_htons(ETH_P_IP):
+        case __constant_htons(ETH_P_IPV6):
+        case __constant_htons(ETH_P_8021Q):
+                return true;
+        default:
+                return false;
+        }
+}
+
 static int __netif_receive_skb(struct sk_buff *skb)
 {
         struct packet_type *ptype, *pt_prev;
@@ -3165,14 +3182,27 @@ static int __netif_receive_skb(struct sk_buff *skb)
         bool deliver_exact = false;
         int ret = NET_RX_DROP;
         __be16 type;
+        unsigned long pflags = current->flags;
 
         net_timestamp_check(!netdev_tstamp_prequeue, skb);
 
         trace_netif_receive_skb(skb);
 
+        /*
+         * PFMEMALLOC skbs are special, they should
+         * - be delivered to SOCK_MEMALLOC sockets only
+         * - stay away from userspace
+         * - have bounded memory usage
+         *
+         * Use PF_MEMALLOC as this saves us from propagating the allocation
+         * context down to all allocation sites.
+         */
+        if (sk_memalloc_socks() && skb_pfmemalloc(skb))
+                current->flags |= PF_MEMALLOC;
+
         /* if we've gotten here through NAPI, check netpoll */
         if (netpoll_receive_skb(skb))
-                return NET_RX_DROP;
+                goto out;
 
         orig_dev = skb->dev;
 
@@ -3192,7 +3222,7 @@ another_round:
         if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
                 skb = vlan_untag(skb);
                 if (unlikely(!skb))
-                        goto out;
+                        goto unlock;
         }
 
 #ifdef CONFIG_NET_CLS_ACT
@@ -3202,6 +3232,9 @@ another_round:
         }
 #endif
 
+        if (sk_memalloc_socks() && skb_pfmemalloc(skb))
+                goto skip_taps;
+
         list_for_each_entry_rcu(ptype, &ptype_all, list) {
                 if (!ptype->dev || ptype->dev == skb->dev) {
                         if (pt_prev)
@@ -3210,13 +3243,18 @@ another_round:
                 }
         }
 
+skip_taps:
 #ifdef CONFIG_NET_CLS_ACT
         skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
         if (!skb)
-                goto out;
+                goto unlock;
 ncls:
 #endif
 
+        if (sk_memalloc_socks() && skb_pfmemalloc(skb)
+                                && !skb_pfmemalloc_protocol(skb))
+                goto drop;
+
         rx_handler = rcu_dereference(skb->dev->rx_handler);
         if (vlan_tx_tag_present(skb)) {
                 if (pt_prev) {
@@ -3226,7 +3264,7 @@ ncls:
                 if (vlan_do_receive(&skb, !rx_handler))
                         goto another_round;
                 else if (unlikely(!skb))
-                        goto out;
+                        goto unlock;
         }
 
         if (rx_handler) {
@@ -3236,7 +3274,7 @@ ncls:
                 }
                 switch (rx_handler(&skb)) {
                 case RX_HANDLER_CONSUMED:
-                        goto out;
+                        goto unlock;
                 case RX_HANDLER_ANOTHER:
                         goto another_round;
                 case RX_HANDLER_EXACT:
@@ -3269,6 +3307,7 @@ ncls:
                 else
                         ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
         } else {
+drop:
                 atomic_long_inc(&skb->dev->rx_dropped);
                 kfree_skb(skb);
                 /* Jamal, now you will not able to escape explaining
@@ -3277,8 +3316,10 @@ ncls:
                 ret = NET_RX_DROP;
         }
 
-out:
+unlock:
         rcu_read_unlock();
+out:
+        tsk_restore_flags(current, pflags, PF_MEMALLOC);
         return ret;
 }
 
diff --git a/net/core/filter.c b/net/core/filter.c
index d4ce2dc712e3..907efd27ec77 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -83,6 +83,14 @@ int sk_filter(struct sock *sk, struct sk_buff *skb)
         int err;
         struct sk_filter *filter;
 
+        /*
+         * If the skb was allocated from pfmemalloc reserves, only
+         * allow SOCK_MEMALLOC sockets to use it as this socket is
+         * helping free memory
+         */
+        if (skb_pfmemalloc(skb) && !sock_flag(sk, SOCK_MEMALLOC))
+                return -ENOMEM;
+
         err = security_sock_rcv_skb(sk, skb);
         if (err)
                 return err;
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 368f65c15e4f..fe00d1208167 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -145,6 +145,43 @@ static void skb_under_panic(struct sk_buff *skb, int sz, void *here)
         BUG();
 }
 
+
+/*
+ * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells
+ * the caller if emergency pfmemalloc reserves are being used. If it is and
+ * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves
+ * may be used. Otherwise, the packet data may be discarded until enough
+ * memory is free
+ */
+#define kmalloc_reserve(size, gfp, node, pfmemalloc) \
+         __kmalloc_reserve(size, gfp, node, _RET_IP_, pfmemalloc)
+void *__kmalloc_reserve(size_t size, gfp_t flags, int node, unsigned long ip,
+                         bool *pfmemalloc)
+{
+        void *obj;
+        bool ret_pfmemalloc = false;
+
+        /*
+         * Try a regular allocation, when that fails and we're not entitled
+         * to the reserves, fail.
+         */
+        obj = kmalloc_node_track_caller(size,
+                                        flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
+                                        node);
+        if (obj || !(gfp_pfmemalloc_allowed(flags)))
+                goto out;
+
+        /* Try again but now we are using pfmemalloc reserves */
+        ret_pfmemalloc = true;
+        obj = kmalloc_node_track_caller(size, flags, node);
+
+out:
+        if (pfmemalloc)
+                *pfmemalloc = ret_pfmemalloc;
+
+        return obj;
+}
+
 /*      Allocate a new skbuff. We do this ourselves so we can fill in a few
  *      'private' fields and also do memory statistics to find all the
  *      [BEEP] leaks.
@@ -155,8 +192,10 @@ static void skb_under_panic(struct sk_buff *skb, int sz, void *here)
  *      __alloc_skb     -       allocate a network buffer
  *      @size: size to allocate
  *      @gfp_mask: allocation mask
- *      @fclone: allocate from fclone cache instead of head cache
- *              and allocate a cloned (child) skb
+ *      @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache
+ *              instead of head cache and allocate a cloned (child) skb.
+ *              If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for
+ *              allocations in case the data is required for writeback
  *      @node: numa node to allocate memory on
  *
  *      Allocate a new &sk_buff. The returned buffer has no headroom and a
@@ -167,14 +206,19 @@ static void skb_under_panic(struct sk_buff *skb, int sz, void *here)
  *      %GFP_ATOMIC.
  */
 struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
-                            int fclone, int node)
+                            int flags, int node)
 {
         struct kmem_cache *cache;
         struct skb_shared_info *shinfo;
         struct sk_buff *skb;
         u8 *data;
+        bool pfmemalloc;
 
-        cache = fclone ? skbuff_fclone_cache : skbuff_head_cache;
+        cache = (flags & SKB_ALLOC_FCLONE)
+                ? skbuff_fclone_cache : skbuff_head_cache;
+
+        if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))
+                gfp_mask |= __GFP_MEMALLOC;
 
         /* Get the HEAD */
         skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
@@ -189,7 +233,7 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
          */
         size = SKB_DATA_ALIGN(size);
         size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
-        data = kmalloc_node_track_caller(size, gfp_mask, node);
+        data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc);
         if (!data)
                 goto nodata;
         /* kmalloc(size) might give us more room than requested.
@@ -207,6 +251,7 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
         memset(skb, 0, offsetof(struct sk_buff, tail));
         /* Account for allocated memory : skb + skb->head */
         skb->truesize = SKB_TRUESIZE(size);
+        skb->pfmemalloc = pfmemalloc;
         atomic_set(&skb->users, 1);
         skb->head = data;
         skb->data = data;
@@ -222,7 +267,7 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
         atomic_set(&shinfo->dataref, 1);
         kmemcheck_annotate_variable(shinfo->destructor_arg);
 
-        if (fclone) {
+        if (flags & SKB_ALLOC_FCLONE) {
                 struct sk_buff *child = skb + 1;
                 atomic_t *fclone_ref = (atomic_t *) (child + 1);
 
@@ -232,6 +277,7 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
                 atomic_set(fclone_ref, 1);
 
                 child->fclone = SKB_FCLONE_UNAVAILABLE;
+                child->pfmemalloc = pfmemalloc;
         }
 out:
         return skb;
@@ -302,14 +348,7 @@ static DEFINE_PER_CPU(struct netdev_alloc_cache, netdev_alloc_cache);
 
 #define NETDEV_PAGECNT_BIAS (PAGE_SIZE / SMP_CACHE_BYTES)
 
-/**
- * netdev_alloc_frag - allocate a page fragment
- * @fragsz: fragment size
- *
- * Allocates a frag from a page for receive buffer.
- * Uses GFP_ATOMIC allocations.
- */
-void *netdev_alloc_frag(unsigned int fragsz)
+static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
 {
         struct netdev_alloc_cache *nc;
         void *data = NULL;
@@ -319,7 +358,7 @@ void *netdev_alloc_frag(unsigned int fragsz)
         nc = &__get_cpu_var(netdev_alloc_cache);
         if (unlikely(!nc->page)) {
 refill:
-                nc->page = alloc_page(GFP_ATOMIC | __GFP_COLD);
+                nc->page = alloc_page(gfp_mask);
                 if (unlikely(!nc->page))
                         goto end;
 recycle:
@@ -343,6 +382,18 @@ end:
         local_irq_restore(flags);
         return data;
 }
+
+/**
+ * netdev_alloc_frag - allocate a page fragment
+ * @fragsz: fragment size
+ *
+ * Allocates a frag from a page for receive buffer.
+ * Uses GFP_ATOMIC allocations.
+ */
+void *netdev_alloc_frag(unsigned int fragsz)
+{
+        return __netdev_alloc_frag(fragsz, GFP_ATOMIC | __GFP_COLD);
+}
 EXPORT_SYMBOL(netdev_alloc_frag);
 
 /**
@@ -366,7 +417,12 @@ struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
                               SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 
         if (fragsz <= PAGE_SIZE && !(gfp_mask & (__GFP_WAIT | GFP_DMA))) {
-                void *data = netdev_alloc_frag(fragsz);
+                void *data;
+
+                if (sk_memalloc_socks())
+                        gfp_mask |= __GFP_MEMALLOC;
+
+                data = __netdev_alloc_frag(fragsz, gfp_mask);
 
                 if (likely(data)) {
                         skb = build_skb(data, fragsz);
@@ -374,7 +430,8 @@ struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
                                 put_page(virt_to_head_page(data));
                 }
         } else {
-                skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask, 0, NUMA_NO_NODE);
+                skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask,
+                                  SKB_ALLOC_RX, NUMA_NO_NODE);
         }
         if (likely(skb)) {
                 skb_reserve(skb, NET_SKB_PAD);
@@ -656,6 +713,7 @@ static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
 #if IS_ENABLED(CONFIG_IP_VS)
         new->ipvs_property      = old->ipvs_property;
 #endif
+        new->pfmemalloc         = old->pfmemalloc;
         new->protocol           = old->protocol;
         new->mark               = old->mark;
         new->skb_iif            = old->skb_iif;
@@ -814,6 +872,9 @@ struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
                 n->fclone = SKB_FCLONE_CLONE;
                 atomic_inc(fclone_ref);
         } else {
+                if (skb_pfmemalloc(skb))
+                        gfp_mask |= __GFP_MEMALLOC;
+
                 n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
                 if (!n)
                         return NULL;
@@ -850,6 +911,13 @@ static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
         skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
 }
 
+static inline int skb_alloc_rx_flag(const struct sk_buff *skb)
+{
+        if (skb_pfmemalloc(skb))
+                return SKB_ALLOC_RX;
+        return 0;
+}
+
 /**
  *      skb_copy        -       create private copy of an sk_buff
  *      @skb: buffer to copy
@@ -871,7 +939,8 @@ struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
 {
         int headerlen = skb_headroom(skb);
         unsigned int size = skb_end_offset(skb) + skb->data_len;
-        struct sk_buff *n = alloc_skb(size, gfp_mask);
+        struct sk_buff *n = __alloc_skb(size, gfp_mask,
+                                        skb_alloc_rx_flag(skb), NUMA_NO_NODE);
 
         if (!n)
                 return NULL;
@@ -906,7 +975,8 @@ EXPORT_SYMBOL(skb_copy);
 struct sk_buff *__pskb_copy(struct sk_buff *skb, int headroom, gfp_t gfp_mask)
 {
         unsigned int size = skb_headlen(skb) + headroom;
-        struct sk_buff *n = alloc_skb(size, gfp_mask);
+        struct sk_buff *n = __alloc_skb(size, gfp_mask,
+                                        skb_alloc_rx_flag(skb), NUMA_NO_NODE);
 
         if (!n)
                 goto out;
@@ -979,8 +1049,10 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
 
         size = SKB_DATA_ALIGN(size);
 
-        data = kmalloc(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
-                       gfp_mask);
+        if (skb_pfmemalloc(skb))
+                gfp_mask |= __GFP_MEMALLOC;
+        data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
+                               gfp_mask, NUMA_NO_NODE, NULL);
         if (!data)
                 goto nodata;
         size = SKB_WITH_OVERHEAD(ksize(data));
@@ -1092,8 +1164,9 @@ struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
         /*
          *      Allocate the copy buffer
          */
-        struct sk_buff *n = alloc_skb(newheadroom + skb->len + newtailroom,
-                                      gfp_mask);
+        struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom,
+                                        gfp_mask, skb_alloc_rx_flag(skb),
+                                        NUMA_NO_NODE);
         int oldheadroom = skb_headroom(skb);
         int head_copy_len, head_copy_off;
         int off;
@@ -2775,8 +2848,9 @@ struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features)
                         skb_release_head_state(nskb);
                         __skb_push(nskb, doffset);
                 } else {
-                        nskb = alloc_skb(hsize + doffset + headroom,
-                                         GFP_ATOMIC);
+                        nskb = __alloc_skb(hsize + doffset + headroom,
+                                           GFP_ATOMIC, skb_alloc_rx_flag(skb),
+                                           NUMA_NO_NODE);
 
                         if (unlikely(!nskb))
                                 goto err;
diff --git a/net/core/sock.c b/net/core/sock.c
index 2676a88f533e..6b654b3ddfda 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -142,7 +142,7 @@
 static DEFINE_MUTEX(proto_list_mutex);
 static LIST_HEAD(proto_list);
 
-#ifdef CONFIG_CGROUP_MEM_RES_CTLR_KMEM
+#ifdef CONFIG_MEMCG_KMEM
 int mem_cgroup_sockets_init(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
 {
         struct proto *proto;
@@ -271,6 +271,61 @@ __u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX;
 int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512);
 EXPORT_SYMBOL(sysctl_optmem_max);
 
+struct static_key memalloc_socks = STATIC_KEY_INIT_FALSE;
+EXPORT_SYMBOL_GPL(memalloc_socks);
+
+/**
+ * sk_set_memalloc - sets %SOCK_MEMALLOC
+ * @sk: socket to set it on
+ *
+ * Set %SOCK_MEMALLOC on a socket for access to emergency reserves.
+ * It's the responsibility of the admin to adjust min_free_kbytes
+ * to meet the requirements
+ */
+void sk_set_memalloc(struct sock *sk)
+{
+        sock_set_flag(sk, SOCK_MEMALLOC);
+        sk->sk_allocation |= __GFP_MEMALLOC;
+        static_key_slow_inc(&memalloc_socks);
+}
+EXPORT_SYMBOL_GPL(sk_set_memalloc);
+
+void sk_clear_memalloc(struct sock *sk)
+{
+        sock_reset_flag(sk, SOCK_MEMALLOC);
+        sk->sk_allocation &= ~__GFP_MEMALLOC;
+        static_key_slow_dec(&memalloc_socks);
+
+        /*
+         * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward
+         * progress of swapping. However, if SOCK_MEMALLOC is cleared while
+         * it has rmem allocations there is a risk that the user of the
+         * socket cannot make forward progress due to exceeding the rmem
+         * limits. By rights, sk_clear_memalloc() should only be called
+         * on sockets being torn down but warn and reset the accounting if
+         * that assumption breaks.
+         */
+        if (WARN_ON(sk->sk_forward_alloc))
+                sk_mem_reclaim(sk);
+}
+EXPORT_SYMBOL_GPL(sk_clear_memalloc);
+
+int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
+{
+        int ret;
+        unsigned long pflags = current->flags;
+
+        /* these should have been dropped before queueing */
+        BUG_ON(!sock_flag(sk, SOCK_MEMALLOC));
+
+        current->flags |= PF_MEMALLOC;
+        ret = sk->sk_backlog_rcv(sk, skb);
+        tsk_restore_flags(current, pflags, PF_MEMALLOC);
+
+        return ret;
+}
+EXPORT_SYMBOL(__sk_backlog_rcv);
+
 #if defined(CONFIG_CGROUPS)
 #if !defined(CONFIG_NET_CLS_CGROUP)
 int net_cls_subsys_id = -1;
@@ -353,7 +408,7 @@ int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
         if (err)
                 return err;
 
-        if (!sk_rmem_schedule(sk, skb->truesize)) {
+        if (!sk_rmem_schedule(sk, skb, skb->truesize)) {
                 atomic_inc(&sk->sk_drops);
                 return -ENOBUFS;
         }