Added missing tegra files.HEADmaster

1 files changed, 1020 insertions, 0 deletions

diff --git a/drivers/video/tegra/nvmap/nvmap_handle.c b/drivers/video/tegra/nvmap/nvmap_handle.c
new file mode 100644
index 00000000000..539b7ce9801
--- /dev/null
+++ b/drivers/video/tegra/nvmap/nvmap_handle.c
@@ -0,0 +1,1020 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap_handle.c
+ *
+ * Handle allocation and freeing routines for nvmap
+ *
+ * Copyright (c) 2009-2012, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#define pr_fmt(fmt)     "%s: " fmt, __func__
+
+#include <linux/err.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/mm.h>
+#include <linux/rbtree.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/fs.h>
+
+#include <asm/cacheflush.h>
+#include <asm/outercache.h>
+#include <asm/pgtable.h>
+
+#include <mach/iovmm.h>
+#include <mach/nvmap.h>
+
+#include <linux/vmstat.h>
+#include <linux/swap.h>
+#include <linux/shrinker.h>
+#include <linux/moduleparam.h>
+
+#include "nvmap.h"
+#include "nvmap_mru.h"
+#include "nvmap_common.h"
+
+#define PRINT_CARVEOUT_CONVERSION 0
+#if PRINT_CARVEOUT_CONVERSION
+#define PR_INFO pr_info
+#else
+#define PR_INFO(...)
+#endif
+
+#define NVMAP_SECURE_HEAPS      (NVMAP_HEAP_CARVEOUT_IRAM | NVMAP_HEAP_IOVMM | \
+                                 NVMAP_HEAP_CARVEOUT_VPR)
+#ifdef CONFIG_NVMAP_HIGHMEM_ONLY
+#define GFP_NVMAP               (__GFP_HIGHMEM | __GFP_NOWARN)
+#else
+#define GFP_NVMAP               (GFP_KERNEL | __GFP_HIGHMEM | __GFP_NOWARN)
+#endif
+/* handles may be arbitrarily large (16+MiB), and any handle allocated from
+ * the kernel (i.e., not a carveout handle) includes its array of pages. to
+ * preserve kmalloc space, if the array of pages exceeds PAGELIST_VMALLOC_MIN,
+ * the array is allocated using vmalloc. */
+#define PAGELIST_VMALLOC_MIN    (PAGE_SIZE * 2)
+#define NVMAP_TEST_PAGE_POOL_SHRINKER 1
+static bool enable_pp = 1;
+static int pool_size[NVMAP_NUM_POOLS];
+
+static char *s_memtype_str[] = {
+        "uc",
+        "wc",
+        "iwb",
+        "wb",
+};
+
+static inline void nvmap_page_pool_lock(struct nvmap_page_pool *pool)
+{
+        mutex_lock(&pool->lock);
+}
+
+static inline void nvmap_page_pool_unlock(struct nvmap_page_pool *pool)
+{
+        mutex_unlock(&pool->lock);
+}
+
+static struct page *nvmap_page_pool_alloc_locked(struct nvmap_page_pool *pool)
+{
+        struct page *page = NULL;
+
+        if (pool->npages > 0)
+                page = pool->page_array[--pool->npages];
+        return page;
+}
+
+static struct page *nvmap_page_pool_alloc(struct nvmap_page_pool *pool)
+{
+        struct page *page = NULL;
+
+        if (pool) {
+                nvmap_page_pool_lock(pool);
+                page = nvmap_page_pool_alloc_locked(pool);
+                nvmap_page_pool_unlock(pool);
+        }
+        return page;
+}
+
+static bool nvmap_page_pool_release_locked(struct nvmap_page_pool *pool,
+                                            struct page *page)
+{
+        int ret = false;
+
+        if (enable_pp && pool->npages < pool->max_pages) {
+                pool->page_array[pool->npages++] = page;
+                ret = true;
+        }
+        return ret;
+}
+
+static bool nvmap_page_pool_release(struct nvmap_page_pool *pool,
+                                          struct page *page)
+{
+        int ret = false;
+
+        if (pool) {
+                nvmap_page_pool_lock(pool);
+                ret = nvmap_page_pool_release_locked(pool, page);
+                nvmap_page_pool_unlock(pool);
+        }
+        return ret;
+}
+
+static int nvmap_page_pool_get_available_count(struct nvmap_page_pool *pool)
+{
+        return pool->npages;
+}
+
+static int nvmap_page_pool_free(struct nvmap_page_pool *pool, int nr_free)
+{
+        int i = nr_free;
+        int idx = 0;
+        struct page *page;
+
+        if (!nr_free)
+                return nr_free;
+        nvmap_page_pool_lock(pool);
+        while (i) {
+                page = nvmap_page_pool_alloc_locked(pool);
+                if (!page)
+                        break;
+                pool->shrink_array[idx++] = page;
+                i--;
+        }
+
+        if (idx)
+                set_pages_array_wb(pool->shrink_array, idx);
+        while (idx--)
+                __free_page(pool->shrink_array[idx]);
+        nvmap_page_pool_unlock(pool);
+        return i;
+}
+
+static int nvmap_page_pool_get_unused_pages(void)
+{
+        unsigned int i;
+        int total = 0;
+        struct nvmap_share *share = nvmap_get_share_from_dev(nvmap_dev);
+
+        for (i = 0; i < NVMAP_NUM_POOLS; i++)
+                total += nvmap_page_pool_get_available_count(&share->pools[i]);
+
+        return total;
+}
+
+static void nvmap_page_pool_resize(struct nvmap_page_pool *pool, int size)
+{
+        int available_pages;
+        int pages_to_release = 0;
+        struct page **page_array = NULL;
+        struct page **shrink_array = NULL;
+
+        if (size == pool->max_pages)
+                return;
+repeat:
+        nvmap_page_pool_free(pool, pages_to_release);
+        nvmap_page_pool_lock(pool);
+        available_pages = nvmap_page_pool_get_available_count(pool);
+        if (available_pages > size) {
+                nvmap_page_pool_unlock(pool);
+                pages_to_release = available_pages - size;
+                goto repeat;
+        }
+
+        if (size == 0) {
+                vfree(pool->page_array);
+                vfree(pool->shrink_array);
+                pool->page_array = pool->shrink_array = NULL;
+                goto out;
+        }
+
+        page_array = vmalloc(sizeof(struct page *) * size);
+        shrink_array = vmalloc(sizeof(struct page *) * size);
+        if (!page_array || !shrink_array)
+                goto fail;
+
+        memcpy(page_array, pool->page_array,
+                pool->npages * sizeof(struct page *));
+        vfree(pool->page_array);
+        vfree(pool->shrink_array);
+        pool->page_array = page_array;
+        pool->shrink_array = shrink_array;
+out:
+        pr_debug("%s pool resized to %d from %d pages",
+                s_memtype_str[pool->flags], size, pool->max_pages);
+        pool->max_pages = size;
+        goto exit;
+fail:
+        vfree(page_array);
+        vfree(shrink_array);
+        pr_err("failed");
+exit:
+        nvmap_page_pool_unlock(pool);
+}
+
+static int nvmap_page_pool_shrink(struct shrinker *shrinker,
+                                  struct shrink_control *sc)
+{
+        unsigned int i;
+        unsigned int pool_offset;
+        struct nvmap_page_pool *pool;
+        int shrink_pages = sc->nr_to_scan;
+        static atomic_t start_pool = ATOMIC_INIT(-1);
+        struct nvmap_share *share = nvmap_get_share_from_dev(nvmap_dev);
+
+        if (!shrink_pages)
+                goto out;
+
+        pr_debug("sh_pages=%d", shrink_pages);
+
+        for (i = 0; i < NVMAP_NUM_POOLS && shrink_pages; i++) {
+                pool_offset = atomic_add_return(1, &start_pool) %
+                                NVMAP_NUM_POOLS;
+                pool = &share->pools[pool_offset];
+                shrink_pages = nvmap_page_pool_free(pool, shrink_pages);
+        }
+out:
+        return nvmap_page_pool_get_unused_pages();
+}
+
+static struct shrinker nvmap_page_pool_shrinker = {
+        .shrink = nvmap_page_pool_shrink,
+        .seeks = 1,
+};
+
+static void shrink_page_pools(int *total_pages, int *available_pages)
+{
+        struct shrink_control sc;
+
+        sc.gfp_mask = GFP_KERNEL;
+        sc.nr_to_scan = 0;
+        *total_pages = nvmap_page_pool_shrink(NULL, &sc);
+        sc.nr_to_scan = *total_pages * 2;
+        *available_pages = nvmap_page_pool_shrink(NULL, &sc);
+}
+
+#if NVMAP_TEST_PAGE_POOL_SHRINKER
+static bool shrink_pp;
+static int shrink_set(const char *arg, const struct kernel_param *kp)
+{
+        int cpu = smp_processor_id();
+        unsigned long long t1, t2;
+        int total_pages, available_pages;
+
+        param_set_bool(arg, kp);
+
+        if (shrink_pp) {
+                t1 = cpu_clock(cpu);
+                shrink_page_pools(&total_pages, &available_pages);
+                t2 = cpu_clock(cpu);
+                pr_info("shrink page pools: time=%lldns, "
+                        "total_pages_released=%d, free_pages_available=%d",
+                        t2-t1, total_pages, available_pages);
+        }
+        return 0;
+}
+
+static int shrink_get(char *buff, const struct kernel_param *kp)
+{
+        return param_get_bool(buff, kp);
+}
+
+static struct kernel_param_ops shrink_ops = {
+        .get = shrink_get,
+        .set = shrink_set,
+};
+
+module_param_cb(shrink_page_pools, &shrink_ops, &shrink_pp, 0644);
+#endif
+
+static int enable_pp_set(const char *arg, const struct kernel_param *kp)
+{
+        int total_pages, available_pages;
+
+        param_set_bool(arg, kp);
+
+        if (!enable_pp) {
+                shrink_page_pools(&total_pages, &available_pages);
+                pr_info("disabled page pools and released pages, "
+                        "total_pages_released=%d, free_pages_available=%d",
+                        total_pages, available_pages);
+        }
+        return 0;
+}
+
+static int enable_pp_get(char *buff, const struct kernel_param *kp)
+{
+        return param_get_int(buff, kp);
+}
+
+static struct kernel_param_ops enable_pp_ops = {
+        .get = enable_pp_get,
+        .set = enable_pp_set,
+};
+
+module_param_cb(enable_page_pools, &enable_pp_ops, &enable_pp, 0644);
+
+#define POOL_SIZE_SET(m, i) \
+static int pool_size_##m##_set(const char *arg, const struct kernel_param *kp) \
+{ \
+        struct nvmap_share *share = nvmap_get_share_from_dev(nvmap_dev); \
+        param_set_int(arg, kp); \
+        nvmap_page_pool_resize(&share->pools[i], pool_size[i]); \
+        return 0; \
+}
+
+#define POOL_SIZE_GET(m) \
+static int pool_size_##m##_get(char *buff, const struct kernel_param *kp) \
+{ \
+        return param_get_int(buff, kp); \
+}
+
+#define POOL_SIZE_OPS(m) \
+static struct kernel_param_ops pool_size_##m##_ops = { \
+        .get = pool_size_##m##_get, \
+        .set = pool_size_##m##_set, \
+};
+
+#define POOL_SIZE_MOUDLE_PARAM_CB(m, i) \
+module_param_cb(m##_pool_size, &pool_size_##m##_ops, &pool_size[i], 0644)
+
+POOL_SIZE_SET(uc, NVMAP_HANDLE_UNCACHEABLE);
+POOL_SIZE_GET(uc);
+POOL_SIZE_OPS(uc);
+POOL_SIZE_MOUDLE_PARAM_CB(uc, NVMAP_HANDLE_UNCACHEABLE);
+
+POOL_SIZE_SET(wc, NVMAP_HANDLE_WRITE_COMBINE);
+POOL_SIZE_GET(wc);
+POOL_SIZE_OPS(wc);
+POOL_SIZE_MOUDLE_PARAM_CB(wc, NVMAP_HANDLE_WRITE_COMBINE);
+
+POOL_SIZE_SET(iwb, NVMAP_HANDLE_INNER_CACHEABLE);
+POOL_SIZE_GET(iwb);
+POOL_SIZE_OPS(iwb);
+POOL_SIZE_MOUDLE_PARAM_CB(iwb, NVMAP_HANDLE_INNER_CACHEABLE);
+
+POOL_SIZE_SET(wb, NVMAP_HANDLE_CACHEABLE);
+POOL_SIZE_GET(wb);
+POOL_SIZE_OPS(wb);
+POOL_SIZE_MOUDLE_PARAM_CB(wb, NVMAP_HANDLE_CACHEABLE);
+
+int nvmap_page_pool_init(struct nvmap_page_pool *pool, int flags)
+{
+        struct page *page;
+        int i;
+        static int reg = 1;
+        struct sysinfo info;
+        typedef int (*set_pages_array) (struct page **pages, int addrinarray);
+        set_pages_array s_cpa[] = {
+                set_pages_array_uc,
+                set_pages_array_wc,
+                set_pages_array_iwb,
+                set_pages_array_wb
+        };
+
+        BUG_ON(flags >= NVMAP_NUM_POOLS);
+        memset(pool, 0x0, sizeof(*pool));
+        mutex_init(&pool->lock);
+        pool->flags = flags;
+
+        /* No default pool for cached memory. */
+        if (flags == NVMAP_HANDLE_CACHEABLE)
+                return 0;
+
+        si_meminfo(&info);
+        if (!pool_size[flags]) {
+                /* Use 3/8th of total ram for page pools.
+                 * 1/8th for uc, 1/8th for wc and 1/8th for iwb.
+                 */
+                pool->max_pages = info.totalram >> 3;
+        }
+        if (pool->max_pages <= 0 || pool->max_pages >= info.totalram)
+                pool->max_pages = NVMAP_DEFAULT_PAGE_POOL_SIZE;
+        pool_size[flags] = pool->max_pages;
+        pr_info("nvmap %s page pool size=%d pages",
+                s_memtype_str[flags], pool->max_pages);
+        pool->page_array = vmalloc(sizeof(void *) * pool->max_pages);
+        pool->shrink_array = vmalloc(sizeof(struct page *) * pool->max_pages);
+        if (!pool->page_array || !pool->shrink_array)
+                goto fail;
+
+        if (reg) {
+                reg = 0;
+                register_shrinker(&nvmap_page_pool_shrinker);
+        }
+
+        nvmap_page_pool_lock(pool);
+        for (i = 0; i < pool->max_pages; i++) {
+                page = alloc_page(GFP_NVMAP);
+                if (!page)
+                        goto do_cpa;
+                if (!nvmap_page_pool_release_locked(pool, page)) {
+                        __free_page(page);
+                        goto do_cpa;
+                }
+        }
+do_cpa:
+        (*s_cpa[flags])(pool->page_array, pool->npages);
+        nvmap_page_pool_unlock(pool);
+        return 0;
+fail:
+        pool->max_pages = 0;
+        vfree(pool->shrink_array);
+        vfree(pool->page_array);
+        return -ENOMEM;
+}
+
+static inline void *altalloc(size_t len)
+{
+        if (len >= PAGELIST_VMALLOC_MIN)
+                return vmalloc(len);
+        else
+                return kmalloc(len, GFP_KERNEL);
+}
+
+static inline void altfree(void *ptr, size_t len)
+{
+        if (!ptr)
+                return;
+
+        if (len >= PAGELIST_VMALLOC_MIN)
+                vfree(ptr);
+        else
+                kfree(ptr);
+}
+
+void _nvmap_handle_free(struct nvmap_handle *h)
+{
+        struct nvmap_share *share = nvmap_get_share_from_dev(h->dev);
+        unsigned int i, nr_page, page_index = 0;
+        struct nvmap_page_pool *pool = NULL;
+
+        if (nvmap_handle_remove(h->dev, h) != 0)
+                return;
+
+        if (!h->alloc)
+                goto out;
+
+        if (!h->heap_pgalloc) {
+                nvmap_usecount_inc(h);
+                nvmap_heap_free(h->carveout);
+                goto out;
+        }
+
+        nr_page = DIV_ROUND_UP(h->size, PAGE_SIZE);
+
+        BUG_ON(h->size & ~PAGE_MASK);
+        BUG_ON(!h->pgalloc.pages);
+
+        nvmap_mru_remove(share, h);
+
+        if (h->flags < NVMAP_NUM_POOLS)
+                pool = &share->pools[h->flags];
+
+        while (page_index < nr_page) {
+                if (!nvmap_page_pool_release(pool,
+                    h->pgalloc.pages[page_index]))
+                        break;
+                page_index++;
+        }
+
+        if (page_index == nr_page)
+                goto skip_attr_restore;
+
+        /* Restore page attributes. */
+        if (h->flags == NVMAP_HANDLE_WRITE_COMBINE ||
+            h->flags == NVMAP_HANDLE_UNCACHEABLE ||
+            h->flags == NVMAP_HANDLE_INNER_CACHEABLE)
+                set_pages_array_wb(&h->pgalloc.pages[page_index],
+                                nr_page - page_index);
+
+skip_attr_restore:
+        if (h->pgalloc.area)
+                tegra_iovmm_free_vm(h->pgalloc.area);
+
+        for (i = page_index; i < nr_page; i++)
+                __free_page(h->pgalloc.pages[i]);
+
+        altfree(h->pgalloc.pages, nr_page * sizeof(struct page *));
+
+out:
+        kfree(h);
+}
+
+static struct page *nvmap_alloc_pages_exact(gfp_t gfp, size_t size)
+{
+        struct page *page, *p, *e;
+        unsigned int order;
+
+        size = PAGE_ALIGN(size);
+        order = get_order(size);
+        page = alloc_pages(gfp, order);
+
+        if (!page)
+                return NULL;
+
+        split_page(page, order);
+        e = page + (1 << order);
+        for (p = page + (size >> PAGE_SHIFT); p < e; p++)
+                __free_page(p);
+
+        return page;
+}
+
+static int handle_page_alloc(struct nvmap_client *client,
+                             struct nvmap_handle *h, bool contiguous)
+{
+        size_t size = PAGE_ALIGN(h->size);
+        struct nvmap_share *share = nvmap_get_share_from_dev(h->dev);
+        unsigned int nr_page = size >> PAGE_SHIFT;
+        pgprot_t prot;
+        unsigned int i = 0, page_index = 0;
+        struct page **pages;
+        struct nvmap_page_pool *pool = NULL;
+
+        pages = altalloc(nr_page * sizeof(*pages));
+        if (!pages)
+                return -ENOMEM;
+
+        prot = nvmap_pgprot(h, pgprot_kernel);
+
+        h->pgalloc.area = NULL;
+        if (contiguous) {
+                struct page *page;
+                page = nvmap_alloc_pages_exact(GFP_NVMAP, size);
+                if (!page)
+                        goto fail;
+
+                for (i = 0; i < nr_page; i++)
+                        pages[i] = nth_page(page, i);
+
+        } else {
+                if (h->flags < NVMAP_NUM_POOLS)
+                        pool = &share->pools[h->flags];
+
+                for (i = 0; i < nr_page; i++) {
+                        /* Get pages from pool, if available. */
+                        pages[i] = nvmap_page_pool_alloc(pool);
+                        if (!pages[i])
+                                break;
+                        page_index++;
+                }
+
+                for (; i < nr_page; i++) {
+                        pages[i] = nvmap_alloc_pages_exact(GFP_NVMAP,
+                                PAGE_SIZE);
+                        if (!pages[i])
+                                goto fail;
+                }
+
+#ifndef CONFIG_NVMAP_RECLAIM_UNPINNED_VM
+                h->pgalloc.area = tegra_iovmm_create_vm(client->share->iovmm,
+                                        NULL, size, h->align, prot,
+                                        h->pgalloc.iovm_addr);
+                if (!h->pgalloc.area)
+                        goto fail;
+
+                h->pgalloc.dirty = true;
+#endif
+        }
+
+        if (nr_page == page_index)
+                goto skip_attr_change;
+
+        /* Update the pages mapping in kernel page table. */
+        if (h->flags == NVMAP_HANDLE_WRITE_COMBINE)
+                set_pages_array_wc(&pages[page_index],
+                                nr_page - page_index);
+        else if (h->flags == NVMAP_HANDLE_UNCACHEABLE)
+                set_pages_array_uc(&pages[page_index],
+                                nr_page - page_index);
+        else if (h->flags == NVMAP_HANDLE_INNER_CACHEABLE)
+                set_pages_array_iwb(&pages[page_index],
+                                nr_page - page_index);
+
+skip_attr_change:
+        h->size = size;
+        h->pgalloc.pages = pages;
+        h->pgalloc.contig = contiguous;
+        INIT_LIST_HEAD(&h->pgalloc.mru_list);
+        return 0;
+
+fail:
+        while (i--) {
+                set_pages_array_wb(&pages[i], 1);
+                __free_page(pages[i]);
+        }
+        altfree(pages, nr_page * sizeof(*pages));
+        wmb();
+        return -ENOMEM;
+}
+
+static void alloc_handle(struct nvmap_client *client,
+                         struct nvmap_handle *h, unsigned int type)
+{
+        BUG_ON(type & (type - 1));
+
+#ifdef CONFIG_NVMAP_CONVERT_CARVEOUT_TO_IOVMM
+#define __NVMAP_HEAP_CARVEOUT   (NVMAP_HEAP_CARVEOUT_IRAM | NVMAP_HEAP_CARVEOUT_VPR)
+#define __NVMAP_HEAP_IOVMM      (NVMAP_HEAP_IOVMM | NVMAP_HEAP_CARVEOUT_GENERIC)
+        if (type & NVMAP_HEAP_CARVEOUT_GENERIC) {
+#ifdef CONFIG_NVMAP_ALLOW_SYSMEM
+                if (h->size <= PAGE_SIZE) {
+                        PR_INFO("###CARVEOUT CONVERTED TO SYSMEM "
+                                "0x%x bytes %s(%d)###\n",
+                                h->size, current->comm, current->pid);
+                        goto sysheap;
+                }
+#endif
+                PR_INFO("###CARVEOUT CONVERTED TO IOVM "
+                        "0x%x bytes %s(%d)###\n",
+                        h->size, current->comm, current->pid);
+        }
+#else
+#define __NVMAP_HEAP_CARVEOUT   NVMAP_HEAP_CARVEOUT_MASK
+#define __NVMAP_HEAP_IOVMM      NVMAP_HEAP_IOVMM
+#endif
+
+        if (type & __NVMAP_HEAP_CARVEOUT) {
+                struct nvmap_heap_block *b;
+#ifdef CONFIG_NVMAP_CONVERT_CARVEOUT_TO_IOVMM
+                PR_INFO("###IRAM REQUEST RETAINED "
+                        "0x%x bytes %s(%d)###\n",
+                        h->size, current->comm, current->pid);
+#endif
+                /* Protect handle from relocation */
+                nvmap_usecount_inc(h);
+
+                b = nvmap_carveout_alloc(client, h, type);
+                if (b) {
+                        h->heap_pgalloc = false;
+                        h->alloc = true;
+                        nvmap_carveout_commit_add(client,
+                                nvmap_heap_to_arg(nvmap_block_to_heap(b)),
+                                h->size);
+                }
+                nvmap_usecount_dec(h);
+
+        } else if (type & __NVMAP_HEAP_IOVMM) {
+                size_t reserved = PAGE_ALIGN(h->size);
+                int commit = 0;
+                int ret;
+
+                /* increment the committed IOVM space prior to allocation
+                 * to avoid race conditions with other threads simultaneously
+                 * allocating. */
+                commit = atomic_add_return(reserved,
+                                            &client->iovm_commit);
+
+                if (commit < client->iovm_limit)
+                        ret = handle_page_alloc(client, h, false);
+                else
+                        ret = -ENOMEM;
+
+                if (!ret) {
+                        h->heap_pgalloc = true;
+                        h->alloc = true;
+                } else {
+                        atomic_sub(reserved, &client->iovm_commit);
+                }
+
+        } else if (type & NVMAP_HEAP_SYSMEM) {
+#if defined(CONFIG_NVMAP_CONVERT_CARVEOUT_TO_IOVMM) && \
+        defined(CONFIG_NVMAP_ALLOW_SYSMEM)
+sysheap:
+#endif
+                if (handle_page_alloc(client, h, true) == 0) {
+                        BUG_ON(!h->pgalloc.contig);
+                        h->heap_pgalloc = true;
+                        h->alloc = true;
+                }
+        }
+}
+
+/* small allocations will try to allocate from generic OS memory before
+ * any of the limited heaps, to increase the effective memory for graphics
+ * allocations, and to reduce fragmentation of the graphics heaps with
+ * sub-page splinters */
+static const unsigned int heap_policy_small[] = {
+        NVMAP_HEAP_CARVEOUT_VPR,
+        NVMAP_HEAP_CARVEOUT_IRAM,
+#ifdef CONFIG_NVMAP_ALLOW_SYSMEM
+        NVMAP_HEAP_SYSMEM,
+#endif
+        NVMAP_HEAP_CARVEOUT_MASK,
+        NVMAP_HEAP_IOVMM,
+        0,
+};
+
+static const unsigned int heap_policy_large[] = {
+        NVMAP_HEAP_CARVEOUT_VPR,
+        NVMAP_HEAP_CARVEOUT_IRAM,
+        NVMAP_HEAP_IOVMM,
+        NVMAP_HEAP_CARVEOUT_MASK,
+#ifdef CONFIG_NVMAP_ALLOW_SYSMEM
+        NVMAP_HEAP_SYSMEM,
+#endif
+        0,
+};
+
+/* Do not override single page policy if there is not much space to
+avoid invoking system oom killer. */
+#define NVMAP_SMALL_POLICY_SYSMEM_THRESHOLD 50000000
+
+int nvmap_alloc_handle_id(struct nvmap_client *client,
+                          unsigned long id, unsigned int heap_mask,
+                          size_t align, unsigned int flags)
+{
+        struct nvmap_handle *h = NULL;
+        const unsigned int *alloc_policy;
+        int nr_page;
+        int err = -ENOMEM;
+
+        h = nvmap_get_handle_id(client, id);
+
+        if (!h)
+                return -EINVAL;
+
+        if (h->alloc)
+                goto out;
+
+        h->userflags = flags;
+        nr_page = ((h->size + PAGE_SIZE - 1) >> PAGE_SHIFT);
+        h->secure = !!(flags & NVMAP_HANDLE_SECURE);
+        h->flags = (flags & NVMAP_HANDLE_CACHE_FLAG);
+        h->align = max_t(size_t, align, L1_CACHE_BYTES);
+
+#ifndef CONFIG_TEGRA_IOVMM
+        if (heap_mask & NVMAP_HEAP_IOVMM) {
+                heap_mask &= NVMAP_HEAP_IOVMM;
+                heap_mask |= NVMAP_HEAP_CARVEOUT_GENERIC;
+        }
+#endif
+#ifndef CONFIG_NVMAP_CONVERT_CARVEOUT_TO_IOVMM
+#ifdef CONFIG_NVMAP_ALLOW_SYSMEM
+        /* Allow single pages allocations in system memory to save
+         * carveout space and avoid extra iovm mappings */
+        if (nr_page == 1) {
+                if (heap_mask & NVMAP_HEAP_IOVMM)
+                        heap_mask |= NVMAP_HEAP_SYSMEM;
+                else if (heap_mask & NVMAP_HEAP_CARVEOUT_GENERIC) {
+                        /* Calculate size of free physical pages
+                         * managed by kernel */
+                        unsigned long freeMem =
+                                (global_page_state(NR_FREE_PAGES) +
+                                global_page_state(NR_FILE_PAGES) -
+                                total_swapcache_pages) << PAGE_SHIFT;
+
+                        if (freeMem > NVMAP_SMALL_POLICY_SYSMEM_THRESHOLD)
+                                heap_mask |= NVMAP_HEAP_SYSMEM;
+                }
+        }
+#endif
+
+        /* This restriction is deprecated as alignments greater than
+           PAGE_SIZE are now correctly handled, but it is retained for
+           AP20 compatibility. */
+        if (h->align > PAGE_SIZE)
+                heap_mask &= NVMAP_HEAP_CARVEOUT_MASK;
+#endif
+        /* secure allocations can only be served from secure heaps */
+        if (h->secure)
+                heap_mask &= NVMAP_SECURE_HEAPS;
+
+        if (!heap_mask) {
+                err = -EINVAL;
+                goto out;
+        }
+
+        alloc_policy = (nr_page == 1) ? heap_policy_small : heap_policy_large;
+
+        while (!h->alloc && *alloc_policy) {
+                unsigned int heap_type;
+
+                heap_type = *alloc_policy++;
+                heap_type &= heap_mask;
+
+                if (!heap_type)
+                        continue;
+
+                heap_mask &= ~heap_type;
+
+                while (heap_type && !h->alloc) {
+                        unsigned int heap;
+
+                        /* iterate possible heaps MSB-to-LSB, since higher-
+                         * priority carveouts will have higher usage masks */
+                        heap = 1 << __fls(heap_type);
+                        alloc_handle(client, h, heap);
+                        heap_type &= ~heap;
+                }
+        }
+
+out:
+        err = (h->alloc) ? 0 : err;
+        nvmap_handle_put(h);
+        return err;
+}
+
+void nvmap_free_handle_id(struct nvmap_client *client, unsigned long id)
+{
+        struct nvmap_handle_ref *ref;
+        struct nvmap_handle *h;
+        int pins;
+
+        nvmap_ref_lock(client);
+
+        ref = _nvmap_validate_id_locked(client, id);
+        if (!ref) {
+                nvmap_ref_unlock(client);
+                return;
+        }
+
+        BUG_ON(!ref->handle);
+        h = ref->handle;
+
+        if (atomic_dec_return(&ref->dupes)) {
+                nvmap_ref_unlock(client);
+                goto out;
+        }
+
+        smp_rmb();
+        pins = atomic_read(&ref->pin);
+        rb_erase(&ref->node, &client->handle_refs);
+
+        if (h->alloc && h->heap_pgalloc && !h->pgalloc.contig)
+                atomic_sub(h->size, &client->iovm_commit);
+
+        if (h->alloc && !h->heap_pgalloc) {
+                mutex_lock(&h->lock);
+                nvmap_carveout_commit_subtract(client,
+                        nvmap_heap_to_arg(nvmap_block_to_heap(h->carveout)),
+                        h->size);
+                mutex_unlock(&h->lock);
+        }
+
+        nvmap_ref_unlock(client);
+
+        if (pins)
+                nvmap_err(client, "%s freeing pinned handle %p\n",
+                          current->group_leader->comm, h);
+
+        while (pins--)
+                nvmap_unpin_handles(client, &ref->handle, 1);
+
+        if (h->owner == client)
+                h->owner = NULL;
+
+        kfree(ref);
+
+out:
+        BUG_ON(!atomic_read(&h->ref));
+        nvmap_handle_put(h);
+}
+
+static void add_handle_ref(struct nvmap_client *client,
+                           struct nvmap_handle_ref *ref)
+{
+        struct rb_node **p, *parent = NULL;
+
+        nvmap_ref_lock(client);
+        p = &client->handle_refs.rb_node;
+        while (*p) {
+                struct nvmap_handle_ref *node;
+                parent = *p;
+                node = rb_entry(parent, struct nvmap_handle_ref, node);
+                if (ref->handle > node->handle)
+                        p = &parent->rb_right;
+                else
+                        p = &parent->rb_left;
+        }
+        rb_link_node(&ref->node, parent, p);
+        rb_insert_color(&ref->node, &client->handle_refs);
+        nvmap_ref_unlock(client);
+}
+
+struct nvmap_handle_ref *nvmap_create_handle(struct nvmap_client *client,
+                                             size_t size)
+{
+        struct nvmap_handle *h;
+        struct nvmap_handle_ref *ref = NULL;
+
+        if (!client)
+                return ERR_PTR(-EINVAL);
+
+        if (!size)
+                return ERR_PTR(-EINVAL);
+
+        h = kzalloc(sizeof(*h), GFP_KERNEL);
+        if (!h)
+                return ERR_PTR(-ENOMEM);
+
+        ref = kzalloc(sizeof(*ref), GFP_KERNEL);
+        if (!ref) {
+                kfree(h);
+                return ERR_PTR(-ENOMEM);
+        }
+
+        atomic_set(&h->ref, 1);
+        atomic_set(&h->pin, 0);
+        h->owner = client;
+        h->dev = client->dev;
+        BUG_ON(!h->owner);
+        h->size = h->orig_size = size;
+        h->flags = NVMAP_HANDLE_WRITE_COMBINE;
+        mutex_init(&h->lock);
+
+        nvmap_handle_add(client->dev, h);
+
+        atomic_set(&ref->dupes, 1);
+        ref->handle = h;
+        atomic_set(&ref->pin, 0);
+        add_handle_ref(client, ref);
+        return ref;
+}
+
+struct nvmap_handle_ref *nvmap_duplicate_handle_id(struct nvmap_client *client,
+                                                   unsigned long id)
+{
+        struct nvmap_handle_ref *ref = NULL;
+        struct nvmap_handle *h = NULL;
+
+        BUG_ON(!client || client->dev != nvmap_dev);
+        /* on success, the reference count for the handle should be
+         * incremented, so the success paths will not call nvmap_handle_put */
+        h = nvmap_validate_get(client, id);
+
+        if (!h) {
+                nvmap_debug(client, "%s duplicate handle failed\n",
+                            current->group_leader->comm);
+                return ERR_PTR(-EPERM);
+        }
+
+        if (!h->alloc) {
+                nvmap_err(client, "%s duplicating unallocated handle\n",
+                          current->group_leader->comm);
+                nvmap_handle_put(h);
+                return ERR_PTR(-EINVAL);
+        }
+
+        nvmap_ref_lock(client);
+        ref = _nvmap_validate_id_locked(client, (unsigned long)h);
+
+        if (ref) {
+                /* handle already duplicated in client; just increment
+                 * the reference count rather than re-duplicating it */
+                atomic_inc(&ref->dupes);
+                nvmap_ref_unlock(client);
+                return ref;
+        }
+
+        nvmap_ref_unlock(client);
+
+        /* verify that adding this handle to the process' access list
+         * won't exceed the IOVM limit */
+        if (h->heap_pgalloc && !h->pgalloc.contig) {
+                int oc;
+                oc = atomic_add_return(h->size, &client->iovm_commit);
+                if (oc > client->iovm_limit && !client->super) {
+                        atomic_sub(h->size, &client->iovm_commit);
+                        nvmap_handle_put(h);
+                        nvmap_err(client, "duplicating %p in %s over-commits"
+                                  " IOVMM space\n", (void *)id,
+                                  current->group_leader->comm);
+                        return ERR_PTR(-ENOMEM);
+                }
+        }
+
+        ref = kzalloc(sizeof(*ref), GFP_KERNEL);
+        if (!ref) {
+                nvmap_handle_put(h);
+                return ERR_PTR(-ENOMEM);
+        }
+
+        if (!h->heap_pgalloc) {
+                mutex_lock(&h->lock);
+                nvmap_carveout_commit_add(client,
+                        nvmap_heap_to_arg(nvmap_block_to_heap(h->carveout)),
+                        h->size);
+                mutex_unlock(&h->lock);
+        }
+
+        atomic_set(&ref->dupes, 1);
+        ref->handle = h;
+        atomic_set(&ref->pin, 0);
+        add_handle_ref(client, ref);
+        return ref;
+}