1 files changed, 871 insertions, 0 deletions

diff --git a/drivers/video/tegra/nvmap/nvmap.c b/drivers/video/tegra/nvmap/nvmap.c
new file mode 100644
index 00000000000..b4b6241618d
--- /dev/null
+++ b/drivers/video/tegra/nvmap/nvmap.c
@@ -0,0 +1,871 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap.c
+ *
+ * Memory manager for Tegra GPU
+ *
+ * Copyright (c) 2009-2011, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#include <linux/err.h>
+#include <linux/highmem.h>
+#include <linux/io.h>
+#include <linux/rbtree.h>
+#include <linux/vmalloc.h>
+#include <linux/wait.h>
+#include <linux/slab.h>
+
+#include <asm/pgtable.h>
+#include <asm/tlbflush.h>
+
+#include <mach/iovmm.h>
+#include <mach/nvmap.h>
+
+#include "nvmap.h"
+#include "nvmap_mru.h"
+
+/* private nvmap_handle flag for pinning duplicate detection */
+#define NVMAP_HANDLE_VISITED (0x1ul << 31)
+
+/* map the backing pages for a heap_pgalloc handle into its IOVMM area */
+static void map_iovmm_area(struct nvmap_handle *h)
+{
+        tegra_iovmm_addr_t va;
+        unsigned long i;
+
+        BUG_ON(!h->heap_pgalloc || !h->pgalloc.area);
+        BUG_ON(h->size & ~PAGE_MASK);
+        WARN_ON(!h->pgalloc.dirty);
+
+        for (va = h->pgalloc.area->iovm_start, i = 0;
+             va < (h->pgalloc.area->iovm_start + h->size);
+             i++, va += PAGE_SIZE) {
+                BUG_ON(!pfn_valid(page_to_pfn(h->pgalloc.pages[i])));
+                tegra_iovmm_vm_insert_pfn(h->pgalloc.area, va,
+                                          page_to_pfn(h->pgalloc.pages[i]));
+        }
+        h->pgalloc.dirty = false;
+}
+
+/* must be called inside nvmap_pin_lock, to ensure that an entire stream
+ * of pins will complete without racing with a second stream. handle should
+ * have nvmap_handle_get (or nvmap_validate_get) called before calling
+ * this function. */
+static int pin_locked(struct nvmap_client *client, struct nvmap_handle *h)
+{
+        struct tegra_iovmm_area *area;
+        BUG_ON(!h->alloc);
+
+        nvmap_mru_lock(client->share);
+        if (atomic_inc_return(&h->pin) == 1) {
+                if (h->heap_pgalloc && !h->pgalloc.contig) {
+                        area = nvmap_handle_iovmm_locked(client, h);
+                        if (!area) {
+                                /* no race here, inside the pin mutex */
+                                atomic_dec(&h->pin);
+                                nvmap_mru_unlock(client->share);
+                                return -ENOMEM;
+                        }
+                        if (area != h->pgalloc.area)
+                                h->pgalloc.dirty = true;
+                        h->pgalloc.area = area;
+                }
+        }
+        nvmap_mru_unlock(client->share);
+        return 0;
+}
+
+/* doesn't need to be called inside nvmap_pin_lock, since this will only
+ * expand the available VM area */
+static int handle_unpin(struct nvmap_client *client,
+                struct nvmap_handle *h, int free_vm)
+{
+        int ret = 0;
+        nvmap_mru_lock(client->share);
+
+        if (atomic_read(&h->pin) == 0) {
+                nvmap_err(client, "%s unpinning unpinned handle %p\n",
+                          current->group_leader->comm, h);
+                nvmap_mru_unlock(client->share);
+                return 0;
+        }
+
+        BUG_ON(!h->alloc);
+
+        if (!atomic_dec_return(&h->pin)) {
+                if (h->heap_pgalloc && h->pgalloc.area) {
+                        /* if a secure handle is clean (i.e., mapped into
+                         * IOVMM, it needs to be zapped on unpin. */
+                        if (h->secure && !h->pgalloc.dirty) {
+                                tegra_iovmm_zap_vm(h->pgalloc.area);
+                                h->pgalloc.dirty = true;
+                        }
+                        if (free_vm) {
+                                tegra_iovmm_free_vm(h->pgalloc.area);
+                                h->pgalloc.area = NULL;
+                        } else
+                                nvmap_mru_insert_locked(client->share, h);
+                        ret = 1;
+                }
+        }
+
+        nvmap_mru_unlock(client->share);
+        nvmap_handle_put(h);
+        return ret;
+}
+
+static int pin_array_locked(struct nvmap_client *client,
+                struct nvmap_handle **h, int count)
+{
+        int pinned;
+        int i;
+        int err = 0;
+
+        for (pinned = 0; pinned < count; pinned++) {
+                err = pin_locked(client, h[pinned]);
+                if (err)
+                        break;
+        }
+
+        if (err) {
+                /* unpin pinned handles */
+                for (i = 0; i < pinned; i++) {
+                        /* inc ref counter, because
+                         * handle_unpin decrements it */
+                        nvmap_handle_get(h[i]);
+                        /* unpin handles and free vm */
+                        handle_unpin(client, h[i], true);
+                }
+        }
+
+        if (err && tegra_iovmm_get_max_free(client->share->iovmm) >=
+                                                        client->iovm_limit) {
+                /* First attempt to pin in empty iovmm
+                 * may still fail because of fragmentation caused by
+                 * placing handles in MRU areas. After such failure
+                 * all MRU gets cleaned and iovm space is freed.
+                 *
+                 * We have to do pinning again here since there might be is
+                 * no more incoming pin_wait wakeup calls from unpin
+                 * operations */
+                for (pinned = 0; pinned < count; pinned++) {
+                        err = pin_locked(client, h[pinned]);
+                        if (err)
+                                break;
+                }
+                if (err) {
+                        pr_err("Pinning in empty iovmm failed!!!\n");
+                        BUG_ON(1);
+                }
+        }
+        return err;
+}
+
+static int wait_pin_array_locked(struct nvmap_client *client,
+                struct nvmap_handle **h, int count)
+{
+        int ret = 0;
+
+        ret = pin_array_locked(client, h, count);
+
+        if (ret) {
+                ret = wait_event_interruptible(client->share->pin_wait,
+                                !pin_array_locked(client, h, count));
+        }
+        return ret ? -EINTR : 0;
+}
+
+static int handle_unpin_noref(struct nvmap_client *client, unsigned long id)
+{
+        struct nvmap_handle *h;
+        int w;
+
+        h = nvmap_validate_get(client, id);
+        if (unlikely(!h)) {
+                nvmap_err(client, "%s attempting to unpin invalid handle %p\n",
+                          current->group_leader->comm, (void *)id);
+                return 0;
+        }
+
+        nvmap_err(client, "%s unpinning unreferenced handle %p\n",
+                  current->group_leader->comm, h);
+        WARN_ON(1);
+
+        w = handle_unpin(client, h, false);
+        nvmap_handle_put(h);
+        return w;
+}
+
+void nvmap_unpin_ids(struct nvmap_client *client,
+                     unsigned int nr, const unsigned long *ids)
+{
+        unsigned int i;
+        int do_wake = 0;
+
+        for (i = 0; i < nr; i++) {
+                struct nvmap_handle_ref *ref;
+
+                if (!ids[i])
+                        continue;
+
+                nvmap_ref_lock(client);
+                ref = _nvmap_validate_id_locked(client, ids[i]);
+                if (ref) {
+                        struct nvmap_handle *h = ref->handle;
+                        int e = atomic_add_unless(&ref->pin, -1, 0);
+
+                        nvmap_ref_unlock(client);
+
+                        if (!e) {
+                                nvmap_err(client, "%s unpinning unpinned "
+                                          "handle %08lx\n",
+                                          current->group_leader->comm, ids[i]);
+                        } else {
+                                do_wake |= handle_unpin(client, h, false);
+                        }
+                } else {
+                        nvmap_ref_unlock(client);
+                        if (client->super)
+                                do_wake |= handle_unpin_noref(client, ids[i]);
+                        else
+                                nvmap_err(client, "%s unpinning invalid "
+                                          "handle %08lx\n",
+                                          current->group_leader->comm, ids[i]);
+                }
+        }
+
+        if (do_wake)
+                wake_up(&client->share->pin_wait);
+}
+
+/* pins a list of handle_ref objects; same conditions apply as to
+ * _nvmap_handle_pin, but also bumps the pin count of each handle_ref. */
+int nvmap_pin_ids(struct nvmap_client *client,
+                  unsigned int nr, const unsigned long *ids)
+{
+        int ret = 0;
+        unsigned int i;
+        struct nvmap_handle **h = (struct nvmap_handle **)ids;
+        struct nvmap_handle_ref *ref;
+
+        /* to optimize for the common case (client provided valid handle
+         * references and the pin succeeds), increment the handle_ref pin
+         * count during validation. in error cases, the tree will need to
+         * be re-walked, since the handle_ref is discarded so that an
+         * allocation isn't required. if a handle_ref is not found,
+         * locally validate that the caller has permission to pin the handle;
+         * handle_refs are not created in this case, so it is possible that
+         * if the caller crashes after pinning a global handle, the handle
+         * will be permanently leaked. */
+        nvmap_ref_lock(client);
+        for (i = 0; i < nr && !ret; i++) {
+                ref = _nvmap_validate_id_locked(client, ids[i]);
+                if (ref) {
+                        atomic_inc(&ref->pin);
+                        nvmap_handle_get(h[i]);
+                } else {
+                        struct nvmap_handle *verify;
+                        nvmap_ref_unlock(client);
+                        verify = nvmap_validate_get(client, ids[i]);
+                        if (verify)
+                                nvmap_warn(client, "%s pinning unreferenced "
+                                           "handle %p\n",
+                                           current->group_leader->comm, h[i]);
+                        else
+                                ret = -EPERM;
+                        nvmap_ref_lock(client);
+                }
+        }
+        nvmap_ref_unlock(client);
+
+        nr = i;
+
+        if (ret)
+                goto out;
+
+        ret = mutex_lock_interruptible(&client->share->pin_lock);
+        if (WARN_ON(ret))
+                goto out;
+
+        ret = wait_pin_array_locked(client, h, nr);
+
+        mutex_unlock(&client->share->pin_lock);
+
+        if (ret) {
+                ret = -EINTR;
+        } else {
+                for (i = 0; i < nr; i++) {
+                        if (h[i]->heap_pgalloc && h[i]->pgalloc.dirty)
+                                map_iovmm_area(h[i]);
+                }
+        }
+
+out:
+        if (ret) {
+                nvmap_ref_lock(client);
+                for (i = 0; i < nr; i++) {
+                        ref = _nvmap_validate_id_locked(client, ids[i]);
+                        if (!ref) {
+                                nvmap_warn(client, "%s freed handle %p "
+                                           "during pinning\n",
+                                           current->group_leader->comm,
+                                           (void *)ids[i]);
+                                continue;
+                        }
+                        atomic_dec(&ref->pin);
+                }
+                nvmap_ref_unlock(client);
+
+                for (i = 0; i < nr; i++)
+                        nvmap_handle_put(h[i]);
+        }
+
+        return ret;
+}
+
+static phys_addr_t handle_phys(struct nvmap_handle *h)
+{
+        phys_addr_t addr;
+
+        if (h->heap_pgalloc && h->pgalloc.contig) {
+                addr = page_to_phys(h->pgalloc.pages[0]);
+        } else if (h->heap_pgalloc) {
+                BUG_ON(!h->pgalloc.area);
+                addr = h->pgalloc.area->iovm_start;
+        } else {
+                addr = h->carveout->base;
+        }
+
+        return addr;
+}
+
+/* stores the physical address (+offset) of each handle relocation entry
+ * into its output location. see nvmap_pin_array for more details.
+ *
+ * each entry in arr (i.e., each relocation request) specifies two handles:
+ * the handle to pin (pin), and the handle where the address of pin should be
+ * written (patch). in pseudocode, this loop basically looks like:
+ *
+ * for (i = 0; i < nr; i++) {
+ *     (pin, pin_offset, patch, patch_offset) = arr[i];
+ *     patch[patch_offset] = address_of(pin) + pin_offset;
+ * }
+ */
+static int nvmap_reloc_pin_array(struct nvmap_client *client,
+                                 const struct nvmap_pinarray_elem *arr,
+                                 int nr, struct nvmap_handle *gather)
+{
+        struct nvmap_handle *last_patch = NULL;
+        unsigned int last_pfn = 0;
+        pte_t **pte;
+        void *addr;
+        int i;
+
+        pte = nvmap_alloc_pte(client->dev, &addr);
+        if (IS_ERR(pte))
+                return PTR_ERR(pte);
+
+        for (i = 0; i < nr; i++) {
+                struct nvmap_handle *patch;
+                struct nvmap_handle *pin;
+                phys_addr_t reloc_addr;
+                phys_addr_t phys;
+                unsigned int pfn;
+
+                /* all of the handles are validated and get'ted prior to
+                 * calling this function, so casting is safe here */
+                pin = (struct nvmap_handle *)arr[i].pin_mem;
+
+                if (arr[i].patch_mem == (unsigned long)last_patch) {
+                        patch = last_patch;
+                } else if (arr[i].patch_mem == (unsigned long)gather) {
+                        patch = gather;
+                } else {
+                        if (last_patch)
+                                nvmap_handle_put(last_patch);
+
+                        patch = nvmap_get_handle_id(client, arr[i].patch_mem);
+                        if (!patch) {
+                                nvmap_free_pte(client->dev, pte);
+                                return -EPERM;
+                        }
+                        last_patch = patch;
+                }
+
+                if (patch->heap_pgalloc) {
+                        unsigned int page = arr[i].patch_offset >> PAGE_SHIFT;
+                        phys = page_to_phys(patch->pgalloc.pages[page]);
+                        phys += (arr[i].patch_offset & ~PAGE_MASK);
+                } else {
+                        phys = patch->carveout->base + arr[i].patch_offset;
+                }
+
+                pfn = __phys_to_pfn(phys);
+                if (pfn != last_pfn) {
+                        pgprot_t prot = nvmap_pgprot(patch, pgprot_kernel);
+                        phys_addr_t kaddr = (phys_addr_t)addr;
+                        set_pte_at(&init_mm, kaddr, *pte, pfn_pte(pfn, prot));
+                        flush_tlb_kernel_page(kaddr);
+                        last_pfn = pfn;
+                }
+
+                reloc_addr = handle_phys(pin) + arr[i].pin_offset;
+                reloc_addr >>= arr[i].reloc_shift;
+                __raw_writel(reloc_addr, addr + (phys & ~PAGE_MASK));
+        }
+
+        nvmap_free_pte(client->dev, pte);
+
+        if (last_patch)
+                nvmap_handle_put(last_patch);
+
+        wmb();
+
+        return 0;
+}
+
+static int nvmap_validate_get_pin_array(struct nvmap_client *client,
+                                        const struct nvmap_pinarray_elem *arr,
+                                        int nr, struct nvmap_handle **h)
+{
+        int i;
+        int ret = 0;
+        int count = 0;
+
+        nvmap_ref_lock(client);
+
+        for (i = 0; i < nr; i++) {
+                struct nvmap_handle_ref *ref;
+
+                if (need_resched()) {
+                        nvmap_ref_unlock(client);
+                        schedule();
+                        nvmap_ref_lock(client);
+                }
+
+                ref = _nvmap_validate_id_locked(client, arr[i].pin_mem);
+
+                if (!ref)
+                        nvmap_warn(client, "falied to validate id\n");
+                else if (!ref->handle)
+                        nvmap_warn(client, "id had no associated handle\n");
+                else if (!ref->handle->alloc)
+                        nvmap_warn(client, "handle had no allocation\n");
+
+                if (!ref || !ref->handle || !ref->handle->alloc) {
+                        ret = -EPERM;
+                        break;
+                }
+
+                /* a handle may be referenced multiple times in arr, but
+                 * it will only be pinned once; this ensures that the
+                 * minimum number of sync-queue slots in the host driver
+                 * are dedicated to storing unpin lists, which allows
+                 * for greater parallelism between the CPU and graphics
+                 * processor */
+                if (ref->handle->flags & NVMAP_HANDLE_VISITED)
+                        continue;
+
+                ref->handle->flags |= NVMAP_HANDLE_VISITED;
+
+                h[count] = nvmap_handle_get(ref->handle);
+                BUG_ON(!h[count]);
+                count++;
+        }
+
+        nvmap_ref_unlock(client);
+
+        if (ret) {
+                for (i = 0; i < count; i++) {
+                        h[i]->flags &= ~NVMAP_HANDLE_VISITED;
+                        nvmap_handle_put(h[i]);
+                }
+        }
+
+        return ret ?: count;
+}
+
+/* a typical mechanism host1x clients use for using the Tegra graphics
+ * processor is to build a command buffer which contains relocatable
+ * memory handle commands, and rely on the kernel to convert these in-place
+ * to addresses which are understood by the GPU hardware.
+ *
+ * this is implemented by having clients provide a sideband array
+ * of relocatable handles (+ offsets) and the location in the command
+ * buffer handle to patch with the GPU address when the client submits
+ * its command buffer to the host1x driver.
+ *
+ * the host driver also uses this relocation mechanism internally to
+ * relocate the client's (unpinned) command buffers into host-addressable
+ * memory.
+ *
+ * @client: nvmap_client which should be used for validation; should be
+ *          owned by the process which is submitting command buffers
+ * @gather: special handle for relocated command buffer outputs used
+ *          internally by the host driver. if this handle is encountered
+ *          as an output handle in the relocation array, it is assumed
+ *          to be a known-good output and is not validated.
+ * @arr:    array of ((relocatable handle, offset), (output handle, offset))
+ *          tuples.
+ * @nr:     number of entries in arr
+ * @unique_arr: list of nvmap_handle objects which were pinned by
+ *              nvmap_pin_array. must be unpinned by the caller after the
+ *              command buffers referenced in gather have completed.
+ */
+int nvmap_pin_array(struct nvmap_client *client, struct nvmap_handle *gather,
+                    const struct nvmap_pinarray_elem *arr, int nr,
+                    struct nvmap_handle **unique_arr)
+{
+        int count = 0;
+        int ret = 0;
+        int i;
+
+        if (mutex_lock_interruptible(&client->share->pin_lock)) {
+                nvmap_warn(client, "%s interrupted when acquiring pin lock\n",
+                           current->group_leader->comm);
+                return -EINTR;
+        }
+
+        count = nvmap_validate_get_pin_array(client, arr, nr, unique_arr);
+        if (count < 0) {
+                mutex_unlock(&client->share->pin_lock);
+                nvmap_warn(client, "failed to validate pin array\n");
+                return count;
+        }
+
+        for (i = 0; i < count; i++)
+                unique_arr[i]->flags &= ~NVMAP_HANDLE_VISITED;
+
+        ret = wait_pin_array_locked(client, unique_arr, count);
+
+        mutex_unlock(&client->share->pin_lock);
+
+        if (!ret)
+                ret = nvmap_reloc_pin_array(client, arr, nr, gather);
+
+        if (WARN_ON(ret)) {
+                for (i = 0; i < count; i++)
+                        nvmap_handle_put(unique_arr[i]);
+                return ret;
+        } else {
+                for (i = 0; i < count; i++) {
+                        if (unique_arr[i]->heap_pgalloc &&
+                            unique_arr[i]->pgalloc.dirty)
+                                map_iovmm_area(unique_arr[i]);
+                }
+        }
+
+        return count;
+}
+
+phys_addr_t nvmap_pin(struct nvmap_client *client,
+                        struct nvmap_handle_ref *ref)
+{
+        struct nvmap_handle *h;
+        phys_addr_t phys;
+        int ret = 0;
+
+        h = nvmap_handle_get(ref->handle);
+        if (WARN_ON(!h))
+                return -EINVAL;
+
+        atomic_inc(&ref->pin);
+
+        if (WARN_ON(mutex_lock_interruptible(&client->share->pin_lock))) {
+                ret = -EINTR;
+        } else {
+                ret = wait_pin_array_locked(client, &h, 1);
+                mutex_unlock(&client->share->pin_lock);
+        }
+
+        if (ret) {
+                atomic_dec(&ref->pin);
+                nvmap_handle_put(h);
+        } else {
+                if (h->heap_pgalloc && h->pgalloc.dirty)
+                        map_iovmm_area(h);
+                phys = handle_phys(h);
+        }
+
+        return ret ?: phys;
+}
+
+phys_addr_t nvmap_handle_address(struct nvmap_client *c, unsigned long id)
+{
+        struct nvmap_handle *h;
+        phys_addr_t phys;
+
+        h = nvmap_get_handle_id(c, id);
+        if (!h)
+                return -EPERM;
+        mutex_lock(&h->lock);
+        phys = handle_phys(h);
+        mutex_unlock(&h->lock);
+        nvmap_handle_put(h);
+
+        return phys;
+}
+
+void nvmap_unpin(struct nvmap_client *client, struct nvmap_handle_ref *ref)
+{
+        if (!ref)
+                return;
+
+        atomic_dec(&ref->pin);
+        if (handle_unpin(client, ref->handle, false))
+                wake_up(&client->share->pin_wait);
+}
+
+void nvmap_unpin_handles(struct nvmap_client *client,
+                         struct nvmap_handle **h, int nr)
+{
+        int i;
+        int do_wake = 0;
+
+        for (i = 0; i < nr; i++) {
+                if (WARN_ON(!h[i]))
+                        continue;
+                do_wake |= handle_unpin(client, h[i], false);
+        }
+
+        if (do_wake)
+                wake_up(&client->share->pin_wait);
+}
+
+void *nvmap_mmap(struct nvmap_handle_ref *ref)
+{
+        struct nvmap_handle *h;
+        pgprot_t prot;
+        unsigned long adj_size;
+        unsigned long offs;
+        struct vm_struct *v;
+        void *p;
+
+        h = nvmap_handle_get(ref->handle);
+        if (!h)
+                return NULL;
+
+        prot = nvmap_pgprot(h, pgprot_kernel);
+
+        if (h->heap_pgalloc)
+                return vm_map_ram(h->pgalloc.pages, h->size >> PAGE_SHIFT,
+                                  -1, prot);
+
+        /* carveout - explicitly map the pfns into a vmalloc area */
+
+        nvmap_usecount_inc(h);
+
+        adj_size = h->carveout->base & ~PAGE_MASK;
+        adj_size += h->size;
+        adj_size = PAGE_ALIGN(adj_size);
+
+        v = alloc_vm_area(adj_size);
+        if (!v) {
+                nvmap_usecount_dec(h);
+                nvmap_handle_put(h);
+                return NULL;
+        }
+
+        p = v->addr + (h->carveout->base & ~PAGE_MASK);
+
+        for (offs = 0; offs < adj_size; offs += PAGE_SIZE) {
+                unsigned long addr = (unsigned long) v->addr + offs;
+                unsigned int pfn;
+                pgd_t *pgd;
+                pud_t *pud;
+                pmd_t *pmd;
+                pte_t *pte;
+
+                pfn = __phys_to_pfn(h->carveout->base + offs);
+                pgd = pgd_offset_k(addr);
+                pud = pud_alloc(&init_mm, pgd, addr);
+                if (!pud)
+                        break;
+                pmd = pmd_alloc(&init_mm, pud, addr);
+                if (!pmd)
+                        break;
+                pte = pte_alloc_kernel(pmd, addr);
+                if (!pte)
+                        break;
+                set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot));
+                flush_tlb_kernel_page(addr);
+        }
+
+        if (offs != adj_size) {
+                free_vm_area(v);
+                nvmap_usecount_dec(h);
+                nvmap_handle_put(h);
+                return NULL;
+        }
+
+        /* leave the handle ref count incremented by 1, so that
+         * the handle will not be freed while the kernel mapping exists.
+         * nvmap_handle_put will be called by unmapping this address */
+        return p;
+}
+
+void nvmap_munmap(struct nvmap_handle_ref *ref, void *addr)
+{
+        struct nvmap_handle *h;
+
+        if (!ref)
+                return;
+
+        h = ref->handle;
+
+        if (h->heap_pgalloc) {
+                vm_unmap_ram(addr, h->size >> PAGE_SHIFT);
+        } else {
+                struct vm_struct *vm;
+                addr -= (h->carveout->base & ~PAGE_MASK);
+                vm = remove_vm_area(addr);
+                BUG_ON(!vm);
+                kfree(vm);
+                nvmap_usecount_dec(h);
+        }
+        nvmap_handle_put(h);
+}
+
+struct nvmap_handle_ref *nvmap_alloc(struct nvmap_client *client, size_t size,
+                                     size_t align, unsigned int flags,
+                                     unsigned int heap_mask)
+{
+        const unsigned int default_heap = (NVMAP_HEAP_SYSMEM |
+                                           NVMAP_HEAP_CARVEOUT_GENERIC);
+        struct nvmap_handle_ref *r = NULL;
+        int err;
+
+        if (heap_mask == 0)
+                heap_mask = default_heap;
+
+        r = nvmap_create_handle(client, size);
+        if (IS_ERR(r))
+                return r;
+
+        err = nvmap_alloc_handle_id(client, nvmap_ref_to_id(r),
+                                    heap_mask, align, flags);
+
+        if (err) {
+                nvmap_free_handle_id(client, nvmap_ref_to_id(r));
+                return ERR_PTR(err);
+        }
+
+        return r;
+}
+
+/* allocates memory with specifed iovm_start address. */
+struct nvmap_handle_ref *nvmap_alloc_iovm(struct nvmap_client *client,
+        size_t size, size_t align, unsigned int flags, unsigned int iovm_start)
+{
+        int err;
+        struct nvmap_handle *h;
+        struct nvmap_handle_ref *r;
+        const unsigned int default_heap = NVMAP_HEAP_IOVMM;
+
+        /* size need to be more than one page.
+         * otherwise heap preference would change to system heap.
+         */
+        if (size <= PAGE_SIZE)
+                size = PAGE_SIZE << 1;
+        r = nvmap_create_handle(client, size);
+        if (IS_ERR_OR_NULL(r))
+                return r;
+
+        h = r->handle;
+        h->pgalloc.iovm_addr = iovm_start;
+        err = nvmap_alloc_handle_id(client, nvmap_ref_to_id(r),
+                        default_heap, align, flags);
+        if (err)
+                goto fail;
+
+        err = mutex_lock_interruptible(&client->share->pin_lock);
+        if (WARN_ON(err))
+                goto fail;
+        err = pin_locked(client, h);
+        mutex_unlock(&client->share->pin_lock);
+        if (err)
+                goto fail;
+        return r;
+
+fail:
+        nvmap_free_handle_id(client, nvmap_ref_to_id(r));
+        return ERR_PTR(err);
+}
+
+void nvmap_free_iovm(struct nvmap_client *client, struct nvmap_handle_ref *r)
+{
+        unsigned long ref_id = nvmap_ref_to_id(r);
+
+        nvmap_unpin_ids(client, 1, &ref_id);
+        nvmap_free_handle_id(client, ref_id);
+}
+
+void nvmap_free(struct nvmap_client *client, struct nvmap_handle_ref *r)
+{
+        if (!r)
+                return;
+
+        nvmap_free_handle_id(client, nvmap_ref_to_id(r));
+}
+
+/*
+ * create a mapping to the user's buffer and write it
+ * (uses similar logic from nvmap_reloc_pin_array to map the cmdbuf)
+ */
+int nvmap_patch_word(struct nvmap_client *client,
+                                struct nvmap_handle *patch,
+                                u32 patch_offset, u32 patch_value)
+{
+        phys_addr_t phys;
+        unsigned long kaddr;
+        unsigned int pfn;
+        void *addr;
+        pte_t **pte;
+        pgprot_t prot;
+
+        if (patch_offset >= patch->size) {
+                nvmap_warn(client, "read/write outside of handle\n");
+                return -EFAULT;
+        }
+
+        pte = nvmap_alloc_pte(client->dev, &addr);
+        if (IS_ERR(pte))
+                return PTR_ERR(pte);
+
+        /* derive physaddr of cmdbuf WAIT to patch */
+        if (patch->heap_pgalloc) {
+                unsigned int page = patch_offset >> PAGE_SHIFT;
+                phys = page_to_phys(patch->pgalloc.pages[page]);
+                phys += (patch_offset & ~PAGE_MASK);
+        } else {
+                phys = patch->carveout->base + patch_offset;
+        }
+
+        pfn = __phys_to_pfn(phys);
+        prot = nvmap_pgprot(patch, pgprot_kernel);
+        kaddr = (unsigned long)addr;
+
+        /* write PTE, so addr points to cmdbuf PFN */
+        set_pte_at(&init_mm, kaddr, *pte, pfn_pte(pfn, prot));
+        flush_tlb_kernel_page(kaddr);
+
+        /* write patch_value to addr + page offset */
+        __raw_writel(patch_value, addr + (phys & ~PAGE_MASK));
+
+        nvmap_free_pte(client->dev, pte);
+        wmb();
+        return 0;
+}