1 files changed, 1132 insertions, 0 deletions

diff --git a/drivers/gpu/pvr/mmap.c b/drivers/gpu/pvr/mmap.c
new file mode 100644
index 00000000000..66cef26e522
--- /dev/null
+++ b/drivers/gpu/pvr/mmap.c
@@ -0,0 +1,1132 @@
+/**********************************************************************
+ *
+ * Copyright(c) 2008 Imagination Technologies Ltd. All rights reserved.
+ * 
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ * 
+ * This program is distributed in the hope it will be useful but, except 
+ * as otherwise stated in writing, without any warranty; without even the 
+ * implied warranty of merchantability or fitness for a particular purpose. 
+ * See the GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ * 
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ *
+ * Contact Information:
+ * Imagination Technologies Ltd. <gpl-support@imgtec.com>
+ * Home Park Estate, Kings Langley, Herts, WD4 8LZ, UK 
+ *
+ ******************************************************************************/
+
+#ifndef AUTOCONF_INCLUDED
+ #include <linux/config.h>
+#endif
+
+#include <linux/version.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/vmalloc.h>
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0))
+#include <linux/wrapper.h>
+#endif
+#include <linux/slab.h>
+#include <asm/io.h>
+#include <asm/page.h>
+#include <asm/shmparam.h>
+#include <asm/pgtable.h>
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,22))
+#include <linux/sched.h>
+#include <asm/current.h>
+#endif
+#if defined(SUPPORT_DRI_DRM)
+#include <drm/drmP.h>
+#endif
+
+#include "img_defs.h"
+#include "services.h"
+#include "servicesint.h"
+#include "pvrmmap.h"
+#include "mutils.h"
+#include "mmap.h"
+#include "mm.h"
+#include "pvr_debug.h"
+#include "osfunc.h"
+#include "proc.h"
+#include "mutex.h"
+#include "handle.h"
+#include "perproc.h"
+#include "env_perproc.h"
+#include "bridged_support.h"
+#if defined(SUPPORT_DRI_DRM)
+#include "pvr_drm.h"
+#endif
+
+#if !defined(PVR_SECURE_HANDLES)
+#error "The mmap code requires PVR_SECURE_HANDLES"
+#endif
+
+static PVRSRV_LINUX_MUTEX g_sMMapMutex;
+
+static LinuxKMemCache *g_psMemmapCache = NULL;
+static LIST_HEAD(g_sMMapAreaList);
+static LIST_HEAD(g_sMMapOffsetStructList);
+#if defined(DEBUG_LINUX_MMAP_AREAS)
+static IMG_UINT32 g_ui32RegisteredAreas = 0;
+static IMG_UINT32 g_ui32TotalByteSize = 0;
+#endif
+
+
+#if defined(DEBUG_LINUX_MMAP_AREAS)
+static struct proc_dir_entry *g_ProcMMap;
+#endif 
+
+#if !defined(PVR_MAKE_ALL_PFNS_SPECIAL)
+#define MMAP2_PGOFF_RESOLUTION (32-PAGE_SHIFT+12)
+#define RESERVED_PGOFF_BITS 1
+#define MAX_MMAP_HANDLE         ((1UL<<(MMAP2_PGOFF_RESOLUTION-RESERVED_PGOFF_BITS))-1)
+
+#define FIRST_PHYSICAL_PFN      0
+#define LAST_PHYSICAL_PFN       (FIRST_PHYSICAL_PFN + MAX_MMAP_HANDLE)
+#define FIRST_SPECIAL_PFN       (LAST_PHYSICAL_PFN + 1)
+#define LAST_SPECIAL_PFN        (FIRST_SPECIAL_PFN + MAX_MMAP_HANDLE)
+
+#else   
+
+#if PAGE_SHIFT != 12
+#error This build variant has not yet been made non-4KB page-size aware
+#endif
+
+#if defined(PVR_MMAP_OFFSET_BASE)
+#define FIRST_SPECIAL_PFN       PVR_MMAP_OFFSET_BASE
+#else
+#define FIRST_SPECIAL_PFN       0x80000000UL
+#endif
+
+#if defined(PVR_NUM_MMAP_HANDLES)
+#define MAX_MMAP_HANDLE         PVR_NUM_MMAP_HANDLES
+#else
+#define MAX_MMAP_HANDLE         0x7fffffffUL
+#endif
+
+#endif  
+
+#if !defined(PVR_MAKE_ALL_PFNS_SPECIAL)
+static inline IMG_BOOL
+PFNIsPhysical(IMG_UINT32 pfn)
+{
+        
+        return ( (pfn <= LAST_PHYSICAL_PFN)) ? IMG_TRUE : IMG_FALSE;
+}
+
+static inline IMG_BOOL
+PFNIsSpecial(IMG_UINT32 pfn)
+{
+        
+        return ((pfn >= FIRST_SPECIAL_PFN) ) ? IMG_TRUE : IMG_FALSE;
+}
+#endif
+
+#if !defined(PVR_MAKE_ALL_PFNS_SPECIAL)
+static inline IMG_HANDLE
+MMapOffsetToHandle(IMG_UINT32 pfn)
+{
+        if (PFNIsPhysical(pfn))
+        {
+                PVR_ASSERT(PFNIsPhysical(pfn));
+                return IMG_NULL;
+        }
+        return (IMG_HANDLE)(pfn - FIRST_SPECIAL_PFN);
+}
+#endif
+
+static inline IMG_UINT32
+HandleToMMapOffset(IMG_HANDLE hHandle)
+{
+        IMG_UINT32 ulHandle = (IMG_UINT32)hHandle;
+
+#if !defined(PVR_MAKE_ALL_PFNS_SPECIAL)
+        if (PFNIsSpecial(ulHandle))
+        {
+                PVR_ASSERT(PFNIsSpecial(ulHandle));
+                return 0;
+        }
+#endif
+        return ulHandle + FIRST_SPECIAL_PFN;
+}
+
+#if !defined(PVR_MAKE_ALL_PFNS_SPECIAL)
+static inline IMG_BOOL
+LinuxMemAreaUsesPhysicalMap(LinuxMemArea *psLinuxMemArea)
+{
+    return LinuxMemAreaPhysIsContig(psLinuxMemArea);
+}
+#endif
+
+#if !defined(PVR_MAKE_ALL_PFNS_SPECIAL)
+static inline IMG_UINT32
+GetCurrentThreadID(IMG_VOID)
+{
+        
+        return (IMG_UINT32)current->pid;
+}
+#endif
+
+static PKV_OFFSET_STRUCT
+CreateOffsetStruct(LinuxMemArea *psLinuxMemArea, IMG_UINT32 ui32Offset, IMG_UINT32 ui32RealByteSize)
+{
+    PKV_OFFSET_STRUCT psOffsetStruct;
+#if defined(DEBUG) || defined(DEBUG_LINUX_MMAP_AREAS)
+    const IMG_CHAR *pszName = LinuxMemAreaTypeToString(LinuxMemAreaRootType(psLinuxMemArea));
+#endif
+
+#if defined(DEBUG) || defined(DEBUG_LINUX_MMAP_AREAS)
+    PVR_DPF((PVR_DBG_MESSAGE,
+             "%s(%s, psLinuxMemArea: 0x%p, ui32AllocFlags: 0x%8x)",
+             __FUNCTION__, pszName, psLinuxMemArea, psLinuxMemArea->ui32AreaFlags));
+#endif
+
+    PVR_ASSERT(psLinuxMemArea->eAreaType != LINUX_MEM_AREA_SUB_ALLOC || LinuxMemAreaRoot(psLinuxMemArea)->eAreaType != LINUX_MEM_AREA_SUB_ALLOC);
+
+    PVR_ASSERT(psLinuxMemArea->bMMapRegistered);
+
+    psOffsetStruct = KMemCacheAllocWrapper(g_psMemmapCache, GFP_KERNEL);
+    if(psOffsetStruct == IMG_NULL)
+    {
+        PVR_DPF((PVR_DBG_ERROR,"PVRMMapRegisterArea: Couldn't alloc another mapping record from cache"));
+        return IMG_NULL;
+    }
+    
+    psOffsetStruct->ui32MMapOffset = ui32Offset;
+
+    psOffsetStruct->psLinuxMemArea = psLinuxMemArea;
+
+    psOffsetStruct->ui32RealByteSize = ui32RealByteSize;
+
+    
+#if !defined(PVR_MAKE_ALL_PFNS_SPECIAL)
+    psOffsetStruct->ui32TID = GetCurrentThreadID();
+#endif
+    psOffsetStruct->ui32PID = OSGetCurrentProcessIDKM();
+
+#if defined(DEBUG_LINUX_MMAP_AREAS)
+    
+    psOffsetStruct->pszName = pszName;
+#endif
+
+    list_add_tail(&psOffsetStruct->sAreaItem, &psLinuxMemArea->sMMapOffsetStructList);
+
+    return psOffsetStruct;
+}
+
+
+static IMG_VOID
+DestroyOffsetStruct(PKV_OFFSET_STRUCT psOffsetStruct)
+{
+#ifdef DEBUG
+    IMG_CPU_PHYADDR CpuPAddr;
+    CpuPAddr = LinuxMemAreaToCpuPAddr(psOffsetStruct->psLinuxMemArea, 0);
+#endif
+
+    list_del(&psOffsetStruct->sAreaItem);
+
+    if (psOffsetStruct->bOnMMapList)
+    {
+        list_del(&psOffsetStruct->sMMapItem);
+    }
+
+#ifdef DEBUG
+    PVR_DPF((PVR_DBG_MESSAGE, "%s: Table entry: "
+             "psLinuxMemArea=%p, CpuPAddr=0x%08X", __FUNCTION__,
+             psOffsetStruct->psLinuxMemArea,
+             CpuPAddr.uiAddr));
+#endif
+    
+    KMemCacheFreeWrapper(g_psMemmapCache, psOffsetStruct);
+}
+
+
+static inline IMG_VOID
+DetermineUsersSizeAndByteOffset(LinuxMemArea *psLinuxMemArea,
+                               IMG_UINT32 *pui32RealByteSize,
+                               IMG_UINT32 *pui32ByteOffset)
+{
+    IMG_UINT32 ui32PageAlignmentOffset;
+    IMG_CPU_PHYADDR CpuPAddr;
+    
+    CpuPAddr = LinuxMemAreaToCpuPAddr(psLinuxMemArea, 0);
+    ui32PageAlignmentOffset = ADDR_TO_PAGE_OFFSET(CpuPAddr.uiAddr);
+    
+    *pui32ByteOffset = ui32PageAlignmentOffset;
+
+    *pui32RealByteSize = PAGE_ALIGN(psLinuxMemArea->ui32ByteSize + ui32PageAlignmentOffset);
+}
+
+
+PVRSRV_ERROR
+PVRMMapOSMemHandleToMMapData(PVRSRV_PER_PROCESS_DATA *psPerProc,
+                                IMG_HANDLE hMHandle,
+                                IMG_UINT32 *pui32MMapOffset,
+                                IMG_UINT32 *pui32ByteOffset,
+                                IMG_UINT32 *pui32RealByteSize,
+                                IMG_UINT32 *pui32UserVAddr)
+{
+    LinuxMemArea *psLinuxMemArea;
+    PKV_OFFSET_STRUCT psOffsetStruct;
+    IMG_HANDLE hOSMemHandle;
+    PVRSRV_ERROR eError;
+
+    LinuxLockMutex(&g_sMMapMutex);
+
+    PVR_ASSERT(PVRSRVGetMaxHandle(psPerProc->psHandleBase) <= MAX_MMAP_HANDLE);
+
+    eError = PVRSRVLookupOSMemHandle(psPerProc->psHandleBase, &hOSMemHandle, hMHandle);
+    if (eError != PVRSRV_OK)
+    {
+        PVR_DPF((PVR_DBG_ERROR, "%s: Lookup of handle %p failed", __FUNCTION__, hMHandle));
+
+        goto exit_unlock;
+    }
+
+    psLinuxMemArea = (LinuxMemArea *)hOSMemHandle;
+
+    DetermineUsersSizeAndByteOffset(psLinuxMemArea,
+                                   pui32RealByteSize,
+                                   pui32ByteOffset);
+
+    
+    list_for_each_entry(psOffsetStruct, &psLinuxMemArea->sMMapOffsetStructList, sAreaItem)
+    {
+        if (psPerProc->ui32PID == psOffsetStruct->ui32PID)
+        {
+
+           PVR_ASSERT(*pui32RealByteSize == psOffsetStruct->ui32RealByteSize);
+           
+           *pui32MMapOffset = psOffsetStruct->ui32MMapOffset;
+           *pui32UserVAddr = psOffsetStruct->ui32UserVAddr;
+           psOffsetStruct->ui32RefCount++;
+
+           eError = PVRSRV_OK;
+           goto exit_unlock;
+        }
+    }
+
+    
+    *pui32UserVAddr = 0;
+
+#if !defined(PVR_MAKE_ALL_PFNS_SPECIAL)
+    if (LinuxMemAreaUsesPhysicalMap(psLinuxMemArea))
+    {
+        *pui32MMapOffset = LinuxMemAreaToCpuPFN(psLinuxMemArea, 0);
+        PVR_ASSERT(PFNIsPhysical(*pui32MMapOffset));
+    }
+    else
+#endif
+    {
+        *pui32MMapOffset = HandleToMMapOffset(hMHandle);
+#if !defined(PVR_MAKE_ALL_PFNS_SPECIAL)
+        PVR_ASSERT(PFNIsSpecial(*pui32MMapOffset));
+#endif
+    }
+
+    psOffsetStruct = CreateOffsetStruct(psLinuxMemArea, *pui32MMapOffset, *pui32RealByteSize);
+    if (psOffsetStruct == IMG_NULL)
+    {
+        eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+        goto exit_unlock;
+    }
+
+    
+    list_add_tail(&psOffsetStruct->sMMapItem, &g_sMMapOffsetStructList);
+
+    psOffsetStruct->bOnMMapList = IMG_TRUE;
+
+    psOffsetStruct->ui32RefCount++;
+
+    eError = PVRSRV_OK;
+
+        
+
+
+        *pui32MMapOffset = *pui32MMapOffset << (PAGE_SHIFT - 12);
+
+exit_unlock:
+    LinuxUnLockMutex(&g_sMMapMutex);
+
+    return eError;
+}
+
+
+PVRSRV_ERROR
+PVRMMapReleaseMMapData(PVRSRV_PER_PROCESS_DATA *psPerProc,
+                                IMG_HANDLE hMHandle,
+                                IMG_BOOL *pbMUnmap,
+                                IMG_UINT32 *pui32RealByteSize,
+                                IMG_UINT32 *pui32UserVAddr)
+{
+    LinuxMemArea *psLinuxMemArea;
+    PKV_OFFSET_STRUCT psOffsetStruct;
+    IMG_HANDLE hOSMemHandle;
+    PVRSRV_ERROR eError;
+    IMG_UINT32 ui32PID = OSGetCurrentProcessIDKM();
+
+    LinuxLockMutex(&g_sMMapMutex);
+
+    PVR_ASSERT(PVRSRVGetMaxHandle(psPerProc->psHandleBase) <= MAX_MMAP_HANDLE);
+
+    eError = PVRSRVLookupOSMemHandle(psPerProc->psHandleBase, &hOSMemHandle, hMHandle);
+    if (eError != PVRSRV_OK)
+    {
+        PVR_DPF((PVR_DBG_ERROR, "%s: Lookup of handle %p failed", __FUNCTION__, hMHandle));
+
+        goto exit_unlock;
+    }
+
+    psLinuxMemArea = (LinuxMemArea *)hOSMemHandle;
+
+    
+    list_for_each_entry(psOffsetStruct, &psLinuxMemArea->sMMapOffsetStructList, sAreaItem)
+    {
+        if (psOffsetStruct->ui32PID == ui32PID)
+        {
+            if (psOffsetStruct->ui32RefCount == 0)
+            {
+                PVR_DPF((PVR_DBG_ERROR, "%s: Attempt to release mmap data with zero reference count for offset struct 0x%p, memory area %p", __FUNCTION__, psOffsetStruct, psLinuxMemArea));
+                eError = PVRSRV_ERROR_STILL_MAPPED;
+                goto exit_unlock;
+            }
+
+            psOffsetStruct->ui32RefCount--;
+
+            *pbMUnmap = (IMG_BOOL)((psOffsetStruct->ui32RefCount == 0) && (psOffsetStruct->ui32UserVAddr != 0));
+
+            *pui32UserVAddr = (*pbMUnmap) ? psOffsetStruct->ui32UserVAddr : 0;
+            *pui32RealByteSize = (*pbMUnmap) ? psOffsetStruct->ui32RealByteSize : 0;
+
+            eError = PVRSRV_OK;
+            goto exit_unlock;
+        }
+    }
+
+    
+    PVR_DPF((PVR_DBG_ERROR, "%s: Mapping data not found for handle %p (memory area %p)", __FUNCTION__, hMHandle, psLinuxMemArea));
+
+    eError =  PVRSRV_ERROR_MAPPING_NOT_FOUND;
+
+exit_unlock:
+    LinuxUnLockMutex(&g_sMMapMutex);
+
+    return eError;
+}
+
+static inline PKV_OFFSET_STRUCT
+FindOffsetStructByOffset(IMG_UINT32 ui32Offset, IMG_UINT32 ui32RealByteSize)
+{
+    PKV_OFFSET_STRUCT psOffsetStruct;
+#if !defined(PVR_MAKE_ALL_PFNS_SPECIAL)
+    IMG_UINT32 ui32TID = GetCurrentThreadID();
+#endif
+    IMG_UINT32 ui32PID = OSGetCurrentProcessIDKM();
+
+    list_for_each_entry(psOffsetStruct, &g_sMMapOffsetStructList, sMMapItem)
+    {
+        if (ui32Offset == psOffsetStruct->ui32MMapOffset && ui32RealByteSize == psOffsetStruct->ui32RealByteSize && psOffsetStruct->ui32PID == ui32PID)
+        {
+#if !defined(PVR_MAKE_ALL_PFNS_SPECIAL)
+            
+            if (!PFNIsPhysical(ui32Offset) || psOffsetStruct->ui32TID == ui32TID)
+#endif
+            {
+                return psOffsetStruct;
+            }
+        }
+    }
+
+    return IMG_NULL;
+}
+
+
+static IMG_BOOL
+DoMapToUser(LinuxMemArea *psLinuxMemArea,
+            struct vm_area_struct* ps_vma,
+            IMG_UINT32 ui32ByteOffset)
+{
+    IMG_UINT32 ui32ByteSize;
+
+    if (psLinuxMemArea->eAreaType == LINUX_MEM_AREA_SUB_ALLOC)
+    {
+        return DoMapToUser(LinuxMemAreaRoot(psLinuxMemArea),             
+                    ps_vma,
+                    psLinuxMemArea->uData.sSubAlloc.ui32ByteOffset + ui32ByteOffset);
+    }
+
+    
+    ui32ByteSize = ps_vma->vm_end - ps_vma->vm_start;
+    PVR_ASSERT(ADDR_TO_PAGE_OFFSET(ui32ByteSize) == 0);
+
+#if defined (__sparc__)
+    
+#error "SPARC not supported"
+#endif
+
+#if !defined(PVR_MAKE_ALL_PFNS_SPECIAL)
+    if (PFNIsPhysical(ps_vma->vm_pgoff))
+    {
+        IMG_INT result;
+
+        PVR_ASSERT(LinuxMemAreaPhysIsContig(psLinuxMemArea));
+        PVR_ASSERT(LinuxMemAreaToCpuPFN(psLinuxMemArea, ui32ByteOffset) == ps_vma->vm_pgoff);
+        
+        result = IO_REMAP_PFN_RANGE(ps_vma, ps_vma->vm_start, ps_vma->vm_pgoff, ui32ByteSize, ps_vma->vm_page_prot);
+
+        if(result == 0)
+        {
+            return IMG_TRUE;
+        }
+
+        PVR_DPF((PVR_DBG_MESSAGE, "%s: Failed to map contiguous physical address range (%d), trying non-contiguous path", __FUNCTION__, result));
+    }
+#endif
+
+    {
+        
+        IMG_UINT32 ulVMAPos;
+        IMG_UINT32 ui32ByteEnd = ui32ByteOffset + ui32ByteSize;
+        IMG_UINT32 ui32PA;
+#if defined(PVR_MAKE_ALL_PFNS_SPECIAL)
+        IMG_BOOL bMixedMap = IMG_FALSE;
+#endif
+        
+        for(ui32PA = ui32ByteOffset; ui32PA < ui32ByteEnd; ui32PA += PAGE_SIZE)
+        {
+            IMG_UINT32 pfn =  LinuxMemAreaToCpuPFN(psLinuxMemArea, ui32PA);
+
+            if (!pfn_valid(pfn))
+            {
+#if !defined(PVR_MAKE_ALL_PFNS_SPECIAL)
+                PVR_DPF((PVR_DBG_ERROR,"%s: Error - PFN invalid: 0x%x", __FUNCTION__, pfn));
+                return IMG_FALSE;
+#else
+                bMixedMap = IMG_TRUE;
+#endif
+            }
+        }
+
+#if defined(PVR_MAKE_ALL_PFNS_SPECIAL)
+        if (bMixedMap)
+        {
+            ps_vma->vm_flags |= VM_MIXEDMAP;
+        }
+#endif
+        
+        ulVMAPos = ps_vma->vm_start;
+        for(ui32PA = ui32ByteOffset; ui32PA < ui32ByteEnd; ui32PA += PAGE_SIZE)
+        {
+            IMG_UINT32 pfn;
+            IMG_INT result;
+
+            pfn =  LinuxMemAreaToCpuPFN(psLinuxMemArea, ui32PA);
+
+#if defined(PVR_MAKE_ALL_PFNS_SPECIAL)
+            if (bMixedMap)
+            {
+                result = vm_insert_mixed(ps_vma, ulVMAPos, pfn);
+                if(result != 0)
+                {
+                    PVR_DPF((PVR_DBG_ERROR,"%s: Error - vm_insert_mixed failed (%d)", __FUNCTION__, result));
+                    return IMG_FALSE;
+                }
+            }
+            else
+#endif
+            {
+                struct page *psPage;
+
+                PVR_ASSERT(pfn_valid(pfn));
+
+                psPage = pfn_to_page(pfn);
+
+                result = VM_INSERT_PAGE(ps_vma,  ulVMAPos, psPage);
+                if(result != 0)
+                {
+                    PVR_DPF((PVR_DBG_ERROR,"%s: Error - VM_INSERT_PAGE failed (%d)", __FUNCTION__, result));
+                    return IMG_FALSE;
+                }
+            }
+            ulVMAPos += PAGE_SIZE;
+        }
+    }
+
+    return IMG_TRUE;
+}
+
+
+static IMG_VOID
+MMapVOpenNoLock(struct vm_area_struct* ps_vma)
+{
+    PKV_OFFSET_STRUCT psOffsetStruct = (PKV_OFFSET_STRUCT)ps_vma->vm_private_data;
+    PVR_ASSERT(psOffsetStruct != IMG_NULL)
+    psOffsetStruct->ui32Mapped++;
+    PVR_ASSERT(!psOffsetStruct->bOnMMapList);
+
+    if (psOffsetStruct->ui32Mapped > 1)
+    {
+        PVR_DPF((PVR_DBG_WARNING, "%s: Offset structure 0x%p is being shared across processes (psOffsetStruct->ui32Mapped: %u)", __FUNCTION__, psOffsetStruct, psOffsetStruct->ui32Mapped));
+        PVR_ASSERT((ps_vma->vm_flags & VM_DONTCOPY) == 0);
+    }
+
+#if defined(DEBUG_LINUX_MMAP_AREAS)
+
+    PVR_DPF((PVR_DBG_MESSAGE,
+             "%s: psLinuxMemArea 0x%p, KVAddress 0x%p MMapOffset %d, ui32Mapped %d",
+             __FUNCTION__,
+             psOffsetStruct->psLinuxMemArea,
+             LinuxMemAreaToCpuVAddr(psOffsetStruct->psLinuxMemArea),
+             psOffsetStruct->ui32MMapOffset,
+             psOffsetStruct->ui32Mapped));
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0))
+    MOD_INC_USE_COUNT;
+#endif
+}
+
+
+static void
+MMapVOpen(struct vm_area_struct* ps_vma)
+{
+    LinuxLockMutex(&g_sMMapMutex);
+
+    MMapVOpenNoLock(ps_vma);
+
+    LinuxUnLockMutex(&g_sMMapMutex);
+}
+
+
+static IMG_VOID
+MMapVCloseNoLock(struct vm_area_struct* ps_vma)
+{
+    PKV_OFFSET_STRUCT psOffsetStruct = (PKV_OFFSET_STRUCT)ps_vma->vm_private_data;
+    PVR_ASSERT(psOffsetStruct != IMG_NULL)
+
+#if defined(DEBUG_LINUX_MMAP_AREAS)
+    PVR_DPF((PVR_DBG_MESSAGE,
+             "%s: psLinuxMemArea %p, CpuVAddr %p ui32MMapOffset %d, ui32Mapped %d",
+             __FUNCTION__,
+             psOffsetStruct->psLinuxMemArea,
+             LinuxMemAreaToCpuVAddr(psOffsetStruct->psLinuxMemArea),
+             psOffsetStruct->ui32MMapOffset,
+             psOffsetStruct->ui32Mapped));
+#endif
+
+    PVR_ASSERT(!psOffsetStruct->bOnMMapList);
+    psOffsetStruct->ui32Mapped--;
+    if (psOffsetStruct->ui32Mapped == 0)
+    {
+        if (psOffsetStruct->ui32RefCount != 0)
+        {
+                PVR_DPF((PVR_DBG_MESSAGE, "%s: psOffsetStruct %p has non-zero reference count (ui32RefCount = %u). User mode address of start of mapping: 0x%x", __FUNCTION__, psOffsetStruct, psOffsetStruct->ui32RefCount, psOffsetStruct->ui32UserVAddr));
+        }
+
+        DestroyOffsetStruct(psOffsetStruct);
+    }
+
+    ps_vma->vm_private_data = NULL;
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0))
+    MOD_DEC_USE_COUNT;
+#endif
+}
+
+static void
+MMapVClose(struct vm_area_struct* ps_vma)
+{
+    LinuxLockMutex(&g_sMMapMutex);
+
+    MMapVCloseNoLock(ps_vma);
+
+    LinuxUnLockMutex(&g_sMMapMutex);
+}
+
+
+static struct vm_operations_struct MMapIOOps =
+{
+        .open=MMapVOpen,
+        .close=MMapVClose
+};
+
+
+int
+PVRMMap(struct file* pFile, struct vm_area_struct* ps_vma)
+{
+    IMG_UINT32 ui32ByteSize;
+    PKV_OFFSET_STRUCT psOffsetStruct;
+    int iRetVal = 0;
+
+    PVR_UNREFERENCED_PARAMETER(pFile);
+
+    LinuxLockMutex(&g_sMMapMutex);
+    
+    ui32ByteSize = ps_vma->vm_end - ps_vma->vm_start;
+    
+    PVR_DPF((PVR_DBG_MESSAGE, "%s: Received mmap(2) request with ui32MMapOffset 0x%08lx,"
+                              " and ui32ByteSize %d(0x%08x)",
+            __FUNCTION__,
+            ps_vma->vm_pgoff,
+            ui32ByteSize, ui32ByteSize));
+   
+    psOffsetStruct = FindOffsetStructByOffset(ps_vma->vm_pgoff, ui32ByteSize);
+    if (psOffsetStruct == IMG_NULL)
+    {
+#if defined(SUPPORT_DRI_DRM)
+        LinuxUnLockMutex(&g_sMMapMutex);
+
+#if !defined(SUPPORT_DRI_DRM_EXT)
+        
+        return drm_mmap(pFile, ps_vma);
+#else
+        
+        return -ENOENT;
+#endif
+#else
+        PVR_UNREFERENCED_PARAMETER(pFile);
+
+        PVR_DPF((PVR_DBG_ERROR,
+             "%s: Attempted to mmap unregistered area at vm_pgoff 0x%lx",
+             __FUNCTION__, ps_vma->vm_pgoff));
+        iRetVal = -EINVAL;
+#endif
+        goto unlock_and_return;
+    }
+    list_del(&psOffsetStruct->sMMapItem);
+    psOffsetStruct->bOnMMapList = IMG_FALSE;
+
+    
+    if (((ps_vma->vm_flags & VM_WRITE) != 0) &&
+        ((ps_vma->vm_flags & VM_SHARED) == 0))
+    {
+        PVR_DPF((PVR_DBG_ERROR, "%s: Cannot mmap non-shareable writable areas", __FUNCTION__));
+        iRetVal = -EINVAL;
+        goto unlock_and_return;
+    }
+   
+    PVR_DPF((PVR_DBG_MESSAGE, "%s: Mapped psLinuxMemArea 0x%p\n",
+         __FUNCTION__, psOffsetStruct->psLinuxMemArea));
+
+    ps_vma->vm_flags |= VM_RESERVED;
+    ps_vma->vm_flags |= VM_IO;
+
+    
+    ps_vma->vm_flags |= VM_DONTEXPAND;
+    
+    
+    ps_vma->vm_flags |= VM_DONTCOPY;
+
+    ps_vma->vm_private_data = (void *)psOffsetStruct;
+    
+    switch(psOffsetStruct->psLinuxMemArea->ui32AreaFlags & PVRSRV_HAP_CACHETYPE_MASK)
+    {
+        case PVRSRV_HAP_CACHED:
+            
+            break;
+        case PVRSRV_HAP_WRITECOMBINE:
+            ps_vma->vm_page_prot = PGPROT_WC(ps_vma->vm_page_prot);
+            break;
+        case PVRSRV_HAP_UNCACHED:
+            ps_vma->vm_page_prot = PGPROT_UC(ps_vma->vm_page_prot);
+            break;
+        default:
+            PVR_DPF((PVR_DBG_ERROR, "%s: unknown cache type", __FUNCTION__));
+            iRetVal = -EINVAL;
+            goto unlock_and_return;
+    }
+    
+    
+    ps_vma->vm_ops = &MMapIOOps;
+    
+    if(!DoMapToUser(psOffsetStruct->psLinuxMemArea, ps_vma, 0))
+    {
+        iRetVal = -EAGAIN;
+        goto unlock_and_return;
+    }
+    
+    PVR_ASSERT(psOffsetStruct->ui32UserVAddr == 0)
+
+    psOffsetStruct->ui32UserVAddr = ps_vma->vm_start;
+
+    
+    if(psOffsetStruct->psLinuxMemArea->bNeedsCacheInvalidate)
+    {
+        IMG_UINT32 ui32RealByteSize, ui32ByteOffset;
+        IMG_VOID *pvBase;
+
+        DetermineUsersSizeAndByteOffset(psOffsetStruct->psLinuxMemArea,
+                                        &ui32RealByteSize,
+                                        &ui32ByteOffset);
+
+        ui32RealByteSize = psOffsetStruct->psLinuxMemArea->ui32ByteSize;
+        pvBase = (IMG_VOID *)ps_vma->vm_start + ui32ByteOffset;
+
+        OSInvalidateCPUCacheRangeKM(psOffsetStruct->psLinuxMemArea,
+                                    pvBase, ui32RealByteSize);
+        psOffsetStruct->psLinuxMemArea->bNeedsCacheInvalidate = IMG_FALSE;
+    }
+
+    
+    MMapVOpenNoLock(ps_vma);
+    
+    PVR_DPF((PVR_DBG_MESSAGE, "%s: Mapped area at offset 0x%08lx\n",
+             __FUNCTION__, ps_vma->vm_pgoff));
+    
+unlock_and_return:
+    if (iRetVal != 0 && psOffsetStruct != IMG_NULL)
+    {
+        DestroyOffsetStruct(psOffsetStruct);
+    }
+
+    LinuxUnLockMutex(&g_sMMapMutex);
+    
+    return iRetVal;
+}
+
+
+#if defined(DEBUG_LINUX_MMAP_AREAS)
+
+static void ProcSeqStartstopMMapRegistations(struct seq_file *sfile,IMG_BOOL start) 
+{
+        if(start) 
+        {
+            LinuxLockMutex(&g_sMMapMutex);              
+        }
+        else
+        {
+            LinuxUnLockMutex(&g_sMMapMutex);
+        }
+}
+
+
+static void* ProcSeqOff2ElementMMapRegistrations(struct seq_file *sfile, loff_t off)
+{
+    LinuxMemArea *psLinuxMemArea;
+        if(!off) 
+        {
+                return PVR_PROC_SEQ_START_TOKEN;
+        }
+
+    list_for_each_entry(psLinuxMemArea, &g_sMMapAreaList, sMMapItem)
+    {
+        PKV_OFFSET_STRUCT psOffsetStruct;
+
+                list_for_each_entry(psOffsetStruct, &psLinuxMemArea->sMMapOffsetStructList, sAreaItem)
+        {
+                off--;
+                if (off == 0)
+                {                               
+                                PVR_ASSERT(psOffsetStruct->psLinuxMemArea == psLinuxMemArea);
+                                return (void*)psOffsetStruct;
+                    }
+        }
+    }
+        return (void*)0;
+}
+
+static void* ProcSeqNextMMapRegistrations(struct seq_file *sfile,void* el,loff_t off)
+{
+        return ProcSeqOff2ElementMMapRegistrations(sfile,off);
+}
+
+
+static void ProcSeqShowMMapRegistrations(struct seq_file *sfile, void *el)
+{
+        KV_OFFSET_STRUCT *psOffsetStruct = (KV_OFFSET_STRUCT*)el;
+    LinuxMemArea *psLinuxMemArea;
+        IMG_UINT32 ui32RealByteSize;
+        IMG_UINT32 ui32ByteOffset;
+
+        if(el == PVR_PROC_SEQ_START_TOKEN) 
+        {
+        seq_printf( sfile,
+#if !defined(DEBUG_LINUX_XML_PROC_FILES)
+                                                  "Allocations registered for mmap: %u\n"
+                          "In total these areas correspond to %u bytes\n"
+                          "psLinuxMemArea "
+                                                  "UserVAddr "
+                                                  "KernelVAddr "
+                                                  "CpuPAddr "
+                          "MMapOffset "
+                          "ByteLength "
+                          "LinuxMemType             "
+                                                  "Pid   Name     Flags\n",
+#else
+                          "<mmap_header>\n"
+                          "\t<count>%u</count>\n"
+                          "\t<bytes>%u</bytes>\n"
+                          "</mmap_header>\n",
+#endif
+                                                  g_ui32RegisteredAreas,
+                          g_ui32TotalByteSize
+                          );
+                return;
+        }
+
+        psLinuxMemArea = psOffsetStruct->psLinuxMemArea;
+
+        DetermineUsersSizeAndByteOffset(psLinuxMemArea,
+                                                                        &ui32RealByteSize,
+                                                                        &ui32ByteOffset);
+
+        seq_printf( sfile,
+#if !defined(DEBUG_LINUX_XML_PROC_FILES)
+                                                "%-8p       %08x %-8p %08x %08x   %-8d   %-24s %-5u %-8s %08x(%s)\n",
+#else
+                        "<mmap_record>\n"
+                                                "\t<pointer>%-8p</pointer>\n"
+                        "\t<user_virtual>%-8x</user_virtual>\n"
+                        "\t<kernel_virtual>%-8p</kernel_virtual>\n"
+                        "\t<cpu_physical>%08x</cpu_physical>\n"
+                        "\t<mmap_offset>%08x</mmap_offset>\n"
+                        "\t<bytes>%-8d</bytes>\n"
+                        "\t<linux_mem_area_type>%-24s</linux_mem_area_type>\n"
+                        "\t<pid>%-5u</pid>\n"
+                        "\t<name>%-8s</name>\n"
+                        "\t<flags>%08x</flags>\n"
+                        "\t<flags_string>%s</flags_string>\n"
+                        "</mmap_record>\n",
+#endif
+                        psLinuxMemArea,
+                                                psOffsetStruct->ui32UserVAddr + ui32ByteOffset,
+                                                LinuxMemAreaToCpuVAddr(psLinuxMemArea),
+                        LinuxMemAreaToCpuPAddr(psLinuxMemArea,0).uiAddr,
+                                                psOffsetStruct->ui32MMapOffset,
+                                                psLinuxMemArea->ui32ByteSize,
+                        LinuxMemAreaTypeToString(psLinuxMemArea->eAreaType),
+                                                psOffsetStruct->ui32PID,
+                                                psOffsetStruct->pszName,
+                                                psLinuxMemArea->ui32AreaFlags,
+                        HAPFlagsToString(psLinuxMemArea->ui32AreaFlags));
+}
+
+#endif
+
+
+PVRSRV_ERROR
+PVRMMapRegisterArea(LinuxMemArea *psLinuxMemArea)
+{
+    PVRSRV_ERROR eError;
+#if defined(DEBUG) || defined(DEBUG_LINUX_MMAP_AREAS)
+    const IMG_CHAR *pszName = LinuxMemAreaTypeToString(LinuxMemAreaRootType(psLinuxMemArea));
+#endif
+
+    LinuxLockMutex(&g_sMMapMutex);
+
+#if defined(DEBUG) || defined(DEBUG_LINUX_MMAP_AREAS)
+    PVR_DPF((PVR_DBG_MESSAGE,
+             "%s(%s, psLinuxMemArea 0x%p, ui32AllocFlags 0x%8x)",
+             __FUNCTION__, pszName, psLinuxMemArea,  psLinuxMemArea->ui32AreaFlags));
+#endif
+
+    PVR_ASSERT(psLinuxMemArea->eAreaType != LINUX_MEM_AREA_SUB_ALLOC || LinuxMemAreaRoot(psLinuxMemArea)->eAreaType != LINUX_MEM_AREA_SUB_ALLOC);
+
+    
+    if(psLinuxMemArea->bMMapRegistered)
+    {
+        PVR_DPF((PVR_DBG_ERROR, "%s: psLinuxMemArea 0x%p is already registered",
+                __FUNCTION__, psLinuxMemArea));
+        eError = PVRSRV_ERROR_INVALID_PARAMS;
+        goto exit_unlock;
+    }
+
+    list_add_tail(&psLinuxMemArea->sMMapItem, &g_sMMapAreaList);
+
+    psLinuxMemArea->bMMapRegistered = IMG_TRUE;
+
+#if defined(DEBUG_LINUX_MMAP_AREAS)
+    g_ui32RegisteredAreas++;
+    
+    if (psLinuxMemArea->eAreaType != LINUX_MEM_AREA_SUB_ALLOC)
+    {
+        g_ui32TotalByteSize += psLinuxMemArea->ui32ByteSize;
+    }
+#endif
+
+    eError = PVRSRV_OK;
+
+exit_unlock:
+    LinuxUnLockMutex(&g_sMMapMutex);
+
+    return eError;
+}
+
+
+PVRSRV_ERROR
+PVRMMapRemoveRegisteredArea(LinuxMemArea *psLinuxMemArea)
+{
+    PVRSRV_ERROR eError;
+    PKV_OFFSET_STRUCT psOffsetStruct, psTmpOffsetStruct;
+
+    LinuxLockMutex(&g_sMMapMutex);
+
+    PVR_ASSERT(psLinuxMemArea->bMMapRegistered);
+
+    list_for_each_entry_safe(psOffsetStruct, psTmpOffsetStruct, &psLinuxMemArea->sMMapOffsetStructList, sAreaItem)
+    {
+        if (psOffsetStruct->ui32Mapped != 0)
+        {
+             PVR_DPF((PVR_DBG_ERROR, "%s: psOffsetStruct 0x%p for memory area 0x0x%p is still mapped; psOffsetStruct->ui32Mapped %u",  __FUNCTION__, psOffsetStruct, psLinuxMemArea, psOffsetStruct->ui32Mapped));
+                eError = PVRSRV_ERROR_STILL_MAPPED;
+                goto exit_unlock;
+        }
+        else
+        {
+              
+             PVR_DPF((PVR_DBG_WARNING, "%s: psOffsetStruct 0x%p was never mapped",  __FUNCTION__, psOffsetStruct));
+        }
+
+        PVR_ASSERT((psOffsetStruct->ui32Mapped == 0) && psOffsetStruct->bOnMMapList);
+
+        DestroyOffsetStruct(psOffsetStruct);
+    }
+
+    list_del(&psLinuxMemArea->sMMapItem);
+
+    psLinuxMemArea->bMMapRegistered = IMG_FALSE;
+
+#if defined(DEBUG_LINUX_MMAP_AREAS)
+    g_ui32RegisteredAreas--;
+    if (psLinuxMemArea->eAreaType != LINUX_MEM_AREA_SUB_ALLOC)
+    {
+        g_ui32TotalByteSize -= psLinuxMemArea->ui32ByteSize;
+    }
+#endif
+
+    eError = PVRSRV_OK;
+
+exit_unlock:
+    LinuxUnLockMutex(&g_sMMapMutex);
+    return eError;
+}
+
+
+PVRSRV_ERROR
+LinuxMMapPerProcessConnect(PVRSRV_ENV_PER_PROCESS_DATA *psEnvPerProc)
+{
+    PVR_UNREFERENCED_PARAMETER(psEnvPerProc);
+
+    return PVRSRV_OK;
+}
+
+IMG_VOID
+LinuxMMapPerProcessDisconnect(PVRSRV_ENV_PER_PROCESS_DATA *psEnvPerProc)
+{
+    PKV_OFFSET_STRUCT psOffsetStruct, psTmpOffsetStruct;
+    IMG_BOOL bWarn = IMG_FALSE;
+    IMG_UINT32 ui32PID = OSGetCurrentProcessIDKM();
+
+    PVR_UNREFERENCED_PARAMETER(psEnvPerProc);
+
+    LinuxLockMutex(&g_sMMapMutex);
+
+    list_for_each_entry_safe(psOffsetStruct, psTmpOffsetStruct, &g_sMMapOffsetStructList, sMMapItem)
+    {
+        if (psOffsetStruct->ui32PID == ui32PID)
+        {
+            if (!bWarn)
+            {
+                PVR_DPF((PVR_DBG_WARNING, "%s: process has unmapped offset structures. Removing them", __FUNCTION__));
+                bWarn = IMG_TRUE;
+            }
+            PVR_ASSERT(psOffsetStruct->ui32Mapped == 0);
+            PVR_ASSERT(psOffsetStruct->bOnMMapList);
+
+            DestroyOffsetStruct(psOffsetStruct);
+        }
+    }
+
+    LinuxUnLockMutex(&g_sMMapMutex);
+}
+
+
+PVRSRV_ERROR LinuxMMapPerProcessHandleOptions(PVRSRV_HANDLE_BASE *psHandleBase)
+{
+    PVRSRV_ERROR eError;
+
+    eError = PVRSRVSetMaxHandle(psHandleBase, MAX_MMAP_HANDLE);
+    if (eError != PVRSRV_OK)
+    {
+        PVR_DPF((PVR_DBG_ERROR,"%s: failed to set handle limit (%d)", __FUNCTION__, eError));
+        return eError;
+    }
+
+    return eError;
+}
+
+
+IMG_VOID
+PVRMMapInit(IMG_VOID)
+{
+    LinuxInitMutex(&g_sMMapMutex);
+
+    g_psMemmapCache = KMemCacheCreateWrapper("img-mmap", sizeof(KV_OFFSET_STRUCT), 0, 0);
+    if (!g_psMemmapCache)
+    {
+        PVR_DPF((PVR_DBG_ERROR,"%s: failed to allocate kmem_cache", __FUNCTION__));
+        goto error;
+    }
+
+#if defined(DEBUG_LINUX_MMAP_AREAS)
+        g_ProcMMap = CreateProcReadEntrySeq("mmap", NULL, 
+                                                  ProcSeqNextMMapRegistrations,
+                                                  ProcSeqShowMMapRegistrations,
+                                                  ProcSeqOff2ElementMMapRegistrations,
+                                                  ProcSeqStartstopMMapRegistations
+                                                 );
+#endif  
+    return;
+
+error:
+    PVRMMapCleanup();
+    return;
+}
+
+
+IMG_VOID
+PVRMMapCleanup(IMG_VOID)
+{
+    PVRSRV_ERROR eError;
+
+    if (!list_empty(&g_sMMapAreaList))
+    {
+        LinuxMemArea *psLinuxMemArea, *psTmpMemArea;
+
+        PVR_DPF((PVR_DBG_ERROR, "%s: Memory areas are still registered with MMap", __FUNCTION__));
+        
+        PVR_TRACE(("%s: Unregistering memory areas", __FUNCTION__));
+        list_for_each_entry_safe(psLinuxMemArea, psTmpMemArea, &g_sMMapAreaList, sMMapItem)
+        {
+                eError = PVRMMapRemoveRegisteredArea(psLinuxMemArea);
+                if (eError != PVRSRV_OK)
+                {
+                        PVR_DPF((PVR_DBG_ERROR, "%s: PVRMMapRemoveRegisteredArea failed (%d)", __FUNCTION__, eError));
+                }
+                PVR_ASSERT(eError == PVRSRV_OK);
+
+                LinuxMemAreaDeepFree(psLinuxMemArea);
+        }
+    }
+    PVR_ASSERT(list_empty((&g_sMMapAreaList)));
+
+#if defined(DEBUG_LINUX_MMAP_AREAS)
+    RemoveProcEntrySeq(g_ProcMMap);
+#endif 
+
+    if(g_psMemmapCache)
+    {
+        KMemCacheDestroyWrapper(g_psMemmapCache);
+        g_psMemmapCache = NULL;
+    }
+}