1 files changed, 442 insertions, 0 deletions

diff --git a/drivers/misc/mic/host/mic_smpt.c b/drivers/misc/mic/host/mic_smpt.c
new file mode 100644
index 000000000000..fae474c4899e
--- /dev/null
+++ b/drivers/misc/mic/host/mic_smpt.c
@@ -0,0 +1,442 @@
+/*
+ * Intel MIC Platform Software Stack (MPSS)
+ *
+ * Copyright(c) 2013 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ *
+ * Intel MIC Host driver.
+ *
+ */
+#include <linux/pci.h>
+
+#include "../common/mic_dev.h"
+#include "mic_device.h"
+#include "mic_smpt.h"
+
+static inline u64 mic_system_page_mask(struct mic_device *mdev)
+{
+        return (1ULL << mdev->smpt->info.page_shift) - 1ULL;
+}
+
+static inline u8 mic_sys_addr_to_smpt(struct mic_device *mdev, dma_addr_t pa)
+{
+        return (pa - mdev->smpt->info.base) >> mdev->smpt->info.page_shift;
+}
+
+static inline u64 mic_smpt_to_pa(struct mic_device *mdev, u8 index)
+{
+        return mdev->smpt->info.base + (index * mdev->smpt->info.page_size);
+}
+
+static inline u64 mic_smpt_offset(struct mic_device *mdev, dma_addr_t pa)
+{
+        return pa & mic_system_page_mask(mdev);
+}
+
+static inline u64 mic_smpt_align_low(struct mic_device *mdev, dma_addr_t pa)
+{
+        return ALIGN(pa - mic_system_page_mask(mdev),
+                mdev->smpt->info.page_size);
+}
+
+static inline u64 mic_smpt_align_high(struct mic_device *mdev, dma_addr_t pa)
+{
+        return ALIGN(pa, mdev->smpt->info.page_size);
+}
+
+/* Total Cumulative system memory accessible by MIC across all SMPT entries */
+static inline u64 mic_max_system_memory(struct mic_device *mdev)
+{
+        return mdev->smpt->info.num_reg * mdev->smpt->info.page_size;
+}
+
+/* Maximum system memory address accessible by MIC */
+static inline u64 mic_max_system_addr(struct mic_device *mdev)
+{
+        return mdev->smpt->info.base + mic_max_system_memory(mdev) - 1ULL;
+}
+
+/* Check if the DMA address is a MIC system memory address */
+static inline bool
+mic_is_system_addr(struct mic_device *mdev, dma_addr_t pa)
+{
+        return pa >= mdev->smpt->info.base && pa <= mic_max_system_addr(mdev);
+}
+
+/* Populate an SMPT entry and update the reference counts. */
+static void mic_add_smpt_entry(int spt, s64 *ref, u64 addr,
+                int entries, struct mic_device *mdev)
+{
+        struct mic_smpt_info *smpt_info = mdev->smpt;
+        int i;
+
+        for (i = spt; i < spt + entries; i++,
+                addr += smpt_info->info.page_size) {
+                if (!smpt_info->entry[i].ref_count &&
+                    (smpt_info->entry[i].dma_addr != addr)) {
+                        mdev->smpt_ops->set(mdev, addr, i);
+                        smpt_info->entry[i].dma_addr = addr;
+                }
+                smpt_info->entry[i].ref_count += ref[i - spt];
+        }
+}
+
+/*
+ * Find an available MIC address in MIC SMPT address space
+ * for a given DMA address and size.
+ */
+static dma_addr_t mic_smpt_op(struct mic_device *mdev, u64 dma_addr,
+                                int entries, s64 *ref, size_t size)
+{
+        int spt;
+        int ae = 0;
+        int i;
+        unsigned long flags;
+        dma_addr_t mic_addr = 0;
+        dma_addr_t addr = dma_addr;
+        struct mic_smpt_info *smpt_info = mdev->smpt;
+
+        spin_lock_irqsave(&smpt_info->smpt_lock, flags);
+
+        /* find existing entries */
+        for (i = 0; i < smpt_info->info.num_reg; i++) {
+                if (smpt_info->entry[i].dma_addr == addr) {
+                        ae++;
+                        addr += smpt_info->info.page_size;
+                } else if (ae) /* cannot find contiguous entries */
+                        goto not_found;
+
+                if (ae == entries)
+                        goto found;
+        }
+
+        /* find free entry */
+        for (ae = 0, i = 0; i < smpt_info->info.num_reg; i++) {
+                ae = (smpt_info->entry[i].ref_count == 0) ? ae + 1 : 0;
+                if (ae == entries)
+                        goto found;
+        }
+
+not_found:
+        spin_unlock_irqrestore(&smpt_info->smpt_lock, flags);
+        return mic_addr;
+
+found:
+        spt = i - entries + 1;
+        mic_addr = mic_smpt_to_pa(mdev, spt);
+        mic_add_smpt_entry(spt, ref, dma_addr, entries, mdev);
+        smpt_info->map_count++;
+        smpt_info->ref_count += (s64)size;
+        spin_unlock_irqrestore(&smpt_info->smpt_lock, flags);
+        return mic_addr;
+}
+
+/*
+ * Returns number of smpt entries needed for dma_addr to dma_addr + size
+ * also returns the reference count array for each of those entries
+ * and the starting smpt address
+ */
+static int mic_get_smpt_ref_count(struct mic_device *mdev, dma_addr_t dma_addr,
+                                size_t size, s64 *ref,  u64 *smpt_start)
+{
+        u64 start =  dma_addr;
+        u64 end = dma_addr + size;
+        int i = 0;
+
+        while (start < end) {
+                ref[i++] = min(mic_smpt_align_high(mdev, start + 1),
+                        end) - start;
+                start = mic_smpt_align_high(mdev, start + 1);
+        }
+
+        if (smpt_start)
+                *smpt_start = mic_smpt_align_low(mdev, dma_addr);
+
+        return i;
+}
+
+/*
+ * mic_to_dma_addr - Converts a MIC address to a DMA address.
+ *
+ * @mdev: pointer to mic_device instance.
+ * @mic_addr: MIC address.
+ *
+ * returns a DMA address.
+ */
+static dma_addr_t
+mic_to_dma_addr(struct mic_device *mdev, dma_addr_t mic_addr)
+{
+        struct mic_smpt_info *smpt_info = mdev->smpt;
+        int spt;
+        dma_addr_t dma_addr;
+
+        if (!mic_is_system_addr(mdev, mic_addr)) {
+                dev_err(mdev->sdev->parent,
+                        "mic_addr is invalid. mic_addr = 0x%llx\n", mic_addr);
+                return -EINVAL;
+        }
+        spt = mic_sys_addr_to_smpt(mdev, mic_addr);
+        dma_addr = smpt_info->entry[spt].dma_addr +
+                mic_smpt_offset(mdev, mic_addr);
+        return dma_addr;
+}
+
+/**
+ * mic_map - Maps a DMA address to a MIC physical address.
+ *
+ * @mdev: pointer to mic_device instance.
+ * @dma_addr: DMA address.
+ * @size: Size of the region to be mapped.
+ *
+ * This API converts the DMA address provided to a DMA address understood
+ * by MIC. Caller should check for errors by calling mic_map_error(..).
+ *
+ * returns DMA address as required by MIC.
+ */
+dma_addr_t mic_map(struct mic_device *mdev, dma_addr_t dma_addr, size_t size)
+{
+        dma_addr_t mic_addr = 0;
+        int num_entries;
+        s64 *ref;
+        u64 smpt_start;
+
+        if (!size || size > mic_max_system_memory(mdev))
+                return mic_addr;
+
+        ref = kmalloc(mdev->smpt->info.num_reg * sizeof(s64), GFP_KERNEL);
+        if (!ref)
+                return mic_addr;
+
+        num_entries = mic_get_smpt_ref_count(mdev, dma_addr, size,
+                ref, &smpt_start);
+
+        /* Set the smpt table appropriately and get 16G aligned mic address */
+        mic_addr = mic_smpt_op(mdev, smpt_start, num_entries, ref, size);
+
+        kfree(ref);
+
+        /*
+         * If mic_addr is zero then its an error case
+         * since mic_addr can never be zero.
+         * else generate mic_addr by adding the 16G offset in dma_addr
+         */
+        if (!mic_addr && MIC_FAMILY_X100 == mdev->family) {
+                dev_err(mdev->sdev->parent,
+                        "mic_map failed dma_addr 0x%llx size 0x%lx\n",
+                        dma_addr, size);
+                return mic_addr;
+        } else {
+                return mic_addr + mic_smpt_offset(mdev, dma_addr);
+        }
+}
+
+/**
+ * mic_unmap - Unmaps a MIC physical address.
+ *
+ * @mdev: pointer to mic_device instance.
+ * @mic_addr: MIC physical address.
+ * @size: Size of the region to be unmapped.
+ *
+ * This API unmaps the mappings created by mic_map(..).
+ *
+ * returns None.
+ */
+void mic_unmap(struct mic_device *mdev, dma_addr_t mic_addr, size_t size)
+{
+        struct mic_smpt_info *smpt_info = mdev->smpt;
+        s64 *ref;
+        int num_smpt;
+        int spt;
+        int i;
+        unsigned long flags;
+
+        if (!size)
+                return;
+
+        if (!mic_is_system_addr(mdev, mic_addr)) {
+                dev_err(mdev->sdev->parent,
+                        "invalid address: 0x%llx\n", mic_addr);
+                return;
+        }
+
+        spt = mic_sys_addr_to_smpt(mdev, mic_addr);
+        ref = kmalloc(mdev->smpt->info.num_reg * sizeof(s64), GFP_KERNEL);
+        if (!ref)
+                return;
+
+        /* Get number of smpt entries to be mapped, ref count array */
+        num_smpt = mic_get_smpt_ref_count(mdev, mic_addr, size, ref, NULL);
+
+        spin_lock_irqsave(&smpt_info->smpt_lock, flags);
+        smpt_info->unmap_count++;
+        smpt_info->ref_count -= (s64)size;
+
+        for (i = spt; i < spt + num_smpt; i++) {
+                smpt_info->entry[i].ref_count -= ref[i - spt];
+                if (smpt_info->entry[i].ref_count < 0)
+                        dev_warn(mdev->sdev->parent,
+                                 "ref count for entry %d is negative\n", i);
+        }
+        spin_unlock_irqrestore(&smpt_info->smpt_lock, flags);
+        kfree(ref);
+}
+
+/**
+ * mic_map_single - Maps a virtual address to a MIC physical address.
+ *
+ * @mdev: pointer to mic_device instance.
+ * @va: Kernel direct mapped virtual address.
+ * @size: Size of the region to be mapped.
+ *
+ * This API calls pci_map_single(..) for the direct mapped virtual address
+ * and then converts the DMA address provided to a DMA address understood
+ * by MIC. Caller should check for errors by calling mic_map_error(..).
+ *
+ * returns DMA address as required by MIC.
+ */
+dma_addr_t mic_map_single(struct mic_device *mdev, void *va, size_t size)
+{
+        dma_addr_t mic_addr = 0;
+        struct pci_dev *pdev = container_of(mdev->sdev->parent,
+                struct pci_dev, dev);
+        dma_addr_t dma_addr =
+                pci_map_single(pdev, va, size, PCI_DMA_BIDIRECTIONAL);
+
+        if (!pci_dma_mapping_error(pdev, dma_addr)) {
+                mic_addr = mic_map(mdev, dma_addr, size);
+                if (!mic_addr) {
+                        dev_err(mdev->sdev->parent,
+                                "mic_map failed dma_addr 0x%llx size 0x%lx\n",
+                                dma_addr, size);
+                        pci_unmap_single(pdev, dma_addr,
+                                         size, PCI_DMA_BIDIRECTIONAL);
+                }
+        }
+        return mic_addr;
+}
+
+/**
+ * mic_unmap_single - Unmaps a MIC physical address.
+ *
+ * @mdev: pointer to mic_device instance.
+ * @mic_addr: MIC physical address.
+ * @size: Size of the region to be unmapped.
+ *
+ * This API unmaps the mappings created by mic_map_single(..).
+ *
+ * returns None.
+ */
+void
+mic_unmap_single(struct mic_device *mdev, dma_addr_t mic_addr, size_t size)
+{
+        struct pci_dev *pdev = container_of(mdev->sdev->parent,
+                struct pci_dev, dev);
+        dma_addr_t dma_addr = mic_to_dma_addr(mdev, mic_addr);
+        mic_unmap(mdev, mic_addr, size);
+        pci_unmap_single(pdev, dma_addr, size, PCI_DMA_BIDIRECTIONAL);
+}
+
+/**
+ * mic_smpt_init - Initialize MIC System Memory Page Tables.
+ *
+ * @mdev: pointer to mic_device instance.
+ *
+ * returns 0 for success and -errno for error.
+ */
+int mic_smpt_init(struct mic_device *mdev)
+{
+        int i, err = 0;
+        dma_addr_t dma_addr;
+        struct mic_smpt_info *smpt_info;
+
+        mdev->smpt = kmalloc(sizeof(*mdev->smpt), GFP_KERNEL);
+        if (!mdev->smpt)
+                return -ENOMEM;
+
+        smpt_info = mdev->smpt;
+        mdev->smpt_ops->init(mdev);
+        smpt_info->entry = kmalloc_array(smpt_info->info.num_reg,
+                                         sizeof(*smpt_info->entry), GFP_KERNEL);
+        if (!smpt_info->entry) {
+                err = -ENOMEM;
+                goto free_smpt;
+        }
+        spin_lock_init(&smpt_info->smpt_lock);
+        for (i = 0; i < smpt_info->info.num_reg; i++) {
+                dma_addr = i * smpt_info->info.page_size;
+                smpt_info->entry[i].dma_addr = dma_addr;
+                smpt_info->entry[i].ref_count = 0;
+                mdev->smpt_ops->set(mdev, dma_addr, i);
+        }
+        smpt_info->ref_count = 0;
+        smpt_info->map_count = 0;
+        smpt_info->unmap_count = 0;
+        return 0;
+free_smpt:
+        kfree(smpt_info);
+        return err;
+}
+
+/**
+ * mic_smpt_uninit - UnInitialize MIC System Memory Page Tables.
+ *
+ * @mdev: pointer to mic_device instance.
+ *
+ * returns None.
+ */
+void mic_smpt_uninit(struct mic_device *mdev)
+{
+        struct mic_smpt_info *smpt_info = mdev->smpt;
+        int i;
+
+        dev_dbg(mdev->sdev->parent,
+                "nodeid %d SMPT ref count %lld map %lld unmap %lld\n",
+                mdev->id, smpt_info->ref_count,
+                smpt_info->map_count, smpt_info->unmap_count);
+
+        for (i = 0; i < smpt_info->info.num_reg; i++) {
+                dev_dbg(mdev->sdev->parent,
+                        "SMPT entry[%d] dma_addr = 0x%llx ref_count = %lld\n",
+                        i, smpt_info->entry[i].dma_addr,
+                        smpt_info->entry[i].ref_count);
+                if (smpt_info->entry[i].ref_count)
+                        dev_warn(mdev->sdev->parent,
+                                 "ref count for entry %d is not zero\n", i);
+        }
+        kfree(smpt_info->entry);
+        kfree(smpt_info);
+}
+
+/**
+ * mic_smpt_restore - Restore MIC System Memory Page Tables.
+ *
+ * @mdev: pointer to mic_device instance.
+ *
+ * Restore the SMPT registers to values previously stored in the
+ * SW data structures. Some MIC steppings lose register state
+ * across resets and this API should be called for performing
+ * a restore operation if required.
+ *
+ * returns None.
+ */
+void mic_smpt_restore(struct mic_device *mdev)
+{
+        int i;
+        dma_addr_t dma_addr;
+
+        for (i = 0; i < mdev->smpt->info.num_reg; i++) {
+                dma_addr = mdev->smpt->entry[i].dma_addr;
+                mdev->smpt_ops->set(mdev, dma_addr, i);
+        }
+}