/*
* Copyright (c) 2006 ARM Ltd.
* Copyright (c) 2010 ST-Ericsson SA
*
* Author: Peter Pearse <peter.pearse@arm.com>
* Author: Linus Walleij <linus.walleij@stericsson.com>
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation; either version 2 of the License, or (at your option)
* any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program; if not, write to the Free Software Foundation, Inc., 59
* Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
* The full GNU General Public License is in this distribution in the file
* called COPYING.
*
* Documentation: ARM DDI 0196G == PL080
* Documentation: ARM DDI 0218E == PL081
* Documentation: S3C6410 User's Manual == PL080S
*
* PL080 & PL081 both have 16 sets of DMA signals that can be routed to any
* channel.
*
* The PL080 has 8 channels available for simultaneous use, and the PL081
* has only two channels. So on these DMA controllers the number of channels
* and the number of incoming DMA signals are two totally different things.
* It is usually not possible to theoretically handle all physical signals,
* so a multiplexing scheme with possible denial of use is necessary.
*
* The PL080 has a dual bus master, PL081 has a single master.
*
* PL080S is a version modified by Samsung and used in S3C64xx SoCs.
* It differs in following aspects:
* - CH_CONFIG register at different offset,
* - separate CH_CONTROL2 register for transfer size,
* - bigger maximum transfer size,
* - 8-word aligned LLI, instead of 4-word, due to extra CCTL2 word,
* - no support for peripheral flow control.
*
* Memory to peripheral transfer may be visualized as
* Get data from memory to DMAC
* Until no data left
* On burst request from peripheral
* Destination burst from DMAC to peripheral
* Clear burst request
* Raise terminal count interrupt
*
* For peripherals with a FIFO:
* Source burst size == half the depth of the peripheral FIFO
* Destination burst size == the depth of the peripheral FIFO
*
* (Bursts are irrelevant for mem to mem transfers - there are no burst
* signals, the DMA controller will simply facilitate its AHB master.)
*
* ASSUMES default (little) endianness for DMA transfers
*
* The PL08x has two flow control settings:
* - DMAC flow control: the transfer size defines the number of transfers
* which occur for the current LLI entry, and the DMAC raises TC at the
* end of every LLI entry. Observed behaviour shows the DMAC listening
* to both the BREQ and SREQ signals (contrary to documented),
* transferring data if either is active. The LBREQ and LSREQ signals
* are ignored.
*
* - Peripheral flow control: the transfer size is ignored (and should be
* zero). The data is transferred from the current LLI entry, until
* after the final transfer signalled by LBREQ or LSREQ. The DMAC
* will then move to the next LLI entry. Unsupported by PL080S.
*/
#include <linux/amba/bus.h>
#include <linux/amba/pl08x.h>
#include <linux/debugfs.h>
#include <linux/delay.h>
#include <linux/device.h>
#include <linux/dmaengine.h>
#include <linux/dmapool.h>
#include <linux/dma-mapping.h>
#include <linux/export.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/pm_runtime.h>
#include <linux/seq_file.h>
#include <linux/slab.h>
#include <linux/amba/pl080.h>
#include "dmaengine.h"
#include "virt-dma.h"
#define DRIVER_NAME "pl08xdmac"
static struct amba_driver pl08x_amba_driver;
struct pl08x_driver_data;
/**
* struct vendor_data - vendor-specific config parameters for PL08x derivatives
* @channels: the number of channels available in this variant
* @dualmaster: whether this version supports dual AHB masters or not.
* @nomadik: whether the channels have Nomadik security extension bits
* that need to be checked for permission before use and some registers are
* missing
* @pl080s: whether this version is a PL080S, which has separate register and
* LLI word for transfer size.
*/
struct vendor_data {
u8 config_offset;
u8 channels;
bool dualmaster;
bool nomadik;
bool pl080s;
u32 max_transfer_size;
};
/**
* struct pl08x_bus_data - information of source or destination
* busses for a transfer
* @addr: current address
* @maxwidth: the maximum width of a transfer on this bus
* @buswidth: the width of this bus in bytes: 1, 2 or 4
*/
struct pl08x_bus_data {
dma_addr_t addr;
u8 maxwidth;
u8 buswidth;
};
#define IS_BUS_ALIGNED(bus) IS_ALIGNED((bus)->addr, (bus)->buswidth)
/**
* struct pl08x_phy_chan - holder for the physical channels
* @id: physical index to this channel
* @lock: a lock to use when altering an instance of this struct
* @serving: the virtual channel currently being served by this physical
* channel
* @locked: channel unavailable for the system, e.g. dedicated to secure
* world
*/
struct pl08x_phy_chan {
unsigned int id;
void __iomem *base;
void __iomem *reg_config;
spinlock_t lock;
struct pl08x_dma_chan *serving;
bool locked;
};
/**
* struct pl08x_sg - structure containing data per sg
* @src_addr: src address of sg
* @dst_addr: dst address of sg
* @len: transfer len in bytes
* @node: node for txd's dsg_list
*/
struct pl08x_sg {
dma_addr_t src_addr;
dma_addr_t dst_addr;
size_t len;
struct list_head node;
};
/**
* struct pl08x_txd - wrapper for struct dma_async_tx_descriptor
* @vd: virtual DMA descriptor
* @dsg_list: list of children sg's
* @llis_bus: DMA memory address (physical) start for the LLIs
* @llis_va: virtual memory address start for the LLIs
* @cctl: control reg values for current txd
* @ccfg: config reg values for current txd
* @done: this marks completed descriptors, which should not have their
* mux released.
* @cyclic: indicate cyclic transfers
*/
struct pl08x_txd {
struct virt_dma_desc vd;
struct list_head dsg_list;
dma_addr_t llis_bus;
u32 *llis_va;
/* Default cctl value for LLIs */
u32 cctl;
/*
* Settings to be put into the physical channel when we
* trigger this txd. Other registers are in llis_va[0].
*/
u32 ccfg;
bool done;
bool cyclic;
};
/**
* struct pl08x_dma_chan_state - holds the PL08x specific virtual channel
* states
* @PL08X_CHAN_IDLE: the channel is idle
* @PL08X_CHAN_RUNNING: the channel has allocated a physical transport
* channel and is running a transfer on it
* @PL08X_CHAN_PAUSED: the channel has allocated a physical transport
* channel, but the transfer is currently paused
* @PL08X_CHAN_WAITING: the channel is waiting for a physical transport
* channel to become available (only pertains to memcpy channels)
*/
enum pl08x_dma_chan_state {
PL08X_CHAN_IDLE,
PL08X_CHAN_RUNNING,
PL08X_CHAN_PAUSED,
PL08X_CHAN_WAITING,
};
/**
* struct pl08x_dma_chan - this structure wraps a DMA ENGINE channel
* @vc: wrappped virtual channel
* @phychan: the physical channel utilized by this channel, if there is one
* @name: name of channel
* @cd: channel platform data
* @runtime_addr: address for RX/TX according to the runtime config
* @at: active transaction on this channel
* @lock: a lock for this channel data
* @host: a pointer to the host (internal use)
* @state: whether the channel is idle, paused, running etc
* @slave: whether this channel is a device (slave) or for memcpy
* @signal: the physical DMA request signal which this channel is using
* @mux_use: count of descriptors using this DMA request signal setting
*/
struct pl08x_dma_chan {
struct virt_dma_chan vc;
struct pl08x_phy_chan *phychan;
const char *name;
const struct pl08x_channel_data *cd;
struct dma_slave_config cfg;
struct pl08x_txd *at;
struct pl08x_driver_data *host;
enum pl08x_dma_chan_state state;
bool slave;
int signal;
unsigned mux_use;
};
/**
* struct pl08x_driver_data - the local state holder for the PL08x
* @slave: slave engine for this instance
* @memcpy: memcpy engine for this instance
* @base: virtual memory base (remapped) for the PL08x
* @adev: the corresponding AMBA (PrimeCell) bus entry
* @vd: vendor data for this PL08x variant
* @pd: platform data passed in from the platform/machine
* @phy_chans: array of data for the physical channels
* @pool: a pool for the LLI descriptors
* @lli_buses: bitmask to or in to LLI pointer selecting AHB port for LLI
* fetches
* @mem_buses: set to indicate memory transfers on AHB2.
* @lock: a spinlock for this struct
*/
struct pl08x_driver_data {
struct dma_device slave;
struct dma_device memcpy;
void __iomem *base;
struct amba_device *adev;
const struct vendor_data *vd;
struct pl08x_platform_data *pd;
struct pl08x_phy_chan *phy_chans;
struct dma_pool *pool;
u8 lli_buses;
u8 mem_buses;
u8 lli_words;
};
/*
* PL08X specific defines
*/
/* The order of words in an LLI. */
#define PL080_LLI_SRC 0
#define PL080_LLI_DST 1
#define PL080_LLI_LLI 2
#define PL080_LLI_CCTL 3
#define PL080S_LLI_CCTL2 4
/* Total words in an LLI. */
#define PL080_LLI_WORDS 4
#define PL080S_LLI_WORDS 8
/*
* Number of LLIs in each LLI buffer allocated for one transfer
* (maximum times we call dma_pool_alloc on this pool without freeing)
*/
#define MAX_NUM_TSFR_LLIS 512
#define PL08X_ALIGN 8
static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan)
{
return container_of(chan, struct pl08x_dma_chan, vc.chan);
}
static inline struct pl08x_txd *to_pl08x_txd(struct dma_async_tx_descriptor *tx)
{
return container_of(tx, struct pl08x_txd, vd.tx);
}
/*
* Mux handling.
*
* This gives us the DMA request input to the PL08x primecell which the
* peripheral described by the channel data will be routed to, possibly
* via a board/SoC specific external MUX. One important point to note
* here is that this does not depend on the physical channel.
*/
static int pl08x_request_mux(struct pl08x_dma_chan *plchan)
{
const struct pl08x_platform_data *pd = plchan->host->pd;
int ret;
if (plchan->mux_use++ == 0 && pd->get_xfer_signal) {
ret = pd->get_xfer_signal(plchan->cd);
if (ret < 0) {
plchan->mux_use = 0;
return ret;
}
plchan->signal = ret;
}
return 0;
}
static void pl08x_release_mux(struct pl08x_dma_chan *plchan)
{
const struct pl08x_platform_data *pd = plchan->host->pd;
if (plchan->signal >= 0) {
WARN_ON(plchan->mux_use == 0);
if (--plchan->mux_use == 0 && pd->put_xfer_signal) {
pd->put_xfer_signal(plchan->cd, plchan->signal);
plchan->signal = -1;
}
}
}
/*
* Physical channel handling
*/
/* Whether a certain channel is busy or not */
static int pl08x_phy_channel_busy(struct pl08x_phy_chan *ch)
{
unsigned int val;
val = readl(ch->reg_config);
return val & PL080_CONFIG_ACTIVE;
}
static void pl08x_write_lli(struct pl08x_driver_data *pl08x,
struct pl08x_phy_chan *phychan, const u32 *lli, u32 ccfg)
{
if (pl08x->vd->pl080s)
dev_vdbg(&pl08x->adev->dev,
"WRITE channel %d: csrc=0x%08x, cdst=0x%08x, "
"clli=0x%08x, cctl=0x%08x, cctl2=0x%08x, ccfg=0x%08x\n",
phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST],
lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL],
lli[PL080S_LLI_CCTL2], ccfg);
else
dev_vdbg(&pl08x->adev->dev,
"WRITE channel %d: csrc=0x%08x, cdst=0x%08x, "
"clli=0x%08x, cctl=0x%08x, ccfg=0x%08x\n",
phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST],
lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL], ccfg);
writel_relaxed(lli[PL080_LLI_SRC], phychan->base + PL080_CH_SRC_ADDR);
writel_relaxed(lli[PL080_LLI_DST], phychan->base + PL080_CH_DST_ADDR);
writel_relaxed(lli[PL080_LLI_LLI], phychan->base + PL080_CH_LLI);
writel_relaxed(lli[PL080_LLI_CCTL], phychan->base + PL080_CH_CONTROL);
if (pl08x->vd->pl080s)
writel_relaxed(lli[PL080S_LLI_CCTL2],
phychan->base + PL080S_CH_CONTROL2);
writel(ccfg, phychan->reg_config);
}
/*
* Set the initial DMA register values i.e. those for the first LLI
* The next LLI pointer and the configuration interrupt bit have
* been set when the LLIs were constructed. Poke them into the hardware
* and start the transfer.
*/
static void pl08x_start_next_txd(struct pl08x_dma_chan *plchan)
{
struct pl08x_driver_data *pl08x = plchan->host;
struct pl08x_phy_chan *phychan = plchan->phychan;
struct virt_dma_desc *vd = vchan_next_desc(&plchan->vc);
struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
u32 val;
list_del(&txd->vd.node);
plchan->at = txd;
/* Wait for channel inactive */
while (pl08x_phy_channel_busy(phychan))
cpu_relax();
pl08x_write_lli(pl08x, phychan, &txd->llis_va[0], txd->ccfg);
/* Enable the DMA channel */
/* Do not access config register until channel shows as disabled */
while (readl(pl08x->base + PL080_EN_CHAN) & (1 << phychan->id))
cpu_relax();
/* Do not access config register until channel shows as inactive */
val = readl(phychan->reg_config);
while ((val & PL080_CONFIG_ACTIVE) || (val & PL080_CONFIG_ENABLE))
val = readl(phychan->reg_config);
writel(val | PL080_CONFIG_ENABLE, phychan->reg_config);
}
/*
* Pause the channel by setting the HALT bit.
*
* For M->P transfers, pause the DMAC first and then stop the peripheral -
* the FIFO can only drain if the peripheral is still requesting data.
* (note: this can still timeout if the DMAC FIFO never drains of data.)
*
* For P->M transfers, disable the peripheral first to stop it filling
* the DMAC FIFO, and then pause the DMAC.
*/
static void pl08x_pause_phy_chan(struct pl08x_phy_chan *ch)
{
u32 val;
int timeout;
/* Set the HALT bit and wait for the FIFO to drain */
val = readl(ch->reg_config);
val |= PL080_CONFIG_HALT;
writel(val, ch->reg_config);
/* Wait for channel inactive */
for (timeout = 1000; timeout; timeout--) {
if (!pl08x_phy_channel_busy(ch))
break;
udelay(1);
}
if (pl08x_phy_channel_busy(ch))
pr_err("pl08x: channel%u timeout waiting for pause\n", ch->id);
}
static void pl08x_resume_phy_chan(struct pl08x_phy_chan *ch)
{
u32 val;
/* Clear the HALT bit */
val = readl(ch->reg_config);
val &= ~PL080_CONFIG_HALT;
writel(val, ch->reg_config);
}
/*
* pl08x_terminate_phy_chan() stops the channel, clears the FIFO and
* clears any pending interrupt status. This should not be used for
* an on-going transfer, but as a method of shutting down a channel
* (eg, when it's no longer used) or terminating a transfer.
*/
static void pl08x_terminate_phy_chan(struct pl08x_driver_data *pl08x,
struct pl08x_phy_chan *ch)
{
u32 val = readl(ch->reg_config);
val &= ~(PL080_CONFIG_ENABLE | PL080_CONFIG_ERR_IRQ_MASK |
PL080_CONFIG_TC_IRQ_MASK);
writel(val, ch->reg_config);
writel(1 << ch->id, pl08x->base + PL080_ERR_CLEAR);
writel(1 << ch->id, pl08x->base + PL080_TC_CLEAR);
}
static inline u32 get_bytes_in_cctl(u32 cctl)
{
/* The source width defines the number of bytes */
u32 bytes = cctl & PL080_CONTROL_TRANSFER_SIZE_MASK;
cctl &= PL080_CONTROL_SWIDTH_MASK;
switch (cctl >> PL080_CONTROL_SWIDTH_SHIFT) {
case PL080_WIDTH_8BIT:
break;
case PL080_WIDTH_16BIT:
bytes *= 2;
break;
case PL080_WIDTH_32BIT:
bytes *= 4;
break;
}
return bytes;
}
static inline u32 get_bytes_in_cctl_pl080s(u32 cctl, u32 cctl1)
{
/* The source width defines the number of bytes */
u32 bytes = cctl1 & PL080S_CONTROL_TRANSFER_SIZE_MASK;
cctl &= PL080_CONTROL_SWIDTH_MASK;
switch (cctl >> PL080_CONTROL_SWIDTH_SHIFT) {
case PL080_WIDTH_8BIT:
break;
case PL080_WIDTH_16BIT:
bytes *= 2;
break;
case PL080_WIDTH_32BIT:
bytes *= 4;
break;
}
return bytes;
}
/* The channel should be paused when calling this */
static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan)
{
struct pl08x_driver_data *pl08x = plchan->host;
const u32 *llis_va, *llis_va_limit;
struct pl08x_phy_chan *ch;
dma_addr_t llis_bus;
struct pl08x_txd *txd;
u32 llis_max_words;
size_t bytes;
u32 clli;
ch = plchan->phychan;
txd = plchan->at;
if (!ch || !txd)
return 0;