dmaengine: dma40: Add support to split up large elements

The maximum transfer size of the stedma40 is (64k-1) x data-width.
If the transfer size of one element exceeds this limit
the job is split up and sent as linked transfer.

Signed-off-by: Per Forlin <per.forlin@linaro.org>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>

1 files changed, 172 insertions, 74 deletions

diff --git a/drivers/dma/ste_dma40_ll.c b/drivers/dma/ste_dma40_ll.c
index 8557cb88b255..0b096a38322d 100644
--- a/drivers/dma/ste_dma40_ll.c
+++ b/drivers/dma/ste_dma40_ll.c
@@ -1,6 +1,6 @@
 /*
  * Copyright (C) ST-Ericsson SA 2007-2010
- * Author: Per Friden <per.friden@stericsson.com> for ST-Ericsson
+ * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson
  * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson
  * License terms: GNU General Public License (GPL) version 2
  */
@@ -122,15 +122,15 @@ void d40_phy_cfg(struct stedma40_chan_cfg *cfg,
         *dst_cfg = dst;
 }
 
-int d40_phy_fill_lli(struct d40_phy_lli *lli,
-                     dma_addr_t data,
-                     u32 data_size,
-                     int psize,
-                     dma_addr_t next_lli,
-                     u32 reg_cfg,
-                     bool term_int,
-                     u32 data_width,
-                     bool is_device)
+static int d40_phy_fill_lli(struct d40_phy_lli *lli,
+                            dma_addr_t data,
+                            u32 data_size,
+                            int psize,
+                            dma_addr_t next_lli,
+                            u32 reg_cfg,
+                            bool term_int,
+                            u32 data_width,
+                            bool is_device)
 {
         int num_elems;
 
@@ -139,13 +139,6 @@ int d40_phy_fill_lli(struct d40_phy_lli *lli,
         else
                 num_elems = 2 << psize;
 
-        /*
-         * Size is 16bit. data_width is 8, 16, 32 or 64 bit
-         * Block large than 64 KiB must be split.
-         */
-        if (data_size > (0xffff << data_width))
-                return -EINVAL;
-
         /* Must be aligned */
         if (!IS_ALIGNED(data, 0x1 << data_width))
                 return -EINVAL;
@@ -187,55 +180,118 @@ int d40_phy_fill_lli(struct d40_phy_lli *lli,
         return 0;
 }
 
+static int d40_seg_size(int size, int data_width1, int data_width2)
+{
+        u32 max_w = max(data_width1, data_width2);
+        u32 min_w = min(data_width1, data_width2);
+        u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE << min_w, 1 << max_w);
+
+        if (seg_max > STEDMA40_MAX_SEG_SIZE)
+                seg_max -= (1 << max_w);
+
+        if (size <= seg_max)
+                return size;
+
+        if (size <= 2 * seg_max)
+                return ALIGN(size / 2, 1 << max_w);
+
+        return seg_max;
+}
+
+struct d40_phy_lli *d40_phy_buf_to_lli(struct d40_phy_lli *lli,
+                                       dma_addr_t addr,
+                                       u32 size,
+                                       int psize,
+                                       dma_addr_t lli_phys,
+                                       u32 reg_cfg,
+                                       bool term_int,
+                                       u32 data_width1,
+                                       u32 data_width2,
+                                       bool is_device)
+{
+        int err;
+        dma_addr_t next = lli_phys;
+        int size_rest = size;
+        int size_seg = 0;
+
+        do {
+                size_seg = d40_seg_size(size_rest, data_width1, data_width2);
+                size_rest -= size_seg;
+
+                if (term_int && size_rest == 0)
+                        next = 0;
+                else
+                        next = ALIGN(next + sizeof(struct d40_phy_lli),
+                                     D40_LLI_ALIGN);
+
+                err = d40_phy_fill_lli(lli,
+                                       addr,
+                                       size_seg,
+                                       psize,
+                                       next,
+                                       reg_cfg,
+                                       !next,
+                                       data_width1,
+                                       is_device);
+
+                if (err)
+                        goto err;
+
+                lli++;
+                if (!is_device)
+                        addr += size_seg;
+        } while (size_rest);
+
+        return lli;
+
+ err:
+        return NULL;
+}
+
 int d40_phy_sg_to_lli(struct scatterlist *sg,
                       int sg_len,
                       dma_addr_t target,
-                      struct d40_phy_lli *lli,
+                      struct d40_phy_lli *lli_sg,
                       dma_addr_t lli_phys,
                       u32 reg_cfg,
-                      u32 data_width,
+                      u32 data_width1,
+                      u32 data_width2,
                       int psize)
 {
         int total_size = 0;
         int i;
         struct scatterlist *current_sg = sg;
-        dma_addr_t next_lli_phys;
         dma_addr_t dst;
-        int err = 0;
+        struct d40_phy_lli *lli = lli_sg;
+        dma_addr_t l_phys = lli_phys;
 
         for_each_sg(sg, current_sg, sg_len, i) {
 
                 total_size += sg_dma_len(current_sg);
 
-                /* If this scatter list entry is the last one, no next link */
-                if (sg_len - 1 == i)
-                        next_lli_phys = 0;
-                else
-                        next_lli_phys = ALIGN(lli_phys + (i + 1) *
-                                              sizeof(struct d40_phy_lli),
-                                              D40_LLI_ALIGN);
-
                 if (target)
                         dst = target;
                 else
                         dst = sg_phys(current_sg);
 
-                err = d40_phy_fill_lli(&lli[i],
-                                       dst,
-                                       sg_dma_len(current_sg),
-                                       psize,
-                                       next_lli_phys,
-                                       reg_cfg,
-                                       !next_lli_phys,
-                                       data_width,
-                                       target == dst);
-                if (err)
-                        goto err;
+                l_phys = ALIGN(lli_phys + (lli - lli_sg) *
+                               sizeof(struct d40_phy_lli), D40_LLI_ALIGN);
+
+                lli = d40_phy_buf_to_lli(lli,
+                                         dst,
+                                         sg_dma_len(current_sg),
+                                         psize,
+                                         l_phys,
+                                         reg_cfg,
+                                         sg_len - 1 == i,
+                                         data_width1,
+                                         data_width2,
+                                         target == dst);
+                if (lli == NULL)
+                        return -EINVAL;
         }
 
         return total_size;
-err:
-        return err;
 }
 
 
@@ -315,17 +371,20 @@ void d40_log_lli_lcla_write(struct d40_log_lli *lcla,
         writel(lli_dst->lcsp13, &lcla[1].lcsp13);
 }
 
-void d40_log_fill_lli(struct d40_log_lli *lli,
-                      dma_addr_t data, u32 data_size,
-                      u32 reg_cfg,
-                      u32 data_width,
-                      bool addr_inc)
+static void d40_log_fill_lli(struct d40_log_lli *lli,
+                             dma_addr_t data, u32 data_size,
+                             u32 reg_cfg,
+                             u32 data_width,
+                             bool addr_inc)
 {
         lli->lcsp13 = reg_cfg;
 
         /* The number of elements to transfer */
         lli->lcsp02 = ((data_size >> data_width) <<
                        D40_MEM_LCSP0_ECNT_POS) & D40_MEM_LCSP0_ECNT_MASK;
+
+        BUG_ON((data_size >> data_width) > STEDMA40_MAX_SEG_SIZE);
+
         /* 16 LSBs address of the current element */
         lli->lcsp02 |= data & D40_MEM_LCSP0_SPTR_MASK;
         /* 16 MSBs address of the current element */
@@ -348,55 +407,94 @@ int d40_log_sg_to_dev(struct scatterlist *sg,
         int total_size = 0;
         struct scatterlist *current_sg = sg;
         int i;
+        struct d40_log_lli *lli_src = lli->src;
+        struct d40_log_lli *lli_dst = lli->dst;
 
         for_each_sg(sg, current_sg, sg_len, i) {
                 total_size += sg_dma_len(current_sg);
 
                 if (direction == DMA_TO_DEVICE) {
-                        d40_log_fill_lli(&lli->src[i],
-                                         sg_phys(current_sg),
-                                         sg_dma_len(current_sg),
-                                         lcsp->lcsp1, src_data_width,
-                                         true);
-                        d40_log_fill_lli(&lli->dst[i],
-                                         dev_addr,
-                                         sg_dma_len(current_sg),
-                                         lcsp->lcsp3, dst_data_width,
-                                         false);
+                        lli_src =
+                                d40_log_buf_to_lli(lli_src,
+                                                   sg_phys(current_sg),
+                                                   sg_dma_len(current_sg),
+                                                   lcsp->lcsp1, src_data_width,
+                                                   dst_data_width,
+                                                   true);
+                        lli_dst =
+                                d40_log_buf_to_lli(lli_dst,
+                                                   dev_addr,
+                                                   sg_dma_len(current_sg),
+                                                   lcsp->lcsp3, dst_data_width,
+                                                   src_data_width,
+                                                   false);
                 } else {
-                        d40_log_fill_lli(&lli->dst[i],
-                                         sg_phys(current_sg),
-                                         sg_dma_len(current_sg),
-                                         lcsp->lcsp3, dst_data_width,
-                                         true);
-                        d40_log_fill_lli(&lli->src[i],
-                                         dev_addr,
-                                         sg_dma_len(current_sg),
-                                         lcsp->lcsp1, src_data_width,
-                                         false);
+                        lli_dst =
+                                d40_log_buf_to_lli(lli_dst,
+                                                   sg_phys(current_sg),
+                                                   sg_dma_len(current_sg),
+                                                   lcsp->lcsp3, dst_data_width,
+                                                   src_data_width,
+                                                   true);
+                        lli_src =
+                                d40_log_buf_to_lli(lli_src,
+                                                   dev_addr,
+                                                   sg_dma_len(current_sg),
+                                                   lcsp->lcsp1, src_data_width,
+                                                   dst_data_width,
+                                                   false);
                 }
         }
         return total_size;
 }
 
+struct d40_log_lli *d40_log_buf_to_lli(struct d40_log_lli *lli_sg,
+                                       dma_addr_t addr,
+                                       int size,
+                                       u32 lcsp13, /* src or dst*/
+                                       u32 data_width1,
+                                       u32 data_width2,
+                                       bool addr_inc)
+{
+        struct d40_log_lli *lli = lli_sg;
+        int size_rest = size;
+        int size_seg = 0;
+
+        do {
+                size_seg = d40_seg_size(size_rest, data_width1, data_width2);
+                size_rest -= size_seg;
+
+                d40_log_fill_lli(lli,
+                                 addr,
+                                 size_seg,
+                                 lcsp13, data_width1,
+                                 addr_inc);
+                if (addr_inc)
+                        addr += size_seg;
+                lli++;
+        } while (size_rest);
+
+        return lli;
+}
+
 int d40_log_sg_to_lli(struct scatterlist *sg,
                       int sg_len,
                       struct d40_log_lli *lli_sg,
                       u32 lcsp13, /* src or dst*/
-                      u32 data_width)
+                      u32 data_width1, u32 data_width2)
 {
         int total_size = 0;
         struct scatterlist *current_sg = sg;
         int i;
+        struct d40_log_lli *lli = lli_sg;
 
         for_each_sg(sg, current_sg, sg_len, i) {
                 total_size += sg_dma_len(current_sg);
-
-                d40_log_fill_lli(&lli_sg[i],
-                                 sg_phys(current_sg),
-                                 sg_dma_len(current_sg),
-                                 lcsp13, data_width,
-                                 true);
+                lli = d40_log_buf_to_lli(lli,
+                                         sg_phys(current_sg),
+                                         sg_dma_len(current_sg),
+                                         lcsp13,
+                                         data_width1, data_width2, true);
         }
         return total_size;
 }