aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDave Jiang <dave.jiang@intel.com>2015-08-11 11:48:32 -0400
committerVinod Koul <vinod.koul@intel.com>2015-08-17 04:07:30 -0400
commit885b201056e942f7deb66496b5c501d2a35d6c04 (patch)
tree7d82fe9fd8b0a6f257a531dfe8c1dcdfc84b768c
parent55f878ec47e3ab560a046c9030a97b1048b74e8b (diff)
dmaengine: ioatdma: remove dma_v2.*
Clean out dma_v2 and remove ioat2 calls since we are moving everything to just ioat. Signed-off-by: Dave Jiang <dave.jiang@intel.com> Acked-by: Dan Williams <dan.j.williams@intel.com> Signed-off-by: Vinod Koul <vinod.koul@intel.com>
Diffstat
-rw-r--r--drivers/dma/ioat/Makefile2
-rw-r--r--drivers/dma/ioat/dca.c1
-rw-r--r--drivers/dma/ioat/dma.c667
-rw-r--r--drivers/dma/ioat/dma.h119
-rw-r--r--drivers/dma/ioat/dma_v2.c695
-rw-r--r--drivers/dma/ioat/dma_v2.h143
-rw-r--r--drivers/dma/ioat/dma_v3.c63
-rw-r--r--drivers/dma/ioat/pci.c15
8 files changed, 816 insertions, 889 deletions
diff --git a/drivers/dma/ioat/Makefile b/drivers/dma/ioat/Makefile
index 0ff7270af25b..655df9188387 100644
--- a/drivers/dma/ioat/Makefile
+++ b/drivers/dma/ioat/Makefile
@@ -1,2 +1,2 @@
1obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o 1obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o
2ioatdma-y := pci.o dma.o dma_v2.o dma_v3.o dca.o 2ioatdma-y := pci.o dma.o dma_v3.o dca.o
diff --git a/drivers/dma/ioat/dca.c b/drivers/dma/ioat/dca.c
index 540d94ce9736..f2b9a421985a 100644
--- a/drivers/dma/ioat/dca.c
+++ b/drivers/dma/ioat/dca.c
@@ -31,7 +31,6 @@
31 31
32#include "dma.h" 32#include "dma.h"
33#include "registers.h" 33#include "registers.h"
34#include "dma_v2.h"
35 34
36/* 35/*
37 * Bit 7 of a tag map entry is the "valid" bit, if it is set then bits 0:6 36 * Bit 7 of a tag map entry is the "valid" bit, if it is set then bits 0:6
diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
index 3cf2639fb06a..02b5c04dea8a 100644
--- a/drivers/dma/ioat/dma.c
+++ b/drivers/dma/ioat/dma.c
@@ -41,6 +41,19 @@ int ioat_pending_level = 4;
41module_param(ioat_pending_level, int, 0644); 41module_param(ioat_pending_level, int, 0644);
42MODULE_PARM_DESC(ioat_pending_level, 42MODULE_PARM_DESC(ioat_pending_level,
43 "high-water mark for pushing ioat descriptors (default: 4)"); 43 "high-water mark for pushing ioat descriptors (default: 4)");
44int ioat_ring_alloc_order = 8;
45module_param(ioat_ring_alloc_order, int, 0644);
46MODULE_PARM_DESC(ioat_ring_alloc_order,
47 "ioat+: allocate 2^n descriptors per channel (default: 8 max: 16)");
48static int ioat_ring_max_alloc_order = IOAT_MAX_ORDER;
49module_param(ioat_ring_max_alloc_order, int, 0644);
50MODULE_PARM_DESC(ioat_ring_max_alloc_order,
51 "ioat+: upper limit for ring size (default: 16)");
52static char ioat_interrupt_style[32] = "msix";
53module_param_string(ioat_interrupt_style, ioat_interrupt_style,
54 sizeof(ioat_interrupt_style), 0644);
55MODULE_PARM_DESC(ioat_interrupt_style,
56 "set ioat interrupt style: msix (default), msi, intx");
44 57
45/** 58/**
46 * ioat_dma_do_interrupt - handler used for single vector interrupt mode 59 * ioat_dma_do_interrupt - handler used for single vector interrupt mode
@@ -314,12 +327,6 @@ out:
314 return err; 327 return err;
315} 328}
316 329
317static char ioat_interrupt_style[32] = "msix";
318module_param_string(ioat_interrupt_style, ioat_interrupt_style,
319 sizeof(ioat_interrupt_style), 0644);
320MODULE_PARM_DESC(ioat_interrupt_style,
321 "set ioat interrupt style: msix (default), msi, intx");
322
323/** 330/**
324 * ioat_dma_setup_interrupts - setup interrupt handler 331 * ioat_dma_setup_interrupts - setup interrupt handler
325 * @ioat_dma: ioat dma device 332 * @ioat_dma: ioat dma device
@@ -577,3 +584,651 @@ void ioat_dma_remove(struct ioatdma_device *ioat_dma)
577 584
578 INIT_LIST_HEAD(&dma->channels); 585 INIT_LIST_HEAD(&dma->channels);
579} 586}
587
588void __ioat_issue_pending(struct ioatdma_chan *ioat_chan)
589{
590 ioat_chan->dmacount += ioat_ring_pending(ioat_chan);
591 ioat_chan->issued = ioat_chan->head;
592 writew(ioat_chan->dmacount,
593 ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
594 dev_dbg(to_dev(ioat_chan),
595 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
596 __func__, ioat_chan->head, ioat_chan->tail,
597 ioat_chan->issued, ioat_chan->dmacount);
598}
599
600void ioat_issue_pending(struct dma_chan *c)
601{
602 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
603
604 if (ioat_ring_pending(ioat_chan)) {
605 spin_lock_bh(&ioat_chan->prep_lock);
606 __ioat_issue_pending(ioat_chan);
607 spin_unlock_bh(&ioat_chan->prep_lock);
608 }
609}
610
611/**
612 * ioat_update_pending - log pending descriptors
613 * @ioat: ioat+ channel
614 *
615 * Check if the number of unsubmitted descriptors has exceeded the
616 * watermark. Called with prep_lock held
617 */
618static void ioat_update_pending(struct ioatdma_chan *ioat_chan)
619{
620 if (ioat_ring_pending(ioat_chan) > ioat_pending_level)
621 __ioat_issue_pending(ioat_chan);
622}
623
624static void __ioat_start_null_desc(struct ioatdma_chan *ioat_chan)
625{
626 struct ioat_ring_ent *desc;
627 struct ioat_dma_descriptor *hw;
628
629 if (ioat_ring_space(ioat_chan) < 1) {
630 dev_err(to_dev(ioat_chan),
631 "Unable to start null desc - ring full\n");
632 return;
633 }
634
635 dev_dbg(to_dev(ioat_chan),
636 "%s: head: %#x tail: %#x issued: %#x\n",
637 __func__, ioat_chan->head, ioat_chan->tail, ioat_chan->issued);
638 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head);
639
640 hw = desc->hw;
641 hw->ctl = 0;
642 hw->ctl_f.null = 1;
643 hw->ctl_f.int_en = 1;
644 hw->ctl_f.compl_write = 1;
645 /* set size to non-zero value (channel returns error when size is 0) */
646 hw->size = NULL_DESC_BUFFER_SIZE;
647 hw->src_addr = 0;
648 hw->dst_addr = 0;
649 async_tx_ack(&desc->txd);
650 ioat_set_chainaddr(ioat_chan, desc->txd.phys);
651 dump_desc_dbg(ioat_chan, desc);
652 /* make sure descriptors are written before we submit */
653 wmb();
654 ioat_chan->head += 1;
655 __ioat_issue_pending(ioat_chan);
656}
657
658static void ioat_start_null_desc(struct ioatdma_chan *ioat_chan)
659{
660 spin_lock_bh(&ioat_chan->prep_lock);
661 __ioat_start_null_desc(ioat_chan);
662 spin_unlock_bh(&ioat_chan->prep_lock);
663}
664
665void __ioat_restart_chan(struct ioatdma_chan *ioat_chan)
666{
667 /* set the tail to be re-issued */
668 ioat_chan->issued = ioat_chan->tail;
669 ioat_chan->dmacount = 0;
670 set_bit(IOAT_COMPLETION_PENDING, &ioat_chan->state);
671 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
672
673 dev_dbg(to_dev(ioat_chan),
674 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
675 __func__, ioat_chan->head, ioat_chan->tail,
676 ioat_chan->issued, ioat_chan->dmacount);
677
678 if (ioat_ring_pending(ioat_chan)) {
679 struct ioat_ring_ent *desc;
680
681 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail);
682 ioat_set_chainaddr(ioat_chan, desc->txd.phys);
683 __ioat_issue_pending(ioat_chan);
684 } else
685 __ioat_start_null_desc(ioat_chan);
686}
687
688int ioat_quiesce(struct ioatdma_chan *ioat_chan, unsigned long tmo)
689{
690 unsigned long end = jiffies + tmo;
691 int err = 0;
692 u32 status;
693
694 status = ioat_chansts(ioat_chan);
695 if (is_ioat_active(status) || is_ioat_idle(status))
696 ioat_suspend(ioat_chan);
697 while (is_ioat_active(status) || is_ioat_idle(status)) {
698 if (tmo && time_after(jiffies, end)) {
699 err = -ETIMEDOUT;
700 break;
701 }
702 status = ioat_chansts(ioat_chan);
703 cpu_relax();
704 }
705
706 return err;
707}
708
709int ioat_reset_sync(struct ioatdma_chan *ioat_chan, unsigned long tmo)
710{
711 unsigned long end = jiffies + tmo;
712 int err = 0;
713
714 ioat_reset(ioat_chan);
715 while (ioat_reset_pending(ioat_chan)) {
716 if (end && time_after(jiffies, end)) {
717 err = -ETIMEDOUT;
718 break;
719 }
720 cpu_relax();
721 }
722
723 return err;
724}
725
726/**
727 * ioat_enumerate_channels - find and initialize the device's channels
728 * @ioat_dma: the ioat dma device to be enumerated
729 */
730int ioat_enumerate_channels(struct ioatdma_device *ioat_dma)
731{
732 struct ioatdma_chan *ioat_chan;
733 struct device *dev = &ioat_dma->pdev->dev;
734 struct dma_device *dma = &ioat_dma->dma_dev;
735 u8 xfercap_log;
736 int i;
737
738 INIT_LIST_HEAD(&dma->channels);
739 dma->chancnt = readb(ioat_dma->reg_base + IOAT_CHANCNT_OFFSET);
740 dma->chancnt &= 0x1f; /* bits [4:0] valid */
741 if (dma->chancnt > ARRAY_SIZE(ioat_dma->idx)) {
742 dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n",
743 dma->chancnt, ARRAY_SIZE(ioat_dma->idx));
744 dma->chancnt = ARRAY_SIZE(ioat_dma->idx);
745 }
746 xfercap_log = readb(ioat_dma->reg_base + IOAT_XFERCAP_OFFSET);
747 xfercap_log &= 0x1f; /* bits [4:0] valid */
748 if (xfercap_log == 0)
749 return 0;
750 dev_dbg(dev, "%s: xfercap = %d\n", __func__, 1 << xfercap_log);
751
752 for (i = 0; i < dma->chancnt; i++) {
753 ioat_chan = devm_kzalloc(dev, sizeof(*ioat_chan), GFP_KERNEL);
754 if (!ioat_chan)
755 break;
756
757 ioat_init_channel(ioat_dma, ioat_chan, i);
758 ioat_chan->xfercap_log = xfercap_log;
759 spin_lock_init(&ioat_chan->prep_lock);
760 if (ioat_dma->reset_hw(ioat_chan)) {
761 i = 0;
762 break;
763 }
764 }
765 dma->chancnt = i;
766 return i;
767}
768
769static dma_cookie_t ioat_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
770{
771 struct dma_chan *c = tx->chan;
772 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
773 dma_cookie_t cookie;
774
775 cookie = dma_cookie_assign(tx);
776 dev_dbg(to_dev(ioat_chan), "%s: cookie: %d\n", __func__, cookie);
777
778 if (!test_and_set_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state))
779 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
780
781 /* make descriptor updates visible before advancing ioat->head,
782 * this is purposefully not smp_wmb() since we are also
783 * publishing the descriptor updates to a dma device
784 */
785 wmb();
786
787 ioat_chan->head += ioat_chan->produce;
788
789 ioat_update_pending(ioat_chan);
790 spin_unlock_bh(&ioat_chan->prep_lock);
791
792 return cookie;
793}
794
795static struct ioat_ring_ent *
796ioat_alloc_ring_ent(struct dma_chan *chan, gfp_t flags)
797{
798 struct ioat_dma_descriptor *hw;
799 struct ioat_ring_ent *desc;
800 struct ioatdma_device *ioat_dma;
801 dma_addr_t phys;
802
803 ioat_dma = to_ioatdma_device(chan->device);
804 hw = pci_pool_alloc(ioat_dma->dma_pool, flags, &phys);
805 if (!hw)
806 return NULL;
807 memset(hw, 0, sizeof(*hw));
808
809 desc = kmem_cache_zalloc(ioat_cache, flags);
810 if (!desc) {
811 pci_pool_free(ioat_dma->dma_pool, hw, phys);
812 return NULL;
813 }
814
815 dma_async_tx_descriptor_init(&desc->txd, chan);
816 desc->txd.tx_submit = ioat_tx_submit_unlock;
817 desc->hw = hw;
818 desc->txd.phys = phys;
819 return desc;
820}
821
822static void
823ioat_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan)
824{
825 struct ioatdma_device *ioat_dma;
826
827 ioat_dma = to_ioatdma_device(chan->device);
828 pci_pool_free(ioat_dma->dma_pool, desc->hw, desc->txd.phys);
829 kmem_cache_free(ioat_cache, desc);
830}
831
832static struct ioat_ring_ent **
833ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags)
834{
835 struct ioat_ring_ent **ring;
836 int descs = 1 << order;
837 int i;
838
839 if (order > ioat_get_max_alloc_order())
840 return NULL;
841
842 /* allocate the array to hold the software ring */
843 ring = kcalloc(descs, sizeof(*ring), flags);
844 if (!ring)
845 return NULL;
846 for (i = 0; i < descs; i++) {
847 ring[i] = ioat_alloc_ring_ent(c, flags);
848 if (!ring[i]) {
849 while (i--)
850 ioat_free_ring_ent(ring[i], c);
851 kfree(ring);
852 return NULL;
853 }
854 set_desc_id(ring[i], i);
855 }
856
857 /* link descs */
858 for (i = 0; i < descs-1; i++) {
859 struct ioat_ring_ent *next = ring[i+1];
860 struct ioat_dma_descriptor *hw = ring[i]->hw;
861
862 hw->next = next->txd.phys;
863 }
864 ring[i]->hw->next = ring[0]->txd.phys;
865
866 return ring;
867}
868
869/**
870 * ioat_free_chan_resources - release all the descriptors
871 * @chan: the channel to be cleaned
872 */
873void ioat_free_chan_resources(struct dma_chan *c)
874{
875 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
876 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
877 struct ioat_ring_ent *desc;
878 const int total_descs = 1 << ioat_chan->alloc_order;
879 int descs;
880 int i;
881
882 /* Before freeing channel resources first check
883 * if they have been previously allocated for this channel.
884 */
885 if (!ioat_chan->ring)
886 return;
887
888 ioat_stop(ioat_chan);
889 ioat_dma->reset_hw(ioat_chan);
890
891 spin_lock_bh(&ioat_chan->cleanup_lock);
892 spin_lock_bh(&ioat_chan->prep_lock);
893 descs = ioat_ring_space(ioat_chan);
894 dev_dbg(to_dev(ioat_chan), "freeing %d idle descriptors\n", descs);
895 for (i = 0; i < descs; i++) {
896 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head + i);
897 ioat_free_ring_ent(desc, c);
898 }
899
900 if (descs < total_descs)
901 dev_err(to_dev(ioat_chan), "Freeing %d in use descriptors!\n",
902 total_descs - descs);
903
904 for (i = 0; i < total_descs - descs; i++) {
905 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail + i);
906 dump_desc_dbg(ioat_chan, desc);
907 ioat_free_ring_ent(desc, c);
908 }
909
910 kfree(ioat_chan->ring);
911 ioat_chan->ring = NULL;
912 ioat_chan->alloc_order = 0;
913 pci_pool_free(ioat_dma->completion_pool, ioat_chan->completion,
914 ioat_chan->completion_dma);
915 spin_unlock_bh(&ioat_chan->prep_lock);
916 spin_unlock_bh(&ioat_chan->cleanup_lock);
917
918 ioat_chan->last_completion = 0;
919 ioat_chan->completion_dma = 0;
920 ioat_chan->dmacount = 0;
921}
922
923/* ioat_alloc_chan_resources - allocate/initialize ioat descriptor ring
924 * @chan: channel to be initialized
925 */
926int ioat_alloc_chan_resources(struct dma_chan *c)
927{
928 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
929 struct ioat_ring_ent **ring;
930 u64 status;
931 int order;
932 int i = 0;
933 u32 chanerr;
934
935 /* have we already been set up? */
936 if (ioat_chan->ring)
937 return 1 << ioat_chan->alloc_order;
938
939 /* Setup register to interrupt and write completion status on error */
940 writew(IOAT_CHANCTRL_RUN, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
941
942 /* allocate a completion writeback area */
943 /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
944 ioat_chan->completion =
945 pci_pool_alloc(ioat_chan->ioat_dma->completion_pool,
946 GFP_KERNEL, &ioat_chan->completion_dma);
947 if (!ioat_chan->completion)
948 return -ENOMEM;
949
950 memset(ioat_chan->completion, 0, sizeof(*ioat_chan->completion));
951 writel(((u64)ioat_chan->completion_dma) & 0x00000000FFFFFFFF,
952 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
953 writel(((u64)ioat_chan->completion_dma) >> 32,
954 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
955
956 order = ioat_get_alloc_order();
957 ring = ioat_alloc_ring(c, order, GFP_KERNEL);
958 if (!ring)
959 return -ENOMEM;
960
961 spin_lock_bh(&ioat_chan->cleanup_lock);
962 spin_lock_bh(&ioat_chan->prep_lock);
963 ioat_chan->ring = ring;
964 ioat_chan->head = 0;
965 ioat_chan->issued = 0;
966 ioat_chan->tail = 0;
967 ioat_chan->alloc_order = order;
968 set_bit(IOAT_RUN, &ioat_chan->state);
969 spin_unlock_bh(&ioat_chan->prep_lock);
970 spin_unlock_bh(&ioat_chan->cleanup_lock);
971
972 ioat_start_null_desc(ioat_chan);
973
974 /* check that we got off the ground */
975 do {
976 udelay(1);
977 status = ioat_chansts(ioat_chan);
978 } while (i++ < 20 && !is_ioat_active(status) && !is_ioat_idle(status));
979
980 if (is_ioat_active(status) || is_ioat_idle(status))
981 return 1 << ioat_chan->alloc_order;
982
983 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
984
985 dev_WARN(to_dev(ioat_chan),
986 "failed to start channel chanerr: %#x\n", chanerr);
987 ioat_free_chan_resources(c);
988 return -EFAULT;
989}
990
991bool reshape_ring(struct ioatdma_chan *ioat_chan, int order)
992{
993 /* reshape differs from normal ring allocation in that we want
994 * to allocate a new software ring while only
995 * extending/truncating the hardware ring
996 */
997 struct dma_chan *c = &ioat_chan->dma_chan;
998 const u32 curr_size = ioat_ring_size(ioat_chan);
999 const u16 active = ioat_ring_active(ioat_chan);
1000 const u32 new_size = 1 << order;
1001 struct ioat_ring_ent **ring;
1002 u32 i;
1003
1004 if (order > ioat_get_max_alloc_order())
1005 return false;
1006
1007 /* double check that we have at least 1 free descriptor */
1008 if (active == curr_size)
1009 return false;
1010
1011 /* when shrinking, verify that we can hold the current active
1012 * set in the new ring
1013 */
1014 if (active >= new_size)
1015 return false;
1016
1017 /* allocate the array to hold the software ring */
1018 ring = kcalloc(new_size, sizeof(*ring), GFP_NOWAIT);
1019 if (!ring)
1020 return false;
1021
1022 /* allocate/trim descriptors as needed */
1023 if (new_size > curr_size) {
1024 /* copy current descriptors to the new ring */
1025 for (i = 0; i < curr_size; i++) {
1026 u16 curr_idx = (ioat_chan->tail+i) & (curr_size-1);
1027 u16 new_idx = (ioat_chan->tail+i) & (new_size-1);
1028
1029 ring[new_idx] = ioat_chan->ring[curr_idx];
1030 set_desc_id(ring[new_idx], new_idx);
1031 }
1032
1033 /* add new descriptors to the ring */
1034 for (i = curr_size; i < new_size; i++) {
1035 u16 new_idx = (ioat_chan->tail+i) & (new_size-1);
1036
1037 ring[new_idx] = ioat_alloc_ring_ent(c, GFP_NOWAIT);
1038 if (!ring[new_idx]) {
1039 while (i--) {
1040 u16 new_idx = (ioat_chan->tail+i) &
1041 (new_size-1);
1042
1043 ioat_free_ring_ent(ring[new_idx], c);
1044 }
1045 kfree(ring);
1046 return false;
1047 }
1048 set_desc_id(ring[new_idx], new_idx);
1049 }
1050
1051 /* hw link new descriptors */
1052 for (i = curr_size-1; i < new_size; i++) {
1053 u16 new_idx = (ioat_chan->tail+i) & (new_size-1);
1054 struct ioat_ring_ent *next =
1055 ring[(new_idx+1) & (new_size-1)];
1056 struct ioat_dma_descriptor *hw = ring[new_idx]->hw;
1057
1058 hw->next = next->txd.phys;
1059 }
1060 } else {
1061 struct ioat_dma_descriptor *hw;
1062 struct ioat_ring_ent *next;
1063
1064 /* copy current descriptors to the new ring, dropping the
1065 * removed descriptors
1066 */
1067 for (i = 0; i < new_size; i++) {
1068 u16 curr_idx = (ioat_chan->tail+i) & (curr_size-1);
1069 u16 new_idx = (ioat_chan->tail+i) & (new_size-1);
1070
1071 ring[new_idx] = ioat_chan->ring[curr_idx];
1072 set_desc_id(ring[new_idx], new_idx);
1073 }
1074
1075 /* free deleted descriptors */
1076 for (i = new_size; i < curr_size; i++) {
1077 struct ioat_ring_ent *ent;
1078
1079 ent = ioat_get_ring_ent(ioat_chan, ioat_chan->tail+i);
1080 ioat_free_ring_ent(ent, c);
1081 }
1082
1083 /* fix up hardware ring */
1084 hw = ring[(ioat_chan->tail+new_size-1) & (new_size-1)]->hw;
1085 next = ring[(ioat_chan->tail+new_size) & (new_size-1)];
1086 hw->next = next->txd.phys;
1087 }
1088
1089 dev_dbg(to_dev(ioat_chan), "%s: allocated %d descriptors\n",
1090 __func__, new_size);
1091
1092 kfree(ioat_chan->ring);
1093 ioat_chan->ring = ring;
1094 ioat_chan->alloc_order = order;
1095
1096 return true;
1097}
1098
1099/**
1100 * ioat_check_space_lock - verify space and grab ring producer lock
1101 * @ioat: ioat,3 channel (ring) to operate on
1102 * @num_descs: allocation length
1103 */
1104int ioat_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs)
1105{
1106 bool retry;
1107
1108 retry:
1109 spin_lock_bh(&ioat_chan->prep_lock);
1110 /* never allow the last descriptor to be consumed, we need at
1111 * least one free at all times to allow for on-the-fly ring
1112 * resizing.
1113 */
1114 if (likely(ioat_ring_space(ioat_chan) > num_descs)) {
1115 dev_dbg(to_dev(ioat_chan), "%s: num_descs: %d (%x:%x:%x)\n",
1116 __func__, num_descs, ioat_chan->head,
1117 ioat_chan->tail, ioat_chan->issued);
1118 ioat_chan->produce = num_descs;
1119 return 0; /* with ioat->prep_lock held */
1120 }
1121 retry = test_and_set_bit(IOAT_RESHAPE_PENDING, &ioat_chan->state);
1122 spin_unlock_bh(&ioat_chan->prep_lock);
1123
1124 /* is another cpu already trying to expand the ring? */
1125 if (retry)
1126 goto retry;
1127
1128 spin_lock_bh(&ioat_chan->cleanup_lock);
1129 spin_lock_bh(&ioat_chan->prep_lock);
1130 retry = reshape_ring(ioat_chan, ioat_chan->alloc_order + 1);
1131 clear_bit(IOAT_RESHAPE_PENDING, &ioat_chan->state);
1132 spin_unlock_bh(&ioat_chan->prep_lock);
1133 spin_unlock_bh(&ioat_chan->cleanup_lock);
1134
1135 /* if we were able to expand the ring retry the allocation */
1136 if (retry)
1137 goto retry;
1138
1139 dev_dbg_ratelimited(to_dev(ioat_chan),
1140 "%s: ring full! num_descs: %d (%x:%x:%x)\n",
1141 __func__, num_descs, ioat_chan->head,
1142 ioat_chan->tail, ioat_chan->issued);
1143
1144 /* progress reclaim in the allocation failure case we may be
1145 * called under bh_disabled so we need to trigger the timer
1146 * event directly
1147 */
1148 if (time_is_before_jiffies(ioat_chan->timer.expires)
1149 && timer_pending(&ioat_chan->timer)) {
1150 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
1151
1152 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
1153 ioat_dma->timer_fn((unsigned long)ioat_chan);
1154 }
1155
1156 return -ENOMEM;
1157}
1158
1159struct dma_async_tx_descriptor *
1160ioat_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
1161 dma_addr_t dma_src, size_t len, unsigned long flags)
1162{
1163 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
1164 struct ioat_dma_descriptor *hw;
1165 struct ioat_ring_ent *desc;
1166 dma_addr_t dst = dma_dest;
1167 dma_addr_t src = dma_src;
1168 size_t total_len = len;
1169 int num_descs, idx, i;
1170
1171 num_descs = ioat_xferlen_to_descs(ioat_chan, len);
1172 if (likely(num_descs) &&
1173 ioat_check_space_lock(ioat_chan, num_descs) == 0)
1174 idx = ioat_chan->head;
1175 else
1176 return NULL;
1177 i = 0;
1178 do {
1179 size_t copy = min_t(size_t, len, 1 << ioat_chan->xfercap_log);
1180
1181 desc = ioat_get_ring_ent(ioat_chan, idx + i);
1182 hw = desc->hw;
1183
1184 hw->size = copy;
1185 hw->ctl = 0;
1186 hw->src_addr = src;
1187 hw->dst_addr = dst;
1188
1189 len -= copy;
1190 dst += copy;
1191 src += copy;
1192 dump_desc_dbg(ioat_chan, desc);
1193 } while (++i < num_descs);
1194
1195 desc->txd.flags = flags;
1196 desc->len = total_len;
1197 hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
1198 hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
1199 hw->ctl_f.compl_write = 1;
1200 dump_desc_dbg(ioat_chan, desc);
1201 /* we leave the channel locked to ensure in order submission */
1202
1203 return &desc->txd;
1204}
1205
1206static ssize_t ring_size_show(struct dma_chan *c, char *page)
1207{
1208 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
1209
1210 return sprintf(page, "%d\n", (1 << ioat_chan->alloc_order) & ~1);
1211}
1212static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size);
1213
1214static ssize_t ring_active_show(struct dma_chan *c, char *page)
1215{
1216 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
1217
1218 /* ...taken outside the lock, no need to be precise */
1219 return sprintf(page, "%d\n", ioat_ring_active(ioat_chan));
1220}
1221static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active);
1222
1223static struct attribute *ioat_attrs[] = {
1224 &ring_size_attr.attr,
1225 &ring_active_attr.attr,
1226 &ioat_cap_attr.attr,
1227 &ioat_version_attr.attr,
1228 NULL,
1229};
1230
1231struct kobj_type ioat_ktype = {
1232 .sysfs_ops = &ioat_sysfs_ops,
1233 .default_attrs = ioat_attrs,
1234};
diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h
index 11bbcf27f86f..2566ec6ae8a4 100644
--- a/drivers/dma/ioat/dma.h
+++ b/drivers/dma/ioat/dma.h
@@ -18,13 +18,14 @@
18#define IOATDMA_H 18#define IOATDMA_H
19 19
20#include <linux/dmaengine.h> 20#include <linux/dmaengine.h>
21#include "hw.h"
22#include "registers.h"
23#include <linux/init.h> 21#include <linux/init.h>
24#include <linux/dmapool.h> 22#include <linux/dmapool.h>
25#include <linux/cache.h> 23#include <linux/cache.h>
26#include <linux/pci_ids.h> 24#include <linux/pci_ids.h>
27#include <net/tcp.h> 25#include <linux/circ_buf.h>
26#include <linux/interrupt.h>
27#include "registers.h"
28#include "hw.h"
28 29
29#define IOAT_DMA_VERSION "4.00" 30#define IOAT_DMA_VERSION "4.00"
30 31
@@ -154,6 +155,41 @@ struct ioat_sed_ent {
154 unsigned int hw_pool; 155 unsigned int hw_pool;
155}; 156};
156 157
158/**
159 * struct ioat_ring_ent - wrapper around hardware descriptor
160 * @hw: hardware DMA descriptor (for memcpy)
161 * @fill: hardware fill descriptor
162 * @xor: hardware xor descriptor
163 * @xor_ex: hardware xor extension descriptor
164 * @pq: hardware pq descriptor
165 * @pq_ex: hardware pq extension descriptor
166 * @pqu: hardware pq update descriptor
167 * @raw: hardware raw (un-typed) descriptor
168 * @txd: the generic software descriptor for all engines
169 * @len: total transaction length for unmap
170 * @result: asynchronous result of validate operations
171 * @id: identifier for debug
172 */
173
174struct ioat_ring_ent {
175 union {
176 struct ioat_dma_descriptor *hw;
177 struct ioat_xor_descriptor *xor;
178 struct ioat_xor_ext_descriptor *xor_ex;
179 struct ioat_pq_descriptor *pq;
180 struct ioat_pq_ext_descriptor *pq_ex;
181 struct ioat_pq_update_descriptor *pqu;
182 struct ioat_raw_descriptor *raw;
183 };
184 size_t len;
185 struct dma_async_tx_descriptor txd;
186 enum sum_check_flags *result;
187 #ifdef DEBUG
188 int id;
189 #endif
190 struct ioat_sed_ent *sed;
191};
192
157static inline struct ioatdma_chan *to_ioat_chan(struct dma_chan *c) 193static inline struct ioatdma_chan *to_ioat_chan(struct dma_chan *c)
158{ 194{
159 return container_of(c, struct ioatdma_chan, dma_chan); 195 return container_of(c, struct ioatdma_chan, dma_chan);
@@ -291,6 +327,60 @@ static inline bool is_ioat_bug(unsigned long err)
291 return !!err; 327 return !!err;
292} 328}
293 329
330#define IOAT_MAX_ORDER 16
331#define ioat_get_alloc_order() \
332 (min(ioat_ring_alloc_order, IOAT_MAX_ORDER))
333#define ioat_get_max_alloc_order() \
334 (min(ioat_ring_max_alloc_order, IOAT_MAX_ORDER))
335
336static inline u32 ioat_ring_size(struct ioatdma_chan *ioat_chan)
337{
338 return 1 << ioat_chan->alloc_order;
339}
340
341/* count of descriptors in flight with the engine */
342static inline u16 ioat_ring_active(struct ioatdma_chan *ioat_chan)
343{
344 return CIRC_CNT(ioat_chan->head, ioat_chan->tail,
345 ioat_ring_size(ioat_chan));
346}
347
348/* count of descriptors pending submission to hardware */
349static inline u16 ioat_ring_pending(struct ioatdma_chan *ioat_chan)
350{
351 return CIRC_CNT(ioat_chan->head, ioat_chan->issued,
352 ioat_ring_size(ioat_chan));
353}
354
355static inline u32 ioat_ring_space(struct ioatdma_chan *ioat_chan)
356{
357 return ioat_ring_size(ioat_chan) - ioat_ring_active(ioat_chan);
358}
359
360static inline u16
361ioat_xferlen_to_descs(struct ioatdma_chan *ioat_chan, size_t len)
362{
363 u16 num_descs = len >> ioat_chan->xfercap_log;
364
365 num_descs += !!(len & ((1 << ioat_chan->xfercap_log) - 1));
366 return num_descs;
367}
368
369static inline struct ioat_ring_ent *
370ioat_get_ring_ent(struct ioatdma_chan *ioat_chan, u16 idx)
371{
372 return ioat_chan->ring[idx & (ioat_ring_size(ioat_chan) - 1)];
373}
374
375static inline void
376ioat_set_chainaddr(struct ioatdma_chan *ioat_chan, u64 addr)
377{
378 writel(addr & 0x00000000FFFFFFFF,
379 ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
380 writel(addr >> 32,
381 ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
382}
383
294int ioat_probe(struct ioatdma_device *ioat_dma); 384int ioat_probe(struct ioatdma_device *ioat_dma);
295int ioat_register(struct ioatdma_device *ioat_dma); 385int ioat_register(struct ioatdma_device *ioat_dma);
296int ioat_dma_self_test(struct ioatdma_device *ioat_dma); 386int ioat_dma_self_test(struct ioatdma_device *ioat_dma);
@@ -306,7 +396,30 @@ void ioat_kobject_add(struct ioatdma_device *ioat_dma, struct kobj_type *type);
306void ioat_kobject_del(struct ioatdma_device *ioat_dma); 396void ioat_kobject_del(struct ioatdma_device *ioat_dma);
307int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma); 397int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma);
308void ioat_stop(struct ioatdma_chan *ioat_chan); 398void ioat_stop(struct ioatdma_chan *ioat_chan);
399int ioat_dma_probe(struct ioatdma_device *ioat_dma, int dca);
400int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca);
401struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase);
402int ioat_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs);
403int ioat_enumerate_channels(struct ioatdma_device *ioat_dma);
404struct dma_async_tx_descriptor *
405ioat_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
406 dma_addr_t dma_src, size_t len, unsigned long flags);
407void ioat_issue_pending(struct dma_chan *chan);
408int ioat_alloc_chan_resources(struct dma_chan *c);
409void ioat_free_chan_resources(struct dma_chan *c);
410void __ioat_restart_chan(struct ioatdma_chan *ioat_chan);
411bool reshape_ring(struct ioatdma_chan *ioat, int order);
412void __ioat_issue_pending(struct ioatdma_chan *ioat_chan);
413void ioat_timer_event(unsigned long data);
414int ioat_quiesce(struct ioatdma_chan *ioat_chan, unsigned long tmo);
415int ioat_reset_sync(struct ioatdma_chan *ioat_chan, unsigned long tmo);
416
309extern const struct sysfs_ops ioat_sysfs_ops; 417extern const struct sysfs_ops ioat_sysfs_ops;
310extern struct ioat_sysfs_entry ioat_version_attr; 418extern struct ioat_sysfs_entry ioat_version_attr;
311extern struct ioat_sysfs_entry ioat_cap_attr; 419extern struct ioat_sysfs_entry ioat_cap_attr;
420extern int ioat_pending_level;
421extern int ioat_ring_alloc_order;
422extern struct kobj_type ioat_ktype;
423extern struct kmem_cache *ioat_cache;
424
312#endif /* IOATDMA_H */ 425#endif /* IOATDMA_H */
diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c
deleted file mode 100644
index 020c1fe31ca1..000000000000
--- a/drivers/dma/ioat/dma_v2.c
+++ /dev/null
@@ -1,695 +0,0 @@
1/*
2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2004 - 2009 Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * The full GNU General Public License is included in this distribution in
15 * the file called "COPYING".
16 *
17 */
18
19/*
20 * This driver supports an Intel I/OAT DMA engine (versions >= 2), which
21 * does asynchronous data movement and checksumming operations.
22 */
23
24#include <linux/init.h>
25#include <linux/module.h>
26#include <linux/slab.h>
27#include <linux/pci.h>
28#include <linux/interrupt.h>
29#include <linux/dmaengine.h>
30#include <linux/delay.h>
31#include <linux/dma-mapping.h>
32#include <linux/workqueue.h>
33#include <linux/prefetch.h>
34#include "dma.h"
35#include "dma_v2.h"
36#include "registers.h"
37#include "hw.h"
38
39#include "../dmaengine.h"
40
41int ioat_ring_alloc_order = 8;
42module_param(ioat_ring_alloc_order, int, 0644);
43MODULE_PARM_DESC(ioat_ring_alloc_order,
44 "ioat2+: allocate 2^n descriptors per channel"
45 " (default: 8 max: 16)");
46static int ioat_ring_max_alloc_order = IOAT_MAX_ORDER;
47module_param(ioat_ring_max_alloc_order, int, 0644);
48MODULE_PARM_DESC(ioat_ring_max_alloc_order,
49 "ioat2+: upper limit for ring size (default: 16)");
50
51void __ioat2_issue_pending(struct ioatdma_chan *ioat_chan)
52{
53 ioat_chan->dmacount += ioat2_ring_pending(ioat_chan);
54 ioat_chan->issued = ioat_chan->head;
55 writew(ioat_chan->dmacount,
56 ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
57 dev_dbg(to_dev(ioat_chan),
58 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
59 __func__, ioat_chan->head, ioat_chan->tail,
60 ioat_chan->issued, ioat_chan->dmacount);
61}
62
63void ioat2_issue_pending(struct dma_chan *c)
64{
65 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
66
67 if (ioat2_ring_pending(ioat_chan)) {
68 spin_lock_bh(&ioat_chan->prep_lock);
69 __ioat2_issue_pending(ioat_chan);
70 spin_unlock_bh(&ioat_chan->prep_lock);
71 }
72}
73
74/**
75 * ioat2_update_pending - log pending descriptors
76 * @ioat: ioat2+ channel
77 *
78 * Check if the number of unsubmitted descriptors has exceeded the
79 * watermark. Called with prep_lock held
80 */
81static void ioat2_update_pending(struct ioatdma_chan *ioat_chan)
82{
83 if (ioat2_ring_pending(ioat_chan) > ioat_pending_level)
84 __ioat2_issue_pending(ioat_chan);
85}
86
87static void __ioat2_start_null_desc(struct ioatdma_chan *ioat_chan)
88{
89 struct ioat_ring_ent *desc;
90 struct ioat_dma_descriptor *hw;
91
92 if (ioat2_ring_space(ioat_chan) < 1) {
93 dev_err(to_dev(ioat_chan),
94 "Unable to start null desc - ring full\n");
95 return;
96 }
97
98 dev_dbg(to_dev(ioat_chan),
99 "%s: head: %#x tail: %#x issued: %#x\n",
100 __func__, ioat_chan->head, ioat_chan->tail, ioat_chan->issued);
101 desc = ioat2_get_ring_ent(ioat_chan, ioat_chan->head);
102
103 hw = desc->hw;
104 hw->ctl = 0;
105 hw->ctl_f.null = 1;
106 hw->ctl_f.int_en = 1;
107 hw->ctl_f.compl_write = 1;
108 /* set size to non-zero value (channel returns error when size is 0) */
109 hw->size = NULL_DESC_BUFFER_SIZE;
110 hw->src_addr = 0;
111 hw->dst_addr = 0;
112 async_tx_ack(&desc->txd);
113 ioat2_set_chainaddr(ioat_chan, desc->txd.phys);
114 dump_desc_dbg(ioat_chan, desc);
115 wmb();
116 ioat_chan->head += 1;
117 __ioat2_issue_pending(ioat_chan);
118}
119
120static void ioat2_start_null_desc(struct ioatdma_chan *ioat_chan)
121{
122 spin_lock_bh(&ioat_chan->prep_lock);
123 __ioat2_start_null_desc(ioat_chan);
124 spin_unlock_bh(&ioat_chan->prep_lock);
125}
126
127void __ioat2_restart_chan(struct ioatdma_chan *ioat_chan)
128{
129 /* set the tail to be re-issued */
130 ioat_chan->issued = ioat_chan->tail;
131 ioat_chan->dmacount = 0;
132 set_bit(IOAT_COMPLETION_PENDING, &ioat_chan->state);
133 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
134
135 dev_dbg(to_dev(ioat_chan),
136 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
137 __func__, ioat_chan->head, ioat_chan->tail,
138 ioat_chan->issued, ioat_chan->dmacount);
139
140 if (ioat2_ring_pending(ioat_chan)) {
141 struct ioat_ring_ent *desc;
142
143 desc = ioat2_get_ring_ent(ioat_chan, ioat_chan->tail);
144 ioat2_set_chainaddr(ioat_chan, desc->txd.phys);
145 __ioat2_issue_pending(ioat_chan);
146 } else
147 __ioat2_start_null_desc(ioat_chan);
148}
149
150int ioat2_quiesce(struct ioatdma_chan *ioat_chan, unsigned long tmo)
151{
152 unsigned long end = jiffies + tmo;
153 int err = 0;
154 u32 status;
155
156 status = ioat_chansts(ioat_chan);
157 if (is_ioat_active(status) || is_ioat_idle(status))
158 ioat_suspend(ioat_chan);
159 while (is_ioat_active(status) || is_ioat_idle(status)) {
160 if (tmo && time_after(jiffies, end)) {
161 err = -ETIMEDOUT;
162 break;
163 }
164 status = ioat_chansts(ioat_chan);
165 cpu_relax();
166 }
167
168 return err;
169}
170
171int ioat2_reset_sync(struct ioatdma_chan *ioat_chan, unsigned long tmo)
172{
173 unsigned long end = jiffies + tmo;
174 int err = 0;
175
176 ioat_reset(ioat_chan);
177 while (ioat_reset_pending(ioat_chan)) {
178 if (end && time_after(jiffies, end)) {
179 err = -ETIMEDOUT;
180 break;
181 }
182 cpu_relax();
183 }
184
185 return err;
186}
187
188/**
189 * ioat2_enumerate_channels - find and initialize the device's channels
190 * @ioat_dma: the ioat dma device to be enumerated
191 */
192int ioat2_enumerate_channels(struct ioatdma_device *ioat_dma)
193{
194 struct ioatdma_chan *ioat_chan;
195 struct device *dev = &ioat_dma->pdev->dev;
196 struct dma_device *dma = &ioat_dma->dma_dev;
197 u8 xfercap_log;
198 int i;
199
200 INIT_LIST_HEAD(&dma->channels);
201 dma->chancnt = readb(ioat_dma->reg_base + IOAT_CHANCNT_OFFSET);
202 dma->chancnt &= 0x1f; /* bits [4:0] valid */
203 if (dma->chancnt > ARRAY_SIZE(ioat_dma->idx)) {
204 dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n",
205 dma->chancnt, ARRAY_SIZE(ioat_dma->idx));
206 dma->chancnt = ARRAY_SIZE(ioat_dma->idx);
207 }
208 xfercap_log = readb(ioat_dma->reg_base + IOAT_XFERCAP_OFFSET);
209 xfercap_log &= 0x1f; /* bits [4:0] valid */
210 if (xfercap_log == 0)
211 return 0;
212 dev_dbg(dev, "%s: xfercap = %d\n", __func__, 1 << xfercap_log);
213
214 for (i = 0; i < dma->chancnt; i++) {
215 ioat_chan = devm_kzalloc(dev, sizeof(*ioat_chan), GFP_KERNEL);
216 if (!ioat_chan)
217 break;
218
219 ioat_init_channel(ioat_dma, ioat_chan, i);
220 ioat_chan->xfercap_log = xfercap_log;
221 spin_lock_init(&ioat_chan->prep_lock);
222 if (ioat_dma->reset_hw(ioat_chan)) {
223 i = 0;
224 break;
225 }
226 }
227 dma->chancnt = i;
228 return i;
229}
230
231static dma_cookie_t ioat2_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
232{
233 struct dma_chan *c = tx->chan;
234 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
235 dma_cookie_t cookie;
236
237 cookie = dma_cookie_assign(tx);
238 dev_dbg(to_dev(ioat_chan), "%s: cookie: %d\n", __func__, cookie);
239
240 if (!test_and_set_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state))
241 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
242
243 /* make descriptor updates visible before advancing ioat->head,
244 * this is purposefully not smp_wmb() since we are also
245 * publishing the descriptor updates to a dma device
246 */
247 wmb();
248
249 ioat_chan->head += ioat_chan->produce;
250
251 ioat2_update_pending(ioat_chan);
252 spin_unlock_bh(&ioat_chan->prep_lock);
253
254 return cookie;
255}
256
257static struct ioat_ring_ent *ioat2_alloc_ring_ent(struct dma_chan *chan, gfp_t flags)
258{
259 struct ioat_dma_descriptor *hw;
260 struct ioat_ring_ent *desc;
261 struct ioatdma_device *ioat_dma;
262 dma_addr_t phys;
263
264 ioat_dma = to_ioatdma_device(chan->device);
265 hw = pci_pool_alloc(ioat_dma->dma_pool, flags, &phys);
266 if (!hw)
267 return NULL;
268 memset(hw, 0, sizeof(*hw));
269
270 desc = kmem_cache_zalloc(ioat2_cache, flags);
271 if (!desc) {
272 pci_pool_free(ioat_dma->dma_pool, hw, phys);
273 return NULL;
274 }
275
276 dma_async_tx_descriptor_init(&desc->txd, chan);
277 desc->txd.tx_submit = ioat2_tx_submit_unlock;
278 desc->hw = hw;
279 desc->txd.phys = phys;
280 return desc;
281}
282
283static void ioat2_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan)
284{
285 struct ioatdma_device *ioat_dma;
286
287 ioat_dma = to_ioatdma_device(chan->device);
288 pci_pool_free(ioat_dma->dma_pool, desc->hw, desc->txd.phys);
289 kmem_cache_free(ioat2_cache, desc);
290}
291
292static struct ioat_ring_ent **ioat2_alloc_ring(struct dma_chan *c, int order, gfp_t flags)
293{
294 struct ioat_ring_ent **ring;
295 int descs = 1 << order;
296 int i;
297
298 if (order > ioat_get_max_alloc_order())
299 return NULL;
300
301 /* allocate the array to hold the software ring */
302 ring = kcalloc(descs, sizeof(*ring), flags);
303 if (!ring)
304 return NULL;
305 for (i = 0; i < descs; i++) {
306 ring[i] = ioat2_alloc_ring_ent(c, flags);
307 if (!ring[i]) {
308 while (i--)
309 ioat2_free_ring_ent(ring[i], c);
310 kfree(ring);
311 return NULL;
312 }
313 set_desc_id(ring[i], i);
314 }
315
316 /* link descs */
317 for (i = 0; i < descs-1; i++) {
318 struct ioat_ring_ent *next = ring[i+1];
319 struct ioat_dma_descriptor *hw = ring[i]->hw;
320
321 hw->next = next->txd.phys;
322 }
323 ring[i]->hw->next = ring[0]->txd.phys;
324
325 return ring;
326}
327
328void ioat2_free_chan_resources(struct dma_chan *c);
329
330/* ioat2_alloc_chan_resources - allocate/initialize ioat2 descriptor ring
331 * @chan: channel to be initialized
332 */
333int ioat2_alloc_chan_resources(struct dma_chan *c)
334{
335 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
336 struct ioat_ring_ent **ring;
337 u64 status;
338 int order;
339 int i = 0;
340
341 /* have we already been set up? */
342 if (ioat_chan->ring)
343 return 1 << ioat_chan->alloc_order;
344
345 /* Setup register to interrupt and write completion status on error */
346 writew(IOAT_CHANCTRL_RUN, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
347
348 /* allocate a completion writeback area */
349 /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
350 ioat_chan->completion =
351 pci_pool_alloc(ioat_chan->ioat_dma->completion_pool,
352 GFP_KERNEL, &ioat_chan->completion_dma);
353 if (!ioat_chan->completion)
354 return -ENOMEM;
355
356 memset(ioat_chan->completion, 0, sizeof(*ioat_chan->completion));
357 writel(((u64)ioat_chan->completion_dma) & 0x00000000FFFFFFFF,
358 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
359 writel(((u64)ioat_chan->completion_dma) >> 32,
360 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
361
362 order = ioat_get_alloc_order();
363 ring = ioat2_alloc_ring(c, order, GFP_KERNEL);
364 if (!ring)
365 return -ENOMEM;
366
367 spin_lock_bh(&ioat_chan->cleanup_lock);
368 spin_lock_bh(&ioat_chan->prep_lock);
369 ioat_chan->ring = ring;
370 ioat_chan->head = 0;
371 ioat_chan->issued = 0;
372 ioat_chan->tail = 0;
373 ioat_chan->alloc_order = order;
374 set_bit(IOAT_RUN, &ioat_chan->state);
375 spin_unlock_bh(&ioat_chan->prep_lock);
376 spin_unlock_bh(&ioat_chan->cleanup_lock);
377
378 ioat2_start_null_desc(ioat_chan);
379
380 /* check that we got off the ground */
381 do {
382 udelay(1);
383 status = ioat_chansts(ioat_chan);
384 } while (i++ < 20 && !is_ioat_active(status) && !is_ioat_idle(status));
385
386 if (is_ioat_active(status) || is_ioat_idle(status)) {
387 return 1 << ioat_chan->alloc_order;
388 } else {
389 u32 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
390
391 dev_WARN(to_dev(ioat_chan),
392 "failed to start channel chanerr: %#x\n", chanerr);
393 ioat2_free_chan_resources(c);
394 return -EFAULT;
395 }
396}
397
398bool reshape_ring(struct ioatdma_chan *ioat_chan, int order)
399{
400 /* reshape differs from normal ring allocation in that we want
401 * to allocate a new software ring while only
402 * extending/truncating the hardware ring
403 */
404 struct dma_chan *c = &ioat_chan->dma_chan;
405 const u32 curr_size = ioat2_ring_size(ioat_chan);
406 const u16 active = ioat2_ring_active(ioat_chan);
407 const u32 new_size = 1 << order;
408 struct ioat_ring_ent **ring;
409 u32 i;
410
411 if (order > ioat_get_max_alloc_order())
412 return false;
413
414 /* double check that we have at least 1 free descriptor */
415 if (active == curr_size)
416 return false;
417
418 /* when shrinking, verify that we can hold the current active
419 * set in the new ring
420 */
421 if (active >= new_size)
422 return false;
423
424 /* allocate the array to hold the software ring */
425 ring = kcalloc(new_size, sizeof(*ring), GFP_NOWAIT);
426 if (!ring)
427 return false;
428
429 /* allocate/trim descriptors as needed */
430 if (new_size > curr_size) {
431 /* copy current descriptors to the new ring */
432 for (i = 0; i < curr_size; i++) {
433 u16 curr_idx = (ioat_chan->tail+i) & (curr_size-1);
434 u16 new_idx = (ioat_chan->tail+i) & (new_size-1);
435
436 ring[new_idx] = ioat_chan->ring[curr_idx];
437 set_desc_id(ring[new_idx], new_idx);
438 }
439
440 /* add new descriptors to the ring */
441 for (i = curr_size; i < new_size; i++) {
442 u16 new_idx = (ioat_chan->tail+i) & (new_size-1);
443
444 ring[new_idx] = ioat2_alloc_ring_ent(c, GFP_NOWAIT);
445 if (!ring[new_idx]) {
446 while (i--) {
447 u16 new_idx = (ioat_chan->tail+i) &
448 (new_size-1);
449
450 ioat2_free_ring_ent(ring[new_idx], c);
451 }
452 kfree(ring);
453 return false;
454 }
455 set_desc_id(ring[new_idx], new_idx);
456 }
457
458 /* hw link new descriptors */
459 for (i = curr_size-1; i < new_size; i++) {
460 u16 new_idx = (ioat_chan->tail+i) & (new_size-1);
461 struct ioat_ring_ent *next = ring[(new_idx+1) & (new_size-1)];
462 struct ioat_dma_descriptor *hw = ring[new_idx]->hw;
463
464 hw->next = next->txd.phys;
465 }
466 } else {
467 struct ioat_dma_descriptor *hw;
468 struct ioat_ring_ent *next;
469
470 /* copy current descriptors to the new ring, dropping the
471 * removed descriptors
472 */
473 for (i = 0; i < new_size; i++) {
474 u16 curr_idx = (ioat_chan->tail+i) & (curr_size-1);
475 u16 new_idx = (ioat_chan->tail+i) & (new_size-1);
476
477 ring[new_idx] = ioat_chan->ring[curr_idx];
478 set_desc_id(ring[new_idx], new_idx);
479 }
480
481 /* free deleted descriptors */
482 for (i = new_size; i < curr_size; i++) {
483 struct ioat_ring_ent *ent;
484
485 ent = ioat2_get_ring_ent(ioat_chan, ioat_chan->tail+i);
486 ioat2_free_ring_ent(ent, c);
487 }
488
489 /* fix up hardware ring */
490 hw = ring[(ioat_chan->tail+new_size-1) & (new_size-1)]->hw;
491 next = ring[(ioat_chan->tail+new_size) & (new_size-1)];
492 hw->next = next->txd.phys;
493 }
494
495 dev_dbg(to_dev(ioat_chan), "%s: allocated %d descriptors\n",
496 __func__, new_size);
497
498 kfree(ioat_chan->ring);
499 ioat_chan->ring = ring;
500 ioat_chan->alloc_order = order;
501
502 return true;
503}
504
505/**
506 * ioat2_check_space_lock - verify space and grab ring producer lock
507 * @ioat: ioat2,3 channel (ring) to operate on
508 * @num_descs: allocation length
509 */
510int ioat2_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs)
511{
512 bool retry;
513
514 retry:
515 spin_lock_bh(&ioat_chan->prep_lock);
516 /* never allow the last descriptor to be consumed, we need at
517 * least one free at all times to allow for on-the-fly ring
518 * resizing.
519 */
520 if (likely(ioat2_ring_space(ioat_chan) > num_descs)) {
521 dev_dbg(to_dev(ioat_chan), "%s: num_descs: %d (%x:%x:%x)\n",
522 __func__, num_descs, ioat_chan->head,
523 ioat_chan->tail, ioat_chan->issued);
524 ioat_chan->produce = num_descs;
525 return 0; /* with ioat->prep_lock held */
526 }
527 retry = test_and_set_bit(IOAT_RESHAPE_PENDING, &ioat_chan->state);
528 spin_unlock_bh(&ioat_chan->prep_lock);
529
530 /* is another cpu already trying to expand the ring? */
531 if (retry)
532 goto retry;
533
534 spin_lock_bh(&ioat_chan->cleanup_lock);
535 spin_lock_bh(&ioat_chan->prep_lock);
536 retry = reshape_ring(ioat_chan, ioat_chan->alloc_order + 1);
537 clear_bit(IOAT_RESHAPE_PENDING, &ioat_chan->state);
538 spin_unlock_bh(&ioat_chan->prep_lock);
539 spin_unlock_bh(&ioat_chan->cleanup_lock);
540
541 /* if we were able to expand the ring retry the allocation */
542 if (retry)
543 goto retry;
544
545 if (printk_ratelimit())
546 dev_dbg(to_dev(ioat_chan),
547 "%s: ring full! num_descs: %d (%x:%x:%x)\n",
548 __func__, num_descs, ioat_chan->head,
549 ioat_chan->tail, ioat_chan->issued);
550
551 /* progress reclaim in the allocation failure case we may be
552 * called under bh_disabled so we need to trigger the timer
553 * event directly
554 */
555 if (time_is_before_jiffies(ioat_chan->timer.expires)
556 && timer_pending(&ioat_chan->timer)) {
557 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
558
559 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
560 ioat_dma->timer_fn((unsigned long)ioat_chan);
561 }
562
563 return -ENOMEM;
564}
565
566struct dma_async_tx_descriptor *
567ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
568 dma_addr_t dma_src, size_t len, unsigned long flags)
569{
570 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
571 struct ioat_dma_descriptor *hw;
572 struct ioat_ring_ent *desc;
573 dma_addr_t dst = dma_dest;
574 dma_addr_t src = dma_src;
575 size_t total_len = len;
576 int num_descs, idx, i;
577
578 num_descs = ioat2_xferlen_to_descs(ioat_chan, len);
579 if (likely(num_descs) &&
580 ioat2_check_space_lock(ioat_chan, num_descs) == 0)
581 idx = ioat_chan->head;
582 else
583 return NULL;
584 i = 0;
585 do {
586 size_t copy = min_t(size_t, len, 1 << ioat_chan->xfercap_log);
587
588 desc = ioat2_get_ring_ent(ioat_chan, idx + i);
589 hw = desc->hw;
590
591 hw->size = copy;
592 hw->ctl = 0;
593 hw->src_addr = src;
594 hw->dst_addr = dst;
595
596 len -= copy;
597 dst += copy;
598 src += copy;
599 dump_desc_dbg(ioat_chan, desc);
600 } while (++i < num_descs);
601
602 desc->txd.flags = flags;
603 desc->len = total_len;
604 hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
605 hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
606 hw->ctl_f.compl_write = 1;
607 dump_desc_dbg(ioat_chan, desc);
608 /* we leave the channel locked to ensure in order submission */
609
610 return &desc->txd;
611}
612
613/**
614 * ioat2_free_chan_resources - release all the descriptors
615 * @chan: the channel to be cleaned
616 */
617void ioat2_free_chan_resources(struct dma_chan *c)
618{
619 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
620 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
621 struct ioat_ring_ent *desc;
622 const int total_descs = 1 << ioat_chan->alloc_order;
623 int descs;
624 int i;
625
626 /* Before freeing channel resources first check
627 * if they have been previously allocated for this channel.
628 */
629 if (!ioat_chan->ring)
630 return;
631
632 ioat_stop(ioat_chan);
633 ioat_dma->reset_hw(ioat_chan);
634
635 spin_lock_bh(&ioat_chan->cleanup_lock);
636 spin_lock_bh(&ioat_chan->prep_lock);
637 descs = ioat2_ring_space(ioat_chan);
638 dev_dbg(to_dev(ioat_chan), "freeing %d idle descriptors\n", descs);
639 for (i = 0; i < descs; i++) {
640 desc = ioat2_get_ring_ent(ioat_chan, ioat_chan->head + i);
641 ioat2_free_ring_ent(desc, c);
642 }
643
644 if (descs < total_descs)
645 dev_err(to_dev(ioat_chan), "Freeing %d in use descriptors!\n",
646 total_descs - descs);
647
648 for (i = 0; i < total_descs - descs; i++) {
649 desc = ioat2_get_ring_ent(ioat_chan, ioat_chan->tail + i);
650 dump_desc_dbg(ioat_chan, desc);
651 ioat2_free_ring_ent(desc, c);
652 }
653
654 kfree(ioat_chan->ring);
655 ioat_chan->ring = NULL;
656 ioat_chan->alloc_order = 0;
657 pci_pool_free(ioat_dma->completion_pool, ioat_chan->completion,
658 ioat_chan->completion_dma);
659 spin_unlock_bh(&ioat_chan->prep_lock);
660 spin_unlock_bh(&ioat_chan->cleanup_lock);
661
662 ioat_chan->last_completion = 0;
663 ioat_chan->completion_dma = 0;
664 ioat_chan->dmacount = 0;
665}
666
667static ssize_t ring_size_show(struct dma_chan *c, char *page)
668{
669 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
670
671 return sprintf(page, "%d\n", (1 << ioat_chan->alloc_order) & ~1);
672}
673static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size);
674
675static ssize_t ring_active_show(struct dma_chan *c, char *page)
676{
677 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
678
679 /* ...taken outside the lock, no need to be precise */
680 return sprintf(page, "%d\n", ioat2_ring_active(ioat_chan));
681}
682static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active);
683
684static struct attribute *ioat2_attrs[] = {
685 &ring_size_attr.attr,
686 &ring_active_attr.attr,
687 &ioat_cap_attr.attr,
688 &ioat_version_attr.attr,
689 NULL,
690};
691
692struct kobj_type ioat2_ktype = {
693 .sysfs_ops = &ioat_sysfs_ops,
694 .default_attrs = ioat2_attrs,
695};
diff --git a/drivers/dma/ioat/dma_v2.h b/drivers/dma/ioat/dma_v2.h
deleted file mode 100644
index 7d69ed3edab4..000000000000
--- a/drivers/dma/ioat/dma_v2.h
+++ /dev/null
@@ -1,143 +0,0 @@
1/*
2 * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the Free
6 * Software Foundation; either version 2 of the License, or (at your option)
7 * any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * The full GNU General Public License is included in this distribution in the
15 * file called COPYING.
16 */
17#ifndef IOATDMA_V2_H
18#define IOATDMA_V2_H
19
20#include <linux/dmaengine.h>
21#include <linux/circ_buf.h>
22#include "dma.h"
23#include "hw.h"
24
25
26extern int ioat_pending_level;
27extern int ioat_ring_alloc_order;
28
29/*
30 * workaround for IOAT ver.3.0 null descriptor issue
31 * (channel returns error when size is 0)
32 */
33#define NULL_DESC_BUFFER_SIZE 1
34
35#define IOAT_MAX_ORDER 16
36#define ioat_get_alloc_order() \
37 (min(ioat_ring_alloc_order, IOAT_MAX_ORDER))
38#define ioat_get_max_alloc_order() \
39 (min(ioat_ring_max_alloc_order, IOAT_MAX_ORDER))
40
41static inline u32 ioat2_ring_size(struct ioatdma_chan *ioat_chan)
42{
43 return 1 << ioat_chan->alloc_order;
44}
45
46/* count of descriptors in flight with the engine */
47static inline u16 ioat2_ring_active(struct ioatdma_chan *ioat_chan)
48{
49 return CIRC_CNT(ioat_chan->head, ioat_chan->tail,
50 ioat2_ring_size(ioat_chan));
51}
52
53/* count of descriptors pending submission to hardware */
54static inline u16 ioat2_ring_pending(struct ioatdma_chan *ioat_chan)
55{
56 return CIRC_CNT(ioat_chan->head, ioat_chan->issued,
57 ioat2_ring_size(ioat_chan));
58}
59
60static inline u32 ioat2_ring_space(struct ioatdma_chan *ioat_chan)
61{
62 return ioat2_ring_size(ioat_chan) - ioat2_ring_active(ioat_chan);
63}
64
65static inline u16
66ioat2_xferlen_to_descs(struct ioatdma_chan *ioat_chan, size_t len)
67{
68 u16 num_descs = len >> ioat_chan->xfercap_log;
69
70 num_descs += !!(len & ((1 << ioat_chan->xfercap_log) - 1));
71 return num_descs;
72}
73
74/**
75 * struct ioat_ring_ent - wrapper around hardware descriptor
76 * @hw: hardware DMA descriptor (for memcpy)
77 * @fill: hardware fill descriptor
78 * @xor: hardware xor descriptor
79 * @xor_ex: hardware xor extension descriptor
80 * @pq: hardware pq descriptor
81 * @pq_ex: hardware pq extension descriptor
82 * @pqu: hardware pq update descriptor
83 * @raw: hardware raw (un-typed) descriptor
84 * @txd: the generic software descriptor for all engines
85 * @len: total transaction length for unmap
86 * @result: asynchronous result of validate operations
87 * @id: identifier for debug
88 */
89
90struct ioat_ring_ent {
91 union {
92 struct ioat_dma_descriptor *hw;
93 struct ioat_xor_descriptor *xor;
94 struct ioat_xor_ext_descriptor *xor_ex;
95 struct ioat_pq_descriptor *pq;
96 struct ioat_pq_ext_descriptor *pq_ex;
97 struct ioat_pq_update_descriptor *pqu;
98 struct ioat_raw_descriptor *raw;
99 };
100 size_t len;
101 struct dma_async_tx_descriptor txd;
102 enum sum_check_flags *result;
103 #ifdef DEBUG
104 int id;
105 #endif
106 struct ioat_sed_ent *sed;
107};
108
109static inline struct ioat_ring_ent *
110ioat2_get_ring_ent(struct ioatdma_chan *ioat_chan, u16 idx)
111{
112 return ioat_chan->ring[idx & (ioat2_ring_size(ioat_chan) - 1)];
113}
114
115static inline void
116ioat2_set_chainaddr(struct ioatdma_chan *ioat_chan, u64 addr)
117{
118 writel(addr & 0x00000000FFFFFFFF,
119 ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
120 writel(addr >> 32,
121 ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
122}
123
124int ioat2_dma_probe(struct ioatdma_device *ioat_dma, int dca);
125int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca);
126struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase);
127int ioat2_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs);
128int ioat2_enumerate_channels(struct ioatdma_device *ioat_dma);
129struct dma_async_tx_descriptor *
130ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
131 dma_addr_t dma_src, size_t len, unsigned long flags);
132void ioat2_issue_pending(struct dma_chan *chan);
133int ioat2_alloc_chan_resources(struct dma_chan *c);
134void ioat2_free_chan_resources(struct dma_chan *c);
135void __ioat2_restart_chan(struct ioatdma_chan *ioat_chan);
136bool reshape_ring(struct ioatdma_chan *ioat, int order);
137void __ioat2_issue_pending(struct ioatdma_chan *ioat_chan);
138void ioat2_timer_event(unsigned long data);
139int ioat2_quiesce(struct ioatdma_chan *ioat_chan, unsigned long tmo);
140int ioat2_reset_sync(struct ioatdma_chan *ioat_chan, unsigned long tmo);
141extern struct kobj_type ioat2_ktype;
142extern struct kmem_cache *ioat2_cache;
143#endif /* IOATDMA_V2_H */
diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c
index 8ad4b07e7b85..2a47f3968ce4 100644
--- a/drivers/dma/ioat/dma_v3.c
+++ b/drivers/dma/ioat/dma_v3.c
@@ -61,7 +61,6 @@
61#include "registers.h" 61#include "registers.h"
62#include "hw.h" 62#include "hw.h"
63#include "dma.h" 63#include "dma.h"
64#include "dma_v2.h"
65 64
66extern struct kmem_cache *ioat3_sed_cache; 65extern struct kmem_cache *ioat3_sed_cache;
67 66
@@ -390,13 +389,13 @@ static void __cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete)
390 if (!phys_complete) 389 if (!phys_complete)
391 return; 390 return;
392 391
393 active = ioat2_ring_active(ioat_chan); 392 active = ioat_ring_active(ioat_chan);
394 for (i = 0; i < active && !seen_current; i++) { 393 for (i = 0; i < active && !seen_current; i++) {
395 struct dma_async_tx_descriptor *tx; 394 struct dma_async_tx_descriptor *tx;
396 395
397 smp_read_barrier_depends(); 396 smp_read_barrier_depends();
398 prefetch(ioat2_get_ring_ent(ioat_chan, idx + i + 1)); 397 prefetch(ioat_get_ring_ent(ioat_chan, idx + i + 1));
399 desc = ioat2_get_ring_ent(ioat_chan, idx + i); 398 desc = ioat_get_ring_ent(ioat_chan, idx + i);
400 dump_desc_dbg(ioat_chan, desc); 399 dump_desc_dbg(ioat_chan, desc);
401 400
402 /* set err stat if we are using dwbes */ 401 /* set err stat if we are using dwbes */
@@ -479,11 +478,11 @@ static void ioat3_restart_channel(struct ioatdma_chan *ioat_chan)
479{ 478{
480 u64 phys_complete; 479 u64 phys_complete;
481 480
482 ioat2_quiesce(ioat_chan, 0); 481 ioat_quiesce(ioat_chan, 0);
483 if (ioat3_cleanup_preamble(ioat_chan, &phys_complete)) 482 if (ioat3_cleanup_preamble(ioat_chan, &phys_complete))
484 __cleanup(ioat_chan, phys_complete); 483 __cleanup(ioat_chan, phys_complete);
485 484
486 __ioat2_restart_chan(ioat_chan); 485 __ioat_restart_chan(ioat_chan);
487} 486}
488 487
489static void ioat3_eh(struct ioatdma_chan *ioat_chan) 488static void ioat3_eh(struct ioatdma_chan *ioat_chan)
@@ -507,7 +506,7 @@ static void ioat3_eh(struct ioatdma_chan *ioat_chan)
507 dev_dbg(to_dev(ioat_chan), "%s: error = %x:%x\n", 506 dev_dbg(to_dev(ioat_chan), "%s: error = %x:%x\n",
508 __func__, chanerr, chanerr_int); 507 __func__, chanerr, chanerr_int);
509 508
510 desc = ioat2_get_ring_ent(ioat_chan, ioat_chan->tail); 509 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail);
511 hw = desc->hw; 510 hw = desc->hw;
512 dump_desc_dbg(ioat_chan, desc); 511 dump_desc_dbg(ioat_chan, desc);
513 512
@@ -561,7 +560,7 @@ static void ioat3_eh(struct ioatdma_chan *ioat_chan)
561 560
562static void check_active(struct ioatdma_chan *ioat_chan) 561static void check_active(struct ioatdma_chan *ioat_chan)
563{ 562{
564 if (ioat2_ring_active(ioat_chan)) { 563 if (ioat_ring_active(ioat_chan)) {
565 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); 564 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
566 return; 565 return;
567 } 566 }
@@ -625,7 +624,7 @@ static void ioat3_timer_event(unsigned long data)
625 } 624 }
626 625
627 626
628 if (ioat2_ring_active(ioat_chan)) 627 if (ioat_ring_active(ioat_chan))
629 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); 628 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
630 else { 629 else {
631 spin_lock_bh(&ioat_chan->prep_lock); 630 spin_lock_bh(&ioat_chan->prep_lock);
@@ -670,7 +669,7 @@ __ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result,
670 669
671 BUG_ON(src_cnt < 2); 670 BUG_ON(src_cnt < 2);
672 671
673 num_descs = ioat2_xferlen_to_descs(ioat_chan, len); 672 num_descs = ioat_xferlen_to_descs(ioat_chan, len);
674 /* we need 2x the number of descriptors to cover greater than 5 673 /* we need 2x the number of descriptors to cover greater than 5
675 * sources 674 * sources
676 */ 675 */
@@ -686,7 +685,7 @@ __ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result,
686 * order. 685 * order.
687 */ 686 */
688 if (likely(num_descs) && 687 if (likely(num_descs) &&
689 ioat2_check_space_lock(ioat_chan, num_descs+1) == 0) 688 ioat_check_space_lock(ioat_chan, num_descs+1) == 0)
690 idx = ioat_chan->head; 689 idx = ioat_chan->head;
691 else 690 else
692 return NULL; 691 return NULL;
@@ -697,14 +696,14 @@ __ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result,
697 len, 1 << ioat_chan->xfercap_log); 696 len, 1 << ioat_chan->xfercap_log);
698 int s; 697 int s;
699 698
700 desc = ioat2_get_ring_ent(ioat_chan, idx + i); 699 desc = ioat_get_ring_ent(ioat_chan, idx + i);
701 xor = desc->xor; 700 xor = desc->xor;
702 701
703 /* save a branch by unconditionally retrieving the 702 /* save a branch by unconditionally retrieving the
704 * extended descriptor xor_set_src() knows to not write 703 * extended descriptor xor_set_src() knows to not write
705 * to it in the single descriptor case 704 * to it in the single descriptor case
706 */ 705 */
707 ext = ioat2_get_ring_ent(ioat_chan, idx + i + 1); 706 ext = ioat_get_ring_ent(ioat_chan, idx + i + 1);
708 xor_ex = ext->xor_ex; 707 xor_ex = ext->xor_ex;
709 708
710 descs[0] = (struct ioat_raw_descriptor *) xor; 709 descs[0] = (struct ioat_raw_descriptor *) xor;
@@ -730,7 +729,7 @@ __ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result,
730 xor->ctl_f.fence = !!(flags & DMA_PREP_FENCE); 729 xor->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
731 730
732 /* completion descriptor carries interrupt bit */ 731 /* completion descriptor carries interrupt bit */
733 compl_desc = ioat2_get_ring_ent(ioat_chan, idx + i); 732 compl_desc = ioat_get_ring_ent(ioat_chan, idx + i);
734 compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT; 733 compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT;
735 hw = compl_desc->hw; 734 hw = compl_desc->hw;
736 hw->ctl = 0; 735 hw->ctl = 0;
@@ -854,7 +853,7 @@ __ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result,
854 */ 853 */
855 BUG_ON(src_cnt + dmaf_continue(flags) < 2); 854 BUG_ON(src_cnt + dmaf_continue(flags) < 2);
856 855
857 num_descs = ioat2_xferlen_to_descs(ioat_chan, len); 856 num_descs = ioat_xferlen_to_descs(ioat_chan, len);
858 /* we need 2x the number of descriptors to cover greater than 3 857 /* we need 2x the number of descriptors to cover greater than 3
859 * sources (we need 1 extra source in the q-only continuation 858 * sources (we need 1 extra source in the q-only continuation
860 * case and 3 extra sources in the p+q continuation case. 859 * case and 3 extra sources in the p+q continuation case.
@@ -872,7 +871,7 @@ __ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result,
872 * order. 871 * order.
873 */ 872 */
874 if (likely(num_descs) && 873 if (likely(num_descs) &&
875 ioat2_check_space_lock(ioat_chan, num_descs + cb32) == 0) 874 ioat_check_space_lock(ioat_chan, num_descs + cb32) == 0)
876 idx = ioat_chan->head; 875 idx = ioat_chan->head;
877 else 876 else
878 return NULL; 877 return NULL;
@@ -882,14 +881,14 @@ __ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result,
882 size_t xfer_size = min_t(size_t, len, 881 size_t xfer_size = min_t(size_t, len,
883 1 << ioat_chan->xfercap_log); 882 1 << ioat_chan->xfercap_log);
884 883
885 desc = ioat2_get_ring_ent(ioat_chan, idx + i); 884 desc = ioat_get_ring_ent(ioat_chan, idx + i);
886 pq = desc->pq; 885 pq = desc->pq;
887 886
888 /* save a branch by unconditionally retrieving the 887 /* save a branch by unconditionally retrieving the
889 * extended descriptor pq_set_src() knows to not write 888 * extended descriptor pq_set_src() knows to not write
890 * to it in the single descriptor case 889 * to it in the single descriptor case
891 */ 890 */
892 ext = ioat2_get_ring_ent(ioat_chan, idx + i + with_ext); 891 ext = ioat_get_ring_ent(ioat_chan, idx + i + with_ext);
893 pq_ex = ext->pq_ex; 892 pq_ex = ext->pq_ex;
894 893
895 descs[0] = (struct ioat_raw_descriptor *) pq; 894 descs[0] = (struct ioat_raw_descriptor *) pq;
@@ -936,7 +935,7 @@ __ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result,
936 compl_desc = desc; 935 compl_desc = desc;
937 } else { 936 } else {
938 /* completion descriptor carries interrupt bit */ 937 /* completion descriptor carries interrupt bit */
939 compl_desc = ioat2_get_ring_ent(ioat_chan, idx + i); 938 compl_desc = ioat_get_ring_ent(ioat_chan, idx + i);
940 compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT; 939 compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT;
941 hw = compl_desc->hw; 940 hw = compl_desc->hw;
942 hw->ctl = 0; 941 hw->ctl = 0;
@@ -972,13 +971,13 @@ __ioat3_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result,
972 971
973 dev_dbg(to_dev(ioat_chan), "%s\n", __func__); 972 dev_dbg(to_dev(ioat_chan), "%s\n", __func__);
974 973
975 num_descs = ioat2_xferlen_to_descs(ioat_chan, len); 974 num_descs = ioat_xferlen_to_descs(ioat_chan, len);
976 975
977 /* 976 /*
978 * 16 source pq is only available on cb3.3 and has no completion 977 * 16 source pq is only available on cb3.3 and has no completion
979 * write hw bug. 978 * write hw bug.
980 */ 979 */
981 if (num_descs && ioat2_check_space_lock(ioat_chan, num_descs) == 0) 980 if (num_descs && ioat_check_space_lock(ioat_chan, num_descs) == 0)
982 idx = ioat_chan->head; 981 idx = ioat_chan->head;
983 else 982 else
984 return NULL; 983 return NULL;
@@ -990,7 +989,7 @@ __ioat3_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result,
990 size_t xfer_size = min_t(size_t, len, 989 size_t xfer_size = min_t(size_t, len,
991 1 << ioat_chan->xfercap_log); 990 1 << ioat_chan->xfercap_log);
992 991
993 desc = ioat2_get_ring_ent(ioat_chan, idx + i); 992 desc = ioat_get_ring_ent(ioat_chan, idx + i);
994 pq = desc->pq; 993 pq = desc->pq;
995 994
996 descs[0] = (struct ioat_raw_descriptor *) pq; 995 descs[0] = (struct ioat_raw_descriptor *) pq;
@@ -1177,8 +1176,8 @@ ioat3_prep_interrupt_lock(struct dma_chan *c, unsigned long flags)
1177 struct ioat_ring_ent *desc; 1176 struct ioat_ring_ent *desc;
1178 struct ioat_dma_descriptor *hw; 1177 struct ioat_dma_descriptor *hw;
1179 1178
1180 if (ioat2_check_space_lock(ioat_chan, 1) == 0) 1179 if (ioat_check_space_lock(ioat_chan, 1) == 0)
1181 desc = ioat2_get_ring_ent(ioat_chan, ioat_chan->head); 1180 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head);
1182 else 1181 else
1183 return NULL; 1182 return NULL;
1184 1183
@@ -1533,7 +1532,7 @@ static int ioat3_reset_hw(struct ioatdma_chan *ioat_chan)
1533 u16 dev_id; 1532 u16 dev_id;
1534 int err; 1533 int err;
1535 1534
1536 ioat2_quiesce(ioat_chan, msecs_to_jiffies(100)); 1535 ioat_quiesce(ioat_chan, msecs_to_jiffies(100));
1537 1536
1538 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); 1537 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
1539 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET); 1538 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
@@ -1561,7 +1560,7 @@ static int ioat3_reset_hw(struct ioatdma_chan *ioat_chan)
1561 } 1560 }
1562 } 1561 }
1563 1562
1564 err = ioat2_reset_sync(ioat_chan, msecs_to_jiffies(200)); 1563 err = ioat_reset_sync(ioat_chan, msecs_to_jiffies(200));
1565 if (!err) 1564 if (!err)
1566 err = ioat3_irq_reinit(ioat_dma); 1565 err = ioat3_irq_reinit(ioat_dma);
1567 1566
@@ -1607,15 +1606,15 @@ int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca)
1607 bool is_raid_device = false; 1606 bool is_raid_device = false;
1608 int err; 1607 int err;
1609 1608
1610 ioat_dma->enumerate_channels = ioat2_enumerate_channels; 1609 ioat_dma->enumerate_channels = ioat_enumerate_channels;
1611 ioat_dma->reset_hw = ioat3_reset_hw; 1610 ioat_dma->reset_hw = ioat3_reset_hw;
1612 ioat_dma->self_test = ioat3_dma_self_test; 1611 ioat_dma->self_test = ioat3_dma_self_test;
1613 ioat_dma->intr_quirk = ioat3_intr_quirk; 1612 ioat_dma->intr_quirk = ioat3_intr_quirk;
1614 dma = &ioat_dma->dma_dev; 1613 dma = &ioat_dma->dma_dev;
1615 dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock; 1614 dma->device_prep_dma_memcpy = ioat_dma_prep_memcpy_lock;
1616 dma->device_issue_pending = ioat2_issue_pending; 1615 dma->device_issue_pending = ioat_issue_pending;
1617 dma->device_alloc_chan_resources = ioat2_alloc_chan_resources; 1616 dma->device_alloc_chan_resources = ioat_alloc_chan_resources;
1618 dma->device_free_chan_resources = ioat2_free_chan_resources; 1617 dma->device_free_chan_resources = ioat_free_chan_resources;
1619 1618
1620 dma_cap_set(DMA_INTERRUPT, dma->cap_mask); 1619 dma_cap_set(DMA_INTERRUPT, dma->cap_mask);
1621 dma->device_prep_dma_interrupt = ioat3_prep_interrupt_lock; 1620 dma->device_prep_dma_interrupt = ioat3_prep_interrupt_lock;
@@ -1708,7 +1707,7 @@ int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca)
1708 if (err) 1707 if (err)
1709 return err; 1708 return err;
1710 1709
1711 ioat_kobject_add(ioat_dma, &ioat2_ktype); 1710 ioat_kobject_add(ioat_dma, &ioat_ktype);
1712 1711
1713 if (dca) 1712 if (dca)
1714 ioat_dma->dca = ioat3_dca_init(pdev, ioat_dma->reg_base); 1713 ioat_dma->dca = ioat3_dca_init(pdev, ioat_dma->reg_base);
diff --git a/drivers/dma/ioat/pci.c b/drivers/dma/ioat/pci.c
index b979a667f501..0ee610224ecd 100644
--- a/drivers/dma/ioat/pci.c
+++ b/drivers/dma/ioat/pci.c
@@ -29,7 +29,6 @@
29#include <linux/slab.h> 29#include <linux/slab.h>
30#include <linux/acpi.h> 30#include <linux/acpi.h>
31#include "dma.h" 31#include "dma.h"
32#include "dma_v2.h"
33#include "registers.h" 32#include "registers.h"
34#include "hw.h" 33#include "hw.h"
35 34
@@ -115,7 +114,7 @@ static int ioat_dca_enabled = 1;
115module_param(ioat_dca_enabled, int, 0644); 114module_param(ioat_dca_enabled, int, 0644);
116MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)"); 115MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)");
117 116
118struct kmem_cache *ioat2_cache; 117struct kmem_cache *ioat_cache;
119struct kmem_cache *ioat3_sed_cache; 118struct kmem_cache *ioat3_sed_cache;
120 119
121#define DRV_NAME "ioatdma" 120#define DRV_NAME "ioatdma"
@@ -246,14 +245,14 @@ static int __init ioat_init_module(void)
246 pr_info("%s: Intel(R) QuickData Technology Driver %s\n", 245 pr_info("%s: Intel(R) QuickData Technology Driver %s\n",
247 DRV_NAME, IOAT_DMA_VERSION); 246 DRV_NAME, IOAT_DMA_VERSION);
248 247
249 ioat2_cache = kmem_cache_create("ioat2", sizeof(struct ioat_ring_ent), 248 ioat_cache = kmem_cache_create("ioat", sizeof(struct ioat_ring_ent),
250 0, SLAB_HWCACHE_ALIGN, NULL); 249 0, SLAB_HWCACHE_ALIGN, NULL);
251 if (!ioat2_cache) 250 if (!ioat_cache)
252 return -ENOMEM; 251 return -ENOMEM;
253 252
254 ioat3_sed_cache = KMEM_CACHE(ioat_sed_ent, 0); 253 ioat3_sed_cache = KMEM_CACHE(ioat_sed_ent, 0);
255 if (!ioat3_sed_cache) 254 if (!ioat3_sed_cache)
256 goto err_ioat2_cache; 255 goto err_ioat_cache;
257 256
258 err = pci_register_driver(&ioat_pci_driver); 257 err = pci_register_driver(&ioat_pci_driver);
259 if (err) 258 if (err)
@@ -264,8 +263,8 @@ static int __init ioat_init_module(void)
264 err_ioat3_cache: 263 err_ioat3_cache:
265 kmem_cache_destroy(ioat3_sed_cache); 264 kmem_cache_destroy(ioat3_sed_cache);
266 265
267 err_ioat2_cache: 266 err_ioat_cache:
268 kmem_cache_destroy(ioat2_cache); 267 kmem_cache_destroy(ioat_cache);
269 268
270 return err; 269 return err;
271} 270}
@@ -274,6 +273,6 @@ module_init(ioat_init_module);
274static void __exit ioat_exit_module(void) 273static void __exit ioat_exit_module(void)
275{ 274{
276 pci_unregister_driver(&ioat_pci_driver); 275 pci_unregister_driver(&ioat_pci_driver);
277 kmem_cache_destroy(ioat2_cache); 276 kmem_cache_destroy(ioat_cache);
278} 277}
279module_exit(ioat_exit_module); 278module_exit(ioat_exit_module);
generated by cgit v1.2.2 (git 2.25.0) at 2025-04-24 22:39:42 -0400