aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--drivers/usb/renesas_usbhs/mod.c34
1 files changed, 19 insertions, 15 deletions
diff --git a/drivers/usb/renesas_usbhs/mod.c b/drivers/usb/renesas_usbhs/mod.c
index 73604a1d6843..d0f5f67e0749 100644
--- a/drivers/usb/renesas_usbhs/mod.c
+++ b/drivers/usb/renesas_usbhs/mod.c
@@ -179,15 +179,17 @@ static void usbhs_status_get_each_irq(struct usbhs_priv *priv,
179 state->intsts0 = usbhs_read(priv, INTSTS0); 179 state->intsts0 = usbhs_read(priv, INTSTS0);
180 state->intsts1 = usbhs_read(priv, INTSTS1); 180 state->intsts1 = usbhs_read(priv, INTSTS1);
181 181
182 state->brdysts = usbhs_read(priv, BRDYSTS);
183 state->nrdysts = usbhs_read(priv, NRDYSTS);
184 state->bempsts = usbhs_read(priv, BEMPSTS);
185
186 state->dvstctr = usbhs_read(priv, DVSTCTR); 182 state->dvstctr = usbhs_read(priv, DVSTCTR);
187 183
188 /* mask */ 184 /* mask */
189 state->bempsts &= mod->irq_bempsts; 185 if (mod) {
190 state->brdysts &= mod->irq_brdysts; 186 state->brdysts = usbhs_read(priv, BRDYSTS);
187 state->nrdysts = usbhs_read(priv, NRDYSTS);
188 state->bempsts = usbhs_read(priv, BEMPSTS);
189
190 state->bempsts &= mod->irq_bempsts;
191 state->brdysts &= mod->irq_brdysts;
192 }
191} 193}
192 194
193/* 195/*
@@ -259,17 +261,19 @@ void usbhs_irq_callback_update(struct usbhs_priv *priv, struct usbhs_mod *mod)
259 * but "mod->irq_dev_state" will be called. 261 * but "mod->irq_dev_state" will be called.
260 */ 262 */
261 263
262 if (mod->irq_ctrl_stage) 264 if (mod) {
263 intenb0 |= CTRE; 265 if (mod->irq_ctrl_stage)
266 intenb0 |= CTRE;
264 267
265 if (mod->irq_empty && mod->irq_bempsts) { 268 if (mod->irq_empty && mod->irq_bempsts) {
266 usbhs_write(priv, BEMPENB, mod->irq_bempsts); 269 usbhs_write(priv, BEMPENB, mod->irq_bempsts);
267 intenb0 |= BEMPE; 270 intenb0 |= BEMPE;
268 } 271 }
269 272
270 if (mod->irq_ready && mod->irq_brdysts) { 273 if (mod->irq_ready && mod->irq_brdysts) {
271 usbhs_write(priv, BRDYENB, mod->irq_brdysts); 274 usbhs_write(priv, BRDYENB, mod->irq_brdysts);
272 intenb0 |= BRDYE; 275 intenb0 |= BRDYE;
276 }
273 } 277 }
274 278
275 usbhs_write(priv, INTENB0, intenb0); 279 usbhs_write(priv, INTENB0, intenb0);
' href='#n539'>539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266
/* $Id: sbus.c,v 1.19 2002/01/23 11:27:32 davem Exp $
 * sbus.c: UltraSparc SBUS controller support.
 *
 * Copyright (C) 1999 David S. Miller (davem@redhat.com)
 */

#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/mm.h>
#include <linux/spinlock.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/interrupt.h>

#include <asm/page.h>
#include <asm/sbus.h>
#include <asm/io.h>
#include <asm/upa.h>
#include <asm/cache.h>
#include <asm/dma.h>
#include <asm/irq.h>
#include <asm/starfire.h>

#include "iommu_common.h"

/* These should be allocated on an SMP_CACHE_BYTES
 * aligned boundary for optimal performance.
 *
 * On SYSIO, using an 8K page size we have 1GB of SBUS
 * DMA space mapped.  We divide this space into equally
 * sized clusters. We allocate a DMA mapping from the
 * cluster that matches the order of the allocation, or
 * if the order is greater than the number of clusters,
 * we try to allocate from the last cluster.
 */

#define NCLUSTERS	8UL
#define ONE_GIG		(1UL * 1024UL * 1024UL * 1024UL)
#define CLUSTER_SIZE	(ONE_GIG / NCLUSTERS)
#define CLUSTER_MASK	(CLUSTER_SIZE - 1)
#define CLUSTER_NPAGES	(CLUSTER_SIZE >> IO_PAGE_SHIFT)
#define MAP_BASE	((u32)0xc0000000)

struct sbus_iommu {
/*0x00*/spinlock_t		lock;

/*0x08*/iopte_t			*page_table;
/*0x10*/unsigned long		strbuf_regs;
/*0x18*/unsigned long		iommu_regs;
/*0x20*/unsigned long		sbus_control_reg;

/*0x28*/volatile unsigned long	strbuf_flushflag;

	/* If NCLUSTERS is ever decresed to 4 or lower,
	 * you must increase the size of the type of
	 * these counters.  You have been duly warned. -DaveM
	 */
/*0x30*/struct {
		u16	next;
		u16	flush;
	} alloc_info[NCLUSTERS];

	/* The lowest used consistent mapping entry.  Since
	 * we allocate consistent maps out of cluster 0 this
	 * is relative to the beginning of closter 0.
	 */
/*0x50*/u32		lowest_consistent_map;
};

/* Offsets from iommu_regs */
#define SYSIO_IOMMUREG_BASE	0x2400UL
#define IOMMU_CONTROL	(0x2400UL - 0x2400UL)	/* IOMMU control register */
#define IOMMU_TSBBASE	(0x2408UL - 0x2400UL)	/* TSB base address register */
#define IOMMU_FLUSH	(0x2410UL - 0x2400UL)	/* IOMMU flush register */
#define IOMMU_VADIAG	(0x4400UL - 0x2400UL)	/* SBUS virtual address diagnostic */
#define IOMMU_TAGCMP	(0x4408UL - 0x2400UL)	/* TLB tag compare diagnostics */
#define IOMMU_LRUDIAG	(0x4500UL - 0x2400UL)	/* IOMMU LRU queue diagnostics */
#define IOMMU_TAGDIAG	(0x4580UL - 0x2400UL)	/* TLB tag diagnostics */
#define IOMMU_DRAMDIAG	(0x4600UL - 0x2400UL)	/* TLB data RAM diagnostics */

#define IOMMU_DRAM_VALID	(1UL << 30UL)

static void __iommu_flushall(struct sbus_iommu *iommu)
{
	unsigned long tag = iommu->iommu_regs + IOMMU_TAGDIAG;
	int entry;

	for (entry = 0; entry < 16; entry++) {
		upa_writeq(0, tag);
		tag += 8UL;
	}
	upa_readq(iommu->sbus_control_reg);

	for (entry = 0; entry < NCLUSTERS; entry++) {
		iommu->alloc_info[entry].flush =
			iommu->alloc_info[entry].next;
	}
}

static void iommu_flush(struct sbus_iommu *iommu, u32 base, unsigned long npages)
{
	while (npages--)
		upa_writeq(base + (npages << IO_PAGE_SHIFT),
			   iommu->iommu_regs + IOMMU_FLUSH);
	upa_readq(iommu->sbus_control_reg);
}

/* Offsets from strbuf_regs */
#define SYSIO_STRBUFREG_BASE	0x2800UL
#define STRBUF_CONTROL	(0x2800UL - 0x2800UL)	/* Control */
#define STRBUF_PFLUSH	(0x2808UL - 0x2800UL)	/* Page flush/invalidate */
#define STRBUF_FSYNC	(0x2810UL - 0x2800UL)	/* Flush synchronization */
#define STRBUF_DRAMDIAG	(0x5000UL - 0x2800UL)	/* data RAM diagnostic */
#define STRBUF_ERRDIAG	(0x5400UL - 0x2800UL)	/* error status diagnostics */
#define STRBUF_PTAGDIAG	(0x5800UL - 0x2800UL)	/* Page tag diagnostics */
#define STRBUF_LTAGDIAG	(0x5900UL - 0x2800UL)	/* Line tag diagnostics */

#define STRBUF_TAG_VALID	0x02UL

static void sbus_strbuf_flush(struct sbus_iommu *iommu, u32 base, unsigned long npages, int direction)
{
	unsigned long n;
	int limit;

	n = npages;
	while (n--)
		upa_writeq(base + (n << IO_PAGE_SHIFT),
			   iommu->strbuf_regs + STRBUF_PFLUSH);

	/* If the device could not have possibly put dirty data into
	 * the streaming cache, no flush-flag synchronization needs
	 * to be performed.
	 */
	if (direction == SBUS_DMA_TODEVICE)
		return;

	iommu->strbuf_flushflag = 0UL;

	/* Whoopee cushion! */
	upa_writeq(__pa(&iommu->strbuf_flushflag),
		   iommu->strbuf_regs + STRBUF_FSYNC);
	upa_readq(iommu->sbus_control_reg);

	limit = 100000;
	while (iommu->strbuf_flushflag == 0UL) {
		limit--;
		if (!limit)
			break;
		udelay(1);
		rmb();
	}
	if (!limit)
		printk(KERN_WARNING "sbus_strbuf_flush: flushflag timeout "
		       "vaddr[%08x] npages[%ld]\n",
		       base, npages);
}

static iopte_t *alloc_streaming_cluster(struct sbus_iommu *iommu, unsigned long npages)
{
	iopte_t *iopte, *limit, *first, *cluster;
	unsigned long cnum, ent, nent, flush_point, found;

	cnum = 0;
	nent = 1;
	while ((1UL << cnum) < npages)
		cnum++;
	if(cnum >= NCLUSTERS) {
		nent = 1UL << (cnum - NCLUSTERS);
		cnum = NCLUSTERS - 1;
	}
	iopte  = iommu->page_table + (cnum * CLUSTER_NPAGES);

	if (cnum == 0)
		limit = (iommu->page_table +
			 iommu->lowest_consistent_map);
	else
		limit = (iopte + CLUSTER_NPAGES);

	iopte += ((ent = iommu->alloc_info[cnum].next) << cnum);
	flush_point = iommu->alloc_info[cnum].flush;

	first = iopte;
	cluster = NULL;
	found = 0;
	for (;;) {
		if (iopte_val(*iopte) == 0UL) {
			found++;
			if (!cluster)
				cluster = iopte;
		} else {
			/* Used cluster in the way */
			cluster = NULL;
			found = 0;
		}

		if (found == nent)
			break;

		iopte += (1 << cnum);
		ent++;
		if (iopte >= limit) {
			iopte = (iommu->page_table + (cnum * CLUSTER_NPAGES));
			ent = 0;

			/* Multiple cluster allocations must not wrap */
			cluster = NULL;
			found = 0;
		}
		if (ent == flush_point)
			__iommu_flushall(iommu);
		if (iopte == first)
			goto bad;
	}

	/* ent/iopte points to the last cluster entry we're going to use,
	 * so save our place for the next allocation.
	 */
	if ((iopte + (1 << cnum)) >= limit)
		ent = 0;
	else
		ent = ent + 1;
	iommu->alloc_info[cnum].next = ent;
	if (ent == flush_point)
		__iommu_flushall(iommu);

	/* I've got your streaming cluster right here buddy boy... */
	return cluster;

bad:
	printk(KERN_EMERG "sbus: alloc_streaming_cluster of npages(%ld) failed!\n",
	       npages);
	return NULL;
}

static void free_streaming_cluster(struct sbus_iommu *iommu, u32 base, unsigned long npages)
{
	unsigned long cnum, ent, nent;
	iopte_t *iopte;

	cnum = 0;
	nent = 1;
	while ((1UL << cnum) < npages)
		cnum++;
	if(cnum >= NCLUSTERS) {
		nent = 1UL << (cnum - NCLUSTERS);
		cnum = NCLUSTERS - 1;
	}
	ent = (base & CLUSTER_MASK) >> (IO_PAGE_SHIFT + cnum);
	iopte = iommu->page_table + ((base - MAP_BASE) >> IO_PAGE_SHIFT);
	do {
		iopte_val(*iopte) = 0UL;
		iopte += 1 << cnum;
	} while(--nent);

	/* If the global flush might not have caught this entry,
	 * adjust the flush point such that we will flush before
	 * ever trying to reuse it.
	 */
#define between(X,Y,Z)	(((Z) - (Y)) >= ((X) - (Y)))
	if (between(ent, iommu->alloc_info[cnum].next, iommu->alloc_info[cnum].flush))
		iommu->alloc_info[cnum].flush = ent;
#undef between
}

/* We allocate consistent mappings from the end of cluster zero. */
static iopte_t *alloc_consistent_cluster(struct sbus_iommu *iommu, unsigned long npages)
{
	iopte_t *iopte;

	iopte = iommu->page_table + (1 * CLUSTER_NPAGES);
	while (iopte > iommu->page_table) {
		iopte--;
		if (!(iopte_val(*iopte) & IOPTE_VALID)) {
			unsigned long tmp = npages;

			while (--tmp) {
				iopte--;
				if (iopte_val(*iopte) & IOPTE_VALID)
					break;
			}
			if (tmp == 0) {
				u32 entry = (iopte - iommu->page_table);

				if (entry < iommu->lowest_consistent_map)
					iommu->lowest_consistent_map = entry;
				return iopte;
			}
		}
	}
	return NULL;
}

static void free_consistent_cluster(struct sbus_iommu *iommu, u32 base, unsigned long npages)
{
	iopte_t *iopte = iommu->page_table + ((base - MAP_BASE) >> IO_PAGE_SHIFT);

	if ((iopte - iommu->page_table) == iommu->lowest_consistent_map) {
		iopte_t *walk = iopte + npages;
		iopte_t *limit;

		limit = iommu->page_table + CLUSTER_NPAGES;
		while (walk < limit) {
			if (iopte_val(*walk) != 0UL)
				break;
			walk++;
		}
		iommu->lowest_consistent_map =
			(walk - iommu->page_table);
	}

	while (npages--)
		*iopte++ = __iopte(0UL);
}

void *sbus_alloc_consistent(struct sbus_dev *sdev, size_t size, dma_addr_t *dvma_addr)
{
	unsigned long order, first_page, flags;
	struct sbus_iommu *iommu;
	iopte_t *iopte;
	void *ret;
	int npages;

	if (size <= 0 || sdev == NULL || dvma_addr == NULL)
		return NULL;

	size = IO_PAGE_ALIGN(size);
	order = get_order(size);
	if (order >= 10)
		return NULL;
	first_page = __get_free_pages(GFP_KERNEL, order);
	if (first_page == 0UL)
		return NULL;
	memset((char *)first_page, 0, PAGE_SIZE << order);

	iommu = sdev->bus->iommu;

	spin_lock_irqsave(&iommu->lock, flags);
	iopte = alloc_consistent_cluster(iommu, size >> IO_PAGE_SHIFT);
	if (iopte == NULL) {
		spin_unlock_irqrestore(&iommu->lock, flags);
		free_pages(first_page, order);
		return NULL;
	}

	/* Ok, we're committed at this point. */
	*dvma_addr = MAP_BASE +	((iopte - iommu->page_table) << IO_PAGE_SHIFT);
	ret = (void *) first_page;
	npages = size >> IO_PAGE_SHIFT;
	while (npages--) {
		*iopte++ = __iopte(IOPTE_VALID | IOPTE_CACHE | IOPTE_WRITE |
				   (__pa(first_page) & IOPTE_PAGE));
		first_page += IO_PAGE_SIZE;
	}
	iommu_flush(iommu, *dvma_addr, size >> IO_PAGE_SHIFT);
	spin_unlock_irqrestore(&iommu->lock, flags);

	return ret;
}

void sbus_free_consistent(struct sbus_dev *sdev, size_t size, void *cpu, dma_addr_t dvma)
{
	unsigned long order, npages;
	struct sbus_iommu *iommu;

	if (size <= 0 || sdev == NULL || cpu == NULL)
		return;

	npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT;
	iommu = sdev->bus->iommu;

	spin_lock_irq(&iommu->lock);
	free_consistent_cluster(iommu, dvma, npages);
	iommu_flush(iommu, dvma, npages);
	spin_unlock_irq(&iommu->lock);

	order = get_order(size);
	if (order < 10)
		free_pages((unsigned long)cpu, order);
}

dma_addr_t sbus_map_single(struct sbus_dev *sdev, void *ptr, size_t size, int dir)
{
	struct sbus_iommu *iommu = sdev->bus->iommu;
	unsigned long npages, pbase, flags;
	iopte_t *iopte;
	u32 dma_base, offset;
	unsigned long iopte_bits;

	if (dir == SBUS_DMA_NONE)
		BUG();

	pbase = (unsigned long) ptr;
	offset = (u32) (pbase & ~IO_PAGE_MASK);
	size = (IO_PAGE_ALIGN(pbase + size) - (pbase & IO_PAGE_MASK));
	pbase = (unsigned long) __pa(pbase & IO_PAGE_MASK);

	spin_lock_irqsave(&iommu->lock, flags);
	npages = size >> IO_PAGE_SHIFT;
	iopte = alloc_streaming_cluster(iommu, npages);
	if (iopte == NULL)
		goto bad;
	dma_base = MAP_BASE + ((iopte - iommu->page_table) << IO_PAGE_SHIFT);
	npages = size >> IO_PAGE_SHIFT;
	iopte_bits = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE;
	if (dir != SBUS_DMA_TODEVICE)
		iopte_bits |= IOPTE_WRITE;
	while (npages--) {
		*iopte++ = __iopte(iopte_bits | (pbase & IOPTE_PAGE));
		pbase += IO_PAGE_SIZE;
	}
	npages = size >> IO_PAGE_SHIFT;
	spin_unlock_irqrestore(&iommu->lock, flags);

	return (dma_base | offset);

bad:
	spin_unlock_irqrestore(&iommu->lock, flags);
	BUG();
	return 0;
}

void sbus_unmap_single(struct sbus_dev *sdev, dma_addr_t dma_addr, size_t size, int direction)
{
	struct sbus_iommu *iommu = sdev->bus->iommu;
	u32 dma_base = dma_addr & IO_PAGE_MASK;
	unsigned long flags;

	size = (IO_PAGE_ALIGN(dma_addr + size) - dma_base);

	spin_lock_irqsave(&iommu->lock, flags);
	free_streaming_cluster(iommu, dma_base, size >> IO_PAGE_SHIFT);
	sbus_strbuf_flush(iommu, dma_base, size >> IO_PAGE_SHIFT, direction);
	spin_unlock_irqrestore(&iommu->lock, flags);
}

#define SG_ENT_PHYS_ADDRESS(SG)	\
	(__pa(page_address((SG)->page)) + (SG)->offset)

static inline void fill_sg(iopte_t *iopte, struct scatterlist *sg, int nused, int nelems, unsigned long iopte_bits)
{
	struct scatterlist *dma_sg = sg;
	struct scatterlist *sg_end = sg + nelems;
	int i;

	for (i = 0; i < nused; i++) {
		unsigned long pteval = ~0UL;
		u32 dma_npages;

		dma_npages = ((dma_sg->dma_address & (IO_PAGE_SIZE - 1UL)) +
			      dma_sg->dma_length +
			      ((IO_PAGE_SIZE - 1UL))) >> IO_PAGE_SHIFT;
		do {
			unsigned long offset;
			signed int len;

			/* If we are here, we know we have at least one
			 * more page to map.  So walk forward until we
			 * hit a page crossing, and begin creating new
			 * mappings from that spot.
			 */
			for (;;) {
				unsigned long tmp;

				tmp = (unsigned long) SG_ENT_PHYS_ADDRESS(sg);
				len = sg->length;
				if (((tmp ^ pteval) >> IO_PAGE_SHIFT) != 0UL) {
					pteval = tmp & IO_PAGE_MASK;
					offset = tmp & (IO_PAGE_SIZE - 1UL);
					break;
				}
				if (((tmp ^ (tmp + len - 1UL)) >> IO_PAGE_SHIFT) != 0UL) {
					pteval = (tmp + IO_PAGE_SIZE) & IO_PAGE_MASK;
					offset = 0UL;
					len -= (IO_PAGE_SIZE - (tmp & (IO_PAGE_SIZE - 1UL)));
					break;
				}
				sg++;
			}

			pteval = ((pteval & IOPTE_PAGE) | iopte_bits);
			while (len > 0) {
				*iopte++ = __iopte(pteval);
				pteval += IO_PAGE_SIZE;
				len -= (IO_PAGE_SIZE - offset);
				offset = 0;
				dma_npages--;
			}

			pteval = (pteval & IOPTE_PAGE) + len;
			sg++;

			/* Skip over any tail mappings we've fully mapped,
			 * adjusting pteval along the way.  Stop when we
			 * detect a page crossing event.
			 */
			while (sg < sg_end &&
			       (pteval << (64 - IO_PAGE_SHIFT)) != 0UL &&
			       (pteval == SG_ENT_PHYS_ADDRESS(sg)) &&
			       ((pteval ^
				 (SG_ENT_PHYS_ADDRESS(sg) + sg->length - 1UL)) >> IO_PAGE_SHIFT) == 0UL) {
				pteval += sg->length;
				sg++;
			}
			if ((pteval << (64 - IO_PAGE_SHIFT)) == 0UL)
				pteval = ~0UL;
		} while (dma_npages != 0);
		dma_sg++;
	}
}

int sbus_map_sg(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int dir)
{
	struct sbus_iommu *iommu = sdev->bus->iommu;
	unsigned long flags, npages;
	iopte_t *iopte;
	u32 dma_base;
	struct scatterlist *sgtmp;
	int used;
	unsigned long iopte_bits;

	if (dir == SBUS_DMA_NONE)
		BUG();

	/* Fast path single entry scatterlists. */
	if (nents == 1) {
		sg->dma_address =
			sbus_map_single(sdev,
					(page_address(sg->page) + sg->offset),
					sg->length, dir);
		sg->dma_length = sg->length;
		return 1;
	}

	npages = prepare_sg(sg, nents);

	spin_lock_irqsave(&iommu->lock, flags);
	iopte = alloc_streaming_cluster(iommu, npages);
	if (iopte == NULL)
		goto bad;
	dma_base = MAP_BASE + ((iopte - iommu->page_table) << IO_PAGE_SHIFT);

	/* Normalize DVMA addresses. */
	sgtmp = sg;
	used = nents;

	while (used && sgtmp->dma_length) {
		sgtmp->dma_address += dma_base;
		sgtmp++;
		used--;
	}
	used = nents - used;

	iopte_bits = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE;
	if (dir != SBUS_DMA_TODEVICE)
		iopte_bits |= IOPTE_WRITE;

	fill_sg(iopte, sg, used, nents, iopte_bits);
#ifdef VERIFY_SG
	verify_sglist(sg, nents, iopte, npages);
#endif
	spin_unlock_irqrestore(&iommu->lock, flags);

	return used;

bad:
	spin_unlock_irqrestore(&iommu->lock, flags);
	BUG();
	return 0;
}

void sbus_unmap_sg(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int direction)
{
	unsigned long size, flags;
	struct sbus_iommu *iommu;
	u32 dvma_base;
	int i;

	/* Fast path single entry scatterlists. */
	if (nents == 1) {
		sbus_unmap_single(sdev, sg->dma_address, sg->dma_length, direction);
		return;
	}

	dvma_base = sg[0].dma_address & IO_PAGE_MASK;
	for (i = 0; i < nents; i++) {
		if (sg[i].dma_length == 0)
			break;
	}
	i--;
	size = IO_PAGE_ALIGN(sg[i].dma_address + sg[i].dma_length) - dvma_base;

	iommu = sdev->bus->iommu;
	spin_lock_irqsave(&iommu->lock, flags);
	free_streaming_cluster(iommu, dvma_base, size >> IO_PAGE_SHIFT);
	sbus_strbuf_flush(iommu, dvma_base, size >> IO_PAGE_SHIFT, direction);
	spin_unlock_irqrestore(&iommu->lock, flags);
}

void sbus_dma_sync_single_for_cpu(struct sbus_dev *sdev, dma_addr_t base, size_t size, int direction)
{
	struct sbus_iommu *iommu = sdev->bus->iommu;
	unsigned long flags;

	size = (IO_PAGE_ALIGN(base + size) - (base & IO_PAGE_MASK));

	spin_lock_irqsave(&iommu->lock, flags);
	sbus_strbuf_flush(iommu, base & IO_PAGE_MASK, size >> IO_PAGE_SHIFT, direction);
	spin_unlock_irqrestore(&iommu->lock, flags);
}

void sbus_dma_sync_single_for_device(struct sbus_dev *sdev, dma_addr_t base, size_t size, int direction)
{
}

void sbus_dma_sync_sg_for_cpu(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int direction)
{
	struct sbus_iommu *iommu = sdev->bus->iommu;
	unsigned long flags, size;
	u32 base;
	int i;

	base = sg[0].dma_address & IO_PAGE_MASK;
	for (i = 0; i < nents; i++) {
		if (sg[i].dma_length == 0)
			break;
	}
	i--;
	size = IO_PAGE_ALIGN(sg[i].dma_address + sg[i].dma_length) - base;

	spin_lock_irqsave(&iommu->lock, flags);
	sbus_strbuf_flush(iommu, base, size >> IO_PAGE_SHIFT, direction);
	spin_unlock_irqrestore(&iommu->lock, flags);
}

void sbus_dma_sync_sg_for_device(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int direction)
{
}

/* Enable 64-bit DVMA mode for the given device. */
void sbus_set_sbus64(struct sbus_dev *sdev, int bursts)
{
	struct sbus_iommu *iommu = sdev->bus->iommu;
	int slot = sdev->slot;
	unsigned long cfg_reg;
	u64 val;

	cfg_reg = iommu->sbus_control_reg;
	switch (slot) {
	case 0:
		cfg_reg += 0x20UL;
		break;
	case 1:
		cfg_reg += 0x28UL;
		break;
	case 2:
		cfg_reg += 0x30UL;
		break;
	case 3:
		cfg_reg += 0x38UL;
		break;
	case 13:
		cfg_reg += 0x40UL;
		break;
	case 14:
		cfg_reg += 0x48UL;
		break;
	case 15:
		cfg_reg += 0x50UL;
		break;

	default:
		return;
	};

	val = upa_readq(cfg_reg);
	if (val & (1UL << 14UL)) {
		/* Extended transfer mode already enabled. */
		return;
	}

	val |= (1UL << 14UL);

	if (bursts & DMA_BURST8)
		val |= (1UL << 1UL);
	if (bursts & DMA_BURST16)
		val |= (1UL << 2UL);
	if (bursts & DMA_BURST32)
		val |= (1UL << 3UL);
	if (bursts & DMA_BURST64)
		val |= (1UL << 4UL);
	upa_writeq(val, cfg_reg);
}

/* SBUS SYSIO INO number to Sparc PIL level. */
static unsigned char sysio_ino_to_pil[] = {
	0, 4, 4, 7, 5, 7, 8, 9,		/* SBUS slot 0 */
	0, 4, 4, 7, 5, 7, 8, 9,		/* SBUS slot 1 */
	0, 4, 4, 7, 5, 7, 8, 9,		/* SBUS slot 2 */
	0, 4, 4, 7, 5, 7, 8, 9,		/* SBUS slot 3 */
	4, /* Onboard SCSI */
	5, /* Onboard Ethernet */
/*XXX*/	8, /* Onboard BPP */
	0, /* Bogon */
       13, /* Audio */
/*XXX*/15, /* PowerFail */
	0, /* Bogon */
	0, /* Bogon */
       12, /* Zilog Serial Channels (incl. Keyboard/Mouse lines) */
       11, /* Floppy */
	0, /* Spare Hardware (bogon for now) */
	0, /* Keyboard (bogon for now) */
	0, /* Mouse (bogon for now) */
	0, /* Serial (bogon for now) */
     0, 0, /* Bogon, Bogon */
       10, /* Timer 0 */
       11, /* Timer 1 */
     0, 0, /* Bogon, Bogon */
       15, /* Uncorrectable SBUS Error */
       15, /* Correctable SBUS Error */
       15, /* SBUS Error */
/*XXX*/ 0, /* Power Management (bogon for now) */
};

/* INO number to IMAP register offset for SYSIO external IRQ's.
 * This should conform to both Sunfire/Wildfire server and Fusion
 * desktop designs.
 */
#define SYSIO_IMAP_SLOT0	0x2c04UL
#define SYSIO_IMAP_SLOT1	0x2c0cUL
#define SYSIO_IMAP_SLOT2	0x2c14UL
#define SYSIO_IMAP_SLOT3	0x2c1cUL
#define SYSIO_IMAP_SCSI		0x3004UL
#define SYSIO_IMAP_ETH		0x300cUL
#define SYSIO_IMAP_BPP		0x3014UL
#define SYSIO_IMAP_AUDIO	0x301cUL
#define SYSIO_IMAP_PFAIL	0x3024UL
#define SYSIO_IMAP_KMS		0x302cUL
#define SYSIO_IMAP_FLPY		0x3034UL
#define SYSIO_IMAP_SHW		0x303cUL
#define SYSIO_IMAP_KBD		0x3044UL
#define SYSIO_IMAP_MS		0x304cUL
#define SYSIO_IMAP_SER		0x3054UL
#define SYSIO_IMAP_TIM0		0x3064UL
#define SYSIO_IMAP_TIM1		0x306cUL
#define SYSIO_IMAP_UE		0x3074UL
#define SYSIO_IMAP_CE		0x307cUL
#define SYSIO_IMAP_SBERR	0x3084UL
#define SYSIO_IMAP_PMGMT	0x308cUL
#define SYSIO_IMAP_GFX		0x3094UL
#define SYSIO_IMAP_EUPA		0x309cUL

#define bogon     ((unsigned long) -1)
static unsigned long sysio_irq_offsets[] = {
	/* SBUS Slot 0 --> 3, level 1 --> 7 */
	SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0,
	SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0,
	SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1,
	SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1,
	SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2,
	SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2,
	SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3,
	SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3,

	/* Onboard devices (not relevant/used on SunFire). */
	SYSIO_IMAP_SCSI,
	SYSIO_IMAP_ETH,
	SYSIO_IMAP_BPP,
	bogon,
	SYSIO_IMAP_AUDIO,
	SYSIO_IMAP_PFAIL,
	bogon,
	bogon,
	SYSIO_IMAP_KMS,
	SYSIO_IMAP_FLPY,
	SYSIO_IMAP_SHW,
	SYSIO_IMAP_KBD,
	SYSIO_IMAP_MS,
	SYSIO_IMAP_SER,
	bogon,
	bogon,
	SYSIO_IMAP_TIM0,
	SYSIO_IMAP_TIM1,
	bogon,
	bogon,
	SYSIO_IMAP_UE,
	SYSIO_IMAP_CE,
	SYSIO_IMAP_SBERR,
	SYSIO_IMAP_PMGMT,
};

#undef bogon

#define NUM_SYSIO_OFFSETS (sizeof(sysio_irq_offsets) / sizeof(sysio_irq_offsets[0]))

/* Convert Interrupt Mapping register pointer to associated
 * Interrupt Clear register pointer, SYSIO specific version.
 */
#define SYSIO_ICLR_UNUSED0	0x3400UL
#define SYSIO_ICLR_SLOT0	0x340cUL
#define SYSIO_ICLR_SLOT1	0x344cUL
#define SYSIO_ICLR_SLOT2	0x348cUL
#define SYSIO_ICLR_SLOT3	0x34ccUL
static unsigned long sysio_imap_to_iclr(unsigned long imap)
{
	unsigned long diff = SYSIO_ICLR_UNUSED0 - SYSIO_IMAP_SLOT0;
	return imap + diff;
}

unsigned int sbus_build_irq(void *buscookie, unsigned int ino)
{
	struct sbus_bus *sbus = (struct sbus_bus *)buscookie;
	struct sbus_iommu *iommu = sbus->iommu;
	unsigned long reg_base = iommu->sbus_control_reg - 0x2000UL;
	unsigned long imap, iclr;
	int pil, sbus_level = 0;

	pil = sysio_ino_to_pil[ino];
	if (!pil) {
		printk("sbus_irq_build: Bad SYSIO INO[%x]\n", ino);
		panic("Bad SYSIO IRQ translations...");
	}

	if (PIL_RESERVED(pil))
		BUG();

	imap = sysio_irq_offsets[ino];
	if (imap == ((unsigned long)-1)) {
		prom_printf("get_irq_translations: Bad SYSIO INO[%x] cpu[%d]\n",
			    ino, pil);
		prom_halt();
	}
	imap += reg_base;

	/* SYSIO inconsistency.  For external SLOTS, we have to select
	 * the right ICLR register based upon the lower SBUS irq level
	 * bits.
	 */
	if (ino >= 0x20) {
		iclr = sysio_imap_to_iclr(imap);
	} else {
		int sbus_slot = (ino & 0x18)>>3;
		
		sbus_level = ino & 0x7;

		switch(sbus_slot) {
		case 0:
			iclr = reg_base + SYSIO_ICLR_SLOT0;
			break;
		case 1:
			iclr = reg_base + SYSIO_ICLR_SLOT1;
			break;
		case 2:
			iclr = reg_base + SYSIO_ICLR_SLOT2;
			break;
		default:
		case 3:
			iclr = reg_base + SYSIO_ICLR_SLOT3;
			break;
		};

		iclr += ((unsigned long)sbus_level - 1UL) * 8UL;
	}
	return build_irq(pil, sbus_level, iclr, imap);
}

/* Error interrupt handling. */
#define SYSIO_UE_AFSR	0x0030UL
#define SYSIO_UE_AFAR	0x0038UL
#define  SYSIO_UEAFSR_PPIO  0x8000000000000000UL /* Primary PIO cause         */
#define  SYSIO_UEAFSR_PDRD  0x4000000000000000UL /* Primary DVMA read cause   */
#define  SYSIO_UEAFSR_PDWR  0x2000000000000000UL /* Primary DVMA write cause  */
#define  SYSIO_UEAFSR_SPIO  0x1000000000000000UL /* Secondary PIO is cause    */
#define  SYSIO_UEAFSR_SDRD  0x0800000000000000UL /* Secondary DVMA read cause */
#define  SYSIO_UEAFSR_SDWR  0x0400000000000000UL /* Secondary DVMA write cause*/
#define  SYSIO_UEAFSR_RESV1 0x03ff000000000000UL /* Reserved                  */
#define  SYSIO_UEAFSR_DOFF  0x0000e00000000000UL /* Doubleword Offset         */
#define  SYSIO_UEAFSR_SIZE  0x00001c0000000000UL /* Bad transfer size 2^SIZE  */
#define  SYSIO_UEAFSR_MID   0x000003e000000000UL /* UPA MID causing the fault */
#define  SYSIO_UEAFSR_RESV2 0x0000001fffffffffUL /* Reserved                  */
static irqreturn_t sysio_ue_handler(int irq, void *dev_id, struct pt_regs *regs)
{
	struct sbus_bus *sbus = dev_id;
	struct sbus_iommu *iommu = sbus->iommu;
	unsigned long reg_base = iommu->sbus_control_reg - 0x2000UL;
	unsigned long afsr_reg, afar_reg;