sbus.c 35 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243
  1. /* $Id: sbus.c,v 1.19 2002/01/23 11:27:32 davem Exp $
  2. * sbus.c: UltraSparc SBUS controller support.
  3. *
  4. * Copyright (C) 1999 David S. Miller (davem@redhat.com)
  5. */
  6. #include <linux/kernel.h>
  7. #include <linux/types.h>
  8. #include <linux/mm.h>
  9. #include <linux/spinlock.h>
  10. #include <linux/slab.h>
  11. #include <linux/init.h>
  12. #include <linux/interrupt.h>
  13. #include <asm/page.h>
  14. #include <asm/sbus.h>
  15. #include <asm/io.h>
  16. #include <asm/upa.h>
  17. #include <asm/cache.h>
  18. #include <asm/dma.h>
  19. #include <asm/irq.h>
  20. #include <asm/starfire.h>
  21. #include "iommu_common.h"
  22. /* These should be allocated on an SMP_CACHE_BYTES
  23. * aligned boundary for optimal performance.
  24. *
  25. * On SYSIO, using an 8K page size we have 1GB of SBUS
  26. * DMA space mapped. We divide this space into equally
  27. * sized clusters. We allocate a DMA mapping from the
  28. * cluster that matches the order of the allocation, or
  29. * if the order is greater than the number of clusters,
  30. * we try to allocate from the last cluster.
  31. */
  32. #define NCLUSTERS 8UL
  33. #define ONE_GIG (1UL * 1024UL * 1024UL * 1024UL)
  34. #define CLUSTER_SIZE (ONE_GIG / NCLUSTERS)
  35. #define CLUSTER_MASK (CLUSTER_SIZE - 1)
  36. #define CLUSTER_NPAGES (CLUSTER_SIZE >> IO_PAGE_SHIFT)
  37. #define MAP_BASE ((u32)0xc0000000)
  38. struct sbus_iommu {
  39. /*0x00*/spinlock_t lock;
  40. /*0x08*/iopte_t *page_table;
  41. /*0x10*/unsigned long strbuf_regs;
  42. /*0x18*/unsigned long iommu_regs;
  43. /*0x20*/unsigned long sbus_control_reg;
  44. /*0x28*/volatile unsigned long strbuf_flushflag;
  45. /* If NCLUSTERS is ever decresed to 4 or lower,
  46. * you must increase the size of the type of
  47. * these counters. You have been duly warned. -DaveM
  48. */
  49. /*0x30*/struct {
  50. u16 next;
  51. u16 flush;
  52. } alloc_info[NCLUSTERS];
  53. /* The lowest used consistent mapping entry. Since
  54. * we allocate consistent maps out of cluster 0 this
  55. * is relative to the beginning of closter 0.
  56. */
  57. /*0x50*/u32 lowest_consistent_map;
  58. };
  59. /* Offsets from iommu_regs */
  60. #define SYSIO_IOMMUREG_BASE 0x2400UL
  61. #define IOMMU_CONTROL (0x2400UL - 0x2400UL) /* IOMMU control register */
  62. #define IOMMU_TSBBASE (0x2408UL - 0x2400UL) /* TSB base address register */
  63. #define IOMMU_FLUSH (0x2410UL - 0x2400UL) /* IOMMU flush register */
  64. #define IOMMU_VADIAG (0x4400UL - 0x2400UL) /* SBUS virtual address diagnostic */
  65. #define IOMMU_TAGCMP (0x4408UL - 0x2400UL) /* TLB tag compare diagnostics */
  66. #define IOMMU_LRUDIAG (0x4500UL - 0x2400UL) /* IOMMU LRU queue diagnostics */
  67. #define IOMMU_TAGDIAG (0x4580UL - 0x2400UL) /* TLB tag diagnostics */
  68. #define IOMMU_DRAMDIAG (0x4600UL - 0x2400UL) /* TLB data RAM diagnostics */
  69. #define IOMMU_DRAM_VALID (1UL << 30UL)
  70. static void __iommu_flushall(struct sbus_iommu *iommu)
  71. {
  72. unsigned long tag = iommu->iommu_regs + IOMMU_TAGDIAG;
  73. int entry;
  74. for (entry = 0; entry < 16; entry++) {
  75. upa_writeq(0, tag);
  76. tag += 8UL;
  77. }
  78. upa_readq(iommu->sbus_control_reg);
  79. for (entry = 0; entry < NCLUSTERS; entry++) {
  80. iommu->alloc_info[entry].flush =
  81. iommu->alloc_info[entry].next;
  82. }
  83. }
  84. static void iommu_flush(struct sbus_iommu *iommu, u32 base, unsigned long npages)
  85. {
  86. while (npages--)
  87. upa_writeq(base + (npages << IO_PAGE_SHIFT),
  88. iommu->iommu_regs + IOMMU_FLUSH);
  89. upa_readq(iommu->sbus_control_reg);
  90. }
  91. /* Offsets from strbuf_regs */
  92. #define SYSIO_STRBUFREG_BASE 0x2800UL
  93. #define STRBUF_CONTROL (0x2800UL - 0x2800UL) /* Control */
  94. #define STRBUF_PFLUSH (0x2808UL - 0x2800UL) /* Page flush/invalidate */
  95. #define STRBUF_FSYNC (0x2810UL - 0x2800UL) /* Flush synchronization */
  96. #define STRBUF_DRAMDIAG (0x5000UL - 0x2800UL) /* data RAM diagnostic */
  97. #define STRBUF_ERRDIAG (0x5400UL - 0x2800UL) /* error status diagnostics */
  98. #define STRBUF_PTAGDIAG (0x5800UL - 0x2800UL) /* Page tag diagnostics */
  99. #define STRBUF_LTAGDIAG (0x5900UL - 0x2800UL) /* Line tag diagnostics */
  100. #define STRBUF_TAG_VALID 0x02UL
  101. static void strbuf_flush(struct sbus_iommu *iommu, u32 base, unsigned long npages)
  102. {
  103. iommu->strbuf_flushflag = 0UL;
  104. while (npages--)
  105. upa_writeq(base + (npages << IO_PAGE_SHIFT),
  106. iommu->strbuf_regs + STRBUF_PFLUSH);
  107. /* Whoopee cushion! */
  108. upa_writeq(__pa(&iommu->strbuf_flushflag),
  109. iommu->strbuf_regs + STRBUF_FSYNC);
  110. upa_readq(iommu->sbus_control_reg);
  111. while (iommu->strbuf_flushflag == 0UL)
  112. membar("#LoadLoad");
  113. }
  114. static iopte_t *alloc_streaming_cluster(struct sbus_iommu *iommu, unsigned long npages)
  115. {
  116. iopte_t *iopte, *limit, *first, *cluster;
  117. unsigned long cnum, ent, nent, flush_point, found;
  118. cnum = 0;
  119. nent = 1;
  120. while ((1UL << cnum) < npages)
  121. cnum++;
  122. if(cnum >= NCLUSTERS) {
  123. nent = 1UL << (cnum - NCLUSTERS);
  124. cnum = NCLUSTERS - 1;
  125. }
  126. iopte = iommu->page_table + (cnum * CLUSTER_NPAGES);
  127. if (cnum == 0)
  128. limit = (iommu->page_table +
  129. iommu->lowest_consistent_map);
  130. else
  131. limit = (iopte + CLUSTER_NPAGES);
  132. iopte += ((ent = iommu->alloc_info[cnum].next) << cnum);
  133. flush_point = iommu->alloc_info[cnum].flush;
  134. first = iopte;
  135. cluster = NULL;
  136. found = 0;
  137. for (;;) {
  138. if (iopte_val(*iopte) == 0UL) {
  139. found++;
  140. if (!cluster)
  141. cluster = iopte;
  142. } else {
  143. /* Used cluster in the way */
  144. cluster = NULL;
  145. found = 0;
  146. }
  147. if (found == nent)
  148. break;
  149. iopte += (1 << cnum);
  150. ent++;
  151. if (iopte >= limit) {
  152. iopte = (iommu->page_table + (cnum * CLUSTER_NPAGES));
  153. ent = 0;
  154. /* Multiple cluster allocations must not wrap */
  155. cluster = NULL;
  156. found = 0;
  157. }
  158. if (ent == flush_point)
  159. __iommu_flushall(iommu);
  160. if (iopte == first)
  161. goto bad;
  162. }
  163. /* ent/iopte points to the last cluster entry we're going to use,
  164. * so save our place for the next allocation.
  165. */
  166. if ((iopte + (1 << cnum)) >= limit)
  167. ent = 0;
  168. else
  169. ent = ent + 1;
  170. iommu->alloc_info[cnum].next = ent;
  171. if (ent == flush_point)
  172. __iommu_flushall(iommu);
  173. /* I've got your streaming cluster right here buddy boy... */
  174. return cluster;
  175. bad:
  176. printk(KERN_EMERG "sbus: alloc_streaming_cluster of npages(%ld) failed!\n",
  177. npages);
  178. return NULL;
  179. }
  180. static void free_streaming_cluster(struct sbus_iommu *iommu, u32 base, unsigned long npages)
  181. {
  182. unsigned long cnum, ent, nent;
  183. iopte_t *iopte;
  184. cnum = 0;
  185. nent = 1;
  186. while ((1UL << cnum) < npages)
  187. cnum++;
  188. if(cnum >= NCLUSTERS) {
  189. nent = 1UL << (cnum - NCLUSTERS);
  190. cnum = NCLUSTERS - 1;
  191. }
  192. ent = (base & CLUSTER_MASK) >> (IO_PAGE_SHIFT + cnum);
  193. iopte = iommu->page_table + ((base - MAP_BASE) >> IO_PAGE_SHIFT);
  194. do {
  195. iopte_val(*iopte) = 0UL;
  196. iopte += 1 << cnum;
  197. } while(--nent);
  198. /* If the global flush might not have caught this entry,
  199. * adjust the flush point such that we will flush before
  200. * ever trying to reuse it.
  201. */
  202. #define between(X,Y,Z) (((Z) - (Y)) >= ((X) - (Y)))
  203. if (between(ent, iommu->alloc_info[cnum].next, iommu->alloc_info[cnum].flush))
  204. iommu->alloc_info[cnum].flush = ent;
  205. #undef between
  206. }
  207. /* We allocate consistent mappings from the end of cluster zero. */
  208. static iopte_t *alloc_consistent_cluster(struct sbus_iommu *iommu, unsigned long npages)
  209. {
  210. iopte_t *iopte;
  211. iopte = iommu->page_table + (1 * CLUSTER_NPAGES);
  212. while (iopte > iommu->page_table) {
  213. iopte--;
  214. if (!(iopte_val(*iopte) & IOPTE_VALID)) {
  215. unsigned long tmp = npages;
  216. while (--tmp) {
  217. iopte--;
  218. if (iopte_val(*iopte) & IOPTE_VALID)
  219. break;
  220. }
  221. if (tmp == 0) {
  222. u32 entry = (iopte - iommu->page_table);
  223. if (entry < iommu->lowest_consistent_map)
  224. iommu->lowest_consistent_map = entry;
  225. return iopte;
  226. }
  227. }
  228. }
  229. return NULL;
  230. }
  231. static void free_consistent_cluster(struct sbus_iommu *iommu, u32 base, unsigned long npages)
  232. {
  233. iopte_t *iopte = iommu->page_table + ((base - MAP_BASE) >> IO_PAGE_SHIFT);
  234. if ((iopte - iommu->page_table) == iommu->lowest_consistent_map) {
  235. iopte_t *walk = iopte + npages;
  236. iopte_t *limit;
  237. limit = iommu->page_table + CLUSTER_NPAGES;
  238. while (walk < limit) {
  239. if (iopte_val(*walk) != 0UL)
  240. break;
  241. walk++;
  242. }
  243. iommu->lowest_consistent_map =
  244. (walk - iommu->page_table);
  245. }
  246. while (npages--)
  247. *iopte++ = __iopte(0UL);
  248. }
  249. void *sbus_alloc_consistent(struct sbus_dev *sdev, size_t size, dma_addr_t *dvma_addr)
  250. {
  251. unsigned long order, first_page, flags;
  252. struct sbus_iommu *iommu;
  253. iopte_t *iopte;
  254. void *ret;
  255. int npages;
  256. if (size <= 0 || sdev == NULL || dvma_addr == NULL)
  257. return NULL;
  258. size = IO_PAGE_ALIGN(size);
  259. order = get_order(size);
  260. if (order >= 10)
  261. return NULL;
  262. first_page = __get_free_pages(GFP_KERNEL, order);
  263. if (first_page == 0UL)
  264. return NULL;
  265. memset((char *)first_page, 0, PAGE_SIZE << order);
  266. iommu = sdev->bus->iommu;
  267. spin_lock_irqsave(&iommu->lock, flags);
  268. iopte = alloc_consistent_cluster(iommu, size >> IO_PAGE_SHIFT);
  269. if (iopte == NULL) {
  270. spin_unlock_irqrestore(&iommu->lock, flags);
  271. free_pages(first_page, order);
  272. return NULL;
  273. }
  274. /* Ok, we're committed at this point. */
  275. *dvma_addr = MAP_BASE + ((iopte - iommu->page_table) << IO_PAGE_SHIFT);
  276. ret = (void *) first_page;
  277. npages = size >> IO_PAGE_SHIFT;
  278. while (npages--) {
  279. *iopte++ = __iopte(IOPTE_VALID | IOPTE_CACHE | IOPTE_WRITE |
  280. (__pa(first_page) & IOPTE_PAGE));
  281. first_page += IO_PAGE_SIZE;
  282. }
  283. iommu_flush(iommu, *dvma_addr, size >> IO_PAGE_SHIFT);
  284. spin_unlock_irqrestore(&iommu->lock, flags);
  285. return ret;
  286. }
  287. void sbus_free_consistent(struct sbus_dev *sdev, size_t size, void *cpu, dma_addr_t dvma)
  288. {
  289. unsigned long order, npages;
  290. struct sbus_iommu *iommu;
  291. if (size <= 0 || sdev == NULL || cpu == NULL)
  292. return;
  293. npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT;
  294. iommu = sdev->bus->iommu;
  295. spin_lock_irq(&iommu->lock);
  296. free_consistent_cluster(iommu, dvma, npages);
  297. iommu_flush(iommu, dvma, npages);
  298. spin_unlock_irq(&iommu->lock);
  299. order = get_order(size);
  300. if (order < 10)
  301. free_pages((unsigned long)cpu, order);
  302. }
  303. dma_addr_t sbus_map_single(struct sbus_dev *sdev, void *ptr, size_t size, int dir)
  304. {
  305. struct sbus_iommu *iommu = sdev->bus->iommu;
  306. unsigned long npages, pbase, flags;
  307. iopte_t *iopte;
  308. u32 dma_base, offset;
  309. unsigned long iopte_bits;
  310. if (dir == SBUS_DMA_NONE)
  311. BUG();
  312. pbase = (unsigned long) ptr;
  313. offset = (u32) (pbase & ~IO_PAGE_MASK);
  314. size = (IO_PAGE_ALIGN(pbase + size) - (pbase & IO_PAGE_MASK));
  315. pbase = (unsigned long) __pa(pbase & IO_PAGE_MASK);
  316. spin_lock_irqsave(&iommu->lock, flags);
  317. npages = size >> IO_PAGE_SHIFT;
  318. iopte = alloc_streaming_cluster(iommu, npages);
  319. if (iopte == NULL)
  320. goto bad;
  321. dma_base = MAP_BASE + ((iopte - iommu->page_table) << IO_PAGE_SHIFT);
  322. npages = size >> IO_PAGE_SHIFT;
  323. iopte_bits = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE;
  324. if (dir != SBUS_DMA_TODEVICE)
  325. iopte_bits |= IOPTE_WRITE;
  326. while (npages--) {
  327. *iopte++ = __iopte(iopte_bits | (pbase & IOPTE_PAGE));
  328. pbase += IO_PAGE_SIZE;
  329. }
  330. npages = size >> IO_PAGE_SHIFT;
  331. spin_unlock_irqrestore(&iommu->lock, flags);
  332. return (dma_base | offset);
  333. bad:
  334. spin_unlock_irqrestore(&iommu->lock, flags);
  335. BUG();
  336. return 0;
  337. }
  338. void sbus_unmap_single(struct sbus_dev *sdev, dma_addr_t dma_addr, size_t size, int direction)
  339. {
  340. struct sbus_iommu *iommu = sdev->bus->iommu;
  341. u32 dma_base = dma_addr & IO_PAGE_MASK;
  342. unsigned long flags;
  343. size = (IO_PAGE_ALIGN(dma_addr + size) - dma_base);
  344. spin_lock_irqsave(&iommu->lock, flags);
  345. free_streaming_cluster(iommu, dma_base, size >> IO_PAGE_SHIFT);
  346. strbuf_flush(iommu, dma_base, size >> IO_PAGE_SHIFT);
  347. spin_unlock_irqrestore(&iommu->lock, flags);
  348. }
  349. #define SG_ENT_PHYS_ADDRESS(SG) \
  350. (__pa(page_address((SG)->page)) + (SG)->offset)
  351. static inline void fill_sg(iopte_t *iopte, struct scatterlist *sg, int nused, int nelems, unsigned long iopte_bits)
  352. {
  353. struct scatterlist *dma_sg = sg;
  354. struct scatterlist *sg_end = sg + nelems;
  355. int i;
  356. for (i = 0; i < nused; i++) {
  357. unsigned long pteval = ~0UL;
  358. u32 dma_npages;
  359. dma_npages = ((dma_sg->dma_address & (IO_PAGE_SIZE - 1UL)) +
  360. dma_sg->dma_length +
  361. ((IO_PAGE_SIZE - 1UL))) >> IO_PAGE_SHIFT;
  362. do {
  363. unsigned long offset;
  364. signed int len;
  365. /* If we are here, we know we have at least one
  366. * more page to map. So walk forward until we
  367. * hit a page crossing, and begin creating new
  368. * mappings from that spot.
  369. */
  370. for (;;) {
  371. unsigned long tmp;
  372. tmp = (unsigned long) SG_ENT_PHYS_ADDRESS(sg);
  373. len = sg->length;
  374. if (((tmp ^ pteval) >> IO_PAGE_SHIFT) != 0UL) {
  375. pteval = tmp & IO_PAGE_MASK;
  376. offset = tmp & (IO_PAGE_SIZE - 1UL);
  377. break;
  378. }
  379. if (((tmp ^ (tmp + len - 1UL)) >> IO_PAGE_SHIFT) != 0UL) {
  380. pteval = (tmp + IO_PAGE_SIZE) & IO_PAGE_MASK;
  381. offset = 0UL;
  382. len -= (IO_PAGE_SIZE - (tmp & (IO_PAGE_SIZE - 1UL)));
  383. break;
  384. }
  385. sg++;
  386. }
  387. pteval = ((pteval & IOPTE_PAGE) | iopte_bits);
  388. while (len > 0) {
  389. *iopte++ = __iopte(pteval);
  390. pteval += IO_PAGE_SIZE;
  391. len -= (IO_PAGE_SIZE - offset);
  392. offset = 0;
  393. dma_npages--;
  394. }
  395. pteval = (pteval & IOPTE_PAGE) + len;
  396. sg++;
  397. /* Skip over any tail mappings we've fully mapped,
  398. * adjusting pteval along the way. Stop when we
  399. * detect a page crossing event.
  400. */
  401. while (sg < sg_end &&
  402. (pteval << (64 - IO_PAGE_SHIFT)) != 0UL &&
  403. (pteval == SG_ENT_PHYS_ADDRESS(sg)) &&
  404. ((pteval ^
  405. (SG_ENT_PHYS_ADDRESS(sg) + sg->length - 1UL)) >> IO_PAGE_SHIFT) == 0UL) {
  406. pteval += sg->length;
  407. sg++;
  408. }
  409. if ((pteval << (64 - IO_PAGE_SHIFT)) == 0UL)
  410. pteval = ~0UL;
  411. } while (dma_npages != 0);
  412. dma_sg++;
  413. }
  414. }
  415. int sbus_map_sg(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int dir)
  416. {
  417. struct sbus_iommu *iommu = sdev->bus->iommu;
  418. unsigned long flags, npages;
  419. iopte_t *iopte;
  420. u32 dma_base;
  421. struct scatterlist *sgtmp;
  422. int used;
  423. unsigned long iopte_bits;
  424. if (dir == SBUS_DMA_NONE)
  425. BUG();
  426. /* Fast path single entry scatterlists. */
  427. if (nents == 1) {
  428. sg->dma_address =
  429. sbus_map_single(sdev,
  430. (page_address(sg->page) + sg->offset),
  431. sg->length, dir);
  432. sg->dma_length = sg->length;
  433. return 1;
  434. }
  435. npages = prepare_sg(sg, nents);
  436. spin_lock_irqsave(&iommu->lock, flags);
  437. iopte = alloc_streaming_cluster(iommu, npages);
  438. if (iopte == NULL)
  439. goto bad;
  440. dma_base = MAP_BASE + ((iopte - iommu->page_table) << IO_PAGE_SHIFT);
  441. /* Normalize DVMA addresses. */
  442. sgtmp = sg;
  443. used = nents;
  444. while (used && sgtmp->dma_length) {
  445. sgtmp->dma_address += dma_base;
  446. sgtmp++;
  447. used--;
  448. }
  449. used = nents - used;
  450. iopte_bits = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE;
  451. if (dir != SBUS_DMA_TODEVICE)
  452. iopte_bits |= IOPTE_WRITE;
  453. fill_sg(iopte, sg, used, nents, iopte_bits);
  454. #ifdef VERIFY_SG
  455. verify_sglist(sg, nents, iopte, npages);
  456. #endif
  457. spin_unlock_irqrestore(&iommu->lock, flags);
  458. return used;
  459. bad:
  460. spin_unlock_irqrestore(&iommu->lock, flags);
  461. BUG();
  462. return 0;
  463. }
  464. void sbus_unmap_sg(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int direction)
  465. {
  466. unsigned long size, flags;
  467. struct sbus_iommu *iommu;
  468. u32 dvma_base;
  469. int i;
  470. /* Fast path single entry scatterlists. */
  471. if (nents == 1) {
  472. sbus_unmap_single(sdev, sg->dma_address, sg->dma_length, direction);
  473. return;
  474. }
  475. dvma_base = sg[0].dma_address & IO_PAGE_MASK;
  476. for (i = 0; i < nents; i++) {
  477. if (sg[i].dma_length == 0)
  478. break;
  479. }
  480. i--;
  481. size = IO_PAGE_ALIGN(sg[i].dma_address + sg[i].dma_length) - dvma_base;
  482. iommu = sdev->bus->iommu;
  483. spin_lock_irqsave(&iommu->lock, flags);
  484. free_streaming_cluster(iommu, dvma_base, size >> IO_PAGE_SHIFT);
  485. strbuf_flush(iommu, dvma_base, size >> IO_PAGE_SHIFT);
  486. spin_unlock_irqrestore(&iommu->lock, flags);
  487. }
  488. void sbus_dma_sync_single_for_cpu(struct sbus_dev *sdev, dma_addr_t base, size_t size, int direction)
  489. {
  490. struct sbus_iommu *iommu = sdev->bus->iommu;
  491. unsigned long flags;
  492. size = (IO_PAGE_ALIGN(base + size) - (base & IO_PAGE_MASK));
  493. spin_lock_irqsave(&iommu->lock, flags);
  494. strbuf_flush(iommu, base & IO_PAGE_MASK, size >> IO_PAGE_SHIFT);
  495. spin_unlock_irqrestore(&iommu->lock, flags);
  496. }
  497. void sbus_dma_sync_single_for_device(struct sbus_dev *sdev, dma_addr_t base, size_t size, int direction)
  498. {
  499. }
  500. void sbus_dma_sync_sg_for_cpu(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int direction)
  501. {
  502. struct sbus_iommu *iommu = sdev->bus->iommu;
  503. unsigned long flags, size;
  504. u32 base;
  505. int i;
  506. base = sg[0].dma_address & IO_PAGE_MASK;
  507. for (i = 0; i < nents; i++) {
  508. if (sg[i].dma_length == 0)
  509. break;
  510. }
  511. i--;
  512. size = IO_PAGE_ALIGN(sg[i].dma_address + sg[i].dma_length) - base;
  513. spin_lock_irqsave(&iommu->lock, flags);
  514. strbuf_flush(iommu, base, size >> IO_PAGE_SHIFT);
  515. spin_unlock_irqrestore(&iommu->lock, flags);
  516. }
  517. void sbus_dma_sync_sg_for_device(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int direction)
  518. {
  519. }
  520. /* Enable 64-bit DVMA mode for the given device. */
  521. void sbus_set_sbus64(struct sbus_dev *sdev, int bursts)
  522. {
  523. struct sbus_iommu *iommu = sdev->bus->iommu;
  524. int slot = sdev->slot;
  525. unsigned long cfg_reg;
  526. u64 val;
  527. cfg_reg = iommu->sbus_control_reg;
  528. switch (slot) {
  529. case 0:
  530. cfg_reg += 0x20UL;
  531. break;
  532. case 1:
  533. cfg_reg += 0x28UL;
  534. break;
  535. case 2:
  536. cfg_reg += 0x30UL;
  537. break;
  538. case 3:
  539. cfg_reg += 0x38UL;
  540. break;
  541. case 13:
  542. cfg_reg += 0x40UL;
  543. break;
  544. case 14:
  545. cfg_reg += 0x48UL;
  546. break;
  547. case 15:
  548. cfg_reg += 0x50UL;
  549. break;
  550. default:
  551. return;
  552. };
  553. val = upa_readq(cfg_reg);
  554. if (val & (1UL << 14UL)) {
  555. /* Extended transfer mode already enabled. */
  556. return;
  557. }
  558. val |= (1UL << 14UL);
  559. if (bursts & DMA_BURST8)
  560. val |= (1UL << 1UL);
  561. if (bursts & DMA_BURST16)
  562. val |= (1UL << 2UL);
  563. if (bursts & DMA_BURST32)
  564. val |= (1UL << 3UL);
  565. if (bursts & DMA_BURST64)
  566. val |= (1UL << 4UL);
  567. upa_writeq(val, cfg_reg);
  568. }
  569. /* SBUS SYSIO INO number to Sparc PIL level. */
  570. static unsigned char sysio_ino_to_pil[] = {
  571. 0, 4, 4, 7, 5, 7, 8, 9, /* SBUS slot 0 */
  572. 0, 4, 4, 7, 5, 7, 8, 9, /* SBUS slot 1 */
  573. 0, 4, 4, 7, 5, 7, 8, 9, /* SBUS slot 2 */
  574. 0, 4, 4, 7, 5, 7, 8, 9, /* SBUS slot 3 */
  575. 4, /* Onboard SCSI */
  576. 5, /* Onboard Ethernet */
  577. /*XXX*/ 8, /* Onboard BPP */
  578. 0, /* Bogon */
  579. 13, /* Audio */
  580. /*XXX*/15, /* PowerFail */
  581. 0, /* Bogon */
  582. 0, /* Bogon */
  583. 12, /* Zilog Serial Channels (incl. Keyboard/Mouse lines) */
  584. 11, /* Floppy */
  585. 0, /* Spare Hardware (bogon for now) */
  586. 0, /* Keyboard (bogon for now) */
  587. 0, /* Mouse (bogon for now) */
  588. 0, /* Serial (bogon for now) */
  589. 0, 0, /* Bogon, Bogon */
  590. 10, /* Timer 0 */
  591. 11, /* Timer 1 */
  592. 0, 0, /* Bogon, Bogon */
  593. 15, /* Uncorrectable SBUS Error */
  594. 15, /* Correctable SBUS Error */
  595. 15, /* SBUS Error */
  596. /*XXX*/ 0, /* Power Management (bogon for now) */
  597. };
  598. /* INO number to IMAP register offset for SYSIO external IRQ's.
  599. * This should conform to both Sunfire/Wildfire server and Fusion
  600. * desktop designs.
  601. */
  602. #define SYSIO_IMAP_SLOT0 0x2c04UL
  603. #define SYSIO_IMAP_SLOT1 0x2c0cUL
  604. #define SYSIO_IMAP_SLOT2 0x2c14UL
  605. #define SYSIO_IMAP_SLOT3 0x2c1cUL
  606. #define SYSIO_IMAP_SCSI 0x3004UL
  607. #define SYSIO_IMAP_ETH 0x300cUL
  608. #define SYSIO_IMAP_BPP 0x3014UL
  609. #define SYSIO_IMAP_AUDIO 0x301cUL
  610. #define SYSIO_IMAP_PFAIL 0x3024UL
  611. #define SYSIO_IMAP_KMS 0x302cUL
  612. #define SYSIO_IMAP_FLPY 0x3034UL
  613. #define SYSIO_IMAP_SHW 0x303cUL
  614. #define SYSIO_IMAP_KBD 0x3044UL
  615. #define SYSIO_IMAP_MS 0x304cUL
  616. #define SYSIO_IMAP_SER 0x3054UL
  617. #define SYSIO_IMAP_TIM0 0x3064UL
  618. #define SYSIO_IMAP_TIM1 0x306cUL
  619. #define SYSIO_IMAP_UE 0x3074UL
  620. #define SYSIO_IMAP_CE 0x307cUL
  621. #define SYSIO_IMAP_SBERR 0x3084UL
  622. #define SYSIO_IMAP_PMGMT 0x308cUL
  623. #define SYSIO_IMAP_GFX 0x3094UL
  624. #define SYSIO_IMAP_EUPA 0x309cUL
  625. #define bogon ((unsigned long) -1)
  626. static unsigned long sysio_irq_offsets[] = {
  627. /* SBUS Slot 0 --> 3, level 1 --> 7 */
  628. SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0,
  629. SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0,
  630. SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1,
  631. SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1,
  632. SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2,
  633. SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2,
  634. SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3,
  635. SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3,
  636. /* Onboard devices (not relevant/used on SunFire). */
  637. SYSIO_IMAP_SCSI,
  638. SYSIO_IMAP_ETH,
  639. SYSIO_IMAP_BPP,
  640. bogon,
  641. SYSIO_IMAP_AUDIO,
  642. SYSIO_IMAP_PFAIL,
  643. bogon,
  644. bogon,
  645. SYSIO_IMAP_KMS,
  646. SYSIO_IMAP_FLPY,
  647. SYSIO_IMAP_SHW,
  648. SYSIO_IMAP_KBD,
  649. SYSIO_IMAP_MS,
  650. SYSIO_IMAP_SER,
  651. bogon,
  652. bogon,
  653. SYSIO_IMAP_TIM0,
  654. SYSIO_IMAP_TIM1,
  655. bogon,
  656. bogon,
  657. SYSIO_IMAP_UE,
  658. SYSIO_IMAP_CE,
  659. SYSIO_IMAP_SBERR,
  660. SYSIO_IMAP_PMGMT,
  661. };
  662. #undef bogon
  663. #define NUM_SYSIO_OFFSETS (sizeof(sysio_irq_offsets) / sizeof(sysio_irq_offsets[0]))
  664. /* Convert Interrupt Mapping register pointer to associated
  665. * Interrupt Clear register pointer, SYSIO specific version.
  666. */
  667. #define SYSIO_ICLR_UNUSED0 0x3400UL
  668. #define SYSIO_ICLR_SLOT0 0x340cUL
  669. #define SYSIO_ICLR_SLOT1 0x344cUL
  670. #define SYSIO_ICLR_SLOT2 0x348cUL
  671. #define SYSIO_ICLR_SLOT3 0x34ccUL
  672. static unsigned long sysio_imap_to_iclr(unsigned long imap)
  673. {
  674. unsigned long diff = SYSIO_ICLR_UNUSED0 - SYSIO_IMAP_SLOT0;
  675. return imap + diff;
  676. }
  677. unsigned int sbus_build_irq(void *buscookie, unsigned int ino)
  678. {
  679. struct sbus_bus *sbus = (struct sbus_bus *)buscookie;
  680. struct sbus_iommu *iommu = sbus->iommu;
  681. unsigned long reg_base = iommu->sbus_control_reg - 0x2000UL;
  682. unsigned long imap, iclr;
  683. int pil, sbus_level = 0;
  684. pil = sysio_ino_to_pil[ino];
  685. if (!pil) {
  686. printk("sbus_irq_build: Bad SYSIO INO[%x]\n", ino);
  687. panic("Bad SYSIO IRQ translations...");
  688. }
  689. if (PIL_RESERVED(pil))
  690. BUG();
  691. imap = sysio_irq_offsets[ino];
  692. if (imap == ((unsigned long)-1)) {
  693. prom_printf("get_irq_translations: Bad SYSIO INO[%x] cpu[%d]\n",
  694. ino, pil);
  695. prom_halt();
  696. }
  697. imap += reg_base;
  698. /* SYSIO inconsistency. For external SLOTS, we have to select
  699. * the right ICLR register based upon the lower SBUS irq level
  700. * bits.
  701. */
  702. if (ino >= 0x20) {
  703. iclr = sysio_imap_to_iclr(imap);
  704. } else {
  705. int sbus_slot = (ino & 0x18)>>3;
  706. sbus_level = ino & 0x7;
  707. switch(sbus_slot) {
  708. case 0:
  709. iclr = reg_base + SYSIO_ICLR_SLOT0;
  710. break;
  711. case 1:
  712. iclr = reg_base + SYSIO_ICLR_SLOT1;
  713. break;
  714. case 2:
  715. iclr = reg_base + SYSIO_ICLR_SLOT2;
  716. break;
  717. default:
  718. case 3:
  719. iclr = reg_base + SYSIO_ICLR_SLOT3;
  720. break;
  721. };
  722. iclr += ((unsigned long)sbus_level - 1UL) * 8UL;
  723. }
  724. return build_irq(pil, sbus_level, iclr, imap);
  725. }
  726. /* Error interrupt handling. */
  727. #define SYSIO_UE_AFSR 0x0030UL
  728. #define SYSIO_UE_AFAR 0x0038UL
  729. #define SYSIO_UEAFSR_PPIO 0x8000000000000000UL /* Primary PIO cause */
  730. #define SYSIO_UEAFSR_PDRD 0x4000000000000000UL /* Primary DVMA read cause */
  731. #define SYSIO_UEAFSR_PDWR 0x2000000000000000UL /* Primary DVMA write cause */
  732. #define SYSIO_UEAFSR_SPIO 0x1000000000000000UL /* Secondary PIO is cause */
  733. #define SYSIO_UEAFSR_SDRD 0x0800000000000000UL /* Secondary DVMA read cause */
  734. #define SYSIO_UEAFSR_SDWR 0x0400000000000000UL /* Secondary DVMA write cause*/
  735. #define SYSIO_UEAFSR_RESV1 0x03ff000000000000UL /* Reserved */
  736. #define SYSIO_UEAFSR_DOFF 0x0000e00000000000UL /* Doubleword Offset */
  737. #define SYSIO_UEAFSR_SIZE 0x00001c0000000000UL /* Bad transfer size 2^SIZE */
  738. #define SYSIO_UEAFSR_MID 0x000003e000000000UL /* UPA MID causing the fault */
  739. #define SYSIO_UEAFSR_RESV2 0x0000001fffffffffUL /* Reserved */
  740. static irqreturn_t sysio_ue_handler(int irq, void *dev_id, struct pt_regs *regs)
  741. {
  742. struct sbus_bus *sbus = dev_id;
  743. struct sbus_iommu *iommu = sbus->iommu;
  744. unsigned long reg_base = iommu->sbus_control_reg - 0x2000UL;
  745. unsigned long afsr_reg, afar_reg;
  746. unsigned long afsr, afar, error_bits;
  747. int reported;
  748. afsr_reg = reg_base + SYSIO_UE_AFSR;
  749. afar_reg = reg_base + SYSIO_UE_AFAR;
  750. /* Latch error status. */
  751. afsr = upa_readq(afsr_reg);
  752. afar = upa_readq(afar_reg);
  753. /* Clear primary/secondary error status bits. */
  754. error_bits = afsr &
  755. (SYSIO_UEAFSR_PPIO | SYSIO_UEAFSR_PDRD | SYSIO_UEAFSR_PDWR |
  756. SYSIO_UEAFSR_SPIO | SYSIO_UEAFSR_SDRD | SYSIO_UEAFSR_SDWR);
  757. upa_writeq(error_bits, afsr_reg);
  758. /* Log the error. */
  759. printk("SYSIO[%x]: Uncorrectable ECC Error, primary error type[%s]\n",
  760. sbus->portid,
  761. (((error_bits & SYSIO_UEAFSR_PPIO) ?
  762. "PIO" :
  763. ((error_bits & SYSIO_UEAFSR_PDRD) ?
  764. "DVMA Read" :
  765. ((error_bits & SYSIO_UEAFSR_PDWR) ?
  766. "DVMA Write" : "???")))));
  767. printk("SYSIO[%x]: DOFF[%lx] SIZE[%lx] MID[%lx]\n",
  768. sbus->portid,
  769. (afsr & SYSIO_UEAFSR_DOFF) >> 45UL,
  770. (afsr & SYSIO_UEAFSR_SIZE) >> 42UL,
  771. (afsr & SYSIO_UEAFSR_MID) >> 37UL);
  772. printk("SYSIO[%x]: AFAR[%016lx]\n", sbus->portid, afar);
  773. printk("SYSIO[%x]: Secondary UE errors [", sbus->portid);
  774. reported = 0;
  775. if (afsr & SYSIO_UEAFSR_SPIO) {
  776. reported++;
  777. printk("(PIO)");
  778. }
  779. if (afsr & SYSIO_UEAFSR_SDRD) {
  780. reported++;
  781. printk("(DVMA Read)");
  782. }
  783. if (afsr & SYSIO_UEAFSR_SDWR) {
  784. reported++;
  785. printk("(DVMA Write)");
  786. }
  787. if (!reported)
  788. printk("(none)");
  789. printk("]\n");
  790. return IRQ_HANDLED;
  791. }
  792. #define SYSIO_CE_AFSR 0x0040UL
  793. #define SYSIO_CE_AFAR 0x0048UL
  794. #define SYSIO_CEAFSR_PPIO 0x8000000000000000UL /* Primary PIO cause */
  795. #define SYSIO_CEAFSR_PDRD 0x4000000000000000UL /* Primary DVMA read cause */
  796. #define SYSIO_CEAFSR_PDWR 0x2000000000000000UL /* Primary DVMA write cause */
  797. #define SYSIO_CEAFSR_SPIO 0x1000000000000000UL /* Secondary PIO cause */
  798. #define SYSIO_CEAFSR_SDRD 0x0800000000000000UL /* Secondary DVMA read cause */
  799. #define SYSIO_CEAFSR_SDWR 0x0400000000000000UL /* Secondary DVMA write cause*/
  800. #define SYSIO_CEAFSR_RESV1 0x0300000000000000UL /* Reserved */
  801. #define SYSIO_CEAFSR_ESYND 0x00ff000000000000UL /* Syndrome Bits */
  802. #define SYSIO_CEAFSR_DOFF 0x0000e00000000000UL /* Double Offset */
  803. #define SYSIO_CEAFSR_SIZE 0x00001c0000000000UL /* Bad transfer size 2^SIZE */
  804. #define SYSIO_CEAFSR_MID 0x000003e000000000UL /* UPA MID causing the fault */
  805. #define SYSIO_CEAFSR_RESV2 0x0000001fffffffffUL /* Reserved */
  806. static irqreturn_t sysio_ce_handler(int irq, void *dev_id, struct pt_regs *regs)
  807. {
  808. struct sbus_bus *sbus = dev_id;
  809. struct sbus_iommu *iommu = sbus->iommu;
  810. unsigned long reg_base = iommu->sbus_control_reg - 0x2000UL;
  811. unsigned long afsr_reg, afar_reg;
  812. unsigned long afsr, afar, error_bits;
  813. int reported;
  814. afsr_reg = reg_base + SYSIO_CE_AFSR;
  815. afar_reg = reg_base + SYSIO_CE_AFAR;
  816. /* Latch error status. */
  817. afsr = upa_readq(afsr_reg);
  818. afar = upa_readq(afar_reg);
  819. /* Clear primary/secondary error status bits. */
  820. error_bits = afsr &
  821. (SYSIO_CEAFSR_PPIO | SYSIO_CEAFSR_PDRD | SYSIO_CEAFSR_PDWR |
  822. SYSIO_CEAFSR_SPIO | SYSIO_CEAFSR_SDRD | SYSIO_CEAFSR_SDWR);
  823. upa_writeq(error_bits, afsr_reg);
  824. printk("SYSIO[%x]: Correctable ECC Error, primary error type[%s]\n",
  825. sbus->portid,
  826. (((error_bits & SYSIO_CEAFSR_PPIO) ?
  827. "PIO" :
  828. ((error_bits & SYSIO_CEAFSR_PDRD) ?
  829. "DVMA Read" :
  830. ((error_bits & SYSIO_CEAFSR_PDWR) ?
  831. "DVMA Write" : "???")))));
  832. /* XXX Use syndrome and afar to print out module string just like
  833. * XXX UDB CE trap handler does... -DaveM
  834. */
  835. printk("SYSIO[%x]: DOFF[%lx] ECC Syndrome[%lx] Size[%lx] MID[%lx]\n",
  836. sbus->portid,
  837. (afsr & SYSIO_CEAFSR_DOFF) >> 45UL,
  838. (afsr & SYSIO_CEAFSR_ESYND) >> 48UL,
  839. (afsr & SYSIO_CEAFSR_SIZE) >> 42UL,
  840. (afsr & SYSIO_CEAFSR_MID) >> 37UL);
  841. printk("SYSIO[%x]: AFAR[%016lx]\n", sbus->portid, afar);
  842. printk("SYSIO[%x]: Secondary CE errors [", sbus->portid);
  843. reported = 0;
  844. if (afsr & SYSIO_CEAFSR_SPIO) {
  845. reported++;
  846. printk("(PIO)");
  847. }
  848. if (afsr & SYSIO_CEAFSR_SDRD) {
  849. reported++;
  850. printk("(DVMA Read)");
  851. }
  852. if (afsr & SYSIO_CEAFSR_SDWR) {
  853. reported++;
  854. printk("(DVMA Write)");
  855. }
  856. if (!reported)
  857. printk("(none)");
  858. printk("]\n");
  859. return IRQ_HANDLED;
  860. }
  861. #define SYSIO_SBUS_AFSR 0x2010UL
  862. #define SYSIO_SBUS_AFAR 0x2018UL
  863. #define SYSIO_SBAFSR_PLE 0x8000000000000000UL /* Primary Late PIO Error */
  864. #define SYSIO_SBAFSR_PTO 0x4000000000000000UL /* Primary SBUS Timeout */
  865. #define SYSIO_SBAFSR_PBERR 0x2000000000000000UL /* Primary SBUS Error ACK */
  866. #define SYSIO_SBAFSR_SLE 0x1000000000000000UL /* Secondary Late PIO Error */
  867. #define SYSIO_SBAFSR_STO 0x0800000000000000UL /* Secondary SBUS Timeout */
  868. #define SYSIO_SBAFSR_SBERR 0x0400000000000000UL /* Secondary SBUS Error ACK */
  869. #define SYSIO_SBAFSR_RESV1 0x03ff000000000000UL /* Reserved */
  870. #define SYSIO_SBAFSR_RD 0x0000800000000000UL /* Primary was late PIO read */
  871. #define SYSIO_SBAFSR_RESV2 0x0000600000000000UL /* Reserved */
  872. #define SYSIO_SBAFSR_SIZE 0x00001c0000000000UL /* Size of transfer */
  873. #define SYSIO_SBAFSR_MID 0x000003e000000000UL /* MID causing the error */
  874. #define SYSIO_SBAFSR_RESV3 0x0000001fffffffffUL /* Reserved */
  875. static irqreturn_t sysio_sbus_error_handler(int irq, void *dev_id, struct pt_regs *regs)
  876. {
  877. struct sbus_bus *sbus = dev_id;
  878. struct sbus_iommu *iommu = sbus->iommu;
  879. unsigned long afsr_reg, afar_reg, reg_base;
  880. unsigned long afsr, afar, error_bits;
  881. int reported;
  882. reg_base = iommu->sbus_control_reg - 0x2000UL;
  883. afsr_reg = reg_base + SYSIO_SBUS_AFSR;
  884. afar_reg = reg_base + SYSIO_SBUS_AFAR;
  885. afsr = upa_readq(afsr_reg);
  886. afar = upa_readq(afar_reg);
  887. /* Clear primary/secondary error status bits. */
  888. error_bits = afsr &
  889. (SYSIO_SBAFSR_PLE | SYSIO_SBAFSR_PTO | SYSIO_SBAFSR_PBERR |
  890. SYSIO_SBAFSR_SLE | SYSIO_SBAFSR_STO | SYSIO_SBAFSR_SBERR);
  891. upa_writeq(error_bits, afsr_reg);
  892. /* Log the error. */
  893. printk("SYSIO[%x]: SBUS Error, primary error type[%s] read(%d)\n",
  894. sbus->portid,
  895. (((error_bits & SYSIO_SBAFSR_PLE) ?
  896. "Late PIO Error" :
  897. ((error_bits & SYSIO_SBAFSR_PTO) ?
  898. "Time Out" :
  899. ((error_bits & SYSIO_SBAFSR_PBERR) ?
  900. "Error Ack" : "???")))),
  901. (afsr & SYSIO_SBAFSR_RD) ? 1 : 0);
  902. printk("SYSIO[%x]: size[%lx] MID[%lx]\n",
  903. sbus->portid,
  904. (afsr & SYSIO_SBAFSR_SIZE) >> 42UL,
  905. (afsr & SYSIO_SBAFSR_MID) >> 37UL);
  906. printk("SYSIO[%x]: AFAR[%016lx]\n", sbus->portid, afar);
  907. printk("SYSIO[%x]: Secondary SBUS errors [", sbus->portid);
  908. reported = 0;
  909. if (afsr & SYSIO_SBAFSR_SLE) {
  910. reported++;
  911. printk("(Late PIO Error)");
  912. }
  913. if (afsr & SYSIO_SBAFSR_STO) {
  914. reported++;
  915. printk("(Time Out)");
  916. }
  917. if (afsr & SYSIO_SBAFSR_SBERR) {
  918. reported++;
  919. printk("(Error Ack)");
  920. }
  921. if (!reported)
  922. printk("(none)");
  923. printk("]\n");
  924. /* XXX check iommu/strbuf for further error status XXX */
  925. return IRQ_HANDLED;
  926. }
  927. #define ECC_CONTROL 0x0020UL
  928. #define SYSIO_ECNTRL_ECCEN 0x8000000000000000UL /* Enable ECC Checking */
  929. #define SYSIO_ECNTRL_UEEN 0x4000000000000000UL /* Enable UE Interrupts */
  930. #define SYSIO_ECNTRL_CEEN 0x2000000000000000UL /* Enable CE Interrupts */
  931. #define SYSIO_UE_INO 0x34
  932. #define SYSIO_CE_INO 0x35
  933. #define SYSIO_SBUSERR_INO 0x36
  934. static void __init sysio_register_error_handlers(struct sbus_bus *sbus)
  935. {
  936. struct sbus_iommu *iommu = sbus->iommu;
  937. unsigned long reg_base = iommu->sbus_control_reg - 0x2000UL;
  938. unsigned int irq;
  939. u64 control;
  940. irq = sbus_build_irq(sbus, SYSIO_UE_INO);
  941. if (request_irq(irq, sysio_ue_handler,
  942. SA_SHIRQ, "SYSIO UE", sbus) < 0) {
  943. prom_printf("SYSIO[%x]: Cannot register UE interrupt.\n",
  944. sbus->portid);
  945. prom_halt();
  946. }
  947. irq = sbus_build_irq(sbus, SYSIO_CE_INO);
  948. if (request_irq(irq, sysio_ce_handler,
  949. SA_SHIRQ, "SYSIO CE", sbus) < 0) {
  950. prom_printf("SYSIO[%x]: Cannot register CE interrupt.\n",
  951. sbus->portid);
  952. prom_halt();
  953. }
  954. irq = sbus_build_irq(sbus, SYSIO_SBUSERR_INO);
  955. if (request_irq(irq, sysio_sbus_error_handler,
  956. SA_SHIRQ, "SYSIO SBUS Error", sbus) < 0) {
  957. prom_printf("SYSIO[%x]: Cannot register SBUS Error interrupt.\n",
  958. sbus->portid);
  959. prom_halt();
  960. }
  961. /* Now turn the error interrupts on and also enable ECC checking. */
  962. upa_writeq((SYSIO_ECNTRL_ECCEN |
  963. SYSIO_ECNTRL_UEEN |
  964. SYSIO_ECNTRL_CEEN),
  965. reg_base + ECC_CONTROL);
  966. control = upa_readq(iommu->sbus_control_reg);
  967. control |= 0x100UL; /* SBUS Error Interrupt Enable */
  968. upa_writeq(control, iommu->sbus_control_reg);
  969. }
  970. /* Boot time initialization. */
  971. void __init sbus_iommu_init(int prom_node, struct sbus_bus *sbus)
  972. {
  973. struct linux_prom64_registers rprop;
  974. struct sbus_iommu *iommu;
  975. unsigned long regs, tsb_base;
  976. u64 control;
  977. int err, i;
  978. sbus->portid = prom_getintdefault(sbus->prom_node,
  979. "upa-portid", -1);
  980. err = prom_getproperty(prom_node, "reg",
  981. (char *)&rprop, sizeof(rprop));
  982. if (err < 0) {
  983. prom_printf("sbus_iommu_init: Cannot map SYSIO control registers.\n");
  984. prom_halt();
  985. }
  986. regs = rprop.phys_addr;
  987. iommu = kmalloc(sizeof(*iommu) + SMP_CACHE_BYTES, GFP_ATOMIC);
  988. if (iommu == NULL) {
  989. prom_printf("sbus_iommu_init: Fatal error, kmalloc(iommu) failed\n");
  990. prom_halt();
  991. }
  992. /* Align on E$ line boundary. */
  993. iommu = (struct sbus_iommu *)
  994. (((unsigned long)iommu + (SMP_CACHE_BYTES - 1UL)) &
  995. ~(SMP_CACHE_BYTES - 1UL));
  996. memset(iommu, 0, sizeof(*iommu));
  997. /* We start with no consistent mappings. */
  998. iommu->lowest_consistent_map = CLUSTER_NPAGES;
  999. for (i = 0; i < NCLUSTERS; i++) {
  1000. iommu->alloc_info[i].flush = 0;
  1001. iommu->alloc_info[i].next = 0;
  1002. }
  1003. /* Setup spinlock. */
  1004. spin_lock_init(&iommu->lock);
  1005. /* Init register offsets. */
  1006. iommu->iommu_regs = regs + SYSIO_IOMMUREG_BASE;
  1007. iommu->strbuf_regs = regs + SYSIO_STRBUFREG_BASE;
  1008. /* The SYSIO SBUS control register is used for dummy reads
  1009. * in order to ensure write completion.
  1010. */
  1011. iommu->sbus_control_reg = regs + 0x2000UL;
  1012. /* Link into SYSIO software state. */
  1013. sbus->iommu = iommu;
  1014. printk("SYSIO: UPA portID %x, at %016lx\n",
  1015. sbus->portid, regs);
  1016. /* Setup for TSB_SIZE=7, TBW_SIZE=0, MMU_DE=1, MMU_EN=1 */
  1017. control = upa_readq(iommu->iommu_regs + IOMMU_CONTROL);
  1018. control = ((7UL << 16UL) |
  1019. (0UL << 2UL) |
  1020. (1UL << 1UL) |
  1021. (1UL << 0UL));
  1022. /* Using the above configuration we need 1MB iommu page
  1023. * table (128K ioptes * 8 bytes per iopte). This is
  1024. * page order 7 on UltraSparc.
  1025. */
  1026. tsb_base = __get_free_pages(GFP_ATOMIC, get_order(IO_TSB_SIZE));
  1027. if (tsb_base == 0UL) {
  1028. prom_printf("sbus_iommu_init: Fatal error, cannot alloc TSB table.\n");
  1029. prom_halt();
  1030. }
  1031. iommu->page_table = (iopte_t *) tsb_base;
  1032. memset(iommu->page_table, 0, IO_TSB_SIZE);
  1033. upa_writeq(control, iommu->iommu_regs + IOMMU_CONTROL);
  1034. /* Clean out any cruft in the IOMMU using
  1035. * diagnostic accesses.
  1036. */
  1037. for (i = 0; i < 16; i++) {
  1038. unsigned long dram = iommu->iommu_regs + IOMMU_DRAMDIAG;
  1039. unsigned long tag = iommu->iommu_regs + IOMMU_TAGDIAG;
  1040. dram += (unsigned long)i * 8UL;
  1041. tag += (unsigned long)i * 8UL;
  1042. upa_writeq(0, dram);
  1043. upa_writeq(0, tag);
  1044. }
  1045. upa_readq(iommu->sbus_control_reg);
  1046. /* Give the TSB to SYSIO. */
  1047. upa_writeq(__pa(tsb_base), iommu->iommu_regs + IOMMU_TSBBASE);
  1048. /* Setup streaming buffer, DE=1 SB_EN=1 */
  1049. control = (1UL << 1UL) | (1UL << 0UL);
  1050. upa_writeq(control, iommu->strbuf_regs + STRBUF_CONTROL);
  1051. /* Clear out the tags using diagnostics. */
  1052. for (i = 0; i < 16; i++) {
  1053. unsigned long ptag, ltag;
  1054. ptag = iommu->strbuf_regs + STRBUF_PTAGDIAG;
  1055. ltag = iommu->strbuf_regs + STRBUF_LTAGDIAG;
  1056. ptag += (unsigned long)i * 8UL;
  1057. ltag += (unsigned long)i * 8UL;
  1058. upa_writeq(0UL, ptag);
  1059. upa_writeq(0UL, ltag);
  1060. }
  1061. /* Enable DVMA arbitration for all devices/slots. */
  1062. control = upa_readq(iommu->sbus_control_reg);
  1063. control |= 0x3fUL;
  1064. upa_writeq(control, iommu->sbus_control_reg);
  1065. /* Now some Xfire specific grot... */
  1066. if (this_is_starfire)
  1067. sbus->starfire_cookie = starfire_hookup(sbus->portid);
  1068. else
  1069. sbus->starfire_cookie = NULL;
  1070. sysio_register_error_handlers(sbus);
  1071. }