pci_iommu.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825
  1. /* $Id: pci_iommu.c,v 1.17 2001/12/17 07:05:09 davem Exp $
  2. * pci_iommu.c: UltraSparc PCI controller IOM/STC support.
  3. *
  4. * Copyright (C) 1999 David S. Miller (davem@redhat.com)
  5. * Copyright (C) 1999, 2000 Jakub Jelinek (jakub@redhat.com)
  6. */
  7. #include <linux/kernel.h>
  8. #include <linux/sched.h>
  9. #include <linux/mm.h>
  10. #include <linux/delay.h>
  11. #include <asm/pbm.h>
  12. #include "iommu_common.h"
  13. #define PCI_STC_CTXMATCH_ADDR(STC, CTX) \
  14. ((STC)->strbuf_ctxmatch_base + ((CTX) << 3))
  15. /* Accessing IOMMU and Streaming Buffer registers.
  16. * REG parameter is a physical address. All registers
  17. * are 64-bits in size.
  18. */
  19. #define pci_iommu_read(__reg) \
  20. ({ u64 __ret; \
  21. __asm__ __volatile__("ldxa [%1] %2, %0" \
  22. : "=r" (__ret) \
  23. : "r" (__reg), "i" (ASI_PHYS_BYPASS_EC_E) \
  24. : "memory"); \
  25. __ret; \
  26. })
  27. #define pci_iommu_write(__reg, __val) \
  28. __asm__ __volatile__("stxa %0, [%1] %2" \
  29. : /* no outputs */ \
  30. : "r" (__val), "r" (__reg), \
  31. "i" (ASI_PHYS_BYPASS_EC_E))
  32. /* Must be invoked under the IOMMU lock. */
  33. static void __iommu_flushall(struct pci_iommu *iommu)
  34. {
  35. unsigned long tag;
  36. int entry;
  37. tag = iommu->iommu_flush + (0xa580UL - 0x0210UL);
  38. for (entry = 0; entry < 16; entry++) {
  39. pci_iommu_write(tag, 0);
  40. tag += 8;
  41. }
  42. /* Ensure completion of previous PIO writes. */
  43. (void) pci_iommu_read(iommu->write_complete_reg);
  44. }
  45. #define IOPTE_CONSISTENT(CTX) \
  46. (IOPTE_VALID | IOPTE_CACHE | \
  47. (((CTX) << 47) & IOPTE_CONTEXT))
  48. #define IOPTE_STREAMING(CTX) \
  49. (IOPTE_CONSISTENT(CTX) | IOPTE_STBUF)
  50. /* Existing mappings are never marked invalid, instead they
  51. * are pointed to a dummy page.
  52. */
  53. #define IOPTE_IS_DUMMY(iommu, iopte) \
  54. ((iopte_val(*iopte) & IOPTE_PAGE) == (iommu)->dummy_page_pa)
  55. static void inline iopte_make_dummy(struct pci_iommu *iommu, iopte_t *iopte)
  56. {
  57. unsigned long val = iopte_val(*iopte);
  58. val &= ~IOPTE_PAGE;
  59. val |= iommu->dummy_page_pa;
  60. iopte_val(*iopte) = val;
  61. }
  62. /* Based largely upon the ppc64 iommu allocator. */
  63. static long pci_arena_alloc(struct pci_iommu *iommu, unsigned long npages)
  64. {
  65. struct pci_iommu_arena *arena = &iommu->arena;
  66. unsigned long n, i, start, end, limit;
  67. int pass;
  68. limit = arena->limit;
  69. start = arena->hint;
  70. pass = 0;
  71. again:
  72. n = find_next_zero_bit(arena->map, limit, start);
  73. end = n + npages;
  74. if (unlikely(end >= limit)) {
  75. if (likely(pass < 1)) {
  76. limit = start;
  77. start = 0;
  78. __iommu_flushall(iommu);
  79. pass++;
  80. goto again;
  81. } else {
  82. /* Scanned the whole thing, give up. */
  83. return -1;
  84. }
  85. }
  86. for (i = n; i < end; i++) {
  87. if (test_bit(i, arena->map)) {
  88. start = i + 1;
  89. goto again;
  90. }
  91. }
  92. for (i = n; i < end; i++)
  93. __set_bit(i, arena->map);
  94. arena->hint = end;
  95. return n;
  96. }
  97. static void pci_arena_free(struct pci_iommu_arena *arena, unsigned long base, unsigned long npages)
  98. {
  99. unsigned long i;
  100. for (i = base; i < (base + npages); i++)
  101. __clear_bit(i, arena->map);
  102. }
  103. void pci_iommu_table_init(struct pci_iommu *iommu, int tsbsize, u32 dma_offset, u32 dma_addr_mask)
  104. {
  105. unsigned long i, tsbbase, order, sz, num_tsb_entries;
  106. num_tsb_entries = tsbsize / sizeof(iopte_t);
  107. /* Setup initial software IOMMU state. */
  108. spin_lock_init(&iommu->lock);
  109. iommu->ctx_lowest_free = 1;
  110. iommu->page_table_map_base = dma_offset;
  111. iommu->dma_addr_mask = dma_addr_mask;
  112. /* Allocate and initialize the free area map. */
  113. sz = num_tsb_entries / 8;
  114. sz = (sz + 7UL) & ~7UL;
  115. iommu->arena.map = kmalloc(sz, GFP_KERNEL);
  116. if (!iommu->arena.map) {
  117. prom_printf("PCI_IOMMU: Error, kmalloc(arena.map) failed.\n");
  118. prom_halt();
  119. }
  120. memset(iommu->arena.map, 0, sz);
  121. iommu->arena.limit = num_tsb_entries;
  122. /* Allocate and initialize the dummy page which we
  123. * set inactive IO PTEs to point to.
  124. */
  125. iommu->dummy_page = __get_free_pages(GFP_KERNEL, 0);
  126. if (!iommu->dummy_page) {
  127. prom_printf("PCI_IOMMU: Error, gfp(dummy_page) failed.\n");
  128. prom_halt();
  129. }
  130. memset((void *)iommu->dummy_page, 0, PAGE_SIZE);
  131. iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page);
  132. /* Now allocate and setup the IOMMU page table itself. */
  133. order = get_order(tsbsize);
  134. tsbbase = __get_free_pages(GFP_KERNEL, order);
  135. if (!tsbbase) {
  136. prom_printf("PCI_IOMMU: Error, gfp(tsb) failed.\n");
  137. prom_halt();
  138. }
  139. iommu->page_table = (iopte_t *)tsbbase;
  140. for (i = 0; i < num_tsb_entries; i++)
  141. iopte_make_dummy(iommu, &iommu->page_table[i]);
  142. }
  143. static inline iopte_t *alloc_npages(struct pci_iommu *iommu, unsigned long npages)
  144. {
  145. long entry;
  146. entry = pci_arena_alloc(iommu, npages);
  147. if (unlikely(entry < 0))
  148. return NULL;
  149. return iommu->page_table + entry;
  150. }
  151. static inline void free_npages(struct pci_iommu *iommu, dma_addr_t base, unsigned long npages)
  152. {
  153. pci_arena_free(&iommu->arena, base >> IO_PAGE_SHIFT, npages);
  154. }
  155. static int iommu_alloc_ctx(struct pci_iommu *iommu)
  156. {
  157. int lowest = iommu->ctx_lowest_free;
  158. int sz = IOMMU_NUM_CTXS - lowest;
  159. int n = find_next_zero_bit(iommu->ctx_bitmap, sz, lowest);
  160. if (unlikely(n == sz)) {
  161. n = find_next_zero_bit(iommu->ctx_bitmap, lowest, 1);
  162. if (unlikely(n == lowest)) {
  163. printk(KERN_WARNING "IOMMU: Ran out of contexts.\n");
  164. n = 0;
  165. }
  166. }
  167. if (n)
  168. __set_bit(n, iommu->ctx_bitmap);
  169. return n;
  170. }
  171. static inline void iommu_free_ctx(struct pci_iommu *iommu, int ctx)
  172. {
  173. if (likely(ctx)) {
  174. __clear_bit(ctx, iommu->ctx_bitmap);
  175. if (ctx < iommu->ctx_lowest_free)
  176. iommu->ctx_lowest_free = ctx;
  177. }
  178. }
  179. /* Allocate and map kernel buffer of size SIZE using consistent mode
  180. * DMA for PCI device PDEV. Return non-NULL cpu-side address if
  181. * successful and set *DMA_ADDRP to the PCI side dma address.
  182. */
  183. void *pci_alloc_consistent(struct pci_dev *pdev, size_t size, dma_addr_t *dma_addrp)
  184. {
  185. struct pcidev_cookie *pcp;
  186. struct pci_iommu *iommu;
  187. iopte_t *iopte;
  188. unsigned long flags, order, first_page;
  189. void *ret;
  190. int npages;
  191. size = IO_PAGE_ALIGN(size);
  192. order = get_order(size);
  193. if (order >= 10)
  194. return NULL;
  195. first_page = __get_free_pages(GFP_ATOMIC, order);
  196. if (first_page == 0UL)
  197. return NULL;
  198. memset((char *)first_page, 0, PAGE_SIZE << order);
  199. pcp = pdev->sysdata;
  200. iommu = pcp->pbm->iommu;
  201. spin_lock_irqsave(&iommu->lock, flags);
  202. iopte = alloc_npages(iommu, size >> IO_PAGE_SHIFT);
  203. spin_unlock_irqrestore(&iommu->lock, flags);
  204. if (unlikely(iopte == NULL)) {
  205. free_pages(first_page, order);
  206. return NULL;
  207. }
  208. *dma_addrp = (iommu->page_table_map_base +
  209. ((iopte - iommu->page_table) << IO_PAGE_SHIFT));
  210. ret = (void *) first_page;
  211. npages = size >> IO_PAGE_SHIFT;
  212. first_page = __pa(first_page);
  213. while (npages--) {
  214. iopte_val(*iopte) = (IOPTE_CONSISTENT(0UL) |
  215. IOPTE_WRITE |
  216. (first_page & IOPTE_PAGE));
  217. iopte++;
  218. first_page += IO_PAGE_SIZE;
  219. }
  220. return ret;
  221. }
  222. /* Free and unmap a consistent DMA translation. */
  223. void pci_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma_addr_t dvma)
  224. {
  225. struct pcidev_cookie *pcp;
  226. struct pci_iommu *iommu;
  227. iopte_t *iopte;
  228. unsigned long flags, order, npages;
  229. npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT;
  230. pcp = pdev->sysdata;
  231. iommu = pcp->pbm->iommu;
  232. iopte = iommu->page_table +
  233. ((dvma - iommu->page_table_map_base) >> IO_PAGE_SHIFT);
  234. spin_lock_irqsave(&iommu->lock, flags);
  235. free_npages(iommu, dvma, npages);
  236. spin_unlock_irqrestore(&iommu->lock, flags);
  237. order = get_order(size);
  238. if (order < 10)
  239. free_pages((unsigned long)cpu, order);
  240. }
  241. /* Map a single buffer at PTR of SZ bytes for PCI DMA
  242. * in streaming mode.
  243. */
  244. dma_addr_t pci_map_single(struct pci_dev *pdev, void *ptr, size_t sz, int direction)
  245. {
  246. struct pcidev_cookie *pcp;
  247. struct pci_iommu *iommu;
  248. struct pci_strbuf *strbuf;
  249. iopte_t *base;
  250. unsigned long flags, npages, oaddr;
  251. unsigned long i, base_paddr, ctx;
  252. u32 bus_addr, ret;
  253. unsigned long iopte_protection;
  254. pcp = pdev->sysdata;
  255. iommu = pcp->pbm->iommu;
  256. strbuf = &pcp->pbm->stc;
  257. if (unlikely(direction == PCI_DMA_NONE))
  258. goto bad_no_ctx;
  259. oaddr = (unsigned long)ptr;
  260. npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK);
  261. npages >>= IO_PAGE_SHIFT;
  262. spin_lock_irqsave(&iommu->lock, flags);
  263. base = alloc_npages(iommu, npages);
  264. ctx = 0;
  265. if (iommu->iommu_ctxflush)
  266. ctx = iommu_alloc_ctx(iommu);
  267. spin_unlock_irqrestore(&iommu->lock, flags);
  268. if (unlikely(!base))
  269. goto bad;
  270. bus_addr = (iommu->page_table_map_base +
  271. ((base - iommu->page_table) << IO_PAGE_SHIFT));
  272. ret = bus_addr | (oaddr & ~IO_PAGE_MASK);
  273. base_paddr = __pa(oaddr & IO_PAGE_MASK);
  274. if (strbuf->strbuf_enabled)
  275. iopte_protection = IOPTE_STREAMING(ctx);
  276. else
  277. iopte_protection = IOPTE_CONSISTENT(ctx);
  278. if (direction != PCI_DMA_TODEVICE)
  279. iopte_protection |= IOPTE_WRITE;
  280. for (i = 0; i < npages; i++, base++, base_paddr += IO_PAGE_SIZE)
  281. iopte_val(*base) = iopte_protection | base_paddr;
  282. return ret;
  283. bad:
  284. iommu_free_ctx(iommu, ctx);
  285. bad_no_ctx:
  286. if (printk_ratelimit())
  287. WARN_ON(1);
  288. return PCI_DMA_ERROR_CODE;
  289. }
  290. static void pci_strbuf_flush(struct pci_strbuf *strbuf, struct pci_iommu *iommu, u32 vaddr, unsigned long ctx, unsigned long npages, int direction)
  291. {
  292. int limit;
  293. if (strbuf->strbuf_ctxflush &&
  294. iommu->iommu_ctxflush) {
  295. unsigned long matchreg, flushreg;
  296. u64 val;
  297. flushreg = strbuf->strbuf_ctxflush;
  298. matchreg = PCI_STC_CTXMATCH_ADDR(strbuf, ctx);
  299. pci_iommu_write(flushreg, ctx);
  300. val = pci_iommu_read(matchreg);
  301. val &= 0xffff;
  302. if (!val)
  303. goto do_flush_sync;
  304. while (val) {
  305. if (val & 0x1)
  306. pci_iommu_write(flushreg, ctx);
  307. val >>= 1;
  308. }
  309. val = pci_iommu_read(matchreg);
  310. if (unlikely(val)) {
  311. printk(KERN_WARNING "pci_strbuf_flush: ctx flush "
  312. "timeout matchreg[%lx] ctx[%lx]\n",
  313. val, ctx);
  314. goto do_page_flush;
  315. }
  316. } else {
  317. unsigned long i;
  318. do_page_flush:
  319. for (i = 0; i < npages; i++, vaddr += IO_PAGE_SIZE)
  320. pci_iommu_write(strbuf->strbuf_pflush, vaddr);
  321. }
  322. do_flush_sync:
  323. /* If the device could not have possibly put dirty data into
  324. * the streaming cache, no flush-flag synchronization needs
  325. * to be performed.
  326. */
  327. if (direction == PCI_DMA_TODEVICE)
  328. return;
  329. PCI_STC_FLUSHFLAG_INIT(strbuf);
  330. pci_iommu_write(strbuf->strbuf_fsync, strbuf->strbuf_flushflag_pa);
  331. (void) pci_iommu_read(iommu->write_complete_reg);
  332. limit = 100000;
  333. while (!PCI_STC_FLUSHFLAG_SET(strbuf)) {
  334. limit--;
  335. if (!limit)
  336. break;
  337. udelay(1);
  338. rmb();
  339. }
  340. if (!limit)
  341. printk(KERN_WARNING "pci_strbuf_flush: flushflag timeout "
  342. "vaddr[%08x] ctx[%lx] npages[%ld]\n",
  343. vaddr, ctx, npages);
  344. }
  345. /* Unmap a single streaming mode DMA translation. */
  346. void pci_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int direction)
  347. {
  348. struct pcidev_cookie *pcp;
  349. struct pci_iommu *iommu;
  350. struct pci_strbuf *strbuf;
  351. iopte_t *base;
  352. unsigned long flags, npages, ctx, i;
  353. if (unlikely(direction == PCI_DMA_NONE)) {
  354. if (printk_ratelimit())
  355. WARN_ON(1);
  356. return;
  357. }
  358. pcp = pdev->sysdata;
  359. iommu = pcp->pbm->iommu;
  360. strbuf = &pcp->pbm->stc;
  361. npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK);
  362. npages >>= IO_PAGE_SHIFT;
  363. base = iommu->page_table +
  364. ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT);
  365. #ifdef DEBUG_PCI_IOMMU
  366. if (IOPTE_IS_DUMMY(iommu, base))
  367. printk("pci_unmap_single called on non-mapped region %08x,%08x from %016lx\n",
  368. bus_addr, sz, __builtin_return_address(0));
  369. #endif
  370. bus_addr &= IO_PAGE_MASK;
  371. spin_lock_irqsave(&iommu->lock, flags);
  372. /* Record the context, if any. */
  373. ctx = 0;
  374. if (iommu->iommu_ctxflush)
  375. ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL;
  376. /* Step 1: Kick data out of streaming buffers if necessary. */
  377. if (strbuf->strbuf_enabled)
  378. pci_strbuf_flush(strbuf, iommu, bus_addr, ctx,
  379. npages, direction);
  380. /* Step 2: Clear out TSB entries. */
  381. for (i = 0; i < npages; i++)
  382. iopte_make_dummy(iommu, base + i);
  383. free_npages(iommu, bus_addr - iommu->page_table_map_base, npages);
  384. iommu_free_ctx(iommu, ctx);
  385. spin_unlock_irqrestore(&iommu->lock, flags);
  386. }
  387. #define SG_ENT_PHYS_ADDRESS(SG) \
  388. (__pa(page_address((SG)->page)) + (SG)->offset)
  389. static inline void fill_sg(iopte_t *iopte, struct scatterlist *sg,
  390. int nused, int nelems, unsigned long iopte_protection)
  391. {
  392. struct scatterlist *dma_sg = sg;
  393. struct scatterlist *sg_end = sg + nelems;
  394. int i;
  395. for (i = 0; i < nused; i++) {
  396. unsigned long pteval = ~0UL;
  397. u32 dma_npages;
  398. dma_npages = ((dma_sg->dma_address & (IO_PAGE_SIZE - 1UL)) +
  399. dma_sg->dma_length +
  400. ((IO_PAGE_SIZE - 1UL))) >> IO_PAGE_SHIFT;
  401. do {
  402. unsigned long offset;
  403. signed int len;
  404. /* If we are here, we know we have at least one
  405. * more page to map. So walk forward until we
  406. * hit a page crossing, and begin creating new
  407. * mappings from that spot.
  408. */
  409. for (;;) {
  410. unsigned long tmp;
  411. tmp = SG_ENT_PHYS_ADDRESS(sg);
  412. len = sg->length;
  413. if (((tmp ^ pteval) >> IO_PAGE_SHIFT) != 0UL) {
  414. pteval = tmp & IO_PAGE_MASK;
  415. offset = tmp & (IO_PAGE_SIZE - 1UL);
  416. break;
  417. }
  418. if (((tmp ^ (tmp + len - 1UL)) >> IO_PAGE_SHIFT) != 0UL) {
  419. pteval = (tmp + IO_PAGE_SIZE) & IO_PAGE_MASK;
  420. offset = 0UL;
  421. len -= (IO_PAGE_SIZE - (tmp & (IO_PAGE_SIZE - 1UL)));
  422. break;
  423. }
  424. sg++;
  425. }
  426. pteval = iopte_protection | (pteval & IOPTE_PAGE);
  427. while (len > 0) {
  428. *iopte++ = __iopte(pteval);
  429. pteval += IO_PAGE_SIZE;
  430. len -= (IO_PAGE_SIZE - offset);
  431. offset = 0;
  432. dma_npages--;
  433. }
  434. pteval = (pteval & IOPTE_PAGE) + len;
  435. sg++;
  436. /* Skip over any tail mappings we've fully mapped,
  437. * adjusting pteval along the way. Stop when we
  438. * detect a page crossing event.
  439. */
  440. while (sg < sg_end &&
  441. (pteval << (64 - IO_PAGE_SHIFT)) != 0UL &&
  442. (pteval == SG_ENT_PHYS_ADDRESS(sg)) &&
  443. ((pteval ^
  444. (SG_ENT_PHYS_ADDRESS(sg) + sg->length - 1UL)) >> IO_PAGE_SHIFT) == 0UL) {
  445. pteval += sg->length;
  446. sg++;
  447. }
  448. if ((pteval << (64 - IO_PAGE_SHIFT)) == 0UL)
  449. pteval = ~0UL;
  450. } while (dma_npages != 0);
  451. dma_sg++;
  452. }
  453. }
  454. /* Map a set of buffers described by SGLIST with NELEMS array
  455. * elements in streaming mode for PCI DMA.
  456. * When making changes here, inspect the assembly output. I was having
  457. * hard time to kepp this routine out of using stack slots for holding variables.
  458. */
  459. int pci_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int direction)
  460. {
  461. struct pcidev_cookie *pcp;
  462. struct pci_iommu *iommu;
  463. struct pci_strbuf *strbuf;
  464. unsigned long flags, ctx, npages, iopte_protection;
  465. iopte_t *base;
  466. u32 dma_base;
  467. struct scatterlist *sgtmp;
  468. int used;
  469. /* Fast path single entry scatterlists. */
  470. if (nelems == 1) {
  471. sglist->dma_address =
  472. pci_map_single(pdev,
  473. (page_address(sglist->page) + sglist->offset),
  474. sglist->length, direction);
  475. if (unlikely(sglist->dma_address == PCI_DMA_ERROR_CODE))
  476. return 0;
  477. sglist->dma_length = sglist->length;
  478. return 1;
  479. }
  480. pcp = pdev->sysdata;
  481. iommu = pcp->pbm->iommu;
  482. strbuf = &pcp->pbm->stc;
  483. if (unlikely(direction == PCI_DMA_NONE))
  484. goto bad_no_ctx;
  485. /* Step 1: Prepare scatter list. */
  486. npages = prepare_sg(sglist, nelems);
  487. /* Step 2: Allocate a cluster and context, if necessary. */
  488. spin_lock_irqsave(&iommu->lock, flags);
  489. base = alloc_npages(iommu, npages);
  490. ctx = 0;
  491. if (iommu->iommu_ctxflush)
  492. ctx = iommu_alloc_ctx(iommu);
  493. spin_unlock_irqrestore(&iommu->lock, flags);
  494. if (base == NULL)
  495. goto bad;
  496. dma_base = iommu->page_table_map_base +
  497. ((base - iommu->page_table) << IO_PAGE_SHIFT);
  498. /* Step 3: Normalize DMA addresses. */
  499. used = nelems;
  500. sgtmp = sglist;
  501. while (used && sgtmp->dma_length) {
  502. sgtmp->dma_address += dma_base;
  503. sgtmp++;
  504. used--;
  505. }
  506. used = nelems - used;
  507. /* Step 4: Create the mappings. */
  508. if (strbuf->strbuf_enabled)
  509. iopte_protection = IOPTE_STREAMING(ctx);
  510. else
  511. iopte_protection = IOPTE_CONSISTENT(ctx);
  512. if (direction != PCI_DMA_TODEVICE)
  513. iopte_protection |= IOPTE_WRITE;
  514. fill_sg(base, sglist, used, nelems, iopte_protection);
  515. #ifdef VERIFY_SG
  516. verify_sglist(sglist, nelems, base, npages);
  517. #endif
  518. return used;
  519. bad:
  520. iommu_free_ctx(iommu, ctx);
  521. bad_no_ctx:
  522. if (printk_ratelimit())
  523. WARN_ON(1);
  524. return 0;
  525. }
  526. /* Unmap a set of streaming mode DMA translations. */
  527. void pci_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int direction)
  528. {
  529. struct pcidev_cookie *pcp;
  530. struct pci_iommu *iommu;
  531. struct pci_strbuf *strbuf;
  532. iopte_t *base;
  533. unsigned long flags, ctx, i, npages;
  534. u32 bus_addr;
  535. if (unlikely(direction == PCI_DMA_NONE)) {
  536. if (printk_ratelimit())
  537. WARN_ON(1);
  538. }
  539. pcp = pdev->sysdata;
  540. iommu = pcp->pbm->iommu;
  541. strbuf = &pcp->pbm->stc;
  542. bus_addr = sglist->dma_address & IO_PAGE_MASK;
  543. for (i = 1; i < nelems; i++)
  544. if (sglist[i].dma_length == 0)
  545. break;
  546. i--;
  547. npages = (IO_PAGE_ALIGN(sglist[i].dma_address + sglist[i].dma_length) -
  548. bus_addr) >> IO_PAGE_SHIFT;
  549. base = iommu->page_table +
  550. ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT);
  551. #ifdef DEBUG_PCI_IOMMU
  552. if (IOPTE_IS_DUMMY(iommu, base))
  553. printk("pci_unmap_sg called on non-mapped region %016lx,%d from %016lx\n", sglist->dma_address, nelems, __builtin_return_address(0));
  554. #endif
  555. spin_lock_irqsave(&iommu->lock, flags);
  556. /* Record the context, if any. */
  557. ctx = 0;
  558. if (iommu->iommu_ctxflush)
  559. ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL;
  560. /* Step 1: Kick data out of streaming buffers if necessary. */
  561. if (strbuf->strbuf_enabled)
  562. pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction);
  563. /* Step 2: Clear out the TSB entries. */
  564. for (i = 0; i < npages; i++)
  565. iopte_make_dummy(iommu, base + i);
  566. free_npages(iommu, bus_addr - iommu->page_table_map_base, npages);
  567. iommu_free_ctx(iommu, ctx);
  568. spin_unlock_irqrestore(&iommu->lock, flags);
  569. }
  570. /* Make physical memory consistent for a single
  571. * streaming mode DMA translation after a transfer.
  572. */
  573. void pci_dma_sync_single_for_cpu(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int direction)
  574. {
  575. struct pcidev_cookie *pcp;
  576. struct pci_iommu *iommu;
  577. struct pci_strbuf *strbuf;
  578. unsigned long flags, ctx, npages;
  579. pcp = pdev->sysdata;
  580. iommu = pcp->pbm->iommu;
  581. strbuf = &pcp->pbm->stc;
  582. if (!strbuf->strbuf_enabled)
  583. return;
  584. spin_lock_irqsave(&iommu->lock, flags);
  585. npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK);
  586. npages >>= IO_PAGE_SHIFT;
  587. bus_addr &= IO_PAGE_MASK;
  588. /* Step 1: Record the context, if any. */
  589. ctx = 0;
  590. if (iommu->iommu_ctxflush &&
  591. strbuf->strbuf_ctxflush) {
  592. iopte_t *iopte;
  593. iopte = iommu->page_table +
  594. ((bus_addr - iommu->page_table_map_base)>>IO_PAGE_SHIFT);
  595. ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL;
  596. }
  597. /* Step 2: Kick data out of streaming buffers. */
  598. pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction);
  599. spin_unlock_irqrestore(&iommu->lock, flags);
  600. }
  601. /* Make physical memory consistent for a set of streaming
  602. * mode DMA translations after a transfer.
  603. */
  604. void pci_dma_sync_sg_for_cpu(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int direction)
  605. {
  606. struct pcidev_cookie *pcp;
  607. struct pci_iommu *iommu;
  608. struct pci_strbuf *strbuf;
  609. unsigned long flags, ctx, npages, i;
  610. u32 bus_addr;
  611. pcp = pdev->sysdata;
  612. iommu = pcp->pbm->iommu;
  613. strbuf = &pcp->pbm->stc;
  614. if (!strbuf->strbuf_enabled)
  615. return;
  616. spin_lock_irqsave(&iommu->lock, flags);
  617. /* Step 1: Record the context, if any. */
  618. ctx = 0;
  619. if (iommu->iommu_ctxflush &&
  620. strbuf->strbuf_ctxflush) {
  621. iopte_t *iopte;
  622. iopte = iommu->page_table +
  623. ((sglist[0].dma_address - iommu->page_table_map_base) >> IO_PAGE_SHIFT);
  624. ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL;
  625. }
  626. /* Step 2: Kick data out of streaming buffers. */
  627. bus_addr = sglist[0].dma_address & IO_PAGE_MASK;
  628. for(i = 1; i < nelems; i++)
  629. if (!sglist[i].dma_length)
  630. break;
  631. i--;
  632. npages = (IO_PAGE_ALIGN(sglist[i].dma_address + sglist[i].dma_length)
  633. - bus_addr) >> IO_PAGE_SHIFT;
  634. pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction);
  635. spin_unlock_irqrestore(&iommu->lock, flags);
  636. }
  637. static void ali_sound_dma_hack(struct pci_dev *pdev, int set_bit)
  638. {
  639. struct pci_dev *ali_isa_bridge;
  640. u8 val;
  641. /* ALI sound chips generate 31-bits of DMA, a special register
  642. * determines what bit 31 is emitted as.
  643. */
  644. ali_isa_bridge = pci_get_device(PCI_VENDOR_ID_AL,
  645. PCI_DEVICE_ID_AL_M1533,
  646. NULL);
  647. pci_read_config_byte(ali_isa_bridge, 0x7e, &val);
  648. if (set_bit)
  649. val |= 0x01;
  650. else
  651. val &= ~0x01;
  652. pci_write_config_byte(ali_isa_bridge, 0x7e, val);
  653. pci_dev_put(ali_isa_bridge);
  654. }
  655. int pci_dma_supported(struct pci_dev *pdev, u64 device_mask)
  656. {
  657. struct pcidev_cookie *pcp = pdev->sysdata;
  658. u64 dma_addr_mask;
  659. if (pdev == NULL) {
  660. dma_addr_mask = 0xffffffff;
  661. } else {
  662. struct pci_iommu *iommu = pcp->pbm->iommu;
  663. dma_addr_mask = iommu->dma_addr_mask;
  664. if (pdev->vendor == PCI_VENDOR_ID_AL &&
  665. pdev->device == PCI_DEVICE_ID_AL_M5451 &&
  666. device_mask == 0x7fffffff) {
  667. ali_sound_dma_hack(pdev,
  668. (dma_addr_mask & 0x80000000) != 0);
  669. return 1;
  670. }
  671. }
  672. if (device_mask >= (1UL << 32UL))
  673. return 0;
  674. return (device_mask & dma_addr_mask) == dma_addr_mask;
  675. }