nouveau_sgdma.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538
  1. #include "drmP.h"
  2. #include "nouveau_drv.h"
  3. #include <linux/pagemap.h>
  4. #include <linux/slab.h>
  5. #define NV_CTXDMA_PAGE_SHIFT 12
  6. #define NV_CTXDMA_PAGE_SIZE (1 << NV_CTXDMA_PAGE_SHIFT)
  7. #define NV_CTXDMA_PAGE_MASK (NV_CTXDMA_PAGE_SIZE - 1)
  8. struct nouveau_sgdma_be {
  9. struct ttm_backend backend;
  10. struct drm_device *dev;
  11. dma_addr_t *pages;
  12. bool *ttm_alloced;
  13. unsigned nr_pages;
  14. u64 offset;
  15. bool bound;
  16. };
  17. static int
  18. nouveau_sgdma_populate(struct ttm_backend *be, unsigned long num_pages,
  19. struct page **pages, struct page *dummy_read_page,
  20. dma_addr_t *dma_addrs)
  21. {
  22. struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
  23. struct drm_device *dev = nvbe->dev;
  24. NV_DEBUG(nvbe->dev, "num_pages = %ld\n", num_pages);
  25. if (nvbe->pages)
  26. return -EINVAL;
  27. nvbe->pages = kmalloc(sizeof(dma_addr_t) * num_pages, GFP_KERNEL);
  28. if (!nvbe->pages)
  29. return -ENOMEM;
  30. nvbe->ttm_alloced = kmalloc(sizeof(bool) * num_pages, GFP_KERNEL);
  31. if (!nvbe->ttm_alloced) {
  32. kfree(nvbe->pages);
  33. nvbe->pages = NULL;
  34. return -ENOMEM;
  35. }
  36. nvbe->nr_pages = 0;
  37. while (num_pages--) {
  38. /* this code path isn't called and is incorrect anyways */
  39. if (0) { /*dma_addrs[nvbe->nr_pages] != DMA_ERROR_CODE)*/
  40. nvbe->pages[nvbe->nr_pages] =
  41. dma_addrs[nvbe->nr_pages];
  42. nvbe->ttm_alloced[nvbe->nr_pages] = true;
  43. } else {
  44. nvbe->pages[nvbe->nr_pages] =
  45. pci_map_page(dev->pdev, pages[nvbe->nr_pages], 0,
  46. PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
  47. if (pci_dma_mapping_error(dev->pdev,
  48. nvbe->pages[nvbe->nr_pages])) {
  49. be->func->clear(be);
  50. return -EFAULT;
  51. }
  52. nvbe->ttm_alloced[nvbe->nr_pages] = false;
  53. }
  54. nvbe->nr_pages++;
  55. }
  56. return 0;
  57. }
  58. static void
  59. nouveau_sgdma_clear(struct ttm_backend *be)
  60. {
  61. struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
  62. struct drm_device *dev;
  63. if (nvbe && nvbe->pages) {
  64. dev = nvbe->dev;
  65. NV_DEBUG(dev, "\n");
  66. if (nvbe->bound)
  67. be->func->unbind(be);
  68. while (nvbe->nr_pages--) {
  69. if (!nvbe->ttm_alloced[nvbe->nr_pages])
  70. pci_unmap_page(dev->pdev, nvbe->pages[nvbe->nr_pages],
  71. PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
  72. }
  73. kfree(nvbe->pages);
  74. kfree(nvbe->ttm_alloced);
  75. nvbe->pages = NULL;
  76. nvbe->ttm_alloced = NULL;
  77. nvbe->nr_pages = 0;
  78. }
  79. }
  80. static void
  81. nouveau_sgdma_destroy(struct ttm_backend *be)
  82. {
  83. struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
  84. if (be) {
  85. NV_DEBUG(nvbe->dev, "\n");
  86. if (nvbe) {
  87. if (nvbe->pages)
  88. be->func->clear(be);
  89. kfree(nvbe);
  90. }
  91. }
  92. }
  93. static int
  94. nv04_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem)
  95. {
  96. struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
  97. struct drm_device *dev = nvbe->dev;
  98. struct drm_nouveau_private *dev_priv = dev->dev_private;
  99. struct nouveau_gpuobj *gpuobj = dev_priv->gart_info.sg_ctxdma;
  100. unsigned i, j, pte;
  101. NV_DEBUG(dev, "pg=0x%lx\n", mem->start);
  102. nvbe->offset = mem->start << PAGE_SHIFT;
  103. pte = (nvbe->offset >> NV_CTXDMA_PAGE_SHIFT) + 2;
  104. for (i = 0; i < nvbe->nr_pages; i++) {
  105. dma_addr_t dma_offset = nvbe->pages[i];
  106. uint32_t offset_l = lower_32_bits(dma_offset);
  107. for (j = 0; j < PAGE_SIZE / NV_CTXDMA_PAGE_SIZE; j++, pte++) {
  108. nv_wo32(gpuobj, (pte * 4) + 0, offset_l | 3);
  109. offset_l += NV_CTXDMA_PAGE_SIZE;
  110. }
  111. }
  112. nvbe->bound = true;
  113. return 0;
  114. }
  115. static int
  116. nv04_sgdma_unbind(struct ttm_backend *be)
  117. {
  118. struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
  119. struct drm_device *dev = nvbe->dev;
  120. struct drm_nouveau_private *dev_priv = dev->dev_private;
  121. struct nouveau_gpuobj *gpuobj = dev_priv->gart_info.sg_ctxdma;
  122. unsigned i, j, pte;
  123. NV_DEBUG(dev, "\n");
  124. if (!nvbe->bound)
  125. return 0;
  126. pte = (nvbe->offset >> NV_CTXDMA_PAGE_SHIFT) + 2;
  127. for (i = 0; i < nvbe->nr_pages; i++) {
  128. for (j = 0; j < PAGE_SIZE / NV_CTXDMA_PAGE_SIZE; j++, pte++)
  129. nv_wo32(gpuobj, (pte * 4) + 0, 0x00000000);
  130. }
  131. nvbe->bound = false;
  132. return 0;
  133. }
  134. static struct ttm_backend_func nv04_sgdma_backend = {
  135. .populate = nouveau_sgdma_populate,
  136. .clear = nouveau_sgdma_clear,
  137. .bind = nv04_sgdma_bind,
  138. .unbind = nv04_sgdma_unbind,
  139. .destroy = nouveau_sgdma_destroy
  140. };
  141. static void
  142. nv41_sgdma_flush(struct nouveau_sgdma_be *nvbe)
  143. {
  144. struct drm_device *dev = nvbe->dev;
  145. nv_wr32(dev, 0x100810, 0x00000022);
  146. if (!nv_wait(dev, 0x100810, 0x00000100, 0x00000100))
  147. NV_ERROR(dev, "vm flush timeout: 0x%08x\n",
  148. nv_rd32(dev, 0x100810));
  149. nv_wr32(dev, 0x100810, 0x00000000);
  150. }
  151. static int
  152. nv41_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem)
  153. {
  154. struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
  155. struct drm_nouveau_private *dev_priv = nvbe->dev->dev_private;
  156. struct nouveau_gpuobj *pgt = dev_priv->gart_info.sg_ctxdma;
  157. dma_addr_t *list = nvbe->pages;
  158. u32 pte = mem->start << 2;
  159. u32 cnt = nvbe->nr_pages;
  160. nvbe->offset = mem->start << PAGE_SHIFT;
  161. while (cnt--) {
  162. nv_wo32(pgt, pte, (*list++ >> 7) | 1);
  163. pte += 4;
  164. }
  165. nv41_sgdma_flush(nvbe);
  166. nvbe->bound = true;
  167. return 0;
  168. }
  169. static int
  170. nv41_sgdma_unbind(struct ttm_backend *be)
  171. {
  172. struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
  173. struct drm_nouveau_private *dev_priv = nvbe->dev->dev_private;
  174. struct nouveau_gpuobj *pgt = dev_priv->gart_info.sg_ctxdma;
  175. u32 pte = (nvbe->offset >> 12) << 2;
  176. u32 cnt = nvbe->nr_pages;
  177. while (cnt--) {
  178. nv_wo32(pgt, pte, 0x00000000);
  179. pte += 4;
  180. }
  181. nv41_sgdma_flush(nvbe);
  182. nvbe->bound = false;
  183. return 0;
  184. }
  185. static struct ttm_backend_func nv41_sgdma_backend = {
  186. .populate = nouveau_sgdma_populate,
  187. .clear = nouveau_sgdma_clear,
  188. .bind = nv41_sgdma_bind,
  189. .unbind = nv41_sgdma_unbind,
  190. .destroy = nouveau_sgdma_destroy
  191. };
  192. static void
  193. nv44_sgdma_flush(struct nouveau_sgdma_be *nvbe)
  194. {
  195. struct drm_device *dev = nvbe->dev;
  196. nv_wr32(dev, 0x100814, (nvbe->nr_pages - 1) << 12);
  197. nv_wr32(dev, 0x100808, nvbe->offset | 0x20);
  198. if (!nv_wait(dev, 0x100808, 0x00000001, 0x00000001))
  199. NV_ERROR(dev, "gart flush timeout: 0x%08x\n",
  200. nv_rd32(dev, 0x100808));
  201. nv_wr32(dev, 0x100808, 0x00000000);
  202. }
  203. static void
  204. nv44_sgdma_fill(struct nouveau_gpuobj *pgt, dma_addr_t *list, u32 base, u32 cnt)
  205. {
  206. struct drm_nouveau_private *dev_priv = pgt->dev->dev_private;
  207. dma_addr_t dummy = dev_priv->gart_info.dummy.addr;
  208. u32 pte, tmp[4];
  209. pte = base >> 2;
  210. base &= ~0x0000000f;
  211. tmp[0] = nv_ro32(pgt, base + 0x0);
  212. tmp[1] = nv_ro32(pgt, base + 0x4);
  213. tmp[2] = nv_ro32(pgt, base + 0x8);
  214. tmp[3] = nv_ro32(pgt, base + 0xc);
  215. while (cnt--) {
  216. u32 addr = list ? (*list++ >> 12) : (dummy >> 12);
  217. switch (pte++ & 0x3) {
  218. case 0:
  219. tmp[0] &= ~0x07ffffff;
  220. tmp[0] |= addr;
  221. break;
  222. case 1:
  223. tmp[0] &= ~0xf8000000;
  224. tmp[0] |= addr << 27;
  225. tmp[1] &= ~0x003fffff;
  226. tmp[1] |= addr >> 5;
  227. break;
  228. case 2:
  229. tmp[1] &= ~0xffc00000;
  230. tmp[1] |= addr << 22;
  231. tmp[2] &= ~0x0001ffff;
  232. tmp[2] |= addr >> 10;
  233. break;
  234. case 3:
  235. tmp[2] &= ~0xfffe0000;
  236. tmp[2] |= addr << 17;
  237. tmp[3] &= ~0x00000fff;
  238. tmp[3] |= addr >> 15;
  239. break;
  240. }
  241. }
  242. tmp[3] |= 0x40000000;
  243. nv_wo32(pgt, base + 0x0, tmp[0]);
  244. nv_wo32(pgt, base + 0x4, tmp[1]);
  245. nv_wo32(pgt, base + 0x8, tmp[2]);
  246. nv_wo32(pgt, base + 0xc, tmp[3]);
  247. }
  248. static int
  249. nv44_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem)
  250. {
  251. struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
  252. struct drm_nouveau_private *dev_priv = nvbe->dev->dev_private;
  253. struct nouveau_gpuobj *pgt = dev_priv->gart_info.sg_ctxdma;
  254. dma_addr_t *list = nvbe->pages;
  255. u32 pte = mem->start << 2, tmp[4];
  256. u32 cnt = nvbe->nr_pages;
  257. int i;
  258. nvbe->offset = mem->start << PAGE_SHIFT;
  259. if (pte & 0x0000000c) {
  260. u32 max = 4 - ((pte >> 2) & 0x3);
  261. u32 part = (cnt > max) ? max : cnt;
  262. nv44_sgdma_fill(pgt, list, pte, part);
  263. pte += (part << 2);
  264. list += part;
  265. cnt -= part;
  266. }
  267. while (cnt >= 4) {
  268. for (i = 0; i < 4; i++)
  269. tmp[i] = *list++ >> 12;
  270. nv_wo32(pgt, pte + 0x0, tmp[0] >> 0 | tmp[1] << 27);
  271. nv_wo32(pgt, pte + 0x4, tmp[1] >> 5 | tmp[2] << 22);
  272. nv_wo32(pgt, pte + 0x8, tmp[2] >> 10 | tmp[3] << 17);
  273. nv_wo32(pgt, pte + 0xc, tmp[3] >> 15 | 0x40000000);
  274. pte += 0x10;
  275. cnt -= 4;
  276. }
  277. if (cnt)
  278. nv44_sgdma_fill(pgt, list, pte, cnt);
  279. nv44_sgdma_flush(nvbe);
  280. nvbe->bound = true;
  281. return 0;
  282. }
  283. static int
  284. nv44_sgdma_unbind(struct ttm_backend *be)
  285. {
  286. struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
  287. struct drm_nouveau_private *dev_priv = nvbe->dev->dev_private;
  288. struct nouveau_gpuobj *pgt = dev_priv->gart_info.sg_ctxdma;
  289. u32 pte = (nvbe->offset >> 12) << 2;
  290. u32 cnt = nvbe->nr_pages;
  291. if (pte & 0x0000000c) {
  292. u32 max = 4 - ((pte >> 2) & 0x3);
  293. u32 part = (cnt > max) ? max : cnt;
  294. nv44_sgdma_fill(pgt, NULL, pte, part);
  295. pte += (part << 2);
  296. cnt -= part;
  297. }
  298. while (cnt >= 4) {
  299. nv_wo32(pgt, pte + 0x0, 0x00000000);
  300. nv_wo32(pgt, pte + 0x4, 0x00000000);
  301. nv_wo32(pgt, pte + 0x8, 0x00000000);
  302. nv_wo32(pgt, pte + 0xc, 0x00000000);
  303. pte += 0x10;
  304. cnt -= 4;
  305. }
  306. if (cnt)
  307. nv44_sgdma_fill(pgt, NULL, pte, cnt);
  308. nv44_sgdma_flush(nvbe);
  309. nvbe->bound = false;
  310. return 0;
  311. }
  312. static struct ttm_backend_func nv44_sgdma_backend = {
  313. .populate = nouveau_sgdma_populate,
  314. .clear = nouveau_sgdma_clear,
  315. .bind = nv44_sgdma_bind,
  316. .unbind = nv44_sgdma_unbind,
  317. .destroy = nouveau_sgdma_destroy
  318. };
  319. static int
  320. nv50_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem)
  321. {
  322. struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
  323. struct nouveau_mem *node = mem->mm_node;
  324. /* noop: bound in move_notify() */
  325. node->pages = nvbe->pages;
  326. nvbe->pages = (dma_addr_t *)node;
  327. nvbe->bound = true;
  328. return 0;
  329. }
  330. static int
  331. nv50_sgdma_unbind(struct ttm_backend *be)
  332. {
  333. struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
  334. struct nouveau_mem *node = (struct nouveau_mem *)nvbe->pages;
  335. /* noop: unbound in move_notify() */
  336. nvbe->pages = node->pages;
  337. node->pages = NULL;
  338. nvbe->bound = false;
  339. return 0;
  340. }
  341. static struct ttm_backend_func nv50_sgdma_backend = {
  342. .populate = nouveau_sgdma_populate,
  343. .clear = nouveau_sgdma_clear,
  344. .bind = nv50_sgdma_bind,
  345. .unbind = nv50_sgdma_unbind,
  346. .destroy = nouveau_sgdma_destroy
  347. };
  348. struct ttm_backend *
  349. nouveau_sgdma_init_ttm(struct drm_device *dev)
  350. {
  351. struct drm_nouveau_private *dev_priv = dev->dev_private;
  352. struct nouveau_sgdma_be *nvbe;
  353. nvbe = kzalloc(sizeof(*nvbe), GFP_KERNEL);
  354. if (!nvbe)
  355. return NULL;
  356. nvbe->dev = dev;
  357. nvbe->backend.func = dev_priv->gart_info.func;
  358. return &nvbe->backend;
  359. }
  360. int
  361. nouveau_sgdma_init(struct drm_device *dev)
  362. {
  363. struct drm_nouveau_private *dev_priv = dev->dev_private;
  364. struct nouveau_gpuobj *gpuobj = NULL;
  365. u32 aper_size, align;
  366. int ret;
  367. if (dev_priv->card_type >= NV_40 && pci_is_pcie(dev->pdev))
  368. aper_size = 512 * 1024 * 1024;
  369. else
  370. aper_size = 64 * 1024 * 1024;
  371. /* Dear NVIDIA, NV44+ would like proper present bits in PTEs for
  372. * christmas. The cards before it have them, the cards after
  373. * it have them, why is NV44 so unloved?
  374. */
  375. dev_priv->gart_info.dummy.page = alloc_page(GFP_DMA32 | GFP_KERNEL);
  376. if (!dev_priv->gart_info.dummy.page)
  377. return -ENOMEM;
  378. dev_priv->gart_info.dummy.addr =
  379. pci_map_page(dev->pdev, dev_priv->gart_info.dummy.page,
  380. 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
  381. if (pci_dma_mapping_error(dev->pdev, dev_priv->gart_info.dummy.addr)) {
  382. NV_ERROR(dev, "error mapping dummy page\n");
  383. __free_page(dev_priv->gart_info.dummy.page);
  384. dev_priv->gart_info.dummy.page = NULL;
  385. return -ENOMEM;
  386. }
  387. if (dev_priv->card_type >= NV_50) {
  388. dev_priv->gart_info.aper_base = 0;
  389. dev_priv->gart_info.aper_size = aper_size;
  390. dev_priv->gart_info.type = NOUVEAU_GART_HW;
  391. dev_priv->gart_info.func = &nv50_sgdma_backend;
  392. } else
  393. if (0 && pci_is_pcie(dev->pdev) &&
  394. dev_priv->chipset > 0x40 && dev_priv->chipset != 0x45) {
  395. if (nv44_graph_class(dev)) {
  396. dev_priv->gart_info.func = &nv44_sgdma_backend;
  397. align = 512 * 1024;
  398. } else {
  399. dev_priv->gart_info.func = &nv41_sgdma_backend;
  400. align = 16;
  401. }
  402. ret = nouveau_gpuobj_new(dev, NULL, aper_size / 1024, align,
  403. NVOBJ_FLAG_ZERO_ALLOC |
  404. NVOBJ_FLAG_ZERO_FREE, &gpuobj);
  405. if (ret) {
  406. NV_ERROR(dev, "Error creating sgdma object: %d\n", ret);
  407. return ret;
  408. }
  409. dev_priv->gart_info.sg_ctxdma = gpuobj;
  410. dev_priv->gart_info.aper_base = 0;
  411. dev_priv->gart_info.aper_size = aper_size;
  412. dev_priv->gart_info.type = NOUVEAU_GART_HW;
  413. } else {
  414. ret = nouveau_gpuobj_new(dev, NULL, (aper_size / 1024) + 8, 16,
  415. NVOBJ_FLAG_ZERO_ALLOC |
  416. NVOBJ_FLAG_ZERO_FREE, &gpuobj);
  417. if (ret) {
  418. NV_ERROR(dev, "Error creating sgdma object: %d\n", ret);
  419. return ret;
  420. }
  421. nv_wo32(gpuobj, 0, NV_CLASS_DMA_IN_MEMORY |
  422. (1 << 12) /* PT present */ |
  423. (0 << 13) /* PT *not* linear */ |
  424. (0 << 14) /* RW */ |
  425. (2 << 16) /* PCI */);
  426. nv_wo32(gpuobj, 4, aper_size - 1);
  427. dev_priv->gart_info.sg_ctxdma = gpuobj;
  428. dev_priv->gart_info.aper_base = 0;
  429. dev_priv->gart_info.aper_size = aper_size;
  430. dev_priv->gart_info.type = NOUVEAU_GART_PDMA;
  431. dev_priv->gart_info.func = &nv04_sgdma_backend;
  432. }
  433. return 0;
  434. }
  435. void
  436. nouveau_sgdma_takedown(struct drm_device *dev)
  437. {
  438. struct drm_nouveau_private *dev_priv = dev->dev_private;
  439. nouveau_gpuobj_ref(NULL, &dev_priv->gart_info.sg_ctxdma);
  440. if (dev_priv->gart_info.dummy.page) {
  441. pci_unmap_page(dev->pdev, dev_priv->gart_info.dummy.addr,
  442. PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
  443. __free_page(dev_priv->gart_info.dummy.page);
  444. dev_priv->gart_info.dummy.page = NULL;
  445. }
  446. }
  447. uint32_t
  448. nouveau_sgdma_get_physical(struct drm_device *dev, uint32_t offset)
  449. {
  450. struct drm_nouveau_private *dev_priv = dev->dev_private;
  451. struct nouveau_gpuobj *gpuobj = dev_priv->gart_info.sg_ctxdma;
  452. int pte = (offset >> NV_CTXDMA_PAGE_SHIFT) + 2;
  453. BUG_ON(dev_priv->card_type >= NV_50);
  454. return (nv_ro32(gpuobj, 4 * pte) & ~NV_CTXDMA_PAGE_MASK) |
  455. (offset & NV_CTXDMA_PAGE_MASK);
  456. }