cik_sdma.c 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785
  1. /*
  2. * Copyright 2013 Advanced Micro Devices, Inc.
  3. *
  4. * Permission is hereby granted, free of charge, to any person obtaining a
  5. * copy of this software and associated documentation files (the "Software"),
  6. * to deal in the Software without restriction, including without limitation
  7. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8. * and/or sell copies of the Software, and to permit persons to whom the
  9. * Software is furnished to do so, subject to the following conditions:
  10. *
  11. * The above copyright notice and this permission notice shall be included in
  12. * all copies or substantial portions of the Software.
  13. *
  14. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  17. * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18. * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19. * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20. * OTHER DEALINGS IN THE SOFTWARE.
  21. *
  22. * Authors: Alex Deucher
  23. */
  24. #include <linux/firmware.h>
  25. #include <drm/drmP.h>
  26. #include "radeon.h"
  27. #include "radeon_asic.h"
  28. #include "cikd.h"
  29. /* sdma */
  30. #define CIK_SDMA_UCODE_SIZE 1050
  31. #define CIK_SDMA_UCODE_VERSION 64
  32. u32 cik_gpu_check_soft_reset(struct radeon_device *rdev);
  33. /*
  34. * sDMA - System DMA
  35. * Starting with CIK, the GPU has new asynchronous
  36. * DMA engines. These engines are used for compute
  37. * and gfx. There are two DMA engines (SDMA0, SDMA1)
  38. * and each one supports 1 ring buffer used for gfx
  39. * and 2 queues used for compute.
  40. *
  41. * The programming model is very similar to the CP
  42. * (ring buffer, IBs, etc.), but sDMA has it's own
  43. * packet format that is different from the PM4 format
  44. * used by the CP. sDMA supports copying data, writing
  45. * embedded data, solid fills, and a number of other
  46. * things. It also has support for tiling/detiling of
  47. * buffers.
  48. */
  49. /**
  50. * cik_sdma_ring_ib_execute - Schedule an IB on the DMA engine
  51. *
  52. * @rdev: radeon_device pointer
  53. * @ib: IB object to schedule
  54. *
  55. * Schedule an IB in the DMA ring (CIK).
  56. */
  57. void cik_sdma_ring_ib_execute(struct radeon_device *rdev,
  58. struct radeon_ib *ib)
  59. {
  60. struct radeon_ring *ring = &rdev->ring[ib->ring];
  61. u32 extra_bits = (ib->vm ? ib->vm->id : 0) & 0xf;
  62. if (rdev->wb.enabled) {
  63. u32 next_rptr = ring->wptr + 5;
  64. while ((next_rptr & 7) != 4)
  65. next_rptr++;
  66. next_rptr += 4;
  67. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0));
  68. radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
  69. radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
  70. radeon_ring_write(ring, 1); /* number of DWs to follow */
  71. radeon_ring_write(ring, next_rptr);
  72. }
  73. /* IB packet must end on a 8 DW boundary */
  74. while ((ring->wptr & 7) != 4)
  75. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
  76. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_INDIRECT_BUFFER, 0, extra_bits));
  77. radeon_ring_write(ring, ib->gpu_addr & 0xffffffe0); /* base must be 32 byte aligned */
  78. radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xffffffff);
  79. radeon_ring_write(ring, ib->length_dw);
  80. }
  81. /**
  82. * cik_sdma_fence_ring_emit - emit a fence on the DMA ring
  83. *
  84. * @rdev: radeon_device pointer
  85. * @fence: radeon fence object
  86. *
  87. * Add a DMA fence packet to the ring to write
  88. * the fence seq number and DMA trap packet to generate
  89. * an interrupt if needed (CIK).
  90. */
  91. void cik_sdma_fence_ring_emit(struct radeon_device *rdev,
  92. struct radeon_fence *fence)
  93. {
  94. struct radeon_ring *ring = &rdev->ring[fence->ring];
  95. u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
  96. u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(1) |
  97. SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */
  98. u32 ref_and_mask;
  99. if (fence->ring == R600_RING_TYPE_DMA_INDEX)
  100. ref_and_mask = SDMA0;
  101. else
  102. ref_and_mask = SDMA1;
  103. /* write the fence */
  104. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_FENCE, 0, 0));
  105. radeon_ring_write(ring, addr & 0xffffffff);
  106. radeon_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
  107. radeon_ring_write(ring, fence->seq);
  108. /* generate an interrupt */
  109. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_TRAP, 0, 0));
  110. /* flush HDP */
  111. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));
  112. radeon_ring_write(ring, GPU_HDP_FLUSH_DONE);
  113. radeon_ring_write(ring, GPU_HDP_FLUSH_REQ);
  114. radeon_ring_write(ring, ref_and_mask); /* REFERENCE */
  115. radeon_ring_write(ring, ref_and_mask); /* MASK */
  116. radeon_ring_write(ring, (4 << 16) | 10); /* RETRY_COUNT, POLL_INTERVAL */
  117. }
  118. /**
  119. * cik_sdma_semaphore_ring_emit - emit a semaphore on the dma ring
  120. *
  121. * @rdev: radeon_device pointer
  122. * @ring: radeon_ring structure holding ring information
  123. * @semaphore: radeon semaphore object
  124. * @emit_wait: wait or signal semaphore
  125. *
  126. * Add a DMA semaphore packet to the ring wait on or signal
  127. * other rings (CIK).
  128. */
  129. void cik_sdma_semaphore_ring_emit(struct radeon_device *rdev,
  130. struct radeon_ring *ring,
  131. struct radeon_semaphore *semaphore,
  132. bool emit_wait)
  133. {
  134. u64 addr = semaphore->gpu_addr;
  135. u32 extra_bits = emit_wait ? 0 : SDMA_SEMAPHORE_EXTRA_S;
  136. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SEMAPHORE, 0, extra_bits));
  137. radeon_ring_write(ring, addr & 0xfffffff8);
  138. radeon_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
  139. }
  140. /**
  141. * cik_sdma_gfx_stop - stop the gfx async dma engines
  142. *
  143. * @rdev: radeon_device pointer
  144. *
  145. * Stop the gfx async dma ring buffers (CIK).
  146. */
  147. static void cik_sdma_gfx_stop(struct radeon_device *rdev)
  148. {
  149. u32 rb_cntl, reg_offset;
  150. int i;
  151. radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
  152. for (i = 0; i < 2; i++) {
  153. if (i == 0)
  154. reg_offset = SDMA0_REGISTER_OFFSET;
  155. else
  156. reg_offset = SDMA1_REGISTER_OFFSET;
  157. rb_cntl = RREG32(SDMA0_GFX_RB_CNTL + reg_offset);
  158. rb_cntl &= ~SDMA_RB_ENABLE;
  159. WREG32(SDMA0_GFX_RB_CNTL + reg_offset, rb_cntl);
  160. WREG32(SDMA0_GFX_IB_CNTL + reg_offset, 0);
  161. }
  162. }
  163. /**
  164. * cik_sdma_rlc_stop - stop the compute async dma engines
  165. *
  166. * @rdev: radeon_device pointer
  167. *
  168. * Stop the compute async dma queues (CIK).
  169. */
  170. static void cik_sdma_rlc_stop(struct radeon_device *rdev)
  171. {
  172. /* XXX todo */
  173. }
  174. /**
  175. * cik_sdma_enable - stop the async dma engines
  176. *
  177. * @rdev: radeon_device pointer
  178. * @enable: enable/disable the DMA MEs.
  179. *
  180. * Halt or unhalt the async dma engines (CIK).
  181. */
  182. void cik_sdma_enable(struct radeon_device *rdev, bool enable)
  183. {
  184. u32 me_cntl, reg_offset;
  185. int i;
  186. for (i = 0; i < 2; i++) {
  187. if (i == 0)
  188. reg_offset = SDMA0_REGISTER_OFFSET;
  189. else
  190. reg_offset = SDMA1_REGISTER_OFFSET;
  191. me_cntl = RREG32(SDMA0_ME_CNTL + reg_offset);
  192. if (enable)
  193. me_cntl &= ~SDMA_HALT;
  194. else
  195. me_cntl |= SDMA_HALT;
  196. WREG32(SDMA0_ME_CNTL + reg_offset, me_cntl);
  197. }
  198. }
  199. /**
  200. * cik_sdma_gfx_resume - setup and start the async dma engines
  201. *
  202. * @rdev: radeon_device pointer
  203. *
  204. * Set up the gfx DMA ring buffers and enable them (CIK).
  205. * Returns 0 for success, error for failure.
  206. */
  207. static int cik_sdma_gfx_resume(struct radeon_device *rdev)
  208. {
  209. struct radeon_ring *ring;
  210. u32 rb_cntl, ib_cntl;
  211. u32 rb_bufsz;
  212. u32 reg_offset, wb_offset;
  213. int i, r;
  214. for (i = 0; i < 2; i++) {
  215. if (i == 0) {
  216. ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
  217. reg_offset = SDMA0_REGISTER_OFFSET;
  218. wb_offset = R600_WB_DMA_RPTR_OFFSET;
  219. } else {
  220. ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
  221. reg_offset = SDMA1_REGISTER_OFFSET;
  222. wb_offset = CAYMAN_WB_DMA1_RPTR_OFFSET;
  223. }
  224. WREG32(SDMA0_SEM_INCOMPLETE_TIMER_CNTL + reg_offset, 0);
  225. WREG32(SDMA0_SEM_WAIT_FAIL_TIMER_CNTL + reg_offset, 0);
  226. /* Set ring buffer size in dwords */
  227. rb_bufsz = order_base_2(ring->ring_size / 4);
  228. rb_cntl = rb_bufsz << 1;
  229. #ifdef __BIG_ENDIAN
  230. rb_cntl |= SDMA_RB_SWAP_ENABLE | SDMA_RPTR_WRITEBACK_SWAP_ENABLE;
  231. #endif
  232. WREG32(SDMA0_GFX_RB_CNTL + reg_offset, rb_cntl);
  233. /* Initialize the ring buffer's read and write pointers */
  234. WREG32(SDMA0_GFX_RB_RPTR + reg_offset, 0);
  235. WREG32(SDMA0_GFX_RB_WPTR + reg_offset, 0);
  236. /* set the wb address whether it's enabled or not */
  237. WREG32(SDMA0_GFX_RB_RPTR_ADDR_HI + reg_offset,
  238. upper_32_bits(rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF);
  239. WREG32(SDMA0_GFX_RB_RPTR_ADDR_LO + reg_offset,
  240. ((rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC));
  241. if (rdev->wb.enabled)
  242. rb_cntl |= SDMA_RPTR_WRITEBACK_ENABLE;
  243. WREG32(SDMA0_GFX_RB_BASE + reg_offset, ring->gpu_addr >> 8);
  244. WREG32(SDMA0_GFX_RB_BASE_HI + reg_offset, ring->gpu_addr >> 40);
  245. ring->wptr = 0;
  246. WREG32(SDMA0_GFX_RB_WPTR + reg_offset, ring->wptr << 2);
  247. ring->rptr = RREG32(SDMA0_GFX_RB_RPTR + reg_offset) >> 2;
  248. /* enable DMA RB */
  249. WREG32(SDMA0_GFX_RB_CNTL + reg_offset, rb_cntl | SDMA_RB_ENABLE);
  250. ib_cntl = SDMA_IB_ENABLE;
  251. #ifdef __BIG_ENDIAN
  252. ib_cntl |= SDMA_IB_SWAP_ENABLE;
  253. #endif
  254. /* enable DMA IBs */
  255. WREG32(SDMA0_GFX_IB_CNTL + reg_offset, ib_cntl);
  256. ring->ready = true;
  257. r = radeon_ring_test(rdev, ring->idx, ring);
  258. if (r) {
  259. ring->ready = false;
  260. return r;
  261. }
  262. }
  263. radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
  264. return 0;
  265. }
  266. /**
  267. * cik_sdma_rlc_resume - setup and start the async dma engines
  268. *
  269. * @rdev: radeon_device pointer
  270. *
  271. * Set up the compute DMA queues and enable them (CIK).
  272. * Returns 0 for success, error for failure.
  273. */
  274. static int cik_sdma_rlc_resume(struct radeon_device *rdev)
  275. {
  276. /* XXX todo */
  277. return 0;
  278. }
  279. /**
  280. * cik_sdma_load_microcode - load the sDMA ME ucode
  281. *
  282. * @rdev: radeon_device pointer
  283. *
  284. * Loads the sDMA0/1 ucode.
  285. * Returns 0 for success, -EINVAL if the ucode is not available.
  286. */
  287. static int cik_sdma_load_microcode(struct radeon_device *rdev)
  288. {
  289. const __be32 *fw_data;
  290. int i;
  291. if (!rdev->sdma_fw)
  292. return -EINVAL;
  293. /* stop the gfx rings and rlc compute queues */
  294. cik_sdma_gfx_stop(rdev);
  295. cik_sdma_rlc_stop(rdev);
  296. /* halt the MEs */
  297. cik_sdma_enable(rdev, false);
  298. /* sdma0 */
  299. fw_data = (const __be32 *)rdev->sdma_fw->data;
  300. WREG32(SDMA0_UCODE_ADDR + SDMA0_REGISTER_OFFSET, 0);
  301. for (i = 0; i < CIK_SDMA_UCODE_SIZE; i++)
  302. WREG32(SDMA0_UCODE_DATA + SDMA0_REGISTER_OFFSET, be32_to_cpup(fw_data++));
  303. WREG32(SDMA0_UCODE_DATA + SDMA0_REGISTER_OFFSET, CIK_SDMA_UCODE_VERSION);
  304. /* sdma1 */
  305. fw_data = (const __be32 *)rdev->sdma_fw->data;
  306. WREG32(SDMA0_UCODE_ADDR + SDMA1_REGISTER_OFFSET, 0);
  307. for (i = 0; i < CIK_SDMA_UCODE_SIZE; i++)
  308. WREG32(SDMA0_UCODE_DATA + SDMA1_REGISTER_OFFSET, be32_to_cpup(fw_data++));
  309. WREG32(SDMA0_UCODE_DATA + SDMA1_REGISTER_OFFSET, CIK_SDMA_UCODE_VERSION);
  310. WREG32(SDMA0_UCODE_ADDR + SDMA0_REGISTER_OFFSET, 0);
  311. WREG32(SDMA0_UCODE_ADDR + SDMA1_REGISTER_OFFSET, 0);
  312. return 0;
  313. }
  314. /**
  315. * cik_sdma_resume - setup and start the async dma engines
  316. *
  317. * @rdev: radeon_device pointer
  318. *
  319. * Set up the DMA engines and enable them (CIK).
  320. * Returns 0 for success, error for failure.
  321. */
  322. int cik_sdma_resume(struct radeon_device *rdev)
  323. {
  324. int r;
  325. /* Reset dma */
  326. WREG32(SRBM_SOFT_RESET, SOFT_RESET_SDMA | SOFT_RESET_SDMA1);
  327. RREG32(SRBM_SOFT_RESET);
  328. udelay(50);
  329. WREG32(SRBM_SOFT_RESET, 0);
  330. RREG32(SRBM_SOFT_RESET);
  331. r = cik_sdma_load_microcode(rdev);
  332. if (r)
  333. return r;
  334. /* unhalt the MEs */
  335. cik_sdma_enable(rdev, true);
  336. /* start the gfx rings and rlc compute queues */
  337. r = cik_sdma_gfx_resume(rdev);
  338. if (r)
  339. return r;
  340. r = cik_sdma_rlc_resume(rdev);
  341. if (r)
  342. return r;
  343. return 0;
  344. }
  345. /**
  346. * cik_sdma_fini - tear down the async dma engines
  347. *
  348. * @rdev: radeon_device pointer
  349. *
  350. * Stop the async dma engines and free the rings (CIK).
  351. */
  352. void cik_sdma_fini(struct radeon_device *rdev)
  353. {
  354. /* stop the gfx rings and rlc compute queues */
  355. cik_sdma_gfx_stop(rdev);
  356. cik_sdma_rlc_stop(rdev);
  357. /* halt the MEs */
  358. cik_sdma_enable(rdev, false);
  359. radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]);
  360. radeon_ring_fini(rdev, &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]);
  361. /* XXX - compute dma queue tear down */
  362. }
  363. /**
  364. * cik_copy_dma - copy pages using the DMA engine
  365. *
  366. * @rdev: radeon_device pointer
  367. * @src_offset: src GPU address
  368. * @dst_offset: dst GPU address
  369. * @num_gpu_pages: number of GPU pages to xfer
  370. * @fence: radeon fence object
  371. *
  372. * Copy GPU paging using the DMA engine (CIK).
  373. * Used by the radeon ttm implementation to move pages if
  374. * registered as the asic copy callback.
  375. */
  376. int cik_copy_dma(struct radeon_device *rdev,
  377. uint64_t src_offset, uint64_t dst_offset,
  378. unsigned num_gpu_pages,
  379. struct radeon_fence **fence)
  380. {
  381. struct radeon_semaphore *sem = NULL;
  382. int ring_index = rdev->asic->copy.dma_ring_index;
  383. struct radeon_ring *ring = &rdev->ring[ring_index];
  384. u32 size_in_bytes, cur_size_in_bytes;
  385. int i, num_loops;
  386. int r = 0;
  387. r = radeon_semaphore_create(rdev, &sem);
  388. if (r) {
  389. DRM_ERROR("radeon: moving bo (%d).\n", r);
  390. return r;
  391. }
  392. size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
  393. num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
  394. r = radeon_ring_lock(rdev, ring, num_loops * 7 + 14);
  395. if (r) {
  396. DRM_ERROR("radeon: moving bo (%d).\n", r);
  397. radeon_semaphore_free(rdev, &sem, NULL);
  398. return r;
  399. }
  400. if (radeon_fence_need_sync(*fence, ring->idx)) {
  401. radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
  402. ring->idx);
  403. radeon_fence_note_sync(*fence, ring->idx);
  404. } else {
  405. radeon_semaphore_free(rdev, &sem, NULL);
  406. }
  407. for (i = 0; i < num_loops; i++) {
  408. cur_size_in_bytes = size_in_bytes;
  409. if (cur_size_in_bytes > 0x1fffff)
  410. cur_size_in_bytes = 0x1fffff;
  411. size_in_bytes -= cur_size_in_bytes;
  412. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0));
  413. radeon_ring_write(ring, cur_size_in_bytes);
  414. radeon_ring_write(ring, 0); /* src/dst endian swap */
  415. radeon_ring_write(ring, src_offset & 0xffffffff);
  416. radeon_ring_write(ring, upper_32_bits(src_offset) & 0xffffffff);
  417. radeon_ring_write(ring, dst_offset & 0xfffffffc);
  418. radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xffffffff);
  419. src_offset += cur_size_in_bytes;
  420. dst_offset += cur_size_in_bytes;
  421. }
  422. r = radeon_fence_emit(rdev, fence, ring->idx);
  423. if (r) {
  424. radeon_ring_unlock_undo(rdev, ring);
  425. return r;
  426. }
  427. radeon_ring_unlock_commit(rdev, ring);
  428. radeon_semaphore_free(rdev, &sem, *fence);
  429. return r;
  430. }
  431. /**
  432. * cik_sdma_ring_test - simple async dma engine test
  433. *
  434. * @rdev: radeon_device pointer
  435. * @ring: radeon_ring structure holding ring information
  436. *
  437. * Test the DMA engine by writing using it to write an
  438. * value to memory. (CIK).
  439. * Returns 0 for success, error for failure.
  440. */
  441. int cik_sdma_ring_test(struct radeon_device *rdev,
  442. struct radeon_ring *ring)
  443. {
  444. unsigned i;
  445. int r;
  446. void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
  447. u32 tmp;
  448. if (!ptr) {
  449. DRM_ERROR("invalid vram scratch pointer\n");
  450. return -EINVAL;
  451. }
  452. tmp = 0xCAFEDEAD;
  453. writel(tmp, ptr);
  454. r = radeon_ring_lock(rdev, ring, 4);
  455. if (r) {
  456. DRM_ERROR("radeon: dma failed to lock ring %d (%d).\n", ring->idx, r);
  457. return r;
  458. }
  459. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0));
  460. radeon_ring_write(ring, rdev->vram_scratch.gpu_addr & 0xfffffffc);
  461. radeon_ring_write(ring, upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xffffffff);
  462. radeon_ring_write(ring, 1); /* number of DWs to follow */
  463. radeon_ring_write(ring, 0xDEADBEEF);
  464. radeon_ring_unlock_commit(rdev, ring);
  465. for (i = 0; i < rdev->usec_timeout; i++) {
  466. tmp = readl(ptr);
  467. if (tmp == 0xDEADBEEF)
  468. break;
  469. DRM_UDELAY(1);
  470. }
  471. if (i < rdev->usec_timeout) {
  472. DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
  473. } else {
  474. DRM_ERROR("radeon: ring %d test failed (0x%08X)\n",
  475. ring->idx, tmp);
  476. r = -EINVAL;
  477. }
  478. return r;
  479. }
  480. /**
  481. * cik_sdma_ib_test - test an IB on the DMA engine
  482. *
  483. * @rdev: radeon_device pointer
  484. * @ring: radeon_ring structure holding ring information
  485. *
  486. * Test a simple IB in the DMA ring (CIK).
  487. * Returns 0 on success, error on failure.
  488. */
  489. int cik_sdma_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
  490. {
  491. struct radeon_ib ib;
  492. unsigned i;
  493. int r;
  494. void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
  495. u32 tmp = 0;
  496. if (!ptr) {
  497. DRM_ERROR("invalid vram scratch pointer\n");
  498. return -EINVAL;
  499. }
  500. tmp = 0xCAFEDEAD;
  501. writel(tmp, ptr);
  502. r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
  503. if (r) {
  504. DRM_ERROR("radeon: failed to get ib (%d).\n", r);
  505. return r;
  506. }
  507. ib.ptr[0] = SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0);
  508. ib.ptr[1] = rdev->vram_scratch.gpu_addr & 0xfffffffc;
  509. ib.ptr[2] = upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xffffffff;
  510. ib.ptr[3] = 1;
  511. ib.ptr[4] = 0xDEADBEEF;
  512. ib.length_dw = 5;
  513. r = radeon_ib_schedule(rdev, &ib, NULL);
  514. if (r) {
  515. radeon_ib_free(rdev, &ib);
  516. DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
  517. return r;
  518. }
  519. r = radeon_fence_wait(ib.fence, false);
  520. if (r) {
  521. DRM_ERROR("radeon: fence wait failed (%d).\n", r);
  522. return r;
  523. }
  524. for (i = 0; i < rdev->usec_timeout; i++) {
  525. tmp = readl(ptr);
  526. if (tmp == 0xDEADBEEF)
  527. break;
  528. DRM_UDELAY(1);
  529. }
  530. if (i < rdev->usec_timeout) {
  531. DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
  532. } else {
  533. DRM_ERROR("radeon: ib test failed (0x%08X)\n", tmp);
  534. r = -EINVAL;
  535. }
  536. radeon_ib_free(rdev, &ib);
  537. return r;
  538. }
  539. /**
  540. * cik_sdma_is_lockup - Check if the DMA engine is locked up
  541. *
  542. * @rdev: radeon_device pointer
  543. * @ring: radeon_ring structure holding ring information
  544. *
  545. * Check if the async DMA engine is locked up (CIK).
  546. * Returns true if the engine appears to be locked up, false if not.
  547. */
  548. bool cik_sdma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
  549. {
  550. u32 reset_mask = cik_gpu_check_soft_reset(rdev);
  551. u32 mask;
  552. if (ring->idx == R600_RING_TYPE_DMA_INDEX)
  553. mask = RADEON_RESET_DMA;
  554. else
  555. mask = RADEON_RESET_DMA1;
  556. if (!(reset_mask & mask)) {
  557. radeon_ring_lockup_update(ring);
  558. return false;
  559. }
  560. /* force ring activities */
  561. radeon_ring_force_activity(rdev, ring);
  562. return radeon_ring_test_lockup(rdev, ring);
  563. }
  564. /**
  565. * cik_sdma_vm_set_page - update the page tables using sDMA
  566. *
  567. * @rdev: radeon_device pointer
  568. * @ib: indirect buffer to fill with commands
  569. * @pe: addr of the page entry
  570. * @addr: dst addr to write into pe
  571. * @count: number of page entries to update
  572. * @incr: increase next addr by incr bytes
  573. * @flags: access flags
  574. *
  575. * Update the page tables using sDMA (CIK).
  576. */
  577. void cik_sdma_vm_set_page(struct radeon_device *rdev,
  578. struct radeon_ib *ib,
  579. uint64_t pe,
  580. uint64_t addr, unsigned count,
  581. uint32_t incr, uint32_t flags)
  582. {
  583. uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
  584. uint64_t value;
  585. unsigned ndw;
  586. if (flags & RADEON_VM_PAGE_SYSTEM) {
  587. while (count) {
  588. ndw = count * 2;
  589. if (ndw > 0xFFFFE)
  590. ndw = 0xFFFFE;
  591. /* for non-physically contiguous pages (system) */
  592. ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0);
  593. ib->ptr[ib->length_dw++] = pe;
  594. ib->ptr[ib->length_dw++] = upper_32_bits(pe);
  595. ib->ptr[ib->length_dw++] = ndw;
  596. for (; ndw > 0; ndw -= 2, --count, pe += 8) {
  597. if (flags & RADEON_VM_PAGE_SYSTEM) {
  598. value = radeon_vm_map_gart(rdev, addr);
  599. value &= 0xFFFFFFFFFFFFF000ULL;
  600. } else if (flags & RADEON_VM_PAGE_VALID) {
  601. value = addr;
  602. } else {
  603. value = 0;
  604. }
  605. addr += incr;
  606. value |= r600_flags;
  607. ib->ptr[ib->length_dw++] = value;
  608. ib->ptr[ib->length_dw++] = upper_32_bits(value);
  609. }
  610. }
  611. } else {
  612. while (count) {
  613. ndw = count;
  614. if (ndw > 0x7FFFF)
  615. ndw = 0x7FFFF;
  616. if (flags & RADEON_VM_PAGE_VALID)
  617. value = addr;
  618. else
  619. value = 0;
  620. /* for physically contiguous pages (vram) */
  621. ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_GENERATE_PTE_PDE, 0, 0);
  622. ib->ptr[ib->length_dw++] = pe; /* dst addr */
  623. ib->ptr[ib->length_dw++] = upper_32_bits(pe);
  624. ib->ptr[ib->length_dw++] = r600_flags; /* mask */
  625. ib->ptr[ib->length_dw++] = 0;
  626. ib->ptr[ib->length_dw++] = value; /* value */
  627. ib->ptr[ib->length_dw++] = upper_32_bits(value);
  628. ib->ptr[ib->length_dw++] = incr; /* increment size */
  629. ib->ptr[ib->length_dw++] = 0;
  630. ib->ptr[ib->length_dw++] = ndw; /* number of entries */
  631. pe += ndw * 8;
  632. addr += ndw * incr;
  633. count -= ndw;
  634. }
  635. }
  636. while (ib->length_dw & 0x7)
  637. ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0);
  638. }
  639. /**
  640. * cik_dma_vm_flush - cik vm flush using sDMA
  641. *
  642. * @rdev: radeon_device pointer
  643. *
  644. * Update the page table base and flush the VM TLB
  645. * using sDMA (CIK).
  646. */
  647. void cik_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
  648. {
  649. struct radeon_ring *ring = &rdev->ring[ridx];
  650. u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(1) |
  651. SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */
  652. u32 ref_and_mask;
  653. if (vm == NULL)
  654. return;
  655. if (ridx == R600_RING_TYPE_DMA_INDEX)
  656. ref_and_mask = SDMA0;
  657. else
  658. ref_and_mask = SDMA1;
  659. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
  660. if (vm->id < 8) {
  661. radeon_ring_write(ring, (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
  662. } else {
  663. radeon_ring_write(ring, (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
  664. }
  665. radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
  666. /* update SH_MEM_* regs */
  667. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
  668. radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
  669. radeon_ring_write(ring, VMID(vm->id));
  670. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
  671. radeon_ring_write(ring, SH_MEM_BASES >> 2);
  672. radeon_ring_write(ring, 0);
  673. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
  674. radeon_ring_write(ring, SH_MEM_CONFIG >> 2);
  675. radeon_ring_write(ring, 0);
  676. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
  677. radeon_ring_write(ring, SH_MEM_APE1_BASE >> 2);
  678. radeon_ring_write(ring, 1);
  679. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
  680. radeon_ring_write(ring, SH_MEM_APE1_LIMIT >> 2);
  681. radeon_ring_write(ring, 0);
  682. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
  683. radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
  684. radeon_ring_write(ring, VMID(0));
  685. /* flush HDP */
  686. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));
  687. radeon_ring_write(ring, GPU_HDP_FLUSH_DONE);
  688. radeon_ring_write(ring, GPU_HDP_FLUSH_REQ);
  689. radeon_ring_write(ring, ref_and_mask); /* REFERENCE */
  690. radeon_ring_write(ring, ref_and_mask); /* MASK */
  691. radeon_ring_write(ring, (4 << 16) | 10); /* RETRY_COUNT, POLL_INTERVAL */
  692. /* flush TLB */
  693. radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
  694. radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
  695. radeon_ring_write(ring, 1 << vm->id);
  696. }