radeon_cs.c 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653
  1. /*
  2. * Copyright 2008 Jerome Glisse.
  3. * All Rights Reserved.
  4. *
  5. * Permission is hereby granted, free of charge, to any person obtaining a
  6. * copy of this software and associated documentation files (the "Software"),
  7. * to deal in the Software without restriction, including without limitation
  8. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  9. * and/or sell copies of the Software, and to permit persons to whom the
  10. * Software is furnished to do so, subject to the following conditions:
  11. *
  12. * The above copyright notice and this permission notice (including the next
  13. * paragraph) shall be included in all copies or substantial portions of the
  14. * Software.
  15. *
  16. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19. * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20. * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21. * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  22. * DEALINGS IN THE SOFTWARE.
  23. *
  24. * Authors:
  25. * Jerome Glisse <glisse@freedesktop.org>
  26. */
  27. #include "drmP.h"
  28. #include "radeon_drm.h"
  29. #include "radeon_reg.h"
  30. #include "radeon.h"
  31. void r100_cs_dump_packet(struct radeon_cs_parser *p,
  32. struct radeon_cs_packet *pkt);
  33. int radeon_cs_parser_relocs(struct radeon_cs_parser *p)
  34. {
  35. struct drm_device *ddev = p->rdev->ddev;
  36. struct radeon_cs_chunk *chunk;
  37. unsigned i, j;
  38. bool duplicate;
  39. if (p->chunk_relocs_idx == -1) {
  40. return 0;
  41. }
  42. chunk = &p->chunks[p->chunk_relocs_idx];
  43. /* FIXME: we assume that each relocs use 4 dwords */
  44. p->nrelocs = chunk->length_dw / 4;
  45. p->relocs_ptr = kcalloc(p->nrelocs, sizeof(void *), GFP_KERNEL);
  46. if (p->relocs_ptr == NULL) {
  47. return -ENOMEM;
  48. }
  49. p->relocs = kcalloc(p->nrelocs, sizeof(struct radeon_cs_reloc), GFP_KERNEL);
  50. if (p->relocs == NULL) {
  51. return -ENOMEM;
  52. }
  53. for (i = 0; i < p->nrelocs; i++) {
  54. struct drm_radeon_cs_reloc *r;
  55. duplicate = false;
  56. r = (struct drm_radeon_cs_reloc *)&chunk->kdata[i*4];
  57. for (j = 0; j < i; j++) {
  58. if (r->handle == p->relocs[j].handle) {
  59. p->relocs_ptr[i] = &p->relocs[j];
  60. duplicate = true;
  61. break;
  62. }
  63. }
  64. if (!duplicate) {
  65. p->relocs[i].gobj = drm_gem_object_lookup(ddev,
  66. p->filp,
  67. r->handle);
  68. if (p->relocs[i].gobj == NULL) {
  69. DRM_ERROR("gem object lookup failed 0x%x\n",
  70. r->handle);
  71. return -ENOENT;
  72. }
  73. p->relocs_ptr[i] = &p->relocs[i];
  74. p->relocs[i].robj = gem_to_radeon_bo(p->relocs[i].gobj);
  75. p->relocs[i].lobj.bo = p->relocs[i].robj;
  76. p->relocs[i].lobj.wdomain = r->write_domain;
  77. p->relocs[i].lobj.rdomain = r->read_domains;
  78. p->relocs[i].lobj.tv.bo = &p->relocs[i].robj->tbo;
  79. p->relocs[i].handle = r->handle;
  80. p->relocs[i].flags = r->flags;
  81. radeon_bo_list_add_object(&p->relocs[i].lobj,
  82. &p->validated);
  83. } else
  84. p->relocs[i].handle = 0;
  85. }
  86. return radeon_bo_list_validate(&p->validated);
  87. }
  88. static int radeon_cs_get_ring(struct radeon_cs_parser *p, u32 ring, s32 priority)
  89. {
  90. p->priority = priority;
  91. switch (ring) {
  92. default:
  93. DRM_ERROR("unknown ring id: %d\n", ring);
  94. return -EINVAL;
  95. case RADEON_CS_RING_GFX:
  96. p->ring = RADEON_RING_TYPE_GFX_INDEX;
  97. break;
  98. case RADEON_CS_RING_COMPUTE:
  99. if (p->rdev->family >= CHIP_TAHITI) {
  100. if (p->priority > 0)
  101. p->ring = CAYMAN_RING_TYPE_CP1_INDEX;
  102. else
  103. p->ring = CAYMAN_RING_TYPE_CP2_INDEX;
  104. } else
  105. p->ring = RADEON_RING_TYPE_GFX_INDEX;
  106. break;
  107. }
  108. return 0;
  109. }
  110. static int radeon_cs_sync_rings(struct radeon_cs_parser *p)
  111. {
  112. bool sync_to_ring[RADEON_NUM_RINGS] = { };
  113. bool need_sync = false;
  114. int i, r;
  115. for (i = 0; i < p->nrelocs; i++) {
  116. struct radeon_fence *fence;
  117. if (!p->relocs[i].robj || !p->relocs[i].robj->tbo.sync_obj)
  118. continue;
  119. fence = p->relocs[i].robj->tbo.sync_obj;
  120. if (fence->ring != p->ring && !radeon_fence_signaled(fence)) {
  121. sync_to_ring[fence->ring] = true;
  122. need_sync = true;
  123. }
  124. }
  125. if (!need_sync) {
  126. return 0;
  127. }
  128. r = radeon_semaphore_create(p->rdev, &p->ib.semaphore);
  129. if (r) {
  130. return r;
  131. }
  132. return radeon_semaphore_sync_rings(p->rdev, p->ib.semaphore,
  133. sync_to_ring, p->ring);
  134. }
  135. /* XXX: note that this is called from the legacy UMS CS ioctl as well */
  136. int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data)
  137. {
  138. struct drm_radeon_cs *cs = data;
  139. uint64_t *chunk_array_ptr;
  140. unsigned size, i;
  141. u32 ring = RADEON_CS_RING_GFX;
  142. s32 priority = 0;
  143. if (!cs->num_chunks) {
  144. return 0;
  145. }
  146. /* get chunks */
  147. INIT_LIST_HEAD(&p->validated);
  148. p->idx = 0;
  149. p->ib.sa_bo = NULL;
  150. p->ib.semaphore = NULL;
  151. p->const_ib.sa_bo = NULL;
  152. p->const_ib.semaphore = NULL;
  153. p->chunk_ib_idx = -1;
  154. p->chunk_relocs_idx = -1;
  155. p->chunk_flags_idx = -1;
  156. p->chunk_const_ib_idx = -1;
  157. p->chunks_array = kcalloc(cs->num_chunks, sizeof(uint64_t), GFP_KERNEL);
  158. if (p->chunks_array == NULL) {
  159. return -ENOMEM;
  160. }
  161. chunk_array_ptr = (uint64_t *)(unsigned long)(cs->chunks);
  162. if (DRM_COPY_FROM_USER(p->chunks_array, chunk_array_ptr,
  163. sizeof(uint64_t)*cs->num_chunks)) {
  164. return -EFAULT;
  165. }
  166. p->cs_flags = 0;
  167. p->nchunks = cs->num_chunks;
  168. p->chunks = kcalloc(p->nchunks, sizeof(struct radeon_cs_chunk), GFP_KERNEL);
  169. if (p->chunks == NULL) {
  170. return -ENOMEM;
  171. }
  172. for (i = 0; i < p->nchunks; i++) {
  173. struct drm_radeon_cs_chunk __user **chunk_ptr = NULL;
  174. struct drm_radeon_cs_chunk user_chunk;
  175. uint32_t __user *cdata;
  176. chunk_ptr = (void __user*)(unsigned long)p->chunks_array[i];
  177. if (DRM_COPY_FROM_USER(&user_chunk, chunk_ptr,
  178. sizeof(struct drm_radeon_cs_chunk))) {
  179. return -EFAULT;
  180. }
  181. p->chunks[i].length_dw = user_chunk.length_dw;
  182. p->chunks[i].kdata = NULL;
  183. p->chunks[i].chunk_id = user_chunk.chunk_id;
  184. if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_RELOCS) {
  185. p->chunk_relocs_idx = i;
  186. }
  187. if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_IB) {
  188. p->chunk_ib_idx = i;
  189. /* zero length IB isn't useful */
  190. if (p->chunks[i].length_dw == 0)
  191. return -EINVAL;
  192. }
  193. if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_CONST_IB) {
  194. p->chunk_const_ib_idx = i;
  195. /* zero length CONST IB isn't useful */
  196. if (p->chunks[i].length_dw == 0)
  197. return -EINVAL;
  198. }
  199. if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS) {
  200. p->chunk_flags_idx = i;
  201. /* zero length flags aren't useful */
  202. if (p->chunks[i].length_dw == 0)
  203. return -EINVAL;
  204. }
  205. p->chunks[i].length_dw = user_chunk.length_dw;
  206. p->chunks[i].user_ptr = (void __user *)(unsigned long)user_chunk.chunk_data;
  207. cdata = (uint32_t *)(unsigned long)user_chunk.chunk_data;
  208. if ((p->chunks[i].chunk_id == RADEON_CHUNK_ID_RELOCS) ||
  209. (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS)) {
  210. size = p->chunks[i].length_dw * sizeof(uint32_t);
  211. p->chunks[i].kdata = kmalloc(size, GFP_KERNEL);
  212. if (p->chunks[i].kdata == NULL) {
  213. return -ENOMEM;
  214. }
  215. if (DRM_COPY_FROM_USER(p->chunks[i].kdata,
  216. p->chunks[i].user_ptr, size)) {
  217. return -EFAULT;
  218. }
  219. if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS) {
  220. p->cs_flags = p->chunks[i].kdata[0];
  221. if (p->chunks[i].length_dw > 1)
  222. ring = p->chunks[i].kdata[1];
  223. if (p->chunks[i].length_dw > 2)
  224. priority = (s32)p->chunks[i].kdata[2];
  225. }
  226. }
  227. }
  228. /* these are KMS only */
  229. if (p->rdev) {
  230. if ((p->cs_flags & RADEON_CS_USE_VM) &&
  231. !p->rdev->vm_manager.enabled) {
  232. DRM_ERROR("VM not active on asic!\n");
  233. return -EINVAL;
  234. }
  235. /* we only support VM on SI+ */
  236. if ((p->rdev->family >= CHIP_TAHITI) &&
  237. ((p->cs_flags & RADEON_CS_USE_VM) == 0)) {
  238. DRM_ERROR("VM required on SI+!\n");
  239. return -EINVAL;
  240. }
  241. if (radeon_cs_get_ring(p, ring, priority))
  242. return -EINVAL;
  243. }
  244. /* deal with non-vm */
  245. if ((p->chunk_ib_idx != -1) &&
  246. ((p->cs_flags & RADEON_CS_USE_VM) == 0) &&
  247. (p->chunks[p->chunk_ib_idx].chunk_id == RADEON_CHUNK_ID_IB)) {
  248. if (p->chunks[p->chunk_ib_idx].length_dw > (16 * 1024)) {
  249. DRM_ERROR("cs IB too big: %d\n",
  250. p->chunks[p->chunk_ib_idx].length_dw);
  251. return -EINVAL;
  252. }
  253. if ((p->rdev->flags & RADEON_IS_AGP)) {
  254. p->chunks[p->chunk_ib_idx].kpage[0] = kmalloc(PAGE_SIZE, GFP_KERNEL);
  255. p->chunks[p->chunk_ib_idx].kpage[1] = kmalloc(PAGE_SIZE, GFP_KERNEL);
  256. if (p->chunks[p->chunk_ib_idx].kpage[0] == NULL ||
  257. p->chunks[p->chunk_ib_idx].kpage[1] == NULL) {
  258. kfree(p->chunks[i].kpage[0]);
  259. kfree(p->chunks[i].kpage[1]);
  260. return -ENOMEM;
  261. }
  262. }
  263. p->chunks[p->chunk_ib_idx].kpage_idx[0] = -1;
  264. p->chunks[p->chunk_ib_idx].kpage_idx[1] = -1;
  265. p->chunks[p->chunk_ib_idx].last_copied_page = -1;
  266. p->chunks[p->chunk_ib_idx].last_page_index =
  267. ((p->chunks[p->chunk_ib_idx].length_dw * 4) - 1) / PAGE_SIZE;
  268. }
  269. return 0;
  270. }
  271. /**
  272. * cs_parser_fini() - clean parser states
  273. * @parser: parser structure holding parsing context.
  274. * @error: error number
  275. *
  276. * If error is set than unvalidate buffer, otherwise just free memory
  277. * used by parsing context.
  278. **/
  279. static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error)
  280. {
  281. unsigned i;
  282. if (!error)
  283. ttm_eu_fence_buffer_objects(&parser->validated,
  284. parser->ib.fence);
  285. else
  286. ttm_eu_backoff_reservation(&parser->validated);
  287. if (parser->relocs != NULL) {
  288. for (i = 0; i < parser->nrelocs; i++) {
  289. if (parser->relocs[i].gobj)
  290. drm_gem_object_unreference_unlocked(parser->relocs[i].gobj);
  291. }
  292. }
  293. kfree(parser->track);
  294. kfree(parser->relocs);
  295. kfree(parser->relocs_ptr);
  296. for (i = 0; i < parser->nchunks; i++) {
  297. kfree(parser->chunks[i].kdata);
  298. if ((parser->rdev->flags & RADEON_IS_AGP)) {
  299. kfree(parser->chunks[i].kpage[0]);
  300. kfree(parser->chunks[i].kpage[1]);
  301. }
  302. }
  303. kfree(parser->chunks);
  304. kfree(parser->chunks_array);
  305. radeon_ib_free(parser->rdev, &parser->ib);
  306. radeon_ib_free(parser->rdev, &parser->const_ib);
  307. }
  308. static int radeon_cs_ib_chunk(struct radeon_device *rdev,
  309. struct radeon_cs_parser *parser)
  310. {
  311. struct radeon_cs_chunk *ib_chunk;
  312. int r;
  313. if (parser->chunk_ib_idx == -1)
  314. return 0;
  315. if (parser->cs_flags & RADEON_CS_USE_VM)
  316. return 0;
  317. ib_chunk = &parser->chunks[parser->chunk_ib_idx];
  318. /* Copy the packet into the IB, the parser will read from the
  319. * input memory (cached) and write to the IB (which can be
  320. * uncached).
  321. */
  322. r = radeon_ib_get(rdev, parser->ring, &parser->ib,
  323. ib_chunk->length_dw * 4);
  324. if (r) {
  325. DRM_ERROR("Failed to get ib !\n");
  326. return r;
  327. }
  328. parser->ib.length_dw = ib_chunk->length_dw;
  329. r = radeon_cs_parse(rdev, parser->ring, parser);
  330. if (r || parser->parser_error) {
  331. DRM_ERROR("Invalid command stream !\n");
  332. return r;
  333. }
  334. r = radeon_cs_finish_pages(parser);
  335. if (r) {
  336. DRM_ERROR("Invalid command stream !\n");
  337. return r;
  338. }
  339. r = radeon_cs_sync_rings(parser);
  340. if (r) {
  341. DRM_ERROR("Failed to synchronize rings !\n");
  342. }
  343. parser->ib.vm_id = 0;
  344. r = radeon_ib_schedule(rdev, &parser->ib);
  345. if (r) {
  346. DRM_ERROR("Failed to schedule IB !\n");
  347. }
  348. return 0;
  349. }
  350. static int radeon_bo_vm_update_pte(struct radeon_cs_parser *parser,
  351. struct radeon_vm *vm)
  352. {
  353. struct radeon_bo_list *lobj;
  354. struct radeon_bo *bo;
  355. int r;
  356. list_for_each_entry(lobj, &parser->validated, tv.head) {
  357. bo = lobj->bo;
  358. r = radeon_vm_bo_update_pte(parser->rdev, vm, bo, &bo->tbo.mem);
  359. if (r) {
  360. return r;
  361. }
  362. }
  363. return 0;
  364. }
  365. static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev,
  366. struct radeon_cs_parser *parser)
  367. {
  368. struct radeon_cs_chunk *ib_chunk;
  369. struct radeon_fpriv *fpriv = parser->filp->driver_priv;
  370. struct radeon_vm *vm = &fpriv->vm;
  371. int r;
  372. if (parser->chunk_ib_idx == -1)
  373. return 0;
  374. if ((parser->cs_flags & RADEON_CS_USE_VM) == 0)
  375. return 0;
  376. if ((rdev->family >= CHIP_TAHITI) &&
  377. (parser->chunk_const_ib_idx != -1)) {
  378. ib_chunk = &parser->chunks[parser->chunk_const_ib_idx];
  379. if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) {
  380. DRM_ERROR("cs IB CONST too big: %d\n", ib_chunk->length_dw);
  381. return -EINVAL;
  382. }
  383. r = radeon_ib_get(rdev, parser->ring, &parser->const_ib,
  384. ib_chunk->length_dw * 4);
  385. if (r) {
  386. DRM_ERROR("Failed to get const ib !\n");
  387. return r;
  388. }
  389. parser->const_ib.is_const_ib = true;
  390. parser->const_ib.length_dw = ib_chunk->length_dw;
  391. /* Copy the packet into the IB */
  392. if (DRM_COPY_FROM_USER(parser->const_ib.ptr, ib_chunk->user_ptr,
  393. ib_chunk->length_dw * 4)) {
  394. return -EFAULT;
  395. }
  396. r = radeon_ring_ib_parse(rdev, parser->ring, &parser->const_ib);
  397. if (r) {
  398. return r;
  399. }
  400. }
  401. ib_chunk = &parser->chunks[parser->chunk_ib_idx];
  402. if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) {
  403. DRM_ERROR("cs IB too big: %d\n", ib_chunk->length_dw);
  404. return -EINVAL;
  405. }
  406. r = radeon_ib_get(rdev, parser->ring, &parser->ib,
  407. ib_chunk->length_dw * 4);
  408. if (r) {
  409. DRM_ERROR("Failed to get ib !\n");
  410. return r;
  411. }
  412. parser->ib.length_dw = ib_chunk->length_dw;
  413. /* Copy the packet into the IB */
  414. if (DRM_COPY_FROM_USER(parser->ib.ptr, ib_chunk->user_ptr,
  415. ib_chunk->length_dw * 4)) {
  416. return -EFAULT;
  417. }
  418. r = radeon_ring_ib_parse(rdev, parser->ring, &parser->ib);
  419. if (r) {
  420. return r;
  421. }
  422. mutex_lock(&vm->mutex);
  423. r = radeon_vm_bind(rdev, vm);
  424. if (r) {
  425. goto out;
  426. }
  427. r = radeon_bo_vm_update_pte(parser, vm);
  428. if (r) {
  429. goto out;
  430. }
  431. r = radeon_cs_sync_rings(parser);
  432. if (r) {
  433. DRM_ERROR("Failed to synchronize rings !\n");
  434. }
  435. if ((rdev->family >= CHIP_TAHITI) &&
  436. (parser->chunk_const_ib_idx != -1)) {
  437. parser->const_ib.vm_id = vm->id;
  438. /* ib pool is bind at 0 in virtual address space to gpu_addr is the
  439. * offset inside the pool bo
  440. */
  441. parser->const_ib.gpu_addr = parser->const_ib.sa_bo->soffset;
  442. r = radeon_ib_schedule(rdev, &parser->const_ib);
  443. if (r)
  444. goto out;
  445. }
  446. parser->ib.vm_id = vm->id;
  447. /* ib pool is bind at 0 in virtual address space to gpu_addr is the
  448. * offset inside the pool bo
  449. */
  450. parser->ib.gpu_addr = parser->ib.sa_bo->soffset;
  451. parser->ib.is_const_ib = false;
  452. r = radeon_ib_schedule(rdev, &parser->ib);
  453. out:
  454. if (!r) {
  455. if (vm->fence) {
  456. radeon_fence_unref(&vm->fence);
  457. }
  458. vm->fence = radeon_fence_ref(parser->ib.fence);
  459. }
  460. mutex_unlock(&fpriv->vm.mutex);
  461. return r;
  462. }
  463. static int radeon_cs_handle_lockup(struct radeon_device *rdev, int r)
  464. {
  465. if (r == -EDEADLK) {
  466. r = radeon_gpu_reset(rdev);
  467. if (!r)
  468. r = -EAGAIN;
  469. }
  470. return r;
  471. }
  472. int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
  473. {
  474. struct radeon_device *rdev = dev->dev_private;
  475. struct radeon_cs_parser parser;
  476. int r;
  477. radeon_mutex_lock(&rdev->cs_mutex);
  478. if (!rdev->accel_working) {
  479. radeon_mutex_unlock(&rdev->cs_mutex);
  480. return -EBUSY;
  481. }
  482. /* initialize parser */
  483. memset(&parser, 0, sizeof(struct radeon_cs_parser));
  484. parser.filp = filp;
  485. parser.rdev = rdev;
  486. parser.dev = rdev->dev;
  487. parser.family = rdev->family;
  488. r = radeon_cs_parser_init(&parser, data);
  489. if (r) {
  490. DRM_ERROR("Failed to initialize parser !\n");
  491. radeon_cs_parser_fini(&parser, r);
  492. r = radeon_cs_handle_lockup(rdev, r);
  493. radeon_mutex_unlock(&rdev->cs_mutex);
  494. return r;
  495. }
  496. r = radeon_cs_parser_relocs(&parser);
  497. if (r) {
  498. if (r != -ERESTARTSYS)
  499. DRM_ERROR("Failed to parse relocation %d!\n", r);
  500. radeon_cs_parser_fini(&parser, r);
  501. r = radeon_cs_handle_lockup(rdev, r);
  502. radeon_mutex_unlock(&rdev->cs_mutex);
  503. return r;
  504. }
  505. r = radeon_cs_ib_chunk(rdev, &parser);
  506. if (r) {
  507. goto out;
  508. }
  509. r = radeon_cs_ib_vm_chunk(rdev, &parser);
  510. if (r) {
  511. goto out;
  512. }
  513. out:
  514. radeon_cs_parser_fini(&parser, r);
  515. r = radeon_cs_handle_lockup(rdev, r);
  516. radeon_mutex_unlock(&rdev->cs_mutex);
  517. return r;
  518. }
  519. int radeon_cs_finish_pages(struct radeon_cs_parser *p)
  520. {
  521. struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx];
  522. int i;
  523. int size = PAGE_SIZE;
  524. for (i = ibc->last_copied_page + 1; i <= ibc->last_page_index; i++) {
  525. if (i == ibc->last_page_index) {
  526. size = (ibc->length_dw * 4) % PAGE_SIZE;
  527. if (size == 0)
  528. size = PAGE_SIZE;
  529. }
  530. if (DRM_COPY_FROM_USER(p->ib.ptr + (i * (PAGE_SIZE/4)),
  531. ibc->user_ptr + (i * PAGE_SIZE),
  532. size))
  533. return -EFAULT;
  534. }
  535. return 0;
  536. }
  537. static int radeon_cs_update_pages(struct radeon_cs_parser *p, int pg_idx)
  538. {
  539. int new_page;
  540. struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx];
  541. int i;
  542. int size = PAGE_SIZE;
  543. bool copy1 = (p->rdev->flags & RADEON_IS_AGP) ? false : true;
  544. for (i = ibc->last_copied_page + 1; i < pg_idx; i++) {
  545. if (DRM_COPY_FROM_USER(p->ib.ptr + (i * (PAGE_SIZE/4)),
  546. ibc->user_ptr + (i * PAGE_SIZE),
  547. PAGE_SIZE)) {
  548. p->parser_error = -EFAULT;
  549. return 0;
  550. }
  551. }
  552. if (pg_idx == ibc->last_page_index) {
  553. size = (ibc->length_dw * 4) % PAGE_SIZE;
  554. if (size == 0)
  555. size = PAGE_SIZE;
  556. }
  557. new_page = ibc->kpage_idx[0] < ibc->kpage_idx[1] ? 0 : 1;
  558. if (copy1)
  559. ibc->kpage[new_page] = p->ib.ptr + (pg_idx * (PAGE_SIZE / 4));
  560. if (DRM_COPY_FROM_USER(ibc->kpage[new_page],
  561. ibc->user_ptr + (pg_idx * PAGE_SIZE),
  562. size)) {
  563. p->parser_error = -EFAULT;
  564. return 0;
  565. }
  566. /* copy to IB for non single case */
  567. if (!copy1)
  568. memcpy((void *)(p->ib.ptr+(pg_idx*(PAGE_SIZE/4))), ibc->kpage[new_page], size);
  569. ibc->last_copied_page = pg_idx;
  570. ibc->kpage_idx[new_page] = pg_idx;
  571. return new_page;
  572. }
  573. u32 radeon_get_ib_value(struct radeon_cs_parser *p, int idx)
  574. {
  575. struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx];
  576. u32 pg_idx, pg_offset;
  577. u32 idx_value = 0;
  578. int new_page;
  579. pg_idx = (idx * 4) / PAGE_SIZE;
  580. pg_offset = (idx * 4) % PAGE_SIZE;
  581. if (ibc->kpage_idx[0] == pg_idx)
  582. return ibc->kpage[0][pg_offset/4];
  583. if (ibc->kpage_idx[1] == pg_idx)
  584. return ibc->kpage[1][pg_offset/4];
  585. new_page = radeon_cs_update_pages(p, pg_idx);
  586. if (new_page < 0) {
  587. p->parser_error = new_page;
  588. return 0;
  589. }
  590. idx_value = ibc->kpage[new_page][pg_offset/4];
  591. return idx_value;
  592. }