radeon_cs.c 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658
  1. /*
  2. * Copyright 2008 Jerome Glisse.
  3. * All Rights Reserved.
  4. *
  5. * Permission is hereby granted, free of charge, to any person obtaining a
  6. * copy of this software and associated documentation files (the "Software"),
  7. * to deal in the Software without restriction, including without limitation
  8. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  9. * and/or sell copies of the Software, and to permit persons to whom the
  10. * Software is furnished to do so, subject to the following conditions:
  11. *
  12. * The above copyright notice and this permission notice (including the next
  13. * paragraph) shall be included in all copies or substantial portions of the
  14. * Software.
  15. *
  16. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19. * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20. * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21. * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  22. * DEALINGS IN THE SOFTWARE.
  23. *
  24. * Authors:
  25. * Jerome Glisse <glisse@freedesktop.org>
  26. */
  27. #include "drmP.h"
  28. #include "radeon_drm.h"
  29. #include "radeon_reg.h"
  30. #include "radeon.h"
  31. void r100_cs_dump_packet(struct radeon_cs_parser *p,
  32. struct radeon_cs_packet *pkt);
  33. int radeon_cs_parser_relocs(struct radeon_cs_parser *p)
  34. {
  35. struct drm_device *ddev = p->rdev->ddev;
  36. struct radeon_cs_chunk *chunk;
  37. unsigned i, j;
  38. bool duplicate;
  39. if (p->chunk_relocs_idx == -1) {
  40. return 0;
  41. }
  42. chunk = &p->chunks[p->chunk_relocs_idx];
  43. /* FIXME: we assume that each relocs use 4 dwords */
  44. p->nrelocs = chunk->length_dw / 4;
  45. p->relocs_ptr = kcalloc(p->nrelocs, sizeof(void *), GFP_KERNEL);
  46. if (p->relocs_ptr == NULL) {
  47. return -ENOMEM;
  48. }
  49. p->relocs = kcalloc(p->nrelocs, sizeof(struct radeon_cs_reloc), GFP_KERNEL);
  50. if (p->relocs == NULL) {
  51. return -ENOMEM;
  52. }
  53. for (i = 0; i < p->nrelocs; i++) {
  54. struct drm_radeon_cs_reloc *r;
  55. duplicate = false;
  56. r = (struct drm_radeon_cs_reloc *)&chunk->kdata[i*4];
  57. for (j = 0; j < i; j++) {
  58. if (r->handle == p->relocs[j].handle) {
  59. p->relocs_ptr[i] = &p->relocs[j];
  60. duplicate = true;
  61. break;
  62. }
  63. }
  64. if (!duplicate) {
  65. p->relocs[i].gobj = drm_gem_object_lookup(ddev,
  66. p->filp,
  67. r->handle);
  68. if (p->relocs[i].gobj == NULL) {
  69. DRM_ERROR("gem object lookup failed 0x%x\n",
  70. r->handle);
  71. return -ENOENT;
  72. }
  73. p->relocs_ptr[i] = &p->relocs[i];
  74. p->relocs[i].robj = gem_to_radeon_bo(p->relocs[i].gobj);
  75. p->relocs[i].lobj.bo = p->relocs[i].robj;
  76. p->relocs[i].lobj.wdomain = r->write_domain;
  77. p->relocs[i].lobj.rdomain = r->read_domains;
  78. p->relocs[i].lobj.tv.bo = &p->relocs[i].robj->tbo;
  79. p->relocs[i].handle = r->handle;
  80. p->relocs[i].flags = r->flags;
  81. radeon_bo_list_add_object(&p->relocs[i].lobj,
  82. &p->validated);
  83. } else
  84. p->relocs[i].handle = 0;
  85. }
  86. return radeon_bo_list_validate(&p->validated);
  87. }
  88. static int radeon_cs_get_ring(struct radeon_cs_parser *p, u32 ring, s32 priority)
  89. {
  90. p->priority = priority;
  91. switch (ring) {
  92. default:
  93. DRM_ERROR("unknown ring id: %d\n", ring);
  94. return -EINVAL;
  95. case RADEON_CS_RING_GFX:
  96. p->ring = RADEON_RING_TYPE_GFX_INDEX;
  97. break;
  98. case RADEON_CS_RING_COMPUTE:
  99. if (p->rdev->family >= CHIP_TAHITI) {
  100. if (p->priority > 0)
  101. p->ring = CAYMAN_RING_TYPE_CP1_INDEX;
  102. else
  103. p->ring = CAYMAN_RING_TYPE_CP2_INDEX;
  104. } else
  105. p->ring = RADEON_RING_TYPE_GFX_INDEX;
  106. break;
  107. }
  108. return 0;
  109. }
  110. static void radeon_cs_sync_rings(struct radeon_cs_parser *p)
  111. {
  112. int i;
  113. for (i = 0; i < p->nrelocs; i++) {
  114. struct radeon_fence *a, *b;
  115. if (!p->relocs[i].robj || !p->relocs[i].robj->tbo.sync_obj)
  116. continue;
  117. a = p->relocs[i].robj->tbo.sync_obj;
  118. b = p->ib.sync_to[a->ring];
  119. p->ib.sync_to[a->ring] = radeon_fence_later(a, b);
  120. }
  121. }
  122. /* XXX: note that this is called from the legacy UMS CS ioctl as well */
  123. int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data)
  124. {
  125. struct drm_radeon_cs *cs = data;
  126. uint64_t *chunk_array_ptr;
  127. unsigned size, i;
  128. u32 ring = RADEON_CS_RING_GFX;
  129. s32 priority = 0;
  130. if (!cs->num_chunks) {
  131. return 0;
  132. }
  133. /* get chunks */
  134. INIT_LIST_HEAD(&p->validated);
  135. p->idx = 0;
  136. p->ib.sa_bo = NULL;
  137. p->ib.semaphore = NULL;
  138. p->const_ib.sa_bo = NULL;
  139. p->const_ib.semaphore = NULL;
  140. p->chunk_ib_idx = -1;
  141. p->chunk_relocs_idx = -1;
  142. p->chunk_flags_idx = -1;
  143. p->chunk_const_ib_idx = -1;
  144. p->chunks_array = kcalloc(cs->num_chunks, sizeof(uint64_t), GFP_KERNEL);
  145. if (p->chunks_array == NULL) {
  146. return -ENOMEM;
  147. }
  148. chunk_array_ptr = (uint64_t *)(unsigned long)(cs->chunks);
  149. if (DRM_COPY_FROM_USER(p->chunks_array, chunk_array_ptr,
  150. sizeof(uint64_t)*cs->num_chunks)) {
  151. return -EFAULT;
  152. }
  153. p->cs_flags = 0;
  154. p->nchunks = cs->num_chunks;
  155. p->chunks = kcalloc(p->nchunks, sizeof(struct radeon_cs_chunk), GFP_KERNEL);
  156. if (p->chunks == NULL) {
  157. return -ENOMEM;
  158. }
  159. for (i = 0; i < p->nchunks; i++) {
  160. struct drm_radeon_cs_chunk __user **chunk_ptr = NULL;
  161. struct drm_radeon_cs_chunk user_chunk;
  162. uint32_t __user *cdata;
  163. chunk_ptr = (void __user*)(unsigned long)p->chunks_array[i];
  164. if (DRM_COPY_FROM_USER(&user_chunk, chunk_ptr,
  165. sizeof(struct drm_radeon_cs_chunk))) {
  166. return -EFAULT;
  167. }
  168. p->chunks[i].length_dw = user_chunk.length_dw;
  169. p->chunks[i].kdata = NULL;
  170. p->chunks[i].chunk_id = user_chunk.chunk_id;
  171. if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_RELOCS) {
  172. p->chunk_relocs_idx = i;
  173. }
  174. if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_IB) {
  175. p->chunk_ib_idx = i;
  176. /* zero length IB isn't useful */
  177. if (p->chunks[i].length_dw == 0)
  178. return -EINVAL;
  179. }
  180. if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_CONST_IB) {
  181. p->chunk_const_ib_idx = i;
  182. /* zero length CONST IB isn't useful */
  183. if (p->chunks[i].length_dw == 0)
  184. return -EINVAL;
  185. }
  186. if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS) {
  187. p->chunk_flags_idx = i;
  188. /* zero length flags aren't useful */
  189. if (p->chunks[i].length_dw == 0)
  190. return -EINVAL;
  191. }
  192. p->chunks[i].length_dw = user_chunk.length_dw;
  193. p->chunks[i].user_ptr = (void __user *)(unsigned long)user_chunk.chunk_data;
  194. cdata = (uint32_t *)(unsigned long)user_chunk.chunk_data;
  195. if ((p->chunks[i].chunk_id == RADEON_CHUNK_ID_RELOCS) ||
  196. (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS)) {
  197. size = p->chunks[i].length_dw * sizeof(uint32_t);
  198. p->chunks[i].kdata = kmalloc(size, GFP_KERNEL);
  199. if (p->chunks[i].kdata == NULL) {
  200. return -ENOMEM;
  201. }
  202. if (DRM_COPY_FROM_USER(p->chunks[i].kdata,
  203. p->chunks[i].user_ptr, size)) {
  204. return -EFAULT;
  205. }
  206. if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS) {
  207. p->cs_flags = p->chunks[i].kdata[0];
  208. if (p->chunks[i].length_dw > 1)
  209. ring = p->chunks[i].kdata[1];
  210. if (p->chunks[i].length_dw > 2)
  211. priority = (s32)p->chunks[i].kdata[2];
  212. }
  213. }
  214. }
  215. /* these are KMS only */
  216. if (p->rdev) {
  217. if ((p->cs_flags & RADEON_CS_USE_VM) &&
  218. !p->rdev->vm_manager.enabled) {
  219. DRM_ERROR("VM not active on asic!\n");
  220. return -EINVAL;
  221. }
  222. /* we only support VM on SI+ */
  223. if ((p->rdev->family >= CHIP_TAHITI) &&
  224. ((p->cs_flags & RADEON_CS_USE_VM) == 0)) {
  225. DRM_ERROR("VM required on SI+!\n");
  226. return -EINVAL;
  227. }
  228. if (radeon_cs_get_ring(p, ring, priority))
  229. return -EINVAL;
  230. }
  231. /* deal with non-vm */
  232. if ((p->chunk_ib_idx != -1) &&
  233. ((p->cs_flags & RADEON_CS_USE_VM) == 0) &&
  234. (p->chunks[p->chunk_ib_idx].chunk_id == RADEON_CHUNK_ID_IB)) {
  235. if (p->chunks[p->chunk_ib_idx].length_dw > (16 * 1024)) {
  236. DRM_ERROR("cs IB too big: %d\n",
  237. p->chunks[p->chunk_ib_idx].length_dw);
  238. return -EINVAL;
  239. }
  240. if ((p->rdev->flags & RADEON_IS_AGP)) {
  241. p->chunks[p->chunk_ib_idx].kpage[0] = kmalloc(PAGE_SIZE, GFP_KERNEL);
  242. p->chunks[p->chunk_ib_idx].kpage[1] = kmalloc(PAGE_SIZE, GFP_KERNEL);
  243. if (p->chunks[p->chunk_ib_idx].kpage[0] == NULL ||
  244. p->chunks[p->chunk_ib_idx].kpage[1] == NULL) {
  245. kfree(p->chunks[i].kpage[0]);
  246. kfree(p->chunks[i].kpage[1]);
  247. return -ENOMEM;
  248. }
  249. }
  250. p->chunks[p->chunk_ib_idx].kpage_idx[0] = -1;
  251. p->chunks[p->chunk_ib_idx].kpage_idx[1] = -1;
  252. p->chunks[p->chunk_ib_idx].last_copied_page = -1;
  253. p->chunks[p->chunk_ib_idx].last_page_index =
  254. ((p->chunks[p->chunk_ib_idx].length_dw * 4) - 1) / PAGE_SIZE;
  255. }
  256. return 0;
  257. }
  258. static void radeon_bo_vm_fence_va(struct radeon_cs_parser *parser,
  259. struct radeon_fence *fence)
  260. {
  261. struct radeon_fpriv *fpriv = parser->filp->driver_priv;
  262. struct radeon_vm *vm = &fpriv->vm;
  263. struct radeon_bo_list *lobj;
  264. if (parser->chunk_ib_idx == -1) {
  265. return;
  266. }
  267. if ((parser->cs_flags & RADEON_CS_USE_VM) == 0) {
  268. return;
  269. }
  270. list_for_each_entry(lobj, &parser->validated, tv.head) {
  271. struct radeon_bo_va *bo_va;
  272. struct radeon_bo *rbo = lobj->bo;
  273. bo_va = radeon_bo_va(rbo, vm);
  274. radeon_fence_unref(&bo_va->fence);
  275. bo_va->fence = radeon_fence_ref(fence);
  276. }
  277. }
  278. /**
  279. * cs_parser_fini() - clean parser states
  280. * @parser: parser structure holding parsing context.
  281. * @error: error number
  282. *
  283. * If error is set than unvalidate buffer, otherwise just free memory
  284. * used by parsing context.
  285. **/
  286. static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error)
  287. {
  288. unsigned i;
  289. if (!error) {
  290. /* fence all bo va before ttm_eu_fence_buffer_objects so bo are still reserved */
  291. radeon_bo_vm_fence_va(parser, parser->ib.fence);
  292. ttm_eu_fence_buffer_objects(&parser->validated,
  293. parser->ib.fence);
  294. } else {
  295. ttm_eu_backoff_reservation(&parser->validated);
  296. }
  297. if (parser->relocs != NULL) {
  298. for (i = 0; i < parser->nrelocs; i++) {
  299. if (parser->relocs[i].gobj)
  300. drm_gem_object_unreference_unlocked(parser->relocs[i].gobj);
  301. }
  302. }
  303. kfree(parser->track);
  304. kfree(parser->relocs);
  305. kfree(parser->relocs_ptr);
  306. for (i = 0; i < parser->nchunks; i++) {
  307. kfree(parser->chunks[i].kdata);
  308. if ((parser->rdev->flags & RADEON_IS_AGP)) {
  309. kfree(parser->chunks[i].kpage[0]);
  310. kfree(parser->chunks[i].kpage[1]);
  311. }
  312. }
  313. kfree(parser->chunks);
  314. kfree(parser->chunks_array);
  315. radeon_ib_free(parser->rdev, &parser->ib);
  316. radeon_ib_free(parser->rdev, &parser->const_ib);
  317. }
  318. static int radeon_cs_ib_chunk(struct radeon_device *rdev,
  319. struct radeon_cs_parser *parser)
  320. {
  321. struct radeon_cs_chunk *ib_chunk;
  322. int r;
  323. if (parser->chunk_ib_idx == -1)
  324. return 0;
  325. if (parser->cs_flags & RADEON_CS_USE_VM)
  326. return 0;
  327. ib_chunk = &parser->chunks[parser->chunk_ib_idx];
  328. /* Copy the packet into the IB, the parser will read from the
  329. * input memory (cached) and write to the IB (which can be
  330. * uncached).
  331. */
  332. r = radeon_ib_get(rdev, parser->ring, &parser->ib,
  333. ib_chunk->length_dw * 4);
  334. if (r) {
  335. DRM_ERROR("Failed to get ib !\n");
  336. return r;
  337. }
  338. parser->ib.length_dw = ib_chunk->length_dw;
  339. r = radeon_cs_parse(rdev, parser->ring, parser);
  340. if (r || parser->parser_error) {
  341. DRM_ERROR("Invalid command stream !\n");
  342. return r;
  343. }
  344. r = radeon_cs_finish_pages(parser);
  345. if (r) {
  346. DRM_ERROR("Invalid command stream !\n");
  347. return r;
  348. }
  349. radeon_cs_sync_rings(parser);
  350. parser->ib.vm_id = 0;
  351. r = radeon_ib_schedule(rdev, &parser->ib, NULL);
  352. if (r) {
  353. DRM_ERROR("Failed to schedule IB !\n");
  354. }
  355. return r;
  356. }
  357. static int radeon_bo_vm_update_pte(struct radeon_cs_parser *parser,
  358. struct radeon_vm *vm)
  359. {
  360. struct radeon_bo_list *lobj;
  361. struct radeon_bo *bo;
  362. int r;
  363. list_for_each_entry(lobj, &parser->validated, tv.head) {
  364. bo = lobj->bo;
  365. r = radeon_vm_bo_update_pte(parser->rdev, vm, bo, &bo->tbo.mem);
  366. if (r) {
  367. return r;
  368. }
  369. }
  370. return 0;
  371. }
  372. static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev,
  373. struct radeon_cs_parser *parser)
  374. {
  375. struct radeon_cs_chunk *ib_chunk;
  376. struct radeon_fpriv *fpriv = parser->filp->driver_priv;
  377. struct radeon_vm *vm = &fpriv->vm;
  378. int r;
  379. if (parser->chunk_ib_idx == -1)
  380. return 0;
  381. if ((parser->cs_flags & RADEON_CS_USE_VM) == 0)
  382. return 0;
  383. if ((rdev->family >= CHIP_TAHITI) &&
  384. (parser->chunk_const_ib_idx != -1)) {
  385. ib_chunk = &parser->chunks[parser->chunk_const_ib_idx];
  386. if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) {
  387. DRM_ERROR("cs IB CONST too big: %d\n", ib_chunk->length_dw);
  388. return -EINVAL;
  389. }
  390. r = radeon_ib_get(rdev, parser->ring, &parser->const_ib,
  391. ib_chunk->length_dw * 4);
  392. if (r) {
  393. DRM_ERROR("Failed to get const ib !\n");
  394. return r;
  395. }
  396. parser->const_ib.is_const_ib = true;
  397. parser->const_ib.length_dw = ib_chunk->length_dw;
  398. /* Copy the packet into the IB */
  399. if (DRM_COPY_FROM_USER(parser->const_ib.ptr, ib_chunk->user_ptr,
  400. ib_chunk->length_dw * 4)) {
  401. return -EFAULT;
  402. }
  403. r = radeon_ring_ib_parse(rdev, parser->ring, &parser->const_ib);
  404. if (r) {
  405. return r;
  406. }
  407. }
  408. ib_chunk = &parser->chunks[parser->chunk_ib_idx];
  409. if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) {
  410. DRM_ERROR("cs IB too big: %d\n", ib_chunk->length_dw);
  411. return -EINVAL;
  412. }
  413. r = radeon_ib_get(rdev, parser->ring, &parser->ib,
  414. ib_chunk->length_dw * 4);
  415. if (r) {
  416. DRM_ERROR("Failed to get ib !\n");
  417. return r;
  418. }
  419. parser->ib.length_dw = ib_chunk->length_dw;
  420. /* Copy the packet into the IB */
  421. if (DRM_COPY_FROM_USER(parser->ib.ptr, ib_chunk->user_ptr,
  422. ib_chunk->length_dw * 4)) {
  423. return -EFAULT;
  424. }
  425. r = radeon_ring_ib_parse(rdev, parser->ring, &parser->ib);
  426. if (r) {
  427. return r;
  428. }
  429. mutex_lock(&rdev->vm_manager.lock);
  430. mutex_lock(&vm->mutex);
  431. r = radeon_vm_bind(rdev, vm);
  432. if (r) {
  433. goto out;
  434. }
  435. r = radeon_bo_vm_update_pte(parser, vm);
  436. if (r) {
  437. goto out;
  438. }
  439. radeon_cs_sync_rings(parser);
  440. parser->ib.vm_id = vm->id;
  441. /* ib pool is bind at 0 in virtual address space,
  442. * so gpu_addr is the offset inside the pool bo
  443. */
  444. parser->ib.gpu_addr = parser->ib.sa_bo->soffset;
  445. if ((rdev->family >= CHIP_TAHITI) &&
  446. (parser->chunk_const_ib_idx != -1)) {
  447. parser->const_ib.vm_id = vm->id;
  448. /* ib pool is bind at 0 in virtual address space,
  449. * so gpu_addr is the offset inside the pool bo
  450. */
  451. parser->const_ib.gpu_addr = parser->const_ib.sa_bo->soffset;
  452. r = radeon_ib_schedule(rdev, &parser->ib, &parser->const_ib);
  453. } else {
  454. r = radeon_ib_schedule(rdev, &parser->ib, NULL);
  455. }
  456. out:
  457. if (!r) {
  458. if (vm->fence) {
  459. radeon_fence_unref(&vm->fence);
  460. }
  461. vm->fence = radeon_fence_ref(parser->ib.fence);
  462. }
  463. mutex_unlock(&vm->mutex);
  464. mutex_unlock(&rdev->vm_manager.lock);
  465. return r;
  466. }
  467. static int radeon_cs_handle_lockup(struct radeon_device *rdev, int r)
  468. {
  469. if (r == -EDEADLK) {
  470. r = radeon_gpu_reset(rdev);
  471. if (!r)
  472. r = -EAGAIN;
  473. }
  474. return r;
  475. }
  476. int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
  477. {
  478. struct radeon_device *rdev = dev->dev_private;
  479. struct radeon_cs_parser parser;
  480. int r;
  481. down_read(&rdev->exclusive_lock);
  482. if (!rdev->accel_working) {
  483. up_read(&rdev->exclusive_lock);
  484. return -EBUSY;
  485. }
  486. /* initialize parser */
  487. memset(&parser, 0, sizeof(struct radeon_cs_parser));
  488. parser.filp = filp;
  489. parser.rdev = rdev;
  490. parser.dev = rdev->dev;
  491. parser.family = rdev->family;
  492. r = radeon_cs_parser_init(&parser, data);
  493. if (r) {
  494. DRM_ERROR("Failed to initialize parser !\n");
  495. radeon_cs_parser_fini(&parser, r);
  496. up_read(&rdev->exclusive_lock);
  497. r = radeon_cs_handle_lockup(rdev, r);
  498. return r;
  499. }
  500. r = radeon_cs_parser_relocs(&parser);
  501. if (r) {
  502. if (r != -ERESTARTSYS)
  503. DRM_ERROR("Failed to parse relocation %d!\n", r);
  504. radeon_cs_parser_fini(&parser, r);
  505. up_read(&rdev->exclusive_lock);
  506. r = radeon_cs_handle_lockup(rdev, r);
  507. return r;
  508. }
  509. r = radeon_cs_ib_chunk(rdev, &parser);
  510. if (r) {
  511. goto out;
  512. }
  513. r = radeon_cs_ib_vm_chunk(rdev, &parser);
  514. if (r) {
  515. goto out;
  516. }
  517. out:
  518. radeon_cs_parser_fini(&parser, r);
  519. up_read(&rdev->exclusive_lock);
  520. r = radeon_cs_handle_lockup(rdev, r);
  521. return r;
  522. }
  523. int radeon_cs_finish_pages(struct radeon_cs_parser *p)
  524. {
  525. struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx];
  526. int i;
  527. int size = PAGE_SIZE;
  528. for (i = ibc->last_copied_page + 1; i <= ibc->last_page_index; i++) {
  529. if (i == ibc->last_page_index) {
  530. size = (ibc->length_dw * 4) % PAGE_SIZE;
  531. if (size == 0)
  532. size = PAGE_SIZE;
  533. }
  534. if (DRM_COPY_FROM_USER(p->ib.ptr + (i * (PAGE_SIZE/4)),
  535. ibc->user_ptr + (i * PAGE_SIZE),
  536. size))
  537. return -EFAULT;
  538. }
  539. return 0;
  540. }
  541. static int radeon_cs_update_pages(struct radeon_cs_parser *p, int pg_idx)
  542. {
  543. int new_page;
  544. struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx];
  545. int i;
  546. int size = PAGE_SIZE;
  547. bool copy1 = (p->rdev->flags & RADEON_IS_AGP) ? false : true;
  548. for (i = ibc->last_copied_page + 1; i < pg_idx; i++) {
  549. if (DRM_COPY_FROM_USER(p->ib.ptr + (i * (PAGE_SIZE/4)),
  550. ibc->user_ptr + (i * PAGE_SIZE),
  551. PAGE_SIZE)) {
  552. p->parser_error = -EFAULT;
  553. return 0;
  554. }
  555. }
  556. if (pg_idx == ibc->last_page_index) {
  557. size = (ibc->length_dw * 4) % PAGE_SIZE;
  558. if (size == 0)
  559. size = PAGE_SIZE;
  560. }
  561. new_page = ibc->kpage_idx[0] < ibc->kpage_idx[1] ? 0 : 1;
  562. if (copy1)
  563. ibc->kpage[new_page] = p->ib.ptr + (pg_idx * (PAGE_SIZE / 4));
  564. if (DRM_COPY_FROM_USER(ibc->kpage[new_page],
  565. ibc->user_ptr + (pg_idx * PAGE_SIZE),
  566. size)) {
  567. p->parser_error = -EFAULT;
  568. return 0;
  569. }
  570. /* copy to IB for non single case */
  571. if (!copy1)
  572. memcpy((void *)(p->ib.ptr+(pg_idx*(PAGE_SIZE/4))), ibc->kpage[new_page], size);
  573. ibc->last_copied_page = pg_idx;
  574. ibc->kpage_idx[new_page] = pg_idx;
  575. return new_page;
  576. }
  577. u32 radeon_get_ib_value(struct radeon_cs_parser *p, int idx)
  578. {
  579. struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx];
  580. u32 pg_idx, pg_offset;
  581. u32 idx_value = 0;
  582. int new_page;
  583. pg_idx = (idx * 4) / PAGE_SIZE;
  584. pg_offset = (idx * 4) % PAGE_SIZE;
  585. if (ibc->kpage_idx[0] == pg_idx)
  586. return ibc->kpage[0][pg_offset/4];
  587. if (ibc->kpage_idx[1] == pg_idx)
  588. return ibc->kpage[1][pg_offset/4];
  589. new_page = radeon_cs_update_pages(p, pg_idx);
  590. if (new_page < 0) {
  591. p->parser_error = new_page;
  592. return 0;
  593. }
  594. idx_value = ibc->kpage[new_page][pg_offset/4];
  595. return idx_value;
  596. }