nouveau_gem.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877
  1. /*
  2. * Copyright (C) 2008 Ben Skeggs.
  3. * All Rights Reserved.
  4. *
  5. * Permission is hereby granted, free of charge, to any person obtaining
  6. * a copy of this software and associated documentation files (the
  7. * "Software"), to deal in the Software without restriction, including
  8. * without limitation the rights to use, copy, modify, merge, publish,
  9. * distribute, sublicense, and/or sell copies of the Software, and to
  10. * permit persons to whom the Software is furnished to do so, subject to
  11. * the following conditions:
  12. *
  13. * The above copyright notice and this permission notice (including the
  14. * next paragraph) shall be included in all copies or substantial
  15. * portions of the Software.
  16. *
  17. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  18. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  19. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  20. * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  21. * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  22. * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  23. * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  24. *
  25. */
  26. #include <linux/dma-buf.h>
  27. #include <subdev/fb.h>
  28. #include "nouveau_drm.h"
  29. #include "nouveau_dma.h"
  30. #include "nouveau_fence.h"
  31. #include "nouveau_abi16.h"
  32. #include "nouveau_ttm.h"
  33. #include "nouveau_gem.h"
  34. int
  35. nouveau_gem_object_new(struct drm_gem_object *gem)
  36. {
  37. return 0;
  38. }
  39. void
  40. nouveau_gem_object_del(struct drm_gem_object *gem)
  41. {
  42. struct nouveau_bo *nvbo = gem->driver_private;
  43. struct ttm_buffer_object *bo = &nvbo->bo;
  44. if (!nvbo)
  45. return;
  46. nvbo->gem = NULL;
  47. if (unlikely(nvbo->pin_refcnt)) {
  48. nvbo->pin_refcnt = 1;
  49. nouveau_bo_unpin(nvbo);
  50. }
  51. if (gem->import_attach)
  52. drm_prime_gem_destroy(gem, nvbo->bo.sg);
  53. ttm_bo_unref(&bo);
  54. drm_gem_object_release(gem);
  55. kfree(gem);
  56. }
  57. int
  58. nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv)
  59. {
  60. struct nouveau_cli *cli = nouveau_cli(file_priv);
  61. struct nouveau_bo *nvbo = nouveau_gem_object(gem);
  62. struct nouveau_vma *vma;
  63. int ret;
  64. if (!cli->base.vm)
  65. return 0;
  66. ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0);
  67. if (ret)
  68. return ret;
  69. vma = nouveau_bo_vma_find(nvbo, cli->base.vm);
  70. if (!vma) {
  71. vma = kzalloc(sizeof(*vma), GFP_KERNEL);
  72. if (!vma) {
  73. ret = -ENOMEM;
  74. goto out;
  75. }
  76. ret = nouveau_bo_vma_add(nvbo, cli->base.vm, vma);
  77. if (ret) {
  78. kfree(vma);
  79. goto out;
  80. }
  81. } else {
  82. vma->refcount++;
  83. }
  84. out:
  85. ttm_bo_unreserve(&nvbo->bo);
  86. return ret;
  87. }
  88. void
  89. nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv)
  90. {
  91. struct nouveau_cli *cli = nouveau_cli(file_priv);
  92. struct nouveau_bo *nvbo = nouveau_gem_object(gem);
  93. struct nouveau_vma *vma;
  94. int ret;
  95. if (!cli->base.vm)
  96. return;
  97. ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0);
  98. if (ret)
  99. return;
  100. vma = nouveau_bo_vma_find(nvbo, cli->base.vm);
  101. if (vma) {
  102. if (--vma->refcount == 0) {
  103. nouveau_bo_vma_del(nvbo, vma);
  104. kfree(vma);
  105. }
  106. }
  107. ttm_bo_unreserve(&nvbo->bo);
  108. }
  109. int
  110. nouveau_gem_new(struct drm_device *dev, int size, int align, uint32_t domain,
  111. uint32_t tile_mode, uint32_t tile_flags,
  112. struct nouveau_bo **pnvbo)
  113. {
  114. struct nouveau_drm *drm = nouveau_drm(dev);
  115. struct nouveau_bo *nvbo;
  116. u32 flags = 0;
  117. int ret;
  118. if (domain & NOUVEAU_GEM_DOMAIN_VRAM)
  119. flags |= TTM_PL_FLAG_VRAM;
  120. if (domain & NOUVEAU_GEM_DOMAIN_GART)
  121. flags |= TTM_PL_FLAG_TT;
  122. if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU)
  123. flags |= TTM_PL_FLAG_SYSTEM;
  124. ret = nouveau_bo_new(dev, size, align, flags, tile_mode,
  125. tile_flags, NULL, pnvbo);
  126. if (ret)
  127. return ret;
  128. nvbo = *pnvbo;
  129. /* we restrict allowed domains on nv50+ to only the types
  130. * that were requested at creation time. not possibly on
  131. * earlier chips without busting the ABI.
  132. */
  133. nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM |
  134. NOUVEAU_GEM_DOMAIN_GART;
  135. if (nv_device(drm->device)->card_type >= NV_50)
  136. nvbo->valid_domains &= domain;
  137. nvbo->gem = drm_gem_object_alloc(dev, nvbo->bo.mem.size);
  138. if (!nvbo->gem) {
  139. nouveau_bo_ref(NULL, pnvbo);
  140. return -ENOMEM;
  141. }
  142. nvbo->bo.persistent_swap_storage = nvbo->gem->filp;
  143. nvbo->gem->driver_private = nvbo;
  144. return 0;
  145. }
  146. static int
  147. nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem,
  148. struct drm_nouveau_gem_info *rep)
  149. {
  150. struct nouveau_cli *cli = nouveau_cli(file_priv);
  151. struct nouveau_bo *nvbo = nouveau_gem_object(gem);
  152. struct nouveau_vma *vma;
  153. if (nvbo->bo.mem.mem_type == TTM_PL_TT)
  154. rep->domain = NOUVEAU_GEM_DOMAIN_GART;
  155. else
  156. rep->domain = NOUVEAU_GEM_DOMAIN_VRAM;
  157. rep->offset = nvbo->bo.offset;
  158. if (cli->base.vm) {
  159. vma = nouveau_bo_vma_find(nvbo, cli->base.vm);
  160. if (!vma)
  161. return -EINVAL;
  162. rep->offset = vma->offset;
  163. }
  164. rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT;
  165. rep->map_handle = nvbo->bo.addr_space_offset;
  166. rep->tile_mode = nvbo->tile_mode;
  167. rep->tile_flags = nvbo->tile_flags;
  168. return 0;
  169. }
  170. int
  171. nouveau_gem_ioctl_new(struct drm_device *dev, void *data,
  172. struct drm_file *file_priv)
  173. {
  174. struct nouveau_drm *drm = nouveau_drm(dev);
  175. struct nouveau_fb *pfb = nouveau_fb(drm->device);
  176. struct drm_nouveau_gem_new *req = data;
  177. struct nouveau_bo *nvbo = NULL;
  178. int ret = 0;
  179. drm->ttm.bdev.dev_mapping = drm->dev->dev_mapping;
  180. if (!pfb->memtype_valid(pfb, req->info.tile_flags)) {
  181. NV_ERROR(drm, "bad page flags: 0x%08x\n", req->info.tile_flags);
  182. return -EINVAL;
  183. }
  184. ret = nouveau_gem_new(dev, req->info.size, req->align,
  185. req->info.domain, req->info.tile_mode,
  186. req->info.tile_flags, &nvbo);
  187. if (ret)
  188. return ret;
  189. ret = drm_gem_handle_create(file_priv, nvbo->gem, &req->info.handle);
  190. if (ret == 0) {
  191. ret = nouveau_gem_info(file_priv, nvbo->gem, &req->info);
  192. if (ret)
  193. drm_gem_handle_delete(file_priv, req->info.handle);
  194. }
  195. /* drop reference from allocate - handle holds it now */
  196. drm_gem_object_unreference_unlocked(nvbo->gem);
  197. return ret;
  198. }
  199. static int
  200. nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains,
  201. uint32_t write_domains, uint32_t valid_domains)
  202. {
  203. struct nouveau_bo *nvbo = gem->driver_private;
  204. struct ttm_buffer_object *bo = &nvbo->bo;
  205. uint32_t domains = valid_domains & nvbo->valid_domains &
  206. (write_domains ? write_domains : read_domains);
  207. uint32_t pref_flags = 0, valid_flags = 0;
  208. if (!domains)
  209. return -EINVAL;
  210. if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM)
  211. valid_flags |= TTM_PL_FLAG_VRAM;
  212. if (valid_domains & NOUVEAU_GEM_DOMAIN_GART)
  213. valid_flags |= TTM_PL_FLAG_TT;
  214. if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
  215. bo->mem.mem_type == TTM_PL_VRAM)
  216. pref_flags |= TTM_PL_FLAG_VRAM;
  217. else if ((domains & NOUVEAU_GEM_DOMAIN_GART) &&
  218. bo->mem.mem_type == TTM_PL_TT)
  219. pref_flags |= TTM_PL_FLAG_TT;
  220. else if (domains & NOUVEAU_GEM_DOMAIN_VRAM)
  221. pref_flags |= TTM_PL_FLAG_VRAM;
  222. else
  223. pref_flags |= TTM_PL_FLAG_TT;
  224. nouveau_bo_placement_set(nvbo, pref_flags, valid_flags);
  225. return 0;
  226. }
  227. struct validate_op {
  228. struct list_head vram_list;
  229. struct list_head gart_list;
  230. struct list_head both_list;
  231. };
  232. static void
  233. validate_fini_list(struct list_head *list, struct nouveau_fence *fence)
  234. {
  235. struct list_head *entry, *tmp;
  236. struct nouveau_bo *nvbo;
  237. list_for_each_safe(entry, tmp, list) {
  238. nvbo = list_entry(entry, struct nouveau_bo, entry);
  239. nouveau_bo_fence(nvbo, fence);
  240. if (unlikely(nvbo->validate_mapped)) {
  241. ttm_bo_kunmap(&nvbo->kmap);
  242. nvbo->validate_mapped = false;
  243. }
  244. list_del(&nvbo->entry);
  245. nvbo->reserved_by = NULL;
  246. ttm_bo_unreserve(&nvbo->bo);
  247. drm_gem_object_unreference_unlocked(nvbo->gem);
  248. }
  249. }
  250. static void
  251. validate_fini(struct validate_op *op, struct nouveau_fence* fence)
  252. {
  253. validate_fini_list(&op->vram_list, fence);
  254. validate_fini_list(&op->gart_list, fence);
  255. validate_fini_list(&op->both_list, fence);
  256. }
  257. static int
  258. validate_init(struct nouveau_channel *chan, struct drm_file *file_priv,
  259. struct drm_nouveau_gem_pushbuf_bo *pbbo,
  260. int nr_buffers, struct validate_op *op)
  261. {
  262. struct drm_device *dev = chan->drm->dev;
  263. struct nouveau_drm *drm = nouveau_drm(dev);
  264. uint32_t sequence;
  265. int trycnt = 0;
  266. int ret, i;
  267. sequence = atomic_add_return(1, &drm->ttm.validate_sequence);
  268. retry:
  269. if (++trycnt > 100000) {
  270. NV_ERROR(drm, "%s failed and gave up.\n", __func__);
  271. return -EINVAL;
  272. }
  273. for (i = 0; i < nr_buffers; i++) {
  274. struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i];
  275. struct drm_gem_object *gem;
  276. struct nouveau_bo *nvbo;
  277. gem = drm_gem_object_lookup(dev, file_priv, b->handle);
  278. if (!gem) {
  279. NV_ERROR(drm, "Unknown handle 0x%08x\n", b->handle);
  280. validate_fini(op, NULL);
  281. return -ENOENT;
  282. }
  283. nvbo = gem->driver_private;
  284. if (nvbo->reserved_by && nvbo->reserved_by == file_priv) {
  285. NV_ERROR(drm, "multiple instances of buffer %d on "
  286. "validation list\n", b->handle);
  287. drm_gem_object_unreference_unlocked(gem);
  288. validate_fini(op, NULL);
  289. return -EINVAL;
  290. }
  291. ret = ttm_bo_reserve(&nvbo->bo, true, false, true, sequence);
  292. if (ret) {
  293. validate_fini(op, NULL);
  294. if (unlikely(ret == -EAGAIN))
  295. ret = ttm_bo_wait_unreserved(&nvbo->bo, true);
  296. drm_gem_object_unreference_unlocked(gem);
  297. if (unlikely(ret)) {
  298. if (ret != -ERESTARTSYS)
  299. NV_ERROR(drm, "fail reserve\n");
  300. return ret;
  301. }
  302. goto retry;
  303. }
  304. b->user_priv = (uint64_t)(unsigned long)nvbo;
  305. nvbo->reserved_by = file_priv;
  306. nvbo->pbbo_index = i;
  307. if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
  308. (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART))
  309. list_add_tail(&nvbo->entry, &op->both_list);
  310. else
  311. if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM)
  312. list_add_tail(&nvbo->entry, &op->vram_list);
  313. else
  314. if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)
  315. list_add_tail(&nvbo->entry, &op->gart_list);
  316. else {
  317. NV_ERROR(drm, "invalid valid domains: 0x%08x\n",
  318. b->valid_domains);
  319. list_add_tail(&nvbo->entry, &op->both_list);
  320. validate_fini(op, NULL);
  321. return -EINVAL;
  322. }
  323. }
  324. return 0;
  325. }
  326. static int
  327. validate_sync(struct nouveau_channel *chan, struct nouveau_bo *nvbo)
  328. {
  329. struct nouveau_fence *fence = NULL;
  330. int ret = 0;
  331. spin_lock(&nvbo->bo.bdev->fence_lock);
  332. if (nvbo->bo.sync_obj)
  333. fence = nouveau_fence_ref(nvbo->bo.sync_obj);
  334. spin_unlock(&nvbo->bo.bdev->fence_lock);
  335. if (fence) {
  336. ret = nouveau_fence_sync(fence, chan);
  337. nouveau_fence_unref(&fence);
  338. }
  339. return ret;
  340. }
  341. static int
  342. validate_list(struct nouveau_channel *chan, struct list_head *list,
  343. struct drm_nouveau_gem_pushbuf_bo *pbbo, uint64_t user_pbbo_ptr)
  344. {
  345. struct nouveau_drm *drm = chan->drm;
  346. struct drm_nouveau_gem_pushbuf_bo __user *upbbo =
  347. (void __force __user *)(uintptr_t)user_pbbo_ptr;
  348. struct nouveau_bo *nvbo;
  349. int ret, relocs = 0;
  350. list_for_each_entry(nvbo, list, entry) {
  351. struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index];
  352. ret = validate_sync(chan, nvbo);
  353. if (unlikely(ret)) {
  354. NV_ERROR(drm, "fail pre-validate sync\n");
  355. return ret;
  356. }
  357. ret = nouveau_gem_set_domain(nvbo->gem, b->read_domains,
  358. b->write_domains,
  359. b->valid_domains);
  360. if (unlikely(ret)) {
  361. NV_ERROR(drm, "fail set_domain\n");
  362. return ret;
  363. }
  364. ret = nouveau_bo_validate(nvbo, true, false);
  365. if (unlikely(ret)) {
  366. if (ret != -ERESTARTSYS)
  367. NV_ERROR(drm, "fail ttm_validate\n");
  368. return ret;
  369. }
  370. ret = validate_sync(chan, nvbo);
  371. if (unlikely(ret)) {
  372. NV_ERROR(drm, "fail post-validate sync\n");
  373. return ret;
  374. }
  375. if (nv_device(drm->device)->card_type < NV_50) {
  376. if (nvbo->bo.offset == b->presumed.offset &&
  377. ((nvbo->bo.mem.mem_type == TTM_PL_VRAM &&
  378. b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) ||
  379. (nvbo->bo.mem.mem_type == TTM_PL_TT &&
  380. b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART)))
  381. continue;
  382. if (nvbo->bo.mem.mem_type == TTM_PL_TT)
  383. b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART;
  384. else
  385. b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM;
  386. b->presumed.offset = nvbo->bo.offset;
  387. b->presumed.valid = 0;
  388. relocs++;
  389. if (DRM_COPY_TO_USER(&upbbo[nvbo->pbbo_index].presumed,
  390. &b->presumed, sizeof(b->presumed)))
  391. return -EFAULT;
  392. }
  393. }
  394. return relocs;
  395. }
  396. static int
  397. nouveau_gem_pushbuf_validate(struct nouveau_channel *chan,
  398. struct drm_file *file_priv,
  399. struct drm_nouveau_gem_pushbuf_bo *pbbo,
  400. uint64_t user_buffers, int nr_buffers,
  401. struct validate_op *op, int *apply_relocs)
  402. {
  403. struct nouveau_drm *drm = chan->drm;
  404. int ret, relocs = 0;
  405. INIT_LIST_HEAD(&op->vram_list);
  406. INIT_LIST_HEAD(&op->gart_list);
  407. INIT_LIST_HEAD(&op->both_list);
  408. if (nr_buffers == 0)
  409. return 0;
  410. ret = validate_init(chan, file_priv, pbbo, nr_buffers, op);
  411. if (unlikely(ret)) {
  412. if (ret != -ERESTARTSYS)
  413. NV_ERROR(drm, "validate_init\n");
  414. return ret;
  415. }
  416. ret = validate_list(chan, &op->vram_list, pbbo, user_buffers);
  417. if (unlikely(ret < 0)) {
  418. if (ret != -ERESTARTSYS)
  419. NV_ERROR(drm, "validate vram_list\n");
  420. validate_fini(op, NULL);
  421. return ret;
  422. }
  423. relocs += ret;
  424. ret = validate_list(chan, &op->gart_list, pbbo, user_buffers);
  425. if (unlikely(ret < 0)) {
  426. if (ret != -ERESTARTSYS)
  427. NV_ERROR(drm, "validate gart_list\n");
  428. validate_fini(op, NULL);
  429. return ret;
  430. }
  431. relocs += ret;
  432. ret = validate_list(chan, &op->both_list, pbbo, user_buffers);
  433. if (unlikely(ret < 0)) {
  434. if (ret != -ERESTARTSYS)
  435. NV_ERROR(drm, "validate both_list\n");
  436. validate_fini(op, NULL);
  437. return ret;
  438. }
  439. relocs += ret;
  440. *apply_relocs = relocs;
  441. return 0;
  442. }
  443. static inline void *
  444. u_memcpya(uint64_t user, unsigned nmemb, unsigned size)
  445. {
  446. void *mem;
  447. void __user *userptr = (void __force __user *)(uintptr_t)user;
  448. mem = kmalloc(nmemb * size, GFP_KERNEL);
  449. if (!mem)
  450. return ERR_PTR(-ENOMEM);
  451. if (DRM_COPY_FROM_USER(mem, userptr, nmemb * size)) {
  452. kfree(mem);
  453. return ERR_PTR(-EFAULT);
  454. }
  455. return mem;
  456. }
  457. static int
  458. nouveau_gem_pushbuf_reloc_apply(struct drm_device *dev,
  459. struct drm_nouveau_gem_pushbuf *req,
  460. struct drm_nouveau_gem_pushbuf_bo *bo)
  461. {
  462. struct nouveau_drm *drm = nouveau_drm(dev);
  463. struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL;
  464. int ret = 0;
  465. unsigned i;
  466. reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc));
  467. if (IS_ERR(reloc))
  468. return PTR_ERR(reloc);
  469. for (i = 0; i < req->nr_relocs; i++) {
  470. struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i];
  471. struct drm_nouveau_gem_pushbuf_bo *b;
  472. struct nouveau_bo *nvbo;
  473. uint32_t data;
  474. if (unlikely(r->bo_index > req->nr_buffers)) {
  475. NV_ERROR(drm, "reloc bo index invalid\n");
  476. ret = -EINVAL;
  477. break;
  478. }
  479. b = &bo[r->bo_index];
  480. if (b->presumed.valid)
  481. continue;
  482. if (unlikely(r->reloc_bo_index > req->nr_buffers)) {
  483. NV_ERROR(drm, "reloc container bo index invalid\n");
  484. ret = -EINVAL;
  485. break;
  486. }
  487. nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv;
  488. if (unlikely(r->reloc_bo_offset + 4 >
  489. nvbo->bo.mem.num_pages << PAGE_SHIFT)) {
  490. NV_ERROR(drm, "reloc outside of bo\n");
  491. ret = -EINVAL;
  492. break;
  493. }
  494. if (!nvbo->kmap.virtual) {
  495. ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages,
  496. &nvbo->kmap);
  497. if (ret) {
  498. NV_ERROR(drm, "failed kmap for reloc\n");
  499. break;
  500. }
  501. nvbo->validate_mapped = true;
  502. }
  503. if (r->flags & NOUVEAU_GEM_RELOC_LOW)
  504. data = b->presumed.offset + r->data;
  505. else
  506. if (r->flags & NOUVEAU_GEM_RELOC_HIGH)
  507. data = (b->presumed.offset + r->data) >> 32;
  508. else
  509. data = r->data;
  510. if (r->flags & NOUVEAU_GEM_RELOC_OR) {
  511. if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART)
  512. data |= r->tor;
  513. else
  514. data |= r->vor;
  515. }
  516. spin_lock(&nvbo->bo.bdev->fence_lock);
  517. ret = ttm_bo_wait(&nvbo->bo, false, false, false);
  518. spin_unlock(&nvbo->bo.bdev->fence_lock);
  519. if (ret) {
  520. NV_ERROR(drm, "reloc wait_idle failed: %d\n", ret);
  521. break;
  522. }
  523. nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data);
  524. }
  525. kfree(reloc);
  526. return ret;
  527. }
  528. int
  529. nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
  530. struct drm_file *file_priv)
  531. {
  532. struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev);
  533. struct nouveau_abi16_chan *temp;
  534. struct nouveau_drm *drm = nouveau_drm(dev);
  535. struct drm_nouveau_gem_pushbuf *req = data;
  536. struct drm_nouveau_gem_pushbuf_push *push;
  537. struct drm_nouveau_gem_pushbuf_bo *bo;
  538. struct nouveau_channel *chan = NULL;
  539. struct validate_op op;
  540. struct nouveau_fence *fence = NULL;
  541. int i, j, ret = 0, do_reloc = 0;
  542. if (unlikely(!abi16))
  543. return -ENOMEM;
  544. list_for_each_entry(temp, &abi16->channels, head) {
  545. if (temp->chan->handle == (NVDRM_CHAN | req->channel)) {
  546. chan = temp->chan;
  547. break;
  548. }
  549. }
  550. if (!chan)
  551. return nouveau_abi16_put(abi16, -ENOENT);
  552. req->vram_available = drm->gem.vram_available;
  553. req->gart_available = drm->gem.gart_available;
  554. if (unlikely(req->nr_push == 0))
  555. goto out_next;
  556. if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) {
  557. NV_ERROR(drm, "pushbuf push count exceeds limit: %d max %d\n",
  558. req->nr_push, NOUVEAU_GEM_MAX_PUSH);
  559. return nouveau_abi16_put(abi16, -EINVAL);
  560. }
  561. if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) {
  562. NV_ERROR(drm, "pushbuf bo count exceeds limit: %d max %d\n",
  563. req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS);
  564. return nouveau_abi16_put(abi16, -EINVAL);
  565. }
  566. if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) {
  567. NV_ERROR(drm, "pushbuf reloc count exceeds limit: %d max %d\n",
  568. req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS);
  569. return nouveau_abi16_put(abi16, -EINVAL);
  570. }
  571. push = u_memcpya(req->push, req->nr_push, sizeof(*push));
  572. if (IS_ERR(push))
  573. return nouveau_abi16_put(abi16, PTR_ERR(push));
  574. bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo));
  575. if (IS_ERR(bo)) {
  576. kfree(push);
  577. return nouveau_abi16_put(abi16, PTR_ERR(bo));
  578. }
  579. /* Ensure all push buffers are on validate list */
  580. for (i = 0; i < req->nr_push; i++) {
  581. if (push[i].bo_index >= req->nr_buffers) {
  582. NV_ERROR(drm, "push %d buffer not in list\n", i);
  583. ret = -EINVAL;
  584. goto out_prevalid;
  585. }
  586. }
  587. /* Validate buffer list */
  588. ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, req->buffers,
  589. req->nr_buffers, &op, &do_reloc);
  590. if (ret) {
  591. if (ret != -ERESTARTSYS)
  592. NV_ERROR(drm, "validate: %d\n", ret);
  593. goto out_prevalid;
  594. }
  595. /* Apply any relocations that are required */
  596. if (do_reloc) {
  597. ret = nouveau_gem_pushbuf_reloc_apply(dev, req, bo);
  598. if (ret) {
  599. NV_ERROR(drm, "reloc apply: %d\n", ret);
  600. goto out;
  601. }
  602. }
  603. if (chan->dma.ib_max) {
  604. ret = nouveau_dma_wait(chan, req->nr_push + 1, 16);
  605. if (ret) {
  606. NV_ERROR(drm, "nv50cal_space: %d\n", ret);
  607. goto out;
  608. }
  609. for (i = 0; i < req->nr_push; i++) {
  610. struct nouveau_bo *nvbo = (void *)(unsigned long)
  611. bo[push[i].bo_index].user_priv;
  612. nv50_dma_push(chan, nvbo, push[i].offset,
  613. push[i].length);
  614. }
  615. } else
  616. if (nv_device(drm->device)->chipset >= 0x25) {
  617. ret = RING_SPACE(chan, req->nr_push * 2);
  618. if (ret) {
  619. NV_ERROR(drm, "cal_space: %d\n", ret);
  620. goto out;
  621. }
  622. for (i = 0; i < req->nr_push; i++) {
  623. struct nouveau_bo *nvbo = (void *)(unsigned long)
  624. bo[push[i].bo_index].user_priv;
  625. OUT_RING(chan, (nvbo->bo.offset + push[i].offset) | 2);
  626. OUT_RING(chan, 0);
  627. }
  628. } else {
  629. ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS));
  630. if (ret) {
  631. NV_ERROR(drm, "jmp_space: %d\n", ret);
  632. goto out;
  633. }
  634. for (i = 0; i < req->nr_push; i++) {
  635. struct nouveau_bo *nvbo = (void *)(unsigned long)
  636. bo[push[i].bo_index].user_priv;
  637. uint32_t cmd;
  638. cmd = chan->push.vma.offset + ((chan->dma.cur + 2) << 2);
  639. cmd |= 0x20000000;
  640. if (unlikely(cmd != req->suffix0)) {
  641. if (!nvbo->kmap.virtual) {
  642. ret = ttm_bo_kmap(&nvbo->bo, 0,
  643. nvbo->bo.mem.
  644. num_pages,
  645. &nvbo->kmap);
  646. if (ret) {
  647. WIND_RING(chan);
  648. goto out;
  649. }
  650. nvbo->validate_mapped = true;
  651. }
  652. nouveau_bo_wr32(nvbo, (push[i].offset +
  653. push[i].length - 8) / 4, cmd);
  654. }
  655. OUT_RING(chan, 0x20000000 |
  656. (nvbo->bo.offset + push[i].offset));
  657. OUT_RING(chan, 0);
  658. for (j = 0; j < NOUVEAU_DMA_SKIPS; j++)
  659. OUT_RING(chan, 0);
  660. }
  661. }
  662. ret = nouveau_fence_new(chan, &fence);
  663. if (ret) {
  664. NV_ERROR(drm, "error fencing pushbuf: %d\n", ret);
  665. WIND_RING(chan);
  666. goto out;
  667. }
  668. out:
  669. validate_fini(&op, fence);
  670. nouveau_fence_unref(&fence);
  671. out_prevalid:
  672. kfree(bo);
  673. kfree(push);
  674. out_next:
  675. if (chan->dma.ib_max) {
  676. req->suffix0 = 0x00000000;
  677. req->suffix1 = 0x00000000;
  678. } else
  679. if (nv_device(drm->device)->chipset >= 0x25) {
  680. req->suffix0 = 0x00020000;
  681. req->suffix1 = 0x00000000;
  682. } else {
  683. req->suffix0 = 0x20000000 |
  684. (chan->push.vma.offset + ((chan->dma.cur + 2) << 2));
  685. req->suffix1 = 0x00000000;
  686. }
  687. return nouveau_abi16_put(abi16, ret);
  688. }
  689. static inline uint32_t
  690. domain_to_ttm(struct nouveau_bo *nvbo, uint32_t domain)
  691. {
  692. uint32_t flags = 0;
  693. if (domain & NOUVEAU_GEM_DOMAIN_VRAM)
  694. flags |= TTM_PL_FLAG_VRAM;
  695. if (domain & NOUVEAU_GEM_DOMAIN_GART)
  696. flags |= TTM_PL_FLAG_TT;
  697. return flags;
  698. }
  699. int
  700. nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data,
  701. struct drm_file *file_priv)
  702. {
  703. struct drm_nouveau_gem_cpu_prep *req = data;
  704. struct drm_gem_object *gem;
  705. struct nouveau_bo *nvbo;
  706. bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT);
  707. int ret = -EINVAL;
  708. gem = drm_gem_object_lookup(dev, file_priv, req->handle);
  709. if (!gem)
  710. return -ENOENT;
  711. nvbo = nouveau_gem_object(gem);
  712. spin_lock(&nvbo->bo.bdev->fence_lock);
  713. ret = ttm_bo_wait(&nvbo->bo, true, true, no_wait);
  714. spin_unlock(&nvbo->bo.bdev->fence_lock);
  715. drm_gem_object_unreference_unlocked(gem);
  716. return ret;
  717. }
  718. int
  719. nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data,
  720. struct drm_file *file_priv)
  721. {
  722. return 0;
  723. }
  724. int
  725. nouveau_gem_ioctl_info(struct drm_device *dev, void *data,
  726. struct drm_file *file_priv)
  727. {
  728. struct drm_nouveau_gem_info *req = data;
  729. struct drm_gem_object *gem;
  730. int ret;
  731. gem = drm_gem_object_lookup(dev, file_priv, req->handle);
  732. if (!gem)
  733. return -ENOENT;
  734. ret = nouveau_gem_info(file_priv, gem, req);
  735. drm_gem_object_unreference_unlocked(gem);
  736. return ret;
  737. }