radeon_ttm.c 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740
  1. /*
  2. * Copyright 2009 Jerome Glisse.
  3. * All Rights Reserved.
  4. *
  5. * Permission is hereby granted, free of charge, to any person obtaining a
  6. * copy of this software and associated documentation files (the
  7. * "Software"), to deal in the Software without restriction, including
  8. * without limitation the rights to use, copy, modify, merge, publish,
  9. * distribute, sub license, and/or sell copies of the Software, and to
  10. * permit persons to whom the Software is furnished to do so, subject to
  11. * the following conditions:
  12. *
  13. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15. * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16. * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17. * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18. * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19. * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20. *
  21. * The above copyright notice and this permission notice (including the
  22. * next paragraph) shall be included in all copies or substantial portions
  23. * of the Software.
  24. *
  25. */
  26. /*
  27. * Authors:
  28. * Jerome Glisse <glisse@freedesktop.org>
  29. * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
  30. * Dave Airlie
  31. */
  32. #include <ttm/ttm_bo_api.h>
  33. #include <ttm/ttm_bo_driver.h>
  34. #include <ttm/ttm_placement.h>
  35. #include <ttm/ttm_module.h>
  36. #include <drm/drmP.h>
  37. #include <drm/radeon_drm.h>
  38. #include <linux/seq_file.h>
  39. #include "radeon_reg.h"
  40. #include "radeon.h"
  41. #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
  42. static int radeon_ttm_debugfs_init(struct radeon_device *rdev);
  43. static struct radeon_device *radeon_get_rdev(struct ttm_bo_device *bdev)
  44. {
  45. struct radeon_mman *mman;
  46. struct radeon_device *rdev;
  47. mman = container_of(bdev, struct radeon_mman, bdev);
  48. rdev = container_of(mman, struct radeon_device, mman);
  49. return rdev;
  50. }
  51. /*
  52. * Global memory.
  53. */
  54. static int radeon_ttm_mem_global_init(struct ttm_global_reference *ref)
  55. {
  56. return ttm_mem_global_init(ref->object);
  57. }
  58. static void radeon_ttm_mem_global_release(struct ttm_global_reference *ref)
  59. {
  60. ttm_mem_global_release(ref->object);
  61. }
  62. static int radeon_ttm_global_init(struct radeon_device *rdev)
  63. {
  64. struct ttm_global_reference *global_ref;
  65. int r;
  66. rdev->mman.mem_global_referenced = false;
  67. global_ref = &rdev->mman.mem_global_ref;
  68. global_ref->global_type = TTM_GLOBAL_TTM_MEM;
  69. global_ref->size = sizeof(struct ttm_mem_global);
  70. global_ref->init = &radeon_ttm_mem_global_init;
  71. global_ref->release = &radeon_ttm_mem_global_release;
  72. r = ttm_global_item_ref(global_ref);
  73. if (r != 0) {
  74. DRM_ERROR("Failed setting up TTM memory accounting "
  75. "subsystem.\n");
  76. return r;
  77. }
  78. rdev->mman.bo_global_ref.mem_glob =
  79. rdev->mman.mem_global_ref.object;
  80. global_ref = &rdev->mman.bo_global_ref.ref;
  81. global_ref->global_type = TTM_GLOBAL_TTM_BO;
  82. global_ref->size = sizeof(struct ttm_bo_global);
  83. global_ref->init = &ttm_bo_global_init;
  84. global_ref->release = &ttm_bo_global_release;
  85. r = ttm_global_item_ref(global_ref);
  86. if (r != 0) {
  87. DRM_ERROR("Failed setting up TTM BO subsystem.\n");
  88. ttm_global_item_unref(&rdev->mman.mem_global_ref);
  89. return r;
  90. }
  91. rdev->mman.mem_global_referenced = true;
  92. return 0;
  93. }
  94. static void radeon_ttm_global_fini(struct radeon_device *rdev)
  95. {
  96. if (rdev->mman.mem_global_referenced) {
  97. ttm_global_item_unref(&rdev->mman.bo_global_ref.ref);
  98. ttm_global_item_unref(&rdev->mman.mem_global_ref);
  99. rdev->mman.mem_global_referenced = false;
  100. }
  101. }
  102. struct ttm_backend *radeon_ttm_backend_create(struct radeon_device *rdev);
  103. static struct ttm_backend*
  104. radeon_create_ttm_backend_entry(struct ttm_bo_device *bdev)
  105. {
  106. struct radeon_device *rdev;
  107. rdev = radeon_get_rdev(bdev);
  108. #if __OS_HAS_AGP
  109. if (rdev->flags & RADEON_IS_AGP) {
  110. return ttm_agp_backend_init(bdev, rdev->ddev->agp->bridge);
  111. } else
  112. #endif
  113. {
  114. return radeon_ttm_backend_create(rdev);
  115. }
  116. }
  117. static int radeon_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
  118. {
  119. return 0;
  120. }
  121. static int radeon_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
  122. struct ttm_mem_type_manager *man)
  123. {
  124. struct radeon_device *rdev;
  125. rdev = radeon_get_rdev(bdev);
  126. switch (type) {
  127. case TTM_PL_SYSTEM:
  128. /* System memory */
  129. man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
  130. man->available_caching = TTM_PL_MASK_CACHING;
  131. man->default_caching = TTM_PL_FLAG_CACHED;
  132. break;
  133. case TTM_PL_TT:
  134. man->gpu_offset = 0;
  135. man->available_caching = TTM_PL_MASK_CACHING;
  136. man->default_caching = TTM_PL_FLAG_CACHED;
  137. man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA;
  138. #if __OS_HAS_AGP
  139. if (rdev->flags & RADEON_IS_AGP) {
  140. if (!(drm_core_has_AGP(rdev->ddev) && rdev->ddev->agp)) {
  141. DRM_ERROR("AGP is not enabled for memory type %u\n",
  142. (unsigned)type);
  143. return -EINVAL;
  144. }
  145. man->io_offset = rdev->mc.agp_base;
  146. man->io_size = rdev->mc.gtt_size;
  147. man->io_addr = NULL;
  148. if (!rdev->ddev->agp->cant_use_aperture)
  149. man->flags = TTM_MEMTYPE_FLAG_NEEDS_IOREMAP |
  150. TTM_MEMTYPE_FLAG_MAPPABLE;
  151. man->available_caching = TTM_PL_FLAG_UNCACHED |
  152. TTM_PL_FLAG_WC;
  153. man->default_caching = TTM_PL_FLAG_WC;
  154. } else
  155. #endif
  156. {
  157. man->io_offset = 0;
  158. man->io_size = 0;
  159. man->io_addr = NULL;
  160. }
  161. break;
  162. case TTM_PL_VRAM:
  163. /* "On-card" video ram */
  164. man->gpu_offset = 0;
  165. man->flags = TTM_MEMTYPE_FLAG_FIXED |
  166. TTM_MEMTYPE_FLAG_NEEDS_IOREMAP |
  167. TTM_MEMTYPE_FLAG_MAPPABLE;
  168. man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC;
  169. man->default_caching = TTM_PL_FLAG_WC;
  170. man->io_addr = NULL;
  171. man->io_offset = rdev->mc.aper_base;
  172. man->io_size = rdev->mc.aper_size;
  173. break;
  174. default:
  175. DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);
  176. return -EINVAL;
  177. }
  178. return 0;
  179. }
  180. static uint32_t radeon_evict_flags(struct ttm_buffer_object *bo)
  181. {
  182. uint32_t cur_placement = bo->mem.placement & ~TTM_PL_MASK_MEMTYPE;
  183. switch (bo->mem.mem_type) {
  184. default:
  185. return (cur_placement & ~TTM_PL_MASK_CACHING) |
  186. TTM_PL_FLAG_SYSTEM |
  187. TTM_PL_FLAG_CACHED;
  188. }
  189. }
  190. static int radeon_verify_access(struct ttm_buffer_object *bo, struct file *filp)
  191. {
  192. return 0;
  193. }
  194. static void radeon_move_null(struct ttm_buffer_object *bo,
  195. struct ttm_mem_reg *new_mem)
  196. {
  197. struct ttm_mem_reg *old_mem = &bo->mem;
  198. BUG_ON(old_mem->mm_node != NULL);
  199. *old_mem = *new_mem;
  200. new_mem->mm_node = NULL;
  201. }
  202. static int radeon_move_blit(struct ttm_buffer_object *bo,
  203. bool evict, int no_wait,
  204. struct ttm_mem_reg *new_mem,
  205. struct ttm_mem_reg *old_mem)
  206. {
  207. struct radeon_device *rdev;
  208. uint64_t old_start, new_start;
  209. struct radeon_fence *fence;
  210. int r;
  211. rdev = radeon_get_rdev(bo->bdev);
  212. r = radeon_fence_create(rdev, &fence);
  213. if (unlikely(r)) {
  214. return r;
  215. }
  216. old_start = old_mem->mm_node->start << PAGE_SHIFT;
  217. new_start = new_mem->mm_node->start << PAGE_SHIFT;
  218. switch (old_mem->mem_type) {
  219. case TTM_PL_VRAM:
  220. old_start += rdev->mc.vram_location;
  221. break;
  222. case TTM_PL_TT:
  223. old_start += rdev->mc.gtt_location;
  224. break;
  225. default:
  226. DRM_ERROR("Unknown placement %d\n", old_mem->mem_type);
  227. return -EINVAL;
  228. }
  229. switch (new_mem->mem_type) {
  230. case TTM_PL_VRAM:
  231. new_start += rdev->mc.vram_location;
  232. break;
  233. case TTM_PL_TT:
  234. new_start += rdev->mc.gtt_location;
  235. break;
  236. default:
  237. DRM_ERROR("Unknown placement %d\n", old_mem->mem_type);
  238. return -EINVAL;
  239. }
  240. if (!rdev->cp.ready) {
  241. DRM_ERROR("Trying to move memory with CP turned off.\n");
  242. return -EINVAL;
  243. }
  244. r = radeon_copy(rdev, old_start, new_start, new_mem->num_pages, fence);
  245. /* FIXME: handle copy error */
  246. r = ttm_bo_move_accel_cleanup(bo, (void *)fence, NULL,
  247. evict, no_wait, new_mem);
  248. radeon_fence_unref(&fence);
  249. return r;
  250. }
  251. static int radeon_move_vram_ram(struct ttm_buffer_object *bo,
  252. bool evict, bool interruptible, bool no_wait,
  253. struct ttm_mem_reg *new_mem)
  254. {
  255. struct radeon_device *rdev;
  256. struct ttm_mem_reg *old_mem = &bo->mem;
  257. struct ttm_mem_reg tmp_mem;
  258. uint32_t proposed_placement;
  259. int r;
  260. rdev = radeon_get_rdev(bo->bdev);
  261. tmp_mem = *new_mem;
  262. tmp_mem.mm_node = NULL;
  263. proposed_placement = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING;
  264. r = ttm_bo_mem_space(bo, proposed_placement, &tmp_mem,
  265. interruptible, no_wait);
  266. if (unlikely(r)) {
  267. return r;
  268. }
  269. r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement);
  270. if (unlikely(r)) {
  271. goto out_cleanup;
  272. }
  273. r = ttm_tt_bind(bo->ttm, &tmp_mem);
  274. if (unlikely(r)) {
  275. goto out_cleanup;
  276. }
  277. r = radeon_move_blit(bo, true, no_wait, &tmp_mem, old_mem);
  278. if (unlikely(r)) {
  279. goto out_cleanup;
  280. }
  281. r = ttm_bo_move_ttm(bo, true, no_wait, new_mem);
  282. out_cleanup:
  283. if (tmp_mem.mm_node) {
  284. struct ttm_bo_global *glob = rdev->mman.bdev.glob;
  285. spin_lock(&glob->lru_lock);
  286. drm_mm_put_block(tmp_mem.mm_node);
  287. spin_unlock(&glob->lru_lock);
  288. return r;
  289. }
  290. return r;
  291. }
  292. static int radeon_move_ram_vram(struct ttm_buffer_object *bo,
  293. bool evict, bool interruptible, bool no_wait,
  294. struct ttm_mem_reg *new_mem)
  295. {
  296. struct radeon_device *rdev;
  297. struct ttm_mem_reg *old_mem = &bo->mem;
  298. struct ttm_mem_reg tmp_mem;
  299. uint32_t proposed_flags;
  300. int r;
  301. rdev = radeon_get_rdev(bo->bdev);
  302. tmp_mem = *new_mem;
  303. tmp_mem.mm_node = NULL;
  304. proposed_flags = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING;
  305. r = ttm_bo_mem_space(bo, proposed_flags, &tmp_mem,
  306. interruptible, no_wait);
  307. if (unlikely(r)) {
  308. return r;
  309. }
  310. r = ttm_bo_move_ttm(bo, true, no_wait, &tmp_mem);
  311. if (unlikely(r)) {
  312. goto out_cleanup;
  313. }
  314. r = radeon_move_blit(bo, true, no_wait, new_mem, old_mem);
  315. if (unlikely(r)) {
  316. goto out_cleanup;
  317. }
  318. out_cleanup:
  319. if (tmp_mem.mm_node) {
  320. struct ttm_bo_global *glob = rdev->mman.bdev.glob;
  321. spin_lock(&glob->lru_lock);
  322. drm_mm_put_block(tmp_mem.mm_node);
  323. spin_unlock(&glob->lru_lock);
  324. return r;
  325. }
  326. return r;
  327. }
  328. static int radeon_bo_move(struct ttm_buffer_object *bo,
  329. bool evict, bool interruptible, bool no_wait,
  330. struct ttm_mem_reg *new_mem)
  331. {
  332. struct radeon_device *rdev;
  333. struct ttm_mem_reg *old_mem = &bo->mem;
  334. int r;
  335. rdev = radeon_get_rdev(bo->bdev);
  336. if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
  337. radeon_move_null(bo, new_mem);
  338. return 0;
  339. }
  340. if ((old_mem->mem_type == TTM_PL_TT &&
  341. new_mem->mem_type == TTM_PL_SYSTEM) ||
  342. (old_mem->mem_type == TTM_PL_SYSTEM &&
  343. new_mem->mem_type == TTM_PL_TT)) {
  344. /* bind is enought */
  345. radeon_move_null(bo, new_mem);
  346. return 0;
  347. }
  348. if (!rdev->cp.ready || rdev->asic->copy == NULL) {
  349. /* use memcpy */
  350. goto memcpy;
  351. }
  352. if (old_mem->mem_type == TTM_PL_VRAM &&
  353. new_mem->mem_type == TTM_PL_SYSTEM) {
  354. r = radeon_move_vram_ram(bo, evict, interruptible,
  355. no_wait, new_mem);
  356. } else if (old_mem->mem_type == TTM_PL_SYSTEM &&
  357. new_mem->mem_type == TTM_PL_VRAM) {
  358. r = radeon_move_ram_vram(bo, evict, interruptible,
  359. no_wait, new_mem);
  360. } else {
  361. r = radeon_move_blit(bo, evict, no_wait, new_mem, old_mem);
  362. }
  363. if (r) {
  364. memcpy:
  365. r = ttm_bo_move_memcpy(bo, evict, no_wait, new_mem);
  366. }
  367. return r;
  368. }
  369. const uint32_t radeon_mem_prios[] = {
  370. TTM_PL_VRAM,
  371. TTM_PL_TT,
  372. TTM_PL_SYSTEM,
  373. };
  374. const uint32_t radeon_busy_prios[] = {
  375. TTM_PL_TT,
  376. TTM_PL_VRAM,
  377. TTM_PL_SYSTEM,
  378. };
  379. static int radeon_sync_obj_wait(void *sync_obj, void *sync_arg,
  380. bool lazy, bool interruptible)
  381. {
  382. return radeon_fence_wait((struct radeon_fence *)sync_obj, interruptible);
  383. }
  384. static int radeon_sync_obj_flush(void *sync_obj, void *sync_arg)
  385. {
  386. return 0;
  387. }
  388. static void radeon_sync_obj_unref(void **sync_obj)
  389. {
  390. radeon_fence_unref((struct radeon_fence **)sync_obj);
  391. }
  392. static void *radeon_sync_obj_ref(void *sync_obj)
  393. {
  394. return radeon_fence_ref((struct radeon_fence *)sync_obj);
  395. }
  396. static bool radeon_sync_obj_signaled(void *sync_obj, void *sync_arg)
  397. {
  398. return radeon_fence_signaled((struct radeon_fence *)sync_obj);
  399. }
  400. static struct ttm_bo_driver radeon_bo_driver = {
  401. .mem_type_prio = radeon_mem_prios,
  402. .mem_busy_prio = radeon_busy_prios,
  403. .num_mem_type_prio = ARRAY_SIZE(radeon_mem_prios),
  404. .num_mem_busy_prio = ARRAY_SIZE(radeon_busy_prios),
  405. .create_ttm_backend_entry = &radeon_create_ttm_backend_entry,
  406. .invalidate_caches = &radeon_invalidate_caches,
  407. .init_mem_type = &radeon_init_mem_type,
  408. .evict_flags = &radeon_evict_flags,
  409. .move = &radeon_bo_move,
  410. .verify_access = &radeon_verify_access,
  411. .sync_obj_signaled = &radeon_sync_obj_signaled,
  412. .sync_obj_wait = &radeon_sync_obj_wait,
  413. .sync_obj_flush = &radeon_sync_obj_flush,
  414. .sync_obj_unref = &radeon_sync_obj_unref,
  415. .sync_obj_ref = &radeon_sync_obj_ref,
  416. .move_notify = &radeon_bo_move_notify,
  417. .fault_reserve_notify = &radeon_bo_fault_reserve_notify,
  418. };
  419. int radeon_ttm_init(struct radeon_device *rdev)
  420. {
  421. int r;
  422. r = radeon_ttm_global_init(rdev);
  423. if (r) {
  424. return r;
  425. }
  426. /* No others user of address space so set it to 0 */
  427. r = ttm_bo_device_init(&rdev->mman.bdev,
  428. rdev->mman.bo_global_ref.ref.object,
  429. &radeon_bo_driver, DRM_FILE_PAGE_OFFSET,
  430. rdev->need_dma32);
  431. if (r) {
  432. DRM_ERROR("failed initializing buffer object driver(%d).\n", r);
  433. return r;
  434. }
  435. r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_VRAM, 0,
  436. ((rdev->mc.real_vram_size) >> PAGE_SHIFT));
  437. if (r) {
  438. DRM_ERROR("Failed initializing VRAM heap.\n");
  439. return r;
  440. }
  441. r = radeon_object_create(rdev, NULL, 256 * 1024, true,
  442. RADEON_GEM_DOMAIN_VRAM, false,
  443. &rdev->stollen_vga_memory);
  444. if (r) {
  445. return r;
  446. }
  447. r = radeon_object_pin(rdev->stollen_vga_memory, RADEON_GEM_DOMAIN_VRAM, NULL);
  448. if (r) {
  449. radeon_object_unref(&rdev->stollen_vga_memory);
  450. return r;
  451. }
  452. DRM_INFO("radeon: %uM of VRAM memory ready\n",
  453. (unsigned)rdev->mc.real_vram_size / (1024 * 1024));
  454. r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_TT, 0,
  455. ((rdev->mc.gtt_size) >> PAGE_SHIFT));
  456. if (r) {
  457. DRM_ERROR("Failed initializing GTT heap.\n");
  458. return r;
  459. }
  460. DRM_INFO("radeon: %uM of GTT memory ready.\n",
  461. (unsigned)(rdev->mc.gtt_size / (1024 * 1024)));
  462. if (unlikely(rdev->mman.bdev.dev_mapping == NULL)) {
  463. rdev->mman.bdev.dev_mapping = rdev->ddev->dev_mapping;
  464. }
  465. r = radeon_ttm_debugfs_init(rdev);
  466. if (r) {
  467. DRM_ERROR("Failed to init debugfs\n");
  468. return r;
  469. }
  470. return 0;
  471. }
  472. void radeon_ttm_fini(struct radeon_device *rdev)
  473. {
  474. if (rdev->stollen_vga_memory) {
  475. radeon_object_unpin(rdev->stollen_vga_memory);
  476. radeon_object_unref(&rdev->stollen_vga_memory);
  477. }
  478. ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_VRAM);
  479. ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_TT);
  480. ttm_bo_device_release(&rdev->mman.bdev);
  481. radeon_gart_fini(rdev);
  482. radeon_ttm_global_fini(rdev);
  483. DRM_INFO("radeon: ttm finalized\n");
  484. }
  485. static struct vm_operations_struct radeon_ttm_vm_ops;
  486. static const struct vm_operations_struct *ttm_vm_ops = NULL;
  487. static int radeon_ttm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  488. {
  489. struct ttm_buffer_object *bo;
  490. int r;
  491. bo = (struct ttm_buffer_object *)vma->vm_private_data;
  492. if (bo == NULL) {
  493. return VM_FAULT_NOPAGE;
  494. }
  495. r = ttm_vm_ops->fault(vma, vmf);
  496. return r;
  497. }
  498. int radeon_mmap(struct file *filp, struct vm_area_struct *vma)
  499. {
  500. struct drm_file *file_priv;
  501. struct radeon_device *rdev;
  502. int r;
  503. if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET)) {
  504. return drm_mmap(filp, vma);
  505. }
  506. file_priv = (struct drm_file *)filp->private_data;
  507. rdev = file_priv->minor->dev->dev_private;
  508. if (rdev == NULL) {
  509. return -EINVAL;
  510. }
  511. r = ttm_bo_mmap(filp, vma, &rdev->mman.bdev);
  512. if (unlikely(r != 0)) {
  513. return r;
  514. }
  515. if (unlikely(ttm_vm_ops == NULL)) {
  516. ttm_vm_ops = vma->vm_ops;
  517. radeon_ttm_vm_ops = *ttm_vm_ops;
  518. radeon_ttm_vm_ops.fault = &radeon_ttm_fault;
  519. }
  520. vma->vm_ops = &radeon_ttm_vm_ops;
  521. return 0;
  522. }
  523. /*
  524. * TTM backend functions.
  525. */
  526. struct radeon_ttm_backend {
  527. struct ttm_backend backend;
  528. struct radeon_device *rdev;
  529. unsigned long num_pages;
  530. struct page **pages;
  531. struct page *dummy_read_page;
  532. bool populated;
  533. bool bound;
  534. unsigned offset;
  535. };
  536. static int radeon_ttm_backend_populate(struct ttm_backend *backend,
  537. unsigned long num_pages,
  538. struct page **pages,
  539. struct page *dummy_read_page)
  540. {
  541. struct radeon_ttm_backend *gtt;
  542. gtt = container_of(backend, struct radeon_ttm_backend, backend);
  543. gtt->pages = pages;
  544. gtt->num_pages = num_pages;
  545. gtt->dummy_read_page = dummy_read_page;
  546. gtt->populated = true;
  547. return 0;
  548. }
  549. static void radeon_ttm_backend_clear(struct ttm_backend *backend)
  550. {
  551. struct radeon_ttm_backend *gtt;
  552. gtt = container_of(backend, struct radeon_ttm_backend, backend);
  553. gtt->pages = NULL;
  554. gtt->num_pages = 0;
  555. gtt->dummy_read_page = NULL;
  556. gtt->populated = false;
  557. gtt->bound = false;
  558. }
  559. static int radeon_ttm_backend_bind(struct ttm_backend *backend,
  560. struct ttm_mem_reg *bo_mem)
  561. {
  562. struct radeon_ttm_backend *gtt;
  563. int r;
  564. gtt = container_of(backend, struct radeon_ttm_backend, backend);
  565. gtt->offset = bo_mem->mm_node->start << PAGE_SHIFT;
  566. if (!gtt->num_pages) {
  567. WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n", gtt->num_pages, bo_mem, backend);
  568. }
  569. r = radeon_gart_bind(gtt->rdev, gtt->offset,
  570. gtt->num_pages, gtt->pages);
  571. if (r) {
  572. DRM_ERROR("failed to bind %lu pages at 0x%08X\n",
  573. gtt->num_pages, gtt->offset);
  574. return r;
  575. }
  576. gtt->bound = true;
  577. return 0;
  578. }
  579. static int radeon_ttm_backend_unbind(struct ttm_backend *backend)
  580. {
  581. struct radeon_ttm_backend *gtt;
  582. gtt = container_of(backend, struct radeon_ttm_backend, backend);
  583. radeon_gart_unbind(gtt->rdev, gtt->offset, gtt->num_pages);
  584. gtt->bound = false;
  585. return 0;
  586. }
  587. static void radeon_ttm_backend_destroy(struct ttm_backend *backend)
  588. {
  589. struct radeon_ttm_backend *gtt;
  590. gtt = container_of(backend, struct radeon_ttm_backend, backend);
  591. if (gtt->bound) {
  592. radeon_ttm_backend_unbind(backend);
  593. }
  594. kfree(gtt);
  595. }
  596. static struct ttm_backend_func radeon_backend_func = {
  597. .populate = &radeon_ttm_backend_populate,
  598. .clear = &radeon_ttm_backend_clear,
  599. .bind = &radeon_ttm_backend_bind,
  600. .unbind = &radeon_ttm_backend_unbind,
  601. .destroy = &radeon_ttm_backend_destroy,
  602. };
  603. struct ttm_backend *radeon_ttm_backend_create(struct radeon_device *rdev)
  604. {
  605. struct radeon_ttm_backend *gtt;
  606. gtt = kzalloc(sizeof(struct radeon_ttm_backend), GFP_KERNEL);
  607. if (gtt == NULL) {
  608. return NULL;
  609. }
  610. gtt->backend.bdev = &rdev->mman.bdev;
  611. gtt->backend.flags = 0;
  612. gtt->backend.func = &radeon_backend_func;
  613. gtt->rdev = rdev;
  614. gtt->pages = NULL;
  615. gtt->num_pages = 0;
  616. gtt->dummy_read_page = NULL;
  617. gtt->populated = false;
  618. gtt->bound = false;
  619. return &gtt->backend;
  620. }
  621. #define RADEON_DEBUGFS_MEM_TYPES 2
  622. #if defined(CONFIG_DEBUG_FS)
  623. static int radeon_mm_dump_table(struct seq_file *m, void *data)
  624. {
  625. struct drm_info_node *node = (struct drm_info_node *)m->private;
  626. struct drm_mm *mm = (struct drm_mm *)node->info_ent->data;
  627. struct drm_device *dev = node->minor->dev;
  628. struct radeon_device *rdev = dev->dev_private;
  629. int ret;
  630. struct ttm_bo_global *glob = rdev->mman.bdev.glob;
  631. spin_lock(&glob->lru_lock);
  632. ret = drm_mm_dump_table(m, mm);
  633. spin_unlock(&glob->lru_lock);
  634. return ret;
  635. }
  636. #endif
  637. static int radeon_ttm_debugfs_init(struct radeon_device *rdev)
  638. {
  639. #if defined(CONFIG_DEBUG_FS)
  640. static struct drm_info_list radeon_mem_types_list[RADEON_DEBUGFS_MEM_TYPES];
  641. static char radeon_mem_types_names[RADEON_DEBUGFS_MEM_TYPES][32];
  642. unsigned i;
  643. for (i = 0; i < RADEON_DEBUGFS_MEM_TYPES; i++) {
  644. if (i == 0)
  645. sprintf(radeon_mem_types_names[i], "radeon_vram_mm");
  646. else
  647. sprintf(radeon_mem_types_names[i], "radeon_gtt_mm");
  648. radeon_mem_types_list[i].name = radeon_mem_types_names[i];
  649. radeon_mem_types_list[i].show = &radeon_mm_dump_table;
  650. radeon_mem_types_list[i].driver_features = 0;
  651. if (i == 0)
  652. radeon_mem_types_list[i].data = &rdev->mman.bdev.man[TTM_PL_VRAM].manager;
  653. else
  654. radeon_mem_types_list[i].data = &rdev->mman.bdev.man[TTM_PL_TT].manager;
  655. }
  656. return radeon_debugfs_add_files(rdev, radeon_mem_types_list, RADEON_DEBUGFS_MEM_TYPES);
  657. #endif
  658. return 0;
  659. }