radeon_ttm.c 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750
  1. /*
  2. * Copyright 2009 Jerome Glisse.
  3. * All Rights Reserved.
  4. *
  5. * Permission is hereby granted, free of charge, to any person obtaining a
  6. * copy of this software and associated documentation files (the
  7. * "Software"), to deal in the Software without restriction, including
  8. * without limitation the rights to use, copy, modify, merge, publish,
  9. * distribute, sub license, and/or sell copies of the Software, and to
  10. * permit persons to whom the Software is furnished to do so, subject to
  11. * the following conditions:
  12. *
  13. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15. * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16. * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17. * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18. * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19. * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20. *
  21. * The above copyright notice and this permission notice (including the
  22. * next paragraph) shall be included in all copies or substantial portions
  23. * of the Software.
  24. *
  25. */
  26. /*
  27. * Authors:
  28. * Jerome Glisse <glisse@freedesktop.org>
  29. * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
  30. * Dave Airlie
  31. */
  32. #include <ttm/ttm_bo_api.h>
  33. #include <ttm/ttm_bo_driver.h>
  34. #include <ttm/ttm_placement.h>
  35. #include <ttm/ttm_module.h>
  36. #include <drm/drmP.h>
  37. #include <drm/radeon_drm.h>
  38. #include <linux/seq_file.h>
  39. #include "radeon_reg.h"
  40. #include "radeon.h"
  41. #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
  42. static int radeon_ttm_debugfs_init(struct radeon_device *rdev);
  43. static struct radeon_device *radeon_get_rdev(struct ttm_bo_device *bdev)
  44. {
  45. struct radeon_mman *mman;
  46. struct radeon_device *rdev;
  47. mman = container_of(bdev, struct radeon_mman, bdev);
  48. rdev = container_of(mman, struct radeon_device, mman);
  49. return rdev;
  50. }
  51. /*
  52. * Global memory.
  53. */
  54. static int radeon_ttm_mem_global_init(struct ttm_global_reference *ref)
  55. {
  56. return ttm_mem_global_init(ref->object);
  57. }
  58. static void radeon_ttm_mem_global_release(struct ttm_global_reference *ref)
  59. {
  60. ttm_mem_global_release(ref->object);
  61. }
  62. static int radeon_ttm_global_init(struct radeon_device *rdev)
  63. {
  64. struct ttm_global_reference *global_ref;
  65. int r;
  66. rdev->mman.mem_global_referenced = false;
  67. global_ref = &rdev->mman.mem_global_ref;
  68. global_ref->global_type = TTM_GLOBAL_TTM_MEM;
  69. global_ref->size = sizeof(struct ttm_mem_global);
  70. global_ref->init = &radeon_ttm_mem_global_init;
  71. global_ref->release = &radeon_ttm_mem_global_release;
  72. r = ttm_global_item_ref(global_ref);
  73. if (r != 0) {
  74. DRM_ERROR("Failed setting up TTM memory accounting "
  75. "subsystem.\n");
  76. return r;
  77. }
  78. rdev->mman.bo_global_ref.mem_glob =
  79. rdev->mman.mem_global_ref.object;
  80. global_ref = &rdev->mman.bo_global_ref.ref;
  81. global_ref->global_type = TTM_GLOBAL_TTM_BO;
  82. global_ref->size = sizeof(struct ttm_bo_global);
  83. global_ref->init = &ttm_bo_global_init;
  84. global_ref->release = &ttm_bo_global_release;
  85. r = ttm_global_item_ref(global_ref);
  86. if (r != 0) {
  87. DRM_ERROR("Failed setting up TTM BO subsystem.\n");
  88. ttm_global_item_unref(&rdev->mman.mem_global_ref);
  89. return r;
  90. }
  91. rdev->mman.mem_global_referenced = true;
  92. return 0;
  93. }
  94. static void radeon_ttm_global_fini(struct radeon_device *rdev)
  95. {
  96. if (rdev->mman.mem_global_referenced) {
  97. ttm_global_item_unref(&rdev->mman.bo_global_ref.ref);
  98. ttm_global_item_unref(&rdev->mman.mem_global_ref);
  99. rdev->mman.mem_global_referenced = false;
  100. }
  101. }
  102. struct ttm_backend *radeon_ttm_backend_create(struct radeon_device *rdev);
  103. static struct ttm_backend*
  104. radeon_create_ttm_backend_entry(struct ttm_bo_device *bdev)
  105. {
  106. struct radeon_device *rdev;
  107. rdev = radeon_get_rdev(bdev);
  108. #if __OS_HAS_AGP
  109. if (rdev->flags & RADEON_IS_AGP) {
  110. return ttm_agp_backend_init(bdev, rdev->ddev->agp->bridge);
  111. } else
  112. #endif
  113. {
  114. return radeon_ttm_backend_create(rdev);
  115. }
  116. }
  117. static int radeon_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
  118. {
  119. return 0;
  120. }
  121. static int radeon_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
  122. struct ttm_mem_type_manager *man)
  123. {
  124. struct radeon_device *rdev;
  125. rdev = radeon_get_rdev(bdev);
  126. switch (type) {
  127. case TTM_PL_SYSTEM:
  128. /* System memory */
  129. man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
  130. man->available_caching = TTM_PL_MASK_CACHING;
  131. man->default_caching = TTM_PL_FLAG_CACHED;
  132. break;
  133. case TTM_PL_TT:
  134. man->gpu_offset = rdev->mc.gtt_location;
  135. man->available_caching = TTM_PL_MASK_CACHING;
  136. man->default_caching = TTM_PL_FLAG_CACHED;
  137. man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA;
  138. #if __OS_HAS_AGP
  139. if (rdev->flags & RADEON_IS_AGP) {
  140. if (!(drm_core_has_AGP(rdev->ddev) && rdev->ddev->agp)) {
  141. DRM_ERROR("AGP is not enabled for memory type %u\n",
  142. (unsigned)type);
  143. return -EINVAL;
  144. }
  145. man->io_offset = rdev->mc.agp_base;
  146. man->io_size = rdev->mc.gtt_size;
  147. man->io_addr = NULL;
  148. if (!rdev->ddev->agp->cant_use_aperture)
  149. man->flags = TTM_MEMTYPE_FLAG_NEEDS_IOREMAP |
  150. TTM_MEMTYPE_FLAG_MAPPABLE;
  151. man->available_caching = TTM_PL_FLAG_UNCACHED |
  152. TTM_PL_FLAG_WC;
  153. man->default_caching = TTM_PL_FLAG_WC;
  154. } else
  155. #endif
  156. {
  157. man->io_offset = 0;
  158. man->io_size = 0;
  159. man->io_addr = NULL;
  160. }
  161. break;
  162. case TTM_PL_VRAM:
  163. /* "On-card" video ram */
  164. man->gpu_offset = rdev->mc.vram_location;
  165. man->flags = TTM_MEMTYPE_FLAG_FIXED |
  166. TTM_MEMTYPE_FLAG_NEEDS_IOREMAP |
  167. TTM_MEMTYPE_FLAG_MAPPABLE;
  168. man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC;
  169. man->default_caching = TTM_PL_FLAG_WC;
  170. man->io_addr = NULL;
  171. man->io_offset = rdev->mc.aper_base;
  172. man->io_size = rdev->mc.aper_size;
  173. break;
  174. default:
  175. DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);
  176. return -EINVAL;
  177. }
  178. return 0;
  179. }
  180. static void radeon_evict_flags(struct ttm_buffer_object *bo,
  181. struct ttm_placement *placement)
  182. {
  183. struct radeon_bo *rbo = container_of(bo, struct radeon_bo, tbo);
  184. switch (bo->mem.mem_type) {
  185. case TTM_PL_VRAM:
  186. radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_GTT);
  187. break;
  188. case TTM_PL_TT:
  189. default:
  190. radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU);
  191. }
  192. *placement = rbo->placement;
  193. }
  194. static int radeon_verify_access(struct ttm_buffer_object *bo, struct file *filp)
  195. {
  196. return 0;
  197. }
  198. static void radeon_move_null(struct ttm_buffer_object *bo,
  199. struct ttm_mem_reg *new_mem)
  200. {
  201. struct ttm_mem_reg *old_mem = &bo->mem;
  202. BUG_ON(old_mem->mm_node != NULL);
  203. *old_mem = *new_mem;
  204. new_mem->mm_node = NULL;
  205. }
  206. static int radeon_move_blit(struct ttm_buffer_object *bo,
  207. bool evict, int no_wait,
  208. struct ttm_mem_reg *new_mem,
  209. struct ttm_mem_reg *old_mem)
  210. {
  211. struct radeon_device *rdev;
  212. uint64_t old_start, new_start;
  213. struct radeon_fence *fence;
  214. int r;
  215. rdev = radeon_get_rdev(bo->bdev);
  216. r = radeon_fence_create(rdev, &fence);
  217. if (unlikely(r)) {
  218. return r;
  219. }
  220. old_start = old_mem->mm_node->start << PAGE_SHIFT;
  221. new_start = new_mem->mm_node->start << PAGE_SHIFT;
  222. switch (old_mem->mem_type) {
  223. case TTM_PL_VRAM:
  224. old_start += rdev->mc.vram_location;
  225. break;
  226. case TTM_PL_TT:
  227. old_start += rdev->mc.gtt_location;
  228. break;
  229. default:
  230. DRM_ERROR("Unknown placement %d\n", old_mem->mem_type);
  231. return -EINVAL;
  232. }
  233. switch (new_mem->mem_type) {
  234. case TTM_PL_VRAM:
  235. new_start += rdev->mc.vram_location;
  236. break;
  237. case TTM_PL_TT:
  238. new_start += rdev->mc.gtt_location;
  239. break;
  240. default:
  241. DRM_ERROR("Unknown placement %d\n", old_mem->mem_type);
  242. return -EINVAL;
  243. }
  244. if (!rdev->cp.ready) {
  245. DRM_ERROR("Trying to move memory with CP turned off.\n");
  246. return -EINVAL;
  247. }
  248. r = radeon_copy(rdev, old_start, new_start, new_mem->num_pages, fence);
  249. /* FIXME: handle copy error */
  250. r = ttm_bo_move_accel_cleanup(bo, (void *)fence, NULL,
  251. evict, no_wait, new_mem);
  252. radeon_fence_unref(&fence);
  253. return r;
  254. }
  255. static int radeon_move_vram_ram(struct ttm_buffer_object *bo,
  256. bool evict, bool interruptible, bool no_wait,
  257. struct ttm_mem_reg *new_mem)
  258. {
  259. struct radeon_device *rdev;
  260. struct ttm_mem_reg *old_mem = &bo->mem;
  261. struct ttm_mem_reg tmp_mem;
  262. u32 placements;
  263. struct ttm_placement placement;
  264. int r;
  265. rdev = radeon_get_rdev(bo->bdev);
  266. tmp_mem = *new_mem;
  267. tmp_mem.mm_node = NULL;
  268. placement.fpfn = 0;
  269. placement.lpfn = 0;
  270. placement.num_placement = 1;
  271. placement.placement = &placements;
  272. placement.num_busy_placement = 1;
  273. placement.busy_placement = &placements;
  274. placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
  275. r = ttm_bo_mem_space(bo, &placement, &tmp_mem,
  276. interruptible, no_wait);
  277. if (unlikely(r)) {
  278. return r;
  279. }
  280. r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement);
  281. if (unlikely(r)) {
  282. goto out_cleanup;
  283. }
  284. r = ttm_tt_bind(bo->ttm, &tmp_mem);
  285. if (unlikely(r)) {
  286. goto out_cleanup;
  287. }
  288. r = radeon_move_blit(bo, true, no_wait, &tmp_mem, old_mem);
  289. if (unlikely(r)) {
  290. goto out_cleanup;
  291. }
  292. r = ttm_bo_move_ttm(bo, true, no_wait, new_mem);
  293. out_cleanup:
  294. if (tmp_mem.mm_node) {
  295. struct ttm_bo_global *glob = rdev->mman.bdev.glob;
  296. spin_lock(&glob->lru_lock);
  297. drm_mm_put_block(tmp_mem.mm_node);
  298. spin_unlock(&glob->lru_lock);
  299. return r;
  300. }
  301. return r;
  302. }
  303. static int radeon_move_ram_vram(struct ttm_buffer_object *bo,
  304. bool evict, bool interruptible, bool no_wait,
  305. struct ttm_mem_reg *new_mem)
  306. {
  307. struct radeon_device *rdev;
  308. struct ttm_mem_reg *old_mem = &bo->mem;
  309. struct ttm_mem_reg tmp_mem;
  310. struct ttm_placement placement;
  311. u32 placements;
  312. int r;
  313. rdev = radeon_get_rdev(bo->bdev);
  314. tmp_mem = *new_mem;
  315. tmp_mem.mm_node = NULL;
  316. placement.fpfn = 0;
  317. placement.lpfn = 0;
  318. placement.num_placement = 1;
  319. placement.placement = &placements;
  320. placement.num_busy_placement = 1;
  321. placement.busy_placement = &placements;
  322. placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
  323. r = ttm_bo_mem_space(bo, &placement, &tmp_mem, interruptible, no_wait);
  324. if (unlikely(r)) {
  325. return r;
  326. }
  327. r = ttm_bo_move_ttm(bo, true, no_wait, &tmp_mem);
  328. if (unlikely(r)) {
  329. goto out_cleanup;
  330. }
  331. r = radeon_move_blit(bo, true, no_wait, new_mem, old_mem);
  332. if (unlikely(r)) {
  333. goto out_cleanup;
  334. }
  335. out_cleanup:
  336. if (tmp_mem.mm_node) {
  337. struct ttm_bo_global *glob = rdev->mman.bdev.glob;
  338. spin_lock(&glob->lru_lock);
  339. drm_mm_put_block(tmp_mem.mm_node);
  340. spin_unlock(&glob->lru_lock);
  341. return r;
  342. }
  343. return r;
  344. }
  345. static int radeon_bo_move(struct ttm_buffer_object *bo,
  346. bool evict, bool interruptible, bool no_wait,
  347. struct ttm_mem_reg *new_mem)
  348. {
  349. struct radeon_device *rdev;
  350. struct ttm_mem_reg *old_mem = &bo->mem;
  351. int r;
  352. rdev = radeon_get_rdev(bo->bdev);
  353. if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
  354. radeon_move_null(bo, new_mem);
  355. return 0;
  356. }
  357. if ((old_mem->mem_type == TTM_PL_TT &&
  358. new_mem->mem_type == TTM_PL_SYSTEM) ||
  359. (old_mem->mem_type == TTM_PL_SYSTEM &&
  360. new_mem->mem_type == TTM_PL_TT)) {
  361. /* bind is enough */
  362. radeon_move_null(bo, new_mem);
  363. return 0;
  364. }
  365. if (!rdev->cp.ready || rdev->asic->copy == NULL) {
  366. /* use memcpy */
  367. goto memcpy;
  368. }
  369. if (old_mem->mem_type == TTM_PL_VRAM &&
  370. new_mem->mem_type == TTM_PL_SYSTEM) {
  371. r = radeon_move_vram_ram(bo, evict, interruptible,
  372. no_wait, new_mem);
  373. } else if (old_mem->mem_type == TTM_PL_SYSTEM &&
  374. new_mem->mem_type == TTM_PL_VRAM) {
  375. r = radeon_move_ram_vram(bo, evict, interruptible,
  376. no_wait, new_mem);
  377. } else {
  378. r = radeon_move_blit(bo, evict, no_wait, new_mem, old_mem);
  379. }
  380. if (r) {
  381. memcpy:
  382. r = ttm_bo_move_memcpy(bo, evict, no_wait, new_mem);
  383. }
  384. return r;
  385. }
  386. static int radeon_sync_obj_wait(void *sync_obj, void *sync_arg,
  387. bool lazy, bool interruptible)
  388. {
  389. return radeon_fence_wait((struct radeon_fence *)sync_obj, interruptible);
  390. }
  391. static int radeon_sync_obj_flush(void *sync_obj, void *sync_arg)
  392. {
  393. return 0;
  394. }
  395. static void radeon_sync_obj_unref(void **sync_obj)
  396. {
  397. radeon_fence_unref((struct radeon_fence **)sync_obj);
  398. }
  399. static void *radeon_sync_obj_ref(void *sync_obj)
  400. {
  401. return radeon_fence_ref((struct radeon_fence *)sync_obj);
  402. }
  403. static bool radeon_sync_obj_signaled(void *sync_obj, void *sync_arg)
  404. {
  405. return radeon_fence_signaled((struct radeon_fence *)sync_obj);
  406. }
  407. static struct ttm_bo_driver radeon_bo_driver = {
  408. .create_ttm_backend_entry = &radeon_create_ttm_backend_entry,
  409. .invalidate_caches = &radeon_invalidate_caches,
  410. .init_mem_type = &radeon_init_mem_type,
  411. .evict_flags = &radeon_evict_flags,
  412. .move = &radeon_bo_move,
  413. .verify_access = &radeon_verify_access,
  414. .sync_obj_signaled = &radeon_sync_obj_signaled,
  415. .sync_obj_wait = &radeon_sync_obj_wait,
  416. .sync_obj_flush = &radeon_sync_obj_flush,
  417. .sync_obj_unref = &radeon_sync_obj_unref,
  418. .sync_obj_ref = &radeon_sync_obj_ref,
  419. .move_notify = &radeon_bo_move_notify,
  420. .fault_reserve_notify = &radeon_bo_fault_reserve_notify,
  421. };
  422. int radeon_ttm_init(struct radeon_device *rdev)
  423. {
  424. int r;
  425. r = radeon_ttm_global_init(rdev);
  426. if (r) {
  427. return r;
  428. }
  429. /* No others user of address space so set it to 0 */
  430. r = ttm_bo_device_init(&rdev->mman.bdev,
  431. rdev->mman.bo_global_ref.ref.object,
  432. &radeon_bo_driver, DRM_FILE_PAGE_OFFSET,
  433. rdev->need_dma32);
  434. if (r) {
  435. DRM_ERROR("failed initializing buffer object driver(%d).\n", r);
  436. return r;
  437. }
  438. r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_VRAM,
  439. rdev->mc.real_vram_size >> PAGE_SHIFT);
  440. if (r) {
  441. DRM_ERROR("Failed initializing VRAM heap.\n");
  442. return r;
  443. }
  444. r = radeon_bo_create(rdev, NULL, 256 * 1024, true,
  445. RADEON_GEM_DOMAIN_VRAM,
  446. &rdev->stollen_vga_memory);
  447. if (r) {
  448. return r;
  449. }
  450. r = radeon_bo_reserve(rdev->stollen_vga_memory, false);
  451. if (r)
  452. return r;
  453. r = radeon_bo_pin(rdev->stollen_vga_memory, RADEON_GEM_DOMAIN_VRAM, NULL);
  454. radeon_bo_unreserve(rdev->stollen_vga_memory);
  455. if (r) {
  456. radeon_bo_unref(&rdev->stollen_vga_memory);
  457. return r;
  458. }
  459. DRM_INFO("radeon: %uM of VRAM memory ready\n",
  460. (unsigned)rdev->mc.real_vram_size / (1024 * 1024));
  461. r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_TT,
  462. rdev->mc.gtt_size >> PAGE_SHIFT);
  463. if (r) {
  464. DRM_ERROR("Failed initializing GTT heap.\n");
  465. return r;
  466. }
  467. DRM_INFO("radeon: %uM of GTT memory ready.\n",
  468. (unsigned)(rdev->mc.gtt_size / (1024 * 1024)));
  469. if (unlikely(rdev->mman.bdev.dev_mapping == NULL)) {
  470. rdev->mman.bdev.dev_mapping = rdev->ddev->dev_mapping;
  471. }
  472. r = radeon_ttm_debugfs_init(rdev);
  473. if (r) {
  474. DRM_ERROR("Failed to init debugfs\n");
  475. return r;
  476. }
  477. return 0;
  478. }
  479. void radeon_ttm_fini(struct radeon_device *rdev)
  480. {
  481. int r;
  482. if (rdev->stollen_vga_memory) {
  483. r = radeon_bo_reserve(rdev->stollen_vga_memory, false);
  484. if (r == 0) {
  485. radeon_bo_unpin(rdev->stollen_vga_memory);
  486. radeon_bo_unreserve(rdev->stollen_vga_memory);
  487. }
  488. radeon_bo_unref(&rdev->stollen_vga_memory);
  489. }
  490. ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_VRAM);
  491. ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_TT);
  492. ttm_bo_device_release(&rdev->mman.bdev);
  493. radeon_gart_fini(rdev);
  494. radeon_ttm_global_fini(rdev);
  495. DRM_INFO("radeon: ttm finalized\n");
  496. }
  497. static struct vm_operations_struct radeon_ttm_vm_ops;
  498. static const struct vm_operations_struct *ttm_vm_ops = NULL;
  499. static int radeon_ttm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  500. {
  501. struct ttm_buffer_object *bo;
  502. int r;
  503. bo = (struct ttm_buffer_object *)vma->vm_private_data;
  504. if (bo == NULL) {
  505. return VM_FAULT_NOPAGE;
  506. }
  507. r = ttm_vm_ops->fault(vma, vmf);
  508. return r;
  509. }
  510. int radeon_mmap(struct file *filp, struct vm_area_struct *vma)
  511. {
  512. struct drm_file *file_priv;
  513. struct radeon_device *rdev;
  514. int r;
  515. if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET)) {
  516. return drm_mmap(filp, vma);
  517. }
  518. file_priv = (struct drm_file *)filp->private_data;
  519. rdev = file_priv->minor->dev->dev_private;
  520. if (rdev == NULL) {
  521. return -EINVAL;
  522. }
  523. r = ttm_bo_mmap(filp, vma, &rdev->mman.bdev);
  524. if (unlikely(r != 0)) {
  525. return r;
  526. }
  527. if (unlikely(ttm_vm_ops == NULL)) {
  528. ttm_vm_ops = vma->vm_ops;
  529. radeon_ttm_vm_ops = *ttm_vm_ops;
  530. radeon_ttm_vm_ops.fault = &radeon_ttm_fault;
  531. }
  532. vma->vm_ops = &radeon_ttm_vm_ops;
  533. return 0;
  534. }
  535. /*
  536. * TTM backend functions.
  537. */
  538. struct radeon_ttm_backend {
  539. struct ttm_backend backend;
  540. struct radeon_device *rdev;
  541. unsigned long num_pages;
  542. struct page **pages;
  543. struct page *dummy_read_page;
  544. bool populated;
  545. bool bound;
  546. unsigned offset;
  547. };
  548. static int radeon_ttm_backend_populate(struct ttm_backend *backend,
  549. unsigned long num_pages,
  550. struct page **pages,
  551. struct page *dummy_read_page)
  552. {
  553. struct radeon_ttm_backend *gtt;
  554. gtt = container_of(backend, struct radeon_ttm_backend, backend);
  555. gtt->pages = pages;
  556. gtt->num_pages = num_pages;
  557. gtt->dummy_read_page = dummy_read_page;
  558. gtt->populated = true;
  559. return 0;
  560. }
  561. static void radeon_ttm_backend_clear(struct ttm_backend *backend)
  562. {
  563. struct radeon_ttm_backend *gtt;
  564. gtt = container_of(backend, struct radeon_ttm_backend, backend);
  565. gtt->pages = NULL;
  566. gtt->num_pages = 0;
  567. gtt->dummy_read_page = NULL;
  568. gtt->populated = false;
  569. gtt->bound = false;
  570. }
  571. static int radeon_ttm_backend_bind(struct ttm_backend *backend,
  572. struct ttm_mem_reg *bo_mem)
  573. {
  574. struct radeon_ttm_backend *gtt;
  575. int r;
  576. gtt = container_of(backend, struct radeon_ttm_backend, backend);
  577. gtt->offset = bo_mem->mm_node->start << PAGE_SHIFT;
  578. if (!gtt->num_pages) {
  579. WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n", gtt->num_pages, bo_mem, backend);
  580. }
  581. r = radeon_gart_bind(gtt->rdev, gtt->offset,
  582. gtt->num_pages, gtt->pages);
  583. if (r) {
  584. DRM_ERROR("failed to bind %lu pages at 0x%08X\n",
  585. gtt->num_pages, gtt->offset);
  586. return r;
  587. }
  588. gtt->bound = true;
  589. return 0;
  590. }
  591. static int radeon_ttm_backend_unbind(struct ttm_backend *backend)
  592. {
  593. struct radeon_ttm_backend *gtt;
  594. gtt = container_of(backend, struct radeon_ttm_backend, backend);
  595. radeon_gart_unbind(gtt->rdev, gtt->offset, gtt->num_pages);
  596. gtt->bound = false;
  597. return 0;
  598. }
  599. static void radeon_ttm_backend_destroy(struct ttm_backend *backend)
  600. {
  601. struct radeon_ttm_backend *gtt;
  602. gtt = container_of(backend, struct radeon_ttm_backend, backend);
  603. if (gtt->bound) {
  604. radeon_ttm_backend_unbind(backend);
  605. }
  606. kfree(gtt);
  607. }
  608. static struct ttm_backend_func radeon_backend_func = {
  609. .populate = &radeon_ttm_backend_populate,
  610. .clear = &radeon_ttm_backend_clear,
  611. .bind = &radeon_ttm_backend_bind,
  612. .unbind = &radeon_ttm_backend_unbind,
  613. .destroy = &radeon_ttm_backend_destroy,
  614. };
  615. struct ttm_backend *radeon_ttm_backend_create(struct radeon_device *rdev)
  616. {
  617. struct radeon_ttm_backend *gtt;
  618. gtt = kzalloc(sizeof(struct radeon_ttm_backend), GFP_KERNEL);
  619. if (gtt == NULL) {
  620. return NULL;
  621. }
  622. gtt->backend.bdev = &rdev->mman.bdev;
  623. gtt->backend.flags = 0;
  624. gtt->backend.func = &radeon_backend_func;
  625. gtt->rdev = rdev;
  626. gtt->pages = NULL;
  627. gtt->num_pages = 0;
  628. gtt->dummy_read_page = NULL;
  629. gtt->populated = false;
  630. gtt->bound = false;
  631. return &gtt->backend;
  632. }
  633. #define RADEON_DEBUGFS_MEM_TYPES 2
  634. #if defined(CONFIG_DEBUG_FS)
  635. static int radeon_mm_dump_table(struct seq_file *m, void *data)
  636. {
  637. struct drm_info_node *node = (struct drm_info_node *)m->private;
  638. struct drm_mm *mm = (struct drm_mm *)node->info_ent->data;
  639. struct drm_device *dev = node->minor->dev;
  640. struct radeon_device *rdev = dev->dev_private;
  641. int ret;
  642. struct ttm_bo_global *glob = rdev->mman.bdev.glob;
  643. spin_lock(&glob->lru_lock);
  644. ret = drm_mm_dump_table(m, mm);
  645. spin_unlock(&glob->lru_lock);
  646. return ret;
  647. }
  648. #endif
  649. static int radeon_ttm_debugfs_init(struct radeon_device *rdev)
  650. {
  651. #if defined(CONFIG_DEBUG_FS)
  652. static struct drm_info_list radeon_mem_types_list[RADEON_DEBUGFS_MEM_TYPES];
  653. static char radeon_mem_types_names[RADEON_DEBUGFS_MEM_TYPES][32];
  654. unsigned i;
  655. for (i = 0; i < RADEON_DEBUGFS_MEM_TYPES; i++) {
  656. if (i == 0)
  657. sprintf(radeon_mem_types_names[i], "radeon_vram_mm");
  658. else
  659. sprintf(radeon_mem_types_names[i], "radeon_gtt_mm");
  660. radeon_mem_types_list[i].name = radeon_mem_types_names[i];
  661. radeon_mem_types_list[i].show = &radeon_mm_dump_table;
  662. radeon_mem_types_list[i].driver_features = 0;
  663. if (i == 0)
  664. radeon_mem_types_list[i].data = &rdev->mman.bdev.man[TTM_PL_VRAM].manager;
  665. else
  666. radeon_mem_types_list[i].data = &rdev->mman.bdev.man[TTM_PL_TT].manager;
  667. }
  668. return radeon_debugfs_add_files(rdev, radeon_mem_types_list, RADEON_DEBUGFS_MEM_TYPES);
  669. #endif
  670. return 0;
  671. }