radeon_ttm.c 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770
  1. /*
  2. * Copyright 2009 Jerome Glisse.
  3. * All Rights Reserved.
  4. *
  5. * Permission is hereby granted, free of charge, to any person obtaining a
  6. * copy of this software and associated documentation files (the
  7. * "Software"), to deal in the Software without restriction, including
  8. * without limitation the rights to use, copy, modify, merge, publish,
  9. * distribute, sub license, and/or sell copies of the Software, and to
  10. * permit persons to whom the Software is furnished to do so, subject to
  11. * the following conditions:
  12. *
  13. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15. * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16. * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17. * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18. * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19. * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20. *
  21. * The above copyright notice and this permission notice (including the
  22. * next paragraph) shall be included in all copies or substantial portions
  23. * of the Software.
  24. *
  25. */
  26. /*
  27. * Authors:
  28. * Jerome Glisse <glisse@freedesktop.org>
  29. * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
  30. * Dave Airlie
  31. */
  32. #include <ttm/ttm_bo_api.h>
  33. #include <ttm/ttm_bo_driver.h>
  34. #include <ttm/ttm_placement.h>
  35. #include <ttm/ttm_module.h>
  36. #include <drm/drmP.h>
  37. #include <drm/radeon_drm.h>
  38. #include <linux/seq_file.h>
  39. #include <linux/slab.h>
  40. #include "radeon_reg.h"
  41. #include "radeon.h"
  42. #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
  43. static int radeon_ttm_debugfs_init(struct radeon_device *rdev);
  44. static struct radeon_device *radeon_get_rdev(struct ttm_bo_device *bdev)
  45. {
  46. struct radeon_mman *mman;
  47. struct radeon_device *rdev;
  48. mman = container_of(bdev, struct radeon_mman, bdev);
  49. rdev = container_of(mman, struct radeon_device, mman);
  50. return rdev;
  51. }
  52. /*
  53. * Global memory.
  54. */
  55. static int radeon_ttm_mem_global_init(struct ttm_global_reference *ref)
  56. {
  57. return ttm_mem_global_init(ref->object);
  58. }
  59. static void radeon_ttm_mem_global_release(struct ttm_global_reference *ref)
  60. {
  61. ttm_mem_global_release(ref->object);
  62. }
  63. static int radeon_ttm_global_init(struct radeon_device *rdev)
  64. {
  65. struct ttm_global_reference *global_ref;
  66. int r;
  67. rdev->mman.mem_global_referenced = false;
  68. global_ref = &rdev->mman.mem_global_ref;
  69. global_ref->global_type = TTM_GLOBAL_TTM_MEM;
  70. global_ref->size = sizeof(struct ttm_mem_global);
  71. global_ref->init = &radeon_ttm_mem_global_init;
  72. global_ref->release = &radeon_ttm_mem_global_release;
  73. r = ttm_global_item_ref(global_ref);
  74. if (r != 0) {
  75. DRM_ERROR("Failed setting up TTM memory accounting "
  76. "subsystem.\n");
  77. return r;
  78. }
  79. rdev->mman.bo_global_ref.mem_glob =
  80. rdev->mman.mem_global_ref.object;
  81. global_ref = &rdev->mman.bo_global_ref.ref;
  82. global_ref->global_type = TTM_GLOBAL_TTM_BO;
  83. global_ref->size = sizeof(struct ttm_bo_global);
  84. global_ref->init = &ttm_bo_global_init;
  85. global_ref->release = &ttm_bo_global_release;
  86. r = ttm_global_item_ref(global_ref);
  87. if (r != 0) {
  88. DRM_ERROR("Failed setting up TTM BO subsystem.\n");
  89. ttm_global_item_unref(&rdev->mman.mem_global_ref);
  90. return r;
  91. }
  92. rdev->mman.mem_global_referenced = true;
  93. return 0;
  94. }
  95. static void radeon_ttm_global_fini(struct radeon_device *rdev)
  96. {
  97. if (rdev->mman.mem_global_referenced) {
  98. ttm_global_item_unref(&rdev->mman.bo_global_ref.ref);
  99. ttm_global_item_unref(&rdev->mman.mem_global_ref);
  100. rdev->mman.mem_global_referenced = false;
  101. }
  102. }
  103. struct ttm_backend *radeon_ttm_backend_create(struct radeon_device *rdev);
  104. static struct ttm_backend*
  105. radeon_create_ttm_backend_entry(struct ttm_bo_device *bdev)
  106. {
  107. struct radeon_device *rdev;
  108. rdev = radeon_get_rdev(bdev);
  109. #if __OS_HAS_AGP
  110. if (rdev->flags & RADEON_IS_AGP) {
  111. return ttm_agp_backend_init(bdev, rdev->ddev->agp->bridge);
  112. } else
  113. #endif
  114. {
  115. return radeon_ttm_backend_create(rdev);
  116. }
  117. }
  118. static int radeon_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
  119. {
  120. return 0;
  121. }
  122. static int radeon_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
  123. struct ttm_mem_type_manager *man)
  124. {
  125. struct radeon_device *rdev;
  126. rdev = radeon_get_rdev(bdev);
  127. switch (type) {
  128. case TTM_PL_SYSTEM:
  129. /* System memory */
  130. man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
  131. man->available_caching = TTM_PL_MASK_CACHING;
  132. man->default_caching = TTM_PL_FLAG_CACHED;
  133. break;
  134. case TTM_PL_TT:
  135. man->gpu_offset = rdev->mc.gtt_start;
  136. man->available_caching = TTM_PL_MASK_CACHING;
  137. man->default_caching = TTM_PL_FLAG_CACHED;
  138. man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA;
  139. #if __OS_HAS_AGP
  140. if (rdev->flags & RADEON_IS_AGP) {
  141. if (!(drm_core_has_AGP(rdev->ddev) && rdev->ddev->agp)) {
  142. DRM_ERROR("AGP is not enabled for memory type %u\n",
  143. (unsigned)type);
  144. return -EINVAL;
  145. }
  146. man->io_offset = rdev->mc.agp_base;
  147. man->io_size = rdev->mc.gtt_size;
  148. man->io_addr = NULL;
  149. if (!rdev->ddev->agp->cant_use_aperture)
  150. man->flags = TTM_MEMTYPE_FLAG_NEEDS_IOREMAP |
  151. TTM_MEMTYPE_FLAG_MAPPABLE;
  152. man->available_caching = TTM_PL_FLAG_UNCACHED |
  153. TTM_PL_FLAG_WC;
  154. man->default_caching = TTM_PL_FLAG_WC;
  155. } else
  156. #endif
  157. {
  158. man->io_offset = 0;
  159. man->io_size = 0;
  160. man->io_addr = NULL;
  161. }
  162. break;
  163. case TTM_PL_VRAM:
  164. /* "On-card" video ram */
  165. man->gpu_offset = rdev->mc.vram_start;
  166. man->flags = TTM_MEMTYPE_FLAG_FIXED |
  167. TTM_MEMTYPE_FLAG_NEEDS_IOREMAP |
  168. TTM_MEMTYPE_FLAG_MAPPABLE;
  169. man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC;
  170. man->default_caching = TTM_PL_FLAG_WC;
  171. man->io_addr = NULL;
  172. man->io_offset = rdev->mc.aper_base;
  173. man->io_size = rdev->mc.aper_size;
  174. break;
  175. default:
  176. DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);
  177. return -EINVAL;
  178. }
  179. return 0;
  180. }
  181. static void radeon_evict_flags(struct ttm_buffer_object *bo,
  182. struct ttm_placement *placement)
  183. {
  184. struct radeon_bo *rbo;
  185. static u32 placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM;
  186. if (!radeon_ttm_bo_is_radeon_bo(bo)) {
  187. placement->fpfn = 0;
  188. placement->lpfn = 0;
  189. placement->placement = &placements;
  190. placement->busy_placement = &placements;
  191. placement->num_placement = 1;
  192. placement->num_busy_placement = 1;
  193. return;
  194. }
  195. rbo = container_of(bo, struct radeon_bo, tbo);
  196. switch (bo->mem.mem_type) {
  197. case TTM_PL_VRAM:
  198. if (rbo->rdev->cp.ready == false)
  199. radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU);
  200. else
  201. radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_GTT);
  202. break;
  203. case TTM_PL_TT:
  204. default:
  205. radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU);
  206. }
  207. *placement = rbo->placement;
  208. }
  209. static int radeon_verify_access(struct ttm_buffer_object *bo, struct file *filp)
  210. {
  211. return 0;
  212. }
  213. static void radeon_move_null(struct ttm_buffer_object *bo,
  214. struct ttm_mem_reg *new_mem)
  215. {
  216. struct ttm_mem_reg *old_mem = &bo->mem;
  217. BUG_ON(old_mem->mm_node != NULL);
  218. *old_mem = *new_mem;
  219. new_mem->mm_node = NULL;
  220. }
  221. static int radeon_move_blit(struct ttm_buffer_object *bo,
  222. bool evict, int no_wait,
  223. struct ttm_mem_reg *new_mem,
  224. struct ttm_mem_reg *old_mem)
  225. {
  226. struct radeon_device *rdev;
  227. uint64_t old_start, new_start;
  228. struct radeon_fence *fence;
  229. int r;
  230. rdev = radeon_get_rdev(bo->bdev);
  231. r = radeon_fence_create(rdev, &fence);
  232. if (unlikely(r)) {
  233. return r;
  234. }
  235. old_start = old_mem->mm_node->start << PAGE_SHIFT;
  236. new_start = new_mem->mm_node->start << PAGE_SHIFT;
  237. switch (old_mem->mem_type) {
  238. case TTM_PL_VRAM:
  239. old_start += rdev->mc.vram_start;
  240. break;
  241. case TTM_PL_TT:
  242. old_start += rdev->mc.gtt_start;
  243. break;
  244. default:
  245. DRM_ERROR("Unknown placement %d\n", old_mem->mem_type);
  246. return -EINVAL;
  247. }
  248. switch (new_mem->mem_type) {
  249. case TTM_PL_VRAM:
  250. new_start += rdev->mc.vram_start;
  251. break;
  252. case TTM_PL_TT:
  253. new_start += rdev->mc.gtt_start;
  254. break;
  255. default:
  256. DRM_ERROR("Unknown placement %d\n", old_mem->mem_type);
  257. return -EINVAL;
  258. }
  259. if (!rdev->cp.ready) {
  260. DRM_ERROR("Trying to move memory with CP turned off.\n");
  261. return -EINVAL;
  262. }
  263. r = radeon_copy(rdev, old_start, new_start, new_mem->num_pages, fence);
  264. /* FIXME: handle copy error */
  265. r = ttm_bo_move_accel_cleanup(bo, (void *)fence, NULL,
  266. evict, no_wait, new_mem);
  267. radeon_fence_unref(&fence);
  268. return r;
  269. }
  270. static int radeon_move_vram_ram(struct ttm_buffer_object *bo,
  271. bool evict, bool interruptible, bool no_wait,
  272. struct ttm_mem_reg *new_mem)
  273. {
  274. struct radeon_device *rdev;
  275. struct ttm_mem_reg *old_mem = &bo->mem;
  276. struct ttm_mem_reg tmp_mem;
  277. u32 placements;
  278. struct ttm_placement placement;
  279. int r;
  280. rdev = radeon_get_rdev(bo->bdev);
  281. tmp_mem = *new_mem;
  282. tmp_mem.mm_node = NULL;
  283. placement.fpfn = 0;
  284. placement.lpfn = 0;
  285. placement.num_placement = 1;
  286. placement.placement = &placements;
  287. placement.num_busy_placement = 1;
  288. placement.busy_placement = &placements;
  289. placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
  290. r = ttm_bo_mem_space(bo, &placement, &tmp_mem,
  291. interruptible, no_wait);
  292. if (unlikely(r)) {
  293. return r;
  294. }
  295. r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement);
  296. if (unlikely(r)) {
  297. goto out_cleanup;
  298. }
  299. r = ttm_tt_bind(bo->ttm, &tmp_mem);
  300. if (unlikely(r)) {
  301. goto out_cleanup;
  302. }
  303. r = radeon_move_blit(bo, true, no_wait, &tmp_mem, old_mem);
  304. if (unlikely(r)) {
  305. goto out_cleanup;
  306. }
  307. r = ttm_bo_move_ttm(bo, true, no_wait, new_mem);
  308. out_cleanup:
  309. if (tmp_mem.mm_node) {
  310. struct ttm_bo_global *glob = rdev->mman.bdev.glob;
  311. spin_lock(&glob->lru_lock);
  312. drm_mm_put_block(tmp_mem.mm_node);
  313. spin_unlock(&glob->lru_lock);
  314. return r;
  315. }
  316. return r;
  317. }
  318. static int radeon_move_ram_vram(struct ttm_buffer_object *bo,
  319. bool evict, bool interruptible, bool no_wait,
  320. struct ttm_mem_reg *new_mem)
  321. {
  322. struct radeon_device *rdev;
  323. struct ttm_mem_reg *old_mem = &bo->mem;
  324. struct ttm_mem_reg tmp_mem;
  325. struct ttm_placement placement;
  326. u32 placements;
  327. int r;
  328. rdev = radeon_get_rdev(bo->bdev);
  329. tmp_mem = *new_mem;
  330. tmp_mem.mm_node = NULL;
  331. placement.fpfn = 0;
  332. placement.lpfn = 0;
  333. placement.num_placement = 1;
  334. placement.placement = &placements;
  335. placement.num_busy_placement = 1;
  336. placement.busy_placement = &placements;
  337. placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
  338. r = ttm_bo_mem_space(bo, &placement, &tmp_mem, interruptible, no_wait);
  339. if (unlikely(r)) {
  340. return r;
  341. }
  342. r = ttm_bo_move_ttm(bo, true, no_wait, &tmp_mem);
  343. if (unlikely(r)) {
  344. goto out_cleanup;
  345. }
  346. r = radeon_move_blit(bo, true, no_wait, new_mem, old_mem);
  347. if (unlikely(r)) {
  348. goto out_cleanup;
  349. }
  350. out_cleanup:
  351. if (tmp_mem.mm_node) {
  352. struct ttm_bo_global *glob = rdev->mman.bdev.glob;
  353. spin_lock(&glob->lru_lock);
  354. drm_mm_put_block(tmp_mem.mm_node);
  355. spin_unlock(&glob->lru_lock);
  356. return r;
  357. }
  358. return r;
  359. }
  360. static int radeon_bo_move(struct ttm_buffer_object *bo,
  361. bool evict, bool interruptible, bool no_wait,
  362. struct ttm_mem_reg *new_mem)
  363. {
  364. struct radeon_device *rdev;
  365. struct ttm_mem_reg *old_mem = &bo->mem;
  366. int r;
  367. rdev = radeon_get_rdev(bo->bdev);
  368. if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
  369. radeon_move_null(bo, new_mem);
  370. return 0;
  371. }
  372. if ((old_mem->mem_type == TTM_PL_TT &&
  373. new_mem->mem_type == TTM_PL_SYSTEM) ||
  374. (old_mem->mem_type == TTM_PL_SYSTEM &&
  375. new_mem->mem_type == TTM_PL_TT)) {
  376. /* bind is enough */
  377. radeon_move_null(bo, new_mem);
  378. return 0;
  379. }
  380. if (!rdev->cp.ready || rdev->asic->copy == NULL) {
  381. /* use memcpy */
  382. goto memcpy;
  383. }
  384. if (old_mem->mem_type == TTM_PL_VRAM &&
  385. new_mem->mem_type == TTM_PL_SYSTEM) {
  386. r = radeon_move_vram_ram(bo, evict, interruptible,
  387. no_wait, new_mem);
  388. } else if (old_mem->mem_type == TTM_PL_SYSTEM &&
  389. new_mem->mem_type == TTM_PL_VRAM) {
  390. r = radeon_move_ram_vram(bo, evict, interruptible,
  391. no_wait, new_mem);
  392. } else {
  393. r = radeon_move_blit(bo, evict, no_wait, new_mem, old_mem);
  394. }
  395. if (r) {
  396. memcpy:
  397. r = ttm_bo_move_memcpy(bo, evict, no_wait, new_mem);
  398. }
  399. return r;
  400. }
  401. static int radeon_sync_obj_wait(void *sync_obj, void *sync_arg,
  402. bool lazy, bool interruptible)
  403. {
  404. return radeon_fence_wait((struct radeon_fence *)sync_obj, interruptible);
  405. }
  406. static int radeon_sync_obj_flush(void *sync_obj, void *sync_arg)
  407. {
  408. return 0;
  409. }
  410. static void radeon_sync_obj_unref(void **sync_obj)
  411. {
  412. radeon_fence_unref((struct radeon_fence **)sync_obj);
  413. }
  414. static void *radeon_sync_obj_ref(void *sync_obj)
  415. {
  416. return radeon_fence_ref((struct radeon_fence *)sync_obj);
  417. }
  418. static bool radeon_sync_obj_signaled(void *sync_obj, void *sync_arg)
  419. {
  420. return radeon_fence_signaled((struct radeon_fence *)sync_obj);
  421. }
  422. static struct ttm_bo_driver radeon_bo_driver = {
  423. .create_ttm_backend_entry = &radeon_create_ttm_backend_entry,
  424. .invalidate_caches = &radeon_invalidate_caches,
  425. .init_mem_type = &radeon_init_mem_type,
  426. .evict_flags = &radeon_evict_flags,
  427. .move = &radeon_bo_move,
  428. .verify_access = &radeon_verify_access,
  429. .sync_obj_signaled = &radeon_sync_obj_signaled,
  430. .sync_obj_wait = &radeon_sync_obj_wait,
  431. .sync_obj_flush = &radeon_sync_obj_flush,
  432. .sync_obj_unref = &radeon_sync_obj_unref,
  433. .sync_obj_ref = &radeon_sync_obj_ref,
  434. .move_notify = &radeon_bo_move_notify,
  435. .fault_reserve_notify = &radeon_bo_fault_reserve_notify,
  436. };
  437. int radeon_ttm_init(struct radeon_device *rdev)
  438. {
  439. int r;
  440. r = radeon_ttm_global_init(rdev);
  441. if (r) {
  442. return r;
  443. }
  444. /* No others user of address space so set it to 0 */
  445. r = ttm_bo_device_init(&rdev->mman.bdev,
  446. rdev->mman.bo_global_ref.ref.object,
  447. &radeon_bo_driver, DRM_FILE_PAGE_OFFSET,
  448. rdev->need_dma32);
  449. if (r) {
  450. DRM_ERROR("failed initializing buffer object driver(%d).\n", r);
  451. return r;
  452. }
  453. rdev->mman.initialized = true;
  454. r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_VRAM,
  455. rdev->mc.real_vram_size >> PAGE_SHIFT);
  456. if (r) {
  457. DRM_ERROR("Failed initializing VRAM heap.\n");
  458. return r;
  459. }
  460. r = radeon_bo_create(rdev, NULL, 256 * 1024, true,
  461. RADEON_GEM_DOMAIN_VRAM,
  462. &rdev->stollen_vga_memory);
  463. if (r) {
  464. return r;
  465. }
  466. r = radeon_bo_reserve(rdev->stollen_vga_memory, false);
  467. if (r)
  468. return r;
  469. r = radeon_bo_pin(rdev->stollen_vga_memory, RADEON_GEM_DOMAIN_VRAM, NULL);
  470. radeon_bo_unreserve(rdev->stollen_vga_memory);
  471. if (r) {
  472. radeon_bo_unref(&rdev->stollen_vga_memory);
  473. return r;
  474. }
  475. DRM_INFO("radeon: %uM of VRAM memory ready\n",
  476. (unsigned)rdev->mc.real_vram_size / (1024 * 1024));
  477. r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_TT,
  478. rdev->mc.gtt_size >> PAGE_SHIFT);
  479. if (r) {
  480. DRM_ERROR("Failed initializing GTT heap.\n");
  481. return r;
  482. }
  483. DRM_INFO("radeon: %uM of GTT memory ready.\n",
  484. (unsigned)(rdev->mc.gtt_size / (1024 * 1024)));
  485. if (unlikely(rdev->mman.bdev.dev_mapping == NULL)) {
  486. rdev->mman.bdev.dev_mapping = rdev->ddev->dev_mapping;
  487. }
  488. r = radeon_ttm_debugfs_init(rdev);
  489. if (r) {
  490. DRM_ERROR("Failed to init debugfs\n");
  491. return r;
  492. }
  493. return 0;
  494. }
  495. void radeon_ttm_fini(struct radeon_device *rdev)
  496. {
  497. int r;
  498. if (!rdev->mman.initialized)
  499. return;
  500. if (rdev->stollen_vga_memory) {
  501. r = radeon_bo_reserve(rdev->stollen_vga_memory, false);
  502. if (r == 0) {
  503. radeon_bo_unpin(rdev->stollen_vga_memory);
  504. radeon_bo_unreserve(rdev->stollen_vga_memory);
  505. }
  506. radeon_bo_unref(&rdev->stollen_vga_memory);
  507. }
  508. ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_VRAM);
  509. ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_TT);
  510. ttm_bo_device_release(&rdev->mman.bdev);
  511. radeon_gart_fini(rdev);
  512. radeon_ttm_global_fini(rdev);
  513. rdev->mman.initialized = false;
  514. DRM_INFO("radeon: ttm finalized\n");
  515. }
  516. static struct vm_operations_struct radeon_ttm_vm_ops;
  517. static const struct vm_operations_struct *ttm_vm_ops = NULL;
  518. static int radeon_ttm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  519. {
  520. struct ttm_buffer_object *bo;
  521. int r;
  522. bo = (struct ttm_buffer_object *)vma->vm_private_data;
  523. if (bo == NULL) {
  524. return VM_FAULT_NOPAGE;
  525. }
  526. r = ttm_vm_ops->fault(vma, vmf);
  527. return r;
  528. }
  529. int radeon_mmap(struct file *filp, struct vm_area_struct *vma)
  530. {
  531. struct drm_file *file_priv;
  532. struct radeon_device *rdev;
  533. int r;
  534. if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET)) {
  535. return drm_mmap(filp, vma);
  536. }
  537. file_priv = (struct drm_file *)filp->private_data;
  538. rdev = file_priv->minor->dev->dev_private;
  539. if (rdev == NULL) {
  540. return -EINVAL;
  541. }
  542. r = ttm_bo_mmap(filp, vma, &rdev->mman.bdev);
  543. if (unlikely(r != 0)) {
  544. return r;
  545. }
  546. if (unlikely(ttm_vm_ops == NULL)) {
  547. ttm_vm_ops = vma->vm_ops;
  548. radeon_ttm_vm_ops = *ttm_vm_ops;
  549. radeon_ttm_vm_ops.fault = &radeon_ttm_fault;
  550. }
  551. vma->vm_ops = &radeon_ttm_vm_ops;
  552. return 0;
  553. }
  554. /*
  555. * TTM backend functions.
  556. */
  557. struct radeon_ttm_backend {
  558. struct ttm_backend backend;
  559. struct radeon_device *rdev;
  560. unsigned long num_pages;
  561. struct page **pages;
  562. struct page *dummy_read_page;
  563. bool populated;
  564. bool bound;
  565. unsigned offset;
  566. };
  567. static int radeon_ttm_backend_populate(struct ttm_backend *backend,
  568. unsigned long num_pages,
  569. struct page **pages,
  570. struct page *dummy_read_page)
  571. {
  572. struct radeon_ttm_backend *gtt;
  573. gtt = container_of(backend, struct radeon_ttm_backend, backend);
  574. gtt->pages = pages;
  575. gtt->num_pages = num_pages;
  576. gtt->dummy_read_page = dummy_read_page;
  577. gtt->populated = true;
  578. return 0;
  579. }
  580. static void radeon_ttm_backend_clear(struct ttm_backend *backend)
  581. {
  582. struct radeon_ttm_backend *gtt;
  583. gtt = container_of(backend, struct radeon_ttm_backend, backend);
  584. gtt->pages = NULL;
  585. gtt->num_pages = 0;
  586. gtt->dummy_read_page = NULL;
  587. gtt->populated = false;
  588. gtt->bound = false;
  589. }
  590. static int radeon_ttm_backend_bind(struct ttm_backend *backend,
  591. struct ttm_mem_reg *bo_mem)
  592. {
  593. struct radeon_ttm_backend *gtt;
  594. int r;
  595. gtt = container_of(backend, struct radeon_ttm_backend, backend);
  596. gtt->offset = bo_mem->mm_node->start << PAGE_SHIFT;
  597. if (!gtt->num_pages) {
  598. WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n", gtt->num_pages, bo_mem, backend);
  599. }
  600. r = radeon_gart_bind(gtt->rdev, gtt->offset,
  601. gtt->num_pages, gtt->pages);
  602. if (r) {
  603. DRM_ERROR("failed to bind %lu pages at 0x%08X\n",
  604. gtt->num_pages, gtt->offset);
  605. return r;
  606. }
  607. gtt->bound = true;
  608. return 0;
  609. }
  610. static int radeon_ttm_backend_unbind(struct ttm_backend *backend)
  611. {
  612. struct radeon_ttm_backend *gtt;
  613. gtt = container_of(backend, struct radeon_ttm_backend, backend);
  614. radeon_gart_unbind(gtt->rdev, gtt->offset, gtt->num_pages);
  615. gtt->bound = false;
  616. return 0;
  617. }
  618. static void radeon_ttm_backend_destroy(struct ttm_backend *backend)
  619. {
  620. struct radeon_ttm_backend *gtt;
  621. gtt = container_of(backend, struct radeon_ttm_backend, backend);
  622. if (gtt->bound) {
  623. radeon_ttm_backend_unbind(backend);
  624. }
  625. kfree(gtt);
  626. }
  627. static struct ttm_backend_func radeon_backend_func = {
  628. .populate = &radeon_ttm_backend_populate,
  629. .clear = &radeon_ttm_backend_clear,
  630. .bind = &radeon_ttm_backend_bind,
  631. .unbind = &radeon_ttm_backend_unbind,
  632. .destroy = &radeon_ttm_backend_destroy,
  633. };
  634. struct ttm_backend *radeon_ttm_backend_create(struct radeon_device *rdev)
  635. {
  636. struct radeon_ttm_backend *gtt;
  637. gtt = kzalloc(sizeof(struct radeon_ttm_backend), GFP_KERNEL);
  638. if (gtt == NULL) {
  639. return NULL;
  640. }
  641. gtt->backend.bdev = &rdev->mman.bdev;
  642. gtt->backend.flags = 0;
  643. gtt->backend.func = &radeon_backend_func;
  644. gtt->rdev = rdev;
  645. gtt->pages = NULL;
  646. gtt->num_pages = 0;
  647. gtt->dummy_read_page = NULL;
  648. gtt->populated = false;
  649. gtt->bound = false;
  650. return &gtt->backend;
  651. }
  652. #define RADEON_DEBUGFS_MEM_TYPES 2
  653. #if defined(CONFIG_DEBUG_FS)
  654. static int radeon_mm_dump_table(struct seq_file *m, void *data)
  655. {
  656. struct drm_info_node *node = (struct drm_info_node *)m->private;
  657. struct drm_mm *mm = (struct drm_mm *)node->info_ent->data;
  658. struct drm_device *dev = node->minor->dev;
  659. struct radeon_device *rdev = dev->dev_private;
  660. int ret;
  661. struct ttm_bo_global *glob = rdev->mman.bdev.glob;
  662. spin_lock(&glob->lru_lock);
  663. ret = drm_mm_dump_table(m, mm);
  664. spin_unlock(&glob->lru_lock);
  665. return ret;
  666. }
  667. #endif
  668. static int radeon_ttm_debugfs_init(struct radeon_device *rdev)
  669. {
  670. #if defined(CONFIG_DEBUG_FS)
  671. static struct drm_info_list radeon_mem_types_list[RADEON_DEBUGFS_MEM_TYPES];
  672. static char radeon_mem_types_names[RADEON_DEBUGFS_MEM_TYPES][32];
  673. unsigned i;
  674. for (i = 0; i < RADEON_DEBUGFS_MEM_TYPES; i++) {
  675. if (i == 0)
  676. sprintf(radeon_mem_types_names[i], "radeon_vram_mm");
  677. else
  678. sprintf(radeon_mem_types_names[i], "radeon_gtt_mm");
  679. radeon_mem_types_list[i].name = radeon_mem_types_names[i];
  680. radeon_mem_types_list[i].show = &radeon_mm_dump_table;
  681. radeon_mem_types_list[i].driver_features = 0;
  682. if (i == 0)
  683. radeon_mem_types_list[i].data = &rdev->mman.bdev.man[TTM_PL_VRAM].manager;
  684. else
  685. radeon_mem_types_list[i].data = &rdev->mman.bdev.man[TTM_PL_TT].manager;
  686. }
  687. return radeon_debugfs_add_files(rdev, radeon_mem_types_list, RADEON_DEBUGFS_MEM_TYPES);
  688. #endif
  689. return 0;
  690. }