ttm_page_alloc.c 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931
  1. /*
  2. * Copyright (c) Red Hat Inc.
  3. * Permission is hereby granted, free of charge, to any person obtaining a
  4. * copy of this software and associated documentation files (the "Software"),
  5. * to deal in the Software without restriction, including without limitation
  6. * the rights to use, copy, modify, merge, publish, distribute, sub license,
  7. * and/or sell copies of the Software, and to permit persons to whom the
  8. * Software is furnished to do so, subject to the following conditions:
  9. *
  10. * The above copyright notice and this permission notice (including the
  11. * next paragraph) shall be included in all copies or substantial portions
  12. * of the Software.
  13. *
  14. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16. * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  17. * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  19. * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  20. * DEALINGS IN THE SOFTWARE.
  21. *
  22. * Authors: Dave Airlie <airlied@redhat.com>
  23. * Jerome Glisse <jglisse@redhat.com>
  24. * Pauli Nieminen <suokkos@gmail.com>
  25. */
  26. /* simple list based uncached page pool
  27. * - Pool collects resently freed pages for reuse
  28. * - Use page->lru to keep a free list
  29. * - doesn't track currently in use pages
  30. */
  31. #define pr_fmt(fmt) "[TTM] " fmt
  32. #include <linux/list.h>
  33. #include <linux/spinlock.h>
  34. #include <linux/highmem.h>
  35. #include <linux/mm_types.h>
  36. #include <linux/module.h>
  37. #include <linux/mm.h>
  38. #include <linux/seq_file.h> /* for seq_printf */
  39. #include <linux/slab.h>
  40. #include <linux/dma-mapping.h>
  41. #include <linux/atomic.h>
  42. #include <drm/ttm/ttm_bo_driver.h>
  43. #include <drm/ttm/ttm_page_alloc.h>
  44. #ifdef TTM_HAS_AGP
  45. #include <asm/agp.h>
  46. #endif
  47. #define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *))
  48. #define SMALL_ALLOCATION 16
  49. #define FREE_ALL_PAGES (~0U)
  50. /* times are in msecs */
  51. #define PAGE_FREE_INTERVAL 1000
  52. /**
  53. * struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages.
  54. *
  55. * @lock: Protects the shared pool from concurrnet access. Must be used with
  56. * irqsave/irqrestore variants because pool allocator maybe called from
  57. * delayed work.
  58. * @fill_lock: Prevent concurrent calls to fill.
  59. * @list: Pool of free uc/wc pages for fast reuse.
  60. * @gfp_flags: Flags to pass for alloc_page.
  61. * @npages: Number of pages in pool.
  62. */
  63. struct ttm_page_pool {
  64. spinlock_t lock;
  65. bool fill_lock;
  66. struct list_head list;
  67. gfp_t gfp_flags;
  68. unsigned npages;
  69. char *name;
  70. unsigned long nfrees;
  71. unsigned long nrefills;
  72. };
  73. /**
  74. * Limits for the pool. They are handled without locks because only place where
  75. * they may change is in sysfs store. They won't have immediate effect anyway
  76. * so forcing serialization to access them is pointless.
  77. */
  78. struct ttm_pool_opts {
  79. unsigned alloc_size;
  80. unsigned max_size;
  81. unsigned small;
  82. };
  83. #define NUM_POOLS 4
  84. /**
  85. * struct ttm_pool_manager - Holds memory pools for fst allocation
  86. *
  87. * Manager is read only object for pool code so it doesn't need locking.
  88. *
  89. * @free_interval: minimum number of jiffies between freeing pages from pool.
  90. * @page_alloc_inited: reference counting for pool allocation.
  91. * @work: Work that is used to shrink the pool. Work is only run when there is
  92. * some pages to free.
  93. * @small_allocation: Limit in number of pages what is small allocation.
  94. *
  95. * @pools: All pool objects in use.
  96. **/
  97. struct ttm_pool_manager {
  98. struct kobject kobj;
  99. struct shrinker mm_shrink;
  100. struct ttm_pool_opts options;
  101. union {
  102. struct ttm_page_pool pools[NUM_POOLS];
  103. struct {
  104. struct ttm_page_pool wc_pool;
  105. struct ttm_page_pool uc_pool;
  106. struct ttm_page_pool wc_pool_dma32;
  107. struct ttm_page_pool uc_pool_dma32;
  108. } ;
  109. };
  110. };
  111. static struct attribute ttm_page_pool_max = {
  112. .name = "pool_max_size",
  113. .mode = S_IRUGO | S_IWUSR
  114. };
  115. static struct attribute ttm_page_pool_small = {
  116. .name = "pool_small_allocation",
  117. .mode = S_IRUGO | S_IWUSR
  118. };
  119. static struct attribute ttm_page_pool_alloc_size = {
  120. .name = "pool_allocation_size",
  121. .mode = S_IRUGO | S_IWUSR
  122. };
  123. static struct attribute *ttm_pool_attrs[] = {
  124. &ttm_page_pool_max,
  125. &ttm_page_pool_small,
  126. &ttm_page_pool_alloc_size,
  127. NULL
  128. };
  129. static void ttm_pool_kobj_release(struct kobject *kobj)
  130. {
  131. struct ttm_pool_manager *m =
  132. container_of(kobj, struct ttm_pool_manager, kobj);
  133. kfree(m);
  134. }
  135. static ssize_t ttm_pool_store(struct kobject *kobj,
  136. struct attribute *attr, const char *buffer, size_t size)
  137. {
  138. struct ttm_pool_manager *m =
  139. container_of(kobj, struct ttm_pool_manager, kobj);
  140. int chars;
  141. unsigned val;
  142. chars = sscanf(buffer, "%u", &val);
  143. if (chars == 0)
  144. return size;
  145. /* Convert kb to number of pages */
  146. val = val / (PAGE_SIZE >> 10);
  147. if (attr == &ttm_page_pool_max)
  148. m->options.max_size = val;
  149. else if (attr == &ttm_page_pool_small)
  150. m->options.small = val;
  151. else if (attr == &ttm_page_pool_alloc_size) {
  152. if (val > NUM_PAGES_TO_ALLOC*8) {
  153. pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n",
  154. NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7),
  155. NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10));
  156. return size;
  157. } else if (val > NUM_PAGES_TO_ALLOC) {
  158. pr_warn("Setting allocation size to larger than %lu is not recommended\n",
  159. NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10));
  160. }
  161. m->options.alloc_size = val;
  162. }
  163. return size;
  164. }
  165. static ssize_t ttm_pool_show(struct kobject *kobj,
  166. struct attribute *attr, char *buffer)
  167. {
  168. struct ttm_pool_manager *m =
  169. container_of(kobj, struct ttm_pool_manager, kobj);
  170. unsigned val = 0;
  171. if (attr == &ttm_page_pool_max)
  172. val = m->options.max_size;
  173. else if (attr == &ttm_page_pool_small)
  174. val = m->options.small;
  175. else if (attr == &ttm_page_pool_alloc_size)
  176. val = m->options.alloc_size;
  177. val = val * (PAGE_SIZE >> 10);
  178. return snprintf(buffer, PAGE_SIZE, "%u\n", val);
  179. }
  180. static const struct sysfs_ops ttm_pool_sysfs_ops = {
  181. .show = &ttm_pool_show,
  182. .store = &ttm_pool_store,
  183. };
  184. static struct kobj_type ttm_pool_kobj_type = {
  185. .release = &ttm_pool_kobj_release,
  186. .sysfs_ops = &ttm_pool_sysfs_ops,
  187. .default_attrs = ttm_pool_attrs,
  188. };
  189. static struct ttm_pool_manager *_manager;
  190. #ifndef CONFIG_X86
  191. static int set_pages_array_wb(struct page **pages, int addrinarray)
  192. {
  193. #ifdef TTM_HAS_AGP
  194. int i;
  195. for (i = 0; i < addrinarray; i++)
  196. unmap_page_from_agp(pages[i]);
  197. #endif
  198. return 0;
  199. }
  200. static int set_pages_array_wc(struct page **pages, int addrinarray)
  201. {
  202. #ifdef TTM_HAS_AGP
  203. int i;
  204. for (i = 0; i < addrinarray; i++)
  205. map_page_into_agp(pages[i]);
  206. #endif
  207. return 0;
  208. }
  209. static int set_pages_array_uc(struct page **pages, int addrinarray)
  210. {
  211. #ifdef TTM_HAS_AGP
  212. int i;
  213. for (i = 0; i < addrinarray; i++)
  214. map_page_into_agp(pages[i]);
  215. #endif
  216. return 0;
  217. }
  218. #endif
  219. /**
  220. * Select the right pool or requested caching state and ttm flags. */
  221. static struct ttm_page_pool *ttm_get_pool(int flags,
  222. enum ttm_caching_state cstate)
  223. {
  224. int pool_index;
  225. if (cstate == tt_cached)
  226. return NULL;
  227. if (cstate == tt_wc)
  228. pool_index = 0x0;
  229. else
  230. pool_index = 0x1;
  231. if (flags & TTM_PAGE_FLAG_DMA32)
  232. pool_index |= 0x2;
  233. return &_manager->pools[pool_index];
  234. }
  235. /* set memory back to wb and free the pages. */
  236. static void ttm_pages_put(struct page *pages[], unsigned npages)
  237. {
  238. unsigned i;
  239. if (set_pages_array_wb(pages, npages))
  240. pr_err("Failed to set %d pages to wb!\n", npages);
  241. for (i = 0; i < npages; ++i)
  242. __free_page(pages[i]);
  243. }
  244. static void ttm_pool_update_free_locked(struct ttm_page_pool *pool,
  245. unsigned freed_pages)
  246. {
  247. pool->npages -= freed_pages;
  248. pool->nfrees += freed_pages;
  249. }
  250. /**
  251. * Free pages from pool.
  252. *
  253. * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC
  254. * number of pages in one go.
  255. *
  256. * @pool: to free the pages from
  257. * @free_all: If set to true will free all pages in pool
  258. **/
  259. static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free)
  260. {
  261. unsigned long irq_flags;
  262. struct page *p;
  263. struct page **pages_to_free;
  264. unsigned freed_pages = 0,
  265. npages_to_free = nr_free;
  266. if (NUM_PAGES_TO_ALLOC < nr_free)
  267. npages_to_free = NUM_PAGES_TO_ALLOC;
  268. pages_to_free = kmalloc(npages_to_free * sizeof(struct page *),
  269. GFP_KERNEL);
  270. if (!pages_to_free) {
  271. pr_err("Failed to allocate memory for pool free operation\n");
  272. return 0;
  273. }
  274. restart:
  275. spin_lock_irqsave(&pool->lock, irq_flags);
  276. list_for_each_entry_reverse(p, &pool->list, lru) {
  277. if (freed_pages >= npages_to_free)
  278. break;
  279. pages_to_free[freed_pages++] = p;
  280. /* We can only remove NUM_PAGES_TO_ALLOC at a time. */
  281. if (freed_pages >= NUM_PAGES_TO_ALLOC) {
  282. /* remove range of pages from the pool */
  283. __list_del(p->lru.prev, &pool->list);
  284. ttm_pool_update_free_locked(pool, freed_pages);
  285. /**
  286. * Because changing page caching is costly
  287. * we unlock the pool to prevent stalling.
  288. */
  289. spin_unlock_irqrestore(&pool->lock, irq_flags);
  290. ttm_pages_put(pages_to_free, freed_pages);
  291. if (likely(nr_free != FREE_ALL_PAGES))
  292. nr_free -= freed_pages;
  293. if (NUM_PAGES_TO_ALLOC >= nr_free)
  294. npages_to_free = nr_free;
  295. else
  296. npages_to_free = NUM_PAGES_TO_ALLOC;
  297. freed_pages = 0;
  298. /* free all so restart the processing */
  299. if (nr_free)
  300. goto restart;
  301. /* Not allowed to fall through or break because
  302. * following context is inside spinlock while we are
  303. * outside here.
  304. */
  305. goto out;
  306. }
  307. }
  308. /* remove range of pages from the pool */
  309. if (freed_pages) {
  310. __list_del(&p->lru, &pool->list);
  311. ttm_pool_update_free_locked(pool, freed_pages);
  312. nr_free -= freed_pages;
  313. }
  314. spin_unlock_irqrestore(&pool->lock, irq_flags);
  315. if (freed_pages)
  316. ttm_pages_put(pages_to_free, freed_pages);
  317. out:
  318. kfree(pages_to_free);
  319. return nr_free;
  320. }
  321. /**
  322. * Callback for mm to request pool to reduce number of page held.
  323. *
  324. * XXX: (dchinner) Deadlock warning!
  325. *
  326. * ttm_page_pool_free() does memory allocation using GFP_KERNEL. that means
  327. * this can deadlock when called a sc->gfp_mask that is not equal to
  328. * GFP_KERNEL.
  329. *
  330. * This code is crying out for a shrinker per pool....
  331. */
  332. static unsigned long
  333. ttm_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
  334. {
  335. static atomic_t start_pool = ATOMIC_INIT(0);
  336. unsigned i;
  337. unsigned pool_offset = atomic_add_return(1, &start_pool);
  338. struct ttm_page_pool *pool;
  339. int shrink_pages = sc->nr_to_scan;
  340. unsigned long freed = 0;
  341. pool_offset = pool_offset % NUM_POOLS;
  342. /* select start pool in round robin fashion */
  343. for (i = 0; i < NUM_POOLS; ++i) {
  344. unsigned nr_free = shrink_pages;
  345. if (shrink_pages == 0)
  346. break;
  347. pool = &_manager->pools[(i + pool_offset)%NUM_POOLS];
  348. shrink_pages = ttm_page_pool_free(pool, nr_free);
  349. freed += nr_free - shrink_pages;
  350. }
  351. return freed;
  352. }
  353. static unsigned long
  354. ttm_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
  355. {
  356. unsigned i;
  357. unsigned long count = 0;
  358. for (i = 0; i < NUM_POOLS; ++i)
  359. count += _manager->pools[i].npages;
  360. return count;
  361. }
  362. static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager)
  363. {
  364. manager->mm_shrink.count_objects = ttm_pool_shrink_count;
  365. manager->mm_shrink.scan_objects = ttm_pool_shrink_scan;
  366. manager->mm_shrink.seeks = 1;
  367. register_shrinker(&manager->mm_shrink);
  368. }
  369. static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager)
  370. {
  371. unregister_shrinker(&manager->mm_shrink);
  372. }
  373. static int ttm_set_pages_caching(struct page **pages,
  374. enum ttm_caching_state cstate, unsigned cpages)
  375. {
  376. int r = 0;
  377. /* Set page caching */
  378. switch (cstate) {
  379. case tt_uncached:
  380. r = set_pages_array_uc(pages, cpages);
  381. if (r)
  382. pr_err("Failed to set %d pages to uc!\n", cpages);
  383. break;
  384. case tt_wc:
  385. r = set_pages_array_wc(pages, cpages);
  386. if (r)
  387. pr_err("Failed to set %d pages to wc!\n", cpages);
  388. break;
  389. default:
  390. break;
  391. }
  392. return r;
  393. }
  394. /**
  395. * Free pages the pages that failed to change the caching state. If there is
  396. * any pages that have changed their caching state already put them to the
  397. * pool.
  398. */
  399. static void ttm_handle_caching_state_failure(struct list_head *pages,
  400. int ttm_flags, enum ttm_caching_state cstate,
  401. struct page **failed_pages, unsigned cpages)
  402. {
  403. unsigned i;
  404. /* Failed pages have to be freed */
  405. for (i = 0; i < cpages; ++i) {
  406. list_del(&failed_pages[i]->lru);
  407. __free_page(failed_pages[i]);
  408. }
  409. }
  410. /**
  411. * Allocate new pages with correct caching.
  412. *
  413. * This function is reentrant if caller updates count depending on number of
  414. * pages returned in pages array.
  415. */
  416. static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags,
  417. int ttm_flags, enum ttm_caching_state cstate, unsigned count)
  418. {
  419. struct page **caching_array;
  420. struct page *p;
  421. int r = 0;
  422. unsigned i, cpages;
  423. unsigned max_cpages = min(count,
  424. (unsigned)(PAGE_SIZE/sizeof(struct page *)));
  425. /* allocate array for page caching change */
  426. caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL);
  427. if (!caching_array) {
  428. pr_err("Unable to allocate table for new pages\n");
  429. return -ENOMEM;
  430. }
  431. for (i = 0, cpages = 0; i < count; ++i) {
  432. p = alloc_page(gfp_flags);
  433. if (!p) {
  434. pr_err("Unable to get page %u\n", i);
  435. /* store already allocated pages in the pool after
  436. * setting the caching state */
  437. if (cpages) {
  438. r = ttm_set_pages_caching(caching_array,
  439. cstate, cpages);
  440. if (r)
  441. ttm_handle_caching_state_failure(pages,
  442. ttm_flags, cstate,
  443. caching_array, cpages);
  444. }
  445. r = -ENOMEM;
  446. goto out;
  447. }
  448. #ifdef CONFIG_HIGHMEM
  449. /* gfp flags of highmem page should never be dma32 so we
  450. * we should be fine in such case
  451. */
  452. if (!PageHighMem(p))
  453. #endif
  454. {
  455. caching_array[cpages++] = p;
  456. if (cpages == max_cpages) {
  457. r = ttm_set_pages_caching(caching_array,
  458. cstate, cpages);
  459. if (r) {
  460. ttm_handle_caching_state_failure(pages,
  461. ttm_flags, cstate,
  462. caching_array, cpages);
  463. goto out;
  464. }
  465. cpages = 0;
  466. }
  467. }
  468. list_add(&p->lru, pages);
  469. }
  470. if (cpages) {
  471. r = ttm_set_pages_caching(caching_array, cstate, cpages);
  472. if (r)
  473. ttm_handle_caching_state_failure(pages,
  474. ttm_flags, cstate,
  475. caching_array, cpages);
  476. }
  477. out:
  478. kfree(caching_array);
  479. return r;
  480. }
  481. /**
  482. * Fill the given pool if there aren't enough pages and the requested number of
  483. * pages is small.
  484. */
  485. static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
  486. int ttm_flags, enum ttm_caching_state cstate, unsigned count,
  487. unsigned long *irq_flags)
  488. {
  489. struct page *p;
  490. int r;
  491. unsigned cpages = 0;
  492. /**
  493. * Only allow one pool fill operation at a time.
  494. * If pool doesn't have enough pages for the allocation new pages are
  495. * allocated from outside of pool.
  496. */
  497. if (pool->fill_lock)
  498. return;
  499. pool->fill_lock = true;
  500. /* If allocation request is small and there are not enough
  501. * pages in a pool we fill the pool up first. */
  502. if (count < _manager->options.small
  503. && count > pool->npages) {
  504. struct list_head new_pages;
  505. unsigned alloc_size = _manager->options.alloc_size;
  506. /**
  507. * Can't change page caching if in irqsave context. We have to
  508. * drop the pool->lock.
  509. */
  510. spin_unlock_irqrestore(&pool->lock, *irq_flags);
  511. INIT_LIST_HEAD(&new_pages);
  512. r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags,
  513. cstate, alloc_size);
  514. spin_lock_irqsave(&pool->lock, *irq_flags);
  515. if (!r) {
  516. list_splice(&new_pages, &pool->list);
  517. ++pool->nrefills;
  518. pool->npages += alloc_size;
  519. } else {
  520. pr_err("Failed to fill pool (%p)\n", pool);
  521. /* If we have any pages left put them to the pool. */
  522. list_for_each_entry(p, &pool->list, lru) {
  523. ++cpages;
  524. }
  525. list_splice(&new_pages, &pool->list);
  526. pool->npages += cpages;
  527. }
  528. }
  529. pool->fill_lock = false;
  530. }
  531. /**
  532. * Cut 'count' number of pages from the pool and put them on the return list.
  533. *
  534. * @return count of pages still required to fulfill the request.
  535. */
  536. static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool,
  537. struct list_head *pages,
  538. int ttm_flags,
  539. enum ttm_caching_state cstate,
  540. unsigned count)
  541. {
  542. unsigned long irq_flags;
  543. struct list_head *p;
  544. unsigned i;
  545. spin_lock_irqsave(&pool->lock, irq_flags);
  546. ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, &irq_flags);
  547. if (count >= pool->npages) {
  548. /* take all pages from the pool */
  549. list_splice_init(&pool->list, pages);
  550. count -= pool->npages;
  551. pool->npages = 0;
  552. goto out;
  553. }
  554. /* find the last pages to include for requested number of pages. Split
  555. * pool to begin and halve it to reduce search space. */
  556. if (count <= pool->npages/2) {
  557. i = 0;
  558. list_for_each(p, &pool->list) {
  559. if (++i == count)
  560. break;
  561. }
  562. } else {
  563. i = pool->npages + 1;
  564. list_for_each_prev(p, &pool->list) {
  565. if (--i == count)
  566. break;
  567. }
  568. }
  569. /* Cut 'count' number of pages from the pool */
  570. list_cut_position(pages, &pool->list, p);
  571. pool->npages -= count;
  572. count = 0;
  573. out:
  574. spin_unlock_irqrestore(&pool->lock, irq_flags);
  575. return count;
  576. }
  577. /* Put all pages in pages list to correct pool to wait for reuse */
  578. static void ttm_put_pages(struct page **pages, unsigned npages, int flags,
  579. enum ttm_caching_state cstate)
  580. {
  581. unsigned long irq_flags;
  582. struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
  583. unsigned i;
  584. if (pool == NULL) {
  585. /* No pool for this memory type so free the pages */
  586. for (i = 0; i < npages; i++) {
  587. if (pages[i]) {
  588. if (page_count(pages[i]) != 1)
  589. pr_err("Erroneous page count. Leaking pages.\n");
  590. __free_page(pages[i]);
  591. pages[i] = NULL;
  592. }
  593. }
  594. return;
  595. }
  596. spin_lock_irqsave(&pool->lock, irq_flags);
  597. for (i = 0; i < npages; i++) {
  598. if (pages[i]) {
  599. if (page_count(pages[i]) != 1)
  600. pr_err("Erroneous page count. Leaking pages.\n");
  601. list_add_tail(&pages[i]->lru, &pool->list);
  602. pages[i] = NULL;
  603. pool->npages++;
  604. }
  605. }
  606. /* Check that we don't go over the pool limit */
  607. npages = 0;
  608. if (pool->npages > _manager->options.max_size) {
  609. npages = pool->npages - _manager->options.max_size;
  610. /* free at least NUM_PAGES_TO_ALLOC number of pages
  611. * to reduce calls to set_memory_wb */
  612. if (npages < NUM_PAGES_TO_ALLOC)
  613. npages = NUM_PAGES_TO_ALLOC;
  614. }
  615. spin_unlock_irqrestore(&pool->lock, irq_flags);
  616. if (npages)
  617. ttm_page_pool_free(pool, npages);
  618. }
  619. /*
  620. * On success pages list will hold count number of correctly
  621. * cached pages.
  622. */
  623. static int ttm_get_pages(struct page **pages, unsigned npages, int flags,
  624. enum ttm_caching_state cstate)
  625. {
  626. struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
  627. struct list_head plist;
  628. struct page *p = NULL;
  629. gfp_t gfp_flags = GFP_USER;
  630. unsigned count;
  631. int r;
  632. /* set zero flag for page allocation if required */
  633. if (flags & TTM_PAGE_FLAG_ZERO_ALLOC)
  634. gfp_flags |= __GFP_ZERO;
  635. /* No pool for cached pages */
  636. if (pool == NULL) {
  637. if (flags & TTM_PAGE_FLAG_DMA32)
  638. gfp_flags |= GFP_DMA32;
  639. else
  640. gfp_flags |= GFP_HIGHUSER;
  641. for (r = 0; r < npages; ++r) {
  642. p = alloc_page(gfp_flags);
  643. if (!p) {
  644. pr_err("Unable to allocate page\n");
  645. return -ENOMEM;
  646. }
  647. pages[r] = p;
  648. }
  649. return 0;
  650. }
  651. /* combine zero flag to pool flags */
  652. gfp_flags |= pool->gfp_flags;
  653. /* First we take pages from the pool */
  654. INIT_LIST_HEAD(&plist);
  655. npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages);
  656. count = 0;
  657. list_for_each_entry(p, &plist, lru) {
  658. pages[count++] = p;
  659. }
  660. /* clear the pages coming from the pool if requested */
  661. if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) {
  662. list_for_each_entry(p, &plist, lru) {
  663. if (PageHighMem(p))
  664. clear_highpage(p);
  665. else
  666. clear_page(page_address(p));
  667. }
  668. }
  669. /* If pool didn't have enough pages allocate new one. */
  670. if (npages > 0) {
  671. /* ttm_alloc_new_pages doesn't reference pool so we can run
  672. * multiple requests in parallel.
  673. **/
  674. INIT_LIST_HEAD(&plist);
  675. r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate, npages);
  676. list_for_each_entry(p, &plist, lru) {
  677. pages[count++] = p;
  678. }
  679. if (r) {
  680. /* If there is any pages in the list put them back to
  681. * the pool. */
  682. pr_err("Failed to allocate extra pages for large request\n");
  683. ttm_put_pages(pages, count, flags, cstate);
  684. return r;
  685. }
  686. }
  687. return 0;
  688. }
  689. static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags,
  690. char *name)
  691. {
  692. spin_lock_init(&pool->lock);
  693. pool->fill_lock = false;
  694. INIT_LIST_HEAD(&pool->list);
  695. pool->npages = pool->nfrees = 0;
  696. pool->gfp_flags = flags;
  697. pool->name = name;
  698. }
  699. int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages)
  700. {
  701. int ret;
  702. WARN_ON(_manager);
  703. pr_info("Initializing pool allocator\n");
  704. _manager = kzalloc(sizeof(*_manager), GFP_KERNEL);
  705. ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc");
  706. ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc");
  707. ttm_page_pool_init_locked(&_manager->wc_pool_dma32,
  708. GFP_USER | GFP_DMA32, "wc dma");
  709. ttm_page_pool_init_locked(&_manager->uc_pool_dma32,
  710. GFP_USER | GFP_DMA32, "uc dma");
  711. _manager->options.max_size = max_pages;
  712. _manager->options.small = SMALL_ALLOCATION;
  713. _manager->options.alloc_size = NUM_PAGES_TO_ALLOC;
  714. ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type,
  715. &glob->kobj, "pool");
  716. if (unlikely(ret != 0)) {
  717. kobject_put(&_manager->kobj);
  718. _manager = NULL;
  719. return ret;
  720. }
  721. ttm_pool_mm_shrink_init(_manager);
  722. return 0;
  723. }
  724. void ttm_page_alloc_fini(void)
  725. {
  726. int i;
  727. pr_info("Finalizing pool allocator\n");
  728. ttm_pool_mm_shrink_fini(_manager);
  729. for (i = 0; i < NUM_POOLS; ++i)
  730. ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES);
  731. kobject_put(&_manager->kobj);
  732. _manager = NULL;
  733. }
  734. int ttm_pool_populate(struct ttm_tt *ttm)
  735. {
  736. struct ttm_mem_global *mem_glob = ttm->glob->mem_glob;
  737. unsigned i;
  738. int ret;
  739. if (ttm->state != tt_unpopulated)
  740. return 0;
  741. for (i = 0; i < ttm->num_pages; ++i) {
  742. ret = ttm_get_pages(&ttm->pages[i], 1,
  743. ttm->page_flags,
  744. ttm->caching_state);
  745. if (ret != 0) {
  746. ttm_pool_unpopulate(ttm);
  747. return -ENOMEM;
  748. }
  749. ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
  750. false, false);
  751. if (unlikely(ret != 0)) {
  752. ttm_pool_unpopulate(ttm);
  753. return -ENOMEM;
  754. }
  755. }
  756. if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {
  757. ret = ttm_tt_swapin(ttm);
  758. if (unlikely(ret != 0)) {
  759. ttm_pool_unpopulate(ttm);
  760. return ret;
  761. }
  762. }
  763. ttm->state = tt_unbound;
  764. return 0;
  765. }
  766. EXPORT_SYMBOL(ttm_pool_populate);
  767. void ttm_pool_unpopulate(struct ttm_tt *ttm)
  768. {
  769. unsigned i;
  770. for (i = 0; i < ttm->num_pages; ++i) {
  771. if (ttm->pages[i]) {
  772. ttm_mem_global_free_page(ttm->glob->mem_glob,
  773. ttm->pages[i]);
  774. ttm_put_pages(&ttm->pages[i], 1,
  775. ttm->page_flags,
  776. ttm->caching_state);
  777. }
  778. }
  779. ttm->state = tt_unpopulated;
  780. }
  781. EXPORT_SYMBOL(ttm_pool_unpopulate);
  782. int ttm_page_alloc_debugfs(struct seq_file *m, void *data)
  783. {
  784. struct ttm_page_pool *p;
  785. unsigned i;
  786. char *h[] = {"pool", "refills", "pages freed", "size"};
  787. if (!_manager) {
  788. seq_printf(m, "No pool allocator running.\n");
  789. return 0;
  790. }
  791. seq_printf(m, "%6s %12s %13s %8s\n",
  792. h[0], h[1], h[2], h[3]);
  793. for (i = 0; i < NUM_POOLS; ++i) {
  794. p = &_manager->pools[i];
  795. seq_printf(m, "%6s %12ld %13ld %8d\n",
  796. p->name, p->nrefills,
  797. p->nfrees, p->npages);
  798. }
  799. return 0;
  800. }
  801. EXPORT_SYMBOL(ttm_page_alloc_debugfs);