zswap.c 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938
  1. /*
  2. * zswap.c - zswap driver file
  3. *
  4. * zswap is a backend for frontswap that takes pages that are in the process
  5. * of being swapped out and attempts to compress and store them in a
  6. * RAM-based memory pool. This can result in a significant I/O reduction on
  7. * the swap device and, in the case where decompressing from RAM is faster
  8. * than reading from the swap device, can also improve workload performance.
  9. *
  10. * Copyright (C) 2012 Seth Jennings <sjenning@linux.vnet.ibm.com>
  11. *
  12. * This program is free software; you can redistribute it and/or
  13. * modify it under the terms of the GNU General Public License
  14. * as published by the Free Software Foundation; either version 2
  15. * of the License, or (at your option) any later version.
  16. *
  17. * This program is distributed in the hope that it will be useful,
  18. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  19. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  20. * GNU General Public License for more details.
  21. */
  22. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  23. #include <linux/module.h>
  24. #include <linux/cpu.h>
  25. #include <linux/highmem.h>
  26. #include <linux/slab.h>
  27. #include <linux/spinlock.h>
  28. #include <linux/types.h>
  29. #include <linux/atomic.h>
  30. #include <linux/frontswap.h>
  31. #include <linux/rbtree.h>
  32. #include <linux/swap.h>
  33. #include <linux/crypto.h>
  34. #include <linux/mempool.h>
  35. #include <linux/zbud.h>
  36. #include <linux/mm_types.h>
  37. #include <linux/page-flags.h>
  38. #include <linux/swapops.h>
  39. #include <linux/writeback.h>
  40. #include <linux/pagemap.h>
  41. /*********************************
  42. * statistics
  43. **********************************/
  44. /* Number of memory pages used by the compressed pool */
  45. static u64 zswap_pool_pages;
  46. /* The number of compressed pages currently stored in zswap */
  47. static atomic_t zswap_stored_pages = ATOMIC_INIT(0);
  48. /*
  49. * The statistics below are not protected from concurrent access for
  50. * performance reasons so they may not be a 100% accurate. However,
  51. * they do provide useful information on roughly how many times a
  52. * certain event is occurring.
  53. */
  54. /* Pool limit was hit (see zswap_max_pool_percent) */
  55. static u64 zswap_pool_limit_hit;
  56. /* Pages written back when pool limit was reached */
  57. static u64 zswap_written_back_pages;
  58. /* Store failed due to a reclaim failure after pool limit was reached */
  59. static u64 zswap_reject_reclaim_fail;
  60. /* Compressed page was too big for the allocator to (optimally) store */
  61. static u64 zswap_reject_compress_poor;
  62. /* Store failed because underlying allocator could not get memory */
  63. static u64 zswap_reject_alloc_fail;
  64. /* Store failed because the entry metadata could not be allocated (rare) */
  65. static u64 zswap_reject_kmemcache_fail;
  66. /* Duplicate store was encountered (rare) */
  67. static u64 zswap_duplicate_entry;
  68. /*********************************
  69. * tunables
  70. **********************************/
  71. /* Enable/disable zswap (disabled by default, fixed at boot for now) */
  72. static bool zswap_enabled __read_mostly;
  73. module_param_named(enabled, zswap_enabled, bool, 0);
  74. /* Compressor to be used by zswap (fixed at boot for now) */
  75. #define ZSWAP_COMPRESSOR_DEFAULT "lzo"
  76. static char *zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT;
  77. module_param_named(compressor, zswap_compressor, charp, 0);
  78. /* The maximum percentage of memory that the compressed pool can occupy */
  79. static unsigned int zswap_max_pool_percent = 20;
  80. module_param_named(max_pool_percent,
  81. zswap_max_pool_percent, uint, 0644);
  82. /*********************************
  83. * compression functions
  84. **********************************/
  85. /* per-cpu compression transforms */
  86. static struct crypto_comp * __percpu *zswap_comp_pcpu_tfms;
  87. enum comp_op {
  88. ZSWAP_COMPOP_COMPRESS,
  89. ZSWAP_COMPOP_DECOMPRESS
  90. };
  91. static int zswap_comp_op(enum comp_op op, const u8 *src, unsigned int slen,
  92. u8 *dst, unsigned int *dlen)
  93. {
  94. struct crypto_comp *tfm;
  95. int ret;
  96. tfm = *per_cpu_ptr(zswap_comp_pcpu_tfms, get_cpu());
  97. switch (op) {
  98. case ZSWAP_COMPOP_COMPRESS:
  99. ret = crypto_comp_compress(tfm, src, slen, dst, dlen);
  100. break;
  101. case ZSWAP_COMPOP_DECOMPRESS:
  102. ret = crypto_comp_decompress(tfm, src, slen, dst, dlen);
  103. break;
  104. default:
  105. ret = -EINVAL;
  106. }
  107. put_cpu();
  108. return ret;
  109. }
  110. static int __init zswap_comp_init(void)
  111. {
  112. if (!crypto_has_comp(zswap_compressor, 0, 0)) {
  113. pr_info("%s compressor not available\n", zswap_compressor);
  114. /* fall back to default compressor */
  115. zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT;
  116. if (!crypto_has_comp(zswap_compressor, 0, 0))
  117. /* can't even load the default compressor */
  118. return -ENODEV;
  119. }
  120. pr_info("using %s compressor\n", zswap_compressor);
  121. /* alloc percpu transforms */
  122. zswap_comp_pcpu_tfms = alloc_percpu(struct crypto_comp *);
  123. if (!zswap_comp_pcpu_tfms)
  124. return -ENOMEM;
  125. return 0;
  126. }
  127. static void zswap_comp_exit(void)
  128. {
  129. /* free percpu transforms */
  130. if (zswap_comp_pcpu_tfms)
  131. free_percpu(zswap_comp_pcpu_tfms);
  132. }
  133. /*********************************
  134. * data structures
  135. **********************************/
  136. /*
  137. * struct zswap_entry
  138. *
  139. * This structure contains the metadata for tracking a single compressed
  140. * page within zswap.
  141. *
  142. * rbnode - links the entry into red-black tree for the appropriate swap type
  143. * refcount - the number of outstanding reference to the entry. This is needed
  144. * to protect against premature freeing of the entry by code
  145. * concurent calls to load, invalidate, and writeback. The lock
  146. * for the zswap_tree structure that contains the entry must
  147. * be held while changing the refcount. Since the lock must
  148. * be held, there is no reason to also make refcount atomic.
  149. * offset - the swap offset for the entry. Index into the red-black tree.
  150. * handle - zsmalloc allocation handle that stores the compressed page data
  151. * length - the length in bytes of the compressed page data. Needed during
  152. * decompression
  153. */
  154. struct zswap_entry {
  155. struct rb_node rbnode;
  156. pgoff_t offset;
  157. int refcount;
  158. unsigned int length;
  159. unsigned long handle;
  160. };
  161. struct zswap_header {
  162. swp_entry_t swpentry;
  163. };
  164. /*
  165. * The tree lock in the zswap_tree struct protects a few things:
  166. * - the rbtree
  167. * - the refcount field of each entry in the tree
  168. */
  169. struct zswap_tree {
  170. struct rb_root rbroot;
  171. spinlock_t lock;
  172. struct zbud_pool *pool;
  173. };
  174. static struct zswap_tree *zswap_trees[MAX_SWAPFILES];
  175. /*********************************
  176. * zswap entry functions
  177. **********************************/
  178. static struct kmem_cache *zswap_entry_cache;
  179. static int zswap_entry_cache_create(void)
  180. {
  181. zswap_entry_cache = KMEM_CACHE(zswap_entry, 0);
  182. return (zswap_entry_cache == NULL);
  183. }
  184. static void zswap_entry_cache_destory(void)
  185. {
  186. kmem_cache_destroy(zswap_entry_cache);
  187. }
  188. static struct zswap_entry *zswap_entry_cache_alloc(gfp_t gfp)
  189. {
  190. struct zswap_entry *entry;
  191. entry = kmem_cache_alloc(zswap_entry_cache, gfp);
  192. if (!entry)
  193. return NULL;
  194. entry->refcount = 1;
  195. RB_CLEAR_NODE(&entry->rbnode);
  196. return entry;
  197. }
  198. static void zswap_entry_cache_free(struct zswap_entry *entry)
  199. {
  200. kmem_cache_free(zswap_entry_cache, entry);
  201. }
  202. /*********************************
  203. * rbtree functions
  204. **********************************/
  205. static struct zswap_entry *zswap_rb_search(struct rb_root *root, pgoff_t offset)
  206. {
  207. struct rb_node *node = root->rb_node;
  208. struct zswap_entry *entry;
  209. while (node) {
  210. entry = rb_entry(node, struct zswap_entry, rbnode);
  211. if (entry->offset > offset)
  212. node = node->rb_left;
  213. else if (entry->offset < offset)
  214. node = node->rb_right;
  215. else
  216. return entry;
  217. }
  218. return NULL;
  219. }
  220. /*
  221. * In the case that a entry with the same offset is found, a pointer to
  222. * the existing entry is stored in dupentry and the function returns -EEXIST
  223. */
  224. static int zswap_rb_insert(struct rb_root *root, struct zswap_entry *entry,
  225. struct zswap_entry **dupentry)
  226. {
  227. struct rb_node **link = &root->rb_node, *parent = NULL;
  228. struct zswap_entry *myentry;
  229. while (*link) {
  230. parent = *link;
  231. myentry = rb_entry(parent, struct zswap_entry, rbnode);
  232. if (myentry->offset > entry->offset)
  233. link = &(*link)->rb_left;
  234. else if (myentry->offset < entry->offset)
  235. link = &(*link)->rb_right;
  236. else {
  237. *dupentry = myentry;
  238. return -EEXIST;
  239. }
  240. }
  241. rb_link_node(&entry->rbnode, parent, link);
  242. rb_insert_color(&entry->rbnode, root);
  243. return 0;
  244. }
  245. static void zswap_rb_erase(struct rb_root *root, struct zswap_entry *entry)
  246. {
  247. if (!RB_EMPTY_NODE(&entry->rbnode)) {
  248. rb_erase(&entry->rbnode, root);
  249. RB_CLEAR_NODE(&entry->rbnode);
  250. }
  251. }
  252. /*
  253. * Carries out the common pattern of freeing and entry's zsmalloc allocation,
  254. * freeing the entry itself, and decrementing the number of stored pages.
  255. */
  256. static void zswap_free_entry(struct zswap_tree *tree,
  257. struct zswap_entry *entry)
  258. {
  259. zbud_free(tree->pool, entry->handle);
  260. zswap_entry_cache_free(entry);
  261. atomic_dec(&zswap_stored_pages);
  262. zswap_pool_pages = zbud_get_pool_size(tree->pool);
  263. }
  264. /* caller must hold the tree lock */
  265. static void zswap_entry_get(struct zswap_entry *entry)
  266. {
  267. entry->refcount++;
  268. }
  269. /* caller must hold the tree lock
  270. * remove from the tree and free it, if nobody reference the entry
  271. */
  272. static void zswap_entry_put(struct zswap_tree *tree,
  273. struct zswap_entry *entry)
  274. {
  275. int refcount = --entry->refcount;
  276. BUG_ON(refcount < 0);
  277. if (refcount == 0) {
  278. zswap_rb_erase(&tree->rbroot, entry);
  279. zswap_free_entry(tree, entry);
  280. }
  281. }
  282. /* caller must hold the tree lock */
  283. static struct zswap_entry *zswap_entry_find_get(struct rb_root *root,
  284. pgoff_t offset)
  285. {
  286. struct zswap_entry *entry = NULL;
  287. entry = zswap_rb_search(root, offset);
  288. if (entry)
  289. zswap_entry_get(entry);
  290. return entry;
  291. }
  292. /*********************************
  293. * per-cpu code
  294. **********************************/
  295. static DEFINE_PER_CPU(u8 *, zswap_dstmem);
  296. static int __zswap_cpu_notifier(unsigned long action, unsigned long cpu)
  297. {
  298. struct crypto_comp *tfm;
  299. u8 *dst;
  300. switch (action) {
  301. case CPU_UP_PREPARE:
  302. tfm = crypto_alloc_comp(zswap_compressor, 0, 0);
  303. if (IS_ERR(tfm)) {
  304. pr_err("can't allocate compressor transform\n");
  305. return NOTIFY_BAD;
  306. }
  307. *per_cpu_ptr(zswap_comp_pcpu_tfms, cpu) = tfm;
  308. dst = kmalloc(PAGE_SIZE * 2, GFP_KERNEL);
  309. if (!dst) {
  310. pr_err("can't allocate compressor buffer\n");
  311. crypto_free_comp(tfm);
  312. *per_cpu_ptr(zswap_comp_pcpu_tfms, cpu) = NULL;
  313. return NOTIFY_BAD;
  314. }
  315. per_cpu(zswap_dstmem, cpu) = dst;
  316. break;
  317. case CPU_DEAD:
  318. case CPU_UP_CANCELED:
  319. tfm = *per_cpu_ptr(zswap_comp_pcpu_tfms, cpu);
  320. if (tfm) {
  321. crypto_free_comp(tfm);
  322. *per_cpu_ptr(zswap_comp_pcpu_tfms, cpu) = NULL;
  323. }
  324. dst = per_cpu(zswap_dstmem, cpu);
  325. kfree(dst);
  326. per_cpu(zswap_dstmem, cpu) = NULL;
  327. break;
  328. default:
  329. break;
  330. }
  331. return NOTIFY_OK;
  332. }
  333. static int zswap_cpu_notifier(struct notifier_block *nb,
  334. unsigned long action, void *pcpu)
  335. {
  336. unsigned long cpu = (unsigned long)pcpu;
  337. return __zswap_cpu_notifier(action, cpu);
  338. }
  339. static struct notifier_block zswap_cpu_notifier_block = {
  340. .notifier_call = zswap_cpu_notifier
  341. };
  342. static int zswap_cpu_init(void)
  343. {
  344. unsigned long cpu;
  345. get_online_cpus();
  346. for_each_online_cpu(cpu)
  347. if (__zswap_cpu_notifier(CPU_UP_PREPARE, cpu) != NOTIFY_OK)
  348. goto cleanup;
  349. register_cpu_notifier(&zswap_cpu_notifier_block);
  350. put_online_cpus();
  351. return 0;
  352. cleanup:
  353. for_each_online_cpu(cpu)
  354. __zswap_cpu_notifier(CPU_UP_CANCELED, cpu);
  355. put_online_cpus();
  356. return -ENOMEM;
  357. }
  358. /*********************************
  359. * helpers
  360. **********************************/
  361. static bool zswap_is_full(void)
  362. {
  363. return (totalram_pages * zswap_max_pool_percent / 100 <
  364. zswap_pool_pages);
  365. }
  366. /*********************************
  367. * writeback code
  368. **********************************/
  369. /* return enum for zswap_get_swap_cache_page */
  370. enum zswap_get_swap_ret {
  371. ZSWAP_SWAPCACHE_NEW,
  372. ZSWAP_SWAPCACHE_EXIST,
  373. ZSWAP_SWAPCACHE_FAIL,
  374. };
  375. /*
  376. * zswap_get_swap_cache_page
  377. *
  378. * This is an adaption of read_swap_cache_async()
  379. *
  380. * This function tries to find a page with the given swap entry
  381. * in the swapper_space address space (the swap cache). If the page
  382. * is found, it is returned in retpage. Otherwise, a page is allocated,
  383. * added to the swap cache, and returned in retpage.
  384. *
  385. * If success, the swap cache page is returned in retpage
  386. * Returns ZSWAP_SWAPCACHE_EXIST if page was already in the swap cache
  387. * Returns ZSWAP_SWAPCACHE_NEW if the new page needs to be populated,
  388. * the new page is added to swapcache and locked
  389. * Returns ZSWAP_SWAPCACHE_FAIL on error
  390. */
  391. static int zswap_get_swap_cache_page(swp_entry_t entry,
  392. struct page **retpage)
  393. {
  394. struct page *found_page, *new_page = NULL;
  395. struct address_space *swapper_space = swap_address_space(entry);
  396. int err;
  397. *retpage = NULL;
  398. do {
  399. /*
  400. * First check the swap cache. Since this is normally
  401. * called after lookup_swap_cache() failed, re-calling
  402. * that would confuse statistics.
  403. */
  404. found_page = find_get_page(swapper_space, entry.val);
  405. if (found_page)
  406. break;
  407. /*
  408. * Get a new page to read into from swap.
  409. */
  410. if (!new_page) {
  411. new_page = alloc_page(GFP_KERNEL);
  412. if (!new_page)
  413. break; /* Out of memory */
  414. }
  415. /*
  416. * call radix_tree_preload() while we can wait.
  417. */
  418. err = radix_tree_preload(GFP_KERNEL);
  419. if (err)
  420. break;
  421. /*
  422. * Swap entry may have been freed since our caller observed it.
  423. */
  424. err = swapcache_prepare(entry);
  425. if (err == -EEXIST) { /* seems racy */
  426. radix_tree_preload_end();
  427. continue;
  428. }
  429. if (err) { /* swp entry is obsolete ? */
  430. radix_tree_preload_end();
  431. break;
  432. }
  433. /* May fail (-ENOMEM) if radix-tree node allocation failed. */
  434. __set_page_locked(new_page);
  435. SetPageSwapBacked(new_page);
  436. err = __add_to_swap_cache(new_page, entry);
  437. if (likely(!err)) {
  438. radix_tree_preload_end();
  439. lru_cache_add_anon(new_page);
  440. *retpage = new_page;
  441. return ZSWAP_SWAPCACHE_NEW;
  442. }
  443. radix_tree_preload_end();
  444. ClearPageSwapBacked(new_page);
  445. __clear_page_locked(new_page);
  446. /*
  447. * add_to_swap_cache() doesn't return -EEXIST, so we can safely
  448. * clear SWAP_HAS_CACHE flag.
  449. */
  450. swapcache_free(entry, NULL);
  451. } while (err != -ENOMEM);
  452. if (new_page)
  453. page_cache_release(new_page);
  454. if (!found_page)
  455. return ZSWAP_SWAPCACHE_FAIL;
  456. *retpage = found_page;
  457. return ZSWAP_SWAPCACHE_EXIST;
  458. }
  459. /*
  460. * Attempts to free an entry by adding a page to the swap cache,
  461. * decompressing the entry data into the page, and issuing a
  462. * bio write to write the page back to the swap device.
  463. *
  464. * This can be thought of as a "resumed writeback" of the page
  465. * to the swap device. We are basically resuming the same swap
  466. * writeback path that was intercepted with the frontswap_store()
  467. * in the first place. After the page has been decompressed into
  468. * the swap cache, the compressed version stored by zswap can be
  469. * freed.
  470. */
  471. static int zswap_writeback_entry(struct zbud_pool *pool, unsigned long handle)
  472. {
  473. struct zswap_header *zhdr;
  474. swp_entry_t swpentry;
  475. struct zswap_tree *tree;
  476. pgoff_t offset;
  477. struct zswap_entry *entry;
  478. struct page *page;
  479. u8 *src, *dst;
  480. unsigned int dlen;
  481. int ret;
  482. struct writeback_control wbc = {
  483. .sync_mode = WB_SYNC_NONE,
  484. };
  485. /* extract swpentry from data */
  486. zhdr = zbud_map(pool, handle);
  487. swpentry = zhdr->swpentry; /* here */
  488. zbud_unmap(pool, handle);
  489. tree = zswap_trees[swp_type(swpentry)];
  490. offset = swp_offset(swpentry);
  491. BUG_ON(pool != tree->pool);
  492. /* find and ref zswap entry */
  493. spin_lock(&tree->lock);
  494. entry = zswap_entry_find_get(&tree->rbroot, offset);
  495. if (!entry) {
  496. /* entry was invalidated */
  497. spin_unlock(&tree->lock);
  498. return 0;
  499. }
  500. spin_unlock(&tree->lock);
  501. BUG_ON(offset != entry->offset);
  502. /* try to allocate swap cache page */
  503. switch (zswap_get_swap_cache_page(swpentry, &page)) {
  504. case ZSWAP_SWAPCACHE_FAIL: /* no memory or invalidate happened */
  505. ret = -ENOMEM;
  506. goto fail;
  507. case ZSWAP_SWAPCACHE_EXIST:
  508. /* page is already in the swap cache, ignore for now */
  509. page_cache_release(page);
  510. ret = -EEXIST;
  511. goto fail;
  512. case ZSWAP_SWAPCACHE_NEW: /* page is locked */
  513. /* decompress */
  514. dlen = PAGE_SIZE;
  515. src = (u8 *)zbud_map(tree->pool, entry->handle) +
  516. sizeof(struct zswap_header);
  517. dst = kmap_atomic(page);
  518. ret = zswap_comp_op(ZSWAP_COMPOP_DECOMPRESS, src,
  519. entry->length, dst, &dlen);
  520. kunmap_atomic(dst);
  521. zbud_unmap(tree->pool, entry->handle);
  522. BUG_ON(ret);
  523. BUG_ON(dlen != PAGE_SIZE);
  524. /* page is up to date */
  525. SetPageUptodate(page);
  526. }
  527. /* move it to the tail of the inactive list after end_writeback */
  528. SetPageReclaim(page);
  529. /* start writeback */
  530. __swap_writepage(page, &wbc, end_swap_bio_write);
  531. page_cache_release(page);
  532. zswap_written_back_pages++;
  533. spin_lock(&tree->lock);
  534. /* drop local reference */
  535. zswap_entry_put(tree, entry);
  536. /*
  537. * There are two possible situations for entry here:
  538. * (1) refcount is 1(normal case), entry is valid and on the tree
  539. * (2) refcount is 0, entry is freed and not on the tree
  540. * because invalidate happened during writeback
  541. * search the tree and free the entry if find entry
  542. */
  543. if (entry == zswap_rb_search(&tree->rbroot, offset))
  544. zswap_entry_put(tree, entry);
  545. spin_unlock(&tree->lock);
  546. goto end;
  547. /*
  548. * if we get here due to ZSWAP_SWAPCACHE_EXIST
  549. * a load may happening concurrently
  550. * it is safe and okay to not free the entry
  551. * if we free the entry in the following put
  552. * it it either okay to return !0
  553. */
  554. fail:
  555. spin_lock(&tree->lock);
  556. zswap_entry_put(tree, entry);
  557. spin_unlock(&tree->lock);
  558. end:
  559. return ret;
  560. }
  561. /*********************************
  562. * frontswap hooks
  563. **********************************/
  564. /* attempts to compress and store an single page */
  565. static int zswap_frontswap_store(unsigned type, pgoff_t offset,
  566. struct page *page)
  567. {
  568. struct zswap_tree *tree = zswap_trees[type];
  569. struct zswap_entry *entry, *dupentry;
  570. int ret;
  571. unsigned int dlen = PAGE_SIZE, len;
  572. unsigned long handle;
  573. char *buf;
  574. u8 *src, *dst;
  575. struct zswap_header *zhdr;
  576. if (!tree) {
  577. ret = -ENODEV;
  578. goto reject;
  579. }
  580. /* reclaim space if needed */
  581. if (zswap_is_full()) {
  582. zswap_pool_limit_hit++;
  583. if (zbud_reclaim_page(tree->pool, 8)) {
  584. zswap_reject_reclaim_fail++;
  585. ret = -ENOMEM;
  586. goto reject;
  587. }
  588. }
  589. /* allocate entry */
  590. entry = zswap_entry_cache_alloc(GFP_KERNEL);
  591. if (!entry) {
  592. zswap_reject_kmemcache_fail++;
  593. ret = -ENOMEM;
  594. goto reject;
  595. }
  596. /* compress */
  597. dst = get_cpu_var(zswap_dstmem);
  598. src = kmap_atomic(page);
  599. ret = zswap_comp_op(ZSWAP_COMPOP_COMPRESS, src, PAGE_SIZE, dst, &dlen);
  600. kunmap_atomic(src);
  601. if (ret) {
  602. ret = -EINVAL;
  603. goto freepage;
  604. }
  605. /* store */
  606. len = dlen + sizeof(struct zswap_header);
  607. ret = zbud_alloc(tree->pool, len, __GFP_NORETRY | __GFP_NOWARN,
  608. &handle);
  609. if (ret == -ENOSPC) {
  610. zswap_reject_compress_poor++;
  611. goto freepage;
  612. }
  613. if (ret) {
  614. zswap_reject_alloc_fail++;
  615. goto freepage;
  616. }
  617. zhdr = zbud_map(tree->pool, handle);
  618. zhdr->swpentry = swp_entry(type, offset);
  619. buf = (u8 *)(zhdr + 1);
  620. memcpy(buf, dst, dlen);
  621. zbud_unmap(tree->pool, handle);
  622. put_cpu_var(zswap_dstmem);
  623. /* populate entry */
  624. entry->offset = offset;
  625. entry->handle = handle;
  626. entry->length = dlen;
  627. /* map */
  628. spin_lock(&tree->lock);
  629. do {
  630. ret = zswap_rb_insert(&tree->rbroot, entry, &dupentry);
  631. if (ret == -EEXIST) {
  632. zswap_duplicate_entry++;
  633. /* remove from rbtree */
  634. zswap_rb_erase(&tree->rbroot, dupentry);
  635. zswap_entry_put(tree, dupentry);
  636. }
  637. } while (ret == -EEXIST);
  638. spin_unlock(&tree->lock);
  639. /* update stats */
  640. atomic_inc(&zswap_stored_pages);
  641. zswap_pool_pages = zbud_get_pool_size(tree->pool);
  642. return 0;
  643. freepage:
  644. put_cpu_var(zswap_dstmem);
  645. zswap_entry_cache_free(entry);
  646. reject:
  647. return ret;
  648. }
  649. /*
  650. * returns 0 if the page was successfully decompressed
  651. * return -1 on entry not found or error
  652. */
  653. static int zswap_frontswap_load(unsigned type, pgoff_t offset,
  654. struct page *page)
  655. {
  656. struct zswap_tree *tree = zswap_trees[type];
  657. struct zswap_entry *entry;
  658. u8 *src, *dst;
  659. unsigned int dlen;
  660. int ret;
  661. /* find */
  662. spin_lock(&tree->lock);
  663. entry = zswap_entry_find_get(&tree->rbroot, offset);
  664. if (!entry) {
  665. /* entry was written back */
  666. spin_unlock(&tree->lock);
  667. return -1;
  668. }
  669. spin_unlock(&tree->lock);
  670. /* decompress */
  671. dlen = PAGE_SIZE;
  672. src = (u8 *)zbud_map(tree->pool, entry->handle) +
  673. sizeof(struct zswap_header);
  674. dst = kmap_atomic(page);
  675. ret = zswap_comp_op(ZSWAP_COMPOP_DECOMPRESS, src, entry->length,
  676. dst, &dlen);
  677. kunmap_atomic(dst);
  678. zbud_unmap(tree->pool, entry->handle);
  679. BUG_ON(ret);
  680. spin_lock(&tree->lock);
  681. zswap_entry_put(tree, entry);
  682. spin_unlock(&tree->lock);
  683. return 0;
  684. }
  685. /* frees an entry in zswap */
  686. static void zswap_frontswap_invalidate_page(unsigned type, pgoff_t offset)
  687. {
  688. struct zswap_tree *tree = zswap_trees[type];
  689. struct zswap_entry *entry;
  690. /* find */
  691. spin_lock(&tree->lock);
  692. entry = zswap_rb_search(&tree->rbroot, offset);
  693. if (!entry) {
  694. /* entry was written back */
  695. spin_unlock(&tree->lock);
  696. return;
  697. }
  698. /* remove from rbtree */
  699. zswap_rb_erase(&tree->rbroot, entry);
  700. /* drop the initial reference from entry creation */
  701. zswap_entry_put(tree, entry);
  702. spin_unlock(&tree->lock);
  703. }
  704. /* frees all zswap entries for the given swap type */
  705. static void zswap_frontswap_invalidate_area(unsigned type)
  706. {
  707. struct zswap_tree *tree = zswap_trees[type];
  708. struct zswap_entry *entry, *n;
  709. if (!tree)
  710. return;
  711. /* walk the tree and free everything */
  712. spin_lock(&tree->lock);
  713. rbtree_postorder_for_each_entry_safe(entry, n, &tree->rbroot, rbnode)
  714. zswap_free_entry(tree, entry);
  715. tree->rbroot = RB_ROOT;
  716. spin_unlock(&tree->lock);
  717. zbud_destroy_pool(tree->pool);
  718. kfree(tree);
  719. zswap_trees[type] = NULL;
  720. }
  721. static struct zbud_ops zswap_zbud_ops = {
  722. .evict = zswap_writeback_entry
  723. };
  724. static void zswap_frontswap_init(unsigned type)
  725. {
  726. struct zswap_tree *tree;
  727. tree = kzalloc(sizeof(struct zswap_tree), GFP_KERNEL);
  728. if (!tree)
  729. goto err;
  730. tree->pool = zbud_create_pool(GFP_KERNEL, &zswap_zbud_ops);
  731. if (!tree->pool)
  732. goto freetree;
  733. tree->rbroot = RB_ROOT;
  734. spin_lock_init(&tree->lock);
  735. zswap_trees[type] = tree;
  736. return;
  737. freetree:
  738. kfree(tree);
  739. err:
  740. pr_err("alloc failed, zswap disabled for swap type %d\n", type);
  741. }
  742. static struct frontswap_ops zswap_frontswap_ops = {
  743. .store = zswap_frontswap_store,
  744. .load = zswap_frontswap_load,
  745. .invalidate_page = zswap_frontswap_invalidate_page,
  746. .invalidate_area = zswap_frontswap_invalidate_area,
  747. .init = zswap_frontswap_init
  748. };
  749. /*********************************
  750. * debugfs functions
  751. **********************************/
  752. #ifdef CONFIG_DEBUG_FS
  753. #include <linux/debugfs.h>
  754. static struct dentry *zswap_debugfs_root;
  755. static int __init zswap_debugfs_init(void)
  756. {
  757. if (!debugfs_initialized())
  758. return -ENODEV;
  759. zswap_debugfs_root = debugfs_create_dir("zswap", NULL);
  760. if (!zswap_debugfs_root)
  761. return -ENOMEM;
  762. debugfs_create_u64("pool_limit_hit", S_IRUGO,
  763. zswap_debugfs_root, &zswap_pool_limit_hit);
  764. debugfs_create_u64("reject_reclaim_fail", S_IRUGO,
  765. zswap_debugfs_root, &zswap_reject_reclaim_fail);
  766. debugfs_create_u64("reject_alloc_fail", S_IRUGO,
  767. zswap_debugfs_root, &zswap_reject_alloc_fail);
  768. debugfs_create_u64("reject_kmemcache_fail", S_IRUGO,
  769. zswap_debugfs_root, &zswap_reject_kmemcache_fail);
  770. debugfs_create_u64("reject_compress_poor", S_IRUGO,
  771. zswap_debugfs_root, &zswap_reject_compress_poor);
  772. debugfs_create_u64("written_back_pages", S_IRUGO,
  773. zswap_debugfs_root, &zswap_written_back_pages);
  774. debugfs_create_u64("duplicate_entry", S_IRUGO,
  775. zswap_debugfs_root, &zswap_duplicate_entry);
  776. debugfs_create_u64("pool_pages", S_IRUGO,
  777. zswap_debugfs_root, &zswap_pool_pages);
  778. debugfs_create_atomic_t("stored_pages", S_IRUGO,
  779. zswap_debugfs_root, &zswap_stored_pages);
  780. return 0;
  781. }
  782. static void __exit zswap_debugfs_exit(void)
  783. {
  784. debugfs_remove_recursive(zswap_debugfs_root);
  785. }
  786. #else
  787. static int __init zswap_debugfs_init(void)
  788. {
  789. return 0;
  790. }
  791. static void __exit zswap_debugfs_exit(void) { }
  792. #endif
  793. /*********************************
  794. * module init and exit
  795. **********************************/
  796. static int __init init_zswap(void)
  797. {
  798. if (!zswap_enabled)
  799. return 0;
  800. pr_info("loading zswap\n");
  801. if (zswap_entry_cache_create()) {
  802. pr_err("entry cache creation failed\n");
  803. goto error;
  804. }
  805. if (zswap_comp_init()) {
  806. pr_err("compressor initialization failed\n");
  807. goto compfail;
  808. }
  809. if (zswap_cpu_init()) {
  810. pr_err("per-cpu initialization failed\n");
  811. goto pcpufail;
  812. }
  813. frontswap_register_ops(&zswap_frontswap_ops);
  814. if (zswap_debugfs_init())
  815. pr_warn("debugfs initialization failed\n");
  816. return 0;
  817. pcpufail:
  818. zswap_comp_exit();
  819. compfail:
  820. zswap_entry_cache_destory();
  821. error:
  822. return -ENOMEM;
  823. }
  824. /* must be late so crypto has time to come up */
  825. late_initcall(init_zswap);
  826. MODULE_LICENSE("GPL");
  827. MODULE_AUTHOR("Seth Jennings <sjenning@linux.vnet.ibm.com>");
  828. MODULE_DESCRIPTION("Compressed cache for swap pages");