swap_state.c 9.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369
  1. /*
  2. * linux/mm/swap_state.c
  3. *
  4. * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
  5. * Swap reorganised 29.12.95, Stephen Tweedie
  6. *
  7. * Rewritten to use page cache, (C) 1998 Stephen Tweedie
  8. */
  9. #include <linux/module.h>
  10. #include <linux/mm.h>
  11. #include <linux/kernel_stat.h>
  12. #include <linux/swap.h>
  13. #include <linux/swapops.h>
  14. #include <linux/init.h>
  15. #include <linux/pagemap.h>
  16. #include <linux/buffer_head.h>
  17. #include <linux/backing-dev.h>
  18. #include <linux/pagevec.h>
  19. #include <linux/migrate.h>
  20. #include <asm/pgtable.h>
  21. /*
  22. * swapper_space is a fiction, retained to simplify the path through
  23. * vmscan's shrink_page_list, to make sync_page look nicer, and to allow
  24. * future use of radix_tree tags in the swap cache.
  25. */
  26. static const struct address_space_operations swap_aops = {
  27. .writepage = swap_writepage,
  28. .sync_page = block_sync_page,
  29. .set_page_dirty = __set_page_dirty_nobuffers,
  30. .migratepage = migrate_page,
  31. };
  32. static struct backing_dev_info swap_backing_dev_info = {
  33. .capabilities = BDI_CAP_NO_ACCT_AND_WRITEBACK,
  34. .unplug_io_fn = swap_unplug_io_fn,
  35. };
  36. struct address_space swapper_space = {
  37. .page_tree = RADIX_TREE_INIT(GFP_ATOMIC|__GFP_NOWARN),
  38. .tree_lock = __SPIN_LOCK_UNLOCKED(swapper_space.tree_lock),
  39. .a_ops = &swap_aops,
  40. .i_mmap_nonlinear = LIST_HEAD_INIT(swapper_space.i_mmap_nonlinear),
  41. .backing_dev_info = &swap_backing_dev_info,
  42. };
  43. #define INC_CACHE_INFO(x) do { swap_cache_info.x++; } while (0)
  44. static struct {
  45. unsigned long add_total;
  46. unsigned long del_total;
  47. unsigned long find_success;
  48. unsigned long find_total;
  49. } swap_cache_info;
  50. void show_swap_cache_info(void)
  51. {
  52. printk("%lu pages in swap cache\n", total_swapcache_pages);
  53. printk("Swap cache stats: add %lu, delete %lu, find %lu/%lu\n",
  54. swap_cache_info.add_total, swap_cache_info.del_total,
  55. swap_cache_info.find_success, swap_cache_info.find_total);
  56. printk("Free swap = %ldkB\n", nr_swap_pages << (PAGE_SHIFT - 10));
  57. printk("Total swap = %lukB\n", total_swap_pages << (PAGE_SHIFT - 10));
  58. }
  59. /*
  60. * add_to_swap_cache resembles add_to_page_cache_locked on swapper_space,
  61. * but sets SwapCache flag and private instead of mapping and index.
  62. */
  63. int add_to_swap_cache(struct page *page, swp_entry_t entry, gfp_t gfp_mask)
  64. {
  65. int error;
  66. BUG_ON(!PageLocked(page));
  67. BUG_ON(PageSwapCache(page));
  68. BUG_ON(PagePrivate(page));
  69. error = radix_tree_preload(gfp_mask);
  70. if (!error) {
  71. page_cache_get(page);
  72. SetPageSwapCache(page);
  73. set_page_private(page, entry.val);
  74. spin_lock_irq(&swapper_space.tree_lock);
  75. error = radix_tree_insert(&swapper_space.page_tree,
  76. entry.val, page);
  77. if (likely(!error)) {
  78. total_swapcache_pages++;
  79. __inc_zone_page_state(page, NR_FILE_PAGES);
  80. INC_CACHE_INFO(add_total);
  81. }
  82. spin_unlock_irq(&swapper_space.tree_lock);
  83. radix_tree_preload_end();
  84. if (unlikely(error)) {
  85. set_page_private(page, 0UL);
  86. ClearPageSwapCache(page);
  87. page_cache_release(page);
  88. }
  89. }
  90. return error;
  91. }
  92. /*
  93. * This must be called only on pages that have
  94. * been verified to be in the swap cache.
  95. */
  96. void __delete_from_swap_cache(struct page *page)
  97. {
  98. BUG_ON(!PageLocked(page));
  99. BUG_ON(!PageSwapCache(page));
  100. BUG_ON(PageWriteback(page));
  101. BUG_ON(PagePrivate(page));
  102. radix_tree_delete(&swapper_space.page_tree, page_private(page));
  103. set_page_private(page, 0);
  104. ClearPageSwapCache(page);
  105. total_swapcache_pages--;
  106. __dec_zone_page_state(page, NR_FILE_PAGES);
  107. INC_CACHE_INFO(del_total);
  108. }
  109. /**
  110. * add_to_swap - allocate swap space for a page
  111. * @page: page we want to move to swap
  112. * @gfp_mask: memory allocation flags
  113. *
  114. * Allocate swap space for the page and add the page to the
  115. * swap cache. Caller needs to hold the page lock.
  116. */
  117. int add_to_swap(struct page * page, gfp_t gfp_mask)
  118. {
  119. swp_entry_t entry;
  120. int err;
  121. BUG_ON(!PageLocked(page));
  122. BUG_ON(!PageUptodate(page));
  123. for (;;) {
  124. entry = get_swap_page();
  125. if (!entry.val)
  126. return 0;
  127. /*
  128. * Radix-tree node allocations from PF_MEMALLOC contexts could
  129. * completely exhaust the page allocator. __GFP_NOMEMALLOC
  130. * stops emergency reserves from being allocated.
  131. *
  132. * TODO: this could cause a theoretical memory reclaim
  133. * deadlock in the swap out path.
  134. */
  135. /*
  136. * Add it to the swap cache and mark it dirty
  137. */
  138. err = add_to_swap_cache(page, entry,
  139. gfp_mask|__GFP_NOMEMALLOC|__GFP_NOWARN);
  140. switch (err) {
  141. case 0: /* Success */
  142. SetPageDirty(page);
  143. return 1;
  144. case -EEXIST:
  145. /* Raced with "speculative" read_swap_cache_async */
  146. swap_free(entry);
  147. continue;
  148. default:
  149. /* -ENOMEM radix-tree allocation failure */
  150. swap_free(entry);
  151. return 0;
  152. }
  153. }
  154. }
  155. /*
  156. * This must be called only on pages that have
  157. * been verified to be in the swap cache and locked.
  158. * It will never put the page into the free list,
  159. * the caller has a reference on the page.
  160. */
  161. void delete_from_swap_cache(struct page *page)
  162. {
  163. swp_entry_t entry;
  164. entry.val = page_private(page);
  165. spin_lock_irq(&swapper_space.tree_lock);
  166. __delete_from_swap_cache(page);
  167. spin_unlock_irq(&swapper_space.tree_lock);
  168. swap_free(entry);
  169. page_cache_release(page);
  170. }
  171. /*
  172. * If we are the only user, then try to free up the swap cache.
  173. *
  174. * Its ok to check for PageSwapCache without the page lock
  175. * here because we are going to recheck again inside
  176. * exclusive_swap_page() _with_ the lock.
  177. * - Marcelo
  178. */
  179. static inline void free_swap_cache(struct page *page)
  180. {
  181. if (PageSwapCache(page) && trylock_page(page)) {
  182. remove_exclusive_swap_page(page);
  183. unlock_page(page);
  184. }
  185. }
  186. /*
  187. * Perform a free_page(), also freeing any swap cache associated with
  188. * this page if it is the last user of the page.
  189. */
  190. void free_page_and_swap_cache(struct page *page)
  191. {
  192. free_swap_cache(page);
  193. page_cache_release(page);
  194. }
  195. /*
  196. * Passed an array of pages, drop them all from swapcache and then release
  197. * them. They are removed from the LRU and freed if this is their last use.
  198. */
  199. void free_pages_and_swap_cache(struct page **pages, int nr)
  200. {
  201. struct page **pagep = pages;
  202. lru_add_drain();
  203. while (nr) {
  204. int todo = min(nr, PAGEVEC_SIZE);
  205. int i;
  206. for (i = 0; i < todo; i++)
  207. free_swap_cache(pagep[i]);
  208. release_pages(pagep, todo, 0);
  209. pagep += todo;
  210. nr -= todo;
  211. }
  212. }
  213. /*
  214. * Lookup a swap entry in the swap cache. A found page will be returned
  215. * unlocked and with its refcount incremented - we rely on the kernel
  216. * lock getting page table operations atomic even if we drop the page
  217. * lock before returning.
  218. */
  219. struct page * lookup_swap_cache(swp_entry_t entry)
  220. {
  221. struct page *page;
  222. page = find_get_page(&swapper_space, entry.val);
  223. if (page)
  224. INC_CACHE_INFO(find_success);
  225. INC_CACHE_INFO(find_total);
  226. return page;
  227. }
  228. /*
  229. * Locate a page of swap in physical memory, reserving swap cache space
  230. * and reading the disk if it is not already cached.
  231. * A failure return means that either the page allocation failed or that
  232. * the swap entry is no longer in use.
  233. */
  234. struct page *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
  235. struct vm_area_struct *vma, unsigned long addr)
  236. {
  237. struct page *found_page, *new_page = NULL;
  238. int err;
  239. do {
  240. /*
  241. * First check the swap cache. Since this is normally
  242. * called after lookup_swap_cache() failed, re-calling
  243. * that would confuse statistics.
  244. */
  245. found_page = find_get_page(&swapper_space, entry.val);
  246. if (found_page)
  247. break;
  248. /*
  249. * Get a new page to read into from swap.
  250. */
  251. if (!new_page) {
  252. new_page = alloc_page_vma(gfp_mask, vma, addr);
  253. if (!new_page)
  254. break; /* Out of memory */
  255. }
  256. /*
  257. * Swap entry may have been freed since our caller observed it.
  258. */
  259. if (!swap_duplicate(entry))
  260. break;
  261. /*
  262. * Associate the page with swap entry in the swap cache.
  263. * May fail (-EEXIST) if there is already a page associated
  264. * with this entry in the swap cache: added by a racing
  265. * read_swap_cache_async, or add_to_swap or shmem_writepage
  266. * re-using the just freed swap entry for an existing page.
  267. * May fail (-ENOMEM) if radix-tree node allocation failed.
  268. */
  269. set_page_locked(new_page);
  270. err = add_to_swap_cache(new_page, entry, gfp_mask & GFP_KERNEL);
  271. if (likely(!err)) {
  272. /*
  273. * Initiate read into locked page and return.
  274. */
  275. lru_cache_add_active(new_page);
  276. swap_readpage(NULL, new_page);
  277. return new_page;
  278. }
  279. clear_page_locked(new_page);
  280. swap_free(entry);
  281. } while (err != -ENOMEM);
  282. if (new_page)
  283. page_cache_release(new_page);
  284. return found_page;
  285. }
  286. /**
  287. * swapin_readahead - swap in pages in hope we need them soon
  288. * @entry: swap entry of this memory
  289. * @gfp_mask: memory allocation flags
  290. * @vma: user vma this address belongs to
  291. * @addr: target address for mempolicy
  292. *
  293. * Returns the struct page for entry and addr, after queueing swapin.
  294. *
  295. * Primitive swap readahead code. We simply read an aligned block of
  296. * (1 << page_cluster) entries in the swap area. This method is chosen
  297. * because it doesn't cost us any seek time. We also make sure to queue
  298. * the 'original' request together with the readahead ones...
  299. *
  300. * This has been extended to use the NUMA policies from the mm triggering
  301. * the readahead.
  302. *
  303. * Caller must hold down_read on the vma->vm_mm if vma is not NULL.
  304. */
  305. struct page *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask,
  306. struct vm_area_struct *vma, unsigned long addr)
  307. {
  308. int nr_pages;
  309. struct page *page;
  310. unsigned long offset;
  311. unsigned long end_offset;
  312. /*
  313. * Get starting offset for readaround, and number of pages to read.
  314. * Adjust starting address by readbehind (for NUMA interleave case)?
  315. * No, it's very unlikely that swap layout would follow vma layout,
  316. * more likely that neighbouring swap pages came from the same node:
  317. * so use the same "addr" to choose the same node for each swap read.
  318. */
  319. nr_pages = valid_swaphandles(entry, &offset);
  320. for (end_offset = offset + nr_pages; offset < end_offset; offset++) {
  321. /* Ok, do the async read-ahead now */
  322. page = read_swap_cache_async(swp_entry(swp_type(entry), offset),
  323. gfp_mask, vma, addr);
  324. if (!page)
  325. break;
  326. page_cache_release(page);
  327. }
  328. lru_add_drain(); /* Push any new pages onto the LRU now */
  329. return read_swap_cache_async(entry, gfp_mask, vma, addr);
  330. }