swapfile.c 42 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699
  1. /*
  2. * linux/mm/swapfile.c
  3. *
  4. * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
  5. * Swap reorganised 29.12.95, Stephen Tweedie
  6. */
  7. #include <linux/config.h>
  8. #include <linux/mm.h>
  9. #include <linux/hugetlb.h>
  10. #include <linux/mman.h>
  11. #include <linux/slab.h>
  12. #include <linux/kernel_stat.h>
  13. #include <linux/swap.h>
  14. #include <linux/vmalloc.h>
  15. #include <linux/pagemap.h>
  16. #include <linux/namei.h>
  17. #include <linux/shm.h>
  18. #include <linux/blkdev.h>
  19. #include <linux/writeback.h>
  20. #include <linux/proc_fs.h>
  21. #include <linux/seq_file.h>
  22. #include <linux/init.h>
  23. #include <linux/module.h>
  24. #include <linux/rmap.h>
  25. #include <linux/security.h>
  26. #include <linux/backing-dev.h>
  27. #include <linux/capability.h>
  28. #include <linux/syscalls.h>
  29. #include <asm/pgtable.h>
  30. #include <asm/tlbflush.h>
  31. #include <linux/swapops.h>
  32. DEFINE_SPINLOCK(swap_lock);
  33. unsigned int nr_swapfiles;
  34. long total_swap_pages;
  35. static int swap_overflow;
  36. static const char Bad_file[] = "Bad swap file entry ";
  37. static const char Unused_file[] = "Unused swap file entry ";
  38. static const char Bad_offset[] = "Bad swap offset entry ";
  39. static const char Unused_offset[] = "Unused swap offset entry ";
  40. struct swap_list_t swap_list = {-1, -1};
  41. struct swap_info_struct swap_info[MAX_SWAPFILES];
  42. static DECLARE_MUTEX(swapon_sem);
  43. /*
  44. * We need this because the bdev->unplug_fn can sleep and we cannot
  45. * hold swap_lock while calling the unplug_fn. And swap_lock
  46. * cannot be turned into a semaphore.
  47. */
  48. static DECLARE_RWSEM(swap_unplug_sem);
  49. void swap_unplug_io_fn(struct backing_dev_info *unused_bdi, struct page *page)
  50. {
  51. swp_entry_t entry;
  52. down_read(&swap_unplug_sem);
  53. entry.val = page_private(page);
  54. if (PageSwapCache(page)) {
  55. struct block_device *bdev = swap_info[swp_type(entry)].bdev;
  56. struct backing_dev_info *bdi;
  57. /*
  58. * If the page is removed from swapcache from under us (with a
  59. * racy try_to_unuse/swapoff) we need an additional reference
  60. * count to avoid reading garbage from page_private(page) above.
  61. * If the WARN_ON triggers during a swapoff it maybe the race
  62. * condition and it's harmless. However if it triggers without
  63. * swapoff it signals a problem.
  64. */
  65. WARN_ON(page_count(page) <= 1);
  66. bdi = bdev->bd_inode->i_mapping->backing_dev_info;
  67. blk_run_backing_dev(bdi, page);
  68. }
  69. up_read(&swap_unplug_sem);
  70. }
  71. #define SWAPFILE_CLUSTER 256
  72. #define LATENCY_LIMIT 256
  73. static inline unsigned long scan_swap_map(struct swap_info_struct *si)
  74. {
  75. unsigned long offset, last_in_cluster;
  76. int latency_ration = LATENCY_LIMIT;
  77. /*
  78. * We try to cluster swap pages by allocating them sequentially
  79. * in swap. Once we've allocated SWAPFILE_CLUSTER pages this
  80. * way, however, we resort to first-free allocation, starting
  81. * a new cluster. This prevents us from scattering swap pages
  82. * all over the entire swap partition, so that we reduce
  83. * overall disk seek times between swap pages. -- sct
  84. * But we do now try to find an empty cluster. -Andrea
  85. */
  86. si->flags += SWP_SCANNING;
  87. if (unlikely(!si->cluster_nr)) {
  88. si->cluster_nr = SWAPFILE_CLUSTER - 1;
  89. if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER)
  90. goto lowest;
  91. spin_unlock(&swap_lock);
  92. offset = si->lowest_bit;
  93. last_in_cluster = offset + SWAPFILE_CLUSTER - 1;
  94. /* Locate the first empty (unaligned) cluster */
  95. for (; last_in_cluster <= si->highest_bit; offset++) {
  96. if (si->swap_map[offset])
  97. last_in_cluster = offset + SWAPFILE_CLUSTER;
  98. else if (offset == last_in_cluster) {
  99. spin_lock(&swap_lock);
  100. si->cluster_next = offset-SWAPFILE_CLUSTER-1;
  101. goto cluster;
  102. }
  103. if (unlikely(--latency_ration < 0)) {
  104. cond_resched();
  105. latency_ration = LATENCY_LIMIT;
  106. }
  107. }
  108. spin_lock(&swap_lock);
  109. goto lowest;
  110. }
  111. si->cluster_nr--;
  112. cluster:
  113. offset = si->cluster_next;
  114. if (offset > si->highest_bit)
  115. lowest: offset = si->lowest_bit;
  116. checks: if (!(si->flags & SWP_WRITEOK))
  117. goto no_page;
  118. if (!si->highest_bit)
  119. goto no_page;
  120. if (!si->swap_map[offset]) {
  121. if (offset == si->lowest_bit)
  122. si->lowest_bit++;
  123. if (offset == si->highest_bit)
  124. si->highest_bit--;
  125. si->inuse_pages++;
  126. if (si->inuse_pages == si->pages) {
  127. si->lowest_bit = si->max;
  128. si->highest_bit = 0;
  129. }
  130. si->swap_map[offset] = 1;
  131. si->cluster_next = offset + 1;
  132. si->flags -= SWP_SCANNING;
  133. return offset;
  134. }
  135. spin_unlock(&swap_lock);
  136. while (++offset <= si->highest_bit) {
  137. if (!si->swap_map[offset]) {
  138. spin_lock(&swap_lock);
  139. goto checks;
  140. }
  141. if (unlikely(--latency_ration < 0)) {
  142. cond_resched();
  143. latency_ration = LATENCY_LIMIT;
  144. }
  145. }
  146. spin_lock(&swap_lock);
  147. goto lowest;
  148. no_page:
  149. si->flags -= SWP_SCANNING;
  150. return 0;
  151. }
  152. swp_entry_t get_swap_page(void)
  153. {
  154. struct swap_info_struct *si;
  155. pgoff_t offset;
  156. int type, next;
  157. int wrapped = 0;
  158. spin_lock(&swap_lock);
  159. if (nr_swap_pages <= 0)
  160. goto noswap;
  161. nr_swap_pages--;
  162. for (type = swap_list.next; type >= 0 && wrapped < 2; type = next) {
  163. si = swap_info + type;
  164. next = si->next;
  165. if (next < 0 ||
  166. (!wrapped && si->prio != swap_info[next].prio)) {
  167. next = swap_list.head;
  168. wrapped++;
  169. }
  170. if (!si->highest_bit)
  171. continue;
  172. if (!(si->flags & SWP_WRITEOK))
  173. continue;
  174. swap_list.next = next;
  175. offset = scan_swap_map(si);
  176. if (offset) {
  177. spin_unlock(&swap_lock);
  178. return swp_entry(type, offset);
  179. }
  180. next = swap_list.next;
  181. }
  182. nr_swap_pages++;
  183. noswap:
  184. spin_unlock(&swap_lock);
  185. return (swp_entry_t) {0};
  186. }
  187. swp_entry_t get_swap_page_of_type(int type)
  188. {
  189. struct swap_info_struct *si;
  190. pgoff_t offset;
  191. spin_lock(&swap_lock);
  192. si = swap_info + type;
  193. if (si->flags & SWP_WRITEOK) {
  194. nr_swap_pages--;
  195. offset = scan_swap_map(si);
  196. if (offset) {
  197. spin_unlock(&swap_lock);
  198. return swp_entry(type, offset);
  199. }
  200. nr_swap_pages++;
  201. }
  202. spin_unlock(&swap_lock);
  203. return (swp_entry_t) {0};
  204. }
  205. static struct swap_info_struct * swap_info_get(swp_entry_t entry)
  206. {
  207. struct swap_info_struct * p;
  208. unsigned long offset, type;
  209. if (!entry.val)
  210. goto out;
  211. type = swp_type(entry);
  212. if (type >= nr_swapfiles)
  213. goto bad_nofile;
  214. p = & swap_info[type];
  215. if (!(p->flags & SWP_USED))
  216. goto bad_device;
  217. offset = swp_offset(entry);
  218. if (offset >= p->max)
  219. goto bad_offset;
  220. if (!p->swap_map[offset])
  221. goto bad_free;
  222. spin_lock(&swap_lock);
  223. return p;
  224. bad_free:
  225. printk(KERN_ERR "swap_free: %s%08lx\n", Unused_offset, entry.val);
  226. goto out;
  227. bad_offset:
  228. printk(KERN_ERR "swap_free: %s%08lx\n", Bad_offset, entry.val);
  229. goto out;
  230. bad_device:
  231. printk(KERN_ERR "swap_free: %s%08lx\n", Unused_file, entry.val);
  232. goto out;
  233. bad_nofile:
  234. printk(KERN_ERR "swap_free: %s%08lx\n", Bad_file, entry.val);
  235. out:
  236. return NULL;
  237. }
  238. static int swap_entry_free(struct swap_info_struct *p, unsigned long offset)
  239. {
  240. int count = p->swap_map[offset];
  241. if (count < SWAP_MAP_MAX) {
  242. count--;
  243. p->swap_map[offset] = count;
  244. if (!count) {
  245. if (offset < p->lowest_bit)
  246. p->lowest_bit = offset;
  247. if (offset > p->highest_bit)
  248. p->highest_bit = offset;
  249. if (p->prio > swap_info[swap_list.next].prio)
  250. swap_list.next = p - swap_info;
  251. nr_swap_pages++;
  252. p->inuse_pages--;
  253. }
  254. }
  255. return count;
  256. }
  257. /*
  258. * Caller has made sure that the swapdevice corresponding to entry
  259. * is still around or has not been recycled.
  260. */
  261. void swap_free(swp_entry_t entry)
  262. {
  263. struct swap_info_struct * p;
  264. p = swap_info_get(entry);
  265. if (p) {
  266. swap_entry_free(p, swp_offset(entry));
  267. spin_unlock(&swap_lock);
  268. }
  269. }
  270. /*
  271. * How many references to page are currently swapped out?
  272. */
  273. static inline int page_swapcount(struct page *page)
  274. {
  275. int count = 0;
  276. struct swap_info_struct *p;
  277. swp_entry_t entry;
  278. entry.val = page_private(page);
  279. p = swap_info_get(entry);
  280. if (p) {
  281. /* Subtract the 1 for the swap cache itself */
  282. count = p->swap_map[swp_offset(entry)] - 1;
  283. spin_unlock(&swap_lock);
  284. }
  285. return count;
  286. }
  287. /*
  288. * We can use this swap cache entry directly
  289. * if there are no other references to it.
  290. */
  291. int can_share_swap_page(struct page *page)
  292. {
  293. int count;
  294. BUG_ON(!PageLocked(page));
  295. count = page_mapcount(page);
  296. if (count <= 1 && PageSwapCache(page))
  297. count += page_swapcount(page);
  298. return count == 1;
  299. }
  300. /*
  301. * Work out if there are any other processes sharing this
  302. * swap cache page. Free it if you can. Return success.
  303. */
  304. int remove_exclusive_swap_page(struct page *page)
  305. {
  306. int retval;
  307. struct swap_info_struct * p;
  308. swp_entry_t entry;
  309. BUG_ON(PagePrivate(page));
  310. BUG_ON(!PageLocked(page));
  311. if (!PageSwapCache(page))
  312. return 0;
  313. if (PageWriteback(page))
  314. return 0;
  315. if (page_count(page) != 2) /* 2: us + cache */
  316. return 0;
  317. entry.val = page_private(page);
  318. p = swap_info_get(entry);
  319. if (!p)
  320. return 0;
  321. /* Is the only swap cache user the cache itself? */
  322. retval = 0;
  323. if (p->swap_map[swp_offset(entry)] == 1) {
  324. /* Recheck the page count with the swapcache lock held.. */
  325. write_lock_irq(&swapper_space.tree_lock);
  326. if ((page_count(page) == 2) && !PageWriteback(page)) {
  327. __delete_from_swap_cache(page);
  328. SetPageDirty(page);
  329. retval = 1;
  330. }
  331. write_unlock_irq(&swapper_space.tree_lock);
  332. }
  333. spin_unlock(&swap_lock);
  334. if (retval) {
  335. swap_free(entry);
  336. page_cache_release(page);
  337. }
  338. return retval;
  339. }
  340. /*
  341. * Free the swap entry like above, but also try to
  342. * free the page cache entry if it is the last user.
  343. */
  344. void free_swap_and_cache(swp_entry_t entry)
  345. {
  346. struct swap_info_struct * p;
  347. struct page *page = NULL;
  348. p = swap_info_get(entry);
  349. if (p) {
  350. if (swap_entry_free(p, swp_offset(entry)) == 1)
  351. page = find_trylock_page(&swapper_space, entry.val);
  352. spin_unlock(&swap_lock);
  353. }
  354. if (page) {
  355. int one_user;
  356. BUG_ON(PagePrivate(page));
  357. page_cache_get(page);
  358. one_user = (page_count(page) == 2);
  359. /* Only cache user (+us), or swap space full? Free it! */
  360. if (!PageWriteback(page) && (one_user || vm_swap_full())) {
  361. delete_from_swap_cache(page);
  362. SetPageDirty(page);
  363. }
  364. unlock_page(page);
  365. page_cache_release(page);
  366. }
  367. }
  368. /*
  369. * No need to decide whether this PTE shares the swap entry with others,
  370. * just let do_wp_page work it out if a write is requested later - to
  371. * force COW, vm_page_prot omits write permission from any private vma.
  372. */
  373. static void unuse_pte(struct vm_area_struct *vma, pte_t *pte,
  374. unsigned long addr, swp_entry_t entry, struct page *page)
  375. {
  376. inc_mm_counter(vma->vm_mm, anon_rss);
  377. get_page(page);
  378. set_pte_at(vma->vm_mm, addr, pte,
  379. pte_mkold(mk_pte(page, vma->vm_page_prot)));
  380. page_add_anon_rmap(page, vma, addr);
  381. swap_free(entry);
  382. /*
  383. * Move the page to the active list so it is not
  384. * immediately swapped out again after swapon.
  385. */
  386. activate_page(page);
  387. }
  388. static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
  389. unsigned long addr, unsigned long end,
  390. swp_entry_t entry, struct page *page)
  391. {
  392. pte_t swp_pte = swp_entry_to_pte(entry);
  393. pte_t *pte;
  394. spinlock_t *ptl;
  395. int found = 0;
  396. pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
  397. do {
  398. /*
  399. * swapoff spends a _lot_ of time in this loop!
  400. * Test inline before going to call unuse_pte.
  401. */
  402. if (unlikely(pte_same(*pte, swp_pte))) {
  403. unuse_pte(vma, pte++, addr, entry, page);
  404. found = 1;
  405. break;
  406. }
  407. } while (pte++, addr += PAGE_SIZE, addr != end);
  408. pte_unmap_unlock(pte - 1, ptl);
  409. return found;
  410. }
  411. static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud,
  412. unsigned long addr, unsigned long end,
  413. swp_entry_t entry, struct page *page)
  414. {
  415. pmd_t *pmd;
  416. unsigned long next;
  417. pmd = pmd_offset(pud, addr);
  418. do {
  419. next = pmd_addr_end(addr, end);
  420. if (pmd_none_or_clear_bad(pmd))
  421. continue;
  422. if (unuse_pte_range(vma, pmd, addr, next, entry, page))
  423. return 1;
  424. } while (pmd++, addr = next, addr != end);
  425. return 0;
  426. }
  427. static inline int unuse_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
  428. unsigned long addr, unsigned long end,
  429. swp_entry_t entry, struct page *page)
  430. {
  431. pud_t *pud;
  432. unsigned long next;
  433. pud = pud_offset(pgd, addr);
  434. do {
  435. next = pud_addr_end(addr, end);
  436. if (pud_none_or_clear_bad(pud))
  437. continue;
  438. if (unuse_pmd_range(vma, pud, addr, next, entry, page))
  439. return 1;
  440. } while (pud++, addr = next, addr != end);
  441. return 0;
  442. }
  443. static int unuse_vma(struct vm_area_struct *vma,
  444. swp_entry_t entry, struct page *page)
  445. {
  446. pgd_t *pgd;
  447. unsigned long addr, end, next;
  448. if (page->mapping) {
  449. addr = page_address_in_vma(page, vma);
  450. if (addr == -EFAULT)
  451. return 0;
  452. else
  453. end = addr + PAGE_SIZE;
  454. } else {
  455. addr = vma->vm_start;
  456. end = vma->vm_end;
  457. }
  458. pgd = pgd_offset(vma->vm_mm, addr);
  459. do {
  460. next = pgd_addr_end(addr, end);
  461. if (pgd_none_or_clear_bad(pgd))
  462. continue;
  463. if (unuse_pud_range(vma, pgd, addr, next, entry, page))
  464. return 1;
  465. } while (pgd++, addr = next, addr != end);
  466. return 0;
  467. }
  468. static int unuse_mm(struct mm_struct *mm,
  469. swp_entry_t entry, struct page *page)
  470. {
  471. struct vm_area_struct *vma;
  472. if (!down_read_trylock(&mm->mmap_sem)) {
  473. /*
  474. * Activate page so shrink_cache is unlikely to unmap its
  475. * ptes while lock is dropped, so swapoff can make progress.
  476. */
  477. activate_page(page);
  478. unlock_page(page);
  479. down_read(&mm->mmap_sem);
  480. lock_page(page);
  481. }
  482. for (vma = mm->mmap; vma; vma = vma->vm_next) {
  483. if (vma->anon_vma && unuse_vma(vma, entry, page))
  484. break;
  485. }
  486. up_read(&mm->mmap_sem);
  487. /*
  488. * Currently unuse_mm cannot fail, but leave error handling
  489. * at call sites for now, since we change it from time to time.
  490. */
  491. return 0;
  492. }
  493. /*
  494. * Scan swap_map from current position to next entry still in use.
  495. * Recycle to start on reaching the end, returning 0 when empty.
  496. */
  497. static unsigned int find_next_to_unuse(struct swap_info_struct *si,
  498. unsigned int prev)
  499. {
  500. unsigned int max = si->max;
  501. unsigned int i = prev;
  502. int count;
  503. /*
  504. * No need for swap_lock here: we're just looking
  505. * for whether an entry is in use, not modifying it; false
  506. * hits are okay, and sys_swapoff() has already prevented new
  507. * allocations from this area (while holding swap_lock).
  508. */
  509. for (;;) {
  510. if (++i >= max) {
  511. if (!prev) {
  512. i = 0;
  513. break;
  514. }
  515. /*
  516. * No entries in use at top of swap_map,
  517. * loop back to start and recheck there.
  518. */
  519. max = prev + 1;
  520. prev = 0;
  521. i = 1;
  522. }
  523. count = si->swap_map[i];
  524. if (count && count != SWAP_MAP_BAD)
  525. break;
  526. }
  527. return i;
  528. }
  529. /*
  530. * We completely avoid races by reading each swap page in advance,
  531. * and then search for the process using it. All the necessary
  532. * page table adjustments can then be made atomically.
  533. */
  534. static int try_to_unuse(unsigned int type)
  535. {
  536. struct swap_info_struct * si = &swap_info[type];
  537. struct mm_struct *start_mm;
  538. unsigned short *swap_map;
  539. unsigned short swcount;
  540. struct page *page;
  541. swp_entry_t entry;
  542. unsigned int i = 0;
  543. int retval = 0;
  544. int reset_overflow = 0;
  545. int shmem;
  546. /*
  547. * When searching mms for an entry, a good strategy is to
  548. * start at the first mm we freed the previous entry from
  549. * (though actually we don't notice whether we or coincidence
  550. * freed the entry). Initialize this start_mm with a hold.
  551. *
  552. * A simpler strategy would be to start at the last mm we
  553. * freed the previous entry from; but that would take less
  554. * advantage of mmlist ordering, which clusters forked mms
  555. * together, child after parent. If we race with dup_mmap(), we
  556. * prefer to resolve parent before child, lest we miss entries
  557. * duplicated after we scanned child: using last mm would invert
  558. * that. Though it's only a serious concern when an overflowed
  559. * swap count is reset from SWAP_MAP_MAX, preventing a rescan.
  560. */
  561. start_mm = &init_mm;
  562. atomic_inc(&init_mm.mm_users);
  563. /*
  564. * Keep on scanning until all entries have gone. Usually,
  565. * one pass through swap_map is enough, but not necessarily:
  566. * there are races when an instance of an entry might be missed.
  567. */
  568. while ((i = find_next_to_unuse(si, i)) != 0) {
  569. if (signal_pending(current)) {
  570. retval = -EINTR;
  571. break;
  572. }
  573. /*
  574. * Get a page for the entry, using the existing swap
  575. * cache page if there is one. Otherwise, get a clean
  576. * page and read the swap into it.
  577. */
  578. swap_map = &si->swap_map[i];
  579. entry = swp_entry(type, i);
  580. page = read_swap_cache_async(entry, NULL, 0);
  581. if (!page) {
  582. /*
  583. * Either swap_duplicate() failed because entry
  584. * has been freed independently, and will not be
  585. * reused since sys_swapoff() already disabled
  586. * allocation from here, or alloc_page() failed.
  587. */
  588. if (!*swap_map)
  589. continue;
  590. retval = -ENOMEM;
  591. break;
  592. }
  593. /*
  594. * Don't hold on to start_mm if it looks like exiting.
  595. */
  596. if (atomic_read(&start_mm->mm_users) == 1) {
  597. mmput(start_mm);
  598. start_mm = &init_mm;
  599. atomic_inc(&init_mm.mm_users);
  600. }
  601. /*
  602. * Wait for and lock page. When do_swap_page races with
  603. * try_to_unuse, do_swap_page can handle the fault much
  604. * faster than try_to_unuse can locate the entry. This
  605. * apparently redundant "wait_on_page_locked" lets try_to_unuse
  606. * defer to do_swap_page in such a case - in some tests,
  607. * do_swap_page and try_to_unuse repeatedly compete.
  608. */
  609. wait_on_page_locked(page);
  610. wait_on_page_writeback(page);
  611. lock_page(page);
  612. wait_on_page_writeback(page);
  613. /*
  614. * Remove all references to entry.
  615. * Whenever we reach init_mm, there's no address space
  616. * to search, but use it as a reminder to search shmem.
  617. */
  618. shmem = 0;
  619. swcount = *swap_map;
  620. if (swcount > 1) {
  621. if (start_mm == &init_mm)
  622. shmem = shmem_unuse(entry, page);
  623. else
  624. retval = unuse_mm(start_mm, entry, page);
  625. }
  626. if (*swap_map > 1) {
  627. int set_start_mm = (*swap_map >= swcount);
  628. struct list_head *p = &start_mm->mmlist;
  629. struct mm_struct *new_start_mm = start_mm;
  630. struct mm_struct *prev_mm = start_mm;
  631. struct mm_struct *mm;
  632. atomic_inc(&new_start_mm->mm_users);
  633. atomic_inc(&prev_mm->mm_users);
  634. spin_lock(&mmlist_lock);
  635. while (*swap_map > 1 && !retval &&
  636. (p = p->next) != &start_mm->mmlist) {
  637. mm = list_entry(p, struct mm_struct, mmlist);
  638. if (atomic_inc_return(&mm->mm_users) == 1) {
  639. atomic_dec(&mm->mm_users);
  640. continue;
  641. }
  642. spin_unlock(&mmlist_lock);
  643. mmput(prev_mm);
  644. prev_mm = mm;
  645. cond_resched();
  646. swcount = *swap_map;
  647. if (swcount <= 1)
  648. ;
  649. else if (mm == &init_mm) {
  650. set_start_mm = 1;
  651. shmem = shmem_unuse(entry, page);
  652. } else
  653. retval = unuse_mm(mm, entry, page);
  654. if (set_start_mm && *swap_map < swcount) {
  655. mmput(new_start_mm);
  656. atomic_inc(&mm->mm_users);
  657. new_start_mm = mm;
  658. set_start_mm = 0;
  659. }
  660. spin_lock(&mmlist_lock);
  661. }
  662. spin_unlock(&mmlist_lock);
  663. mmput(prev_mm);
  664. mmput(start_mm);
  665. start_mm = new_start_mm;
  666. }
  667. if (retval) {
  668. unlock_page(page);
  669. page_cache_release(page);
  670. break;
  671. }
  672. /*
  673. * How could swap count reach 0x7fff when the maximum
  674. * pid is 0x7fff, and there's no way to repeat a swap
  675. * page within an mm (except in shmem, where it's the
  676. * shared object which takes the reference count)?
  677. * We believe SWAP_MAP_MAX cannot occur in Linux 2.4.
  678. *
  679. * If that's wrong, then we should worry more about
  680. * exit_mmap() and do_munmap() cases described above:
  681. * we might be resetting SWAP_MAP_MAX too early here.
  682. * We know "Undead"s can happen, they're okay, so don't
  683. * report them; but do report if we reset SWAP_MAP_MAX.
  684. */
  685. if (*swap_map == SWAP_MAP_MAX) {
  686. spin_lock(&swap_lock);
  687. *swap_map = 1;
  688. spin_unlock(&swap_lock);
  689. reset_overflow = 1;
  690. }
  691. /*
  692. * If a reference remains (rare), we would like to leave
  693. * the page in the swap cache; but try_to_unmap could
  694. * then re-duplicate the entry once we drop page lock,
  695. * so we might loop indefinitely; also, that page could
  696. * not be swapped out to other storage meanwhile. So:
  697. * delete from cache even if there's another reference,
  698. * after ensuring that the data has been saved to disk -
  699. * since if the reference remains (rarer), it will be
  700. * read from disk into another page. Splitting into two
  701. * pages would be incorrect if swap supported "shared
  702. * private" pages, but they are handled by tmpfs files.
  703. *
  704. * Note shmem_unuse already deleted a swappage from
  705. * the swap cache, unless the move to filepage failed:
  706. * in which case it left swappage in cache, lowered its
  707. * swap count to pass quickly through the loops above,
  708. * and now we must reincrement count to try again later.
  709. */
  710. if ((*swap_map > 1) && PageDirty(page) && PageSwapCache(page)) {
  711. struct writeback_control wbc = {
  712. .sync_mode = WB_SYNC_NONE,
  713. };
  714. swap_writepage(page, &wbc);
  715. lock_page(page);
  716. wait_on_page_writeback(page);
  717. }
  718. if (PageSwapCache(page)) {
  719. if (shmem)
  720. swap_duplicate(entry);
  721. else
  722. delete_from_swap_cache(page);
  723. }
  724. /*
  725. * So we could skip searching mms once swap count went
  726. * to 1, we did not mark any present ptes as dirty: must
  727. * mark page dirty so shrink_list will preserve it.
  728. */
  729. SetPageDirty(page);
  730. unlock_page(page);
  731. page_cache_release(page);
  732. /*
  733. * Make sure that we aren't completely killing
  734. * interactive performance.
  735. */
  736. cond_resched();
  737. }
  738. mmput(start_mm);
  739. if (reset_overflow) {
  740. printk(KERN_WARNING "swapoff: cleared swap entry overflow\n");
  741. swap_overflow = 0;
  742. }
  743. return retval;
  744. }
  745. /*
  746. * After a successful try_to_unuse, if no swap is now in use, we know
  747. * we can empty the mmlist. swap_lock must be held on entry and exit.
  748. * Note that mmlist_lock nests inside swap_lock, and an mm must be
  749. * added to the mmlist just after page_duplicate - before would be racy.
  750. */
  751. static void drain_mmlist(void)
  752. {
  753. struct list_head *p, *next;
  754. unsigned int i;
  755. for (i = 0; i < nr_swapfiles; i++)
  756. if (swap_info[i].inuse_pages)
  757. return;
  758. spin_lock(&mmlist_lock);
  759. list_for_each_safe(p, next, &init_mm.mmlist)
  760. list_del_init(p);
  761. spin_unlock(&mmlist_lock);
  762. }
  763. /*
  764. * Use this swapdev's extent info to locate the (PAGE_SIZE) block which
  765. * corresponds to page offset `offset'.
  766. */
  767. sector_t map_swap_page(struct swap_info_struct *sis, pgoff_t offset)
  768. {
  769. struct swap_extent *se = sis->curr_swap_extent;
  770. struct swap_extent *start_se = se;
  771. for ( ; ; ) {
  772. struct list_head *lh;
  773. if (se->start_page <= offset &&
  774. offset < (se->start_page + se->nr_pages)) {
  775. return se->start_block + (offset - se->start_page);
  776. }
  777. lh = se->list.next;
  778. if (lh == &sis->extent_list)
  779. lh = lh->next;
  780. se = list_entry(lh, struct swap_extent, list);
  781. sis->curr_swap_extent = se;
  782. BUG_ON(se == start_se); /* It *must* be present */
  783. }
  784. }
  785. /*
  786. * Free all of a swapdev's extent information
  787. */
  788. static void destroy_swap_extents(struct swap_info_struct *sis)
  789. {
  790. while (!list_empty(&sis->extent_list)) {
  791. struct swap_extent *se;
  792. se = list_entry(sis->extent_list.next,
  793. struct swap_extent, list);
  794. list_del(&se->list);
  795. kfree(se);
  796. }
  797. }
  798. /*
  799. * Add a block range (and the corresponding page range) into this swapdev's
  800. * extent list. The extent list is kept sorted in page order.
  801. *
  802. * This function rather assumes that it is called in ascending page order.
  803. */
  804. static int
  805. add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
  806. unsigned long nr_pages, sector_t start_block)
  807. {
  808. struct swap_extent *se;
  809. struct swap_extent *new_se;
  810. struct list_head *lh;
  811. lh = sis->extent_list.prev; /* The highest page extent */
  812. if (lh != &sis->extent_list) {
  813. se = list_entry(lh, struct swap_extent, list);
  814. BUG_ON(se->start_page + se->nr_pages != start_page);
  815. if (se->start_block + se->nr_pages == start_block) {
  816. /* Merge it */
  817. se->nr_pages += nr_pages;
  818. return 0;
  819. }
  820. }
  821. /*
  822. * No merge. Insert a new extent, preserving ordering.
  823. */
  824. new_se = kmalloc(sizeof(*se), GFP_KERNEL);
  825. if (new_se == NULL)
  826. return -ENOMEM;
  827. new_se->start_page = start_page;
  828. new_se->nr_pages = nr_pages;
  829. new_se->start_block = start_block;
  830. list_add_tail(&new_se->list, &sis->extent_list);
  831. return 1;
  832. }
  833. /*
  834. * A `swap extent' is a simple thing which maps a contiguous range of pages
  835. * onto a contiguous range of disk blocks. An ordered list of swap extents
  836. * is built at swapon time and is then used at swap_writepage/swap_readpage
  837. * time for locating where on disk a page belongs.
  838. *
  839. * If the swapfile is an S_ISBLK block device, a single extent is installed.
  840. * This is done so that the main operating code can treat S_ISBLK and S_ISREG
  841. * swap files identically.
  842. *
  843. * Whether the swapdev is an S_ISREG file or an S_ISBLK blockdev, the swap
  844. * extent list operates in PAGE_SIZE disk blocks. Both S_ISREG and S_ISBLK
  845. * swapfiles are handled *identically* after swapon time.
  846. *
  847. * For S_ISREG swapfiles, setup_swap_extents() will walk all the file's blocks
  848. * and will parse them into an ordered extent list, in PAGE_SIZE chunks. If
  849. * some stray blocks are found which do not fall within the PAGE_SIZE alignment
  850. * requirements, they are simply tossed out - we will never use those blocks
  851. * for swapping.
  852. *
  853. * For S_ISREG swapfiles we set S_SWAPFILE across the life of the swapon. This
  854. * prevents root from shooting her foot off by ftruncating an in-use swapfile,
  855. * which will scribble on the fs.
  856. *
  857. * The amount of disk space which a single swap extent represents varies.
  858. * Typically it is in the 1-4 megabyte range. So we can have hundreds of
  859. * extents in the list. To avoid much list walking, we cache the previous
  860. * search location in `curr_swap_extent', and start new searches from there.
  861. * This is extremely effective. The average number of iterations in
  862. * map_swap_page() has been measured at about 0.3 per page. - akpm.
  863. */
  864. static int setup_swap_extents(struct swap_info_struct *sis, sector_t *span)
  865. {
  866. struct inode *inode;
  867. unsigned blocks_per_page;
  868. unsigned long page_no;
  869. unsigned blkbits;
  870. sector_t probe_block;
  871. sector_t last_block;
  872. sector_t lowest_block = -1;
  873. sector_t highest_block = 0;
  874. int nr_extents = 0;
  875. int ret;
  876. inode = sis->swap_file->f_mapping->host;
  877. if (S_ISBLK(inode->i_mode)) {
  878. ret = add_swap_extent(sis, 0, sis->max, 0);
  879. *span = sis->pages;
  880. goto done;
  881. }
  882. blkbits = inode->i_blkbits;
  883. blocks_per_page = PAGE_SIZE >> blkbits;
  884. /*
  885. * Map all the blocks into the extent list. This code doesn't try
  886. * to be very smart.
  887. */
  888. probe_block = 0;
  889. page_no = 0;
  890. last_block = i_size_read(inode) >> blkbits;
  891. while ((probe_block + blocks_per_page) <= last_block &&
  892. page_no < sis->max) {
  893. unsigned block_in_page;
  894. sector_t first_block;
  895. first_block = bmap(inode, probe_block);
  896. if (first_block == 0)
  897. goto bad_bmap;
  898. /*
  899. * It must be PAGE_SIZE aligned on-disk
  900. */
  901. if (first_block & (blocks_per_page - 1)) {
  902. probe_block++;
  903. goto reprobe;
  904. }
  905. for (block_in_page = 1; block_in_page < blocks_per_page;
  906. block_in_page++) {
  907. sector_t block;
  908. block = bmap(inode, probe_block + block_in_page);
  909. if (block == 0)
  910. goto bad_bmap;
  911. if (block != first_block + block_in_page) {
  912. /* Discontiguity */
  913. probe_block++;
  914. goto reprobe;
  915. }
  916. }
  917. first_block >>= (PAGE_SHIFT - blkbits);
  918. if (page_no) { /* exclude the header page */
  919. if (first_block < lowest_block)
  920. lowest_block = first_block;
  921. if (first_block > highest_block)
  922. highest_block = first_block;
  923. }
  924. /*
  925. * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks
  926. */
  927. ret = add_swap_extent(sis, page_no, 1, first_block);
  928. if (ret < 0)
  929. goto out;
  930. nr_extents += ret;
  931. page_no++;
  932. probe_block += blocks_per_page;
  933. reprobe:
  934. continue;
  935. }
  936. ret = nr_extents;
  937. *span = 1 + highest_block - lowest_block;
  938. if (page_no == 0)
  939. page_no = 1; /* force Empty message */
  940. sis->max = page_no;
  941. sis->pages = page_no - 1;
  942. sis->highest_bit = page_no - 1;
  943. done:
  944. sis->curr_swap_extent = list_entry(sis->extent_list.prev,
  945. struct swap_extent, list);
  946. goto out;
  947. bad_bmap:
  948. printk(KERN_ERR "swapon: swapfile has holes\n");
  949. ret = -EINVAL;
  950. out:
  951. return ret;
  952. }
  953. #if 0 /* We don't need this yet */
  954. #include <linux/backing-dev.h>
  955. int page_queue_congested(struct page *page)
  956. {
  957. struct backing_dev_info *bdi;
  958. BUG_ON(!PageLocked(page)); /* It pins the swap_info_struct */
  959. if (PageSwapCache(page)) {
  960. swp_entry_t entry = { .val = page_private(page) };
  961. struct swap_info_struct *sis;
  962. sis = get_swap_info_struct(swp_type(entry));
  963. bdi = sis->bdev->bd_inode->i_mapping->backing_dev_info;
  964. } else
  965. bdi = page->mapping->backing_dev_info;
  966. return bdi_write_congested(bdi);
  967. }
  968. #endif
  969. asmlinkage long sys_swapoff(const char __user * specialfile)
  970. {
  971. struct swap_info_struct * p = NULL;
  972. unsigned short *swap_map;
  973. struct file *swap_file, *victim;
  974. struct address_space *mapping;
  975. struct inode *inode;
  976. char * pathname;
  977. int i, type, prev;
  978. int err;
  979. if (!capable(CAP_SYS_ADMIN))
  980. return -EPERM;
  981. pathname = getname(specialfile);
  982. err = PTR_ERR(pathname);
  983. if (IS_ERR(pathname))
  984. goto out;
  985. victim = filp_open(pathname, O_RDWR|O_LARGEFILE, 0);
  986. putname(pathname);
  987. err = PTR_ERR(victim);
  988. if (IS_ERR(victim))
  989. goto out;
  990. mapping = victim->f_mapping;
  991. prev = -1;
  992. spin_lock(&swap_lock);
  993. for (type = swap_list.head; type >= 0; type = swap_info[type].next) {
  994. p = swap_info + type;
  995. if ((p->flags & SWP_ACTIVE) == SWP_ACTIVE) {
  996. if (p->swap_file->f_mapping == mapping)
  997. break;
  998. }
  999. prev = type;
  1000. }
  1001. if (type < 0) {
  1002. err = -EINVAL;
  1003. spin_unlock(&swap_lock);
  1004. goto out_dput;
  1005. }
  1006. if (!security_vm_enough_memory(p->pages))
  1007. vm_unacct_memory(p->pages);
  1008. else {
  1009. err = -ENOMEM;
  1010. spin_unlock(&swap_lock);
  1011. goto out_dput;
  1012. }
  1013. if (prev < 0) {
  1014. swap_list.head = p->next;
  1015. } else {
  1016. swap_info[prev].next = p->next;
  1017. }
  1018. if (type == swap_list.next) {
  1019. /* just pick something that's safe... */
  1020. swap_list.next = swap_list.head;
  1021. }
  1022. nr_swap_pages -= p->pages;
  1023. total_swap_pages -= p->pages;
  1024. p->flags &= ~SWP_WRITEOK;
  1025. spin_unlock(&swap_lock);
  1026. current->flags |= PF_SWAPOFF;
  1027. err = try_to_unuse(type);
  1028. current->flags &= ~PF_SWAPOFF;
  1029. if (err) {
  1030. /* re-insert swap space back into swap_list */
  1031. spin_lock(&swap_lock);
  1032. for (prev = -1, i = swap_list.head; i >= 0; prev = i, i = swap_info[i].next)
  1033. if (p->prio >= swap_info[i].prio)
  1034. break;
  1035. p->next = i;
  1036. if (prev < 0)
  1037. swap_list.head = swap_list.next = p - swap_info;
  1038. else
  1039. swap_info[prev].next = p - swap_info;
  1040. nr_swap_pages += p->pages;
  1041. total_swap_pages += p->pages;
  1042. p->flags |= SWP_WRITEOK;
  1043. spin_unlock(&swap_lock);
  1044. goto out_dput;
  1045. }
  1046. /* wait for any unplug function to finish */
  1047. down_write(&swap_unplug_sem);
  1048. up_write(&swap_unplug_sem);
  1049. destroy_swap_extents(p);
  1050. down(&swapon_sem);
  1051. spin_lock(&swap_lock);
  1052. drain_mmlist();
  1053. /* wait for anyone still in scan_swap_map */
  1054. p->highest_bit = 0; /* cuts scans short */
  1055. while (p->flags >= SWP_SCANNING) {
  1056. spin_unlock(&swap_lock);
  1057. schedule_timeout_uninterruptible(1);
  1058. spin_lock(&swap_lock);
  1059. }
  1060. swap_file = p->swap_file;
  1061. p->swap_file = NULL;
  1062. p->max = 0;
  1063. swap_map = p->swap_map;
  1064. p->swap_map = NULL;
  1065. p->flags = 0;
  1066. spin_unlock(&swap_lock);
  1067. up(&swapon_sem);
  1068. vfree(swap_map);
  1069. inode = mapping->host;
  1070. if (S_ISBLK(inode->i_mode)) {
  1071. struct block_device *bdev = I_BDEV(inode);
  1072. set_blocksize(bdev, p->old_block_size);
  1073. bd_release(bdev);
  1074. } else {
  1075. mutex_lock(&inode->i_mutex);
  1076. inode->i_flags &= ~S_SWAPFILE;
  1077. mutex_unlock(&inode->i_mutex);
  1078. }
  1079. filp_close(swap_file, NULL);
  1080. err = 0;
  1081. out_dput:
  1082. filp_close(victim, NULL);
  1083. out:
  1084. return err;
  1085. }
  1086. #ifdef CONFIG_PROC_FS
  1087. /* iterator */
  1088. static void *swap_start(struct seq_file *swap, loff_t *pos)
  1089. {
  1090. struct swap_info_struct *ptr = swap_info;
  1091. int i;
  1092. loff_t l = *pos;
  1093. down(&swapon_sem);
  1094. for (i = 0; i < nr_swapfiles; i++, ptr++) {
  1095. if (!(ptr->flags & SWP_USED) || !ptr->swap_map)
  1096. continue;
  1097. if (!l--)
  1098. return ptr;
  1099. }
  1100. return NULL;
  1101. }
  1102. static void *swap_next(struct seq_file *swap, void *v, loff_t *pos)
  1103. {
  1104. struct swap_info_struct *ptr = v;
  1105. struct swap_info_struct *endptr = swap_info + nr_swapfiles;
  1106. for (++ptr; ptr < endptr; ptr++) {
  1107. if (!(ptr->flags & SWP_USED) || !ptr->swap_map)
  1108. continue;
  1109. ++*pos;
  1110. return ptr;
  1111. }
  1112. return NULL;
  1113. }
  1114. static void swap_stop(struct seq_file *swap, void *v)
  1115. {
  1116. up(&swapon_sem);
  1117. }
  1118. static int swap_show(struct seq_file *swap, void *v)
  1119. {
  1120. struct swap_info_struct *ptr = v;
  1121. struct file *file;
  1122. int len;
  1123. if (v == swap_info)
  1124. seq_puts(swap, "Filename\t\t\t\tType\t\tSize\tUsed\tPriority\n");
  1125. file = ptr->swap_file;
  1126. len = seq_path(swap, file->f_vfsmnt, file->f_dentry, " \t\n\\");
  1127. seq_printf(swap, "%*s%s\t%u\t%u\t%d\n",
  1128. len < 40 ? 40 - len : 1, " ",
  1129. S_ISBLK(file->f_dentry->d_inode->i_mode) ?
  1130. "partition" : "file\t",
  1131. ptr->pages << (PAGE_SHIFT - 10),
  1132. ptr->inuse_pages << (PAGE_SHIFT - 10),
  1133. ptr->prio);
  1134. return 0;
  1135. }
  1136. static struct seq_operations swaps_op = {
  1137. .start = swap_start,
  1138. .next = swap_next,
  1139. .stop = swap_stop,
  1140. .show = swap_show
  1141. };
  1142. static int swaps_open(struct inode *inode, struct file *file)
  1143. {
  1144. return seq_open(file, &swaps_op);
  1145. }
  1146. static struct file_operations proc_swaps_operations = {
  1147. .open = swaps_open,
  1148. .read = seq_read,
  1149. .llseek = seq_lseek,
  1150. .release = seq_release,
  1151. };
  1152. static int __init procswaps_init(void)
  1153. {
  1154. struct proc_dir_entry *entry;
  1155. entry = create_proc_entry("swaps", 0, NULL);
  1156. if (entry)
  1157. entry->proc_fops = &proc_swaps_operations;
  1158. return 0;
  1159. }
  1160. __initcall(procswaps_init);
  1161. #endif /* CONFIG_PROC_FS */
  1162. /*
  1163. * Written 01/25/92 by Simmule Turner, heavily changed by Linus.
  1164. *
  1165. * The swapon system call
  1166. */
  1167. asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags)
  1168. {
  1169. struct swap_info_struct * p;
  1170. char *name = NULL;
  1171. struct block_device *bdev = NULL;
  1172. struct file *swap_file = NULL;
  1173. struct address_space *mapping;
  1174. unsigned int type;
  1175. int i, prev;
  1176. int error;
  1177. static int least_priority;
  1178. union swap_header *swap_header = NULL;
  1179. int swap_header_version;
  1180. unsigned int nr_good_pages = 0;
  1181. int nr_extents = 0;
  1182. sector_t span;
  1183. unsigned long maxpages = 1;
  1184. int swapfilesize;
  1185. unsigned short *swap_map;
  1186. struct page *page = NULL;
  1187. struct inode *inode = NULL;
  1188. int did_down = 0;
  1189. if (!capable(CAP_SYS_ADMIN))
  1190. return -EPERM;
  1191. spin_lock(&swap_lock);
  1192. p = swap_info;
  1193. for (type = 0 ; type < nr_swapfiles ; type++,p++)
  1194. if (!(p->flags & SWP_USED))
  1195. break;
  1196. error = -EPERM;
  1197. /*
  1198. * Test if adding another swap device is possible. There are
  1199. * two limiting factors: 1) the number of bits for the swap
  1200. * type swp_entry_t definition and 2) the number of bits for
  1201. * the swap type in the swap ptes as defined by the different
  1202. * architectures. To honor both limitations a swap entry
  1203. * with swap offset 0 and swap type ~0UL is created, encoded
  1204. * to a swap pte, decoded to a swp_entry_t again and finally
  1205. * the swap type part is extracted. This will mask all bits
  1206. * from the initial ~0UL that can't be encoded in either the
  1207. * swp_entry_t or the architecture definition of a swap pte.
  1208. */
  1209. if (type > swp_type(pte_to_swp_entry(swp_entry_to_pte(swp_entry(~0UL,0))))) {
  1210. spin_unlock(&swap_lock);
  1211. goto out;
  1212. }
  1213. if (type >= nr_swapfiles)
  1214. nr_swapfiles = type+1;
  1215. INIT_LIST_HEAD(&p->extent_list);
  1216. p->flags = SWP_USED;
  1217. p->swap_file = NULL;
  1218. p->old_block_size = 0;
  1219. p->swap_map = NULL;
  1220. p->lowest_bit = 0;
  1221. p->highest_bit = 0;
  1222. p->cluster_nr = 0;
  1223. p->inuse_pages = 0;
  1224. p->next = -1;
  1225. if (swap_flags & SWAP_FLAG_PREFER) {
  1226. p->prio =
  1227. (swap_flags & SWAP_FLAG_PRIO_MASK)>>SWAP_FLAG_PRIO_SHIFT;
  1228. } else {
  1229. p->prio = --least_priority;
  1230. }
  1231. spin_unlock(&swap_lock);
  1232. name = getname(specialfile);
  1233. error = PTR_ERR(name);
  1234. if (IS_ERR(name)) {
  1235. name = NULL;
  1236. goto bad_swap_2;
  1237. }
  1238. swap_file = filp_open(name, O_RDWR|O_LARGEFILE, 0);
  1239. error = PTR_ERR(swap_file);
  1240. if (IS_ERR(swap_file)) {
  1241. swap_file = NULL;
  1242. goto bad_swap_2;
  1243. }
  1244. p->swap_file = swap_file;
  1245. mapping = swap_file->f_mapping;
  1246. inode = mapping->host;
  1247. error = -EBUSY;
  1248. for (i = 0; i < nr_swapfiles; i++) {
  1249. struct swap_info_struct *q = &swap_info[i];
  1250. if (i == type || !q->swap_file)
  1251. continue;
  1252. if (mapping == q->swap_file->f_mapping)
  1253. goto bad_swap;
  1254. }
  1255. error = -EINVAL;
  1256. if (S_ISBLK(inode->i_mode)) {
  1257. bdev = I_BDEV(inode);
  1258. error = bd_claim(bdev, sys_swapon);
  1259. if (error < 0) {
  1260. bdev = NULL;
  1261. error = -EINVAL;
  1262. goto bad_swap;
  1263. }
  1264. p->old_block_size = block_size(bdev);
  1265. error = set_blocksize(bdev, PAGE_SIZE);
  1266. if (error < 0)
  1267. goto bad_swap;
  1268. p->bdev = bdev;
  1269. } else if (S_ISREG(inode->i_mode)) {
  1270. p->bdev = inode->i_sb->s_bdev;
  1271. mutex_lock(&inode->i_mutex);
  1272. did_down = 1;
  1273. if (IS_SWAPFILE(inode)) {
  1274. error = -EBUSY;
  1275. goto bad_swap;
  1276. }
  1277. } else {
  1278. goto bad_swap;
  1279. }
  1280. swapfilesize = i_size_read(inode) >> PAGE_SHIFT;
  1281. /*
  1282. * Read the swap header.
  1283. */
  1284. if (!mapping->a_ops->readpage) {
  1285. error = -EINVAL;
  1286. goto bad_swap;
  1287. }
  1288. page = read_cache_page(mapping, 0,
  1289. (filler_t *)mapping->a_ops->readpage, swap_file);
  1290. if (IS_ERR(page)) {
  1291. error = PTR_ERR(page);
  1292. goto bad_swap;
  1293. }
  1294. wait_on_page_locked(page);
  1295. if (!PageUptodate(page))
  1296. goto bad_swap;
  1297. kmap(page);
  1298. swap_header = page_address(page);
  1299. if (!memcmp("SWAP-SPACE",swap_header->magic.magic,10))
  1300. swap_header_version = 1;
  1301. else if (!memcmp("SWAPSPACE2",swap_header->magic.magic,10))
  1302. swap_header_version = 2;
  1303. else {
  1304. printk(KERN_ERR "Unable to find swap-space signature\n");
  1305. error = -EINVAL;
  1306. goto bad_swap;
  1307. }
  1308. switch (swap_header_version) {
  1309. case 1:
  1310. printk(KERN_ERR "version 0 swap is no longer supported. "
  1311. "Use mkswap -v1 %s\n", name);
  1312. error = -EINVAL;
  1313. goto bad_swap;
  1314. case 2:
  1315. /* Check the swap header's sub-version and the size of
  1316. the swap file and bad block lists */
  1317. if (swap_header->info.version != 1) {
  1318. printk(KERN_WARNING
  1319. "Unable to handle swap header version %d\n",
  1320. swap_header->info.version);
  1321. error = -EINVAL;
  1322. goto bad_swap;
  1323. }
  1324. p->lowest_bit = 1;
  1325. p->cluster_next = 1;
  1326. /*
  1327. * Find out how many pages are allowed for a single swap
  1328. * device. There are two limiting factors: 1) the number of
  1329. * bits for the swap offset in the swp_entry_t type and
  1330. * 2) the number of bits in the a swap pte as defined by
  1331. * the different architectures. In order to find the
  1332. * largest possible bit mask a swap entry with swap type 0
  1333. * and swap offset ~0UL is created, encoded to a swap pte,
  1334. * decoded to a swp_entry_t again and finally the swap
  1335. * offset is extracted. This will mask all the bits from
  1336. * the initial ~0UL mask that can't be encoded in either
  1337. * the swp_entry_t or the architecture definition of a
  1338. * swap pte.
  1339. */
  1340. maxpages = swp_offset(pte_to_swp_entry(swp_entry_to_pte(swp_entry(0,~0UL)))) - 1;
  1341. if (maxpages > swap_header->info.last_page)
  1342. maxpages = swap_header->info.last_page;
  1343. p->highest_bit = maxpages - 1;
  1344. error = -EINVAL;
  1345. if (!maxpages)
  1346. goto bad_swap;
  1347. if (swap_header->info.nr_badpages && S_ISREG(inode->i_mode))
  1348. goto bad_swap;
  1349. if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
  1350. goto bad_swap;
  1351. /* OK, set up the swap map and apply the bad block list */
  1352. if (!(p->swap_map = vmalloc(maxpages * sizeof(short)))) {
  1353. error = -ENOMEM;
  1354. goto bad_swap;
  1355. }
  1356. error = 0;
  1357. memset(p->swap_map, 0, maxpages * sizeof(short));
  1358. for (i = 0; i < swap_header->info.nr_badpages; i++) {
  1359. int page_nr = swap_header->info.badpages[i];
  1360. if (page_nr <= 0 || page_nr >= swap_header->info.last_page)
  1361. error = -EINVAL;
  1362. else
  1363. p->swap_map[page_nr] = SWAP_MAP_BAD;
  1364. }
  1365. nr_good_pages = swap_header->info.last_page -
  1366. swap_header->info.nr_badpages -
  1367. 1 /* header page */;
  1368. if (error)
  1369. goto bad_swap;
  1370. }
  1371. if (swapfilesize && maxpages > swapfilesize) {
  1372. printk(KERN_WARNING
  1373. "Swap area shorter than signature indicates\n");
  1374. error = -EINVAL;
  1375. goto bad_swap;
  1376. }
  1377. if (nr_good_pages) {
  1378. p->swap_map[0] = SWAP_MAP_BAD;
  1379. p->max = maxpages;
  1380. p->pages = nr_good_pages;
  1381. nr_extents = setup_swap_extents(p, &span);
  1382. if (nr_extents < 0) {
  1383. error = nr_extents;
  1384. goto bad_swap;
  1385. }
  1386. nr_good_pages = p->pages;
  1387. }
  1388. if (!nr_good_pages) {
  1389. printk(KERN_WARNING "Empty swap-file\n");
  1390. error = -EINVAL;
  1391. goto bad_swap;
  1392. }
  1393. down(&swapon_sem);
  1394. spin_lock(&swap_lock);
  1395. p->flags = SWP_ACTIVE;
  1396. nr_swap_pages += nr_good_pages;
  1397. total_swap_pages += nr_good_pages;
  1398. printk(KERN_INFO "Adding %uk swap on %s. "
  1399. "Priority:%d extents:%d across:%lluk\n",
  1400. nr_good_pages<<(PAGE_SHIFT-10), name, p->prio,
  1401. nr_extents, (unsigned long long)span<<(PAGE_SHIFT-10));
  1402. /* insert swap space into swap_list: */
  1403. prev = -1;
  1404. for (i = swap_list.head; i >= 0; i = swap_info[i].next) {
  1405. if (p->prio >= swap_info[i].prio) {
  1406. break;
  1407. }
  1408. prev = i;
  1409. }
  1410. p->next = i;
  1411. if (prev < 0) {
  1412. swap_list.head = swap_list.next = p - swap_info;
  1413. } else {
  1414. swap_info[prev].next = p - swap_info;
  1415. }
  1416. spin_unlock(&swap_lock);
  1417. up(&swapon_sem);
  1418. error = 0;
  1419. goto out;
  1420. bad_swap:
  1421. if (bdev) {
  1422. set_blocksize(bdev, p->old_block_size);
  1423. bd_release(bdev);
  1424. }
  1425. destroy_swap_extents(p);
  1426. bad_swap_2:
  1427. spin_lock(&swap_lock);
  1428. swap_map = p->swap_map;
  1429. p->swap_file = NULL;
  1430. p->swap_map = NULL;
  1431. p->flags = 0;
  1432. if (!(swap_flags & SWAP_FLAG_PREFER))
  1433. ++least_priority;
  1434. spin_unlock(&swap_lock);
  1435. vfree(swap_map);
  1436. if (swap_file)
  1437. filp_close(swap_file, NULL);
  1438. out:
  1439. if (page && !IS_ERR(page)) {
  1440. kunmap(page);
  1441. page_cache_release(page);
  1442. }
  1443. if (name)
  1444. putname(name);
  1445. if (did_down) {
  1446. if (!error)
  1447. inode->i_flags |= S_SWAPFILE;
  1448. mutex_unlock(&inode->i_mutex);
  1449. }
  1450. return error;
  1451. }
  1452. void si_swapinfo(struct sysinfo *val)
  1453. {
  1454. unsigned int i;
  1455. unsigned long nr_to_be_unused = 0;
  1456. spin_lock(&swap_lock);
  1457. for (i = 0; i < nr_swapfiles; i++) {
  1458. if (!(swap_info[i].flags & SWP_USED) ||
  1459. (swap_info[i].flags & SWP_WRITEOK))
  1460. continue;
  1461. nr_to_be_unused += swap_info[i].inuse_pages;
  1462. }
  1463. val->freeswap = nr_swap_pages + nr_to_be_unused;
  1464. val->totalswap = total_swap_pages + nr_to_be_unused;
  1465. spin_unlock(&swap_lock);
  1466. }
  1467. /*
  1468. * Verify that a swap entry is valid and increment its swap map count.
  1469. *
  1470. * Note: if swap_map[] reaches SWAP_MAP_MAX the entries are treated as
  1471. * "permanent", but will be reclaimed by the next swapoff.
  1472. */
  1473. int swap_duplicate(swp_entry_t entry)
  1474. {
  1475. struct swap_info_struct * p;
  1476. unsigned long offset, type;
  1477. int result = 0;
  1478. type = swp_type(entry);
  1479. if (type >= nr_swapfiles)
  1480. goto bad_file;
  1481. p = type + swap_info;
  1482. offset = swp_offset(entry);
  1483. spin_lock(&swap_lock);
  1484. if (offset < p->max && p->swap_map[offset]) {
  1485. if (p->swap_map[offset] < SWAP_MAP_MAX - 1) {
  1486. p->swap_map[offset]++;
  1487. result = 1;
  1488. } else if (p->swap_map[offset] <= SWAP_MAP_MAX) {
  1489. if (swap_overflow++ < 5)
  1490. printk(KERN_WARNING "swap_dup: swap entry overflow\n");
  1491. p->swap_map[offset] = SWAP_MAP_MAX;
  1492. result = 1;
  1493. }
  1494. }
  1495. spin_unlock(&swap_lock);
  1496. out:
  1497. return result;
  1498. bad_file:
  1499. printk(KERN_ERR "swap_dup: %s%08lx\n", Bad_file, entry.val);
  1500. goto out;
  1501. }
  1502. struct swap_info_struct *
  1503. get_swap_info_struct(unsigned type)
  1504. {
  1505. return &swap_info[type];
  1506. }
  1507. /*
  1508. * swap_lock prevents swap_map being freed. Don't grab an extra
  1509. * reference on the swaphandle, it doesn't matter if it becomes unused.
  1510. */
  1511. int valid_swaphandles(swp_entry_t entry, unsigned long *offset)
  1512. {
  1513. int ret = 0, i = 1 << page_cluster;
  1514. unsigned long toff;
  1515. struct swap_info_struct *swapdev = swp_type(entry) + swap_info;
  1516. if (!page_cluster) /* no readahead */
  1517. return 0;
  1518. toff = (swp_offset(entry) >> page_cluster) << page_cluster;
  1519. if (!toff) /* first page is swap header */
  1520. toff++, i--;
  1521. *offset = toff;
  1522. spin_lock(&swap_lock);
  1523. do {
  1524. /* Don't read-ahead past the end of the swap area */
  1525. if (toff >= swapdev->max)
  1526. break;
  1527. /* Don't read in free or bad pages */
  1528. if (!swapdev->swap_map[toff])
  1529. break;
  1530. if (swapdev->swap_map[toff] == SWAP_MAP_BAD)
  1531. break;
  1532. toff++;
  1533. ret++;
  1534. } while (--i);
  1535. spin_unlock(&swap_lock);
  1536. return ret;
  1537. }