memory-failure.c 46 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677
  1. /*
  2. * Copyright (C) 2008, 2009 Intel Corporation
  3. * Authors: Andi Kleen, Fengguang Wu
  4. *
  5. * This software may be redistributed and/or modified under the terms of
  6. * the GNU General Public License ("GPL") version 2 only as published by the
  7. * Free Software Foundation.
  8. *
  9. * High level machine check handler. Handles pages reported by the
  10. * hardware as being corrupted usually due to a multi-bit ECC memory or cache
  11. * failure.
  12. *
  13. * In addition there is a "soft offline" entry point that allows stop using
  14. * not-yet-corrupted-by-suspicious pages without killing anything.
  15. *
  16. * Handles page cache pages in various states. The tricky part
  17. * here is that we can access any page asynchronously in respect to
  18. * other VM users, because memory failures could happen anytime and
  19. * anywhere. This could violate some of their assumptions. This is why
  20. * this code has to be extremely careful. Generally it tries to use
  21. * normal locking rules, as in get the standard locks, even if that means
  22. * the error handling takes potentially a long time.
  23. *
  24. * There are several operations here with exponential complexity because
  25. * of unsuitable VM data structures. For example the operation to map back
  26. * from RMAP chains to processes has to walk the complete process list and
  27. * has non linear complexity with the number. But since memory corruptions
  28. * are rare we hope to get away with this. This avoids impacting the core
  29. * VM.
  30. */
  31. /*
  32. * Notebook:
  33. * - hugetlb needs more code
  34. * - kcore/oldmem/vmcore/mem/kmem check for hwpoison pages
  35. * - pass bad pages to kdump next kernel
  36. */
  37. #include <linux/kernel.h>
  38. #include <linux/mm.h>
  39. #include <linux/page-flags.h>
  40. #include <linux/kernel-page-flags.h>
  41. #include <linux/sched.h>
  42. #include <linux/ksm.h>
  43. #include <linux/rmap.h>
  44. #include <linux/export.h>
  45. #include <linux/pagemap.h>
  46. #include <linux/swap.h>
  47. #include <linux/backing-dev.h>
  48. #include <linux/migrate.h>
  49. #include <linux/page-isolation.h>
  50. #include <linux/suspend.h>
  51. #include <linux/slab.h>
  52. #include <linux/swapops.h>
  53. #include <linux/hugetlb.h>
  54. #include <linux/memory_hotplug.h>
  55. #include <linux/mm_inline.h>
  56. #include <linux/kfifo.h>
  57. #include "internal.h"
  58. int sysctl_memory_failure_early_kill __read_mostly = 0;
  59. int sysctl_memory_failure_recovery __read_mostly = 1;
  60. atomic_long_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0);
  61. #if defined(CONFIG_HWPOISON_INJECT) || defined(CONFIG_HWPOISON_INJECT_MODULE)
  62. u32 hwpoison_filter_enable = 0;
  63. u32 hwpoison_filter_dev_major = ~0U;
  64. u32 hwpoison_filter_dev_minor = ~0U;
  65. u64 hwpoison_filter_flags_mask;
  66. u64 hwpoison_filter_flags_value;
  67. EXPORT_SYMBOL_GPL(hwpoison_filter_enable);
  68. EXPORT_SYMBOL_GPL(hwpoison_filter_dev_major);
  69. EXPORT_SYMBOL_GPL(hwpoison_filter_dev_minor);
  70. EXPORT_SYMBOL_GPL(hwpoison_filter_flags_mask);
  71. EXPORT_SYMBOL_GPL(hwpoison_filter_flags_value);
  72. static int hwpoison_filter_dev(struct page *p)
  73. {
  74. struct address_space *mapping;
  75. dev_t dev;
  76. if (hwpoison_filter_dev_major == ~0U &&
  77. hwpoison_filter_dev_minor == ~0U)
  78. return 0;
  79. /*
  80. * page_mapping() does not accept slab pages.
  81. */
  82. if (PageSlab(p))
  83. return -EINVAL;
  84. mapping = page_mapping(p);
  85. if (mapping == NULL || mapping->host == NULL)
  86. return -EINVAL;
  87. dev = mapping->host->i_sb->s_dev;
  88. if (hwpoison_filter_dev_major != ~0U &&
  89. hwpoison_filter_dev_major != MAJOR(dev))
  90. return -EINVAL;
  91. if (hwpoison_filter_dev_minor != ~0U &&
  92. hwpoison_filter_dev_minor != MINOR(dev))
  93. return -EINVAL;
  94. return 0;
  95. }
  96. static int hwpoison_filter_flags(struct page *p)
  97. {
  98. if (!hwpoison_filter_flags_mask)
  99. return 0;
  100. if ((stable_page_flags(p) & hwpoison_filter_flags_mask) ==
  101. hwpoison_filter_flags_value)
  102. return 0;
  103. else
  104. return -EINVAL;
  105. }
  106. /*
  107. * This allows stress tests to limit test scope to a collection of tasks
  108. * by putting them under some memcg. This prevents killing unrelated/important
  109. * processes such as /sbin/init. Note that the target task may share clean
  110. * pages with init (eg. libc text), which is harmless. If the target task
  111. * share _dirty_ pages with another task B, the test scheme must make sure B
  112. * is also included in the memcg. At last, due to race conditions this filter
  113. * can only guarantee that the page either belongs to the memcg tasks, or is
  114. * a freed page.
  115. */
  116. #ifdef CONFIG_MEMCG_SWAP
  117. u64 hwpoison_filter_memcg;
  118. EXPORT_SYMBOL_GPL(hwpoison_filter_memcg);
  119. static int hwpoison_filter_task(struct page *p)
  120. {
  121. struct mem_cgroup *mem;
  122. struct cgroup_subsys_state *css;
  123. unsigned long ino;
  124. if (!hwpoison_filter_memcg)
  125. return 0;
  126. mem = try_get_mem_cgroup_from_page(p);
  127. if (!mem)
  128. return -EINVAL;
  129. css = mem_cgroup_css(mem);
  130. /* root_mem_cgroup has NULL dentries */
  131. if (!css->cgroup->dentry)
  132. return -EINVAL;
  133. ino = css->cgroup->dentry->d_inode->i_ino;
  134. css_put(css);
  135. if (ino != hwpoison_filter_memcg)
  136. return -EINVAL;
  137. return 0;
  138. }
  139. #else
  140. static int hwpoison_filter_task(struct page *p) { return 0; }
  141. #endif
  142. int hwpoison_filter(struct page *p)
  143. {
  144. if (!hwpoison_filter_enable)
  145. return 0;
  146. if (hwpoison_filter_dev(p))
  147. return -EINVAL;
  148. if (hwpoison_filter_flags(p))
  149. return -EINVAL;
  150. if (hwpoison_filter_task(p))
  151. return -EINVAL;
  152. return 0;
  153. }
  154. #else
  155. int hwpoison_filter(struct page *p)
  156. {
  157. return 0;
  158. }
  159. #endif
  160. EXPORT_SYMBOL_GPL(hwpoison_filter);
  161. /*
  162. * Send all the processes who have the page mapped a signal.
  163. * ``action optional'' if they are not immediately affected by the error
  164. * ``action required'' if error happened in current execution context
  165. */
  166. static int kill_proc(struct task_struct *t, unsigned long addr, int trapno,
  167. unsigned long pfn, struct page *page, int flags)
  168. {
  169. struct siginfo si;
  170. int ret;
  171. printk(KERN_ERR
  172. "MCE %#lx: Killing %s:%d due to hardware memory corruption\n",
  173. pfn, t->comm, t->pid);
  174. si.si_signo = SIGBUS;
  175. si.si_errno = 0;
  176. si.si_addr = (void *)addr;
  177. #ifdef __ARCH_SI_TRAPNO
  178. si.si_trapno = trapno;
  179. #endif
  180. si.si_addr_lsb = compound_order(compound_head(page)) + PAGE_SHIFT;
  181. if ((flags & MF_ACTION_REQUIRED) && t == current) {
  182. si.si_code = BUS_MCEERR_AR;
  183. ret = force_sig_info(SIGBUS, &si, t);
  184. } else {
  185. /*
  186. * Don't use force here, it's convenient if the signal
  187. * can be temporarily blocked.
  188. * This could cause a loop when the user sets SIGBUS
  189. * to SIG_IGN, but hopefully no one will do that?
  190. */
  191. si.si_code = BUS_MCEERR_AO;
  192. ret = send_sig_info(SIGBUS, &si, t); /* synchronous? */
  193. }
  194. if (ret < 0)
  195. printk(KERN_INFO "MCE: Error sending signal to %s:%d: %d\n",
  196. t->comm, t->pid, ret);
  197. return ret;
  198. }
  199. /*
  200. * When a unknown page type is encountered drain as many buffers as possible
  201. * in the hope to turn the page into a LRU or free page, which we can handle.
  202. */
  203. void shake_page(struct page *p, int access)
  204. {
  205. if (!PageSlab(p)) {
  206. lru_add_drain_all();
  207. if (PageLRU(p))
  208. return;
  209. drain_all_pages();
  210. if (PageLRU(p) || is_free_buddy_page(p))
  211. return;
  212. }
  213. /*
  214. * Only call shrink_slab here (which would also shrink other caches) if
  215. * access is not potentially fatal.
  216. */
  217. if (access) {
  218. int nr;
  219. int nid = page_to_nid(p);
  220. do {
  221. struct shrink_control shrink = {
  222. .gfp_mask = GFP_KERNEL,
  223. };
  224. node_set(nid, shrink.nodes_to_scan);
  225. nr = shrink_slab(&shrink, 1000, 1000);
  226. if (page_count(p) == 1)
  227. break;
  228. } while (nr > 10);
  229. }
  230. }
  231. EXPORT_SYMBOL_GPL(shake_page);
  232. /*
  233. * Kill all processes that have a poisoned page mapped and then isolate
  234. * the page.
  235. *
  236. * General strategy:
  237. * Find all processes having the page mapped and kill them.
  238. * But we keep a page reference around so that the page is not
  239. * actually freed yet.
  240. * Then stash the page away
  241. *
  242. * There's no convenient way to get back to mapped processes
  243. * from the VMAs. So do a brute-force search over all
  244. * running processes.
  245. *
  246. * Remember that machine checks are not common (or rather
  247. * if they are common you have other problems), so this shouldn't
  248. * be a performance issue.
  249. *
  250. * Also there are some races possible while we get from the
  251. * error detection to actually handle it.
  252. */
  253. struct to_kill {
  254. struct list_head nd;
  255. struct task_struct *tsk;
  256. unsigned long addr;
  257. char addr_valid;
  258. };
  259. /*
  260. * Failure handling: if we can't find or can't kill a process there's
  261. * not much we can do. We just print a message and ignore otherwise.
  262. */
  263. /*
  264. * Schedule a process for later kill.
  265. * Uses GFP_ATOMIC allocations to avoid potential recursions in the VM.
  266. * TBD would GFP_NOIO be enough?
  267. */
  268. static void add_to_kill(struct task_struct *tsk, struct page *p,
  269. struct vm_area_struct *vma,
  270. struct list_head *to_kill,
  271. struct to_kill **tkc)
  272. {
  273. struct to_kill *tk;
  274. if (*tkc) {
  275. tk = *tkc;
  276. *tkc = NULL;
  277. } else {
  278. tk = kmalloc(sizeof(struct to_kill), GFP_ATOMIC);
  279. if (!tk) {
  280. printk(KERN_ERR
  281. "MCE: Out of memory while machine check handling\n");
  282. return;
  283. }
  284. }
  285. tk->addr = page_address_in_vma(p, vma);
  286. tk->addr_valid = 1;
  287. /*
  288. * In theory we don't have to kill when the page was
  289. * munmaped. But it could be also a mremap. Since that's
  290. * likely very rare kill anyways just out of paranoia, but use
  291. * a SIGKILL because the error is not contained anymore.
  292. */
  293. if (tk->addr == -EFAULT) {
  294. pr_info("MCE: Unable to find user space address %lx in %s\n",
  295. page_to_pfn(p), tsk->comm);
  296. tk->addr_valid = 0;
  297. }
  298. get_task_struct(tsk);
  299. tk->tsk = tsk;
  300. list_add_tail(&tk->nd, to_kill);
  301. }
  302. /*
  303. * Kill the processes that have been collected earlier.
  304. *
  305. * Only do anything when DOIT is set, otherwise just free the list
  306. * (this is used for clean pages which do not need killing)
  307. * Also when FAIL is set do a force kill because something went
  308. * wrong earlier.
  309. */
  310. static void kill_procs(struct list_head *to_kill, int forcekill, int trapno,
  311. int fail, struct page *page, unsigned long pfn,
  312. int flags)
  313. {
  314. struct to_kill *tk, *next;
  315. list_for_each_entry_safe (tk, next, to_kill, nd) {
  316. if (forcekill) {
  317. /*
  318. * In case something went wrong with munmapping
  319. * make sure the process doesn't catch the
  320. * signal and then access the memory. Just kill it.
  321. */
  322. if (fail || tk->addr_valid == 0) {
  323. printk(KERN_ERR
  324. "MCE %#lx: forcibly killing %s:%d because of failure to unmap corrupted page\n",
  325. pfn, tk->tsk->comm, tk->tsk->pid);
  326. force_sig(SIGKILL, tk->tsk);
  327. }
  328. /*
  329. * In theory the process could have mapped
  330. * something else on the address in-between. We could
  331. * check for that, but we need to tell the
  332. * process anyways.
  333. */
  334. else if (kill_proc(tk->tsk, tk->addr, trapno,
  335. pfn, page, flags) < 0)
  336. printk(KERN_ERR
  337. "MCE %#lx: Cannot send advisory machine check signal to %s:%d\n",
  338. pfn, tk->tsk->comm, tk->tsk->pid);
  339. }
  340. put_task_struct(tk->tsk);
  341. kfree(tk);
  342. }
  343. }
  344. static int task_early_kill(struct task_struct *tsk)
  345. {
  346. if (!tsk->mm)
  347. return 0;
  348. if (tsk->flags & PF_MCE_PROCESS)
  349. return !!(tsk->flags & PF_MCE_EARLY);
  350. return sysctl_memory_failure_early_kill;
  351. }
  352. /*
  353. * Collect processes when the error hit an anonymous page.
  354. */
  355. static void collect_procs_anon(struct page *page, struct list_head *to_kill,
  356. struct to_kill **tkc)
  357. {
  358. struct vm_area_struct *vma;
  359. struct task_struct *tsk;
  360. struct anon_vma *av;
  361. pgoff_t pgoff;
  362. av = page_lock_anon_vma_read(page);
  363. if (av == NULL) /* Not actually mapped anymore */
  364. return;
  365. pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
  366. read_lock(&tasklist_lock);
  367. for_each_process (tsk) {
  368. struct anon_vma_chain *vmac;
  369. if (!task_early_kill(tsk))
  370. continue;
  371. anon_vma_interval_tree_foreach(vmac, &av->rb_root,
  372. pgoff, pgoff) {
  373. vma = vmac->vma;
  374. if (!page_mapped_in_vma(page, vma))
  375. continue;
  376. if (vma->vm_mm == tsk->mm)
  377. add_to_kill(tsk, page, vma, to_kill, tkc);
  378. }
  379. }
  380. read_unlock(&tasklist_lock);
  381. page_unlock_anon_vma_read(av);
  382. }
  383. /*
  384. * Collect processes when the error hit a file mapped page.
  385. */
  386. static void collect_procs_file(struct page *page, struct list_head *to_kill,
  387. struct to_kill **tkc)
  388. {
  389. struct vm_area_struct *vma;
  390. struct task_struct *tsk;
  391. struct address_space *mapping = page->mapping;
  392. mutex_lock(&mapping->i_mmap_mutex);
  393. read_lock(&tasklist_lock);
  394. for_each_process(tsk) {
  395. pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
  396. if (!task_early_kill(tsk))
  397. continue;
  398. vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff,
  399. pgoff) {
  400. /*
  401. * Send early kill signal to tasks where a vma covers
  402. * the page but the corrupted page is not necessarily
  403. * mapped it in its pte.
  404. * Assume applications who requested early kill want
  405. * to be informed of all such data corruptions.
  406. */
  407. if (vma->vm_mm == tsk->mm)
  408. add_to_kill(tsk, page, vma, to_kill, tkc);
  409. }
  410. }
  411. read_unlock(&tasklist_lock);
  412. mutex_unlock(&mapping->i_mmap_mutex);
  413. }
  414. /*
  415. * Collect the processes who have the corrupted page mapped to kill.
  416. * This is done in two steps for locking reasons.
  417. * First preallocate one tokill structure outside the spin locks,
  418. * so that we can kill at least one process reasonably reliable.
  419. */
  420. static void collect_procs(struct page *page, struct list_head *tokill)
  421. {
  422. struct to_kill *tk;
  423. if (!page->mapping)
  424. return;
  425. tk = kmalloc(sizeof(struct to_kill), GFP_NOIO);
  426. if (!tk)
  427. return;
  428. if (PageAnon(page))
  429. collect_procs_anon(page, tokill, &tk);
  430. else
  431. collect_procs_file(page, tokill, &tk);
  432. kfree(tk);
  433. }
  434. /*
  435. * Error handlers for various types of pages.
  436. */
  437. enum outcome {
  438. IGNORED, /* Error: cannot be handled */
  439. FAILED, /* Error: handling failed */
  440. DELAYED, /* Will be handled later */
  441. RECOVERED, /* Successfully recovered */
  442. };
  443. static const char *action_name[] = {
  444. [IGNORED] = "Ignored",
  445. [FAILED] = "Failed",
  446. [DELAYED] = "Delayed",
  447. [RECOVERED] = "Recovered",
  448. };
  449. /*
  450. * XXX: It is possible that a page is isolated from LRU cache,
  451. * and then kept in swap cache or failed to remove from page cache.
  452. * The page count will stop it from being freed by unpoison.
  453. * Stress tests should be aware of this memory leak problem.
  454. */
  455. static int delete_from_lru_cache(struct page *p)
  456. {
  457. if (!isolate_lru_page(p)) {
  458. /*
  459. * Clear sensible page flags, so that the buddy system won't
  460. * complain when the page is unpoison-and-freed.
  461. */
  462. ClearPageActive(p);
  463. ClearPageUnevictable(p);
  464. /*
  465. * drop the page count elevated by isolate_lru_page()
  466. */
  467. page_cache_release(p);
  468. return 0;
  469. }
  470. return -EIO;
  471. }
  472. /*
  473. * Error hit kernel page.
  474. * Do nothing, try to be lucky and not touch this instead. For a few cases we
  475. * could be more sophisticated.
  476. */
  477. static int me_kernel(struct page *p, unsigned long pfn)
  478. {
  479. return IGNORED;
  480. }
  481. /*
  482. * Page in unknown state. Do nothing.
  483. */
  484. static int me_unknown(struct page *p, unsigned long pfn)
  485. {
  486. printk(KERN_ERR "MCE %#lx: Unknown page state\n", pfn);
  487. return FAILED;
  488. }
  489. /*
  490. * Clean (or cleaned) page cache page.
  491. */
  492. static int me_pagecache_clean(struct page *p, unsigned long pfn)
  493. {
  494. int err;
  495. int ret = FAILED;
  496. struct address_space *mapping;
  497. delete_from_lru_cache(p);
  498. /*
  499. * For anonymous pages we're done the only reference left
  500. * should be the one m_f() holds.
  501. */
  502. if (PageAnon(p))
  503. return RECOVERED;
  504. /*
  505. * Now truncate the page in the page cache. This is really
  506. * more like a "temporary hole punch"
  507. * Don't do this for block devices when someone else
  508. * has a reference, because it could be file system metadata
  509. * and that's not safe to truncate.
  510. */
  511. mapping = page_mapping(p);
  512. if (!mapping) {
  513. /*
  514. * Page has been teared down in the meanwhile
  515. */
  516. return FAILED;
  517. }
  518. /*
  519. * Truncation is a bit tricky. Enable it per file system for now.
  520. *
  521. * Open: to take i_mutex or not for this? Right now we don't.
  522. */
  523. if (mapping->a_ops->error_remove_page) {
  524. err = mapping->a_ops->error_remove_page(mapping, p);
  525. if (err != 0) {
  526. printk(KERN_INFO "MCE %#lx: Failed to punch page: %d\n",
  527. pfn, err);
  528. } else if (page_has_private(p) &&
  529. !try_to_release_page(p, GFP_NOIO)) {
  530. pr_info("MCE %#lx: failed to release buffers\n", pfn);
  531. } else {
  532. ret = RECOVERED;
  533. }
  534. } else {
  535. /*
  536. * If the file system doesn't support it just invalidate
  537. * This fails on dirty or anything with private pages
  538. */
  539. if (invalidate_inode_page(p))
  540. ret = RECOVERED;
  541. else
  542. printk(KERN_INFO "MCE %#lx: Failed to invalidate\n",
  543. pfn);
  544. }
  545. return ret;
  546. }
  547. /*
  548. * Dirty cache page page
  549. * Issues: when the error hit a hole page the error is not properly
  550. * propagated.
  551. */
  552. static int me_pagecache_dirty(struct page *p, unsigned long pfn)
  553. {
  554. struct address_space *mapping = page_mapping(p);
  555. SetPageError(p);
  556. /* TBD: print more information about the file. */
  557. if (mapping) {
  558. /*
  559. * IO error will be reported by write(), fsync(), etc.
  560. * who check the mapping.
  561. * This way the application knows that something went
  562. * wrong with its dirty file data.
  563. *
  564. * There's one open issue:
  565. *
  566. * The EIO will be only reported on the next IO
  567. * operation and then cleared through the IO map.
  568. * Normally Linux has two mechanisms to pass IO error
  569. * first through the AS_EIO flag in the address space
  570. * and then through the PageError flag in the page.
  571. * Since we drop pages on memory failure handling the
  572. * only mechanism open to use is through AS_AIO.
  573. *
  574. * This has the disadvantage that it gets cleared on
  575. * the first operation that returns an error, while
  576. * the PageError bit is more sticky and only cleared
  577. * when the page is reread or dropped. If an
  578. * application assumes it will always get error on
  579. * fsync, but does other operations on the fd before
  580. * and the page is dropped between then the error
  581. * will not be properly reported.
  582. *
  583. * This can already happen even without hwpoisoned
  584. * pages: first on metadata IO errors (which only
  585. * report through AS_EIO) or when the page is dropped
  586. * at the wrong time.
  587. *
  588. * So right now we assume that the application DTRT on
  589. * the first EIO, but we're not worse than other parts
  590. * of the kernel.
  591. */
  592. mapping_set_error(mapping, EIO);
  593. }
  594. return me_pagecache_clean(p, pfn);
  595. }
  596. /*
  597. * Clean and dirty swap cache.
  598. *
  599. * Dirty swap cache page is tricky to handle. The page could live both in page
  600. * cache and swap cache(ie. page is freshly swapped in). So it could be
  601. * referenced concurrently by 2 types of PTEs:
  602. * normal PTEs and swap PTEs. We try to handle them consistently by calling
  603. * try_to_unmap(TTU_IGNORE_HWPOISON) to convert the normal PTEs to swap PTEs,
  604. * and then
  605. * - clear dirty bit to prevent IO
  606. * - remove from LRU
  607. * - but keep in the swap cache, so that when we return to it on
  608. * a later page fault, we know the application is accessing
  609. * corrupted data and shall be killed (we installed simple
  610. * interception code in do_swap_page to catch it).
  611. *
  612. * Clean swap cache pages can be directly isolated. A later page fault will
  613. * bring in the known good data from disk.
  614. */
  615. static int me_swapcache_dirty(struct page *p, unsigned long pfn)
  616. {
  617. ClearPageDirty(p);
  618. /* Trigger EIO in shmem: */
  619. ClearPageUptodate(p);
  620. if (!delete_from_lru_cache(p))
  621. return DELAYED;
  622. else
  623. return FAILED;
  624. }
  625. static int me_swapcache_clean(struct page *p, unsigned long pfn)
  626. {
  627. delete_from_swap_cache(p);
  628. if (!delete_from_lru_cache(p))
  629. return RECOVERED;
  630. else
  631. return FAILED;
  632. }
  633. /*
  634. * Huge pages. Needs work.
  635. * Issues:
  636. * - Error on hugepage is contained in hugepage unit (not in raw page unit.)
  637. * To narrow down kill region to one page, we need to break up pmd.
  638. */
  639. static int me_huge_page(struct page *p, unsigned long pfn)
  640. {
  641. int res = 0;
  642. struct page *hpage = compound_head(p);
  643. /*
  644. * We can safely recover from error on free or reserved (i.e.
  645. * not in-use) hugepage by dequeuing it from freelist.
  646. * To check whether a hugepage is in-use or not, we can't use
  647. * page->lru because it can be used in other hugepage operations,
  648. * such as __unmap_hugepage_range() and gather_surplus_pages().
  649. * So instead we use page_mapping() and PageAnon().
  650. * We assume that this function is called with page lock held,
  651. * so there is no race between isolation and mapping/unmapping.
  652. */
  653. if (!(page_mapping(hpage) || PageAnon(hpage))) {
  654. res = dequeue_hwpoisoned_huge_page(hpage);
  655. if (!res)
  656. return RECOVERED;
  657. }
  658. return DELAYED;
  659. }
  660. /*
  661. * Various page states we can handle.
  662. *
  663. * A page state is defined by its current page->flags bits.
  664. * The table matches them in order and calls the right handler.
  665. *
  666. * This is quite tricky because we can access page at any time
  667. * in its live cycle, so all accesses have to be extremely careful.
  668. *
  669. * This is not complete. More states could be added.
  670. * For any missing state don't attempt recovery.
  671. */
  672. #define dirty (1UL << PG_dirty)
  673. #define sc (1UL << PG_swapcache)
  674. #define unevict (1UL << PG_unevictable)
  675. #define mlock (1UL << PG_mlocked)
  676. #define writeback (1UL << PG_writeback)
  677. #define lru (1UL << PG_lru)
  678. #define swapbacked (1UL << PG_swapbacked)
  679. #define head (1UL << PG_head)
  680. #define tail (1UL << PG_tail)
  681. #define compound (1UL << PG_compound)
  682. #define slab (1UL << PG_slab)
  683. #define reserved (1UL << PG_reserved)
  684. static struct page_state {
  685. unsigned long mask;
  686. unsigned long res;
  687. char *msg;
  688. int (*action)(struct page *p, unsigned long pfn);
  689. } error_states[] = {
  690. { reserved, reserved, "reserved kernel", me_kernel },
  691. /*
  692. * free pages are specially detected outside this table:
  693. * PG_buddy pages only make a small fraction of all free pages.
  694. */
  695. /*
  696. * Could in theory check if slab page is free or if we can drop
  697. * currently unused objects without touching them. But just
  698. * treat it as standard kernel for now.
  699. */
  700. { slab, slab, "kernel slab", me_kernel },
  701. #ifdef CONFIG_PAGEFLAGS_EXTENDED
  702. { head, head, "huge", me_huge_page },
  703. { tail, tail, "huge", me_huge_page },
  704. #else
  705. { compound, compound, "huge", me_huge_page },
  706. #endif
  707. { sc|dirty, sc|dirty, "dirty swapcache", me_swapcache_dirty },
  708. { sc|dirty, sc, "clean swapcache", me_swapcache_clean },
  709. { mlock|dirty, mlock|dirty, "dirty mlocked LRU", me_pagecache_dirty },
  710. { mlock|dirty, mlock, "clean mlocked LRU", me_pagecache_clean },
  711. { unevict|dirty, unevict|dirty, "dirty unevictable LRU", me_pagecache_dirty },
  712. { unevict|dirty, unevict, "clean unevictable LRU", me_pagecache_clean },
  713. { lru|dirty, lru|dirty, "dirty LRU", me_pagecache_dirty },
  714. { lru|dirty, lru, "clean LRU", me_pagecache_clean },
  715. /*
  716. * Catchall entry: must be at end.
  717. */
  718. { 0, 0, "unknown page state", me_unknown },
  719. };
  720. #undef dirty
  721. #undef sc
  722. #undef unevict
  723. #undef mlock
  724. #undef writeback
  725. #undef lru
  726. #undef swapbacked
  727. #undef head
  728. #undef tail
  729. #undef compound
  730. #undef slab
  731. #undef reserved
  732. /*
  733. * "Dirty/Clean" indication is not 100% accurate due to the possibility of
  734. * setting PG_dirty outside page lock. See also comment above set_page_dirty().
  735. */
  736. static void action_result(unsigned long pfn, char *msg, int result)
  737. {
  738. pr_err("MCE %#lx: %s page recovery: %s\n",
  739. pfn, msg, action_name[result]);
  740. }
  741. static int page_action(struct page_state *ps, struct page *p,
  742. unsigned long pfn)
  743. {
  744. int result;
  745. int count;
  746. result = ps->action(p, pfn);
  747. action_result(pfn, ps->msg, result);
  748. count = page_count(p) - 1;
  749. if (ps->action == me_swapcache_dirty && result == DELAYED)
  750. count--;
  751. if (count != 0) {
  752. printk(KERN_ERR
  753. "MCE %#lx: %s page still referenced by %d users\n",
  754. pfn, ps->msg, count);
  755. result = FAILED;
  756. }
  757. /* Could do more checks here if page looks ok */
  758. /*
  759. * Could adjust zone counters here to correct for the missing page.
  760. */
  761. return (result == RECOVERED || result == DELAYED) ? 0 : -EBUSY;
  762. }
  763. /*
  764. * Do all that is necessary to remove user space mappings. Unmap
  765. * the pages and send SIGBUS to the processes if the data was dirty.
  766. */
  767. static int hwpoison_user_mappings(struct page *p, unsigned long pfn,
  768. int trapno, int flags)
  769. {
  770. enum ttu_flags ttu = TTU_UNMAP | TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS;
  771. struct address_space *mapping;
  772. LIST_HEAD(tokill);
  773. int ret;
  774. int kill = 1, forcekill;
  775. struct page *hpage = compound_head(p);
  776. struct page *ppage;
  777. if (PageReserved(p) || PageSlab(p))
  778. return SWAP_SUCCESS;
  779. /*
  780. * This check implies we don't kill processes if their pages
  781. * are in the swap cache early. Those are always late kills.
  782. */
  783. if (!page_mapped(hpage))
  784. return SWAP_SUCCESS;
  785. if (PageKsm(p))
  786. return SWAP_FAIL;
  787. if (PageSwapCache(p)) {
  788. printk(KERN_ERR
  789. "MCE %#lx: keeping poisoned page in swap cache\n", pfn);
  790. ttu |= TTU_IGNORE_HWPOISON;
  791. }
  792. /*
  793. * Propagate the dirty bit from PTEs to struct page first, because we
  794. * need this to decide if we should kill or just drop the page.
  795. * XXX: the dirty test could be racy: set_page_dirty() may not always
  796. * be called inside page lock (it's recommended but not enforced).
  797. */
  798. mapping = page_mapping(hpage);
  799. if (!(flags & MF_MUST_KILL) && !PageDirty(hpage) && mapping &&
  800. mapping_cap_writeback_dirty(mapping)) {
  801. if (page_mkclean(hpage)) {
  802. SetPageDirty(hpage);
  803. } else {
  804. kill = 0;
  805. ttu |= TTU_IGNORE_HWPOISON;
  806. printk(KERN_INFO
  807. "MCE %#lx: corrupted page was clean: dropped without side effects\n",
  808. pfn);
  809. }
  810. }
  811. /*
  812. * ppage: poisoned page
  813. * if p is regular page(4k page)
  814. * ppage == real poisoned page;
  815. * else p is hugetlb or THP, ppage == head page.
  816. */
  817. ppage = hpage;
  818. if (PageTransHuge(hpage)) {
  819. /*
  820. * Verify that this isn't a hugetlbfs head page, the check for
  821. * PageAnon is just for avoid tripping a split_huge_page
  822. * internal debug check, as split_huge_page refuses to deal with
  823. * anything that isn't an anon page. PageAnon can't go away fro
  824. * under us because we hold a refcount on the hpage, without a
  825. * refcount on the hpage. split_huge_page can't be safely called
  826. * in the first place, having a refcount on the tail isn't
  827. * enough * to be safe.
  828. */
  829. if (!PageHuge(hpage) && PageAnon(hpage)) {
  830. if (unlikely(split_huge_page(hpage))) {
  831. /*
  832. * FIXME: if splitting THP is failed, it is
  833. * better to stop the following operation rather
  834. * than causing panic by unmapping. System might
  835. * survive if the page is freed later.
  836. */
  837. printk(KERN_INFO
  838. "MCE %#lx: failed to split THP\n", pfn);
  839. BUG_ON(!PageHWPoison(p));
  840. return SWAP_FAIL;
  841. }
  842. /* THP is split, so ppage should be the real poisoned page. */
  843. ppage = p;
  844. }
  845. }
  846. /*
  847. * First collect all the processes that have the page
  848. * mapped in dirty form. This has to be done before try_to_unmap,
  849. * because ttu takes the rmap data structures down.
  850. *
  851. * Error handling: We ignore errors here because
  852. * there's nothing that can be done.
  853. */
  854. if (kill)
  855. collect_procs(ppage, &tokill);
  856. if (hpage != ppage)
  857. lock_page(ppage);
  858. ret = try_to_unmap(ppage, ttu);
  859. if (ret != SWAP_SUCCESS)
  860. printk(KERN_ERR "MCE %#lx: failed to unmap page (mapcount=%d)\n",
  861. pfn, page_mapcount(ppage));
  862. if (hpage != ppage)
  863. unlock_page(ppage);
  864. /*
  865. * Now that the dirty bit has been propagated to the
  866. * struct page and all unmaps done we can decide if
  867. * killing is needed or not. Only kill when the page
  868. * was dirty or the process is not restartable,
  869. * otherwise the tokill list is merely
  870. * freed. When there was a problem unmapping earlier
  871. * use a more force-full uncatchable kill to prevent
  872. * any accesses to the poisoned memory.
  873. */
  874. forcekill = PageDirty(ppage) || (flags & MF_MUST_KILL);
  875. kill_procs(&tokill, forcekill, trapno,
  876. ret != SWAP_SUCCESS, p, pfn, flags);
  877. return ret;
  878. }
  879. static void set_page_hwpoison_huge_page(struct page *hpage)
  880. {
  881. int i;
  882. int nr_pages = 1 << compound_order(hpage);
  883. for (i = 0; i < nr_pages; i++)
  884. SetPageHWPoison(hpage + i);
  885. }
  886. static void clear_page_hwpoison_huge_page(struct page *hpage)
  887. {
  888. int i;
  889. int nr_pages = 1 << compound_order(hpage);
  890. for (i = 0; i < nr_pages; i++)
  891. ClearPageHWPoison(hpage + i);
  892. }
  893. /**
  894. * memory_failure - Handle memory failure of a page.
  895. * @pfn: Page Number of the corrupted page
  896. * @trapno: Trap number reported in the signal to user space.
  897. * @flags: fine tune action taken
  898. *
  899. * This function is called by the low level machine check code
  900. * of an architecture when it detects hardware memory corruption
  901. * of a page. It tries its best to recover, which includes
  902. * dropping pages, killing processes etc.
  903. *
  904. * The function is primarily of use for corruptions that
  905. * happen outside the current execution context (e.g. when
  906. * detected by a background scrubber)
  907. *
  908. * Must run in process context (e.g. a work queue) with interrupts
  909. * enabled and no spinlocks hold.
  910. */
  911. int memory_failure(unsigned long pfn, int trapno, int flags)
  912. {
  913. struct page_state *ps;
  914. struct page *p;
  915. struct page *hpage;
  916. int res;
  917. unsigned int nr_pages;
  918. unsigned long page_flags;
  919. if (!sysctl_memory_failure_recovery)
  920. panic("Memory failure from trap %d on page %lx", trapno, pfn);
  921. if (!pfn_valid(pfn)) {
  922. printk(KERN_ERR
  923. "MCE %#lx: memory outside kernel control\n",
  924. pfn);
  925. return -ENXIO;
  926. }
  927. p = pfn_to_page(pfn);
  928. hpage = compound_head(p);
  929. if (TestSetPageHWPoison(p)) {
  930. printk(KERN_ERR "MCE %#lx: already hardware poisoned\n", pfn);
  931. return 0;
  932. }
  933. /*
  934. * Currently errors on hugetlbfs pages are measured in hugepage units,
  935. * so nr_pages should be 1 << compound_order. OTOH when errors are on
  936. * transparent hugepages, they are supposed to be split and error
  937. * measurement is done in normal page units. So nr_pages should be one
  938. * in this case.
  939. */
  940. if (PageHuge(p))
  941. nr_pages = 1 << compound_order(hpage);
  942. else /* normal page or thp */
  943. nr_pages = 1;
  944. atomic_long_add(nr_pages, &num_poisoned_pages);
  945. /*
  946. * We need/can do nothing about count=0 pages.
  947. * 1) it's a free page, and therefore in safe hand:
  948. * prep_new_page() will be the gate keeper.
  949. * 2) it's a free hugepage, which is also safe:
  950. * an affected hugepage will be dequeued from hugepage freelist,
  951. * so there's no concern about reusing it ever after.
  952. * 3) it's part of a non-compound high order page.
  953. * Implies some kernel user: cannot stop them from
  954. * R/W the page; let's pray that the page has been
  955. * used and will be freed some time later.
  956. * In fact it's dangerous to directly bump up page count from 0,
  957. * that may make page_freeze_refs()/page_unfreeze_refs() mismatch.
  958. */
  959. if (!(flags & MF_COUNT_INCREASED) &&
  960. !get_page_unless_zero(hpage)) {
  961. if (is_free_buddy_page(p)) {
  962. action_result(pfn, "free buddy", DELAYED);
  963. return 0;
  964. } else if (PageHuge(hpage)) {
  965. /*
  966. * Check "just unpoisoned", "filter hit", and
  967. * "race with other subpage."
  968. */
  969. lock_page(hpage);
  970. if (!PageHWPoison(hpage)
  971. || (hwpoison_filter(p) && TestClearPageHWPoison(p))
  972. || (p != hpage && TestSetPageHWPoison(hpage))) {
  973. atomic_long_sub(nr_pages, &num_poisoned_pages);
  974. return 0;
  975. }
  976. set_page_hwpoison_huge_page(hpage);
  977. res = dequeue_hwpoisoned_huge_page(hpage);
  978. action_result(pfn, "free huge",
  979. res ? IGNORED : DELAYED);
  980. unlock_page(hpage);
  981. return res;
  982. } else {
  983. action_result(pfn, "high order kernel", IGNORED);
  984. return -EBUSY;
  985. }
  986. }
  987. /*
  988. * We ignore non-LRU pages for good reasons.
  989. * - PG_locked is only well defined for LRU pages and a few others
  990. * - to avoid races with __set_page_locked()
  991. * - to avoid races with __SetPageSlab*() (and more non-atomic ops)
  992. * The check (unnecessarily) ignores LRU pages being isolated and
  993. * walked by the page reclaim code, however that's not a big loss.
  994. */
  995. if (!PageHuge(p) && !PageTransTail(p)) {
  996. if (!PageLRU(p))
  997. shake_page(p, 0);
  998. if (!PageLRU(p)) {
  999. /*
  1000. * shake_page could have turned it free.
  1001. */
  1002. if (is_free_buddy_page(p)) {
  1003. if (flags & MF_COUNT_INCREASED)
  1004. action_result(pfn, "free buddy", DELAYED);
  1005. else
  1006. action_result(pfn, "free buddy, 2nd try", DELAYED);
  1007. return 0;
  1008. }
  1009. action_result(pfn, "non LRU", IGNORED);
  1010. put_page(p);
  1011. return -EBUSY;
  1012. }
  1013. }
  1014. /*
  1015. * Lock the page and wait for writeback to finish.
  1016. * It's very difficult to mess with pages currently under IO
  1017. * and in many cases impossible, so we just avoid it here.
  1018. */
  1019. lock_page(hpage);
  1020. /*
  1021. * We use page flags to determine what action should be taken, but
  1022. * the flags can be modified by the error containment action. One
  1023. * example is an mlocked page, where PG_mlocked is cleared by
  1024. * page_remove_rmap() in try_to_unmap_one(). So to determine page status
  1025. * correctly, we save a copy of the page flags at this time.
  1026. */
  1027. page_flags = p->flags;
  1028. /*
  1029. * unpoison always clear PG_hwpoison inside page lock
  1030. */
  1031. if (!PageHWPoison(p)) {
  1032. printk(KERN_ERR "MCE %#lx: just unpoisoned\n", pfn);
  1033. res = 0;
  1034. goto out;
  1035. }
  1036. if (hwpoison_filter(p)) {
  1037. if (TestClearPageHWPoison(p))
  1038. atomic_long_sub(nr_pages, &num_poisoned_pages);
  1039. unlock_page(hpage);
  1040. put_page(hpage);
  1041. return 0;
  1042. }
  1043. /*
  1044. * For error on the tail page, we should set PG_hwpoison
  1045. * on the head page to show that the hugepage is hwpoisoned
  1046. */
  1047. if (PageHuge(p) && PageTail(p) && TestSetPageHWPoison(hpage)) {
  1048. action_result(pfn, "hugepage already hardware poisoned",
  1049. IGNORED);
  1050. unlock_page(hpage);
  1051. put_page(hpage);
  1052. return 0;
  1053. }
  1054. /*
  1055. * Set PG_hwpoison on all pages in an error hugepage,
  1056. * because containment is done in hugepage unit for now.
  1057. * Since we have done TestSetPageHWPoison() for the head page with
  1058. * page lock held, we can safely set PG_hwpoison bits on tail pages.
  1059. */
  1060. if (PageHuge(p))
  1061. set_page_hwpoison_huge_page(hpage);
  1062. wait_on_page_writeback(p);
  1063. /*
  1064. * Now take care of user space mappings.
  1065. * Abort on fail: __delete_from_page_cache() assumes unmapped page.
  1066. */
  1067. if (hwpoison_user_mappings(p, pfn, trapno, flags) != SWAP_SUCCESS) {
  1068. printk(KERN_ERR "MCE %#lx: cannot unmap page, give up\n", pfn);
  1069. res = -EBUSY;
  1070. goto out;
  1071. }
  1072. /*
  1073. * Torn down by someone else?
  1074. */
  1075. if (PageLRU(p) && !PageSwapCache(p) && p->mapping == NULL) {
  1076. action_result(pfn, "already truncated LRU", IGNORED);
  1077. res = -EBUSY;
  1078. goto out;
  1079. }
  1080. res = -EBUSY;
  1081. /*
  1082. * The first check uses the current page flags which may not have any
  1083. * relevant information. The second check with the saved page flagss is
  1084. * carried out only if the first check can't determine the page status.
  1085. */
  1086. for (ps = error_states;; ps++)
  1087. if ((p->flags & ps->mask) == ps->res)
  1088. break;
  1089. page_flags |= (p->flags & (1UL << PG_dirty));
  1090. if (!ps->mask)
  1091. for (ps = error_states;; ps++)
  1092. if ((page_flags & ps->mask) == ps->res)
  1093. break;
  1094. res = page_action(ps, p, pfn);
  1095. out:
  1096. unlock_page(hpage);
  1097. return res;
  1098. }
  1099. EXPORT_SYMBOL_GPL(memory_failure);
  1100. #define MEMORY_FAILURE_FIFO_ORDER 4
  1101. #define MEMORY_FAILURE_FIFO_SIZE (1 << MEMORY_FAILURE_FIFO_ORDER)
  1102. struct memory_failure_entry {
  1103. unsigned long pfn;
  1104. int trapno;
  1105. int flags;
  1106. };
  1107. struct memory_failure_cpu {
  1108. DECLARE_KFIFO(fifo, struct memory_failure_entry,
  1109. MEMORY_FAILURE_FIFO_SIZE);
  1110. spinlock_t lock;
  1111. struct work_struct work;
  1112. };
  1113. static DEFINE_PER_CPU(struct memory_failure_cpu, memory_failure_cpu);
  1114. /**
  1115. * memory_failure_queue - Schedule handling memory failure of a page.
  1116. * @pfn: Page Number of the corrupted page
  1117. * @trapno: Trap number reported in the signal to user space.
  1118. * @flags: Flags for memory failure handling
  1119. *
  1120. * This function is called by the low level hardware error handler
  1121. * when it detects hardware memory corruption of a page. It schedules
  1122. * the recovering of error page, including dropping pages, killing
  1123. * processes etc.
  1124. *
  1125. * The function is primarily of use for corruptions that
  1126. * happen outside the current execution context (e.g. when
  1127. * detected by a background scrubber)
  1128. *
  1129. * Can run in IRQ context.
  1130. */
  1131. void memory_failure_queue(unsigned long pfn, int trapno, int flags)
  1132. {
  1133. struct memory_failure_cpu *mf_cpu;
  1134. unsigned long proc_flags;
  1135. struct memory_failure_entry entry = {
  1136. .pfn = pfn,
  1137. .trapno = trapno,
  1138. .flags = flags,
  1139. };
  1140. mf_cpu = &get_cpu_var(memory_failure_cpu);
  1141. spin_lock_irqsave(&mf_cpu->lock, proc_flags);
  1142. if (kfifo_put(&mf_cpu->fifo, entry))
  1143. schedule_work_on(smp_processor_id(), &mf_cpu->work);
  1144. else
  1145. pr_err("Memory failure: buffer overflow when queuing memory failure at %#lx\n",
  1146. pfn);
  1147. spin_unlock_irqrestore(&mf_cpu->lock, proc_flags);
  1148. put_cpu_var(memory_failure_cpu);
  1149. }
  1150. EXPORT_SYMBOL_GPL(memory_failure_queue);
  1151. static void memory_failure_work_func(struct work_struct *work)
  1152. {
  1153. struct memory_failure_cpu *mf_cpu;
  1154. struct memory_failure_entry entry = { 0, };
  1155. unsigned long proc_flags;
  1156. int gotten;
  1157. mf_cpu = &__get_cpu_var(memory_failure_cpu);
  1158. for (;;) {
  1159. spin_lock_irqsave(&mf_cpu->lock, proc_flags);
  1160. gotten = kfifo_get(&mf_cpu->fifo, &entry);
  1161. spin_unlock_irqrestore(&mf_cpu->lock, proc_flags);
  1162. if (!gotten)
  1163. break;
  1164. if (entry.flags & MF_SOFT_OFFLINE)
  1165. soft_offline_page(pfn_to_page(entry.pfn), entry.flags);
  1166. else
  1167. memory_failure(entry.pfn, entry.trapno, entry.flags);
  1168. }
  1169. }
  1170. static int __init memory_failure_init(void)
  1171. {
  1172. struct memory_failure_cpu *mf_cpu;
  1173. int cpu;
  1174. for_each_possible_cpu(cpu) {
  1175. mf_cpu = &per_cpu(memory_failure_cpu, cpu);
  1176. spin_lock_init(&mf_cpu->lock);
  1177. INIT_KFIFO(mf_cpu->fifo);
  1178. INIT_WORK(&mf_cpu->work, memory_failure_work_func);
  1179. }
  1180. return 0;
  1181. }
  1182. core_initcall(memory_failure_init);
  1183. /**
  1184. * unpoison_memory - Unpoison a previously poisoned page
  1185. * @pfn: Page number of the to be unpoisoned page
  1186. *
  1187. * Software-unpoison a page that has been poisoned by
  1188. * memory_failure() earlier.
  1189. *
  1190. * This is only done on the software-level, so it only works
  1191. * for linux injected failures, not real hardware failures
  1192. *
  1193. * Returns 0 for success, otherwise -errno.
  1194. */
  1195. int unpoison_memory(unsigned long pfn)
  1196. {
  1197. struct page *page;
  1198. struct page *p;
  1199. int freeit = 0;
  1200. unsigned int nr_pages;
  1201. if (!pfn_valid(pfn))
  1202. return -ENXIO;
  1203. p = pfn_to_page(pfn);
  1204. page = compound_head(p);
  1205. if (!PageHWPoison(p)) {
  1206. pr_info("MCE: Page was already unpoisoned %#lx\n", pfn);
  1207. return 0;
  1208. }
  1209. /*
  1210. * unpoison_memory() can encounter thp only when the thp is being
  1211. * worked by memory_failure() and the page lock is not held yet.
  1212. * In such case, we yield to memory_failure() and make unpoison fail.
  1213. */
  1214. if (!PageHuge(page) && PageTransHuge(page)) {
  1215. pr_info("MCE: Memory failure is now running on %#lx\n", pfn);
  1216. return 0;
  1217. }
  1218. nr_pages = 1 << compound_order(page);
  1219. if (!get_page_unless_zero(page)) {
  1220. /*
  1221. * Since HWPoisoned hugepage should have non-zero refcount,
  1222. * race between memory failure and unpoison seems to happen.
  1223. * In such case unpoison fails and memory failure runs
  1224. * to the end.
  1225. */
  1226. if (PageHuge(page)) {
  1227. pr_info("MCE: Memory failure is now running on free hugepage %#lx\n", pfn);
  1228. return 0;
  1229. }
  1230. if (TestClearPageHWPoison(p))
  1231. atomic_long_dec(&num_poisoned_pages);
  1232. pr_info("MCE: Software-unpoisoned free page %#lx\n", pfn);
  1233. return 0;
  1234. }
  1235. lock_page(page);
  1236. /*
  1237. * This test is racy because PG_hwpoison is set outside of page lock.
  1238. * That's acceptable because that won't trigger kernel panic. Instead,
  1239. * the PG_hwpoison page will be caught and isolated on the entrance to
  1240. * the free buddy page pool.
  1241. */
  1242. if (TestClearPageHWPoison(page)) {
  1243. pr_info("MCE: Software-unpoisoned page %#lx\n", pfn);
  1244. atomic_long_sub(nr_pages, &num_poisoned_pages);
  1245. freeit = 1;
  1246. if (PageHuge(page))
  1247. clear_page_hwpoison_huge_page(page);
  1248. }
  1249. unlock_page(page);
  1250. put_page(page);
  1251. if (freeit && !(pfn == my_zero_pfn(0) && page_count(p) == 1))
  1252. put_page(page);
  1253. return 0;
  1254. }
  1255. EXPORT_SYMBOL(unpoison_memory);
  1256. static struct page *new_page(struct page *p, unsigned long private, int **x)
  1257. {
  1258. int nid = page_to_nid(p);
  1259. if (PageHuge(p))
  1260. return alloc_huge_page_node(page_hstate(compound_head(p)),
  1261. nid);
  1262. else
  1263. return alloc_pages_exact_node(nid, GFP_HIGHUSER_MOVABLE, 0);
  1264. }
  1265. /*
  1266. * Safely get reference count of an arbitrary page.
  1267. * Returns 0 for a free page, -EIO for a zero refcount page
  1268. * that is not free, and 1 for any other page type.
  1269. * For 1 the page is returned with increased page count, otherwise not.
  1270. */
  1271. static int __get_any_page(struct page *p, unsigned long pfn, int flags)
  1272. {
  1273. int ret;
  1274. if (flags & MF_COUNT_INCREASED)
  1275. return 1;
  1276. /*
  1277. * When the target page is a free hugepage, just remove it
  1278. * from free hugepage list.
  1279. */
  1280. if (!get_page_unless_zero(compound_head(p))) {
  1281. if (PageHuge(p)) {
  1282. pr_info("%s: %#lx free huge page\n", __func__, pfn);
  1283. ret = 0;
  1284. } else if (is_free_buddy_page(p)) {
  1285. pr_info("%s: %#lx free buddy page\n", __func__, pfn);
  1286. ret = 0;
  1287. } else {
  1288. pr_info("%s: %#lx: unknown zero refcount page type %lx\n",
  1289. __func__, pfn, p->flags);
  1290. ret = -EIO;
  1291. }
  1292. } else {
  1293. /* Not a free page */
  1294. ret = 1;
  1295. }
  1296. return ret;
  1297. }
  1298. static int get_any_page(struct page *page, unsigned long pfn, int flags)
  1299. {
  1300. int ret = __get_any_page(page, pfn, flags);
  1301. if (ret == 1 && !PageHuge(page) && !PageLRU(page)) {
  1302. /*
  1303. * Try to free it.
  1304. */
  1305. put_page(page);
  1306. shake_page(page, 1);
  1307. /*
  1308. * Did it turn free?
  1309. */
  1310. ret = __get_any_page(page, pfn, 0);
  1311. if (!PageLRU(page)) {
  1312. pr_info("soft_offline: %#lx: unknown non LRU page type %lx\n",
  1313. pfn, page->flags);
  1314. return -EIO;
  1315. }
  1316. }
  1317. return ret;
  1318. }
  1319. static int soft_offline_huge_page(struct page *page, int flags)
  1320. {
  1321. int ret;
  1322. unsigned long pfn = page_to_pfn(page);
  1323. struct page *hpage = compound_head(page);
  1324. LIST_HEAD(pagelist);
  1325. /*
  1326. * This double-check of PageHWPoison is to avoid the race with
  1327. * memory_failure(). See also comment in __soft_offline_page().
  1328. */
  1329. lock_page(hpage);
  1330. if (PageHWPoison(hpage)) {
  1331. unlock_page(hpage);
  1332. put_page(hpage);
  1333. pr_info("soft offline: %#lx hugepage already poisoned\n", pfn);
  1334. return -EBUSY;
  1335. }
  1336. unlock_page(hpage);
  1337. /* Keep page count to indicate a given hugepage is isolated. */
  1338. list_move(&hpage->lru, &pagelist);
  1339. ret = migrate_pages(&pagelist, new_page, MPOL_MF_MOVE_ALL,
  1340. MIGRATE_SYNC, MR_MEMORY_FAILURE);
  1341. if (ret) {
  1342. pr_info("soft offline: %#lx: migration failed %d, type %lx\n",
  1343. pfn, ret, page->flags);
  1344. /*
  1345. * We know that soft_offline_huge_page() tries to migrate
  1346. * only one hugepage pointed to by hpage, so we need not
  1347. * run through the pagelist here.
  1348. */
  1349. putback_active_hugepage(hpage);
  1350. if (ret > 0)
  1351. ret = -EIO;
  1352. } else {
  1353. set_page_hwpoison_huge_page(hpage);
  1354. dequeue_hwpoisoned_huge_page(hpage);
  1355. atomic_long_add(1 << compound_order(hpage),
  1356. &num_poisoned_pages);
  1357. }
  1358. return ret;
  1359. }
  1360. static int __soft_offline_page(struct page *page, int flags)
  1361. {
  1362. int ret;
  1363. unsigned long pfn = page_to_pfn(page);
  1364. /*
  1365. * Check PageHWPoison again inside page lock because PageHWPoison
  1366. * is set by memory_failure() outside page lock. Note that
  1367. * memory_failure() also double-checks PageHWPoison inside page lock,
  1368. * so there's no race between soft_offline_page() and memory_failure().
  1369. */
  1370. lock_page(page);
  1371. wait_on_page_writeback(page);
  1372. if (PageHWPoison(page)) {
  1373. unlock_page(page);
  1374. put_page(page);
  1375. pr_info("soft offline: %#lx page already poisoned\n", pfn);
  1376. return -EBUSY;
  1377. }
  1378. /*
  1379. * Try to invalidate first. This should work for
  1380. * non dirty unmapped page cache pages.
  1381. */
  1382. ret = invalidate_inode_page(page);
  1383. unlock_page(page);
  1384. /*
  1385. * RED-PEN would be better to keep it isolated here, but we
  1386. * would need to fix isolation locking first.
  1387. */
  1388. if (ret == 1) {
  1389. put_page(page);
  1390. pr_info("soft_offline: %#lx: invalidated\n", pfn);
  1391. SetPageHWPoison(page);
  1392. atomic_long_inc(&num_poisoned_pages);
  1393. return 0;
  1394. }
  1395. /*
  1396. * Simple invalidation didn't work.
  1397. * Try to migrate to a new page instead. migrate.c
  1398. * handles a large number of cases for us.
  1399. */
  1400. ret = isolate_lru_page(page);
  1401. /*
  1402. * Drop page reference which is came from get_any_page()
  1403. * successful isolate_lru_page() already took another one.
  1404. */
  1405. put_page(page);
  1406. if (!ret) {
  1407. LIST_HEAD(pagelist);
  1408. inc_zone_page_state(page, NR_ISOLATED_ANON +
  1409. page_is_file_cache(page));
  1410. list_add(&page->lru, &pagelist);
  1411. ret = migrate_pages(&pagelist, new_page, MPOL_MF_MOVE_ALL,
  1412. MIGRATE_SYNC, MR_MEMORY_FAILURE);
  1413. if (ret) {
  1414. putback_lru_pages(&pagelist);
  1415. pr_info("soft offline: %#lx: migration failed %d, type %lx\n",
  1416. pfn, ret, page->flags);
  1417. if (ret > 0)
  1418. ret = -EIO;
  1419. } else {
  1420. /*
  1421. * After page migration succeeds, the source page can
  1422. * be trapped in pagevec and actual freeing is delayed.
  1423. * Freeing code works differently based on PG_hwpoison,
  1424. * so there's a race. We need to make sure that the
  1425. * source page should be freed back to buddy before
  1426. * setting PG_hwpoison.
  1427. */
  1428. if (!is_free_buddy_page(page))
  1429. lru_add_drain_all();
  1430. if (!is_free_buddy_page(page))
  1431. drain_all_pages();
  1432. SetPageHWPoison(page);
  1433. if (!is_free_buddy_page(page))
  1434. pr_info("soft offline: %#lx: page leaked\n",
  1435. pfn);
  1436. atomic_long_inc(&num_poisoned_pages);
  1437. }
  1438. } else {
  1439. pr_info("soft offline: %#lx: isolation failed: %d, page count %d, type %lx\n",
  1440. pfn, ret, page_count(page), page->flags);
  1441. }
  1442. return ret;
  1443. }
  1444. /**
  1445. * soft_offline_page - Soft offline a page.
  1446. * @page: page to offline
  1447. * @flags: flags. Same as memory_failure().
  1448. *
  1449. * Returns 0 on success, otherwise negated errno.
  1450. *
  1451. * Soft offline a page, by migration or invalidation,
  1452. * without killing anything. This is for the case when
  1453. * a page is not corrupted yet (so it's still valid to access),
  1454. * but has had a number of corrected errors and is better taken
  1455. * out.
  1456. *
  1457. * The actual policy on when to do that is maintained by
  1458. * user space.
  1459. *
  1460. * This should never impact any application or cause data loss,
  1461. * however it might take some time.
  1462. *
  1463. * This is not a 100% solution for all memory, but tries to be
  1464. * ``good enough'' for the majority of memory.
  1465. */
  1466. int soft_offline_page(struct page *page, int flags)
  1467. {
  1468. int ret;
  1469. unsigned long pfn = page_to_pfn(page);
  1470. struct page *hpage = compound_trans_head(page);
  1471. if (PageHWPoison(page)) {
  1472. pr_info("soft offline: %#lx page already poisoned\n", pfn);
  1473. return -EBUSY;
  1474. }
  1475. if (!PageHuge(page) && PageTransHuge(hpage)) {
  1476. if (PageAnon(hpage) && unlikely(split_huge_page(hpage))) {
  1477. pr_info("soft offline: %#lx: failed to split THP\n",
  1478. pfn);
  1479. return -EBUSY;
  1480. }
  1481. }
  1482. /*
  1483. * The lock_memory_hotplug prevents a race with memory hotplug.
  1484. * This is a big hammer, a better would be nicer.
  1485. */
  1486. lock_memory_hotplug();
  1487. /*
  1488. * Isolate the page, so that it doesn't get reallocated if it
  1489. * was free. This flag should be kept set until the source page
  1490. * is freed and PG_hwpoison on it is set.
  1491. */
  1492. if (get_pageblock_migratetype(page) != MIGRATE_ISOLATE)
  1493. set_migratetype_isolate(page, true);
  1494. ret = get_any_page(page, pfn, flags);
  1495. unlock_memory_hotplug();
  1496. if (ret > 0) { /* for in-use pages */
  1497. if (PageHuge(page))
  1498. ret = soft_offline_huge_page(page, flags);
  1499. else
  1500. ret = __soft_offline_page(page, flags);
  1501. } else if (ret == 0) { /* for free pages */
  1502. if (PageHuge(page)) {
  1503. set_page_hwpoison_huge_page(hpage);
  1504. dequeue_hwpoisoned_huge_page(hpage);
  1505. atomic_long_add(1 << compound_order(hpage),
  1506. &num_poisoned_pages);
  1507. } else {
  1508. SetPageHWPoison(page);
  1509. atomic_long_inc(&num_poisoned_pages);
  1510. }
  1511. }
  1512. unset_migratetype_isolate(page, MIGRATE_MOVABLE);
  1513. return ret;
  1514. }