snapshot.c 50 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976
  1. /*
  2. * linux/kernel/power/snapshot.c
  3. *
  4. * This file provides system snapshot/restore functionality for swsusp.
  5. *
  6. * Copyright (C) 1998-2005 Pavel Machek <pavel@suse.cz>
  7. * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
  8. *
  9. * This file is released under the GPLv2.
  10. *
  11. */
  12. #include <linux/version.h>
  13. #include <linux/module.h>
  14. #include <linux/mm.h>
  15. #include <linux/suspend.h>
  16. #include <linux/delay.h>
  17. #include <linux/bitops.h>
  18. #include <linux/spinlock.h>
  19. #include <linux/kernel.h>
  20. #include <linux/pm.h>
  21. #include <linux/device.h>
  22. #include <linux/init.h>
  23. #include <linux/bootmem.h>
  24. #include <linux/syscalls.h>
  25. #include <linux/console.h>
  26. #include <linux/highmem.h>
  27. #include <asm/uaccess.h>
  28. #include <asm/mmu_context.h>
  29. #include <asm/pgtable.h>
  30. #include <asm/tlbflush.h>
  31. #include <asm/io.h>
  32. #include "power.h"
  33. static int swsusp_page_is_free(struct page *);
  34. static void swsusp_set_page_forbidden(struct page *);
  35. static void swsusp_unset_page_forbidden(struct page *);
  36. /* List of PBEs needed for restoring the pages that were allocated before
  37. * the suspend and included in the suspend image, but have also been
  38. * allocated by the "resume" kernel, so their contents cannot be written
  39. * directly to their "original" page frames.
  40. */
  41. struct pbe *restore_pblist;
  42. /* Pointer to an auxiliary buffer (1 page) */
  43. static void *buffer;
  44. /**
  45. * @safe_needed - on resume, for storing the PBE list and the image,
  46. * we can only use memory pages that do not conflict with the pages
  47. * used before suspend. The unsafe pages have PageNosaveFree set
  48. * and we count them using unsafe_pages.
  49. *
  50. * Each allocated image page is marked as PageNosave and PageNosaveFree
  51. * so that swsusp_free() can release it.
  52. */
  53. #define PG_ANY 0
  54. #define PG_SAFE 1
  55. #define PG_UNSAFE_CLEAR 1
  56. #define PG_UNSAFE_KEEP 0
  57. static unsigned int allocated_unsafe_pages;
  58. static void *get_image_page(gfp_t gfp_mask, int safe_needed)
  59. {
  60. void *res;
  61. res = (void *)get_zeroed_page(gfp_mask);
  62. if (safe_needed)
  63. while (res && swsusp_page_is_free(virt_to_page(res))) {
  64. /* The page is unsafe, mark it for swsusp_free() */
  65. swsusp_set_page_forbidden(virt_to_page(res));
  66. allocated_unsafe_pages++;
  67. res = (void *)get_zeroed_page(gfp_mask);
  68. }
  69. if (res) {
  70. swsusp_set_page_forbidden(virt_to_page(res));
  71. swsusp_set_page_free(virt_to_page(res));
  72. }
  73. return res;
  74. }
  75. unsigned long get_safe_page(gfp_t gfp_mask)
  76. {
  77. return (unsigned long)get_image_page(gfp_mask, PG_SAFE);
  78. }
  79. static struct page *alloc_image_page(gfp_t gfp_mask)
  80. {
  81. struct page *page;
  82. page = alloc_page(gfp_mask);
  83. if (page) {
  84. swsusp_set_page_forbidden(page);
  85. swsusp_set_page_free(page);
  86. }
  87. return page;
  88. }
  89. /**
  90. * free_image_page - free page represented by @addr, allocated with
  91. * get_image_page (page flags set by it must be cleared)
  92. */
  93. static inline void free_image_page(void *addr, int clear_nosave_free)
  94. {
  95. struct page *page;
  96. BUG_ON(!virt_addr_valid(addr));
  97. page = virt_to_page(addr);
  98. swsusp_unset_page_forbidden(page);
  99. if (clear_nosave_free)
  100. swsusp_unset_page_free(page);
  101. __free_page(page);
  102. }
  103. /* struct linked_page is used to build chains of pages */
  104. #define LINKED_PAGE_DATA_SIZE (PAGE_SIZE - sizeof(void *))
  105. struct linked_page {
  106. struct linked_page *next;
  107. char data[LINKED_PAGE_DATA_SIZE];
  108. } __attribute__((packed));
  109. static inline void
  110. free_list_of_pages(struct linked_page *list, int clear_page_nosave)
  111. {
  112. while (list) {
  113. struct linked_page *lp = list->next;
  114. free_image_page(list, clear_page_nosave);
  115. list = lp;
  116. }
  117. }
  118. /**
  119. * struct chain_allocator is used for allocating small objects out of
  120. * a linked list of pages called 'the chain'.
  121. *
  122. * The chain grows each time when there is no room for a new object in
  123. * the current page. The allocated objects cannot be freed individually.
  124. * It is only possible to free them all at once, by freeing the entire
  125. * chain.
  126. *
  127. * NOTE: The chain allocator may be inefficient if the allocated objects
  128. * are not much smaller than PAGE_SIZE.
  129. */
  130. struct chain_allocator {
  131. struct linked_page *chain; /* the chain */
  132. unsigned int used_space; /* total size of objects allocated out
  133. * of the current page
  134. */
  135. gfp_t gfp_mask; /* mask for allocating pages */
  136. int safe_needed; /* if set, only "safe" pages are allocated */
  137. };
  138. static void
  139. chain_init(struct chain_allocator *ca, gfp_t gfp_mask, int safe_needed)
  140. {
  141. ca->chain = NULL;
  142. ca->used_space = LINKED_PAGE_DATA_SIZE;
  143. ca->gfp_mask = gfp_mask;
  144. ca->safe_needed = safe_needed;
  145. }
  146. static void *chain_alloc(struct chain_allocator *ca, unsigned int size)
  147. {
  148. void *ret;
  149. if (LINKED_PAGE_DATA_SIZE - ca->used_space < size) {
  150. struct linked_page *lp;
  151. lp = get_image_page(ca->gfp_mask, ca->safe_needed);
  152. if (!lp)
  153. return NULL;
  154. lp->next = ca->chain;
  155. ca->chain = lp;
  156. ca->used_space = 0;
  157. }
  158. ret = ca->chain->data + ca->used_space;
  159. ca->used_space += size;
  160. return ret;
  161. }
  162. static void chain_free(struct chain_allocator *ca, int clear_page_nosave)
  163. {
  164. free_list_of_pages(ca->chain, clear_page_nosave);
  165. memset(ca, 0, sizeof(struct chain_allocator));
  166. }
  167. /**
  168. * Data types related to memory bitmaps.
  169. *
  170. * Memory bitmap is a structure consiting of many linked lists of
  171. * objects. The main list's elements are of type struct zone_bitmap
  172. * and each of them corresonds to one zone. For each zone bitmap
  173. * object there is a list of objects of type struct bm_block that
  174. * represent each blocks of bit chunks in which information is
  175. * stored.
  176. *
  177. * struct memory_bitmap contains a pointer to the main list of zone
  178. * bitmap objects, a struct bm_position used for browsing the bitmap,
  179. * and a pointer to the list of pages used for allocating all of the
  180. * zone bitmap objects and bitmap block objects.
  181. *
  182. * NOTE: It has to be possible to lay out the bitmap in memory
  183. * using only allocations of order 0. Additionally, the bitmap is
  184. * designed to work with arbitrary number of zones (this is over the
  185. * top for now, but let's avoid making unnecessary assumptions ;-).
  186. *
  187. * struct zone_bitmap contains a pointer to a list of bitmap block
  188. * objects and a pointer to the bitmap block object that has been
  189. * most recently used for setting bits. Additionally, it contains the
  190. * pfns that correspond to the start and end of the represented zone.
  191. *
  192. * struct bm_block contains a pointer to the memory page in which
  193. * information is stored (in the form of a block of bit chunks
  194. * of type unsigned long each). It also contains the pfns that
  195. * correspond to the start and end of the represented memory area and
  196. * the number of bit chunks in the block.
  197. */
  198. #define BM_END_OF_MAP (~0UL)
  199. #define BM_CHUNKS_PER_BLOCK (PAGE_SIZE / sizeof(long))
  200. #define BM_BITS_PER_CHUNK (sizeof(long) << 3)
  201. #define BM_BITS_PER_BLOCK (PAGE_SIZE << 3)
  202. struct bm_block {
  203. struct bm_block *next; /* next element of the list */
  204. unsigned long start_pfn; /* pfn represented by the first bit */
  205. unsigned long end_pfn; /* pfn represented by the last bit plus 1 */
  206. unsigned int size; /* number of bit chunks */
  207. unsigned long *data; /* chunks of bits representing pages */
  208. };
  209. struct zone_bitmap {
  210. struct zone_bitmap *next; /* next element of the list */
  211. unsigned long start_pfn; /* minimal pfn in this zone */
  212. unsigned long end_pfn; /* maximal pfn in this zone plus 1 */
  213. struct bm_block *bm_blocks; /* list of bitmap blocks */
  214. struct bm_block *cur_block; /* recently used bitmap block */
  215. };
  216. /* strcut bm_position is used for browsing memory bitmaps */
  217. struct bm_position {
  218. struct zone_bitmap *zone_bm;
  219. struct bm_block *block;
  220. int chunk;
  221. int bit;
  222. };
  223. struct memory_bitmap {
  224. struct zone_bitmap *zone_bm_list; /* list of zone bitmaps */
  225. struct linked_page *p_list; /* list of pages used to store zone
  226. * bitmap objects and bitmap block
  227. * objects
  228. */
  229. struct bm_position cur; /* most recently used bit position */
  230. };
  231. /* Functions that operate on memory bitmaps */
  232. static inline void memory_bm_reset_chunk(struct memory_bitmap *bm)
  233. {
  234. bm->cur.chunk = 0;
  235. bm->cur.bit = -1;
  236. }
  237. static void memory_bm_position_reset(struct memory_bitmap *bm)
  238. {
  239. struct zone_bitmap *zone_bm;
  240. zone_bm = bm->zone_bm_list;
  241. bm->cur.zone_bm = zone_bm;
  242. bm->cur.block = zone_bm->bm_blocks;
  243. memory_bm_reset_chunk(bm);
  244. }
  245. static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free);
  246. /**
  247. * create_bm_block_list - create a list of block bitmap objects
  248. */
  249. static inline struct bm_block *
  250. create_bm_block_list(unsigned int nr_blocks, struct chain_allocator *ca)
  251. {
  252. struct bm_block *bblist = NULL;
  253. while (nr_blocks-- > 0) {
  254. struct bm_block *bb;
  255. bb = chain_alloc(ca, sizeof(struct bm_block));
  256. if (!bb)
  257. return NULL;
  258. bb->next = bblist;
  259. bblist = bb;
  260. }
  261. return bblist;
  262. }
  263. /**
  264. * create_zone_bm_list - create a list of zone bitmap objects
  265. */
  266. static inline struct zone_bitmap *
  267. create_zone_bm_list(unsigned int nr_zones, struct chain_allocator *ca)
  268. {
  269. struct zone_bitmap *zbmlist = NULL;
  270. while (nr_zones-- > 0) {
  271. struct zone_bitmap *zbm;
  272. zbm = chain_alloc(ca, sizeof(struct zone_bitmap));
  273. if (!zbm)
  274. return NULL;
  275. zbm->next = zbmlist;
  276. zbmlist = zbm;
  277. }
  278. return zbmlist;
  279. }
  280. /**
  281. * memory_bm_create - allocate memory for a memory bitmap
  282. */
  283. static int
  284. memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask, int safe_needed)
  285. {
  286. struct chain_allocator ca;
  287. struct zone *zone;
  288. struct zone_bitmap *zone_bm;
  289. struct bm_block *bb;
  290. unsigned int nr;
  291. chain_init(&ca, gfp_mask, safe_needed);
  292. /* Compute the number of zones */
  293. nr = 0;
  294. for_each_zone(zone)
  295. if (populated_zone(zone))
  296. nr++;
  297. /* Allocate the list of zones bitmap objects */
  298. zone_bm = create_zone_bm_list(nr, &ca);
  299. bm->zone_bm_list = zone_bm;
  300. if (!zone_bm) {
  301. chain_free(&ca, PG_UNSAFE_CLEAR);
  302. return -ENOMEM;
  303. }
  304. /* Initialize the zone bitmap objects */
  305. for_each_zone(zone) {
  306. unsigned long pfn;
  307. if (!populated_zone(zone))
  308. continue;
  309. zone_bm->start_pfn = zone->zone_start_pfn;
  310. zone_bm->end_pfn = zone->zone_start_pfn + zone->spanned_pages;
  311. /* Allocate the list of bitmap block objects */
  312. nr = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK);
  313. bb = create_bm_block_list(nr, &ca);
  314. zone_bm->bm_blocks = bb;
  315. zone_bm->cur_block = bb;
  316. if (!bb)
  317. goto Free;
  318. nr = zone->spanned_pages;
  319. pfn = zone->zone_start_pfn;
  320. /* Initialize the bitmap block objects */
  321. while (bb) {
  322. unsigned long *ptr;
  323. ptr = get_image_page(gfp_mask, safe_needed);
  324. bb->data = ptr;
  325. if (!ptr)
  326. goto Free;
  327. bb->start_pfn = pfn;
  328. if (nr >= BM_BITS_PER_BLOCK) {
  329. pfn += BM_BITS_PER_BLOCK;
  330. bb->size = BM_CHUNKS_PER_BLOCK;
  331. nr -= BM_BITS_PER_BLOCK;
  332. } else {
  333. /* This is executed only once in the loop */
  334. pfn += nr;
  335. bb->size = DIV_ROUND_UP(nr, BM_BITS_PER_CHUNK);
  336. }
  337. bb->end_pfn = pfn;
  338. bb = bb->next;
  339. }
  340. zone_bm = zone_bm->next;
  341. }
  342. bm->p_list = ca.chain;
  343. memory_bm_position_reset(bm);
  344. return 0;
  345. Free:
  346. bm->p_list = ca.chain;
  347. memory_bm_free(bm, PG_UNSAFE_CLEAR);
  348. return -ENOMEM;
  349. }
  350. /**
  351. * memory_bm_free - free memory occupied by the memory bitmap @bm
  352. */
  353. static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free)
  354. {
  355. struct zone_bitmap *zone_bm;
  356. /* Free the list of bit blocks for each zone_bitmap object */
  357. zone_bm = bm->zone_bm_list;
  358. while (zone_bm) {
  359. struct bm_block *bb;
  360. bb = zone_bm->bm_blocks;
  361. while (bb) {
  362. if (bb->data)
  363. free_image_page(bb->data, clear_nosave_free);
  364. bb = bb->next;
  365. }
  366. zone_bm = zone_bm->next;
  367. }
  368. free_list_of_pages(bm->p_list, clear_nosave_free);
  369. bm->zone_bm_list = NULL;
  370. }
  371. /**
  372. * memory_bm_find_bit - find the bit in the bitmap @bm that corresponds
  373. * to given pfn. The cur_zone_bm member of @bm and the cur_block member
  374. * of @bm->cur_zone_bm are updated.
  375. */
  376. static void memory_bm_find_bit(struct memory_bitmap *bm, unsigned long pfn,
  377. void **addr, unsigned int *bit_nr)
  378. {
  379. struct zone_bitmap *zone_bm;
  380. struct bm_block *bb;
  381. /* Check if the pfn is from the current zone */
  382. zone_bm = bm->cur.zone_bm;
  383. if (pfn < zone_bm->start_pfn || pfn >= zone_bm->end_pfn) {
  384. zone_bm = bm->zone_bm_list;
  385. /* We don't assume that the zones are sorted by pfns */
  386. while (pfn < zone_bm->start_pfn || pfn >= zone_bm->end_pfn) {
  387. zone_bm = zone_bm->next;
  388. BUG_ON(!zone_bm);
  389. }
  390. bm->cur.zone_bm = zone_bm;
  391. }
  392. /* Check if the pfn corresponds to the current bitmap block */
  393. bb = zone_bm->cur_block;
  394. if (pfn < bb->start_pfn)
  395. bb = zone_bm->bm_blocks;
  396. while (pfn >= bb->end_pfn) {
  397. bb = bb->next;
  398. BUG_ON(!bb);
  399. }
  400. zone_bm->cur_block = bb;
  401. pfn -= bb->start_pfn;
  402. *bit_nr = pfn % BM_BITS_PER_CHUNK;
  403. *addr = bb->data + pfn / BM_BITS_PER_CHUNK;
  404. }
  405. static void memory_bm_set_bit(struct memory_bitmap *bm, unsigned long pfn)
  406. {
  407. void *addr;
  408. unsigned int bit;
  409. memory_bm_find_bit(bm, pfn, &addr, &bit);
  410. set_bit(bit, addr);
  411. }
  412. static void memory_bm_clear_bit(struct memory_bitmap *bm, unsigned long pfn)
  413. {
  414. void *addr;
  415. unsigned int bit;
  416. memory_bm_find_bit(bm, pfn, &addr, &bit);
  417. clear_bit(bit, addr);
  418. }
  419. static int memory_bm_test_bit(struct memory_bitmap *bm, unsigned long pfn)
  420. {
  421. void *addr;
  422. unsigned int bit;
  423. memory_bm_find_bit(bm, pfn, &addr, &bit);
  424. return test_bit(bit, addr);
  425. }
  426. /* Two auxiliary functions for memory_bm_next_pfn */
  427. /* Find the first set bit in the given chunk, if there is one */
  428. static inline int next_bit_in_chunk(int bit, unsigned long *chunk_p)
  429. {
  430. bit++;
  431. while (bit < BM_BITS_PER_CHUNK) {
  432. if (test_bit(bit, chunk_p))
  433. return bit;
  434. bit++;
  435. }
  436. return -1;
  437. }
  438. /* Find a chunk containing some bits set in given block of bits */
  439. static inline int next_chunk_in_block(int n, struct bm_block *bb)
  440. {
  441. n++;
  442. while (n < bb->size) {
  443. if (bb->data[n])
  444. return n;
  445. n++;
  446. }
  447. return -1;
  448. }
  449. /**
  450. * memory_bm_next_pfn - find the pfn that corresponds to the next set bit
  451. * in the bitmap @bm. If the pfn cannot be found, BM_END_OF_MAP is
  452. * returned.
  453. *
  454. * It is required to run memory_bm_position_reset() before the first call to
  455. * this function.
  456. */
  457. static unsigned long memory_bm_next_pfn(struct memory_bitmap *bm)
  458. {
  459. struct zone_bitmap *zone_bm;
  460. struct bm_block *bb;
  461. int chunk;
  462. int bit;
  463. do {
  464. bb = bm->cur.block;
  465. do {
  466. chunk = bm->cur.chunk;
  467. bit = bm->cur.bit;
  468. do {
  469. bit = next_bit_in_chunk(bit, bb->data + chunk);
  470. if (bit >= 0)
  471. goto Return_pfn;
  472. chunk = next_chunk_in_block(chunk, bb);
  473. bit = -1;
  474. } while (chunk >= 0);
  475. bb = bb->next;
  476. bm->cur.block = bb;
  477. memory_bm_reset_chunk(bm);
  478. } while (bb);
  479. zone_bm = bm->cur.zone_bm->next;
  480. if (zone_bm) {
  481. bm->cur.zone_bm = zone_bm;
  482. bm->cur.block = zone_bm->bm_blocks;
  483. memory_bm_reset_chunk(bm);
  484. }
  485. } while (zone_bm);
  486. memory_bm_position_reset(bm);
  487. return BM_END_OF_MAP;
  488. Return_pfn:
  489. bm->cur.chunk = chunk;
  490. bm->cur.bit = bit;
  491. return bb->start_pfn + chunk * BM_BITS_PER_CHUNK + bit;
  492. }
  493. /**
  494. * This structure represents a range of page frames the contents of which
  495. * should not be saved during the suspend.
  496. */
  497. struct nosave_region {
  498. struct list_head list;
  499. unsigned long start_pfn;
  500. unsigned long end_pfn;
  501. };
  502. static LIST_HEAD(nosave_regions);
  503. /**
  504. * register_nosave_region - register a range of page frames the contents
  505. * of which should not be saved during the suspend (to be used in the early
  506. * initialization code)
  507. */
  508. void __init
  509. __register_nosave_region(unsigned long start_pfn, unsigned long end_pfn,
  510. int use_kmalloc)
  511. {
  512. struct nosave_region *region;
  513. if (start_pfn >= end_pfn)
  514. return;
  515. if (!list_empty(&nosave_regions)) {
  516. /* Try to extend the previous region (they should be sorted) */
  517. region = list_entry(nosave_regions.prev,
  518. struct nosave_region, list);
  519. if (region->end_pfn == start_pfn) {
  520. region->end_pfn = end_pfn;
  521. goto Report;
  522. }
  523. }
  524. if (use_kmalloc) {
  525. /* during init, this shouldn't fail */
  526. region = kmalloc(sizeof(struct nosave_region), GFP_KERNEL);
  527. BUG_ON(!region);
  528. } else
  529. /* This allocation cannot fail */
  530. region = alloc_bootmem_low(sizeof(struct nosave_region));
  531. region->start_pfn = start_pfn;
  532. region->end_pfn = end_pfn;
  533. list_add_tail(&region->list, &nosave_regions);
  534. Report:
  535. printk("swsusp: Registered nosave memory region: %016lx - %016lx\n",
  536. start_pfn << PAGE_SHIFT, end_pfn << PAGE_SHIFT);
  537. }
  538. /*
  539. * Set bits in this map correspond to the page frames the contents of which
  540. * should not be saved during the suspend.
  541. */
  542. static struct memory_bitmap *forbidden_pages_map;
  543. /* Set bits in this map correspond to free page frames. */
  544. static struct memory_bitmap *free_pages_map;
  545. /*
  546. * Each page frame allocated for creating the image is marked by setting the
  547. * corresponding bits in forbidden_pages_map and free_pages_map simultaneously
  548. */
  549. void swsusp_set_page_free(struct page *page)
  550. {
  551. if (free_pages_map)
  552. memory_bm_set_bit(free_pages_map, page_to_pfn(page));
  553. }
  554. static int swsusp_page_is_free(struct page *page)
  555. {
  556. return free_pages_map ?
  557. memory_bm_test_bit(free_pages_map, page_to_pfn(page)) : 0;
  558. }
  559. void swsusp_unset_page_free(struct page *page)
  560. {
  561. if (free_pages_map)
  562. memory_bm_clear_bit(free_pages_map, page_to_pfn(page));
  563. }
  564. static void swsusp_set_page_forbidden(struct page *page)
  565. {
  566. if (forbidden_pages_map)
  567. memory_bm_set_bit(forbidden_pages_map, page_to_pfn(page));
  568. }
  569. int swsusp_page_is_forbidden(struct page *page)
  570. {
  571. return forbidden_pages_map ?
  572. memory_bm_test_bit(forbidden_pages_map, page_to_pfn(page)) : 0;
  573. }
  574. static void swsusp_unset_page_forbidden(struct page *page)
  575. {
  576. if (forbidden_pages_map)
  577. memory_bm_clear_bit(forbidden_pages_map, page_to_pfn(page));
  578. }
  579. /**
  580. * mark_nosave_pages - set bits corresponding to the page frames the
  581. * contents of which should not be saved in a given bitmap.
  582. */
  583. static void mark_nosave_pages(struct memory_bitmap *bm)
  584. {
  585. struct nosave_region *region;
  586. if (list_empty(&nosave_regions))
  587. return;
  588. list_for_each_entry(region, &nosave_regions, list) {
  589. unsigned long pfn;
  590. printk("swsusp: Marking nosave pages: %016lx - %016lx\n",
  591. region->start_pfn << PAGE_SHIFT,
  592. region->end_pfn << PAGE_SHIFT);
  593. for (pfn = region->start_pfn; pfn < region->end_pfn; pfn++)
  594. if (pfn_valid(pfn))
  595. memory_bm_set_bit(bm, pfn);
  596. }
  597. }
  598. /**
  599. * create_basic_memory_bitmaps - create bitmaps needed for marking page
  600. * frames that should not be saved and free page frames. The pointers
  601. * forbidden_pages_map and free_pages_map are only modified if everything
  602. * goes well, because we don't want the bits to be used before both bitmaps
  603. * are set up.
  604. */
  605. int create_basic_memory_bitmaps(void)
  606. {
  607. struct memory_bitmap *bm1, *bm2;
  608. int error = 0;
  609. BUG_ON(forbidden_pages_map || free_pages_map);
  610. bm1 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL);
  611. if (!bm1)
  612. return -ENOMEM;
  613. error = memory_bm_create(bm1, GFP_KERNEL, PG_ANY);
  614. if (error)
  615. goto Free_first_object;
  616. bm2 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL);
  617. if (!bm2)
  618. goto Free_first_bitmap;
  619. error = memory_bm_create(bm2, GFP_KERNEL, PG_ANY);
  620. if (error)
  621. goto Free_second_object;
  622. forbidden_pages_map = bm1;
  623. free_pages_map = bm2;
  624. mark_nosave_pages(forbidden_pages_map);
  625. printk("swsusp: Basic memory bitmaps created\n");
  626. return 0;
  627. Free_second_object:
  628. kfree(bm2);
  629. Free_first_bitmap:
  630. memory_bm_free(bm1, PG_UNSAFE_CLEAR);
  631. Free_first_object:
  632. kfree(bm1);
  633. return -ENOMEM;
  634. }
  635. /**
  636. * free_basic_memory_bitmaps - free memory bitmaps allocated by
  637. * create_basic_memory_bitmaps(). The auxiliary pointers are necessary
  638. * so that the bitmaps themselves are not referred to while they are being
  639. * freed.
  640. */
  641. void free_basic_memory_bitmaps(void)
  642. {
  643. struct memory_bitmap *bm1, *bm2;
  644. BUG_ON(!(forbidden_pages_map && free_pages_map));
  645. bm1 = forbidden_pages_map;
  646. bm2 = free_pages_map;
  647. forbidden_pages_map = NULL;
  648. free_pages_map = NULL;
  649. memory_bm_free(bm1, PG_UNSAFE_CLEAR);
  650. kfree(bm1);
  651. memory_bm_free(bm2, PG_UNSAFE_CLEAR);
  652. kfree(bm2);
  653. printk("swsusp: Basic memory bitmaps freed\n");
  654. }
  655. /**
  656. * snapshot_additional_pages - estimate the number of additional pages
  657. * be needed for setting up the suspend image data structures for given
  658. * zone (usually the returned value is greater than the exact number)
  659. */
  660. unsigned int snapshot_additional_pages(struct zone *zone)
  661. {
  662. unsigned int res;
  663. res = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK);
  664. res += DIV_ROUND_UP(res * sizeof(struct bm_block), PAGE_SIZE);
  665. return 2 * res;
  666. }
  667. #ifdef CONFIG_HIGHMEM
  668. /**
  669. * count_free_highmem_pages - compute the total number of free highmem
  670. * pages, system-wide.
  671. */
  672. static unsigned int count_free_highmem_pages(void)
  673. {
  674. struct zone *zone;
  675. unsigned int cnt = 0;
  676. for_each_zone(zone)
  677. if (populated_zone(zone) && is_highmem(zone))
  678. cnt += zone_page_state(zone, NR_FREE_PAGES);
  679. return cnt;
  680. }
  681. /**
  682. * saveable_highmem_page - Determine whether a highmem page should be
  683. * included in the suspend image.
  684. *
  685. * We should save the page if it isn't Nosave or NosaveFree, or Reserved,
  686. * and it isn't a part of a free chunk of pages.
  687. */
  688. static struct page *saveable_highmem_page(unsigned long pfn)
  689. {
  690. struct page *page;
  691. if (!pfn_valid(pfn))
  692. return NULL;
  693. page = pfn_to_page(pfn);
  694. BUG_ON(!PageHighMem(page));
  695. if (swsusp_page_is_forbidden(page) || swsusp_page_is_free(page) ||
  696. PageReserved(page))
  697. return NULL;
  698. return page;
  699. }
  700. /**
  701. * count_highmem_pages - compute the total number of saveable highmem
  702. * pages.
  703. */
  704. unsigned int count_highmem_pages(void)
  705. {
  706. struct zone *zone;
  707. unsigned int n = 0;
  708. for_each_zone(zone) {
  709. unsigned long pfn, max_zone_pfn;
  710. if (!is_highmem(zone))
  711. continue;
  712. mark_free_pages(zone);
  713. max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
  714. for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
  715. if (saveable_highmem_page(pfn))
  716. n++;
  717. }
  718. return n;
  719. }
  720. #else
  721. static inline void *saveable_highmem_page(unsigned long pfn) { return NULL; }
  722. #endif /* CONFIG_HIGHMEM */
  723. /**
  724. * saveable - Determine whether a non-highmem page should be included in
  725. * the suspend image.
  726. *
  727. * We should save the page if it isn't Nosave, and is not in the range
  728. * of pages statically defined as 'unsaveable', and it isn't a part of
  729. * a free chunk of pages.
  730. */
  731. static struct page *saveable_page(unsigned long pfn)
  732. {
  733. struct page *page;
  734. if (!pfn_valid(pfn))
  735. return NULL;
  736. page = pfn_to_page(pfn);
  737. BUG_ON(PageHighMem(page));
  738. if (swsusp_page_is_forbidden(page) || swsusp_page_is_free(page))
  739. return NULL;
  740. if (PageReserved(page) && pfn_is_nosave(pfn))
  741. return NULL;
  742. return page;
  743. }
  744. /**
  745. * count_data_pages - compute the total number of saveable non-highmem
  746. * pages.
  747. */
  748. unsigned int count_data_pages(void)
  749. {
  750. struct zone *zone;
  751. unsigned long pfn, max_zone_pfn;
  752. unsigned int n = 0;
  753. for_each_zone(zone) {
  754. if (is_highmem(zone))
  755. continue;
  756. mark_free_pages(zone);
  757. max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
  758. for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
  759. if(saveable_page(pfn))
  760. n++;
  761. }
  762. return n;
  763. }
  764. /* This is needed, because copy_page and memcpy are not usable for copying
  765. * task structs.
  766. */
  767. static inline void do_copy_page(long *dst, long *src)
  768. {
  769. int n;
  770. for (n = PAGE_SIZE / sizeof(long); n; n--)
  771. *dst++ = *src++;
  772. }
  773. #ifdef CONFIG_HIGHMEM
  774. static inline struct page *
  775. page_is_saveable(struct zone *zone, unsigned long pfn)
  776. {
  777. return is_highmem(zone) ?
  778. saveable_highmem_page(pfn) : saveable_page(pfn);
  779. }
  780. static inline void
  781. copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)
  782. {
  783. struct page *s_page, *d_page;
  784. void *src, *dst;
  785. s_page = pfn_to_page(src_pfn);
  786. d_page = pfn_to_page(dst_pfn);
  787. if (PageHighMem(s_page)) {
  788. src = kmap_atomic(s_page, KM_USER0);
  789. dst = kmap_atomic(d_page, KM_USER1);
  790. do_copy_page(dst, src);
  791. kunmap_atomic(src, KM_USER0);
  792. kunmap_atomic(dst, KM_USER1);
  793. } else {
  794. src = page_address(s_page);
  795. if (PageHighMem(d_page)) {
  796. /* Page pointed to by src may contain some kernel
  797. * data modified by kmap_atomic()
  798. */
  799. do_copy_page(buffer, src);
  800. dst = kmap_atomic(pfn_to_page(dst_pfn), KM_USER0);
  801. memcpy(dst, buffer, PAGE_SIZE);
  802. kunmap_atomic(dst, KM_USER0);
  803. } else {
  804. dst = page_address(d_page);
  805. do_copy_page(dst, src);
  806. }
  807. }
  808. }
  809. #else
  810. #define page_is_saveable(zone, pfn) saveable_page(pfn)
  811. static inline void
  812. copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)
  813. {
  814. do_copy_page(page_address(pfn_to_page(dst_pfn)),
  815. page_address(pfn_to_page(src_pfn)));
  816. }
  817. #endif /* CONFIG_HIGHMEM */
  818. static void
  819. copy_data_pages(struct memory_bitmap *copy_bm, struct memory_bitmap *orig_bm)
  820. {
  821. struct zone *zone;
  822. unsigned long pfn;
  823. for_each_zone(zone) {
  824. unsigned long max_zone_pfn;
  825. mark_free_pages(zone);
  826. max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
  827. for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
  828. if (page_is_saveable(zone, pfn))
  829. memory_bm_set_bit(orig_bm, pfn);
  830. }
  831. memory_bm_position_reset(orig_bm);
  832. memory_bm_position_reset(copy_bm);
  833. for(;;) {
  834. pfn = memory_bm_next_pfn(orig_bm);
  835. if (unlikely(pfn == BM_END_OF_MAP))
  836. break;
  837. copy_data_page(memory_bm_next_pfn(copy_bm), pfn);
  838. }
  839. }
  840. /* Total number of image pages */
  841. static unsigned int nr_copy_pages;
  842. /* Number of pages needed for saving the original pfns of the image pages */
  843. static unsigned int nr_meta_pages;
  844. /**
  845. * swsusp_free - free pages allocated for the suspend.
  846. *
  847. * Suspend pages are alocated before the atomic copy is made, so we
  848. * need to release them after the resume.
  849. */
  850. void swsusp_free(void)
  851. {
  852. struct zone *zone;
  853. unsigned long pfn, max_zone_pfn;
  854. for_each_zone(zone) {
  855. max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
  856. for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
  857. if (pfn_valid(pfn)) {
  858. struct page *page = pfn_to_page(pfn);
  859. if (swsusp_page_is_forbidden(page) &&
  860. swsusp_page_is_free(page)) {
  861. swsusp_unset_page_forbidden(page);
  862. swsusp_unset_page_free(page);
  863. __free_page(page);
  864. }
  865. }
  866. }
  867. nr_copy_pages = 0;
  868. nr_meta_pages = 0;
  869. restore_pblist = NULL;
  870. buffer = NULL;
  871. }
  872. #ifdef CONFIG_HIGHMEM
  873. /**
  874. * count_pages_for_highmem - compute the number of non-highmem pages
  875. * that will be necessary for creating copies of highmem pages.
  876. */
  877. static unsigned int count_pages_for_highmem(unsigned int nr_highmem)
  878. {
  879. unsigned int free_highmem = count_free_highmem_pages();
  880. if (free_highmem >= nr_highmem)
  881. nr_highmem = 0;
  882. else
  883. nr_highmem -= free_highmem;
  884. return nr_highmem;
  885. }
  886. #else
  887. static unsigned int
  888. count_pages_for_highmem(unsigned int nr_highmem) { return 0; }
  889. #endif /* CONFIG_HIGHMEM */
  890. /**
  891. * enough_free_mem - Make sure we have enough free memory for the
  892. * snapshot image.
  893. */
  894. static int enough_free_mem(unsigned int nr_pages, unsigned int nr_highmem)
  895. {
  896. struct zone *zone;
  897. unsigned int free = 0, meta = 0;
  898. for_each_zone(zone) {
  899. meta += snapshot_additional_pages(zone);
  900. if (!is_highmem(zone))
  901. free += zone_page_state(zone, NR_FREE_PAGES);
  902. }
  903. nr_pages += count_pages_for_highmem(nr_highmem);
  904. pr_debug("swsusp: Normal pages needed: %u + %u + %u, available pages: %u\n",
  905. nr_pages, PAGES_FOR_IO, meta, free);
  906. return free > nr_pages + PAGES_FOR_IO + meta;
  907. }
  908. #ifdef CONFIG_HIGHMEM
  909. /**
  910. * get_highmem_buffer - if there are some highmem pages in the suspend
  911. * image, we may need the buffer to copy them and/or load their data.
  912. */
  913. static inline int get_highmem_buffer(int safe_needed)
  914. {
  915. buffer = get_image_page(GFP_ATOMIC | __GFP_COLD, safe_needed);
  916. return buffer ? 0 : -ENOMEM;
  917. }
  918. /**
  919. * alloc_highmem_image_pages - allocate some highmem pages for the image.
  920. * Try to allocate as many pages as needed, but if the number of free
  921. * highmem pages is lesser than that, allocate them all.
  922. */
  923. static inline unsigned int
  924. alloc_highmem_image_pages(struct memory_bitmap *bm, unsigned int nr_highmem)
  925. {
  926. unsigned int to_alloc = count_free_highmem_pages();
  927. if (to_alloc > nr_highmem)
  928. to_alloc = nr_highmem;
  929. nr_highmem -= to_alloc;
  930. while (to_alloc-- > 0) {
  931. struct page *page;
  932. page = alloc_image_page(__GFP_HIGHMEM);
  933. memory_bm_set_bit(bm, page_to_pfn(page));
  934. }
  935. return nr_highmem;
  936. }
  937. #else
  938. static inline int get_highmem_buffer(int safe_needed) { return 0; }
  939. static inline unsigned int
  940. alloc_highmem_image_pages(struct memory_bitmap *bm, unsigned int n) { return 0; }
  941. #endif /* CONFIG_HIGHMEM */
  942. /**
  943. * swsusp_alloc - allocate memory for the suspend image
  944. *
  945. * We first try to allocate as many highmem pages as there are
  946. * saveable highmem pages in the system. If that fails, we allocate
  947. * non-highmem pages for the copies of the remaining highmem ones.
  948. *
  949. * In this approach it is likely that the copies of highmem pages will
  950. * also be located in the high memory, because of the way in which
  951. * copy_data_pages() works.
  952. */
  953. static int
  954. swsusp_alloc(struct memory_bitmap *orig_bm, struct memory_bitmap *copy_bm,
  955. unsigned int nr_pages, unsigned int nr_highmem)
  956. {
  957. int error;
  958. error = memory_bm_create(orig_bm, GFP_ATOMIC | __GFP_COLD, PG_ANY);
  959. if (error)
  960. goto Free;
  961. error = memory_bm_create(copy_bm, GFP_ATOMIC | __GFP_COLD, PG_ANY);
  962. if (error)
  963. goto Free;
  964. if (nr_highmem > 0) {
  965. error = get_highmem_buffer(PG_ANY);
  966. if (error)
  967. goto Free;
  968. nr_pages += alloc_highmem_image_pages(copy_bm, nr_highmem);
  969. }
  970. while (nr_pages-- > 0) {
  971. struct page *page = alloc_image_page(GFP_ATOMIC | __GFP_COLD);
  972. if (!page)
  973. goto Free;
  974. memory_bm_set_bit(copy_bm, page_to_pfn(page));
  975. }
  976. return 0;
  977. Free:
  978. swsusp_free();
  979. return -ENOMEM;
  980. }
  981. /* Memory bitmap used for marking saveable pages (during suspend) or the
  982. * suspend image pages (during resume)
  983. */
  984. static struct memory_bitmap orig_bm;
  985. /* Memory bitmap used on suspend for marking allocated pages that will contain
  986. * the copies of saveable pages. During resume it is initially used for
  987. * marking the suspend image pages, but then its set bits are duplicated in
  988. * @orig_bm and it is released. Next, on systems with high memory, it may be
  989. * used for marking "safe" highmem pages, but it has to be reinitialized for
  990. * this purpose.
  991. */
  992. static struct memory_bitmap copy_bm;
  993. asmlinkage int swsusp_save(void)
  994. {
  995. unsigned int nr_pages, nr_highmem;
  996. printk("swsusp: critical section: \n");
  997. drain_local_pages();
  998. nr_pages = count_data_pages();
  999. nr_highmem = count_highmem_pages();
  1000. printk("swsusp: Need to copy %u pages\n", nr_pages + nr_highmem);
  1001. if (!enough_free_mem(nr_pages, nr_highmem)) {
  1002. printk(KERN_ERR "swsusp: Not enough free memory\n");
  1003. return -ENOMEM;
  1004. }
  1005. if (swsusp_alloc(&orig_bm, &copy_bm, nr_pages, nr_highmem)) {
  1006. printk(KERN_ERR "swsusp: Memory allocation failed\n");
  1007. return -ENOMEM;
  1008. }
  1009. /* During allocating of suspend pagedir, new cold pages may appear.
  1010. * Kill them.
  1011. */
  1012. drain_local_pages();
  1013. copy_data_pages(&copy_bm, &orig_bm);
  1014. /*
  1015. * End of critical section. From now on, we can write to memory,
  1016. * but we should not touch disk. This specially means we must _not_
  1017. * touch swap space! Except we must write out our image of course.
  1018. */
  1019. nr_pages += nr_highmem;
  1020. nr_copy_pages = nr_pages;
  1021. nr_meta_pages = DIV_ROUND_UP(nr_pages * sizeof(long), PAGE_SIZE);
  1022. printk("swsusp: critical section: done (%d pages copied)\n", nr_pages);
  1023. return 0;
  1024. }
  1025. #ifndef CONFIG_ARCH_HIBERNATION_HEADER
  1026. static int init_header_complete(struct swsusp_info *info)
  1027. {
  1028. memcpy(&info->uts, init_utsname(), sizeof(struct new_utsname));
  1029. info->version_code = LINUX_VERSION_CODE;
  1030. return 0;
  1031. }
  1032. static char *check_image_kernel(struct swsusp_info *info)
  1033. {
  1034. if (info->version_code != LINUX_VERSION_CODE)
  1035. return "kernel version";
  1036. if (strcmp(info->uts.sysname,init_utsname()->sysname))
  1037. return "system type";
  1038. if (strcmp(info->uts.release,init_utsname()->release))
  1039. return "kernel release";
  1040. if (strcmp(info->uts.version,init_utsname()->version))
  1041. return "version";
  1042. if (strcmp(info->uts.machine,init_utsname()->machine))
  1043. return "machine";
  1044. return NULL;
  1045. }
  1046. #endif /* CONFIG_ARCH_HIBERNATION_HEADER */
  1047. unsigned long snapshot_get_image_size(void)
  1048. {
  1049. return nr_copy_pages + nr_meta_pages + 1;
  1050. }
  1051. static int init_header(struct swsusp_info *info)
  1052. {
  1053. memset(info, 0, sizeof(struct swsusp_info));
  1054. info->num_physpages = num_physpages;
  1055. info->image_pages = nr_copy_pages;
  1056. info->pages = snapshot_get_image_size();
  1057. info->size = info->pages;
  1058. info->size <<= PAGE_SHIFT;
  1059. return init_header_complete(info);
  1060. }
  1061. /**
  1062. * pack_pfns - pfns corresponding to the set bits found in the bitmap @bm
  1063. * are stored in the array @buf[] (1 page at a time)
  1064. */
  1065. static inline void
  1066. pack_pfns(unsigned long *buf, struct memory_bitmap *bm)
  1067. {
  1068. int j;
  1069. for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {
  1070. buf[j] = memory_bm_next_pfn(bm);
  1071. if (unlikely(buf[j] == BM_END_OF_MAP))
  1072. break;
  1073. }
  1074. }
  1075. /**
  1076. * snapshot_read_next - used for reading the system memory snapshot.
  1077. *
  1078. * On the first call to it @handle should point to a zeroed
  1079. * snapshot_handle structure. The structure gets updated and a pointer
  1080. * to it should be passed to this function every next time.
  1081. *
  1082. * The @count parameter should contain the number of bytes the caller
  1083. * wants to read from the snapshot. It must not be zero.
  1084. *
  1085. * On success the function returns a positive number. Then, the caller
  1086. * is allowed to read up to the returned number of bytes from the memory
  1087. * location computed by the data_of() macro. The number returned
  1088. * may be smaller than @count, but this only happens if the read would
  1089. * cross a page boundary otherwise.
  1090. *
  1091. * The function returns 0 to indicate the end of data stream condition,
  1092. * and a negative number is returned on error. In such cases the
  1093. * structure pointed to by @handle is not updated and should not be used
  1094. * any more.
  1095. */
  1096. int snapshot_read_next(struct snapshot_handle *handle, size_t count)
  1097. {
  1098. if (handle->cur > nr_meta_pages + nr_copy_pages)
  1099. return 0;
  1100. if (!buffer) {
  1101. /* This makes the buffer be freed by swsusp_free() */
  1102. buffer = get_image_page(GFP_ATOMIC, PG_ANY);
  1103. if (!buffer)
  1104. return -ENOMEM;
  1105. }
  1106. if (!handle->offset) {
  1107. int error;
  1108. error = init_header((struct swsusp_info *)buffer);
  1109. if (error)
  1110. return error;
  1111. handle->buffer = buffer;
  1112. memory_bm_position_reset(&orig_bm);
  1113. memory_bm_position_reset(&copy_bm);
  1114. }
  1115. if (handle->prev < handle->cur) {
  1116. if (handle->cur <= nr_meta_pages) {
  1117. memset(buffer, 0, PAGE_SIZE);
  1118. pack_pfns(buffer, &orig_bm);
  1119. } else {
  1120. struct page *page;
  1121. page = pfn_to_page(memory_bm_next_pfn(&copy_bm));
  1122. if (PageHighMem(page)) {
  1123. /* Highmem pages are copied to the buffer,
  1124. * because we can't return with a kmapped
  1125. * highmem page (we may not be called again).
  1126. */
  1127. void *kaddr;
  1128. kaddr = kmap_atomic(page, KM_USER0);
  1129. memcpy(buffer, kaddr, PAGE_SIZE);
  1130. kunmap_atomic(kaddr, KM_USER0);
  1131. handle->buffer = buffer;
  1132. } else {
  1133. handle->buffer = page_address(page);
  1134. }
  1135. }
  1136. handle->prev = handle->cur;
  1137. }
  1138. handle->buf_offset = handle->cur_offset;
  1139. if (handle->cur_offset + count >= PAGE_SIZE) {
  1140. count = PAGE_SIZE - handle->cur_offset;
  1141. handle->cur_offset = 0;
  1142. handle->cur++;
  1143. } else {
  1144. handle->cur_offset += count;
  1145. }
  1146. handle->offset += count;
  1147. return count;
  1148. }
  1149. /**
  1150. * mark_unsafe_pages - mark the pages that cannot be used for storing
  1151. * the image during resume, because they conflict with the pages that
  1152. * had been used before suspend
  1153. */
  1154. static int mark_unsafe_pages(struct memory_bitmap *bm)
  1155. {
  1156. struct zone *zone;
  1157. unsigned long pfn, max_zone_pfn;
  1158. /* Clear page flags */
  1159. for_each_zone(zone) {
  1160. max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
  1161. for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
  1162. if (pfn_valid(pfn))
  1163. swsusp_unset_page_free(pfn_to_page(pfn));
  1164. }
  1165. /* Mark pages that correspond to the "original" pfns as "unsafe" */
  1166. memory_bm_position_reset(bm);
  1167. do {
  1168. pfn = memory_bm_next_pfn(bm);
  1169. if (likely(pfn != BM_END_OF_MAP)) {
  1170. if (likely(pfn_valid(pfn)))
  1171. swsusp_set_page_free(pfn_to_page(pfn));
  1172. else
  1173. return -EFAULT;
  1174. }
  1175. } while (pfn != BM_END_OF_MAP);
  1176. allocated_unsafe_pages = 0;
  1177. return 0;
  1178. }
  1179. static void
  1180. duplicate_memory_bitmap(struct memory_bitmap *dst, struct memory_bitmap *src)
  1181. {
  1182. unsigned long pfn;
  1183. memory_bm_position_reset(src);
  1184. pfn = memory_bm_next_pfn(src);
  1185. while (pfn != BM_END_OF_MAP) {
  1186. memory_bm_set_bit(dst, pfn);
  1187. pfn = memory_bm_next_pfn(src);
  1188. }
  1189. }
  1190. static int check_header(struct swsusp_info *info)
  1191. {
  1192. char *reason;
  1193. reason = check_image_kernel(info);
  1194. if (!reason && info->num_physpages != num_physpages)
  1195. reason = "memory size";
  1196. if (reason) {
  1197. printk(KERN_ERR "swsusp: Resume mismatch: %s\n", reason);
  1198. return -EPERM;
  1199. }
  1200. return 0;
  1201. }
  1202. /**
  1203. * load header - check the image header and copy data from it
  1204. */
  1205. static int
  1206. load_header(struct swsusp_info *info)
  1207. {
  1208. int error;
  1209. restore_pblist = NULL;
  1210. error = check_header(info);
  1211. if (!error) {
  1212. nr_copy_pages = info->image_pages;
  1213. nr_meta_pages = info->pages - info->image_pages - 1;
  1214. }
  1215. return error;
  1216. }
  1217. /**
  1218. * unpack_orig_pfns - for each element of @buf[] (1 page at a time) set
  1219. * the corresponding bit in the memory bitmap @bm
  1220. */
  1221. static inline void
  1222. unpack_orig_pfns(unsigned long *buf, struct memory_bitmap *bm)
  1223. {
  1224. int j;
  1225. for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {
  1226. if (unlikely(buf[j] == BM_END_OF_MAP))
  1227. break;
  1228. memory_bm_set_bit(bm, buf[j]);
  1229. }
  1230. }
  1231. /* List of "safe" pages that may be used to store data loaded from the suspend
  1232. * image
  1233. */
  1234. static struct linked_page *safe_pages_list;
  1235. #ifdef CONFIG_HIGHMEM
  1236. /* struct highmem_pbe is used for creating the list of highmem pages that
  1237. * should be restored atomically during the resume from disk, because the page
  1238. * frames they have occupied before the suspend are in use.
  1239. */
  1240. struct highmem_pbe {
  1241. struct page *copy_page; /* data is here now */
  1242. struct page *orig_page; /* data was here before the suspend */
  1243. struct highmem_pbe *next;
  1244. };
  1245. /* List of highmem PBEs needed for restoring the highmem pages that were
  1246. * allocated before the suspend and included in the suspend image, but have
  1247. * also been allocated by the "resume" kernel, so their contents cannot be
  1248. * written directly to their "original" page frames.
  1249. */
  1250. static struct highmem_pbe *highmem_pblist;
  1251. /**
  1252. * count_highmem_image_pages - compute the number of highmem pages in the
  1253. * suspend image. The bits in the memory bitmap @bm that correspond to the
  1254. * image pages are assumed to be set.
  1255. */
  1256. static unsigned int count_highmem_image_pages(struct memory_bitmap *bm)
  1257. {
  1258. unsigned long pfn;
  1259. unsigned int cnt = 0;
  1260. memory_bm_position_reset(bm);
  1261. pfn = memory_bm_next_pfn(bm);
  1262. while (pfn != BM_END_OF_MAP) {
  1263. if (PageHighMem(pfn_to_page(pfn)))
  1264. cnt++;
  1265. pfn = memory_bm_next_pfn(bm);
  1266. }
  1267. return cnt;
  1268. }
  1269. /**
  1270. * prepare_highmem_image - try to allocate as many highmem pages as
  1271. * there are highmem image pages (@nr_highmem_p points to the variable
  1272. * containing the number of highmem image pages). The pages that are
  1273. * "safe" (ie. will not be overwritten when the suspend image is
  1274. * restored) have the corresponding bits set in @bm (it must be
  1275. * unitialized).
  1276. *
  1277. * NOTE: This function should not be called if there are no highmem
  1278. * image pages.
  1279. */
  1280. static unsigned int safe_highmem_pages;
  1281. static struct memory_bitmap *safe_highmem_bm;
  1282. static int
  1283. prepare_highmem_image(struct memory_bitmap *bm, unsigned int *nr_highmem_p)
  1284. {
  1285. unsigned int to_alloc;
  1286. if (memory_bm_create(bm, GFP_ATOMIC, PG_SAFE))
  1287. return -ENOMEM;
  1288. if (get_highmem_buffer(PG_SAFE))
  1289. return -ENOMEM;
  1290. to_alloc = count_free_highmem_pages();
  1291. if (to_alloc > *nr_highmem_p)
  1292. to_alloc = *nr_highmem_p;
  1293. else
  1294. *nr_highmem_p = to_alloc;
  1295. safe_highmem_pages = 0;
  1296. while (to_alloc-- > 0) {
  1297. struct page *page;
  1298. page = alloc_page(__GFP_HIGHMEM);
  1299. if (!swsusp_page_is_free(page)) {
  1300. /* The page is "safe", set its bit the bitmap */
  1301. memory_bm_set_bit(bm, page_to_pfn(page));
  1302. safe_highmem_pages++;
  1303. }
  1304. /* Mark the page as allocated */
  1305. swsusp_set_page_forbidden(page);
  1306. swsusp_set_page_free(page);
  1307. }
  1308. memory_bm_position_reset(bm);
  1309. safe_highmem_bm = bm;
  1310. return 0;
  1311. }
  1312. /**
  1313. * get_highmem_page_buffer - for given highmem image page find the buffer
  1314. * that suspend_write_next() should set for its caller to write to.
  1315. *
  1316. * If the page is to be saved to its "original" page frame or a copy of
  1317. * the page is to be made in the highmem, @buffer is returned. Otherwise,
  1318. * the copy of the page is to be made in normal memory, so the address of
  1319. * the copy is returned.
  1320. *
  1321. * If @buffer is returned, the caller of suspend_write_next() will write
  1322. * the page's contents to @buffer, so they will have to be copied to the
  1323. * right location on the next call to suspend_write_next() and it is done
  1324. * with the help of copy_last_highmem_page(). For this purpose, if
  1325. * @buffer is returned, @last_highmem page is set to the page to which
  1326. * the data will have to be copied from @buffer.
  1327. */
  1328. static struct page *last_highmem_page;
  1329. static void *
  1330. get_highmem_page_buffer(struct page *page, struct chain_allocator *ca)
  1331. {
  1332. struct highmem_pbe *pbe;
  1333. void *kaddr;
  1334. if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page)) {
  1335. /* We have allocated the "original" page frame and we can
  1336. * use it directly to store the loaded page.
  1337. */
  1338. last_highmem_page = page;
  1339. return buffer;
  1340. }
  1341. /* The "original" page frame has not been allocated and we have to
  1342. * use a "safe" page frame to store the loaded page.
  1343. */
  1344. pbe = chain_alloc(ca, sizeof(struct highmem_pbe));
  1345. if (!pbe) {
  1346. swsusp_free();
  1347. return NULL;
  1348. }
  1349. pbe->orig_page = page;
  1350. if (safe_highmem_pages > 0) {
  1351. struct page *tmp;
  1352. /* Copy of the page will be stored in high memory */
  1353. kaddr = buffer;
  1354. tmp = pfn_to_page(memory_bm_next_pfn(safe_highmem_bm));
  1355. safe_highmem_pages--;
  1356. last_highmem_page = tmp;
  1357. pbe->copy_page = tmp;
  1358. } else {
  1359. /* Copy of the page will be stored in normal memory */
  1360. kaddr = safe_pages_list;
  1361. safe_pages_list = safe_pages_list->next;
  1362. pbe->copy_page = virt_to_page(kaddr);
  1363. }
  1364. pbe->next = highmem_pblist;
  1365. highmem_pblist = pbe;
  1366. return kaddr;
  1367. }
  1368. /**
  1369. * copy_last_highmem_page - copy the contents of a highmem image from
  1370. * @buffer, where the caller of snapshot_write_next() has place them,
  1371. * to the right location represented by @last_highmem_page .
  1372. */
  1373. static void copy_last_highmem_page(void)
  1374. {
  1375. if (last_highmem_page) {
  1376. void *dst;
  1377. dst = kmap_atomic(last_highmem_page, KM_USER0);
  1378. memcpy(dst, buffer, PAGE_SIZE);
  1379. kunmap_atomic(dst, KM_USER0);
  1380. last_highmem_page = NULL;
  1381. }
  1382. }
  1383. static inline int last_highmem_page_copied(void)
  1384. {
  1385. return !last_highmem_page;
  1386. }
  1387. static inline void free_highmem_data(void)
  1388. {
  1389. if (safe_highmem_bm)
  1390. memory_bm_free(safe_highmem_bm, PG_UNSAFE_CLEAR);
  1391. if (buffer)
  1392. free_image_page(buffer, PG_UNSAFE_CLEAR);
  1393. }
  1394. #else
  1395. static inline int get_safe_write_buffer(void) { return 0; }
  1396. static unsigned int
  1397. count_highmem_image_pages(struct memory_bitmap *bm) { return 0; }
  1398. static inline int
  1399. prepare_highmem_image(struct memory_bitmap *bm, unsigned int *nr_highmem_p)
  1400. {
  1401. return 0;
  1402. }
  1403. static inline void *
  1404. get_highmem_page_buffer(struct page *page, struct chain_allocator *ca)
  1405. {
  1406. return NULL;
  1407. }
  1408. static inline void copy_last_highmem_page(void) {}
  1409. static inline int last_highmem_page_copied(void) { return 1; }
  1410. static inline void free_highmem_data(void) {}
  1411. #endif /* CONFIG_HIGHMEM */
  1412. /**
  1413. * prepare_image - use the memory bitmap @bm to mark the pages that will
  1414. * be overwritten in the process of restoring the system memory state
  1415. * from the suspend image ("unsafe" pages) and allocate memory for the
  1416. * image.
  1417. *
  1418. * The idea is to allocate a new memory bitmap first and then allocate
  1419. * as many pages as needed for the image data, but not to assign these
  1420. * pages to specific tasks initially. Instead, we just mark them as
  1421. * allocated and create a lists of "safe" pages that will be used
  1422. * later. On systems with high memory a list of "safe" highmem pages is
  1423. * also created.
  1424. */
  1425. #define PBES_PER_LINKED_PAGE (LINKED_PAGE_DATA_SIZE / sizeof(struct pbe))
  1426. static int
  1427. prepare_image(struct memory_bitmap *new_bm, struct memory_bitmap *bm)
  1428. {
  1429. unsigned int nr_pages, nr_highmem;
  1430. struct linked_page *sp_list, *lp;
  1431. int error;
  1432. /* If there is no highmem, the buffer will not be necessary */
  1433. free_image_page(buffer, PG_UNSAFE_CLEAR);
  1434. buffer = NULL;
  1435. nr_highmem = count_highmem_image_pages(bm);
  1436. error = mark_unsafe_pages(bm);
  1437. if (error)
  1438. goto Free;
  1439. error = memory_bm_create(new_bm, GFP_ATOMIC, PG_SAFE);
  1440. if (error)
  1441. goto Free;
  1442. duplicate_memory_bitmap(new_bm, bm);
  1443. memory_bm_free(bm, PG_UNSAFE_KEEP);
  1444. if (nr_highmem > 0) {
  1445. error = prepare_highmem_image(bm, &nr_highmem);
  1446. if (error)
  1447. goto Free;
  1448. }
  1449. /* Reserve some safe pages for potential later use.
  1450. *
  1451. * NOTE: This way we make sure there will be enough safe pages for the
  1452. * chain_alloc() in get_buffer(). It is a bit wasteful, but
  1453. * nr_copy_pages cannot be greater than 50% of the memory anyway.
  1454. */
  1455. sp_list = NULL;
  1456. /* nr_copy_pages cannot be lesser than allocated_unsafe_pages */
  1457. nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages;
  1458. nr_pages = DIV_ROUND_UP(nr_pages, PBES_PER_LINKED_PAGE);
  1459. while (nr_pages > 0) {
  1460. lp = get_image_page(GFP_ATOMIC, PG_SAFE);
  1461. if (!lp) {
  1462. error = -ENOMEM;
  1463. goto Free;
  1464. }
  1465. lp->next = sp_list;
  1466. sp_list = lp;
  1467. nr_pages--;
  1468. }
  1469. /* Preallocate memory for the image */
  1470. safe_pages_list = NULL;
  1471. nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages;
  1472. while (nr_pages > 0) {
  1473. lp = (struct linked_page *)get_zeroed_page(GFP_ATOMIC);
  1474. if (!lp) {
  1475. error = -ENOMEM;
  1476. goto Free;
  1477. }
  1478. if (!swsusp_page_is_free(virt_to_page(lp))) {
  1479. /* The page is "safe", add it to the list */
  1480. lp->next = safe_pages_list;
  1481. safe_pages_list = lp;
  1482. }
  1483. /* Mark the page as allocated */
  1484. swsusp_set_page_forbidden(virt_to_page(lp));
  1485. swsusp_set_page_free(virt_to_page(lp));
  1486. nr_pages--;
  1487. }
  1488. /* Free the reserved safe pages so that chain_alloc() can use them */
  1489. while (sp_list) {
  1490. lp = sp_list->next;
  1491. free_image_page(sp_list, PG_UNSAFE_CLEAR);
  1492. sp_list = lp;
  1493. }
  1494. return 0;
  1495. Free:
  1496. swsusp_free();
  1497. return error;
  1498. }
  1499. /**
  1500. * get_buffer - compute the address that snapshot_write_next() should
  1501. * set for its caller to write to.
  1502. */
  1503. static void *get_buffer(struct memory_bitmap *bm, struct chain_allocator *ca)
  1504. {
  1505. struct pbe *pbe;
  1506. struct page *page = pfn_to_page(memory_bm_next_pfn(bm));
  1507. if (PageHighMem(page))
  1508. return get_highmem_page_buffer(page, ca);
  1509. if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page))
  1510. /* We have allocated the "original" page frame and we can
  1511. * use it directly to store the loaded page.
  1512. */
  1513. return page_address(page);
  1514. /* The "original" page frame has not been allocated and we have to
  1515. * use a "safe" page frame to store the loaded page.
  1516. */
  1517. pbe = chain_alloc(ca, sizeof(struct pbe));
  1518. if (!pbe) {
  1519. swsusp_free();
  1520. return NULL;
  1521. }
  1522. pbe->orig_address = page_address(page);
  1523. pbe->address = safe_pages_list;
  1524. safe_pages_list = safe_pages_list->next;
  1525. pbe->next = restore_pblist;
  1526. restore_pblist = pbe;
  1527. return pbe->address;
  1528. }
  1529. /**
  1530. * snapshot_write_next - used for writing the system memory snapshot.
  1531. *
  1532. * On the first call to it @handle should point to a zeroed
  1533. * snapshot_handle structure. The structure gets updated and a pointer
  1534. * to it should be passed to this function every next time.
  1535. *
  1536. * The @count parameter should contain the number of bytes the caller
  1537. * wants to write to the image. It must not be zero.
  1538. *
  1539. * On success the function returns a positive number. Then, the caller
  1540. * is allowed to write up to the returned number of bytes to the memory
  1541. * location computed by the data_of() macro. The number returned
  1542. * may be smaller than @count, but this only happens if the write would
  1543. * cross a page boundary otherwise.
  1544. *
  1545. * The function returns 0 to indicate the "end of file" condition,
  1546. * and a negative number is returned on error. In such cases the
  1547. * structure pointed to by @handle is not updated and should not be used
  1548. * any more.
  1549. */
  1550. int snapshot_write_next(struct snapshot_handle *handle, size_t count)
  1551. {
  1552. static struct chain_allocator ca;
  1553. int error = 0;
  1554. /* Check if we have already loaded the entire image */
  1555. if (handle->prev && handle->cur > nr_meta_pages + nr_copy_pages)
  1556. return 0;
  1557. if (handle->offset == 0) {
  1558. if (!buffer)
  1559. /* This makes the buffer be freed by swsusp_free() */
  1560. buffer = get_image_page(GFP_ATOMIC, PG_ANY);
  1561. if (!buffer)
  1562. return -ENOMEM;
  1563. handle->buffer = buffer;
  1564. }
  1565. handle->sync_read = 1;
  1566. if (handle->prev < handle->cur) {
  1567. if (handle->prev == 0) {
  1568. error = load_header(buffer);
  1569. if (error)
  1570. return error;
  1571. error = memory_bm_create(&copy_bm, GFP_ATOMIC, PG_ANY);
  1572. if (error)
  1573. return error;
  1574. } else if (handle->prev <= nr_meta_pages) {
  1575. unpack_orig_pfns(buffer, &copy_bm);
  1576. if (handle->prev == nr_meta_pages) {
  1577. error = prepare_image(&orig_bm, &copy_bm);
  1578. if (error)
  1579. return error;
  1580. chain_init(&ca, GFP_ATOMIC, PG_SAFE);
  1581. memory_bm_position_reset(&orig_bm);
  1582. restore_pblist = NULL;
  1583. handle->buffer = get_buffer(&orig_bm, &ca);
  1584. handle->sync_read = 0;
  1585. if (!handle->buffer)
  1586. return -ENOMEM;
  1587. }
  1588. } else {
  1589. copy_last_highmem_page();
  1590. handle->buffer = get_buffer(&orig_bm, &ca);
  1591. if (handle->buffer != buffer)
  1592. handle->sync_read = 0;
  1593. }
  1594. handle->prev = handle->cur;
  1595. }
  1596. handle->buf_offset = handle->cur_offset;
  1597. if (handle->cur_offset + count >= PAGE_SIZE) {
  1598. count = PAGE_SIZE - handle->cur_offset;
  1599. handle->cur_offset = 0;
  1600. handle->cur++;
  1601. } else {
  1602. handle->cur_offset += count;
  1603. }
  1604. handle->offset += count;
  1605. return count;
  1606. }
  1607. /**
  1608. * snapshot_write_finalize - must be called after the last call to
  1609. * snapshot_write_next() in case the last page in the image happens
  1610. * to be a highmem page and its contents should be stored in the
  1611. * highmem. Additionally, it releases the memory that will not be
  1612. * used any more.
  1613. */
  1614. void snapshot_write_finalize(struct snapshot_handle *handle)
  1615. {
  1616. copy_last_highmem_page();
  1617. /* Free only if we have loaded the image entirely */
  1618. if (handle->prev && handle->cur > nr_meta_pages + nr_copy_pages) {
  1619. memory_bm_free(&orig_bm, PG_UNSAFE_CLEAR);
  1620. free_highmem_data();
  1621. }
  1622. }
  1623. int snapshot_image_loaded(struct snapshot_handle *handle)
  1624. {
  1625. return !(!nr_copy_pages || !last_highmem_page_copied() ||
  1626. handle->cur <= nr_meta_pages + nr_copy_pages);
  1627. }
  1628. #ifdef CONFIG_HIGHMEM
  1629. /* Assumes that @buf is ready and points to a "safe" page */
  1630. static inline void
  1631. swap_two_pages_data(struct page *p1, struct page *p2, void *buf)
  1632. {
  1633. void *kaddr1, *kaddr2;
  1634. kaddr1 = kmap_atomic(p1, KM_USER0);
  1635. kaddr2 = kmap_atomic(p2, KM_USER1);
  1636. memcpy(buf, kaddr1, PAGE_SIZE);
  1637. memcpy(kaddr1, kaddr2, PAGE_SIZE);
  1638. memcpy(kaddr2, buf, PAGE_SIZE);
  1639. kunmap_atomic(kaddr1, KM_USER0);
  1640. kunmap_atomic(kaddr2, KM_USER1);
  1641. }
  1642. /**
  1643. * restore_highmem - for each highmem page that was allocated before
  1644. * the suspend and included in the suspend image, and also has been
  1645. * allocated by the "resume" kernel swap its current (ie. "before
  1646. * resume") contents with the previous (ie. "before suspend") one.
  1647. *
  1648. * If the resume eventually fails, we can call this function once
  1649. * again and restore the "before resume" highmem state.
  1650. */
  1651. int restore_highmem(void)
  1652. {
  1653. struct highmem_pbe *pbe = highmem_pblist;
  1654. void *buf;
  1655. if (!pbe)
  1656. return 0;
  1657. buf = get_image_page(GFP_ATOMIC, PG_SAFE);
  1658. if (!buf)
  1659. return -ENOMEM;
  1660. while (pbe) {
  1661. swap_two_pages_data(pbe->copy_page, pbe->orig_page, buf);
  1662. pbe = pbe->next;
  1663. }
  1664. free_image_page(buf, PG_UNSAFE_CLEAR);
  1665. return 0;
  1666. }
  1667. #endif /* CONFIG_HIGHMEM */