mmu.c 110 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * This module enables machines with Intel VT-x extensions to run virtual
  5. * machines without emulation or binary translation.
  6. *
  7. * MMU support
  8. *
  9. * Copyright (C) 2006 Qumranet, Inc.
  10. * Copyright 2010 Red Hat, Inc. and/or its affiliates.
  11. *
  12. * Authors:
  13. * Yaniv Kamay <yaniv@qumranet.com>
  14. * Avi Kivity <avi@qumranet.com>
  15. *
  16. * This work is licensed under the terms of the GNU GPL, version 2. See
  17. * the COPYING file in the top-level directory.
  18. *
  19. */
  20. #include "irq.h"
  21. #include "mmu.h"
  22. #include "x86.h"
  23. #include "kvm_cache_regs.h"
  24. #include <linux/kvm_host.h>
  25. #include <linux/types.h>
  26. #include <linux/string.h>
  27. #include <linux/mm.h>
  28. #include <linux/highmem.h>
  29. #include <linux/module.h>
  30. #include <linux/swap.h>
  31. #include <linux/hugetlb.h>
  32. #include <linux/compiler.h>
  33. #include <linux/srcu.h>
  34. #include <linux/slab.h>
  35. #include <linux/uaccess.h>
  36. #include <asm/page.h>
  37. #include <asm/cmpxchg.h>
  38. #include <asm/io.h>
  39. #include <asm/vmx.h>
  40. /*
  41. * When setting this variable to true it enables Two-Dimensional-Paging
  42. * where the hardware walks 2 page tables:
  43. * 1. the guest-virtual to guest-physical
  44. * 2. while doing 1. it walks guest-physical to host-physical
  45. * If the hardware supports that we don't need to do shadow paging.
  46. */
  47. bool tdp_enabled = false;
  48. enum {
  49. AUDIT_PRE_PAGE_FAULT,
  50. AUDIT_POST_PAGE_FAULT,
  51. AUDIT_PRE_PTE_WRITE,
  52. AUDIT_POST_PTE_WRITE,
  53. AUDIT_PRE_SYNC,
  54. AUDIT_POST_SYNC
  55. };
  56. #undef MMU_DEBUG
  57. #ifdef MMU_DEBUG
  58. #define pgprintk(x...) do { if (dbg) printk(x); } while (0)
  59. #define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
  60. #else
  61. #define pgprintk(x...) do { } while (0)
  62. #define rmap_printk(x...) do { } while (0)
  63. #endif
  64. #ifdef MMU_DEBUG
  65. static bool dbg = 0;
  66. module_param(dbg, bool, 0644);
  67. #endif
  68. #ifndef MMU_DEBUG
  69. #define ASSERT(x) do { } while (0)
  70. #else
  71. #define ASSERT(x) \
  72. if (!(x)) { \
  73. printk(KERN_WARNING "assertion failed %s:%d: %s\n", \
  74. __FILE__, __LINE__, #x); \
  75. }
  76. #endif
  77. #define PTE_PREFETCH_NUM 8
  78. #define PT_FIRST_AVAIL_BITS_SHIFT 10
  79. #define PT64_SECOND_AVAIL_BITS_SHIFT 52
  80. #define PT64_LEVEL_BITS 9
  81. #define PT64_LEVEL_SHIFT(level) \
  82. (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
  83. #define PT64_INDEX(address, level)\
  84. (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
  85. #define PT32_LEVEL_BITS 10
  86. #define PT32_LEVEL_SHIFT(level) \
  87. (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
  88. #define PT32_LVL_OFFSET_MASK(level) \
  89. (PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
  90. * PT32_LEVEL_BITS))) - 1))
  91. #define PT32_INDEX(address, level)\
  92. (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
  93. #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
  94. #define PT64_DIR_BASE_ADDR_MASK \
  95. (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
  96. #define PT64_LVL_ADDR_MASK(level) \
  97. (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
  98. * PT64_LEVEL_BITS))) - 1))
  99. #define PT64_LVL_OFFSET_MASK(level) \
  100. (PT64_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
  101. * PT64_LEVEL_BITS))) - 1))
  102. #define PT32_BASE_ADDR_MASK PAGE_MASK
  103. #define PT32_DIR_BASE_ADDR_MASK \
  104. (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
  105. #define PT32_LVL_ADDR_MASK(level) \
  106. (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
  107. * PT32_LEVEL_BITS))) - 1))
  108. #define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK \
  109. | PT64_NX_MASK)
  110. #define ACC_EXEC_MASK 1
  111. #define ACC_WRITE_MASK PT_WRITABLE_MASK
  112. #define ACC_USER_MASK PT_USER_MASK
  113. #define ACC_ALL (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
  114. #include <trace/events/kvm.h>
  115. #define CREATE_TRACE_POINTS
  116. #include "mmutrace.h"
  117. #define SPTE_HOST_WRITEABLE (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
  118. #define SPTE_MMU_WRITEABLE (1ULL << (PT_FIRST_AVAIL_BITS_SHIFT + 1))
  119. #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)
  120. /* make pte_list_desc fit well in cache line */
  121. #define PTE_LIST_EXT 3
  122. struct pte_list_desc {
  123. u64 *sptes[PTE_LIST_EXT];
  124. struct pte_list_desc *more;
  125. };
  126. struct kvm_shadow_walk_iterator {
  127. u64 addr;
  128. hpa_t shadow_addr;
  129. u64 *sptep;
  130. int level;
  131. unsigned index;
  132. };
  133. #define for_each_shadow_entry(_vcpu, _addr, _walker) \
  134. for (shadow_walk_init(&(_walker), _vcpu, _addr); \
  135. shadow_walk_okay(&(_walker)); \
  136. shadow_walk_next(&(_walker)))
  137. #define for_each_shadow_entry_lockless(_vcpu, _addr, _walker, spte) \
  138. for (shadow_walk_init(&(_walker), _vcpu, _addr); \
  139. shadow_walk_okay(&(_walker)) && \
  140. ({ spte = mmu_spte_get_lockless(_walker.sptep); 1; }); \
  141. __shadow_walk_next(&(_walker), spte))
  142. static struct kmem_cache *pte_list_desc_cache;
  143. static struct kmem_cache *mmu_page_header_cache;
  144. static struct percpu_counter kvm_total_used_mmu_pages;
  145. static u64 __read_mostly shadow_nx_mask;
  146. static u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
  147. static u64 __read_mostly shadow_user_mask;
  148. static u64 __read_mostly shadow_accessed_mask;
  149. static u64 __read_mostly shadow_dirty_mask;
  150. static u64 __read_mostly shadow_mmio_mask;
  151. static void mmu_spte_set(u64 *sptep, u64 spte);
  152. static void mmu_free_roots(struct kvm_vcpu *vcpu);
  153. void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask)
  154. {
  155. shadow_mmio_mask = mmio_mask;
  156. }
  157. EXPORT_SYMBOL_GPL(kvm_mmu_set_mmio_spte_mask);
  158. /*
  159. * spte bits of bit 3 ~ bit 11 are used as low 9 bits of generation number,
  160. * the bits of bits 52 ~ bit 61 are used as high 10 bits of generation
  161. * number.
  162. */
  163. #define MMIO_SPTE_GEN_LOW_SHIFT 3
  164. #define MMIO_SPTE_GEN_HIGH_SHIFT 52
  165. #define MMIO_GEN_SHIFT 19
  166. #define MMIO_GEN_LOW_SHIFT 9
  167. #define MMIO_GEN_LOW_MASK ((1 << MMIO_GEN_LOW_SHIFT) - 1)
  168. #define MMIO_GEN_MASK ((1 << MMIO_GEN_SHIFT) - 1)
  169. #define MMIO_MAX_GEN ((1 << MMIO_GEN_SHIFT) - 1)
  170. static u64 generation_mmio_spte_mask(unsigned int gen)
  171. {
  172. u64 mask;
  173. WARN_ON(gen > MMIO_MAX_GEN);
  174. mask = (gen & MMIO_GEN_LOW_MASK) << MMIO_SPTE_GEN_LOW_SHIFT;
  175. mask |= ((u64)gen >> MMIO_GEN_LOW_SHIFT) << MMIO_SPTE_GEN_HIGH_SHIFT;
  176. return mask;
  177. }
  178. static unsigned int get_mmio_spte_generation(u64 spte)
  179. {
  180. unsigned int gen;
  181. spte &= ~shadow_mmio_mask;
  182. gen = (spte >> MMIO_SPTE_GEN_LOW_SHIFT) & MMIO_GEN_LOW_MASK;
  183. gen |= (spte >> MMIO_SPTE_GEN_HIGH_SHIFT) << MMIO_GEN_LOW_SHIFT;
  184. return gen;
  185. }
  186. static unsigned int kvm_current_mmio_generation(struct kvm *kvm)
  187. {
  188. /*
  189. * Init kvm generation close to MMIO_MAX_GEN to easily test the
  190. * code of handling generation number wrap-around.
  191. */
  192. return (kvm_memslots(kvm)->generation +
  193. MMIO_MAX_GEN - 150) & MMIO_GEN_MASK;
  194. }
  195. static void mark_mmio_spte(struct kvm *kvm, u64 *sptep, u64 gfn,
  196. unsigned access)
  197. {
  198. unsigned int gen = kvm_current_mmio_generation(kvm);
  199. u64 mask = generation_mmio_spte_mask(gen);
  200. access &= ACC_WRITE_MASK | ACC_USER_MASK;
  201. mask |= shadow_mmio_mask | access | gfn << PAGE_SHIFT;
  202. trace_mark_mmio_spte(sptep, gfn, access, gen);
  203. mmu_spte_set(sptep, mask);
  204. }
  205. static bool is_mmio_spte(u64 spte)
  206. {
  207. return (spte & shadow_mmio_mask) == shadow_mmio_mask;
  208. }
  209. static gfn_t get_mmio_spte_gfn(u64 spte)
  210. {
  211. u64 mask = generation_mmio_spte_mask(MMIO_MAX_GEN) | shadow_mmio_mask;
  212. return (spte & ~mask) >> PAGE_SHIFT;
  213. }
  214. static unsigned get_mmio_spte_access(u64 spte)
  215. {
  216. u64 mask = generation_mmio_spte_mask(MMIO_MAX_GEN) | shadow_mmio_mask;
  217. return (spte & ~mask) & ~PAGE_MASK;
  218. }
  219. static bool set_mmio_spte(struct kvm *kvm, u64 *sptep, gfn_t gfn,
  220. pfn_t pfn, unsigned access)
  221. {
  222. if (unlikely(is_noslot_pfn(pfn))) {
  223. mark_mmio_spte(kvm, sptep, gfn, access);
  224. return true;
  225. }
  226. return false;
  227. }
  228. static bool check_mmio_spte(struct kvm *kvm, u64 spte)
  229. {
  230. unsigned int kvm_gen, spte_gen;
  231. kvm_gen = kvm_current_mmio_generation(kvm);
  232. spte_gen = get_mmio_spte_generation(spte);
  233. trace_check_mmio_spte(spte, kvm_gen, spte_gen);
  234. return likely(kvm_gen == spte_gen);
  235. }
  236. static inline u64 rsvd_bits(int s, int e)
  237. {
  238. return ((1ULL << (e - s + 1)) - 1) << s;
  239. }
  240. void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
  241. u64 dirty_mask, u64 nx_mask, u64 x_mask)
  242. {
  243. shadow_user_mask = user_mask;
  244. shadow_accessed_mask = accessed_mask;
  245. shadow_dirty_mask = dirty_mask;
  246. shadow_nx_mask = nx_mask;
  247. shadow_x_mask = x_mask;
  248. }
  249. EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);
  250. static int is_cpuid_PSE36(void)
  251. {
  252. return 1;
  253. }
  254. static int is_nx(struct kvm_vcpu *vcpu)
  255. {
  256. return vcpu->arch.efer & EFER_NX;
  257. }
  258. static int is_shadow_present_pte(u64 pte)
  259. {
  260. return pte & PT_PRESENT_MASK && !is_mmio_spte(pte);
  261. }
  262. static int is_large_pte(u64 pte)
  263. {
  264. return pte & PT_PAGE_SIZE_MASK;
  265. }
  266. static int is_dirty_gpte(unsigned long pte)
  267. {
  268. return pte & PT_DIRTY_MASK;
  269. }
  270. static int is_rmap_spte(u64 pte)
  271. {
  272. return is_shadow_present_pte(pte);
  273. }
  274. static int is_last_spte(u64 pte, int level)
  275. {
  276. if (level == PT_PAGE_TABLE_LEVEL)
  277. return 1;
  278. if (is_large_pte(pte))
  279. return 1;
  280. return 0;
  281. }
  282. static pfn_t spte_to_pfn(u64 pte)
  283. {
  284. return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
  285. }
  286. static gfn_t pse36_gfn_delta(u32 gpte)
  287. {
  288. int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;
  289. return (gpte & PT32_DIR_PSE36_MASK) << shift;
  290. }
  291. #ifdef CONFIG_X86_64
  292. static void __set_spte(u64 *sptep, u64 spte)
  293. {
  294. *sptep = spte;
  295. }
  296. static void __update_clear_spte_fast(u64 *sptep, u64 spte)
  297. {
  298. *sptep = spte;
  299. }
  300. static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
  301. {
  302. return xchg(sptep, spte);
  303. }
  304. static u64 __get_spte_lockless(u64 *sptep)
  305. {
  306. return ACCESS_ONCE(*sptep);
  307. }
  308. static bool __check_direct_spte_mmio_pf(u64 spte)
  309. {
  310. /* It is valid if the spte is zapped. */
  311. return spte == 0ull;
  312. }
  313. #else
  314. union split_spte {
  315. struct {
  316. u32 spte_low;
  317. u32 spte_high;
  318. };
  319. u64 spte;
  320. };
  321. static void count_spte_clear(u64 *sptep, u64 spte)
  322. {
  323. struct kvm_mmu_page *sp = page_header(__pa(sptep));
  324. if (is_shadow_present_pte(spte))
  325. return;
  326. /* Ensure the spte is completely set before we increase the count */
  327. smp_wmb();
  328. sp->clear_spte_count++;
  329. }
  330. static void __set_spte(u64 *sptep, u64 spte)
  331. {
  332. union split_spte *ssptep, sspte;
  333. ssptep = (union split_spte *)sptep;
  334. sspte = (union split_spte)spte;
  335. ssptep->spte_high = sspte.spte_high;
  336. /*
  337. * If we map the spte from nonpresent to present, We should store
  338. * the high bits firstly, then set present bit, so cpu can not
  339. * fetch this spte while we are setting the spte.
  340. */
  341. smp_wmb();
  342. ssptep->spte_low = sspte.spte_low;
  343. }
  344. static void __update_clear_spte_fast(u64 *sptep, u64 spte)
  345. {
  346. union split_spte *ssptep, sspte;
  347. ssptep = (union split_spte *)sptep;
  348. sspte = (union split_spte)spte;
  349. ssptep->spte_low = sspte.spte_low;
  350. /*
  351. * If we map the spte from present to nonpresent, we should clear
  352. * present bit firstly to avoid vcpu fetch the old high bits.
  353. */
  354. smp_wmb();
  355. ssptep->spte_high = sspte.spte_high;
  356. count_spte_clear(sptep, spte);
  357. }
  358. static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
  359. {
  360. union split_spte *ssptep, sspte, orig;
  361. ssptep = (union split_spte *)sptep;
  362. sspte = (union split_spte)spte;
  363. /* xchg acts as a barrier before the setting of the high bits */
  364. orig.spte_low = xchg(&ssptep->spte_low, sspte.spte_low);
  365. orig.spte_high = ssptep->spte_high;
  366. ssptep->spte_high = sspte.spte_high;
  367. count_spte_clear(sptep, spte);
  368. return orig.spte;
  369. }
  370. /*
  371. * The idea using the light way get the spte on x86_32 guest is from
  372. * gup_get_pte(arch/x86/mm/gup.c).
  373. *
  374. * An spte tlb flush may be pending, because kvm_set_pte_rmapp
  375. * coalesces them and we are running out of the MMU lock. Therefore
  376. * we need to protect against in-progress updates of the spte.
  377. *
  378. * Reading the spte while an update is in progress may get the old value
  379. * for the high part of the spte. The race is fine for a present->non-present
  380. * change (because the high part of the spte is ignored for non-present spte),
  381. * but for a present->present change we must reread the spte.
  382. *
  383. * All such changes are done in two steps (present->non-present and
  384. * non-present->present), hence it is enough to count the number of
  385. * present->non-present updates: if it changed while reading the spte,
  386. * we might have hit the race. This is done using clear_spte_count.
  387. */
  388. static u64 __get_spte_lockless(u64 *sptep)
  389. {
  390. struct kvm_mmu_page *sp = page_header(__pa(sptep));
  391. union split_spte spte, *orig = (union split_spte *)sptep;
  392. int count;
  393. retry:
  394. count = sp->clear_spte_count;
  395. smp_rmb();
  396. spte.spte_low = orig->spte_low;
  397. smp_rmb();
  398. spte.spte_high = orig->spte_high;
  399. smp_rmb();
  400. if (unlikely(spte.spte_low != orig->spte_low ||
  401. count != sp->clear_spte_count))
  402. goto retry;
  403. return spte.spte;
  404. }
  405. static bool __check_direct_spte_mmio_pf(u64 spte)
  406. {
  407. union split_spte sspte = (union split_spte)spte;
  408. u32 high_mmio_mask = shadow_mmio_mask >> 32;
  409. /* It is valid if the spte is zapped. */
  410. if (spte == 0ull)
  411. return true;
  412. /* It is valid if the spte is being zapped. */
  413. if (sspte.spte_low == 0ull &&
  414. (sspte.spte_high & high_mmio_mask) == high_mmio_mask)
  415. return true;
  416. return false;
  417. }
  418. #endif
  419. static bool spte_is_locklessly_modifiable(u64 spte)
  420. {
  421. return (spte & (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE)) ==
  422. (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE);
  423. }
  424. static bool spte_has_volatile_bits(u64 spte)
  425. {
  426. /*
  427. * Always atomicly update spte if it can be updated
  428. * out of mmu-lock, it can ensure dirty bit is not lost,
  429. * also, it can help us to get a stable is_writable_pte()
  430. * to ensure tlb flush is not missed.
  431. */
  432. if (spte_is_locklessly_modifiable(spte))
  433. return true;
  434. if (!shadow_accessed_mask)
  435. return false;
  436. if (!is_shadow_present_pte(spte))
  437. return false;
  438. if ((spte & shadow_accessed_mask) &&
  439. (!is_writable_pte(spte) || (spte & shadow_dirty_mask)))
  440. return false;
  441. return true;
  442. }
  443. static bool spte_is_bit_cleared(u64 old_spte, u64 new_spte, u64 bit_mask)
  444. {
  445. return (old_spte & bit_mask) && !(new_spte & bit_mask);
  446. }
  447. /* Rules for using mmu_spte_set:
  448. * Set the sptep from nonpresent to present.
  449. * Note: the sptep being assigned *must* be either not present
  450. * or in a state where the hardware will not attempt to update
  451. * the spte.
  452. */
  453. static void mmu_spte_set(u64 *sptep, u64 new_spte)
  454. {
  455. WARN_ON(is_shadow_present_pte(*sptep));
  456. __set_spte(sptep, new_spte);
  457. }
  458. /* Rules for using mmu_spte_update:
  459. * Update the state bits, it means the mapped pfn is not changged.
  460. *
  461. * Whenever we overwrite a writable spte with a read-only one we
  462. * should flush remote TLBs. Otherwise rmap_write_protect
  463. * will find a read-only spte, even though the writable spte
  464. * might be cached on a CPU's TLB, the return value indicates this
  465. * case.
  466. */
  467. static bool mmu_spte_update(u64 *sptep, u64 new_spte)
  468. {
  469. u64 old_spte = *sptep;
  470. bool ret = false;
  471. WARN_ON(!is_rmap_spte(new_spte));
  472. if (!is_shadow_present_pte(old_spte)) {
  473. mmu_spte_set(sptep, new_spte);
  474. return ret;
  475. }
  476. if (!spte_has_volatile_bits(old_spte))
  477. __update_clear_spte_fast(sptep, new_spte);
  478. else
  479. old_spte = __update_clear_spte_slow(sptep, new_spte);
  480. /*
  481. * For the spte updated out of mmu-lock is safe, since
  482. * we always atomicly update it, see the comments in
  483. * spte_has_volatile_bits().
  484. */
  485. if (is_writable_pte(old_spte) && !is_writable_pte(new_spte))
  486. ret = true;
  487. if (!shadow_accessed_mask)
  488. return ret;
  489. if (spte_is_bit_cleared(old_spte, new_spte, shadow_accessed_mask))
  490. kvm_set_pfn_accessed(spte_to_pfn(old_spte));
  491. if (spte_is_bit_cleared(old_spte, new_spte, shadow_dirty_mask))
  492. kvm_set_pfn_dirty(spte_to_pfn(old_spte));
  493. return ret;
  494. }
  495. /*
  496. * Rules for using mmu_spte_clear_track_bits:
  497. * It sets the sptep from present to nonpresent, and track the
  498. * state bits, it is used to clear the last level sptep.
  499. */
  500. static int mmu_spte_clear_track_bits(u64 *sptep)
  501. {
  502. pfn_t pfn;
  503. u64 old_spte = *sptep;
  504. if (!spte_has_volatile_bits(old_spte))
  505. __update_clear_spte_fast(sptep, 0ull);
  506. else
  507. old_spte = __update_clear_spte_slow(sptep, 0ull);
  508. if (!is_rmap_spte(old_spte))
  509. return 0;
  510. pfn = spte_to_pfn(old_spte);
  511. /*
  512. * KVM does not hold the refcount of the page used by
  513. * kvm mmu, before reclaiming the page, we should
  514. * unmap it from mmu first.
  515. */
  516. WARN_ON(!kvm_is_mmio_pfn(pfn) && !page_count(pfn_to_page(pfn)));
  517. if (!shadow_accessed_mask || old_spte & shadow_accessed_mask)
  518. kvm_set_pfn_accessed(pfn);
  519. if (!shadow_dirty_mask || (old_spte & shadow_dirty_mask))
  520. kvm_set_pfn_dirty(pfn);
  521. return 1;
  522. }
  523. /*
  524. * Rules for using mmu_spte_clear_no_track:
  525. * Directly clear spte without caring the state bits of sptep,
  526. * it is used to set the upper level spte.
  527. */
  528. static void mmu_spte_clear_no_track(u64 *sptep)
  529. {
  530. __update_clear_spte_fast(sptep, 0ull);
  531. }
  532. static u64 mmu_spte_get_lockless(u64 *sptep)
  533. {
  534. return __get_spte_lockless(sptep);
  535. }
  536. static void walk_shadow_page_lockless_begin(struct kvm_vcpu *vcpu)
  537. {
  538. /*
  539. * Prevent page table teardown by making any free-er wait during
  540. * kvm_flush_remote_tlbs() IPI to all active vcpus.
  541. */
  542. local_irq_disable();
  543. vcpu->mode = READING_SHADOW_PAGE_TABLES;
  544. /*
  545. * Make sure a following spte read is not reordered ahead of the write
  546. * to vcpu->mode.
  547. */
  548. smp_mb();
  549. }
  550. static void walk_shadow_page_lockless_end(struct kvm_vcpu *vcpu)
  551. {
  552. /*
  553. * Make sure the write to vcpu->mode is not reordered in front of
  554. * reads to sptes. If it does, kvm_commit_zap_page() can see us
  555. * OUTSIDE_GUEST_MODE and proceed to free the shadow page table.
  556. */
  557. smp_mb();
  558. vcpu->mode = OUTSIDE_GUEST_MODE;
  559. local_irq_enable();
  560. }
  561. static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
  562. struct kmem_cache *base_cache, int min)
  563. {
  564. void *obj;
  565. if (cache->nobjs >= min)
  566. return 0;
  567. while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
  568. obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
  569. if (!obj)
  570. return -ENOMEM;
  571. cache->objects[cache->nobjs++] = obj;
  572. }
  573. return 0;
  574. }
  575. static int mmu_memory_cache_free_objects(struct kvm_mmu_memory_cache *cache)
  576. {
  577. return cache->nobjs;
  578. }
  579. static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc,
  580. struct kmem_cache *cache)
  581. {
  582. while (mc->nobjs)
  583. kmem_cache_free(cache, mc->objects[--mc->nobjs]);
  584. }
  585. static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
  586. int min)
  587. {
  588. void *page;
  589. if (cache->nobjs >= min)
  590. return 0;
  591. while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
  592. page = (void *)__get_free_page(GFP_KERNEL);
  593. if (!page)
  594. return -ENOMEM;
  595. cache->objects[cache->nobjs++] = page;
  596. }
  597. return 0;
  598. }
  599. static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
  600. {
  601. while (mc->nobjs)
  602. free_page((unsigned long)mc->objects[--mc->nobjs]);
  603. }
  604. static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
  605. {
  606. int r;
  607. r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
  608. pte_list_desc_cache, 8 + PTE_PREFETCH_NUM);
  609. if (r)
  610. goto out;
  611. r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
  612. if (r)
  613. goto out;
  614. r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
  615. mmu_page_header_cache, 4);
  616. out:
  617. return r;
  618. }
  619. static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
  620. {
  621. mmu_free_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
  622. pte_list_desc_cache);
  623. mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
  624. mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache,
  625. mmu_page_header_cache);
  626. }
  627. static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
  628. {
  629. void *p;
  630. BUG_ON(!mc->nobjs);
  631. p = mc->objects[--mc->nobjs];
  632. return p;
  633. }
  634. static struct pte_list_desc *mmu_alloc_pte_list_desc(struct kvm_vcpu *vcpu)
  635. {
  636. return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_list_desc_cache);
  637. }
  638. static void mmu_free_pte_list_desc(struct pte_list_desc *pte_list_desc)
  639. {
  640. kmem_cache_free(pte_list_desc_cache, pte_list_desc);
  641. }
  642. static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index)
  643. {
  644. if (!sp->role.direct)
  645. return sp->gfns[index];
  646. return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS));
  647. }
  648. static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn)
  649. {
  650. if (sp->role.direct)
  651. BUG_ON(gfn != kvm_mmu_page_get_gfn(sp, index));
  652. else
  653. sp->gfns[index] = gfn;
  654. }
  655. /*
  656. * Return the pointer to the large page information for a given gfn,
  657. * handling slots that are not large page aligned.
  658. */
  659. static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn,
  660. struct kvm_memory_slot *slot,
  661. int level)
  662. {
  663. unsigned long idx;
  664. idx = gfn_to_index(gfn, slot->base_gfn, level);
  665. return &slot->arch.lpage_info[level - 2][idx];
  666. }
  667. static void account_shadowed(struct kvm *kvm, gfn_t gfn)
  668. {
  669. struct kvm_memory_slot *slot;
  670. struct kvm_lpage_info *linfo;
  671. int i;
  672. slot = gfn_to_memslot(kvm, gfn);
  673. for (i = PT_DIRECTORY_LEVEL;
  674. i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
  675. linfo = lpage_info_slot(gfn, slot, i);
  676. linfo->write_count += 1;
  677. }
  678. kvm->arch.indirect_shadow_pages++;
  679. }
  680. static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn)
  681. {
  682. struct kvm_memory_slot *slot;
  683. struct kvm_lpage_info *linfo;
  684. int i;
  685. slot = gfn_to_memslot(kvm, gfn);
  686. for (i = PT_DIRECTORY_LEVEL;
  687. i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
  688. linfo = lpage_info_slot(gfn, slot, i);
  689. linfo->write_count -= 1;
  690. WARN_ON(linfo->write_count < 0);
  691. }
  692. kvm->arch.indirect_shadow_pages--;
  693. }
  694. static int has_wrprotected_page(struct kvm *kvm,
  695. gfn_t gfn,
  696. int level)
  697. {
  698. struct kvm_memory_slot *slot;
  699. struct kvm_lpage_info *linfo;
  700. slot = gfn_to_memslot(kvm, gfn);
  701. if (slot) {
  702. linfo = lpage_info_slot(gfn, slot, level);
  703. return linfo->write_count;
  704. }
  705. return 1;
  706. }
  707. static int host_mapping_level(struct kvm *kvm, gfn_t gfn)
  708. {
  709. unsigned long page_size;
  710. int i, ret = 0;
  711. page_size = kvm_host_page_size(kvm, gfn);
  712. for (i = PT_PAGE_TABLE_LEVEL;
  713. i < (PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES); ++i) {
  714. if (page_size >= KVM_HPAGE_SIZE(i))
  715. ret = i;
  716. else
  717. break;
  718. }
  719. return ret;
  720. }
  721. static struct kvm_memory_slot *
  722. gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t gfn,
  723. bool no_dirty_log)
  724. {
  725. struct kvm_memory_slot *slot;
  726. slot = gfn_to_memslot(vcpu->kvm, gfn);
  727. if (!slot || slot->flags & KVM_MEMSLOT_INVALID ||
  728. (no_dirty_log && slot->dirty_bitmap))
  729. slot = NULL;
  730. return slot;
  731. }
  732. static bool mapping_level_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t large_gfn)
  733. {
  734. return !gfn_to_memslot_dirty_bitmap(vcpu, large_gfn, true);
  735. }
  736. static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn)
  737. {
  738. int host_level, level, max_level;
  739. host_level = host_mapping_level(vcpu->kvm, large_gfn);
  740. if (host_level == PT_PAGE_TABLE_LEVEL)
  741. return host_level;
  742. max_level = min(kvm_x86_ops->get_lpage_level(), host_level);
  743. for (level = PT_DIRECTORY_LEVEL; level <= max_level; ++level)
  744. if (has_wrprotected_page(vcpu->kvm, large_gfn, level))
  745. break;
  746. return level - 1;
  747. }
  748. /*
  749. * Pte mapping structures:
  750. *
  751. * If pte_list bit zero is zero, then pte_list point to the spte.
  752. *
  753. * If pte_list bit zero is one, (then pte_list & ~1) points to a struct
  754. * pte_list_desc containing more mappings.
  755. *
  756. * Returns the number of pte entries before the spte was added or zero if
  757. * the spte was not added.
  758. *
  759. */
  760. static int pte_list_add(struct kvm_vcpu *vcpu, u64 *spte,
  761. unsigned long *pte_list)
  762. {
  763. struct pte_list_desc *desc;
  764. int i, count = 0;
  765. if (!*pte_list) {
  766. rmap_printk("pte_list_add: %p %llx 0->1\n", spte, *spte);
  767. *pte_list = (unsigned long)spte;
  768. } else if (!(*pte_list & 1)) {
  769. rmap_printk("pte_list_add: %p %llx 1->many\n", spte, *spte);
  770. desc = mmu_alloc_pte_list_desc(vcpu);
  771. desc->sptes[0] = (u64 *)*pte_list;
  772. desc->sptes[1] = spte;
  773. *pte_list = (unsigned long)desc | 1;
  774. ++count;
  775. } else {
  776. rmap_printk("pte_list_add: %p %llx many->many\n", spte, *spte);
  777. desc = (struct pte_list_desc *)(*pte_list & ~1ul);
  778. while (desc->sptes[PTE_LIST_EXT-1] && desc->more) {
  779. desc = desc->more;
  780. count += PTE_LIST_EXT;
  781. }
  782. if (desc->sptes[PTE_LIST_EXT-1]) {
  783. desc->more = mmu_alloc_pte_list_desc(vcpu);
  784. desc = desc->more;
  785. }
  786. for (i = 0; desc->sptes[i]; ++i)
  787. ++count;
  788. desc->sptes[i] = spte;
  789. }
  790. return count;
  791. }
  792. static void
  793. pte_list_desc_remove_entry(unsigned long *pte_list, struct pte_list_desc *desc,
  794. int i, struct pte_list_desc *prev_desc)
  795. {
  796. int j;
  797. for (j = PTE_LIST_EXT - 1; !desc->sptes[j] && j > i; --j)
  798. ;
  799. desc->sptes[i] = desc->sptes[j];
  800. desc->sptes[j] = NULL;
  801. if (j != 0)
  802. return;
  803. if (!prev_desc && !desc->more)
  804. *pte_list = (unsigned long)desc->sptes[0];
  805. else
  806. if (prev_desc)
  807. prev_desc->more = desc->more;
  808. else
  809. *pte_list = (unsigned long)desc->more | 1;
  810. mmu_free_pte_list_desc(desc);
  811. }
  812. static void pte_list_remove(u64 *spte, unsigned long *pte_list)
  813. {
  814. struct pte_list_desc *desc;
  815. struct pte_list_desc *prev_desc;
  816. int i;
  817. if (!*pte_list) {
  818. printk(KERN_ERR "pte_list_remove: %p 0->BUG\n", spte);
  819. BUG();
  820. } else if (!(*pte_list & 1)) {
  821. rmap_printk("pte_list_remove: %p 1->0\n", spte);
  822. if ((u64 *)*pte_list != spte) {
  823. printk(KERN_ERR "pte_list_remove: %p 1->BUG\n", spte);
  824. BUG();
  825. }
  826. *pte_list = 0;
  827. } else {
  828. rmap_printk("pte_list_remove: %p many->many\n", spte);
  829. desc = (struct pte_list_desc *)(*pte_list & ~1ul);
  830. prev_desc = NULL;
  831. while (desc) {
  832. for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i)
  833. if (desc->sptes[i] == spte) {
  834. pte_list_desc_remove_entry(pte_list,
  835. desc, i,
  836. prev_desc);
  837. return;
  838. }
  839. prev_desc = desc;
  840. desc = desc->more;
  841. }
  842. pr_err("pte_list_remove: %p many->many\n", spte);
  843. BUG();
  844. }
  845. }
  846. typedef void (*pte_list_walk_fn) (u64 *spte);
  847. static void pte_list_walk(unsigned long *pte_list, pte_list_walk_fn fn)
  848. {
  849. struct pte_list_desc *desc;
  850. int i;
  851. if (!*pte_list)
  852. return;
  853. if (!(*pte_list & 1))
  854. return fn((u64 *)*pte_list);
  855. desc = (struct pte_list_desc *)(*pte_list & ~1ul);
  856. while (desc) {
  857. for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i)
  858. fn(desc->sptes[i]);
  859. desc = desc->more;
  860. }
  861. }
  862. static unsigned long *__gfn_to_rmap(gfn_t gfn, int level,
  863. struct kvm_memory_slot *slot)
  864. {
  865. unsigned long idx;
  866. idx = gfn_to_index(gfn, slot->base_gfn, level);
  867. return &slot->arch.rmap[level - PT_PAGE_TABLE_LEVEL][idx];
  868. }
  869. /*
  870. * Take gfn and return the reverse mapping to it.
  871. */
  872. static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int level)
  873. {
  874. struct kvm_memory_slot *slot;
  875. slot = gfn_to_memslot(kvm, gfn);
  876. return __gfn_to_rmap(gfn, level, slot);
  877. }
  878. static bool rmap_can_add(struct kvm_vcpu *vcpu)
  879. {
  880. struct kvm_mmu_memory_cache *cache;
  881. cache = &vcpu->arch.mmu_pte_list_desc_cache;
  882. return mmu_memory_cache_free_objects(cache);
  883. }
  884. static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
  885. {
  886. struct kvm_mmu_page *sp;
  887. unsigned long *rmapp;
  888. sp = page_header(__pa(spte));
  889. kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn);
  890. rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
  891. return pte_list_add(vcpu, spte, rmapp);
  892. }
  893. static void rmap_remove(struct kvm *kvm, u64 *spte)
  894. {
  895. struct kvm_mmu_page *sp;
  896. gfn_t gfn;
  897. unsigned long *rmapp;
  898. sp = page_header(__pa(spte));
  899. gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt);
  900. rmapp = gfn_to_rmap(kvm, gfn, sp->role.level);
  901. pte_list_remove(spte, rmapp);
  902. }
  903. /*
  904. * Used by the following functions to iterate through the sptes linked by a
  905. * rmap. All fields are private and not assumed to be used outside.
  906. */
  907. struct rmap_iterator {
  908. /* private fields */
  909. struct pte_list_desc *desc; /* holds the sptep if not NULL */
  910. int pos; /* index of the sptep */
  911. };
  912. /*
  913. * Iteration must be started by this function. This should also be used after
  914. * removing/dropping sptes from the rmap link because in such cases the
  915. * information in the itererator may not be valid.
  916. *
  917. * Returns sptep if found, NULL otherwise.
  918. */
  919. static u64 *rmap_get_first(unsigned long rmap, struct rmap_iterator *iter)
  920. {
  921. if (!rmap)
  922. return NULL;
  923. if (!(rmap & 1)) {
  924. iter->desc = NULL;
  925. return (u64 *)rmap;
  926. }
  927. iter->desc = (struct pte_list_desc *)(rmap & ~1ul);
  928. iter->pos = 0;
  929. return iter->desc->sptes[iter->pos];
  930. }
  931. /*
  932. * Must be used with a valid iterator: e.g. after rmap_get_first().
  933. *
  934. * Returns sptep if found, NULL otherwise.
  935. */
  936. static u64 *rmap_get_next(struct rmap_iterator *iter)
  937. {
  938. if (iter->desc) {
  939. if (iter->pos < PTE_LIST_EXT - 1) {
  940. u64 *sptep;
  941. ++iter->pos;
  942. sptep = iter->desc->sptes[iter->pos];
  943. if (sptep)
  944. return sptep;
  945. }
  946. iter->desc = iter->desc->more;
  947. if (iter->desc) {
  948. iter->pos = 0;
  949. /* desc->sptes[0] cannot be NULL */
  950. return iter->desc->sptes[iter->pos];
  951. }
  952. }
  953. return NULL;
  954. }
  955. static void drop_spte(struct kvm *kvm, u64 *sptep)
  956. {
  957. if (mmu_spte_clear_track_bits(sptep))
  958. rmap_remove(kvm, sptep);
  959. }
  960. static bool __drop_large_spte(struct kvm *kvm, u64 *sptep)
  961. {
  962. if (is_large_pte(*sptep)) {
  963. WARN_ON(page_header(__pa(sptep))->role.level ==
  964. PT_PAGE_TABLE_LEVEL);
  965. drop_spte(kvm, sptep);
  966. --kvm->stat.lpages;
  967. return true;
  968. }
  969. return false;
  970. }
  971. static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
  972. {
  973. if (__drop_large_spte(vcpu->kvm, sptep))
  974. kvm_flush_remote_tlbs(vcpu->kvm);
  975. }
  976. /*
  977. * Write-protect on the specified @sptep, @pt_protect indicates whether
  978. * spte writ-protection is caused by protecting shadow page table.
  979. * @flush indicates whether tlb need be flushed.
  980. *
  981. * Note: write protection is difference between drity logging and spte
  982. * protection:
  983. * - for dirty logging, the spte can be set to writable at anytime if
  984. * its dirty bitmap is properly set.
  985. * - for spte protection, the spte can be writable only after unsync-ing
  986. * shadow page.
  987. *
  988. * Return true if the spte is dropped.
  989. */
  990. static bool
  991. spte_write_protect(struct kvm *kvm, u64 *sptep, bool *flush, bool pt_protect)
  992. {
  993. u64 spte = *sptep;
  994. if (!is_writable_pte(spte) &&
  995. !(pt_protect && spte_is_locklessly_modifiable(spte)))
  996. return false;
  997. rmap_printk("rmap_write_protect: spte %p %llx\n", sptep, *sptep);
  998. if (__drop_large_spte(kvm, sptep)) {
  999. *flush |= true;
  1000. return true;
  1001. }
  1002. if (pt_protect)
  1003. spte &= ~SPTE_MMU_WRITEABLE;
  1004. spte = spte & ~PT_WRITABLE_MASK;
  1005. *flush |= mmu_spte_update(sptep, spte);
  1006. return false;
  1007. }
  1008. static bool __rmap_write_protect(struct kvm *kvm, unsigned long *rmapp,
  1009. bool pt_protect)
  1010. {
  1011. u64 *sptep;
  1012. struct rmap_iterator iter;
  1013. bool flush = false;
  1014. for (sptep = rmap_get_first(*rmapp, &iter); sptep;) {
  1015. BUG_ON(!(*sptep & PT_PRESENT_MASK));
  1016. if (spte_write_protect(kvm, sptep, &flush, pt_protect)) {
  1017. sptep = rmap_get_first(*rmapp, &iter);
  1018. continue;
  1019. }
  1020. sptep = rmap_get_next(&iter);
  1021. }
  1022. return flush;
  1023. }
  1024. /**
  1025. * kvm_mmu_write_protect_pt_masked - write protect selected PT level pages
  1026. * @kvm: kvm instance
  1027. * @slot: slot to protect
  1028. * @gfn_offset: start of the BITS_PER_LONG pages we care about
  1029. * @mask: indicates which pages we should protect
  1030. *
  1031. * Used when we do not need to care about huge page mappings: e.g. during dirty
  1032. * logging we do not have any such mappings.
  1033. */
  1034. void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
  1035. struct kvm_memory_slot *slot,
  1036. gfn_t gfn_offset, unsigned long mask)
  1037. {
  1038. unsigned long *rmapp;
  1039. while (mask) {
  1040. rmapp = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
  1041. PT_PAGE_TABLE_LEVEL, slot);
  1042. __rmap_write_protect(kvm, rmapp, false);
  1043. /* clear the first set bit */
  1044. mask &= mask - 1;
  1045. }
  1046. }
  1047. static bool rmap_write_protect(struct kvm *kvm, u64 gfn)
  1048. {
  1049. struct kvm_memory_slot *slot;
  1050. unsigned long *rmapp;
  1051. int i;
  1052. bool write_protected = false;
  1053. slot = gfn_to_memslot(kvm, gfn);
  1054. for (i = PT_PAGE_TABLE_LEVEL;
  1055. i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
  1056. rmapp = __gfn_to_rmap(gfn, i, slot);
  1057. write_protected |= __rmap_write_protect(kvm, rmapp, true);
  1058. }
  1059. return write_protected;
  1060. }
  1061. static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
  1062. struct kvm_memory_slot *slot, unsigned long data)
  1063. {
  1064. u64 *sptep;
  1065. struct rmap_iterator iter;
  1066. int need_tlb_flush = 0;
  1067. while ((sptep = rmap_get_first(*rmapp, &iter))) {
  1068. BUG_ON(!(*sptep & PT_PRESENT_MASK));
  1069. rmap_printk("kvm_rmap_unmap_hva: spte %p %llx\n", sptep, *sptep);
  1070. drop_spte(kvm, sptep);
  1071. need_tlb_flush = 1;
  1072. }
  1073. return need_tlb_flush;
  1074. }
  1075. static int kvm_set_pte_rmapp(struct kvm *kvm, unsigned long *rmapp,
  1076. struct kvm_memory_slot *slot, unsigned long data)
  1077. {
  1078. u64 *sptep;
  1079. struct rmap_iterator iter;
  1080. int need_flush = 0;
  1081. u64 new_spte;
  1082. pte_t *ptep = (pte_t *)data;
  1083. pfn_t new_pfn;
  1084. WARN_ON(pte_huge(*ptep));
  1085. new_pfn = pte_pfn(*ptep);
  1086. for (sptep = rmap_get_first(*rmapp, &iter); sptep;) {
  1087. BUG_ON(!is_shadow_present_pte(*sptep));
  1088. rmap_printk("kvm_set_pte_rmapp: spte %p %llx\n", sptep, *sptep);
  1089. need_flush = 1;
  1090. if (pte_write(*ptep)) {
  1091. drop_spte(kvm, sptep);
  1092. sptep = rmap_get_first(*rmapp, &iter);
  1093. } else {
  1094. new_spte = *sptep & ~PT64_BASE_ADDR_MASK;
  1095. new_spte |= (u64)new_pfn << PAGE_SHIFT;
  1096. new_spte &= ~PT_WRITABLE_MASK;
  1097. new_spte &= ~SPTE_HOST_WRITEABLE;
  1098. new_spte &= ~shadow_accessed_mask;
  1099. mmu_spte_clear_track_bits(sptep);
  1100. mmu_spte_set(sptep, new_spte);
  1101. sptep = rmap_get_next(&iter);
  1102. }
  1103. }
  1104. if (need_flush)
  1105. kvm_flush_remote_tlbs(kvm);
  1106. return 0;
  1107. }
  1108. static int kvm_handle_hva_range(struct kvm *kvm,
  1109. unsigned long start,
  1110. unsigned long end,
  1111. unsigned long data,
  1112. int (*handler)(struct kvm *kvm,
  1113. unsigned long *rmapp,
  1114. struct kvm_memory_slot *slot,
  1115. unsigned long data))
  1116. {
  1117. int j;
  1118. int ret = 0;
  1119. struct kvm_memslots *slots;
  1120. struct kvm_memory_slot *memslot;
  1121. slots = kvm_memslots(kvm);
  1122. kvm_for_each_memslot(memslot, slots) {
  1123. unsigned long hva_start, hva_end;
  1124. gfn_t gfn_start, gfn_end;
  1125. hva_start = max(start, memslot->userspace_addr);
  1126. hva_end = min(end, memslot->userspace_addr +
  1127. (memslot->npages << PAGE_SHIFT));
  1128. if (hva_start >= hva_end)
  1129. continue;
  1130. /*
  1131. * {gfn(page) | page intersects with [hva_start, hva_end)} =
  1132. * {gfn_start, gfn_start+1, ..., gfn_end-1}.
  1133. */
  1134. gfn_start = hva_to_gfn_memslot(hva_start, memslot);
  1135. gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
  1136. for (j = PT_PAGE_TABLE_LEVEL;
  1137. j < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++j) {
  1138. unsigned long idx, idx_end;
  1139. unsigned long *rmapp;
  1140. /*
  1141. * {idx(page_j) | page_j intersects with
  1142. * [hva_start, hva_end)} = {idx, idx+1, ..., idx_end}.
  1143. */
  1144. idx = gfn_to_index(gfn_start, memslot->base_gfn, j);
  1145. idx_end = gfn_to_index(gfn_end - 1, memslot->base_gfn, j);
  1146. rmapp = __gfn_to_rmap(gfn_start, j, memslot);
  1147. for (; idx <= idx_end; ++idx)
  1148. ret |= handler(kvm, rmapp++, memslot, data);
  1149. }
  1150. }
  1151. return ret;
  1152. }
  1153. static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
  1154. unsigned long data,
  1155. int (*handler)(struct kvm *kvm, unsigned long *rmapp,
  1156. struct kvm_memory_slot *slot,
  1157. unsigned long data))
  1158. {
  1159. return kvm_handle_hva_range(kvm, hva, hva + 1, data, handler);
  1160. }
  1161. int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
  1162. {
  1163. return kvm_handle_hva(kvm, hva, 0, kvm_unmap_rmapp);
  1164. }
  1165. int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end)
  1166. {
  1167. return kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp);
  1168. }
  1169. void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
  1170. {
  1171. kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);
  1172. }
  1173. static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
  1174. struct kvm_memory_slot *slot, unsigned long data)
  1175. {
  1176. u64 *sptep;
  1177. struct rmap_iterator uninitialized_var(iter);
  1178. int young = 0;
  1179. /*
  1180. * In case of absence of EPT Access and Dirty Bits supports,
  1181. * emulate the accessed bit for EPT, by checking if this page has
  1182. * an EPT mapping, and clearing it if it does. On the next access,
  1183. * a new EPT mapping will be established.
  1184. * This has some overhead, but not as much as the cost of swapping
  1185. * out actively used pages or breaking up actively used hugepages.
  1186. */
  1187. if (!shadow_accessed_mask) {
  1188. young = kvm_unmap_rmapp(kvm, rmapp, slot, data);
  1189. goto out;
  1190. }
  1191. for (sptep = rmap_get_first(*rmapp, &iter); sptep;
  1192. sptep = rmap_get_next(&iter)) {
  1193. BUG_ON(!is_shadow_present_pte(*sptep));
  1194. if (*sptep & shadow_accessed_mask) {
  1195. young = 1;
  1196. clear_bit((ffs(shadow_accessed_mask) - 1),
  1197. (unsigned long *)sptep);
  1198. }
  1199. }
  1200. out:
  1201. /* @data has hva passed to kvm_age_hva(). */
  1202. trace_kvm_age_page(data, slot, young);
  1203. return young;
  1204. }
  1205. static int kvm_test_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
  1206. struct kvm_memory_slot *slot, unsigned long data)
  1207. {
  1208. u64 *sptep;
  1209. struct rmap_iterator iter;
  1210. int young = 0;
  1211. /*
  1212. * If there's no access bit in the secondary pte set by the
  1213. * hardware it's up to gup-fast/gup to set the access bit in
  1214. * the primary pte or in the page structure.
  1215. */
  1216. if (!shadow_accessed_mask)
  1217. goto out;
  1218. for (sptep = rmap_get_first(*rmapp, &iter); sptep;
  1219. sptep = rmap_get_next(&iter)) {
  1220. BUG_ON(!is_shadow_present_pte(*sptep));
  1221. if (*sptep & shadow_accessed_mask) {
  1222. young = 1;
  1223. break;
  1224. }
  1225. }
  1226. out:
  1227. return young;
  1228. }
  1229. #define RMAP_RECYCLE_THRESHOLD 1000
  1230. static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
  1231. {
  1232. unsigned long *rmapp;
  1233. struct kvm_mmu_page *sp;
  1234. sp = page_header(__pa(spte));
  1235. rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
  1236. kvm_unmap_rmapp(vcpu->kvm, rmapp, NULL, 0);
  1237. kvm_flush_remote_tlbs(vcpu->kvm);
  1238. }
  1239. int kvm_age_hva(struct kvm *kvm, unsigned long hva)
  1240. {
  1241. return kvm_handle_hva(kvm, hva, hva, kvm_age_rmapp);
  1242. }
  1243. int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
  1244. {
  1245. return kvm_handle_hva(kvm, hva, 0, kvm_test_age_rmapp);
  1246. }
  1247. #ifdef MMU_DEBUG
  1248. static int is_empty_shadow_page(u64 *spt)
  1249. {
  1250. u64 *pos;
  1251. u64 *end;
  1252. for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
  1253. if (is_shadow_present_pte(*pos)) {
  1254. printk(KERN_ERR "%s: %p %llx\n", __func__,
  1255. pos, *pos);
  1256. return 0;
  1257. }
  1258. return 1;
  1259. }
  1260. #endif
  1261. /*
  1262. * This value is the sum of all of the kvm instances's
  1263. * kvm->arch.n_used_mmu_pages values. We need a global,
  1264. * aggregate version in order to make the slab shrinker
  1265. * faster
  1266. */
  1267. static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, int nr)
  1268. {
  1269. kvm->arch.n_used_mmu_pages += nr;
  1270. percpu_counter_add(&kvm_total_used_mmu_pages, nr);
  1271. }
  1272. static void kvm_mmu_free_page(struct kvm_mmu_page *sp)
  1273. {
  1274. ASSERT(is_empty_shadow_page(sp->spt));
  1275. hlist_del(&sp->hash_link);
  1276. list_del(&sp->link);
  1277. free_page((unsigned long)sp->spt);
  1278. if (!sp->role.direct)
  1279. free_page((unsigned long)sp->gfns);
  1280. kmem_cache_free(mmu_page_header_cache, sp);
  1281. }
  1282. static unsigned kvm_page_table_hashfn(gfn_t gfn)
  1283. {
  1284. return gfn & ((1 << KVM_MMU_HASH_SHIFT) - 1);
  1285. }
  1286. static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
  1287. struct kvm_mmu_page *sp, u64 *parent_pte)
  1288. {
  1289. if (!parent_pte)
  1290. return;
  1291. pte_list_add(vcpu, parent_pte, &sp->parent_ptes);
  1292. }
  1293. static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
  1294. u64 *parent_pte)
  1295. {
  1296. pte_list_remove(parent_pte, &sp->parent_ptes);
  1297. }
  1298. static void drop_parent_pte(struct kvm_mmu_page *sp,
  1299. u64 *parent_pte)
  1300. {
  1301. mmu_page_remove_parent_pte(sp, parent_pte);
  1302. mmu_spte_clear_no_track(parent_pte);
  1303. }
  1304. static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
  1305. u64 *parent_pte, int direct)
  1306. {
  1307. struct kvm_mmu_page *sp;
  1308. sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache);
  1309. sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache);
  1310. if (!direct)
  1311. sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache);
  1312. set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
  1313. /*
  1314. * The active_mmu_pages list is the FIFO list, do not move the
  1315. * page until it is zapped. kvm_zap_obsolete_pages depends on
  1316. * this feature. See the comments in kvm_zap_obsolete_pages().
  1317. */
  1318. list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
  1319. sp->parent_ptes = 0;
  1320. mmu_page_add_parent_pte(vcpu, sp, parent_pte);
  1321. kvm_mod_used_mmu_pages(vcpu->kvm, +1);
  1322. return sp;
  1323. }
  1324. static void mark_unsync(u64 *spte);
  1325. static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)
  1326. {
  1327. pte_list_walk(&sp->parent_ptes, mark_unsync);
  1328. }
  1329. static void mark_unsync(u64 *spte)
  1330. {
  1331. struct kvm_mmu_page *sp;
  1332. unsigned int index;
  1333. sp = page_header(__pa(spte));
  1334. index = spte - sp->spt;
  1335. if (__test_and_set_bit(index, sp->unsync_child_bitmap))
  1336. return;
  1337. if (sp->unsync_children++)
  1338. return;
  1339. kvm_mmu_mark_parents_unsync(sp);
  1340. }
  1341. static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
  1342. struct kvm_mmu_page *sp)
  1343. {
  1344. return 1;
  1345. }
  1346. static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
  1347. {
  1348. }
  1349. static void nonpaging_update_pte(struct kvm_vcpu *vcpu,
  1350. struct kvm_mmu_page *sp, u64 *spte,
  1351. const void *pte)
  1352. {
  1353. WARN_ON(1);
  1354. }
  1355. #define KVM_PAGE_ARRAY_NR 16
  1356. struct kvm_mmu_pages {
  1357. struct mmu_page_and_offset {
  1358. struct kvm_mmu_page *sp;
  1359. unsigned int idx;
  1360. } page[KVM_PAGE_ARRAY_NR];
  1361. unsigned int nr;
  1362. };
  1363. static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
  1364. int idx)
  1365. {
  1366. int i;
  1367. if (sp->unsync)
  1368. for (i=0; i < pvec->nr; i++)
  1369. if (pvec->page[i].sp == sp)
  1370. return 0;
  1371. pvec->page[pvec->nr].sp = sp;
  1372. pvec->page[pvec->nr].idx = idx;
  1373. pvec->nr++;
  1374. return (pvec->nr == KVM_PAGE_ARRAY_NR);
  1375. }
  1376. static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
  1377. struct kvm_mmu_pages *pvec)
  1378. {
  1379. int i, ret, nr_unsync_leaf = 0;
  1380. for_each_set_bit(i, sp->unsync_child_bitmap, 512) {
  1381. struct kvm_mmu_page *child;
  1382. u64 ent = sp->spt[i];
  1383. if (!is_shadow_present_pte(ent) || is_large_pte(ent))
  1384. goto clear_child_bitmap;
  1385. child = page_header(ent & PT64_BASE_ADDR_MASK);
  1386. if (child->unsync_children) {
  1387. if (mmu_pages_add(pvec, child, i))
  1388. return -ENOSPC;
  1389. ret = __mmu_unsync_walk(child, pvec);
  1390. if (!ret)
  1391. goto clear_child_bitmap;
  1392. else if (ret > 0)
  1393. nr_unsync_leaf += ret;
  1394. else
  1395. return ret;
  1396. } else if (child->unsync) {
  1397. nr_unsync_leaf++;
  1398. if (mmu_pages_add(pvec, child, i))
  1399. return -ENOSPC;
  1400. } else
  1401. goto clear_child_bitmap;
  1402. continue;
  1403. clear_child_bitmap:
  1404. __clear_bit(i, sp->unsync_child_bitmap);
  1405. sp->unsync_children--;
  1406. WARN_ON((int)sp->unsync_children < 0);
  1407. }
  1408. return nr_unsync_leaf;
  1409. }
  1410. static int mmu_unsync_walk(struct kvm_mmu_page *sp,
  1411. struct kvm_mmu_pages *pvec)
  1412. {
  1413. if (!sp->unsync_children)
  1414. return 0;
  1415. mmu_pages_add(pvec, sp, 0);
  1416. return __mmu_unsync_walk(sp, pvec);
  1417. }
  1418. static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
  1419. {
  1420. WARN_ON(!sp->unsync);
  1421. trace_kvm_mmu_sync_page(sp);
  1422. sp->unsync = 0;
  1423. --kvm->stat.mmu_unsync;
  1424. }
  1425. static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
  1426. struct list_head *invalid_list);
  1427. static void kvm_mmu_commit_zap_page(struct kvm *kvm,
  1428. struct list_head *invalid_list);
  1429. /*
  1430. * NOTE: we should pay more attention on the zapped-obsolete page
  1431. * (is_obsolete_sp(sp) && sp->role.invalid) when you do hash list walk
  1432. * since it has been deleted from active_mmu_pages but still can be found
  1433. * at hast list.
  1434. *
  1435. * for_each_gfn_indirect_valid_sp has skipped that kind of page and
  1436. * kvm_mmu_get_page(), the only user of for_each_gfn_sp(), has skipped
  1437. * all the obsolete pages.
  1438. */
  1439. #define for_each_gfn_sp(_kvm, _sp, _gfn) \
  1440. hlist_for_each_entry(_sp, \
  1441. &(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)], hash_link) \
  1442. if ((_sp)->gfn != (_gfn)) {} else
  1443. #define for_each_gfn_indirect_valid_sp(_kvm, _sp, _gfn) \
  1444. for_each_gfn_sp(_kvm, _sp, _gfn) \
  1445. if ((_sp)->role.direct || (_sp)->role.invalid) {} else
  1446. /* @sp->gfn should be write-protected at the call site */
  1447. static int __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
  1448. struct list_head *invalid_list, bool clear_unsync)
  1449. {
  1450. if (sp->role.cr4_pae != !!is_pae(vcpu)) {
  1451. kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
  1452. return 1;
  1453. }
  1454. if (clear_unsync)
  1455. kvm_unlink_unsync_page(vcpu->kvm, sp);
  1456. if (vcpu->arch.mmu.sync_page(vcpu, sp)) {
  1457. kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
  1458. return 1;
  1459. }
  1460. kvm_mmu_flush_tlb(vcpu);
  1461. return 0;
  1462. }
  1463. static int kvm_sync_page_transient(struct kvm_vcpu *vcpu,
  1464. struct kvm_mmu_page *sp)
  1465. {
  1466. LIST_HEAD(invalid_list);
  1467. int ret;
  1468. ret = __kvm_sync_page(vcpu, sp, &invalid_list, false);
  1469. if (ret)
  1470. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  1471. return ret;
  1472. }
  1473. #ifdef CONFIG_KVM_MMU_AUDIT
  1474. #include "mmu_audit.c"
  1475. #else
  1476. static void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) { }
  1477. static void mmu_audit_disable(void) { }
  1478. #endif
  1479. static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
  1480. struct list_head *invalid_list)
  1481. {
  1482. return __kvm_sync_page(vcpu, sp, invalid_list, true);
  1483. }
  1484. /* @gfn should be write-protected at the call site */
  1485. static void kvm_sync_pages(struct kvm_vcpu *vcpu, gfn_t gfn)
  1486. {
  1487. struct kvm_mmu_page *s;
  1488. LIST_HEAD(invalid_list);
  1489. bool flush = false;
  1490. for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
  1491. if (!s->unsync)
  1492. continue;
  1493. WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
  1494. kvm_unlink_unsync_page(vcpu->kvm, s);
  1495. if ((s->role.cr4_pae != !!is_pae(vcpu)) ||
  1496. (vcpu->arch.mmu.sync_page(vcpu, s))) {
  1497. kvm_mmu_prepare_zap_page(vcpu->kvm, s, &invalid_list);
  1498. continue;
  1499. }
  1500. flush = true;
  1501. }
  1502. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  1503. if (flush)
  1504. kvm_mmu_flush_tlb(vcpu);
  1505. }
  1506. struct mmu_page_path {
  1507. struct kvm_mmu_page *parent[PT64_ROOT_LEVEL-1];
  1508. unsigned int idx[PT64_ROOT_LEVEL-1];
  1509. };
  1510. #define for_each_sp(pvec, sp, parents, i) \
  1511. for (i = mmu_pages_next(&pvec, &parents, -1), \
  1512. sp = pvec.page[i].sp; \
  1513. i < pvec.nr && ({ sp = pvec.page[i].sp; 1;}); \
  1514. i = mmu_pages_next(&pvec, &parents, i))
  1515. static int mmu_pages_next(struct kvm_mmu_pages *pvec,
  1516. struct mmu_page_path *parents,
  1517. int i)
  1518. {
  1519. int n;
  1520. for (n = i+1; n < pvec->nr; n++) {
  1521. struct kvm_mmu_page *sp = pvec->page[n].sp;
  1522. if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
  1523. parents->idx[0] = pvec->page[n].idx;
  1524. return n;
  1525. }
  1526. parents->parent[sp->role.level-2] = sp;
  1527. parents->idx[sp->role.level-1] = pvec->page[n].idx;
  1528. }
  1529. return n;
  1530. }
  1531. static void mmu_pages_clear_parents(struct mmu_page_path *parents)
  1532. {
  1533. struct kvm_mmu_page *sp;
  1534. unsigned int level = 0;
  1535. do {
  1536. unsigned int idx = parents->idx[level];
  1537. sp = parents->parent[level];
  1538. if (!sp)
  1539. return;
  1540. --sp->unsync_children;
  1541. WARN_ON((int)sp->unsync_children < 0);
  1542. __clear_bit(idx, sp->unsync_child_bitmap);
  1543. level++;
  1544. } while (level < PT64_ROOT_LEVEL-1 && !sp->unsync_children);
  1545. }
  1546. static void kvm_mmu_pages_init(struct kvm_mmu_page *parent,
  1547. struct mmu_page_path *parents,
  1548. struct kvm_mmu_pages *pvec)
  1549. {
  1550. parents->parent[parent->role.level-1] = NULL;
  1551. pvec->nr = 0;
  1552. }
  1553. static void mmu_sync_children(struct kvm_vcpu *vcpu,
  1554. struct kvm_mmu_page *parent)
  1555. {
  1556. int i;
  1557. struct kvm_mmu_page *sp;
  1558. struct mmu_page_path parents;
  1559. struct kvm_mmu_pages pages;
  1560. LIST_HEAD(invalid_list);
  1561. kvm_mmu_pages_init(parent, &parents, &pages);
  1562. while (mmu_unsync_walk(parent, &pages)) {
  1563. bool protected = false;
  1564. for_each_sp(pages, sp, parents, i)
  1565. protected |= rmap_write_protect(vcpu->kvm, sp->gfn);
  1566. if (protected)
  1567. kvm_flush_remote_tlbs(vcpu->kvm);
  1568. for_each_sp(pages, sp, parents, i) {
  1569. kvm_sync_page(vcpu, sp, &invalid_list);
  1570. mmu_pages_clear_parents(&parents);
  1571. }
  1572. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  1573. cond_resched_lock(&vcpu->kvm->mmu_lock);
  1574. kvm_mmu_pages_init(parent, &parents, &pages);
  1575. }
  1576. }
  1577. static void init_shadow_page_table(struct kvm_mmu_page *sp)
  1578. {
  1579. int i;
  1580. for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
  1581. sp->spt[i] = 0ull;
  1582. }
  1583. static void __clear_sp_write_flooding_count(struct kvm_mmu_page *sp)
  1584. {
  1585. sp->write_flooding_count = 0;
  1586. }
  1587. static void clear_sp_write_flooding_count(u64 *spte)
  1588. {
  1589. struct kvm_mmu_page *sp = page_header(__pa(spte));
  1590. __clear_sp_write_flooding_count(sp);
  1591. }
  1592. static bool is_obsolete_sp(struct kvm *kvm, struct kvm_mmu_page *sp)
  1593. {
  1594. return unlikely(sp->mmu_valid_gen != kvm->arch.mmu_valid_gen);
  1595. }
  1596. static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
  1597. gfn_t gfn,
  1598. gva_t gaddr,
  1599. unsigned level,
  1600. int direct,
  1601. unsigned access,
  1602. u64 *parent_pte)
  1603. {
  1604. union kvm_mmu_page_role role;
  1605. unsigned quadrant;
  1606. struct kvm_mmu_page *sp;
  1607. bool need_sync = false;
  1608. role = vcpu->arch.mmu.base_role;
  1609. role.level = level;
  1610. role.direct = direct;
  1611. if (role.direct)
  1612. role.cr4_pae = 0;
  1613. role.access = access;
  1614. if (!vcpu->arch.mmu.direct_map
  1615. && vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) {
  1616. quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
  1617. quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
  1618. role.quadrant = quadrant;
  1619. }
  1620. for_each_gfn_sp(vcpu->kvm, sp, gfn) {
  1621. if (is_obsolete_sp(vcpu->kvm, sp))
  1622. continue;
  1623. if (!need_sync && sp->unsync)
  1624. need_sync = true;
  1625. if (sp->role.word != role.word)
  1626. continue;
  1627. if (sp->unsync && kvm_sync_page_transient(vcpu, sp))
  1628. break;
  1629. mmu_page_add_parent_pte(vcpu, sp, parent_pte);
  1630. if (sp->unsync_children) {
  1631. kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
  1632. kvm_mmu_mark_parents_unsync(sp);
  1633. } else if (sp->unsync)
  1634. kvm_mmu_mark_parents_unsync(sp);
  1635. __clear_sp_write_flooding_count(sp);
  1636. trace_kvm_mmu_get_page(sp, false);
  1637. return sp;
  1638. }
  1639. ++vcpu->kvm->stat.mmu_cache_miss;
  1640. sp = kvm_mmu_alloc_page(vcpu, parent_pte, direct);
  1641. if (!sp)
  1642. return sp;
  1643. sp->gfn = gfn;
  1644. sp->role = role;
  1645. hlist_add_head(&sp->hash_link,
  1646. &vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)]);
  1647. if (!direct) {
  1648. if (rmap_write_protect(vcpu->kvm, gfn))
  1649. kvm_flush_remote_tlbs(vcpu->kvm);
  1650. if (level > PT_PAGE_TABLE_LEVEL && need_sync)
  1651. kvm_sync_pages(vcpu, gfn);
  1652. account_shadowed(vcpu->kvm, gfn);
  1653. }
  1654. sp->mmu_valid_gen = vcpu->kvm->arch.mmu_valid_gen;
  1655. init_shadow_page_table(sp);
  1656. trace_kvm_mmu_get_page(sp, true);
  1657. return sp;
  1658. }
  1659. static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
  1660. struct kvm_vcpu *vcpu, u64 addr)
  1661. {
  1662. iterator->addr = addr;
  1663. iterator->shadow_addr = vcpu->arch.mmu.root_hpa;
  1664. iterator->level = vcpu->arch.mmu.shadow_root_level;
  1665. if (iterator->level == PT64_ROOT_LEVEL &&
  1666. vcpu->arch.mmu.root_level < PT64_ROOT_LEVEL &&
  1667. !vcpu->arch.mmu.direct_map)
  1668. --iterator->level;
  1669. if (iterator->level == PT32E_ROOT_LEVEL) {
  1670. iterator->shadow_addr
  1671. = vcpu->arch.mmu.pae_root[(addr >> 30) & 3];
  1672. iterator->shadow_addr &= PT64_BASE_ADDR_MASK;
  1673. --iterator->level;
  1674. if (!iterator->shadow_addr)
  1675. iterator->level = 0;
  1676. }
  1677. }
  1678. static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
  1679. {
  1680. if (iterator->level < PT_PAGE_TABLE_LEVEL)
  1681. return false;
  1682. iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
  1683. iterator->sptep = ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
  1684. return true;
  1685. }
  1686. static void __shadow_walk_next(struct kvm_shadow_walk_iterator *iterator,
  1687. u64 spte)
  1688. {
  1689. if (is_last_spte(spte, iterator->level)) {
  1690. iterator->level = 0;
  1691. return;
  1692. }
  1693. iterator->shadow_addr = spte & PT64_BASE_ADDR_MASK;
  1694. --iterator->level;
  1695. }
  1696. static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
  1697. {
  1698. return __shadow_walk_next(iterator, *iterator->sptep);
  1699. }
  1700. static void link_shadow_page(u64 *sptep, struct kvm_mmu_page *sp)
  1701. {
  1702. u64 spte;
  1703. spte = __pa(sp->spt) | PT_PRESENT_MASK | PT_WRITABLE_MASK |
  1704. shadow_user_mask | shadow_x_mask | shadow_accessed_mask;
  1705. mmu_spte_set(sptep, spte);
  1706. }
  1707. static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
  1708. unsigned direct_access)
  1709. {
  1710. if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {
  1711. struct kvm_mmu_page *child;
  1712. /*
  1713. * For the direct sp, if the guest pte's dirty bit
  1714. * changed form clean to dirty, it will corrupt the
  1715. * sp's access: allow writable in the read-only sp,
  1716. * so we should update the spte at this point to get
  1717. * a new sp with the correct access.
  1718. */
  1719. child = page_header(*sptep & PT64_BASE_ADDR_MASK);
  1720. if (child->role.access == direct_access)
  1721. return;
  1722. drop_parent_pte(child, sptep);
  1723. kvm_flush_remote_tlbs(vcpu->kvm);
  1724. }
  1725. }
  1726. static bool mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
  1727. u64 *spte)
  1728. {
  1729. u64 pte;
  1730. struct kvm_mmu_page *child;
  1731. pte = *spte;
  1732. if (is_shadow_present_pte(pte)) {
  1733. if (is_last_spte(pte, sp->role.level)) {
  1734. drop_spte(kvm, spte);
  1735. if (is_large_pte(pte))
  1736. --kvm->stat.lpages;
  1737. } else {
  1738. child = page_header(pte & PT64_BASE_ADDR_MASK);
  1739. drop_parent_pte(child, spte);
  1740. }
  1741. return true;
  1742. }
  1743. if (is_mmio_spte(pte))
  1744. mmu_spte_clear_no_track(spte);
  1745. return false;
  1746. }
  1747. static void kvm_mmu_page_unlink_children(struct kvm *kvm,
  1748. struct kvm_mmu_page *sp)
  1749. {
  1750. unsigned i;
  1751. for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
  1752. mmu_page_zap_pte(kvm, sp, sp->spt + i);
  1753. }
  1754. static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte)
  1755. {
  1756. mmu_page_remove_parent_pte(sp, parent_pte);
  1757. }
  1758. static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
  1759. {
  1760. u64 *sptep;
  1761. struct rmap_iterator iter;
  1762. while ((sptep = rmap_get_first(sp->parent_ptes, &iter)))
  1763. drop_parent_pte(sp, sptep);
  1764. }
  1765. static int mmu_zap_unsync_children(struct kvm *kvm,
  1766. struct kvm_mmu_page *parent,
  1767. struct list_head *invalid_list)
  1768. {
  1769. int i, zapped = 0;
  1770. struct mmu_page_path parents;
  1771. struct kvm_mmu_pages pages;
  1772. if (parent->role.level == PT_PAGE_TABLE_LEVEL)
  1773. return 0;
  1774. kvm_mmu_pages_init(parent, &parents, &pages);
  1775. while (mmu_unsync_walk(parent, &pages)) {
  1776. struct kvm_mmu_page *sp;
  1777. for_each_sp(pages, sp, parents, i) {
  1778. kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
  1779. mmu_pages_clear_parents(&parents);
  1780. zapped++;
  1781. }
  1782. kvm_mmu_pages_init(parent, &parents, &pages);
  1783. }
  1784. return zapped;
  1785. }
  1786. static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
  1787. struct list_head *invalid_list)
  1788. {
  1789. int ret;
  1790. trace_kvm_mmu_prepare_zap_page(sp);
  1791. ++kvm->stat.mmu_shadow_zapped;
  1792. ret = mmu_zap_unsync_children(kvm, sp, invalid_list);
  1793. kvm_mmu_page_unlink_children(kvm, sp);
  1794. kvm_mmu_unlink_parents(kvm, sp);
  1795. if (!sp->role.invalid && !sp->role.direct)
  1796. unaccount_shadowed(kvm, sp->gfn);
  1797. if (sp->unsync)
  1798. kvm_unlink_unsync_page(kvm, sp);
  1799. if (!sp->root_count) {
  1800. /* Count self */
  1801. ret++;
  1802. list_move(&sp->link, invalid_list);
  1803. kvm_mod_used_mmu_pages(kvm, -1);
  1804. } else {
  1805. list_move(&sp->link, &kvm->arch.active_mmu_pages);
  1806. /*
  1807. * The obsolete pages can not be used on any vcpus.
  1808. * See the comments in kvm_mmu_invalidate_zap_all_pages().
  1809. */
  1810. if (!sp->role.invalid && !is_obsolete_sp(kvm, sp))
  1811. kvm_reload_remote_mmus(kvm);
  1812. }
  1813. sp->role.invalid = 1;
  1814. return ret;
  1815. }
  1816. static void kvm_mmu_commit_zap_page(struct kvm *kvm,
  1817. struct list_head *invalid_list)
  1818. {
  1819. struct kvm_mmu_page *sp, *nsp;
  1820. if (list_empty(invalid_list))
  1821. return;
  1822. /*
  1823. * wmb: make sure everyone sees our modifications to the page tables
  1824. * rmb: make sure we see changes to vcpu->mode
  1825. */
  1826. smp_mb();
  1827. /*
  1828. * Wait for all vcpus to exit guest mode and/or lockless shadow
  1829. * page table walks.
  1830. */
  1831. kvm_flush_remote_tlbs(kvm);
  1832. list_for_each_entry_safe(sp, nsp, invalid_list, link) {
  1833. WARN_ON(!sp->role.invalid || sp->root_count);
  1834. kvm_mmu_free_page(sp);
  1835. }
  1836. }
  1837. static bool prepare_zap_oldest_mmu_page(struct kvm *kvm,
  1838. struct list_head *invalid_list)
  1839. {
  1840. struct kvm_mmu_page *sp;
  1841. if (list_empty(&kvm->arch.active_mmu_pages))
  1842. return false;
  1843. sp = list_entry(kvm->arch.active_mmu_pages.prev,
  1844. struct kvm_mmu_page, link);
  1845. kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
  1846. return true;
  1847. }
  1848. /*
  1849. * Changing the number of mmu pages allocated to the vm
  1850. * Note: if goal_nr_mmu_pages is too small, you will get dead lock
  1851. */
  1852. void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int goal_nr_mmu_pages)
  1853. {
  1854. LIST_HEAD(invalid_list);
  1855. spin_lock(&kvm->mmu_lock);
  1856. if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
  1857. /* Need to free some mmu pages to achieve the goal. */
  1858. while (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages)
  1859. if (!prepare_zap_oldest_mmu_page(kvm, &invalid_list))
  1860. break;
  1861. kvm_mmu_commit_zap_page(kvm, &invalid_list);
  1862. goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages;
  1863. }
  1864. kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
  1865. spin_unlock(&kvm->mmu_lock);
  1866. }
  1867. int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
  1868. {
  1869. struct kvm_mmu_page *sp;
  1870. LIST_HEAD(invalid_list);
  1871. int r;
  1872. pgprintk("%s: looking for gfn %llx\n", __func__, gfn);
  1873. r = 0;
  1874. spin_lock(&kvm->mmu_lock);
  1875. for_each_gfn_indirect_valid_sp(kvm, sp, gfn) {
  1876. pgprintk("%s: gfn %llx role %x\n", __func__, gfn,
  1877. sp->role.word);
  1878. r = 1;
  1879. kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
  1880. }
  1881. kvm_mmu_commit_zap_page(kvm, &invalid_list);
  1882. spin_unlock(&kvm->mmu_lock);
  1883. return r;
  1884. }
  1885. EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page);
  1886. /*
  1887. * The function is based on mtrr_type_lookup() in
  1888. * arch/x86/kernel/cpu/mtrr/generic.c
  1889. */
  1890. static int get_mtrr_type(struct mtrr_state_type *mtrr_state,
  1891. u64 start, u64 end)
  1892. {
  1893. int i;
  1894. u64 base, mask;
  1895. u8 prev_match, curr_match;
  1896. int num_var_ranges = KVM_NR_VAR_MTRR;
  1897. if (!mtrr_state->enabled)
  1898. return 0xFF;
  1899. /* Make end inclusive end, instead of exclusive */
  1900. end--;
  1901. /* Look in fixed ranges. Just return the type as per start */
  1902. if (mtrr_state->have_fixed && (start < 0x100000)) {
  1903. int idx;
  1904. if (start < 0x80000) {
  1905. idx = 0;
  1906. idx += (start >> 16);
  1907. return mtrr_state->fixed_ranges[idx];
  1908. } else if (start < 0xC0000) {
  1909. idx = 1 * 8;
  1910. idx += ((start - 0x80000) >> 14);
  1911. return mtrr_state->fixed_ranges[idx];
  1912. } else if (start < 0x1000000) {
  1913. idx = 3 * 8;
  1914. idx += ((start - 0xC0000) >> 12);
  1915. return mtrr_state->fixed_ranges[idx];
  1916. }
  1917. }
  1918. /*
  1919. * Look in variable ranges
  1920. * Look of multiple ranges matching this address and pick type
  1921. * as per MTRR precedence
  1922. */
  1923. if (!(mtrr_state->enabled & 2))
  1924. return mtrr_state->def_type;
  1925. prev_match = 0xFF;
  1926. for (i = 0; i < num_var_ranges; ++i) {
  1927. unsigned short start_state, end_state;
  1928. if (!(mtrr_state->var_ranges[i].mask_lo & (1 << 11)))
  1929. continue;
  1930. base = (((u64)mtrr_state->var_ranges[i].base_hi) << 32) +
  1931. (mtrr_state->var_ranges[i].base_lo & PAGE_MASK);
  1932. mask = (((u64)mtrr_state->var_ranges[i].mask_hi) << 32) +
  1933. (mtrr_state->var_ranges[i].mask_lo & PAGE_MASK);
  1934. start_state = ((start & mask) == (base & mask));
  1935. end_state = ((end & mask) == (base & mask));
  1936. if (start_state != end_state)
  1937. return 0xFE;
  1938. if ((start & mask) != (base & mask))
  1939. continue;
  1940. curr_match = mtrr_state->var_ranges[i].base_lo & 0xff;
  1941. if (prev_match == 0xFF) {
  1942. prev_match = curr_match;
  1943. continue;
  1944. }
  1945. if (prev_match == MTRR_TYPE_UNCACHABLE ||
  1946. curr_match == MTRR_TYPE_UNCACHABLE)
  1947. return MTRR_TYPE_UNCACHABLE;
  1948. if ((prev_match == MTRR_TYPE_WRBACK &&
  1949. curr_match == MTRR_TYPE_WRTHROUGH) ||
  1950. (prev_match == MTRR_TYPE_WRTHROUGH &&
  1951. curr_match == MTRR_TYPE_WRBACK)) {
  1952. prev_match = MTRR_TYPE_WRTHROUGH;
  1953. curr_match = MTRR_TYPE_WRTHROUGH;
  1954. }
  1955. if (prev_match != curr_match)
  1956. return MTRR_TYPE_UNCACHABLE;
  1957. }
  1958. if (prev_match != 0xFF)
  1959. return prev_match;
  1960. return mtrr_state->def_type;
  1961. }
  1962. u8 kvm_get_guest_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn)
  1963. {
  1964. u8 mtrr;
  1965. mtrr = get_mtrr_type(&vcpu->arch.mtrr_state, gfn << PAGE_SHIFT,
  1966. (gfn << PAGE_SHIFT) + PAGE_SIZE);
  1967. if (mtrr == 0xfe || mtrr == 0xff)
  1968. mtrr = MTRR_TYPE_WRBACK;
  1969. return mtrr;
  1970. }
  1971. EXPORT_SYMBOL_GPL(kvm_get_guest_memory_type);
  1972. static void __kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
  1973. {
  1974. trace_kvm_mmu_unsync_page(sp);
  1975. ++vcpu->kvm->stat.mmu_unsync;
  1976. sp->unsync = 1;
  1977. kvm_mmu_mark_parents_unsync(sp);
  1978. }
  1979. static void kvm_unsync_pages(struct kvm_vcpu *vcpu, gfn_t gfn)
  1980. {
  1981. struct kvm_mmu_page *s;
  1982. for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
  1983. if (s->unsync)
  1984. continue;
  1985. WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
  1986. __kvm_unsync_page(vcpu, s);
  1987. }
  1988. }
  1989. static int mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
  1990. bool can_unsync)
  1991. {
  1992. struct kvm_mmu_page *s;
  1993. bool need_unsync = false;
  1994. for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
  1995. if (!can_unsync)
  1996. return 1;
  1997. if (s->role.level != PT_PAGE_TABLE_LEVEL)
  1998. return 1;
  1999. if (!s->unsync)
  2000. need_unsync = true;
  2001. }
  2002. if (need_unsync)
  2003. kvm_unsync_pages(vcpu, gfn);
  2004. return 0;
  2005. }
  2006. static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
  2007. unsigned pte_access, int level,
  2008. gfn_t gfn, pfn_t pfn, bool speculative,
  2009. bool can_unsync, bool host_writable)
  2010. {
  2011. u64 spte;
  2012. int ret = 0;
  2013. if (set_mmio_spte(vcpu->kvm, sptep, gfn, pfn, pte_access))
  2014. return 0;
  2015. spte = PT_PRESENT_MASK;
  2016. if (!speculative)
  2017. spte |= shadow_accessed_mask;
  2018. if (pte_access & ACC_EXEC_MASK)
  2019. spte |= shadow_x_mask;
  2020. else
  2021. spte |= shadow_nx_mask;
  2022. if (pte_access & ACC_USER_MASK)
  2023. spte |= shadow_user_mask;
  2024. if (level > PT_PAGE_TABLE_LEVEL)
  2025. spte |= PT_PAGE_SIZE_MASK;
  2026. if (tdp_enabled)
  2027. spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn,
  2028. kvm_is_mmio_pfn(pfn));
  2029. if (host_writable)
  2030. spte |= SPTE_HOST_WRITEABLE;
  2031. else
  2032. pte_access &= ~ACC_WRITE_MASK;
  2033. spte |= (u64)pfn << PAGE_SHIFT;
  2034. if (pte_access & ACC_WRITE_MASK) {
  2035. /*
  2036. * Other vcpu creates new sp in the window between
  2037. * mapping_level() and acquiring mmu-lock. We can
  2038. * allow guest to retry the access, the mapping can
  2039. * be fixed if guest refault.
  2040. */
  2041. if (level > PT_PAGE_TABLE_LEVEL &&
  2042. has_wrprotected_page(vcpu->kvm, gfn, level))
  2043. goto done;
  2044. spte |= PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE;
  2045. /*
  2046. * Optimization: for pte sync, if spte was writable the hash
  2047. * lookup is unnecessary (and expensive). Write protection
  2048. * is responsibility of mmu_get_page / kvm_sync_page.
  2049. * Same reasoning can be applied to dirty page accounting.
  2050. */
  2051. if (!can_unsync && is_writable_pte(*sptep))
  2052. goto set_pte;
  2053. if (mmu_need_write_protect(vcpu, gfn, can_unsync)) {
  2054. pgprintk("%s: found shadow page for %llx, marking ro\n",
  2055. __func__, gfn);
  2056. ret = 1;
  2057. pte_access &= ~ACC_WRITE_MASK;
  2058. spte &= ~(PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE);
  2059. }
  2060. }
  2061. if (pte_access & ACC_WRITE_MASK)
  2062. mark_page_dirty(vcpu->kvm, gfn);
  2063. set_pte:
  2064. if (mmu_spte_update(sptep, spte))
  2065. kvm_flush_remote_tlbs(vcpu->kvm);
  2066. done:
  2067. return ret;
  2068. }
  2069. static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
  2070. unsigned pte_access, int write_fault, int *emulate,
  2071. int level, gfn_t gfn, pfn_t pfn, bool speculative,
  2072. bool host_writable)
  2073. {
  2074. int was_rmapped = 0;
  2075. int rmap_count;
  2076. pgprintk("%s: spte %llx write_fault %d gfn %llx\n", __func__,
  2077. *sptep, write_fault, gfn);
  2078. if (is_rmap_spte(*sptep)) {
  2079. /*
  2080. * If we overwrite a PTE page pointer with a 2MB PMD, unlink
  2081. * the parent of the now unreachable PTE.
  2082. */
  2083. if (level > PT_PAGE_TABLE_LEVEL &&
  2084. !is_large_pte(*sptep)) {
  2085. struct kvm_mmu_page *child;
  2086. u64 pte = *sptep;
  2087. child = page_header(pte & PT64_BASE_ADDR_MASK);
  2088. drop_parent_pte(child, sptep);
  2089. kvm_flush_remote_tlbs(vcpu->kvm);
  2090. } else if (pfn != spte_to_pfn(*sptep)) {
  2091. pgprintk("hfn old %llx new %llx\n",
  2092. spte_to_pfn(*sptep), pfn);
  2093. drop_spte(vcpu->kvm, sptep);
  2094. kvm_flush_remote_tlbs(vcpu->kvm);
  2095. } else
  2096. was_rmapped = 1;
  2097. }
  2098. if (set_spte(vcpu, sptep, pte_access, level, gfn, pfn, speculative,
  2099. true, host_writable)) {
  2100. if (write_fault)
  2101. *emulate = 1;
  2102. kvm_mmu_flush_tlb(vcpu);
  2103. }
  2104. if (unlikely(is_mmio_spte(*sptep) && emulate))
  2105. *emulate = 1;
  2106. pgprintk("%s: setting spte %llx\n", __func__, *sptep);
  2107. pgprintk("instantiating %s PTE (%s) at %llx (%llx) addr %p\n",
  2108. is_large_pte(*sptep)? "2MB" : "4kB",
  2109. *sptep & PT_PRESENT_MASK ?"RW":"R", gfn,
  2110. *sptep, sptep);
  2111. if (!was_rmapped && is_large_pte(*sptep))
  2112. ++vcpu->kvm->stat.lpages;
  2113. if (is_shadow_present_pte(*sptep)) {
  2114. if (!was_rmapped) {
  2115. rmap_count = rmap_add(vcpu, sptep, gfn);
  2116. if (rmap_count > RMAP_RECYCLE_THRESHOLD)
  2117. rmap_recycle(vcpu, sptep, gfn);
  2118. }
  2119. }
  2120. kvm_release_pfn_clean(pfn);
  2121. }
  2122. static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
  2123. {
  2124. mmu_free_roots(vcpu);
  2125. }
  2126. static bool is_rsvd_bits_set(struct kvm_mmu *mmu, u64 gpte, int level)
  2127. {
  2128. int bit7;
  2129. bit7 = (gpte >> 7) & 1;
  2130. return (gpte & mmu->rsvd_bits_mask[bit7][level-1]) != 0;
  2131. }
  2132. static pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn,
  2133. bool no_dirty_log)
  2134. {
  2135. struct kvm_memory_slot *slot;
  2136. slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, no_dirty_log);
  2137. if (!slot)
  2138. return KVM_PFN_ERR_FAULT;
  2139. return gfn_to_pfn_memslot_atomic(slot, gfn);
  2140. }
  2141. static bool prefetch_invalid_gpte(struct kvm_vcpu *vcpu,
  2142. struct kvm_mmu_page *sp, u64 *spte,
  2143. u64 gpte)
  2144. {
  2145. if (is_rsvd_bits_set(&vcpu->arch.mmu, gpte, PT_PAGE_TABLE_LEVEL))
  2146. goto no_present;
  2147. if (!is_present_gpte(gpte))
  2148. goto no_present;
  2149. if (!(gpte & PT_ACCESSED_MASK))
  2150. goto no_present;
  2151. return false;
  2152. no_present:
  2153. drop_spte(vcpu->kvm, spte);
  2154. return true;
  2155. }
  2156. static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
  2157. struct kvm_mmu_page *sp,
  2158. u64 *start, u64 *end)
  2159. {
  2160. struct page *pages[PTE_PREFETCH_NUM];
  2161. unsigned access = sp->role.access;
  2162. int i, ret;
  2163. gfn_t gfn;
  2164. gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt);
  2165. if (!gfn_to_memslot_dirty_bitmap(vcpu, gfn, access & ACC_WRITE_MASK))
  2166. return -1;
  2167. ret = gfn_to_page_many_atomic(vcpu->kvm, gfn, pages, end - start);
  2168. if (ret <= 0)
  2169. return -1;
  2170. for (i = 0; i < ret; i++, gfn++, start++)
  2171. mmu_set_spte(vcpu, start, access, 0, NULL,
  2172. sp->role.level, gfn, page_to_pfn(pages[i]),
  2173. true, true);
  2174. return 0;
  2175. }
  2176. static void __direct_pte_prefetch(struct kvm_vcpu *vcpu,
  2177. struct kvm_mmu_page *sp, u64 *sptep)
  2178. {
  2179. u64 *spte, *start = NULL;
  2180. int i;
  2181. WARN_ON(!sp->role.direct);
  2182. i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
  2183. spte = sp->spt + i;
  2184. for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
  2185. if (is_shadow_present_pte(*spte) || spte == sptep) {
  2186. if (!start)
  2187. continue;
  2188. if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0)
  2189. break;
  2190. start = NULL;
  2191. } else if (!start)
  2192. start = spte;
  2193. }
  2194. }
  2195. static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
  2196. {
  2197. struct kvm_mmu_page *sp;
  2198. /*
  2199. * Since it's no accessed bit on EPT, it's no way to
  2200. * distinguish between actually accessed translations
  2201. * and prefetched, so disable pte prefetch if EPT is
  2202. * enabled.
  2203. */
  2204. if (!shadow_accessed_mask)
  2205. return;
  2206. sp = page_header(__pa(sptep));
  2207. if (sp->role.level > PT_PAGE_TABLE_LEVEL)
  2208. return;
  2209. __direct_pte_prefetch(vcpu, sp, sptep);
  2210. }
  2211. static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
  2212. int map_writable, int level, gfn_t gfn, pfn_t pfn,
  2213. bool prefault)
  2214. {
  2215. struct kvm_shadow_walk_iterator iterator;
  2216. struct kvm_mmu_page *sp;
  2217. int emulate = 0;
  2218. gfn_t pseudo_gfn;
  2219. for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
  2220. if (iterator.level == level) {
  2221. mmu_set_spte(vcpu, iterator.sptep, ACC_ALL,
  2222. write, &emulate, level, gfn, pfn,
  2223. prefault, map_writable);
  2224. direct_pte_prefetch(vcpu, iterator.sptep);
  2225. ++vcpu->stat.pf_fixed;
  2226. break;
  2227. }
  2228. if (!is_shadow_present_pte(*iterator.sptep)) {
  2229. u64 base_addr = iterator.addr;
  2230. base_addr &= PT64_LVL_ADDR_MASK(iterator.level);
  2231. pseudo_gfn = base_addr >> PAGE_SHIFT;
  2232. sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr,
  2233. iterator.level - 1,
  2234. 1, ACC_ALL, iterator.sptep);
  2235. link_shadow_page(iterator.sptep, sp);
  2236. }
  2237. }
  2238. return emulate;
  2239. }
  2240. static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *tsk)
  2241. {
  2242. siginfo_t info;
  2243. info.si_signo = SIGBUS;
  2244. info.si_errno = 0;
  2245. info.si_code = BUS_MCEERR_AR;
  2246. info.si_addr = (void __user *)address;
  2247. info.si_addr_lsb = PAGE_SHIFT;
  2248. send_sig_info(SIGBUS, &info, tsk);
  2249. }
  2250. static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, pfn_t pfn)
  2251. {
  2252. /*
  2253. * Do not cache the mmio info caused by writing the readonly gfn
  2254. * into the spte otherwise read access on readonly gfn also can
  2255. * caused mmio page fault and treat it as mmio access.
  2256. * Return 1 to tell kvm to emulate it.
  2257. */
  2258. if (pfn == KVM_PFN_ERR_RO_FAULT)
  2259. return 1;
  2260. if (pfn == KVM_PFN_ERR_HWPOISON) {
  2261. kvm_send_hwpoison_signal(gfn_to_hva(vcpu->kvm, gfn), current);
  2262. return 0;
  2263. }
  2264. return -EFAULT;
  2265. }
  2266. static void transparent_hugepage_adjust(struct kvm_vcpu *vcpu,
  2267. gfn_t *gfnp, pfn_t *pfnp, int *levelp)
  2268. {
  2269. pfn_t pfn = *pfnp;
  2270. gfn_t gfn = *gfnp;
  2271. int level = *levelp;
  2272. /*
  2273. * Check if it's a transparent hugepage. If this would be an
  2274. * hugetlbfs page, level wouldn't be set to
  2275. * PT_PAGE_TABLE_LEVEL and there would be no adjustment done
  2276. * here.
  2277. */
  2278. if (!is_error_noslot_pfn(pfn) && !kvm_is_mmio_pfn(pfn) &&
  2279. level == PT_PAGE_TABLE_LEVEL &&
  2280. PageTransCompound(pfn_to_page(pfn)) &&
  2281. !has_wrprotected_page(vcpu->kvm, gfn, PT_DIRECTORY_LEVEL)) {
  2282. unsigned long mask;
  2283. /*
  2284. * mmu_notifier_retry was successful and we hold the
  2285. * mmu_lock here, so the pmd can't become splitting
  2286. * from under us, and in turn
  2287. * __split_huge_page_refcount() can't run from under
  2288. * us and we can safely transfer the refcount from
  2289. * PG_tail to PG_head as we switch the pfn to tail to
  2290. * head.
  2291. */
  2292. *levelp = level = PT_DIRECTORY_LEVEL;
  2293. mask = KVM_PAGES_PER_HPAGE(level) - 1;
  2294. VM_BUG_ON((gfn & mask) != (pfn & mask));
  2295. if (pfn & mask) {
  2296. gfn &= ~mask;
  2297. *gfnp = gfn;
  2298. kvm_release_pfn_clean(pfn);
  2299. pfn &= ~mask;
  2300. kvm_get_pfn(pfn);
  2301. *pfnp = pfn;
  2302. }
  2303. }
  2304. }
  2305. static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn,
  2306. pfn_t pfn, unsigned access, int *ret_val)
  2307. {
  2308. bool ret = true;
  2309. /* The pfn is invalid, report the error! */
  2310. if (unlikely(is_error_pfn(pfn))) {
  2311. *ret_val = kvm_handle_bad_page(vcpu, gfn, pfn);
  2312. goto exit;
  2313. }
  2314. if (unlikely(is_noslot_pfn(pfn)))
  2315. vcpu_cache_mmio_info(vcpu, gva, gfn, access);
  2316. ret = false;
  2317. exit:
  2318. return ret;
  2319. }
  2320. static bool page_fault_can_be_fast(struct kvm_vcpu *vcpu, u32 error_code)
  2321. {
  2322. /*
  2323. * Do not fix the mmio spte with invalid generation number which
  2324. * need to be updated by slow page fault path.
  2325. */
  2326. if (unlikely(error_code & PFERR_RSVD_MASK))
  2327. return false;
  2328. /*
  2329. * #PF can be fast only if the shadow page table is present and it
  2330. * is caused by write-protect, that means we just need change the
  2331. * W bit of the spte which can be done out of mmu-lock.
  2332. */
  2333. if (!(error_code & PFERR_PRESENT_MASK) ||
  2334. !(error_code & PFERR_WRITE_MASK))
  2335. return false;
  2336. return true;
  2337. }
  2338. static bool
  2339. fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 spte)
  2340. {
  2341. struct kvm_mmu_page *sp = page_header(__pa(sptep));
  2342. gfn_t gfn;
  2343. WARN_ON(!sp->role.direct);
  2344. /*
  2345. * The gfn of direct spte is stable since it is calculated
  2346. * by sp->gfn.
  2347. */
  2348. gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt);
  2349. if (cmpxchg64(sptep, spte, spte | PT_WRITABLE_MASK) == spte)
  2350. mark_page_dirty(vcpu->kvm, gfn);
  2351. return true;
  2352. }
  2353. /*
  2354. * Return value:
  2355. * - true: let the vcpu to access on the same address again.
  2356. * - false: let the real page fault path to fix it.
  2357. */
  2358. static bool fast_page_fault(struct kvm_vcpu *vcpu, gva_t gva, int level,
  2359. u32 error_code)
  2360. {
  2361. struct kvm_shadow_walk_iterator iterator;
  2362. bool ret = false;
  2363. u64 spte = 0ull;
  2364. if (!page_fault_can_be_fast(vcpu, error_code))
  2365. return false;
  2366. walk_shadow_page_lockless_begin(vcpu);
  2367. for_each_shadow_entry_lockless(vcpu, gva, iterator, spte)
  2368. if (!is_shadow_present_pte(spte) || iterator.level < level)
  2369. break;
  2370. /*
  2371. * If the mapping has been changed, let the vcpu fault on the
  2372. * same address again.
  2373. */
  2374. if (!is_rmap_spte(spte)) {
  2375. ret = true;
  2376. goto exit;
  2377. }
  2378. if (!is_last_spte(spte, level))
  2379. goto exit;
  2380. /*
  2381. * Check if it is a spurious fault caused by TLB lazily flushed.
  2382. *
  2383. * Need not check the access of upper level table entries since
  2384. * they are always ACC_ALL.
  2385. */
  2386. if (is_writable_pte(spte)) {
  2387. ret = true;
  2388. goto exit;
  2389. }
  2390. /*
  2391. * Currently, to simplify the code, only the spte write-protected
  2392. * by dirty-log can be fast fixed.
  2393. */
  2394. if (!spte_is_locklessly_modifiable(spte))
  2395. goto exit;
  2396. /*
  2397. * Currently, fast page fault only works for direct mapping since
  2398. * the gfn is not stable for indirect shadow page.
  2399. * See Documentation/virtual/kvm/locking.txt to get more detail.
  2400. */
  2401. ret = fast_pf_fix_direct_spte(vcpu, iterator.sptep, spte);
  2402. exit:
  2403. trace_fast_page_fault(vcpu, gva, error_code, iterator.sptep,
  2404. spte, ret);
  2405. walk_shadow_page_lockless_end(vcpu);
  2406. return ret;
  2407. }
  2408. static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
  2409. gva_t gva, pfn_t *pfn, bool write, bool *writable);
  2410. static void make_mmu_pages_available(struct kvm_vcpu *vcpu);
  2411. static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, u32 error_code,
  2412. gfn_t gfn, bool prefault)
  2413. {
  2414. int r;
  2415. int level;
  2416. int force_pt_level;
  2417. pfn_t pfn;
  2418. unsigned long mmu_seq;
  2419. bool map_writable, write = error_code & PFERR_WRITE_MASK;
  2420. force_pt_level = mapping_level_dirty_bitmap(vcpu, gfn);
  2421. if (likely(!force_pt_level)) {
  2422. level = mapping_level(vcpu, gfn);
  2423. /*
  2424. * This path builds a PAE pagetable - so we can map
  2425. * 2mb pages at maximum. Therefore check if the level
  2426. * is larger than that.
  2427. */
  2428. if (level > PT_DIRECTORY_LEVEL)
  2429. level = PT_DIRECTORY_LEVEL;
  2430. gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
  2431. } else
  2432. level = PT_PAGE_TABLE_LEVEL;
  2433. if (fast_page_fault(vcpu, v, level, error_code))
  2434. return 0;
  2435. mmu_seq = vcpu->kvm->mmu_notifier_seq;
  2436. smp_rmb();
  2437. if (try_async_pf(vcpu, prefault, gfn, v, &pfn, write, &map_writable))
  2438. return 0;
  2439. if (handle_abnormal_pfn(vcpu, v, gfn, pfn, ACC_ALL, &r))
  2440. return r;
  2441. spin_lock(&vcpu->kvm->mmu_lock);
  2442. if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
  2443. goto out_unlock;
  2444. make_mmu_pages_available(vcpu);
  2445. if (likely(!force_pt_level))
  2446. transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
  2447. r = __direct_map(vcpu, v, write, map_writable, level, gfn, pfn,
  2448. prefault);
  2449. spin_unlock(&vcpu->kvm->mmu_lock);
  2450. return r;
  2451. out_unlock:
  2452. spin_unlock(&vcpu->kvm->mmu_lock);
  2453. kvm_release_pfn_clean(pfn);
  2454. return 0;
  2455. }
  2456. static void mmu_free_roots(struct kvm_vcpu *vcpu)
  2457. {
  2458. int i;
  2459. struct kvm_mmu_page *sp;
  2460. LIST_HEAD(invalid_list);
  2461. if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
  2462. return;
  2463. if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL &&
  2464. (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL ||
  2465. vcpu->arch.mmu.direct_map)) {
  2466. hpa_t root = vcpu->arch.mmu.root_hpa;
  2467. spin_lock(&vcpu->kvm->mmu_lock);
  2468. sp = page_header(root);
  2469. --sp->root_count;
  2470. if (!sp->root_count && sp->role.invalid) {
  2471. kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
  2472. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  2473. }
  2474. spin_unlock(&vcpu->kvm->mmu_lock);
  2475. vcpu->arch.mmu.root_hpa = INVALID_PAGE;
  2476. return;
  2477. }
  2478. spin_lock(&vcpu->kvm->mmu_lock);
  2479. for (i = 0; i < 4; ++i) {
  2480. hpa_t root = vcpu->arch.mmu.pae_root[i];
  2481. if (root) {
  2482. root &= PT64_BASE_ADDR_MASK;
  2483. sp = page_header(root);
  2484. --sp->root_count;
  2485. if (!sp->root_count && sp->role.invalid)
  2486. kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
  2487. &invalid_list);
  2488. }
  2489. vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
  2490. }
  2491. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  2492. spin_unlock(&vcpu->kvm->mmu_lock);
  2493. vcpu->arch.mmu.root_hpa = INVALID_PAGE;
  2494. }
  2495. static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
  2496. {
  2497. int ret = 0;
  2498. if (!kvm_is_visible_gfn(vcpu->kvm, root_gfn)) {
  2499. kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
  2500. ret = 1;
  2501. }
  2502. return ret;
  2503. }
  2504. static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu)
  2505. {
  2506. struct kvm_mmu_page *sp;
  2507. unsigned i;
  2508. if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
  2509. spin_lock(&vcpu->kvm->mmu_lock);
  2510. make_mmu_pages_available(vcpu);
  2511. sp = kvm_mmu_get_page(vcpu, 0, 0, PT64_ROOT_LEVEL,
  2512. 1, ACC_ALL, NULL);
  2513. ++sp->root_count;
  2514. spin_unlock(&vcpu->kvm->mmu_lock);
  2515. vcpu->arch.mmu.root_hpa = __pa(sp->spt);
  2516. } else if (vcpu->arch.mmu.shadow_root_level == PT32E_ROOT_LEVEL) {
  2517. for (i = 0; i < 4; ++i) {
  2518. hpa_t root = vcpu->arch.mmu.pae_root[i];
  2519. ASSERT(!VALID_PAGE(root));
  2520. spin_lock(&vcpu->kvm->mmu_lock);
  2521. make_mmu_pages_available(vcpu);
  2522. sp = kvm_mmu_get_page(vcpu, i << (30 - PAGE_SHIFT),
  2523. i << 30,
  2524. PT32_ROOT_LEVEL, 1, ACC_ALL,
  2525. NULL);
  2526. root = __pa(sp->spt);
  2527. ++sp->root_count;
  2528. spin_unlock(&vcpu->kvm->mmu_lock);
  2529. vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
  2530. }
  2531. vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
  2532. } else
  2533. BUG();
  2534. return 0;
  2535. }
  2536. static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
  2537. {
  2538. struct kvm_mmu_page *sp;
  2539. u64 pdptr, pm_mask;
  2540. gfn_t root_gfn;
  2541. int i;
  2542. root_gfn = vcpu->arch.mmu.get_cr3(vcpu) >> PAGE_SHIFT;
  2543. if (mmu_check_root(vcpu, root_gfn))
  2544. return 1;
  2545. /*
  2546. * Do we shadow a long mode page table? If so we need to
  2547. * write-protect the guests page table root.
  2548. */
  2549. if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
  2550. hpa_t root = vcpu->arch.mmu.root_hpa;
  2551. ASSERT(!VALID_PAGE(root));
  2552. spin_lock(&vcpu->kvm->mmu_lock);
  2553. make_mmu_pages_available(vcpu);
  2554. sp = kvm_mmu_get_page(vcpu, root_gfn, 0, PT64_ROOT_LEVEL,
  2555. 0, ACC_ALL, NULL);
  2556. root = __pa(sp->spt);
  2557. ++sp->root_count;
  2558. spin_unlock(&vcpu->kvm->mmu_lock);
  2559. vcpu->arch.mmu.root_hpa = root;
  2560. return 0;
  2561. }
  2562. /*
  2563. * We shadow a 32 bit page table. This may be a legacy 2-level
  2564. * or a PAE 3-level page table. In either case we need to be aware that
  2565. * the shadow page table may be a PAE or a long mode page table.
  2566. */
  2567. pm_mask = PT_PRESENT_MASK;
  2568. if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL)
  2569. pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK;
  2570. for (i = 0; i < 4; ++i) {
  2571. hpa_t root = vcpu->arch.mmu.pae_root[i];
  2572. ASSERT(!VALID_PAGE(root));
  2573. if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
  2574. pdptr = vcpu->arch.mmu.get_pdptr(vcpu, i);
  2575. if (!is_present_gpte(pdptr)) {
  2576. vcpu->arch.mmu.pae_root[i] = 0;
  2577. continue;
  2578. }
  2579. root_gfn = pdptr >> PAGE_SHIFT;
  2580. if (mmu_check_root(vcpu, root_gfn))
  2581. return 1;
  2582. }
  2583. spin_lock(&vcpu->kvm->mmu_lock);
  2584. make_mmu_pages_available(vcpu);
  2585. sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
  2586. PT32_ROOT_LEVEL, 0,
  2587. ACC_ALL, NULL);
  2588. root = __pa(sp->spt);
  2589. ++sp->root_count;
  2590. spin_unlock(&vcpu->kvm->mmu_lock);
  2591. vcpu->arch.mmu.pae_root[i] = root | pm_mask;
  2592. }
  2593. vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
  2594. /*
  2595. * If we shadow a 32 bit page table with a long mode page
  2596. * table we enter this path.
  2597. */
  2598. if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
  2599. if (vcpu->arch.mmu.lm_root == NULL) {
  2600. /*
  2601. * The additional page necessary for this is only
  2602. * allocated on demand.
  2603. */
  2604. u64 *lm_root;
  2605. lm_root = (void*)get_zeroed_page(GFP_KERNEL);
  2606. if (lm_root == NULL)
  2607. return 1;
  2608. lm_root[0] = __pa(vcpu->arch.mmu.pae_root) | pm_mask;
  2609. vcpu->arch.mmu.lm_root = lm_root;
  2610. }
  2611. vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.lm_root);
  2612. }
  2613. return 0;
  2614. }
  2615. static int mmu_alloc_roots(struct kvm_vcpu *vcpu)
  2616. {
  2617. if (vcpu->arch.mmu.direct_map)
  2618. return mmu_alloc_direct_roots(vcpu);
  2619. else
  2620. return mmu_alloc_shadow_roots(vcpu);
  2621. }
  2622. static void mmu_sync_roots(struct kvm_vcpu *vcpu)
  2623. {
  2624. int i;
  2625. struct kvm_mmu_page *sp;
  2626. if (vcpu->arch.mmu.direct_map)
  2627. return;
  2628. if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
  2629. return;
  2630. vcpu_clear_mmio_info(vcpu, ~0ul);
  2631. kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);
  2632. if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
  2633. hpa_t root = vcpu->arch.mmu.root_hpa;
  2634. sp = page_header(root);
  2635. mmu_sync_children(vcpu, sp);
  2636. kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
  2637. return;
  2638. }
  2639. for (i = 0; i < 4; ++i) {
  2640. hpa_t root = vcpu->arch.mmu.pae_root[i];
  2641. if (root && VALID_PAGE(root)) {
  2642. root &= PT64_BASE_ADDR_MASK;
  2643. sp = page_header(root);
  2644. mmu_sync_children(vcpu, sp);
  2645. }
  2646. }
  2647. kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
  2648. }
  2649. void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
  2650. {
  2651. spin_lock(&vcpu->kvm->mmu_lock);
  2652. mmu_sync_roots(vcpu);
  2653. spin_unlock(&vcpu->kvm->mmu_lock);
  2654. }
  2655. static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr,
  2656. u32 access, struct x86_exception *exception)
  2657. {
  2658. if (exception)
  2659. exception->error_code = 0;
  2660. return vaddr;
  2661. }
  2662. static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gva_t vaddr,
  2663. u32 access,
  2664. struct x86_exception *exception)
  2665. {
  2666. if (exception)
  2667. exception->error_code = 0;
  2668. return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access);
  2669. }
  2670. static bool quickly_check_mmio_pf(struct kvm_vcpu *vcpu, u64 addr, bool direct)
  2671. {
  2672. if (direct)
  2673. return vcpu_match_mmio_gpa(vcpu, addr);
  2674. return vcpu_match_mmio_gva(vcpu, addr);
  2675. }
  2676. /*
  2677. * On direct hosts, the last spte is only allows two states
  2678. * for mmio page fault:
  2679. * - It is the mmio spte
  2680. * - It is zapped or it is being zapped.
  2681. *
  2682. * This function completely checks the spte when the last spte
  2683. * is not the mmio spte.
  2684. */
  2685. static bool check_direct_spte_mmio_pf(u64 spte)
  2686. {
  2687. return __check_direct_spte_mmio_pf(spte);
  2688. }
  2689. static u64 walk_shadow_page_get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr)
  2690. {
  2691. struct kvm_shadow_walk_iterator iterator;
  2692. u64 spte = 0ull;
  2693. walk_shadow_page_lockless_begin(vcpu);
  2694. for_each_shadow_entry_lockless(vcpu, addr, iterator, spte)
  2695. if (!is_shadow_present_pte(spte))
  2696. break;
  2697. walk_shadow_page_lockless_end(vcpu);
  2698. return spte;
  2699. }
  2700. int handle_mmio_page_fault_common(struct kvm_vcpu *vcpu, u64 addr, bool direct)
  2701. {
  2702. u64 spte;
  2703. if (quickly_check_mmio_pf(vcpu, addr, direct))
  2704. return RET_MMIO_PF_EMULATE;
  2705. spte = walk_shadow_page_get_mmio_spte(vcpu, addr);
  2706. if (is_mmio_spte(spte)) {
  2707. gfn_t gfn = get_mmio_spte_gfn(spte);
  2708. unsigned access = get_mmio_spte_access(spte);
  2709. if (!check_mmio_spte(vcpu->kvm, spte))
  2710. return RET_MMIO_PF_INVALID;
  2711. if (direct)
  2712. addr = 0;
  2713. trace_handle_mmio_page_fault(addr, gfn, access);
  2714. vcpu_cache_mmio_info(vcpu, addr, gfn, access);
  2715. return RET_MMIO_PF_EMULATE;
  2716. }
  2717. /*
  2718. * It's ok if the gva is remapped by other cpus on shadow guest,
  2719. * it's a BUG if the gfn is not a mmio page.
  2720. */
  2721. if (direct && !check_direct_spte_mmio_pf(spte))
  2722. return RET_MMIO_PF_BUG;
  2723. /*
  2724. * If the page table is zapped by other cpus, let CPU fault again on
  2725. * the address.
  2726. */
  2727. return RET_MMIO_PF_RETRY;
  2728. }
  2729. EXPORT_SYMBOL_GPL(handle_mmio_page_fault_common);
  2730. static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr,
  2731. u32 error_code, bool direct)
  2732. {
  2733. int ret;
  2734. ret = handle_mmio_page_fault_common(vcpu, addr, direct);
  2735. WARN_ON(ret == RET_MMIO_PF_BUG);
  2736. return ret;
  2737. }
  2738. static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
  2739. u32 error_code, bool prefault)
  2740. {
  2741. gfn_t gfn;
  2742. int r;
  2743. pgprintk("%s: gva %lx error %x\n", __func__, gva, error_code);
  2744. if (unlikely(error_code & PFERR_RSVD_MASK)) {
  2745. r = handle_mmio_page_fault(vcpu, gva, error_code, true);
  2746. if (likely(r != RET_MMIO_PF_INVALID))
  2747. return r;
  2748. }
  2749. r = mmu_topup_memory_caches(vcpu);
  2750. if (r)
  2751. return r;
  2752. ASSERT(vcpu);
  2753. ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
  2754. gfn = gva >> PAGE_SHIFT;
  2755. return nonpaging_map(vcpu, gva & PAGE_MASK,
  2756. error_code, gfn, prefault);
  2757. }
  2758. static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn)
  2759. {
  2760. struct kvm_arch_async_pf arch;
  2761. arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id;
  2762. arch.gfn = gfn;
  2763. arch.direct_map = vcpu->arch.mmu.direct_map;
  2764. arch.cr3 = vcpu->arch.mmu.get_cr3(vcpu);
  2765. return kvm_setup_async_pf(vcpu, gva, gfn, &arch);
  2766. }
  2767. static bool can_do_async_pf(struct kvm_vcpu *vcpu)
  2768. {
  2769. if (unlikely(!irqchip_in_kernel(vcpu->kvm) ||
  2770. kvm_event_needs_reinjection(vcpu)))
  2771. return false;
  2772. return kvm_x86_ops->interrupt_allowed(vcpu);
  2773. }
  2774. static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
  2775. gva_t gva, pfn_t *pfn, bool write, bool *writable)
  2776. {
  2777. bool async;
  2778. *pfn = gfn_to_pfn_async(vcpu->kvm, gfn, &async, write, writable);
  2779. if (!async)
  2780. return false; /* *pfn has correct page already */
  2781. if (!prefault && can_do_async_pf(vcpu)) {
  2782. trace_kvm_try_async_get_page(gva, gfn);
  2783. if (kvm_find_async_pf_gfn(vcpu, gfn)) {
  2784. trace_kvm_async_pf_doublefault(gva, gfn);
  2785. kvm_make_request(KVM_REQ_APF_HALT, vcpu);
  2786. return true;
  2787. } else if (kvm_arch_setup_async_pf(vcpu, gva, gfn))
  2788. return true;
  2789. }
  2790. *pfn = gfn_to_pfn_prot(vcpu->kvm, gfn, write, writable);
  2791. return false;
  2792. }
  2793. static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, u32 error_code,
  2794. bool prefault)
  2795. {
  2796. pfn_t pfn;
  2797. int r;
  2798. int level;
  2799. int force_pt_level;
  2800. gfn_t gfn = gpa >> PAGE_SHIFT;
  2801. unsigned long mmu_seq;
  2802. int write = error_code & PFERR_WRITE_MASK;
  2803. bool map_writable;
  2804. ASSERT(vcpu);
  2805. ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
  2806. if (unlikely(error_code & PFERR_RSVD_MASK)) {
  2807. r = handle_mmio_page_fault(vcpu, gpa, error_code, true);
  2808. if (likely(r != RET_MMIO_PF_INVALID))
  2809. return r;
  2810. }
  2811. r = mmu_topup_memory_caches(vcpu);
  2812. if (r)
  2813. return r;
  2814. force_pt_level = mapping_level_dirty_bitmap(vcpu, gfn);
  2815. if (likely(!force_pt_level)) {
  2816. level = mapping_level(vcpu, gfn);
  2817. gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
  2818. } else
  2819. level = PT_PAGE_TABLE_LEVEL;
  2820. if (fast_page_fault(vcpu, gpa, level, error_code))
  2821. return 0;
  2822. mmu_seq = vcpu->kvm->mmu_notifier_seq;
  2823. smp_rmb();
  2824. if (try_async_pf(vcpu, prefault, gfn, gpa, &pfn, write, &map_writable))
  2825. return 0;
  2826. if (handle_abnormal_pfn(vcpu, 0, gfn, pfn, ACC_ALL, &r))
  2827. return r;
  2828. spin_lock(&vcpu->kvm->mmu_lock);
  2829. if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
  2830. goto out_unlock;
  2831. make_mmu_pages_available(vcpu);
  2832. if (likely(!force_pt_level))
  2833. transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
  2834. r = __direct_map(vcpu, gpa, write, map_writable,
  2835. level, gfn, pfn, prefault);
  2836. spin_unlock(&vcpu->kvm->mmu_lock);
  2837. return r;
  2838. out_unlock:
  2839. spin_unlock(&vcpu->kvm->mmu_lock);
  2840. kvm_release_pfn_clean(pfn);
  2841. return 0;
  2842. }
  2843. static void nonpaging_free(struct kvm_vcpu *vcpu)
  2844. {
  2845. mmu_free_roots(vcpu);
  2846. }
  2847. static int nonpaging_init_context(struct kvm_vcpu *vcpu,
  2848. struct kvm_mmu *context)
  2849. {
  2850. context->new_cr3 = nonpaging_new_cr3;
  2851. context->page_fault = nonpaging_page_fault;
  2852. context->gva_to_gpa = nonpaging_gva_to_gpa;
  2853. context->free = nonpaging_free;
  2854. context->sync_page = nonpaging_sync_page;
  2855. context->invlpg = nonpaging_invlpg;
  2856. context->update_pte = nonpaging_update_pte;
  2857. context->root_level = 0;
  2858. context->shadow_root_level = PT32E_ROOT_LEVEL;
  2859. context->root_hpa = INVALID_PAGE;
  2860. context->direct_map = true;
  2861. context->nx = false;
  2862. return 0;
  2863. }
  2864. void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
  2865. {
  2866. ++vcpu->stat.tlb_flush;
  2867. kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
  2868. }
  2869. static void paging_new_cr3(struct kvm_vcpu *vcpu)
  2870. {
  2871. pgprintk("%s: cr3 %lx\n", __func__, kvm_read_cr3(vcpu));
  2872. mmu_free_roots(vcpu);
  2873. }
  2874. static unsigned long get_cr3(struct kvm_vcpu *vcpu)
  2875. {
  2876. return kvm_read_cr3(vcpu);
  2877. }
  2878. static void inject_page_fault(struct kvm_vcpu *vcpu,
  2879. struct x86_exception *fault)
  2880. {
  2881. vcpu->arch.mmu.inject_page_fault(vcpu, fault);
  2882. }
  2883. static void paging_free(struct kvm_vcpu *vcpu)
  2884. {
  2885. nonpaging_free(vcpu);
  2886. }
  2887. static inline void protect_clean_gpte(unsigned *access, unsigned gpte)
  2888. {
  2889. unsigned mask;
  2890. BUILD_BUG_ON(PT_WRITABLE_MASK != ACC_WRITE_MASK);
  2891. mask = (unsigned)~ACC_WRITE_MASK;
  2892. /* Allow write access to dirty gptes */
  2893. mask |= (gpte >> (PT_DIRTY_SHIFT - PT_WRITABLE_SHIFT)) & PT_WRITABLE_MASK;
  2894. *access &= mask;
  2895. }
  2896. static bool sync_mmio_spte(struct kvm *kvm, u64 *sptep, gfn_t gfn,
  2897. unsigned access, int *nr_present)
  2898. {
  2899. if (unlikely(is_mmio_spte(*sptep))) {
  2900. if (gfn != get_mmio_spte_gfn(*sptep)) {
  2901. mmu_spte_clear_no_track(sptep);
  2902. return true;
  2903. }
  2904. (*nr_present)++;
  2905. mark_mmio_spte(kvm, sptep, gfn, access);
  2906. return true;
  2907. }
  2908. return false;
  2909. }
  2910. static inline unsigned gpte_access(struct kvm_vcpu *vcpu, u64 gpte)
  2911. {
  2912. unsigned access;
  2913. access = (gpte & (PT_WRITABLE_MASK | PT_USER_MASK)) | ACC_EXEC_MASK;
  2914. access &= ~(gpte >> PT64_NX_SHIFT);
  2915. return access;
  2916. }
  2917. static inline bool is_last_gpte(struct kvm_mmu *mmu, unsigned level, unsigned gpte)
  2918. {
  2919. unsigned index;
  2920. index = level - 1;
  2921. index |= (gpte & PT_PAGE_SIZE_MASK) >> (PT_PAGE_SIZE_SHIFT - 2);
  2922. return mmu->last_pte_bitmap & (1 << index);
  2923. }
  2924. #define PTTYPE 64
  2925. #include "paging_tmpl.h"
  2926. #undef PTTYPE
  2927. #define PTTYPE 32
  2928. #include "paging_tmpl.h"
  2929. #undef PTTYPE
  2930. static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
  2931. struct kvm_mmu *context)
  2932. {
  2933. int maxphyaddr = cpuid_maxphyaddr(vcpu);
  2934. u64 exb_bit_rsvd = 0;
  2935. if (!context->nx)
  2936. exb_bit_rsvd = rsvd_bits(63, 63);
  2937. switch (context->root_level) {
  2938. case PT32_ROOT_LEVEL:
  2939. /* no rsvd bits for 2 level 4K page table entries */
  2940. context->rsvd_bits_mask[0][1] = 0;
  2941. context->rsvd_bits_mask[0][0] = 0;
  2942. context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
  2943. if (!is_pse(vcpu)) {
  2944. context->rsvd_bits_mask[1][1] = 0;
  2945. break;
  2946. }
  2947. if (is_cpuid_PSE36())
  2948. /* 36bits PSE 4MB page */
  2949. context->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
  2950. else
  2951. /* 32 bits PSE 4MB page */
  2952. context->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
  2953. break;
  2954. case PT32E_ROOT_LEVEL:
  2955. context->rsvd_bits_mask[0][2] =
  2956. rsvd_bits(maxphyaddr, 63) |
  2957. rsvd_bits(7, 8) | rsvd_bits(1, 2); /* PDPTE */
  2958. context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
  2959. rsvd_bits(maxphyaddr, 62); /* PDE */
  2960. context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
  2961. rsvd_bits(maxphyaddr, 62); /* PTE */
  2962. context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
  2963. rsvd_bits(maxphyaddr, 62) |
  2964. rsvd_bits(13, 20); /* large page */
  2965. context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
  2966. break;
  2967. case PT64_ROOT_LEVEL:
  2968. context->rsvd_bits_mask[0][3] = exb_bit_rsvd |
  2969. rsvd_bits(maxphyaddr, 51) | rsvd_bits(7, 8);
  2970. context->rsvd_bits_mask[0][2] = exb_bit_rsvd |
  2971. rsvd_bits(maxphyaddr, 51) | rsvd_bits(7, 8);
  2972. context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
  2973. rsvd_bits(maxphyaddr, 51);
  2974. context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
  2975. rsvd_bits(maxphyaddr, 51);
  2976. context->rsvd_bits_mask[1][3] = context->rsvd_bits_mask[0][3];
  2977. context->rsvd_bits_mask[1][2] = exb_bit_rsvd |
  2978. rsvd_bits(maxphyaddr, 51) |
  2979. rsvd_bits(13, 29);
  2980. context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
  2981. rsvd_bits(maxphyaddr, 51) |
  2982. rsvd_bits(13, 20); /* large page */
  2983. context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
  2984. break;
  2985. }
  2986. }
  2987. static void update_permission_bitmask(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
  2988. {
  2989. unsigned bit, byte, pfec;
  2990. u8 map;
  2991. bool fault, x, w, u, wf, uf, ff, smep;
  2992. smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
  2993. for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) {
  2994. pfec = byte << 1;
  2995. map = 0;
  2996. wf = pfec & PFERR_WRITE_MASK;
  2997. uf = pfec & PFERR_USER_MASK;
  2998. ff = pfec & PFERR_FETCH_MASK;
  2999. for (bit = 0; bit < 8; ++bit) {
  3000. x = bit & ACC_EXEC_MASK;
  3001. w = bit & ACC_WRITE_MASK;
  3002. u = bit & ACC_USER_MASK;
  3003. /* Not really needed: !nx will cause pte.nx to fault */
  3004. x |= !mmu->nx;
  3005. /* Allow supervisor writes if !cr0.wp */
  3006. w |= !is_write_protection(vcpu) && !uf;
  3007. /* Disallow supervisor fetches of user code if cr4.smep */
  3008. x &= !(smep && u && !uf);
  3009. fault = (ff && !x) || (uf && !u) || (wf && !w);
  3010. map |= fault << bit;
  3011. }
  3012. mmu->permissions[byte] = map;
  3013. }
  3014. }
  3015. static void update_last_pte_bitmap(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
  3016. {
  3017. u8 map;
  3018. unsigned level, root_level = mmu->root_level;
  3019. const unsigned ps_set_index = 1 << 2; /* bit 2 of index: ps */
  3020. if (root_level == PT32E_ROOT_LEVEL)
  3021. --root_level;
  3022. /* PT_PAGE_TABLE_LEVEL always terminates */
  3023. map = 1 | (1 << ps_set_index);
  3024. for (level = PT_DIRECTORY_LEVEL; level <= root_level; ++level) {
  3025. if (level <= PT_PDPE_LEVEL
  3026. && (mmu->root_level >= PT32E_ROOT_LEVEL || is_pse(vcpu)))
  3027. map |= 1 << (ps_set_index | (level - 1));
  3028. }
  3029. mmu->last_pte_bitmap = map;
  3030. }
  3031. static int paging64_init_context_common(struct kvm_vcpu *vcpu,
  3032. struct kvm_mmu *context,
  3033. int level)
  3034. {
  3035. context->nx = is_nx(vcpu);
  3036. context->root_level = level;
  3037. reset_rsvds_bits_mask(vcpu, context);
  3038. update_permission_bitmask(vcpu, context);
  3039. update_last_pte_bitmap(vcpu, context);
  3040. ASSERT(is_pae(vcpu));
  3041. context->new_cr3 = paging_new_cr3;
  3042. context->page_fault = paging64_page_fault;
  3043. context->gva_to_gpa = paging64_gva_to_gpa;
  3044. context->sync_page = paging64_sync_page;
  3045. context->invlpg = paging64_invlpg;
  3046. context->update_pte = paging64_update_pte;
  3047. context->free = paging_free;
  3048. context->shadow_root_level = level;
  3049. context->root_hpa = INVALID_PAGE;
  3050. context->direct_map = false;
  3051. return 0;
  3052. }
  3053. static int paging64_init_context(struct kvm_vcpu *vcpu,
  3054. struct kvm_mmu *context)
  3055. {
  3056. return paging64_init_context_common(vcpu, context, PT64_ROOT_LEVEL);
  3057. }
  3058. static int paging32_init_context(struct kvm_vcpu *vcpu,
  3059. struct kvm_mmu *context)
  3060. {
  3061. context->nx = false;
  3062. context->root_level = PT32_ROOT_LEVEL;
  3063. reset_rsvds_bits_mask(vcpu, context);
  3064. update_permission_bitmask(vcpu, context);
  3065. update_last_pte_bitmap(vcpu, context);
  3066. context->new_cr3 = paging_new_cr3;
  3067. context->page_fault = paging32_page_fault;
  3068. context->gva_to_gpa = paging32_gva_to_gpa;
  3069. context->free = paging_free;
  3070. context->sync_page = paging32_sync_page;
  3071. context->invlpg = paging32_invlpg;
  3072. context->update_pte = paging32_update_pte;
  3073. context->shadow_root_level = PT32E_ROOT_LEVEL;
  3074. context->root_hpa = INVALID_PAGE;
  3075. context->direct_map = false;
  3076. return 0;
  3077. }
  3078. static int paging32E_init_context(struct kvm_vcpu *vcpu,
  3079. struct kvm_mmu *context)
  3080. {
  3081. return paging64_init_context_common(vcpu, context, PT32E_ROOT_LEVEL);
  3082. }
  3083. static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
  3084. {
  3085. struct kvm_mmu *context = vcpu->arch.walk_mmu;
  3086. context->base_role.word = 0;
  3087. context->new_cr3 = nonpaging_new_cr3;
  3088. context->page_fault = tdp_page_fault;
  3089. context->free = nonpaging_free;
  3090. context->sync_page = nonpaging_sync_page;
  3091. context->invlpg = nonpaging_invlpg;
  3092. context->update_pte = nonpaging_update_pte;
  3093. context->shadow_root_level = kvm_x86_ops->get_tdp_level();
  3094. context->root_hpa = INVALID_PAGE;
  3095. context->direct_map = true;
  3096. context->set_cr3 = kvm_x86_ops->set_tdp_cr3;
  3097. context->get_cr3 = get_cr3;
  3098. context->get_pdptr = kvm_pdptr_read;
  3099. context->inject_page_fault = kvm_inject_page_fault;
  3100. if (!is_paging(vcpu)) {
  3101. context->nx = false;
  3102. context->gva_to_gpa = nonpaging_gva_to_gpa;
  3103. context->root_level = 0;
  3104. } else if (is_long_mode(vcpu)) {
  3105. context->nx = is_nx(vcpu);
  3106. context->root_level = PT64_ROOT_LEVEL;
  3107. reset_rsvds_bits_mask(vcpu, context);
  3108. context->gva_to_gpa = paging64_gva_to_gpa;
  3109. } else if (is_pae(vcpu)) {
  3110. context->nx = is_nx(vcpu);
  3111. context->root_level = PT32E_ROOT_LEVEL;
  3112. reset_rsvds_bits_mask(vcpu, context);
  3113. context->gva_to_gpa = paging64_gva_to_gpa;
  3114. } else {
  3115. context->nx = false;
  3116. context->root_level = PT32_ROOT_LEVEL;
  3117. reset_rsvds_bits_mask(vcpu, context);
  3118. context->gva_to_gpa = paging32_gva_to_gpa;
  3119. }
  3120. update_permission_bitmask(vcpu, context);
  3121. update_last_pte_bitmap(vcpu, context);
  3122. return 0;
  3123. }
  3124. int kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *context)
  3125. {
  3126. int r;
  3127. bool smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
  3128. ASSERT(vcpu);
  3129. ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
  3130. if (!is_paging(vcpu))
  3131. r = nonpaging_init_context(vcpu, context);
  3132. else if (is_long_mode(vcpu))
  3133. r = paging64_init_context(vcpu, context);
  3134. else if (is_pae(vcpu))
  3135. r = paging32E_init_context(vcpu, context);
  3136. else
  3137. r = paging32_init_context(vcpu, context);
  3138. vcpu->arch.mmu.base_role.nxe = is_nx(vcpu);
  3139. vcpu->arch.mmu.base_role.cr4_pae = !!is_pae(vcpu);
  3140. vcpu->arch.mmu.base_role.cr0_wp = is_write_protection(vcpu);
  3141. vcpu->arch.mmu.base_role.smep_andnot_wp
  3142. = smep && !is_write_protection(vcpu);
  3143. return r;
  3144. }
  3145. EXPORT_SYMBOL_GPL(kvm_init_shadow_mmu);
  3146. static int init_kvm_softmmu(struct kvm_vcpu *vcpu)
  3147. {
  3148. int r = kvm_init_shadow_mmu(vcpu, vcpu->arch.walk_mmu);
  3149. vcpu->arch.walk_mmu->set_cr3 = kvm_x86_ops->set_cr3;
  3150. vcpu->arch.walk_mmu->get_cr3 = get_cr3;
  3151. vcpu->arch.walk_mmu->get_pdptr = kvm_pdptr_read;
  3152. vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault;
  3153. return r;
  3154. }
  3155. static int init_kvm_nested_mmu(struct kvm_vcpu *vcpu)
  3156. {
  3157. struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;
  3158. g_context->get_cr3 = get_cr3;
  3159. g_context->get_pdptr = kvm_pdptr_read;
  3160. g_context->inject_page_fault = kvm_inject_page_fault;
  3161. /*
  3162. * Note that arch.mmu.gva_to_gpa translates l2_gva to l1_gpa. The
  3163. * translation of l2_gpa to l1_gpa addresses is done using the
  3164. * arch.nested_mmu.gva_to_gpa function. Basically the gva_to_gpa
  3165. * functions between mmu and nested_mmu are swapped.
  3166. */
  3167. if (!is_paging(vcpu)) {
  3168. g_context->nx = false;
  3169. g_context->root_level = 0;
  3170. g_context->gva_to_gpa = nonpaging_gva_to_gpa_nested;
  3171. } else if (is_long_mode(vcpu)) {
  3172. g_context->nx = is_nx(vcpu);
  3173. g_context->root_level = PT64_ROOT_LEVEL;
  3174. reset_rsvds_bits_mask(vcpu, g_context);
  3175. g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
  3176. } else if (is_pae(vcpu)) {
  3177. g_context->nx = is_nx(vcpu);
  3178. g_context->root_level = PT32E_ROOT_LEVEL;
  3179. reset_rsvds_bits_mask(vcpu, g_context);
  3180. g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
  3181. } else {
  3182. g_context->nx = false;
  3183. g_context->root_level = PT32_ROOT_LEVEL;
  3184. reset_rsvds_bits_mask(vcpu, g_context);
  3185. g_context->gva_to_gpa = paging32_gva_to_gpa_nested;
  3186. }
  3187. update_permission_bitmask(vcpu, g_context);
  3188. update_last_pte_bitmap(vcpu, g_context);
  3189. return 0;
  3190. }
  3191. static int init_kvm_mmu(struct kvm_vcpu *vcpu)
  3192. {
  3193. if (mmu_is_nested(vcpu))
  3194. return init_kvm_nested_mmu(vcpu);
  3195. else if (tdp_enabled)
  3196. return init_kvm_tdp_mmu(vcpu);
  3197. else
  3198. return init_kvm_softmmu(vcpu);
  3199. }
  3200. static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
  3201. {
  3202. ASSERT(vcpu);
  3203. if (VALID_PAGE(vcpu->arch.mmu.root_hpa))
  3204. /* mmu.free() should set root_hpa = INVALID_PAGE */
  3205. vcpu->arch.mmu.free(vcpu);
  3206. }
  3207. int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
  3208. {
  3209. destroy_kvm_mmu(vcpu);
  3210. return init_kvm_mmu(vcpu);
  3211. }
  3212. EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
  3213. int kvm_mmu_load(struct kvm_vcpu *vcpu)
  3214. {
  3215. int r;
  3216. r = mmu_topup_memory_caches(vcpu);
  3217. if (r)
  3218. goto out;
  3219. r = mmu_alloc_roots(vcpu);
  3220. kvm_mmu_sync_roots(vcpu);
  3221. if (r)
  3222. goto out;
  3223. /* set_cr3() should ensure TLB has been flushed */
  3224. vcpu->arch.mmu.set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
  3225. out:
  3226. return r;
  3227. }
  3228. EXPORT_SYMBOL_GPL(kvm_mmu_load);
  3229. void kvm_mmu_unload(struct kvm_vcpu *vcpu)
  3230. {
  3231. mmu_free_roots(vcpu);
  3232. }
  3233. EXPORT_SYMBOL_GPL(kvm_mmu_unload);
  3234. static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
  3235. struct kvm_mmu_page *sp, u64 *spte,
  3236. const void *new)
  3237. {
  3238. if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
  3239. ++vcpu->kvm->stat.mmu_pde_zapped;
  3240. return;
  3241. }
  3242. ++vcpu->kvm->stat.mmu_pte_updated;
  3243. vcpu->arch.mmu.update_pte(vcpu, sp, spte, new);
  3244. }
  3245. static bool need_remote_flush(u64 old, u64 new)
  3246. {
  3247. if (!is_shadow_present_pte(old))
  3248. return false;
  3249. if (!is_shadow_present_pte(new))
  3250. return true;
  3251. if ((old ^ new) & PT64_BASE_ADDR_MASK)
  3252. return true;
  3253. old ^= PT64_NX_MASK;
  3254. new ^= PT64_NX_MASK;
  3255. return (old & ~new & PT64_PERM_MASK) != 0;
  3256. }
  3257. static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, bool zap_page,
  3258. bool remote_flush, bool local_flush)
  3259. {
  3260. if (zap_page)
  3261. return;
  3262. if (remote_flush)
  3263. kvm_flush_remote_tlbs(vcpu->kvm);
  3264. else if (local_flush)
  3265. kvm_mmu_flush_tlb(vcpu);
  3266. }
  3267. static u64 mmu_pte_write_fetch_gpte(struct kvm_vcpu *vcpu, gpa_t *gpa,
  3268. const u8 *new, int *bytes)
  3269. {
  3270. u64 gentry;
  3271. int r;
  3272. /*
  3273. * Assume that the pte write on a page table of the same type
  3274. * as the current vcpu paging mode since we update the sptes only
  3275. * when they have the same mode.
  3276. */
  3277. if (is_pae(vcpu) && *bytes == 4) {
  3278. /* Handle a 32-bit guest writing two halves of a 64-bit gpte */
  3279. *gpa &= ~(gpa_t)7;
  3280. *bytes = 8;
  3281. r = kvm_read_guest(vcpu->kvm, *gpa, &gentry, 8);
  3282. if (r)
  3283. gentry = 0;
  3284. new = (const u8 *)&gentry;
  3285. }
  3286. switch (*bytes) {
  3287. case 4:
  3288. gentry = *(const u32 *)new;
  3289. break;
  3290. case 8:
  3291. gentry = *(const u64 *)new;
  3292. break;
  3293. default:
  3294. gentry = 0;
  3295. break;
  3296. }
  3297. return gentry;
  3298. }
  3299. /*
  3300. * If we're seeing too many writes to a page, it may no longer be a page table,
  3301. * or we may be forking, in which case it is better to unmap the page.
  3302. */
  3303. static bool detect_write_flooding(struct kvm_mmu_page *sp)
  3304. {
  3305. /*
  3306. * Skip write-flooding detected for the sp whose level is 1, because
  3307. * it can become unsync, then the guest page is not write-protected.
  3308. */
  3309. if (sp->role.level == PT_PAGE_TABLE_LEVEL)
  3310. return false;
  3311. return ++sp->write_flooding_count >= 3;
  3312. }
  3313. /*
  3314. * Misaligned accesses are too much trouble to fix up; also, they usually
  3315. * indicate a page is not used as a page table.
  3316. */
  3317. static bool detect_write_misaligned(struct kvm_mmu_page *sp, gpa_t gpa,
  3318. int bytes)
  3319. {
  3320. unsigned offset, pte_size, misaligned;
  3321. pgprintk("misaligned: gpa %llx bytes %d role %x\n",
  3322. gpa, bytes, sp->role.word);
  3323. offset = offset_in_page(gpa);
  3324. pte_size = sp->role.cr4_pae ? 8 : 4;
  3325. /*
  3326. * Sometimes, the OS only writes the last one bytes to update status
  3327. * bits, for example, in linux, andb instruction is used in clear_bit().
  3328. */
  3329. if (!(offset & (pte_size - 1)) && bytes == 1)
  3330. return false;
  3331. misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
  3332. misaligned |= bytes < 4;
  3333. return misaligned;
  3334. }
  3335. static u64 *get_written_sptes(struct kvm_mmu_page *sp, gpa_t gpa, int *nspte)
  3336. {
  3337. unsigned page_offset, quadrant;
  3338. u64 *spte;
  3339. int level;
  3340. page_offset = offset_in_page(gpa);
  3341. level = sp->role.level;
  3342. *nspte = 1;
  3343. if (!sp->role.cr4_pae) {
  3344. page_offset <<= 1; /* 32->64 */
  3345. /*
  3346. * A 32-bit pde maps 4MB while the shadow pdes map
  3347. * only 2MB. So we need to double the offset again
  3348. * and zap two pdes instead of one.
  3349. */
  3350. if (level == PT32_ROOT_LEVEL) {
  3351. page_offset &= ~7; /* kill rounding error */
  3352. page_offset <<= 1;
  3353. *nspte = 2;
  3354. }
  3355. quadrant = page_offset >> PAGE_SHIFT;
  3356. page_offset &= ~PAGE_MASK;
  3357. if (quadrant != sp->role.quadrant)
  3358. return NULL;
  3359. }
  3360. spte = &sp->spt[page_offset / sizeof(*spte)];
  3361. return spte;
  3362. }
  3363. void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
  3364. const u8 *new, int bytes)
  3365. {
  3366. gfn_t gfn = gpa >> PAGE_SHIFT;
  3367. union kvm_mmu_page_role mask = { .word = 0 };
  3368. struct kvm_mmu_page *sp;
  3369. LIST_HEAD(invalid_list);
  3370. u64 entry, gentry, *spte;
  3371. int npte;
  3372. bool remote_flush, local_flush, zap_page;
  3373. /*
  3374. * If we don't have indirect shadow pages, it means no page is
  3375. * write-protected, so we can exit simply.
  3376. */
  3377. if (!ACCESS_ONCE(vcpu->kvm->arch.indirect_shadow_pages))
  3378. return;
  3379. zap_page = remote_flush = local_flush = false;
  3380. pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);
  3381. gentry = mmu_pte_write_fetch_gpte(vcpu, &gpa, new, &bytes);
  3382. /*
  3383. * No need to care whether allocation memory is successful
  3384. * or not since pte prefetch is skiped if it does not have
  3385. * enough objects in the cache.
  3386. */
  3387. mmu_topup_memory_caches(vcpu);
  3388. spin_lock(&vcpu->kvm->mmu_lock);
  3389. ++vcpu->kvm->stat.mmu_pte_write;
  3390. kvm_mmu_audit(vcpu, AUDIT_PRE_PTE_WRITE);
  3391. mask.cr0_wp = mask.cr4_pae = mask.nxe = 1;
  3392. for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) {
  3393. if (detect_write_misaligned(sp, gpa, bytes) ||
  3394. detect_write_flooding(sp)) {
  3395. zap_page |= !!kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
  3396. &invalid_list);
  3397. ++vcpu->kvm->stat.mmu_flooded;
  3398. continue;
  3399. }
  3400. spte = get_written_sptes(sp, gpa, &npte);
  3401. if (!spte)
  3402. continue;
  3403. local_flush = true;
  3404. while (npte--) {
  3405. entry = *spte;
  3406. mmu_page_zap_pte(vcpu->kvm, sp, spte);
  3407. if (gentry &&
  3408. !((sp->role.word ^ vcpu->arch.mmu.base_role.word)
  3409. & mask.word) && rmap_can_add(vcpu))
  3410. mmu_pte_write_new_pte(vcpu, sp, spte, &gentry);
  3411. if (need_remote_flush(entry, *spte))
  3412. remote_flush = true;
  3413. ++spte;
  3414. }
  3415. }
  3416. mmu_pte_write_flush_tlb(vcpu, zap_page, remote_flush, local_flush);
  3417. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  3418. kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE);
  3419. spin_unlock(&vcpu->kvm->mmu_lock);
  3420. }
  3421. int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
  3422. {
  3423. gpa_t gpa;
  3424. int r;
  3425. if (vcpu->arch.mmu.direct_map)
  3426. return 0;
  3427. gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL);
  3428. r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
  3429. return r;
  3430. }
  3431. EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt);
  3432. static void make_mmu_pages_available(struct kvm_vcpu *vcpu)
  3433. {
  3434. LIST_HEAD(invalid_list);
  3435. if (likely(kvm_mmu_available_pages(vcpu->kvm) >= KVM_MIN_FREE_MMU_PAGES))
  3436. return;
  3437. while (kvm_mmu_available_pages(vcpu->kvm) < KVM_REFILL_PAGES) {
  3438. if (!prepare_zap_oldest_mmu_page(vcpu->kvm, &invalid_list))
  3439. break;
  3440. ++vcpu->kvm->stat.mmu_recycled;
  3441. }
  3442. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  3443. }
  3444. static bool is_mmio_page_fault(struct kvm_vcpu *vcpu, gva_t addr)
  3445. {
  3446. if (vcpu->arch.mmu.direct_map || mmu_is_nested(vcpu))
  3447. return vcpu_match_mmio_gpa(vcpu, addr);
  3448. return vcpu_match_mmio_gva(vcpu, addr);
  3449. }
  3450. int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code,
  3451. void *insn, int insn_len)
  3452. {
  3453. int r, emulation_type = EMULTYPE_RETRY;
  3454. enum emulation_result er;
  3455. r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code, false);
  3456. if (r < 0)
  3457. goto out;
  3458. if (!r) {
  3459. r = 1;
  3460. goto out;
  3461. }
  3462. if (is_mmio_page_fault(vcpu, cr2))
  3463. emulation_type = 0;
  3464. er = x86_emulate_instruction(vcpu, cr2, emulation_type, insn, insn_len);
  3465. switch (er) {
  3466. case EMULATE_DONE:
  3467. return 1;
  3468. case EMULATE_DO_MMIO:
  3469. ++vcpu->stat.mmio_exits;
  3470. /* fall through */
  3471. case EMULATE_FAIL:
  3472. return 0;
  3473. default:
  3474. BUG();
  3475. }
  3476. out:
  3477. return r;
  3478. }
  3479. EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
  3480. void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
  3481. {
  3482. vcpu->arch.mmu.invlpg(vcpu, gva);
  3483. kvm_mmu_flush_tlb(vcpu);
  3484. ++vcpu->stat.invlpg;
  3485. }
  3486. EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);
  3487. void kvm_enable_tdp(void)
  3488. {
  3489. tdp_enabled = true;
  3490. }
  3491. EXPORT_SYMBOL_GPL(kvm_enable_tdp);
  3492. void kvm_disable_tdp(void)
  3493. {
  3494. tdp_enabled = false;
  3495. }
  3496. EXPORT_SYMBOL_GPL(kvm_disable_tdp);
  3497. static void free_mmu_pages(struct kvm_vcpu *vcpu)
  3498. {
  3499. free_page((unsigned long)vcpu->arch.mmu.pae_root);
  3500. if (vcpu->arch.mmu.lm_root != NULL)
  3501. free_page((unsigned long)vcpu->arch.mmu.lm_root);
  3502. }
  3503. static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
  3504. {
  3505. struct page *page;
  3506. int i;
  3507. ASSERT(vcpu);
  3508. /*
  3509. * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
  3510. * Therefore we need to allocate shadow page tables in the first
  3511. * 4GB of memory, which happens to fit the DMA32 zone.
  3512. */
  3513. page = alloc_page(GFP_KERNEL | __GFP_DMA32);
  3514. if (!page)
  3515. return -ENOMEM;
  3516. vcpu->arch.mmu.pae_root = page_address(page);
  3517. for (i = 0; i < 4; ++i)
  3518. vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
  3519. return 0;
  3520. }
  3521. int kvm_mmu_create(struct kvm_vcpu *vcpu)
  3522. {
  3523. ASSERT(vcpu);
  3524. vcpu->arch.walk_mmu = &vcpu->arch.mmu;
  3525. vcpu->arch.mmu.root_hpa = INVALID_PAGE;
  3526. vcpu->arch.mmu.translate_gpa = translate_gpa;
  3527. vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa;
  3528. return alloc_mmu_pages(vcpu);
  3529. }
  3530. int kvm_mmu_setup(struct kvm_vcpu *vcpu)
  3531. {
  3532. ASSERT(vcpu);
  3533. ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
  3534. return init_kvm_mmu(vcpu);
  3535. }
  3536. void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
  3537. {
  3538. struct kvm_memory_slot *memslot;
  3539. gfn_t last_gfn;
  3540. int i;
  3541. memslot = id_to_memslot(kvm->memslots, slot);
  3542. last_gfn = memslot->base_gfn + memslot->npages - 1;
  3543. spin_lock(&kvm->mmu_lock);
  3544. for (i = PT_PAGE_TABLE_LEVEL;
  3545. i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
  3546. unsigned long *rmapp;
  3547. unsigned long last_index, index;
  3548. rmapp = memslot->arch.rmap[i - PT_PAGE_TABLE_LEVEL];
  3549. last_index = gfn_to_index(last_gfn, memslot->base_gfn, i);
  3550. for (index = 0; index <= last_index; ++index, ++rmapp) {
  3551. if (*rmapp)
  3552. __rmap_write_protect(kvm, rmapp, false);
  3553. if (need_resched() || spin_needbreak(&kvm->mmu_lock)) {
  3554. kvm_flush_remote_tlbs(kvm);
  3555. cond_resched_lock(&kvm->mmu_lock);
  3556. }
  3557. }
  3558. }
  3559. kvm_flush_remote_tlbs(kvm);
  3560. spin_unlock(&kvm->mmu_lock);
  3561. }
  3562. #define BATCH_ZAP_PAGES 10
  3563. static void kvm_zap_obsolete_pages(struct kvm *kvm)
  3564. {
  3565. struct kvm_mmu_page *sp, *node;
  3566. int batch = 0;
  3567. restart:
  3568. list_for_each_entry_safe_reverse(sp, node,
  3569. &kvm->arch.active_mmu_pages, link) {
  3570. int ret;
  3571. /*
  3572. * No obsolete page exists before new created page since
  3573. * active_mmu_pages is the FIFO list.
  3574. */
  3575. if (!is_obsolete_sp(kvm, sp))
  3576. break;
  3577. /*
  3578. * Since we are reversely walking the list and the invalid
  3579. * list will be moved to the head, skip the invalid page
  3580. * can help us to avoid the infinity list walking.
  3581. */
  3582. if (sp->role.invalid)
  3583. continue;
  3584. /*
  3585. * Need not flush tlb since we only zap the sp with invalid
  3586. * generation number.
  3587. */
  3588. if (batch >= BATCH_ZAP_PAGES &&
  3589. cond_resched_lock(&kvm->mmu_lock)) {
  3590. batch = 0;
  3591. goto restart;
  3592. }
  3593. ret = kvm_mmu_prepare_zap_page(kvm, sp,
  3594. &kvm->arch.zapped_obsolete_pages);
  3595. batch += ret;
  3596. if (ret)
  3597. goto restart;
  3598. }
  3599. /*
  3600. * Should flush tlb before free page tables since lockless-walking
  3601. * may use the pages.
  3602. */
  3603. kvm_mmu_commit_zap_page(kvm, &kvm->arch.zapped_obsolete_pages);
  3604. }
  3605. /*
  3606. * Fast invalidate all shadow pages and use lock-break technique
  3607. * to zap obsolete pages.
  3608. *
  3609. * It's required when memslot is being deleted or VM is being
  3610. * destroyed, in these cases, we should ensure that KVM MMU does
  3611. * not use any resource of the being-deleted slot or all slots
  3612. * after calling the function.
  3613. */
  3614. void kvm_mmu_invalidate_zap_all_pages(struct kvm *kvm)
  3615. {
  3616. spin_lock(&kvm->mmu_lock);
  3617. trace_kvm_mmu_invalidate_zap_all_pages(kvm);
  3618. kvm->arch.mmu_valid_gen++;
  3619. /*
  3620. * Notify all vcpus to reload its shadow page table
  3621. * and flush TLB. Then all vcpus will switch to new
  3622. * shadow page table with the new mmu_valid_gen.
  3623. *
  3624. * Note: we should do this under the protection of
  3625. * mmu-lock, otherwise, vcpu would purge shadow page
  3626. * but miss tlb flush.
  3627. */
  3628. kvm_reload_remote_mmus(kvm);
  3629. kvm_zap_obsolete_pages(kvm);
  3630. spin_unlock(&kvm->mmu_lock);
  3631. }
  3632. static bool kvm_has_zapped_obsolete_pages(struct kvm *kvm)
  3633. {
  3634. return unlikely(!list_empty_careful(&kvm->arch.zapped_obsolete_pages));
  3635. }
  3636. void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm)
  3637. {
  3638. /*
  3639. * The very rare case: if the generation-number is round,
  3640. * zap all shadow pages.
  3641. *
  3642. * The max value is MMIO_MAX_GEN - 1 since it is not called
  3643. * when mark memslot invalid.
  3644. */
  3645. if (unlikely(kvm_current_mmio_generation(kvm) >= (MMIO_MAX_GEN - 1))) {
  3646. printk_ratelimited(KERN_INFO "kvm: zapping shadow pages for mmio generation wraparound\n");
  3647. kvm_mmu_invalidate_zap_all_pages(kvm);
  3648. }
  3649. }
  3650. static int mmu_shrink(struct shrinker *shrink, struct shrink_control *sc)
  3651. {
  3652. struct kvm *kvm;
  3653. int nr_to_scan = sc->nr_to_scan;
  3654. if (nr_to_scan == 0)
  3655. goto out;
  3656. raw_spin_lock(&kvm_lock);
  3657. list_for_each_entry(kvm, &vm_list, vm_list) {
  3658. int idx;
  3659. LIST_HEAD(invalid_list);
  3660. /*
  3661. * Never scan more than sc->nr_to_scan VM instances.
  3662. * Will not hit this condition practically since we do not try
  3663. * to shrink more than one VM and it is very unlikely to see
  3664. * !n_used_mmu_pages so many times.
  3665. */
  3666. if (!nr_to_scan--)
  3667. break;
  3668. /*
  3669. * n_used_mmu_pages is accessed without holding kvm->mmu_lock
  3670. * here. We may skip a VM instance errorneosly, but we do not
  3671. * want to shrink a VM that only started to populate its MMU
  3672. * anyway.
  3673. */
  3674. if (!kvm->arch.n_used_mmu_pages &&
  3675. !kvm_has_zapped_obsolete_pages(kvm))
  3676. continue;
  3677. idx = srcu_read_lock(&kvm->srcu);
  3678. spin_lock(&kvm->mmu_lock);
  3679. if (kvm_has_zapped_obsolete_pages(kvm)) {
  3680. kvm_mmu_commit_zap_page(kvm,
  3681. &kvm->arch.zapped_obsolete_pages);
  3682. goto unlock;
  3683. }
  3684. prepare_zap_oldest_mmu_page(kvm, &invalid_list);
  3685. kvm_mmu_commit_zap_page(kvm, &invalid_list);
  3686. unlock:
  3687. spin_unlock(&kvm->mmu_lock);
  3688. srcu_read_unlock(&kvm->srcu, idx);
  3689. list_move_tail(&kvm->vm_list, &vm_list);
  3690. break;
  3691. }
  3692. raw_spin_unlock(&kvm_lock);
  3693. out:
  3694. return percpu_counter_read_positive(&kvm_total_used_mmu_pages);
  3695. }
  3696. static struct shrinker mmu_shrinker = {
  3697. .shrink = mmu_shrink,
  3698. .seeks = DEFAULT_SEEKS * 10,
  3699. };
  3700. static void mmu_destroy_caches(void)
  3701. {
  3702. if (pte_list_desc_cache)
  3703. kmem_cache_destroy(pte_list_desc_cache);
  3704. if (mmu_page_header_cache)
  3705. kmem_cache_destroy(mmu_page_header_cache);
  3706. }
  3707. int kvm_mmu_module_init(void)
  3708. {
  3709. pte_list_desc_cache = kmem_cache_create("pte_list_desc",
  3710. sizeof(struct pte_list_desc),
  3711. 0, 0, NULL);
  3712. if (!pte_list_desc_cache)
  3713. goto nomem;
  3714. mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
  3715. sizeof(struct kvm_mmu_page),
  3716. 0, 0, NULL);
  3717. if (!mmu_page_header_cache)
  3718. goto nomem;
  3719. if (percpu_counter_init(&kvm_total_used_mmu_pages, 0))
  3720. goto nomem;
  3721. register_shrinker(&mmu_shrinker);
  3722. return 0;
  3723. nomem:
  3724. mmu_destroy_caches();
  3725. return -ENOMEM;
  3726. }
  3727. /*
  3728. * Caculate mmu pages needed for kvm.
  3729. */
  3730. unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
  3731. {
  3732. unsigned int nr_mmu_pages;
  3733. unsigned int nr_pages = 0;
  3734. struct kvm_memslots *slots;
  3735. struct kvm_memory_slot *memslot;
  3736. slots = kvm_memslots(kvm);
  3737. kvm_for_each_memslot(memslot, slots)
  3738. nr_pages += memslot->npages;
  3739. nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
  3740. nr_mmu_pages = max(nr_mmu_pages,
  3741. (unsigned int) KVM_MIN_ALLOC_MMU_PAGES);
  3742. return nr_mmu_pages;
  3743. }
  3744. int kvm_mmu_get_spte_hierarchy(struct kvm_vcpu *vcpu, u64 addr, u64 sptes[4])
  3745. {
  3746. struct kvm_shadow_walk_iterator iterator;
  3747. u64 spte;
  3748. int nr_sptes = 0;
  3749. walk_shadow_page_lockless_begin(vcpu);
  3750. for_each_shadow_entry_lockless(vcpu, addr, iterator, spte) {
  3751. sptes[iterator.level-1] = spte;
  3752. nr_sptes++;
  3753. if (!is_shadow_present_pte(spte))
  3754. break;
  3755. }
  3756. walk_shadow_page_lockless_end(vcpu);
  3757. return nr_sptes;
  3758. }
  3759. EXPORT_SYMBOL_GPL(kvm_mmu_get_spte_hierarchy);
  3760. void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
  3761. {
  3762. ASSERT(vcpu);
  3763. destroy_kvm_mmu(vcpu);
  3764. free_mmu_pages(vcpu);
  3765. mmu_free_memory_caches(vcpu);
  3766. }
  3767. void kvm_mmu_module_exit(void)
  3768. {
  3769. mmu_destroy_caches();
  3770. percpu_counter_destroy(&kvm_total_used_mmu_pages);
  3771. unregister_shrinker(&mmu_shrinker);
  3772. mmu_audit_disable();
  3773. }