mmu.c 109 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * This module enables machines with Intel VT-x extensions to run virtual
  5. * machines without emulation or binary translation.
  6. *
  7. * MMU support
  8. *
  9. * Copyright (C) 2006 Qumranet, Inc.
  10. * Copyright 2010 Red Hat, Inc. and/or its affiliates.
  11. *
  12. * Authors:
  13. * Yaniv Kamay <yaniv@qumranet.com>
  14. * Avi Kivity <avi@qumranet.com>
  15. *
  16. * This work is licensed under the terms of the GNU GPL, version 2. See
  17. * the COPYING file in the top-level directory.
  18. *
  19. */
  20. #include "irq.h"
  21. #include "mmu.h"
  22. #include "x86.h"
  23. #include "kvm_cache_regs.h"
  24. #include <linux/kvm_host.h>
  25. #include <linux/types.h>
  26. #include <linux/string.h>
  27. #include <linux/mm.h>
  28. #include <linux/highmem.h>
  29. #include <linux/module.h>
  30. #include <linux/swap.h>
  31. #include <linux/hugetlb.h>
  32. #include <linux/compiler.h>
  33. #include <linux/srcu.h>
  34. #include <linux/slab.h>
  35. #include <linux/uaccess.h>
  36. #include <asm/page.h>
  37. #include <asm/cmpxchg.h>
  38. #include <asm/io.h>
  39. #include <asm/vmx.h>
  40. /*
  41. * When setting this variable to true it enables Two-Dimensional-Paging
  42. * where the hardware walks 2 page tables:
  43. * 1. the guest-virtual to guest-physical
  44. * 2. while doing 1. it walks guest-physical to host-physical
  45. * If the hardware supports that we don't need to do shadow paging.
  46. */
  47. bool tdp_enabled = false;
  48. enum {
  49. AUDIT_PRE_PAGE_FAULT,
  50. AUDIT_POST_PAGE_FAULT,
  51. AUDIT_PRE_PTE_WRITE,
  52. AUDIT_POST_PTE_WRITE,
  53. AUDIT_PRE_SYNC,
  54. AUDIT_POST_SYNC
  55. };
  56. #undef MMU_DEBUG
  57. #ifdef MMU_DEBUG
  58. #define pgprintk(x...) do { if (dbg) printk(x); } while (0)
  59. #define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
  60. #else
  61. #define pgprintk(x...) do { } while (0)
  62. #define rmap_printk(x...) do { } while (0)
  63. #endif
  64. #ifdef MMU_DEBUG
  65. static bool dbg = 0;
  66. module_param(dbg, bool, 0644);
  67. #endif
  68. #ifndef MMU_DEBUG
  69. #define ASSERT(x) do { } while (0)
  70. #else
  71. #define ASSERT(x) \
  72. if (!(x)) { \
  73. printk(KERN_WARNING "assertion failed %s:%d: %s\n", \
  74. __FILE__, __LINE__, #x); \
  75. }
  76. #endif
  77. #define PTE_PREFETCH_NUM 8
  78. #define PT_FIRST_AVAIL_BITS_SHIFT 10
  79. #define PT64_SECOND_AVAIL_BITS_SHIFT 52
  80. #define PT64_LEVEL_BITS 9
  81. #define PT64_LEVEL_SHIFT(level) \
  82. (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
  83. #define PT64_INDEX(address, level)\
  84. (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
  85. #define PT32_LEVEL_BITS 10
  86. #define PT32_LEVEL_SHIFT(level) \
  87. (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
  88. #define PT32_LVL_OFFSET_MASK(level) \
  89. (PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
  90. * PT32_LEVEL_BITS))) - 1))
  91. #define PT32_INDEX(address, level)\
  92. (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
  93. #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
  94. #define PT64_DIR_BASE_ADDR_MASK \
  95. (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
  96. #define PT64_LVL_ADDR_MASK(level) \
  97. (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
  98. * PT64_LEVEL_BITS))) - 1))
  99. #define PT64_LVL_OFFSET_MASK(level) \
  100. (PT64_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
  101. * PT64_LEVEL_BITS))) - 1))
  102. #define PT32_BASE_ADDR_MASK PAGE_MASK
  103. #define PT32_DIR_BASE_ADDR_MASK \
  104. (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
  105. #define PT32_LVL_ADDR_MASK(level) \
  106. (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
  107. * PT32_LEVEL_BITS))) - 1))
  108. #define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK \
  109. | PT64_NX_MASK)
  110. #define ACC_EXEC_MASK 1
  111. #define ACC_WRITE_MASK PT_WRITABLE_MASK
  112. #define ACC_USER_MASK PT_USER_MASK
  113. #define ACC_ALL (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
  114. #include <trace/events/kvm.h>
  115. #define CREATE_TRACE_POINTS
  116. #include "mmutrace.h"
  117. #define SPTE_HOST_WRITEABLE (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
  118. #define SPTE_MMU_WRITEABLE (1ULL << (PT_FIRST_AVAIL_BITS_SHIFT + 1))
  119. #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)
  120. /* make pte_list_desc fit well in cache line */
  121. #define PTE_LIST_EXT 3
  122. struct pte_list_desc {
  123. u64 *sptes[PTE_LIST_EXT];
  124. struct pte_list_desc *more;
  125. };
  126. struct kvm_shadow_walk_iterator {
  127. u64 addr;
  128. hpa_t shadow_addr;
  129. u64 *sptep;
  130. int level;
  131. unsigned index;
  132. };
  133. #define for_each_shadow_entry(_vcpu, _addr, _walker) \
  134. for (shadow_walk_init(&(_walker), _vcpu, _addr); \
  135. shadow_walk_okay(&(_walker)); \
  136. shadow_walk_next(&(_walker)))
  137. #define for_each_shadow_entry_lockless(_vcpu, _addr, _walker, spte) \
  138. for (shadow_walk_init(&(_walker), _vcpu, _addr); \
  139. shadow_walk_okay(&(_walker)) && \
  140. ({ spte = mmu_spte_get_lockless(_walker.sptep); 1; }); \
  141. __shadow_walk_next(&(_walker), spte))
  142. static struct kmem_cache *pte_list_desc_cache;
  143. static struct kmem_cache *mmu_page_header_cache;
  144. static struct percpu_counter kvm_total_used_mmu_pages;
  145. static u64 __read_mostly shadow_nx_mask;
  146. static u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
  147. static u64 __read_mostly shadow_user_mask;
  148. static u64 __read_mostly shadow_accessed_mask;
  149. static u64 __read_mostly shadow_dirty_mask;
  150. static u64 __read_mostly shadow_mmio_mask;
  151. static void mmu_spte_set(u64 *sptep, u64 spte);
  152. static void mmu_free_roots(struct kvm_vcpu *vcpu);
  153. void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask)
  154. {
  155. shadow_mmio_mask = mmio_mask;
  156. }
  157. EXPORT_SYMBOL_GPL(kvm_mmu_set_mmio_spte_mask);
  158. /*
  159. * spte bits of bit 3 ~ bit 11 are used as low 9 bits of generation number,
  160. * the bits of bits 52 ~ bit 61 are used as high 10 bits of generation
  161. * number.
  162. */
  163. #define MMIO_SPTE_GEN_LOW_SHIFT 3
  164. #define MMIO_SPTE_GEN_HIGH_SHIFT 52
  165. #define MMIO_GEN_SHIFT 19
  166. #define MMIO_GEN_LOW_SHIFT 9
  167. #define MMIO_GEN_LOW_MASK ((1 << MMIO_GEN_LOW_SHIFT) - 1)
  168. #define MMIO_GEN_MASK ((1 << MMIO_GEN_SHIFT) - 1)
  169. #define MMIO_MAX_GEN ((1 << MMIO_GEN_SHIFT) - 1)
  170. static u64 generation_mmio_spte_mask(unsigned int gen)
  171. {
  172. u64 mask;
  173. WARN_ON(gen > MMIO_MAX_GEN);
  174. mask = (gen & MMIO_GEN_LOW_MASK) << MMIO_SPTE_GEN_LOW_SHIFT;
  175. mask |= ((u64)gen >> MMIO_GEN_LOW_SHIFT) << MMIO_SPTE_GEN_HIGH_SHIFT;
  176. return mask;
  177. }
  178. static unsigned int get_mmio_spte_generation(u64 spte)
  179. {
  180. unsigned int gen;
  181. spte &= ~shadow_mmio_mask;
  182. gen = (spte >> MMIO_SPTE_GEN_LOW_SHIFT) & MMIO_GEN_LOW_MASK;
  183. gen |= (spte >> MMIO_SPTE_GEN_HIGH_SHIFT) << MMIO_GEN_LOW_SHIFT;
  184. return gen;
  185. }
  186. static unsigned int kvm_current_mmio_generation(struct kvm *kvm)
  187. {
  188. /*
  189. * Init kvm generation close to MMIO_MAX_GEN to easily test the
  190. * code of handling generation number wrap-around.
  191. */
  192. return (kvm_memslots(kvm)->generation +
  193. MMIO_MAX_GEN - 150) & MMIO_GEN_MASK;
  194. }
  195. static void mark_mmio_spte(struct kvm *kvm, u64 *sptep, u64 gfn,
  196. unsigned access)
  197. {
  198. unsigned int gen = kvm_current_mmio_generation(kvm);
  199. u64 mask = generation_mmio_spte_mask(gen);
  200. access &= ACC_WRITE_MASK | ACC_USER_MASK;
  201. mask |= shadow_mmio_mask | access | gfn << PAGE_SHIFT;
  202. trace_mark_mmio_spte(sptep, gfn, access, gen);
  203. mmu_spte_set(sptep, mask);
  204. }
  205. static bool is_mmio_spte(u64 spte)
  206. {
  207. return (spte & shadow_mmio_mask) == shadow_mmio_mask;
  208. }
  209. static gfn_t get_mmio_spte_gfn(u64 spte)
  210. {
  211. u64 mask = generation_mmio_spte_mask(MMIO_MAX_GEN) | shadow_mmio_mask;
  212. return (spte & ~mask) >> PAGE_SHIFT;
  213. }
  214. static unsigned get_mmio_spte_access(u64 spte)
  215. {
  216. u64 mask = generation_mmio_spte_mask(MMIO_MAX_GEN) | shadow_mmio_mask;
  217. return (spte & ~mask) & ~PAGE_MASK;
  218. }
  219. static bool set_mmio_spte(struct kvm *kvm, u64 *sptep, gfn_t gfn,
  220. pfn_t pfn, unsigned access)
  221. {
  222. if (unlikely(is_noslot_pfn(pfn))) {
  223. mark_mmio_spte(kvm, sptep, gfn, access);
  224. return true;
  225. }
  226. return false;
  227. }
  228. static bool check_mmio_spte(struct kvm *kvm, u64 spte)
  229. {
  230. unsigned int kvm_gen, spte_gen;
  231. kvm_gen = kvm_current_mmio_generation(kvm);
  232. spte_gen = get_mmio_spte_generation(spte);
  233. trace_check_mmio_spte(spte, kvm_gen, spte_gen);
  234. return likely(kvm_gen == spte_gen);
  235. }
  236. static inline u64 rsvd_bits(int s, int e)
  237. {
  238. return ((1ULL << (e - s + 1)) - 1) << s;
  239. }
  240. void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
  241. u64 dirty_mask, u64 nx_mask, u64 x_mask)
  242. {
  243. shadow_user_mask = user_mask;
  244. shadow_accessed_mask = accessed_mask;
  245. shadow_dirty_mask = dirty_mask;
  246. shadow_nx_mask = nx_mask;
  247. shadow_x_mask = x_mask;
  248. }
  249. EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);
  250. static int is_cpuid_PSE36(void)
  251. {
  252. return 1;
  253. }
  254. static int is_nx(struct kvm_vcpu *vcpu)
  255. {
  256. return vcpu->arch.efer & EFER_NX;
  257. }
  258. static int is_shadow_present_pte(u64 pte)
  259. {
  260. return pte & PT_PRESENT_MASK && !is_mmio_spte(pte);
  261. }
  262. static int is_large_pte(u64 pte)
  263. {
  264. return pte & PT_PAGE_SIZE_MASK;
  265. }
  266. static int is_dirty_gpte(unsigned long pte)
  267. {
  268. return pte & PT_DIRTY_MASK;
  269. }
  270. static int is_rmap_spte(u64 pte)
  271. {
  272. return is_shadow_present_pte(pte);
  273. }
  274. static int is_last_spte(u64 pte, int level)
  275. {
  276. if (level == PT_PAGE_TABLE_LEVEL)
  277. return 1;
  278. if (is_large_pte(pte))
  279. return 1;
  280. return 0;
  281. }
  282. static pfn_t spte_to_pfn(u64 pte)
  283. {
  284. return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
  285. }
  286. static gfn_t pse36_gfn_delta(u32 gpte)
  287. {
  288. int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;
  289. return (gpte & PT32_DIR_PSE36_MASK) << shift;
  290. }
  291. #ifdef CONFIG_X86_64
  292. static void __set_spte(u64 *sptep, u64 spte)
  293. {
  294. *sptep = spte;
  295. }
  296. static void __update_clear_spte_fast(u64 *sptep, u64 spte)
  297. {
  298. *sptep = spte;
  299. }
  300. static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
  301. {
  302. return xchg(sptep, spte);
  303. }
  304. static u64 __get_spte_lockless(u64 *sptep)
  305. {
  306. return ACCESS_ONCE(*sptep);
  307. }
  308. static bool __check_direct_spte_mmio_pf(u64 spte)
  309. {
  310. /* It is valid if the spte is zapped. */
  311. return spte == 0ull;
  312. }
  313. #else
  314. union split_spte {
  315. struct {
  316. u32 spte_low;
  317. u32 spte_high;
  318. };
  319. u64 spte;
  320. };
  321. static void count_spte_clear(u64 *sptep, u64 spte)
  322. {
  323. struct kvm_mmu_page *sp = page_header(__pa(sptep));
  324. if (is_shadow_present_pte(spte))
  325. return;
  326. /* Ensure the spte is completely set before we increase the count */
  327. smp_wmb();
  328. sp->clear_spte_count++;
  329. }
  330. static void __set_spte(u64 *sptep, u64 spte)
  331. {
  332. union split_spte *ssptep, sspte;
  333. ssptep = (union split_spte *)sptep;
  334. sspte = (union split_spte)spte;
  335. ssptep->spte_high = sspte.spte_high;
  336. /*
  337. * If we map the spte from nonpresent to present, We should store
  338. * the high bits firstly, then set present bit, so cpu can not
  339. * fetch this spte while we are setting the spte.
  340. */
  341. smp_wmb();
  342. ssptep->spte_low = sspte.spte_low;
  343. }
  344. static void __update_clear_spte_fast(u64 *sptep, u64 spte)
  345. {
  346. union split_spte *ssptep, sspte;
  347. ssptep = (union split_spte *)sptep;
  348. sspte = (union split_spte)spte;
  349. ssptep->spte_low = sspte.spte_low;
  350. /*
  351. * If we map the spte from present to nonpresent, we should clear
  352. * present bit firstly to avoid vcpu fetch the old high bits.
  353. */
  354. smp_wmb();
  355. ssptep->spte_high = sspte.spte_high;
  356. count_spte_clear(sptep, spte);
  357. }
  358. static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
  359. {
  360. union split_spte *ssptep, sspte, orig;
  361. ssptep = (union split_spte *)sptep;
  362. sspte = (union split_spte)spte;
  363. /* xchg acts as a barrier before the setting of the high bits */
  364. orig.spte_low = xchg(&ssptep->spte_low, sspte.spte_low);
  365. orig.spte_high = ssptep->spte_high;
  366. ssptep->spte_high = sspte.spte_high;
  367. count_spte_clear(sptep, spte);
  368. return orig.spte;
  369. }
  370. /*
  371. * The idea using the light way get the spte on x86_32 guest is from
  372. * gup_get_pte(arch/x86/mm/gup.c).
  373. * The difference is we can not catch the spte tlb flush if we leave
  374. * guest mode, so we emulate it by increase clear_spte_count when spte
  375. * is cleared.
  376. */
  377. static u64 __get_spte_lockless(u64 *sptep)
  378. {
  379. struct kvm_mmu_page *sp = page_header(__pa(sptep));
  380. union split_spte spte, *orig = (union split_spte *)sptep;
  381. int count;
  382. retry:
  383. count = sp->clear_spte_count;
  384. smp_rmb();
  385. spte.spte_low = orig->spte_low;
  386. smp_rmb();
  387. spte.spte_high = orig->spte_high;
  388. smp_rmb();
  389. if (unlikely(spte.spte_low != orig->spte_low ||
  390. count != sp->clear_spte_count))
  391. goto retry;
  392. return spte.spte;
  393. }
  394. static bool __check_direct_spte_mmio_pf(u64 spte)
  395. {
  396. union split_spte sspte = (union split_spte)spte;
  397. u32 high_mmio_mask = shadow_mmio_mask >> 32;
  398. /* It is valid if the spte is zapped. */
  399. if (spte == 0ull)
  400. return true;
  401. /* It is valid if the spte is being zapped. */
  402. if (sspte.spte_low == 0ull &&
  403. (sspte.spte_high & high_mmio_mask) == high_mmio_mask)
  404. return true;
  405. return false;
  406. }
  407. #endif
  408. static bool spte_is_locklessly_modifiable(u64 spte)
  409. {
  410. return (spte & (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE)) ==
  411. (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE);
  412. }
  413. static bool spte_has_volatile_bits(u64 spte)
  414. {
  415. /*
  416. * Always atomicly update spte if it can be updated
  417. * out of mmu-lock, it can ensure dirty bit is not lost,
  418. * also, it can help us to get a stable is_writable_pte()
  419. * to ensure tlb flush is not missed.
  420. */
  421. if (spte_is_locklessly_modifiable(spte))
  422. return true;
  423. if (!shadow_accessed_mask)
  424. return false;
  425. if (!is_shadow_present_pte(spte))
  426. return false;
  427. if ((spte & shadow_accessed_mask) &&
  428. (!is_writable_pte(spte) || (spte & shadow_dirty_mask)))
  429. return false;
  430. return true;
  431. }
  432. static bool spte_is_bit_cleared(u64 old_spte, u64 new_spte, u64 bit_mask)
  433. {
  434. return (old_spte & bit_mask) && !(new_spte & bit_mask);
  435. }
  436. /* Rules for using mmu_spte_set:
  437. * Set the sptep from nonpresent to present.
  438. * Note: the sptep being assigned *must* be either not present
  439. * or in a state where the hardware will not attempt to update
  440. * the spte.
  441. */
  442. static void mmu_spte_set(u64 *sptep, u64 new_spte)
  443. {
  444. WARN_ON(is_shadow_present_pte(*sptep));
  445. __set_spte(sptep, new_spte);
  446. }
  447. /* Rules for using mmu_spte_update:
  448. * Update the state bits, it means the mapped pfn is not changged.
  449. *
  450. * Whenever we overwrite a writable spte with a read-only one we
  451. * should flush remote TLBs. Otherwise rmap_write_protect
  452. * will find a read-only spte, even though the writable spte
  453. * might be cached on a CPU's TLB, the return value indicates this
  454. * case.
  455. */
  456. static bool mmu_spte_update(u64 *sptep, u64 new_spte)
  457. {
  458. u64 old_spte = *sptep;
  459. bool ret = false;
  460. WARN_ON(!is_rmap_spte(new_spte));
  461. if (!is_shadow_present_pte(old_spte)) {
  462. mmu_spte_set(sptep, new_spte);
  463. return ret;
  464. }
  465. if (!spte_has_volatile_bits(old_spte))
  466. __update_clear_spte_fast(sptep, new_spte);
  467. else
  468. old_spte = __update_clear_spte_slow(sptep, new_spte);
  469. /*
  470. * For the spte updated out of mmu-lock is safe, since
  471. * we always atomicly update it, see the comments in
  472. * spte_has_volatile_bits().
  473. */
  474. if (is_writable_pte(old_spte) && !is_writable_pte(new_spte))
  475. ret = true;
  476. if (!shadow_accessed_mask)
  477. return ret;
  478. if (spte_is_bit_cleared(old_spte, new_spte, shadow_accessed_mask))
  479. kvm_set_pfn_accessed(spte_to_pfn(old_spte));
  480. if (spte_is_bit_cleared(old_spte, new_spte, shadow_dirty_mask))
  481. kvm_set_pfn_dirty(spte_to_pfn(old_spte));
  482. return ret;
  483. }
  484. /*
  485. * Rules for using mmu_spte_clear_track_bits:
  486. * It sets the sptep from present to nonpresent, and track the
  487. * state bits, it is used to clear the last level sptep.
  488. */
  489. static int mmu_spte_clear_track_bits(u64 *sptep)
  490. {
  491. pfn_t pfn;
  492. u64 old_spte = *sptep;
  493. if (!spte_has_volatile_bits(old_spte))
  494. __update_clear_spte_fast(sptep, 0ull);
  495. else
  496. old_spte = __update_clear_spte_slow(sptep, 0ull);
  497. if (!is_rmap_spte(old_spte))
  498. return 0;
  499. pfn = spte_to_pfn(old_spte);
  500. /*
  501. * KVM does not hold the refcount of the page used by
  502. * kvm mmu, before reclaiming the page, we should
  503. * unmap it from mmu first.
  504. */
  505. WARN_ON(!kvm_is_mmio_pfn(pfn) && !page_count(pfn_to_page(pfn)));
  506. if (!shadow_accessed_mask || old_spte & shadow_accessed_mask)
  507. kvm_set_pfn_accessed(pfn);
  508. if (!shadow_dirty_mask || (old_spte & shadow_dirty_mask))
  509. kvm_set_pfn_dirty(pfn);
  510. return 1;
  511. }
  512. /*
  513. * Rules for using mmu_spte_clear_no_track:
  514. * Directly clear spte without caring the state bits of sptep,
  515. * it is used to set the upper level spte.
  516. */
  517. static void mmu_spte_clear_no_track(u64 *sptep)
  518. {
  519. __update_clear_spte_fast(sptep, 0ull);
  520. }
  521. static u64 mmu_spte_get_lockless(u64 *sptep)
  522. {
  523. return __get_spte_lockless(sptep);
  524. }
  525. static void walk_shadow_page_lockless_begin(struct kvm_vcpu *vcpu)
  526. {
  527. /*
  528. * Prevent page table teardown by making any free-er wait during
  529. * kvm_flush_remote_tlbs() IPI to all active vcpus.
  530. */
  531. local_irq_disable();
  532. vcpu->mode = READING_SHADOW_PAGE_TABLES;
  533. /*
  534. * Make sure a following spte read is not reordered ahead of the write
  535. * to vcpu->mode.
  536. */
  537. smp_mb();
  538. }
  539. static void walk_shadow_page_lockless_end(struct kvm_vcpu *vcpu)
  540. {
  541. /*
  542. * Make sure the write to vcpu->mode is not reordered in front of
  543. * reads to sptes. If it does, kvm_commit_zap_page() can see us
  544. * OUTSIDE_GUEST_MODE and proceed to free the shadow page table.
  545. */
  546. smp_mb();
  547. vcpu->mode = OUTSIDE_GUEST_MODE;
  548. local_irq_enable();
  549. }
  550. static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
  551. struct kmem_cache *base_cache, int min)
  552. {
  553. void *obj;
  554. if (cache->nobjs >= min)
  555. return 0;
  556. while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
  557. obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
  558. if (!obj)
  559. return -ENOMEM;
  560. cache->objects[cache->nobjs++] = obj;
  561. }
  562. return 0;
  563. }
  564. static int mmu_memory_cache_free_objects(struct kvm_mmu_memory_cache *cache)
  565. {
  566. return cache->nobjs;
  567. }
  568. static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc,
  569. struct kmem_cache *cache)
  570. {
  571. while (mc->nobjs)
  572. kmem_cache_free(cache, mc->objects[--mc->nobjs]);
  573. }
  574. static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
  575. int min)
  576. {
  577. void *page;
  578. if (cache->nobjs >= min)
  579. return 0;
  580. while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
  581. page = (void *)__get_free_page(GFP_KERNEL);
  582. if (!page)
  583. return -ENOMEM;
  584. cache->objects[cache->nobjs++] = page;
  585. }
  586. return 0;
  587. }
  588. static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
  589. {
  590. while (mc->nobjs)
  591. free_page((unsigned long)mc->objects[--mc->nobjs]);
  592. }
  593. static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
  594. {
  595. int r;
  596. r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
  597. pte_list_desc_cache, 8 + PTE_PREFETCH_NUM);
  598. if (r)
  599. goto out;
  600. r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
  601. if (r)
  602. goto out;
  603. r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
  604. mmu_page_header_cache, 4);
  605. out:
  606. return r;
  607. }
  608. static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
  609. {
  610. mmu_free_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
  611. pte_list_desc_cache);
  612. mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
  613. mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache,
  614. mmu_page_header_cache);
  615. }
  616. static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
  617. {
  618. void *p;
  619. BUG_ON(!mc->nobjs);
  620. p = mc->objects[--mc->nobjs];
  621. return p;
  622. }
  623. static struct pte_list_desc *mmu_alloc_pte_list_desc(struct kvm_vcpu *vcpu)
  624. {
  625. return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_list_desc_cache);
  626. }
  627. static void mmu_free_pte_list_desc(struct pte_list_desc *pte_list_desc)
  628. {
  629. kmem_cache_free(pte_list_desc_cache, pte_list_desc);
  630. }
  631. static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index)
  632. {
  633. if (!sp->role.direct)
  634. return sp->gfns[index];
  635. return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS));
  636. }
  637. static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn)
  638. {
  639. if (sp->role.direct)
  640. BUG_ON(gfn != kvm_mmu_page_get_gfn(sp, index));
  641. else
  642. sp->gfns[index] = gfn;
  643. }
  644. /*
  645. * Return the pointer to the large page information for a given gfn,
  646. * handling slots that are not large page aligned.
  647. */
  648. static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn,
  649. struct kvm_memory_slot *slot,
  650. int level)
  651. {
  652. unsigned long idx;
  653. idx = gfn_to_index(gfn, slot->base_gfn, level);
  654. return &slot->arch.lpage_info[level - 2][idx];
  655. }
  656. static void account_shadowed(struct kvm *kvm, gfn_t gfn)
  657. {
  658. struct kvm_memory_slot *slot;
  659. struct kvm_lpage_info *linfo;
  660. int i;
  661. slot = gfn_to_memslot(kvm, gfn);
  662. for (i = PT_DIRECTORY_LEVEL;
  663. i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
  664. linfo = lpage_info_slot(gfn, slot, i);
  665. linfo->write_count += 1;
  666. }
  667. kvm->arch.indirect_shadow_pages++;
  668. }
  669. static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn)
  670. {
  671. struct kvm_memory_slot *slot;
  672. struct kvm_lpage_info *linfo;
  673. int i;
  674. slot = gfn_to_memslot(kvm, gfn);
  675. for (i = PT_DIRECTORY_LEVEL;
  676. i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
  677. linfo = lpage_info_slot(gfn, slot, i);
  678. linfo->write_count -= 1;
  679. WARN_ON(linfo->write_count < 0);
  680. }
  681. kvm->arch.indirect_shadow_pages--;
  682. }
  683. static int has_wrprotected_page(struct kvm *kvm,
  684. gfn_t gfn,
  685. int level)
  686. {
  687. struct kvm_memory_slot *slot;
  688. struct kvm_lpage_info *linfo;
  689. slot = gfn_to_memslot(kvm, gfn);
  690. if (slot) {
  691. linfo = lpage_info_slot(gfn, slot, level);
  692. return linfo->write_count;
  693. }
  694. return 1;
  695. }
  696. static int host_mapping_level(struct kvm *kvm, gfn_t gfn)
  697. {
  698. unsigned long page_size;
  699. int i, ret = 0;
  700. page_size = kvm_host_page_size(kvm, gfn);
  701. for (i = PT_PAGE_TABLE_LEVEL;
  702. i < (PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES); ++i) {
  703. if (page_size >= KVM_HPAGE_SIZE(i))
  704. ret = i;
  705. else
  706. break;
  707. }
  708. return ret;
  709. }
  710. static struct kvm_memory_slot *
  711. gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t gfn,
  712. bool no_dirty_log)
  713. {
  714. struct kvm_memory_slot *slot;
  715. slot = gfn_to_memslot(vcpu->kvm, gfn);
  716. if (!slot || slot->flags & KVM_MEMSLOT_INVALID ||
  717. (no_dirty_log && slot->dirty_bitmap))
  718. slot = NULL;
  719. return slot;
  720. }
  721. static bool mapping_level_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t large_gfn)
  722. {
  723. return !gfn_to_memslot_dirty_bitmap(vcpu, large_gfn, true);
  724. }
  725. static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn)
  726. {
  727. int host_level, level, max_level;
  728. host_level = host_mapping_level(vcpu->kvm, large_gfn);
  729. if (host_level == PT_PAGE_TABLE_LEVEL)
  730. return host_level;
  731. max_level = min(kvm_x86_ops->get_lpage_level(), host_level);
  732. for (level = PT_DIRECTORY_LEVEL; level <= max_level; ++level)
  733. if (has_wrprotected_page(vcpu->kvm, large_gfn, level))
  734. break;
  735. return level - 1;
  736. }
  737. /*
  738. * Pte mapping structures:
  739. *
  740. * If pte_list bit zero is zero, then pte_list point to the spte.
  741. *
  742. * If pte_list bit zero is one, (then pte_list & ~1) points to a struct
  743. * pte_list_desc containing more mappings.
  744. *
  745. * Returns the number of pte entries before the spte was added or zero if
  746. * the spte was not added.
  747. *
  748. */
  749. static int pte_list_add(struct kvm_vcpu *vcpu, u64 *spte,
  750. unsigned long *pte_list)
  751. {
  752. struct pte_list_desc *desc;
  753. int i, count = 0;
  754. if (!*pte_list) {
  755. rmap_printk("pte_list_add: %p %llx 0->1\n", spte, *spte);
  756. *pte_list = (unsigned long)spte;
  757. } else if (!(*pte_list & 1)) {
  758. rmap_printk("pte_list_add: %p %llx 1->many\n", spte, *spte);
  759. desc = mmu_alloc_pte_list_desc(vcpu);
  760. desc->sptes[0] = (u64 *)*pte_list;
  761. desc->sptes[1] = spte;
  762. *pte_list = (unsigned long)desc | 1;
  763. ++count;
  764. } else {
  765. rmap_printk("pte_list_add: %p %llx many->many\n", spte, *spte);
  766. desc = (struct pte_list_desc *)(*pte_list & ~1ul);
  767. while (desc->sptes[PTE_LIST_EXT-1] && desc->more) {
  768. desc = desc->more;
  769. count += PTE_LIST_EXT;
  770. }
  771. if (desc->sptes[PTE_LIST_EXT-1]) {
  772. desc->more = mmu_alloc_pte_list_desc(vcpu);
  773. desc = desc->more;
  774. }
  775. for (i = 0; desc->sptes[i]; ++i)
  776. ++count;
  777. desc->sptes[i] = spte;
  778. }
  779. return count;
  780. }
  781. static void
  782. pte_list_desc_remove_entry(unsigned long *pte_list, struct pte_list_desc *desc,
  783. int i, struct pte_list_desc *prev_desc)
  784. {
  785. int j;
  786. for (j = PTE_LIST_EXT - 1; !desc->sptes[j] && j > i; --j)
  787. ;
  788. desc->sptes[i] = desc->sptes[j];
  789. desc->sptes[j] = NULL;
  790. if (j != 0)
  791. return;
  792. if (!prev_desc && !desc->more)
  793. *pte_list = (unsigned long)desc->sptes[0];
  794. else
  795. if (prev_desc)
  796. prev_desc->more = desc->more;
  797. else
  798. *pte_list = (unsigned long)desc->more | 1;
  799. mmu_free_pte_list_desc(desc);
  800. }
  801. static void pte_list_remove(u64 *spte, unsigned long *pte_list)
  802. {
  803. struct pte_list_desc *desc;
  804. struct pte_list_desc *prev_desc;
  805. int i;
  806. if (!*pte_list) {
  807. printk(KERN_ERR "pte_list_remove: %p 0->BUG\n", spte);
  808. BUG();
  809. } else if (!(*pte_list & 1)) {
  810. rmap_printk("pte_list_remove: %p 1->0\n", spte);
  811. if ((u64 *)*pte_list != spte) {
  812. printk(KERN_ERR "pte_list_remove: %p 1->BUG\n", spte);
  813. BUG();
  814. }
  815. *pte_list = 0;
  816. } else {
  817. rmap_printk("pte_list_remove: %p many->many\n", spte);
  818. desc = (struct pte_list_desc *)(*pte_list & ~1ul);
  819. prev_desc = NULL;
  820. while (desc) {
  821. for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i)
  822. if (desc->sptes[i] == spte) {
  823. pte_list_desc_remove_entry(pte_list,
  824. desc, i,
  825. prev_desc);
  826. return;
  827. }
  828. prev_desc = desc;
  829. desc = desc->more;
  830. }
  831. pr_err("pte_list_remove: %p many->many\n", spte);
  832. BUG();
  833. }
  834. }
  835. typedef void (*pte_list_walk_fn) (u64 *spte);
  836. static void pte_list_walk(unsigned long *pte_list, pte_list_walk_fn fn)
  837. {
  838. struct pte_list_desc *desc;
  839. int i;
  840. if (!*pte_list)
  841. return;
  842. if (!(*pte_list & 1))
  843. return fn((u64 *)*pte_list);
  844. desc = (struct pte_list_desc *)(*pte_list & ~1ul);
  845. while (desc) {
  846. for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i)
  847. fn(desc->sptes[i]);
  848. desc = desc->more;
  849. }
  850. }
  851. static unsigned long *__gfn_to_rmap(gfn_t gfn, int level,
  852. struct kvm_memory_slot *slot)
  853. {
  854. unsigned long idx;
  855. idx = gfn_to_index(gfn, slot->base_gfn, level);
  856. return &slot->arch.rmap[level - PT_PAGE_TABLE_LEVEL][idx];
  857. }
  858. /*
  859. * Take gfn and return the reverse mapping to it.
  860. */
  861. static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int level)
  862. {
  863. struct kvm_memory_slot *slot;
  864. slot = gfn_to_memslot(kvm, gfn);
  865. return __gfn_to_rmap(gfn, level, slot);
  866. }
  867. static bool rmap_can_add(struct kvm_vcpu *vcpu)
  868. {
  869. struct kvm_mmu_memory_cache *cache;
  870. cache = &vcpu->arch.mmu_pte_list_desc_cache;
  871. return mmu_memory_cache_free_objects(cache);
  872. }
  873. static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
  874. {
  875. struct kvm_mmu_page *sp;
  876. unsigned long *rmapp;
  877. sp = page_header(__pa(spte));
  878. kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn);
  879. rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
  880. return pte_list_add(vcpu, spte, rmapp);
  881. }
  882. static void rmap_remove(struct kvm *kvm, u64 *spte)
  883. {
  884. struct kvm_mmu_page *sp;
  885. gfn_t gfn;
  886. unsigned long *rmapp;
  887. sp = page_header(__pa(spte));
  888. gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt);
  889. rmapp = gfn_to_rmap(kvm, gfn, sp->role.level);
  890. pte_list_remove(spte, rmapp);
  891. }
  892. /*
  893. * Used by the following functions to iterate through the sptes linked by a
  894. * rmap. All fields are private and not assumed to be used outside.
  895. */
  896. struct rmap_iterator {
  897. /* private fields */
  898. struct pte_list_desc *desc; /* holds the sptep if not NULL */
  899. int pos; /* index of the sptep */
  900. };
  901. /*
  902. * Iteration must be started by this function. This should also be used after
  903. * removing/dropping sptes from the rmap link because in such cases the
  904. * information in the itererator may not be valid.
  905. *
  906. * Returns sptep if found, NULL otherwise.
  907. */
  908. static u64 *rmap_get_first(unsigned long rmap, struct rmap_iterator *iter)
  909. {
  910. if (!rmap)
  911. return NULL;
  912. if (!(rmap & 1)) {
  913. iter->desc = NULL;
  914. return (u64 *)rmap;
  915. }
  916. iter->desc = (struct pte_list_desc *)(rmap & ~1ul);
  917. iter->pos = 0;
  918. return iter->desc->sptes[iter->pos];
  919. }
  920. /*
  921. * Must be used with a valid iterator: e.g. after rmap_get_first().
  922. *
  923. * Returns sptep if found, NULL otherwise.
  924. */
  925. static u64 *rmap_get_next(struct rmap_iterator *iter)
  926. {
  927. if (iter->desc) {
  928. if (iter->pos < PTE_LIST_EXT - 1) {
  929. u64 *sptep;
  930. ++iter->pos;
  931. sptep = iter->desc->sptes[iter->pos];
  932. if (sptep)
  933. return sptep;
  934. }
  935. iter->desc = iter->desc->more;
  936. if (iter->desc) {
  937. iter->pos = 0;
  938. /* desc->sptes[0] cannot be NULL */
  939. return iter->desc->sptes[iter->pos];
  940. }
  941. }
  942. return NULL;
  943. }
  944. static void drop_spte(struct kvm *kvm, u64 *sptep)
  945. {
  946. if (mmu_spte_clear_track_bits(sptep))
  947. rmap_remove(kvm, sptep);
  948. }
  949. static bool __drop_large_spte(struct kvm *kvm, u64 *sptep)
  950. {
  951. if (is_large_pte(*sptep)) {
  952. WARN_ON(page_header(__pa(sptep))->role.level ==
  953. PT_PAGE_TABLE_LEVEL);
  954. drop_spte(kvm, sptep);
  955. --kvm->stat.lpages;
  956. return true;
  957. }
  958. return false;
  959. }
  960. static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
  961. {
  962. if (__drop_large_spte(vcpu->kvm, sptep))
  963. kvm_flush_remote_tlbs(vcpu->kvm);
  964. }
  965. /*
  966. * Write-protect on the specified @sptep, @pt_protect indicates whether
  967. * spte writ-protection is caused by protecting shadow page table.
  968. * @flush indicates whether tlb need be flushed.
  969. *
  970. * Note: write protection is difference between drity logging and spte
  971. * protection:
  972. * - for dirty logging, the spte can be set to writable at anytime if
  973. * its dirty bitmap is properly set.
  974. * - for spte protection, the spte can be writable only after unsync-ing
  975. * shadow page.
  976. *
  977. * Return true if the spte is dropped.
  978. */
  979. static bool
  980. spte_write_protect(struct kvm *kvm, u64 *sptep, bool *flush, bool pt_protect)
  981. {
  982. u64 spte = *sptep;
  983. if (!is_writable_pte(spte) &&
  984. !(pt_protect && spte_is_locklessly_modifiable(spte)))
  985. return false;
  986. rmap_printk("rmap_write_protect: spte %p %llx\n", sptep, *sptep);
  987. if (__drop_large_spte(kvm, sptep)) {
  988. *flush |= true;
  989. return true;
  990. }
  991. if (pt_protect)
  992. spte &= ~SPTE_MMU_WRITEABLE;
  993. spte = spte & ~PT_WRITABLE_MASK;
  994. *flush |= mmu_spte_update(sptep, spte);
  995. return false;
  996. }
  997. static bool __rmap_write_protect(struct kvm *kvm, unsigned long *rmapp,
  998. bool pt_protect)
  999. {
  1000. u64 *sptep;
  1001. struct rmap_iterator iter;
  1002. bool flush = false;
  1003. for (sptep = rmap_get_first(*rmapp, &iter); sptep;) {
  1004. BUG_ON(!(*sptep & PT_PRESENT_MASK));
  1005. if (spte_write_protect(kvm, sptep, &flush, pt_protect)) {
  1006. sptep = rmap_get_first(*rmapp, &iter);
  1007. continue;
  1008. }
  1009. sptep = rmap_get_next(&iter);
  1010. }
  1011. return flush;
  1012. }
  1013. /**
  1014. * kvm_mmu_write_protect_pt_masked - write protect selected PT level pages
  1015. * @kvm: kvm instance
  1016. * @slot: slot to protect
  1017. * @gfn_offset: start of the BITS_PER_LONG pages we care about
  1018. * @mask: indicates which pages we should protect
  1019. *
  1020. * Used when we do not need to care about huge page mappings: e.g. during dirty
  1021. * logging we do not have any such mappings.
  1022. */
  1023. void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
  1024. struct kvm_memory_slot *slot,
  1025. gfn_t gfn_offset, unsigned long mask)
  1026. {
  1027. unsigned long *rmapp;
  1028. while (mask) {
  1029. rmapp = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
  1030. PT_PAGE_TABLE_LEVEL, slot);
  1031. __rmap_write_protect(kvm, rmapp, false);
  1032. /* clear the first set bit */
  1033. mask &= mask - 1;
  1034. }
  1035. }
  1036. static bool rmap_write_protect(struct kvm *kvm, u64 gfn)
  1037. {
  1038. struct kvm_memory_slot *slot;
  1039. unsigned long *rmapp;
  1040. int i;
  1041. bool write_protected = false;
  1042. slot = gfn_to_memslot(kvm, gfn);
  1043. for (i = PT_PAGE_TABLE_LEVEL;
  1044. i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
  1045. rmapp = __gfn_to_rmap(gfn, i, slot);
  1046. write_protected |= __rmap_write_protect(kvm, rmapp, true);
  1047. }
  1048. return write_protected;
  1049. }
  1050. static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
  1051. struct kvm_memory_slot *slot, unsigned long data)
  1052. {
  1053. u64 *sptep;
  1054. struct rmap_iterator iter;
  1055. int need_tlb_flush = 0;
  1056. while ((sptep = rmap_get_first(*rmapp, &iter))) {
  1057. BUG_ON(!(*sptep & PT_PRESENT_MASK));
  1058. rmap_printk("kvm_rmap_unmap_hva: spte %p %llx\n", sptep, *sptep);
  1059. drop_spte(kvm, sptep);
  1060. need_tlb_flush = 1;
  1061. }
  1062. return need_tlb_flush;
  1063. }
  1064. static int kvm_set_pte_rmapp(struct kvm *kvm, unsigned long *rmapp,
  1065. struct kvm_memory_slot *slot, unsigned long data)
  1066. {
  1067. u64 *sptep;
  1068. struct rmap_iterator iter;
  1069. int need_flush = 0;
  1070. u64 new_spte;
  1071. pte_t *ptep = (pte_t *)data;
  1072. pfn_t new_pfn;
  1073. WARN_ON(pte_huge(*ptep));
  1074. new_pfn = pte_pfn(*ptep);
  1075. for (sptep = rmap_get_first(*rmapp, &iter); sptep;) {
  1076. BUG_ON(!is_shadow_present_pte(*sptep));
  1077. rmap_printk("kvm_set_pte_rmapp: spte %p %llx\n", sptep, *sptep);
  1078. need_flush = 1;
  1079. if (pte_write(*ptep)) {
  1080. drop_spte(kvm, sptep);
  1081. sptep = rmap_get_first(*rmapp, &iter);
  1082. } else {
  1083. new_spte = *sptep & ~PT64_BASE_ADDR_MASK;
  1084. new_spte |= (u64)new_pfn << PAGE_SHIFT;
  1085. new_spte &= ~PT_WRITABLE_MASK;
  1086. new_spte &= ~SPTE_HOST_WRITEABLE;
  1087. new_spte &= ~shadow_accessed_mask;
  1088. mmu_spte_clear_track_bits(sptep);
  1089. mmu_spte_set(sptep, new_spte);
  1090. sptep = rmap_get_next(&iter);
  1091. }
  1092. }
  1093. if (need_flush)
  1094. kvm_flush_remote_tlbs(kvm);
  1095. return 0;
  1096. }
  1097. static int kvm_handle_hva_range(struct kvm *kvm,
  1098. unsigned long start,
  1099. unsigned long end,
  1100. unsigned long data,
  1101. int (*handler)(struct kvm *kvm,
  1102. unsigned long *rmapp,
  1103. struct kvm_memory_slot *slot,
  1104. unsigned long data))
  1105. {
  1106. int j;
  1107. int ret = 0;
  1108. struct kvm_memslots *slots;
  1109. struct kvm_memory_slot *memslot;
  1110. slots = kvm_memslots(kvm);
  1111. kvm_for_each_memslot(memslot, slots) {
  1112. unsigned long hva_start, hva_end;
  1113. gfn_t gfn_start, gfn_end;
  1114. hva_start = max(start, memslot->userspace_addr);
  1115. hva_end = min(end, memslot->userspace_addr +
  1116. (memslot->npages << PAGE_SHIFT));
  1117. if (hva_start >= hva_end)
  1118. continue;
  1119. /*
  1120. * {gfn(page) | page intersects with [hva_start, hva_end)} =
  1121. * {gfn_start, gfn_start+1, ..., gfn_end-1}.
  1122. */
  1123. gfn_start = hva_to_gfn_memslot(hva_start, memslot);
  1124. gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
  1125. for (j = PT_PAGE_TABLE_LEVEL;
  1126. j < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++j) {
  1127. unsigned long idx, idx_end;
  1128. unsigned long *rmapp;
  1129. /*
  1130. * {idx(page_j) | page_j intersects with
  1131. * [hva_start, hva_end)} = {idx, idx+1, ..., idx_end}.
  1132. */
  1133. idx = gfn_to_index(gfn_start, memslot->base_gfn, j);
  1134. idx_end = gfn_to_index(gfn_end - 1, memslot->base_gfn, j);
  1135. rmapp = __gfn_to_rmap(gfn_start, j, memslot);
  1136. for (; idx <= idx_end; ++idx)
  1137. ret |= handler(kvm, rmapp++, memslot, data);
  1138. }
  1139. }
  1140. return ret;
  1141. }
  1142. static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
  1143. unsigned long data,
  1144. int (*handler)(struct kvm *kvm, unsigned long *rmapp,
  1145. struct kvm_memory_slot *slot,
  1146. unsigned long data))
  1147. {
  1148. return kvm_handle_hva_range(kvm, hva, hva + 1, data, handler);
  1149. }
  1150. int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
  1151. {
  1152. return kvm_handle_hva(kvm, hva, 0, kvm_unmap_rmapp);
  1153. }
  1154. int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end)
  1155. {
  1156. return kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp);
  1157. }
  1158. void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
  1159. {
  1160. kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);
  1161. }
  1162. static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
  1163. struct kvm_memory_slot *slot, unsigned long data)
  1164. {
  1165. u64 *sptep;
  1166. struct rmap_iterator uninitialized_var(iter);
  1167. int young = 0;
  1168. /*
  1169. * In case of absence of EPT Access and Dirty Bits supports,
  1170. * emulate the accessed bit for EPT, by checking if this page has
  1171. * an EPT mapping, and clearing it if it does. On the next access,
  1172. * a new EPT mapping will be established.
  1173. * This has some overhead, but not as much as the cost of swapping
  1174. * out actively used pages or breaking up actively used hugepages.
  1175. */
  1176. if (!shadow_accessed_mask) {
  1177. young = kvm_unmap_rmapp(kvm, rmapp, slot, data);
  1178. goto out;
  1179. }
  1180. for (sptep = rmap_get_first(*rmapp, &iter); sptep;
  1181. sptep = rmap_get_next(&iter)) {
  1182. BUG_ON(!is_shadow_present_pte(*sptep));
  1183. if (*sptep & shadow_accessed_mask) {
  1184. young = 1;
  1185. clear_bit((ffs(shadow_accessed_mask) - 1),
  1186. (unsigned long *)sptep);
  1187. }
  1188. }
  1189. out:
  1190. /* @data has hva passed to kvm_age_hva(). */
  1191. trace_kvm_age_page(data, slot, young);
  1192. return young;
  1193. }
  1194. static int kvm_test_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
  1195. struct kvm_memory_slot *slot, unsigned long data)
  1196. {
  1197. u64 *sptep;
  1198. struct rmap_iterator iter;
  1199. int young = 0;
  1200. /*
  1201. * If there's no access bit in the secondary pte set by the
  1202. * hardware it's up to gup-fast/gup to set the access bit in
  1203. * the primary pte or in the page structure.
  1204. */
  1205. if (!shadow_accessed_mask)
  1206. goto out;
  1207. for (sptep = rmap_get_first(*rmapp, &iter); sptep;
  1208. sptep = rmap_get_next(&iter)) {
  1209. BUG_ON(!is_shadow_present_pte(*sptep));
  1210. if (*sptep & shadow_accessed_mask) {
  1211. young = 1;
  1212. break;
  1213. }
  1214. }
  1215. out:
  1216. return young;
  1217. }
  1218. #define RMAP_RECYCLE_THRESHOLD 1000
  1219. static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
  1220. {
  1221. unsigned long *rmapp;
  1222. struct kvm_mmu_page *sp;
  1223. sp = page_header(__pa(spte));
  1224. rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
  1225. kvm_unmap_rmapp(vcpu->kvm, rmapp, NULL, 0);
  1226. kvm_flush_remote_tlbs(vcpu->kvm);
  1227. }
  1228. int kvm_age_hva(struct kvm *kvm, unsigned long hva)
  1229. {
  1230. return kvm_handle_hva(kvm, hva, hva, kvm_age_rmapp);
  1231. }
  1232. int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
  1233. {
  1234. return kvm_handle_hva(kvm, hva, 0, kvm_test_age_rmapp);
  1235. }
  1236. #ifdef MMU_DEBUG
  1237. static int is_empty_shadow_page(u64 *spt)
  1238. {
  1239. u64 *pos;
  1240. u64 *end;
  1241. for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
  1242. if (is_shadow_present_pte(*pos)) {
  1243. printk(KERN_ERR "%s: %p %llx\n", __func__,
  1244. pos, *pos);
  1245. return 0;
  1246. }
  1247. return 1;
  1248. }
  1249. #endif
  1250. /*
  1251. * This value is the sum of all of the kvm instances's
  1252. * kvm->arch.n_used_mmu_pages values. We need a global,
  1253. * aggregate version in order to make the slab shrinker
  1254. * faster
  1255. */
  1256. static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, int nr)
  1257. {
  1258. kvm->arch.n_used_mmu_pages += nr;
  1259. percpu_counter_add(&kvm_total_used_mmu_pages, nr);
  1260. }
  1261. static void kvm_mmu_free_page(struct kvm_mmu_page *sp)
  1262. {
  1263. ASSERT(is_empty_shadow_page(sp->spt));
  1264. hlist_del(&sp->hash_link);
  1265. list_del(&sp->link);
  1266. free_page((unsigned long)sp->spt);
  1267. if (!sp->role.direct)
  1268. free_page((unsigned long)sp->gfns);
  1269. kmem_cache_free(mmu_page_header_cache, sp);
  1270. }
  1271. static unsigned kvm_page_table_hashfn(gfn_t gfn)
  1272. {
  1273. return gfn & ((1 << KVM_MMU_HASH_SHIFT) - 1);
  1274. }
  1275. static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
  1276. struct kvm_mmu_page *sp, u64 *parent_pte)
  1277. {
  1278. if (!parent_pte)
  1279. return;
  1280. pte_list_add(vcpu, parent_pte, &sp->parent_ptes);
  1281. }
  1282. static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
  1283. u64 *parent_pte)
  1284. {
  1285. pte_list_remove(parent_pte, &sp->parent_ptes);
  1286. }
  1287. static void drop_parent_pte(struct kvm_mmu_page *sp,
  1288. u64 *parent_pte)
  1289. {
  1290. mmu_page_remove_parent_pte(sp, parent_pte);
  1291. mmu_spte_clear_no_track(parent_pte);
  1292. }
  1293. static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
  1294. u64 *parent_pte, int direct)
  1295. {
  1296. struct kvm_mmu_page *sp;
  1297. sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache);
  1298. sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache);
  1299. if (!direct)
  1300. sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache);
  1301. set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
  1302. /*
  1303. * The active_mmu_pages list is the FIFO list, do not move the
  1304. * page until it is zapped. kvm_zap_obsolete_pages depends on
  1305. * this feature. See the comments in kvm_zap_obsolete_pages().
  1306. */
  1307. list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
  1308. sp->parent_ptes = 0;
  1309. mmu_page_add_parent_pte(vcpu, sp, parent_pte);
  1310. kvm_mod_used_mmu_pages(vcpu->kvm, +1);
  1311. return sp;
  1312. }
  1313. static void mark_unsync(u64 *spte);
  1314. static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)
  1315. {
  1316. pte_list_walk(&sp->parent_ptes, mark_unsync);
  1317. }
  1318. static void mark_unsync(u64 *spte)
  1319. {
  1320. struct kvm_mmu_page *sp;
  1321. unsigned int index;
  1322. sp = page_header(__pa(spte));
  1323. index = spte - sp->spt;
  1324. if (__test_and_set_bit(index, sp->unsync_child_bitmap))
  1325. return;
  1326. if (sp->unsync_children++)
  1327. return;
  1328. kvm_mmu_mark_parents_unsync(sp);
  1329. }
  1330. static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
  1331. struct kvm_mmu_page *sp)
  1332. {
  1333. return 1;
  1334. }
  1335. static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
  1336. {
  1337. }
  1338. static void nonpaging_update_pte(struct kvm_vcpu *vcpu,
  1339. struct kvm_mmu_page *sp, u64 *spte,
  1340. const void *pte)
  1341. {
  1342. WARN_ON(1);
  1343. }
  1344. #define KVM_PAGE_ARRAY_NR 16
  1345. struct kvm_mmu_pages {
  1346. struct mmu_page_and_offset {
  1347. struct kvm_mmu_page *sp;
  1348. unsigned int idx;
  1349. } page[KVM_PAGE_ARRAY_NR];
  1350. unsigned int nr;
  1351. };
  1352. static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
  1353. int idx)
  1354. {
  1355. int i;
  1356. if (sp->unsync)
  1357. for (i=0; i < pvec->nr; i++)
  1358. if (pvec->page[i].sp == sp)
  1359. return 0;
  1360. pvec->page[pvec->nr].sp = sp;
  1361. pvec->page[pvec->nr].idx = idx;
  1362. pvec->nr++;
  1363. return (pvec->nr == KVM_PAGE_ARRAY_NR);
  1364. }
  1365. static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
  1366. struct kvm_mmu_pages *pvec)
  1367. {
  1368. int i, ret, nr_unsync_leaf = 0;
  1369. for_each_set_bit(i, sp->unsync_child_bitmap, 512) {
  1370. struct kvm_mmu_page *child;
  1371. u64 ent = sp->spt[i];
  1372. if (!is_shadow_present_pte(ent) || is_large_pte(ent))
  1373. goto clear_child_bitmap;
  1374. child = page_header(ent & PT64_BASE_ADDR_MASK);
  1375. if (child->unsync_children) {
  1376. if (mmu_pages_add(pvec, child, i))
  1377. return -ENOSPC;
  1378. ret = __mmu_unsync_walk(child, pvec);
  1379. if (!ret)
  1380. goto clear_child_bitmap;
  1381. else if (ret > 0)
  1382. nr_unsync_leaf += ret;
  1383. else
  1384. return ret;
  1385. } else if (child->unsync) {
  1386. nr_unsync_leaf++;
  1387. if (mmu_pages_add(pvec, child, i))
  1388. return -ENOSPC;
  1389. } else
  1390. goto clear_child_bitmap;
  1391. continue;
  1392. clear_child_bitmap:
  1393. __clear_bit(i, sp->unsync_child_bitmap);
  1394. sp->unsync_children--;
  1395. WARN_ON((int)sp->unsync_children < 0);
  1396. }
  1397. return nr_unsync_leaf;
  1398. }
  1399. static int mmu_unsync_walk(struct kvm_mmu_page *sp,
  1400. struct kvm_mmu_pages *pvec)
  1401. {
  1402. if (!sp->unsync_children)
  1403. return 0;
  1404. mmu_pages_add(pvec, sp, 0);
  1405. return __mmu_unsync_walk(sp, pvec);
  1406. }
  1407. static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
  1408. {
  1409. WARN_ON(!sp->unsync);
  1410. trace_kvm_mmu_sync_page(sp);
  1411. sp->unsync = 0;
  1412. --kvm->stat.mmu_unsync;
  1413. }
  1414. static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
  1415. struct list_head *invalid_list);
  1416. static void kvm_mmu_commit_zap_page(struct kvm *kvm,
  1417. struct list_head *invalid_list);
  1418. /*
  1419. * NOTE: we should pay more attention on the zapped-obsolete page
  1420. * (is_obsolete_sp(sp) && sp->role.invalid) when you do hash list walk
  1421. * since it has been deleted from active_mmu_pages but still can be found
  1422. * at hast list.
  1423. *
  1424. * for_each_gfn_indirect_valid_sp has skipped that kind of page and
  1425. * kvm_mmu_get_page(), the only user of for_each_gfn_sp(), has skipped
  1426. * all the obsolete pages.
  1427. */
  1428. #define for_each_gfn_sp(_kvm, _sp, _gfn) \
  1429. hlist_for_each_entry(_sp, \
  1430. &(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)], hash_link) \
  1431. if ((_sp)->gfn != (_gfn)) {} else
  1432. #define for_each_gfn_indirect_valid_sp(_kvm, _sp, _gfn) \
  1433. for_each_gfn_sp(_kvm, _sp, _gfn) \
  1434. if ((_sp)->role.direct || (_sp)->role.invalid) {} else
  1435. /* @sp->gfn should be write-protected at the call site */
  1436. static int __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
  1437. struct list_head *invalid_list, bool clear_unsync)
  1438. {
  1439. if (sp->role.cr4_pae != !!is_pae(vcpu)) {
  1440. kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
  1441. return 1;
  1442. }
  1443. if (clear_unsync)
  1444. kvm_unlink_unsync_page(vcpu->kvm, sp);
  1445. if (vcpu->arch.mmu.sync_page(vcpu, sp)) {
  1446. kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
  1447. return 1;
  1448. }
  1449. kvm_mmu_flush_tlb(vcpu);
  1450. return 0;
  1451. }
  1452. static int kvm_sync_page_transient(struct kvm_vcpu *vcpu,
  1453. struct kvm_mmu_page *sp)
  1454. {
  1455. LIST_HEAD(invalid_list);
  1456. int ret;
  1457. ret = __kvm_sync_page(vcpu, sp, &invalid_list, false);
  1458. if (ret)
  1459. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  1460. return ret;
  1461. }
  1462. #ifdef CONFIG_KVM_MMU_AUDIT
  1463. #include "mmu_audit.c"
  1464. #else
  1465. static void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) { }
  1466. static void mmu_audit_disable(void) { }
  1467. #endif
  1468. static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
  1469. struct list_head *invalid_list)
  1470. {
  1471. return __kvm_sync_page(vcpu, sp, invalid_list, true);
  1472. }
  1473. /* @gfn should be write-protected at the call site */
  1474. static void kvm_sync_pages(struct kvm_vcpu *vcpu, gfn_t gfn)
  1475. {
  1476. struct kvm_mmu_page *s;
  1477. LIST_HEAD(invalid_list);
  1478. bool flush = false;
  1479. for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
  1480. if (!s->unsync)
  1481. continue;
  1482. WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
  1483. kvm_unlink_unsync_page(vcpu->kvm, s);
  1484. if ((s->role.cr4_pae != !!is_pae(vcpu)) ||
  1485. (vcpu->arch.mmu.sync_page(vcpu, s))) {
  1486. kvm_mmu_prepare_zap_page(vcpu->kvm, s, &invalid_list);
  1487. continue;
  1488. }
  1489. flush = true;
  1490. }
  1491. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  1492. if (flush)
  1493. kvm_mmu_flush_tlb(vcpu);
  1494. }
  1495. struct mmu_page_path {
  1496. struct kvm_mmu_page *parent[PT64_ROOT_LEVEL-1];
  1497. unsigned int idx[PT64_ROOT_LEVEL-1];
  1498. };
  1499. #define for_each_sp(pvec, sp, parents, i) \
  1500. for (i = mmu_pages_next(&pvec, &parents, -1), \
  1501. sp = pvec.page[i].sp; \
  1502. i < pvec.nr && ({ sp = pvec.page[i].sp; 1;}); \
  1503. i = mmu_pages_next(&pvec, &parents, i))
  1504. static int mmu_pages_next(struct kvm_mmu_pages *pvec,
  1505. struct mmu_page_path *parents,
  1506. int i)
  1507. {
  1508. int n;
  1509. for (n = i+1; n < pvec->nr; n++) {
  1510. struct kvm_mmu_page *sp = pvec->page[n].sp;
  1511. if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
  1512. parents->idx[0] = pvec->page[n].idx;
  1513. return n;
  1514. }
  1515. parents->parent[sp->role.level-2] = sp;
  1516. parents->idx[sp->role.level-1] = pvec->page[n].idx;
  1517. }
  1518. return n;
  1519. }
  1520. static void mmu_pages_clear_parents(struct mmu_page_path *parents)
  1521. {
  1522. struct kvm_mmu_page *sp;
  1523. unsigned int level = 0;
  1524. do {
  1525. unsigned int idx = parents->idx[level];
  1526. sp = parents->parent[level];
  1527. if (!sp)
  1528. return;
  1529. --sp->unsync_children;
  1530. WARN_ON((int)sp->unsync_children < 0);
  1531. __clear_bit(idx, sp->unsync_child_bitmap);
  1532. level++;
  1533. } while (level < PT64_ROOT_LEVEL-1 && !sp->unsync_children);
  1534. }
  1535. static void kvm_mmu_pages_init(struct kvm_mmu_page *parent,
  1536. struct mmu_page_path *parents,
  1537. struct kvm_mmu_pages *pvec)
  1538. {
  1539. parents->parent[parent->role.level-1] = NULL;
  1540. pvec->nr = 0;
  1541. }
  1542. static void mmu_sync_children(struct kvm_vcpu *vcpu,
  1543. struct kvm_mmu_page *parent)
  1544. {
  1545. int i;
  1546. struct kvm_mmu_page *sp;
  1547. struct mmu_page_path parents;
  1548. struct kvm_mmu_pages pages;
  1549. LIST_HEAD(invalid_list);
  1550. kvm_mmu_pages_init(parent, &parents, &pages);
  1551. while (mmu_unsync_walk(parent, &pages)) {
  1552. bool protected = false;
  1553. for_each_sp(pages, sp, parents, i)
  1554. protected |= rmap_write_protect(vcpu->kvm, sp->gfn);
  1555. if (protected)
  1556. kvm_flush_remote_tlbs(vcpu->kvm);
  1557. for_each_sp(pages, sp, parents, i) {
  1558. kvm_sync_page(vcpu, sp, &invalid_list);
  1559. mmu_pages_clear_parents(&parents);
  1560. }
  1561. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  1562. cond_resched_lock(&vcpu->kvm->mmu_lock);
  1563. kvm_mmu_pages_init(parent, &parents, &pages);
  1564. }
  1565. }
  1566. static void init_shadow_page_table(struct kvm_mmu_page *sp)
  1567. {
  1568. int i;
  1569. for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
  1570. sp->spt[i] = 0ull;
  1571. }
  1572. static void __clear_sp_write_flooding_count(struct kvm_mmu_page *sp)
  1573. {
  1574. sp->write_flooding_count = 0;
  1575. }
  1576. static void clear_sp_write_flooding_count(u64 *spte)
  1577. {
  1578. struct kvm_mmu_page *sp = page_header(__pa(spte));
  1579. __clear_sp_write_flooding_count(sp);
  1580. }
  1581. static bool is_obsolete_sp(struct kvm *kvm, struct kvm_mmu_page *sp)
  1582. {
  1583. return unlikely(sp->mmu_valid_gen != kvm->arch.mmu_valid_gen);
  1584. }
  1585. static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
  1586. gfn_t gfn,
  1587. gva_t gaddr,
  1588. unsigned level,
  1589. int direct,
  1590. unsigned access,
  1591. u64 *parent_pte)
  1592. {
  1593. union kvm_mmu_page_role role;
  1594. unsigned quadrant;
  1595. struct kvm_mmu_page *sp;
  1596. bool need_sync = false;
  1597. role = vcpu->arch.mmu.base_role;
  1598. role.level = level;
  1599. role.direct = direct;
  1600. if (role.direct)
  1601. role.cr4_pae = 0;
  1602. role.access = access;
  1603. if (!vcpu->arch.mmu.direct_map
  1604. && vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) {
  1605. quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
  1606. quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
  1607. role.quadrant = quadrant;
  1608. }
  1609. for_each_gfn_sp(vcpu->kvm, sp, gfn) {
  1610. if (is_obsolete_sp(vcpu->kvm, sp))
  1611. continue;
  1612. if (!need_sync && sp->unsync)
  1613. need_sync = true;
  1614. if (sp->role.word != role.word)
  1615. continue;
  1616. if (sp->unsync && kvm_sync_page_transient(vcpu, sp))
  1617. break;
  1618. mmu_page_add_parent_pte(vcpu, sp, parent_pte);
  1619. if (sp->unsync_children) {
  1620. kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
  1621. kvm_mmu_mark_parents_unsync(sp);
  1622. } else if (sp->unsync)
  1623. kvm_mmu_mark_parents_unsync(sp);
  1624. __clear_sp_write_flooding_count(sp);
  1625. trace_kvm_mmu_get_page(sp, false);
  1626. return sp;
  1627. }
  1628. ++vcpu->kvm->stat.mmu_cache_miss;
  1629. sp = kvm_mmu_alloc_page(vcpu, parent_pte, direct);
  1630. if (!sp)
  1631. return sp;
  1632. sp->gfn = gfn;
  1633. sp->role = role;
  1634. hlist_add_head(&sp->hash_link,
  1635. &vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)]);
  1636. if (!direct) {
  1637. if (rmap_write_protect(vcpu->kvm, gfn))
  1638. kvm_flush_remote_tlbs(vcpu->kvm);
  1639. if (level > PT_PAGE_TABLE_LEVEL && need_sync)
  1640. kvm_sync_pages(vcpu, gfn);
  1641. account_shadowed(vcpu->kvm, gfn);
  1642. }
  1643. sp->mmu_valid_gen = vcpu->kvm->arch.mmu_valid_gen;
  1644. init_shadow_page_table(sp);
  1645. trace_kvm_mmu_get_page(sp, true);
  1646. return sp;
  1647. }
  1648. static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
  1649. struct kvm_vcpu *vcpu, u64 addr)
  1650. {
  1651. iterator->addr = addr;
  1652. iterator->shadow_addr = vcpu->arch.mmu.root_hpa;
  1653. iterator->level = vcpu->arch.mmu.shadow_root_level;
  1654. if (iterator->level == PT64_ROOT_LEVEL &&
  1655. vcpu->arch.mmu.root_level < PT64_ROOT_LEVEL &&
  1656. !vcpu->arch.mmu.direct_map)
  1657. --iterator->level;
  1658. if (iterator->level == PT32E_ROOT_LEVEL) {
  1659. iterator->shadow_addr
  1660. = vcpu->arch.mmu.pae_root[(addr >> 30) & 3];
  1661. iterator->shadow_addr &= PT64_BASE_ADDR_MASK;
  1662. --iterator->level;
  1663. if (!iterator->shadow_addr)
  1664. iterator->level = 0;
  1665. }
  1666. }
  1667. static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
  1668. {
  1669. if (iterator->level < PT_PAGE_TABLE_LEVEL)
  1670. return false;
  1671. iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
  1672. iterator->sptep = ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
  1673. return true;
  1674. }
  1675. static void __shadow_walk_next(struct kvm_shadow_walk_iterator *iterator,
  1676. u64 spte)
  1677. {
  1678. if (is_last_spte(spte, iterator->level)) {
  1679. iterator->level = 0;
  1680. return;
  1681. }
  1682. iterator->shadow_addr = spte & PT64_BASE_ADDR_MASK;
  1683. --iterator->level;
  1684. }
  1685. static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
  1686. {
  1687. return __shadow_walk_next(iterator, *iterator->sptep);
  1688. }
  1689. static void link_shadow_page(u64 *sptep, struct kvm_mmu_page *sp)
  1690. {
  1691. u64 spte;
  1692. spte = __pa(sp->spt) | PT_PRESENT_MASK | PT_WRITABLE_MASK |
  1693. shadow_user_mask | shadow_x_mask | shadow_accessed_mask;
  1694. mmu_spte_set(sptep, spte);
  1695. }
  1696. static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
  1697. unsigned direct_access)
  1698. {
  1699. if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {
  1700. struct kvm_mmu_page *child;
  1701. /*
  1702. * For the direct sp, if the guest pte's dirty bit
  1703. * changed form clean to dirty, it will corrupt the
  1704. * sp's access: allow writable in the read-only sp,
  1705. * so we should update the spte at this point to get
  1706. * a new sp with the correct access.
  1707. */
  1708. child = page_header(*sptep & PT64_BASE_ADDR_MASK);
  1709. if (child->role.access == direct_access)
  1710. return;
  1711. drop_parent_pte(child, sptep);
  1712. kvm_flush_remote_tlbs(vcpu->kvm);
  1713. }
  1714. }
  1715. static bool mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
  1716. u64 *spte)
  1717. {
  1718. u64 pte;
  1719. struct kvm_mmu_page *child;
  1720. pte = *spte;
  1721. if (is_shadow_present_pte(pte)) {
  1722. if (is_last_spte(pte, sp->role.level)) {
  1723. drop_spte(kvm, spte);
  1724. if (is_large_pte(pte))
  1725. --kvm->stat.lpages;
  1726. } else {
  1727. child = page_header(pte & PT64_BASE_ADDR_MASK);
  1728. drop_parent_pte(child, spte);
  1729. }
  1730. return true;
  1731. }
  1732. if (is_mmio_spte(pte))
  1733. mmu_spte_clear_no_track(spte);
  1734. return false;
  1735. }
  1736. static void kvm_mmu_page_unlink_children(struct kvm *kvm,
  1737. struct kvm_mmu_page *sp)
  1738. {
  1739. unsigned i;
  1740. for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
  1741. mmu_page_zap_pte(kvm, sp, sp->spt + i);
  1742. }
  1743. static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte)
  1744. {
  1745. mmu_page_remove_parent_pte(sp, parent_pte);
  1746. }
  1747. static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
  1748. {
  1749. u64 *sptep;
  1750. struct rmap_iterator iter;
  1751. while ((sptep = rmap_get_first(sp->parent_ptes, &iter)))
  1752. drop_parent_pte(sp, sptep);
  1753. }
  1754. static int mmu_zap_unsync_children(struct kvm *kvm,
  1755. struct kvm_mmu_page *parent,
  1756. struct list_head *invalid_list)
  1757. {
  1758. int i, zapped = 0;
  1759. struct mmu_page_path parents;
  1760. struct kvm_mmu_pages pages;
  1761. if (parent->role.level == PT_PAGE_TABLE_LEVEL)
  1762. return 0;
  1763. kvm_mmu_pages_init(parent, &parents, &pages);
  1764. while (mmu_unsync_walk(parent, &pages)) {
  1765. struct kvm_mmu_page *sp;
  1766. for_each_sp(pages, sp, parents, i) {
  1767. kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
  1768. mmu_pages_clear_parents(&parents);
  1769. zapped++;
  1770. }
  1771. kvm_mmu_pages_init(parent, &parents, &pages);
  1772. }
  1773. return zapped;
  1774. }
  1775. static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
  1776. struct list_head *invalid_list)
  1777. {
  1778. int ret;
  1779. trace_kvm_mmu_prepare_zap_page(sp);
  1780. ++kvm->stat.mmu_shadow_zapped;
  1781. ret = mmu_zap_unsync_children(kvm, sp, invalid_list);
  1782. kvm_mmu_page_unlink_children(kvm, sp);
  1783. kvm_mmu_unlink_parents(kvm, sp);
  1784. if (!sp->role.invalid && !sp->role.direct)
  1785. unaccount_shadowed(kvm, sp->gfn);
  1786. if (sp->unsync)
  1787. kvm_unlink_unsync_page(kvm, sp);
  1788. if (!sp->root_count) {
  1789. /* Count self */
  1790. ret++;
  1791. list_move(&sp->link, invalid_list);
  1792. kvm_mod_used_mmu_pages(kvm, -1);
  1793. } else {
  1794. list_move(&sp->link, &kvm->arch.active_mmu_pages);
  1795. /*
  1796. * The obsolete pages can not be used on any vcpus.
  1797. * See the comments in kvm_mmu_invalidate_zap_all_pages().
  1798. */
  1799. if (!sp->role.invalid && !is_obsolete_sp(kvm, sp))
  1800. kvm_reload_remote_mmus(kvm);
  1801. }
  1802. sp->role.invalid = 1;
  1803. return ret;
  1804. }
  1805. static void kvm_mmu_commit_zap_page(struct kvm *kvm,
  1806. struct list_head *invalid_list)
  1807. {
  1808. struct kvm_mmu_page *sp, *nsp;
  1809. if (list_empty(invalid_list))
  1810. return;
  1811. /*
  1812. * wmb: make sure everyone sees our modifications to the page tables
  1813. * rmb: make sure we see changes to vcpu->mode
  1814. */
  1815. smp_mb();
  1816. /*
  1817. * Wait for all vcpus to exit guest mode and/or lockless shadow
  1818. * page table walks.
  1819. */
  1820. kvm_flush_remote_tlbs(kvm);
  1821. list_for_each_entry_safe(sp, nsp, invalid_list, link) {
  1822. WARN_ON(!sp->role.invalid || sp->root_count);
  1823. kvm_mmu_free_page(sp);
  1824. }
  1825. }
  1826. static bool prepare_zap_oldest_mmu_page(struct kvm *kvm,
  1827. struct list_head *invalid_list)
  1828. {
  1829. struct kvm_mmu_page *sp;
  1830. if (list_empty(&kvm->arch.active_mmu_pages))
  1831. return false;
  1832. sp = list_entry(kvm->arch.active_mmu_pages.prev,
  1833. struct kvm_mmu_page, link);
  1834. kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
  1835. return true;
  1836. }
  1837. /*
  1838. * Changing the number of mmu pages allocated to the vm
  1839. * Note: if goal_nr_mmu_pages is too small, you will get dead lock
  1840. */
  1841. void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int goal_nr_mmu_pages)
  1842. {
  1843. LIST_HEAD(invalid_list);
  1844. spin_lock(&kvm->mmu_lock);
  1845. if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
  1846. /* Need to free some mmu pages to achieve the goal. */
  1847. while (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages)
  1848. if (!prepare_zap_oldest_mmu_page(kvm, &invalid_list))
  1849. break;
  1850. kvm_mmu_commit_zap_page(kvm, &invalid_list);
  1851. goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages;
  1852. }
  1853. kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
  1854. spin_unlock(&kvm->mmu_lock);
  1855. }
  1856. int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
  1857. {
  1858. struct kvm_mmu_page *sp;
  1859. LIST_HEAD(invalid_list);
  1860. int r;
  1861. pgprintk("%s: looking for gfn %llx\n", __func__, gfn);
  1862. r = 0;
  1863. spin_lock(&kvm->mmu_lock);
  1864. for_each_gfn_indirect_valid_sp(kvm, sp, gfn) {
  1865. pgprintk("%s: gfn %llx role %x\n", __func__, gfn,
  1866. sp->role.word);
  1867. r = 1;
  1868. kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
  1869. }
  1870. kvm_mmu_commit_zap_page(kvm, &invalid_list);
  1871. spin_unlock(&kvm->mmu_lock);
  1872. return r;
  1873. }
  1874. EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page);
  1875. /*
  1876. * The function is based on mtrr_type_lookup() in
  1877. * arch/x86/kernel/cpu/mtrr/generic.c
  1878. */
  1879. static int get_mtrr_type(struct mtrr_state_type *mtrr_state,
  1880. u64 start, u64 end)
  1881. {
  1882. int i;
  1883. u64 base, mask;
  1884. u8 prev_match, curr_match;
  1885. int num_var_ranges = KVM_NR_VAR_MTRR;
  1886. if (!mtrr_state->enabled)
  1887. return 0xFF;
  1888. /* Make end inclusive end, instead of exclusive */
  1889. end--;
  1890. /* Look in fixed ranges. Just return the type as per start */
  1891. if (mtrr_state->have_fixed && (start < 0x100000)) {
  1892. int idx;
  1893. if (start < 0x80000) {
  1894. idx = 0;
  1895. idx += (start >> 16);
  1896. return mtrr_state->fixed_ranges[idx];
  1897. } else if (start < 0xC0000) {
  1898. idx = 1 * 8;
  1899. idx += ((start - 0x80000) >> 14);
  1900. return mtrr_state->fixed_ranges[idx];
  1901. } else if (start < 0x1000000) {
  1902. idx = 3 * 8;
  1903. idx += ((start - 0xC0000) >> 12);
  1904. return mtrr_state->fixed_ranges[idx];
  1905. }
  1906. }
  1907. /*
  1908. * Look in variable ranges
  1909. * Look of multiple ranges matching this address and pick type
  1910. * as per MTRR precedence
  1911. */
  1912. if (!(mtrr_state->enabled & 2))
  1913. return mtrr_state->def_type;
  1914. prev_match = 0xFF;
  1915. for (i = 0; i < num_var_ranges; ++i) {
  1916. unsigned short start_state, end_state;
  1917. if (!(mtrr_state->var_ranges[i].mask_lo & (1 << 11)))
  1918. continue;
  1919. base = (((u64)mtrr_state->var_ranges[i].base_hi) << 32) +
  1920. (mtrr_state->var_ranges[i].base_lo & PAGE_MASK);
  1921. mask = (((u64)mtrr_state->var_ranges[i].mask_hi) << 32) +
  1922. (mtrr_state->var_ranges[i].mask_lo & PAGE_MASK);
  1923. start_state = ((start & mask) == (base & mask));
  1924. end_state = ((end & mask) == (base & mask));
  1925. if (start_state != end_state)
  1926. return 0xFE;
  1927. if ((start & mask) != (base & mask))
  1928. continue;
  1929. curr_match = mtrr_state->var_ranges[i].base_lo & 0xff;
  1930. if (prev_match == 0xFF) {
  1931. prev_match = curr_match;
  1932. continue;
  1933. }
  1934. if (prev_match == MTRR_TYPE_UNCACHABLE ||
  1935. curr_match == MTRR_TYPE_UNCACHABLE)
  1936. return MTRR_TYPE_UNCACHABLE;
  1937. if ((prev_match == MTRR_TYPE_WRBACK &&
  1938. curr_match == MTRR_TYPE_WRTHROUGH) ||
  1939. (prev_match == MTRR_TYPE_WRTHROUGH &&
  1940. curr_match == MTRR_TYPE_WRBACK)) {
  1941. prev_match = MTRR_TYPE_WRTHROUGH;
  1942. curr_match = MTRR_TYPE_WRTHROUGH;
  1943. }
  1944. if (prev_match != curr_match)
  1945. return MTRR_TYPE_UNCACHABLE;
  1946. }
  1947. if (prev_match != 0xFF)
  1948. return prev_match;
  1949. return mtrr_state->def_type;
  1950. }
  1951. u8 kvm_get_guest_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn)
  1952. {
  1953. u8 mtrr;
  1954. mtrr = get_mtrr_type(&vcpu->arch.mtrr_state, gfn << PAGE_SHIFT,
  1955. (gfn << PAGE_SHIFT) + PAGE_SIZE);
  1956. if (mtrr == 0xfe || mtrr == 0xff)
  1957. mtrr = MTRR_TYPE_WRBACK;
  1958. return mtrr;
  1959. }
  1960. EXPORT_SYMBOL_GPL(kvm_get_guest_memory_type);
  1961. static void __kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
  1962. {
  1963. trace_kvm_mmu_unsync_page(sp);
  1964. ++vcpu->kvm->stat.mmu_unsync;
  1965. sp->unsync = 1;
  1966. kvm_mmu_mark_parents_unsync(sp);
  1967. }
  1968. static void kvm_unsync_pages(struct kvm_vcpu *vcpu, gfn_t gfn)
  1969. {
  1970. struct kvm_mmu_page *s;
  1971. for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
  1972. if (s->unsync)
  1973. continue;
  1974. WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
  1975. __kvm_unsync_page(vcpu, s);
  1976. }
  1977. }
  1978. static int mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
  1979. bool can_unsync)
  1980. {
  1981. struct kvm_mmu_page *s;
  1982. bool need_unsync = false;
  1983. for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
  1984. if (!can_unsync)
  1985. return 1;
  1986. if (s->role.level != PT_PAGE_TABLE_LEVEL)
  1987. return 1;
  1988. if (!s->unsync)
  1989. need_unsync = true;
  1990. }
  1991. if (need_unsync)
  1992. kvm_unsync_pages(vcpu, gfn);
  1993. return 0;
  1994. }
  1995. static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
  1996. unsigned pte_access, int level,
  1997. gfn_t gfn, pfn_t pfn, bool speculative,
  1998. bool can_unsync, bool host_writable)
  1999. {
  2000. u64 spte;
  2001. int ret = 0;
  2002. if (set_mmio_spte(vcpu->kvm, sptep, gfn, pfn, pte_access))
  2003. return 0;
  2004. spte = PT_PRESENT_MASK;
  2005. if (!speculative)
  2006. spte |= shadow_accessed_mask;
  2007. if (pte_access & ACC_EXEC_MASK)
  2008. spte |= shadow_x_mask;
  2009. else
  2010. spte |= shadow_nx_mask;
  2011. if (pte_access & ACC_USER_MASK)
  2012. spte |= shadow_user_mask;
  2013. if (level > PT_PAGE_TABLE_LEVEL)
  2014. spte |= PT_PAGE_SIZE_MASK;
  2015. if (tdp_enabled)
  2016. spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn,
  2017. kvm_is_mmio_pfn(pfn));
  2018. if (host_writable)
  2019. spte |= SPTE_HOST_WRITEABLE;
  2020. else
  2021. pte_access &= ~ACC_WRITE_MASK;
  2022. spte |= (u64)pfn << PAGE_SHIFT;
  2023. if (pte_access & ACC_WRITE_MASK) {
  2024. /*
  2025. * Other vcpu creates new sp in the window between
  2026. * mapping_level() and acquiring mmu-lock. We can
  2027. * allow guest to retry the access, the mapping can
  2028. * be fixed if guest refault.
  2029. */
  2030. if (level > PT_PAGE_TABLE_LEVEL &&
  2031. has_wrprotected_page(vcpu->kvm, gfn, level))
  2032. goto done;
  2033. spte |= PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE;
  2034. /*
  2035. * Optimization: for pte sync, if spte was writable the hash
  2036. * lookup is unnecessary (and expensive). Write protection
  2037. * is responsibility of mmu_get_page / kvm_sync_page.
  2038. * Same reasoning can be applied to dirty page accounting.
  2039. */
  2040. if (!can_unsync && is_writable_pte(*sptep))
  2041. goto set_pte;
  2042. if (mmu_need_write_protect(vcpu, gfn, can_unsync)) {
  2043. pgprintk("%s: found shadow page for %llx, marking ro\n",
  2044. __func__, gfn);
  2045. ret = 1;
  2046. pte_access &= ~ACC_WRITE_MASK;
  2047. spte &= ~(PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE);
  2048. }
  2049. }
  2050. if (pte_access & ACC_WRITE_MASK)
  2051. mark_page_dirty(vcpu->kvm, gfn);
  2052. set_pte:
  2053. if (mmu_spte_update(sptep, spte))
  2054. kvm_flush_remote_tlbs(vcpu->kvm);
  2055. done:
  2056. return ret;
  2057. }
  2058. static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
  2059. unsigned pte_access, int write_fault, int *emulate,
  2060. int level, gfn_t gfn, pfn_t pfn, bool speculative,
  2061. bool host_writable)
  2062. {
  2063. int was_rmapped = 0;
  2064. int rmap_count;
  2065. pgprintk("%s: spte %llx write_fault %d gfn %llx\n", __func__,
  2066. *sptep, write_fault, gfn);
  2067. if (is_rmap_spte(*sptep)) {
  2068. /*
  2069. * If we overwrite a PTE page pointer with a 2MB PMD, unlink
  2070. * the parent of the now unreachable PTE.
  2071. */
  2072. if (level > PT_PAGE_TABLE_LEVEL &&
  2073. !is_large_pte(*sptep)) {
  2074. struct kvm_mmu_page *child;
  2075. u64 pte = *sptep;
  2076. child = page_header(pte & PT64_BASE_ADDR_MASK);
  2077. drop_parent_pte(child, sptep);
  2078. kvm_flush_remote_tlbs(vcpu->kvm);
  2079. } else if (pfn != spte_to_pfn(*sptep)) {
  2080. pgprintk("hfn old %llx new %llx\n",
  2081. spte_to_pfn(*sptep), pfn);
  2082. drop_spte(vcpu->kvm, sptep);
  2083. kvm_flush_remote_tlbs(vcpu->kvm);
  2084. } else
  2085. was_rmapped = 1;
  2086. }
  2087. if (set_spte(vcpu, sptep, pte_access, level, gfn, pfn, speculative,
  2088. true, host_writable)) {
  2089. if (write_fault)
  2090. *emulate = 1;
  2091. kvm_mmu_flush_tlb(vcpu);
  2092. }
  2093. if (unlikely(is_mmio_spte(*sptep) && emulate))
  2094. *emulate = 1;
  2095. pgprintk("%s: setting spte %llx\n", __func__, *sptep);
  2096. pgprintk("instantiating %s PTE (%s) at %llx (%llx) addr %p\n",
  2097. is_large_pte(*sptep)? "2MB" : "4kB",
  2098. *sptep & PT_PRESENT_MASK ?"RW":"R", gfn,
  2099. *sptep, sptep);
  2100. if (!was_rmapped && is_large_pte(*sptep))
  2101. ++vcpu->kvm->stat.lpages;
  2102. if (is_shadow_present_pte(*sptep)) {
  2103. if (!was_rmapped) {
  2104. rmap_count = rmap_add(vcpu, sptep, gfn);
  2105. if (rmap_count > RMAP_RECYCLE_THRESHOLD)
  2106. rmap_recycle(vcpu, sptep, gfn);
  2107. }
  2108. }
  2109. kvm_release_pfn_clean(pfn);
  2110. }
  2111. static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
  2112. {
  2113. mmu_free_roots(vcpu);
  2114. }
  2115. static bool is_rsvd_bits_set(struct kvm_mmu *mmu, u64 gpte, int level)
  2116. {
  2117. int bit7;
  2118. bit7 = (gpte >> 7) & 1;
  2119. return (gpte & mmu->rsvd_bits_mask[bit7][level-1]) != 0;
  2120. }
  2121. static pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn,
  2122. bool no_dirty_log)
  2123. {
  2124. struct kvm_memory_slot *slot;
  2125. slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, no_dirty_log);
  2126. if (!slot)
  2127. return KVM_PFN_ERR_FAULT;
  2128. return gfn_to_pfn_memslot_atomic(slot, gfn);
  2129. }
  2130. static bool prefetch_invalid_gpte(struct kvm_vcpu *vcpu,
  2131. struct kvm_mmu_page *sp, u64 *spte,
  2132. u64 gpte)
  2133. {
  2134. if (is_rsvd_bits_set(&vcpu->arch.mmu, gpte, PT_PAGE_TABLE_LEVEL))
  2135. goto no_present;
  2136. if (!is_present_gpte(gpte))
  2137. goto no_present;
  2138. if (!(gpte & PT_ACCESSED_MASK))
  2139. goto no_present;
  2140. return false;
  2141. no_present:
  2142. drop_spte(vcpu->kvm, spte);
  2143. return true;
  2144. }
  2145. static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
  2146. struct kvm_mmu_page *sp,
  2147. u64 *start, u64 *end)
  2148. {
  2149. struct page *pages[PTE_PREFETCH_NUM];
  2150. unsigned access = sp->role.access;
  2151. int i, ret;
  2152. gfn_t gfn;
  2153. gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt);
  2154. if (!gfn_to_memslot_dirty_bitmap(vcpu, gfn, access & ACC_WRITE_MASK))
  2155. return -1;
  2156. ret = gfn_to_page_many_atomic(vcpu->kvm, gfn, pages, end - start);
  2157. if (ret <= 0)
  2158. return -1;
  2159. for (i = 0; i < ret; i++, gfn++, start++)
  2160. mmu_set_spte(vcpu, start, access, 0, NULL,
  2161. sp->role.level, gfn, page_to_pfn(pages[i]),
  2162. true, true);
  2163. return 0;
  2164. }
  2165. static void __direct_pte_prefetch(struct kvm_vcpu *vcpu,
  2166. struct kvm_mmu_page *sp, u64 *sptep)
  2167. {
  2168. u64 *spte, *start = NULL;
  2169. int i;
  2170. WARN_ON(!sp->role.direct);
  2171. i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
  2172. spte = sp->spt + i;
  2173. for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
  2174. if (is_shadow_present_pte(*spte) || spte == sptep) {
  2175. if (!start)
  2176. continue;
  2177. if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0)
  2178. break;
  2179. start = NULL;
  2180. } else if (!start)
  2181. start = spte;
  2182. }
  2183. }
  2184. static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
  2185. {
  2186. struct kvm_mmu_page *sp;
  2187. /*
  2188. * Since it's no accessed bit on EPT, it's no way to
  2189. * distinguish between actually accessed translations
  2190. * and prefetched, so disable pte prefetch if EPT is
  2191. * enabled.
  2192. */
  2193. if (!shadow_accessed_mask)
  2194. return;
  2195. sp = page_header(__pa(sptep));
  2196. if (sp->role.level > PT_PAGE_TABLE_LEVEL)
  2197. return;
  2198. __direct_pte_prefetch(vcpu, sp, sptep);
  2199. }
  2200. static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
  2201. int map_writable, int level, gfn_t gfn, pfn_t pfn,
  2202. bool prefault)
  2203. {
  2204. struct kvm_shadow_walk_iterator iterator;
  2205. struct kvm_mmu_page *sp;
  2206. int emulate = 0;
  2207. gfn_t pseudo_gfn;
  2208. for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
  2209. if (iterator.level == level) {
  2210. mmu_set_spte(vcpu, iterator.sptep, ACC_ALL,
  2211. write, &emulate, level, gfn, pfn,
  2212. prefault, map_writable);
  2213. direct_pte_prefetch(vcpu, iterator.sptep);
  2214. ++vcpu->stat.pf_fixed;
  2215. break;
  2216. }
  2217. if (!is_shadow_present_pte(*iterator.sptep)) {
  2218. u64 base_addr = iterator.addr;
  2219. base_addr &= PT64_LVL_ADDR_MASK(iterator.level);
  2220. pseudo_gfn = base_addr >> PAGE_SHIFT;
  2221. sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr,
  2222. iterator.level - 1,
  2223. 1, ACC_ALL, iterator.sptep);
  2224. link_shadow_page(iterator.sptep, sp);
  2225. }
  2226. }
  2227. return emulate;
  2228. }
  2229. static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *tsk)
  2230. {
  2231. siginfo_t info;
  2232. info.si_signo = SIGBUS;
  2233. info.si_errno = 0;
  2234. info.si_code = BUS_MCEERR_AR;
  2235. info.si_addr = (void __user *)address;
  2236. info.si_addr_lsb = PAGE_SHIFT;
  2237. send_sig_info(SIGBUS, &info, tsk);
  2238. }
  2239. static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, pfn_t pfn)
  2240. {
  2241. /*
  2242. * Do not cache the mmio info caused by writing the readonly gfn
  2243. * into the spte otherwise read access on readonly gfn also can
  2244. * caused mmio page fault and treat it as mmio access.
  2245. * Return 1 to tell kvm to emulate it.
  2246. */
  2247. if (pfn == KVM_PFN_ERR_RO_FAULT)
  2248. return 1;
  2249. if (pfn == KVM_PFN_ERR_HWPOISON) {
  2250. kvm_send_hwpoison_signal(gfn_to_hva(vcpu->kvm, gfn), current);
  2251. return 0;
  2252. }
  2253. return -EFAULT;
  2254. }
  2255. static void transparent_hugepage_adjust(struct kvm_vcpu *vcpu,
  2256. gfn_t *gfnp, pfn_t *pfnp, int *levelp)
  2257. {
  2258. pfn_t pfn = *pfnp;
  2259. gfn_t gfn = *gfnp;
  2260. int level = *levelp;
  2261. /*
  2262. * Check if it's a transparent hugepage. If this would be an
  2263. * hugetlbfs page, level wouldn't be set to
  2264. * PT_PAGE_TABLE_LEVEL and there would be no adjustment done
  2265. * here.
  2266. */
  2267. if (!is_error_noslot_pfn(pfn) && !kvm_is_mmio_pfn(pfn) &&
  2268. level == PT_PAGE_TABLE_LEVEL &&
  2269. PageTransCompound(pfn_to_page(pfn)) &&
  2270. !has_wrprotected_page(vcpu->kvm, gfn, PT_DIRECTORY_LEVEL)) {
  2271. unsigned long mask;
  2272. /*
  2273. * mmu_notifier_retry was successful and we hold the
  2274. * mmu_lock here, so the pmd can't become splitting
  2275. * from under us, and in turn
  2276. * __split_huge_page_refcount() can't run from under
  2277. * us and we can safely transfer the refcount from
  2278. * PG_tail to PG_head as we switch the pfn to tail to
  2279. * head.
  2280. */
  2281. *levelp = level = PT_DIRECTORY_LEVEL;
  2282. mask = KVM_PAGES_PER_HPAGE(level) - 1;
  2283. VM_BUG_ON((gfn & mask) != (pfn & mask));
  2284. if (pfn & mask) {
  2285. gfn &= ~mask;
  2286. *gfnp = gfn;
  2287. kvm_release_pfn_clean(pfn);
  2288. pfn &= ~mask;
  2289. kvm_get_pfn(pfn);
  2290. *pfnp = pfn;
  2291. }
  2292. }
  2293. }
  2294. static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn,
  2295. pfn_t pfn, unsigned access, int *ret_val)
  2296. {
  2297. bool ret = true;
  2298. /* The pfn is invalid, report the error! */
  2299. if (unlikely(is_error_pfn(pfn))) {
  2300. *ret_val = kvm_handle_bad_page(vcpu, gfn, pfn);
  2301. goto exit;
  2302. }
  2303. if (unlikely(is_noslot_pfn(pfn)))
  2304. vcpu_cache_mmio_info(vcpu, gva, gfn, access);
  2305. ret = false;
  2306. exit:
  2307. return ret;
  2308. }
  2309. static bool page_fault_can_be_fast(struct kvm_vcpu *vcpu, u32 error_code)
  2310. {
  2311. /*
  2312. * #PF can be fast only if the shadow page table is present and it
  2313. * is caused by write-protect, that means we just need change the
  2314. * W bit of the spte which can be done out of mmu-lock.
  2315. */
  2316. if (!(error_code & PFERR_PRESENT_MASK) ||
  2317. !(error_code & PFERR_WRITE_MASK))
  2318. return false;
  2319. return true;
  2320. }
  2321. static bool
  2322. fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 spte)
  2323. {
  2324. struct kvm_mmu_page *sp = page_header(__pa(sptep));
  2325. gfn_t gfn;
  2326. WARN_ON(!sp->role.direct);
  2327. /*
  2328. * The gfn of direct spte is stable since it is calculated
  2329. * by sp->gfn.
  2330. */
  2331. gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt);
  2332. if (cmpxchg64(sptep, spte, spte | PT_WRITABLE_MASK) == spte)
  2333. mark_page_dirty(vcpu->kvm, gfn);
  2334. return true;
  2335. }
  2336. /*
  2337. * Return value:
  2338. * - true: let the vcpu to access on the same address again.
  2339. * - false: let the real page fault path to fix it.
  2340. */
  2341. static bool fast_page_fault(struct kvm_vcpu *vcpu, gva_t gva, int level,
  2342. u32 error_code)
  2343. {
  2344. struct kvm_shadow_walk_iterator iterator;
  2345. bool ret = false;
  2346. u64 spte = 0ull;
  2347. if (!page_fault_can_be_fast(vcpu, error_code))
  2348. return false;
  2349. walk_shadow_page_lockless_begin(vcpu);
  2350. for_each_shadow_entry_lockless(vcpu, gva, iterator, spte)
  2351. if (!is_shadow_present_pte(spte) || iterator.level < level)
  2352. break;
  2353. /*
  2354. * If the mapping has been changed, let the vcpu fault on the
  2355. * same address again.
  2356. */
  2357. if (!is_rmap_spte(spte)) {
  2358. ret = true;
  2359. goto exit;
  2360. }
  2361. if (!is_last_spte(spte, level))
  2362. goto exit;
  2363. /*
  2364. * Check if it is a spurious fault caused by TLB lazily flushed.
  2365. *
  2366. * Need not check the access of upper level table entries since
  2367. * they are always ACC_ALL.
  2368. */
  2369. if (is_writable_pte(spte)) {
  2370. ret = true;
  2371. goto exit;
  2372. }
  2373. /*
  2374. * Currently, to simplify the code, only the spte write-protected
  2375. * by dirty-log can be fast fixed.
  2376. */
  2377. if (!spte_is_locklessly_modifiable(spte))
  2378. goto exit;
  2379. /*
  2380. * Currently, fast page fault only works for direct mapping since
  2381. * the gfn is not stable for indirect shadow page.
  2382. * See Documentation/virtual/kvm/locking.txt to get more detail.
  2383. */
  2384. ret = fast_pf_fix_direct_spte(vcpu, iterator.sptep, spte);
  2385. exit:
  2386. trace_fast_page_fault(vcpu, gva, error_code, iterator.sptep,
  2387. spte, ret);
  2388. walk_shadow_page_lockless_end(vcpu);
  2389. return ret;
  2390. }
  2391. static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
  2392. gva_t gva, pfn_t *pfn, bool write, bool *writable);
  2393. static void make_mmu_pages_available(struct kvm_vcpu *vcpu);
  2394. static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, u32 error_code,
  2395. gfn_t gfn, bool prefault)
  2396. {
  2397. int r;
  2398. int level;
  2399. int force_pt_level;
  2400. pfn_t pfn;
  2401. unsigned long mmu_seq;
  2402. bool map_writable, write = error_code & PFERR_WRITE_MASK;
  2403. force_pt_level = mapping_level_dirty_bitmap(vcpu, gfn);
  2404. if (likely(!force_pt_level)) {
  2405. level = mapping_level(vcpu, gfn);
  2406. /*
  2407. * This path builds a PAE pagetable - so we can map
  2408. * 2mb pages at maximum. Therefore check if the level
  2409. * is larger than that.
  2410. */
  2411. if (level > PT_DIRECTORY_LEVEL)
  2412. level = PT_DIRECTORY_LEVEL;
  2413. gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
  2414. } else
  2415. level = PT_PAGE_TABLE_LEVEL;
  2416. if (fast_page_fault(vcpu, v, level, error_code))
  2417. return 0;
  2418. mmu_seq = vcpu->kvm->mmu_notifier_seq;
  2419. smp_rmb();
  2420. if (try_async_pf(vcpu, prefault, gfn, v, &pfn, write, &map_writable))
  2421. return 0;
  2422. if (handle_abnormal_pfn(vcpu, v, gfn, pfn, ACC_ALL, &r))
  2423. return r;
  2424. spin_lock(&vcpu->kvm->mmu_lock);
  2425. if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
  2426. goto out_unlock;
  2427. make_mmu_pages_available(vcpu);
  2428. if (likely(!force_pt_level))
  2429. transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
  2430. r = __direct_map(vcpu, v, write, map_writable, level, gfn, pfn,
  2431. prefault);
  2432. spin_unlock(&vcpu->kvm->mmu_lock);
  2433. return r;
  2434. out_unlock:
  2435. spin_unlock(&vcpu->kvm->mmu_lock);
  2436. kvm_release_pfn_clean(pfn);
  2437. return 0;
  2438. }
  2439. static void mmu_free_roots(struct kvm_vcpu *vcpu)
  2440. {
  2441. int i;
  2442. struct kvm_mmu_page *sp;
  2443. LIST_HEAD(invalid_list);
  2444. if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
  2445. return;
  2446. if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL &&
  2447. (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL ||
  2448. vcpu->arch.mmu.direct_map)) {
  2449. hpa_t root = vcpu->arch.mmu.root_hpa;
  2450. spin_lock(&vcpu->kvm->mmu_lock);
  2451. sp = page_header(root);
  2452. --sp->root_count;
  2453. if (!sp->root_count && sp->role.invalid) {
  2454. kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
  2455. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  2456. }
  2457. spin_unlock(&vcpu->kvm->mmu_lock);
  2458. vcpu->arch.mmu.root_hpa = INVALID_PAGE;
  2459. return;
  2460. }
  2461. spin_lock(&vcpu->kvm->mmu_lock);
  2462. for (i = 0; i < 4; ++i) {
  2463. hpa_t root = vcpu->arch.mmu.pae_root[i];
  2464. if (root) {
  2465. root &= PT64_BASE_ADDR_MASK;
  2466. sp = page_header(root);
  2467. --sp->root_count;
  2468. if (!sp->root_count && sp->role.invalid)
  2469. kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
  2470. &invalid_list);
  2471. }
  2472. vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
  2473. }
  2474. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  2475. spin_unlock(&vcpu->kvm->mmu_lock);
  2476. vcpu->arch.mmu.root_hpa = INVALID_PAGE;
  2477. }
  2478. static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
  2479. {
  2480. int ret = 0;
  2481. if (!kvm_is_visible_gfn(vcpu->kvm, root_gfn)) {
  2482. kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
  2483. ret = 1;
  2484. }
  2485. return ret;
  2486. }
  2487. static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu)
  2488. {
  2489. struct kvm_mmu_page *sp;
  2490. unsigned i;
  2491. if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
  2492. spin_lock(&vcpu->kvm->mmu_lock);
  2493. make_mmu_pages_available(vcpu);
  2494. sp = kvm_mmu_get_page(vcpu, 0, 0, PT64_ROOT_LEVEL,
  2495. 1, ACC_ALL, NULL);
  2496. ++sp->root_count;
  2497. spin_unlock(&vcpu->kvm->mmu_lock);
  2498. vcpu->arch.mmu.root_hpa = __pa(sp->spt);
  2499. } else if (vcpu->arch.mmu.shadow_root_level == PT32E_ROOT_LEVEL) {
  2500. for (i = 0; i < 4; ++i) {
  2501. hpa_t root = vcpu->arch.mmu.pae_root[i];
  2502. ASSERT(!VALID_PAGE(root));
  2503. spin_lock(&vcpu->kvm->mmu_lock);
  2504. make_mmu_pages_available(vcpu);
  2505. sp = kvm_mmu_get_page(vcpu, i << (30 - PAGE_SHIFT),
  2506. i << 30,
  2507. PT32_ROOT_LEVEL, 1, ACC_ALL,
  2508. NULL);
  2509. root = __pa(sp->spt);
  2510. ++sp->root_count;
  2511. spin_unlock(&vcpu->kvm->mmu_lock);
  2512. vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
  2513. }
  2514. vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
  2515. } else
  2516. BUG();
  2517. return 0;
  2518. }
  2519. static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
  2520. {
  2521. struct kvm_mmu_page *sp;
  2522. u64 pdptr, pm_mask;
  2523. gfn_t root_gfn;
  2524. int i;
  2525. root_gfn = vcpu->arch.mmu.get_cr3(vcpu) >> PAGE_SHIFT;
  2526. if (mmu_check_root(vcpu, root_gfn))
  2527. return 1;
  2528. /*
  2529. * Do we shadow a long mode page table? If so we need to
  2530. * write-protect the guests page table root.
  2531. */
  2532. if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
  2533. hpa_t root = vcpu->arch.mmu.root_hpa;
  2534. ASSERT(!VALID_PAGE(root));
  2535. spin_lock(&vcpu->kvm->mmu_lock);
  2536. make_mmu_pages_available(vcpu);
  2537. sp = kvm_mmu_get_page(vcpu, root_gfn, 0, PT64_ROOT_LEVEL,
  2538. 0, ACC_ALL, NULL);
  2539. root = __pa(sp->spt);
  2540. ++sp->root_count;
  2541. spin_unlock(&vcpu->kvm->mmu_lock);
  2542. vcpu->arch.mmu.root_hpa = root;
  2543. return 0;
  2544. }
  2545. /*
  2546. * We shadow a 32 bit page table. This may be a legacy 2-level
  2547. * or a PAE 3-level page table. In either case we need to be aware that
  2548. * the shadow page table may be a PAE or a long mode page table.
  2549. */
  2550. pm_mask = PT_PRESENT_MASK;
  2551. if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL)
  2552. pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK;
  2553. for (i = 0; i < 4; ++i) {
  2554. hpa_t root = vcpu->arch.mmu.pae_root[i];
  2555. ASSERT(!VALID_PAGE(root));
  2556. if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
  2557. pdptr = vcpu->arch.mmu.get_pdptr(vcpu, i);
  2558. if (!is_present_gpte(pdptr)) {
  2559. vcpu->arch.mmu.pae_root[i] = 0;
  2560. continue;
  2561. }
  2562. root_gfn = pdptr >> PAGE_SHIFT;
  2563. if (mmu_check_root(vcpu, root_gfn))
  2564. return 1;
  2565. }
  2566. spin_lock(&vcpu->kvm->mmu_lock);
  2567. make_mmu_pages_available(vcpu);
  2568. sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
  2569. PT32_ROOT_LEVEL, 0,
  2570. ACC_ALL, NULL);
  2571. root = __pa(sp->spt);
  2572. ++sp->root_count;
  2573. spin_unlock(&vcpu->kvm->mmu_lock);
  2574. vcpu->arch.mmu.pae_root[i] = root | pm_mask;
  2575. }
  2576. vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
  2577. /*
  2578. * If we shadow a 32 bit page table with a long mode page
  2579. * table we enter this path.
  2580. */
  2581. if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
  2582. if (vcpu->arch.mmu.lm_root == NULL) {
  2583. /*
  2584. * The additional page necessary for this is only
  2585. * allocated on demand.
  2586. */
  2587. u64 *lm_root;
  2588. lm_root = (void*)get_zeroed_page(GFP_KERNEL);
  2589. if (lm_root == NULL)
  2590. return 1;
  2591. lm_root[0] = __pa(vcpu->arch.mmu.pae_root) | pm_mask;
  2592. vcpu->arch.mmu.lm_root = lm_root;
  2593. }
  2594. vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.lm_root);
  2595. }
  2596. return 0;
  2597. }
  2598. static int mmu_alloc_roots(struct kvm_vcpu *vcpu)
  2599. {
  2600. if (vcpu->arch.mmu.direct_map)
  2601. return mmu_alloc_direct_roots(vcpu);
  2602. else
  2603. return mmu_alloc_shadow_roots(vcpu);
  2604. }
  2605. static void mmu_sync_roots(struct kvm_vcpu *vcpu)
  2606. {
  2607. int i;
  2608. struct kvm_mmu_page *sp;
  2609. if (vcpu->arch.mmu.direct_map)
  2610. return;
  2611. if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
  2612. return;
  2613. vcpu_clear_mmio_info(vcpu, ~0ul);
  2614. kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);
  2615. if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
  2616. hpa_t root = vcpu->arch.mmu.root_hpa;
  2617. sp = page_header(root);
  2618. mmu_sync_children(vcpu, sp);
  2619. kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
  2620. return;
  2621. }
  2622. for (i = 0; i < 4; ++i) {
  2623. hpa_t root = vcpu->arch.mmu.pae_root[i];
  2624. if (root && VALID_PAGE(root)) {
  2625. root &= PT64_BASE_ADDR_MASK;
  2626. sp = page_header(root);
  2627. mmu_sync_children(vcpu, sp);
  2628. }
  2629. }
  2630. kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
  2631. }
  2632. void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
  2633. {
  2634. spin_lock(&vcpu->kvm->mmu_lock);
  2635. mmu_sync_roots(vcpu);
  2636. spin_unlock(&vcpu->kvm->mmu_lock);
  2637. }
  2638. static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr,
  2639. u32 access, struct x86_exception *exception)
  2640. {
  2641. if (exception)
  2642. exception->error_code = 0;
  2643. return vaddr;
  2644. }
  2645. static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gva_t vaddr,
  2646. u32 access,
  2647. struct x86_exception *exception)
  2648. {
  2649. if (exception)
  2650. exception->error_code = 0;
  2651. return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access);
  2652. }
  2653. static bool quickly_check_mmio_pf(struct kvm_vcpu *vcpu, u64 addr, bool direct)
  2654. {
  2655. if (direct)
  2656. return vcpu_match_mmio_gpa(vcpu, addr);
  2657. return vcpu_match_mmio_gva(vcpu, addr);
  2658. }
  2659. /*
  2660. * On direct hosts, the last spte is only allows two states
  2661. * for mmio page fault:
  2662. * - It is the mmio spte
  2663. * - It is zapped or it is being zapped.
  2664. *
  2665. * This function completely checks the spte when the last spte
  2666. * is not the mmio spte.
  2667. */
  2668. static bool check_direct_spte_mmio_pf(u64 spte)
  2669. {
  2670. return __check_direct_spte_mmio_pf(spte);
  2671. }
  2672. static u64 walk_shadow_page_get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr)
  2673. {
  2674. struct kvm_shadow_walk_iterator iterator;
  2675. u64 spte = 0ull;
  2676. walk_shadow_page_lockless_begin(vcpu);
  2677. for_each_shadow_entry_lockless(vcpu, addr, iterator, spte)
  2678. if (!is_shadow_present_pte(spte))
  2679. break;
  2680. walk_shadow_page_lockless_end(vcpu);
  2681. return spte;
  2682. }
  2683. int handle_mmio_page_fault_common(struct kvm_vcpu *vcpu, u64 addr, bool direct)
  2684. {
  2685. u64 spte;
  2686. if (quickly_check_mmio_pf(vcpu, addr, direct))
  2687. return RET_MMIO_PF_EMULATE;
  2688. spte = walk_shadow_page_get_mmio_spte(vcpu, addr);
  2689. if (is_mmio_spte(spte)) {
  2690. gfn_t gfn = get_mmio_spte_gfn(spte);
  2691. unsigned access = get_mmio_spte_access(spte);
  2692. if (!check_mmio_spte(vcpu->kvm, spte))
  2693. return RET_MMIO_PF_INVALID;
  2694. if (direct)
  2695. addr = 0;
  2696. trace_handle_mmio_page_fault(addr, gfn, access);
  2697. vcpu_cache_mmio_info(vcpu, addr, gfn, access);
  2698. return RET_MMIO_PF_EMULATE;
  2699. }
  2700. /*
  2701. * It's ok if the gva is remapped by other cpus on shadow guest,
  2702. * it's a BUG if the gfn is not a mmio page.
  2703. */
  2704. if (direct && !check_direct_spte_mmio_pf(spte))
  2705. return RET_MMIO_PF_BUG;
  2706. /*
  2707. * If the page table is zapped by other cpus, let CPU fault again on
  2708. * the address.
  2709. */
  2710. return RET_MMIO_PF_RETRY;
  2711. }
  2712. EXPORT_SYMBOL_GPL(handle_mmio_page_fault_common);
  2713. static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr,
  2714. u32 error_code, bool direct)
  2715. {
  2716. int ret;
  2717. ret = handle_mmio_page_fault_common(vcpu, addr, direct);
  2718. WARN_ON(ret == RET_MMIO_PF_BUG);
  2719. return ret;
  2720. }
  2721. static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
  2722. u32 error_code, bool prefault)
  2723. {
  2724. gfn_t gfn;
  2725. int r;
  2726. pgprintk("%s: gva %lx error %x\n", __func__, gva, error_code);
  2727. if (unlikely(error_code & PFERR_RSVD_MASK)) {
  2728. r = handle_mmio_page_fault(vcpu, gva, error_code, true);
  2729. if (likely(r != RET_MMIO_PF_INVALID))
  2730. return r;
  2731. }
  2732. r = mmu_topup_memory_caches(vcpu);
  2733. if (r)
  2734. return r;
  2735. ASSERT(vcpu);
  2736. ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
  2737. gfn = gva >> PAGE_SHIFT;
  2738. return nonpaging_map(vcpu, gva & PAGE_MASK,
  2739. error_code, gfn, prefault);
  2740. }
  2741. static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn)
  2742. {
  2743. struct kvm_arch_async_pf arch;
  2744. arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id;
  2745. arch.gfn = gfn;
  2746. arch.direct_map = vcpu->arch.mmu.direct_map;
  2747. arch.cr3 = vcpu->arch.mmu.get_cr3(vcpu);
  2748. return kvm_setup_async_pf(vcpu, gva, gfn, &arch);
  2749. }
  2750. static bool can_do_async_pf(struct kvm_vcpu *vcpu)
  2751. {
  2752. if (unlikely(!irqchip_in_kernel(vcpu->kvm) ||
  2753. kvm_event_needs_reinjection(vcpu)))
  2754. return false;
  2755. return kvm_x86_ops->interrupt_allowed(vcpu);
  2756. }
  2757. static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
  2758. gva_t gva, pfn_t *pfn, bool write, bool *writable)
  2759. {
  2760. bool async;
  2761. *pfn = gfn_to_pfn_async(vcpu->kvm, gfn, &async, write, writable);
  2762. if (!async)
  2763. return false; /* *pfn has correct page already */
  2764. if (!prefault && can_do_async_pf(vcpu)) {
  2765. trace_kvm_try_async_get_page(gva, gfn);
  2766. if (kvm_find_async_pf_gfn(vcpu, gfn)) {
  2767. trace_kvm_async_pf_doublefault(gva, gfn);
  2768. kvm_make_request(KVM_REQ_APF_HALT, vcpu);
  2769. return true;
  2770. } else if (kvm_arch_setup_async_pf(vcpu, gva, gfn))
  2771. return true;
  2772. }
  2773. *pfn = gfn_to_pfn_prot(vcpu->kvm, gfn, write, writable);
  2774. return false;
  2775. }
  2776. static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, u32 error_code,
  2777. bool prefault)
  2778. {
  2779. pfn_t pfn;
  2780. int r;
  2781. int level;
  2782. int force_pt_level;
  2783. gfn_t gfn = gpa >> PAGE_SHIFT;
  2784. unsigned long mmu_seq;
  2785. int write = error_code & PFERR_WRITE_MASK;
  2786. bool map_writable;
  2787. ASSERT(vcpu);
  2788. ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
  2789. if (unlikely(error_code & PFERR_RSVD_MASK)) {
  2790. r = handle_mmio_page_fault(vcpu, gpa, error_code, true);
  2791. if (likely(r != RET_MMIO_PF_INVALID))
  2792. return r;
  2793. }
  2794. r = mmu_topup_memory_caches(vcpu);
  2795. if (r)
  2796. return r;
  2797. force_pt_level = mapping_level_dirty_bitmap(vcpu, gfn);
  2798. if (likely(!force_pt_level)) {
  2799. level = mapping_level(vcpu, gfn);
  2800. gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
  2801. } else
  2802. level = PT_PAGE_TABLE_LEVEL;
  2803. if (fast_page_fault(vcpu, gpa, level, error_code))
  2804. return 0;
  2805. mmu_seq = vcpu->kvm->mmu_notifier_seq;
  2806. smp_rmb();
  2807. if (try_async_pf(vcpu, prefault, gfn, gpa, &pfn, write, &map_writable))
  2808. return 0;
  2809. if (handle_abnormal_pfn(vcpu, 0, gfn, pfn, ACC_ALL, &r))
  2810. return r;
  2811. spin_lock(&vcpu->kvm->mmu_lock);
  2812. if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
  2813. goto out_unlock;
  2814. make_mmu_pages_available(vcpu);
  2815. if (likely(!force_pt_level))
  2816. transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
  2817. r = __direct_map(vcpu, gpa, write, map_writable,
  2818. level, gfn, pfn, prefault);
  2819. spin_unlock(&vcpu->kvm->mmu_lock);
  2820. return r;
  2821. out_unlock:
  2822. spin_unlock(&vcpu->kvm->mmu_lock);
  2823. kvm_release_pfn_clean(pfn);
  2824. return 0;
  2825. }
  2826. static void nonpaging_free(struct kvm_vcpu *vcpu)
  2827. {
  2828. mmu_free_roots(vcpu);
  2829. }
  2830. static int nonpaging_init_context(struct kvm_vcpu *vcpu,
  2831. struct kvm_mmu *context)
  2832. {
  2833. context->new_cr3 = nonpaging_new_cr3;
  2834. context->page_fault = nonpaging_page_fault;
  2835. context->gva_to_gpa = nonpaging_gva_to_gpa;
  2836. context->free = nonpaging_free;
  2837. context->sync_page = nonpaging_sync_page;
  2838. context->invlpg = nonpaging_invlpg;
  2839. context->update_pte = nonpaging_update_pte;
  2840. context->root_level = 0;
  2841. context->shadow_root_level = PT32E_ROOT_LEVEL;
  2842. context->root_hpa = INVALID_PAGE;
  2843. context->direct_map = true;
  2844. context->nx = false;
  2845. return 0;
  2846. }
  2847. void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
  2848. {
  2849. ++vcpu->stat.tlb_flush;
  2850. kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
  2851. }
  2852. static void paging_new_cr3(struct kvm_vcpu *vcpu)
  2853. {
  2854. pgprintk("%s: cr3 %lx\n", __func__, kvm_read_cr3(vcpu));
  2855. mmu_free_roots(vcpu);
  2856. }
  2857. static unsigned long get_cr3(struct kvm_vcpu *vcpu)
  2858. {
  2859. return kvm_read_cr3(vcpu);
  2860. }
  2861. static void inject_page_fault(struct kvm_vcpu *vcpu,
  2862. struct x86_exception *fault)
  2863. {
  2864. vcpu->arch.mmu.inject_page_fault(vcpu, fault);
  2865. }
  2866. static void paging_free(struct kvm_vcpu *vcpu)
  2867. {
  2868. nonpaging_free(vcpu);
  2869. }
  2870. static inline void protect_clean_gpte(unsigned *access, unsigned gpte)
  2871. {
  2872. unsigned mask;
  2873. BUILD_BUG_ON(PT_WRITABLE_MASK != ACC_WRITE_MASK);
  2874. mask = (unsigned)~ACC_WRITE_MASK;
  2875. /* Allow write access to dirty gptes */
  2876. mask |= (gpte >> (PT_DIRTY_SHIFT - PT_WRITABLE_SHIFT)) & PT_WRITABLE_MASK;
  2877. *access &= mask;
  2878. }
  2879. static bool sync_mmio_spte(struct kvm *kvm, u64 *sptep, gfn_t gfn,
  2880. unsigned access, int *nr_present)
  2881. {
  2882. if (unlikely(is_mmio_spte(*sptep))) {
  2883. if (gfn != get_mmio_spte_gfn(*sptep)) {
  2884. mmu_spte_clear_no_track(sptep);
  2885. return true;
  2886. }
  2887. (*nr_present)++;
  2888. mark_mmio_spte(kvm, sptep, gfn, access);
  2889. return true;
  2890. }
  2891. return false;
  2892. }
  2893. static inline unsigned gpte_access(struct kvm_vcpu *vcpu, u64 gpte)
  2894. {
  2895. unsigned access;
  2896. access = (gpte & (PT_WRITABLE_MASK | PT_USER_MASK)) | ACC_EXEC_MASK;
  2897. access &= ~(gpte >> PT64_NX_SHIFT);
  2898. return access;
  2899. }
  2900. static inline bool is_last_gpte(struct kvm_mmu *mmu, unsigned level, unsigned gpte)
  2901. {
  2902. unsigned index;
  2903. index = level - 1;
  2904. index |= (gpte & PT_PAGE_SIZE_MASK) >> (PT_PAGE_SIZE_SHIFT - 2);
  2905. return mmu->last_pte_bitmap & (1 << index);
  2906. }
  2907. #define PTTYPE 64
  2908. #include "paging_tmpl.h"
  2909. #undef PTTYPE
  2910. #define PTTYPE 32
  2911. #include "paging_tmpl.h"
  2912. #undef PTTYPE
  2913. static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
  2914. struct kvm_mmu *context)
  2915. {
  2916. int maxphyaddr = cpuid_maxphyaddr(vcpu);
  2917. u64 exb_bit_rsvd = 0;
  2918. if (!context->nx)
  2919. exb_bit_rsvd = rsvd_bits(63, 63);
  2920. switch (context->root_level) {
  2921. case PT32_ROOT_LEVEL:
  2922. /* no rsvd bits for 2 level 4K page table entries */
  2923. context->rsvd_bits_mask[0][1] = 0;
  2924. context->rsvd_bits_mask[0][0] = 0;
  2925. context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
  2926. if (!is_pse(vcpu)) {
  2927. context->rsvd_bits_mask[1][1] = 0;
  2928. break;
  2929. }
  2930. if (is_cpuid_PSE36())
  2931. /* 36bits PSE 4MB page */
  2932. context->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
  2933. else
  2934. /* 32 bits PSE 4MB page */
  2935. context->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
  2936. break;
  2937. case PT32E_ROOT_LEVEL:
  2938. context->rsvd_bits_mask[0][2] =
  2939. rsvd_bits(maxphyaddr, 63) |
  2940. rsvd_bits(7, 8) | rsvd_bits(1, 2); /* PDPTE */
  2941. context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
  2942. rsvd_bits(maxphyaddr, 62); /* PDE */
  2943. context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
  2944. rsvd_bits(maxphyaddr, 62); /* PTE */
  2945. context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
  2946. rsvd_bits(maxphyaddr, 62) |
  2947. rsvd_bits(13, 20); /* large page */
  2948. context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
  2949. break;
  2950. case PT64_ROOT_LEVEL:
  2951. context->rsvd_bits_mask[0][3] = exb_bit_rsvd |
  2952. rsvd_bits(maxphyaddr, 51) | rsvd_bits(7, 8);
  2953. context->rsvd_bits_mask[0][2] = exb_bit_rsvd |
  2954. rsvd_bits(maxphyaddr, 51) | rsvd_bits(7, 8);
  2955. context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
  2956. rsvd_bits(maxphyaddr, 51);
  2957. context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
  2958. rsvd_bits(maxphyaddr, 51);
  2959. context->rsvd_bits_mask[1][3] = context->rsvd_bits_mask[0][3];
  2960. context->rsvd_bits_mask[1][2] = exb_bit_rsvd |
  2961. rsvd_bits(maxphyaddr, 51) |
  2962. rsvd_bits(13, 29);
  2963. context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
  2964. rsvd_bits(maxphyaddr, 51) |
  2965. rsvd_bits(13, 20); /* large page */
  2966. context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
  2967. break;
  2968. }
  2969. }
  2970. static void update_permission_bitmask(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
  2971. {
  2972. unsigned bit, byte, pfec;
  2973. u8 map;
  2974. bool fault, x, w, u, wf, uf, ff, smep;
  2975. smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
  2976. for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) {
  2977. pfec = byte << 1;
  2978. map = 0;
  2979. wf = pfec & PFERR_WRITE_MASK;
  2980. uf = pfec & PFERR_USER_MASK;
  2981. ff = pfec & PFERR_FETCH_MASK;
  2982. for (bit = 0; bit < 8; ++bit) {
  2983. x = bit & ACC_EXEC_MASK;
  2984. w = bit & ACC_WRITE_MASK;
  2985. u = bit & ACC_USER_MASK;
  2986. /* Not really needed: !nx will cause pte.nx to fault */
  2987. x |= !mmu->nx;
  2988. /* Allow supervisor writes if !cr0.wp */
  2989. w |= !is_write_protection(vcpu) && !uf;
  2990. /* Disallow supervisor fetches of user code if cr4.smep */
  2991. x &= !(smep && u && !uf);
  2992. fault = (ff && !x) || (uf && !u) || (wf && !w);
  2993. map |= fault << bit;
  2994. }
  2995. mmu->permissions[byte] = map;
  2996. }
  2997. }
  2998. static void update_last_pte_bitmap(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
  2999. {
  3000. u8 map;
  3001. unsigned level, root_level = mmu->root_level;
  3002. const unsigned ps_set_index = 1 << 2; /* bit 2 of index: ps */
  3003. if (root_level == PT32E_ROOT_LEVEL)
  3004. --root_level;
  3005. /* PT_PAGE_TABLE_LEVEL always terminates */
  3006. map = 1 | (1 << ps_set_index);
  3007. for (level = PT_DIRECTORY_LEVEL; level <= root_level; ++level) {
  3008. if (level <= PT_PDPE_LEVEL
  3009. && (mmu->root_level >= PT32E_ROOT_LEVEL || is_pse(vcpu)))
  3010. map |= 1 << (ps_set_index | (level - 1));
  3011. }
  3012. mmu->last_pte_bitmap = map;
  3013. }
  3014. static int paging64_init_context_common(struct kvm_vcpu *vcpu,
  3015. struct kvm_mmu *context,
  3016. int level)
  3017. {
  3018. context->nx = is_nx(vcpu);
  3019. context->root_level = level;
  3020. reset_rsvds_bits_mask(vcpu, context);
  3021. update_permission_bitmask(vcpu, context);
  3022. update_last_pte_bitmap(vcpu, context);
  3023. ASSERT(is_pae(vcpu));
  3024. context->new_cr3 = paging_new_cr3;
  3025. context->page_fault = paging64_page_fault;
  3026. context->gva_to_gpa = paging64_gva_to_gpa;
  3027. context->sync_page = paging64_sync_page;
  3028. context->invlpg = paging64_invlpg;
  3029. context->update_pte = paging64_update_pte;
  3030. context->free = paging_free;
  3031. context->shadow_root_level = level;
  3032. context->root_hpa = INVALID_PAGE;
  3033. context->direct_map = false;
  3034. return 0;
  3035. }
  3036. static int paging64_init_context(struct kvm_vcpu *vcpu,
  3037. struct kvm_mmu *context)
  3038. {
  3039. return paging64_init_context_common(vcpu, context, PT64_ROOT_LEVEL);
  3040. }
  3041. static int paging32_init_context(struct kvm_vcpu *vcpu,
  3042. struct kvm_mmu *context)
  3043. {
  3044. context->nx = false;
  3045. context->root_level = PT32_ROOT_LEVEL;
  3046. reset_rsvds_bits_mask(vcpu, context);
  3047. update_permission_bitmask(vcpu, context);
  3048. update_last_pte_bitmap(vcpu, context);
  3049. context->new_cr3 = paging_new_cr3;
  3050. context->page_fault = paging32_page_fault;
  3051. context->gva_to_gpa = paging32_gva_to_gpa;
  3052. context->free = paging_free;
  3053. context->sync_page = paging32_sync_page;
  3054. context->invlpg = paging32_invlpg;
  3055. context->update_pte = paging32_update_pte;
  3056. context->shadow_root_level = PT32E_ROOT_LEVEL;
  3057. context->root_hpa = INVALID_PAGE;
  3058. context->direct_map = false;
  3059. return 0;
  3060. }
  3061. static int paging32E_init_context(struct kvm_vcpu *vcpu,
  3062. struct kvm_mmu *context)
  3063. {
  3064. return paging64_init_context_common(vcpu, context, PT32E_ROOT_LEVEL);
  3065. }
  3066. static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
  3067. {
  3068. struct kvm_mmu *context = vcpu->arch.walk_mmu;
  3069. context->base_role.word = 0;
  3070. context->new_cr3 = nonpaging_new_cr3;
  3071. context->page_fault = tdp_page_fault;
  3072. context->free = nonpaging_free;
  3073. context->sync_page = nonpaging_sync_page;
  3074. context->invlpg = nonpaging_invlpg;
  3075. context->update_pte = nonpaging_update_pte;
  3076. context->shadow_root_level = kvm_x86_ops->get_tdp_level();
  3077. context->root_hpa = INVALID_PAGE;
  3078. context->direct_map = true;
  3079. context->set_cr3 = kvm_x86_ops->set_tdp_cr3;
  3080. context->get_cr3 = get_cr3;
  3081. context->get_pdptr = kvm_pdptr_read;
  3082. context->inject_page_fault = kvm_inject_page_fault;
  3083. if (!is_paging(vcpu)) {
  3084. context->nx = false;
  3085. context->gva_to_gpa = nonpaging_gva_to_gpa;
  3086. context->root_level = 0;
  3087. } else if (is_long_mode(vcpu)) {
  3088. context->nx = is_nx(vcpu);
  3089. context->root_level = PT64_ROOT_LEVEL;
  3090. reset_rsvds_bits_mask(vcpu, context);
  3091. context->gva_to_gpa = paging64_gva_to_gpa;
  3092. } else if (is_pae(vcpu)) {
  3093. context->nx = is_nx(vcpu);
  3094. context->root_level = PT32E_ROOT_LEVEL;
  3095. reset_rsvds_bits_mask(vcpu, context);
  3096. context->gva_to_gpa = paging64_gva_to_gpa;
  3097. } else {
  3098. context->nx = false;
  3099. context->root_level = PT32_ROOT_LEVEL;
  3100. reset_rsvds_bits_mask(vcpu, context);
  3101. context->gva_to_gpa = paging32_gva_to_gpa;
  3102. }
  3103. update_permission_bitmask(vcpu, context);
  3104. update_last_pte_bitmap(vcpu, context);
  3105. return 0;
  3106. }
  3107. int kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *context)
  3108. {
  3109. int r;
  3110. bool smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
  3111. ASSERT(vcpu);
  3112. ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
  3113. if (!is_paging(vcpu))
  3114. r = nonpaging_init_context(vcpu, context);
  3115. else if (is_long_mode(vcpu))
  3116. r = paging64_init_context(vcpu, context);
  3117. else if (is_pae(vcpu))
  3118. r = paging32E_init_context(vcpu, context);
  3119. else
  3120. r = paging32_init_context(vcpu, context);
  3121. vcpu->arch.mmu.base_role.nxe = is_nx(vcpu);
  3122. vcpu->arch.mmu.base_role.cr4_pae = !!is_pae(vcpu);
  3123. vcpu->arch.mmu.base_role.cr0_wp = is_write_protection(vcpu);
  3124. vcpu->arch.mmu.base_role.smep_andnot_wp
  3125. = smep && !is_write_protection(vcpu);
  3126. return r;
  3127. }
  3128. EXPORT_SYMBOL_GPL(kvm_init_shadow_mmu);
  3129. static int init_kvm_softmmu(struct kvm_vcpu *vcpu)
  3130. {
  3131. int r = kvm_init_shadow_mmu(vcpu, vcpu->arch.walk_mmu);
  3132. vcpu->arch.walk_mmu->set_cr3 = kvm_x86_ops->set_cr3;
  3133. vcpu->arch.walk_mmu->get_cr3 = get_cr3;
  3134. vcpu->arch.walk_mmu->get_pdptr = kvm_pdptr_read;
  3135. vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault;
  3136. return r;
  3137. }
  3138. static int init_kvm_nested_mmu(struct kvm_vcpu *vcpu)
  3139. {
  3140. struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;
  3141. g_context->get_cr3 = get_cr3;
  3142. g_context->get_pdptr = kvm_pdptr_read;
  3143. g_context->inject_page_fault = kvm_inject_page_fault;
  3144. /*
  3145. * Note that arch.mmu.gva_to_gpa translates l2_gva to l1_gpa. The
  3146. * translation of l2_gpa to l1_gpa addresses is done using the
  3147. * arch.nested_mmu.gva_to_gpa function. Basically the gva_to_gpa
  3148. * functions between mmu and nested_mmu are swapped.
  3149. */
  3150. if (!is_paging(vcpu)) {
  3151. g_context->nx = false;
  3152. g_context->root_level = 0;
  3153. g_context->gva_to_gpa = nonpaging_gva_to_gpa_nested;
  3154. } else if (is_long_mode(vcpu)) {
  3155. g_context->nx = is_nx(vcpu);
  3156. g_context->root_level = PT64_ROOT_LEVEL;
  3157. reset_rsvds_bits_mask(vcpu, g_context);
  3158. g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
  3159. } else if (is_pae(vcpu)) {
  3160. g_context->nx = is_nx(vcpu);
  3161. g_context->root_level = PT32E_ROOT_LEVEL;
  3162. reset_rsvds_bits_mask(vcpu, g_context);
  3163. g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
  3164. } else {
  3165. g_context->nx = false;
  3166. g_context->root_level = PT32_ROOT_LEVEL;
  3167. reset_rsvds_bits_mask(vcpu, g_context);
  3168. g_context->gva_to_gpa = paging32_gva_to_gpa_nested;
  3169. }
  3170. update_permission_bitmask(vcpu, g_context);
  3171. update_last_pte_bitmap(vcpu, g_context);
  3172. return 0;
  3173. }
  3174. static int init_kvm_mmu(struct kvm_vcpu *vcpu)
  3175. {
  3176. if (mmu_is_nested(vcpu))
  3177. return init_kvm_nested_mmu(vcpu);
  3178. else if (tdp_enabled)
  3179. return init_kvm_tdp_mmu(vcpu);
  3180. else
  3181. return init_kvm_softmmu(vcpu);
  3182. }
  3183. static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
  3184. {
  3185. ASSERT(vcpu);
  3186. if (VALID_PAGE(vcpu->arch.mmu.root_hpa))
  3187. /* mmu.free() should set root_hpa = INVALID_PAGE */
  3188. vcpu->arch.mmu.free(vcpu);
  3189. }
  3190. int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
  3191. {
  3192. destroy_kvm_mmu(vcpu);
  3193. return init_kvm_mmu(vcpu);
  3194. }
  3195. EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
  3196. int kvm_mmu_load(struct kvm_vcpu *vcpu)
  3197. {
  3198. int r;
  3199. r = mmu_topup_memory_caches(vcpu);
  3200. if (r)
  3201. goto out;
  3202. r = mmu_alloc_roots(vcpu);
  3203. kvm_mmu_sync_roots(vcpu);
  3204. if (r)
  3205. goto out;
  3206. /* set_cr3() should ensure TLB has been flushed */
  3207. vcpu->arch.mmu.set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
  3208. out:
  3209. return r;
  3210. }
  3211. EXPORT_SYMBOL_GPL(kvm_mmu_load);
  3212. void kvm_mmu_unload(struct kvm_vcpu *vcpu)
  3213. {
  3214. mmu_free_roots(vcpu);
  3215. }
  3216. EXPORT_SYMBOL_GPL(kvm_mmu_unload);
  3217. static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
  3218. struct kvm_mmu_page *sp, u64 *spte,
  3219. const void *new)
  3220. {
  3221. if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
  3222. ++vcpu->kvm->stat.mmu_pde_zapped;
  3223. return;
  3224. }
  3225. ++vcpu->kvm->stat.mmu_pte_updated;
  3226. vcpu->arch.mmu.update_pte(vcpu, sp, spte, new);
  3227. }
  3228. static bool need_remote_flush(u64 old, u64 new)
  3229. {
  3230. if (!is_shadow_present_pte(old))
  3231. return false;
  3232. if (!is_shadow_present_pte(new))
  3233. return true;
  3234. if ((old ^ new) & PT64_BASE_ADDR_MASK)
  3235. return true;
  3236. old ^= PT64_NX_MASK;
  3237. new ^= PT64_NX_MASK;
  3238. return (old & ~new & PT64_PERM_MASK) != 0;
  3239. }
  3240. static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, bool zap_page,
  3241. bool remote_flush, bool local_flush)
  3242. {
  3243. if (zap_page)
  3244. return;
  3245. if (remote_flush)
  3246. kvm_flush_remote_tlbs(vcpu->kvm);
  3247. else if (local_flush)
  3248. kvm_mmu_flush_tlb(vcpu);
  3249. }
  3250. static u64 mmu_pte_write_fetch_gpte(struct kvm_vcpu *vcpu, gpa_t *gpa,
  3251. const u8 *new, int *bytes)
  3252. {
  3253. u64 gentry;
  3254. int r;
  3255. /*
  3256. * Assume that the pte write on a page table of the same type
  3257. * as the current vcpu paging mode since we update the sptes only
  3258. * when they have the same mode.
  3259. */
  3260. if (is_pae(vcpu) && *bytes == 4) {
  3261. /* Handle a 32-bit guest writing two halves of a 64-bit gpte */
  3262. *gpa &= ~(gpa_t)7;
  3263. *bytes = 8;
  3264. r = kvm_read_guest(vcpu->kvm, *gpa, &gentry, 8);
  3265. if (r)
  3266. gentry = 0;
  3267. new = (const u8 *)&gentry;
  3268. }
  3269. switch (*bytes) {
  3270. case 4:
  3271. gentry = *(const u32 *)new;
  3272. break;
  3273. case 8:
  3274. gentry = *(const u64 *)new;
  3275. break;
  3276. default:
  3277. gentry = 0;
  3278. break;
  3279. }
  3280. return gentry;
  3281. }
  3282. /*
  3283. * If we're seeing too many writes to a page, it may no longer be a page table,
  3284. * or we may be forking, in which case it is better to unmap the page.
  3285. */
  3286. static bool detect_write_flooding(struct kvm_mmu_page *sp)
  3287. {
  3288. /*
  3289. * Skip write-flooding detected for the sp whose level is 1, because
  3290. * it can become unsync, then the guest page is not write-protected.
  3291. */
  3292. if (sp->role.level == PT_PAGE_TABLE_LEVEL)
  3293. return false;
  3294. return ++sp->write_flooding_count >= 3;
  3295. }
  3296. /*
  3297. * Misaligned accesses are too much trouble to fix up; also, they usually
  3298. * indicate a page is not used as a page table.
  3299. */
  3300. static bool detect_write_misaligned(struct kvm_mmu_page *sp, gpa_t gpa,
  3301. int bytes)
  3302. {
  3303. unsigned offset, pte_size, misaligned;
  3304. pgprintk("misaligned: gpa %llx bytes %d role %x\n",
  3305. gpa, bytes, sp->role.word);
  3306. offset = offset_in_page(gpa);
  3307. pte_size = sp->role.cr4_pae ? 8 : 4;
  3308. /*
  3309. * Sometimes, the OS only writes the last one bytes to update status
  3310. * bits, for example, in linux, andb instruction is used in clear_bit().
  3311. */
  3312. if (!(offset & (pte_size - 1)) && bytes == 1)
  3313. return false;
  3314. misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
  3315. misaligned |= bytes < 4;
  3316. return misaligned;
  3317. }
  3318. static u64 *get_written_sptes(struct kvm_mmu_page *sp, gpa_t gpa, int *nspte)
  3319. {
  3320. unsigned page_offset, quadrant;
  3321. u64 *spte;
  3322. int level;
  3323. page_offset = offset_in_page(gpa);
  3324. level = sp->role.level;
  3325. *nspte = 1;
  3326. if (!sp->role.cr4_pae) {
  3327. page_offset <<= 1; /* 32->64 */
  3328. /*
  3329. * A 32-bit pde maps 4MB while the shadow pdes map
  3330. * only 2MB. So we need to double the offset again
  3331. * and zap two pdes instead of one.
  3332. */
  3333. if (level == PT32_ROOT_LEVEL) {
  3334. page_offset &= ~7; /* kill rounding error */
  3335. page_offset <<= 1;
  3336. *nspte = 2;
  3337. }
  3338. quadrant = page_offset >> PAGE_SHIFT;
  3339. page_offset &= ~PAGE_MASK;
  3340. if (quadrant != sp->role.quadrant)
  3341. return NULL;
  3342. }
  3343. spte = &sp->spt[page_offset / sizeof(*spte)];
  3344. return spte;
  3345. }
  3346. void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
  3347. const u8 *new, int bytes)
  3348. {
  3349. gfn_t gfn = gpa >> PAGE_SHIFT;
  3350. union kvm_mmu_page_role mask = { .word = 0 };
  3351. struct kvm_mmu_page *sp;
  3352. LIST_HEAD(invalid_list);
  3353. u64 entry, gentry, *spte;
  3354. int npte;
  3355. bool remote_flush, local_flush, zap_page;
  3356. /*
  3357. * If we don't have indirect shadow pages, it means no page is
  3358. * write-protected, so we can exit simply.
  3359. */
  3360. if (!ACCESS_ONCE(vcpu->kvm->arch.indirect_shadow_pages))
  3361. return;
  3362. zap_page = remote_flush = local_flush = false;
  3363. pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);
  3364. gentry = mmu_pte_write_fetch_gpte(vcpu, &gpa, new, &bytes);
  3365. /*
  3366. * No need to care whether allocation memory is successful
  3367. * or not since pte prefetch is skiped if it does not have
  3368. * enough objects in the cache.
  3369. */
  3370. mmu_topup_memory_caches(vcpu);
  3371. spin_lock(&vcpu->kvm->mmu_lock);
  3372. ++vcpu->kvm->stat.mmu_pte_write;
  3373. kvm_mmu_audit(vcpu, AUDIT_PRE_PTE_WRITE);
  3374. mask.cr0_wp = mask.cr4_pae = mask.nxe = 1;
  3375. for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) {
  3376. if (detect_write_misaligned(sp, gpa, bytes) ||
  3377. detect_write_flooding(sp)) {
  3378. zap_page |= !!kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
  3379. &invalid_list);
  3380. ++vcpu->kvm->stat.mmu_flooded;
  3381. continue;
  3382. }
  3383. spte = get_written_sptes(sp, gpa, &npte);
  3384. if (!spte)
  3385. continue;
  3386. local_flush = true;
  3387. while (npte--) {
  3388. entry = *spte;
  3389. mmu_page_zap_pte(vcpu->kvm, sp, spte);
  3390. if (gentry &&
  3391. !((sp->role.word ^ vcpu->arch.mmu.base_role.word)
  3392. & mask.word) && rmap_can_add(vcpu))
  3393. mmu_pte_write_new_pte(vcpu, sp, spte, &gentry);
  3394. if (need_remote_flush(entry, *spte))
  3395. remote_flush = true;
  3396. ++spte;
  3397. }
  3398. }
  3399. mmu_pte_write_flush_tlb(vcpu, zap_page, remote_flush, local_flush);
  3400. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  3401. kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE);
  3402. spin_unlock(&vcpu->kvm->mmu_lock);
  3403. }
  3404. int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
  3405. {
  3406. gpa_t gpa;
  3407. int r;
  3408. if (vcpu->arch.mmu.direct_map)
  3409. return 0;
  3410. gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL);
  3411. r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
  3412. return r;
  3413. }
  3414. EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt);
  3415. static void make_mmu_pages_available(struct kvm_vcpu *vcpu)
  3416. {
  3417. LIST_HEAD(invalid_list);
  3418. if (likely(kvm_mmu_available_pages(vcpu->kvm) >= KVM_MIN_FREE_MMU_PAGES))
  3419. return;
  3420. while (kvm_mmu_available_pages(vcpu->kvm) < KVM_REFILL_PAGES) {
  3421. if (!prepare_zap_oldest_mmu_page(vcpu->kvm, &invalid_list))
  3422. break;
  3423. ++vcpu->kvm->stat.mmu_recycled;
  3424. }
  3425. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  3426. }
  3427. static bool is_mmio_page_fault(struct kvm_vcpu *vcpu, gva_t addr)
  3428. {
  3429. if (vcpu->arch.mmu.direct_map || mmu_is_nested(vcpu))
  3430. return vcpu_match_mmio_gpa(vcpu, addr);
  3431. return vcpu_match_mmio_gva(vcpu, addr);
  3432. }
  3433. int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code,
  3434. void *insn, int insn_len)
  3435. {
  3436. int r, emulation_type = EMULTYPE_RETRY;
  3437. enum emulation_result er;
  3438. r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code, false);
  3439. if (r < 0)
  3440. goto out;
  3441. if (!r) {
  3442. r = 1;
  3443. goto out;
  3444. }
  3445. if (is_mmio_page_fault(vcpu, cr2))
  3446. emulation_type = 0;
  3447. er = x86_emulate_instruction(vcpu, cr2, emulation_type, insn, insn_len);
  3448. switch (er) {
  3449. case EMULATE_DONE:
  3450. return 1;
  3451. case EMULATE_DO_MMIO:
  3452. ++vcpu->stat.mmio_exits;
  3453. /* fall through */
  3454. case EMULATE_FAIL:
  3455. return 0;
  3456. default:
  3457. BUG();
  3458. }
  3459. out:
  3460. return r;
  3461. }
  3462. EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
  3463. void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
  3464. {
  3465. vcpu->arch.mmu.invlpg(vcpu, gva);
  3466. kvm_mmu_flush_tlb(vcpu);
  3467. ++vcpu->stat.invlpg;
  3468. }
  3469. EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);
  3470. void kvm_enable_tdp(void)
  3471. {
  3472. tdp_enabled = true;
  3473. }
  3474. EXPORT_SYMBOL_GPL(kvm_enable_tdp);
  3475. void kvm_disable_tdp(void)
  3476. {
  3477. tdp_enabled = false;
  3478. }
  3479. EXPORT_SYMBOL_GPL(kvm_disable_tdp);
  3480. static void free_mmu_pages(struct kvm_vcpu *vcpu)
  3481. {
  3482. free_page((unsigned long)vcpu->arch.mmu.pae_root);
  3483. if (vcpu->arch.mmu.lm_root != NULL)
  3484. free_page((unsigned long)vcpu->arch.mmu.lm_root);
  3485. }
  3486. static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
  3487. {
  3488. struct page *page;
  3489. int i;
  3490. ASSERT(vcpu);
  3491. /*
  3492. * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
  3493. * Therefore we need to allocate shadow page tables in the first
  3494. * 4GB of memory, which happens to fit the DMA32 zone.
  3495. */
  3496. page = alloc_page(GFP_KERNEL | __GFP_DMA32);
  3497. if (!page)
  3498. return -ENOMEM;
  3499. vcpu->arch.mmu.pae_root = page_address(page);
  3500. for (i = 0; i < 4; ++i)
  3501. vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
  3502. return 0;
  3503. }
  3504. int kvm_mmu_create(struct kvm_vcpu *vcpu)
  3505. {
  3506. ASSERT(vcpu);
  3507. vcpu->arch.walk_mmu = &vcpu->arch.mmu;
  3508. vcpu->arch.mmu.root_hpa = INVALID_PAGE;
  3509. vcpu->arch.mmu.translate_gpa = translate_gpa;
  3510. vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa;
  3511. return alloc_mmu_pages(vcpu);
  3512. }
  3513. int kvm_mmu_setup(struct kvm_vcpu *vcpu)
  3514. {
  3515. ASSERT(vcpu);
  3516. ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
  3517. return init_kvm_mmu(vcpu);
  3518. }
  3519. void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
  3520. {
  3521. struct kvm_memory_slot *memslot;
  3522. gfn_t last_gfn;
  3523. int i;
  3524. memslot = id_to_memslot(kvm->memslots, slot);
  3525. last_gfn = memslot->base_gfn + memslot->npages - 1;
  3526. spin_lock(&kvm->mmu_lock);
  3527. for (i = PT_PAGE_TABLE_LEVEL;
  3528. i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
  3529. unsigned long *rmapp;
  3530. unsigned long last_index, index;
  3531. rmapp = memslot->arch.rmap[i - PT_PAGE_TABLE_LEVEL];
  3532. last_index = gfn_to_index(last_gfn, memslot->base_gfn, i);
  3533. for (index = 0; index <= last_index; ++index, ++rmapp) {
  3534. if (*rmapp)
  3535. __rmap_write_protect(kvm, rmapp, false);
  3536. if (need_resched() || spin_needbreak(&kvm->mmu_lock)) {
  3537. kvm_flush_remote_tlbs(kvm);
  3538. cond_resched_lock(&kvm->mmu_lock);
  3539. }
  3540. }
  3541. }
  3542. kvm_flush_remote_tlbs(kvm);
  3543. spin_unlock(&kvm->mmu_lock);
  3544. }
  3545. #define BATCH_ZAP_PAGES 10
  3546. static void kvm_zap_obsolete_pages(struct kvm *kvm)
  3547. {
  3548. struct kvm_mmu_page *sp, *node;
  3549. int batch = 0;
  3550. restart:
  3551. list_for_each_entry_safe_reverse(sp, node,
  3552. &kvm->arch.active_mmu_pages, link) {
  3553. int ret;
  3554. /*
  3555. * No obsolete page exists before new created page since
  3556. * active_mmu_pages is the FIFO list.
  3557. */
  3558. if (!is_obsolete_sp(kvm, sp))
  3559. break;
  3560. /*
  3561. * Since we are reversely walking the list and the invalid
  3562. * list will be moved to the head, skip the invalid page
  3563. * can help us to avoid the infinity list walking.
  3564. */
  3565. if (sp->role.invalid)
  3566. continue;
  3567. /*
  3568. * Need not flush tlb since we only zap the sp with invalid
  3569. * generation number.
  3570. */
  3571. if (batch >= BATCH_ZAP_PAGES &&
  3572. cond_resched_lock(&kvm->mmu_lock)) {
  3573. batch = 0;
  3574. goto restart;
  3575. }
  3576. ret = kvm_mmu_prepare_zap_page(kvm, sp,
  3577. &kvm->arch.zapped_obsolete_pages);
  3578. batch += ret;
  3579. if (ret)
  3580. goto restart;
  3581. }
  3582. /*
  3583. * Should flush tlb before free page tables since lockless-walking
  3584. * may use the pages.
  3585. */
  3586. kvm_mmu_commit_zap_page(kvm, &kvm->arch.zapped_obsolete_pages);
  3587. }
  3588. /*
  3589. * Fast invalidate all shadow pages and use lock-break technique
  3590. * to zap obsolete pages.
  3591. *
  3592. * It's required when memslot is being deleted or VM is being
  3593. * destroyed, in these cases, we should ensure that KVM MMU does
  3594. * not use any resource of the being-deleted slot or all slots
  3595. * after calling the function.
  3596. */
  3597. void kvm_mmu_invalidate_zap_all_pages(struct kvm *kvm)
  3598. {
  3599. spin_lock(&kvm->mmu_lock);
  3600. trace_kvm_mmu_invalidate_zap_all_pages(kvm);
  3601. kvm->arch.mmu_valid_gen++;
  3602. /*
  3603. * Notify all vcpus to reload its shadow page table
  3604. * and flush TLB. Then all vcpus will switch to new
  3605. * shadow page table with the new mmu_valid_gen.
  3606. *
  3607. * Note: we should do this under the protection of
  3608. * mmu-lock, otherwise, vcpu would purge shadow page
  3609. * but miss tlb flush.
  3610. */
  3611. kvm_reload_remote_mmus(kvm);
  3612. kvm_zap_obsolete_pages(kvm);
  3613. spin_unlock(&kvm->mmu_lock);
  3614. }
  3615. static bool kvm_has_zapped_obsolete_pages(struct kvm *kvm)
  3616. {
  3617. return unlikely(!list_empty_careful(&kvm->arch.zapped_obsolete_pages));
  3618. }
  3619. void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm)
  3620. {
  3621. /*
  3622. * The very rare case: if the generation-number is round,
  3623. * zap all shadow pages.
  3624. *
  3625. * The max value is MMIO_MAX_GEN - 1 since it is not called
  3626. * when mark memslot invalid.
  3627. */
  3628. if (unlikely(kvm_current_mmio_generation(kvm) >= (MMIO_MAX_GEN - 1)))
  3629. kvm_mmu_invalidate_zap_all_pages(kvm);
  3630. }
  3631. static int mmu_shrink(struct shrinker *shrink, struct shrink_control *sc)
  3632. {
  3633. struct kvm *kvm;
  3634. int nr_to_scan = sc->nr_to_scan;
  3635. if (nr_to_scan == 0)
  3636. goto out;
  3637. raw_spin_lock(&kvm_lock);
  3638. list_for_each_entry(kvm, &vm_list, vm_list) {
  3639. int idx;
  3640. LIST_HEAD(invalid_list);
  3641. /*
  3642. * Never scan more than sc->nr_to_scan VM instances.
  3643. * Will not hit this condition practically since we do not try
  3644. * to shrink more than one VM and it is very unlikely to see
  3645. * !n_used_mmu_pages so many times.
  3646. */
  3647. if (!nr_to_scan--)
  3648. break;
  3649. /*
  3650. * n_used_mmu_pages is accessed without holding kvm->mmu_lock
  3651. * here. We may skip a VM instance errorneosly, but we do not
  3652. * want to shrink a VM that only started to populate its MMU
  3653. * anyway.
  3654. */
  3655. if (!kvm->arch.n_used_mmu_pages &&
  3656. !kvm_has_zapped_obsolete_pages(kvm))
  3657. continue;
  3658. idx = srcu_read_lock(&kvm->srcu);
  3659. spin_lock(&kvm->mmu_lock);
  3660. if (kvm_has_zapped_obsolete_pages(kvm)) {
  3661. kvm_mmu_commit_zap_page(kvm,
  3662. &kvm->arch.zapped_obsolete_pages);
  3663. goto unlock;
  3664. }
  3665. prepare_zap_oldest_mmu_page(kvm, &invalid_list);
  3666. kvm_mmu_commit_zap_page(kvm, &invalid_list);
  3667. unlock:
  3668. spin_unlock(&kvm->mmu_lock);
  3669. srcu_read_unlock(&kvm->srcu, idx);
  3670. list_move_tail(&kvm->vm_list, &vm_list);
  3671. break;
  3672. }
  3673. raw_spin_unlock(&kvm_lock);
  3674. out:
  3675. return percpu_counter_read_positive(&kvm_total_used_mmu_pages);
  3676. }
  3677. static struct shrinker mmu_shrinker = {
  3678. .shrink = mmu_shrink,
  3679. .seeks = DEFAULT_SEEKS * 10,
  3680. };
  3681. static void mmu_destroy_caches(void)
  3682. {
  3683. if (pte_list_desc_cache)
  3684. kmem_cache_destroy(pte_list_desc_cache);
  3685. if (mmu_page_header_cache)
  3686. kmem_cache_destroy(mmu_page_header_cache);
  3687. }
  3688. int kvm_mmu_module_init(void)
  3689. {
  3690. pte_list_desc_cache = kmem_cache_create("pte_list_desc",
  3691. sizeof(struct pte_list_desc),
  3692. 0, 0, NULL);
  3693. if (!pte_list_desc_cache)
  3694. goto nomem;
  3695. mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
  3696. sizeof(struct kvm_mmu_page),
  3697. 0, 0, NULL);
  3698. if (!mmu_page_header_cache)
  3699. goto nomem;
  3700. if (percpu_counter_init(&kvm_total_used_mmu_pages, 0))
  3701. goto nomem;
  3702. register_shrinker(&mmu_shrinker);
  3703. return 0;
  3704. nomem:
  3705. mmu_destroy_caches();
  3706. return -ENOMEM;
  3707. }
  3708. /*
  3709. * Caculate mmu pages needed for kvm.
  3710. */
  3711. unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
  3712. {
  3713. unsigned int nr_mmu_pages;
  3714. unsigned int nr_pages = 0;
  3715. struct kvm_memslots *slots;
  3716. struct kvm_memory_slot *memslot;
  3717. slots = kvm_memslots(kvm);
  3718. kvm_for_each_memslot(memslot, slots)
  3719. nr_pages += memslot->npages;
  3720. nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
  3721. nr_mmu_pages = max(nr_mmu_pages,
  3722. (unsigned int) KVM_MIN_ALLOC_MMU_PAGES);
  3723. return nr_mmu_pages;
  3724. }
  3725. int kvm_mmu_get_spte_hierarchy(struct kvm_vcpu *vcpu, u64 addr, u64 sptes[4])
  3726. {
  3727. struct kvm_shadow_walk_iterator iterator;
  3728. u64 spte;
  3729. int nr_sptes = 0;
  3730. walk_shadow_page_lockless_begin(vcpu);
  3731. for_each_shadow_entry_lockless(vcpu, addr, iterator, spte) {
  3732. sptes[iterator.level-1] = spte;
  3733. nr_sptes++;
  3734. if (!is_shadow_present_pte(spte))
  3735. break;
  3736. }
  3737. walk_shadow_page_lockless_end(vcpu);
  3738. return nr_sptes;
  3739. }
  3740. EXPORT_SYMBOL_GPL(kvm_mmu_get_spte_hierarchy);
  3741. void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
  3742. {
  3743. ASSERT(vcpu);
  3744. destroy_kvm_mmu(vcpu);
  3745. free_mmu_pages(vcpu);
  3746. mmu_free_memory_caches(vcpu);
  3747. }
  3748. void kvm_mmu_module_exit(void)
  3749. {
  3750. mmu_destroy_caches();
  3751. percpu_counter_destroy(&kvm_total_used_mmu_pages);
  3752. unregister_shrinker(&mmu_shrinker);
  3753. mmu_audit_disable();
  3754. }