mmu.c 89 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * This module enables machines with Intel VT-x extensions to run virtual
  5. * machines without emulation or binary translation.
  6. *
  7. * MMU support
  8. *
  9. * Copyright (C) 2006 Qumranet, Inc.
  10. * Copyright 2010 Red Hat, Inc. and/or its affiliates.
  11. *
  12. * Authors:
  13. * Yaniv Kamay <yaniv@qumranet.com>
  14. * Avi Kivity <avi@qumranet.com>
  15. *
  16. * This work is licensed under the terms of the GNU GPL, version 2. See
  17. * the COPYING file in the top-level directory.
  18. *
  19. */
  20. #include "irq.h"
  21. #include "mmu.h"
  22. #include "x86.h"
  23. #include "kvm_cache_regs.h"
  24. #include "x86.h"
  25. #include <linux/kvm_host.h>
  26. #include <linux/types.h>
  27. #include <linux/string.h>
  28. #include <linux/mm.h>
  29. #include <linux/highmem.h>
  30. #include <linux/module.h>
  31. #include <linux/swap.h>
  32. #include <linux/hugetlb.h>
  33. #include <linux/compiler.h>
  34. #include <linux/srcu.h>
  35. #include <linux/slab.h>
  36. #include <linux/uaccess.h>
  37. #include <asm/page.h>
  38. #include <asm/cmpxchg.h>
  39. #include <asm/io.h>
  40. #include <asm/vmx.h>
  41. /*
  42. * When setting this variable to true it enables Two-Dimensional-Paging
  43. * where the hardware walks 2 page tables:
  44. * 1. the guest-virtual to guest-physical
  45. * 2. while doing 1. it walks guest-physical to host-physical
  46. * If the hardware supports that we don't need to do shadow paging.
  47. */
  48. bool tdp_enabled = false;
  49. enum {
  50. AUDIT_PRE_PAGE_FAULT,
  51. AUDIT_POST_PAGE_FAULT,
  52. AUDIT_PRE_PTE_WRITE,
  53. AUDIT_POST_PTE_WRITE,
  54. AUDIT_PRE_SYNC,
  55. AUDIT_POST_SYNC
  56. };
  57. char *audit_point_name[] = {
  58. "pre page fault",
  59. "post page fault",
  60. "pre pte write",
  61. "post pte write",
  62. "pre sync",
  63. "post sync"
  64. };
  65. #undef MMU_DEBUG
  66. #ifdef MMU_DEBUG
  67. #define pgprintk(x...) do { if (dbg) printk(x); } while (0)
  68. #define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
  69. #else
  70. #define pgprintk(x...) do { } while (0)
  71. #define rmap_printk(x...) do { } while (0)
  72. #endif
  73. #ifdef MMU_DEBUG
  74. static int dbg = 0;
  75. module_param(dbg, bool, 0644);
  76. #endif
  77. static int oos_shadow = 1;
  78. module_param(oos_shadow, bool, 0644);
  79. #ifndef MMU_DEBUG
  80. #define ASSERT(x) do { } while (0)
  81. #else
  82. #define ASSERT(x) \
  83. if (!(x)) { \
  84. printk(KERN_WARNING "assertion failed %s:%d: %s\n", \
  85. __FILE__, __LINE__, #x); \
  86. }
  87. #endif
  88. #define PTE_PREFETCH_NUM 8
  89. #define PT_FIRST_AVAIL_BITS_SHIFT 9
  90. #define PT64_SECOND_AVAIL_BITS_SHIFT 52
  91. #define PT64_LEVEL_BITS 9
  92. #define PT64_LEVEL_SHIFT(level) \
  93. (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
  94. #define PT64_INDEX(address, level)\
  95. (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
  96. #define PT32_LEVEL_BITS 10
  97. #define PT32_LEVEL_SHIFT(level) \
  98. (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
  99. #define PT32_LVL_OFFSET_MASK(level) \
  100. (PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
  101. * PT32_LEVEL_BITS))) - 1))
  102. #define PT32_INDEX(address, level)\
  103. (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
  104. #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
  105. #define PT64_DIR_BASE_ADDR_MASK \
  106. (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
  107. #define PT64_LVL_ADDR_MASK(level) \
  108. (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
  109. * PT64_LEVEL_BITS))) - 1))
  110. #define PT64_LVL_OFFSET_MASK(level) \
  111. (PT64_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
  112. * PT64_LEVEL_BITS))) - 1))
  113. #define PT32_BASE_ADDR_MASK PAGE_MASK
  114. #define PT32_DIR_BASE_ADDR_MASK \
  115. (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
  116. #define PT32_LVL_ADDR_MASK(level) \
  117. (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
  118. * PT32_LEVEL_BITS))) - 1))
  119. #define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK \
  120. | PT64_NX_MASK)
  121. #define PTE_LIST_EXT 4
  122. #define ACC_EXEC_MASK 1
  123. #define ACC_WRITE_MASK PT_WRITABLE_MASK
  124. #define ACC_USER_MASK PT_USER_MASK
  125. #define ACC_ALL (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
  126. #include <trace/events/kvm.h>
  127. #define CREATE_TRACE_POINTS
  128. #include "mmutrace.h"
  129. #define SPTE_HOST_WRITEABLE (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
  130. #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)
  131. struct pte_list_desc {
  132. u64 *sptes[PTE_LIST_EXT];
  133. struct pte_list_desc *more;
  134. };
  135. struct kvm_shadow_walk_iterator {
  136. u64 addr;
  137. hpa_t shadow_addr;
  138. int level;
  139. u64 *sptep;
  140. unsigned index;
  141. };
  142. #define for_each_shadow_entry(_vcpu, _addr, _walker) \
  143. for (shadow_walk_init(&(_walker), _vcpu, _addr); \
  144. shadow_walk_okay(&(_walker)); \
  145. shadow_walk_next(&(_walker)))
  146. static struct kmem_cache *pte_list_desc_cache;
  147. static struct kmem_cache *mmu_page_header_cache;
  148. static struct percpu_counter kvm_total_used_mmu_pages;
  149. static u64 __read_mostly shadow_nx_mask;
  150. static u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
  151. static u64 __read_mostly shadow_user_mask;
  152. static u64 __read_mostly shadow_accessed_mask;
  153. static u64 __read_mostly shadow_dirty_mask;
  154. static inline u64 rsvd_bits(int s, int e)
  155. {
  156. return ((1ULL << (e - s + 1)) - 1) << s;
  157. }
  158. void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
  159. u64 dirty_mask, u64 nx_mask, u64 x_mask)
  160. {
  161. shadow_user_mask = user_mask;
  162. shadow_accessed_mask = accessed_mask;
  163. shadow_dirty_mask = dirty_mask;
  164. shadow_nx_mask = nx_mask;
  165. shadow_x_mask = x_mask;
  166. }
  167. EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);
  168. static int is_cpuid_PSE36(void)
  169. {
  170. return 1;
  171. }
  172. static int is_nx(struct kvm_vcpu *vcpu)
  173. {
  174. return vcpu->arch.efer & EFER_NX;
  175. }
  176. static int is_shadow_present_pte(u64 pte)
  177. {
  178. return pte & PT_PRESENT_MASK;
  179. }
  180. static int is_large_pte(u64 pte)
  181. {
  182. return pte & PT_PAGE_SIZE_MASK;
  183. }
  184. static int is_dirty_gpte(unsigned long pte)
  185. {
  186. return pte & PT_DIRTY_MASK;
  187. }
  188. static int is_rmap_spte(u64 pte)
  189. {
  190. return is_shadow_present_pte(pte);
  191. }
  192. static int is_last_spte(u64 pte, int level)
  193. {
  194. if (level == PT_PAGE_TABLE_LEVEL)
  195. return 1;
  196. if (is_large_pte(pte))
  197. return 1;
  198. return 0;
  199. }
  200. static pfn_t spte_to_pfn(u64 pte)
  201. {
  202. return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
  203. }
  204. static gfn_t pse36_gfn_delta(u32 gpte)
  205. {
  206. int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;
  207. return (gpte & PT32_DIR_PSE36_MASK) << shift;
  208. }
  209. static void __set_spte(u64 *sptep, u64 spte)
  210. {
  211. set_64bit(sptep, spte);
  212. }
  213. static u64 __xchg_spte(u64 *sptep, u64 new_spte)
  214. {
  215. #ifdef CONFIG_X86_64
  216. return xchg(sptep, new_spte);
  217. #else
  218. u64 old_spte;
  219. do {
  220. old_spte = *sptep;
  221. } while (cmpxchg64(sptep, old_spte, new_spte) != old_spte);
  222. return old_spte;
  223. #endif
  224. }
  225. static bool spte_has_volatile_bits(u64 spte)
  226. {
  227. if (!shadow_accessed_mask)
  228. return false;
  229. if (!is_shadow_present_pte(spte))
  230. return false;
  231. if ((spte & shadow_accessed_mask) &&
  232. (!is_writable_pte(spte) || (spte & shadow_dirty_mask)))
  233. return false;
  234. return true;
  235. }
  236. static bool spte_is_bit_cleared(u64 old_spte, u64 new_spte, u64 bit_mask)
  237. {
  238. return (old_spte & bit_mask) && !(new_spte & bit_mask);
  239. }
  240. static void update_spte(u64 *sptep, u64 new_spte)
  241. {
  242. u64 mask, old_spte = *sptep;
  243. WARN_ON(!is_rmap_spte(new_spte));
  244. new_spte |= old_spte & shadow_dirty_mask;
  245. mask = shadow_accessed_mask;
  246. if (is_writable_pte(old_spte))
  247. mask |= shadow_dirty_mask;
  248. if (!spte_has_volatile_bits(old_spte) || (new_spte & mask) == mask)
  249. __set_spte(sptep, new_spte);
  250. else
  251. old_spte = __xchg_spte(sptep, new_spte);
  252. if (!shadow_accessed_mask)
  253. return;
  254. if (spte_is_bit_cleared(old_spte, new_spte, shadow_accessed_mask))
  255. kvm_set_pfn_accessed(spte_to_pfn(old_spte));
  256. if (spte_is_bit_cleared(old_spte, new_spte, shadow_dirty_mask))
  257. kvm_set_pfn_dirty(spte_to_pfn(old_spte));
  258. }
  259. static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
  260. struct kmem_cache *base_cache, int min)
  261. {
  262. void *obj;
  263. if (cache->nobjs >= min)
  264. return 0;
  265. while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
  266. obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
  267. if (!obj)
  268. return -ENOMEM;
  269. cache->objects[cache->nobjs++] = obj;
  270. }
  271. return 0;
  272. }
  273. static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc,
  274. struct kmem_cache *cache)
  275. {
  276. while (mc->nobjs)
  277. kmem_cache_free(cache, mc->objects[--mc->nobjs]);
  278. }
  279. static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
  280. int min)
  281. {
  282. void *page;
  283. if (cache->nobjs >= min)
  284. return 0;
  285. while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
  286. page = (void *)__get_free_page(GFP_KERNEL);
  287. if (!page)
  288. return -ENOMEM;
  289. cache->objects[cache->nobjs++] = page;
  290. }
  291. return 0;
  292. }
  293. static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
  294. {
  295. while (mc->nobjs)
  296. free_page((unsigned long)mc->objects[--mc->nobjs]);
  297. }
  298. static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
  299. {
  300. int r;
  301. r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
  302. pte_list_desc_cache, 8 + PTE_PREFETCH_NUM);
  303. if (r)
  304. goto out;
  305. r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
  306. if (r)
  307. goto out;
  308. r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
  309. mmu_page_header_cache, 4);
  310. out:
  311. return r;
  312. }
  313. static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
  314. {
  315. mmu_free_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
  316. pte_list_desc_cache);
  317. mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
  318. mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache,
  319. mmu_page_header_cache);
  320. }
  321. static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
  322. size_t size)
  323. {
  324. void *p;
  325. BUG_ON(!mc->nobjs);
  326. p = mc->objects[--mc->nobjs];
  327. return p;
  328. }
  329. static struct pte_list_desc *mmu_alloc_pte_list_desc(struct kvm_vcpu *vcpu)
  330. {
  331. return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_list_desc_cache,
  332. sizeof(struct pte_list_desc));
  333. }
  334. static void mmu_free_pte_list_desc(struct pte_list_desc *pte_list_desc)
  335. {
  336. kmem_cache_free(pte_list_desc_cache, pte_list_desc);
  337. }
  338. static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index)
  339. {
  340. if (!sp->role.direct)
  341. return sp->gfns[index];
  342. return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS));
  343. }
  344. static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn)
  345. {
  346. if (sp->role.direct)
  347. BUG_ON(gfn != kvm_mmu_page_get_gfn(sp, index));
  348. else
  349. sp->gfns[index] = gfn;
  350. }
  351. /*
  352. * Return the pointer to the large page information for a given gfn,
  353. * handling slots that are not large page aligned.
  354. */
  355. static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn,
  356. struct kvm_memory_slot *slot,
  357. int level)
  358. {
  359. unsigned long idx;
  360. idx = (gfn >> KVM_HPAGE_GFN_SHIFT(level)) -
  361. (slot->base_gfn >> KVM_HPAGE_GFN_SHIFT(level));
  362. return &slot->lpage_info[level - 2][idx];
  363. }
  364. static void account_shadowed(struct kvm *kvm, gfn_t gfn)
  365. {
  366. struct kvm_memory_slot *slot;
  367. struct kvm_lpage_info *linfo;
  368. int i;
  369. slot = gfn_to_memslot(kvm, gfn);
  370. for (i = PT_DIRECTORY_LEVEL;
  371. i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
  372. linfo = lpage_info_slot(gfn, slot, i);
  373. linfo->write_count += 1;
  374. }
  375. kvm->arch.indirect_shadow_pages++;
  376. }
  377. static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn)
  378. {
  379. struct kvm_memory_slot *slot;
  380. struct kvm_lpage_info *linfo;
  381. int i;
  382. slot = gfn_to_memslot(kvm, gfn);
  383. for (i = PT_DIRECTORY_LEVEL;
  384. i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
  385. linfo = lpage_info_slot(gfn, slot, i);
  386. linfo->write_count -= 1;
  387. WARN_ON(linfo->write_count < 0);
  388. }
  389. kvm->arch.indirect_shadow_pages--;
  390. }
  391. static int has_wrprotected_page(struct kvm *kvm,
  392. gfn_t gfn,
  393. int level)
  394. {
  395. struct kvm_memory_slot *slot;
  396. struct kvm_lpage_info *linfo;
  397. slot = gfn_to_memslot(kvm, gfn);
  398. if (slot) {
  399. linfo = lpage_info_slot(gfn, slot, level);
  400. return linfo->write_count;
  401. }
  402. return 1;
  403. }
  404. static int host_mapping_level(struct kvm *kvm, gfn_t gfn)
  405. {
  406. unsigned long page_size;
  407. int i, ret = 0;
  408. page_size = kvm_host_page_size(kvm, gfn);
  409. for (i = PT_PAGE_TABLE_LEVEL;
  410. i < (PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES); ++i) {
  411. if (page_size >= KVM_HPAGE_SIZE(i))
  412. ret = i;
  413. else
  414. break;
  415. }
  416. return ret;
  417. }
  418. static struct kvm_memory_slot *
  419. gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t gfn,
  420. bool no_dirty_log)
  421. {
  422. struct kvm_memory_slot *slot;
  423. slot = gfn_to_memslot(vcpu->kvm, gfn);
  424. if (!slot || slot->flags & KVM_MEMSLOT_INVALID ||
  425. (no_dirty_log && slot->dirty_bitmap))
  426. slot = NULL;
  427. return slot;
  428. }
  429. static bool mapping_level_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t large_gfn)
  430. {
  431. return !gfn_to_memslot_dirty_bitmap(vcpu, large_gfn, true);
  432. }
  433. static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn)
  434. {
  435. int host_level, level, max_level;
  436. host_level = host_mapping_level(vcpu->kvm, large_gfn);
  437. if (host_level == PT_PAGE_TABLE_LEVEL)
  438. return host_level;
  439. max_level = kvm_x86_ops->get_lpage_level() < host_level ?
  440. kvm_x86_ops->get_lpage_level() : host_level;
  441. for (level = PT_DIRECTORY_LEVEL; level <= max_level; ++level)
  442. if (has_wrprotected_page(vcpu->kvm, large_gfn, level))
  443. break;
  444. return level - 1;
  445. }
  446. /*
  447. * Pte mapping structures:
  448. *
  449. * If pte_list bit zero is zero, then pte_list point to the spte.
  450. *
  451. * If pte_list bit zero is one, (then pte_list & ~1) points to a struct
  452. * pte_list_desc containing more mappings.
  453. *
  454. * Returns the number of pte entries before the spte was added or zero if
  455. * the spte was not added.
  456. *
  457. */
  458. static int pte_list_add(struct kvm_vcpu *vcpu, u64 *spte,
  459. unsigned long *pte_list)
  460. {
  461. struct pte_list_desc *desc;
  462. int i, count = 0;
  463. if (!*pte_list) {
  464. rmap_printk("pte_list_add: %p %llx 0->1\n", spte, *spte);
  465. *pte_list = (unsigned long)spte;
  466. } else if (!(*pte_list & 1)) {
  467. rmap_printk("pte_list_add: %p %llx 1->many\n", spte, *spte);
  468. desc = mmu_alloc_pte_list_desc(vcpu);
  469. desc->sptes[0] = (u64 *)*pte_list;
  470. desc->sptes[1] = spte;
  471. *pte_list = (unsigned long)desc | 1;
  472. ++count;
  473. } else {
  474. rmap_printk("pte_list_add: %p %llx many->many\n", spte, *spte);
  475. desc = (struct pte_list_desc *)(*pte_list & ~1ul);
  476. while (desc->sptes[PTE_LIST_EXT-1] && desc->more) {
  477. desc = desc->more;
  478. count += PTE_LIST_EXT;
  479. }
  480. if (desc->sptes[PTE_LIST_EXT-1]) {
  481. desc->more = mmu_alloc_pte_list_desc(vcpu);
  482. desc = desc->more;
  483. }
  484. for (i = 0; desc->sptes[i]; ++i)
  485. ++count;
  486. desc->sptes[i] = spte;
  487. }
  488. return count;
  489. }
  490. static u64 *pte_list_next(unsigned long *pte_list, u64 *spte)
  491. {
  492. struct pte_list_desc *desc;
  493. u64 *prev_spte;
  494. int i;
  495. if (!*pte_list)
  496. return NULL;
  497. else if (!(*pte_list & 1)) {
  498. if (!spte)
  499. return (u64 *)*pte_list;
  500. return NULL;
  501. }
  502. desc = (struct pte_list_desc *)(*pte_list & ~1ul);
  503. prev_spte = NULL;
  504. while (desc) {
  505. for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i) {
  506. if (prev_spte == spte)
  507. return desc->sptes[i];
  508. prev_spte = desc->sptes[i];
  509. }
  510. desc = desc->more;
  511. }
  512. return NULL;
  513. }
  514. static void
  515. pte_list_desc_remove_entry(unsigned long *pte_list, struct pte_list_desc *desc,
  516. int i, struct pte_list_desc *prev_desc)
  517. {
  518. int j;
  519. for (j = PTE_LIST_EXT - 1; !desc->sptes[j] && j > i; --j)
  520. ;
  521. desc->sptes[i] = desc->sptes[j];
  522. desc->sptes[j] = NULL;
  523. if (j != 0)
  524. return;
  525. if (!prev_desc && !desc->more)
  526. *pte_list = (unsigned long)desc->sptes[0];
  527. else
  528. if (prev_desc)
  529. prev_desc->more = desc->more;
  530. else
  531. *pte_list = (unsigned long)desc->more | 1;
  532. mmu_free_pte_list_desc(desc);
  533. }
  534. static void pte_list_remove(u64 *spte, unsigned long *pte_list)
  535. {
  536. struct pte_list_desc *desc;
  537. struct pte_list_desc *prev_desc;
  538. int i;
  539. if (!*pte_list) {
  540. printk(KERN_ERR "pte_list_remove: %p 0->BUG\n", spte);
  541. BUG();
  542. } else if (!(*pte_list & 1)) {
  543. rmap_printk("pte_list_remove: %p 1->0\n", spte);
  544. if ((u64 *)*pte_list != spte) {
  545. printk(KERN_ERR "pte_list_remove: %p 1->BUG\n", spte);
  546. BUG();
  547. }
  548. *pte_list = 0;
  549. } else {
  550. rmap_printk("pte_list_remove: %p many->many\n", spte);
  551. desc = (struct pte_list_desc *)(*pte_list & ~1ul);
  552. prev_desc = NULL;
  553. while (desc) {
  554. for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i)
  555. if (desc->sptes[i] == spte) {
  556. pte_list_desc_remove_entry(pte_list,
  557. desc, i,
  558. prev_desc);
  559. return;
  560. }
  561. prev_desc = desc;
  562. desc = desc->more;
  563. }
  564. pr_err("pte_list_remove: %p many->many\n", spte);
  565. BUG();
  566. }
  567. }
  568. typedef void (*pte_list_walk_fn) (u64 *spte);
  569. static void pte_list_walk(unsigned long *pte_list, pte_list_walk_fn fn)
  570. {
  571. struct pte_list_desc *desc;
  572. int i;
  573. if (!*pte_list)
  574. return;
  575. if (!(*pte_list & 1))
  576. return fn((u64 *)*pte_list);
  577. desc = (struct pte_list_desc *)(*pte_list & ~1ul);
  578. while (desc) {
  579. for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i)
  580. fn(desc->sptes[i]);
  581. desc = desc->more;
  582. }
  583. }
  584. /*
  585. * Take gfn and return the reverse mapping to it.
  586. */
  587. static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int level)
  588. {
  589. struct kvm_memory_slot *slot;
  590. struct kvm_lpage_info *linfo;
  591. slot = gfn_to_memslot(kvm, gfn);
  592. if (likely(level == PT_PAGE_TABLE_LEVEL))
  593. return &slot->rmap[gfn - slot->base_gfn];
  594. linfo = lpage_info_slot(gfn, slot, level);
  595. return &linfo->rmap_pde;
  596. }
  597. static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
  598. {
  599. struct kvm_mmu_page *sp;
  600. unsigned long *rmapp;
  601. sp = page_header(__pa(spte));
  602. kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn);
  603. rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
  604. return pte_list_add(vcpu, spte, rmapp);
  605. }
  606. static u64 *rmap_next(struct kvm *kvm, unsigned long *rmapp, u64 *spte)
  607. {
  608. return pte_list_next(rmapp, spte);
  609. }
  610. static void rmap_remove(struct kvm *kvm, u64 *spte)
  611. {
  612. struct kvm_mmu_page *sp;
  613. gfn_t gfn;
  614. unsigned long *rmapp;
  615. sp = page_header(__pa(spte));
  616. gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt);
  617. rmapp = gfn_to_rmap(kvm, gfn, sp->role.level);
  618. pte_list_remove(spte, rmapp);
  619. }
  620. static int set_spte_track_bits(u64 *sptep, u64 new_spte)
  621. {
  622. pfn_t pfn;
  623. u64 old_spte = *sptep;
  624. if (!spte_has_volatile_bits(old_spte))
  625. __set_spte(sptep, new_spte);
  626. else
  627. old_spte = __xchg_spte(sptep, new_spte);
  628. if (!is_rmap_spte(old_spte))
  629. return 0;
  630. pfn = spte_to_pfn(old_spte);
  631. if (!shadow_accessed_mask || old_spte & shadow_accessed_mask)
  632. kvm_set_pfn_accessed(pfn);
  633. if (!shadow_dirty_mask || (old_spte & shadow_dirty_mask))
  634. kvm_set_pfn_dirty(pfn);
  635. return 1;
  636. }
  637. static void drop_spte(struct kvm *kvm, u64 *sptep)
  638. {
  639. if (set_spte_track_bits(sptep, 0ull))
  640. rmap_remove(kvm, sptep);
  641. }
  642. static int rmap_write_protect(struct kvm *kvm, u64 gfn)
  643. {
  644. unsigned long *rmapp;
  645. u64 *spte;
  646. int i, write_protected = 0;
  647. rmapp = gfn_to_rmap(kvm, gfn, PT_PAGE_TABLE_LEVEL);
  648. spte = rmap_next(kvm, rmapp, NULL);
  649. while (spte) {
  650. BUG_ON(!spte);
  651. BUG_ON(!(*spte & PT_PRESENT_MASK));
  652. rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
  653. if (is_writable_pte(*spte)) {
  654. update_spte(spte, *spte & ~PT_WRITABLE_MASK);
  655. write_protected = 1;
  656. }
  657. spte = rmap_next(kvm, rmapp, spte);
  658. }
  659. /* check for huge page mappings */
  660. for (i = PT_DIRECTORY_LEVEL;
  661. i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
  662. rmapp = gfn_to_rmap(kvm, gfn, i);
  663. spte = rmap_next(kvm, rmapp, NULL);
  664. while (spte) {
  665. BUG_ON(!spte);
  666. BUG_ON(!(*spte & PT_PRESENT_MASK));
  667. BUG_ON((*spte & (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK)) != (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK));
  668. pgprintk("rmap_write_protect(large): spte %p %llx %lld\n", spte, *spte, gfn);
  669. if (is_writable_pte(*spte)) {
  670. drop_spte(kvm, spte);
  671. --kvm->stat.lpages;
  672. spte = NULL;
  673. write_protected = 1;
  674. }
  675. spte = rmap_next(kvm, rmapp, spte);
  676. }
  677. }
  678. return write_protected;
  679. }
  680. static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
  681. unsigned long data)
  682. {
  683. u64 *spte;
  684. int need_tlb_flush = 0;
  685. while ((spte = rmap_next(kvm, rmapp, NULL))) {
  686. BUG_ON(!(*spte & PT_PRESENT_MASK));
  687. rmap_printk("kvm_rmap_unmap_hva: spte %p %llx\n", spte, *spte);
  688. drop_spte(kvm, spte);
  689. need_tlb_flush = 1;
  690. }
  691. return need_tlb_flush;
  692. }
  693. static int kvm_set_pte_rmapp(struct kvm *kvm, unsigned long *rmapp,
  694. unsigned long data)
  695. {
  696. int need_flush = 0;
  697. u64 *spte, new_spte;
  698. pte_t *ptep = (pte_t *)data;
  699. pfn_t new_pfn;
  700. WARN_ON(pte_huge(*ptep));
  701. new_pfn = pte_pfn(*ptep);
  702. spte = rmap_next(kvm, rmapp, NULL);
  703. while (spte) {
  704. BUG_ON(!is_shadow_present_pte(*spte));
  705. rmap_printk("kvm_set_pte_rmapp: spte %p %llx\n", spte, *spte);
  706. need_flush = 1;
  707. if (pte_write(*ptep)) {
  708. drop_spte(kvm, spte);
  709. spte = rmap_next(kvm, rmapp, NULL);
  710. } else {
  711. new_spte = *spte &~ (PT64_BASE_ADDR_MASK);
  712. new_spte |= (u64)new_pfn << PAGE_SHIFT;
  713. new_spte &= ~PT_WRITABLE_MASK;
  714. new_spte &= ~SPTE_HOST_WRITEABLE;
  715. new_spte &= ~shadow_accessed_mask;
  716. set_spte_track_bits(spte, new_spte);
  717. spte = rmap_next(kvm, rmapp, spte);
  718. }
  719. }
  720. if (need_flush)
  721. kvm_flush_remote_tlbs(kvm);
  722. return 0;
  723. }
  724. static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
  725. unsigned long data,
  726. int (*handler)(struct kvm *kvm, unsigned long *rmapp,
  727. unsigned long data))
  728. {
  729. int i, j;
  730. int ret;
  731. int retval = 0;
  732. struct kvm_memslots *slots;
  733. slots = kvm_memslots(kvm);
  734. for (i = 0; i < slots->nmemslots; i++) {
  735. struct kvm_memory_slot *memslot = &slots->memslots[i];
  736. unsigned long start = memslot->userspace_addr;
  737. unsigned long end;
  738. end = start + (memslot->npages << PAGE_SHIFT);
  739. if (hva >= start && hva < end) {
  740. gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT;
  741. gfn_t gfn = memslot->base_gfn + gfn_offset;
  742. ret = handler(kvm, &memslot->rmap[gfn_offset], data);
  743. for (j = 0; j < KVM_NR_PAGE_SIZES - 1; ++j) {
  744. struct kvm_lpage_info *linfo;
  745. linfo = lpage_info_slot(gfn, memslot,
  746. PT_DIRECTORY_LEVEL + j);
  747. ret |= handler(kvm, &linfo->rmap_pde, data);
  748. }
  749. trace_kvm_age_page(hva, memslot, ret);
  750. retval |= ret;
  751. }
  752. }
  753. return retval;
  754. }
  755. int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
  756. {
  757. return kvm_handle_hva(kvm, hva, 0, kvm_unmap_rmapp);
  758. }
  759. void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
  760. {
  761. kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);
  762. }
  763. static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
  764. unsigned long data)
  765. {
  766. u64 *spte;
  767. int young = 0;
  768. /*
  769. * Emulate the accessed bit for EPT, by checking if this page has
  770. * an EPT mapping, and clearing it if it does. On the next access,
  771. * a new EPT mapping will be established.
  772. * This has some overhead, but not as much as the cost of swapping
  773. * out actively used pages or breaking up actively used hugepages.
  774. */
  775. if (!shadow_accessed_mask)
  776. return kvm_unmap_rmapp(kvm, rmapp, data);
  777. spte = rmap_next(kvm, rmapp, NULL);
  778. while (spte) {
  779. int _young;
  780. u64 _spte = *spte;
  781. BUG_ON(!(_spte & PT_PRESENT_MASK));
  782. _young = _spte & PT_ACCESSED_MASK;
  783. if (_young) {
  784. young = 1;
  785. clear_bit(PT_ACCESSED_SHIFT, (unsigned long *)spte);
  786. }
  787. spte = rmap_next(kvm, rmapp, spte);
  788. }
  789. return young;
  790. }
  791. static int kvm_test_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
  792. unsigned long data)
  793. {
  794. u64 *spte;
  795. int young = 0;
  796. /*
  797. * If there's no access bit in the secondary pte set by the
  798. * hardware it's up to gup-fast/gup to set the access bit in
  799. * the primary pte or in the page structure.
  800. */
  801. if (!shadow_accessed_mask)
  802. goto out;
  803. spte = rmap_next(kvm, rmapp, NULL);
  804. while (spte) {
  805. u64 _spte = *spte;
  806. BUG_ON(!(_spte & PT_PRESENT_MASK));
  807. young = _spte & PT_ACCESSED_MASK;
  808. if (young) {
  809. young = 1;
  810. break;
  811. }
  812. spte = rmap_next(kvm, rmapp, spte);
  813. }
  814. out:
  815. return young;
  816. }
  817. #define RMAP_RECYCLE_THRESHOLD 1000
  818. static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
  819. {
  820. unsigned long *rmapp;
  821. struct kvm_mmu_page *sp;
  822. sp = page_header(__pa(spte));
  823. rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
  824. kvm_unmap_rmapp(vcpu->kvm, rmapp, 0);
  825. kvm_flush_remote_tlbs(vcpu->kvm);
  826. }
  827. int kvm_age_hva(struct kvm *kvm, unsigned long hva)
  828. {
  829. return kvm_handle_hva(kvm, hva, 0, kvm_age_rmapp);
  830. }
  831. int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
  832. {
  833. return kvm_handle_hva(kvm, hva, 0, kvm_test_age_rmapp);
  834. }
  835. #ifdef MMU_DEBUG
  836. static int is_empty_shadow_page(u64 *spt)
  837. {
  838. u64 *pos;
  839. u64 *end;
  840. for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
  841. if (is_shadow_present_pte(*pos)) {
  842. printk(KERN_ERR "%s: %p %llx\n", __func__,
  843. pos, *pos);
  844. return 0;
  845. }
  846. return 1;
  847. }
  848. #endif
  849. /*
  850. * This value is the sum of all of the kvm instances's
  851. * kvm->arch.n_used_mmu_pages values. We need a global,
  852. * aggregate version in order to make the slab shrinker
  853. * faster
  854. */
  855. static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, int nr)
  856. {
  857. kvm->arch.n_used_mmu_pages += nr;
  858. percpu_counter_add(&kvm_total_used_mmu_pages, nr);
  859. }
  860. /*
  861. * Remove the sp from shadow page cache, after call it,
  862. * we can not find this sp from the cache, and the shadow
  863. * page table is still valid.
  864. * It should be under the protection of mmu lock.
  865. */
  866. static void kvm_mmu_isolate_page(struct kvm_mmu_page *sp)
  867. {
  868. ASSERT(is_empty_shadow_page(sp->spt));
  869. hlist_del(&sp->hash_link);
  870. if (!sp->role.direct)
  871. free_page((unsigned long)sp->gfns);
  872. }
  873. /*
  874. * Free the shadow page table and the sp, we can do it
  875. * out of the protection of mmu lock.
  876. */
  877. static void kvm_mmu_free_page(struct kvm_mmu_page *sp)
  878. {
  879. list_del(&sp->link);
  880. free_page((unsigned long)sp->spt);
  881. kmem_cache_free(mmu_page_header_cache, sp);
  882. }
  883. static unsigned kvm_page_table_hashfn(gfn_t gfn)
  884. {
  885. return gfn & ((1 << KVM_MMU_HASH_SHIFT) - 1);
  886. }
  887. static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
  888. struct kvm_mmu_page *sp, u64 *parent_pte)
  889. {
  890. if (!parent_pte)
  891. return;
  892. pte_list_add(vcpu, parent_pte, &sp->parent_ptes);
  893. }
  894. static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
  895. u64 *parent_pte)
  896. {
  897. pte_list_remove(parent_pte, &sp->parent_ptes);
  898. }
  899. static void drop_parent_pte(struct kvm_mmu_page *sp,
  900. u64 *parent_pte)
  901. {
  902. mmu_page_remove_parent_pte(sp, parent_pte);
  903. __set_spte(parent_pte, 0ull);
  904. }
  905. static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
  906. u64 *parent_pte, int direct)
  907. {
  908. struct kvm_mmu_page *sp;
  909. sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache,
  910. sizeof *sp);
  911. sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE);
  912. if (!direct)
  913. sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache,
  914. PAGE_SIZE);
  915. set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
  916. list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
  917. bitmap_zero(sp->slot_bitmap, KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS);
  918. sp->parent_ptes = 0;
  919. mmu_page_add_parent_pte(vcpu, sp, parent_pte);
  920. kvm_mod_used_mmu_pages(vcpu->kvm, +1);
  921. return sp;
  922. }
  923. static void mark_unsync(u64 *spte);
  924. static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)
  925. {
  926. pte_list_walk(&sp->parent_ptes, mark_unsync);
  927. }
  928. static void mark_unsync(u64 *spte)
  929. {
  930. struct kvm_mmu_page *sp;
  931. unsigned int index;
  932. sp = page_header(__pa(spte));
  933. index = spte - sp->spt;
  934. if (__test_and_set_bit(index, sp->unsync_child_bitmap))
  935. return;
  936. if (sp->unsync_children++)
  937. return;
  938. kvm_mmu_mark_parents_unsync(sp);
  939. }
  940. static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
  941. struct kvm_mmu_page *sp)
  942. {
  943. return 1;
  944. }
  945. static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
  946. {
  947. }
  948. static void nonpaging_update_pte(struct kvm_vcpu *vcpu,
  949. struct kvm_mmu_page *sp, u64 *spte,
  950. const void *pte)
  951. {
  952. WARN_ON(1);
  953. }
  954. #define KVM_PAGE_ARRAY_NR 16
  955. struct kvm_mmu_pages {
  956. struct mmu_page_and_offset {
  957. struct kvm_mmu_page *sp;
  958. unsigned int idx;
  959. } page[KVM_PAGE_ARRAY_NR];
  960. unsigned int nr;
  961. };
  962. #define for_each_unsync_children(bitmap, idx) \
  963. for (idx = find_first_bit(bitmap, 512); \
  964. idx < 512; \
  965. idx = find_next_bit(bitmap, 512, idx+1))
  966. static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
  967. int idx)
  968. {
  969. int i;
  970. if (sp->unsync)
  971. for (i=0; i < pvec->nr; i++)
  972. if (pvec->page[i].sp == sp)
  973. return 0;
  974. pvec->page[pvec->nr].sp = sp;
  975. pvec->page[pvec->nr].idx = idx;
  976. pvec->nr++;
  977. return (pvec->nr == KVM_PAGE_ARRAY_NR);
  978. }
  979. static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
  980. struct kvm_mmu_pages *pvec)
  981. {
  982. int i, ret, nr_unsync_leaf = 0;
  983. for_each_unsync_children(sp->unsync_child_bitmap, i) {
  984. struct kvm_mmu_page *child;
  985. u64 ent = sp->spt[i];
  986. if (!is_shadow_present_pte(ent) || is_large_pte(ent))
  987. goto clear_child_bitmap;
  988. child = page_header(ent & PT64_BASE_ADDR_MASK);
  989. if (child->unsync_children) {
  990. if (mmu_pages_add(pvec, child, i))
  991. return -ENOSPC;
  992. ret = __mmu_unsync_walk(child, pvec);
  993. if (!ret)
  994. goto clear_child_bitmap;
  995. else if (ret > 0)
  996. nr_unsync_leaf += ret;
  997. else
  998. return ret;
  999. } else if (child->unsync) {
  1000. nr_unsync_leaf++;
  1001. if (mmu_pages_add(pvec, child, i))
  1002. return -ENOSPC;
  1003. } else
  1004. goto clear_child_bitmap;
  1005. continue;
  1006. clear_child_bitmap:
  1007. __clear_bit(i, sp->unsync_child_bitmap);
  1008. sp->unsync_children--;
  1009. WARN_ON((int)sp->unsync_children < 0);
  1010. }
  1011. return nr_unsync_leaf;
  1012. }
  1013. static int mmu_unsync_walk(struct kvm_mmu_page *sp,
  1014. struct kvm_mmu_pages *pvec)
  1015. {
  1016. if (!sp->unsync_children)
  1017. return 0;
  1018. mmu_pages_add(pvec, sp, 0);
  1019. return __mmu_unsync_walk(sp, pvec);
  1020. }
  1021. static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
  1022. {
  1023. WARN_ON(!sp->unsync);
  1024. trace_kvm_mmu_sync_page(sp);
  1025. sp->unsync = 0;
  1026. --kvm->stat.mmu_unsync;
  1027. }
  1028. static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
  1029. struct list_head *invalid_list);
  1030. static void kvm_mmu_commit_zap_page(struct kvm *kvm,
  1031. struct list_head *invalid_list);
  1032. #define for_each_gfn_sp(kvm, sp, gfn, pos) \
  1033. hlist_for_each_entry(sp, pos, \
  1034. &(kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)], hash_link) \
  1035. if ((sp)->gfn != (gfn)) {} else
  1036. #define for_each_gfn_indirect_valid_sp(kvm, sp, gfn, pos) \
  1037. hlist_for_each_entry(sp, pos, \
  1038. &(kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)], hash_link) \
  1039. if ((sp)->gfn != (gfn) || (sp)->role.direct || \
  1040. (sp)->role.invalid) {} else
  1041. /* @sp->gfn should be write-protected at the call site */
  1042. static int __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
  1043. struct list_head *invalid_list, bool clear_unsync)
  1044. {
  1045. if (sp->role.cr4_pae != !!is_pae(vcpu)) {
  1046. kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
  1047. return 1;
  1048. }
  1049. if (clear_unsync)
  1050. kvm_unlink_unsync_page(vcpu->kvm, sp);
  1051. if (vcpu->arch.mmu.sync_page(vcpu, sp)) {
  1052. kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
  1053. return 1;
  1054. }
  1055. kvm_mmu_flush_tlb(vcpu);
  1056. return 0;
  1057. }
  1058. static int kvm_sync_page_transient(struct kvm_vcpu *vcpu,
  1059. struct kvm_mmu_page *sp)
  1060. {
  1061. LIST_HEAD(invalid_list);
  1062. int ret;
  1063. ret = __kvm_sync_page(vcpu, sp, &invalid_list, false);
  1064. if (ret)
  1065. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  1066. return ret;
  1067. }
  1068. static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
  1069. struct list_head *invalid_list)
  1070. {
  1071. return __kvm_sync_page(vcpu, sp, invalid_list, true);
  1072. }
  1073. /* @gfn should be write-protected at the call site */
  1074. static void kvm_sync_pages(struct kvm_vcpu *vcpu, gfn_t gfn)
  1075. {
  1076. struct kvm_mmu_page *s;
  1077. struct hlist_node *node;
  1078. LIST_HEAD(invalid_list);
  1079. bool flush = false;
  1080. for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn, node) {
  1081. if (!s->unsync)
  1082. continue;
  1083. WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
  1084. kvm_unlink_unsync_page(vcpu->kvm, s);
  1085. if ((s->role.cr4_pae != !!is_pae(vcpu)) ||
  1086. (vcpu->arch.mmu.sync_page(vcpu, s))) {
  1087. kvm_mmu_prepare_zap_page(vcpu->kvm, s, &invalid_list);
  1088. continue;
  1089. }
  1090. flush = true;
  1091. }
  1092. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  1093. if (flush)
  1094. kvm_mmu_flush_tlb(vcpu);
  1095. }
  1096. struct mmu_page_path {
  1097. struct kvm_mmu_page *parent[PT64_ROOT_LEVEL-1];
  1098. unsigned int idx[PT64_ROOT_LEVEL-1];
  1099. };
  1100. #define for_each_sp(pvec, sp, parents, i) \
  1101. for (i = mmu_pages_next(&pvec, &parents, -1), \
  1102. sp = pvec.page[i].sp; \
  1103. i < pvec.nr && ({ sp = pvec.page[i].sp; 1;}); \
  1104. i = mmu_pages_next(&pvec, &parents, i))
  1105. static int mmu_pages_next(struct kvm_mmu_pages *pvec,
  1106. struct mmu_page_path *parents,
  1107. int i)
  1108. {
  1109. int n;
  1110. for (n = i+1; n < pvec->nr; n++) {
  1111. struct kvm_mmu_page *sp = pvec->page[n].sp;
  1112. if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
  1113. parents->idx[0] = pvec->page[n].idx;
  1114. return n;
  1115. }
  1116. parents->parent[sp->role.level-2] = sp;
  1117. parents->idx[sp->role.level-1] = pvec->page[n].idx;
  1118. }
  1119. return n;
  1120. }
  1121. static void mmu_pages_clear_parents(struct mmu_page_path *parents)
  1122. {
  1123. struct kvm_mmu_page *sp;
  1124. unsigned int level = 0;
  1125. do {
  1126. unsigned int idx = parents->idx[level];
  1127. sp = parents->parent[level];
  1128. if (!sp)
  1129. return;
  1130. --sp->unsync_children;
  1131. WARN_ON((int)sp->unsync_children < 0);
  1132. __clear_bit(idx, sp->unsync_child_bitmap);
  1133. level++;
  1134. } while (level < PT64_ROOT_LEVEL-1 && !sp->unsync_children);
  1135. }
  1136. static void kvm_mmu_pages_init(struct kvm_mmu_page *parent,
  1137. struct mmu_page_path *parents,
  1138. struct kvm_mmu_pages *pvec)
  1139. {
  1140. parents->parent[parent->role.level-1] = NULL;
  1141. pvec->nr = 0;
  1142. }
  1143. static void mmu_sync_children(struct kvm_vcpu *vcpu,
  1144. struct kvm_mmu_page *parent)
  1145. {
  1146. int i;
  1147. struct kvm_mmu_page *sp;
  1148. struct mmu_page_path parents;
  1149. struct kvm_mmu_pages pages;
  1150. LIST_HEAD(invalid_list);
  1151. kvm_mmu_pages_init(parent, &parents, &pages);
  1152. while (mmu_unsync_walk(parent, &pages)) {
  1153. int protected = 0;
  1154. for_each_sp(pages, sp, parents, i)
  1155. protected |= rmap_write_protect(vcpu->kvm, sp->gfn);
  1156. if (protected)
  1157. kvm_flush_remote_tlbs(vcpu->kvm);
  1158. for_each_sp(pages, sp, parents, i) {
  1159. kvm_sync_page(vcpu, sp, &invalid_list);
  1160. mmu_pages_clear_parents(&parents);
  1161. }
  1162. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  1163. cond_resched_lock(&vcpu->kvm->mmu_lock);
  1164. kvm_mmu_pages_init(parent, &parents, &pages);
  1165. }
  1166. }
  1167. static void init_shadow_page_table(struct kvm_mmu_page *sp)
  1168. {
  1169. int i;
  1170. for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
  1171. sp->spt[i] = 0ull;
  1172. }
  1173. static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
  1174. gfn_t gfn,
  1175. gva_t gaddr,
  1176. unsigned level,
  1177. int direct,
  1178. unsigned access,
  1179. u64 *parent_pte)
  1180. {
  1181. union kvm_mmu_page_role role;
  1182. unsigned quadrant;
  1183. struct kvm_mmu_page *sp;
  1184. struct hlist_node *node;
  1185. bool need_sync = false;
  1186. role = vcpu->arch.mmu.base_role;
  1187. role.level = level;
  1188. role.direct = direct;
  1189. if (role.direct)
  1190. role.cr4_pae = 0;
  1191. role.access = access;
  1192. if (!vcpu->arch.mmu.direct_map
  1193. && vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) {
  1194. quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
  1195. quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
  1196. role.quadrant = quadrant;
  1197. }
  1198. for_each_gfn_sp(vcpu->kvm, sp, gfn, node) {
  1199. if (!need_sync && sp->unsync)
  1200. need_sync = true;
  1201. if (sp->role.word != role.word)
  1202. continue;
  1203. if (sp->unsync && kvm_sync_page_transient(vcpu, sp))
  1204. break;
  1205. mmu_page_add_parent_pte(vcpu, sp, parent_pte);
  1206. if (sp->unsync_children) {
  1207. kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
  1208. kvm_mmu_mark_parents_unsync(sp);
  1209. } else if (sp->unsync)
  1210. kvm_mmu_mark_parents_unsync(sp);
  1211. trace_kvm_mmu_get_page(sp, false);
  1212. return sp;
  1213. }
  1214. ++vcpu->kvm->stat.mmu_cache_miss;
  1215. sp = kvm_mmu_alloc_page(vcpu, parent_pte, direct);
  1216. if (!sp)
  1217. return sp;
  1218. sp->gfn = gfn;
  1219. sp->role = role;
  1220. hlist_add_head(&sp->hash_link,
  1221. &vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)]);
  1222. if (!direct) {
  1223. if (rmap_write_protect(vcpu->kvm, gfn))
  1224. kvm_flush_remote_tlbs(vcpu->kvm);
  1225. if (level > PT_PAGE_TABLE_LEVEL && need_sync)
  1226. kvm_sync_pages(vcpu, gfn);
  1227. account_shadowed(vcpu->kvm, gfn);
  1228. }
  1229. init_shadow_page_table(sp);
  1230. trace_kvm_mmu_get_page(sp, true);
  1231. return sp;
  1232. }
  1233. static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
  1234. struct kvm_vcpu *vcpu, u64 addr)
  1235. {
  1236. iterator->addr = addr;
  1237. iterator->shadow_addr = vcpu->arch.mmu.root_hpa;
  1238. iterator->level = vcpu->arch.mmu.shadow_root_level;
  1239. if (iterator->level == PT64_ROOT_LEVEL &&
  1240. vcpu->arch.mmu.root_level < PT64_ROOT_LEVEL &&
  1241. !vcpu->arch.mmu.direct_map)
  1242. --iterator->level;
  1243. if (iterator->level == PT32E_ROOT_LEVEL) {
  1244. iterator->shadow_addr
  1245. = vcpu->arch.mmu.pae_root[(addr >> 30) & 3];
  1246. iterator->shadow_addr &= PT64_BASE_ADDR_MASK;
  1247. --iterator->level;
  1248. if (!iterator->shadow_addr)
  1249. iterator->level = 0;
  1250. }
  1251. }
  1252. static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
  1253. {
  1254. if (iterator->level < PT_PAGE_TABLE_LEVEL)
  1255. return false;
  1256. iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
  1257. iterator->sptep = ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
  1258. return true;
  1259. }
  1260. static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
  1261. {
  1262. if (is_last_spte(*iterator->sptep, iterator->level)) {
  1263. iterator->level = 0;
  1264. return;
  1265. }
  1266. iterator->shadow_addr = *iterator->sptep & PT64_BASE_ADDR_MASK;
  1267. --iterator->level;
  1268. }
  1269. static void link_shadow_page(u64 *sptep, struct kvm_mmu_page *sp)
  1270. {
  1271. u64 spte;
  1272. spte = __pa(sp->spt)
  1273. | PT_PRESENT_MASK | PT_ACCESSED_MASK
  1274. | PT_WRITABLE_MASK | PT_USER_MASK;
  1275. __set_spte(sptep, spte);
  1276. }
  1277. static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
  1278. {
  1279. if (is_large_pte(*sptep)) {
  1280. drop_spte(vcpu->kvm, sptep);
  1281. kvm_flush_remote_tlbs(vcpu->kvm);
  1282. }
  1283. }
  1284. static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
  1285. unsigned direct_access)
  1286. {
  1287. if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {
  1288. struct kvm_mmu_page *child;
  1289. /*
  1290. * For the direct sp, if the guest pte's dirty bit
  1291. * changed form clean to dirty, it will corrupt the
  1292. * sp's access: allow writable in the read-only sp,
  1293. * so we should update the spte at this point to get
  1294. * a new sp with the correct access.
  1295. */
  1296. child = page_header(*sptep & PT64_BASE_ADDR_MASK);
  1297. if (child->role.access == direct_access)
  1298. return;
  1299. drop_parent_pte(child, sptep);
  1300. kvm_flush_remote_tlbs(vcpu->kvm);
  1301. }
  1302. }
  1303. static void mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
  1304. u64 *spte)
  1305. {
  1306. u64 pte;
  1307. struct kvm_mmu_page *child;
  1308. pte = *spte;
  1309. if (is_shadow_present_pte(pte)) {
  1310. if (is_last_spte(pte, sp->role.level))
  1311. drop_spte(kvm, spte);
  1312. else {
  1313. child = page_header(pte & PT64_BASE_ADDR_MASK);
  1314. drop_parent_pte(child, spte);
  1315. }
  1316. }
  1317. if (is_large_pte(pte))
  1318. --kvm->stat.lpages;
  1319. }
  1320. static void kvm_mmu_page_unlink_children(struct kvm *kvm,
  1321. struct kvm_mmu_page *sp)
  1322. {
  1323. unsigned i;
  1324. for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
  1325. mmu_page_zap_pte(kvm, sp, sp->spt + i);
  1326. }
  1327. static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte)
  1328. {
  1329. mmu_page_remove_parent_pte(sp, parent_pte);
  1330. }
  1331. static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm)
  1332. {
  1333. int i;
  1334. struct kvm_vcpu *vcpu;
  1335. kvm_for_each_vcpu(i, vcpu, kvm)
  1336. vcpu->arch.last_pte_updated = NULL;
  1337. }
  1338. static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
  1339. {
  1340. u64 *parent_pte;
  1341. while ((parent_pte = pte_list_next(&sp->parent_ptes, NULL)))
  1342. drop_parent_pte(sp, parent_pte);
  1343. }
  1344. static int mmu_zap_unsync_children(struct kvm *kvm,
  1345. struct kvm_mmu_page *parent,
  1346. struct list_head *invalid_list)
  1347. {
  1348. int i, zapped = 0;
  1349. struct mmu_page_path parents;
  1350. struct kvm_mmu_pages pages;
  1351. if (parent->role.level == PT_PAGE_TABLE_LEVEL)
  1352. return 0;
  1353. kvm_mmu_pages_init(parent, &parents, &pages);
  1354. while (mmu_unsync_walk(parent, &pages)) {
  1355. struct kvm_mmu_page *sp;
  1356. for_each_sp(pages, sp, parents, i) {
  1357. kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
  1358. mmu_pages_clear_parents(&parents);
  1359. zapped++;
  1360. }
  1361. kvm_mmu_pages_init(parent, &parents, &pages);
  1362. }
  1363. return zapped;
  1364. }
  1365. static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
  1366. struct list_head *invalid_list)
  1367. {
  1368. int ret;
  1369. trace_kvm_mmu_prepare_zap_page(sp);
  1370. ++kvm->stat.mmu_shadow_zapped;
  1371. ret = mmu_zap_unsync_children(kvm, sp, invalid_list);
  1372. kvm_mmu_page_unlink_children(kvm, sp);
  1373. kvm_mmu_unlink_parents(kvm, sp);
  1374. if (!sp->role.invalid && !sp->role.direct)
  1375. unaccount_shadowed(kvm, sp->gfn);
  1376. if (sp->unsync)
  1377. kvm_unlink_unsync_page(kvm, sp);
  1378. if (!sp->root_count) {
  1379. /* Count self */
  1380. ret++;
  1381. list_move(&sp->link, invalid_list);
  1382. kvm_mod_used_mmu_pages(kvm, -1);
  1383. } else {
  1384. list_move(&sp->link, &kvm->arch.active_mmu_pages);
  1385. kvm_reload_remote_mmus(kvm);
  1386. }
  1387. sp->role.invalid = 1;
  1388. kvm_mmu_reset_last_pte_updated(kvm);
  1389. return ret;
  1390. }
  1391. static void kvm_mmu_commit_zap_page(struct kvm *kvm,
  1392. struct list_head *invalid_list)
  1393. {
  1394. struct kvm_mmu_page *sp;
  1395. if (list_empty(invalid_list))
  1396. return;
  1397. kvm_flush_remote_tlbs(kvm);
  1398. do {
  1399. sp = list_first_entry(invalid_list, struct kvm_mmu_page, link);
  1400. WARN_ON(!sp->role.invalid || sp->root_count);
  1401. kvm_mmu_isolate_page(sp);
  1402. kvm_mmu_free_page(sp);
  1403. } while (!list_empty(invalid_list));
  1404. }
  1405. /*
  1406. * Changing the number of mmu pages allocated to the vm
  1407. * Note: if goal_nr_mmu_pages is too small, you will get dead lock
  1408. */
  1409. void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int goal_nr_mmu_pages)
  1410. {
  1411. LIST_HEAD(invalid_list);
  1412. /*
  1413. * If we set the number of mmu pages to be smaller be than the
  1414. * number of actived pages , we must to free some mmu pages before we
  1415. * change the value
  1416. */
  1417. if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
  1418. while (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages &&
  1419. !list_empty(&kvm->arch.active_mmu_pages)) {
  1420. struct kvm_mmu_page *page;
  1421. page = container_of(kvm->arch.active_mmu_pages.prev,
  1422. struct kvm_mmu_page, link);
  1423. kvm_mmu_prepare_zap_page(kvm, page, &invalid_list);
  1424. }
  1425. kvm_mmu_commit_zap_page(kvm, &invalid_list);
  1426. goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages;
  1427. }
  1428. kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
  1429. }
  1430. static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
  1431. {
  1432. struct kvm_mmu_page *sp;
  1433. struct hlist_node *node;
  1434. LIST_HEAD(invalid_list);
  1435. int r;
  1436. pgprintk("%s: looking for gfn %llx\n", __func__, gfn);
  1437. r = 0;
  1438. for_each_gfn_indirect_valid_sp(kvm, sp, gfn, node) {
  1439. pgprintk("%s: gfn %llx role %x\n", __func__, gfn,
  1440. sp->role.word);
  1441. r = 1;
  1442. kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
  1443. }
  1444. kvm_mmu_commit_zap_page(kvm, &invalid_list);
  1445. return r;
  1446. }
  1447. static void mmu_unshadow(struct kvm *kvm, gfn_t gfn)
  1448. {
  1449. struct kvm_mmu_page *sp;
  1450. struct hlist_node *node;
  1451. LIST_HEAD(invalid_list);
  1452. for_each_gfn_indirect_valid_sp(kvm, sp, gfn, node) {
  1453. pgprintk("%s: zap %llx %x\n",
  1454. __func__, gfn, sp->role.word);
  1455. kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
  1456. }
  1457. kvm_mmu_commit_zap_page(kvm, &invalid_list);
  1458. }
  1459. static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn)
  1460. {
  1461. int slot = memslot_id(kvm, gfn);
  1462. struct kvm_mmu_page *sp = page_header(__pa(pte));
  1463. __set_bit(slot, sp->slot_bitmap);
  1464. }
  1465. /*
  1466. * The function is based on mtrr_type_lookup() in
  1467. * arch/x86/kernel/cpu/mtrr/generic.c
  1468. */
  1469. static int get_mtrr_type(struct mtrr_state_type *mtrr_state,
  1470. u64 start, u64 end)
  1471. {
  1472. int i;
  1473. u64 base, mask;
  1474. u8 prev_match, curr_match;
  1475. int num_var_ranges = KVM_NR_VAR_MTRR;
  1476. if (!mtrr_state->enabled)
  1477. return 0xFF;
  1478. /* Make end inclusive end, instead of exclusive */
  1479. end--;
  1480. /* Look in fixed ranges. Just return the type as per start */
  1481. if (mtrr_state->have_fixed && (start < 0x100000)) {
  1482. int idx;
  1483. if (start < 0x80000) {
  1484. idx = 0;
  1485. idx += (start >> 16);
  1486. return mtrr_state->fixed_ranges[idx];
  1487. } else if (start < 0xC0000) {
  1488. idx = 1 * 8;
  1489. idx += ((start - 0x80000) >> 14);
  1490. return mtrr_state->fixed_ranges[idx];
  1491. } else if (start < 0x1000000) {
  1492. idx = 3 * 8;
  1493. idx += ((start - 0xC0000) >> 12);
  1494. return mtrr_state->fixed_ranges[idx];
  1495. }
  1496. }
  1497. /*
  1498. * Look in variable ranges
  1499. * Look of multiple ranges matching this address and pick type
  1500. * as per MTRR precedence
  1501. */
  1502. if (!(mtrr_state->enabled & 2))
  1503. return mtrr_state->def_type;
  1504. prev_match = 0xFF;
  1505. for (i = 0; i < num_var_ranges; ++i) {
  1506. unsigned short start_state, end_state;
  1507. if (!(mtrr_state->var_ranges[i].mask_lo & (1 << 11)))
  1508. continue;
  1509. base = (((u64)mtrr_state->var_ranges[i].base_hi) << 32) +
  1510. (mtrr_state->var_ranges[i].base_lo & PAGE_MASK);
  1511. mask = (((u64)mtrr_state->var_ranges[i].mask_hi) << 32) +
  1512. (mtrr_state->var_ranges[i].mask_lo & PAGE_MASK);
  1513. start_state = ((start & mask) == (base & mask));
  1514. end_state = ((end & mask) == (base & mask));
  1515. if (start_state != end_state)
  1516. return 0xFE;
  1517. if ((start & mask) != (base & mask))
  1518. continue;
  1519. curr_match = mtrr_state->var_ranges[i].base_lo & 0xff;
  1520. if (prev_match == 0xFF) {
  1521. prev_match = curr_match;
  1522. continue;
  1523. }
  1524. if (prev_match == MTRR_TYPE_UNCACHABLE ||
  1525. curr_match == MTRR_TYPE_UNCACHABLE)
  1526. return MTRR_TYPE_UNCACHABLE;
  1527. if ((prev_match == MTRR_TYPE_WRBACK &&
  1528. curr_match == MTRR_TYPE_WRTHROUGH) ||
  1529. (prev_match == MTRR_TYPE_WRTHROUGH &&
  1530. curr_match == MTRR_TYPE_WRBACK)) {
  1531. prev_match = MTRR_TYPE_WRTHROUGH;
  1532. curr_match = MTRR_TYPE_WRTHROUGH;
  1533. }
  1534. if (prev_match != curr_match)
  1535. return MTRR_TYPE_UNCACHABLE;
  1536. }
  1537. if (prev_match != 0xFF)
  1538. return prev_match;
  1539. return mtrr_state->def_type;
  1540. }
  1541. u8 kvm_get_guest_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn)
  1542. {
  1543. u8 mtrr;
  1544. mtrr = get_mtrr_type(&vcpu->arch.mtrr_state, gfn << PAGE_SHIFT,
  1545. (gfn << PAGE_SHIFT) + PAGE_SIZE);
  1546. if (mtrr == 0xfe || mtrr == 0xff)
  1547. mtrr = MTRR_TYPE_WRBACK;
  1548. return mtrr;
  1549. }
  1550. EXPORT_SYMBOL_GPL(kvm_get_guest_memory_type);
  1551. static void __kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
  1552. {
  1553. trace_kvm_mmu_unsync_page(sp);
  1554. ++vcpu->kvm->stat.mmu_unsync;
  1555. sp->unsync = 1;
  1556. kvm_mmu_mark_parents_unsync(sp);
  1557. }
  1558. static void kvm_unsync_pages(struct kvm_vcpu *vcpu, gfn_t gfn)
  1559. {
  1560. struct kvm_mmu_page *s;
  1561. struct hlist_node *node;
  1562. for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn, node) {
  1563. if (s->unsync)
  1564. continue;
  1565. WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
  1566. __kvm_unsync_page(vcpu, s);
  1567. }
  1568. }
  1569. static int mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
  1570. bool can_unsync)
  1571. {
  1572. struct kvm_mmu_page *s;
  1573. struct hlist_node *node;
  1574. bool need_unsync = false;
  1575. for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn, node) {
  1576. if (!can_unsync)
  1577. return 1;
  1578. if (s->role.level != PT_PAGE_TABLE_LEVEL)
  1579. return 1;
  1580. if (!need_unsync && !s->unsync) {
  1581. if (!oos_shadow)
  1582. return 1;
  1583. need_unsync = true;
  1584. }
  1585. }
  1586. if (need_unsync)
  1587. kvm_unsync_pages(vcpu, gfn);
  1588. return 0;
  1589. }
  1590. static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
  1591. unsigned pte_access, int user_fault,
  1592. int write_fault, int level,
  1593. gfn_t gfn, pfn_t pfn, bool speculative,
  1594. bool can_unsync, bool host_writable)
  1595. {
  1596. u64 spte, entry = *sptep;
  1597. int ret = 0;
  1598. /*
  1599. * We don't set the accessed bit, since we sometimes want to see
  1600. * whether the guest actually used the pte (in order to detect
  1601. * demand paging).
  1602. */
  1603. spte = PT_PRESENT_MASK;
  1604. if (!speculative)
  1605. spte |= shadow_accessed_mask;
  1606. if (pte_access & ACC_EXEC_MASK)
  1607. spte |= shadow_x_mask;
  1608. else
  1609. spte |= shadow_nx_mask;
  1610. if (pte_access & ACC_USER_MASK)
  1611. spte |= shadow_user_mask;
  1612. if (level > PT_PAGE_TABLE_LEVEL)
  1613. spte |= PT_PAGE_SIZE_MASK;
  1614. if (tdp_enabled)
  1615. spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn,
  1616. kvm_is_mmio_pfn(pfn));
  1617. if (host_writable)
  1618. spte |= SPTE_HOST_WRITEABLE;
  1619. else
  1620. pte_access &= ~ACC_WRITE_MASK;
  1621. spte |= (u64)pfn << PAGE_SHIFT;
  1622. if ((pte_access & ACC_WRITE_MASK)
  1623. || (!vcpu->arch.mmu.direct_map && write_fault
  1624. && !is_write_protection(vcpu) && !user_fault)) {
  1625. if (level > PT_PAGE_TABLE_LEVEL &&
  1626. has_wrprotected_page(vcpu->kvm, gfn, level)) {
  1627. ret = 1;
  1628. drop_spte(vcpu->kvm, sptep);
  1629. goto done;
  1630. }
  1631. spte |= PT_WRITABLE_MASK;
  1632. if (!vcpu->arch.mmu.direct_map
  1633. && !(pte_access & ACC_WRITE_MASK)) {
  1634. spte &= ~PT_USER_MASK;
  1635. /*
  1636. * If we converted a user page to a kernel page,
  1637. * so that the kernel can write to it when cr0.wp=0,
  1638. * then we should prevent the kernel from executing it
  1639. * if SMEP is enabled.
  1640. */
  1641. if (kvm_read_cr4_bits(vcpu, X86_CR4_SMEP))
  1642. spte |= PT64_NX_MASK;
  1643. }
  1644. /*
  1645. * Optimization: for pte sync, if spte was writable the hash
  1646. * lookup is unnecessary (and expensive). Write protection
  1647. * is responsibility of mmu_get_page / kvm_sync_page.
  1648. * Same reasoning can be applied to dirty page accounting.
  1649. */
  1650. if (!can_unsync && is_writable_pte(*sptep))
  1651. goto set_pte;
  1652. if (mmu_need_write_protect(vcpu, gfn, can_unsync)) {
  1653. pgprintk("%s: found shadow page for %llx, marking ro\n",
  1654. __func__, gfn);
  1655. ret = 1;
  1656. pte_access &= ~ACC_WRITE_MASK;
  1657. if (is_writable_pte(spte))
  1658. spte &= ~PT_WRITABLE_MASK;
  1659. }
  1660. }
  1661. if (pte_access & ACC_WRITE_MASK)
  1662. mark_page_dirty(vcpu->kvm, gfn);
  1663. set_pte:
  1664. update_spte(sptep, spte);
  1665. /*
  1666. * If we overwrite a writable spte with a read-only one we
  1667. * should flush remote TLBs. Otherwise rmap_write_protect
  1668. * will find a read-only spte, even though the writable spte
  1669. * might be cached on a CPU's TLB.
  1670. */
  1671. if (is_writable_pte(entry) && !is_writable_pte(*sptep))
  1672. kvm_flush_remote_tlbs(vcpu->kvm);
  1673. done:
  1674. return ret;
  1675. }
  1676. static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
  1677. unsigned pt_access, unsigned pte_access,
  1678. int user_fault, int write_fault,
  1679. int *emulate, int level, gfn_t gfn,
  1680. pfn_t pfn, bool speculative,
  1681. bool host_writable)
  1682. {
  1683. int was_rmapped = 0;
  1684. int rmap_count;
  1685. pgprintk("%s: spte %llx access %x write_fault %d"
  1686. " user_fault %d gfn %llx\n",
  1687. __func__, *sptep, pt_access,
  1688. write_fault, user_fault, gfn);
  1689. if (is_rmap_spte(*sptep)) {
  1690. /*
  1691. * If we overwrite a PTE page pointer with a 2MB PMD, unlink
  1692. * the parent of the now unreachable PTE.
  1693. */
  1694. if (level > PT_PAGE_TABLE_LEVEL &&
  1695. !is_large_pte(*sptep)) {
  1696. struct kvm_mmu_page *child;
  1697. u64 pte = *sptep;
  1698. child = page_header(pte & PT64_BASE_ADDR_MASK);
  1699. drop_parent_pte(child, sptep);
  1700. kvm_flush_remote_tlbs(vcpu->kvm);
  1701. } else if (pfn != spte_to_pfn(*sptep)) {
  1702. pgprintk("hfn old %llx new %llx\n",
  1703. spte_to_pfn(*sptep), pfn);
  1704. drop_spte(vcpu->kvm, sptep);
  1705. kvm_flush_remote_tlbs(vcpu->kvm);
  1706. } else
  1707. was_rmapped = 1;
  1708. }
  1709. if (set_spte(vcpu, sptep, pte_access, user_fault, write_fault,
  1710. level, gfn, pfn, speculative, true,
  1711. host_writable)) {
  1712. if (write_fault)
  1713. *emulate = 1;
  1714. kvm_mmu_flush_tlb(vcpu);
  1715. }
  1716. pgprintk("%s: setting spte %llx\n", __func__, *sptep);
  1717. pgprintk("instantiating %s PTE (%s) at %llx (%llx) addr %p\n",
  1718. is_large_pte(*sptep)? "2MB" : "4kB",
  1719. *sptep & PT_PRESENT_MASK ?"RW":"R", gfn,
  1720. *sptep, sptep);
  1721. if (!was_rmapped && is_large_pte(*sptep))
  1722. ++vcpu->kvm->stat.lpages;
  1723. if (is_shadow_present_pte(*sptep)) {
  1724. page_header_update_slot(vcpu->kvm, sptep, gfn);
  1725. if (!was_rmapped) {
  1726. rmap_count = rmap_add(vcpu, sptep, gfn);
  1727. if (rmap_count > RMAP_RECYCLE_THRESHOLD)
  1728. rmap_recycle(vcpu, sptep, gfn);
  1729. }
  1730. }
  1731. kvm_release_pfn_clean(pfn);
  1732. if (speculative) {
  1733. vcpu->arch.last_pte_updated = sptep;
  1734. vcpu->arch.last_pte_gfn = gfn;
  1735. }
  1736. }
  1737. static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
  1738. {
  1739. }
  1740. static pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn,
  1741. bool no_dirty_log)
  1742. {
  1743. struct kvm_memory_slot *slot;
  1744. unsigned long hva;
  1745. slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, no_dirty_log);
  1746. if (!slot) {
  1747. get_page(fault_page);
  1748. return page_to_pfn(fault_page);
  1749. }
  1750. hva = gfn_to_hva_memslot(slot, gfn);
  1751. return hva_to_pfn_atomic(vcpu->kvm, hva);
  1752. }
  1753. static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
  1754. struct kvm_mmu_page *sp,
  1755. u64 *start, u64 *end)
  1756. {
  1757. struct page *pages[PTE_PREFETCH_NUM];
  1758. unsigned access = sp->role.access;
  1759. int i, ret;
  1760. gfn_t gfn;
  1761. gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt);
  1762. if (!gfn_to_memslot_dirty_bitmap(vcpu, gfn, access & ACC_WRITE_MASK))
  1763. return -1;
  1764. ret = gfn_to_page_many_atomic(vcpu->kvm, gfn, pages, end - start);
  1765. if (ret <= 0)
  1766. return -1;
  1767. for (i = 0; i < ret; i++, gfn++, start++)
  1768. mmu_set_spte(vcpu, start, ACC_ALL,
  1769. access, 0, 0, NULL,
  1770. sp->role.level, gfn,
  1771. page_to_pfn(pages[i]), true, true);
  1772. return 0;
  1773. }
  1774. static void __direct_pte_prefetch(struct kvm_vcpu *vcpu,
  1775. struct kvm_mmu_page *sp, u64 *sptep)
  1776. {
  1777. u64 *spte, *start = NULL;
  1778. int i;
  1779. WARN_ON(!sp->role.direct);
  1780. i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
  1781. spte = sp->spt + i;
  1782. for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
  1783. if (is_shadow_present_pte(*spte) || spte == sptep) {
  1784. if (!start)
  1785. continue;
  1786. if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0)
  1787. break;
  1788. start = NULL;
  1789. } else if (!start)
  1790. start = spte;
  1791. }
  1792. }
  1793. static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
  1794. {
  1795. struct kvm_mmu_page *sp;
  1796. /*
  1797. * Since it's no accessed bit on EPT, it's no way to
  1798. * distinguish between actually accessed translations
  1799. * and prefetched, so disable pte prefetch if EPT is
  1800. * enabled.
  1801. */
  1802. if (!shadow_accessed_mask)
  1803. return;
  1804. sp = page_header(__pa(sptep));
  1805. if (sp->role.level > PT_PAGE_TABLE_LEVEL)
  1806. return;
  1807. __direct_pte_prefetch(vcpu, sp, sptep);
  1808. }
  1809. static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
  1810. int map_writable, int level, gfn_t gfn, pfn_t pfn,
  1811. bool prefault)
  1812. {
  1813. struct kvm_shadow_walk_iterator iterator;
  1814. struct kvm_mmu_page *sp;
  1815. int emulate = 0;
  1816. gfn_t pseudo_gfn;
  1817. for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
  1818. if (iterator.level == level) {
  1819. unsigned pte_access = ACC_ALL;
  1820. mmu_set_spte(vcpu, iterator.sptep, ACC_ALL, pte_access,
  1821. 0, write, &emulate,
  1822. level, gfn, pfn, prefault, map_writable);
  1823. direct_pte_prefetch(vcpu, iterator.sptep);
  1824. ++vcpu->stat.pf_fixed;
  1825. break;
  1826. }
  1827. if (!is_shadow_present_pte(*iterator.sptep)) {
  1828. u64 base_addr = iterator.addr;
  1829. base_addr &= PT64_LVL_ADDR_MASK(iterator.level);
  1830. pseudo_gfn = base_addr >> PAGE_SHIFT;
  1831. sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr,
  1832. iterator.level - 1,
  1833. 1, ACC_ALL, iterator.sptep);
  1834. if (!sp) {
  1835. pgprintk("nonpaging_map: ENOMEM\n");
  1836. kvm_release_pfn_clean(pfn);
  1837. return -ENOMEM;
  1838. }
  1839. __set_spte(iterator.sptep,
  1840. __pa(sp->spt)
  1841. | PT_PRESENT_MASK | PT_WRITABLE_MASK
  1842. | shadow_user_mask | shadow_x_mask
  1843. | shadow_accessed_mask);
  1844. }
  1845. }
  1846. return emulate;
  1847. }
  1848. static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *tsk)
  1849. {
  1850. siginfo_t info;
  1851. info.si_signo = SIGBUS;
  1852. info.si_errno = 0;
  1853. info.si_code = BUS_MCEERR_AR;
  1854. info.si_addr = (void __user *)address;
  1855. info.si_addr_lsb = PAGE_SHIFT;
  1856. send_sig_info(SIGBUS, &info, tsk);
  1857. }
  1858. static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, pfn_t pfn)
  1859. {
  1860. kvm_release_pfn_clean(pfn);
  1861. if (is_hwpoison_pfn(pfn)) {
  1862. kvm_send_hwpoison_signal(gfn_to_hva(vcpu->kvm, gfn), current);
  1863. return 0;
  1864. }
  1865. return -EFAULT;
  1866. }
  1867. static void transparent_hugepage_adjust(struct kvm_vcpu *vcpu,
  1868. gfn_t *gfnp, pfn_t *pfnp, int *levelp)
  1869. {
  1870. pfn_t pfn = *pfnp;
  1871. gfn_t gfn = *gfnp;
  1872. int level = *levelp;
  1873. /*
  1874. * Check if it's a transparent hugepage. If this would be an
  1875. * hugetlbfs page, level wouldn't be set to
  1876. * PT_PAGE_TABLE_LEVEL and there would be no adjustment done
  1877. * here.
  1878. */
  1879. if (!is_error_pfn(pfn) && !kvm_is_mmio_pfn(pfn) &&
  1880. level == PT_PAGE_TABLE_LEVEL &&
  1881. PageTransCompound(pfn_to_page(pfn)) &&
  1882. !has_wrprotected_page(vcpu->kvm, gfn, PT_DIRECTORY_LEVEL)) {
  1883. unsigned long mask;
  1884. /*
  1885. * mmu_notifier_retry was successful and we hold the
  1886. * mmu_lock here, so the pmd can't become splitting
  1887. * from under us, and in turn
  1888. * __split_huge_page_refcount() can't run from under
  1889. * us and we can safely transfer the refcount from
  1890. * PG_tail to PG_head as we switch the pfn to tail to
  1891. * head.
  1892. */
  1893. *levelp = level = PT_DIRECTORY_LEVEL;
  1894. mask = KVM_PAGES_PER_HPAGE(level) - 1;
  1895. VM_BUG_ON((gfn & mask) != (pfn & mask));
  1896. if (pfn & mask) {
  1897. gfn &= ~mask;
  1898. *gfnp = gfn;
  1899. kvm_release_pfn_clean(pfn);
  1900. pfn &= ~mask;
  1901. if (!get_page_unless_zero(pfn_to_page(pfn)))
  1902. BUG();
  1903. *pfnp = pfn;
  1904. }
  1905. }
  1906. }
  1907. static bool mmu_invalid_pfn(pfn_t pfn)
  1908. {
  1909. return unlikely(is_invalid_pfn(pfn) || is_noslot_pfn(pfn));
  1910. }
  1911. static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn,
  1912. pfn_t pfn, unsigned access, int *ret_val)
  1913. {
  1914. bool ret = true;
  1915. /* The pfn is invalid, report the error! */
  1916. if (unlikely(is_invalid_pfn(pfn))) {
  1917. *ret_val = kvm_handle_bad_page(vcpu, gfn, pfn);
  1918. goto exit;
  1919. }
  1920. if (unlikely(is_noslot_pfn(pfn))) {
  1921. vcpu_cache_mmio_info(vcpu, gva, gfn, access);
  1922. *ret_val = 1;
  1923. goto exit;
  1924. }
  1925. ret = false;
  1926. exit:
  1927. return ret;
  1928. }
  1929. static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
  1930. gva_t gva, pfn_t *pfn, bool write, bool *writable);
  1931. static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn,
  1932. bool prefault)
  1933. {
  1934. int r;
  1935. int level;
  1936. int force_pt_level;
  1937. pfn_t pfn;
  1938. unsigned long mmu_seq;
  1939. bool map_writable;
  1940. force_pt_level = mapping_level_dirty_bitmap(vcpu, gfn);
  1941. if (likely(!force_pt_level)) {
  1942. level = mapping_level(vcpu, gfn);
  1943. /*
  1944. * This path builds a PAE pagetable - so we can map
  1945. * 2mb pages at maximum. Therefore check if the level
  1946. * is larger than that.
  1947. */
  1948. if (level > PT_DIRECTORY_LEVEL)
  1949. level = PT_DIRECTORY_LEVEL;
  1950. gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
  1951. } else
  1952. level = PT_PAGE_TABLE_LEVEL;
  1953. mmu_seq = vcpu->kvm->mmu_notifier_seq;
  1954. smp_rmb();
  1955. if (try_async_pf(vcpu, prefault, gfn, v, &pfn, write, &map_writable))
  1956. return 0;
  1957. if (handle_abnormal_pfn(vcpu, v, gfn, pfn, ACC_ALL, &r))
  1958. return r;
  1959. spin_lock(&vcpu->kvm->mmu_lock);
  1960. if (mmu_notifier_retry(vcpu, mmu_seq))
  1961. goto out_unlock;
  1962. kvm_mmu_free_some_pages(vcpu);
  1963. if (likely(!force_pt_level))
  1964. transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
  1965. r = __direct_map(vcpu, v, write, map_writable, level, gfn, pfn,
  1966. prefault);
  1967. spin_unlock(&vcpu->kvm->mmu_lock);
  1968. return r;
  1969. out_unlock:
  1970. spin_unlock(&vcpu->kvm->mmu_lock);
  1971. kvm_release_pfn_clean(pfn);
  1972. return 0;
  1973. }
  1974. static void mmu_free_roots(struct kvm_vcpu *vcpu)
  1975. {
  1976. int i;
  1977. struct kvm_mmu_page *sp;
  1978. LIST_HEAD(invalid_list);
  1979. if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
  1980. return;
  1981. spin_lock(&vcpu->kvm->mmu_lock);
  1982. if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL &&
  1983. (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL ||
  1984. vcpu->arch.mmu.direct_map)) {
  1985. hpa_t root = vcpu->arch.mmu.root_hpa;
  1986. sp = page_header(root);
  1987. --sp->root_count;
  1988. if (!sp->root_count && sp->role.invalid) {
  1989. kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
  1990. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  1991. }
  1992. vcpu->arch.mmu.root_hpa = INVALID_PAGE;
  1993. spin_unlock(&vcpu->kvm->mmu_lock);
  1994. return;
  1995. }
  1996. for (i = 0; i < 4; ++i) {
  1997. hpa_t root = vcpu->arch.mmu.pae_root[i];
  1998. if (root) {
  1999. root &= PT64_BASE_ADDR_MASK;
  2000. sp = page_header(root);
  2001. --sp->root_count;
  2002. if (!sp->root_count && sp->role.invalid)
  2003. kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
  2004. &invalid_list);
  2005. }
  2006. vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
  2007. }
  2008. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  2009. spin_unlock(&vcpu->kvm->mmu_lock);
  2010. vcpu->arch.mmu.root_hpa = INVALID_PAGE;
  2011. }
  2012. static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
  2013. {
  2014. int ret = 0;
  2015. if (!kvm_is_visible_gfn(vcpu->kvm, root_gfn)) {
  2016. kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
  2017. ret = 1;
  2018. }
  2019. return ret;
  2020. }
  2021. static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu)
  2022. {
  2023. struct kvm_mmu_page *sp;
  2024. unsigned i;
  2025. if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
  2026. spin_lock(&vcpu->kvm->mmu_lock);
  2027. kvm_mmu_free_some_pages(vcpu);
  2028. sp = kvm_mmu_get_page(vcpu, 0, 0, PT64_ROOT_LEVEL,
  2029. 1, ACC_ALL, NULL);
  2030. ++sp->root_count;
  2031. spin_unlock(&vcpu->kvm->mmu_lock);
  2032. vcpu->arch.mmu.root_hpa = __pa(sp->spt);
  2033. } else if (vcpu->arch.mmu.shadow_root_level == PT32E_ROOT_LEVEL) {
  2034. for (i = 0; i < 4; ++i) {
  2035. hpa_t root = vcpu->arch.mmu.pae_root[i];
  2036. ASSERT(!VALID_PAGE(root));
  2037. spin_lock(&vcpu->kvm->mmu_lock);
  2038. kvm_mmu_free_some_pages(vcpu);
  2039. sp = kvm_mmu_get_page(vcpu, i << (30 - PAGE_SHIFT),
  2040. i << 30,
  2041. PT32_ROOT_LEVEL, 1, ACC_ALL,
  2042. NULL);
  2043. root = __pa(sp->spt);
  2044. ++sp->root_count;
  2045. spin_unlock(&vcpu->kvm->mmu_lock);
  2046. vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
  2047. }
  2048. vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
  2049. } else
  2050. BUG();
  2051. return 0;
  2052. }
  2053. static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
  2054. {
  2055. struct kvm_mmu_page *sp;
  2056. u64 pdptr, pm_mask;
  2057. gfn_t root_gfn;
  2058. int i;
  2059. root_gfn = vcpu->arch.mmu.get_cr3(vcpu) >> PAGE_SHIFT;
  2060. if (mmu_check_root(vcpu, root_gfn))
  2061. return 1;
  2062. /*
  2063. * Do we shadow a long mode page table? If so we need to
  2064. * write-protect the guests page table root.
  2065. */
  2066. if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
  2067. hpa_t root = vcpu->arch.mmu.root_hpa;
  2068. ASSERT(!VALID_PAGE(root));
  2069. spin_lock(&vcpu->kvm->mmu_lock);
  2070. kvm_mmu_free_some_pages(vcpu);
  2071. sp = kvm_mmu_get_page(vcpu, root_gfn, 0, PT64_ROOT_LEVEL,
  2072. 0, ACC_ALL, NULL);
  2073. root = __pa(sp->spt);
  2074. ++sp->root_count;
  2075. spin_unlock(&vcpu->kvm->mmu_lock);
  2076. vcpu->arch.mmu.root_hpa = root;
  2077. return 0;
  2078. }
  2079. /*
  2080. * We shadow a 32 bit page table. This may be a legacy 2-level
  2081. * or a PAE 3-level page table. In either case we need to be aware that
  2082. * the shadow page table may be a PAE or a long mode page table.
  2083. */
  2084. pm_mask = PT_PRESENT_MASK;
  2085. if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL)
  2086. pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK;
  2087. for (i = 0; i < 4; ++i) {
  2088. hpa_t root = vcpu->arch.mmu.pae_root[i];
  2089. ASSERT(!VALID_PAGE(root));
  2090. if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
  2091. pdptr = kvm_pdptr_read_mmu(vcpu, &vcpu->arch.mmu, i);
  2092. if (!is_present_gpte(pdptr)) {
  2093. vcpu->arch.mmu.pae_root[i] = 0;
  2094. continue;
  2095. }
  2096. root_gfn = pdptr >> PAGE_SHIFT;
  2097. if (mmu_check_root(vcpu, root_gfn))
  2098. return 1;
  2099. }
  2100. spin_lock(&vcpu->kvm->mmu_lock);
  2101. kvm_mmu_free_some_pages(vcpu);
  2102. sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
  2103. PT32_ROOT_LEVEL, 0,
  2104. ACC_ALL, NULL);
  2105. root = __pa(sp->spt);
  2106. ++sp->root_count;
  2107. spin_unlock(&vcpu->kvm->mmu_lock);
  2108. vcpu->arch.mmu.pae_root[i] = root | pm_mask;
  2109. }
  2110. vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
  2111. /*
  2112. * If we shadow a 32 bit page table with a long mode page
  2113. * table we enter this path.
  2114. */
  2115. if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
  2116. if (vcpu->arch.mmu.lm_root == NULL) {
  2117. /*
  2118. * The additional page necessary for this is only
  2119. * allocated on demand.
  2120. */
  2121. u64 *lm_root;
  2122. lm_root = (void*)get_zeroed_page(GFP_KERNEL);
  2123. if (lm_root == NULL)
  2124. return 1;
  2125. lm_root[0] = __pa(vcpu->arch.mmu.pae_root) | pm_mask;
  2126. vcpu->arch.mmu.lm_root = lm_root;
  2127. }
  2128. vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.lm_root);
  2129. }
  2130. return 0;
  2131. }
  2132. static int mmu_alloc_roots(struct kvm_vcpu *vcpu)
  2133. {
  2134. if (vcpu->arch.mmu.direct_map)
  2135. return mmu_alloc_direct_roots(vcpu);
  2136. else
  2137. return mmu_alloc_shadow_roots(vcpu);
  2138. }
  2139. static void mmu_sync_roots(struct kvm_vcpu *vcpu)
  2140. {
  2141. int i;
  2142. struct kvm_mmu_page *sp;
  2143. if (vcpu->arch.mmu.direct_map)
  2144. return;
  2145. if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
  2146. return;
  2147. vcpu_clear_mmio_info(vcpu, ~0ul);
  2148. trace_kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);
  2149. if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
  2150. hpa_t root = vcpu->arch.mmu.root_hpa;
  2151. sp = page_header(root);
  2152. mmu_sync_children(vcpu, sp);
  2153. trace_kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
  2154. return;
  2155. }
  2156. for (i = 0; i < 4; ++i) {
  2157. hpa_t root = vcpu->arch.mmu.pae_root[i];
  2158. if (root && VALID_PAGE(root)) {
  2159. root &= PT64_BASE_ADDR_MASK;
  2160. sp = page_header(root);
  2161. mmu_sync_children(vcpu, sp);
  2162. }
  2163. }
  2164. trace_kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
  2165. }
  2166. void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
  2167. {
  2168. spin_lock(&vcpu->kvm->mmu_lock);
  2169. mmu_sync_roots(vcpu);
  2170. spin_unlock(&vcpu->kvm->mmu_lock);
  2171. }
  2172. static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr,
  2173. u32 access, struct x86_exception *exception)
  2174. {
  2175. if (exception)
  2176. exception->error_code = 0;
  2177. return vaddr;
  2178. }
  2179. static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gva_t vaddr,
  2180. u32 access,
  2181. struct x86_exception *exception)
  2182. {
  2183. if (exception)
  2184. exception->error_code = 0;
  2185. return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access);
  2186. }
  2187. static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
  2188. u32 error_code, bool prefault)
  2189. {
  2190. gfn_t gfn;
  2191. int r;
  2192. pgprintk("%s: gva %lx error %x\n", __func__, gva, error_code);
  2193. r = mmu_topup_memory_caches(vcpu);
  2194. if (r)
  2195. return r;
  2196. ASSERT(vcpu);
  2197. ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
  2198. gfn = gva >> PAGE_SHIFT;
  2199. return nonpaging_map(vcpu, gva & PAGE_MASK,
  2200. error_code & PFERR_WRITE_MASK, gfn, prefault);
  2201. }
  2202. static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn)
  2203. {
  2204. struct kvm_arch_async_pf arch;
  2205. arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id;
  2206. arch.gfn = gfn;
  2207. arch.direct_map = vcpu->arch.mmu.direct_map;
  2208. arch.cr3 = vcpu->arch.mmu.get_cr3(vcpu);
  2209. return kvm_setup_async_pf(vcpu, gva, gfn, &arch);
  2210. }
  2211. static bool can_do_async_pf(struct kvm_vcpu *vcpu)
  2212. {
  2213. if (unlikely(!irqchip_in_kernel(vcpu->kvm) ||
  2214. kvm_event_needs_reinjection(vcpu)))
  2215. return false;
  2216. return kvm_x86_ops->interrupt_allowed(vcpu);
  2217. }
  2218. static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
  2219. gva_t gva, pfn_t *pfn, bool write, bool *writable)
  2220. {
  2221. bool async;
  2222. *pfn = gfn_to_pfn_async(vcpu->kvm, gfn, &async, write, writable);
  2223. if (!async)
  2224. return false; /* *pfn has correct page already */
  2225. put_page(pfn_to_page(*pfn));
  2226. if (!prefault && can_do_async_pf(vcpu)) {
  2227. trace_kvm_try_async_get_page(gva, gfn);
  2228. if (kvm_find_async_pf_gfn(vcpu, gfn)) {
  2229. trace_kvm_async_pf_doublefault(gva, gfn);
  2230. kvm_make_request(KVM_REQ_APF_HALT, vcpu);
  2231. return true;
  2232. } else if (kvm_arch_setup_async_pf(vcpu, gva, gfn))
  2233. return true;
  2234. }
  2235. *pfn = gfn_to_pfn_prot(vcpu->kvm, gfn, write, writable);
  2236. return false;
  2237. }
  2238. static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, u32 error_code,
  2239. bool prefault)
  2240. {
  2241. pfn_t pfn;
  2242. int r;
  2243. int level;
  2244. int force_pt_level;
  2245. gfn_t gfn = gpa >> PAGE_SHIFT;
  2246. unsigned long mmu_seq;
  2247. int write = error_code & PFERR_WRITE_MASK;
  2248. bool map_writable;
  2249. ASSERT(vcpu);
  2250. ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
  2251. r = mmu_topup_memory_caches(vcpu);
  2252. if (r)
  2253. return r;
  2254. force_pt_level = mapping_level_dirty_bitmap(vcpu, gfn);
  2255. if (likely(!force_pt_level)) {
  2256. level = mapping_level(vcpu, gfn);
  2257. gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
  2258. } else
  2259. level = PT_PAGE_TABLE_LEVEL;
  2260. mmu_seq = vcpu->kvm->mmu_notifier_seq;
  2261. smp_rmb();
  2262. if (try_async_pf(vcpu, prefault, gfn, gpa, &pfn, write, &map_writable))
  2263. return 0;
  2264. if (handle_abnormal_pfn(vcpu, 0, gfn, pfn, ACC_ALL, &r))
  2265. return r;
  2266. spin_lock(&vcpu->kvm->mmu_lock);
  2267. if (mmu_notifier_retry(vcpu, mmu_seq))
  2268. goto out_unlock;
  2269. kvm_mmu_free_some_pages(vcpu);
  2270. if (likely(!force_pt_level))
  2271. transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
  2272. r = __direct_map(vcpu, gpa, write, map_writable,
  2273. level, gfn, pfn, prefault);
  2274. spin_unlock(&vcpu->kvm->mmu_lock);
  2275. return r;
  2276. out_unlock:
  2277. spin_unlock(&vcpu->kvm->mmu_lock);
  2278. kvm_release_pfn_clean(pfn);
  2279. return 0;
  2280. }
  2281. static void nonpaging_free(struct kvm_vcpu *vcpu)
  2282. {
  2283. mmu_free_roots(vcpu);
  2284. }
  2285. static int nonpaging_init_context(struct kvm_vcpu *vcpu,
  2286. struct kvm_mmu *context)
  2287. {
  2288. context->new_cr3 = nonpaging_new_cr3;
  2289. context->page_fault = nonpaging_page_fault;
  2290. context->gva_to_gpa = nonpaging_gva_to_gpa;
  2291. context->free = nonpaging_free;
  2292. context->sync_page = nonpaging_sync_page;
  2293. context->invlpg = nonpaging_invlpg;
  2294. context->update_pte = nonpaging_update_pte;
  2295. context->root_level = 0;
  2296. context->shadow_root_level = PT32E_ROOT_LEVEL;
  2297. context->root_hpa = INVALID_PAGE;
  2298. context->direct_map = true;
  2299. context->nx = false;
  2300. return 0;
  2301. }
  2302. void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
  2303. {
  2304. ++vcpu->stat.tlb_flush;
  2305. kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
  2306. }
  2307. static void paging_new_cr3(struct kvm_vcpu *vcpu)
  2308. {
  2309. pgprintk("%s: cr3 %lx\n", __func__, kvm_read_cr3(vcpu));
  2310. mmu_free_roots(vcpu);
  2311. }
  2312. static unsigned long get_cr3(struct kvm_vcpu *vcpu)
  2313. {
  2314. return kvm_read_cr3(vcpu);
  2315. }
  2316. static void inject_page_fault(struct kvm_vcpu *vcpu,
  2317. struct x86_exception *fault)
  2318. {
  2319. vcpu->arch.mmu.inject_page_fault(vcpu, fault);
  2320. }
  2321. static void paging_free(struct kvm_vcpu *vcpu)
  2322. {
  2323. nonpaging_free(vcpu);
  2324. }
  2325. static bool is_rsvd_bits_set(struct kvm_mmu *mmu, u64 gpte, int level)
  2326. {
  2327. int bit7;
  2328. bit7 = (gpte >> 7) & 1;
  2329. return (gpte & mmu->rsvd_bits_mask[bit7][level-1]) != 0;
  2330. }
  2331. #define PTTYPE 64
  2332. #include "paging_tmpl.h"
  2333. #undef PTTYPE
  2334. #define PTTYPE 32
  2335. #include "paging_tmpl.h"
  2336. #undef PTTYPE
  2337. static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
  2338. struct kvm_mmu *context,
  2339. int level)
  2340. {
  2341. int maxphyaddr = cpuid_maxphyaddr(vcpu);
  2342. u64 exb_bit_rsvd = 0;
  2343. if (!context->nx)
  2344. exb_bit_rsvd = rsvd_bits(63, 63);
  2345. switch (level) {
  2346. case PT32_ROOT_LEVEL:
  2347. /* no rsvd bits for 2 level 4K page table entries */
  2348. context->rsvd_bits_mask[0][1] = 0;
  2349. context->rsvd_bits_mask[0][0] = 0;
  2350. context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
  2351. if (!is_pse(vcpu)) {
  2352. context->rsvd_bits_mask[1][1] = 0;
  2353. break;
  2354. }
  2355. if (is_cpuid_PSE36())
  2356. /* 36bits PSE 4MB page */
  2357. context->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
  2358. else
  2359. /* 32 bits PSE 4MB page */
  2360. context->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
  2361. break;
  2362. case PT32E_ROOT_LEVEL:
  2363. context->rsvd_bits_mask[0][2] =
  2364. rsvd_bits(maxphyaddr, 63) |
  2365. rsvd_bits(7, 8) | rsvd_bits(1, 2); /* PDPTE */
  2366. context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
  2367. rsvd_bits(maxphyaddr, 62); /* PDE */
  2368. context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
  2369. rsvd_bits(maxphyaddr, 62); /* PTE */
  2370. context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
  2371. rsvd_bits(maxphyaddr, 62) |
  2372. rsvd_bits(13, 20); /* large page */
  2373. context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
  2374. break;
  2375. case PT64_ROOT_LEVEL:
  2376. context->rsvd_bits_mask[0][3] = exb_bit_rsvd |
  2377. rsvd_bits(maxphyaddr, 51) | rsvd_bits(7, 8);
  2378. context->rsvd_bits_mask[0][2] = exb_bit_rsvd |
  2379. rsvd_bits(maxphyaddr, 51) | rsvd_bits(7, 8);
  2380. context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
  2381. rsvd_bits(maxphyaddr, 51);
  2382. context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
  2383. rsvd_bits(maxphyaddr, 51);
  2384. context->rsvd_bits_mask[1][3] = context->rsvd_bits_mask[0][3];
  2385. context->rsvd_bits_mask[1][2] = exb_bit_rsvd |
  2386. rsvd_bits(maxphyaddr, 51) |
  2387. rsvd_bits(13, 29);
  2388. context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
  2389. rsvd_bits(maxphyaddr, 51) |
  2390. rsvd_bits(13, 20); /* large page */
  2391. context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
  2392. break;
  2393. }
  2394. }
  2395. static int paging64_init_context_common(struct kvm_vcpu *vcpu,
  2396. struct kvm_mmu *context,
  2397. int level)
  2398. {
  2399. context->nx = is_nx(vcpu);
  2400. reset_rsvds_bits_mask(vcpu, context, level);
  2401. ASSERT(is_pae(vcpu));
  2402. context->new_cr3 = paging_new_cr3;
  2403. context->page_fault = paging64_page_fault;
  2404. context->gva_to_gpa = paging64_gva_to_gpa;
  2405. context->sync_page = paging64_sync_page;
  2406. context->invlpg = paging64_invlpg;
  2407. context->update_pte = paging64_update_pte;
  2408. context->free = paging_free;
  2409. context->root_level = level;
  2410. context->shadow_root_level = level;
  2411. context->root_hpa = INVALID_PAGE;
  2412. context->direct_map = false;
  2413. return 0;
  2414. }
  2415. static int paging64_init_context(struct kvm_vcpu *vcpu,
  2416. struct kvm_mmu *context)
  2417. {
  2418. return paging64_init_context_common(vcpu, context, PT64_ROOT_LEVEL);
  2419. }
  2420. static int paging32_init_context(struct kvm_vcpu *vcpu,
  2421. struct kvm_mmu *context)
  2422. {
  2423. context->nx = false;
  2424. reset_rsvds_bits_mask(vcpu, context, PT32_ROOT_LEVEL);
  2425. context->new_cr3 = paging_new_cr3;
  2426. context->page_fault = paging32_page_fault;
  2427. context->gva_to_gpa = paging32_gva_to_gpa;
  2428. context->free = paging_free;
  2429. context->sync_page = paging32_sync_page;
  2430. context->invlpg = paging32_invlpg;
  2431. context->update_pte = paging32_update_pte;
  2432. context->root_level = PT32_ROOT_LEVEL;
  2433. context->shadow_root_level = PT32E_ROOT_LEVEL;
  2434. context->root_hpa = INVALID_PAGE;
  2435. context->direct_map = false;
  2436. return 0;
  2437. }
  2438. static int paging32E_init_context(struct kvm_vcpu *vcpu,
  2439. struct kvm_mmu *context)
  2440. {
  2441. return paging64_init_context_common(vcpu, context, PT32E_ROOT_LEVEL);
  2442. }
  2443. static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
  2444. {
  2445. struct kvm_mmu *context = vcpu->arch.walk_mmu;
  2446. context->base_role.word = 0;
  2447. context->new_cr3 = nonpaging_new_cr3;
  2448. context->page_fault = tdp_page_fault;
  2449. context->free = nonpaging_free;
  2450. context->sync_page = nonpaging_sync_page;
  2451. context->invlpg = nonpaging_invlpg;
  2452. context->update_pte = nonpaging_update_pte;
  2453. context->shadow_root_level = kvm_x86_ops->get_tdp_level();
  2454. context->root_hpa = INVALID_PAGE;
  2455. context->direct_map = true;
  2456. context->set_cr3 = kvm_x86_ops->set_tdp_cr3;
  2457. context->get_cr3 = get_cr3;
  2458. context->inject_page_fault = kvm_inject_page_fault;
  2459. context->nx = is_nx(vcpu);
  2460. if (!is_paging(vcpu)) {
  2461. context->nx = false;
  2462. context->gva_to_gpa = nonpaging_gva_to_gpa;
  2463. context->root_level = 0;
  2464. } else if (is_long_mode(vcpu)) {
  2465. context->nx = is_nx(vcpu);
  2466. reset_rsvds_bits_mask(vcpu, context, PT64_ROOT_LEVEL);
  2467. context->gva_to_gpa = paging64_gva_to_gpa;
  2468. context->root_level = PT64_ROOT_LEVEL;
  2469. } else if (is_pae(vcpu)) {
  2470. context->nx = is_nx(vcpu);
  2471. reset_rsvds_bits_mask(vcpu, context, PT32E_ROOT_LEVEL);
  2472. context->gva_to_gpa = paging64_gva_to_gpa;
  2473. context->root_level = PT32E_ROOT_LEVEL;
  2474. } else {
  2475. context->nx = false;
  2476. reset_rsvds_bits_mask(vcpu, context, PT32_ROOT_LEVEL);
  2477. context->gva_to_gpa = paging32_gva_to_gpa;
  2478. context->root_level = PT32_ROOT_LEVEL;
  2479. }
  2480. return 0;
  2481. }
  2482. int kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *context)
  2483. {
  2484. int r;
  2485. bool smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
  2486. ASSERT(vcpu);
  2487. ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
  2488. if (!is_paging(vcpu))
  2489. r = nonpaging_init_context(vcpu, context);
  2490. else if (is_long_mode(vcpu))
  2491. r = paging64_init_context(vcpu, context);
  2492. else if (is_pae(vcpu))
  2493. r = paging32E_init_context(vcpu, context);
  2494. else
  2495. r = paging32_init_context(vcpu, context);
  2496. vcpu->arch.mmu.base_role.cr4_pae = !!is_pae(vcpu);
  2497. vcpu->arch.mmu.base_role.cr0_wp = is_write_protection(vcpu);
  2498. vcpu->arch.mmu.base_role.smep_andnot_wp
  2499. = smep && !is_write_protection(vcpu);
  2500. return r;
  2501. }
  2502. EXPORT_SYMBOL_GPL(kvm_init_shadow_mmu);
  2503. static int init_kvm_softmmu(struct kvm_vcpu *vcpu)
  2504. {
  2505. int r = kvm_init_shadow_mmu(vcpu, vcpu->arch.walk_mmu);
  2506. vcpu->arch.walk_mmu->set_cr3 = kvm_x86_ops->set_cr3;
  2507. vcpu->arch.walk_mmu->get_cr3 = get_cr3;
  2508. vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault;
  2509. return r;
  2510. }
  2511. static int init_kvm_nested_mmu(struct kvm_vcpu *vcpu)
  2512. {
  2513. struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;
  2514. g_context->get_cr3 = get_cr3;
  2515. g_context->inject_page_fault = kvm_inject_page_fault;
  2516. /*
  2517. * Note that arch.mmu.gva_to_gpa translates l2_gva to l1_gpa. The
  2518. * translation of l2_gpa to l1_gpa addresses is done using the
  2519. * arch.nested_mmu.gva_to_gpa function. Basically the gva_to_gpa
  2520. * functions between mmu and nested_mmu are swapped.
  2521. */
  2522. if (!is_paging(vcpu)) {
  2523. g_context->nx = false;
  2524. g_context->root_level = 0;
  2525. g_context->gva_to_gpa = nonpaging_gva_to_gpa_nested;
  2526. } else if (is_long_mode(vcpu)) {
  2527. g_context->nx = is_nx(vcpu);
  2528. reset_rsvds_bits_mask(vcpu, g_context, PT64_ROOT_LEVEL);
  2529. g_context->root_level = PT64_ROOT_LEVEL;
  2530. g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
  2531. } else if (is_pae(vcpu)) {
  2532. g_context->nx = is_nx(vcpu);
  2533. reset_rsvds_bits_mask(vcpu, g_context, PT32E_ROOT_LEVEL);
  2534. g_context->root_level = PT32E_ROOT_LEVEL;
  2535. g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
  2536. } else {
  2537. g_context->nx = false;
  2538. reset_rsvds_bits_mask(vcpu, g_context, PT32_ROOT_LEVEL);
  2539. g_context->root_level = PT32_ROOT_LEVEL;
  2540. g_context->gva_to_gpa = paging32_gva_to_gpa_nested;
  2541. }
  2542. return 0;
  2543. }
  2544. static int init_kvm_mmu(struct kvm_vcpu *vcpu)
  2545. {
  2546. if (mmu_is_nested(vcpu))
  2547. return init_kvm_nested_mmu(vcpu);
  2548. else if (tdp_enabled)
  2549. return init_kvm_tdp_mmu(vcpu);
  2550. else
  2551. return init_kvm_softmmu(vcpu);
  2552. }
  2553. static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
  2554. {
  2555. ASSERT(vcpu);
  2556. if (VALID_PAGE(vcpu->arch.mmu.root_hpa))
  2557. /* mmu.free() should set root_hpa = INVALID_PAGE */
  2558. vcpu->arch.mmu.free(vcpu);
  2559. }
  2560. int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
  2561. {
  2562. destroy_kvm_mmu(vcpu);
  2563. return init_kvm_mmu(vcpu);
  2564. }
  2565. EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
  2566. int kvm_mmu_load(struct kvm_vcpu *vcpu)
  2567. {
  2568. int r;
  2569. r = mmu_topup_memory_caches(vcpu);
  2570. if (r)
  2571. goto out;
  2572. r = mmu_alloc_roots(vcpu);
  2573. spin_lock(&vcpu->kvm->mmu_lock);
  2574. mmu_sync_roots(vcpu);
  2575. spin_unlock(&vcpu->kvm->mmu_lock);
  2576. if (r)
  2577. goto out;
  2578. /* set_cr3() should ensure TLB has been flushed */
  2579. vcpu->arch.mmu.set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
  2580. out:
  2581. return r;
  2582. }
  2583. EXPORT_SYMBOL_GPL(kvm_mmu_load);
  2584. void kvm_mmu_unload(struct kvm_vcpu *vcpu)
  2585. {
  2586. mmu_free_roots(vcpu);
  2587. }
  2588. EXPORT_SYMBOL_GPL(kvm_mmu_unload);
  2589. static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
  2590. struct kvm_mmu_page *sp, u64 *spte,
  2591. const void *new)
  2592. {
  2593. if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
  2594. ++vcpu->kvm->stat.mmu_pde_zapped;
  2595. return;
  2596. }
  2597. ++vcpu->kvm->stat.mmu_pte_updated;
  2598. vcpu->arch.mmu.update_pte(vcpu, sp, spte, new);
  2599. }
  2600. static bool need_remote_flush(u64 old, u64 new)
  2601. {
  2602. if (!is_shadow_present_pte(old))
  2603. return false;
  2604. if (!is_shadow_present_pte(new))
  2605. return true;
  2606. if ((old ^ new) & PT64_BASE_ADDR_MASK)
  2607. return true;
  2608. old ^= PT64_NX_MASK;
  2609. new ^= PT64_NX_MASK;
  2610. return (old & ~new & PT64_PERM_MASK) != 0;
  2611. }
  2612. static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, bool zap_page,
  2613. bool remote_flush, bool local_flush)
  2614. {
  2615. if (zap_page)
  2616. return;
  2617. if (remote_flush)
  2618. kvm_flush_remote_tlbs(vcpu->kvm);
  2619. else if (local_flush)
  2620. kvm_mmu_flush_tlb(vcpu);
  2621. }
  2622. static bool last_updated_pte_accessed(struct kvm_vcpu *vcpu)
  2623. {
  2624. u64 *spte = vcpu->arch.last_pte_updated;
  2625. return !!(spte && (*spte & shadow_accessed_mask));
  2626. }
  2627. static void kvm_mmu_access_page(struct kvm_vcpu *vcpu, gfn_t gfn)
  2628. {
  2629. u64 *spte = vcpu->arch.last_pte_updated;
  2630. if (spte
  2631. && vcpu->arch.last_pte_gfn == gfn
  2632. && shadow_accessed_mask
  2633. && !(*spte & shadow_accessed_mask)
  2634. && is_shadow_present_pte(*spte))
  2635. set_bit(PT_ACCESSED_SHIFT, (unsigned long *)spte);
  2636. }
  2637. void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
  2638. const u8 *new, int bytes,
  2639. bool guest_initiated)
  2640. {
  2641. gfn_t gfn = gpa >> PAGE_SHIFT;
  2642. union kvm_mmu_page_role mask = { .word = 0 };
  2643. struct kvm_mmu_page *sp;
  2644. struct hlist_node *node;
  2645. LIST_HEAD(invalid_list);
  2646. u64 entry, gentry, *spte;
  2647. unsigned pte_size, page_offset, misaligned, quadrant, offset;
  2648. int level, npte, invlpg_counter, r, flooded = 0;
  2649. bool remote_flush, local_flush, zap_page;
  2650. /*
  2651. * If we don't have indirect shadow pages, it means no page is
  2652. * write-protected, so we can exit simply.
  2653. */
  2654. if (!ACCESS_ONCE(vcpu->kvm->arch.indirect_shadow_pages))
  2655. return;
  2656. zap_page = remote_flush = local_flush = false;
  2657. offset = offset_in_page(gpa);
  2658. pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);
  2659. invlpg_counter = atomic_read(&vcpu->kvm->arch.invlpg_counter);
  2660. /*
  2661. * Assume that the pte write on a page table of the same type
  2662. * as the current vcpu paging mode since we update the sptes only
  2663. * when they have the same mode.
  2664. */
  2665. if ((is_pae(vcpu) && bytes == 4) || !new) {
  2666. /* Handle a 32-bit guest writing two halves of a 64-bit gpte */
  2667. if (is_pae(vcpu)) {
  2668. gpa &= ~(gpa_t)7;
  2669. bytes = 8;
  2670. }
  2671. r = kvm_read_guest(vcpu->kvm, gpa, &gentry, min(bytes, 8));
  2672. if (r)
  2673. gentry = 0;
  2674. new = (const u8 *)&gentry;
  2675. }
  2676. switch (bytes) {
  2677. case 4:
  2678. gentry = *(const u32 *)new;
  2679. break;
  2680. case 8:
  2681. gentry = *(const u64 *)new;
  2682. break;
  2683. default:
  2684. gentry = 0;
  2685. break;
  2686. }
  2687. spin_lock(&vcpu->kvm->mmu_lock);
  2688. if (atomic_read(&vcpu->kvm->arch.invlpg_counter) != invlpg_counter)
  2689. gentry = 0;
  2690. kvm_mmu_free_some_pages(vcpu);
  2691. ++vcpu->kvm->stat.mmu_pte_write;
  2692. trace_kvm_mmu_audit(vcpu, AUDIT_PRE_PTE_WRITE);
  2693. if (guest_initiated) {
  2694. kvm_mmu_access_page(vcpu, gfn);
  2695. if (gfn == vcpu->arch.last_pt_write_gfn
  2696. && !last_updated_pte_accessed(vcpu)) {
  2697. ++vcpu->arch.last_pt_write_count;
  2698. if (vcpu->arch.last_pt_write_count >= 3)
  2699. flooded = 1;
  2700. } else {
  2701. vcpu->arch.last_pt_write_gfn = gfn;
  2702. vcpu->arch.last_pt_write_count = 1;
  2703. vcpu->arch.last_pte_updated = NULL;
  2704. }
  2705. }
  2706. mask.cr0_wp = mask.cr4_pae = mask.nxe = 1;
  2707. for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn, node) {
  2708. pte_size = sp->role.cr4_pae ? 8 : 4;
  2709. misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
  2710. misaligned |= bytes < 4;
  2711. if (misaligned || flooded) {
  2712. /*
  2713. * Misaligned accesses are too much trouble to fix
  2714. * up; also, they usually indicate a page is not used
  2715. * as a page table.
  2716. *
  2717. * If we're seeing too many writes to a page,
  2718. * it may no longer be a page table, or we may be
  2719. * forking, in which case it is better to unmap the
  2720. * page.
  2721. */
  2722. pgprintk("misaligned: gpa %llx bytes %d role %x\n",
  2723. gpa, bytes, sp->role.word);
  2724. zap_page |= !!kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
  2725. &invalid_list);
  2726. ++vcpu->kvm->stat.mmu_flooded;
  2727. continue;
  2728. }
  2729. page_offset = offset;
  2730. level = sp->role.level;
  2731. npte = 1;
  2732. if (!sp->role.cr4_pae) {
  2733. page_offset <<= 1; /* 32->64 */
  2734. /*
  2735. * A 32-bit pde maps 4MB while the shadow pdes map
  2736. * only 2MB. So we need to double the offset again
  2737. * and zap two pdes instead of one.
  2738. */
  2739. if (level == PT32_ROOT_LEVEL) {
  2740. page_offset &= ~7; /* kill rounding error */
  2741. page_offset <<= 1;
  2742. npte = 2;
  2743. }
  2744. quadrant = page_offset >> PAGE_SHIFT;
  2745. page_offset &= ~PAGE_MASK;
  2746. if (quadrant != sp->role.quadrant)
  2747. continue;
  2748. }
  2749. local_flush = true;
  2750. spte = &sp->spt[page_offset / sizeof(*spte)];
  2751. while (npte--) {
  2752. entry = *spte;
  2753. mmu_page_zap_pte(vcpu->kvm, sp, spte);
  2754. if (gentry &&
  2755. !((sp->role.word ^ vcpu->arch.mmu.base_role.word)
  2756. & mask.word))
  2757. mmu_pte_write_new_pte(vcpu, sp, spte, &gentry);
  2758. if (!remote_flush && need_remote_flush(entry, *spte))
  2759. remote_flush = true;
  2760. ++spte;
  2761. }
  2762. }
  2763. mmu_pte_write_flush_tlb(vcpu, zap_page, remote_flush, local_flush);
  2764. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  2765. trace_kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE);
  2766. spin_unlock(&vcpu->kvm->mmu_lock);
  2767. }
  2768. int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
  2769. {
  2770. gpa_t gpa;
  2771. int r;
  2772. if (vcpu->arch.mmu.direct_map)
  2773. return 0;
  2774. gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL);
  2775. spin_lock(&vcpu->kvm->mmu_lock);
  2776. r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
  2777. spin_unlock(&vcpu->kvm->mmu_lock);
  2778. return r;
  2779. }
  2780. EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt);
  2781. void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
  2782. {
  2783. LIST_HEAD(invalid_list);
  2784. while (kvm_mmu_available_pages(vcpu->kvm) < KVM_REFILL_PAGES &&
  2785. !list_empty(&vcpu->kvm->arch.active_mmu_pages)) {
  2786. struct kvm_mmu_page *sp;
  2787. sp = container_of(vcpu->kvm->arch.active_mmu_pages.prev,
  2788. struct kvm_mmu_page, link);
  2789. kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
  2790. ++vcpu->kvm->stat.mmu_recycled;
  2791. }
  2792. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  2793. }
  2794. int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code,
  2795. void *insn, int insn_len)
  2796. {
  2797. int r;
  2798. enum emulation_result er;
  2799. r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code, false);
  2800. if (r < 0)
  2801. goto out;
  2802. if (!r) {
  2803. r = 1;
  2804. goto out;
  2805. }
  2806. r = mmu_topup_memory_caches(vcpu);
  2807. if (r)
  2808. goto out;
  2809. er = x86_emulate_instruction(vcpu, cr2, 0, insn, insn_len);
  2810. switch (er) {
  2811. case EMULATE_DONE:
  2812. return 1;
  2813. case EMULATE_DO_MMIO:
  2814. ++vcpu->stat.mmio_exits;
  2815. /* fall through */
  2816. case EMULATE_FAIL:
  2817. return 0;
  2818. default:
  2819. BUG();
  2820. }
  2821. out:
  2822. return r;
  2823. }
  2824. EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
  2825. void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
  2826. {
  2827. vcpu->arch.mmu.invlpg(vcpu, gva);
  2828. kvm_mmu_flush_tlb(vcpu);
  2829. ++vcpu->stat.invlpg;
  2830. }
  2831. EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);
  2832. void kvm_enable_tdp(void)
  2833. {
  2834. tdp_enabled = true;
  2835. }
  2836. EXPORT_SYMBOL_GPL(kvm_enable_tdp);
  2837. void kvm_disable_tdp(void)
  2838. {
  2839. tdp_enabled = false;
  2840. }
  2841. EXPORT_SYMBOL_GPL(kvm_disable_tdp);
  2842. static void free_mmu_pages(struct kvm_vcpu *vcpu)
  2843. {
  2844. free_page((unsigned long)vcpu->arch.mmu.pae_root);
  2845. if (vcpu->arch.mmu.lm_root != NULL)
  2846. free_page((unsigned long)vcpu->arch.mmu.lm_root);
  2847. }
  2848. static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
  2849. {
  2850. struct page *page;
  2851. int i;
  2852. ASSERT(vcpu);
  2853. /*
  2854. * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
  2855. * Therefore we need to allocate shadow page tables in the first
  2856. * 4GB of memory, which happens to fit the DMA32 zone.
  2857. */
  2858. page = alloc_page(GFP_KERNEL | __GFP_DMA32);
  2859. if (!page)
  2860. return -ENOMEM;
  2861. vcpu->arch.mmu.pae_root = page_address(page);
  2862. for (i = 0; i < 4; ++i)
  2863. vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
  2864. return 0;
  2865. }
  2866. int kvm_mmu_create(struct kvm_vcpu *vcpu)
  2867. {
  2868. ASSERT(vcpu);
  2869. ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
  2870. return alloc_mmu_pages(vcpu);
  2871. }
  2872. int kvm_mmu_setup(struct kvm_vcpu *vcpu)
  2873. {
  2874. ASSERT(vcpu);
  2875. ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
  2876. return init_kvm_mmu(vcpu);
  2877. }
  2878. void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
  2879. {
  2880. struct kvm_mmu_page *sp;
  2881. list_for_each_entry(sp, &kvm->arch.active_mmu_pages, link) {
  2882. int i;
  2883. u64 *pt;
  2884. if (!test_bit(slot, sp->slot_bitmap))
  2885. continue;
  2886. pt = sp->spt;
  2887. for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
  2888. if (!is_shadow_present_pte(pt[i]) ||
  2889. !is_last_spte(pt[i], sp->role.level))
  2890. continue;
  2891. if (is_large_pte(pt[i])) {
  2892. drop_spte(kvm, &pt[i]);
  2893. --kvm->stat.lpages;
  2894. continue;
  2895. }
  2896. /* avoid RMW */
  2897. if (is_writable_pte(pt[i]))
  2898. update_spte(&pt[i], pt[i] & ~PT_WRITABLE_MASK);
  2899. }
  2900. }
  2901. kvm_flush_remote_tlbs(kvm);
  2902. }
  2903. void kvm_mmu_zap_all(struct kvm *kvm)
  2904. {
  2905. struct kvm_mmu_page *sp, *node;
  2906. LIST_HEAD(invalid_list);
  2907. spin_lock(&kvm->mmu_lock);
  2908. restart:
  2909. list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link)
  2910. if (kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list))
  2911. goto restart;
  2912. kvm_mmu_commit_zap_page(kvm, &invalid_list);
  2913. spin_unlock(&kvm->mmu_lock);
  2914. }
  2915. static int kvm_mmu_remove_some_alloc_mmu_pages(struct kvm *kvm,
  2916. struct list_head *invalid_list)
  2917. {
  2918. struct kvm_mmu_page *page;
  2919. page = container_of(kvm->arch.active_mmu_pages.prev,
  2920. struct kvm_mmu_page, link);
  2921. return kvm_mmu_prepare_zap_page(kvm, page, invalid_list);
  2922. }
  2923. static int mmu_shrink(struct shrinker *shrink, struct shrink_control *sc)
  2924. {
  2925. struct kvm *kvm;
  2926. struct kvm *kvm_freed = NULL;
  2927. int nr_to_scan = sc->nr_to_scan;
  2928. if (nr_to_scan == 0)
  2929. goto out;
  2930. raw_spin_lock(&kvm_lock);
  2931. list_for_each_entry(kvm, &vm_list, vm_list) {
  2932. int idx, freed_pages;
  2933. LIST_HEAD(invalid_list);
  2934. idx = srcu_read_lock(&kvm->srcu);
  2935. spin_lock(&kvm->mmu_lock);
  2936. if (!kvm_freed && nr_to_scan > 0 &&
  2937. kvm->arch.n_used_mmu_pages > 0) {
  2938. freed_pages = kvm_mmu_remove_some_alloc_mmu_pages(kvm,
  2939. &invalid_list);
  2940. kvm_freed = kvm;
  2941. }
  2942. nr_to_scan--;
  2943. kvm_mmu_commit_zap_page(kvm, &invalid_list);
  2944. spin_unlock(&kvm->mmu_lock);
  2945. srcu_read_unlock(&kvm->srcu, idx);
  2946. }
  2947. if (kvm_freed)
  2948. list_move_tail(&kvm_freed->vm_list, &vm_list);
  2949. raw_spin_unlock(&kvm_lock);
  2950. out:
  2951. return percpu_counter_read_positive(&kvm_total_used_mmu_pages);
  2952. }
  2953. static struct shrinker mmu_shrinker = {
  2954. .shrink = mmu_shrink,
  2955. .seeks = DEFAULT_SEEKS * 10,
  2956. };
  2957. static void mmu_destroy_caches(void)
  2958. {
  2959. if (pte_list_desc_cache)
  2960. kmem_cache_destroy(pte_list_desc_cache);
  2961. if (mmu_page_header_cache)
  2962. kmem_cache_destroy(mmu_page_header_cache);
  2963. }
  2964. int kvm_mmu_module_init(void)
  2965. {
  2966. pte_list_desc_cache = kmem_cache_create("pte_list_desc",
  2967. sizeof(struct pte_list_desc),
  2968. 0, 0, NULL);
  2969. if (!pte_list_desc_cache)
  2970. goto nomem;
  2971. mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
  2972. sizeof(struct kvm_mmu_page),
  2973. 0, 0, NULL);
  2974. if (!mmu_page_header_cache)
  2975. goto nomem;
  2976. if (percpu_counter_init(&kvm_total_used_mmu_pages, 0))
  2977. goto nomem;
  2978. register_shrinker(&mmu_shrinker);
  2979. return 0;
  2980. nomem:
  2981. mmu_destroy_caches();
  2982. return -ENOMEM;
  2983. }
  2984. /*
  2985. * Caculate mmu pages needed for kvm.
  2986. */
  2987. unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
  2988. {
  2989. int i;
  2990. unsigned int nr_mmu_pages;
  2991. unsigned int nr_pages = 0;
  2992. struct kvm_memslots *slots;
  2993. slots = kvm_memslots(kvm);
  2994. for (i = 0; i < slots->nmemslots; i++)
  2995. nr_pages += slots->memslots[i].npages;
  2996. nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
  2997. nr_mmu_pages = max(nr_mmu_pages,
  2998. (unsigned int) KVM_MIN_ALLOC_MMU_PAGES);
  2999. return nr_mmu_pages;
  3000. }
  3001. static void *pv_mmu_peek_buffer(struct kvm_pv_mmu_op_buffer *buffer,
  3002. unsigned len)
  3003. {
  3004. if (len > buffer->len)
  3005. return NULL;
  3006. return buffer->ptr;
  3007. }
  3008. static void *pv_mmu_read_buffer(struct kvm_pv_mmu_op_buffer *buffer,
  3009. unsigned len)
  3010. {
  3011. void *ret;
  3012. ret = pv_mmu_peek_buffer(buffer, len);
  3013. if (!ret)
  3014. return ret;
  3015. buffer->ptr += len;
  3016. buffer->len -= len;
  3017. buffer->processed += len;
  3018. return ret;
  3019. }
  3020. static int kvm_pv_mmu_write(struct kvm_vcpu *vcpu,
  3021. gpa_t addr, gpa_t value)
  3022. {
  3023. int bytes = 8;
  3024. int r;
  3025. if (!is_long_mode(vcpu) && !is_pae(vcpu))
  3026. bytes = 4;
  3027. r = mmu_topup_memory_caches(vcpu);
  3028. if (r)
  3029. return r;
  3030. if (!emulator_write_phys(vcpu, addr, &value, bytes))
  3031. return -EFAULT;
  3032. return 1;
  3033. }
  3034. static int kvm_pv_mmu_flush_tlb(struct kvm_vcpu *vcpu)
  3035. {
  3036. (void)kvm_set_cr3(vcpu, kvm_read_cr3(vcpu));
  3037. return 1;
  3038. }
  3039. static int kvm_pv_mmu_release_pt(struct kvm_vcpu *vcpu, gpa_t addr)
  3040. {
  3041. spin_lock(&vcpu->kvm->mmu_lock);
  3042. mmu_unshadow(vcpu->kvm, addr >> PAGE_SHIFT);
  3043. spin_unlock(&vcpu->kvm->mmu_lock);
  3044. return 1;
  3045. }
  3046. static int kvm_pv_mmu_op_one(struct kvm_vcpu *vcpu,
  3047. struct kvm_pv_mmu_op_buffer *buffer)
  3048. {
  3049. struct kvm_mmu_op_header *header;
  3050. header = pv_mmu_peek_buffer(buffer, sizeof *header);
  3051. if (!header)
  3052. return 0;
  3053. switch (header->op) {
  3054. case KVM_MMU_OP_WRITE_PTE: {
  3055. struct kvm_mmu_op_write_pte *wpte;
  3056. wpte = pv_mmu_read_buffer(buffer, sizeof *wpte);
  3057. if (!wpte)
  3058. return 0;
  3059. return kvm_pv_mmu_write(vcpu, wpte->pte_phys,
  3060. wpte->pte_val);
  3061. }
  3062. case KVM_MMU_OP_FLUSH_TLB: {
  3063. struct kvm_mmu_op_flush_tlb *ftlb;
  3064. ftlb = pv_mmu_read_buffer(buffer, sizeof *ftlb);
  3065. if (!ftlb)
  3066. return 0;
  3067. return kvm_pv_mmu_flush_tlb(vcpu);
  3068. }
  3069. case KVM_MMU_OP_RELEASE_PT: {
  3070. struct kvm_mmu_op_release_pt *rpt;
  3071. rpt = pv_mmu_read_buffer(buffer, sizeof *rpt);
  3072. if (!rpt)
  3073. return 0;
  3074. return kvm_pv_mmu_release_pt(vcpu, rpt->pt_phys);
  3075. }
  3076. default: return 0;
  3077. }
  3078. }
  3079. int kvm_pv_mmu_op(struct kvm_vcpu *vcpu, unsigned long bytes,
  3080. gpa_t addr, unsigned long *ret)
  3081. {
  3082. int r;
  3083. struct kvm_pv_mmu_op_buffer *buffer = &vcpu->arch.mmu_op_buffer;
  3084. buffer->ptr = buffer->buf;
  3085. buffer->len = min_t(unsigned long, bytes, sizeof buffer->buf);
  3086. buffer->processed = 0;
  3087. r = kvm_read_guest(vcpu->kvm, addr, buffer->buf, buffer->len);
  3088. if (r)
  3089. goto out;
  3090. while (buffer->len) {
  3091. r = kvm_pv_mmu_op_one(vcpu, buffer);
  3092. if (r < 0)
  3093. goto out;
  3094. if (r == 0)
  3095. break;
  3096. }
  3097. r = 1;
  3098. out:
  3099. *ret = buffer->processed;
  3100. return r;
  3101. }
  3102. int kvm_mmu_get_spte_hierarchy(struct kvm_vcpu *vcpu, u64 addr, u64 sptes[4])
  3103. {
  3104. struct kvm_shadow_walk_iterator iterator;
  3105. int nr_sptes = 0;
  3106. spin_lock(&vcpu->kvm->mmu_lock);
  3107. for_each_shadow_entry(vcpu, addr, iterator) {
  3108. sptes[iterator.level-1] = *iterator.sptep;
  3109. nr_sptes++;
  3110. if (!is_shadow_present_pte(*iterator.sptep))
  3111. break;
  3112. }
  3113. spin_unlock(&vcpu->kvm->mmu_lock);
  3114. return nr_sptes;
  3115. }
  3116. EXPORT_SYMBOL_GPL(kvm_mmu_get_spte_hierarchy);
  3117. void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
  3118. {
  3119. ASSERT(vcpu);
  3120. destroy_kvm_mmu(vcpu);
  3121. free_mmu_pages(vcpu);
  3122. mmu_free_memory_caches(vcpu);
  3123. }
  3124. #ifdef CONFIG_KVM_MMU_AUDIT
  3125. #include "mmu_audit.c"
  3126. #else
  3127. static void mmu_audit_disable(void) { }
  3128. #endif
  3129. void kvm_mmu_module_exit(void)
  3130. {
  3131. mmu_destroy_caches();
  3132. percpu_counter_destroy(&kvm_total_used_mmu_pages);
  3133. unregister_shrinker(&mmu_shrinker);
  3134. mmu_audit_disable();
  3135. }