mmu.c 90 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * This module enables machines with Intel VT-x extensions to run virtual
  5. * machines without emulation or binary translation.
  6. *
  7. * MMU support
  8. *
  9. * Copyright (C) 2006 Qumranet, Inc.
  10. * Copyright 2010 Red Hat, Inc. and/or its affiliates.
  11. *
  12. * Authors:
  13. * Yaniv Kamay <yaniv@qumranet.com>
  14. * Avi Kivity <avi@qumranet.com>
  15. *
  16. * This work is licensed under the terms of the GNU GPL, version 2. See
  17. * the COPYING file in the top-level directory.
  18. *
  19. */
  20. #include "irq.h"
  21. #include "mmu.h"
  22. #include "x86.h"
  23. #include "kvm_cache_regs.h"
  24. #include "x86.h"
  25. #include <linux/kvm_host.h>
  26. #include <linux/types.h>
  27. #include <linux/string.h>
  28. #include <linux/mm.h>
  29. #include <linux/highmem.h>
  30. #include <linux/module.h>
  31. #include <linux/swap.h>
  32. #include <linux/hugetlb.h>
  33. #include <linux/compiler.h>
  34. #include <linux/srcu.h>
  35. #include <linux/slab.h>
  36. #include <linux/uaccess.h>
  37. #include <asm/page.h>
  38. #include <asm/cmpxchg.h>
  39. #include <asm/io.h>
  40. #include <asm/vmx.h>
  41. /*
  42. * When setting this variable to true it enables Two-Dimensional-Paging
  43. * where the hardware walks 2 page tables:
  44. * 1. the guest-virtual to guest-physical
  45. * 2. while doing 1. it walks guest-physical to host-physical
  46. * If the hardware supports that we don't need to do shadow paging.
  47. */
  48. bool tdp_enabled = false;
  49. enum {
  50. AUDIT_PRE_PAGE_FAULT,
  51. AUDIT_POST_PAGE_FAULT,
  52. AUDIT_PRE_PTE_WRITE,
  53. AUDIT_POST_PTE_WRITE,
  54. AUDIT_PRE_SYNC,
  55. AUDIT_POST_SYNC
  56. };
  57. char *audit_point_name[] = {
  58. "pre page fault",
  59. "post page fault",
  60. "pre pte write",
  61. "post pte write",
  62. "pre sync",
  63. "post sync"
  64. };
  65. #undef MMU_DEBUG
  66. #ifdef MMU_DEBUG
  67. #define pgprintk(x...) do { if (dbg) printk(x); } while (0)
  68. #define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
  69. #else
  70. #define pgprintk(x...) do { } while (0)
  71. #define rmap_printk(x...) do { } while (0)
  72. #endif
  73. #ifdef MMU_DEBUG
  74. static int dbg = 0;
  75. module_param(dbg, bool, 0644);
  76. #endif
  77. static int oos_shadow = 1;
  78. module_param(oos_shadow, bool, 0644);
  79. #ifndef MMU_DEBUG
  80. #define ASSERT(x) do { } while (0)
  81. #else
  82. #define ASSERT(x) \
  83. if (!(x)) { \
  84. printk(KERN_WARNING "assertion failed %s:%d: %s\n", \
  85. __FILE__, __LINE__, #x); \
  86. }
  87. #endif
  88. #define PTE_PREFETCH_NUM 8
  89. #define PT_FIRST_AVAIL_BITS_SHIFT 9
  90. #define PT64_SECOND_AVAIL_BITS_SHIFT 52
  91. #define PT64_LEVEL_BITS 9
  92. #define PT64_LEVEL_SHIFT(level) \
  93. (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
  94. #define PT64_LEVEL_MASK(level) \
  95. (((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level))
  96. #define PT64_INDEX(address, level)\
  97. (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
  98. #define PT32_LEVEL_BITS 10
  99. #define PT32_LEVEL_SHIFT(level) \
  100. (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
  101. #define PT32_LEVEL_MASK(level) \
  102. (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))
  103. #define PT32_LVL_OFFSET_MASK(level) \
  104. (PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
  105. * PT32_LEVEL_BITS))) - 1))
  106. #define PT32_INDEX(address, level)\
  107. (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
  108. #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
  109. #define PT64_DIR_BASE_ADDR_MASK \
  110. (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
  111. #define PT64_LVL_ADDR_MASK(level) \
  112. (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
  113. * PT64_LEVEL_BITS))) - 1))
  114. #define PT64_LVL_OFFSET_MASK(level) \
  115. (PT64_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
  116. * PT64_LEVEL_BITS))) - 1))
  117. #define PT32_BASE_ADDR_MASK PAGE_MASK
  118. #define PT32_DIR_BASE_ADDR_MASK \
  119. (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
  120. #define PT32_LVL_ADDR_MASK(level) \
  121. (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
  122. * PT32_LEVEL_BITS))) - 1))
  123. #define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK \
  124. | PT64_NX_MASK)
  125. #define RMAP_EXT 4
  126. #define ACC_EXEC_MASK 1
  127. #define ACC_WRITE_MASK PT_WRITABLE_MASK
  128. #define ACC_USER_MASK PT_USER_MASK
  129. #define ACC_ALL (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
  130. #include <trace/events/kvm.h>
  131. #define CREATE_TRACE_POINTS
  132. #include "mmutrace.h"
  133. #define SPTE_HOST_WRITEABLE (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
  134. #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)
  135. struct kvm_rmap_desc {
  136. u64 *sptes[RMAP_EXT];
  137. struct kvm_rmap_desc *more;
  138. };
  139. struct kvm_shadow_walk_iterator {
  140. u64 addr;
  141. hpa_t shadow_addr;
  142. int level;
  143. u64 *sptep;
  144. unsigned index;
  145. };
  146. #define for_each_shadow_entry(_vcpu, _addr, _walker) \
  147. for (shadow_walk_init(&(_walker), _vcpu, _addr); \
  148. shadow_walk_okay(&(_walker)); \
  149. shadow_walk_next(&(_walker)))
  150. typedef void (*mmu_parent_walk_fn) (struct kvm_mmu_page *sp, u64 *spte);
  151. static struct kmem_cache *pte_chain_cache;
  152. static struct kmem_cache *rmap_desc_cache;
  153. static struct kmem_cache *mmu_page_header_cache;
  154. static struct percpu_counter kvm_total_used_mmu_pages;
  155. static u64 __read_mostly shadow_trap_nonpresent_pte;
  156. static u64 __read_mostly shadow_notrap_nonpresent_pte;
  157. static u64 __read_mostly shadow_nx_mask;
  158. static u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
  159. static u64 __read_mostly shadow_user_mask;
  160. static u64 __read_mostly shadow_accessed_mask;
  161. static u64 __read_mostly shadow_dirty_mask;
  162. static inline u64 rsvd_bits(int s, int e)
  163. {
  164. return ((1ULL << (e - s + 1)) - 1) << s;
  165. }
  166. void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte)
  167. {
  168. shadow_trap_nonpresent_pte = trap_pte;
  169. shadow_notrap_nonpresent_pte = notrap_pte;
  170. }
  171. EXPORT_SYMBOL_GPL(kvm_mmu_set_nonpresent_ptes);
  172. void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
  173. u64 dirty_mask, u64 nx_mask, u64 x_mask)
  174. {
  175. shadow_user_mask = user_mask;
  176. shadow_accessed_mask = accessed_mask;
  177. shadow_dirty_mask = dirty_mask;
  178. shadow_nx_mask = nx_mask;
  179. shadow_x_mask = x_mask;
  180. }
  181. EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);
  182. static bool is_write_protection(struct kvm_vcpu *vcpu)
  183. {
  184. return kvm_read_cr0_bits(vcpu, X86_CR0_WP);
  185. }
  186. static int is_cpuid_PSE36(void)
  187. {
  188. return 1;
  189. }
  190. static int is_nx(struct kvm_vcpu *vcpu)
  191. {
  192. return vcpu->arch.efer & EFER_NX;
  193. }
  194. static int is_shadow_present_pte(u64 pte)
  195. {
  196. return pte != shadow_trap_nonpresent_pte
  197. && pte != shadow_notrap_nonpresent_pte;
  198. }
  199. static int is_large_pte(u64 pte)
  200. {
  201. return pte & PT_PAGE_SIZE_MASK;
  202. }
  203. static int is_writable_pte(unsigned long pte)
  204. {
  205. return pte & PT_WRITABLE_MASK;
  206. }
  207. static int is_dirty_gpte(unsigned long pte)
  208. {
  209. return pte & PT_DIRTY_MASK;
  210. }
  211. static int is_rmap_spte(u64 pte)
  212. {
  213. return is_shadow_present_pte(pte);
  214. }
  215. static int is_last_spte(u64 pte, int level)
  216. {
  217. if (level == PT_PAGE_TABLE_LEVEL)
  218. return 1;
  219. if (is_large_pte(pte))
  220. return 1;
  221. return 0;
  222. }
  223. static pfn_t spte_to_pfn(u64 pte)
  224. {
  225. return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
  226. }
  227. static gfn_t pse36_gfn_delta(u32 gpte)
  228. {
  229. int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;
  230. return (gpte & PT32_DIR_PSE36_MASK) << shift;
  231. }
  232. static void __set_spte(u64 *sptep, u64 spte)
  233. {
  234. set_64bit(sptep, spte);
  235. }
  236. static u64 __xchg_spte(u64 *sptep, u64 new_spte)
  237. {
  238. #ifdef CONFIG_X86_64
  239. return xchg(sptep, new_spte);
  240. #else
  241. u64 old_spte;
  242. do {
  243. old_spte = *sptep;
  244. } while (cmpxchg64(sptep, old_spte, new_spte) != old_spte);
  245. return old_spte;
  246. #endif
  247. }
  248. static bool spte_has_volatile_bits(u64 spte)
  249. {
  250. if (!shadow_accessed_mask)
  251. return false;
  252. if (!is_shadow_present_pte(spte))
  253. return false;
  254. if ((spte & shadow_accessed_mask) &&
  255. (!is_writable_pte(spte) || (spte & shadow_dirty_mask)))
  256. return false;
  257. return true;
  258. }
  259. static bool spte_is_bit_cleared(u64 old_spte, u64 new_spte, u64 bit_mask)
  260. {
  261. return (old_spte & bit_mask) && !(new_spte & bit_mask);
  262. }
  263. static void update_spte(u64 *sptep, u64 new_spte)
  264. {
  265. u64 mask, old_spte = *sptep;
  266. WARN_ON(!is_rmap_spte(new_spte));
  267. new_spte |= old_spte & shadow_dirty_mask;
  268. mask = shadow_accessed_mask;
  269. if (is_writable_pte(old_spte))
  270. mask |= shadow_dirty_mask;
  271. if (!spte_has_volatile_bits(old_spte) || (new_spte & mask) == mask)
  272. __set_spte(sptep, new_spte);
  273. else
  274. old_spte = __xchg_spte(sptep, new_spte);
  275. if (!shadow_accessed_mask)
  276. return;
  277. if (spte_is_bit_cleared(old_spte, new_spte, shadow_accessed_mask))
  278. kvm_set_pfn_accessed(spte_to_pfn(old_spte));
  279. if (spte_is_bit_cleared(old_spte, new_spte, shadow_dirty_mask))
  280. kvm_set_pfn_dirty(spte_to_pfn(old_spte));
  281. }
  282. static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
  283. struct kmem_cache *base_cache, int min)
  284. {
  285. void *obj;
  286. if (cache->nobjs >= min)
  287. return 0;
  288. while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
  289. obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
  290. if (!obj)
  291. return -ENOMEM;
  292. cache->objects[cache->nobjs++] = obj;
  293. }
  294. return 0;
  295. }
  296. static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc,
  297. struct kmem_cache *cache)
  298. {
  299. while (mc->nobjs)
  300. kmem_cache_free(cache, mc->objects[--mc->nobjs]);
  301. }
  302. static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
  303. int min)
  304. {
  305. struct page *page;
  306. if (cache->nobjs >= min)
  307. return 0;
  308. while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
  309. page = alloc_page(GFP_KERNEL);
  310. if (!page)
  311. return -ENOMEM;
  312. cache->objects[cache->nobjs++] = page_address(page);
  313. }
  314. return 0;
  315. }
  316. static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
  317. {
  318. while (mc->nobjs)
  319. free_page((unsigned long)mc->objects[--mc->nobjs]);
  320. }
  321. static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
  322. {
  323. int r;
  324. r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_chain_cache,
  325. pte_chain_cache, 4);
  326. if (r)
  327. goto out;
  328. r = mmu_topup_memory_cache(&vcpu->arch.mmu_rmap_desc_cache,
  329. rmap_desc_cache, 4 + PTE_PREFETCH_NUM);
  330. if (r)
  331. goto out;
  332. r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
  333. if (r)
  334. goto out;
  335. r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
  336. mmu_page_header_cache, 4);
  337. out:
  338. return r;
  339. }
  340. static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
  341. {
  342. mmu_free_memory_cache(&vcpu->arch.mmu_pte_chain_cache, pte_chain_cache);
  343. mmu_free_memory_cache(&vcpu->arch.mmu_rmap_desc_cache, rmap_desc_cache);
  344. mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
  345. mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache,
  346. mmu_page_header_cache);
  347. }
  348. static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
  349. size_t size)
  350. {
  351. void *p;
  352. BUG_ON(!mc->nobjs);
  353. p = mc->objects[--mc->nobjs];
  354. return p;
  355. }
  356. static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu)
  357. {
  358. return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_chain_cache,
  359. sizeof(struct kvm_pte_chain));
  360. }
  361. static void mmu_free_pte_chain(struct kvm_pte_chain *pc)
  362. {
  363. kmem_cache_free(pte_chain_cache, pc);
  364. }
  365. static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu)
  366. {
  367. return mmu_memory_cache_alloc(&vcpu->arch.mmu_rmap_desc_cache,
  368. sizeof(struct kvm_rmap_desc));
  369. }
  370. static void mmu_free_rmap_desc(struct kvm_rmap_desc *rd)
  371. {
  372. kmem_cache_free(rmap_desc_cache, rd);
  373. }
  374. static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index)
  375. {
  376. if (!sp->role.direct)
  377. return sp->gfns[index];
  378. return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS));
  379. }
  380. static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn)
  381. {
  382. if (sp->role.direct)
  383. BUG_ON(gfn != kvm_mmu_page_get_gfn(sp, index));
  384. else
  385. sp->gfns[index] = gfn;
  386. }
  387. /*
  388. * Return the pointer to the largepage write count for a given
  389. * gfn, handling slots that are not large page aligned.
  390. */
  391. static int *slot_largepage_idx(gfn_t gfn,
  392. struct kvm_memory_slot *slot,
  393. int level)
  394. {
  395. unsigned long idx;
  396. idx = (gfn >> KVM_HPAGE_GFN_SHIFT(level)) -
  397. (slot->base_gfn >> KVM_HPAGE_GFN_SHIFT(level));
  398. return &slot->lpage_info[level - 2][idx].write_count;
  399. }
  400. static void account_shadowed(struct kvm *kvm, gfn_t gfn)
  401. {
  402. struct kvm_memory_slot *slot;
  403. int *write_count;
  404. int i;
  405. slot = gfn_to_memslot(kvm, gfn);
  406. for (i = PT_DIRECTORY_LEVEL;
  407. i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
  408. write_count = slot_largepage_idx(gfn, slot, i);
  409. *write_count += 1;
  410. }
  411. }
  412. static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn)
  413. {
  414. struct kvm_memory_slot *slot;
  415. int *write_count;
  416. int i;
  417. slot = gfn_to_memslot(kvm, gfn);
  418. for (i = PT_DIRECTORY_LEVEL;
  419. i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
  420. write_count = slot_largepage_idx(gfn, slot, i);
  421. *write_count -= 1;
  422. WARN_ON(*write_count < 0);
  423. }
  424. }
  425. static int has_wrprotected_page(struct kvm *kvm,
  426. gfn_t gfn,
  427. int level)
  428. {
  429. struct kvm_memory_slot *slot;
  430. int *largepage_idx;
  431. slot = gfn_to_memslot(kvm, gfn);
  432. if (slot) {
  433. largepage_idx = slot_largepage_idx(gfn, slot, level);
  434. return *largepage_idx;
  435. }
  436. return 1;
  437. }
  438. static int host_mapping_level(struct kvm *kvm, gfn_t gfn)
  439. {
  440. unsigned long page_size;
  441. int i, ret = 0;
  442. page_size = kvm_host_page_size(kvm, gfn);
  443. for (i = PT_PAGE_TABLE_LEVEL;
  444. i < (PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES); ++i) {
  445. if (page_size >= KVM_HPAGE_SIZE(i))
  446. ret = i;
  447. else
  448. break;
  449. }
  450. return ret;
  451. }
  452. static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn)
  453. {
  454. struct kvm_memory_slot *slot;
  455. int host_level, level, max_level;
  456. slot = gfn_to_memslot(vcpu->kvm, large_gfn);
  457. if (slot && slot->dirty_bitmap)
  458. return PT_PAGE_TABLE_LEVEL;
  459. host_level = host_mapping_level(vcpu->kvm, large_gfn);
  460. if (host_level == PT_PAGE_TABLE_LEVEL)
  461. return host_level;
  462. max_level = kvm_x86_ops->get_lpage_level() < host_level ?
  463. kvm_x86_ops->get_lpage_level() : host_level;
  464. for (level = PT_DIRECTORY_LEVEL; level <= max_level; ++level)
  465. if (has_wrprotected_page(vcpu->kvm, large_gfn, level))
  466. break;
  467. return level - 1;
  468. }
  469. /*
  470. * Take gfn and return the reverse mapping to it.
  471. */
  472. static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int level)
  473. {
  474. struct kvm_memory_slot *slot;
  475. unsigned long idx;
  476. slot = gfn_to_memslot(kvm, gfn);
  477. if (likely(level == PT_PAGE_TABLE_LEVEL))
  478. return &slot->rmap[gfn - slot->base_gfn];
  479. idx = (gfn >> KVM_HPAGE_GFN_SHIFT(level)) -
  480. (slot->base_gfn >> KVM_HPAGE_GFN_SHIFT(level));
  481. return &slot->lpage_info[level - 2][idx].rmap_pde;
  482. }
  483. /*
  484. * Reverse mapping data structures:
  485. *
  486. * If rmapp bit zero is zero, then rmapp point to the shadw page table entry
  487. * that points to page_address(page).
  488. *
  489. * If rmapp bit zero is one, (then rmap & ~1) points to a struct kvm_rmap_desc
  490. * containing more mappings.
  491. *
  492. * Returns the number of rmap entries before the spte was added or zero if
  493. * the spte was not added.
  494. *
  495. */
  496. static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
  497. {
  498. struct kvm_mmu_page *sp;
  499. struct kvm_rmap_desc *desc;
  500. unsigned long *rmapp;
  501. int i, count = 0;
  502. if (!is_rmap_spte(*spte))
  503. return count;
  504. sp = page_header(__pa(spte));
  505. kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn);
  506. rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
  507. if (!*rmapp) {
  508. rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
  509. *rmapp = (unsigned long)spte;
  510. } else if (!(*rmapp & 1)) {
  511. rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
  512. desc = mmu_alloc_rmap_desc(vcpu);
  513. desc->sptes[0] = (u64 *)*rmapp;
  514. desc->sptes[1] = spte;
  515. *rmapp = (unsigned long)desc | 1;
  516. ++count;
  517. } else {
  518. rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
  519. desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
  520. while (desc->sptes[RMAP_EXT-1] && desc->more) {
  521. desc = desc->more;
  522. count += RMAP_EXT;
  523. }
  524. if (desc->sptes[RMAP_EXT-1]) {
  525. desc->more = mmu_alloc_rmap_desc(vcpu);
  526. desc = desc->more;
  527. }
  528. for (i = 0; desc->sptes[i]; ++i)
  529. ++count;
  530. desc->sptes[i] = spte;
  531. }
  532. return count;
  533. }
  534. static void rmap_desc_remove_entry(unsigned long *rmapp,
  535. struct kvm_rmap_desc *desc,
  536. int i,
  537. struct kvm_rmap_desc *prev_desc)
  538. {
  539. int j;
  540. for (j = RMAP_EXT - 1; !desc->sptes[j] && j > i; --j)
  541. ;
  542. desc->sptes[i] = desc->sptes[j];
  543. desc->sptes[j] = NULL;
  544. if (j != 0)
  545. return;
  546. if (!prev_desc && !desc->more)
  547. *rmapp = (unsigned long)desc->sptes[0];
  548. else
  549. if (prev_desc)
  550. prev_desc->more = desc->more;
  551. else
  552. *rmapp = (unsigned long)desc->more | 1;
  553. mmu_free_rmap_desc(desc);
  554. }
  555. static void rmap_remove(struct kvm *kvm, u64 *spte)
  556. {
  557. struct kvm_rmap_desc *desc;
  558. struct kvm_rmap_desc *prev_desc;
  559. struct kvm_mmu_page *sp;
  560. gfn_t gfn;
  561. unsigned long *rmapp;
  562. int i;
  563. sp = page_header(__pa(spte));
  564. gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt);
  565. rmapp = gfn_to_rmap(kvm, gfn, sp->role.level);
  566. if (!*rmapp) {
  567. printk(KERN_ERR "rmap_remove: %p 0->BUG\n", spte);
  568. BUG();
  569. } else if (!(*rmapp & 1)) {
  570. rmap_printk("rmap_remove: %p 1->0\n", spte);
  571. if ((u64 *)*rmapp != spte) {
  572. printk(KERN_ERR "rmap_remove: %p 1->BUG\n", spte);
  573. BUG();
  574. }
  575. *rmapp = 0;
  576. } else {
  577. rmap_printk("rmap_remove: %p many->many\n", spte);
  578. desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
  579. prev_desc = NULL;
  580. while (desc) {
  581. for (i = 0; i < RMAP_EXT && desc->sptes[i]; ++i)
  582. if (desc->sptes[i] == spte) {
  583. rmap_desc_remove_entry(rmapp,
  584. desc, i,
  585. prev_desc);
  586. return;
  587. }
  588. prev_desc = desc;
  589. desc = desc->more;
  590. }
  591. pr_err("rmap_remove: %p many->many\n", spte);
  592. BUG();
  593. }
  594. }
  595. static int set_spte_track_bits(u64 *sptep, u64 new_spte)
  596. {
  597. pfn_t pfn;
  598. u64 old_spte = *sptep;
  599. if (!spte_has_volatile_bits(old_spte))
  600. __set_spte(sptep, new_spte);
  601. else
  602. old_spte = __xchg_spte(sptep, new_spte);
  603. if (!is_rmap_spte(old_spte))
  604. return 0;
  605. pfn = spte_to_pfn(old_spte);
  606. if (!shadow_accessed_mask || old_spte & shadow_accessed_mask)
  607. kvm_set_pfn_accessed(pfn);
  608. if (!shadow_dirty_mask || (old_spte & shadow_dirty_mask))
  609. kvm_set_pfn_dirty(pfn);
  610. return 1;
  611. }
  612. static void drop_spte(struct kvm *kvm, u64 *sptep, u64 new_spte)
  613. {
  614. if (set_spte_track_bits(sptep, new_spte))
  615. rmap_remove(kvm, sptep);
  616. }
  617. static u64 *rmap_next(struct kvm *kvm, unsigned long *rmapp, u64 *spte)
  618. {
  619. struct kvm_rmap_desc *desc;
  620. u64 *prev_spte;
  621. int i;
  622. if (!*rmapp)
  623. return NULL;
  624. else if (!(*rmapp & 1)) {
  625. if (!spte)
  626. return (u64 *)*rmapp;
  627. return NULL;
  628. }
  629. desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
  630. prev_spte = NULL;
  631. while (desc) {
  632. for (i = 0; i < RMAP_EXT && desc->sptes[i]; ++i) {
  633. if (prev_spte == spte)
  634. return desc->sptes[i];
  635. prev_spte = desc->sptes[i];
  636. }
  637. desc = desc->more;
  638. }
  639. return NULL;
  640. }
  641. static int rmap_write_protect(struct kvm *kvm, u64 gfn)
  642. {
  643. unsigned long *rmapp;
  644. u64 *spte;
  645. int i, write_protected = 0;
  646. rmapp = gfn_to_rmap(kvm, gfn, PT_PAGE_TABLE_LEVEL);
  647. spte = rmap_next(kvm, rmapp, NULL);
  648. while (spte) {
  649. BUG_ON(!spte);
  650. BUG_ON(!(*spte & PT_PRESENT_MASK));
  651. rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
  652. if (is_writable_pte(*spte)) {
  653. update_spte(spte, *spte & ~PT_WRITABLE_MASK);
  654. write_protected = 1;
  655. }
  656. spte = rmap_next(kvm, rmapp, spte);
  657. }
  658. /* check for huge page mappings */
  659. for (i = PT_DIRECTORY_LEVEL;
  660. i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
  661. rmapp = gfn_to_rmap(kvm, gfn, i);
  662. spte = rmap_next(kvm, rmapp, NULL);
  663. while (spte) {
  664. BUG_ON(!spte);
  665. BUG_ON(!(*spte & PT_PRESENT_MASK));
  666. BUG_ON((*spte & (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK)) != (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK));
  667. pgprintk("rmap_write_protect(large): spte %p %llx %lld\n", spte, *spte, gfn);
  668. if (is_writable_pte(*spte)) {
  669. drop_spte(kvm, spte,
  670. shadow_trap_nonpresent_pte);
  671. --kvm->stat.lpages;
  672. spte = NULL;
  673. write_protected = 1;
  674. }
  675. spte = rmap_next(kvm, rmapp, spte);
  676. }
  677. }
  678. return write_protected;
  679. }
  680. static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
  681. unsigned long data)
  682. {
  683. u64 *spte;
  684. int need_tlb_flush = 0;
  685. while ((spte = rmap_next(kvm, rmapp, NULL))) {
  686. BUG_ON(!(*spte & PT_PRESENT_MASK));
  687. rmap_printk("kvm_rmap_unmap_hva: spte %p %llx\n", spte, *spte);
  688. drop_spte(kvm, spte, shadow_trap_nonpresent_pte);
  689. need_tlb_flush = 1;
  690. }
  691. return need_tlb_flush;
  692. }
  693. static int kvm_set_pte_rmapp(struct kvm *kvm, unsigned long *rmapp,
  694. unsigned long data)
  695. {
  696. int need_flush = 0;
  697. u64 *spte, new_spte;
  698. pte_t *ptep = (pte_t *)data;
  699. pfn_t new_pfn;
  700. WARN_ON(pte_huge(*ptep));
  701. new_pfn = pte_pfn(*ptep);
  702. spte = rmap_next(kvm, rmapp, NULL);
  703. while (spte) {
  704. BUG_ON(!is_shadow_present_pte(*spte));
  705. rmap_printk("kvm_set_pte_rmapp: spte %p %llx\n", spte, *spte);
  706. need_flush = 1;
  707. if (pte_write(*ptep)) {
  708. drop_spte(kvm, spte, shadow_trap_nonpresent_pte);
  709. spte = rmap_next(kvm, rmapp, NULL);
  710. } else {
  711. new_spte = *spte &~ (PT64_BASE_ADDR_MASK);
  712. new_spte |= (u64)new_pfn << PAGE_SHIFT;
  713. new_spte &= ~PT_WRITABLE_MASK;
  714. new_spte &= ~SPTE_HOST_WRITEABLE;
  715. new_spte &= ~shadow_accessed_mask;
  716. set_spte_track_bits(spte, new_spte);
  717. spte = rmap_next(kvm, rmapp, spte);
  718. }
  719. }
  720. if (need_flush)
  721. kvm_flush_remote_tlbs(kvm);
  722. return 0;
  723. }
  724. static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
  725. unsigned long data,
  726. int (*handler)(struct kvm *kvm, unsigned long *rmapp,
  727. unsigned long data))
  728. {
  729. int i, j;
  730. int ret;
  731. int retval = 0;
  732. struct kvm_memslots *slots;
  733. slots = kvm_memslots(kvm);
  734. for (i = 0; i < slots->nmemslots; i++) {
  735. struct kvm_memory_slot *memslot = &slots->memslots[i];
  736. unsigned long start = memslot->userspace_addr;
  737. unsigned long end;
  738. end = start + (memslot->npages << PAGE_SHIFT);
  739. if (hva >= start && hva < end) {
  740. gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT;
  741. ret = handler(kvm, &memslot->rmap[gfn_offset], data);
  742. for (j = 0; j < KVM_NR_PAGE_SIZES - 1; ++j) {
  743. unsigned long idx;
  744. int sh;
  745. sh = KVM_HPAGE_GFN_SHIFT(PT_DIRECTORY_LEVEL+j);
  746. idx = ((memslot->base_gfn+gfn_offset) >> sh) -
  747. (memslot->base_gfn >> sh);
  748. ret |= handler(kvm,
  749. &memslot->lpage_info[j][idx].rmap_pde,
  750. data);
  751. }
  752. trace_kvm_age_page(hva, memslot, ret);
  753. retval |= ret;
  754. }
  755. }
  756. return retval;
  757. }
  758. int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
  759. {
  760. return kvm_handle_hva(kvm, hva, 0, kvm_unmap_rmapp);
  761. }
  762. void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
  763. {
  764. kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);
  765. }
  766. static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
  767. unsigned long data)
  768. {
  769. u64 *spte;
  770. int young = 0;
  771. /*
  772. * Emulate the accessed bit for EPT, by checking if this page has
  773. * an EPT mapping, and clearing it if it does. On the next access,
  774. * a new EPT mapping will be established.
  775. * This has some overhead, but not as much as the cost of swapping
  776. * out actively used pages or breaking up actively used hugepages.
  777. */
  778. if (!shadow_accessed_mask)
  779. return kvm_unmap_rmapp(kvm, rmapp, data);
  780. spte = rmap_next(kvm, rmapp, NULL);
  781. while (spte) {
  782. int _young;
  783. u64 _spte = *spte;
  784. BUG_ON(!(_spte & PT_PRESENT_MASK));
  785. _young = _spte & PT_ACCESSED_MASK;
  786. if (_young) {
  787. young = 1;
  788. clear_bit(PT_ACCESSED_SHIFT, (unsigned long *)spte);
  789. }
  790. spte = rmap_next(kvm, rmapp, spte);
  791. }
  792. return young;
  793. }
  794. #define RMAP_RECYCLE_THRESHOLD 1000
  795. static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
  796. {
  797. unsigned long *rmapp;
  798. struct kvm_mmu_page *sp;
  799. sp = page_header(__pa(spte));
  800. rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
  801. kvm_unmap_rmapp(vcpu->kvm, rmapp, 0);
  802. kvm_flush_remote_tlbs(vcpu->kvm);
  803. }
  804. int kvm_age_hva(struct kvm *kvm, unsigned long hva)
  805. {
  806. return kvm_handle_hva(kvm, hva, 0, kvm_age_rmapp);
  807. }
  808. #ifdef MMU_DEBUG
  809. static int is_empty_shadow_page(u64 *spt)
  810. {
  811. u64 *pos;
  812. u64 *end;
  813. for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
  814. if (is_shadow_present_pte(*pos)) {
  815. printk(KERN_ERR "%s: %p %llx\n", __func__,
  816. pos, *pos);
  817. return 0;
  818. }
  819. return 1;
  820. }
  821. #endif
  822. /*
  823. * This value is the sum of all of the kvm instances's
  824. * kvm->arch.n_used_mmu_pages values. We need a global,
  825. * aggregate version in order to make the slab shrinker
  826. * faster
  827. */
  828. static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, int nr)
  829. {
  830. kvm->arch.n_used_mmu_pages += nr;
  831. percpu_counter_add(&kvm_total_used_mmu_pages, nr);
  832. }
  833. static void kvm_mmu_free_page(struct kvm *kvm, struct kvm_mmu_page *sp)
  834. {
  835. ASSERT(is_empty_shadow_page(sp->spt));
  836. hlist_del(&sp->hash_link);
  837. list_del(&sp->link);
  838. __free_page(virt_to_page(sp->spt));
  839. if (!sp->role.direct)
  840. __free_page(virt_to_page(sp->gfns));
  841. kmem_cache_free(mmu_page_header_cache, sp);
  842. kvm_mod_used_mmu_pages(kvm, -1);
  843. }
  844. static unsigned kvm_page_table_hashfn(gfn_t gfn)
  845. {
  846. return gfn & ((1 << KVM_MMU_HASH_SHIFT) - 1);
  847. }
  848. static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
  849. u64 *parent_pte, int direct)
  850. {
  851. struct kvm_mmu_page *sp;
  852. sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache, sizeof *sp);
  853. sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE);
  854. if (!direct)
  855. sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache,
  856. PAGE_SIZE);
  857. set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
  858. list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
  859. bitmap_zero(sp->slot_bitmap, KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS);
  860. sp->multimapped = 0;
  861. sp->parent_pte = parent_pte;
  862. kvm_mod_used_mmu_pages(vcpu->kvm, +1);
  863. return sp;
  864. }
  865. static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
  866. struct kvm_mmu_page *sp, u64 *parent_pte)
  867. {
  868. struct kvm_pte_chain *pte_chain;
  869. struct hlist_node *node;
  870. int i;
  871. if (!parent_pte)
  872. return;
  873. if (!sp->multimapped) {
  874. u64 *old = sp->parent_pte;
  875. if (!old) {
  876. sp->parent_pte = parent_pte;
  877. return;
  878. }
  879. sp->multimapped = 1;
  880. pte_chain = mmu_alloc_pte_chain(vcpu);
  881. INIT_HLIST_HEAD(&sp->parent_ptes);
  882. hlist_add_head(&pte_chain->link, &sp->parent_ptes);
  883. pte_chain->parent_ptes[0] = old;
  884. }
  885. hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link) {
  886. if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1])
  887. continue;
  888. for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i)
  889. if (!pte_chain->parent_ptes[i]) {
  890. pte_chain->parent_ptes[i] = parent_pte;
  891. return;
  892. }
  893. }
  894. pte_chain = mmu_alloc_pte_chain(vcpu);
  895. BUG_ON(!pte_chain);
  896. hlist_add_head(&pte_chain->link, &sp->parent_ptes);
  897. pte_chain->parent_ptes[0] = parent_pte;
  898. }
  899. static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
  900. u64 *parent_pte)
  901. {
  902. struct kvm_pte_chain *pte_chain;
  903. struct hlist_node *node;
  904. int i;
  905. if (!sp->multimapped) {
  906. BUG_ON(sp->parent_pte != parent_pte);
  907. sp->parent_pte = NULL;
  908. return;
  909. }
  910. hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
  911. for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
  912. if (!pte_chain->parent_ptes[i])
  913. break;
  914. if (pte_chain->parent_ptes[i] != parent_pte)
  915. continue;
  916. while (i + 1 < NR_PTE_CHAIN_ENTRIES
  917. && pte_chain->parent_ptes[i + 1]) {
  918. pte_chain->parent_ptes[i]
  919. = pte_chain->parent_ptes[i + 1];
  920. ++i;
  921. }
  922. pte_chain->parent_ptes[i] = NULL;
  923. if (i == 0) {
  924. hlist_del(&pte_chain->link);
  925. mmu_free_pte_chain(pte_chain);
  926. if (hlist_empty(&sp->parent_ptes)) {
  927. sp->multimapped = 0;
  928. sp->parent_pte = NULL;
  929. }
  930. }
  931. return;
  932. }
  933. BUG();
  934. }
  935. static void mmu_parent_walk(struct kvm_mmu_page *sp, mmu_parent_walk_fn fn)
  936. {
  937. struct kvm_pte_chain *pte_chain;
  938. struct hlist_node *node;
  939. struct kvm_mmu_page *parent_sp;
  940. int i;
  941. if (!sp->multimapped && sp->parent_pte) {
  942. parent_sp = page_header(__pa(sp->parent_pte));
  943. fn(parent_sp, sp->parent_pte);
  944. return;
  945. }
  946. hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
  947. for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
  948. u64 *spte = pte_chain->parent_ptes[i];
  949. if (!spte)
  950. break;
  951. parent_sp = page_header(__pa(spte));
  952. fn(parent_sp, spte);
  953. }
  954. }
  955. static void mark_unsync(struct kvm_mmu_page *sp, u64 *spte);
  956. static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)
  957. {
  958. mmu_parent_walk(sp, mark_unsync);
  959. }
  960. static void mark_unsync(struct kvm_mmu_page *sp, u64 *spte)
  961. {
  962. unsigned int index;
  963. index = spte - sp->spt;
  964. if (__test_and_set_bit(index, sp->unsync_child_bitmap))
  965. return;
  966. if (sp->unsync_children++)
  967. return;
  968. kvm_mmu_mark_parents_unsync(sp);
  969. }
  970. static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu,
  971. struct kvm_mmu_page *sp)
  972. {
  973. int i;
  974. for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
  975. sp->spt[i] = shadow_trap_nonpresent_pte;
  976. }
  977. static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
  978. struct kvm_mmu_page *sp, bool clear_unsync)
  979. {
  980. return 1;
  981. }
  982. static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
  983. {
  984. }
  985. #define KVM_PAGE_ARRAY_NR 16
  986. struct kvm_mmu_pages {
  987. struct mmu_page_and_offset {
  988. struct kvm_mmu_page *sp;
  989. unsigned int idx;
  990. } page[KVM_PAGE_ARRAY_NR];
  991. unsigned int nr;
  992. };
  993. #define for_each_unsync_children(bitmap, idx) \
  994. for (idx = find_first_bit(bitmap, 512); \
  995. idx < 512; \
  996. idx = find_next_bit(bitmap, 512, idx+1))
  997. static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
  998. int idx)
  999. {
  1000. int i;
  1001. if (sp->unsync)
  1002. for (i=0; i < pvec->nr; i++)
  1003. if (pvec->page[i].sp == sp)
  1004. return 0;
  1005. pvec->page[pvec->nr].sp = sp;
  1006. pvec->page[pvec->nr].idx = idx;
  1007. pvec->nr++;
  1008. return (pvec->nr == KVM_PAGE_ARRAY_NR);
  1009. }
  1010. static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
  1011. struct kvm_mmu_pages *pvec)
  1012. {
  1013. int i, ret, nr_unsync_leaf = 0;
  1014. for_each_unsync_children(sp->unsync_child_bitmap, i) {
  1015. struct kvm_mmu_page *child;
  1016. u64 ent = sp->spt[i];
  1017. if (!is_shadow_present_pte(ent) || is_large_pte(ent))
  1018. goto clear_child_bitmap;
  1019. child = page_header(ent & PT64_BASE_ADDR_MASK);
  1020. if (child->unsync_children) {
  1021. if (mmu_pages_add(pvec, child, i))
  1022. return -ENOSPC;
  1023. ret = __mmu_unsync_walk(child, pvec);
  1024. if (!ret)
  1025. goto clear_child_bitmap;
  1026. else if (ret > 0)
  1027. nr_unsync_leaf += ret;
  1028. else
  1029. return ret;
  1030. } else if (child->unsync) {
  1031. nr_unsync_leaf++;
  1032. if (mmu_pages_add(pvec, child, i))
  1033. return -ENOSPC;
  1034. } else
  1035. goto clear_child_bitmap;
  1036. continue;
  1037. clear_child_bitmap:
  1038. __clear_bit(i, sp->unsync_child_bitmap);
  1039. sp->unsync_children--;
  1040. WARN_ON((int)sp->unsync_children < 0);
  1041. }
  1042. return nr_unsync_leaf;
  1043. }
  1044. static int mmu_unsync_walk(struct kvm_mmu_page *sp,
  1045. struct kvm_mmu_pages *pvec)
  1046. {
  1047. if (!sp->unsync_children)
  1048. return 0;
  1049. mmu_pages_add(pvec, sp, 0);
  1050. return __mmu_unsync_walk(sp, pvec);
  1051. }
  1052. static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
  1053. {
  1054. WARN_ON(!sp->unsync);
  1055. trace_kvm_mmu_sync_page(sp);
  1056. sp->unsync = 0;
  1057. --kvm->stat.mmu_unsync;
  1058. }
  1059. static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
  1060. struct list_head *invalid_list);
  1061. static void kvm_mmu_commit_zap_page(struct kvm *kvm,
  1062. struct list_head *invalid_list);
  1063. #define for_each_gfn_sp(kvm, sp, gfn, pos) \
  1064. hlist_for_each_entry(sp, pos, \
  1065. &(kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)], hash_link) \
  1066. if ((sp)->gfn != (gfn)) {} else
  1067. #define for_each_gfn_indirect_valid_sp(kvm, sp, gfn, pos) \
  1068. hlist_for_each_entry(sp, pos, \
  1069. &(kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)], hash_link) \
  1070. if ((sp)->gfn != (gfn) || (sp)->role.direct || \
  1071. (sp)->role.invalid) {} else
  1072. /* @sp->gfn should be write-protected at the call site */
  1073. static int __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
  1074. struct list_head *invalid_list, bool clear_unsync)
  1075. {
  1076. if (sp->role.cr4_pae != !!is_pae(vcpu)) {
  1077. kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
  1078. return 1;
  1079. }
  1080. if (clear_unsync)
  1081. kvm_unlink_unsync_page(vcpu->kvm, sp);
  1082. if (vcpu->arch.mmu.sync_page(vcpu, sp, clear_unsync)) {
  1083. kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
  1084. return 1;
  1085. }
  1086. kvm_mmu_flush_tlb(vcpu);
  1087. return 0;
  1088. }
  1089. static int kvm_sync_page_transient(struct kvm_vcpu *vcpu,
  1090. struct kvm_mmu_page *sp)
  1091. {
  1092. LIST_HEAD(invalid_list);
  1093. int ret;
  1094. ret = __kvm_sync_page(vcpu, sp, &invalid_list, false);
  1095. if (ret)
  1096. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  1097. return ret;
  1098. }
  1099. static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
  1100. struct list_head *invalid_list)
  1101. {
  1102. return __kvm_sync_page(vcpu, sp, invalid_list, true);
  1103. }
  1104. /* @gfn should be write-protected at the call site */
  1105. static void kvm_sync_pages(struct kvm_vcpu *vcpu, gfn_t gfn)
  1106. {
  1107. struct kvm_mmu_page *s;
  1108. struct hlist_node *node;
  1109. LIST_HEAD(invalid_list);
  1110. bool flush = false;
  1111. for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn, node) {
  1112. if (!s->unsync)
  1113. continue;
  1114. WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
  1115. if ((s->role.cr4_pae != !!is_pae(vcpu)) ||
  1116. (vcpu->arch.mmu.sync_page(vcpu, s, true))) {
  1117. kvm_mmu_prepare_zap_page(vcpu->kvm, s, &invalid_list);
  1118. continue;
  1119. }
  1120. kvm_unlink_unsync_page(vcpu->kvm, s);
  1121. flush = true;
  1122. }
  1123. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  1124. if (flush)
  1125. kvm_mmu_flush_tlb(vcpu);
  1126. }
  1127. struct mmu_page_path {
  1128. struct kvm_mmu_page *parent[PT64_ROOT_LEVEL-1];
  1129. unsigned int idx[PT64_ROOT_LEVEL-1];
  1130. };
  1131. #define for_each_sp(pvec, sp, parents, i) \
  1132. for (i = mmu_pages_next(&pvec, &parents, -1), \
  1133. sp = pvec.page[i].sp; \
  1134. i < pvec.nr && ({ sp = pvec.page[i].sp; 1;}); \
  1135. i = mmu_pages_next(&pvec, &parents, i))
  1136. static int mmu_pages_next(struct kvm_mmu_pages *pvec,
  1137. struct mmu_page_path *parents,
  1138. int i)
  1139. {
  1140. int n;
  1141. for (n = i+1; n < pvec->nr; n++) {
  1142. struct kvm_mmu_page *sp = pvec->page[n].sp;
  1143. if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
  1144. parents->idx[0] = pvec->page[n].idx;
  1145. return n;
  1146. }
  1147. parents->parent[sp->role.level-2] = sp;
  1148. parents->idx[sp->role.level-1] = pvec->page[n].idx;
  1149. }
  1150. return n;
  1151. }
  1152. static void mmu_pages_clear_parents(struct mmu_page_path *parents)
  1153. {
  1154. struct kvm_mmu_page *sp;
  1155. unsigned int level = 0;
  1156. do {
  1157. unsigned int idx = parents->idx[level];
  1158. sp = parents->parent[level];
  1159. if (!sp)
  1160. return;
  1161. --sp->unsync_children;
  1162. WARN_ON((int)sp->unsync_children < 0);
  1163. __clear_bit(idx, sp->unsync_child_bitmap);
  1164. level++;
  1165. } while (level < PT64_ROOT_LEVEL-1 && !sp->unsync_children);
  1166. }
  1167. static void kvm_mmu_pages_init(struct kvm_mmu_page *parent,
  1168. struct mmu_page_path *parents,
  1169. struct kvm_mmu_pages *pvec)
  1170. {
  1171. parents->parent[parent->role.level-1] = NULL;
  1172. pvec->nr = 0;
  1173. }
  1174. static void mmu_sync_children(struct kvm_vcpu *vcpu,
  1175. struct kvm_mmu_page *parent)
  1176. {
  1177. int i;
  1178. struct kvm_mmu_page *sp;
  1179. struct mmu_page_path parents;
  1180. struct kvm_mmu_pages pages;
  1181. LIST_HEAD(invalid_list);
  1182. kvm_mmu_pages_init(parent, &parents, &pages);
  1183. while (mmu_unsync_walk(parent, &pages)) {
  1184. int protected = 0;
  1185. for_each_sp(pages, sp, parents, i)
  1186. protected |= rmap_write_protect(vcpu->kvm, sp->gfn);
  1187. if (protected)
  1188. kvm_flush_remote_tlbs(vcpu->kvm);
  1189. for_each_sp(pages, sp, parents, i) {
  1190. kvm_sync_page(vcpu, sp, &invalid_list);
  1191. mmu_pages_clear_parents(&parents);
  1192. }
  1193. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  1194. cond_resched_lock(&vcpu->kvm->mmu_lock);
  1195. kvm_mmu_pages_init(parent, &parents, &pages);
  1196. }
  1197. }
  1198. static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
  1199. gfn_t gfn,
  1200. gva_t gaddr,
  1201. unsigned level,
  1202. int direct,
  1203. unsigned access,
  1204. u64 *parent_pte)
  1205. {
  1206. union kvm_mmu_page_role role;
  1207. unsigned quadrant;
  1208. struct kvm_mmu_page *sp;
  1209. struct hlist_node *node;
  1210. bool need_sync = false;
  1211. role = vcpu->arch.mmu.base_role;
  1212. role.level = level;
  1213. role.direct = direct;
  1214. if (role.direct)
  1215. role.cr4_pae = 0;
  1216. role.access = access;
  1217. if (!vcpu->arch.mmu.direct_map
  1218. && vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) {
  1219. quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
  1220. quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
  1221. role.quadrant = quadrant;
  1222. }
  1223. for_each_gfn_sp(vcpu->kvm, sp, gfn, node) {
  1224. if (!need_sync && sp->unsync)
  1225. need_sync = true;
  1226. if (sp->role.word != role.word)
  1227. continue;
  1228. if (sp->unsync && kvm_sync_page_transient(vcpu, sp))
  1229. break;
  1230. mmu_page_add_parent_pte(vcpu, sp, parent_pte);
  1231. if (sp->unsync_children) {
  1232. kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
  1233. kvm_mmu_mark_parents_unsync(sp);
  1234. } else if (sp->unsync)
  1235. kvm_mmu_mark_parents_unsync(sp);
  1236. trace_kvm_mmu_get_page(sp, false);
  1237. return sp;
  1238. }
  1239. ++vcpu->kvm->stat.mmu_cache_miss;
  1240. sp = kvm_mmu_alloc_page(vcpu, parent_pte, direct);
  1241. if (!sp)
  1242. return sp;
  1243. sp->gfn = gfn;
  1244. sp->role = role;
  1245. hlist_add_head(&sp->hash_link,
  1246. &vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)]);
  1247. if (!direct) {
  1248. if (rmap_write_protect(vcpu->kvm, gfn))
  1249. kvm_flush_remote_tlbs(vcpu->kvm);
  1250. if (level > PT_PAGE_TABLE_LEVEL && need_sync)
  1251. kvm_sync_pages(vcpu, gfn);
  1252. account_shadowed(vcpu->kvm, gfn);
  1253. }
  1254. if (shadow_trap_nonpresent_pte != shadow_notrap_nonpresent_pte)
  1255. vcpu->arch.mmu.prefetch_page(vcpu, sp);
  1256. else
  1257. nonpaging_prefetch_page(vcpu, sp);
  1258. trace_kvm_mmu_get_page(sp, true);
  1259. return sp;
  1260. }
  1261. static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
  1262. struct kvm_vcpu *vcpu, u64 addr)
  1263. {
  1264. iterator->addr = addr;
  1265. iterator->shadow_addr = vcpu->arch.mmu.root_hpa;
  1266. iterator->level = vcpu->arch.mmu.shadow_root_level;
  1267. if (iterator->level == PT64_ROOT_LEVEL &&
  1268. vcpu->arch.mmu.root_level < PT64_ROOT_LEVEL &&
  1269. !vcpu->arch.mmu.direct_map)
  1270. --iterator->level;
  1271. if (iterator->level == PT32E_ROOT_LEVEL) {
  1272. iterator->shadow_addr
  1273. = vcpu->arch.mmu.pae_root[(addr >> 30) & 3];
  1274. iterator->shadow_addr &= PT64_BASE_ADDR_MASK;
  1275. --iterator->level;
  1276. if (!iterator->shadow_addr)
  1277. iterator->level = 0;
  1278. }
  1279. }
  1280. static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
  1281. {
  1282. if (iterator->level < PT_PAGE_TABLE_LEVEL)
  1283. return false;
  1284. if (iterator->level == PT_PAGE_TABLE_LEVEL)
  1285. if (is_large_pte(*iterator->sptep))
  1286. return false;
  1287. iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
  1288. iterator->sptep = ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
  1289. return true;
  1290. }
  1291. static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
  1292. {
  1293. iterator->shadow_addr = *iterator->sptep & PT64_BASE_ADDR_MASK;
  1294. --iterator->level;
  1295. }
  1296. static void link_shadow_page(u64 *sptep, struct kvm_mmu_page *sp)
  1297. {
  1298. u64 spte;
  1299. spte = __pa(sp->spt)
  1300. | PT_PRESENT_MASK | PT_ACCESSED_MASK
  1301. | PT_WRITABLE_MASK | PT_USER_MASK;
  1302. __set_spte(sptep, spte);
  1303. }
  1304. static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
  1305. {
  1306. if (is_large_pte(*sptep)) {
  1307. drop_spte(vcpu->kvm, sptep, shadow_trap_nonpresent_pte);
  1308. kvm_flush_remote_tlbs(vcpu->kvm);
  1309. }
  1310. }
  1311. static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
  1312. unsigned direct_access)
  1313. {
  1314. if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {
  1315. struct kvm_mmu_page *child;
  1316. /*
  1317. * For the direct sp, if the guest pte's dirty bit
  1318. * changed form clean to dirty, it will corrupt the
  1319. * sp's access: allow writable in the read-only sp,
  1320. * so we should update the spte at this point to get
  1321. * a new sp with the correct access.
  1322. */
  1323. child = page_header(*sptep & PT64_BASE_ADDR_MASK);
  1324. if (child->role.access == direct_access)
  1325. return;
  1326. mmu_page_remove_parent_pte(child, sptep);
  1327. __set_spte(sptep, shadow_trap_nonpresent_pte);
  1328. kvm_flush_remote_tlbs(vcpu->kvm);
  1329. }
  1330. }
  1331. static void kvm_mmu_page_unlink_children(struct kvm *kvm,
  1332. struct kvm_mmu_page *sp)
  1333. {
  1334. unsigned i;
  1335. u64 *pt;
  1336. u64 ent;
  1337. pt = sp->spt;
  1338. for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
  1339. ent = pt[i];
  1340. if (is_shadow_present_pte(ent)) {
  1341. if (!is_last_spte(ent, sp->role.level)) {
  1342. ent &= PT64_BASE_ADDR_MASK;
  1343. mmu_page_remove_parent_pte(page_header(ent),
  1344. &pt[i]);
  1345. } else {
  1346. if (is_large_pte(ent))
  1347. --kvm->stat.lpages;
  1348. drop_spte(kvm, &pt[i],
  1349. shadow_trap_nonpresent_pte);
  1350. }
  1351. }
  1352. pt[i] = shadow_trap_nonpresent_pte;
  1353. }
  1354. }
  1355. static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte)
  1356. {
  1357. mmu_page_remove_parent_pte(sp, parent_pte);
  1358. }
  1359. static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm)
  1360. {
  1361. int i;
  1362. struct kvm_vcpu *vcpu;
  1363. kvm_for_each_vcpu(i, vcpu, kvm)
  1364. vcpu->arch.last_pte_updated = NULL;
  1365. }
  1366. static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
  1367. {
  1368. u64 *parent_pte;
  1369. while (sp->multimapped || sp->parent_pte) {
  1370. if (!sp->multimapped)
  1371. parent_pte = sp->parent_pte;
  1372. else {
  1373. struct kvm_pte_chain *chain;
  1374. chain = container_of(sp->parent_ptes.first,
  1375. struct kvm_pte_chain, link);
  1376. parent_pte = chain->parent_ptes[0];
  1377. }
  1378. BUG_ON(!parent_pte);
  1379. kvm_mmu_put_page(sp, parent_pte);
  1380. __set_spte(parent_pte, shadow_trap_nonpresent_pte);
  1381. }
  1382. }
  1383. static int mmu_zap_unsync_children(struct kvm *kvm,
  1384. struct kvm_mmu_page *parent,
  1385. struct list_head *invalid_list)
  1386. {
  1387. int i, zapped = 0;
  1388. struct mmu_page_path parents;
  1389. struct kvm_mmu_pages pages;
  1390. if (parent->role.level == PT_PAGE_TABLE_LEVEL)
  1391. return 0;
  1392. kvm_mmu_pages_init(parent, &parents, &pages);
  1393. while (mmu_unsync_walk(parent, &pages)) {
  1394. struct kvm_mmu_page *sp;
  1395. for_each_sp(pages, sp, parents, i) {
  1396. kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
  1397. mmu_pages_clear_parents(&parents);
  1398. zapped++;
  1399. }
  1400. kvm_mmu_pages_init(parent, &parents, &pages);
  1401. }
  1402. return zapped;
  1403. }
  1404. static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
  1405. struct list_head *invalid_list)
  1406. {
  1407. int ret;
  1408. trace_kvm_mmu_prepare_zap_page(sp);
  1409. ++kvm->stat.mmu_shadow_zapped;
  1410. ret = mmu_zap_unsync_children(kvm, sp, invalid_list);
  1411. kvm_mmu_page_unlink_children(kvm, sp);
  1412. kvm_mmu_unlink_parents(kvm, sp);
  1413. if (!sp->role.invalid && !sp->role.direct)
  1414. unaccount_shadowed(kvm, sp->gfn);
  1415. if (sp->unsync)
  1416. kvm_unlink_unsync_page(kvm, sp);
  1417. if (!sp->root_count) {
  1418. /* Count self */
  1419. ret++;
  1420. list_move(&sp->link, invalid_list);
  1421. } else {
  1422. list_move(&sp->link, &kvm->arch.active_mmu_pages);
  1423. kvm_reload_remote_mmus(kvm);
  1424. }
  1425. sp->role.invalid = 1;
  1426. kvm_mmu_reset_last_pte_updated(kvm);
  1427. return ret;
  1428. }
  1429. static void kvm_mmu_commit_zap_page(struct kvm *kvm,
  1430. struct list_head *invalid_list)
  1431. {
  1432. struct kvm_mmu_page *sp;
  1433. if (list_empty(invalid_list))
  1434. return;
  1435. kvm_flush_remote_tlbs(kvm);
  1436. do {
  1437. sp = list_first_entry(invalid_list, struct kvm_mmu_page, link);
  1438. WARN_ON(!sp->role.invalid || sp->root_count);
  1439. kvm_mmu_free_page(kvm, sp);
  1440. } while (!list_empty(invalid_list));
  1441. }
  1442. /*
  1443. * Changing the number of mmu pages allocated to the vm
  1444. * Note: if goal_nr_mmu_pages is too small, you will get dead lock
  1445. */
  1446. void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int goal_nr_mmu_pages)
  1447. {
  1448. LIST_HEAD(invalid_list);
  1449. /*
  1450. * If we set the number of mmu pages to be smaller be than the
  1451. * number of actived pages , we must to free some mmu pages before we
  1452. * change the value
  1453. */
  1454. if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
  1455. while (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages &&
  1456. !list_empty(&kvm->arch.active_mmu_pages)) {
  1457. struct kvm_mmu_page *page;
  1458. page = container_of(kvm->arch.active_mmu_pages.prev,
  1459. struct kvm_mmu_page, link);
  1460. kvm_mmu_prepare_zap_page(kvm, page, &invalid_list);
  1461. kvm_mmu_commit_zap_page(kvm, &invalid_list);
  1462. }
  1463. goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages;
  1464. }
  1465. kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
  1466. }
  1467. static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
  1468. {
  1469. struct kvm_mmu_page *sp;
  1470. struct hlist_node *node;
  1471. LIST_HEAD(invalid_list);
  1472. int r;
  1473. pgprintk("%s: looking for gfn %llx\n", __func__, gfn);
  1474. r = 0;
  1475. for_each_gfn_indirect_valid_sp(kvm, sp, gfn, node) {
  1476. pgprintk("%s: gfn %llx role %x\n", __func__, gfn,
  1477. sp->role.word);
  1478. r = 1;
  1479. kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
  1480. }
  1481. kvm_mmu_commit_zap_page(kvm, &invalid_list);
  1482. return r;
  1483. }
  1484. static void mmu_unshadow(struct kvm *kvm, gfn_t gfn)
  1485. {
  1486. struct kvm_mmu_page *sp;
  1487. struct hlist_node *node;
  1488. LIST_HEAD(invalid_list);
  1489. for_each_gfn_indirect_valid_sp(kvm, sp, gfn, node) {
  1490. pgprintk("%s: zap %llx %x\n",
  1491. __func__, gfn, sp->role.word);
  1492. kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
  1493. }
  1494. kvm_mmu_commit_zap_page(kvm, &invalid_list);
  1495. }
  1496. static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn)
  1497. {
  1498. int slot = memslot_id(kvm, gfn);
  1499. struct kvm_mmu_page *sp = page_header(__pa(pte));
  1500. __set_bit(slot, sp->slot_bitmap);
  1501. }
  1502. static void mmu_convert_notrap(struct kvm_mmu_page *sp)
  1503. {
  1504. int i;
  1505. u64 *pt = sp->spt;
  1506. if (shadow_trap_nonpresent_pte == shadow_notrap_nonpresent_pte)
  1507. return;
  1508. for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
  1509. if (pt[i] == shadow_notrap_nonpresent_pte)
  1510. __set_spte(&pt[i], shadow_trap_nonpresent_pte);
  1511. }
  1512. }
  1513. /*
  1514. * The function is based on mtrr_type_lookup() in
  1515. * arch/x86/kernel/cpu/mtrr/generic.c
  1516. */
  1517. static int get_mtrr_type(struct mtrr_state_type *mtrr_state,
  1518. u64 start, u64 end)
  1519. {
  1520. int i;
  1521. u64 base, mask;
  1522. u8 prev_match, curr_match;
  1523. int num_var_ranges = KVM_NR_VAR_MTRR;
  1524. if (!mtrr_state->enabled)
  1525. return 0xFF;
  1526. /* Make end inclusive end, instead of exclusive */
  1527. end--;
  1528. /* Look in fixed ranges. Just return the type as per start */
  1529. if (mtrr_state->have_fixed && (start < 0x100000)) {
  1530. int idx;
  1531. if (start < 0x80000) {
  1532. idx = 0;
  1533. idx += (start >> 16);
  1534. return mtrr_state->fixed_ranges[idx];
  1535. } else if (start < 0xC0000) {
  1536. idx = 1 * 8;
  1537. idx += ((start - 0x80000) >> 14);
  1538. return mtrr_state->fixed_ranges[idx];
  1539. } else if (start < 0x1000000) {
  1540. idx = 3 * 8;
  1541. idx += ((start - 0xC0000) >> 12);
  1542. return mtrr_state->fixed_ranges[idx];
  1543. }
  1544. }
  1545. /*
  1546. * Look in variable ranges
  1547. * Look of multiple ranges matching this address and pick type
  1548. * as per MTRR precedence
  1549. */
  1550. if (!(mtrr_state->enabled & 2))
  1551. return mtrr_state->def_type;
  1552. prev_match = 0xFF;
  1553. for (i = 0; i < num_var_ranges; ++i) {
  1554. unsigned short start_state, end_state;
  1555. if (!(mtrr_state->var_ranges[i].mask_lo & (1 << 11)))
  1556. continue;
  1557. base = (((u64)mtrr_state->var_ranges[i].base_hi) << 32) +
  1558. (mtrr_state->var_ranges[i].base_lo & PAGE_MASK);
  1559. mask = (((u64)mtrr_state->var_ranges[i].mask_hi) << 32) +
  1560. (mtrr_state->var_ranges[i].mask_lo & PAGE_MASK);
  1561. start_state = ((start & mask) == (base & mask));
  1562. end_state = ((end & mask) == (base & mask));
  1563. if (start_state != end_state)
  1564. return 0xFE;
  1565. if ((start & mask) != (base & mask))
  1566. continue;
  1567. curr_match = mtrr_state->var_ranges[i].base_lo & 0xff;
  1568. if (prev_match == 0xFF) {
  1569. prev_match = curr_match;
  1570. continue;
  1571. }
  1572. if (prev_match == MTRR_TYPE_UNCACHABLE ||
  1573. curr_match == MTRR_TYPE_UNCACHABLE)
  1574. return MTRR_TYPE_UNCACHABLE;
  1575. if ((prev_match == MTRR_TYPE_WRBACK &&
  1576. curr_match == MTRR_TYPE_WRTHROUGH) ||
  1577. (prev_match == MTRR_TYPE_WRTHROUGH &&
  1578. curr_match == MTRR_TYPE_WRBACK)) {
  1579. prev_match = MTRR_TYPE_WRTHROUGH;
  1580. curr_match = MTRR_TYPE_WRTHROUGH;
  1581. }
  1582. if (prev_match != curr_match)
  1583. return MTRR_TYPE_UNCACHABLE;
  1584. }
  1585. if (prev_match != 0xFF)
  1586. return prev_match;
  1587. return mtrr_state->def_type;
  1588. }
  1589. u8 kvm_get_guest_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn)
  1590. {
  1591. u8 mtrr;
  1592. mtrr = get_mtrr_type(&vcpu->arch.mtrr_state, gfn << PAGE_SHIFT,
  1593. (gfn << PAGE_SHIFT) + PAGE_SIZE);
  1594. if (mtrr == 0xfe || mtrr == 0xff)
  1595. mtrr = MTRR_TYPE_WRBACK;
  1596. return mtrr;
  1597. }
  1598. EXPORT_SYMBOL_GPL(kvm_get_guest_memory_type);
  1599. static void __kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
  1600. {
  1601. trace_kvm_mmu_unsync_page(sp);
  1602. ++vcpu->kvm->stat.mmu_unsync;
  1603. sp->unsync = 1;
  1604. kvm_mmu_mark_parents_unsync(sp);
  1605. mmu_convert_notrap(sp);
  1606. }
  1607. static void kvm_unsync_pages(struct kvm_vcpu *vcpu, gfn_t gfn)
  1608. {
  1609. struct kvm_mmu_page *s;
  1610. struct hlist_node *node;
  1611. for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn, node) {
  1612. if (s->unsync)
  1613. continue;
  1614. WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
  1615. __kvm_unsync_page(vcpu, s);
  1616. }
  1617. }
  1618. static int mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
  1619. bool can_unsync)
  1620. {
  1621. struct kvm_mmu_page *s;
  1622. struct hlist_node *node;
  1623. bool need_unsync = false;
  1624. for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn, node) {
  1625. if (!can_unsync)
  1626. return 1;
  1627. if (s->role.level != PT_PAGE_TABLE_LEVEL)
  1628. return 1;
  1629. if (!need_unsync && !s->unsync) {
  1630. if (!oos_shadow)
  1631. return 1;
  1632. need_unsync = true;
  1633. }
  1634. }
  1635. if (need_unsync)
  1636. kvm_unsync_pages(vcpu, gfn);
  1637. return 0;
  1638. }
  1639. static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
  1640. unsigned pte_access, int user_fault,
  1641. int write_fault, int dirty, int level,
  1642. gfn_t gfn, pfn_t pfn, bool speculative,
  1643. bool can_unsync, bool reset_host_protection)
  1644. {
  1645. u64 spte, entry = *sptep;
  1646. int ret = 0;
  1647. /*
  1648. * We don't set the accessed bit, since we sometimes want to see
  1649. * whether the guest actually used the pte (in order to detect
  1650. * demand paging).
  1651. */
  1652. spte = PT_PRESENT_MASK;
  1653. if (!speculative)
  1654. spte |= shadow_accessed_mask;
  1655. if (!dirty)
  1656. pte_access &= ~ACC_WRITE_MASK;
  1657. if (pte_access & ACC_EXEC_MASK)
  1658. spte |= shadow_x_mask;
  1659. else
  1660. spte |= shadow_nx_mask;
  1661. if (pte_access & ACC_USER_MASK)
  1662. spte |= shadow_user_mask;
  1663. if (level > PT_PAGE_TABLE_LEVEL)
  1664. spte |= PT_PAGE_SIZE_MASK;
  1665. if (tdp_enabled)
  1666. spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn,
  1667. kvm_is_mmio_pfn(pfn));
  1668. if (reset_host_protection)
  1669. spte |= SPTE_HOST_WRITEABLE;
  1670. spte |= (u64)pfn << PAGE_SHIFT;
  1671. if ((pte_access & ACC_WRITE_MASK)
  1672. || (!vcpu->arch.mmu.direct_map && write_fault
  1673. && !is_write_protection(vcpu) && !user_fault)) {
  1674. if (level > PT_PAGE_TABLE_LEVEL &&
  1675. has_wrprotected_page(vcpu->kvm, gfn, level)) {
  1676. ret = 1;
  1677. drop_spte(vcpu->kvm, sptep, shadow_trap_nonpresent_pte);
  1678. goto done;
  1679. }
  1680. spte |= PT_WRITABLE_MASK;
  1681. if (!vcpu->arch.mmu.direct_map
  1682. && !(pte_access & ACC_WRITE_MASK))
  1683. spte &= ~PT_USER_MASK;
  1684. /*
  1685. * Optimization: for pte sync, if spte was writable the hash
  1686. * lookup is unnecessary (and expensive). Write protection
  1687. * is responsibility of mmu_get_page / kvm_sync_page.
  1688. * Same reasoning can be applied to dirty page accounting.
  1689. */
  1690. if (!can_unsync && is_writable_pte(*sptep))
  1691. goto set_pte;
  1692. if (mmu_need_write_protect(vcpu, gfn, can_unsync)) {
  1693. pgprintk("%s: found shadow page for %llx, marking ro\n",
  1694. __func__, gfn);
  1695. ret = 1;
  1696. pte_access &= ~ACC_WRITE_MASK;
  1697. if (is_writable_pte(spte))
  1698. spte &= ~PT_WRITABLE_MASK;
  1699. }
  1700. }
  1701. if (pte_access & ACC_WRITE_MASK)
  1702. mark_page_dirty(vcpu->kvm, gfn);
  1703. set_pte:
  1704. update_spte(sptep, spte);
  1705. /*
  1706. * If we overwrite a writable spte with a read-only one we
  1707. * should flush remote TLBs. Otherwise rmap_write_protect
  1708. * will find a read-only spte, even though the writable spte
  1709. * might be cached on a CPU's TLB.
  1710. */
  1711. if (is_writable_pte(entry) && !is_writable_pte(*sptep))
  1712. kvm_flush_remote_tlbs(vcpu->kvm);
  1713. done:
  1714. return ret;
  1715. }
  1716. static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
  1717. unsigned pt_access, unsigned pte_access,
  1718. int user_fault, int write_fault, int dirty,
  1719. int *ptwrite, int level, gfn_t gfn,
  1720. pfn_t pfn, bool speculative,
  1721. bool reset_host_protection)
  1722. {
  1723. int was_rmapped = 0;
  1724. int rmap_count;
  1725. pgprintk("%s: spte %llx access %x write_fault %d"
  1726. " user_fault %d gfn %llx\n",
  1727. __func__, *sptep, pt_access,
  1728. write_fault, user_fault, gfn);
  1729. if (is_rmap_spte(*sptep)) {
  1730. /*
  1731. * If we overwrite a PTE page pointer with a 2MB PMD, unlink
  1732. * the parent of the now unreachable PTE.
  1733. */
  1734. if (level > PT_PAGE_TABLE_LEVEL &&
  1735. !is_large_pte(*sptep)) {
  1736. struct kvm_mmu_page *child;
  1737. u64 pte = *sptep;
  1738. child = page_header(pte & PT64_BASE_ADDR_MASK);
  1739. mmu_page_remove_parent_pte(child, sptep);
  1740. __set_spte(sptep, shadow_trap_nonpresent_pte);
  1741. kvm_flush_remote_tlbs(vcpu->kvm);
  1742. } else if (pfn != spte_to_pfn(*sptep)) {
  1743. pgprintk("hfn old %llx new %llx\n",
  1744. spte_to_pfn(*sptep), pfn);
  1745. drop_spte(vcpu->kvm, sptep, shadow_trap_nonpresent_pte);
  1746. kvm_flush_remote_tlbs(vcpu->kvm);
  1747. } else
  1748. was_rmapped = 1;
  1749. }
  1750. if (set_spte(vcpu, sptep, pte_access, user_fault, write_fault,
  1751. dirty, level, gfn, pfn, speculative, true,
  1752. reset_host_protection)) {
  1753. if (write_fault)
  1754. *ptwrite = 1;
  1755. kvm_mmu_flush_tlb(vcpu);
  1756. }
  1757. pgprintk("%s: setting spte %llx\n", __func__, *sptep);
  1758. pgprintk("instantiating %s PTE (%s) at %llx (%llx) addr %p\n",
  1759. is_large_pte(*sptep)? "2MB" : "4kB",
  1760. *sptep & PT_PRESENT_MASK ?"RW":"R", gfn,
  1761. *sptep, sptep);
  1762. if (!was_rmapped && is_large_pte(*sptep))
  1763. ++vcpu->kvm->stat.lpages;
  1764. page_header_update_slot(vcpu->kvm, sptep, gfn);
  1765. if (!was_rmapped) {
  1766. rmap_count = rmap_add(vcpu, sptep, gfn);
  1767. if (rmap_count > RMAP_RECYCLE_THRESHOLD)
  1768. rmap_recycle(vcpu, sptep, gfn);
  1769. }
  1770. kvm_release_pfn_clean(pfn);
  1771. if (speculative) {
  1772. vcpu->arch.last_pte_updated = sptep;
  1773. vcpu->arch.last_pte_gfn = gfn;
  1774. }
  1775. }
  1776. static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
  1777. {
  1778. }
  1779. static struct kvm_memory_slot *
  1780. pte_prefetch_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn, bool no_dirty_log)
  1781. {
  1782. struct kvm_memory_slot *slot;
  1783. slot = gfn_to_memslot(vcpu->kvm, gfn);
  1784. if (!slot || slot->flags & KVM_MEMSLOT_INVALID ||
  1785. (no_dirty_log && slot->dirty_bitmap))
  1786. slot = NULL;
  1787. return slot;
  1788. }
  1789. static pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn,
  1790. bool no_dirty_log)
  1791. {
  1792. struct kvm_memory_slot *slot;
  1793. unsigned long hva;
  1794. slot = pte_prefetch_gfn_to_memslot(vcpu, gfn, no_dirty_log);
  1795. if (!slot) {
  1796. get_page(bad_page);
  1797. return page_to_pfn(bad_page);
  1798. }
  1799. hva = gfn_to_hva_memslot(slot, gfn);
  1800. return hva_to_pfn_atomic(vcpu->kvm, hva);
  1801. }
  1802. static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
  1803. struct kvm_mmu_page *sp,
  1804. u64 *start, u64 *end)
  1805. {
  1806. struct page *pages[PTE_PREFETCH_NUM];
  1807. unsigned access = sp->role.access;
  1808. int i, ret;
  1809. gfn_t gfn;
  1810. gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt);
  1811. if (!pte_prefetch_gfn_to_memslot(vcpu, gfn, access & ACC_WRITE_MASK))
  1812. return -1;
  1813. ret = gfn_to_page_many_atomic(vcpu->kvm, gfn, pages, end - start);
  1814. if (ret <= 0)
  1815. return -1;
  1816. for (i = 0; i < ret; i++, gfn++, start++)
  1817. mmu_set_spte(vcpu, start, ACC_ALL,
  1818. access, 0, 0, 1, NULL,
  1819. sp->role.level, gfn,
  1820. page_to_pfn(pages[i]), true, true);
  1821. return 0;
  1822. }
  1823. static void __direct_pte_prefetch(struct kvm_vcpu *vcpu,
  1824. struct kvm_mmu_page *sp, u64 *sptep)
  1825. {
  1826. u64 *spte, *start = NULL;
  1827. int i;
  1828. WARN_ON(!sp->role.direct);
  1829. i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
  1830. spte = sp->spt + i;
  1831. for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
  1832. if (*spte != shadow_trap_nonpresent_pte || spte == sptep) {
  1833. if (!start)
  1834. continue;
  1835. if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0)
  1836. break;
  1837. start = NULL;
  1838. } else if (!start)
  1839. start = spte;
  1840. }
  1841. }
  1842. static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
  1843. {
  1844. struct kvm_mmu_page *sp;
  1845. /*
  1846. * Since it's no accessed bit on EPT, it's no way to
  1847. * distinguish between actually accessed translations
  1848. * and prefetched, so disable pte prefetch if EPT is
  1849. * enabled.
  1850. */
  1851. if (!shadow_accessed_mask)
  1852. return;
  1853. sp = page_header(__pa(sptep));
  1854. if (sp->role.level > PT_PAGE_TABLE_LEVEL)
  1855. return;
  1856. __direct_pte_prefetch(vcpu, sp, sptep);
  1857. }
  1858. static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
  1859. int map_writable, int level, gfn_t gfn, pfn_t pfn)
  1860. {
  1861. struct kvm_shadow_walk_iterator iterator;
  1862. struct kvm_mmu_page *sp;
  1863. int pt_write = 0;
  1864. gfn_t pseudo_gfn;
  1865. for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
  1866. if (iterator.level == level) {
  1867. unsigned pte_access = ACC_ALL;
  1868. if (!map_writable)
  1869. pte_access &= ~ACC_WRITE_MASK;
  1870. mmu_set_spte(vcpu, iterator.sptep, ACC_ALL, pte_access,
  1871. 0, write, 1, &pt_write,
  1872. level, gfn, pfn, false, map_writable);
  1873. direct_pte_prefetch(vcpu, iterator.sptep);
  1874. ++vcpu->stat.pf_fixed;
  1875. break;
  1876. }
  1877. if (*iterator.sptep == shadow_trap_nonpresent_pte) {
  1878. u64 base_addr = iterator.addr;
  1879. base_addr &= PT64_LVL_ADDR_MASK(iterator.level);
  1880. pseudo_gfn = base_addr >> PAGE_SHIFT;
  1881. sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr,
  1882. iterator.level - 1,
  1883. 1, ACC_ALL, iterator.sptep);
  1884. if (!sp) {
  1885. pgprintk("nonpaging_map: ENOMEM\n");
  1886. kvm_release_pfn_clean(pfn);
  1887. return -ENOMEM;
  1888. }
  1889. __set_spte(iterator.sptep,
  1890. __pa(sp->spt)
  1891. | PT_PRESENT_MASK | PT_WRITABLE_MASK
  1892. | shadow_user_mask | shadow_x_mask
  1893. | shadow_accessed_mask);
  1894. }
  1895. }
  1896. return pt_write;
  1897. }
  1898. static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *tsk)
  1899. {
  1900. siginfo_t info;
  1901. info.si_signo = SIGBUS;
  1902. info.si_errno = 0;
  1903. info.si_code = BUS_MCEERR_AR;
  1904. info.si_addr = (void __user *)address;
  1905. info.si_addr_lsb = PAGE_SHIFT;
  1906. send_sig_info(SIGBUS, &info, tsk);
  1907. }
  1908. static int kvm_handle_bad_page(struct kvm *kvm, gfn_t gfn, pfn_t pfn)
  1909. {
  1910. kvm_release_pfn_clean(pfn);
  1911. if (is_hwpoison_pfn(pfn)) {
  1912. kvm_send_hwpoison_signal(gfn_to_hva(kvm, gfn), current);
  1913. return 0;
  1914. } else if (is_fault_pfn(pfn))
  1915. return -EFAULT;
  1916. return 1;
  1917. }
  1918. static bool try_async_pf(struct kvm_vcpu *vcpu, bool no_apf, gfn_t gfn,
  1919. gva_t gva, pfn_t *pfn, bool write, bool *writable);
  1920. static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn,
  1921. bool no_apf)
  1922. {
  1923. int r;
  1924. int level;
  1925. pfn_t pfn;
  1926. unsigned long mmu_seq;
  1927. bool map_writable;
  1928. level = mapping_level(vcpu, gfn);
  1929. /*
  1930. * This path builds a PAE pagetable - so we can map 2mb pages at
  1931. * maximum. Therefore check if the level is larger than that.
  1932. */
  1933. if (level > PT_DIRECTORY_LEVEL)
  1934. level = PT_DIRECTORY_LEVEL;
  1935. gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
  1936. mmu_seq = vcpu->kvm->mmu_notifier_seq;
  1937. smp_rmb();
  1938. if (try_async_pf(vcpu, no_apf, gfn, v, &pfn, write, &map_writable))
  1939. return 0;
  1940. /* mmio */
  1941. if (is_error_pfn(pfn))
  1942. return kvm_handle_bad_page(vcpu->kvm, gfn, pfn);
  1943. spin_lock(&vcpu->kvm->mmu_lock);
  1944. if (mmu_notifier_retry(vcpu, mmu_seq))
  1945. goto out_unlock;
  1946. kvm_mmu_free_some_pages(vcpu);
  1947. r = __direct_map(vcpu, v, write, map_writable, level, gfn, pfn);
  1948. spin_unlock(&vcpu->kvm->mmu_lock);
  1949. return r;
  1950. out_unlock:
  1951. spin_unlock(&vcpu->kvm->mmu_lock);
  1952. kvm_release_pfn_clean(pfn);
  1953. return 0;
  1954. }
  1955. static void mmu_free_roots(struct kvm_vcpu *vcpu)
  1956. {
  1957. int i;
  1958. struct kvm_mmu_page *sp;
  1959. LIST_HEAD(invalid_list);
  1960. if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
  1961. return;
  1962. spin_lock(&vcpu->kvm->mmu_lock);
  1963. if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL &&
  1964. (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL ||
  1965. vcpu->arch.mmu.direct_map)) {
  1966. hpa_t root = vcpu->arch.mmu.root_hpa;
  1967. sp = page_header(root);
  1968. --sp->root_count;
  1969. if (!sp->root_count && sp->role.invalid) {
  1970. kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
  1971. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  1972. }
  1973. vcpu->arch.mmu.root_hpa = INVALID_PAGE;
  1974. spin_unlock(&vcpu->kvm->mmu_lock);
  1975. return;
  1976. }
  1977. for (i = 0; i < 4; ++i) {
  1978. hpa_t root = vcpu->arch.mmu.pae_root[i];
  1979. if (root) {
  1980. root &= PT64_BASE_ADDR_MASK;
  1981. sp = page_header(root);
  1982. --sp->root_count;
  1983. if (!sp->root_count && sp->role.invalid)
  1984. kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
  1985. &invalid_list);
  1986. }
  1987. vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
  1988. }
  1989. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  1990. spin_unlock(&vcpu->kvm->mmu_lock);
  1991. vcpu->arch.mmu.root_hpa = INVALID_PAGE;
  1992. }
  1993. static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
  1994. {
  1995. int ret = 0;
  1996. if (!kvm_is_visible_gfn(vcpu->kvm, root_gfn)) {
  1997. kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
  1998. ret = 1;
  1999. }
  2000. return ret;
  2001. }
  2002. static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu)
  2003. {
  2004. struct kvm_mmu_page *sp;
  2005. unsigned i;
  2006. if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
  2007. spin_lock(&vcpu->kvm->mmu_lock);
  2008. kvm_mmu_free_some_pages(vcpu);
  2009. sp = kvm_mmu_get_page(vcpu, 0, 0, PT64_ROOT_LEVEL,
  2010. 1, ACC_ALL, NULL);
  2011. ++sp->root_count;
  2012. spin_unlock(&vcpu->kvm->mmu_lock);
  2013. vcpu->arch.mmu.root_hpa = __pa(sp->spt);
  2014. } else if (vcpu->arch.mmu.shadow_root_level == PT32E_ROOT_LEVEL) {
  2015. for (i = 0; i < 4; ++i) {
  2016. hpa_t root = vcpu->arch.mmu.pae_root[i];
  2017. ASSERT(!VALID_PAGE(root));
  2018. spin_lock(&vcpu->kvm->mmu_lock);
  2019. kvm_mmu_free_some_pages(vcpu);
  2020. sp = kvm_mmu_get_page(vcpu, i << (30 - PAGE_SHIFT),
  2021. i << 30,
  2022. PT32_ROOT_LEVEL, 1, ACC_ALL,
  2023. NULL);
  2024. root = __pa(sp->spt);
  2025. ++sp->root_count;
  2026. spin_unlock(&vcpu->kvm->mmu_lock);
  2027. vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
  2028. }
  2029. vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
  2030. } else
  2031. BUG();
  2032. return 0;
  2033. }
  2034. static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
  2035. {
  2036. struct kvm_mmu_page *sp;
  2037. u64 pdptr, pm_mask;
  2038. gfn_t root_gfn;
  2039. int i;
  2040. root_gfn = vcpu->arch.mmu.get_cr3(vcpu) >> PAGE_SHIFT;
  2041. if (mmu_check_root(vcpu, root_gfn))
  2042. return 1;
  2043. /*
  2044. * Do we shadow a long mode page table? If so we need to
  2045. * write-protect the guests page table root.
  2046. */
  2047. if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
  2048. hpa_t root = vcpu->arch.mmu.root_hpa;
  2049. ASSERT(!VALID_PAGE(root));
  2050. spin_lock(&vcpu->kvm->mmu_lock);
  2051. kvm_mmu_free_some_pages(vcpu);
  2052. sp = kvm_mmu_get_page(vcpu, root_gfn, 0, PT64_ROOT_LEVEL,
  2053. 0, ACC_ALL, NULL);
  2054. root = __pa(sp->spt);
  2055. ++sp->root_count;
  2056. spin_unlock(&vcpu->kvm->mmu_lock);
  2057. vcpu->arch.mmu.root_hpa = root;
  2058. return 0;
  2059. }
  2060. /*
  2061. * We shadow a 32 bit page table. This may be a legacy 2-level
  2062. * or a PAE 3-level page table. In either case we need to be aware that
  2063. * the shadow page table may be a PAE or a long mode page table.
  2064. */
  2065. pm_mask = PT_PRESENT_MASK;
  2066. if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL)
  2067. pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK;
  2068. for (i = 0; i < 4; ++i) {
  2069. hpa_t root = vcpu->arch.mmu.pae_root[i];
  2070. ASSERT(!VALID_PAGE(root));
  2071. if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
  2072. pdptr = kvm_pdptr_read_mmu(vcpu, &vcpu->arch.mmu, i);
  2073. if (!is_present_gpte(pdptr)) {
  2074. vcpu->arch.mmu.pae_root[i] = 0;
  2075. continue;
  2076. }
  2077. root_gfn = pdptr >> PAGE_SHIFT;
  2078. if (mmu_check_root(vcpu, root_gfn))
  2079. return 1;
  2080. }
  2081. spin_lock(&vcpu->kvm->mmu_lock);
  2082. kvm_mmu_free_some_pages(vcpu);
  2083. sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
  2084. PT32_ROOT_LEVEL, 0,
  2085. ACC_ALL, NULL);
  2086. root = __pa(sp->spt);
  2087. ++sp->root_count;
  2088. spin_unlock(&vcpu->kvm->mmu_lock);
  2089. vcpu->arch.mmu.pae_root[i] = root | pm_mask;
  2090. }
  2091. vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
  2092. /*
  2093. * If we shadow a 32 bit page table with a long mode page
  2094. * table we enter this path.
  2095. */
  2096. if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
  2097. if (vcpu->arch.mmu.lm_root == NULL) {
  2098. /*
  2099. * The additional page necessary for this is only
  2100. * allocated on demand.
  2101. */
  2102. u64 *lm_root;
  2103. lm_root = (void*)get_zeroed_page(GFP_KERNEL);
  2104. if (lm_root == NULL)
  2105. return 1;
  2106. lm_root[0] = __pa(vcpu->arch.mmu.pae_root) | pm_mask;
  2107. vcpu->arch.mmu.lm_root = lm_root;
  2108. }
  2109. vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.lm_root);
  2110. }
  2111. return 0;
  2112. }
  2113. static int mmu_alloc_roots(struct kvm_vcpu *vcpu)
  2114. {
  2115. if (vcpu->arch.mmu.direct_map)
  2116. return mmu_alloc_direct_roots(vcpu);
  2117. else
  2118. return mmu_alloc_shadow_roots(vcpu);
  2119. }
  2120. static void mmu_sync_roots(struct kvm_vcpu *vcpu)
  2121. {
  2122. int i;
  2123. struct kvm_mmu_page *sp;
  2124. if (vcpu->arch.mmu.direct_map)
  2125. return;
  2126. if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
  2127. return;
  2128. trace_kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);
  2129. if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
  2130. hpa_t root = vcpu->arch.mmu.root_hpa;
  2131. sp = page_header(root);
  2132. mmu_sync_children(vcpu, sp);
  2133. trace_kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
  2134. return;
  2135. }
  2136. for (i = 0; i < 4; ++i) {
  2137. hpa_t root = vcpu->arch.mmu.pae_root[i];
  2138. if (root && VALID_PAGE(root)) {
  2139. root &= PT64_BASE_ADDR_MASK;
  2140. sp = page_header(root);
  2141. mmu_sync_children(vcpu, sp);
  2142. }
  2143. }
  2144. trace_kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
  2145. }
  2146. void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
  2147. {
  2148. spin_lock(&vcpu->kvm->mmu_lock);
  2149. mmu_sync_roots(vcpu);
  2150. spin_unlock(&vcpu->kvm->mmu_lock);
  2151. }
  2152. static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr,
  2153. u32 access, u32 *error)
  2154. {
  2155. if (error)
  2156. *error = 0;
  2157. return vaddr;
  2158. }
  2159. static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gva_t vaddr,
  2160. u32 access, u32 *error)
  2161. {
  2162. if (error)
  2163. *error = 0;
  2164. return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access);
  2165. }
  2166. static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
  2167. u32 error_code, bool no_apf)
  2168. {
  2169. gfn_t gfn;
  2170. int r;
  2171. pgprintk("%s: gva %lx error %x\n", __func__, gva, error_code);
  2172. r = mmu_topup_memory_caches(vcpu);
  2173. if (r)
  2174. return r;
  2175. ASSERT(vcpu);
  2176. ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
  2177. gfn = gva >> PAGE_SHIFT;
  2178. return nonpaging_map(vcpu, gva & PAGE_MASK,
  2179. error_code & PFERR_WRITE_MASK, gfn, no_apf);
  2180. }
  2181. static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn)
  2182. {
  2183. struct kvm_arch_async_pf arch;
  2184. arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id;
  2185. arch.gfn = gfn;
  2186. arch.direct_map = vcpu->arch.mmu.direct_map;
  2187. return kvm_setup_async_pf(vcpu, gva, gfn, &arch);
  2188. }
  2189. static bool can_do_async_pf(struct kvm_vcpu *vcpu)
  2190. {
  2191. if (unlikely(!irqchip_in_kernel(vcpu->kvm) ||
  2192. kvm_event_needs_reinjection(vcpu)))
  2193. return false;
  2194. return kvm_x86_ops->interrupt_allowed(vcpu);
  2195. }
  2196. static bool try_async_pf(struct kvm_vcpu *vcpu, bool no_apf, gfn_t gfn,
  2197. gva_t gva, pfn_t *pfn, bool write, bool *writable)
  2198. {
  2199. bool async;
  2200. *pfn = gfn_to_pfn_async(vcpu->kvm, gfn, &async, write, writable);
  2201. if (!async)
  2202. return false; /* *pfn has correct page already */
  2203. put_page(pfn_to_page(*pfn));
  2204. if (!no_apf && can_do_async_pf(vcpu)) {
  2205. trace_kvm_try_async_get_page(gva, gfn);
  2206. if (kvm_find_async_pf_gfn(vcpu, gfn)) {
  2207. trace_kvm_async_pf_doublefault(gva, gfn);
  2208. kvm_make_request(KVM_REQ_APF_HALT, vcpu);
  2209. return true;
  2210. } else if (kvm_arch_setup_async_pf(vcpu, gva, gfn))
  2211. return true;
  2212. }
  2213. *pfn = gfn_to_pfn_prot(vcpu->kvm, gfn, write, writable);
  2214. return false;
  2215. }
  2216. static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, u32 error_code,
  2217. bool no_apf)
  2218. {
  2219. pfn_t pfn;
  2220. int r;
  2221. int level;
  2222. gfn_t gfn = gpa >> PAGE_SHIFT;
  2223. unsigned long mmu_seq;
  2224. int write = error_code & PFERR_WRITE_MASK;
  2225. bool map_writable;
  2226. ASSERT(vcpu);
  2227. ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
  2228. r = mmu_topup_memory_caches(vcpu);
  2229. if (r)
  2230. return r;
  2231. level = mapping_level(vcpu, gfn);
  2232. gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
  2233. mmu_seq = vcpu->kvm->mmu_notifier_seq;
  2234. smp_rmb();
  2235. if (try_async_pf(vcpu, no_apf, gfn, gpa, &pfn, write, &map_writable))
  2236. return 0;
  2237. /* mmio */
  2238. if (is_error_pfn(pfn))
  2239. return kvm_handle_bad_page(vcpu->kvm, gfn, pfn);
  2240. spin_lock(&vcpu->kvm->mmu_lock);
  2241. if (mmu_notifier_retry(vcpu, mmu_seq))
  2242. goto out_unlock;
  2243. kvm_mmu_free_some_pages(vcpu);
  2244. r = __direct_map(vcpu, gpa, write, map_writable,
  2245. level, gfn, pfn);
  2246. spin_unlock(&vcpu->kvm->mmu_lock);
  2247. return r;
  2248. out_unlock:
  2249. spin_unlock(&vcpu->kvm->mmu_lock);
  2250. kvm_release_pfn_clean(pfn);
  2251. return 0;
  2252. }
  2253. static void nonpaging_free(struct kvm_vcpu *vcpu)
  2254. {
  2255. mmu_free_roots(vcpu);
  2256. }
  2257. static int nonpaging_init_context(struct kvm_vcpu *vcpu,
  2258. struct kvm_mmu *context)
  2259. {
  2260. context->new_cr3 = nonpaging_new_cr3;
  2261. context->page_fault = nonpaging_page_fault;
  2262. context->gva_to_gpa = nonpaging_gva_to_gpa;
  2263. context->free = nonpaging_free;
  2264. context->prefetch_page = nonpaging_prefetch_page;
  2265. context->sync_page = nonpaging_sync_page;
  2266. context->invlpg = nonpaging_invlpg;
  2267. context->root_level = 0;
  2268. context->shadow_root_level = PT32E_ROOT_LEVEL;
  2269. context->root_hpa = INVALID_PAGE;
  2270. context->direct_map = true;
  2271. context->nx = false;
  2272. return 0;
  2273. }
  2274. void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
  2275. {
  2276. ++vcpu->stat.tlb_flush;
  2277. kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
  2278. }
  2279. static void paging_new_cr3(struct kvm_vcpu *vcpu)
  2280. {
  2281. pgprintk("%s: cr3 %lx\n", __func__, vcpu->arch.cr3);
  2282. mmu_free_roots(vcpu);
  2283. }
  2284. static unsigned long get_cr3(struct kvm_vcpu *vcpu)
  2285. {
  2286. return vcpu->arch.cr3;
  2287. }
  2288. static void inject_page_fault(struct kvm_vcpu *vcpu)
  2289. {
  2290. vcpu->arch.mmu.inject_page_fault(vcpu);
  2291. }
  2292. static void paging_free(struct kvm_vcpu *vcpu)
  2293. {
  2294. nonpaging_free(vcpu);
  2295. }
  2296. static bool is_rsvd_bits_set(struct kvm_mmu *mmu, u64 gpte, int level)
  2297. {
  2298. int bit7;
  2299. bit7 = (gpte >> 7) & 1;
  2300. return (gpte & mmu->rsvd_bits_mask[bit7][level-1]) != 0;
  2301. }
  2302. #define PTTYPE 64
  2303. #include "paging_tmpl.h"
  2304. #undef PTTYPE
  2305. #define PTTYPE 32
  2306. #include "paging_tmpl.h"
  2307. #undef PTTYPE
  2308. static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
  2309. struct kvm_mmu *context,
  2310. int level)
  2311. {
  2312. int maxphyaddr = cpuid_maxphyaddr(vcpu);
  2313. u64 exb_bit_rsvd = 0;
  2314. if (!context->nx)
  2315. exb_bit_rsvd = rsvd_bits(63, 63);
  2316. switch (level) {
  2317. case PT32_ROOT_LEVEL:
  2318. /* no rsvd bits for 2 level 4K page table entries */
  2319. context->rsvd_bits_mask[0][1] = 0;
  2320. context->rsvd_bits_mask[0][0] = 0;
  2321. context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
  2322. if (!is_pse(vcpu)) {
  2323. context->rsvd_bits_mask[1][1] = 0;
  2324. break;
  2325. }
  2326. if (is_cpuid_PSE36())
  2327. /* 36bits PSE 4MB page */
  2328. context->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
  2329. else
  2330. /* 32 bits PSE 4MB page */
  2331. context->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
  2332. break;
  2333. case PT32E_ROOT_LEVEL:
  2334. context->rsvd_bits_mask[0][2] =
  2335. rsvd_bits(maxphyaddr, 63) |
  2336. rsvd_bits(7, 8) | rsvd_bits(1, 2); /* PDPTE */
  2337. context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
  2338. rsvd_bits(maxphyaddr, 62); /* PDE */
  2339. context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
  2340. rsvd_bits(maxphyaddr, 62); /* PTE */
  2341. context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
  2342. rsvd_bits(maxphyaddr, 62) |
  2343. rsvd_bits(13, 20); /* large page */
  2344. context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
  2345. break;
  2346. case PT64_ROOT_LEVEL:
  2347. context->rsvd_bits_mask[0][3] = exb_bit_rsvd |
  2348. rsvd_bits(maxphyaddr, 51) | rsvd_bits(7, 8);
  2349. context->rsvd_bits_mask[0][2] = exb_bit_rsvd |
  2350. rsvd_bits(maxphyaddr, 51) | rsvd_bits(7, 8);
  2351. context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
  2352. rsvd_bits(maxphyaddr, 51);
  2353. context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
  2354. rsvd_bits(maxphyaddr, 51);
  2355. context->rsvd_bits_mask[1][3] = context->rsvd_bits_mask[0][3];
  2356. context->rsvd_bits_mask[1][2] = exb_bit_rsvd |
  2357. rsvd_bits(maxphyaddr, 51) |
  2358. rsvd_bits(13, 29);
  2359. context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
  2360. rsvd_bits(maxphyaddr, 51) |
  2361. rsvd_bits(13, 20); /* large page */
  2362. context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
  2363. break;
  2364. }
  2365. }
  2366. static int paging64_init_context_common(struct kvm_vcpu *vcpu,
  2367. struct kvm_mmu *context,
  2368. int level)
  2369. {
  2370. context->nx = is_nx(vcpu);
  2371. reset_rsvds_bits_mask(vcpu, context, level);
  2372. ASSERT(is_pae(vcpu));
  2373. context->new_cr3 = paging_new_cr3;
  2374. context->page_fault = paging64_page_fault;
  2375. context->gva_to_gpa = paging64_gva_to_gpa;
  2376. context->prefetch_page = paging64_prefetch_page;
  2377. context->sync_page = paging64_sync_page;
  2378. context->invlpg = paging64_invlpg;
  2379. context->free = paging_free;
  2380. context->root_level = level;
  2381. context->shadow_root_level = level;
  2382. context->root_hpa = INVALID_PAGE;
  2383. context->direct_map = false;
  2384. return 0;
  2385. }
  2386. static int paging64_init_context(struct kvm_vcpu *vcpu,
  2387. struct kvm_mmu *context)
  2388. {
  2389. return paging64_init_context_common(vcpu, context, PT64_ROOT_LEVEL);
  2390. }
  2391. static int paging32_init_context(struct kvm_vcpu *vcpu,
  2392. struct kvm_mmu *context)
  2393. {
  2394. context->nx = false;
  2395. reset_rsvds_bits_mask(vcpu, context, PT32_ROOT_LEVEL);
  2396. context->new_cr3 = paging_new_cr3;
  2397. context->page_fault = paging32_page_fault;
  2398. context->gva_to_gpa = paging32_gva_to_gpa;
  2399. context->free = paging_free;
  2400. context->prefetch_page = paging32_prefetch_page;
  2401. context->sync_page = paging32_sync_page;
  2402. context->invlpg = paging32_invlpg;
  2403. context->root_level = PT32_ROOT_LEVEL;
  2404. context->shadow_root_level = PT32E_ROOT_LEVEL;
  2405. context->root_hpa = INVALID_PAGE;
  2406. context->direct_map = false;
  2407. return 0;
  2408. }
  2409. static int paging32E_init_context(struct kvm_vcpu *vcpu,
  2410. struct kvm_mmu *context)
  2411. {
  2412. return paging64_init_context_common(vcpu, context, PT32E_ROOT_LEVEL);
  2413. }
  2414. static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
  2415. {
  2416. struct kvm_mmu *context = vcpu->arch.walk_mmu;
  2417. context->new_cr3 = nonpaging_new_cr3;
  2418. context->page_fault = tdp_page_fault;
  2419. context->free = nonpaging_free;
  2420. context->prefetch_page = nonpaging_prefetch_page;
  2421. context->sync_page = nonpaging_sync_page;
  2422. context->invlpg = nonpaging_invlpg;
  2423. context->shadow_root_level = kvm_x86_ops->get_tdp_level();
  2424. context->root_hpa = INVALID_PAGE;
  2425. context->direct_map = true;
  2426. context->set_cr3 = kvm_x86_ops->set_tdp_cr3;
  2427. context->get_cr3 = get_cr3;
  2428. context->inject_page_fault = kvm_inject_page_fault;
  2429. context->nx = is_nx(vcpu);
  2430. if (!is_paging(vcpu)) {
  2431. context->nx = false;
  2432. context->gva_to_gpa = nonpaging_gva_to_gpa;
  2433. context->root_level = 0;
  2434. } else if (is_long_mode(vcpu)) {
  2435. context->nx = is_nx(vcpu);
  2436. reset_rsvds_bits_mask(vcpu, context, PT64_ROOT_LEVEL);
  2437. context->gva_to_gpa = paging64_gva_to_gpa;
  2438. context->root_level = PT64_ROOT_LEVEL;
  2439. } else if (is_pae(vcpu)) {
  2440. context->nx = is_nx(vcpu);
  2441. reset_rsvds_bits_mask(vcpu, context, PT32E_ROOT_LEVEL);
  2442. context->gva_to_gpa = paging64_gva_to_gpa;
  2443. context->root_level = PT32E_ROOT_LEVEL;
  2444. } else {
  2445. context->nx = false;
  2446. reset_rsvds_bits_mask(vcpu, context, PT32_ROOT_LEVEL);
  2447. context->gva_to_gpa = paging32_gva_to_gpa;
  2448. context->root_level = PT32_ROOT_LEVEL;
  2449. }
  2450. return 0;
  2451. }
  2452. int kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *context)
  2453. {
  2454. int r;
  2455. ASSERT(vcpu);
  2456. ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
  2457. if (!is_paging(vcpu))
  2458. r = nonpaging_init_context(vcpu, context);
  2459. else if (is_long_mode(vcpu))
  2460. r = paging64_init_context(vcpu, context);
  2461. else if (is_pae(vcpu))
  2462. r = paging32E_init_context(vcpu, context);
  2463. else
  2464. r = paging32_init_context(vcpu, context);
  2465. vcpu->arch.mmu.base_role.cr4_pae = !!is_pae(vcpu);
  2466. vcpu->arch.mmu.base_role.cr0_wp = is_write_protection(vcpu);
  2467. return r;
  2468. }
  2469. EXPORT_SYMBOL_GPL(kvm_init_shadow_mmu);
  2470. static int init_kvm_softmmu(struct kvm_vcpu *vcpu)
  2471. {
  2472. int r = kvm_init_shadow_mmu(vcpu, vcpu->arch.walk_mmu);
  2473. vcpu->arch.walk_mmu->set_cr3 = kvm_x86_ops->set_cr3;
  2474. vcpu->arch.walk_mmu->get_cr3 = get_cr3;
  2475. vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault;
  2476. return r;
  2477. }
  2478. static int init_kvm_nested_mmu(struct kvm_vcpu *vcpu)
  2479. {
  2480. struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;
  2481. g_context->get_cr3 = get_cr3;
  2482. g_context->inject_page_fault = kvm_inject_page_fault;
  2483. /*
  2484. * Note that arch.mmu.gva_to_gpa translates l2_gva to l1_gpa. The
  2485. * translation of l2_gpa to l1_gpa addresses is done using the
  2486. * arch.nested_mmu.gva_to_gpa function. Basically the gva_to_gpa
  2487. * functions between mmu and nested_mmu are swapped.
  2488. */
  2489. if (!is_paging(vcpu)) {
  2490. g_context->nx = false;
  2491. g_context->root_level = 0;
  2492. g_context->gva_to_gpa = nonpaging_gva_to_gpa_nested;
  2493. } else if (is_long_mode(vcpu)) {
  2494. g_context->nx = is_nx(vcpu);
  2495. reset_rsvds_bits_mask(vcpu, g_context, PT64_ROOT_LEVEL);
  2496. g_context->root_level = PT64_ROOT_LEVEL;
  2497. g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
  2498. } else if (is_pae(vcpu)) {
  2499. g_context->nx = is_nx(vcpu);
  2500. reset_rsvds_bits_mask(vcpu, g_context, PT32E_ROOT_LEVEL);
  2501. g_context->root_level = PT32E_ROOT_LEVEL;
  2502. g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
  2503. } else {
  2504. g_context->nx = false;
  2505. reset_rsvds_bits_mask(vcpu, g_context, PT32_ROOT_LEVEL);
  2506. g_context->root_level = PT32_ROOT_LEVEL;
  2507. g_context->gva_to_gpa = paging32_gva_to_gpa_nested;
  2508. }
  2509. return 0;
  2510. }
  2511. static int init_kvm_mmu(struct kvm_vcpu *vcpu)
  2512. {
  2513. vcpu->arch.update_pte.pfn = bad_pfn;
  2514. if (mmu_is_nested(vcpu))
  2515. return init_kvm_nested_mmu(vcpu);
  2516. else if (tdp_enabled)
  2517. return init_kvm_tdp_mmu(vcpu);
  2518. else
  2519. return init_kvm_softmmu(vcpu);
  2520. }
  2521. static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
  2522. {
  2523. ASSERT(vcpu);
  2524. if (VALID_PAGE(vcpu->arch.mmu.root_hpa))
  2525. /* mmu.free() should set root_hpa = INVALID_PAGE */
  2526. vcpu->arch.mmu.free(vcpu);
  2527. }
  2528. int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
  2529. {
  2530. destroy_kvm_mmu(vcpu);
  2531. return init_kvm_mmu(vcpu);
  2532. }
  2533. EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
  2534. int kvm_mmu_load(struct kvm_vcpu *vcpu)
  2535. {
  2536. int r;
  2537. r = mmu_topup_memory_caches(vcpu);
  2538. if (r)
  2539. goto out;
  2540. r = mmu_alloc_roots(vcpu);
  2541. spin_lock(&vcpu->kvm->mmu_lock);
  2542. mmu_sync_roots(vcpu);
  2543. spin_unlock(&vcpu->kvm->mmu_lock);
  2544. if (r)
  2545. goto out;
  2546. /* set_cr3() should ensure TLB has been flushed */
  2547. vcpu->arch.mmu.set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
  2548. out:
  2549. return r;
  2550. }
  2551. EXPORT_SYMBOL_GPL(kvm_mmu_load);
  2552. void kvm_mmu_unload(struct kvm_vcpu *vcpu)
  2553. {
  2554. mmu_free_roots(vcpu);
  2555. }
  2556. EXPORT_SYMBOL_GPL(kvm_mmu_unload);
  2557. static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu,
  2558. struct kvm_mmu_page *sp,
  2559. u64 *spte)
  2560. {
  2561. u64 pte;
  2562. struct kvm_mmu_page *child;
  2563. pte = *spte;
  2564. if (is_shadow_present_pte(pte)) {
  2565. if (is_last_spte(pte, sp->role.level))
  2566. drop_spte(vcpu->kvm, spte, shadow_trap_nonpresent_pte);
  2567. else {
  2568. child = page_header(pte & PT64_BASE_ADDR_MASK);
  2569. mmu_page_remove_parent_pte(child, spte);
  2570. }
  2571. }
  2572. __set_spte(spte, shadow_trap_nonpresent_pte);
  2573. if (is_large_pte(pte))
  2574. --vcpu->kvm->stat.lpages;
  2575. }
  2576. static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
  2577. struct kvm_mmu_page *sp,
  2578. u64 *spte,
  2579. const void *new)
  2580. {
  2581. if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
  2582. ++vcpu->kvm->stat.mmu_pde_zapped;
  2583. return;
  2584. }
  2585. if (is_rsvd_bits_set(&vcpu->arch.mmu, *(u64 *)new, PT_PAGE_TABLE_LEVEL))
  2586. return;
  2587. ++vcpu->kvm->stat.mmu_pte_updated;
  2588. if (!sp->role.cr4_pae)
  2589. paging32_update_pte(vcpu, sp, spte, new);
  2590. else
  2591. paging64_update_pte(vcpu, sp, spte, new);
  2592. }
  2593. static bool need_remote_flush(u64 old, u64 new)
  2594. {
  2595. if (!is_shadow_present_pte(old))
  2596. return false;
  2597. if (!is_shadow_present_pte(new))
  2598. return true;
  2599. if ((old ^ new) & PT64_BASE_ADDR_MASK)
  2600. return true;
  2601. old ^= PT64_NX_MASK;
  2602. new ^= PT64_NX_MASK;
  2603. return (old & ~new & PT64_PERM_MASK) != 0;
  2604. }
  2605. static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, bool zap_page,
  2606. bool remote_flush, bool local_flush)
  2607. {
  2608. if (zap_page)
  2609. return;
  2610. if (remote_flush)
  2611. kvm_flush_remote_tlbs(vcpu->kvm);
  2612. else if (local_flush)
  2613. kvm_mmu_flush_tlb(vcpu);
  2614. }
  2615. static bool last_updated_pte_accessed(struct kvm_vcpu *vcpu)
  2616. {
  2617. u64 *spte = vcpu->arch.last_pte_updated;
  2618. return !!(spte && (*spte & shadow_accessed_mask));
  2619. }
  2620. static void mmu_guess_page_from_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
  2621. u64 gpte)
  2622. {
  2623. gfn_t gfn;
  2624. pfn_t pfn;
  2625. if (!is_present_gpte(gpte))
  2626. return;
  2627. gfn = (gpte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
  2628. vcpu->arch.update_pte.mmu_seq = vcpu->kvm->mmu_notifier_seq;
  2629. smp_rmb();
  2630. pfn = gfn_to_pfn(vcpu->kvm, gfn);
  2631. if (is_error_pfn(pfn)) {
  2632. kvm_release_pfn_clean(pfn);
  2633. return;
  2634. }
  2635. vcpu->arch.update_pte.gfn = gfn;
  2636. vcpu->arch.update_pte.pfn = pfn;
  2637. }
  2638. static void kvm_mmu_access_page(struct kvm_vcpu *vcpu, gfn_t gfn)
  2639. {
  2640. u64 *spte = vcpu->arch.last_pte_updated;
  2641. if (spte
  2642. && vcpu->arch.last_pte_gfn == gfn
  2643. && shadow_accessed_mask
  2644. && !(*spte & shadow_accessed_mask)
  2645. && is_shadow_present_pte(*spte))
  2646. set_bit(PT_ACCESSED_SHIFT, (unsigned long *)spte);
  2647. }
  2648. void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
  2649. const u8 *new, int bytes,
  2650. bool guest_initiated)
  2651. {
  2652. gfn_t gfn = gpa >> PAGE_SHIFT;
  2653. union kvm_mmu_page_role mask = { .word = 0 };
  2654. struct kvm_mmu_page *sp;
  2655. struct hlist_node *node;
  2656. LIST_HEAD(invalid_list);
  2657. u64 entry, gentry;
  2658. u64 *spte;
  2659. unsigned offset = offset_in_page(gpa);
  2660. unsigned pte_size;
  2661. unsigned page_offset;
  2662. unsigned misaligned;
  2663. unsigned quadrant;
  2664. int level;
  2665. int flooded = 0;
  2666. int npte;
  2667. int r;
  2668. int invlpg_counter;
  2669. bool remote_flush, local_flush, zap_page;
  2670. zap_page = remote_flush = local_flush = false;
  2671. pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);
  2672. invlpg_counter = atomic_read(&vcpu->kvm->arch.invlpg_counter);
  2673. /*
  2674. * Assume that the pte write on a page table of the same type
  2675. * as the current vcpu paging mode. This is nearly always true
  2676. * (might be false while changing modes). Note it is verified later
  2677. * by update_pte().
  2678. */
  2679. if ((is_pae(vcpu) && bytes == 4) || !new) {
  2680. /* Handle a 32-bit guest writing two halves of a 64-bit gpte */
  2681. if (is_pae(vcpu)) {
  2682. gpa &= ~(gpa_t)7;
  2683. bytes = 8;
  2684. }
  2685. r = kvm_read_guest(vcpu->kvm, gpa, &gentry, min(bytes, 8));
  2686. if (r)
  2687. gentry = 0;
  2688. new = (const u8 *)&gentry;
  2689. }
  2690. switch (bytes) {
  2691. case 4:
  2692. gentry = *(const u32 *)new;
  2693. break;
  2694. case 8:
  2695. gentry = *(const u64 *)new;
  2696. break;
  2697. default:
  2698. gentry = 0;
  2699. break;
  2700. }
  2701. mmu_guess_page_from_pte_write(vcpu, gpa, gentry);
  2702. spin_lock(&vcpu->kvm->mmu_lock);
  2703. if (atomic_read(&vcpu->kvm->arch.invlpg_counter) != invlpg_counter)
  2704. gentry = 0;
  2705. kvm_mmu_access_page(vcpu, gfn);
  2706. kvm_mmu_free_some_pages(vcpu);
  2707. ++vcpu->kvm->stat.mmu_pte_write;
  2708. trace_kvm_mmu_audit(vcpu, AUDIT_PRE_PTE_WRITE);
  2709. if (guest_initiated) {
  2710. if (gfn == vcpu->arch.last_pt_write_gfn
  2711. && !last_updated_pte_accessed(vcpu)) {
  2712. ++vcpu->arch.last_pt_write_count;
  2713. if (vcpu->arch.last_pt_write_count >= 3)
  2714. flooded = 1;
  2715. } else {
  2716. vcpu->arch.last_pt_write_gfn = gfn;
  2717. vcpu->arch.last_pt_write_count = 1;
  2718. vcpu->arch.last_pte_updated = NULL;
  2719. }
  2720. }
  2721. mask.cr0_wp = mask.cr4_pae = mask.nxe = 1;
  2722. for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn, node) {
  2723. pte_size = sp->role.cr4_pae ? 8 : 4;
  2724. misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
  2725. misaligned |= bytes < 4;
  2726. if (misaligned || flooded) {
  2727. /*
  2728. * Misaligned accesses are too much trouble to fix
  2729. * up; also, they usually indicate a page is not used
  2730. * as a page table.
  2731. *
  2732. * If we're seeing too many writes to a page,
  2733. * it may no longer be a page table, or we may be
  2734. * forking, in which case it is better to unmap the
  2735. * page.
  2736. */
  2737. pgprintk("misaligned: gpa %llx bytes %d role %x\n",
  2738. gpa, bytes, sp->role.word);
  2739. zap_page |= !!kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
  2740. &invalid_list);
  2741. ++vcpu->kvm->stat.mmu_flooded;
  2742. continue;
  2743. }
  2744. page_offset = offset;
  2745. level = sp->role.level;
  2746. npte = 1;
  2747. if (!sp->role.cr4_pae) {
  2748. page_offset <<= 1; /* 32->64 */
  2749. /*
  2750. * A 32-bit pde maps 4MB while the shadow pdes map
  2751. * only 2MB. So we need to double the offset again
  2752. * and zap two pdes instead of one.
  2753. */
  2754. if (level == PT32_ROOT_LEVEL) {
  2755. page_offset &= ~7; /* kill rounding error */
  2756. page_offset <<= 1;
  2757. npte = 2;
  2758. }
  2759. quadrant = page_offset >> PAGE_SHIFT;
  2760. page_offset &= ~PAGE_MASK;
  2761. if (quadrant != sp->role.quadrant)
  2762. continue;
  2763. }
  2764. local_flush = true;
  2765. spte = &sp->spt[page_offset / sizeof(*spte)];
  2766. while (npte--) {
  2767. entry = *spte;
  2768. mmu_pte_write_zap_pte(vcpu, sp, spte);
  2769. if (gentry &&
  2770. !((sp->role.word ^ vcpu->arch.mmu.base_role.word)
  2771. & mask.word))
  2772. mmu_pte_write_new_pte(vcpu, sp, spte, &gentry);
  2773. if (!remote_flush && need_remote_flush(entry, *spte))
  2774. remote_flush = true;
  2775. ++spte;
  2776. }
  2777. }
  2778. mmu_pte_write_flush_tlb(vcpu, zap_page, remote_flush, local_flush);
  2779. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  2780. trace_kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE);
  2781. spin_unlock(&vcpu->kvm->mmu_lock);
  2782. if (!is_error_pfn(vcpu->arch.update_pte.pfn)) {
  2783. kvm_release_pfn_clean(vcpu->arch.update_pte.pfn);
  2784. vcpu->arch.update_pte.pfn = bad_pfn;
  2785. }
  2786. }
  2787. int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
  2788. {
  2789. gpa_t gpa;
  2790. int r;
  2791. if (vcpu->arch.mmu.direct_map)
  2792. return 0;
  2793. gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL);
  2794. spin_lock(&vcpu->kvm->mmu_lock);
  2795. r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
  2796. spin_unlock(&vcpu->kvm->mmu_lock);
  2797. return r;
  2798. }
  2799. EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt);
  2800. void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
  2801. {
  2802. LIST_HEAD(invalid_list);
  2803. while (kvm_mmu_available_pages(vcpu->kvm) < KVM_REFILL_PAGES &&
  2804. !list_empty(&vcpu->kvm->arch.active_mmu_pages)) {
  2805. struct kvm_mmu_page *sp;
  2806. sp = container_of(vcpu->kvm->arch.active_mmu_pages.prev,
  2807. struct kvm_mmu_page, link);
  2808. kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
  2809. kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
  2810. ++vcpu->kvm->stat.mmu_recycled;
  2811. }
  2812. }
  2813. int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code)
  2814. {
  2815. int r;
  2816. enum emulation_result er;
  2817. r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code, false);
  2818. if (r < 0)
  2819. goto out;
  2820. if (!r) {
  2821. r = 1;
  2822. goto out;
  2823. }
  2824. r = mmu_topup_memory_caches(vcpu);
  2825. if (r)
  2826. goto out;
  2827. er = emulate_instruction(vcpu, cr2, error_code, 0);
  2828. switch (er) {
  2829. case EMULATE_DONE:
  2830. return 1;
  2831. case EMULATE_DO_MMIO:
  2832. ++vcpu->stat.mmio_exits;
  2833. /* fall through */
  2834. case EMULATE_FAIL:
  2835. return 0;
  2836. default:
  2837. BUG();
  2838. }
  2839. out:
  2840. return r;
  2841. }
  2842. EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
  2843. void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
  2844. {
  2845. vcpu->arch.mmu.invlpg(vcpu, gva);
  2846. kvm_mmu_flush_tlb(vcpu);
  2847. ++vcpu->stat.invlpg;
  2848. }
  2849. EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);
  2850. void kvm_enable_tdp(void)
  2851. {
  2852. tdp_enabled = true;
  2853. }
  2854. EXPORT_SYMBOL_GPL(kvm_enable_tdp);
  2855. void kvm_disable_tdp(void)
  2856. {
  2857. tdp_enabled = false;
  2858. }
  2859. EXPORT_SYMBOL_GPL(kvm_disable_tdp);
  2860. static void free_mmu_pages(struct kvm_vcpu *vcpu)
  2861. {
  2862. free_page((unsigned long)vcpu->arch.mmu.pae_root);
  2863. if (vcpu->arch.mmu.lm_root != NULL)
  2864. free_page((unsigned long)vcpu->arch.mmu.lm_root);
  2865. }
  2866. static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
  2867. {
  2868. struct page *page;
  2869. int i;
  2870. ASSERT(vcpu);
  2871. /*
  2872. * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
  2873. * Therefore we need to allocate shadow page tables in the first
  2874. * 4GB of memory, which happens to fit the DMA32 zone.
  2875. */
  2876. page = alloc_page(GFP_KERNEL | __GFP_DMA32);
  2877. if (!page)
  2878. return -ENOMEM;
  2879. vcpu->arch.mmu.pae_root = page_address(page);
  2880. for (i = 0; i < 4; ++i)
  2881. vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
  2882. return 0;
  2883. }
  2884. int kvm_mmu_create(struct kvm_vcpu *vcpu)
  2885. {
  2886. ASSERT(vcpu);
  2887. ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
  2888. return alloc_mmu_pages(vcpu);
  2889. }
  2890. int kvm_mmu_setup(struct kvm_vcpu *vcpu)
  2891. {
  2892. ASSERT(vcpu);
  2893. ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
  2894. return init_kvm_mmu(vcpu);
  2895. }
  2896. void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
  2897. {
  2898. struct kvm_mmu_page *sp;
  2899. list_for_each_entry(sp, &kvm->arch.active_mmu_pages, link) {
  2900. int i;
  2901. u64 *pt;
  2902. if (!test_bit(slot, sp->slot_bitmap))
  2903. continue;
  2904. pt = sp->spt;
  2905. for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
  2906. /* avoid RMW */
  2907. if (is_writable_pte(pt[i]))
  2908. pt[i] &= ~PT_WRITABLE_MASK;
  2909. }
  2910. kvm_flush_remote_tlbs(kvm);
  2911. }
  2912. void kvm_mmu_zap_all(struct kvm *kvm)
  2913. {
  2914. struct kvm_mmu_page *sp, *node;
  2915. LIST_HEAD(invalid_list);
  2916. spin_lock(&kvm->mmu_lock);
  2917. restart:
  2918. list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link)
  2919. if (kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list))
  2920. goto restart;
  2921. kvm_mmu_commit_zap_page(kvm, &invalid_list);
  2922. spin_unlock(&kvm->mmu_lock);
  2923. }
  2924. static int kvm_mmu_remove_some_alloc_mmu_pages(struct kvm *kvm,
  2925. struct list_head *invalid_list)
  2926. {
  2927. struct kvm_mmu_page *page;
  2928. page = container_of(kvm->arch.active_mmu_pages.prev,
  2929. struct kvm_mmu_page, link);
  2930. return kvm_mmu_prepare_zap_page(kvm, page, invalid_list);
  2931. }
  2932. static int mmu_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
  2933. {
  2934. struct kvm *kvm;
  2935. struct kvm *kvm_freed = NULL;
  2936. if (nr_to_scan == 0)
  2937. goto out;
  2938. spin_lock(&kvm_lock);
  2939. list_for_each_entry(kvm, &vm_list, vm_list) {
  2940. int idx, freed_pages;
  2941. LIST_HEAD(invalid_list);
  2942. idx = srcu_read_lock(&kvm->srcu);
  2943. spin_lock(&kvm->mmu_lock);
  2944. if (!kvm_freed && nr_to_scan > 0 &&
  2945. kvm->arch.n_used_mmu_pages > 0) {
  2946. freed_pages = kvm_mmu_remove_some_alloc_mmu_pages(kvm,
  2947. &invalid_list);
  2948. kvm_freed = kvm;
  2949. }
  2950. nr_to_scan--;
  2951. kvm_mmu_commit_zap_page(kvm, &invalid_list);
  2952. spin_unlock(&kvm->mmu_lock);
  2953. srcu_read_unlock(&kvm->srcu, idx);
  2954. }
  2955. if (kvm_freed)
  2956. list_move_tail(&kvm_freed->vm_list, &vm_list);
  2957. spin_unlock(&kvm_lock);
  2958. out:
  2959. return percpu_counter_read_positive(&kvm_total_used_mmu_pages);
  2960. }
  2961. static struct shrinker mmu_shrinker = {
  2962. .shrink = mmu_shrink,
  2963. .seeks = DEFAULT_SEEKS * 10,
  2964. };
  2965. static void mmu_destroy_caches(void)
  2966. {
  2967. if (pte_chain_cache)
  2968. kmem_cache_destroy(pte_chain_cache);
  2969. if (rmap_desc_cache)
  2970. kmem_cache_destroy(rmap_desc_cache);
  2971. if (mmu_page_header_cache)
  2972. kmem_cache_destroy(mmu_page_header_cache);
  2973. }
  2974. void kvm_mmu_module_exit(void)
  2975. {
  2976. mmu_destroy_caches();
  2977. percpu_counter_destroy(&kvm_total_used_mmu_pages);
  2978. unregister_shrinker(&mmu_shrinker);
  2979. }
  2980. int kvm_mmu_module_init(void)
  2981. {
  2982. pte_chain_cache = kmem_cache_create("kvm_pte_chain",
  2983. sizeof(struct kvm_pte_chain),
  2984. 0, 0, NULL);
  2985. if (!pte_chain_cache)
  2986. goto nomem;
  2987. rmap_desc_cache = kmem_cache_create("kvm_rmap_desc",
  2988. sizeof(struct kvm_rmap_desc),
  2989. 0, 0, NULL);
  2990. if (!rmap_desc_cache)
  2991. goto nomem;
  2992. mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
  2993. sizeof(struct kvm_mmu_page),
  2994. 0, 0, NULL);
  2995. if (!mmu_page_header_cache)
  2996. goto nomem;
  2997. if (percpu_counter_init(&kvm_total_used_mmu_pages, 0))
  2998. goto nomem;
  2999. register_shrinker(&mmu_shrinker);
  3000. return 0;
  3001. nomem:
  3002. mmu_destroy_caches();
  3003. return -ENOMEM;
  3004. }
  3005. /*
  3006. * Caculate mmu pages needed for kvm.
  3007. */
  3008. unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
  3009. {
  3010. int i;
  3011. unsigned int nr_mmu_pages;
  3012. unsigned int nr_pages = 0;
  3013. struct kvm_memslots *slots;
  3014. slots = kvm_memslots(kvm);
  3015. for (i = 0; i < slots->nmemslots; i++)
  3016. nr_pages += slots->memslots[i].npages;
  3017. nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
  3018. nr_mmu_pages = max(nr_mmu_pages,
  3019. (unsigned int) KVM_MIN_ALLOC_MMU_PAGES);
  3020. return nr_mmu_pages;
  3021. }
  3022. static void *pv_mmu_peek_buffer(struct kvm_pv_mmu_op_buffer *buffer,
  3023. unsigned len)
  3024. {
  3025. if (len > buffer->len)
  3026. return NULL;
  3027. return buffer->ptr;
  3028. }
  3029. static void *pv_mmu_read_buffer(struct kvm_pv_mmu_op_buffer *buffer,
  3030. unsigned len)
  3031. {
  3032. void *ret;
  3033. ret = pv_mmu_peek_buffer(buffer, len);
  3034. if (!ret)
  3035. return ret;
  3036. buffer->ptr += len;
  3037. buffer->len -= len;
  3038. buffer->processed += len;
  3039. return ret;
  3040. }
  3041. static int kvm_pv_mmu_write(struct kvm_vcpu *vcpu,
  3042. gpa_t addr, gpa_t value)
  3043. {
  3044. int bytes = 8;
  3045. int r;
  3046. if (!is_long_mode(vcpu) && !is_pae(vcpu))
  3047. bytes = 4;
  3048. r = mmu_topup_memory_caches(vcpu);
  3049. if (r)
  3050. return r;
  3051. if (!emulator_write_phys(vcpu, addr, &value, bytes))
  3052. return -EFAULT;
  3053. return 1;
  3054. }
  3055. static int kvm_pv_mmu_flush_tlb(struct kvm_vcpu *vcpu)
  3056. {
  3057. (void)kvm_set_cr3(vcpu, vcpu->arch.cr3);
  3058. return 1;
  3059. }
  3060. static int kvm_pv_mmu_release_pt(struct kvm_vcpu *vcpu, gpa_t addr)
  3061. {
  3062. spin_lock(&vcpu->kvm->mmu_lock);
  3063. mmu_unshadow(vcpu->kvm, addr >> PAGE_SHIFT);
  3064. spin_unlock(&vcpu->kvm->mmu_lock);
  3065. return 1;
  3066. }
  3067. static int kvm_pv_mmu_op_one(struct kvm_vcpu *vcpu,
  3068. struct kvm_pv_mmu_op_buffer *buffer)
  3069. {
  3070. struct kvm_mmu_op_header *header;
  3071. header = pv_mmu_peek_buffer(buffer, sizeof *header);
  3072. if (!header)
  3073. return 0;
  3074. switch (header->op) {
  3075. case KVM_MMU_OP_WRITE_PTE: {
  3076. struct kvm_mmu_op_write_pte *wpte;
  3077. wpte = pv_mmu_read_buffer(buffer, sizeof *wpte);
  3078. if (!wpte)
  3079. return 0;
  3080. return kvm_pv_mmu_write(vcpu, wpte->pte_phys,
  3081. wpte->pte_val);
  3082. }
  3083. case KVM_MMU_OP_FLUSH_TLB: {
  3084. struct kvm_mmu_op_flush_tlb *ftlb;
  3085. ftlb = pv_mmu_read_buffer(buffer, sizeof *ftlb);
  3086. if (!ftlb)
  3087. return 0;
  3088. return kvm_pv_mmu_flush_tlb(vcpu);
  3089. }
  3090. case KVM_MMU_OP_RELEASE_PT: {
  3091. struct kvm_mmu_op_release_pt *rpt;
  3092. rpt = pv_mmu_read_buffer(buffer, sizeof *rpt);
  3093. if (!rpt)
  3094. return 0;
  3095. return kvm_pv_mmu_release_pt(vcpu, rpt->pt_phys);
  3096. }
  3097. default: return 0;
  3098. }
  3099. }
  3100. int kvm_pv_mmu_op(struct kvm_vcpu *vcpu, unsigned long bytes,
  3101. gpa_t addr, unsigned long *ret)
  3102. {
  3103. int r;
  3104. struct kvm_pv_mmu_op_buffer *buffer = &vcpu->arch.mmu_op_buffer;
  3105. buffer->ptr = buffer->buf;
  3106. buffer->len = min_t(unsigned long, bytes, sizeof buffer->buf);
  3107. buffer->processed = 0;
  3108. r = kvm_read_guest(vcpu->kvm, addr, buffer->buf, buffer->len);
  3109. if (r)
  3110. goto out;
  3111. while (buffer->len) {
  3112. r = kvm_pv_mmu_op_one(vcpu, buffer);
  3113. if (r < 0)
  3114. goto out;
  3115. if (r == 0)
  3116. break;
  3117. }
  3118. r = 1;
  3119. out:
  3120. *ret = buffer->processed;
  3121. return r;
  3122. }
  3123. int kvm_mmu_get_spte_hierarchy(struct kvm_vcpu *vcpu, u64 addr, u64 sptes[4])
  3124. {
  3125. struct kvm_shadow_walk_iterator iterator;
  3126. int nr_sptes = 0;
  3127. spin_lock(&vcpu->kvm->mmu_lock);
  3128. for_each_shadow_entry(vcpu, addr, iterator) {
  3129. sptes[iterator.level-1] = *iterator.sptep;
  3130. nr_sptes++;
  3131. if (!is_shadow_present_pte(*iterator.sptep))
  3132. break;
  3133. }
  3134. spin_unlock(&vcpu->kvm->mmu_lock);
  3135. return nr_sptes;
  3136. }
  3137. EXPORT_SYMBOL_GPL(kvm_mmu_get_spte_hierarchy);
  3138. #ifdef CONFIG_KVM_MMU_AUDIT
  3139. #include "mmu_audit.c"
  3140. #else
  3141. static void mmu_audit_disable(void) { }
  3142. #endif
  3143. void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
  3144. {
  3145. ASSERT(vcpu);
  3146. destroy_kvm_mmu(vcpu);
  3147. free_mmu_pages(vcpu);
  3148. mmu_free_memory_caches(vcpu);
  3149. mmu_audit_disable();
  3150. }