x86.c 143 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * derived from drivers/kvm/kvm_main.c
  5. *
  6. * Copyright (C) 2006 Qumranet, Inc.
  7. * Copyright (C) 2008 Qumranet, Inc.
  8. * Copyright IBM Corporation, 2008
  9. *
  10. * Authors:
  11. * Avi Kivity <avi@qumranet.com>
  12. * Yaniv Kamay <yaniv@qumranet.com>
  13. * Amit Shah <amit.shah@qumranet.com>
  14. * Ben-Ami Yassour <benami@il.ibm.com>
  15. *
  16. * This work is licensed under the terms of the GNU GPL, version 2. See
  17. * the COPYING file in the top-level directory.
  18. *
  19. */
  20. #include <linux/kvm_host.h>
  21. #include "irq.h"
  22. #include "mmu.h"
  23. #include "i8254.h"
  24. #include "tss.h"
  25. #include "kvm_cache_regs.h"
  26. #include "x86.h"
  27. #include <linux/clocksource.h>
  28. #include <linux/interrupt.h>
  29. #include <linux/kvm.h>
  30. #include <linux/fs.h>
  31. #include <linux/vmalloc.h>
  32. #include <linux/module.h>
  33. #include <linux/mman.h>
  34. #include <linux/highmem.h>
  35. #include <linux/iommu.h>
  36. #include <linux/intel-iommu.h>
  37. #include <linux/cpufreq.h>
  38. #include <linux/user-return-notifier.h>
  39. #include <linux/srcu.h>
  40. #include <trace/events/kvm.h>
  41. #undef TRACE_INCLUDE_FILE
  42. #define CREATE_TRACE_POINTS
  43. #include "trace.h"
  44. #include <asm/debugreg.h>
  45. #include <asm/uaccess.h>
  46. #include <asm/msr.h>
  47. #include <asm/desc.h>
  48. #include <asm/mtrr.h>
  49. #include <asm/mce.h>
  50. #define MAX_IO_MSRS 256
  51. #define CR0_RESERVED_BITS \
  52. (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \
  53. | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \
  54. | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG))
  55. #define CR4_RESERVED_BITS \
  56. (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\
  57. | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE \
  58. | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR \
  59. | X86_CR4_OSXMMEXCPT | X86_CR4_VMXE))
  60. #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
  61. #define KVM_MAX_MCE_BANKS 32
  62. #define KVM_MCE_CAP_SUPPORTED MCG_CTL_P
  63. /* EFER defaults:
  64. * - enable syscall per default because its emulated by KVM
  65. * - enable LME and LMA per default on 64 bit KVM
  66. */
  67. #ifdef CONFIG_X86_64
  68. static u64 __read_mostly efer_reserved_bits = 0xfffffffffffffafeULL;
  69. #else
  70. static u64 __read_mostly efer_reserved_bits = 0xfffffffffffffffeULL;
  71. #endif
  72. #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
  73. #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  74. static void update_cr8_intercept(struct kvm_vcpu *vcpu);
  75. static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid,
  76. struct kvm_cpuid_entry2 __user *entries);
  77. struct kvm_x86_ops *kvm_x86_ops;
  78. EXPORT_SYMBOL_GPL(kvm_x86_ops);
  79. int ignore_msrs = 0;
  80. module_param_named(ignore_msrs, ignore_msrs, bool, S_IRUGO | S_IWUSR);
  81. #define KVM_NR_SHARED_MSRS 16
  82. struct kvm_shared_msrs_global {
  83. int nr;
  84. u32 msrs[KVM_NR_SHARED_MSRS];
  85. };
  86. struct kvm_shared_msrs {
  87. struct user_return_notifier urn;
  88. bool registered;
  89. struct kvm_shared_msr_values {
  90. u64 host;
  91. u64 curr;
  92. } values[KVM_NR_SHARED_MSRS];
  93. };
  94. static struct kvm_shared_msrs_global __read_mostly shared_msrs_global;
  95. static DEFINE_PER_CPU(struct kvm_shared_msrs, shared_msrs);
  96. struct kvm_stats_debugfs_item debugfs_entries[] = {
  97. { "pf_fixed", VCPU_STAT(pf_fixed) },
  98. { "pf_guest", VCPU_STAT(pf_guest) },
  99. { "tlb_flush", VCPU_STAT(tlb_flush) },
  100. { "invlpg", VCPU_STAT(invlpg) },
  101. { "exits", VCPU_STAT(exits) },
  102. { "io_exits", VCPU_STAT(io_exits) },
  103. { "mmio_exits", VCPU_STAT(mmio_exits) },
  104. { "signal_exits", VCPU_STAT(signal_exits) },
  105. { "irq_window", VCPU_STAT(irq_window_exits) },
  106. { "nmi_window", VCPU_STAT(nmi_window_exits) },
  107. { "halt_exits", VCPU_STAT(halt_exits) },
  108. { "halt_wakeup", VCPU_STAT(halt_wakeup) },
  109. { "hypercalls", VCPU_STAT(hypercalls) },
  110. { "request_irq", VCPU_STAT(request_irq_exits) },
  111. { "irq_exits", VCPU_STAT(irq_exits) },
  112. { "host_state_reload", VCPU_STAT(host_state_reload) },
  113. { "efer_reload", VCPU_STAT(efer_reload) },
  114. { "fpu_reload", VCPU_STAT(fpu_reload) },
  115. { "insn_emulation", VCPU_STAT(insn_emulation) },
  116. { "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) },
  117. { "irq_injections", VCPU_STAT(irq_injections) },
  118. { "nmi_injections", VCPU_STAT(nmi_injections) },
  119. { "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) },
  120. { "mmu_pte_write", VM_STAT(mmu_pte_write) },
  121. { "mmu_pte_updated", VM_STAT(mmu_pte_updated) },
  122. { "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) },
  123. { "mmu_flooded", VM_STAT(mmu_flooded) },
  124. { "mmu_recycled", VM_STAT(mmu_recycled) },
  125. { "mmu_cache_miss", VM_STAT(mmu_cache_miss) },
  126. { "mmu_unsync", VM_STAT(mmu_unsync) },
  127. { "remote_tlb_flush", VM_STAT(remote_tlb_flush) },
  128. { "largepages", VM_STAT(lpages) },
  129. { NULL }
  130. };
  131. static void kvm_on_user_return(struct user_return_notifier *urn)
  132. {
  133. unsigned slot;
  134. struct kvm_shared_msrs *locals
  135. = container_of(urn, struct kvm_shared_msrs, urn);
  136. struct kvm_shared_msr_values *values;
  137. for (slot = 0; slot < shared_msrs_global.nr; ++slot) {
  138. values = &locals->values[slot];
  139. if (values->host != values->curr) {
  140. wrmsrl(shared_msrs_global.msrs[slot], values->host);
  141. values->curr = values->host;
  142. }
  143. }
  144. locals->registered = false;
  145. user_return_notifier_unregister(urn);
  146. }
  147. static void shared_msr_update(unsigned slot, u32 msr)
  148. {
  149. struct kvm_shared_msrs *smsr;
  150. u64 value;
  151. smsr = &__get_cpu_var(shared_msrs);
  152. /* only read, and nobody should modify it at this time,
  153. * so don't need lock */
  154. if (slot >= shared_msrs_global.nr) {
  155. printk(KERN_ERR "kvm: invalid MSR slot!");
  156. return;
  157. }
  158. rdmsrl_safe(msr, &value);
  159. smsr->values[slot].host = value;
  160. smsr->values[slot].curr = value;
  161. }
  162. void kvm_define_shared_msr(unsigned slot, u32 msr)
  163. {
  164. if (slot >= shared_msrs_global.nr)
  165. shared_msrs_global.nr = slot + 1;
  166. shared_msrs_global.msrs[slot] = msr;
  167. /* we need ensured the shared_msr_global have been updated */
  168. smp_wmb();
  169. }
  170. EXPORT_SYMBOL_GPL(kvm_define_shared_msr);
  171. static void kvm_shared_msr_cpu_online(void)
  172. {
  173. unsigned i;
  174. for (i = 0; i < shared_msrs_global.nr; ++i)
  175. shared_msr_update(i, shared_msrs_global.msrs[i]);
  176. }
  177. void kvm_set_shared_msr(unsigned slot, u64 value, u64 mask)
  178. {
  179. struct kvm_shared_msrs *smsr = &__get_cpu_var(shared_msrs);
  180. if (((value ^ smsr->values[slot].curr) & mask) == 0)
  181. return;
  182. smsr->values[slot].curr = value;
  183. wrmsrl(shared_msrs_global.msrs[slot], value);
  184. if (!smsr->registered) {
  185. smsr->urn.on_user_return = kvm_on_user_return;
  186. user_return_notifier_register(&smsr->urn);
  187. smsr->registered = true;
  188. }
  189. }
  190. EXPORT_SYMBOL_GPL(kvm_set_shared_msr);
  191. static void drop_user_return_notifiers(void *ignore)
  192. {
  193. struct kvm_shared_msrs *smsr = &__get_cpu_var(shared_msrs);
  194. if (smsr->registered)
  195. kvm_on_user_return(&smsr->urn);
  196. }
  197. unsigned long segment_base(u16 selector)
  198. {
  199. struct descriptor_table gdt;
  200. struct desc_struct *d;
  201. unsigned long table_base;
  202. unsigned long v;
  203. if (selector == 0)
  204. return 0;
  205. kvm_get_gdt(&gdt);
  206. table_base = gdt.base;
  207. if (selector & 4) { /* from ldt */
  208. u16 ldt_selector = kvm_read_ldt();
  209. table_base = segment_base(ldt_selector);
  210. }
  211. d = (struct desc_struct *)(table_base + (selector & ~7));
  212. v = get_desc_base(d);
  213. #ifdef CONFIG_X86_64
  214. if (d->s == 0 && (d->type == 2 || d->type == 9 || d->type == 11))
  215. v |= ((unsigned long)((struct ldttss_desc64 *)d)->base3) << 32;
  216. #endif
  217. return v;
  218. }
  219. EXPORT_SYMBOL_GPL(segment_base);
  220. u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
  221. {
  222. if (irqchip_in_kernel(vcpu->kvm))
  223. return vcpu->arch.apic_base;
  224. else
  225. return vcpu->arch.apic_base;
  226. }
  227. EXPORT_SYMBOL_GPL(kvm_get_apic_base);
  228. void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data)
  229. {
  230. /* TODO: reserve bits check */
  231. if (irqchip_in_kernel(vcpu->kvm))
  232. kvm_lapic_set_base(vcpu, data);
  233. else
  234. vcpu->arch.apic_base = data;
  235. }
  236. EXPORT_SYMBOL_GPL(kvm_set_apic_base);
  237. #define EXCPT_BENIGN 0
  238. #define EXCPT_CONTRIBUTORY 1
  239. #define EXCPT_PF 2
  240. static int exception_class(int vector)
  241. {
  242. switch (vector) {
  243. case PF_VECTOR:
  244. return EXCPT_PF;
  245. case DE_VECTOR:
  246. case TS_VECTOR:
  247. case NP_VECTOR:
  248. case SS_VECTOR:
  249. case GP_VECTOR:
  250. return EXCPT_CONTRIBUTORY;
  251. default:
  252. break;
  253. }
  254. return EXCPT_BENIGN;
  255. }
  256. static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
  257. unsigned nr, bool has_error, u32 error_code)
  258. {
  259. u32 prev_nr;
  260. int class1, class2;
  261. if (!vcpu->arch.exception.pending) {
  262. queue:
  263. vcpu->arch.exception.pending = true;
  264. vcpu->arch.exception.has_error_code = has_error;
  265. vcpu->arch.exception.nr = nr;
  266. vcpu->arch.exception.error_code = error_code;
  267. return;
  268. }
  269. /* to check exception */
  270. prev_nr = vcpu->arch.exception.nr;
  271. if (prev_nr == DF_VECTOR) {
  272. /* triple fault -> shutdown */
  273. set_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests);
  274. return;
  275. }
  276. class1 = exception_class(prev_nr);
  277. class2 = exception_class(nr);
  278. if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY)
  279. || (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) {
  280. /* generate double fault per SDM Table 5-5 */
  281. vcpu->arch.exception.pending = true;
  282. vcpu->arch.exception.has_error_code = true;
  283. vcpu->arch.exception.nr = DF_VECTOR;
  284. vcpu->arch.exception.error_code = 0;
  285. } else
  286. /* replace previous exception with a new one in a hope
  287. that instruction re-execution will regenerate lost
  288. exception */
  289. goto queue;
  290. }
  291. void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
  292. {
  293. kvm_multiple_exception(vcpu, nr, false, 0);
  294. }
  295. EXPORT_SYMBOL_GPL(kvm_queue_exception);
  296. void kvm_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long addr,
  297. u32 error_code)
  298. {
  299. ++vcpu->stat.pf_guest;
  300. vcpu->arch.cr2 = addr;
  301. kvm_queue_exception_e(vcpu, PF_VECTOR, error_code);
  302. }
  303. void kvm_inject_nmi(struct kvm_vcpu *vcpu)
  304. {
  305. vcpu->arch.nmi_pending = 1;
  306. }
  307. EXPORT_SYMBOL_GPL(kvm_inject_nmi);
  308. void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
  309. {
  310. kvm_multiple_exception(vcpu, nr, true, error_code);
  311. }
  312. EXPORT_SYMBOL_GPL(kvm_queue_exception_e);
  313. /*
  314. * Checks if cpl <= required_cpl; if true, return true. Otherwise queue
  315. * a #GP and return false.
  316. */
  317. bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
  318. {
  319. if (kvm_x86_ops->get_cpl(vcpu) <= required_cpl)
  320. return true;
  321. kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
  322. return false;
  323. }
  324. EXPORT_SYMBOL_GPL(kvm_require_cpl);
  325. /*
  326. * Load the pae pdptrs. Return true is they are all valid.
  327. */
  328. int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
  329. {
  330. gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
  331. unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
  332. int i;
  333. int ret;
  334. u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)];
  335. ret = kvm_read_guest_page(vcpu->kvm, pdpt_gfn, pdpte,
  336. offset * sizeof(u64), sizeof(pdpte));
  337. if (ret < 0) {
  338. ret = 0;
  339. goto out;
  340. }
  341. for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
  342. if (is_present_gpte(pdpte[i]) &&
  343. (pdpte[i] & vcpu->arch.mmu.rsvd_bits_mask[0][2])) {
  344. ret = 0;
  345. goto out;
  346. }
  347. }
  348. ret = 1;
  349. memcpy(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs));
  350. __set_bit(VCPU_EXREG_PDPTR,
  351. (unsigned long *)&vcpu->arch.regs_avail);
  352. __set_bit(VCPU_EXREG_PDPTR,
  353. (unsigned long *)&vcpu->arch.regs_dirty);
  354. out:
  355. return ret;
  356. }
  357. EXPORT_SYMBOL_GPL(load_pdptrs);
  358. static bool pdptrs_changed(struct kvm_vcpu *vcpu)
  359. {
  360. u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)];
  361. bool changed = true;
  362. int r;
  363. if (is_long_mode(vcpu) || !is_pae(vcpu))
  364. return false;
  365. if (!test_bit(VCPU_EXREG_PDPTR,
  366. (unsigned long *)&vcpu->arch.regs_avail))
  367. return true;
  368. r = kvm_read_guest(vcpu->kvm, vcpu->arch.cr3 & ~31u, pdpte, sizeof(pdpte));
  369. if (r < 0)
  370. goto out;
  371. changed = memcmp(pdpte, vcpu->arch.pdptrs, sizeof(pdpte)) != 0;
  372. out:
  373. return changed;
  374. }
  375. void kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
  376. {
  377. cr0 |= X86_CR0_ET;
  378. #ifdef CONFIG_X86_64
  379. if (cr0 & 0xffffffff00000000UL) {
  380. printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
  381. cr0, kvm_read_cr0(vcpu));
  382. kvm_inject_gp(vcpu, 0);
  383. return;
  384. }
  385. #endif
  386. cr0 &= ~CR0_RESERVED_BITS;
  387. if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) {
  388. printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n");
  389. kvm_inject_gp(vcpu, 0);
  390. return;
  391. }
  392. if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) {
  393. printk(KERN_DEBUG "set_cr0: #GP, set PG flag "
  394. "and a clear PE flag\n");
  395. kvm_inject_gp(vcpu, 0);
  396. return;
  397. }
  398. if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
  399. #ifdef CONFIG_X86_64
  400. if ((vcpu->arch.efer & EFER_LME)) {
  401. int cs_db, cs_l;
  402. if (!is_pae(vcpu)) {
  403. printk(KERN_DEBUG "set_cr0: #GP, start paging "
  404. "in long mode while PAE is disabled\n");
  405. kvm_inject_gp(vcpu, 0);
  406. return;
  407. }
  408. kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
  409. if (cs_l) {
  410. printk(KERN_DEBUG "set_cr0: #GP, start paging "
  411. "in long mode while CS.L == 1\n");
  412. kvm_inject_gp(vcpu, 0);
  413. return;
  414. }
  415. } else
  416. #endif
  417. if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.cr3)) {
  418. printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
  419. "reserved bits\n");
  420. kvm_inject_gp(vcpu, 0);
  421. return;
  422. }
  423. }
  424. kvm_x86_ops->set_cr0(vcpu, cr0);
  425. vcpu->arch.cr0 = cr0;
  426. kvm_mmu_reset_context(vcpu);
  427. return;
  428. }
  429. EXPORT_SYMBOL_GPL(kvm_set_cr0);
  430. void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
  431. {
  432. kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0ful) | (msw & 0x0f));
  433. }
  434. EXPORT_SYMBOL_GPL(kvm_lmsw);
  435. void kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
  436. {
  437. unsigned long old_cr4 = kvm_read_cr4(vcpu);
  438. unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PAE;
  439. if (cr4 & CR4_RESERVED_BITS) {
  440. printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n");
  441. kvm_inject_gp(vcpu, 0);
  442. return;
  443. }
  444. if (is_long_mode(vcpu)) {
  445. if (!(cr4 & X86_CR4_PAE)) {
  446. printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while "
  447. "in long mode\n");
  448. kvm_inject_gp(vcpu, 0);
  449. return;
  450. }
  451. } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
  452. && ((cr4 ^ old_cr4) & pdptr_bits)
  453. && !load_pdptrs(vcpu, vcpu->arch.cr3)) {
  454. printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
  455. kvm_inject_gp(vcpu, 0);
  456. return;
  457. }
  458. if (cr4 & X86_CR4_VMXE) {
  459. printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n");
  460. kvm_inject_gp(vcpu, 0);
  461. return;
  462. }
  463. kvm_x86_ops->set_cr4(vcpu, cr4);
  464. vcpu->arch.cr4 = cr4;
  465. vcpu->arch.mmu.base_role.cr4_pge = (cr4 & X86_CR4_PGE) && !tdp_enabled;
  466. kvm_mmu_reset_context(vcpu);
  467. }
  468. EXPORT_SYMBOL_GPL(kvm_set_cr4);
  469. void kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
  470. {
  471. if (cr3 == vcpu->arch.cr3 && !pdptrs_changed(vcpu)) {
  472. kvm_mmu_sync_roots(vcpu);
  473. kvm_mmu_flush_tlb(vcpu);
  474. return;
  475. }
  476. if (is_long_mode(vcpu)) {
  477. if (cr3 & CR3_L_MODE_RESERVED_BITS) {
  478. printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
  479. kvm_inject_gp(vcpu, 0);
  480. return;
  481. }
  482. } else {
  483. if (is_pae(vcpu)) {
  484. if (cr3 & CR3_PAE_RESERVED_BITS) {
  485. printk(KERN_DEBUG
  486. "set_cr3: #GP, reserved bits\n");
  487. kvm_inject_gp(vcpu, 0);
  488. return;
  489. }
  490. if (is_paging(vcpu) && !load_pdptrs(vcpu, cr3)) {
  491. printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
  492. "reserved bits\n");
  493. kvm_inject_gp(vcpu, 0);
  494. return;
  495. }
  496. }
  497. /*
  498. * We don't check reserved bits in nonpae mode, because
  499. * this isn't enforced, and VMware depends on this.
  500. */
  501. }
  502. /*
  503. * Does the new cr3 value map to physical memory? (Note, we
  504. * catch an invalid cr3 even in real-mode, because it would
  505. * cause trouble later on when we turn on paging anyway.)
  506. *
  507. * A real CPU would silently accept an invalid cr3 and would
  508. * attempt to use it - with largely undefined (and often hard
  509. * to debug) behavior on the guest side.
  510. */
  511. if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
  512. kvm_inject_gp(vcpu, 0);
  513. else {
  514. vcpu->arch.cr3 = cr3;
  515. vcpu->arch.mmu.new_cr3(vcpu);
  516. }
  517. }
  518. EXPORT_SYMBOL_GPL(kvm_set_cr3);
  519. void kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
  520. {
  521. if (cr8 & CR8_RESERVED_BITS) {
  522. printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8);
  523. kvm_inject_gp(vcpu, 0);
  524. return;
  525. }
  526. if (irqchip_in_kernel(vcpu->kvm))
  527. kvm_lapic_set_tpr(vcpu, cr8);
  528. else
  529. vcpu->arch.cr8 = cr8;
  530. }
  531. EXPORT_SYMBOL_GPL(kvm_set_cr8);
  532. unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
  533. {
  534. if (irqchip_in_kernel(vcpu->kvm))
  535. return kvm_lapic_get_cr8(vcpu);
  536. else
  537. return vcpu->arch.cr8;
  538. }
  539. EXPORT_SYMBOL_GPL(kvm_get_cr8);
  540. static inline u32 bit(int bitno)
  541. {
  542. return 1 << (bitno & 31);
  543. }
  544. /*
  545. * List of msr numbers which we expose to userspace through KVM_GET_MSRS
  546. * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
  547. *
  548. * This list is modified at module load time to reflect the
  549. * capabilities of the host cpu. This capabilities test skips MSRs that are
  550. * kvm-specific. Those are put in the beginning of the list.
  551. */
  552. #define KVM_SAVE_MSRS_BEGIN 5
  553. static u32 msrs_to_save[] = {
  554. MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
  555. HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
  556. HV_X64_MSR_APIC_ASSIST_PAGE,
  557. MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
  558. MSR_K6_STAR,
  559. #ifdef CONFIG_X86_64
  560. MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
  561. #endif
  562. MSR_IA32_TSC, MSR_IA32_PERF_STATUS, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA
  563. };
  564. static unsigned num_msrs_to_save;
  565. static u32 emulated_msrs[] = {
  566. MSR_IA32_MISC_ENABLE,
  567. };
  568. static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
  569. {
  570. if (efer & efer_reserved_bits) {
  571. printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n",
  572. efer);
  573. kvm_inject_gp(vcpu, 0);
  574. return;
  575. }
  576. if (is_paging(vcpu)
  577. && (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME)) {
  578. printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n");
  579. kvm_inject_gp(vcpu, 0);
  580. return;
  581. }
  582. if (efer & EFER_FFXSR) {
  583. struct kvm_cpuid_entry2 *feat;
  584. feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
  585. if (!feat || !(feat->edx & bit(X86_FEATURE_FXSR_OPT))) {
  586. printk(KERN_DEBUG "set_efer: #GP, enable FFXSR w/o CPUID capability\n");
  587. kvm_inject_gp(vcpu, 0);
  588. return;
  589. }
  590. }
  591. if (efer & EFER_SVME) {
  592. struct kvm_cpuid_entry2 *feat;
  593. feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
  594. if (!feat || !(feat->ecx & bit(X86_FEATURE_SVM))) {
  595. printk(KERN_DEBUG "set_efer: #GP, enable SVM w/o SVM\n");
  596. kvm_inject_gp(vcpu, 0);
  597. return;
  598. }
  599. }
  600. kvm_x86_ops->set_efer(vcpu, efer);
  601. efer &= ~EFER_LMA;
  602. efer |= vcpu->arch.efer & EFER_LMA;
  603. vcpu->arch.efer = efer;
  604. vcpu->arch.mmu.base_role.nxe = (efer & EFER_NX) && !tdp_enabled;
  605. kvm_mmu_reset_context(vcpu);
  606. }
  607. void kvm_enable_efer_bits(u64 mask)
  608. {
  609. efer_reserved_bits &= ~mask;
  610. }
  611. EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);
  612. /*
  613. * Writes msr value into into the appropriate "register".
  614. * Returns 0 on success, non-0 otherwise.
  615. * Assumes vcpu_load() was already called.
  616. */
  617. int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
  618. {
  619. return kvm_x86_ops->set_msr(vcpu, msr_index, data);
  620. }
  621. /*
  622. * Adapt set_msr() to msr_io()'s calling convention
  623. */
  624. static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
  625. {
  626. return kvm_set_msr(vcpu, index, *data);
  627. }
  628. static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock)
  629. {
  630. static int version;
  631. struct pvclock_wall_clock wc;
  632. struct timespec boot;
  633. if (!wall_clock)
  634. return;
  635. version++;
  636. kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
  637. /*
  638. * The guest calculates current wall clock time by adding
  639. * system time (updated by kvm_write_guest_time below) to the
  640. * wall clock specified here. guest system time equals host
  641. * system time for us, thus we must fill in host boot time here.
  642. */
  643. getboottime(&boot);
  644. wc.sec = boot.tv_sec;
  645. wc.nsec = boot.tv_nsec;
  646. wc.version = version;
  647. kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));
  648. version++;
  649. kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
  650. }
  651. static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
  652. {
  653. uint32_t quotient, remainder;
  654. /* Don't try to replace with do_div(), this one calculates
  655. * "(dividend << 32) / divisor" */
  656. __asm__ ( "divl %4"
  657. : "=a" (quotient), "=d" (remainder)
  658. : "0" (0), "1" (dividend), "r" (divisor) );
  659. return quotient;
  660. }
  661. static void kvm_set_time_scale(uint32_t tsc_khz, struct pvclock_vcpu_time_info *hv_clock)
  662. {
  663. uint64_t nsecs = 1000000000LL;
  664. int32_t shift = 0;
  665. uint64_t tps64;
  666. uint32_t tps32;
  667. tps64 = tsc_khz * 1000LL;
  668. while (tps64 > nsecs*2) {
  669. tps64 >>= 1;
  670. shift--;
  671. }
  672. tps32 = (uint32_t)tps64;
  673. while (tps32 <= (uint32_t)nsecs) {
  674. tps32 <<= 1;
  675. shift++;
  676. }
  677. hv_clock->tsc_shift = shift;
  678. hv_clock->tsc_to_system_mul = div_frac(nsecs, tps32);
  679. pr_debug("%s: tsc_khz %u, tsc_shift %d, tsc_mul %u\n",
  680. __func__, tsc_khz, hv_clock->tsc_shift,
  681. hv_clock->tsc_to_system_mul);
  682. }
  683. static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
  684. static void kvm_write_guest_time(struct kvm_vcpu *v)
  685. {
  686. struct timespec ts;
  687. unsigned long flags;
  688. struct kvm_vcpu_arch *vcpu = &v->arch;
  689. void *shared_kaddr;
  690. unsigned long this_tsc_khz;
  691. if ((!vcpu->time_page))
  692. return;
  693. this_tsc_khz = get_cpu_var(cpu_tsc_khz);
  694. if (unlikely(vcpu->hv_clock_tsc_khz != this_tsc_khz)) {
  695. kvm_set_time_scale(this_tsc_khz, &vcpu->hv_clock);
  696. vcpu->hv_clock_tsc_khz = this_tsc_khz;
  697. }
  698. put_cpu_var(cpu_tsc_khz);
  699. /* Keep irq disabled to prevent changes to the clock */
  700. local_irq_save(flags);
  701. kvm_get_msr(v, MSR_IA32_TSC, &vcpu->hv_clock.tsc_timestamp);
  702. ktime_get_ts(&ts);
  703. monotonic_to_bootbased(&ts);
  704. local_irq_restore(flags);
  705. /* With all the info we got, fill in the values */
  706. vcpu->hv_clock.system_time = ts.tv_nsec +
  707. (NSEC_PER_SEC * (u64)ts.tv_sec) + v->kvm->arch.kvmclock_offset;
  708. /*
  709. * The interface expects us to write an even number signaling that the
  710. * update is finished. Since the guest won't see the intermediate
  711. * state, we just increase by 2 at the end.
  712. */
  713. vcpu->hv_clock.version += 2;
  714. shared_kaddr = kmap_atomic(vcpu->time_page, KM_USER0);
  715. memcpy(shared_kaddr + vcpu->time_offset, &vcpu->hv_clock,
  716. sizeof(vcpu->hv_clock));
  717. kunmap_atomic(shared_kaddr, KM_USER0);
  718. mark_page_dirty(v->kvm, vcpu->time >> PAGE_SHIFT);
  719. }
  720. static int kvm_request_guest_time_update(struct kvm_vcpu *v)
  721. {
  722. struct kvm_vcpu_arch *vcpu = &v->arch;
  723. if (!vcpu->time_page)
  724. return 0;
  725. set_bit(KVM_REQ_KVMCLOCK_UPDATE, &v->requests);
  726. return 1;
  727. }
  728. static bool msr_mtrr_valid(unsigned msr)
  729. {
  730. switch (msr) {
  731. case 0x200 ... 0x200 + 2 * KVM_NR_VAR_MTRR - 1:
  732. case MSR_MTRRfix64K_00000:
  733. case MSR_MTRRfix16K_80000:
  734. case MSR_MTRRfix16K_A0000:
  735. case MSR_MTRRfix4K_C0000:
  736. case MSR_MTRRfix4K_C8000:
  737. case MSR_MTRRfix4K_D0000:
  738. case MSR_MTRRfix4K_D8000:
  739. case MSR_MTRRfix4K_E0000:
  740. case MSR_MTRRfix4K_E8000:
  741. case MSR_MTRRfix4K_F0000:
  742. case MSR_MTRRfix4K_F8000:
  743. case MSR_MTRRdefType:
  744. case MSR_IA32_CR_PAT:
  745. return true;
  746. case 0x2f8:
  747. return true;
  748. }
  749. return false;
  750. }
  751. static bool valid_pat_type(unsigned t)
  752. {
  753. return t < 8 && (1 << t) & 0xf3; /* 0, 1, 4, 5, 6, 7 */
  754. }
  755. static bool valid_mtrr_type(unsigned t)
  756. {
  757. return t < 8 && (1 << t) & 0x73; /* 0, 1, 4, 5, 6 */
  758. }
  759. static bool mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  760. {
  761. int i;
  762. if (!msr_mtrr_valid(msr))
  763. return false;
  764. if (msr == MSR_IA32_CR_PAT) {
  765. for (i = 0; i < 8; i++)
  766. if (!valid_pat_type((data >> (i * 8)) & 0xff))
  767. return false;
  768. return true;
  769. } else if (msr == MSR_MTRRdefType) {
  770. if (data & ~0xcff)
  771. return false;
  772. return valid_mtrr_type(data & 0xff);
  773. } else if (msr >= MSR_MTRRfix64K_00000 && msr <= MSR_MTRRfix4K_F8000) {
  774. for (i = 0; i < 8 ; i++)
  775. if (!valid_mtrr_type((data >> (i * 8)) & 0xff))
  776. return false;
  777. return true;
  778. }
  779. /* variable MTRRs */
  780. return valid_mtrr_type(data & 0xff);
  781. }
  782. static int set_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  783. {
  784. u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges;
  785. if (!mtrr_valid(vcpu, msr, data))
  786. return 1;
  787. if (msr == MSR_MTRRdefType) {
  788. vcpu->arch.mtrr_state.def_type = data;
  789. vcpu->arch.mtrr_state.enabled = (data & 0xc00) >> 10;
  790. } else if (msr == MSR_MTRRfix64K_00000)
  791. p[0] = data;
  792. else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000)
  793. p[1 + msr - MSR_MTRRfix16K_80000] = data;
  794. else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000)
  795. p[3 + msr - MSR_MTRRfix4K_C0000] = data;
  796. else if (msr == MSR_IA32_CR_PAT)
  797. vcpu->arch.pat = data;
  798. else { /* Variable MTRRs */
  799. int idx, is_mtrr_mask;
  800. u64 *pt;
  801. idx = (msr - 0x200) / 2;
  802. is_mtrr_mask = msr - 0x200 - 2 * idx;
  803. if (!is_mtrr_mask)
  804. pt =
  805. (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo;
  806. else
  807. pt =
  808. (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo;
  809. *pt = data;
  810. }
  811. kvm_mmu_reset_context(vcpu);
  812. return 0;
  813. }
  814. static int set_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  815. {
  816. u64 mcg_cap = vcpu->arch.mcg_cap;
  817. unsigned bank_num = mcg_cap & 0xff;
  818. switch (msr) {
  819. case MSR_IA32_MCG_STATUS:
  820. vcpu->arch.mcg_status = data;
  821. break;
  822. case MSR_IA32_MCG_CTL:
  823. if (!(mcg_cap & MCG_CTL_P))
  824. return 1;
  825. if (data != 0 && data != ~(u64)0)
  826. return -1;
  827. vcpu->arch.mcg_ctl = data;
  828. break;
  829. default:
  830. if (msr >= MSR_IA32_MC0_CTL &&
  831. msr < MSR_IA32_MC0_CTL + 4 * bank_num) {
  832. u32 offset = msr - MSR_IA32_MC0_CTL;
  833. /* only 0 or all 1s can be written to IA32_MCi_CTL */
  834. if ((offset & 0x3) == 0 &&
  835. data != 0 && data != ~(u64)0)
  836. return -1;
  837. vcpu->arch.mce_banks[offset] = data;
  838. break;
  839. }
  840. return 1;
  841. }
  842. return 0;
  843. }
  844. static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data)
  845. {
  846. struct kvm *kvm = vcpu->kvm;
  847. int lm = is_long_mode(vcpu);
  848. u8 *blob_addr = lm ? (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_64
  849. : (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_32;
  850. u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
  851. : kvm->arch.xen_hvm_config.blob_size_32;
  852. u32 page_num = data & ~PAGE_MASK;
  853. u64 page_addr = data & PAGE_MASK;
  854. u8 *page;
  855. int r;
  856. r = -E2BIG;
  857. if (page_num >= blob_size)
  858. goto out;
  859. r = -ENOMEM;
  860. page = kzalloc(PAGE_SIZE, GFP_KERNEL);
  861. if (!page)
  862. goto out;
  863. r = -EFAULT;
  864. if (copy_from_user(page, blob_addr + (page_num * PAGE_SIZE), PAGE_SIZE))
  865. goto out_free;
  866. if (kvm_write_guest(kvm, page_addr, page, PAGE_SIZE))
  867. goto out_free;
  868. r = 0;
  869. out_free:
  870. kfree(page);
  871. out:
  872. return r;
  873. }
  874. static bool kvm_hv_hypercall_enabled(struct kvm *kvm)
  875. {
  876. return kvm->arch.hv_hypercall & HV_X64_MSR_HYPERCALL_ENABLE;
  877. }
  878. static bool kvm_hv_msr_partition_wide(u32 msr)
  879. {
  880. bool r = false;
  881. switch (msr) {
  882. case HV_X64_MSR_GUEST_OS_ID:
  883. case HV_X64_MSR_HYPERCALL:
  884. r = true;
  885. break;
  886. }
  887. return r;
  888. }
  889. static int set_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  890. {
  891. struct kvm *kvm = vcpu->kvm;
  892. switch (msr) {
  893. case HV_X64_MSR_GUEST_OS_ID:
  894. kvm->arch.hv_guest_os_id = data;
  895. /* setting guest os id to zero disables hypercall page */
  896. if (!kvm->arch.hv_guest_os_id)
  897. kvm->arch.hv_hypercall &= ~HV_X64_MSR_HYPERCALL_ENABLE;
  898. break;
  899. case HV_X64_MSR_HYPERCALL: {
  900. u64 gfn;
  901. unsigned long addr;
  902. u8 instructions[4];
  903. /* if guest os id is not set hypercall should remain disabled */
  904. if (!kvm->arch.hv_guest_os_id)
  905. break;
  906. if (!(data & HV_X64_MSR_HYPERCALL_ENABLE)) {
  907. kvm->arch.hv_hypercall = data;
  908. break;
  909. }
  910. gfn = data >> HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT;
  911. addr = gfn_to_hva(kvm, gfn);
  912. if (kvm_is_error_hva(addr))
  913. return 1;
  914. kvm_x86_ops->patch_hypercall(vcpu, instructions);
  915. ((unsigned char *)instructions)[3] = 0xc3; /* ret */
  916. if (copy_to_user((void __user *)addr, instructions, 4))
  917. return 1;
  918. kvm->arch.hv_hypercall = data;
  919. break;
  920. }
  921. default:
  922. pr_unimpl(vcpu, "HYPER-V unimplemented wrmsr: 0x%x "
  923. "data 0x%llx\n", msr, data);
  924. return 1;
  925. }
  926. return 0;
  927. }
  928. static int set_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  929. {
  930. switch (msr) {
  931. case HV_X64_MSR_APIC_ASSIST_PAGE: {
  932. unsigned long addr;
  933. if (!(data & HV_X64_MSR_APIC_ASSIST_PAGE_ENABLE)) {
  934. vcpu->arch.hv_vapic = data;
  935. break;
  936. }
  937. addr = gfn_to_hva(vcpu->kvm, data >>
  938. HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT);
  939. if (kvm_is_error_hva(addr))
  940. return 1;
  941. if (clear_user((void __user *)addr, PAGE_SIZE))
  942. return 1;
  943. vcpu->arch.hv_vapic = data;
  944. break;
  945. }
  946. case HV_X64_MSR_EOI:
  947. return kvm_hv_vapic_msr_write(vcpu, APIC_EOI, data);
  948. case HV_X64_MSR_ICR:
  949. return kvm_hv_vapic_msr_write(vcpu, APIC_ICR, data);
  950. case HV_X64_MSR_TPR:
  951. return kvm_hv_vapic_msr_write(vcpu, APIC_TASKPRI, data);
  952. default:
  953. pr_unimpl(vcpu, "HYPER-V unimplemented wrmsr: 0x%x "
  954. "data 0x%llx\n", msr, data);
  955. return 1;
  956. }
  957. return 0;
  958. }
  959. int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  960. {
  961. switch (msr) {
  962. case MSR_EFER:
  963. set_efer(vcpu, data);
  964. break;
  965. case MSR_K7_HWCR:
  966. data &= ~(u64)0x40; /* ignore flush filter disable */
  967. if (data != 0) {
  968. pr_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n",
  969. data);
  970. return 1;
  971. }
  972. break;
  973. case MSR_FAM10H_MMIO_CONF_BASE:
  974. if (data != 0) {
  975. pr_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: "
  976. "0x%llx\n", data);
  977. return 1;
  978. }
  979. break;
  980. case MSR_AMD64_NB_CFG:
  981. break;
  982. case MSR_IA32_DEBUGCTLMSR:
  983. if (!data) {
  984. /* We support the non-activated case already */
  985. break;
  986. } else if (data & ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_BTF)) {
  987. /* Values other than LBR and BTF are vendor-specific,
  988. thus reserved and should throw a #GP */
  989. return 1;
  990. }
  991. pr_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
  992. __func__, data);
  993. break;
  994. case MSR_IA32_UCODE_REV:
  995. case MSR_IA32_UCODE_WRITE:
  996. case MSR_VM_HSAVE_PA:
  997. case MSR_AMD64_PATCH_LOADER:
  998. break;
  999. case 0x200 ... 0x2ff:
  1000. return set_msr_mtrr(vcpu, msr, data);
  1001. case MSR_IA32_APICBASE:
  1002. kvm_set_apic_base(vcpu, data);
  1003. break;
  1004. case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
  1005. return kvm_x2apic_msr_write(vcpu, msr, data);
  1006. case MSR_IA32_MISC_ENABLE:
  1007. vcpu->arch.ia32_misc_enable_msr = data;
  1008. break;
  1009. case MSR_KVM_WALL_CLOCK:
  1010. vcpu->kvm->arch.wall_clock = data;
  1011. kvm_write_wall_clock(vcpu->kvm, data);
  1012. break;
  1013. case MSR_KVM_SYSTEM_TIME: {
  1014. if (vcpu->arch.time_page) {
  1015. kvm_release_page_dirty(vcpu->arch.time_page);
  1016. vcpu->arch.time_page = NULL;
  1017. }
  1018. vcpu->arch.time = data;
  1019. /* we verify if the enable bit is set... */
  1020. if (!(data & 1))
  1021. break;
  1022. /* ...but clean it before doing the actual write */
  1023. vcpu->arch.time_offset = data & ~(PAGE_MASK | 1);
  1024. vcpu->arch.time_page =
  1025. gfn_to_page(vcpu->kvm, data >> PAGE_SHIFT);
  1026. if (is_error_page(vcpu->arch.time_page)) {
  1027. kvm_release_page_clean(vcpu->arch.time_page);
  1028. vcpu->arch.time_page = NULL;
  1029. }
  1030. kvm_request_guest_time_update(vcpu);
  1031. break;
  1032. }
  1033. case MSR_IA32_MCG_CTL:
  1034. case MSR_IA32_MCG_STATUS:
  1035. case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
  1036. return set_msr_mce(vcpu, msr, data);
  1037. /* Performance counters are not protected by a CPUID bit,
  1038. * so we should check all of them in the generic path for the sake of
  1039. * cross vendor migration.
  1040. * Writing a zero into the event select MSRs disables them,
  1041. * which we perfectly emulate ;-). Any other value should be at least
  1042. * reported, some guests depend on them.
  1043. */
  1044. case MSR_P6_EVNTSEL0:
  1045. case MSR_P6_EVNTSEL1:
  1046. case MSR_K7_EVNTSEL0:
  1047. case MSR_K7_EVNTSEL1:
  1048. case MSR_K7_EVNTSEL2:
  1049. case MSR_K7_EVNTSEL3:
  1050. if (data != 0)
  1051. pr_unimpl(vcpu, "unimplemented perfctr wrmsr: "
  1052. "0x%x data 0x%llx\n", msr, data);
  1053. break;
  1054. /* at least RHEL 4 unconditionally writes to the perfctr registers,
  1055. * so we ignore writes to make it happy.
  1056. */
  1057. case MSR_P6_PERFCTR0:
  1058. case MSR_P6_PERFCTR1:
  1059. case MSR_K7_PERFCTR0:
  1060. case MSR_K7_PERFCTR1:
  1061. case MSR_K7_PERFCTR2:
  1062. case MSR_K7_PERFCTR3:
  1063. pr_unimpl(vcpu, "unimplemented perfctr wrmsr: "
  1064. "0x%x data 0x%llx\n", msr, data);
  1065. break;
  1066. case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
  1067. if (kvm_hv_msr_partition_wide(msr)) {
  1068. int r;
  1069. mutex_lock(&vcpu->kvm->lock);
  1070. r = set_msr_hyperv_pw(vcpu, msr, data);
  1071. mutex_unlock(&vcpu->kvm->lock);
  1072. return r;
  1073. } else
  1074. return set_msr_hyperv(vcpu, msr, data);
  1075. break;
  1076. default:
  1077. if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr))
  1078. return xen_hvm_config(vcpu, data);
  1079. if (!ignore_msrs) {
  1080. pr_unimpl(vcpu, "unhandled wrmsr: 0x%x data %llx\n",
  1081. msr, data);
  1082. return 1;
  1083. } else {
  1084. pr_unimpl(vcpu, "ignored wrmsr: 0x%x data %llx\n",
  1085. msr, data);
  1086. break;
  1087. }
  1088. }
  1089. return 0;
  1090. }
  1091. EXPORT_SYMBOL_GPL(kvm_set_msr_common);
  1092. /*
  1093. * Reads an msr value (of 'msr_index') into 'pdata'.
  1094. * Returns 0 on success, non-0 otherwise.
  1095. * Assumes vcpu_load() was already called.
  1096. */
  1097. int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
  1098. {
  1099. return kvm_x86_ops->get_msr(vcpu, msr_index, pdata);
  1100. }
  1101. static int get_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
  1102. {
  1103. u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges;
  1104. if (!msr_mtrr_valid(msr))
  1105. return 1;
  1106. if (msr == MSR_MTRRdefType)
  1107. *pdata = vcpu->arch.mtrr_state.def_type +
  1108. (vcpu->arch.mtrr_state.enabled << 10);
  1109. else if (msr == MSR_MTRRfix64K_00000)
  1110. *pdata = p[0];
  1111. else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000)
  1112. *pdata = p[1 + msr - MSR_MTRRfix16K_80000];
  1113. else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000)
  1114. *pdata = p[3 + msr - MSR_MTRRfix4K_C0000];
  1115. else if (msr == MSR_IA32_CR_PAT)
  1116. *pdata = vcpu->arch.pat;
  1117. else { /* Variable MTRRs */
  1118. int idx, is_mtrr_mask;
  1119. u64 *pt;
  1120. idx = (msr - 0x200) / 2;
  1121. is_mtrr_mask = msr - 0x200 - 2 * idx;
  1122. if (!is_mtrr_mask)
  1123. pt =
  1124. (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo;
  1125. else
  1126. pt =
  1127. (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo;
  1128. *pdata = *pt;
  1129. }
  1130. return 0;
  1131. }
  1132. static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
  1133. {
  1134. u64 data;
  1135. u64 mcg_cap = vcpu->arch.mcg_cap;
  1136. unsigned bank_num = mcg_cap & 0xff;
  1137. switch (msr) {
  1138. case MSR_IA32_P5_MC_ADDR:
  1139. case MSR_IA32_P5_MC_TYPE:
  1140. data = 0;
  1141. break;
  1142. case MSR_IA32_MCG_CAP:
  1143. data = vcpu->arch.mcg_cap;
  1144. break;
  1145. case MSR_IA32_MCG_CTL:
  1146. if (!(mcg_cap & MCG_CTL_P))
  1147. return 1;
  1148. data = vcpu->arch.mcg_ctl;
  1149. break;
  1150. case MSR_IA32_MCG_STATUS:
  1151. data = vcpu->arch.mcg_status;
  1152. break;
  1153. default:
  1154. if (msr >= MSR_IA32_MC0_CTL &&
  1155. msr < MSR_IA32_MC0_CTL + 4 * bank_num) {
  1156. u32 offset = msr - MSR_IA32_MC0_CTL;
  1157. data = vcpu->arch.mce_banks[offset];
  1158. break;
  1159. }
  1160. return 1;
  1161. }
  1162. *pdata = data;
  1163. return 0;
  1164. }
  1165. static int get_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
  1166. {
  1167. u64 data = 0;
  1168. struct kvm *kvm = vcpu->kvm;
  1169. switch (msr) {
  1170. case HV_X64_MSR_GUEST_OS_ID:
  1171. data = kvm->arch.hv_guest_os_id;
  1172. break;
  1173. case HV_X64_MSR_HYPERCALL:
  1174. data = kvm->arch.hv_hypercall;
  1175. break;
  1176. default:
  1177. pr_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr);
  1178. return 1;
  1179. }
  1180. *pdata = data;
  1181. return 0;
  1182. }
  1183. static int get_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
  1184. {
  1185. u64 data = 0;
  1186. switch (msr) {
  1187. case HV_X64_MSR_VP_INDEX: {
  1188. int r;
  1189. struct kvm_vcpu *v;
  1190. kvm_for_each_vcpu(r, v, vcpu->kvm)
  1191. if (v == vcpu)
  1192. data = r;
  1193. break;
  1194. }
  1195. case HV_X64_MSR_EOI:
  1196. return kvm_hv_vapic_msr_read(vcpu, APIC_EOI, pdata);
  1197. case HV_X64_MSR_ICR:
  1198. return kvm_hv_vapic_msr_read(vcpu, APIC_ICR, pdata);
  1199. case HV_X64_MSR_TPR:
  1200. return kvm_hv_vapic_msr_read(vcpu, APIC_TASKPRI, pdata);
  1201. default:
  1202. pr_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr);
  1203. return 1;
  1204. }
  1205. *pdata = data;
  1206. return 0;
  1207. }
  1208. int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
  1209. {
  1210. u64 data;
  1211. switch (msr) {
  1212. case MSR_IA32_PLATFORM_ID:
  1213. case MSR_IA32_UCODE_REV:
  1214. case MSR_IA32_EBL_CR_POWERON:
  1215. case MSR_IA32_DEBUGCTLMSR:
  1216. case MSR_IA32_LASTBRANCHFROMIP:
  1217. case MSR_IA32_LASTBRANCHTOIP:
  1218. case MSR_IA32_LASTINTFROMIP:
  1219. case MSR_IA32_LASTINTTOIP:
  1220. case MSR_K8_SYSCFG:
  1221. case MSR_K7_HWCR:
  1222. case MSR_VM_HSAVE_PA:
  1223. case MSR_P6_PERFCTR0:
  1224. case MSR_P6_PERFCTR1:
  1225. case MSR_P6_EVNTSEL0:
  1226. case MSR_P6_EVNTSEL1:
  1227. case MSR_K7_EVNTSEL0:
  1228. case MSR_K7_PERFCTR0:
  1229. case MSR_K8_INT_PENDING_MSG:
  1230. case MSR_AMD64_NB_CFG:
  1231. case MSR_FAM10H_MMIO_CONF_BASE:
  1232. data = 0;
  1233. break;
  1234. case MSR_MTRRcap:
  1235. data = 0x500 | KVM_NR_VAR_MTRR;
  1236. break;
  1237. case 0x200 ... 0x2ff:
  1238. return get_msr_mtrr(vcpu, msr, pdata);
  1239. case 0xcd: /* fsb frequency */
  1240. data = 3;
  1241. break;
  1242. case MSR_IA32_APICBASE:
  1243. data = kvm_get_apic_base(vcpu);
  1244. break;
  1245. case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
  1246. return kvm_x2apic_msr_read(vcpu, msr, pdata);
  1247. break;
  1248. case MSR_IA32_MISC_ENABLE:
  1249. data = vcpu->arch.ia32_misc_enable_msr;
  1250. break;
  1251. case MSR_IA32_PERF_STATUS:
  1252. /* TSC increment by tick */
  1253. data = 1000ULL;
  1254. /* CPU multiplier */
  1255. data |= (((uint64_t)4ULL) << 40);
  1256. break;
  1257. case MSR_EFER:
  1258. data = vcpu->arch.efer;
  1259. break;
  1260. case MSR_KVM_WALL_CLOCK:
  1261. data = vcpu->kvm->arch.wall_clock;
  1262. break;
  1263. case MSR_KVM_SYSTEM_TIME:
  1264. data = vcpu->arch.time;
  1265. break;
  1266. case MSR_IA32_P5_MC_ADDR:
  1267. case MSR_IA32_P5_MC_TYPE:
  1268. case MSR_IA32_MCG_CAP:
  1269. case MSR_IA32_MCG_CTL:
  1270. case MSR_IA32_MCG_STATUS:
  1271. case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
  1272. return get_msr_mce(vcpu, msr, pdata);
  1273. case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
  1274. if (kvm_hv_msr_partition_wide(msr)) {
  1275. int r;
  1276. mutex_lock(&vcpu->kvm->lock);
  1277. r = get_msr_hyperv_pw(vcpu, msr, pdata);
  1278. mutex_unlock(&vcpu->kvm->lock);
  1279. return r;
  1280. } else
  1281. return get_msr_hyperv(vcpu, msr, pdata);
  1282. break;
  1283. default:
  1284. if (!ignore_msrs) {
  1285. pr_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr);
  1286. return 1;
  1287. } else {
  1288. pr_unimpl(vcpu, "ignored rdmsr: 0x%x\n", msr);
  1289. data = 0;
  1290. }
  1291. break;
  1292. }
  1293. *pdata = data;
  1294. return 0;
  1295. }
  1296. EXPORT_SYMBOL_GPL(kvm_get_msr_common);
  1297. /*
  1298. * Read or write a bunch of msrs. All parameters are kernel addresses.
  1299. *
  1300. * @return number of msrs set successfully.
  1301. */
  1302. static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
  1303. struct kvm_msr_entry *entries,
  1304. int (*do_msr)(struct kvm_vcpu *vcpu,
  1305. unsigned index, u64 *data))
  1306. {
  1307. int i, idx;
  1308. vcpu_load(vcpu);
  1309. idx = srcu_read_lock(&vcpu->kvm->srcu);
  1310. for (i = 0; i < msrs->nmsrs; ++i)
  1311. if (do_msr(vcpu, entries[i].index, &entries[i].data))
  1312. break;
  1313. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  1314. vcpu_put(vcpu);
  1315. return i;
  1316. }
  1317. /*
  1318. * Read or write a bunch of msrs. Parameters are user addresses.
  1319. *
  1320. * @return number of msrs set successfully.
  1321. */
  1322. static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
  1323. int (*do_msr)(struct kvm_vcpu *vcpu,
  1324. unsigned index, u64 *data),
  1325. int writeback)
  1326. {
  1327. struct kvm_msrs msrs;
  1328. struct kvm_msr_entry *entries;
  1329. int r, n;
  1330. unsigned size;
  1331. r = -EFAULT;
  1332. if (copy_from_user(&msrs, user_msrs, sizeof msrs))
  1333. goto out;
  1334. r = -E2BIG;
  1335. if (msrs.nmsrs >= MAX_IO_MSRS)
  1336. goto out;
  1337. r = -ENOMEM;
  1338. size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
  1339. entries = vmalloc(size);
  1340. if (!entries)
  1341. goto out;
  1342. r = -EFAULT;
  1343. if (copy_from_user(entries, user_msrs->entries, size))
  1344. goto out_free;
  1345. r = n = __msr_io(vcpu, &msrs, entries, do_msr);
  1346. if (r < 0)
  1347. goto out_free;
  1348. r = -EFAULT;
  1349. if (writeback && copy_to_user(user_msrs->entries, entries, size))
  1350. goto out_free;
  1351. r = n;
  1352. out_free:
  1353. vfree(entries);
  1354. out:
  1355. return r;
  1356. }
  1357. int kvm_dev_ioctl_check_extension(long ext)
  1358. {
  1359. int r;
  1360. switch (ext) {
  1361. case KVM_CAP_IRQCHIP:
  1362. case KVM_CAP_HLT:
  1363. case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
  1364. case KVM_CAP_SET_TSS_ADDR:
  1365. case KVM_CAP_EXT_CPUID:
  1366. case KVM_CAP_CLOCKSOURCE:
  1367. case KVM_CAP_PIT:
  1368. case KVM_CAP_NOP_IO_DELAY:
  1369. case KVM_CAP_MP_STATE:
  1370. case KVM_CAP_SYNC_MMU:
  1371. case KVM_CAP_REINJECT_CONTROL:
  1372. case KVM_CAP_IRQ_INJECT_STATUS:
  1373. case KVM_CAP_ASSIGN_DEV_IRQ:
  1374. case KVM_CAP_IRQFD:
  1375. case KVM_CAP_IOEVENTFD:
  1376. case KVM_CAP_PIT2:
  1377. case KVM_CAP_PIT_STATE2:
  1378. case KVM_CAP_SET_IDENTITY_MAP_ADDR:
  1379. case KVM_CAP_XEN_HVM:
  1380. case KVM_CAP_ADJUST_CLOCK:
  1381. case KVM_CAP_VCPU_EVENTS:
  1382. case KVM_CAP_HYPERV:
  1383. case KVM_CAP_HYPERV_VAPIC:
  1384. case KVM_CAP_HYPERV_SPIN:
  1385. case KVM_CAP_PCI_SEGMENT:
  1386. case KVM_CAP_X86_ROBUST_SINGLESTEP:
  1387. r = 1;
  1388. break;
  1389. case KVM_CAP_COALESCED_MMIO:
  1390. r = KVM_COALESCED_MMIO_PAGE_OFFSET;
  1391. break;
  1392. case KVM_CAP_VAPIC:
  1393. r = !kvm_x86_ops->cpu_has_accelerated_tpr();
  1394. break;
  1395. case KVM_CAP_NR_VCPUS:
  1396. r = KVM_MAX_VCPUS;
  1397. break;
  1398. case KVM_CAP_NR_MEMSLOTS:
  1399. r = KVM_MEMORY_SLOTS;
  1400. break;
  1401. case KVM_CAP_PV_MMU: /* obsolete */
  1402. r = 0;
  1403. break;
  1404. case KVM_CAP_IOMMU:
  1405. r = iommu_found();
  1406. break;
  1407. case KVM_CAP_MCE:
  1408. r = KVM_MAX_MCE_BANKS;
  1409. break;
  1410. default:
  1411. r = 0;
  1412. break;
  1413. }
  1414. return r;
  1415. }
  1416. long kvm_arch_dev_ioctl(struct file *filp,
  1417. unsigned int ioctl, unsigned long arg)
  1418. {
  1419. void __user *argp = (void __user *)arg;
  1420. long r;
  1421. switch (ioctl) {
  1422. case KVM_GET_MSR_INDEX_LIST: {
  1423. struct kvm_msr_list __user *user_msr_list = argp;
  1424. struct kvm_msr_list msr_list;
  1425. unsigned n;
  1426. r = -EFAULT;
  1427. if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
  1428. goto out;
  1429. n = msr_list.nmsrs;
  1430. msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
  1431. if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
  1432. goto out;
  1433. r = -E2BIG;
  1434. if (n < msr_list.nmsrs)
  1435. goto out;
  1436. r = -EFAULT;
  1437. if (copy_to_user(user_msr_list->indices, &msrs_to_save,
  1438. num_msrs_to_save * sizeof(u32)))
  1439. goto out;
  1440. if (copy_to_user(user_msr_list->indices + num_msrs_to_save,
  1441. &emulated_msrs,
  1442. ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
  1443. goto out;
  1444. r = 0;
  1445. break;
  1446. }
  1447. case KVM_GET_SUPPORTED_CPUID: {
  1448. struct kvm_cpuid2 __user *cpuid_arg = argp;
  1449. struct kvm_cpuid2 cpuid;
  1450. r = -EFAULT;
  1451. if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
  1452. goto out;
  1453. r = kvm_dev_ioctl_get_supported_cpuid(&cpuid,
  1454. cpuid_arg->entries);
  1455. if (r)
  1456. goto out;
  1457. r = -EFAULT;
  1458. if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
  1459. goto out;
  1460. r = 0;
  1461. break;
  1462. }
  1463. case KVM_X86_GET_MCE_CAP_SUPPORTED: {
  1464. u64 mce_cap;
  1465. mce_cap = KVM_MCE_CAP_SUPPORTED;
  1466. r = -EFAULT;
  1467. if (copy_to_user(argp, &mce_cap, sizeof mce_cap))
  1468. goto out;
  1469. r = 0;
  1470. break;
  1471. }
  1472. default:
  1473. r = -EINVAL;
  1474. }
  1475. out:
  1476. return r;
  1477. }
  1478. void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
  1479. {
  1480. kvm_x86_ops->vcpu_load(vcpu, cpu);
  1481. if (unlikely(per_cpu(cpu_tsc_khz, cpu) == 0)) {
  1482. unsigned long khz = cpufreq_quick_get(cpu);
  1483. if (!khz)
  1484. khz = tsc_khz;
  1485. per_cpu(cpu_tsc_khz, cpu) = khz;
  1486. }
  1487. kvm_request_guest_time_update(vcpu);
  1488. }
  1489. void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
  1490. {
  1491. kvm_put_guest_fpu(vcpu);
  1492. kvm_x86_ops->vcpu_put(vcpu);
  1493. }
  1494. static int is_efer_nx(void)
  1495. {
  1496. unsigned long long efer = 0;
  1497. rdmsrl_safe(MSR_EFER, &efer);
  1498. return efer & EFER_NX;
  1499. }
  1500. static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu)
  1501. {
  1502. int i;
  1503. struct kvm_cpuid_entry2 *e, *entry;
  1504. entry = NULL;
  1505. for (i = 0; i < vcpu->arch.cpuid_nent; ++i) {
  1506. e = &vcpu->arch.cpuid_entries[i];
  1507. if (e->function == 0x80000001) {
  1508. entry = e;
  1509. break;
  1510. }
  1511. }
  1512. if (entry && (entry->edx & (1 << 20)) && !is_efer_nx()) {
  1513. entry->edx &= ~(1 << 20);
  1514. printk(KERN_INFO "kvm: guest NX capability removed\n");
  1515. }
  1516. }
  1517. /* when an old userspace process fills a new kernel module */
  1518. static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
  1519. struct kvm_cpuid *cpuid,
  1520. struct kvm_cpuid_entry __user *entries)
  1521. {
  1522. int r, i;
  1523. struct kvm_cpuid_entry *cpuid_entries;
  1524. r = -E2BIG;
  1525. if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
  1526. goto out;
  1527. r = -ENOMEM;
  1528. cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry) * cpuid->nent);
  1529. if (!cpuid_entries)
  1530. goto out;
  1531. r = -EFAULT;
  1532. if (copy_from_user(cpuid_entries, entries,
  1533. cpuid->nent * sizeof(struct kvm_cpuid_entry)))
  1534. goto out_free;
  1535. for (i = 0; i < cpuid->nent; i++) {
  1536. vcpu->arch.cpuid_entries[i].function = cpuid_entries[i].function;
  1537. vcpu->arch.cpuid_entries[i].eax = cpuid_entries[i].eax;
  1538. vcpu->arch.cpuid_entries[i].ebx = cpuid_entries[i].ebx;
  1539. vcpu->arch.cpuid_entries[i].ecx = cpuid_entries[i].ecx;
  1540. vcpu->arch.cpuid_entries[i].edx = cpuid_entries[i].edx;
  1541. vcpu->arch.cpuid_entries[i].index = 0;
  1542. vcpu->arch.cpuid_entries[i].flags = 0;
  1543. vcpu->arch.cpuid_entries[i].padding[0] = 0;
  1544. vcpu->arch.cpuid_entries[i].padding[1] = 0;
  1545. vcpu->arch.cpuid_entries[i].padding[2] = 0;
  1546. }
  1547. vcpu->arch.cpuid_nent = cpuid->nent;
  1548. cpuid_fix_nx_cap(vcpu);
  1549. r = 0;
  1550. kvm_apic_set_version(vcpu);
  1551. kvm_x86_ops->cpuid_update(vcpu);
  1552. out_free:
  1553. vfree(cpuid_entries);
  1554. out:
  1555. return r;
  1556. }
  1557. static int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu,
  1558. struct kvm_cpuid2 *cpuid,
  1559. struct kvm_cpuid_entry2 __user *entries)
  1560. {
  1561. int r;
  1562. r = -E2BIG;
  1563. if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
  1564. goto out;
  1565. r = -EFAULT;
  1566. if (copy_from_user(&vcpu->arch.cpuid_entries, entries,
  1567. cpuid->nent * sizeof(struct kvm_cpuid_entry2)))
  1568. goto out;
  1569. vcpu->arch.cpuid_nent = cpuid->nent;
  1570. kvm_apic_set_version(vcpu);
  1571. kvm_x86_ops->cpuid_update(vcpu);
  1572. return 0;
  1573. out:
  1574. return r;
  1575. }
  1576. static int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu,
  1577. struct kvm_cpuid2 *cpuid,
  1578. struct kvm_cpuid_entry2 __user *entries)
  1579. {
  1580. int r;
  1581. r = -E2BIG;
  1582. if (cpuid->nent < vcpu->arch.cpuid_nent)
  1583. goto out;
  1584. r = -EFAULT;
  1585. if (copy_to_user(entries, &vcpu->arch.cpuid_entries,
  1586. vcpu->arch.cpuid_nent * sizeof(struct kvm_cpuid_entry2)))
  1587. goto out;
  1588. return 0;
  1589. out:
  1590. cpuid->nent = vcpu->arch.cpuid_nent;
  1591. return r;
  1592. }
  1593. static void do_cpuid_1_ent(struct kvm_cpuid_entry2 *entry, u32 function,
  1594. u32 index)
  1595. {
  1596. entry->function = function;
  1597. entry->index = index;
  1598. cpuid_count(entry->function, entry->index,
  1599. &entry->eax, &entry->ebx, &entry->ecx, &entry->edx);
  1600. entry->flags = 0;
  1601. }
  1602. #define F(x) bit(X86_FEATURE_##x)
  1603. static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
  1604. u32 index, int *nent, int maxnent)
  1605. {
  1606. unsigned f_nx = is_efer_nx() ? F(NX) : 0;
  1607. #ifdef CONFIG_X86_64
  1608. unsigned f_gbpages = (kvm_x86_ops->get_lpage_level() == PT_PDPE_LEVEL)
  1609. ? F(GBPAGES) : 0;
  1610. unsigned f_lm = F(LM);
  1611. #else
  1612. unsigned f_gbpages = 0;
  1613. unsigned f_lm = 0;
  1614. #endif
  1615. unsigned f_rdtscp = kvm_x86_ops->rdtscp_supported() ? F(RDTSCP) : 0;
  1616. /* cpuid 1.edx */
  1617. const u32 kvm_supported_word0_x86_features =
  1618. F(FPU) | F(VME) | F(DE) | F(PSE) |
  1619. F(TSC) | F(MSR) | F(PAE) | F(MCE) |
  1620. F(CX8) | F(APIC) | 0 /* Reserved */ | F(SEP) |
  1621. F(MTRR) | F(PGE) | F(MCA) | F(CMOV) |
  1622. F(PAT) | F(PSE36) | 0 /* PSN */ | F(CLFLSH) |
  1623. 0 /* Reserved, DS, ACPI */ | F(MMX) |
  1624. F(FXSR) | F(XMM) | F(XMM2) | F(SELFSNOOP) |
  1625. 0 /* HTT, TM, Reserved, PBE */;
  1626. /* cpuid 0x80000001.edx */
  1627. const u32 kvm_supported_word1_x86_features =
  1628. F(FPU) | F(VME) | F(DE) | F(PSE) |
  1629. F(TSC) | F(MSR) | F(PAE) | F(MCE) |
  1630. F(CX8) | F(APIC) | 0 /* Reserved */ | F(SYSCALL) |
  1631. F(MTRR) | F(PGE) | F(MCA) | F(CMOV) |
  1632. F(PAT) | F(PSE36) | 0 /* Reserved */ |
  1633. f_nx | 0 /* Reserved */ | F(MMXEXT) | F(MMX) |
  1634. F(FXSR) | F(FXSR_OPT) | f_gbpages | f_rdtscp |
  1635. 0 /* Reserved */ | f_lm | F(3DNOWEXT) | F(3DNOW);
  1636. /* cpuid 1.ecx */
  1637. const u32 kvm_supported_word4_x86_features =
  1638. F(XMM3) | 0 /* Reserved, DTES64, MONITOR */ |
  1639. 0 /* DS-CPL, VMX, SMX, EST */ |
  1640. 0 /* TM2 */ | F(SSSE3) | 0 /* CNXT-ID */ | 0 /* Reserved */ |
  1641. 0 /* Reserved */ | F(CX16) | 0 /* xTPR Update, PDCM */ |
  1642. 0 /* Reserved, DCA */ | F(XMM4_1) |
  1643. F(XMM4_2) | F(X2APIC) | F(MOVBE) | F(POPCNT) |
  1644. 0 /* Reserved, XSAVE, OSXSAVE */;
  1645. /* cpuid 0x80000001.ecx */
  1646. const u32 kvm_supported_word6_x86_features =
  1647. F(LAHF_LM) | F(CMP_LEGACY) | F(SVM) | 0 /* ExtApicSpace */ |
  1648. F(CR8_LEGACY) | F(ABM) | F(SSE4A) | F(MISALIGNSSE) |
  1649. F(3DNOWPREFETCH) | 0 /* OSVW */ | 0 /* IBS */ | F(SSE5) |
  1650. 0 /* SKINIT */ | 0 /* WDT */;
  1651. /* all calls to cpuid_count() should be made on the same cpu */
  1652. get_cpu();
  1653. do_cpuid_1_ent(entry, function, index);
  1654. ++*nent;
  1655. switch (function) {
  1656. case 0:
  1657. entry->eax = min(entry->eax, (u32)0xb);
  1658. break;
  1659. case 1:
  1660. entry->edx &= kvm_supported_word0_x86_features;
  1661. entry->ecx &= kvm_supported_word4_x86_features;
  1662. /* we support x2apic emulation even if host does not support
  1663. * it since we emulate x2apic in software */
  1664. entry->ecx |= F(X2APIC);
  1665. break;
  1666. /* function 2 entries are STATEFUL. That is, repeated cpuid commands
  1667. * may return different values. This forces us to get_cpu() before
  1668. * issuing the first command, and also to emulate this annoying behavior
  1669. * in kvm_emulate_cpuid() using KVM_CPUID_FLAG_STATE_READ_NEXT */
  1670. case 2: {
  1671. int t, times = entry->eax & 0xff;
  1672. entry->flags |= KVM_CPUID_FLAG_STATEFUL_FUNC;
  1673. entry->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT;
  1674. for (t = 1; t < times && *nent < maxnent; ++t) {
  1675. do_cpuid_1_ent(&entry[t], function, 0);
  1676. entry[t].flags |= KVM_CPUID_FLAG_STATEFUL_FUNC;
  1677. ++*nent;
  1678. }
  1679. break;
  1680. }
  1681. /* function 4 and 0xb have additional index. */
  1682. case 4: {
  1683. int i, cache_type;
  1684. entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
  1685. /* read more entries until cache_type is zero */
  1686. for (i = 1; *nent < maxnent; ++i) {
  1687. cache_type = entry[i - 1].eax & 0x1f;
  1688. if (!cache_type)
  1689. break;
  1690. do_cpuid_1_ent(&entry[i], function, i);
  1691. entry[i].flags |=
  1692. KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
  1693. ++*nent;
  1694. }
  1695. break;
  1696. }
  1697. case 0xb: {
  1698. int i, level_type;
  1699. entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
  1700. /* read more entries until level_type is zero */
  1701. for (i = 1; *nent < maxnent; ++i) {
  1702. level_type = entry[i - 1].ecx & 0xff00;
  1703. if (!level_type)
  1704. break;
  1705. do_cpuid_1_ent(&entry[i], function, i);
  1706. entry[i].flags |=
  1707. KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
  1708. ++*nent;
  1709. }
  1710. break;
  1711. }
  1712. case 0x80000000:
  1713. entry->eax = min(entry->eax, 0x8000001a);
  1714. break;
  1715. case 0x80000001:
  1716. entry->edx &= kvm_supported_word1_x86_features;
  1717. entry->ecx &= kvm_supported_word6_x86_features;
  1718. break;
  1719. }
  1720. put_cpu();
  1721. }
  1722. #undef F
  1723. static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid,
  1724. struct kvm_cpuid_entry2 __user *entries)
  1725. {
  1726. struct kvm_cpuid_entry2 *cpuid_entries;
  1727. int limit, nent = 0, r = -E2BIG;
  1728. u32 func;
  1729. if (cpuid->nent < 1)
  1730. goto out;
  1731. if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
  1732. cpuid->nent = KVM_MAX_CPUID_ENTRIES;
  1733. r = -ENOMEM;
  1734. cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry2) * cpuid->nent);
  1735. if (!cpuid_entries)
  1736. goto out;
  1737. do_cpuid_ent(&cpuid_entries[0], 0, 0, &nent, cpuid->nent);
  1738. limit = cpuid_entries[0].eax;
  1739. for (func = 1; func <= limit && nent < cpuid->nent; ++func)
  1740. do_cpuid_ent(&cpuid_entries[nent], func, 0,
  1741. &nent, cpuid->nent);
  1742. r = -E2BIG;
  1743. if (nent >= cpuid->nent)
  1744. goto out_free;
  1745. do_cpuid_ent(&cpuid_entries[nent], 0x80000000, 0, &nent, cpuid->nent);
  1746. limit = cpuid_entries[nent - 1].eax;
  1747. for (func = 0x80000001; func <= limit && nent < cpuid->nent; ++func)
  1748. do_cpuid_ent(&cpuid_entries[nent], func, 0,
  1749. &nent, cpuid->nent);
  1750. r = -E2BIG;
  1751. if (nent >= cpuid->nent)
  1752. goto out_free;
  1753. r = -EFAULT;
  1754. if (copy_to_user(entries, cpuid_entries,
  1755. nent * sizeof(struct kvm_cpuid_entry2)))
  1756. goto out_free;
  1757. cpuid->nent = nent;
  1758. r = 0;
  1759. out_free:
  1760. vfree(cpuid_entries);
  1761. out:
  1762. return r;
  1763. }
  1764. static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
  1765. struct kvm_lapic_state *s)
  1766. {
  1767. vcpu_load(vcpu);
  1768. memcpy(s->regs, vcpu->arch.apic->regs, sizeof *s);
  1769. vcpu_put(vcpu);
  1770. return 0;
  1771. }
  1772. static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
  1773. struct kvm_lapic_state *s)
  1774. {
  1775. vcpu_load(vcpu);
  1776. memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s);
  1777. kvm_apic_post_state_restore(vcpu);
  1778. update_cr8_intercept(vcpu);
  1779. vcpu_put(vcpu);
  1780. return 0;
  1781. }
  1782. static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
  1783. struct kvm_interrupt *irq)
  1784. {
  1785. if (irq->irq < 0 || irq->irq >= 256)
  1786. return -EINVAL;
  1787. if (irqchip_in_kernel(vcpu->kvm))
  1788. return -ENXIO;
  1789. vcpu_load(vcpu);
  1790. kvm_queue_interrupt(vcpu, irq->irq, false);
  1791. vcpu_put(vcpu);
  1792. return 0;
  1793. }
  1794. static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)
  1795. {
  1796. vcpu_load(vcpu);
  1797. kvm_inject_nmi(vcpu);
  1798. vcpu_put(vcpu);
  1799. return 0;
  1800. }
  1801. static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
  1802. struct kvm_tpr_access_ctl *tac)
  1803. {
  1804. if (tac->flags)
  1805. return -EINVAL;
  1806. vcpu->arch.tpr_access_reporting = !!tac->enabled;
  1807. return 0;
  1808. }
  1809. static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
  1810. u64 mcg_cap)
  1811. {
  1812. int r;
  1813. unsigned bank_num = mcg_cap & 0xff, bank;
  1814. r = -EINVAL;
  1815. if (!bank_num || bank_num >= KVM_MAX_MCE_BANKS)
  1816. goto out;
  1817. if (mcg_cap & ~(KVM_MCE_CAP_SUPPORTED | 0xff | 0xff0000))
  1818. goto out;
  1819. r = 0;
  1820. vcpu->arch.mcg_cap = mcg_cap;
  1821. /* Init IA32_MCG_CTL to all 1s */
  1822. if (mcg_cap & MCG_CTL_P)
  1823. vcpu->arch.mcg_ctl = ~(u64)0;
  1824. /* Init IA32_MCi_CTL to all 1s */
  1825. for (bank = 0; bank < bank_num; bank++)
  1826. vcpu->arch.mce_banks[bank*4] = ~(u64)0;
  1827. out:
  1828. return r;
  1829. }
  1830. static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
  1831. struct kvm_x86_mce *mce)
  1832. {
  1833. u64 mcg_cap = vcpu->arch.mcg_cap;
  1834. unsigned bank_num = mcg_cap & 0xff;
  1835. u64 *banks = vcpu->arch.mce_banks;
  1836. if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))
  1837. return -EINVAL;
  1838. /*
  1839. * if IA32_MCG_CTL is not all 1s, the uncorrected error
  1840. * reporting is disabled
  1841. */
  1842. if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) &&
  1843. vcpu->arch.mcg_ctl != ~(u64)0)
  1844. return 0;
  1845. banks += 4 * mce->bank;
  1846. /*
  1847. * if IA32_MCi_CTL is not all 1s, the uncorrected error
  1848. * reporting is disabled for the bank
  1849. */
  1850. if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0)
  1851. return 0;
  1852. if (mce->status & MCI_STATUS_UC) {
  1853. if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
  1854. !kvm_read_cr4_bits(vcpu, X86_CR4_MCE)) {
  1855. printk(KERN_DEBUG "kvm: set_mce: "
  1856. "injects mce exception while "
  1857. "previous one is in progress!\n");
  1858. set_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests);
  1859. return 0;
  1860. }
  1861. if (banks[1] & MCI_STATUS_VAL)
  1862. mce->status |= MCI_STATUS_OVER;
  1863. banks[2] = mce->addr;
  1864. banks[3] = mce->misc;
  1865. vcpu->arch.mcg_status = mce->mcg_status;
  1866. banks[1] = mce->status;
  1867. kvm_queue_exception(vcpu, MC_VECTOR);
  1868. } else if (!(banks[1] & MCI_STATUS_VAL)
  1869. || !(banks[1] & MCI_STATUS_UC)) {
  1870. if (banks[1] & MCI_STATUS_VAL)
  1871. mce->status |= MCI_STATUS_OVER;
  1872. banks[2] = mce->addr;
  1873. banks[3] = mce->misc;
  1874. banks[1] = mce->status;
  1875. } else
  1876. banks[1] |= MCI_STATUS_OVER;
  1877. return 0;
  1878. }
  1879. static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
  1880. struct kvm_vcpu_events *events)
  1881. {
  1882. vcpu_load(vcpu);
  1883. events->exception.injected = vcpu->arch.exception.pending;
  1884. events->exception.nr = vcpu->arch.exception.nr;
  1885. events->exception.has_error_code = vcpu->arch.exception.has_error_code;
  1886. events->exception.error_code = vcpu->arch.exception.error_code;
  1887. events->interrupt.injected = vcpu->arch.interrupt.pending;
  1888. events->interrupt.nr = vcpu->arch.interrupt.nr;
  1889. events->interrupt.soft = vcpu->arch.interrupt.soft;
  1890. events->nmi.injected = vcpu->arch.nmi_injected;
  1891. events->nmi.pending = vcpu->arch.nmi_pending;
  1892. events->nmi.masked = kvm_x86_ops->get_nmi_mask(vcpu);
  1893. events->sipi_vector = vcpu->arch.sipi_vector;
  1894. events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING
  1895. | KVM_VCPUEVENT_VALID_SIPI_VECTOR);
  1896. vcpu_put(vcpu);
  1897. }
  1898. static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
  1899. struct kvm_vcpu_events *events)
  1900. {
  1901. if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING
  1902. | KVM_VCPUEVENT_VALID_SIPI_VECTOR))
  1903. return -EINVAL;
  1904. vcpu_load(vcpu);
  1905. vcpu->arch.exception.pending = events->exception.injected;
  1906. vcpu->arch.exception.nr = events->exception.nr;
  1907. vcpu->arch.exception.has_error_code = events->exception.has_error_code;
  1908. vcpu->arch.exception.error_code = events->exception.error_code;
  1909. vcpu->arch.interrupt.pending = events->interrupt.injected;
  1910. vcpu->arch.interrupt.nr = events->interrupt.nr;
  1911. vcpu->arch.interrupt.soft = events->interrupt.soft;
  1912. if (vcpu->arch.interrupt.pending && irqchip_in_kernel(vcpu->kvm))
  1913. kvm_pic_clear_isr_ack(vcpu->kvm);
  1914. vcpu->arch.nmi_injected = events->nmi.injected;
  1915. if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING)
  1916. vcpu->arch.nmi_pending = events->nmi.pending;
  1917. kvm_x86_ops->set_nmi_mask(vcpu, events->nmi.masked);
  1918. if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR)
  1919. vcpu->arch.sipi_vector = events->sipi_vector;
  1920. vcpu_put(vcpu);
  1921. return 0;
  1922. }
  1923. long kvm_arch_vcpu_ioctl(struct file *filp,
  1924. unsigned int ioctl, unsigned long arg)
  1925. {
  1926. struct kvm_vcpu *vcpu = filp->private_data;
  1927. void __user *argp = (void __user *)arg;
  1928. int r;
  1929. struct kvm_lapic_state *lapic = NULL;
  1930. switch (ioctl) {
  1931. case KVM_GET_LAPIC: {
  1932. r = -EINVAL;
  1933. if (!vcpu->arch.apic)
  1934. goto out;
  1935. lapic = kzalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL);
  1936. r = -ENOMEM;
  1937. if (!lapic)
  1938. goto out;
  1939. r = kvm_vcpu_ioctl_get_lapic(vcpu, lapic);
  1940. if (r)
  1941. goto out;
  1942. r = -EFAULT;
  1943. if (copy_to_user(argp, lapic, sizeof(struct kvm_lapic_state)))
  1944. goto out;
  1945. r = 0;
  1946. break;
  1947. }
  1948. case KVM_SET_LAPIC: {
  1949. r = -EINVAL;
  1950. if (!vcpu->arch.apic)
  1951. goto out;
  1952. lapic = kmalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL);
  1953. r = -ENOMEM;
  1954. if (!lapic)
  1955. goto out;
  1956. r = -EFAULT;
  1957. if (copy_from_user(lapic, argp, sizeof(struct kvm_lapic_state)))
  1958. goto out;
  1959. r = kvm_vcpu_ioctl_set_lapic(vcpu, lapic);
  1960. if (r)
  1961. goto out;
  1962. r = 0;
  1963. break;
  1964. }
  1965. case KVM_INTERRUPT: {
  1966. struct kvm_interrupt irq;
  1967. r = -EFAULT;
  1968. if (copy_from_user(&irq, argp, sizeof irq))
  1969. goto out;
  1970. r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
  1971. if (r)
  1972. goto out;
  1973. r = 0;
  1974. break;
  1975. }
  1976. case KVM_NMI: {
  1977. r = kvm_vcpu_ioctl_nmi(vcpu);
  1978. if (r)
  1979. goto out;
  1980. r = 0;
  1981. break;
  1982. }
  1983. case KVM_SET_CPUID: {
  1984. struct kvm_cpuid __user *cpuid_arg = argp;
  1985. struct kvm_cpuid cpuid;
  1986. r = -EFAULT;
  1987. if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
  1988. goto out;
  1989. r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
  1990. if (r)
  1991. goto out;
  1992. break;
  1993. }
  1994. case KVM_SET_CPUID2: {
  1995. struct kvm_cpuid2 __user *cpuid_arg = argp;
  1996. struct kvm_cpuid2 cpuid;
  1997. r = -EFAULT;
  1998. if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
  1999. goto out;
  2000. r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
  2001. cpuid_arg->entries);
  2002. if (r)
  2003. goto out;
  2004. break;
  2005. }
  2006. case KVM_GET_CPUID2: {
  2007. struct kvm_cpuid2 __user *cpuid_arg = argp;
  2008. struct kvm_cpuid2 cpuid;
  2009. r = -EFAULT;
  2010. if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
  2011. goto out;
  2012. r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
  2013. cpuid_arg->entries);
  2014. if (r)
  2015. goto out;
  2016. r = -EFAULT;
  2017. if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
  2018. goto out;
  2019. r = 0;
  2020. break;
  2021. }
  2022. case KVM_GET_MSRS:
  2023. r = msr_io(vcpu, argp, kvm_get_msr, 1);
  2024. break;
  2025. case KVM_SET_MSRS:
  2026. r = msr_io(vcpu, argp, do_set_msr, 0);
  2027. break;
  2028. case KVM_TPR_ACCESS_REPORTING: {
  2029. struct kvm_tpr_access_ctl tac;
  2030. r = -EFAULT;
  2031. if (copy_from_user(&tac, argp, sizeof tac))
  2032. goto out;
  2033. r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
  2034. if (r)
  2035. goto out;
  2036. r = -EFAULT;
  2037. if (copy_to_user(argp, &tac, sizeof tac))
  2038. goto out;
  2039. r = 0;
  2040. break;
  2041. };
  2042. case KVM_SET_VAPIC_ADDR: {
  2043. struct kvm_vapic_addr va;
  2044. r = -EINVAL;
  2045. if (!irqchip_in_kernel(vcpu->kvm))
  2046. goto out;
  2047. r = -EFAULT;
  2048. if (copy_from_user(&va, argp, sizeof va))
  2049. goto out;
  2050. r = 0;
  2051. kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
  2052. break;
  2053. }
  2054. case KVM_X86_SETUP_MCE: {
  2055. u64 mcg_cap;
  2056. r = -EFAULT;
  2057. if (copy_from_user(&mcg_cap, argp, sizeof mcg_cap))
  2058. goto out;
  2059. r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
  2060. break;
  2061. }
  2062. case KVM_X86_SET_MCE: {
  2063. struct kvm_x86_mce mce;
  2064. r = -EFAULT;
  2065. if (copy_from_user(&mce, argp, sizeof mce))
  2066. goto out;
  2067. r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
  2068. break;
  2069. }
  2070. case KVM_GET_VCPU_EVENTS: {
  2071. struct kvm_vcpu_events events;
  2072. kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);
  2073. r = -EFAULT;
  2074. if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events)))
  2075. break;
  2076. r = 0;
  2077. break;
  2078. }
  2079. case KVM_SET_VCPU_EVENTS: {
  2080. struct kvm_vcpu_events events;
  2081. r = -EFAULT;
  2082. if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events)))
  2083. break;
  2084. r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);
  2085. break;
  2086. }
  2087. default:
  2088. r = -EINVAL;
  2089. }
  2090. out:
  2091. kfree(lapic);
  2092. return r;
  2093. }
  2094. static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
  2095. {
  2096. int ret;
  2097. if (addr > (unsigned int)(-3 * PAGE_SIZE))
  2098. return -1;
  2099. ret = kvm_x86_ops->set_tss_addr(kvm, addr);
  2100. return ret;
  2101. }
  2102. static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,
  2103. u64 ident_addr)
  2104. {
  2105. kvm->arch.ept_identity_map_addr = ident_addr;
  2106. return 0;
  2107. }
  2108. static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
  2109. u32 kvm_nr_mmu_pages)
  2110. {
  2111. if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
  2112. return -EINVAL;
  2113. mutex_lock(&kvm->slots_lock);
  2114. spin_lock(&kvm->mmu_lock);
  2115. kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
  2116. kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
  2117. spin_unlock(&kvm->mmu_lock);
  2118. mutex_unlock(&kvm->slots_lock);
  2119. return 0;
  2120. }
  2121. static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm)
  2122. {
  2123. return kvm->arch.n_alloc_mmu_pages;
  2124. }
  2125. gfn_t unalias_gfn_instantiation(struct kvm *kvm, gfn_t gfn)
  2126. {
  2127. int i;
  2128. struct kvm_mem_alias *alias;
  2129. struct kvm_mem_aliases *aliases;
  2130. aliases = rcu_dereference(kvm->arch.aliases);
  2131. for (i = 0; i < aliases->naliases; ++i) {
  2132. alias = &aliases->aliases[i];
  2133. if (alias->flags & KVM_ALIAS_INVALID)
  2134. continue;
  2135. if (gfn >= alias->base_gfn
  2136. && gfn < alias->base_gfn + alias->npages)
  2137. return alias->target_gfn + gfn - alias->base_gfn;
  2138. }
  2139. return gfn;
  2140. }
  2141. gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
  2142. {
  2143. int i;
  2144. struct kvm_mem_alias *alias;
  2145. struct kvm_mem_aliases *aliases;
  2146. aliases = rcu_dereference(kvm->arch.aliases);
  2147. for (i = 0; i < aliases->naliases; ++i) {
  2148. alias = &aliases->aliases[i];
  2149. if (gfn >= alias->base_gfn
  2150. && gfn < alias->base_gfn + alias->npages)
  2151. return alias->target_gfn + gfn - alias->base_gfn;
  2152. }
  2153. return gfn;
  2154. }
  2155. /*
  2156. * Set a new alias region. Aliases map a portion of physical memory into
  2157. * another portion. This is useful for memory windows, for example the PC
  2158. * VGA region.
  2159. */
  2160. static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm,
  2161. struct kvm_memory_alias *alias)
  2162. {
  2163. int r, n;
  2164. struct kvm_mem_alias *p;
  2165. struct kvm_mem_aliases *aliases, *old_aliases;
  2166. r = -EINVAL;
  2167. /* General sanity checks */
  2168. if (alias->memory_size & (PAGE_SIZE - 1))
  2169. goto out;
  2170. if (alias->guest_phys_addr & (PAGE_SIZE - 1))
  2171. goto out;
  2172. if (alias->slot >= KVM_ALIAS_SLOTS)
  2173. goto out;
  2174. if (alias->guest_phys_addr + alias->memory_size
  2175. < alias->guest_phys_addr)
  2176. goto out;
  2177. if (alias->target_phys_addr + alias->memory_size
  2178. < alias->target_phys_addr)
  2179. goto out;
  2180. r = -ENOMEM;
  2181. aliases = kzalloc(sizeof(struct kvm_mem_aliases), GFP_KERNEL);
  2182. if (!aliases)
  2183. goto out;
  2184. mutex_lock(&kvm->slots_lock);
  2185. /* invalidate any gfn reference in case of deletion/shrinking */
  2186. memcpy(aliases, kvm->arch.aliases, sizeof(struct kvm_mem_aliases));
  2187. aliases->aliases[alias->slot].flags |= KVM_ALIAS_INVALID;
  2188. old_aliases = kvm->arch.aliases;
  2189. rcu_assign_pointer(kvm->arch.aliases, aliases);
  2190. synchronize_srcu_expedited(&kvm->srcu);
  2191. kvm_mmu_zap_all(kvm);
  2192. kfree(old_aliases);
  2193. r = -ENOMEM;
  2194. aliases = kzalloc(sizeof(struct kvm_mem_aliases), GFP_KERNEL);
  2195. if (!aliases)
  2196. goto out_unlock;
  2197. memcpy(aliases, kvm->arch.aliases, sizeof(struct kvm_mem_aliases));
  2198. p = &aliases->aliases[alias->slot];
  2199. p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT;
  2200. p->npages = alias->memory_size >> PAGE_SHIFT;
  2201. p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT;
  2202. p->flags &= ~(KVM_ALIAS_INVALID);
  2203. for (n = KVM_ALIAS_SLOTS; n > 0; --n)
  2204. if (aliases->aliases[n - 1].npages)
  2205. break;
  2206. aliases->naliases = n;
  2207. old_aliases = kvm->arch.aliases;
  2208. rcu_assign_pointer(kvm->arch.aliases, aliases);
  2209. synchronize_srcu_expedited(&kvm->srcu);
  2210. kfree(old_aliases);
  2211. r = 0;
  2212. out_unlock:
  2213. mutex_unlock(&kvm->slots_lock);
  2214. out:
  2215. return r;
  2216. }
  2217. static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
  2218. {
  2219. int r;
  2220. r = 0;
  2221. switch (chip->chip_id) {
  2222. case KVM_IRQCHIP_PIC_MASTER:
  2223. memcpy(&chip->chip.pic,
  2224. &pic_irqchip(kvm)->pics[0],
  2225. sizeof(struct kvm_pic_state));
  2226. break;
  2227. case KVM_IRQCHIP_PIC_SLAVE:
  2228. memcpy(&chip->chip.pic,
  2229. &pic_irqchip(kvm)->pics[1],
  2230. sizeof(struct kvm_pic_state));
  2231. break;
  2232. case KVM_IRQCHIP_IOAPIC:
  2233. r = kvm_get_ioapic(kvm, &chip->chip.ioapic);
  2234. break;
  2235. default:
  2236. r = -EINVAL;
  2237. break;
  2238. }
  2239. return r;
  2240. }
  2241. static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
  2242. {
  2243. int r;
  2244. r = 0;
  2245. switch (chip->chip_id) {
  2246. case KVM_IRQCHIP_PIC_MASTER:
  2247. raw_spin_lock(&pic_irqchip(kvm)->lock);
  2248. memcpy(&pic_irqchip(kvm)->pics[0],
  2249. &chip->chip.pic,
  2250. sizeof(struct kvm_pic_state));
  2251. raw_spin_unlock(&pic_irqchip(kvm)->lock);
  2252. break;
  2253. case KVM_IRQCHIP_PIC_SLAVE:
  2254. raw_spin_lock(&pic_irqchip(kvm)->lock);
  2255. memcpy(&pic_irqchip(kvm)->pics[1],
  2256. &chip->chip.pic,
  2257. sizeof(struct kvm_pic_state));
  2258. raw_spin_unlock(&pic_irqchip(kvm)->lock);
  2259. break;
  2260. case KVM_IRQCHIP_IOAPIC:
  2261. r = kvm_set_ioapic(kvm, &chip->chip.ioapic);
  2262. break;
  2263. default:
  2264. r = -EINVAL;
  2265. break;
  2266. }
  2267. kvm_pic_update_irq(pic_irqchip(kvm));
  2268. return r;
  2269. }
  2270. static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
  2271. {
  2272. int r = 0;
  2273. mutex_lock(&kvm->arch.vpit->pit_state.lock);
  2274. memcpy(ps, &kvm->arch.vpit->pit_state, sizeof(struct kvm_pit_state));
  2275. mutex_unlock(&kvm->arch.vpit->pit_state.lock);
  2276. return r;
  2277. }
  2278. static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
  2279. {
  2280. int r = 0;
  2281. mutex_lock(&kvm->arch.vpit->pit_state.lock);
  2282. memcpy(&kvm->arch.vpit->pit_state, ps, sizeof(struct kvm_pit_state));
  2283. kvm_pit_load_count(kvm, 0, ps->channels[0].count, 0);
  2284. mutex_unlock(&kvm->arch.vpit->pit_state.lock);
  2285. return r;
  2286. }
  2287. static int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
  2288. {
  2289. int r = 0;
  2290. mutex_lock(&kvm->arch.vpit->pit_state.lock);
  2291. memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,
  2292. sizeof(ps->channels));
  2293. ps->flags = kvm->arch.vpit->pit_state.flags;
  2294. mutex_unlock(&kvm->arch.vpit->pit_state.lock);
  2295. return r;
  2296. }
  2297. static int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
  2298. {
  2299. int r = 0, start = 0;
  2300. u32 prev_legacy, cur_legacy;
  2301. mutex_lock(&kvm->arch.vpit->pit_state.lock);
  2302. prev_legacy = kvm->arch.vpit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY;
  2303. cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY;
  2304. if (!prev_legacy && cur_legacy)
  2305. start = 1;
  2306. memcpy(&kvm->arch.vpit->pit_state.channels, &ps->channels,
  2307. sizeof(kvm->arch.vpit->pit_state.channels));
  2308. kvm->arch.vpit->pit_state.flags = ps->flags;
  2309. kvm_pit_load_count(kvm, 0, kvm->arch.vpit->pit_state.channels[0].count, start);
  2310. mutex_unlock(&kvm->arch.vpit->pit_state.lock);
  2311. return r;
  2312. }
  2313. static int kvm_vm_ioctl_reinject(struct kvm *kvm,
  2314. struct kvm_reinject_control *control)
  2315. {
  2316. if (!kvm->arch.vpit)
  2317. return -ENXIO;
  2318. mutex_lock(&kvm->arch.vpit->pit_state.lock);
  2319. kvm->arch.vpit->pit_state.pit_timer.reinject = control->pit_reinject;
  2320. mutex_unlock(&kvm->arch.vpit->pit_state.lock);
  2321. return 0;
  2322. }
  2323. /*
  2324. * Get (and clear) the dirty memory log for a memory slot.
  2325. */
  2326. int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
  2327. struct kvm_dirty_log *log)
  2328. {
  2329. int r, n, i;
  2330. struct kvm_memory_slot *memslot;
  2331. unsigned long is_dirty = 0;
  2332. unsigned long *dirty_bitmap = NULL;
  2333. mutex_lock(&kvm->slots_lock);
  2334. r = -EINVAL;
  2335. if (log->slot >= KVM_MEMORY_SLOTS)
  2336. goto out;
  2337. memslot = &kvm->memslots->memslots[log->slot];
  2338. r = -ENOENT;
  2339. if (!memslot->dirty_bitmap)
  2340. goto out;
  2341. n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
  2342. r = -ENOMEM;
  2343. dirty_bitmap = vmalloc(n);
  2344. if (!dirty_bitmap)
  2345. goto out;
  2346. memset(dirty_bitmap, 0, n);
  2347. for (i = 0; !is_dirty && i < n/sizeof(long); i++)
  2348. is_dirty = memslot->dirty_bitmap[i];
  2349. /* If nothing is dirty, don't bother messing with page tables. */
  2350. if (is_dirty) {
  2351. struct kvm_memslots *slots, *old_slots;
  2352. spin_lock(&kvm->mmu_lock);
  2353. kvm_mmu_slot_remove_write_access(kvm, log->slot);
  2354. spin_unlock(&kvm->mmu_lock);
  2355. slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
  2356. if (!slots)
  2357. goto out_free;
  2358. memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots));
  2359. slots->memslots[log->slot].dirty_bitmap = dirty_bitmap;
  2360. old_slots = kvm->memslots;
  2361. rcu_assign_pointer(kvm->memslots, slots);
  2362. synchronize_srcu_expedited(&kvm->srcu);
  2363. dirty_bitmap = old_slots->memslots[log->slot].dirty_bitmap;
  2364. kfree(old_slots);
  2365. }
  2366. r = 0;
  2367. if (copy_to_user(log->dirty_bitmap, dirty_bitmap, n))
  2368. r = -EFAULT;
  2369. out_free:
  2370. vfree(dirty_bitmap);
  2371. out:
  2372. mutex_unlock(&kvm->slots_lock);
  2373. return r;
  2374. }
  2375. long kvm_arch_vm_ioctl(struct file *filp,
  2376. unsigned int ioctl, unsigned long arg)
  2377. {
  2378. struct kvm *kvm = filp->private_data;
  2379. void __user *argp = (void __user *)arg;
  2380. int r = -ENOTTY;
  2381. /*
  2382. * This union makes it completely explicit to gcc-3.x
  2383. * that these two variables' stack usage should be
  2384. * combined, not added together.
  2385. */
  2386. union {
  2387. struct kvm_pit_state ps;
  2388. struct kvm_pit_state2 ps2;
  2389. struct kvm_memory_alias alias;
  2390. struct kvm_pit_config pit_config;
  2391. } u;
  2392. switch (ioctl) {
  2393. case KVM_SET_TSS_ADDR:
  2394. r = kvm_vm_ioctl_set_tss_addr(kvm, arg);
  2395. if (r < 0)
  2396. goto out;
  2397. break;
  2398. case KVM_SET_IDENTITY_MAP_ADDR: {
  2399. u64 ident_addr;
  2400. r = -EFAULT;
  2401. if (copy_from_user(&ident_addr, argp, sizeof ident_addr))
  2402. goto out;
  2403. r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr);
  2404. if (r < 0)
  2405. goto out;
  2406. break;
  2407. }
  2408. case KVM_SET_MEMORY_REGION: {
  2409. struct kvm_memory_region kvm_mem;
  2410. struct kvm_userspace_memory_region kvm_userspace_mem;
  2411. r = -EFAULT;
  2412. if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem))
  2413. goto out;
  2414. kvm_userspace_mem.slot = kvm_mem.slot;
  2415. kvm_userspace_mem.flags = kvm_mem.flags;
  2416. kvm_userspace_mem.guest_phys_addr = kvm_mem.guest_phys_addr;
  2417. kvm_userspace_mem.memory_size = kvm_mem.memory_size;
  2418. r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 0);
  2419. if (r)
  2420. goto out;
  2421. break;
  2422. }
  2423. case KVM_SET_NR_MMU_PAGES:
  2424. r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
  2425. if (r)
  2426. goto out;
  2427. break;
  2428. case KVM_GET_NR_MMU_PAGES:
  2429. r = kvm_vm_ioctl_get_nr_mmu_pages(kvm);
  2430. break;
  2431. case KVM_SET_MEMORY_ALIAS:
  2432. r = -EFAULT;
  2433. if (copy_from_user(&u.alias, argp, sizeof(struct kvm_memory_alias)))
  2434. goto out;
  2435. r = kvm_vm_ioctl_set_memory_alias(kvm, &u.alias);
  2436. if (r)
  2437. goto out;
  2438. break;
  2439. case KVM_CREATE_IRQCHIP: {
  2440. struct kvm_pic *vpic;
  2441. mutex_lock(&kvm->lock);
  2442. r = -EEXIST;
  2443. if (kvm->arch.vpic)
  2444. goto create_irqchip_unlock;
  2445. r = -ENOMEM;
  2446. vpic = kvm_create_pic(kvm);
  2447. if (vpic) {
  2448. r = kvm_ioapic_init(kvm);
  2449. if (r) {
  2450. kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS,
  2451. &vpic->dev);
  2452. kfree(vpic);
  2453. goto create_irqchip_unlock;
  2454. }
  2455. } else
  2456. goto create_irqchip_unlock;
  2457. smp_wmb();
  2458. kvm->arch.vpic = vpic;
  2459. smp_wmb();
  2460. r = kvm_setup_default_irq_routing(kvm);
  2461. if (r) {
  2462. mutex_lock(&kvm->irq_lock);
  2463. kvm_ioapic_destroy(kvm);
  2464. kvm_destroy_pic(kvm);
  2465. mutex_unlock(&kvm->irq_lock);
  2466. }
  2467. create_irqchip_unlock:
  2468. mutex_unlock(&kvm->lock);
  2469. break;
  2470. }
  2471. case KVM_CREATE_PIT:
  2472. u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY;
  2473. goto create_pit;
  2474. case KVM_CREATE_PIT2:
  2475. r = -EFAULT;
  2476. if (copy_from_user(&u.pit_config, argp,
  2477. sizeof(struct kvm_pit_config)))
  2478. goto out;
  2479. create_pit:
  2480. mutex_lock(&kvm->slots_lock);
  2481. r = -EEXIST;
  2482. if (kvm->arch.vpit)
  2483. goto create_pit_unlock;
  2484. r = -ENOMEM;
  2485. kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);
  2486. if (kvm->arch.vpit)
  2487. r = 0;
  2488. create_pit_unlock:
  2489. mutex_unlock(&kvm->slots_lock);
  2490. break;
  2491. case KVM_IRQ_LINE_STATUS:
  2492. case KVM_IRQ_LINE: {
  2493. struct kvm_irq_level irq_event;
  2494. r = -EFAULT;
  2495. if (copy_from_user(&irq_event, argp, sizeof irq_event))
  2496. goto out;
  2497. if (irqchip_in_kernel(kvm)) {
  2498. __s32 status;
  2499. status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
  2500. irq_event.irq, irq_event.level);
  2501. if (ioctl == KVM_IRQ_LINE_STATUS) {
  2502. irq_event.status = status;
  2503. if (copy_to_user(argp, &irq_event,
  2504. sizeof irq_event))
  2505. goto out;
  2506. }
  2507. r = 0;
  2508. }
  2509. break;
  2510. }
  2511. case KVM_GET_IRQCHIP: {
  2512. /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
  2513. struct kvm_irqchip *chip = kmalloc(sizeof(*chip), GFP_KERNEL);
  2514. r = -ENOMEM;
  2515. if (!chip)
  2516. goto out;
  2517. r = -EFAULT;
  2518. if (copy_from_user(chip, argp, sizeof *chip))
  2519. goto get_irqchip_out;
  2520. r = -ENXIO;
  2521. if (!irqchip_in_kernel(kvm))
  2522. goto get_irqchip_out;
  2523. r = kvm_vm_ioctl_get_irqchip(kvm, chip);
  2524. if (r)
  2525. goto get_irqchip_out;
  2526. r = -EFAULT;
  2527. if (copy_to_user(argp, chip, sizeof *chip))
  2528. goto get_irqchip_out;
  2529. r = 0;
  2530. get_irqchip_out:
  2531. kfree(chip);
  2532. if (r)
  2533. goto out;
  2534. break;
  2535. }
  2536. case KVM_SET_IRQCHIP: {
  2537. /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
  2538. struct kvm_irqchip *chip = kmalloc(sizeof(*chip), GFP_KERNEL);
  2539. r = -ENOMEM;
  2540. if (!chip)
  2541. goto out;
  2542. r = -EFAULT;
  2543. if (copy_from_user(chip, argp, sizeof *chip))
  2544. goto set_irqchip_out;
  2545. r = -ENXIO;
  2546. if (!irqchip_in_kernel(kvm))
  2547. goto set_irqchip_out;
  2548. r = kvm_vm_ioctl_set_irqchip(kvm, chip);
  2549. if (r)
  2550. goto set_irqchip_out;
  2551. r = 0;
  2552. set_irqchip_out:
  2553. kfree(chip);
  2554. if (r)
  2555. goto out;
  2556. break;
  2557. }
  2558. case KVM_GET_PIT: {
  2559. r = -EFAULT;
  2560. if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state)))
  2561. goto out;
  2562. r = -ENXIO;
  2563. if (!kvm->arch.vpit)
  2564. goto out;
  2565. r = kvm_vm_ioctl_get_pit(kvm, &u.ps);
  2566. if (r)
  2567. goto out;
  2568. r = -EFAULT;
  2569. if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state)))
  2570. goto out;
  2571. r = 0;
  2572. break;
  2573. }
  2574. case KVM_SET_PIT: {
  2575. r = -EFAULT;
  2576. if (copy_from_user(&u.ps, argp, sizeof u.ps))
  2577. goto out;
  2578. r = -ENXIO;
  2579. if (!kvm->arch.vpit)
  2580. goto out;
  2581. r = kvm_vm_ioctl_set_pit(kvm, &u.ps);
  2582. if (r)
  2583. goto out;
  2584. r = 0;
  2585. break;
  2586. }
  2587. case KVM_GET_PIT2: {
  2588. r = -ENXIO;
  2589. if (!kvm->arch.vpit)
  2590. goto out;
  2591. r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2);
  2592. if (r)
  2593. goto out;
  2594. r = -EFAULT;
  2595. if (copy_to_user(argp, &u.ps2, sizeof(u.ps2)))
  2596. goto out;
  2597. r = 0;
  2598. break;
  2599. }
  2600. case KVM_SET_PIT2: {
  2601. r = -EFAULT;
  2602. if (copy_from_user(&u.ps2, argp, sizeof(u.ps2)))
  2603. goto out;
  2604. r = -ENXIO;
  2605. if (!kvm->arch.vpit)
  2606. goto out;
  2607. r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2);
  2608. if (r)
  2609. goto out;
  2610. r = 0;
  2611. break;
  2612. }
  2613. case KVM_REINJECT_CONTROL: {
  2614. struct kvm_reinject_control control;
  2615. r = -EFAULT;
  2616. if (copy_from_user(&control, argp, sizeof(control)))
  2617. goto out;
  2618. r = kvm_vm_ioctl_reinject(kvm, &control);
  2619. if (r)
  2620. goto out;
  2621. r = 0;
  2622. break;
  2623. }
  2624. case KVM_XEN_HVM_CONFIG: {
  2625. r = -EFAULT;
  2626. if (copy_from_user(&kvm->arch.xen_hvm_config, argp,
  2627. sizeof(struct kvm_xen_hvm_config)))
  2628. goto out;
  2629. r = -EINVAL;
  2630. if (kvm->arch.xen_hvm_config.flags)
  2631. goto out;
  2632. r = 0;
  2633. break;
  2634. }
  2635. case KVM_SET_CLOCK: {
  2636. struct timespec now;
  2637. struct kvm_clock_data user_ns;
  2638. u64 now_ns;
  2639. s64 delta;
  2640. r = -EFAULT;
  2641. if (copy_from_user(&user_ns, argp, sizeof(user_ns)))
  2642. goto out;
  2643. r = -EINVAL;
  2644. if (user_ns.flags)
  2645. goto out;
  2646. r = 0;
  2647. ktime_get_ts(&now);
  2648. now_ns = timespec_to_ns(&now);
  2649. delta = user_ns.clock - now_ns;
  2650. kvm->arch.kvmclock_offset = delta;
  2651. break;
  2652. }
  2653. case KVM_GET_CLOCK: {
  2654. struct timespec now;
  2655. struct kvm_clock_data user_ns;
  2656. u64 now_ns;
  2657. ktime_get_ts(&now);
  2658. now_ns = timespec_to_ns(&now);
  2659. user_ns.clock = kvm->arch.kvmclock_offset + now_ns;
  2660. user_ns.flags = 0;
  2661. r = -EFAULT;
  2662. if (copy_to_user(argp, &user_ns, sizeof(user_ns)))
  2663. goto out;
  2664. r = 0;
  2665. break;
  2666. }
  2667. default:
  2668. ;
  2669. }
  2670. out:
  2671. return r;
  2672. }
  2673. static void kvm_init_msr_list(void)
  2674. {
  2675. u32 dummy[2];
  2676. unsigned i, j;
  2677. /* skip the first msrs in the list. KVM-specific */
  2678. for (i = j = KVM_SAVE_MSRS_BEGIN; i < ARRAY_SIZE(msrs_to_save); i++) {
  2679. if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
  2680. continue;
  2681. if (j < i)
  2682. msrs_to_save[j] = msrs_to_save[i];
  2683. j++;
  2684. }
  2685. num_msrs_to_save = j;
  2686. }
  2687. static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
  2688. const void *v)
  2689. {
  2690. if (vcpu->arch.apic &&
  2691. !kvm_iodevice_write(&vcpu->arch.apic->dev, addr, len, v))
  2692. return 0;
  2693. return kvm_io_bus_write(vcpu->kvm, KVM_MMIO_BUS, addr, len, v);
  2694. }
  2695. static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
  2696. {
  2697. if (vcpu->arch.apic &&
  2698. !kvm_iodevice_read(&vcpu->arch.apic->dev, addr, len, v))
  2699. return 0;
  2700. return kvm_io_bus_read(vcpu->kvm, KVM_MMIO_BUS, addr, len, v);
  2701. }
  2702. gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, u32 *error)
  2703. {
  2704. u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
  2705. return vcpu->arch.mmu.gva_to_gpa(vcpu, gva, access, error);
  2706. }
  2707. gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva, u32 *error)
  2708. {
  2709. u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
  2710. access |= PFERR_FETCH_MASK;
  2711. return vcpu->arch.mmu.gva_to_gpa(vcpu, gva, access, error);
  2712. }
  2713. gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, u32 *error)
  2714. {
  2715. u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
  2716. access |= PFERR_WRITE_MASK;
  2717. return vcpu->arch.mmu.gva_to_gpa(vcpu, gva, access, error);
  2718. }
  2719. /* uses this to access any guest's mapped memory without checking CPL */
  2720. gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, u32 *error)
  2721. {
  2722. return vcpu->arch.mmu.gva_to_gpa(vcpu, gva, 0, error);
  2723. }
  2724. static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
  2725. struct kvm_vcpu *vcpu, u32 access,
  2726. u32 *error)
  2727. {
  2728. void *data = val;
  2729. int r = X86EMUL_CONTINUE;
  2730. while (bytes) {
  2731. gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr, access, error);
  2732. unsigned offset = addr & (PAGE_SIZE-1);
  2733. unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
  2734. int ret;
  2735. if (gpa == UNMAPPED_GVA) {
  2736. r = X86EMUL_PROPAGATE_FAULT;
  2737. goto out;
  2738. }
  2739. ret = kvm_read_guest(vcpu->kvm, gpa, data, toread);
  2740. if (ret < 0) {
  2741. r = X86EMUL_UNHANDLEABLE;
  2742. goto out;
  2743. }
  2744. bytes -= toread;
  2745. data += toread;
  2746. addr += toread;
  2747. }
  2748. out:
  2749. return r;
  2750. }
  2751. /* used for instruction fetching */
  2752. static int kvm_fetch_guest_virt(gva_t addr, void *val, unsigned int bytes,
  2753. struct kvm_vcpu *vcpu, u32 *error)
  2754. {
  2755. u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
  2756. return kvm_read_guest_virt_helper(addr, val, bytes, vcpu,
  2757. access | PFERR_FETCH_MASK, error);
  2758. }
  2759. static int kvm_read_guest_virt(gva_t addr, void *val, unsigned int bytes,
  2760. struct kvm_vcpu *vcpu, u32 *error)
  2761. {
  2762. u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
  2763. return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
  2764. error);
  2765. }
  2766. static int kvm_read_guest_virt_system(gva_t addr, void *val, unsigned int bytes,
  2767. struct kvm_vcpu *vcpu, u32 *error)
  2768. {
  2769. return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, 0, error);
  2770. }
  2771. static int kvm_write_guest_virt(gva_t addr, void *val, unsigned int bytes,
  2772. struct kvm_vcpu *vcpu, u32 *error)
  2773. {
  2774. void *data = val;
  2775. int r = X86EMUL_CONTINUE;
  2776. while (bytes) {
  2777. gpa_t gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, error);
  2778. unsigned offset = addr & (PAGE_SIZE-1);
  2779. unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
  2780. int ret;
  2781. if (gpa == UNMAPPED_GVA) {
  2782. r = X86EMUL_PROPAGATE_FAULT;
  2783. goto out;
  2784. }
  2785. ret = kvm_write_guest(vcpu->kvm, gpa, data, towrite);
  2786. if (ret < 0) {
  2787. r = X86EMUL_UNHANDLEABLE;
  2788. goto out;
  2789. }
  2790. bytes -= towrite;
  2791. data += towrite;
  2792. addr += towrite;
  2793. }
  2794. out:
  2795. return r;
  2796. }
  2797. static int emulator_read_emulated(unsigned long addr,
  2798. void *val,
  2799. unsigned int bytes,
  2800. struct kvm_vcpu *vcpu)
  2801. {
  2802. gpa_t gpa;
  2803. u32 error_code;
  2804. if (vcpu->mmio_read_completed) {
  2805. memcpy(val, vcpu->mmio_data, bytes);
  2806. trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
  2807. vcpu->mmio_phys_addr, *(u64 *)val);
  2808. vcpu->mmio_read_completed = 0;
  2809. return X86EMUL_CONTINUE;
  2810. }
  2811. gpa = kvm_mmu_gva_to_gpa_read(vcpu, addr, &error_code);
  2812. if (gpa == UNMAPPED_GVA) {
  2813. kvm_inject_page_fault(vcpu, addr, error_code);
  2814. return X86EMUL_PROPAGATE_FAULT;
  2815. }
  2816. /* For APIC access vmexit */
  2817. if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
  2818. goto mmio;
  2819. if (kvm_read_guest_virt(addr, val, bytes, vcpu, NULL)
  2820. == X86EMUL_CONTINUE)
  2821. return X86EMUL_CONTINUE;
  2822. mmio:
  2823. /*
  2824. * Is this MMIO handled locally?
  2825. */
  2826. if (!vcpu_mmio_read(vcpu, gpa, bytes, val)) {
  2827. trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes, gpa, *(u64 *)val);
  2828. return X86EMUL_CONTINUE;
  2829. }
  2830. trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, 0);
  2831. vcpu->mmio_needed = 1;
  2832. vcpu->mmio_phys_addr = gpa;
  2833. vcpu->mmio_size = bytes;
  2834. vcpu->mmio_is_write = 0;
  2835. return X86EMUL_UNHANDLEABLE;
  2836. }
  2837. int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
  2838. const void *val, int bytes)
  2839. {
  2840. int ret;
  2841. ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes);
  2842. if (ret < 0)
  2843. return 0;
  2844. kvm_mmu_pte_write(vcpu, gpa, val, bytes, 1);
  2845. return 1;
  2846. }
  2847. static int emulator_write_emulated_onepage(unsigned long addr,
  2848. const void *val,
  2849. unsigned int bytes,
  2850. struct kvm_vcpu *vcpu)
  2851. {
  2852. gpa_t gpa;
  2853. u32 error_code;
  2854. gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, &error_code);
  2855. if (gpa == UNMAPPED_GVA) {
  2856. kvm_inject_page_fault(vcpu, addr, error_code);
  2857. return X86EMUL_PROPAGATE_FAULT;
  2858. }
  2859. /* For APIC access vmexit */
  2860. if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
  2861. goto mmio;
  2862. if (emulator_write_phys(vcpu, gpa, val, bytes))
  2863. return X86EMUL_CONTINUE;
  2864. mmio:
  2865. trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, *(u64 *)val);
  2866. /*
  2867. * Is this MMIO handled locally?
  2868. */
  2869. if (!vcpu_mmio_write(vcpu, gpa, bytes, val))
  2870. return X86EMUL_CONTINUE;
  2871. vcpu->mmio_needed = 1;
  2872. vcpu->mmio_phys_addr = gpa;
  2873. vcpu->mmio_size = bytes;
  2874. vcpu->mmio_is_write = 1;
  2875. memcpy(vcpu->mmio_data, val, bytes);
  2876. return X86EMUL_CONTINUE;
  2877. }
  2878. int emulator_write_emulated(unsigned long addr,
  2879. const void *val,
  2880. unsigned int bytes,
  2881. struct kvm_vcpu *vcpu)
  2882. {
  2883. /* Crossing a page boundary? */
  2884. if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
  2885. int rc, now;
  2886. now = -addr & ~PAGE_MASK;
  2887. rc = emulator_write_emulated_onepage(addr, val, now, vcpu);
  2888. if (rc != X86EMUL_CONTINUE)
  2889. return rc;
  2890. addr += now;
  2891. val += now;
  2892. bytes -= now;
  2893. }
  2894. return emulator_write_emulated_onepage(addr, val, bytes, vcpu);
  2895. }
  2896. EXPORT_SYMBOL_GPL(emulator_write_emulated);
  2897. static int emulator_cmpxchg_emulated(unsigned long addr,
  2898. const void *old,
  2899. const void *new,
  2900. unsigned int bytes,
  2901. struct kvm_vcpu *vcpu)
  2902. {
  2903. printk_once(KERN_WARNING "kvm: emulating exchange as write\n");
  2904. #ifndef CONFIG_X86_64
  2905. /* guests cmpxchg8b have to be emulated atomically */
  2906. if (bytes == 8) {
  2907. gpa_t gpa;
  2908. struct page *page;
  2909. char *kaddr;
  2910. u64 val;
  2911. gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);
  2912. if (gpa == UNMAPPED_GVA ||
  2913. (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
  2914. goto emul_write;
  2915. if (((gpa + bytes - 1) & PAGE_MASK) != (gpa & PAGE_MASK))
  2916. goto emul_write;
  2917. val = *(u64 *)new;
  2918. page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
  2919. kaddr = kmap_atomic(page, KM_USER0);
  2920. set_64bit((u64 *)(kaddr + offset_in_page(gpa)), val);
  2921. kunmap_atomic(kaddr, KM_USER0);
  2922. kvm_release_page_dirty(page);
  2923. }
  2924. emul_write:
  2925. #endif
  2926. return emulator_write_emulated(addr, new, bytes, vcpu);
  2927. }
  2928. static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
  2929. {
  2930. return kvm_x86_ops->get_segment_base(vcpu, seg);
  2931. }
  2932. int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
  2933. {
  2934. kvm_mmu_invlpg(vcpu, address);
  2935. return X86EMUL_CONTINUE;
  2936. }
  2937. int emulate_clts(struct kvm_vcpu *vcpu)
  2938. {
  2939. kvm_x86_ops->set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~X86_CR0_TS));
  2940. kvm_x86_ops->fpu_activate(vcpu);
  2941. return X86EMUL_CONTINUE;
  2942. }
  2943. int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest)
  2944. {
  2945. return kvm_x86_ops->get_dr(ctxt->vcpu, dr, dest);
  2946. }
  2947. int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
  2948. {
  2949. unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
  2950. return kvm_x86_ops->set_dr(ctxt->vcpu, dr, value & mask);
  2951. }
  2952. void kvm_report_emulation_failure(struct kvm_vcpu *vcpu, const char *context)
  2953. {
  2954. u8 opcodes[4];
  2955. unsigned long rip = kvm_rip_read(vcpu);
  2956. unsigned long rip_linear;
  2957. if (!printk_ratelimit())
  2958. return;
  2959. rip_linear = rip + get_segment_base(vcpu, VCPU_SREG_CS);
  2960. kvm_read_guest_virt(rip_linear, (void *)opcodes, 4, vcpu, NULL);
  2961. printk(KERN_ERR "emulation failed (%s) rip %lx %02x %02x %02x %02x\n",
  2962. context, rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]);
  2963. }
  2964. EXPORT_SYMBOL_GPL(kvm_report_emulation_failure);
  2965. static struct x86_emulate_ops emulate_ops = {
  2966. .read_std = kvm_read_guest_virt_system,
  2967. .fetch = kvm_fetch_guest_virt,
  2968. .read_emulated = emulator_read_emulated,
  2969. .write_emulated = emulator_write_emulated,
  2970. .cmpxchg_emulated = emulator_cmpxchg_emulated,
  2971. };
  2972. static void cache_all_regs(struct kvm_vcpu *vcpu)
  2973. {
  2974. kvm_register_read(vcpu, VCPU_REGS_RAX);
  2975. kvm_register_read(vcpu, VCPU_REGS_RSP);
  2976. kvm_register_read(vcpu, VCPU_REGS_RIP);
  2977. vcpu->arch.regs_dirty = ~0;
  2978. }
  2979. int emulate_instruction(struct kvm_vcpu *vcpu,
  2980. unsigned long cr2,
  2981. u16 error_code,
  2982. int emulation_type)
  2983. {
  2984. int r, shadow_mask;
  2985. struct decode_cache *c;
  2986. struct kvm_run *run = vcpu->run;
  2987. kvm_clear_exception_queue(vcpu);
  2988. vcpu->arch.mmio_fault_cr2 = cr2;
  2989. /*
  2990. * TODO: fix emulate.c to use guest_read/write_register
  2991. * instead of direct ->regs accesses, can save hundred cycles
  2992. * on Intel for instructions that don't read/change RSP, for
  2993. * for example.
  2994. */
  2995. cache_all_regs(vcpu);
  2996. vcpu->mmio_is_write = 0;
  2997. vcpu->arch.pio.string = 0;
  2998. if (!(emulation_type & EMULTYPE_NO_DECODE)) {
  2999. int cs_db, cs_l;
  3000. kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
  3001. vcpu->arch.emulate_ctxt.vcpu = vcpu;
  3002. vcpu->arch.emulate_ctxt.eflags = kvm_get_rflags(vcpu);
  3003. vcpu->arch.emulate_ctxt.mode =
  3004. (!is_protmode(vcpu)) ? X86EMUL_MODE_REAL :
  3005. (vcpu->arch.emulate_ctxt.eflags & X86_EFLAGS_VM)
  3006. ? X86EMUL_MODE_VM86 : cs_l
  3007. ? X86EMUL_MODE_PROT64 : cs_db
  3008. ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
  3009. r = x86_decode_insn(&vcpu->arch.emulate_ctxt, &emulate_ops);
  3010. /* Only allow emulation of specific instructions on #UD
  3011. * (namely VMMCALL, sysenter, sysexit, syscall)*/
  3012. c = &vcpu->arch.emulate_ctxt.decode;
  3013. if (emulation_type & EMULTYPE_TRAP_UD) {
  3014. if (!c->twobyte)
  3015. return EMULATE_FAIL;
  3016. switch (c->b) {
  3017. case 0x01: /* VMMCALL */
  3018. if (c->modrm_mod != 3 || c->modrm_rm != 1)
  3019. return EMULATE_FAIL;
  3020. break;
  3021. case 0x34: /* sysenter */
  3022. case 0x35: /* sysexit */
  3023. if (c->modrm_mod != 0 || c->modrm_rm != 0)
  3024. return EMULATE_FAIL;
  3025. break;
  3026. case 0x05: /* syscall */
  3027. if (c->modrm_mod != 0 || c->modrm_rm != 0)
  3028. return EMULATE_FAIL;
  3029. break;
  3030. default:
  3031. return EMULATE_FAIL;
  3032. }
  3033. if (!(c->modrm_reg == 0 || c->modrm_reg == 3))
  3034. return EMULATE_FAIL;
  3035. }
  3036. ++vcpu->stat.insn_emulation;
  3037. if (r) {
  3038. ++vcpu->stat.insn_emulation_fail;
  3039. if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
  3040. return EMULATE_DONE;
  3041. return EMULATE_FAIL;
  3042. }
  3043. }
  3044. if (emulation_type & EMULTYPE_SKIP) {
  3045. kvm_rip_write(vcpu, vcpu->arch.emulate_ctxt.decode.eip);
  3046. return EMULATE_DONE;
  3047. }
  3048. r = x86_emulate_insn(&vcpu->arch.emulate_ctxt, &emulate_ops);
  3049. shadow_mask = vcpu->arch.emulate_ctxt.interruptibility;
  3050. if (r == 0)
  3051. kvm_x86_ops->set_interrupt_shadow(vcpu, shadow_mask);
  3052. if (vcpu->arch.pio.string)
  3053. return EMULATE_DO_MMIO;
  3054. if ((r || vcpu->mmio_is_write) && run) {
  3055. run->exit_reason = KVM_EXIT_MMIO;
  3056. run->mmio.phys_addr = vcpu->mmio_phys_addr;
  3057. memcpy(run->mmio.data, vcpu->mmio_data, 8);
  3058. run->mmio.len = vcpu->mmio_size;
  3059. run->mmio.is_write = vcpu->mmio_is_write;
  3060. }
  3061. if (r) {
  3062. if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
  3063. return EMULATE_DONE;
  3064. if (!vcpu->mmio_needed) {
  3065. kvm_report_emulation_failure(vcpu, "mmio");
  3066. return EMULATE_FAIL;
  3067. }
  3068. return EMULATE_DO_MMIO;
  3069. }
  3070. kvm_set_rflags(vcpu, vcpu->arch.emulate_ctxt.eflags);
  3071. if (vcpu->mmio_is_write) {
  3072. vcpu->mmio_needed = 0;
  3073. return EMULATE_DO_MMIO;
  3074. }
  3075. return EMULATE_DONE;
  3076. }
  3077. EXPORT_SYMBOL_GPL(emulate_instruction);
  3078. static int pio_copy_data(struct kvm_vcpu *vcpu)
  3079. {
  3080. void *p = vcpu->arch.pio_data;
  3081. gva_t q = vcpu->arch.pio.guest_gva;
  3082. unsigned bytes;
  3083. int ret;
  3084. u32 error_code;
  3085. bytes = vcpu->arch.pio.size * vcpu->arch.pio.cur_count;
  3086. if (vcpu->arch.pio.in)
  3087. ret = kvm_write_guest_virt(q, p, bytes, vcpu, &error_code);
  3088. else
  3089. ret = kvm_read_guest_virt(q, p, bytes, vcpu, &error_code);
  3090. if (ret == X86EMUL_PROPAGATE_FAULT)
  3091. kvm_inject_page_fault(vcpu, q, error_code);
  3092. return ret;
  3093. }
  3094. int complete_pio(struct kvm_vcpu *vcpu)
  3095. {
  3096. struct kvm_pio_request *io = &vcpu->arch.pio;
  3097. long delta;
  3098. int r;
  3099. unsigned long val;
  3100. if (!io->string) {
  3101. if (io->in) {
  3102. val = kvm_register_read(vcpu, VCPU_REGS_RAX);
  3103. memcpy(&val, vcpu->arch.pio_data, io->size);
  3104. kvm_register_write(vcpu, VCPU_REGS_RAX, val);
  3105. }
  3106. } else {
  3107. if (io->in) {
  3108. r = pio_copy_data(vcpu);
  3109. if (r)
  3110. goto out;
  3111. }
  3112. delta = 1;
  3113. if (io->rep) {
  3114. delta *= io->cur_count;
  3115. /*
  3116. * The size of the register should really depend on
  3117. * current address size.
  3118. */
  3119. val = kvm_register_read(vcpu, VCPU_REGS_RCX);
  3120. val -= delta;
  3121. kvm_register_write(vcpu, VCPU_REGS_RCX, val);
  3122. }
  3123. if (io->down)
  3124. delta = -delta;
  3125. delta *= io->size;
  3126. if (io->in) {
  3127. val = kvm_register_read(vcpu, VCPU_REGS_RDI);
  3128. val += delta;
  3129. kvm_register_write(vcpu, VCPU_REGS_RDI, val);
  3130. } else {
  3131. val = kvm_register_read(vcpu, VCPU_REGS_RSI);
  3132. val += delta;
  3133. kvm_register_write(vcpu, VCPU_REGS_RSI, val);
  3134. }
  3135. }
  3136. out:
  3137. io->count -= io->cur_count;
  3138. io->cur_count = 0;
  3139. return 0;
  3140. }
  3141. static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
  3142. {
  3143. /* TODO: String I/O for in kernel device */
  3144. int r;
  3145. if (vcpu->arch.pio.in)
  3146. r = kvm_io_bus_read(vcpu->kvm, KVM_PIO_BUS, vcpu->arch.pio.port,
  3147. vcpu->arch.pio.size, pd);
  3148. else
  3149. r = kvm_io_bus_write(vcpu->kvm, KVM_PIO_BUS,
  3150. vcpu->arch.pio.port, vcpu->arch.pio.size,
  3151. pd);
  3152. return r;
  3153. }
  3154. static int pio_string_write(struct kvm_vcpu *vcpu)
  3155. {
  3156. struct kvm_pio_request *io = &vcpu->arch.pio;
  3157. void *pd = vcpu->arch.pio_data;
  3158. int i, r = 0;
  3159. for (i = 0; i < io->cur_count; i++) {
  3160. if (kvm_io_bus_write(vcpu->kvm, KVM_PIO_BUS,
  3161. io->port, io->size, pd)) {
  3162. r = -EOPNOTSUPP;
  3163. break;
  3164. }
  3165. pd += io->size;
  3166. }
  3167. return r;
  3168. }
  3169. int kvm_emulate_pio(struct kvm_vcpu *vcpu, int in, int size, unsigned port)
  3170. {
  3171. unsigned long val;
  3172. trace_kvm_pio(!in, port, size, 1);
  3173. vcpu->run->exit_reason = KVM_EXIT_IO;
  3174. vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
  3175. vcpu->run->io.size = vcpu->arch.pio.size = size;
  3176. vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
  3177. vcpu->run->io.count = vcpu->arch.pio.count = vcpu->arch.pio.cur_count = 1;
  3178. vcpu->run->io.port = vcpu->arch.pio.port = port;
  3179. vcpu->arch.pio.in = in;
  3180. vcpu->arch.pio.string = 0;
  3181. vcpu->arch.pio.down = 0;
  3182. vcpu->arch.pio.rep = 0;
  3183. if (!vcpu->arch.pio.in) {
  3184. val = kvm_register_read(vcpu, VCPU_REGS_RAX);
  3185. memcpy(vcpu->arch.pio_data, &val, 4);
  3186. }
  3187. if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
  3188. complete_pio(vcpu);
  3189. return 1;
  3190. }
  3191. return 0;
  3192. }
  3193. EXPORT_SYMBOL_GPL(kvm_emulate_pio);
  3194. int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, int in,
  3195. int size, unsigned long count, int down,
  3196. gva_t address, int rep, unsigned port)
  3197. {
  3198. unsigned now, in_page;
  3199. int ret = 0;
  3200. trace_kvm_pio(!in, port, size, count);
  3201. vcpu->run->exit_reason = KVM_EXIT_IO;
  3202. vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
  3203. vcpu->run->io.size = vcpu->arch.pio.size = size;
  3204. vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
  3205. vcpu->run->io.count = vcpu->arch.pio.count = vcpu->arch.pio.cur_count = count;
  3206. vcpu->run->io.port = vcpu->arch.pio.port = port;
  3207. vcpu->arch.pio.in = in;
  3208. vcpu->arch.pio.string = 1;
  3209. vcpu->arch.pio.down = down;
  3210. vcpu->arch.pio.rep = rep;
  3211. if (!count) {
  3212. kvm_x86_ops->skip_emulated_instruction(vcpu);
  3213. return 1;
  3214. }
  3215. if (!down)
  3216. in_page = PAGE_SIZE - offset_in_page(address);
  3217. else
  3218. in_page = offset_in_page(address) + size;
  3219. now = min(count, (unsigned long)in_page / size);
  3220. if (!now)
  3221. now = 1;
  3222. if (down) {
  3223. /*
  3224. * String I/O in reverse. Yuck. Kill the guest, fix later.
  3225. */
  3226. pr_unimpl(vcpu, "guest string pio down\n");
  3227. kvm_inject_gp(vcpu, 0);
  3228. return 1;
  3229. }
  3230. vcpu->run->io.count = now;
  3231. vcpu->arch.pio.cur_count = now;
  3232. if (vcpu->arch.pio.cur_count == vcpu->arch.pio.count)
  3233. kvm_x86_ops->skip_emulated_instruction(vcpu);
  3234. vcpu->arch.pio.guest_gva = address;
  3235. if (!vcpu->arch.pio.in) {
  3236. /* string PIO write */
  3237. ret = pio_copy_data(vcpu);
  3238. if (ret == X86EMUL_PROPAGATE_FAULT)
  3239. return 1;
  3240. if (ret == 0 && !pio_string_write(vcpu)) {
  3241. complete_pio(vcpu);
  3242. if (vcpu->arch.pio.count == 0)
  3243. ret = 1;
  3244. }
  3245. }
  3246. /* no string PIO read support yet */
  3247. return ret;
  3248. }
  3249. EXPORT_SYMBOL_GPL(kvm_emulate_pio_string);
  3250. static void bounce_off(void *info)
  3251. {
  3252. /* nothing */
  3253. }
  3254. static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
  3255. void *data)
  3256. {
  3257. struct cpufreq_freqs *freq = data;
  3258. struct kvm *kvm;
  3259. struct kvm_vcpu *vcpu;
  3260. int i, send_ipi = 0;
  3261. if (val == CPUFREQ_PRECHANGE && freq->old > freq->new)
  3262. return 0;
  3263. if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new)
  3264. return 0;
  3265. per_cpu(cpu_tsc_khz, freq->cpu) = freq->new;
  3266. spin_lock(&kvm_lock);
  3267. list_for_each_entry(kvm, &vm_list, vm_list) {
  3268. kvm_for_each_vcpu(i, vcpu, kvm) {
  3269. if (vcpu->cpu != freq->cpu)
  3270. continue;
  3271. if (!kvm_request_guest_time_update(vcpu))
  3272. continue;
  3273. if (vcpu->cpu != smp_processor_id())
  3274. send_ipi++;
  3275. }
  3276. }
  3277. spin_unlock(&kvm_lock);
  3278. if (freq->old < freq->new && send_ipi) {
  3279. /*
  3280. * We upscale the frequency. Must make the guest
  3281. * doesn't see old kvmclock values while running with
  3282. * the new frequency, otherwise we risk the guest sees
  3283. * time go backwards.
  3284. *
  3285. * In case we update the frequency for another cpu
  3286. * (which might be in guest context) send an interrupt
  3287. * to kick the cpu out of guest context. Next time
  3288. * guest context is entered kvmclock will be updated,
  3289. * so the guest will not see stale values.
  3290. */
  3291. smp_call_function_single(freq->cpu, bounce_off, NULL, 1);
  3292. }
  3293. return 0;
  3294. }
  3295. static struct notifier_block kvmclock_cpufreq_notifier_block = {
  3296. .notifier_call = kvmclock_cpufreq_notifier
  3297. };
  3298. static void kvm_timer_init(void)
  3299. {
  3300. int cpu;
  3301. if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
  3302. cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block,
  3303. CPUFREQ_TRANSITION_NOTIFIER);
  3304. for_each_online_cpu(cpu) {
  3305. unsigned long khz = cpufreq_get(cpu);
  3306. if (!khz)
  3307. khz = tsc_khz;
  3308. per_cpu(cpu_tsc_khz, cpu) = khz;
  3309. }
  3310. } else {
  3311. for_each_possible_cpu(cpu)
  3312. per_cpu(cpu_tsc_khz, cpu) = tsc_khz;
  3313. }
  3314. }
  3315. int kvm_arch_init(void *opaque)
  3316. {
  3317. int r;
  3318. struct kvm_x86_ops *ops = (struct kvm_x86_ops *)opaque;
  3319. if (kvm_x86_ops) {
  3320. printk(KERN_ERR "kvm: already loaded the other module\n");
  3321. r = -EEXIST;
  3322. goto out;
  3323. }
  3324. if (!ops->cpu_has_kvm_support()) {
  3325. printk(KERN_ERR "kvm: no hardware support\n");
  3326. r = -EOPNOTSUPP;
  3327. goto out;
  3328. }
  3329. if (ops->disabled_by_bios()) {
  3330. printk(KERN_ERR "kvm: disabled by bios\n");
  3331. r = -EOPNOTSUPP;
  3332. goto out;
  3333. }
  3334. r = kvm_mmu_module_init();
  3335. if (r)
  3336. goto out;
  3337. kvm_init_msr_list();
  3338. kvm_x86_ops = ops;
  3339. kvm_mmu_set_nonpresent_ptes(0ull, 0ull);
  3340. kvm_mmu_set_base_ptes(PT_PRESENT_MASK);
  3341. kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK,
  3342. PT_DIRTY_MASK, PT64_NX_MASK, 0);
  3343. kvm_timer_init();
  3344. return 0;
  3345. out:
  3346. return r;
  3347. }
  3348. void kvm_arch_exit(void)
  3349. {
  3350. if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
  3351. cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
  3352. CPUFREQ_TRANSITION_NOTIFIER);
  3353. kvm_x86_ops = NULL;
  3354. kvm_mmu_module_exit();
  3355. }
  3356. int kvm_emulate_halt(struct kvm_vcpu *vcpu)
  3357. {
  3358. ++vcpu->stat.halt_exits;
  3359. if (irqchip_in_kernel(vcpu->kvm)) {
  3360. vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
  3361. return 1;
  3362. } else {
  3363. vcpu->run->exit_reason = KVM_EXIT_HLT;
  3364. return 0;
  3365. }
  3366. }
  3367. EXPORT_SYMBOL_GPL(kvm_emulate_halt);
  3368. static inline gpa_t hc_gpa(struct kvm_vcpu *vcpu, unsigned long a0,
  3369. unsigned long a1)
  3370. {
  3371. if (is_long_mode(vcpu))
  3372. return a0;
  3373. else
  3374. return a0 | ((gpa_t)a1 << 32);
  3375. }
  3376. int kvm_hv_hypercall(struct kvm_vcpu *vcpu)
  3377. {
  3378. u64 param, ingpa, outgpa, ret;
  3379. uint16_t code, rep_idx, rep_cnt, res = HV_STATUS_SUCCESS, rep_done = 0;
  3380. bool fast, longmode;
  3381. int cs_db, cs_l;
  3382. /*
  3383. * hypercall generates UD from non zero cpl and real mode
  3384. * per HYPER-V spec
  3385. */
  3386. if (kvm_x86_ops->get_cpl(vcpu) != 0 || !is_protmode(vcpu)) {
  3387. kvm_queue_exception(vcpu, UD_VECTOR);
  3388. return 0;
  3389. }
  3390. kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
  3391. longmode = is_long_mode(vcpu) && cs_l == 1;
  3392. if (!longmode) {
  3393. param = ((u64)kvm_register_read(vcpu, VCPU_REGS_RDX) << 32) |
  3394. (kvm_register_read(vcpu, VCPU_REGS_RAX) & 0xffffffff);
  3395. ingpa = ((u64)kvm_register_read(vcpu, VCPU_REGS_RBX) << 32) |
  3396. (kvm_register_read(vcpu, VCPU_REGS_RCX) & 0xffffffff);
  3397. outgpa = ((u64)kvm_register_read(vcpu, VCPU_REGS_RDI) << 32) |
  3398. (kvm_register_read(vcpu, VCPU_REGS_RSI) & 0xffffffff);
  3399. }
  3400. #ifdef CONFIG_X86_64
  3401. else {
  3402. param = kvm_register_read(vcpu, VCPU_REGS_RCX);
  3403. ingpa = kvm_register_read(vcpu, VCPU_REGS_RDX);
  3404. outgpa = kvm_register_read(vcpu, VCPU_REGS_R8);
  3405. }
  3406. #endif
  3407. code = param & 0xffff;
  3408. fast = (param >> 16) & 0x1;
  3409. rep_cnt = (param >> 32) & 0xfff;
  3410. rep_idx = (param >> 48) & 0xfff;
  3411. trace_kvm_hv_hypercall(code, fast, rep_cnt, rep_idx, ingpa, outgpa);
  3412. switch (code) {
  3413. case HV_X64_HV_NOTIFY_LONG_SPIN_WAIT:
  3414. kvm_vcpu_on_spin(vcpu);
  3415. break;
  3416. default:
  3417. res = HV_STATUS_INVALID_HYPERCALL_CODE;
  3418. break;
  3419. }
  3420. ret = res | (((u64)rep_done & 0xfff) << 32);
  3421. if (longmode) {
  3422. kvm_register_write(vcpu, VCPU_REGS_RAX, ret);
  3423. } else {
  3424. kvm_register_write(vcpu, VCPU_REGS_RDX, ret >> 32);
  3425. kvm_register_write(vcpu, VCPU_REGS_RAX, ret & 0xffffffff);
  3426. }
  3427. return 1;
  3428. }
  3429. int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
  3430. {
  3431. unsigned long nr, a0, a1, a2, a3, ret;
  3432. int r = 1;
  3433. if (kvm_hv_hypercall_enabled(vcpu->kvm))
  3434. return kvm_hv_hypercall(vcpu);
  3435. nr = kvm_register_read(vcpu, VCPU_REGS_RAX);
  3436. a0 = kvm_register_read(vcpu, VCPU_REGS_RBX);
  3437. a1 = kvm_register_read(vcpu, VCPU_REGS_RCX);
  3438. a2 = kvm_register_read(vcpu, VCPU_REGS_RDX);
  3439. a3 = kvm_register_read(vcpu, VCPU_REGS_RSI);
  3440. trace_kvm_hypercall(nr, a0, a1, a2, a3);
  3441. if (!is_long_mode(vcpu)) {
  3442. nr &= 0xFFFFFFFF;
  3443. a0 &= 0xFFFFFFFF;
  3444. a1 &= 0xFFFFFFFF;
  3445. a2 &= 0xFFFFFFFF;
  3446. a3 &= 0xFFFFFFFF;
  3447. }
  3448. if (kvm_x86_ops->get_cpl(vcpu) != 0) {
  3449. ret = -KVM_EPERM;
  3450. goto out;
  3451. }
  3452. switch (nr) {
  3453. case KVM_HC_VAPIC_POLL_IRQ:
  3454. ret = 0;
  3455. break;
  3456. case KVM_HC_MMU_OP:
  3457. r = kvm_pv_mmu_op(vcpu, a0, hc_gpa(vcpu, a1, a2), &ret);
  3458. break;
  3459. default:
  3460. ret = -KVM_ENOSYS;
  3461. break;
  3462. }
  3463. out:
  3464. kvm_register_write(vcpu, VCPU_REGS_RAX, ret);
  3465. ++vcpu->stat.hypercalls;
  3466. return r;
  3467. }
  3468. EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);
  3469. int kvm_fix_hypercall(struct kvm_vcpu *vcpu)
  3470. {
  3471. char instruction[3];
  3472. unsigned long rip = kvm_rip_read(vcpu);
  3473. /*
  3474. * Blow out the MMU to ensure that no other VCPU has an active mapping
  3475. * to ensure that the updated hypercall appears atomically across all
  3476. * VCPUs.
  3477. */
  3478. kvm_mmu_zap_all(vcpu->kvm);
  3479. kvm_x86_ops->patch_hypercall(vcpu, instruction);
  3480. return emulator_write_emulated(rip, instruction, 3, vcpu);
  3481. }
  3482. static u64 mk_cr_64(u64 curr_cr, u32 new_val)
  3483. {
  3484. return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
  3485. }
  3486. void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
  3487. {
  3488. struct descriptor_table dt = { limit, base };
  3489. kvm_x86_ops->set_gdt(vcpu, &dt);
  3490. }
  3491. void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
  3492. {
  3493. struct descriptor_table dt = { limit, base };
  3494. kvm_x86_ops->set_idt(vcpu, &dt);
  3495. }
  3496. void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
  3497. unsigned long *rflags)
  3498. {
  3499. kvm_lmsw(vcpu, msw);
  3500. *rflags = kvm_get_rflags(vcpu);
  3501. }
  3502. unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
  3503. {
  3504. unsigned long value;
  3505. switch (cr) {
  3506. case 0:
  3507. value = kvm_read_cr0(vcpu);
  3508. break;
  3509. case 2:
  3510. value = vcpu->arch.cr2;
  3511. break;
  3512. case 3:
  3513. value = vcpu->arch.cr3;
  3514. break;
  3515. case 4:
  3516. value = kvm_read_cr4(vcpu);
  3517. break;
  3518. case 8:
  3519. value = kvm_get_cr8(vcpu);
  3520. break;
  3521. default:
  3522. vcpu_printf(vcpu, "%s: unexpected cr %u\n", __func__, cr);
  3523. return 0;
  3524. }
  3525. return value;
  3526. }
  3527. void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
  3528. unsigned long *rflags)
  3529. {
  3530. switch (cr) {
  3531. case 0:
  3532. kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));
  3533. *rflags = kvm_get_rflags(vcpu);
  3534. break;
  3535. case 2:
  3536. vcpu->arch.cr2 = val;
  3537. break;
  3538. case 3:
  3539. kvm_set_cr3(vcpu, val);
  3540. break;
  3541. case 4:
  3542. kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));
  3543. break;
  3544. case 8:
  3545. kvm_set_cr8(vcpu, val & 0xfUL);
  3546. break;
  3547. default:
  3548. vcpu_printf(vcpu, "%s: unexpected cr %u\n", __func__, cr);
  3549. }
  3550. }
  3551. static int move_to_next_stateful_cpuid_entry(struct kvm_vcpu *vcpu, int i)
  3552. {
  3553. struct kvm_cpuid_entry2 *e = &vcpu->arch.cpuid_entries[i];
  3554. int j, nent = vcpu->arch.cpuid_nent;
  3555. e->flags &= ~KVM_CPUID_FLAG_STATE_READ_NEXT;
  3556. /* when no next entry is found, the current entry[i] is reselected */
  3557. for (j = i + 1; ; j = (j + 1) % nent) {
  3558. struct kvm_cpuid_entry2 *ej = &vcpu->arch.cpuid_entries[j];
  3559. if (ej->function == e->function) {
  3560. ej->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT;
  3561. return j;
  3562. }
  3563. }
  3564. return 0; /* silence gcc, even though control never reaches here */
  3565. }
  3566. /* find an entry with matching function, matching index (if needed), and that
  3567. * should be read next (if it's stateful) */
  3568. static int is_matching_cpuid_entry(struct kvm_cpuid_entry2 *e,
  3569. u32 function, u32 index)
  3570. {
  3571. if (e->function != function)
  3572. return 0;
  3573. if ((e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX) && e->index != index)
  3574. return 0;
  3575. if ((e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) &&
  3576. !(e->flags & KVM_CPUID_FLAG_STATE_READ_NEXT))
  3577. return 0;
  3578. return 1;
  3579. }
  3580. struct kvm_cpuid_entry2 *kvm_find_cpuid_entry(struct kvm_vcpu *vcpu,
  3581. u32 function, u32 index)
  3582. {
  3583. int i;
  3584. struct kvm_cpuid_entry2 *best = NULL;
  3585. for (i = 0; i < vcpu->arch.cpuid_nent; ++i) {
  3586. struct kvm_cpuid_entry2 *e;
  3587. e = &vcpu->arch.cpuid_entries[i];
  3588. if (is_matching_cpuid_entry(e, function, index)) {
  3589. if (e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC)
  3590. move_to_next_stateful_cpuid_entry(vcpu, i);
  3591. best = e;
  3592. break;
  3593. }
  3594. /*
  3595. * Both basic or both extended?
  3596. */
  3597. if (((e->function ^ function) & 0x80000000) == 0)
  3598. if (!best || e->function > best->function)
  3599. best = e;
  3600. }
  3601. return best;
  3602. }
  3603. EXPORT_SYMBOL_GPL(kvm_find_cpuid_entry);
  3604. int cpuid_maxphyaddr(struct kvm_vcpu *vcpu)
  3605. {
  3606. struct kvm_cpuid_entry2 *best;
  3607. best = kvm_find_cpuid_entry(vcpu, 0x80000008, 0);
  3608. if (best)
  3609. return best->eax & 0xff;
  3610. return 36;
  3611. }
  3612. void kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
  3613. {
  3614. u32 function, index;
  3615. struct kvm_cpuid_entry2 *best;
  3616. function = kvm_register_read(vcpu, VCPU_REGS_RAX);
  3617. index = kvm_register_read(vcpu, VCPU_REGS_RCX);
  3618. kvm_register_write(vcpu, VCPU_REGS_RAX, 0);
  3619. kvm_register_write(vcpu, VCPU_REGS_RBX, 0);
  3620. kvm_register_write(vcpu, VCPU_REGS_RCX, 0);
  3621. kvm_register_write(vcpu, VCPU_REGS_RDX, 0);
  3622. best = kvm_find_cpuid_entry(vcpu, function, index);
  3623. if (best) {
  3624. kvm_register_write(vcpu, VCPU_REGS_RAX, best->eax);
  3625. kvm_register_write(vcpu, VCPU_REGS_RBX, best->ebx);
  3626. kvm_register_write(vcpu, VCPU_REGS_RCX, best->ecx);
  3627. kvm_register_write(vcpu, VCPU_REGS_RDX, best->edx);
  3628. }
  3629. kvm_x86_ops->skip_emulated_instruction(vcpu);
  3630. trace_kvm_cpuid(function,
  3631. kvm_register_read(vcpu, VCPU_REGS_RAX),
  3632. kvm_register_read(vcpu, VCPU_REGS_RBX),
  3633. kvm_register_read(vcpu, VCPU_REGS_RCX),
  3634. kvm_register_read(vcpu, VCPU_REGS_RDX));
  3635. }
  3636. EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);
  3637. /*
  3638. * Check if userspace requested an interrupt window, and that the
  3639. * interrupt window is open.
  3640. *
  3641. * No need to exit to userspace if we already have an interrupt queued.
  3642. */
  3643. static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)
  3644. {
  3645. return (!irqchip_in_kernel(vcpu->kvm) && !kvm_cpu_has_interrupt(vcpu) &&
  3646. vcpu->run->request_interrupt_window &&
  3647. kvm_arch_interrupt_allowed(vcpu));
  3648. }
  3649. static void post_kvm_run_save(struct kvm_vcpu *vcpu)
  3650. {
  3651. struct kvm_run *kvm_run = vcpu->run;
  3652. kvm_run->if_flag = (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
  3653. kvm_run->cr8 = kvm_get_cr8(vcpu);
  3654. kvm_run->apic_base = kvm_get_apic_base(vcpu);
  3655. if (irqchip_in_kernel(vcpu->kvm))
  3656. kvm_run->ready_for_interrupt_injection = 1;
  3657. else
  3658. kvm_run->ready_for_interrupt_injection =
  3659. kvm_arch_interrupt_allowed(vcpu) &&
  3660. !kvm_cpu_has_interrupt(vcpu) &&
  3661. !kvm_event_needs_reinjection(vcpu);
  3662. }
  3663. static void vapic_enter(struct kvm_vcpu *vcpu)
  3664. {
  3665. struct kvm_lapic *apic = vcpu->arch.apic;
  3666. struct page *page;
  3667. if (!apic || !apic->vapic_addr)
  3668. return;
  3669. page = gfn_to_page(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT);
  3670. vcpu->arch.apic->vapic_page = page;
  3671. }
  3672. static void vapic_exit(struct kvm_vcpu *vcpu)
  3673. {
  3674. struct kvm_lapic *apic = vcpu->arch.apic;
  3675. int idx;
  3676. if (!apic || !apic->vapic_addr)
  3677. return;
  3678. idx = srcu_read_lock(&vcpu->kvm->srcu);
  3679. kvm_release_page_dirty(apic->vapic_page);
  3680. mark_page_dirty(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT);
  3681. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  3682. }
  3683. static void update_cr8_intercept(struct kvm_vcpu *vcpu)
  3684. {
  3685. int max_irr, tpr;
  3686. if (!kvm_x86_ops->update_cr8_intercept)
  3687. return;
  3688. if (!vcpu->arch.apic)
  3689. return;
  3690. if (!vcpu->arch.apic->vapic_addr)
  3691. max_irr = kvm_lapic_find_highest_irr(vcpu);
  3692. else
  3693. max_irr = -1;
  3694. if (max_irr != -1)
  3695. max_irr >>= 4;
  3696. tpr = kvm_lapic_get_cr8(vcpu);
  3697. kvm_x86_ops->update_cr8_intercept(vcpu, tpr, max_irr);
  3698. }
  3699. static void inject_pending_event(struct kvm_vcpu *vcpu)
  3700. {
  3701. /* try to reinject previous events if any */
  3702. if (vcpu->arch.exception.pending) {
  3703. kvm_x86_ops->queue_exception(vcpu, vcpu->arch.exception.nr,
  3704. vcpu->arch.exception.has_error_code,
  3705. vcpu->arch.exception.error_code);
  3706. return;
  3707. }
  3708. if (vcpu->arch.nmi_injected) {
  3709. kvm_x86_ops->set_nmi(vcpu);
  3710. return;
  3711. }
  3712. if (vcpu->arch.interrupt.pending) {
  3713. kvm_x86_ops->set_irq(vcpu);
  3714. return;
  3715. }
  3716. /* try to inject new event if pending */
  3717. if (vcpu->arch.nmi_pending) {
  3718. if (kvm_x86_ops->nmi_allowed(vcpu)) {
  3719. vcpu->arch.nmi_pending = false;
  3720. vcpu->arch.nmi_injected = true;
  3721. kvm_x86_ops->set_nmi(vcpu);
  3722. }
  3723. } else if (kvm_cpu_has_interrupt(vcpu)) {
  3724. if (kvm_x86_ops->interrupt_allowed(vcpu)) {
  3725. kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu),
  3726. false);
  3727. kvm_x86_ops->set_irq(vcpu);
  3728. }
  3729. }
  3730. }
  3731. static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
  3732. {
  3733. int r;
  3734. bool req_int_win = !irqchip_in_kernel(vcpu->kvm) &&
  3735. vcpu->run->request_interrupt_window;
  3736. if (vcpu->requests)
  3737. if (test_and_clear_bit(KVM_REQ_MMU_RELOAD, &vcpu->requests))
  3738. kvm_mmu_unload(vcpu);
  3739. r = kvm_mmu_reload(vcpu);
  3740. if (unlikely(r))
  3741. goto out;
  3742. if (vcpu->requests) {
  3743. if (test_and_clear_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests))
  3744. __kvm_migrate_timers(vcpu);
  3745. if (test_and_clear_bit(KVM_REQ_KVMCLOCK_UPDATE, &vcpu->requests))
  3746. kvm_write_guest_time(vcpu);
  3747. if (test_and_clear_bit(KVM_REQ_MMU_SYNC, &vcpu->requests))
  3748. kvm_mmu_sync_roots(vcpu);
  3749. if (test_and_clear_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests))
  3750. kvm_x86_ops->tlb_flush(vcpu);
  3751. if (test_and_clear_bit(KVM_REQ_REPORT_TPR_ACCESS,
  3752. &vcpu->requests)) {
  3753. vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;
  3754. r = 0;
  3755. goto out;
  3756. }
  3757. if (test_and_clear_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests)) {
  3758. vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
  3759. r = 0;
  3760. goto out;
  3761. }
  3762. if (test_and_clear_bit(KVM_REQ_DEACTIVATE_FPU, &vcpu->requests)) {
  3763. vcpu->fpu_active = 0;
  3764. kvm_x86_ops->fpu_deactivate(vcpu);
  3765. }
  3766. }
  3767. preempt_disable();
  3768. kvm_x86_ops->prepare_guest_switch(vcpu);
  3769. if (vcpu->fpu_active)
  3770. kvm_load_guest_fpu(vcpu);
  3771. local_irq_disable();
  3772. clear_bit(KVM_REQ_KICK, &vcpu->requests);
  3773. smp_mb__after_clear_bit();
  3774. if (vcpu->requests || need_resched() || signal_pending(current)) {
  3775. set_bit(KVM_REQ_KICK, &vcpu->requests);
  3776. local_irq_enable();
  3777. preempt_enable();
  3778. r = 1;
  3779. goto out;
  3780. }
  3781. inject_pending_event(vcpu);
  3782. /* enable NMI/IRQ window open exits if needed */
  3783. if (vcpu->arch.nmi_pending)
  3784. kvm_x86_ops->enable_nmi_window(vcpu);
  3785. else if (kvm_cpu_has_interrupt(vcpu) || req_int_win)
  3786. kvm_x86_ops->enable_irq_window(vcpu);
  3787. if (kvm_lapic_enabled(vcpu)) {
  3788. update_cr8_intercept(vcpu);
  3789. kvm_lapic_sync_to_vapic(vcpu);
  3790. }
  3791. srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
  3792. kvm_guest_enter();
  3793. if (unlikely(vcpu->arch.switch_db_regs)) {
  3794. set_debugreg(0, 7);
  3795. set_debugreg(vcpu->arch.eff_db[0], 0);
  3796. set_debugreg(vcpu->arch.eff_db[1], 1);
  3797. set_debugreg(vcpu->arch.eff_db[2], 2);
  3798. set_debugreg(vcpu->arch.eff_db[3], 3);
  3799. }
  3800. trace_kvm_entry(vcpu->vcpu_id);
  3801. kvm_x86_ops->run(vcpu);
  3802. /*
  3803. * If the guest has used debug registers, at least dr7
  3804. * will be disabled while returning to the host.
  3805. * If we don't have active breakpoints in the host, we don't
  3806. * care about the messed up debug address registers. But if
  3807. * we have some of them active, restore the old state.
  3808. */
  3809. if (hw_breakpoint_active())
  3810. hw_breakpoint_restore();
  3811. set_bit(KVM_REQ_KICK, &vcpu->requests);
  3812. local_irq_enable();
  3813. ++vcpu->stat.exits;
  3814. /*
  3815. * We must have an instruction between local_irq_enable() and
  3816. * kvm_guest_exit(), so the timer interrupt isn't delayed by
  3817. * the interrupt shadow. The stat.exits increment will do nicely.
  3818. * But we need to prevent reordering, hence this barrier():
  3819. */
  3820. barrier();
  3821. kvm_guest_exit();
  3822. preempt_enable();
  3823. vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
  3824. /*
  3825. * Profile KVM exit RIPs:
  3826. */
  3827. if (unlikely(prof_on == KVM_PROFILING)) {
  3828. unsigned long rip = kvm_rip_read(vcpu);
  3829. profile_hit(KVM_PROFILING, (void *)rip);
  3830. }
  3831. kvm_lapic_sync_from_vapic(vcpu);
  3832. r = kvm_x86_ops->handle_exit(vcpu);
  3833. out:
  3834. return r;
  3835. }
  3836. static int __vcpu_run(struct kvm_vcpu *vcpu)
  3837. {
  3838. int r;
  3839. struct kvm *kvm = vcpu->kvm;
  3840. if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED)) {
  3841. pr_debug("vcpu %d received sipi with vector # %x\n",
  3842. vcpu->vcpu_id, vcpu->arch.sipi_vector);
  3843. kvm_lapic_reset(vcpu);
  3844. r = kvm_arch_vcpu_reset(vcpu);
  3845. if (r)
  3846. return r;
  3847. vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
  3848. }
  3849. vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
  3850. vapic_enter(vcpu);
  3851. r = 1;
  3852. while (r > 0) {
  3853. if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE)
  3854. r = vcpu_enter_guest(vcpu);
  3855. else {
  3856. srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
  3857. kvm_vcpu_block(vcpu);
  3858. vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
  3859. if (test_and_clear_bit(KVM_REQ_UNHALT, &vcpu->requests))
  3860. {
  3861. switch(vcpu->arch.mp_state) {
  3862. case KVM_MP_STATE_HALTED:
  3863. vcpu->arch.mp_state =
  3864. KVM_MP_STATE_RUNNABLE;
  3865. case KVM_MP_STATE_RUNNABLE:
  3866. break;
  3867. case KVM_MP_STATE_SIPI_RECEIVED:
  3868. default:
  3869. r = -EINTR;
  3870. break;
  3871. }
  3872. }
  3873. }
  3874. if (r <= 0)
  3875. break;
  3876. clear_bit(KVM_REQ_PENDING_TIMER, &vcpu->requests);
  3877. if (kvm_cpu_has_pending_timer(vcpu))
  3878. kvm_inject_pending_timer_irqs(vcpu);
  3879. if (dm_request_for_irq_injection(vcpu)) {
  3880. r = -EINTR;
  3881. vcpu->run->exit_reason = KVM_EXIT_INTR;
  3882. ++vcpu->stat.request_irq_exits;
  3883. }
  3884. if (signal_pending(current)) {
  3885. r = -EINTR;
  3886. vcpu->run->exit_reason = KVM_EXIT_INTR;
  3887. ++vcpu->stat.signal_exits;
  3888. }
  3889. if (need_resched()) {
  3890. srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
  3891. kvm_resched(vcpu);
  3892. vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
  3893. }
  3894. }
  3895. srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
  3896. post_kvm_run_save(vcpu);
  3897. vapic_exit(vcpu);
  3898. return r;
  3899. }
  3900. int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  3901. {
  3902. int r;
  3903. sigset_t sigsaved;
  3904. vcpu_load(vcpu);
  3905. if (vcpu->sigset_active)
  3906. sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
  3907. if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
  3908. kvm_vcpu_block(vcpu);
  3909. clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
  3910. r = -EAGAIN;
  3911. goto out;
  3912. }
  3913. /* re-sync apic's tpr */
  3914. if (!irqchip_in_kernel(vcpu->kvm))
  3915. kvm_set_cr8(vcpu, kvm_run->cr8);
  3916. if (vcpu->arch.pio.cur_count) {
  3917. r = complete_pio(vcpu);
  3918. if (r)
  3919. goto out;
  3920. }
  3921. if (vcpu->mmio_needed) {
  3922. memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
  3923. vcpu->mmio_read_completed = 1;
  3924. vcpu->mmio_needed = 0;
  3925. vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
  3926. r = emulate_instruction(vcpu, vcpu->arch.mmio_fault_cr2, 0,
  3927. EMULTYPE_NO_DECODE);
  3928. srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
  3929. if (r == EMULATE_DO_MMIO) {
  3930. /*
  3931. * Read-modify-write. Back to userspace.
  3932. */
  3933. r = 0;
  3934. goto out;
  3935. }
  3936. }
  3937. if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL)
  3938. kvm_register_write(vcpu, VCPU_REGS_RAX,
  3939. kvm_run->hypercall.ret);
  3940. r = __vcpu_run(vcpu);
  3941. out:
  3942. if (vcpu->sigset_active)
  3943. sigprocmask(SIG_SETMASK, &sigsaved, NULL);
  3944. vcpu_put(vcpu);
  3945. return r;
  3946. }
  3947. int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
  3948. {
  3949. vcpu_load(vcpu);
  3950. regs->rax = kvm_register_read(vcpu, VCPU_REGS_RAX);
  3951. regs->rbx = kvm_register_read(vcpu, VCPU_REGS_RBX);
  3952. regs->rcx = kvm_register_read(vcpu, VCPU_REGS_RCX);
  3953. regs->rdx = kvm_register_read(vcpu, VCPU_REGS_RDX);
  3954. regs->rsi = kvm_register_read(vcpu, VCPU_REGS_RSI);
  3955. regs->rdi = kvm_register_read(vcpu, VCPU_REGS_RDI);
  3956. regs->rsp = kvm_register_read(vcpu, VCPU_REGS_RSP);
  3957. regs->rbp = kvm_register_read(vcpu, VCPU_REGS_RBP);
  3958. #ifdef CONFIG_X86_64
  3959. regs->r8 = kvm_register_read(vcpu, VCPU_REGS_R8);
  3960. regs->r9 = kvm_register_read(vcpu, VCPU_REGS_R9);
  3961. regs->r10 = kvm_register_read(vcpu, VCPU_REGS_R10);
  3962. regs->r11 = kvm_register_read(vcpu, VCPU_REGS_R11);
  3963. regs->r12 = kvm_register_read(vcpu, VCPU_REGS_R12);
  3964. regs->r13 = kvm_register_read(vcpu, VCPU_REGS_R13);
  3965. regs->r14 = kvm_register_read(vcpu, VCPU_REGS_R14);
  3966. regs->r15 = kvm_register_read(vcpu, VCPU_REGS_R15);
  3967. #endif
  3968. regs->rip = kvm_rip_read(vcpu);
  3969. regs->rflags = kvm_get_rflags(vcpu);
  3970. vcpu_put(vcpu);
  3971. return 0;
  3972. }
  3973. int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
  3974. {
  3975. vcpu_load(vcpu);
  3976. kvm_register_write(vcpu, VCPU_REGS_RAX, regs->rax);
  3977. kvm_register_write(vcpu, VCPU_REGS_RBX, regs->rbx);
  3978. kvm_register_write(vcpu, VCPU_REGS_RCX, regs->rcx);
  3979. kvm_register_write(vcpu, VCPU_REGS_RDX, regs->rdx);
  3980. kvm_register_write(vcpu, VCPU_REGS_RSI, regs->rsi);
  3981. kvm_register_write(vcpu, VCPU_REGS_RDI, regs->rdi);
  3982. kvm_register_write(vcpu, VCPU_REGS_RSP, regs->rsp);
  3983. kvm_register_write(vcpu, VCPU_REGS_RBP, regs->rbp);
  3984. #ifdef CONFIG_X86_64
  3985. kvm_register_write(vcpu, VCPU_REGS_R8, regs->r8);
  3986. kvm_register_write(vcpu, VCPU_REGS_R9, regs->r9);
  3987. kvm_register_write(vcpu, VCPU_REGS_R10, regs->r10);
  3988. kvm_register_write(vcpu, VCPU_REGS_R11, regs->r11);
  3989. kvm_register_write(vcpu, VCPU_REGS_R12, regs->r12);
  3990. kvm_register_write(vcpu, VCPU_REGS_R13, regs->r13);
  3991. kvm_register_write(vcpu, VCPU_REGS_R14, regs->r14);
  3992. kvm_register_write(vcpu, VCPU_REGS_R15, regs->r15);
  3993. #endif
  3994. kvm_rip_write(vcpu, regs->rip);
  3995. kvm_set_rflags(vcpu, regs->rflags);
  3996. vcpu->arch.exception.pending = false;
  3997. vcpu_put(vcpu);
  3998. return 0;
  3999. }
  4000. void kvm_get_segment(struct kvm_vcpu *vcpu,
  4001. struct kvm_segment *var, int seg)
  4002. {
  4003. kvm_x86_ops->get_segment(vcpu, var, seg);
  4004. }
  4005. void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
  4006. {
  4007. struct kvm_segment cs;
  4008. kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
  4009. *db = cs.db;
  4010. *l = cs.l;
  4011. }
  4012. EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits);
  4013. int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
  4014. struct kvm_sregs *sregs)
  4015. {
  4016. struct descriptor_table dt;
  4017. vcpu_load(vcpu);
  4018. kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
  4019. kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
  4020. kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
  4021. kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
  4022. kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
  4023. kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
  4024. kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
  4025. kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
  4026. kvm_x86_ops->get_idt(vcpu, &dt);
  4027. sregs->idt.limit = dt.limit;
  4028. sregs->idt.base = dt.base;
  4029. kvm_x86_ops->get_gdt(vcpu, &dt);
  4030. sregs->gdt.limit = dt.limit;
  4031. sregs->gdt.base = dt.base;
  4032. sregs->cr0 = kvm_read_cr0(vcpu);
  4033. sregs->cr2 = vcpu->arch.cr2;
  4034. sregs->cr3 = vcpu->arch.cr3;
  4035. sregs->cr4 = kvm_read_cr4(vcpu);
  4036. sregs->cr8 = kvm_get_cr8(vcpu);
  4037. sregs->efer = vcpu->arch.efer;
  4038. sregs->apic_base = kvm_get_apic_base(vcpu);
  4039. memset(sregs->interrupt_bitmap, 0, sizeof sregs->interrupt_bitmap);
  4040. if (vcpu->arch.interrupt.pending && !vcpu->arch.interrupt.soft)
  4041. set_bit(vcpu->arch.interrupt.nr,
  4042. (unsigned long *)sregs->interrupt_bitmap);
  4043. vcpu_put(vcpu);
  4044. return 0;
  4045. }
  4046. int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
  4047. struct kvm_mp_state *mp_state)
  4048. {
  4049. vcpu_load(vcpu);
  4050. mp_state->mp_state = vcpu->arch.mp_state;
  4051. vcpu_put(vcpu);
  4052. return 0;
  4053. }
  4054. int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
  4055. struct kvm_mp_state *mp_state)
  4056. {
  4057. vcpu_load(vcpu);
  4058. vcpu->arch.mp_state = mp_state->mp_state;
  4059. vcpu_put(vcpu);
  4060. return 0;
  4061. }
  4062. static void kvm_set_segment(struct kvm_vcpu *vcpu,
  4063. struct kvm_segment *var, int seg)
  4064. {
  4065. kvm_x86_ops->set_segment(vcpu, var, seg);
  4066. }
  4067. static void seg_desct_to_kvm_desct(struct desc_struct *seg_desc, u16 selector,
  4068. struct kvm_segment *kvm_desct)
  4069. {
  4070. kvm_desct->base = get_desc_base(seg_desc);
  4071. kvm_desct->limit = get_desc_limit(seg_desc);
  4072. if (seg_desc->g) {
  4073. kvm_desct->limit <<= 12;
  4074. kvm_desct->limit |= 0xfff;
  4075. }
  4076. kvm_desct->selector = selector;
  4077. kvm_desct->type = seg_desc->type;
  4078. kvm_desct->present = seg_desc->p;
  4079. kvm_desct->dpl = seg_desc->dpl;
  4080. kvm_desct->db = seg_desc->d;
  4081. kvm_desct->s = seg_desc->s;
  4082. kvm_desct->l = seg_desc->l;
  4083. kvm_desct->g = seg_desc->g;
  4084. kvm_desct->avl = seg_desc->avl;
  4085. if (!selector)
  4086. kvm_desct->unusable = 1;
  4087. else
  4088. kvm_desct->unusable = 0;
  4089. kvm_desct->padding = 0;
  4090. }
  4091. static void get_segment_descriptor_dtable(struct kvm_vcpu *vcpu,
  4092. u16 selector,
  4093. struct descriptor_table *dtable)
  4094. {
  4095. if (selector & 1 << 2) {
  4096. struct kvm_segment kvm_seg;
  4097. kvm_get_segment(vcpu, &kvm_seg, VCPU_SREG_LDTR);
  4098. if (kvm_seg.unusable)
  4099. dtable->limit = 0;
  4100. else
  4101. dtable->limit = kvm_seg.limit;
  4102. dtable->base = kvm_seg.base;
  4103. }
  4104. else
  4105. kvm_x86_ops->get_gdt(vcpu, dtable);
  4106. }
  4107. /* allowed just for 8 bytes segments */
  4108. static int load_guest_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector,
  4109. struct desc_struct *seg_desc)
  4110. {
  4111. struct descriptor_table dtable;
  4112. u16 index = selector >> 3;
  4113. int ret;
  4114. u32 err;
  4115. gva_t addr;
  4116. get_segment_descriptor_dtable(vcpu, selector, &dtable);
  4117. if (dtable.limit < index * 8 + 7) {
  4118. kvm_queue_exception_e(vcpu, GP_VECTOR, selector & 0xfffc);
  4119. return X86EMUL_PROPAGATE_FAULT;
  4120. }
  4121. addr = dtable.base + index * 8;
  4122. ret = kvm_read_guest_virt_system(addr, seg_desc, sizeof(*seg_desc),
  4123. vcpu, &err);
  4124. if (ret == X86EMUL_PROPAGATE_FAULT)
  4125. kvm_inject_page_fault(vcpu, addr, err);
  4126. return ret;
  4127. }
  4128. /* allowed just for 8 bytes segments */
  4129. static int save_guest_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector,
  4130. struct desc_struct *seg_desc)
  4131. {
  4132. struct descriptor_table dtable;
  4133. u16 index = selector >> 3;
  4134. get_segment_descriptor_dtable(vcpu, selector, &dtable);
  4135. if (dtable.limit < index * 8 + 7)
  4136. return 1;
  4137. return kvm_write_guest_virt(dtable.base + index*8, seg_desc, sizeof(*seg_desc), vcpu, NULL);
  4138. }
  4139. static gpa_t get_tss_base_addr_write(struct kvm_vcpu *vcpu,
  4140. struct desc_struct *seg_desc)
  4141. {
  4142. u32 base_addr = get_desc_base(seg_desc);
  4143. return kvm_mmu_gva_to_gpa_write(vcpu, base_addr, NULL);
  4144. }
  4145. static gpa_t get_tss_base_addr_read(struct kvm_vcpu *vcpu,
  4146. struct desc_struct *seg_desc)
  4147. {
  4148. u32 base_addr = get_desc_base(seg_desc);
  4149. return kvm_mmu_gva_to_gpa_read(vcpu, base_addr, NULL);
  4150. }
  4151. static u16 get_segment_selector(struct kvm_vcpu *vcpu, int seg)
  4152. {
  4153. struct kvm_segment kvm_seg;
  4154. kvm_get_segment(vcpu, &kvm_seg, seg);
  4155. return kvm_seg.selector;
  4156. }
  4157. static int kvm_load_realmode_segment(struct kvm_vcpu *vcpu, u16 selector, int seg)
  4158. {
  4159. struct kvm_segment segvar = {
  4160. .base = selector << 4,
  4161. .limit = 0xffff,
  4162. .selector = selector,
  4163. .type = 3,
  4164. .present = 1,
  4165. .dpl = 3,
  4166. .db = 0,
  4167. .s = 1,
  4168. .l = 0,
  4169. .g = 0,
  4170. .avl = 0,
  4171. .unusable = 0,
  4172. };
  4173. kvm_x86_ops->set_segment(vcpu, &segvar, seg);
  4174. return X86EMUL_CONTINUE;
  4175. }
  4176. static int is_vm86_segment(struct kvm_vcpu *vcpu, int seg)
  4177. {
  4178. return (seg != VCPU_SREG_LDTR) &&
  4179. (seg != VCPU_SREG_TR) &&
  4180. (kvm_get_rflags(vcpu) & X86_EFLAGS_VM);
  4181. }
  4182. int kvm_load_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector, int seg)
  4183. {
  4184. struct kvm_segment kvm_seg;
  4185. struct desc_struct seg_desc;
  4186. u8 dpl, rpl, cpl;
  4187. unsigned err_vec = GP_VECTOR;
  4188. u32 err_code = 0;
  4189. bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
  4190. int ret;
  4191. if (is_vm86_segment(vcpu, seg) || !is_protmode(vcpu))
  4192. return kvm_load_realmode_segment(vcpu, selector, seg);
  4193. /* NULL selector is not valid for TR, CS and SS */
  4194. if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
  4195. && null_selector)
  4196. goto exception;
  4197. /* TR should be in GDT only */
  4198. if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
  4199. goto exception;
  4200. ret = load_guest_segment_descriptor(vcpu, selector, &seg_desc);
  4201. if (ret)
  4202. return ret;
  4203. seg_desct_to_kvm_desct(&seg_desc, selector, &kvm_seg);
  4204. if (null_selector) { /* for NULL selector skip all following checks */
  4205. kvm_seg.unusable = 1;
  4206. goto load;
  4207. }
  4208. err_code = selector & 0xfffc;
  4209. err_vec = GP_VECTOR;
  4210. /* can't load system descriptor into segment selecor */
  4211. if (seg <= VCPU_SREG_GS && !kvm_seg.s)
  4212. goto exception;
  4213. if (!kvm_seg.present) {
  4214. err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
  4215. goto exception;
  4216. }
  4217. rpl = selector & 3;
  4218. dpl = kvm_seg.dpl;
  4219. cpl = kvm_x86_ops->get_cpl(vcpu);
  4220. switch (seg) {
  4221. case VCPU_SREG_SS:
  4222. /*
  4223. * segment is not a writable data segment or segment
  4224. * selector's RPL != CPL or segment selector's RPL != CPL
  4225. */
  4226. if (rpl != cpl || (kvm_seg.type & 0xa) != 0x2 || dpl != cpl)
  4227. goto exception;
  4228. break;
  4229. case VCPU_SREG_CS:
  4230. if (!(kvm_seg.type & 8))
  4231. goto exception;
  4232. if (kvm_seg.type & 4) {
  4233. /* conforming */
  4234. if (dpl > cpl)
  4235. goto exception;
  4236. } else {
  4237. /* nonconforming */
  4238. if (rpl > cpl || dpl != cpl)
  4239. goto exception;
  4240. }
  4241. /* CS(RPL) <- CPL */
  4242. selector = (selector & 0xfffc) | cpl;
  4243. break;
  4244. case VCPU_SREG_TR:
  4245. if (kvm_seg.s || (kvm_seg.type != 1 && kvm_seg.type != 9))
  4246. goto exception;
  4247. break;
  4248. case VCPU_SREG_LDTR:
  4249. if (kvm_seg.s || kvm_seg.type != 2)
  4250. goto exception;
  4251. break;
  4252. default: /* DS, ES, FS, or GS */
  4253. /*
  4254. * segment is not a data or readable code segment or
  4255. * ((segment is a data or nonconforming code segment)
  4256. * and (both RPL and CPL > DPL))
  4257. */
  4258. if ((kvm_seg.type & 0xa) == 0x8 ||
  4259. (((kvm_seg.type & 0xc) != 0xc) && (rpl > dpl && cpl > dpl)))
  4260. goto exception;
  4261. break;
  4262. }
  4263. if (!kvm_seg.unusable && kvm_seg.s) {
  4264. /* mark segment as accessed */
  4265. kvm_seg.type |= 1;
  4266. seg_desc.type |= 1;
  4267. save_guest_segment_descriptor(vcpu, selector, &seg_desc);
  4268. }
  4269. load:
  4270. kvm_set_segment(vcpu, &kvm_seg, seg);
  4271. return X86EMUL_CONTINUE;
  4272. exception:
  4273. kvm_queue_exception_e(vcpu, err_vec, err_code);
  4274. return X86EMUL_PROPAGATE_FAULT;
  4275. }
  4276. static void save_state_to_tss32(struct kvm_vcpu *vcpu,
  4277. struct tss_segment_32 *tss)
  4278. {
  4279. tss->cr3 = vcpu->arch.cr3;
  4280. tss->eip = kvm_rip_read(vcpu);
  4281. tss->eflags = kvm_get_rflags(vcpu);
  4282. tss->eax = kvm_register_read(vcpu, VCPU_REGS_RAX);
  4283. tss->ecx = kvm_register_read(vcpu, VCPU_REGS_RCX);
  4284. tss->edx = kvm_register_read(vcpu, VCPU_REGS_RDX);
  4285. tss->ebx = kvm_register_read(vcpu, VCPU_REGS_RBX);
  4286. tss->esp = kvm_register_read(vcpu, VCPU_REGS_RSP);
  4287. tss->ebp = kvm_register_read(vcpu, VCPU_REGS_RBP);
  4288. tss->esi = kvm_register_read(vcpu, VCPU_REGS_RSI);
  4289. tss->edi = kvm_register_read(vcpu, VCPU_REGS_RDI);
  4290. tss->es = get_segment_selector(vcpu, VCPU_SREG_ES);
  4291. tss->cs = get_segment_selector(vcpu, VCPU_SREG_CS);
  4292. tss->ss = get_segment_selector(vcpu, VCPU_SREG_SS);
  4293. tss->ds = get_segment_selector(vcpu, VCPU_SREG_DS);
  4294. tss->fs = get_segment_selector(vcpu, VCPU_SREG_FS);
  4295. tss->gs = get_segment_selector(vcpu, VCPU_SREG_GS);
  4296. tss->ldt_selector = get_segment_selector(vcpu, VCPU_SREG_LDTR);
  4297. }
  4298. static void kvm_load_segment_selector(struct kvm_vcpu *vcpu, u16 sel, int seg)
  4299. {
  4300. struct kvm_segment kvm_seg;
  4301. kvm_get_segment(vcpu, &kvm_seg, seg);
  4302. kvm_seg.selector = sel;
  4303. kvm_set_segment(vcpu, &kvm_seg, seg);
  4304. }
  4305. static int load_state_from_tss32(struct kvm_vcpu *vcpu,
  4306. struct tss_segment_32 *tss)
  4307. {
  4308. kvm_set_cr3(vcpu, tss->cr3);
  4309. kvm_rip_write(vcpu, tss->eip);
  4310. kvm_set_rflags(vcpu, tss->eflags | 2);
  4311. kvm_register_write(vcpu, VCPU_REGS_RAX, tss->eax);
  4312. kvm_register_write(vcpu, VCPU_REGS_RCX, tss->ecx);
  4313. kvm_register_write(vcpu, VCPU_REGS_RDX, tss->edx);
  4314. kvm_register_write(vcpu, VCPU_REGS_RBX, tss->ebx);
  4315. kvm_register_write(vcpu, VCPU_REGS_RSP, tss->esp);
  4316. kvm_register_write(vcpu, VCPU_REGS_RBP, tss->ebp);
  4317. kvm_register_write(vcpu, VCPU_REGS_RSI, tss->esi);
  4318. kvm_register_write(vcpu, VCPU_REGS_RDI, tss->edi);
  4319. /*
  4320. * SDM says that segment selectors are loaded before segment
  4321. * descriptors
  4322. */
  4323. kvm_load_segment_selector(vcpu, tss->ldt_selector, VCPU_SREG_LDTR);
  4324. kvm_load_segment_selector(vcpu, tss->es, VCPU_SREG_ES);
  4325. kvm_load_segment_selector(vcpu, tss->cs, VCPU_SREG_CS);
  4326. kvm_load_segment_selector(vcpu, tss->ss, VCPU_SREG_SS);
  4327. kvm_load_segment_selector(vcpu, tss->ds, VCPU_SREG_DS);
  4328. kvm_load_segment_selector(vcpu, tss->fs, VCPU_SREG_FS);
  4329. kvm_load_segment_selector(vcpu, tss->gs, VCPU_SREG_GS);
  4330. /*
  4331. * Now load segment descriptors. If fault happenes at this stage
  4332. * it is handled in a context of new task
  4333. */
  4334. if (kvm_load_segment_descriptor(vcpu, tss->ldt_selector, VCPU_SREG_LDTR))
  4335. return 1;
  4336. if (kvm_load_segment_descriptor(vcpu, tss->es, VCPU_SREG_ES))
  4337. return 1;
  4338. if (kvm_load_segment_descriptor(vcpu, tss->cs, VCPU_SREG_CS))
  4339. return 1;
  4340. if (kvm_load_segment_descriptor(vcpu, tss->ss, VCPU_SREG_SS))
  4341. return 1;
  4342. if (kvm_load_segment_descriptor(vcpu, tss->ds, VCPU_SREG_DS))
  4343. return 1;
  4344. if (kvm_load_segment_descriptor(vcpu, tss->fs, VCPU_SREG_FS))
  4345. return 1;
  4346. if (kvm_load_segment_descriptor(vcpu, tss->gs, VCPU_SREG_GS))
  4347. return 1;
  4348. return 0;
  4349. }
  4350. static void save_state_to_tss16(struct kvm_vcpu *vcpu,
  4351. struct tss_segment_16 *tss)
  4352. {
  4353. tss->ip = kvm_rip_read(vcpu);
  4354. tss->flag = kvm_get_rflags(vcpu);
  4355. tss->ax = kvm_register_read(vcpu, VCPU_REGS_RAX);
  4356. tss->cx = kvm_register_read(vcpu, VCPU_REGS_RCX);
  4357. tss->dx = kvm_register_read(vcpu, VCPU_REGS_RDX);
  4358. tss->bx = kvm_register_read(vcpu, VCPU_REGS_RBX);
  4359. tss->sp = kvm_register_read(vcpu, VCPU_REGS_RSP);
  4360. tss->bp = kvm_register_read(vcpu, VCPU_REGS_RBP);
  4361. tss->si = kvm_register_read(vcpu, VCPU_REGS_RSI);
  4362. tss->di = kvm_register_read(vcpu, VCPU_REGS_RDI);
  4363. tss->es = get_segment_selector(vcpu, VCPU_SREG_ES);
  4364. tss->cs = get_segment_selector(vcpu, VCPU_SREG_CS);
  4365. tss->ss = get_segment_selector(vcpu, VCPU_SREG_SS);
  4366. tss->ds = get_segment_selector(vcpu, VCPU_SREG_DS);
  4367. tss->ldt = get_segment_selector(vcpu, VCPU_SREG_LDTR);
  4368. }
  4369. static int load_state_from_tss16(struct kvm_vcpu *vcpu,
  4370. struct tss_segment_16 *tss)
  4371. {
  4372. kvm_rip_write(vcpu, tss->ip);
  4373. kvm_set_rflags(vcpu, tss->flag | 2);
  4374. kvm_register_write(vcpu, VCPU_REGS_RAX, tss->ax);
  4375. kvm_register_write(vcpu, VCPU_REGS_RCX, tss->cx);
  4376. kvm_register_write(vcpu, VCPU_REGS_RDX, tss->dx);
  4377. kvm_register_write(vcpu, VCPU_REGS_RBX, tss->bx);
  4378. kvm_register_write(vcpu, VCPU_REGS_RSP, tss->sp);
  4379. kvm_register_write(vcpu, VCPU_REGS_RBP, tss->bp);
  4380. kvm_register_write(vcpu, VCPU_REGS_RSI, tss->si);
  4381. kvm_register_write(vcpu, VCPU_REGS_RDI, tss->di);
  4382. /*
  4383. * SDM says that segment selectors are loaded before segment
  4384. * descriptors
  4385. */
  4386. kvm_load_segment_selector(vcpu, tss->ldt, VCPU_SREG_LDTR);
  4387. kvm_load_segment_selector(vcpu, tss->es, VCPU_SREG_ES);
  4388. kvm_load_segment_selector(vcpu, tss->cs, VCPU_SREG_CS);
  4389. kvm_load_segment_selector(vcpu, tss->ss, VCPU_SREG_SS);
  4390. kvm_load_segment_selector(vcpu, tss->ds, VCPU_SREG_DS);
  4391. /*
  4392. * Now load segment descriptors. If fault happenes at this stage
  4393. * it is handled in a context of new task
  4394. */
  4395. if (kvm_load_segment_descriptor(vcpu, tss->ldt, VCPU_SREG_LDTR))
  4396. return 1;
  4397. if (kvm_load_segment_descriptor(vcpu, tss->es, VCPU_SREG_ES))
  4398. return 1;
  4399. if (kvm_load_segment_descriptor(vcpu, tss->cs, VCPU_SREG_CS))
  4400. return 1;
  4401. if (kvm_load_segment_descriptor(vcpu, tss->ss, VCPU_SREG_SS))
  4402. return 1;
  4403. if (kvm_load_segment_descriptor(vcpu, tss->ds, VCPU_SREG_DS))
  4404. return 1;
  4405. return 0;
  4406. }
  4407. static int kvm_task_switch_16(struct kvm_vcpu *vcpu, u16 tss_selector,
  4408. u16 old_tss_sel, u32 old_tss_base,
  4409. struct desc_struct *nseg_desc)
  4410. {
  4411. struct tss_segment_16 tss_segment_16;
  4412. int ret = 0;
  4413. if (kvm_read_guest(vcpu->kvm, old_tss_base, &tss_segment_16,
  4414. sizeof tss_segment_16))
  4415. goto out;
  4416. save_state_to_tss16(vcpu, &tss_segment_16);
  4417. if (kvm_write_guest(vcpu->kvm, old_tss_base, &tss_segment_16,
  4418. sizeof tss_segment_16))
  4419. goto out;
  4420. if (kvm_read_guest(vcpu->kvm, get_tss_base_addr_read(vcpu, nseg_desc),
  4421. &tss_segment_16, sizeof tss_segment_16))
  4422. goto out;
  4423. if (old_tss_sel != 0xffff) {
  4424. tss_segment_16.prev_task_link = old_tss_sel;
  4425. if (kvm_write_guest(vcpu->kvm,
  4426. get_tss_base_addr_write(vcpu, nseg_desc),
  4427. &tss_segment_16.prev_task_link,
  4428. sizeof tss_segment_16.prev_task_link))
  4429. goto out;
  4430. }
  4431. if (load_state_from_tss16(vcpu, &tss_segment_16))
  4432. goto out;
  4433. ret = 1;
  4434. out:
  4435. return ret;
  4436. }
  4437. static int kvm_task_switch_32(struct kvm_vcpu *vcpu, u16 tss_selector,
  4438. u16 old_tss_sel, u32 old_tss_base,
  4439. struct desc_struct *nseg_desc)
  4440. {
  4441. struct tss_segment_32 tss_segment_32;
  4442. int ret = 0;
  4443. if (kvm_read_guest(vcpu->kvm, old_tss_base, &tss_segment_32,
  4444. sizeof tss_segment_32))
  4445. goto out;
  4446. save_state_to_tss32(vcpu, &tss_segment_32);
  4447. if (kvm_write_guest(vcpu->kvm, old_tss_base, &tss_segment_32,
  4448. sizeof tss_segment_32))
  4449. goto out;
  4450. if (kvm_read_guest(vcpu->kvm, get_tss_base_addr_read(vcpu, nseg_desc),
  4451. &tss_segment_32, sizeof tss_segment_32))
  4452. goto out;
  4453. if (old_tss_sel != 0xffff) {
  4454. tss_segment_32.prev_task_link = old_tss_sel;
  4455. if (kvm_write_guest(vcpu->kvm,
  4456. get_tss_base_addr_write(vcpu, nseg_desc),
  4457. &tss_segment_32.prev_task_link,
  4458. sizeof tss_segment_32.prev_task_link))
  4459. goto out;
  4460. }
  4461. if (load_state_from_tss32(vcpu, &tss_segment_32))
  4462. goto out;
  4463. ret = 1;
  4464. out:
  4465. return ret;
  4466. }
  4467. int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason)
  4468. {
  4469. struct kvm_segment tr_seg;
  4470. struct desc_struct cseg_desc;
  4471. struct desc_struct nseg_desc;
  4472. int ret = 0;
  4473. u32 old_tss_base = get_segment_base(vcpu, VCPU_SREG_TR);
  4474. u16 old_tss_sel = get_segment_selector(vcpu, VCPU_SREG_TR);
  4475. old_tss_base = kvm_mmu_gva_to_gpa_write(vcpu, old_tss_base, NULL);
  4476. /* FIXME: Handle errors. Failure to read either TSS or their
  4477. * descriptors should generate a pagefault.
  4478. */
  4479. if (load_guest_segment_descriptor(vcpu, tss_selector, &nseg_desc))
  4480. goto out;
  4481. if (load_guest_segment_descriptor(vcpu, old_tss_sel, &cseg_desc))
  4482. goto out;
  4483. if (reason != TASK_SWITCH_IRET) {
  4484. int cpl;
  4485. cpl = kvm_x86_ops->get_cpl(vcpu);
  4486. if ((tss_selector & 3) > nseg_desc.dpl || cpl > nseg_desc.dpl) {
  4487. kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
  4488. return 1;
  4489. }
  4490. }
  4491. if (!nseg_desc.p || get_desc_limit(&nseg_desc) < 0x67) {
  4492. kvm_queue_exception_e(vcpu, TS_VECTOR, tss_selector & 0xfffc);
  4493. return 1;
  4494. }
  4495. if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
  4496. cseg_desc.type &= ~(1 << 1); //clear the B flag
  4497. save_guest_segment_descriptor(vcpu, old_tss_sel, &cseg_desc);
  4498. }
  4499. if (reason == TASK_SWITCH_IRET) {
  4500. u32 eflags = kvm_get_rflags(vcpu);
  4501. kvm_set_rflags(vcpu, eflags & ~X86_EFLAGS_NT);
  4502. }
  4503. /* set back link to prev task only if NT bit is set in eflags
  4504. note that old_tss_sel is not used afetr this point */
  4505. if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
  4506. old_tss_sel = 0xffff;
  4507. if (nseg_desc.type & 8)
  4508. ret = kvm_task_switch_32(vcpu, tss_selector, old_tss_sel,
  4509. old_tss_base, &nseg_desc);
  4510. else
  4511. ret = kvm_task_switch_16(vcpu, tss_selector, old_tss_sel,
  4512. old_tss_base, &nseg_desc);
  4513. if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE) {
  4514. u32 eflags = kvm_get_rflags(vcpu);
  4515. kvm_set_rflags(vcpu, eflags | X86_EFLAGS_NT);
  4516. }
  4517. if (reason != TASK_SWITCH_IRET) {
  4518. nseg_desc.type |= (1 << 1);
  4519. save_guest_segment_descriptor(vcpu, tss_selector,
  4520. &nseg_desc);
  4521. }
  4522. kvm_x86_ops->set_cr0(vcpu, kvm_read_cr0(vcpu) | X86_CR0_TS);
  4523. seg_desct_to_kvm_desct(&nseg_desc, tss_selector, &tr_seg);
  4524. tr_seg.type = 11;
  4525. kvm_set_segment(vcpu, &tr_seg, VCPU_SREG_TR);
  4526. out:
  4527. return ret;
  4528. }
  4529. EXPORT_SYMBOL_GPL(kvm_task_switch);
  4530. int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
  4531. struct kvm_sregs *sregs)
  4532. {
  4533. int mmu_reset_needed = 0;
  4534. int pending_vec, max_bits;
  4535. struct descriptor_table dt;
  4536. vcpu_load(vcpu);
  4537. dt.limit = sregs->idt.limit;
  4538. dt.base = sregs->idt.base;
  4539. kvm_x86_ops->set_idt(vcpu, &dt);
  4540. dt.limit = sregs->gdt.limit;
  4541. dt.base = sregs->gdt.base;
  4542. kvm_x86_ops->set_gdt(vcpu, &dt);
  4543. vcpu->arch.cr2 = sregs->cr2;
  4544. mmu_reset_needed |= vcpu->arch.cr3 != sregs->cr3;
  4545. vcpu->arch.cr3 = sregs->cr3;
  4546. kvm_set_cr8(vcpu, sregs->cr8);
  4547. mmu_reset_needed |= vcpu->arch.efer != sregs->efer;
  4548. kvm_x86_ops->set_efer(vcpu, sregs->efer);
  4549. kvm_set_apic_base(vcpu, sregs->apic_base);
  4550. mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;
  4551. kvm_x86_ops->set_cr0(vcpu, sregs->cr0);
  4552. vcpu->arch.cr0 = sregs->cr0;
  4553. mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
  4554. kvm_x86_ops->set_cr4(vcpu, sregs->cr4);
  4555. if (!is_long_mode(vcpu) && is_pae(vcpu)) {
  4556. load_pdptrs(vcpu, vcpu->arch.cr3);
  4557. mmu_reset_needed = 1;
  4558. }
  4559. if (mmu_reset_needed)
  4560. kvm_mmu_reset_context(vcpu);
  4561. max_bits = (sizeof sregs->interrupt_bitmap) << 3;
  4562. pending_vec = find_first_bit(
  4563. (const unsigned long *)sregs->interrupt_bitmap, max_bits);
  4564. if (pending_vec < max_bits) {
  4565. kvm_queue_interrupt(vcpu, pending_vec, false);
  4566. pr_debug("Set back pending irq %d\n", pending_vec);
  4567. if (irqchip_in_kernel(vcpu->kvm))
  4568. kvm_pic_clear_isr_ack(vcpu->kvm);
  4569. }
  4570. kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
  4571. kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
  4572. kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
  4573. kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
  4574. kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
  4575. kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
  4576. kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
  4577. kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
  4578. update_cr8_intercept(vcpu);
  4579. /* Older userspace won't unhalt the vcpu on reset. */
  4580. if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
  4581. sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&
  4582. !is_protmode(vcpu))
  4583. vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
  4584. vcpu_put(vcpu);
  4585. return 0;
  4586. }
  4587. int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
  4588. struct kvm_guest_debug *dbg)
  4589. {
  4590. unsigned long rflags;
  4591. int i, r;
  4592. vcpu_load(vcpu);
  4593. if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) {
  4594. r = -EBUSY;
  4595. if (vcpu->arch.exception.pending)
  4596. goto unlock_out;
  4597. if (dbg->control & KVM_GUESTDBG_INJECT_DB)
  4598. kvm_queue_exception(vcpu, DB_VECTOR);
  4599. else
  4600. kvm_queue_exception(vcpu, BP_VECTOR);
  4601. }
  4602. /*
  4603. * Read rflags as long as potentially injected trace flags are still
  4604. * filtered out.
  4605. */
  4606. rflags = kvm_get_rflags(vcpu);
  4607. vcpu->guest_debug = dbg->control;
  4608. if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))
  4609. vcpu->guest_debug = 0;
  4610. if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
  4611. for (i = 0; i < KVM_NR_DB_REGS; ++i)
  4612. vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
  4613. vcpu->arch.switch_db_regs =
  4614. (dbg->arch.debugreg[7] & DR7_BP_EN_MASK);
  4615. } else {
  4616. for (i = 0; i < KVM_NR_DB_REGS; i++)
  4617. vcpu->arch.eff_db[i] = vcpu->arch.db[i];
  4618. vcpu->arch.switch_db_regs = (vcpu->arch.dr7 & DR7_BP_EN_MASK);
  4619. }
  4620. if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
  4621. vcpu->arch.singlestep_cs =
  4622. get_segment_selector(vcpu, VCPU_SREG_CS);
  4623. vcpu->arch.singlestep_rip = kvm_rip_read(vcpu);
  4624. }
  4625. /*
  4626. * Trigger an rflags update that will inject or remove the trace
  4627. * flags.
  4628. */
  4629. kvm_set_rflags(vcpu, rflags);
  4630. kvm_x86_ops->set_guest_debug(vcpu, dbg);
  4631. r = 0;
  4632. unlock_out:
  4633. vcpu_put(vcpu);
  4634. return r;
  4635. }
  4636. /*
  4637. * fxsave fpu state. Taken from x86_64/processor.h. To be killed when
  4638. * we have asm/x86/processor.h
  4639. */
  4640. struct fxsave {
  4641. u16 cwd;
  4642. u16 swd;
  4643. u16 twd;
  4644. u16 fop;
  4645. u64 rip;
  4646. u64 rdp;
  4647. u32 mxcsr;
  4648. u32 mxcsr_mask;
  4649. u32 st_space[32]; /* 8*16 bytes for each FP-reg = 128 bytes */
  4650. #ifdef CONFIG_X86_64
  4651. u32 xmm_space[64]; /* 16*16 bytes for each XMM-reg = 256 bytes */
  4652. #else
  4653. u32 xmm_space[32]; /* 8*16 bytes for each XMM-reg = 128 bytes */
  4654. #endif
  4655. };
  4656. /*
  4657. * Translate a guest virtual address to a guest physical address.
  4658. */
  4659. int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
  4660. struct kvm_translation *tr)
  4661. {
  4662. unsigned long vaddr = tr->linear_address;
  4663. gpa_t gpa;
  4664. int idx;
  4665. vcpu_load(vcpu);
  4666. idx = srcu_read_lock(&vcpu->kvm->srcu);
  4667. gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);
  4668. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  4669. tr->physical_address = gpa;
  4670. tr->valid = gpa != UNMAPPED_GVA;
  4671. tr->writeable = 1;
  4672. tr->usermode = 0;
  4673. vcpu_put(vcpu);
  4674. return 0;
  4675. }
  4676. int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
  4677. {
  4678. struct fxsave *fxsave = (struct fxsave *)&vcpu->arch.guest_fx_image;
  4679. vcpu_load(vcpu);
  4680. memcpy(fpu->fpr, fxsave->st_space, 128);
  4681. fpu->fcw = fxsave->cwd;
  4682. fpu->fsw = fxsave->swd;
  4683. fpu->ftwx = fxsave->twd;
  4684. fpu->last_opcode = fxsave->fop;
  4685. fpu->last_ip = fxsave->rip;
  4686. fpu->last_dp = fxsave->rdp;
  4687. memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space);
  4688. vcpu_put(vcpu);
  4689. return 0;
  4690. }
  4691. int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
  4692. {
  4693. struct fxsave *fxsave = (struct fxsave *)&vcpu->arch.guest_fx_image;
  4694. vcpu_load(vcpu);
  4695. memcpy(fxsave->st_space, fpu->fpr, 128);
  4696. fxsave->cwd = fpu->fcw;
  4697. fxsave->swd = fpu->fsw;
  4698. fxsave->twd = fpu->ftwx;
  4699. fxsave->fop = fpu->last_opcode;
  4700. fxsave->rip = fpu->last_ip;
  4701. fxsave->rdp = fpu->last_dp;
  4702. memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space);
  4703. vcpu_put(vcpu);
  4704. return 0;
  4705. }
  4706. void fx_init(struct kvm_vcpu *vcpu)
  4707. {
  4708. unsigned after_mxcsr_mask;
  4709. /*
  4710. * Touch the fpu the first time in non atomic context as if
  4711. * this is the first fpu instruction the exception handler
  4712. * will fire before the instruction returns and it'll have to
  4713. * allocate ram with GFP_KERNEL.
  4714. */
  4715. if (!used_math())
  4716. kvm_fx_save(&vcpu->arch.host_fx_image);
  4717. /* Initialize guest FPU by resetting ours and saving into guest's */
  4718. preempt_disable();
  4719. kvm_fx_save(&vcpu->arch.host_fx_image);
  4720. kvm_fx_finit();
  4721. kvm_fx_save(&vcpu->arch.guest_fx_image);
  4722. kvm_fx_restore(&vcpu->arch.host_fx_image);
  4723. preempt_enable();
  4724. vcpu->arch.cr0 |= X86_CR0_ET;
  4725. after_mxcsr_mask = offsetof(struct i387_fxsave_struct, st_space);
  4726. vcpu->arch.guest_fx_image.mxcsr = 0x1f80;
  4727. memset((void *)&vcpu->arch.guest_fx_image + after_mxcsr_mask,
  4728. 0, sizeof(struct i387_fxsave_struct) - after_mxcsr_mask);
  4729. }
  4730. EXPORT_SYMBOL_GPL(fx_init);
  4731. void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
  4732. {
  4733. if (vcpu->guest_fpu_loaded)
  4734. return;
  4735. vcpu->guest_fpu_loaded = 1;
  4736. kvm_fx_save(&vcpu->arch.host_fx_image);
  4737. kvm_fx_restore(&vcpu->arch.guest_fx_image);
  4738. trace_kvm_fpu(1);
  4739. }
  4740. void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
  4741. {
  4742. if (!vcpu->guest_fpu_loaded)
  4743. return;
  4744. vcpu->guest_fpu_loaded = 0;
  4745. kvm_fx_save(&vcpu->arch.guest_fx_image);
  4746. kvm_fx_restore(&vcpu->arch.host_fx_image);
  4747. ++vcpu->stat.fpu_reload;
  4748. set_bit(KVM_REQ_DEACTIVATE_FPU, &vcpu->requests);
  4749. trace_kvm_fpu(0);
  4750. }
  4751. void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
  4752. {
  4753. if (vcpu->arch.time_page) {
  4754. kvm_release_page_dirty(vcpu->arch.time_page);
  4755. vcpu->arch.time_page = NULL;
  4756. }
  4757. kvm_x86_ops->vcpu_free(vcpu);
  4758. }
  4759. struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
  4760. unsigned int id)
  4761. {
  4762. return kvm_x86_ops->vcpu_create(kvm, id);
  4763. }
  4764. int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
  4765. {
  4766. int r;
  4767. /* We do fxsave: this must be aligned. */
  4768. BUG_ON((unsigned long)&vcpu->arch.host_fx_image & 0xF);
  4769. vcpu->arch.mtrr_state.have_fixed = 1;
  4770. vcpu_load(vcpu);
  4771. r = kvm_arch_vcpu_reset(vcpu);
  4772. if (r == 0)
  4773. r = kvm_mmu_setup(vcpu);
  4774. vcpu_put(vcpu);
  4775. if (r < 0)
  4776. goto free_vcpu;
  4777. return 0;
  4778. free_vcpu:
  4779. kvm_x86_ops->vcpu_free(vcpu);
  4780. return r;
  4781. }
  4782. void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
  4783. {
  4784. vcpu_load(vcpu);
  4785. kvm_mmu_unload(vcpu);
  4786. vcpu_put(vcpu);
  4787. kvm_x86_ops->vcpu_free(vcpu);
  4788. }
  4789. int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu)
  4790. {
  4791. vcpu->arch.nmi_pending = false;
  4792. vcpu->arch.nmi_injected = false;
  4793. vcpu->arch.switch_db_regs = 0;
  4794. memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
  4795. vcpu->arch.dr6 = DR6_FIXED_1;
  4796. vcpu->arch.dr7 = DR7_FIXED_1;
  4797. return kvm_x86_ops->vcpu_reset(vcpu);
  4798. }
  4799. int kvm_arch_hardware_enable(void *garbage)
  4800. {
  4801. /*
  4802. * Since this may be called from a hotplug notifcation,
  4803. * we can't get the CPU frequency directly.
  4804. */
  4805. if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
  4806. int cpu = raw_smp_processor_id();
  4807. per_cpu(cpu_tsc_khz, cpu) = 0;
  4808. }
  4809. kvm_shared_msr_cpu_online();
  4810. return kvm_x86_ops->hardware_enable(garbage);
  4811. }
  4812. void kvm_arch_hardware_disable(void *garbage)
  4813. {
  4814. kvm_x86_ops->hardware_disable(garbage);
  4815. drop_user_return_notifiers(garbage);
  4816. }
  4817. int kvm_arch_hardware_setup(void)
  4818. {
  4819. return kvm_x86_ops->hardware_setup();
  4820. }
  4821. void kvm_arch_hardware_unsetup(void)
  4822. {
  4823. kvm_x86_ops->hardware_unsetup();
  4824. }
  4825. void kvm_arch_check_processor_compat(void *rtn)
  4826. {
  4827. kvm_x86_ops->check_processor_compatibility(rtn);
  4828. }
  4829. int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
  4830. {
  4831. struct page *page;
  4832. struct kvm *kvm;
  4833. int r;
  4834. BUG_ON(vcpu->kvm == NULL);
  4835. kvm = vcpu->kvm;
  4836. vcpu->arch.mmu.root_hpa = INVALID_PAGE;
  4837. if (!irqchip_in_kernel(kvm) || kvm_vcpu_is_bsp(vcpu))
  4838. vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
  4839. else
  4840. vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
  4841. page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  4842. if (!page) {
  4843. r = -ENOMEM;
  4844. goto fail;
  4845. }
  4846. vcpu->arch.pio_data = page_address(page);
  4847. r = kvm_mmu_create(vcpu);
  4848. if (r < 0)
  4849. goto fail_free_pio_data;
  4850. if (irqchip_in_kernel(kvm)) {
  4851. r = kvm_create_lapic(vcpu);
  4852. if (r < 0)
  4853. goto fail_mmu_destroy;
  4854. }
  4855. vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4,
  4856. GFP_KERNEL);
  4857. if (!vcpu->arch.mce_banks) {
  4858. r = -ENOMEM;
  4859. goto fail_free_lapic;
  4860. }
  4861. vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;
  4862. return 0;
  4863. fail_free_lapic:
  4864. kvm_free_lapic(vcpu);
  4865. fail_mmu_destroy:
  4866. kvm_mmu_destroy(vcpu);
  4867. fail_free_pio_data:
  4868. free_page((unsigned long)vcpu->arch.pio_data);
  4869. fail:
  4870. return r;
  4871. }
  4872. void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
  4873. {
  4874. int idx;
  4875. kfree(vcpu->arch.mce_banks);
  4876. kvm_free_lapic(vcpu);
  4877. idx = srcu_read_lock(&vcpu->kvm->srcu);
  4878. kvm_mmu_destroy(vcpu);
  4879. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  4880. free_page((unsigned long)vcpu->arch.pio_data);
  4881. }
  4882. struct kvm *kvm_arch_create_vm(void)
  4883. {
  4884. struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
  4885. if (!kvm)
  4886. return ERR_PTR(-ENOMEM);
  4887. kvm->arch.aliases = kzalloc(sizeof(struct kvm_mem_aliases), GFP_KERNEL);
  4888. if (!kvm->arch.aliases) {
  4889. kfree(kvm);
  4890. return ERR_PTR(-ENOMEM);
  4891. }
  4892. INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
  4893. INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
  4894. /* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
  4895. set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
  4896. rdtscll(kvm->arch.vm_init_tsc);
  4897. return kvm;
  4898. }
  4899. static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
  4900. {
  4901. vcpu_load(vcpu);
  4902. kvm_mmu_unload(vcpu);
  4903. vcpu_put(vcpu);
  4904. }
  4905. static void kvm_free_vcpus(struct kvm *kvm)
  4906. {
  4907. unsigned int i;
  4908. struct kvm_vcpu *vcpu;
  4909. /*
  4910. * Unpin any mmu pages first.
  4911. */
  4912. kvm_for_each_vcpu(i, vcpu, kvm)
  4913. kvm_unload_vcpu_mmu(vcpu);
  4914. kvm_for_each_vcpu(i, vcpu, kvm)
  4915. kvm_arch_vcpu_free(vcpu);
  4916. mutex_lock(&kvm->lock);
  4917. for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
  4918. kvm->vcpus[i] = NULL;
  4919. atomic_set(&kvm->online_vcpus, 0);
  4920. mutex_unlock(&kvm->lock);
  4921. }
  4922. void kvm_arch_sync_events(struct kvm *kvm)
  4923. {
  4924. kvm_free_all_assigned_devices(kvm);
  4925. }
  4926. void kvm_arch_destroy_vm(struct kvm *kvm)
  4927. {
  4928. kvm_iommu_unmap_guest(kvm);
  4929. kvm_free_pit(kvm);
  4930. kfree(kvm->arch.vpic);
  4931. kfree(kvm->arch.vioapic);
  4932. kvm_free_vcpus(kvm);
  4933. kvm_free_physmem(kvm);
  4934. if (kvm->arch.apic_access_page)
  4935. put_page(kvm->arch.apic_access_page);
  4936. if (kvm->arch.ept_identity_pagetable)
  4937. put_page(kvm->arch.ept_identity_pagetable);
  4938. cleanup_srcu_struct(&kvm->srcu);
  4939. kfree(kvm->arch.aliases);
  4940. kfree(kvm);
  4941. }
  4942. int kvm_arch_prepare_memory_region(struct kvm *kvm,
  4943. struct kvm_memory_slot *memslot,
  4944. struct kvm_memory_slot old,
  4945. struct kvm_userspace_memory_region *mem,
  4946. int user_alloc)
  4947. {
  4948. int npages = memslot->npages;
  4949. /*To keep backward compatibility with older userspace,
  4950. *x86 needs to hanlde !user_alloc case.
  4951. */
  4952. if (!user_alloc) {
  4953. if (npages && !old.rmap) {
  4954. unsigned long userspace_addr;
  4955. down_write(&current->mm->mmap_sem);
  4956. userspace_addr = do_mmap(NULL, 0,
  4957. npages * PAGE_SIZE,
  4958. PROT_READ | PROT_WRITE,
  4959. MAP_PRIVATE | MAP_ANONYMOUS,
  4960. 0);
  4961. up_write(&current->mm->mmap_sem);
  4962. if (IS_ERR((void *)userspace_addr))
  4963. return PTR_ERR((void *)userspace_addr);
  4964. memslot->userspace_addr = userspace_addr;
  4965. }
  4966. }
  4967. return 0;
  4968. }
  4969. void kvm_arch_commit_memory_region(struct kvm *kvm,
  4970. struct kvm_userspace_memory_region *mem,
  4971. struct kvm_memory_slot old,
  4972. int user_alloc)
  4973. {
  4974. int npages = mem->memory_size >> PAGE_SHIFT;
  4975. if (!user_alloc && !old.user_alloc && old.rmap && !npages) {
  4976. int ret;
  4977. down_write(&current->mm->mmap_sem);
  4978. ret = do_munmap(current->mm, old.userspace_addr,
  4979. old.npages * PAGE_SIZE);
  4980. up_write(&current->mm->mmap_sem);
  4981. if (ret < 0)
  4982. printk(KERN_WARNING
  4983. "kvm_vm_ioctl_set_memory_region: "
  4984. "failed to munmap memory\n");
  4985. }
  4986. spin_lock(&kvm->mmu_lock);
  4987. if (!kvm->arch.n_requested_mmu_pages) {
  4988. unsigned int nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm);
  4989. kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
  4990. }
  4991. kvm_mmu_slot_remove_write_access(kvm, mem->slot);
  4992. spin_unlock(&kvm->mmu_lock);
  4993. }
  4994. void kvm_arch_flush_shadow(struct kvm *kvm)
  4995. {
  4996. kvm_mmu_zap_all(kvm);
  4997. kvm_reload_remote_mmus(kvm);
  4998. }
  4999. int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
  5000. {
  5001. return vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE
  5002. || vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED
  5003. || vcpu->arch.nmi_pending ||
  5004. (kvm_arch_interrupt_allowed(vcpu) &&
  5005. kvm_cpu_has_interrupt(vcpu));
  5006. }
  5007. void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
  5008. {
  5009. int me;
  5010. int cpu = vcpu->cpu;
  5011. if (waitqueue_active(&vcpu->wq)) {
  5012. wake_up_interruptible(&vcpu->wq);
  5013. ++vcpu->stat.halt_wakeup;
  5014. }
  5015. me = get_cpu();
  5016. if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
  5017. if (!test_and_set_bit(KVM_REQ_KICK, &vcpu->requests))
  5018. smp_send_reschedule(cpu);
  5019. put_cpu();
  5020. }
  5021. int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
  5022. {
  5023. return kvm_x86_ops->interrupt_allowed(vcpu);
  5024. }
  5025. unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)
  5026. {
  5027. unsigned long rflags;
  5028. rflags = kvm_x86_ops->get_rflags(vcpu);
  5029. if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
  5030. rflags &= ~(unsigned long)(X86_EFLAGS_TF | X86_EFLAGS_RF);
  5031. return rflags;
  5032. }
  5033. EXPORT_SYMBOL_GPL(kvm_get_rflags);
  5034. void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
  5035. {
  5036. if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
  5037. vcpu->arch.singlestep_cs ==
  5038. get_segment_selector(vcpu, VCPU_SREG_CS) &&
  5039. vcpu->arch.singlestep_rip == kvm_rip_read(vcpu))
  5040. rflags |= X86_EFLAGS_TF | X86_EFLAGS_RF;
  5041. kvm_x86_ops->set_rflags(vcpu, rflags);
  5042. }
  5043. EXPORT_SYMBOL_GPL(kvm_set_rflags);
  5044. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
  5045. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
  5046. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);
  5047. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr);
  5048. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr);
  5049. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmrun);
  5050. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit);
  5051. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject);
  5052. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit);
  5053. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga);
  5054. EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit);