emulate.c 121 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734
  1. /******************************************************************************
  2. * emulate.c
  3. *
  4. * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
  5. *
  6. * Copyright (c) 2005 Keir Fraser
  7. *
  8. * Linux coding style, mod r/m decoder, segment base fixes, real-mode
  9. * privileged instructions:
  10. *
  11. * Copyright (C) 2006 Qumranet
  12. * Copyright 2010 Red Hat, Inc. and/or its affiliates.
  13. *
  14. * Avi Kivity <avi@qumranet.com>
  15. * Yaniv Kamay <yaniv@qumranet.com>
  16. *
  17. * This work is licensed under the terms of the GNU GPL, version 2. See
  18. * the COPYING file in the top-level directory.
  19. *
  20. * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
  21. */
  22. #include <linux/kvm_host.h>
  23. #include "kvm_cache_regs.h"
  24. #include <linux/module.h>
  25. #include <asm/kvm_emulate.h>
  26. #include <linux/stringify.h>
  27. #include "x86.h"
  28. #include "tss.h"
  29. /*
  30. * Operand types
  31. */
  32. #define OpNone 0ull
  33. #define OpImplicit 1ull /* No generic decode */
  34. #define OpReg 2ull /* Register */
  35. #define OpMem 3ull /* Memory */
  36. #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
  37. #define OpDI 5ull /* ES:DI/EDI/RDI */
  38. #define OpMem64 6ull /* Memory, 64-bit */
  39. #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
  40. #define OpDX 8ull /* DX register */
  41. #define OpCL 9ull /* CL register (for shifts) */
  42. #define OpImmByte 10ull /* 8-bit sign extended immediate */
  43. #define OpOne 11ull /* Implied 1 */
  44. #define OpImm 12ull /* Sign extended up to 32-bit immediate */
  45. #define OpMem16 13ull /* Memory operand (16-bit). */
  46. #define OpMem32 14ull /* Memory operand (32-bit). */
  47. #define OpImmU 15ull /* Immediate operand, zero extended */
  48. #define OpSI 16ull /* SI/ESI/RSI */
  49. #define OpImmFAddr 17ull /* Immediate far address */
  50. #define OpMemFAddr 18ull /* Far address in memory */
  51. #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
  52. #define OpES 20ull /* ES */
  53. #define OpCS 21ull /* CS */
  54. #define OpSS 22ull /* SS */
  55. #define OpDS 23ull /* DS */
  56. #define OpFS 24ull /* FS */
  57. #define OpGS 25ull /* GS */
  58. #define OpMem8 26ull /* 8-bit zero extended memory operand */
  59. #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
  60. #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
  61. #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
  62. #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
  63. #define OpBits 5 /* Width of operand field */
  64. #define OpMask ((1ull << OpBits) - 1)
  65. /*
  66. * Opcode effective-address decode tables.
  67. * Note that we only emulate instructions that have at least one memory
  68. * operand (excluding implicit stack references). We assume that stack
  69. * references and instruction fetches will never occur in special memory
  70. * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
  71. * not be handled.
  72. */
  73. /* Operand sizes: 8-bit operands or specified/overridden size. */
  74. #define ByteOp (1<<0) /* 8-bit operands. */
  75. /* Destination operand type. */
  76. #define DstShift 1
  77. #define ImplicitOps (OpImplicit << DstShift)
  78. #define DstReg (OpReg << DstShift)
  79. #define DstMem (OpMem << DstShift)
  80. #define DstAcc (OpAcc << DstShift)
  81. #define DstDI (OpDI << DstShift)
  82. #define DstMem64 (OpMem64 << DstShift)
  83. #define DstImmUByte (OpImmUByte << DstShift)
  84. #define DstDX (OpDX << DstShift)
  85. #define DstAccLo (OpAccLo << DstShift)
  86. #define DstMask (OpMask << DstShift)
  87. /* Source operand type. */
  88. #define SrcShift 6
  89. #define SrcNone (OpNone << SrcShift)
  90. #define SrcReg (OpReg << SrcShift)
  91. #define SrcMem (OpMem << SrcShift)
  92. #define SrcMem16 (OpMem16 << SrcShift)
  93. #define SrcMem32 (OpMem32 << SrcShift)
  94. #define SrcImm (OpImm << SrcShift)
  95. #define SrcImmByte (OpImmByte << SrcShift)
  96. #define SrcOne (OpOne << SrcShift)
  97. #define SrcImmUByte (OpImmUByte << SrcShift)
  98. #define SrcImmU (OpImmU << SrcShift)
  99. #define SrcSI (OpSI << SrcShift)
  100. #define SrcXLat (OpXLat << SrcShift)
  101. #define SrcImmFAddr (OpImmFAddr << SrcShift)
  102. #define SrcMemFAddr (OpMemFAddr << SrcShift)
  103. #define SrcAcc (OpAcc << SrcShift)
  104. #define SrcImmU16 (OpImmU16 << SrcShift)
  105. #define SrcImm64 (OpImm64 << SrcShift)
  106. #define SrcDX (OpDX << SrcShift)
  107. #define SrcMem8 (OpMem8 << SrcShift)
  108. #define SrcAccHi (OpAccHi << SrcShift)
  109. #define SrcMask (OpMask << SrcShift)
  110. #define BitOp (1<<11)
  111. #define MemAbs (1<<12) /* Memory operand is absolute displacement */
  112. #define String (1<<13) /* String instruction (rep capable) */
  113. #define Stack (1<<14) /* Stack instruction (push/pop) */
  114. #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
  115. #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
  116. #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
  117. #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
  118. #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
  119. #define Escape (5<<15) /* Escape to coprocessor instruction */
  120. #define Sse (1<<18) /* SSE Vector instruction */
  121. /* Generic ModRM decode. */
  122. #define ModRM (1<<19)
  123. /* Destination is only written; never read. */
  124. #define Mov (1<<20)
  125. /* Misc flags */
  126. #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
  127. #define VendorSpecific (1<<22) /* Vendor specific instruction */
  128. #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
  129. #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
  130. #define Undefined (1<<25) /* No Such Instruction */
  131. #define Lock (1<<26) /* lock prefix is allowed for the instruction */
  132. #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
  133. #define No64 (1<<28)
  134. #define PageTable (1 << 29) /* instruction used to write page table */
  135. #define NotImpl (1 << 30) /* instruction is not implemented */
  136. /* Source 2 operand type */
  137. #define Src2Shift (31)
  138. #define Src2None (OpNone << Src2Shift)
  139. #define Src2Mem (OpMem << Src2Shift)
  140. #define Src2CL (OpCL << Src2Shift)
  141. #define Src2ImmByte (OpImmByte << Src2Shift)
  142. #define Src2One (OpOne << Src2Shift)
  143. #define Src2Imm (OpImm << Src2Shift)
  144. #define Src2ES (OpES << Src2Shift)
  145. #define Src2CS (OpCS << Src2Shift)
  146. #define Src2SS (OpSS << Src2Shift)
  147. #define Src2DS (OpDS << Src2Shift)
  148. #define Src2FS (OpFS << Src2Shift)
  149. #define Src2GS (OpGS << Src2Shift)
  150. #define Src2Mask (OpMask << Src2Shift)
  151. #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
  152. #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
  153. #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
  154. #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
  155. #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
  156. #define NoWrite ((u64)1 << 45) /* No writeback */
  157. #define SrcWrite ((u64)1 << 46) /* Write back src operand */
  158. #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
  159. #define X2(x...) x, x
  160. #define X3(x...) X2(x), x
  161. #define X4(x...) X2(x), X2(x)
  162. #define X5(x...) X4(x), x
  163. #define X6(x...) X4(x), X2(x)
  164. #define X7(x...) X4(x), X3(x)
  165. #define X8(x...) X4(x), X4(x)
  166. #define X16(x...) X8(x), X8(x)
  167. #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
  168. #define FASTOP_SIZE 8
  169. /*
  170. * fastop functions have a special calling convention:
  171. *
  172. * dst: rax (in/out)
  173. * src: rdx (in/out)
  174. * src2: rcx (in)
  175. * flags: rflags (in/out)
  176. * ex: rsi (in:fastop pointer, out:zero if exception)
  177. *
  178. * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
  179. * different operand sizes can be reached by calculation, rather than a jump
  180. * table (which would be bigger than the code).
  181. *
  182. * fastop functions are declared as taking a never-defined fastop parameter,
  183. * so they can't be called from C directly.
  184. */
  185. struct fastop;
  186. struct opcode {
  187. u64 flags : 56;
  188. u64 intercept : 8;
  189. union {
  190. int (*execute)(struct x86_emulate_ctxt *ctxt);
  191. const struct opcode *group;
  192. const struct group_dual *gdual;
  193. const struct gprefix *gprefix;
  194. const struct escape *esc;
  195. void (*fastop)(struct fastop *fake);
  196. } u;
  197. int (*check_perm)(struct x86_emulate_ctxt *ctxt);
  198. };
  199. struct group_dual {
  200. struct opcode mod012[8];
  201. struct opcode mod3[8];
  202. };
  203. struct gprefix {
  204. struct opcode pfx_no;
  205. struct opcode pfx_66;
  206. struct opcode pfx_f2;
  207. struct opcode pfx_f3;
  208. };
  209. struct escape {
  210. struct opcode op[8];
  211. struct opcode high[64];
  212. };
  213. /* EFLAGS bit definitions. */
  214. #define EFLG_ID (1<<21)
  215. #define EFLG_VIP (1<<20)
  216. #define EFLG_VIF (1<<19)
  217. #define EFLG_AC (1<<18)
  218. #define EFLG_VM (1<<17)
  219. #define EFLG_RF (1<<16)
  220. #define EFLG_IOPL (3<<12)
  221. #define EFLG_NT (1<<14)
  222. #define EFLG_OF (1<<11)
  223. #define EFLG_DF (1<<10)
  224. #define EFLG_IF (1<<9)
  225. #define EFLG_TF (1<<8)
  226. #define EFLG_SF (1<<7)
  227. #define EFLG_ZF (1<<6)
  228. #define EFLG_AF (1<<4)
  229. #define EFLG_PF (1<<2)
  230. #define EFLG_CF (1<<0)
  231. #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
  232. #define EFLG_RESERVED_ONE_MASK 2
  233. static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
  234. {
  235. if (!(ctxt->regs_valid & (1 << nr))) {
  236. ctxt->regs_valid |= 1 << nr;
  237. ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
  238. }
  239. return ctxt->_regs[nr];
  240. }
  241. static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
  242. {
  243. ctxt->regs_valid |= 1 << nr;
  244. ctxt->regs_dirty |= 1 << nr;
  245. return &ctxt->_regs[nr];
  246. }
  247. static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
  248. {
  249. reg_read(ctxt, nr);
  250. return reg_write(ctxt, nr);
  251. }
  252. static void writeback_registers(struct x86_emulate_ctxt *ctxt)
  253. {
  254. unsigned reg;
  255. for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
  256. ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
  257. }
  258. static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
  259. {
  260. ctxt->regs_dirty = 0;
  261. ctxt->regs_valid = 0;
  262. }
  263. /*
  264. * These EFLAGS bits are restored from saved value during emulation, and
  265. * any changes are written back to the saved value after emulation.
  266. */
  267. #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
  268. #ifdef CONFIG_X86_64
  269. #define ON64(x) x
  270. #else
  271. #define ON64(x)
  272. #endif
  273. static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
  274. #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
  275. #define FOP_RET "ret \n\t"
  276. #define FOP_START(op) \
  277. extern void em_##op(struct fastop *fake); \
  278. asm(".pushsection .text, \"ax\" \n\t" \
  279. ".global em_" #op " \n\t" \
  280. FOP_ALIGN \
  281. "em_" #op ": \n\t"
  282. #define FOP_END \
  283. ".popsection")
  284. #define FOPNOP() FOP_ALIGN FOP_RET
  285. #define FOP1E(op, dst) \
  286. FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
  287. #define FOP1EEX(op, dst) \
  288. FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
  289. #define FASTOP1(op) \
  290. FOP_START(op) \
  291. FOP1E(op##b, al) \
  292. FOP1E(op##w, ax) \
  293. FOP1E(op##l, eax) \
  294. ON64(FOP1E(op##q, rax)) \
  295. FOP_END
  296. /* 1-operand, using src2 (for MUL/DIV r/m) */
  297. #define FASTOP1SRC2(op, name) \
  298. FOP_START(name) \
  299. FOP1E(op, cl) \
  300. FOP1E(op, cx) \
  301. FOP1E(op, ecx) \
  302. ON64(FOP1E(op, rcx)) \
  303. FOP_END
  304. /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
  305. #define FASTOP1SRC2EX(op, name) \
  306. FOP_START(name) \
  307. FOP1EEX(op, cl) \
  308. FOP1EEX(op, cx) \
  309. FOP1EEX(op, ecx) \
  310. ON64(FOP1EEX(op, rcx)) \
  311. FOP_END
  312. #define FOP2E(op, dst, src) \
  313. FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
  314. #define FASTOP2(op) \
  315. FOP_START(op) \
  316. FOP2E(op##b, al, dl) \
  317. FOP2E(op##w, ax, dx) \
  318. FOP2E(op##l, eax, edx) \
  319. ON64(FOP2E(op##q, rax, rdx)) \
  320. FOP_END
  321. /* 2 operand, word only */
  322. #define FASTOP2W(op) \
  323. FOP_START(op) \
  324. FOPNOP() \
  325. FOP2E(op##w, ax, dx) \
  326. FOP2E(op##l, eax, edx) \
  327. ON64(FOP2E(op##q, rax, rdx)) \
  328. FOP_END
  329. /* 2 operand, src is CL */
  330. #define FASTOP2CL(op) \
  331. FOP_START(op) \
  332. FOP2E(op##b, al, cl) \
  333. FOP2E(op##w, ax, cl) \
  334. FOP2E(op##l, eax, cl) \
  335. ON64(FOP2E(op##q, rax, cl)) \
  336. FOP_END
  337. #define FOP3E(op, dst, src, src2) \
  338. FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
  339. /* 3-operand, word-only, src2=cl */
  340. #define FASTOP3WCL(op) \
  341. FOP_START(op) \
  342. FOPNOP() \
  343. FOP3E(op##w, ax, dx, cl) \
  344. FOP3E(op##l, eax, edx, cl) \
  345. ON64(FOP3E(op##q, rax, rdx, cl)) \
  346. FOP_END
  347. /* Special case for SETcc - 1 instruction per cc */
  348. #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
  349. asm(".global kvm_fastop_exception \n"
  350. "kvm_fastop_exception: xor %esi, %esi; ret");
  351. FOP_START(setcc)
  352. FOP_SETCC(seto)
  353. FOP_SETCC(setno)
  354. FOP_SETCC(setc)
  355. FOP_SETCC(setnc)
  356. FOP_SETCC(setz)
  357. FOP_SETCC(setnz)
  358. FOP_SETCC(setbe)
  359. FOP_SETCC(setnbe)
  360. FOP_SETCC(sets)
  361. FOP_SETCC(setns)
  362. FOP_SETCC(setp)
  363. FOP_SETCC(setnp)
  364. FOP_SETCC(setl)
  365. FOP_SETCC(setnl)
  366. FOP_SETCC(setle)
  367. FOP_SETCC(setnle)
  368. FOP_END;
  369. FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
  370. FOP_END;
  371. static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
  372. enum x86_intercept intercept,
  373. enum x86_intercept_stage stage)
  374. {
  375. struct x86_instruction_info info = {
  376. .intercept = intercept,
  377. .rep_prefix = ctxt->rep_prefix,
  378. .modrm_mod = ctxt->modrm_mod,
  379. .modrm_reg = ctxt->modrm_reg,
  380. .modrm_rm = ctxt->modrm_rm,
  381. .src_val = ctxt->src.val64,
  382. .src_bytes = ctxt->src.bytes,
  383. .dst_bytes = ctxt->dst.bytes,
  384. .ad_bytes = ctxt->ad_bytes,
  385. .next_rip = ctxt->eip,
  386. };
  387. return ctxt->ops->intercept(ctxt, &info, stage);
  388. }
  389. static void assign_masked(ulong *dest, ulong src, ulong mask)
  390. {
  391. *dest = (*dest & ~mask) | (src & mask);
  392. }
  393. static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
  394. {
  395. return (1UL << (ctxt->ad_bytes << 3)) - 1;
  396. }
  397. static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
  398. {
  399. u16 sel;
  400. struct desc_struct ss;
  401. if (ctxt->mode == X86EMUL_MODE_PROT64)
  402. return ~0UL;
  403. ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
  404. return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
  405. }
  406. static int stack_size(struct x86_emulate_ctxt *ctxt)
  407. {
  408. return (__fls(stack_mask(ctxt)) + 1) >> 3;
  409. }
  410. /* Access/update address held in a register, based on addressing mode. */
  411. static inline unsigned long
  412. address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
  413. {
  414. if (ctxt->ad_bytes == sizeof(unsigned long))
  415. return reg;
  416. else
  417. return reg & ad_mask(ctxt);
  418. }
  419. static inline unsigned long
  420. register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
  421. {
  422. return address_mask(ctxt, reg);
  423. }
  424. static void masked_increment(ulong *reg, ulong mask, int inc)
  425. {
  426. assign_masked(reg, *reg + inc, mask);
  427. }
  428. static inline void
  429. register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
  430. {
  431. ulong mask;
  432. if (ctxt->ad_bytes == sizeof(unsigned long))
  433. mask = ~0UL;
  434. else
  435. mask = ad_mask(ctxt);
  436. masked_increment(reg, mask, inc);
  437. }
  438. static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
  439. {
  440. masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
  441. }
  442. static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
  443. {
  444. register_address_increment(ctxt, &ctxt->_eip, rel);
  445. }
  446. static u32 desc_limit_scaled(struct desc_struct *desc)
  447. {
  448. u32 limit = get_desc_limit(desc);
  449. return desc->g ? (limit << 12) | 0xfff : limit;
  450. }
  451. static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
  452. {
  453. ctxt->has_seg_override = true;
  454. ctxt->seg_override = seg;
  455. }
  456. static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
  457. {
  458. if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
  459. return 0;
  460. return ctxt->ops->get_cached_segment_base(ctxt, seg);
  461. }
  462. static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
  463. {
  464. if (!ctxt->has_seg_override)
  465. return 0;
  466. return ctxt->seg_override;
  467. }
  468. static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
  469. u32 error, bool valid)
  470. {
  471. ctxt->exception.vector = vec;
  472. ctxt->exception.error_code = error;
  473. ctxt->exception.error_code_valid = valid;
  474. return X86EMUL_PROPAGATE_FAULT;
  475. }
  476. static int emulate_db(struct x86_emulate_ctxt *ctxt)
  477. {
  478. return emulate_exception(ctxt, DB_VECTOR, 0, false);
  479. }
  480. static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
  481. {
  482. return emulate_exception(ctxt, GP_VECTOR, err, true);
  483. }
  484. static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
  485. {
  486. return emulate_exception(ctxt, SS_VECTOR, err, true);
  487. }
  488. static int emulate_ud(struct x86_emulate_ctxt *ctxt)
  489. {
  490. return emulate_exception(ctxt, UD_VECTOR, 0, false);
  491. }
  492. static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
  493. {
  494. return emulate_exception(ctxt, TS_VECTOR, err, true);
  495. }
  496. static int emulate_de(struct x86_emulate_ctxt *ctxt)
  497. {
  498. return emulate_exception(ctxt, DE_VECTOR, 0, false);
  499. }
  500. static int emulate_nm(struct x86_emulate_ctxt *ctxt)
  501. {
  502. return emulate_exception(ctxt, NM_VECTOR, 0, false);
  503. }
  504. static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
  505. {
  506. u16 selector;
  507. struct desc_struct desc;
  508. ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
  509. return selector;
  510. }
  511. static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
  512. unsigned seg)
  513. {
  514. u16 dummy;
  515. u32 base3;
  516. struct desc_struct desc;
  517. ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
  518. ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
  519. }
  520. /*
  521. * x86 defines three classes of vector instructions: explicitly
  522. * aligned, explicitly unaligned, and the rest, which change behaviour
  523. * depending on whether they're AVX encoded or not.
  524. *
  525. * Also included is CMPXCHG16B which is not a vector instruction, yet it is
  526. * subject to the same check.
  527. */
  528. static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
  529. {
  530. if (likely(size < 16))
  531. return false;
  532. if (ctxt->d & Aligned)
  533. return true;
  534. else if (ctxt->d & Unaligned)
  535. return false;
  536. else if (ctxt->d & Avx)
  537. return false;
  538. else
  539. return true;
  540. }
  541. static int __linearize(struct x86_emulate_ctxt *ctxt,
  542. struct segmented_address addr,
  543. unsigned size, bool write, bool fetch,
  544. ulong *linear)
  545. {
  546. struct desc_struct desc;
  547. bool usable;
  548. ulong la;
  549. u32 lim;
  550. u16 sel;
  551. unsigned cpl;
  552. la = seg_base(ctxt, addr.seg) + addr.ea;
  553. switch (ctxt->mode) {
  554. case X86EMUL_MODE_PROT64:
  555. if (((signed long)la << 16) >> 16 != la)
  556. return emulate_gp(ctxt, 0);
  557. break;
  558. default:
  559. usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
  560. addr.seg);
  561. if (!usable)
  562. goto bad;
  563. /* code segment in protected mode or read-only data segment */
  564. if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
  565. || !(desc.type & 2)) && write)
  566. goto bad;
  567. /* unreadable code segment */
  568. if (!fetch && (desc.type & 8) && !(desc.type & 2))
  569. goto bad;
  570. lim = desc_limit_scaled(&desc);
  571. if ((desc.type & 8) || !(desc.type & 4)) {
  572. /* expand-up segment */
  573. if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
  574. goto bad;
  575. } else {
  576. /* expand-down segment */
  577. if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
  578. goto bad;
  579. lim = desc.d ? 0xffffffff : 0xffff;
  580. if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
  581. goto bad;
  582. }
  583. cpl = ctxt->ops->cpl(ctxt);
  584. if (!(desc.type & 8)) {
  585. /* data segment */
  586. if (cpl > desc.dpl)
  587. goto bad;
  588. } else if ((desc.type & 8) && !(desc.type & 4)) {
  589. /* nonconforming code segment */
  590. if (cpl != desc.dpl)
  591. goto bad;
  592. } else if ((desc.type & 8) && (desc.type & 4)) {
  593. /* conforming code segment */
  594. if (cpl < desc.dpl)
  595. goto bad;
  596. }
  597. break;
  598. }
  599. if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
  600. la &= (u32)-1;
  601. if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
  602. return emulate_gp(ctxt, 0);
  603. *linear = la;
  604. return X86EMUL_CONTINUE;
  605. bad:
  606. if (addr.seg == VCPU_SREG_SS)
  607. return emulate_ss(ctxt, sel);
  608. else
  609. return emulate_gp(ctxt, sel);
  610. }
  611. static int linearize(struct x86_emulate_ctxt *ctxt,
  612. struct segmented_address addr,
  613. unsigned size, bool write,
  614. ulong *linear)
  615. {
  616. return __linearize(ctxt, addr, size, write, false, linear);
  617. }
  618. static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
  619. struct segmented_address addr,
  620. void *data,
  621. unsigned size)
  622. {
  623. int rc;
  624. ulong linear;
  625. rc = linearize(ctxt, addr, size, false, &linear);
  626. if (rc != X86EMUL_CONTINUE)
  627. return rc;
  628. return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
  629. }
  630. /*
  631. * Fetch the next byte of the instruction being emulated which is pointed to
  632. * by ctxt->_eip, then increment ctxt->_eip.
  633. *
  634. * Also prefetch the remaining bytes of the instruction without crossing page
  635. * boundary if they are not in fetch_cache yet.
  636. */
  637. static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
  638. {
  639. struct fetch_cache *fc = &ctxt->fetch;
  640. int rc;
  641. int size, cur_size;
  642. if (ctxt->_eip == fc->end) {
  643. unsigned long linear;
  644. struct segmented_address addr = { .seg = VCPU_SREG_CS,
  645. .ea = ctxt->_eip };
  646. cur_size = fc->end - fc->start;
  647. size = min(15UL - cur_size,
  648. PAGE_SIZE - offset_in_page(ctxt->_eip));
  649. rc = __linearize(ctxt, addr, size, false, true, &linear);
  650. if (unlikely(rc != X86EMUL_CONTINUE))
  651. return rc;
  652. rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
  653. size, &ctxt->exception);
  654. if (unlikely(rc != X86EMUL_CONTINUE))
  655. return rc;
  656. fc->end += size;
  657. }
  658. *dest = fc->data[ctxt->_eip - fc->start];
  659. ctxt->_eip++;
  660. return X86EMUL_CONTINUE;
  661. }
  662. static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
  663. void *dest, unsigned size)
  664. {
  665. int rc;
  666. /* x86 instructions are limited to 15 bytes. */
  667. if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
  668. return X86EMUL_UNHANDLEABLE;
  669. while (size--) {
  670. rc = do_insn_fetch_byte(ctxt, dest++);
  671. if (rc != X86EMUL_CONTINUE)
  672. return rc;
  673. }
  674. return X86EMUL_CONTINUE;
  675. }
  676. /* Fetch next part of the instruction being emulated. */
  677. #define insn_fetch(_type, _ctxt) \
  678. ({ unsigned long _x; \
  679. rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
  680. if (rc != X86EMUL_CONTINUE) \
  681. goto done; \
  682. (_type)_x; \
  683. })
  684. #define insn_fetch_arr(_arr, _size, _ctxt) \
  685. ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
  686. if (rc != X86EMUL_CONTINUE) \
  687. goto done; \
  688. })
  689. /*
  690. * Given the 'reg' portion of a ModRM byte, and a register block, return a
  691. * pointer into the block that addresses the relevant register.
  692. * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
  693. */
  694. static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
  695. int highbyte_regs)
  696. {
  697. void *p;
  698. if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
  699. p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
  700. else
  701. p = reg_rmw(ctxt, modrm_reg);
  702. return p;
  703. }
  704. static int read_descriptor(struct x86_emulate_ctxt *ctxt,
  705. struct segmented_address addr,
  706. u16 *size, unsigned long *address, int op_bytes)
  707. {
  708. int rc;
  709. if (op_bytes == 2)
  710. op_bytes = 3;
  711. *address = 0;
  712. rc = segmented_read_std(ctxt, addr, size, 2);
  713. if (rc != X86EMUL_CONTINUE)
  714. return rc;
  715. addr.ea += 2;
  716. rc = segmented_read_std(ctxt, addr, address, op_bytes);
  717. return rc;
  718. }
  719. FASTOP2(add);
  720. FASTOP2(or);
  721. FASTOP2(adc);
  722. FASTOP2(sbb);
  723. FASTOP2(and);
  724. FASTOP2(sub);
  725. FASTOP2(xor);
  726. FASTOP2(cmp);
  727. FASTOP2(test);
  728. FASTOP1SRC2(mul, mul_ex);
  729. FASTOP1SRC2(imul, imul_ex);
  730. FASTOP1SRC2EX(div, div_ex);
  731. FASTOP1SRC2EX(idiv, idiv_ex);
  732. FASTOP3WCL(shld);
  733. FASTOP3WCL(shrd);
  734. FASTOP2W(imul);
  735. FASTOP1(not);
  736. FASTOP1(neg);
  737. FASTOP1(inc);
  738. FASTOP1(dec);
  739. FASTOP2CL(rol);
  740. FASTOP2CL(ror);
  741. FASTOP2CL(rcl);
  742. FASTOP2CL(rcr);
  743. FASTOP2CL(shl);
  744. FASTOP2CL(shr);
  745. FASTOP2CL(sar);
  746. FASTOP2W(bsf);
  747. FASTOP2W(bsr);
  748. FASTOP2W(bt);
  749. FASTOP2W(bts);
  750. FASTOP2W(btr);
  751. FASTOP2W(btc);
  752. FASTOP2(xadd);
  753. static u8 test_cc(unsigned int condition, unsigned long flags)
  754. {
  755. u8 rc;
  756. void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
  757. flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
  758. asm("push %[flags]; popf; call *%[fastop]"
  759. : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
  760. return rc;
  761. }
  762. static void fetch_register_operand(struct operand *op)
  763. {
  764. switch (op->bytes) {
  765. case 1:
  766. op->val = *(u8 *)op->addr.reg;
  767. break;
  768. case 2:
  769. op->val = *(u16 *)op->addr.reg;
  770. break;
  771. case 4:
  772. op->val = *(u32 *)op->addr.reg;
  773. break;
  774. case 8:
  775. op->val = *(u64 *)op->addr.reg;
  776. break;
  777. }
  778. }
  779. static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
  780. {
  781. ctxt->ops->get_fpu(ctxt);
  782. switch (reg) {
  783. case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
  784. case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
  785. case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
  786. case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
  787. case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
  788. case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
  789. case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
  790. case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
  791. #ifdef CONFIG_X86_64
  792. case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
  793. case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
  794. case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
  795. case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
  796. case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
  797. case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
  798. case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
  799. case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
  800. #endif
  801. default: BUG();
  802. }
  803. ctxt->ops->put_fpu(ctxt);
  804. }
  805. static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
  806. int reg)
  807. {
  808. ctxt->ops->get_fpu(ctxt);
  809. switch (reg) {
  810. case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
  811. case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
  812. case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
  813. case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
  814. case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
  815. case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
  816. case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
  817. case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
  818. #ifdef CONFIG_X86_64
  819. case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
  820. case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
  821. case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
  822. case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
  823. case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
  824. case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
  825. case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
  826. case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
  827. #endif
  828. default: BUG();
  829. }
  830. ctxt->ops->put_fpu(ctxt);
  831. }
  832. static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
  833. {
  834. ctxt->ops->get_fpu(ctxt);
  835. switch (reg) {
  836. case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
  837. case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
  838. case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
  839. case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
  840. case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
  841. case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
  842. case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
  843. case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
  844. default: BUG();
  845. }
  846. ctxt->ops->put_fpu(ctxt);
  847. }
  848. static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
  849. {
  850. ctxt->ops->get_fpu(ctxt);
  851. switch (reg) {
  852. case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
  853. case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
  854. case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
  855. case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
  856. case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
  857. case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
  858. case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
  859. case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
  860. default: BUG();
  861. }
  862. ctxt->ops->put_fpu(ctxt);
  863. }
  864. static int em_fninit(struct x86_emulate_ctxt *ctxt)
  865. {
  866. if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
  867. return emulate_nm(ctxt);
  868. ctxt->ops->get_fpu(ctxt);
  869. asm volatile("fninit");
  870. ctxt->ops->put_fpu(ctxt);
  871. return X86EMUL_CONTINUE;
  872. }
  873. static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
  874. {
  875. u16 fcw;
  876. if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
  877. return emulate_nm(ctxt);
  878. ctxt->ops->get_fpu(ctxt);
  879. asm volatile("fnstcw %0": "+m"(fcw));
  880. ctxt->ops->put_fpu(ctxt);
  881. /* force 2 byte destination */
  882. ctxt->dst.bytes = 2;
  883. ctxt->dst.val = fcw;
  884. return X86EMUL_CONTINUE;
  885. }
  886. static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
  887. {
  888. u16 fsw;
  889. if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
  890. return emulate_nm(ctxt);
  891. ctxt->ops->get_fpu(ctxt);
  892. asm volatile("fnstsw %0": "+m"(fsw));
  893. ctxt->ops->put_fpu(ctxt);
  894. /* force 2 byte destination */
  895. ctxt->dst.bytes = 2;
  896. ctxt->dst.val = fsw;
  897. return X86EMUL_CONTINUE;
  898. }
  899. static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
  900. struct operand *op)
  901. {
  902. unsigned reg = ctxt->modrm_reg;
  903. int highbyte_regs = ctxt->rex_prefix == 0;
  904. if (!(ctxt->d & ModRM))
  905. reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
  906. if (ctxt->d & Sse) {
  907. op->type = OP_XMM;
  908. op->bytes = 16;
  909. op->addr.xmm = reg;
  910. read_sse_reg(ctxt, &op->vec_val, reg);
  911. return;
  912. }
  913. if (ctxt->d & Mmx) {
  914. reg &= 7;
  915. op->type = OP_MM;
  916. op->bytes = 8;
  917. op->addr.mm = reg;
  918. return;
  919. }
  920. op->type = OP_REG;
  921. if (ctxt->d & ByteOp) {
  922. op->addr.reg = decode_register(ctxt, reg, highbyte_regs);
  923. op->bytes = 1;
  924. } else {
  925. op->addr.reg = decode_register(ctxt, reg, 0);
  926. op->bytes = ctxt->op_bytes;
  927. }
  928. fetch_register_operand(op);
  929. op->orig_val = op->val;
  930. }
  931. static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
  932. {
  933. if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
  934. ctxt->modrm_seg = VCPU_SREG_SS;
  935. }
  936. static int decode_modrm(struct x86_emulate_ctxt *ctxt,
  937. struct operand *op)
  938. {
  939. u8 sib;
  940. int index_reg = 0, base_reg = 0, scale;
  941. int rc = X86EMUL_CONTINUE;
  942. ulong modrm_ea = 0;
  943. if (ctxt->rex_prefix) {
  944. ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
  945. index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
  946. ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
  947. }
  948. ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
  949. ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
  950. ctxt->modrm_rm |= (ctxt->modrm & 0x07);
  951. ctxt->modrm_seg = VCPU_SREG_DS;
  952. if (ctxt->modrm_mod == 3) {
  953. int highbyte_regs = ctxt->rex_prefix == 0;
  954. op->type = OP_REG;
  955. op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
  956. op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
  957. highbyte_regs && (ctxt->d & ByteOp));
  958. if (ctxt->d & Sse) {
  959. op->type = OP_XMM;
  960. op->bytes = 16;
  961. op->addr.xmm = ctxt->modrm_rm;
  962. read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
  963. return rc;
  964. }
  965. if (ctxt->d & Mmx) {
  966. op->type = OP_MM;
  967. op->bytes = 8;
  968. op->addr.xmm = ctxt->modrm_rm & 7;
  969. return rc;
  970. }
  971. fetch_register_operand(op);
  972. return rc;
  973. }
  974. op->type = OP_MEM;
  975. if (ctxt->ad_bytes == 2) {
  976. unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
  977. unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
  978. unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
  979. unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
  980. /* 16-bit ModR/M decode. */
  981. switch (ctxt->modrm_mod) {
  982. case 0:
  983. if (ctxt->modrm_rm == 6)
  984. modrm_ea += insn_fetch(u16, ctxt);
  985. break;
  986. case 1:
  987. modrm_ea += insn_fetch(s8, ctxt);
  988. break;
  989. case 2:
  990. modrm_ea += insn_fetch(u16, ctxt);
  991. break;
  992. }
  993. switch (ctxt->modrm_rm) {
  994. case 0:
  995. modrm_ea += bx + si;
  996. break;
  997. case 1:
  998. modrm_ea += bx + di;
  999. break;
  1000. case 2:
  1001. modrm_ea += bp + si;
  1002. break;
  1003. case 3:
  1004. modrm_ea += bp + di;
  1005. break;
  1006. case 4:
  1007. modrm_ea += si;
  1008. break;
  1009. case 5:
  1010. modrm_ea += di;
  1011. break;
  1012. case 6:
  1013. if (ctxt->modrm_mod != 0)
  1014. modrm_ea += bp;
  1015. break;
  1016. case 7:
  1017. modrm_ea += bx;
  1018. break;
  1019. }
  1020. if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
  1021. (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
  1022. ctxt->modrm_seg = VCPU_SREG_SS;
  1023. modrm_ea = (u16)modrm_ea;
  1024. } else {
  1025. /* 32/64-bit ModR/M decode. */
  1026. if ((ctxt->modrm_rm & 7) == 4) {
  1027. sib = insn_fetch(u8, ctxt);
  1028. index_reg |= (sib >> 3) & 7;
  1029. base_reg |= sib & 7;
  1030. scale = sib >> 6;
  1031. if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
  1032. modrm_ea += insn_fetch(s32, ctxt);
  1033. else {
  1034. modrm_ea += reg_read(ctxt, base_reg);
  1035. adjust_modrm_seg(ctxt, base_reg);
  1036. }
  1037. if (index_reg != 4)
  1038. modrm_ea += reg_read(ctxt, index_reg) << scale;
  1039. } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
  1040. if (ctxt->mode == X86EMUL_MODE_PROT64)
  1041. ctxt->rip_relative = 1;
  1042. } else {
  1043. base_reg = ctxt->modrm_rm;
  1044. modrm_ea += reg_read(ctxt, base_reg);
  1045. adjust_modrm_seg(ctxt, base_reg);
  1046. }
  1047. switch (ctxt->modrm_mod) {
  1048. case 0:
  1049. if (ctxt->modrm_rm == 5)
  1050. modrm_ea += insn_fetch(s32, ctxt);
  1051. break;
  1052. case 1:
  1053. modrm_ea += insn_fetch(s8, ctxt);
  1054. break;
  1055. case 2:
  1056. modrm_ea += insn_fetch(s32, ctxt);
  1057. break;
  1058. }
  1059. }
  1060. op->addr.mem.ea = modrm_ea;
  1061. done:
  1062. return rc;
  1063. }
  1064. static int decode_abs(struct x86_emulate_ctxt *ctxt,
  1065. struct operand *op)
  1066. {
  1067. int rc = X86EMUL_CONTINUE;
  1068. op->type = OP_MEM;
  1069. switch (ctxt->ad_bytes) {
  1070. case 2:
  1071. op->addr.mem.ea = insn_fetch(u16, ctxt);
  1072. break;
  1073. case 4:
  1074. op->addr.mem.ea = insn_fetch(u32, ctxt);
  1075. break;
  1076. case 8:
  1077. op->addr.mem.ea = insn_fetch(u64, ctxt);
  1078. break;
  1079. }
  1080. done:
  1081. return rc;
  1082. }
  1083. static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
  1084. {
  1085. long sv = 0, mask;
  1086. if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
  1087. mask = ~(ctxt->dst.bytes * 8 - 1);
  1088. if (ctxt->src.bytes == 2)
  1089. sv = (s16)ctxt->src.val & (s16)mask;
  1090. else if (ctxt->src.bytes == 4)
  1091. sv = (s32)ctxt->src.val & (s32)mask;
  1092. ctxt->dst.addr.mem.ea += (sv >> 3);
  1093. }
  1094. /* only subword offset */
  1095. ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
  1096. }
  1097. static int read_emulated(struct x86_emulate_ctxt *ctxt,
  1098. unsigned long addr, void *dest, unsigned size)
  1099. {
  1100. int rc;
  1101. struct read_cache *mc = &ctxt->mem_read;
  1102. if (mc->pos < mc->end)
  1103. goto read_cached;
  1104. WARN_ON((mc->end + size) >= sizeof(mc->data));
  1105. rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
  1106. &ctxt->exception);
  1107. if (rc != X86EMUL_CONTINUE)
  1108. return rc;
  1109. mc->end += size;
  1110. read_cached:
  1111. memcpy(dest, mc->data + mc->pos, size);
  1112. mc->pos += size;
  1113. return X86EMUL_CONTINUE;
  1114. }
  1115. static int segmented_read(struct x86_emulate_ctxt *ctxt,
  1116. struct segmented_address addr,
  1117. void *data,
  1118. unsigned size)
  1119. {
  1120. int rc;
  1121. ulong linear;
  1122. rc = linearize(ctxt, addr, size, false, &linear);
  1123. if (rc != X86EMUL_CONTINUE)
  1124. return rc;
  1125. return read_emulated(ctxt, linear, data, size);
  1126. }
  1127. static int segmented_write(struct x86_emulate_ctxt *ctxt,
  1128. struct segmented_address addr,
  1129. const void *data,
  1130. unsigned size)
  1131. {
  1132. int rc;
  1133. ulong linear;
  1134. rc = linearize(ctxt, addr, size, true, &linear);
  1135. if (rc != X86EMUL_CONTINUE)
  1136. return rc;
  1137. return ctxt->ops->write_emulated(ctxt, linear, data, size,
  1138. &ctxt->exception);
  1139. }
  1140. static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
  1141. struct segmented_address addr,
  1142. const void *orig_data, const void *data,
  1143. unsigned size)
  1144. {
  1145. int rc;
  1146. ulong linear;
  1147. rc = linearize(ctxt, addr, size, true, &linear);
  1148. if (rc != X86EMUL_CONTINUE)
  1149. return rc;
  1150. return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
  1151. size, &ctxt->exception);
  1152. }
  1153. static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
  1154. unsigned int size, unsigned short port,
  1155. void *dest)
  1156. {
  1157. struct read_cache *rc = &ctxt->io_read;
  1158. if (rc->pos == rc->end) { /* refill pio read ahead */
  1159. unsigned int in_page, n;
  1160. unsigned int count = ctxt->rep_prefix ?
  1161. address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
  1162. in_page = (ctxt->eflags & EFLG_DF) ?
  1163. offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
  1164. PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
  1165. n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
  1166. count);
  1167. if (n == 0)
  1168. n = 1;
  1169. rc->pos = rc->end = 0;
  1170. if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
  1171. return 0;
  1172. rc->end = n * size;
  1173. }
  1174. if (ctxt->rep_prefix && !(ctxt->eflags & EFLG_DF)) {
  1175. ctxt->dst.data = rc->data + rc->pos;
  1176. ctxt->dst.type = OP_MEM_STR;
  1177. ctxt->dst.count = (rc->end - rc->pos) / size;
  1178. rc->pos = rc->end;
  1179. } else {
  1180. memcpy(dest, rc->data + rc->pos, size);
  1181. rc->pos += size;
  1182. }
  1183. return 1;
  1184. }
  1185. static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
  1186. u16 index, struct desc_struct *desc)
  1187. {
  1188. struct desc_ptr dt;
  1189. ulong addr;
  1190. ctxt->ops->get_idt(ctxt, &dt);
  1191. if (dt.size < index * 8 + 7)
  1192. return emulate_gp(ctxt, index << 3 | 0x2);
  1193. addr = dt.address + index * 8;
  1194. return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
  1195. &ctxt->exception);
  1196. }
  1197. static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
  1198. u16 selector, struct desc_ptr *dt)
  1199. {
  1200. const struct x86_emulate_ops *ops = ctxt->ops;
  1201. if (selector & 1 << 2) {
  1202. struct desc_struct desc;
  1203. u16 sel;
  1204. memset (dt, 0, sizeof *dt);
  1205. if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
  1206. return;
  1207. dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
  1208. dt->address = get_desc_base(&desc);
  1209. } else
  1210. ops->get_gdt(ctxt, dt);
  1211. }
  1212. /* allowed just for 8 bytes segments */
  1213. static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
  1214. u16 selector, struct desc_struct *desc,
  1215. ulong *desc_addr_p)
  1216. {
  1217. struct desc_ptr dt;
  1218. u16 index = selector >> 3;
  1219. ulong addr;
  1220. get_descriptor_table_ptr(ctxt, selector, &dt);
  1221. if (dt.size < index * 8 + 7)
  1222. return emulate_gp(ctxt, selector & 0xfffc);
  1223. *desc_addr_p = addr = dt.address + index * 8;
  1224. return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
  1225. &ctxt->exception);
  1226. }
  1227. /* allowed just for 8 bytes segments */
  1228. static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
  1229. u16 selector, struct desc_struct *desc)
  1230. {
  1231. struct desc_ptr dt;
  1232. u16 index = selector >> 3;
  1233. ulong addr;
  1234. get_descriptor_table_ptr(ctxt, selector, &dt);
  1235. if (dt.size < index * 8 + 7)
  1236. return emulate_gp(ctxt, selector & 0xfffc);
  1237. addr = dt.address + index * 8;
  1238. return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
  1239. &ctxt->exception);
  1240. }
  1241. /* Does not support long mode */
  1242. static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
  1243. u16 selector, int seg)
  1244. {
  1245. struct desc_struct seg_desc, old_desc;
  1246. u8 dpl, rpl, cpl;
  1247. unsigned err_vec = GP_VECTOR;
  1248. u32 err_code = 0;
  1249. bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
  1250. ulong desc_addr;
  1251. int ret;
  1252. u16 dummy;
  1253. memset(&seg_desc, 0, sizeof seg_desc);
  1254. if (ctxt->mode == X86EMUL_MODE_REAL) {
  1255. /* set real mode segment descriptor (keep limit etc. for
  1256. * unreal mode) */
  1257. ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
  1258. set_desc_base(&seg_desc, selector << 4);
  1259. goto load;
  1260. } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
  1261. /* VM86 needs a clean new segment descriptor */
  1262. set_desc_base(&seg_desc, selector << 4);
  1263. set_desc_limit(&seg_desc, 0xffff);
  1264. seg_desc.type = 3;
  1265. seg_desc.p = 1;
  1266. seg_desc.s = 1;
  1267. seg_desc.dpl = 3;
  1268. goto load;
  1269. }
  1270. rpl = selector & 3;
  1271. cpl = ctxt->ops->cpl(ctxt);
  1272. /* NULL selector is not valid for TR, CS and SS (except for long mode) */
  1273. if ((seg == VCPU_SREG_CS
  1274. || (seg == VCPU_SREG_SS
  1275. && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
  1276. || seg == VCPU_SREG_TR)
  1277. && null_selector)
  1278. goto exception;
  1279. /* TR should be in GDT only */
  1280. if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
  1281. goto exception;
  1282. if (null_selector) /* for NULL selector skip all following checks */
  1283. goto load;
  1284. ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
  1285. if (ret != X86EMUL_CONTINUE)
  1286. return ret;
  1287. err_code = selector & 0xfffc;
  1288. err_vec = GP_VECTOR;
  1289. /* can't load system descriptor into segment selector */
  1290. if (seg <= VCPU_SREG_GS && !seg_desc.s)
  1291. goto exception;
  1292. if (!seg_desc.p) {
  1293. err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
  1294. goto exception;
  1295. }
  1296. dpl = seg_desc.dpl;
  1297. switch (seg) {
  1298. case VCPU_SREG_SS:
  1299. /*
  1300. * segment is not a writable data segment or segment
  1301. * selector's RPL != CPL or segment selector's RPL != CPL
  1302. */
  1303. if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
  1304. goto exception;
  1305. break;
  1306. case VCPU_SREG_CS:
  1307. if (!(seg_desc.type & 8))
  1308. goto exception;
  1309. if (seg_desc.type & 4) {
  1310. /* conforming */
  1311. if (dpl > cpl)
  1312. goto exception;
  1313. } else {
  1314. /* nonconforming */
  1315. if (rpl > cpl || dpl != cpl)
  1316. goto exception;
  1317. }
  1318. /* CS(RPL) <- CPL */
  1319. selector = (selector & 0xfffc) | cpl;
  1320. break;
  1321. case VCPU_SREG_TR:
  1322. if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
  1323. goto exception;
  1324. old_desc = seg_desc;
  1325. seg_desc.type |= 2; /* busy */
  1326. ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
  1327. sizeof(seg_desc), &ctxt->exception);
  1328. if (ret != X86EMUL_CONTINUE)
  1329. return ret;
  1330. break;
  1331. case VCPU_SREG_LDTR:
  1332. if (seg_desc.s || seg_desc.type != 2)
  1333. goto exception;
  1334. break;
  1335. default: /* DS, ES, FS, or GS */
  1336. /*
  1337. * segment is not a data or readable code segment or
  1338. * ((segment is a data or nonconforming code segment)
  1339. * and (both RPL and CPL > DPL))
  1340. */
  1341. if ((seg_desc.type & 0xa) == 0x8 ||
  1342. (((seg_desc.type & 0xc) != 0xc) &&
  1343. (rpl > dpl && cpl > dpl)))
  1344. goto exception;
  1345. break;
  1346. }
  1347. if (seg_desc.s) {
  1348. /* mark segment as accessed */
  1349. seg_desc.type |= 1;
  1350. ret = write_segment_descriptor(ctxt, selector, &seg_desc);
  1351. if (ret != X86EMUL_CONTINUE)
  1352. return ret;
  1353. }
  1354. load:
  1355. ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
  1356. return X86EMUL_CONTINUE;
  1357. exception:
  1358. emulate_exception(ctxt, err_vec, err_code, true);
  1359. return X86EMUL_PROPAGATE_FAULT;
  1360. }
  1361. static void write_register_operand(struct operand *op)
  1362. {
  1363. /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
  1364. switch (op->bytes) {
  1365. case 1:
  1366. *(u8 *)op->addr.reg = (u8)op->val;
  1367. break;
  1368. case 2:
  1369. *(u16 *)op->addr.reg = (u16)op->val;
  1370. break;
  1371. case 4:
  1372. *op->addr.reg = (u32)op->val;
  1373. break; /* 64b: zero-extend */
  1374. case 8:
  1375. *op->addr.reg = op->val;
  1376. break;
  1377. }
  1378. }
  1379. static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
  1380. {
  1381. int rc;
  1382. switch (op->type) {
  1383. case OP_REG:
  1384. write_register_operand(op);
  1385. break;
  1386. case OP_MEM:
  1387. if (ctxt->lock_prefix)
  1388. rc = segmented_cmpxchg(ctxt,
  1389. op->addr.mem,
  1390. &op->orig_val,
  1391. &op->val,
  1392. op->bytes);
  1393. else
  1394. rc = segmented_write(ctxt,
  1395. op->addr.mem,
  1396. &op->val,
  1397. op->bytes);
  1398. if (rc != X86EMUL_CONTINUE)
  1399. return rc;
  1400. break;
  1401. case OP_MEM_STR:
  1402. rc = segmented_write(ctxt,
  1403. op->addr.mem,
  1404. op->data,
  1405. op->bytes * op->count);
  1406. if (rc != X86EMUL_CONTINUE)
  1407. return rc;
  1408. break;
  1409. case OP_XMM:
  1410. write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
  1411. break;
  1412. case OP_MM:
  1413. write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
  1414. break;
  1415. case OP_NONE:
  1416. /* no writeback */
  1417. break;
  1418. default:
  1419. break;
  1420. }
  1421. return X86EMUL_CONTINUE;
  1422. }
  1423. static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
  1424. {
  1425. struct segmented_address addr;
  1426. rsp_increment(ctxt, -bytes);
  1427. addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
  1428. addr.seg = VCPU_SREG_SS;
  1429. return segmented_write(ctxt, addr, data, bytes);
  1430. }
  1431. static int em_push(struct x86_emulate_ctxt *ctxt)
  1432. {
  1433. /* Disable writeback. */
  1434. ctxt->dst.type = OP_NONE;
  1435. return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
  1436. }
  1437. static int emulate_pop(struct x86_emulate_ctxt *ctxt,
  1438. void *dest, int len)
  1439. {
  1440. int rc;
  1441. struct segmented_address addr;
  1442. addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
  1443. addr.seg = VCPU_SREG_SS;
  1444. rc = segmented_read(ctxt, addr, dest, len);
  1445. if (rc != X86EMUL_CONTINUE)
  1446. return rc;
  1447. rsp_increment(ctxt, len);
  1448. return rc;
  1449. }
  1450. static int em_pop(struct x86_emulate_ctxt *ctxt)
  1451. {
  1452. return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
  1453. }
  1454. static int emulate_popf(struct x86_emulate_ctxt *ctxt,
  1455. void *dest, int len)
  1456. {
  1457. int rc;
  1458. unsigned long val, change_mask;
  1459. int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
  1460. int cpl = ctxt->ops->cpl(ctxt);
  1461. rc = emulate_pop(ctxt, &val, len);
  1462. if (rc != X86EMUL_CONTINUE)
  1463. return rc;
  1464. change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
  1465. | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
  1466. switch(ctxt->mode) {
  1467. case X86EMUL_MODE_PROT64:
  1468. case X86EMUL_MODE_PROT32:
  1469. case X86EMUL_MODE_PROT16:
  1470. if (cpl == 0)
  1471. change_mask |= EFLG_IOPL;
  1472. if (cpl <= iopl)
  1473. change_mask |= EFLG_IF;
  1474. break;
  1475. case X86EMUL_MODE_VM86:
  1476. if (iopl < 3)
  1477. return emulate_gp(ctxt, 0);
  1478. change_mask |= EFLG_IF;
  1479. break;
  1480. default: /* real mode */
  1481. change_mask |= (EFLG_IOPL | EFLG_IF);
  1482. break;
  1483. }
  1484. *(unsigned long *)dest =
  1485. (ctxt->eflags & ~change_mask) | (val & change_mask);
  1486. return rc;
  1487. }
  1488. static int em_popf(struct x86_emulate_ctxt *ctxt)
  1489. {
  1490. ctxt->dst.type = OP_REG;
  1491. ctxt->dst.addr.reg = &ctxt->eflags;
  1492. ctxt->dst.bytes = ctxt->op_bytes;
  1493. return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
  1494. }
  1495. static int em_enter(struct x86_emulate_ctxt *ctxt)
  1496. {
  1497. int rc;
  1498. unsigned frame_size = ctxt->src.val;
  1499. unsigned nesting_level = ctxt->src2.val & 31;
  1500. ulong rbp;
  1501. if (nesting_level)
  1502. return X86EMUL_UNHANDLEABLE;
  1503. rbp = reg_read(ctxt, VCPU_REGS_RBP);
  1504. rc = push(ctxt, &rbp, stack_size(ctxt));
  1505. if (rc != X86EMUL_CONTINUE)
  1506. return rc;
  1507. assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
  1508. stack_mask(ctxt));
  1509. assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
  1510. reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
  1511. stack_mask(ctxt));
  1512. return X86EMUL_CONTINUE;
  1513. }
  1514. static int em_leave(struct x86_emulate_ctxt *ctxt)
  1515. {
  1516. assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
  1517. stack_mask(ctxt));
  1518. return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
  1519. }
  1520. static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
  1521. {
  1522. int seg = ctxt->src2.val;
  1523. ctxt->src.val = get_segment_selector(ctxt, seg);
  1524. return em_push(ctxt);
  1525. }
  1526. static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
  1527. {
  1528. int seg = ctxt->src2.val;
  1529. unsigned long selector;
  1530. int rc;
  1531. rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
  1532. if (rc != X86EMUL_CONTINUE)
  1533. return rc;
  1534. rc = load_segment_descriptor(ctxt, (u16)selector, seg);
  1535. return rc;
  1536. }
  1537. static int em_pusha(struct x86_emulate_ctxt *ctxt)
  1538. {
  1539. unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
  1540. int rc = X86EMUL_CONTINUE;
  1541. int reg = VCPU_REGS_RAX;
  1542. while (reg <= VCPU_REGS_RDI) {
  1543. (reg == VCPU_REGS_RSP) ?
  1544. (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
  1545. rc = em_push(ctxt);
  1546. if (rc != X86EMUL_CONTINUE)
  1547. return rc;
  1548. ++reg;
  1549. }
  1550. return rc;
  1551. }
  1552. static int em_pushf(struct x86_emulate_ctxt *ctxt)
  1553. {
  1554. ctxt->src.val = (unsigned long)ctxt->eflags;
  1555. return em_push(ctxt);
  1556. }
  1557. static int em_popa(struct x86_emulate_ctxt *ctxt)
  1558. {
  1559. int rc = X86EMUL_CONTINUE;
  1560. int reg = VCPU_REGS_RDI;
  1561. while (reg >= VCPU_REGS_RAX) {
  1562. if (reg == VCPU_REGS_RSP) {
  1563. rsp_increment(ctxt, ctxt->op_bytes);
  1564. --reg;
  1565. }
  1566. rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
  1567. if (rc != X86EMUL_CONTINUE)
  1568. break;
  1569. --reg;
  1570. }
  1571. return rc;
  1572. }
  1573. static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
  1574. {
  1575. const struct x86_emulate_ops *ops = ctxt->ops;
  1576. int rc;
  1577. struct desc_ptr dt;
  1578. gva_t cs_addr;
  1579. gva_t eip_addr;
  1580. u16 cs, eip;
  1581. /* TODO: Add limit checks */
  1582. ctxt->src.val = ctxt->eflags;
  1583. rc = em_push(ctxt);
  1584. if (rc != X86EMUL_CONTINUE)
  1585. return rc;
  1586. ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
  1587. ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
  1588. rc = em_push(ctxt);
  1589. if (rc != X86EMUL_CONTINUE)
  1590. return rc;
  1591. ctxt->src.val = ctxt->_eip;
  1592. rc = em_push(ctxt);
  1593. if (rc != X86EMUL_CONTINUE)
  1594. return rc;
  1595. ops->get_idt(ctxt, &dt);
  1596. eip_addr = dt.address + (irq << 2);
  1597. cs_addr = dt.address + (irq << 2) + 2;
  1598. rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
  1599. if (rc != X86EMUL_CONTINUE)
  1600. return rc;
  1601. rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
  1602. if (rc != X86EMUL_CONTINUE)
  1603. return rc;
  1604. rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
  1605. if (rc != X86EMUL_CONTINUE)
  1606. return rc;
  1607. ctxt->_eip = eip;
  1608. return rc;
  1609. }
  1610. int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
  1611. {
  1612. int rc;
  1613. invalidate_registers(ctxt);
  1614. rc = __emulate_int_real(ctxt, irq);
  1615. if (rc == X86EMUL_CONTINUE)
  1616. writeback_registers(ctxt);
  1617. return rc;
  1618. }
  1619. static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
  1620. {
  1621. switch(ctxt->mode) {
  1622. case X86EMUL_MODE_REAL:
  1623. return __emulate_int_real(ctxt, irq);
  1624. case X86EMUL_MODE_VM86:
  1625. case X86EMUL_MODE_PROT16:
  1626. case X86EMUL_MODE_PROT32:
  1627. case X86EMUL_MODE_PROT64:
  1628. default:
  1629. /* Protected mode interrupts unimplemented yet */
  1630. return X86EMUL_UNHANDLEABLE;
  1631. }
  1632. }
  1633. static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
  1634. {
  1635. int rc = X86EMUL_CONTINUE;
  1636. unsigned long temp_eip = 0;
  1637. unsigned long temp_eflags = 0;
  1638. unsigned long cs = 0;
  1639. unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
  1640. EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
  1641. EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
  1642. unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
  1643. /* TODO: Add stack limit check */
  1644. rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
  1645. if (rc != X86EMUL_CONTINUE)
  1646. return rc;
  1647. if (temp_eip & ~0xffff)
  1648. return emulate_gp(ctxt, 0);
  1649. rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
  1650. if (rc != X86EMUL_CONTINUE)
  1651. return rc;
  1652. rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
  1653. if (rc != X86EMUL_CONTINUE)
  1654. return rc;
  1655. rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
  1656. if (rc != X86EMUL_CONTINUE)
  1657. return rc;
  1658. ctxt->_eip = temp_eip;
  1659. if (ctxt->op_bytes == 4)
  1660. ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
  1661. else if (ctxt->op_bytes == 2) {
  1662. ctxt->eflags &= ~0xffff;
  1663. ctxt->eflags |= temp_eflags;
  1664. }
  1665. ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
  1666. ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
  1667. return rc;
  1668. }
  1669. static int em_iret(struct x86_emulate_ctxt *ctxt)
  1670. {
  1671. switch(ctxt->mode) {
  1672. case X86EMUL_MODE_REAL:
  1673. return emulate_iret_real(ctxt);
  1674. case X86EMUL_MODE_VM86:
  1675. case X86EMUL_MODE_PROT16:
  1676. case X86EMUL_MODE_PROT32:
  1677. case X86EMUL_MODE_PROT64:
  1678. default:
  1679. /* iret from protected mode unimplemented yet */
  1680. return X86EMUL_UNHANDLEABLE;
  1681. }
  1682. }
  1683. static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
  1684. {
  1685. int rc;
  1686. unsigned short sel;
  1687. memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
  1688. rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
  1689. if (rc != X86EMUL_CONTINUE)
  1690. return rc;
  1691. ctxt->_eip = 0;
  1692. memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
  1693. return X86EMUL_CONTINUE;
  1694. }
  1695. static int em_grp45(struct x86_emulate_ctxt *ctxt)
  1696. {
  1697. int rc = X86EMUL_CONTINUE;
  1698. switch (ctxt->modrm_reg) {
  1699. case 2: /* call near abs */ {
  1700. long int old_eip;
  1701. old_eip = ctxt->_eip;
  1702. ctxt->_eip = ctxt->src.val;
  1703. ctxt->src.val = old_eip;
  1704. rc = em_push(ctxt);
  1705. break;
  1706. }
  1707. case 4: /* jmp abs */
  1708. ctxt->_eip = ctxt->src.val;
  1709. break;
  1710. case 5: /* jmp far */
  1711. rc = em_jmp_far(ctxt);
  1712. break;
  1713. case 6: /* push */
  1714. rc = em_push(ctxt);
  1715. break;
  1716. }
  1717. return rc;
  1718. }
  1719. static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
  1720. {
  1721. u64 old = ctxt->dst.orig_val64;
  1722. if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
  1723. ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
  1724. *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
  1725. *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
  1726. ctxt->eflags &= ~EFLG_ZF;
  1727. } else {
  1728. ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
  1729. (u32) reg_read(ctxt, VCPU_REGS_RBX);
  1730. ctxt->eflags |= EFLG_ZF;
  1731. }
  1732. return X86EMUL_CONTINUE;
  1733. }
  1734. static int em_ret(struct x86_emulate_ctxt *ctxt)
  1735. {
  1736. ctxt->dst.type = OP_REG;
  1737. ctxt->dst.addr.reg = &ctxt->_eip;
  1738. ctxt->dst.bytes = ctxt->op_bytes;
  1739. return em_pop(ctxt);
  1740. }
  1741. static int em_ret_far(struct x86_emulate_ctxt *ctxt)
  1742. {
  1743. int rc;
  1744. unsigned long cs;
  1745. rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
  1746. if (rc != X86EMUL_CONTINUE)
  1747. return rc;
  1748. if (ctxt->op_bytes == 4)
  1749. ctxt->_eip = (u32)ctxt->_eip;
  1750. rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
  1751. if (rc != X86EMUL_CONTINUE)
  1752. return rc;
  1753. rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
  1754. return rc;
  1755. }
  1756. static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
  1757. {
  1758. /* Save real source value, then compare EAX against destination. */
  1759. ctxt->src.orig_val = ctxt->src.val;
  1760. ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
  1761. fastop(ctxt, em_cmp);
  1762. if (ctxt->eflags & EFLG_ZF) {
  1763. /* Success: write back to memory. */
  1764. ctxt->dst.val = ctxt->src.orig_val;
  1765. } else {
  1766. /* Failure: write the value we saw to EAX. */
  1767. ctxt->dst.type = OP_REG;
  1768. ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
  1769. }
  1770. return X86EMUL_CONTINUE;
  1771. }
  1772. static int em_lseg(struct x86_emulate_ctxt *ctxt)
  1773. {
  1774. int seg = ctxt->src2.val;
  1775. unsigned short sel;
  1776. int rc;
  1777. memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
  1778. rc = load_segment_descriptor(ctxt, sel, seg);
  1779. if (rc != X86EMUL_CONTINUE)
  1780. return rc;
  1781. ctxt->dst.val = ctxt->src.val;
  1782. return rc;
  1783. }
  1784. static void
  1785. setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
  1786. struct desc_struct *cs, struct desc_struct *ss)
  1787. {
  1788. cs->l = 0; /* will be adjusted later */
  1789. set_desc_base(cs, 0); /* flat segment */
  1790. cs->g = 1; /* 4kb granularity */
  1791. set_desc_limit(cs, 0xfffff); /* 4GB limit */
  1792. cs->type = 0x0b; /* Read, Execute, Accessed */
  1793. cs->s = 1;
  1794. cs->dpl = 0; /* will be adjusted later */
  1795. cs->p = 1;
  1796. cs->d = 1;
  1797. cs->avl = 0;
  1798. set_desc_base(ss, 0); /* flat segment */
  1799. set_desc_limit(ss, 0xfffff); /* 4GB limit */
  1800. ss->g = 1; /* 4kb granularity */
  1801. ss->s = 1;
  1802. ss->type = 0x03; /* Read/Write, Accessed */
  1803. ss->d = 1; /* 32bit stack segment */
  1804. ss->dpl = 0;
  1805. ss->p = 1;
  1806. ss->l = 0;
  1807. ss->avl = 0;
  1808. }
  1809. static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
  1810. {
  1811. u32 eax, ebx, ecx, edx;
  1812. eax = ecx = 0;
  1813. ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
  1814. return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
  1815. && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
  1816. && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
  1817. }
  1818. static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
  1819. {
  1820. const struct x86_emulate_ops *ops = ctxt->ops;
  1821. u32 eax, ebx, ecx, edx;
  1822. /*
  1823. * syscall should always be enabled in longmode - so only become
  1824. * vendor specific (cpuid) if other modes are active...
  1825. */
  1826. if (ctxt->mode == X86EMUL_MODE_PROT64)
  1827. return true;
  1828. eax = 0x00000000;
  1829. ecx = 0x00000000;
  1830. ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
  1831. /*
  1832. * Intel ("GenuineIntel")
  1833. * remark: Intel CPUs only support "syscall" in 64bit
  1834. * longmode. Also an 64bit guest with a
  1835. * 32bit compat-app running will #UD !! While this
  1836. * behaviour can be fixed (by emulating) into AMD
  1837. * response - CPUs of AMD can't behave like Intel.
  1838. */
  1839. if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
  1840. ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
  1841. edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
  1842. return false;
  1843. /* AMD ("AuthenticAMD") */
  1844. if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
  1845. ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
  1846. edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
  1847. return true;
  1848. /* AMD ("AMDisbetter!") */
  1849. if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
  1850. ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
  1851. edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
  1852. return true;
  1853. /* default: (not Intel, not AMD), apply Intel's stricter rules... */
  1854. return false;
  1855. }
  1856. static int em_syscall(struct x86_emulate_ctxt *ctxt)
  1857. {
  1858. const struct x86_emulate_ops *ops = ctxt->ops;
  1859. struct desc_struct cs, ss;
  1860. u64 msr_data;
  1861. u16 cs_sel, ss_sel;
  1862. u64 efer = 0;
  1863. /* syscall is not available in real mode */
  1864. if (ctxt->mode == X86EMUL_MODE_REAL ||
  1865. ctxt->mode == X86EMUL_MODE_VM86)
  1866. return emulate_ud(ctxt);
  1867. if (!(em_syscall_is_enabled(ctxt)))
  1868. return emulate_ud(ctxt);
  1869. ops->get_msr(ctxt, MSR_EFER, &efer);
  1870. setup_syscalls_segments(ctxt, &cs, &ss);
  1871. if (!(efer & EFER_SCE))
  1872. return emulate_ud(ctxt);
  1873. ops->get_msr(ctxt, MSR_STAR, &msr_data);
  1874. msr_data >>= 32;
  1875. cs_sel = (u16)(msr_data & 0xfffc);
  1876. ss_sel = (u16)(msr_data + 8);
  1877. if (efer & EFER_LMA) {
  1878. cs.d = 0;
  1879. cs.l = 1;
  1880. }
  1881. ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
  1882. ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
  1883. *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
  1884. if (efer & EFER_LMA) {
  1885. #ifdef CONFIG_X86_64
  1886. *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
  1887. ops->get_msr(ctxt,
  1888. ctxt->mode == X86EMUL_MODE_PROT64 ?
  1889. MSR_LSTAR : MSR_CSTAR, &msr_data);
  1890. ctxt->_eip = msr_data;
  1891. ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
  1892. ctxt->eflags &= ~(msr_data | EFLG_RF);
  1893. #endif
  1894. } else {
  1895. /* legacy mode */
  1896. ops->get_msr(ctxt, MSR_STAR, &msr_data);
  1897. ctxt->_eip = (u32)msr_data;
  1898. ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
  1899. }
  1900. return X86EMUL_CONTINUE;
  1901. }
  1902. static int em_sysenter(struct x86_emulate_ctxt *ctxt)
  1903. {
  1904. const struct x86_emulate_ops *ops = ctxt->ops;
  1905. struct desc_struct cs, ss;
  1906. u64 msr_data;
  1907. u16 cs_sel, ss_sel;
  1908. u64 efer = 0;
  1909. ops->get_msr(ctxt, MSR_EFER, &efer);
  1910. /* inject #GP if in real mode */
  1911. if (ctxt->mode == X86EMUL_MODE_REAL)
  1912. return emulate_gp(ctxt, 0);
  1913. /*
  1914. * Not recognized on AMD in compat mode (but is recognized in legacy
  1915. * mode).
  1916. */
  1917. if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
  1918. && !vendor_intel(ctxt))
  1919. return emulate_ud(ctxt);
  1920. /* XXX sysenter/sysexit have not been tested in 64bit mode.
  1921. * Therefore, we inject an #UD.
  1922. */
  1923. if (ctxt->mode == X86EMUL_MODE_PROT64)
  1924. return emulate_ud(ctxt);
  1925. setup_syscalls_segments(ctxt, &cs, &ss);
  1926. ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
  1927. switch (ctxt->mode) {
  1928. case X86EMUL_MODE_PROT32:
  1929. if ((msr_data & 0xfffc) == 0x0)
  1930. return emulate_gp(ctxt, 0);
  1931. break;
  1932. case X86EMUL_MODE_PROT64:
  1933. if (msr_data == 0x0)
  1934. return emulate_gp(ctxt, 0);
  1935. break;
  1936. default:
  1937. break;
  1938. }
  1939. ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
  1940. cs_sel = (u16)msr_data;
  1941. cs_sel &= ~SELECTOR_RPL_MASK;
  1942. ss_sel = cs_sel + 8;
  1943. ss_sel &= ~SELECTOR_RPL_MASK;
  1944. if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
  1945. cs.d = 0;
  1946. cs.l = 1;
  1947. }
  1948. ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
  1949. ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
  1950. ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
  1951. ctxt->_eip = msr_data;
  1952. ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
  1953. *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
  1954. return X86EMUL_CONTINUE;
  1955. }
  1956. static int em_sysexit(struct x86_emulate_ctxt *ctxt)
  1957. {
  1958. const struct x86_emulate_ops *ops = ctxt->ops;
  1959. struct desc_struct cs, ss;
  1960. u64 msr_data;
  1961. int usermode;
  1962. u16 cs_sel = 0, ss_sel = 0;
  1963. /* inject #GP if in real mode or Virtual 8086 mode */
  1964. if (ctxt->mode == X86EMUL_MODE_REAL ||
  1965. ctxt->mode == X86EMUL_MODE_VM86)
  1966. return emulate_gp(ctxt, 0);
  1967. setup_syscalls_segments(ctxt, &cs, &ss);
  1968. if ((ctxt->rex_prefix & 0x8) != 0x0)
  1969. usermode = X86EMUL_MODE_PROT64;
  1970. else
  1971. usermode = X86EMUL_MODE_PROT32;
  1972. cs.dpl = 3;
  1973. ss.dpl = 3;
  1974. ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
  1975. switch (usermode) {
  1976. case X86EMUL_MODE_PROT32:
  1977. cs_sel = (u16)(msr_data + 16);
  1978. if ((msr_data & 0xfffc) == 0x0)
  1979. return emulate_gp(ctxt, 0);
  1980. ss_sel = (u16)(msr_data + 24);
  1981. break;
  1982. case X86EMUL_MODE_PROT64:
  1983. cs_sel = (u16)(msr_data + 32);
  1984. if (msr_data == 0x0)
  1985. return emulate_gp(ctxt, 0);
  1986. ss_sel = cs_sel + 8;
  1987. cs.d = 0;
  1988. cs.l = 1;
  1989. break;
  1990. }
  1991. cs_sel |= SELECTOR_RPL_MASK;
  1992. ss_sel |= SELECTOR_RPL_MASK;
  1993. ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
  1994. ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
  1995. ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
  1996. *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
  1997. return X86EMUL_CONTINUE;
  1998. }
  1999. static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
  2000. {
  2001. int iopl;
  2002. if (ctxt->mode == X86EMUL_MODE_REAL)
  2003. return false;
  2004. if (ctxt->mode == X86EMUL_MODE_VM86)
  2005. return true;
  2006. iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
  2007. return ctxt->ops->cpl(ctxt) > iopl;
  2008. }
  2009. static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
  2010. u16 port, u16 len)
  2011. {
  2012. const struct x86_emulate_ops *ops = ctxt->ops;
  2013. struct desc_struct tr_seg;
  2014. u32 base3;
  2015. int r;
  2016. u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
  2017. unsigned mask = (1 << len) - 1;
  2018. unsigned long base;
  2019. ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
  2020. if (!tr_seg.p)
  2021. return false;
  2022. if (desc_limit_scaled(&tr_seg) < 103)
  2023. return false;
  2024. base = get_desc_base(&tr_seg);
  2025. #ifdef CONFIG_X86_64
  2026. base |= ((u64)base3) << 32;
  2027. #endif
  2028. r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
  2029. if (r != X86EMUL_CONTINUE)
  2030. return false;
  2031. if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
  2032. return false;
  2033. r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
  2034. if (r != X86EMUL_CONTINUE)
  2035. return false;
  2036. if ((perm >> bit_idx) & mask)
  2037. return false;
  2038. return true;
  2039. }
  2040. static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
  2041. u16 port, u16 len)
  2042. {
  2043. if (ctxt->perm_ok)
  2044. return true;
  2045. if (emulator_bad_iopl(ctxt))
  2046. if (!emulator_io_port_access_allowed(ctxt, port, len))
  2047. return false;
  2048. ctxt->perm_ok = true;
  2049. return true;
  2050. }
  2051. static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
  2052. struct tss_segment_16 *tss)
  2053. {
  2054. tss->ip = ctxt->_eip;
  2055. tss->flag = ctxt->eflags;
  2056. tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
  2057. tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
  2058. tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
  2059. tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
  2060. tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
  2061. tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
  2062. tss->si = reg_read(ctxt, VCPU_REGS_RSI);
  2063. tss->di = reg_read(ctxt, VCPU_REGS_RDI);
  2064. tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
  2065. tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
  2066. tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
  2067. tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
  2068. tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
  2069. }
  2070. static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
  2071. struct tss_segment_16 *tss)
  2072. {
  2073. int ret;
  2074. ctxt->_eip = tss->ip;
  2075. ctxt->eflags = tss->flag | 2;
  2076. *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
  2077. *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
  2078. *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
  2079. *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
  2080. *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
  2081. *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
  2082. *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
  2083. *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
  2084. /*
  2085. * SDM says that segment selectors are loaded before segment
  2086. * descriptors
  2087. */
  2088. set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
  2089. set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
  2090. set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
  2091. set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
  2092. set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
  2093. /*
  2094. * Now load segment descriptors. If fault happens at this stage
  2095. * it is handled in a context of new task
  2096. */
  2097. ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
  2098. if (ret != X86EMUL_CONTINUE)
  2099. return ret;
  2100. ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
  2101. if (ret != X86EMUL_CONTINUE)
  2102. return ret;
  2103. ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
  2104. if (ret != X86EMUL_CONTINUE)
  2105. return ret;
  2106. ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
  2107. if (ret != X86EMUL_CONTINUE)
  2108. return ret;
  2109. ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
  2110. if (ret != X86EMUL_CONTINUE)
  2111. return ret;
  2112. return X86EMUL_CONTINUE;
  2113. }
  2114. static int task_switch_16(struct x86_emulate_ctxt *ctxt,
  2115. u16 tss_selector, u16 old_tss_sel,
  2116. ulong old_tss_base, struct desc_struct *new_desc)
  2117. {
  2118. const struct x86_emulate_ops *ops = ctxt->ops;
  2119. struct tss_segment_16 tss_seg;
  2120. int ret;
  2121. u32 new_tss_base = get_desc_base(new_desc);
  2122. ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
  2123. &ctxt->exception);
  2124. if (ret != X86EMUL_CONTINUE)
  2125. /* FIXME: need to provide precise fault address */
  2126. return ret;
  2127. save_state_to_tss16(ctxt, &tss_seg);
  2128. ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
  2129. &ctxt->exception);
  2130. if (ret != X86EMUL_CONTINUE)
  2131. /* FIXME: need to provide precise fault address */
  2132. return ret;
  2133. ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
  2134. &ctxt->exception);
  2135. if (ret != X86EMUL_CONTINUE)
  2136. /* FIXME: need to provide precise fault address */
  2137. return ret;
  2138. if (old_tss_sel != 0xffff) {
  2139. tss_seg.prev_task_link = old_tss_sel;
  2140. ret = ops->write_std(ctxt, new_tss_base,
  2141. &tss_seg.prev_task_link,
  2142. sizeof tss_seg.prev_task_link,
  2143. &ctxt->exception);
  2144. if (ret != X86EMUL_CONTINUE)
  2145. /* FIXME: need to provide precise fault address */
  2146. return ret;
  2147. }
  2148. return load_state_from_tss16(ctxt, &tss_seg);
  2149. }
  2150. static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
  2151. struct tss_segment_32 *tss)
  2152. {
  2153. tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
  2154. tss->eip = ctxt->_eip;
  2155. tss->eflags = ctxt->eflags;
  2156. tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
  2157. tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
  2158. tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
  2159. tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
  2160. tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
  2161. tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
  2162. tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
  2163. tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
  2164. tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
  2165. tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
  2166. tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
  2167. tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
  2168. tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
  2169. tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
  2170. tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
  2171. }
  2172. static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
  2173. struct tss_segment_32 *tss)
  2174. {
  2175. int ret;
  2176. if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
  2177. return emulate_gp(ctxt, 0);
  2178. ctxt->_eip = tss->eip;
  2179. ctxt->eflags = tss->eflags | 2;
  2180. /* General purpose registers */
  2181. *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
  2182. *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
  2183. *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
  2184. *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
  2185. *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
  2186. *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
  2187. *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
  2188. *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
  2189. /*
  2190. * SDM says that segment selectors are loaded before segment
  2191. * descriptors
  2192. */
  2193. set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
  2194. set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
  2195. set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
  2196. set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
  2197. set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
  2198. set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
  2199. set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
  2200. /*
  2201. * If we're switching between Protected Mode and VM86, we need to make
  2202. * sure to update the mode before loading the segment descriptors so
  2203. * that the selectors are interpreted correctly.
  2204. *
  2205. * Need to get rflags to the vcpu struct immediately because it
  2206. * influences the CPL which is checked at least when loading the segment
  2207. * descriptors and when pushing an error code to the new kernel stack.
  2208. *
  2209. * TODO Introduce a separate ctxt->ops->set_cpl callback
  2210. */
  2211. if (ctxt->eflags & X86_EFLAGS_VM)
  2212. ctxt->mode = X86EMUL_MODE_VM86;
  2213. else
  2214. ctxt->mode = X86EMUL_MODE_PROT32;
  2215. ctxt->ops->set_rflags(ctxt, ctxt->eflags);
  2216. /*
  2217. * Now load segment descriptors. If fault happenes at this stage
  2218. * it is handled in a context of new task
  2219. */
  2220. ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
  2221. if (ret != X86EMUL_CONTINUE)
  2222. return ret;
  2223. ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
  2224. if (ret != X86EMUL_CONTINUE)
  2225. return ret;
  2226. ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
  2227. if (ret != X86EMUL_CONTINUE)
  2228. return ret;
  2229. ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
  2230. if (ret != X86EMUL_CONTINUE)
  2231. return ret;
  2232. ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
  2233. if (ret != X86EMUL_CONTINUE)
  2234. return ret;
  2235. ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
  2236. if (ret != X86EMUL_CONTINUE)
  2237. return ret;
  2238. ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
  2239. if (ret != X86EMUL_CONTINUE)
  2240. return ret;
  2241. return X86EMUL_CONTINUE;
  2242. }
  2243. static int task_switch_32(struct x86_emulate_ctxt *ctxt,
  2244. u16 tss_selector, u16 old_tss_sel,
  2245. ulong old_tss_base, struct desc_struct *new_desc)
  2246. {
  2247. const struct x86_emulate_ops *ops = ctxt->ops;
  2248. struct tss_segment_32 tss_seg;
  2249. int ret;
  2250. u32 new_tss_base = get_desc_base(new_desc);
  2251. ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
  2252. &ctxt->exception);
  2253. if (ret != X86EMUL_CONTINUE)
  2254. /* FIXME: need to provide precise fault address */
  2255. return ret;
  2256. save_state_to_tss32(ctxt, &tss_seg);
  2257. ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
  2258. &ctxt->exception);
  2259. if (ret != X86EMUL_CONTINUE)
  2260. /* FIXME: need to provide precise fault address */
  2261. return ret;
  2262. ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
  2263. &ctxt->exception);
  2264. if (ret != X86EMUL_CONTINUE)
  2265. /* FIXME: need to provide precise fault address */
  2266. return ret;
  2267. if (old_tss_sel != 0xffff) {
  2268. tss_seg.prev_task_link = old_tss_sel;
  2269. ret = ops->write_std(ctxt, new_tss_base,
  2270. &tss_seg.prev_task_link,
  2271. sizeof tss_seg.prev_task_link,
  2272. &ctxt->exception);
  2273. if (ret != X86EMUL_CONTINUE)
  2274. /* FIXME: need to provide precise fault address */
  2275. return ret;
  2276. }
  2277. return load_state_from_tss32(ctxt, &tss_seg);
  2278. }
  2279. static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
  2280. u16 tss_selector, int idt_index, int reason,
  2281. bool has_error_code, u32 error_code)
  2282. {
  2283. const struct x86_emulate_ops *ops = ctxt->ops;
  2284. struct desc_struct curr_tss_desc, next_tss_desc;
  2285. int ret;
  2286. u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
  2287. ulong old_tss_base =
  2288. ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
  2289. u32 desc_limit;
  2290. ulong desc_addr;
  2291. /* FIXME: old_tss_base == ~0 ? */
  2292. ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
  2293. if (ret != X86EMUL_CONTINUE)
  2294. return ret;
  2295. ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
  2296. if (ret != X86EMUL_CONTINUE)
  2297. return ret;
  2298. /* FIXME: check that next_tss_desc is tss */
  2299. /*
  2300. * Check privileges. The three cases are task switch caused by...
  2301. *
  2302. * 1. jmp/call/int to task gate: Check against DPL of the task gate
  2303. * 2. Exception/IRQ/iret: No check is performed
  2304. * 3. jmp/call to TSS: Check against DPL of the TSS
  2305. */
  2306. if (reason == TASK_SWITCH_GATE) {
  2307. if (idt_index != -1) {
  2308. /* Software interrupts */
  2309. struct desc_struct task_gate_desc;
  2310. int dpl;
  2311. ret = read_interrupt_descriptor(ctxt, idt_index,
  2312. &task_gate_desc);
  2313. if (ret != X86EMUL_CONTINUE)
  2314. return ret;
  2315. dpl = task_gate_desc.dpl;
  2316. if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
  2317. return emulate_gp(ctxt, (idt_index << 3) | 0x2);
  2318. }
  2319. } else if (reason != TASK_SWITCH_IRET) {
  2320. int dpl = next_tss_desc.dpl;
  2321. if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
  2322. return emulate_gp(ctxt, tss_selector);
  2323. }
  2324. desc_limit = desc_limit_scaled(&next_tss_desc);
  2325. if (!next_tss_desc.p ||
  2326. ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
  2327. desc_limit < 0x2b)) {
  2328. emulate_ts(ctxt, tss_selector & 0xfffc);
  2329. return X86EMUL_PROPAGATE_FAULT;
  2330. }
  2331. if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
  2332. curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
  2333. write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
  2334. }
  2335. if (reason == TASK_SWITCH_IRET)
  2336. ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
  2337. /* set back link to prev task only if NT bit is set in eflags
  2338. note that old_tss_sel is not used after this point */
  2339. if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
  2340. old_tss_sel = 0xffff;
  2341. if (next_tss_desc.type & 8)
  2342. ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
  2343. old_tss_base, &next_tss_desc);
  2344. else
  2345. ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
  2346. old_tss_base, &next_tss_desc);
  2347. if (ret != X86EMUL_CONTINUE)
  2348. return ret;
  2349. if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
  2350. ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
  2351. if (reason != TASK_SWITCH_IRET) {
  2352. next_tss_desc.type |= (1 << 1); /* set busy flag */
  2353. write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
  2354. }
  2355. ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
  2356. ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
  2357. if (has_error_code) {
  2358. ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
  2359. ctxt->lock_prefix = 0;
  2360. ctxt->src.val = (unsigned long) error_code;
  2361. ret = em_push(ctxt);
  2362. }
  2363. return ret;
  2364. }
  2365. int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
  2366. u16 tss_selector, int idt_index, int reason,
  2367. bool has_error_code, u32 error_code)
  2368. {
  2369. int rc;
  2370. invalidate_registers(ctxt);
  2371. ctxt->_eip = ctxt->eip;
  2372. ctxt->dst.type = OP_NONE;
  2373. rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
  2374. has_error_code, error_code);
  2375. if (rc == X86EMUL_CONTINUE) {
  2376. ctxt->eip = ctxt->_eip;
  2377. writeback_registers(ctxt);
  2378. }
  2379. return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
  2380. }
  2381. static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
  2382. struct operand *op)
  2383. {
  2384. int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
  2385. register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
  2386. op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
  2387. }
  2388. static int em_das(struct x86_emulate_ctxt *ctxt)
  2389. {
  2390. u8 al, old_al;
  2391. bool af, cf, old_cf;
  2392. cf = ctxt->eflags & X86_EFLAGS_CF;
  2393. al = ctxt->dst.val;
  2394. old_al = al;
  2395. old_cf = cf;
  2396. cf = false;
  2397. af = ctxt->eflags & X86_EFLAGS_AF;
  2398. if ((al & 0x0f) > 9 || af) {
  2399. al -= 6;
  2400. cf = old_cf | (al >= 250);
  2401. af = true;
  2402. } else {
  2403. af = false;
  2404. }
  2405. if (old_al > 0x99 || old_cf) {
  2406. al -= 0x60;
  2407. cf = true;
  2408. }
  2409. ctxt->dst.val = al;
  2410. /* Set PF, ZF, SF */
  2411. ctxt->src.type = OP_IMM;
  2412. ctxt->src.val = 0;
  2413. ctxt->src.bytes = 1;
  2414. fastop(ctxt, em_or);
  2415. ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
  2416. if (cf)
  2417. ctxt->eflags |= X86_EFLAGS_CF;
  2418. if (af)
  2419. ctxt->eflags |= X86_EFLAGS_AF;
  2420. return X86EMUL_CONTINUE;
  2421. }
  2422. static int em_aam(struct x86_emulate_ctxt *ctxt)
  2423. {
  2424. u8 al, ah;
  2425. if (ctxt->src.val == 0)
  2426. return emulate_de(ctxt);
  2427. al = ctxt->dst.val & 0xff;
  2428. ah = al / ctxt->src.val;
  2429. al %= ctxt->src.val;
  2430. ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
  2431. /* Set PF, ZF, SF */
  2432. ctxt->src.type = OP_IMM;
  2433. ctxt->src.val = 0;
  2434. ctxt->src.bytes = 1;
  2435. fastop(ctxt, em_or);
  2436. return X86EMUL_CONTINUE;
  2437. }
  2438. static int em_aad(struct x86_emulate_ctxt *ctxt)
  2439. {
  2440. u8 al = ctxt->dst.val & 0xff;
  2441. u8 ah = (ctxt->dst.val >> 8) & 0xff;
  2442. al = (al + (ah * ctxt->src.val)) & 0xff;
  2443. ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
  2444. /* Set PF, ZF, SF */
  2445. ctxt->src.type = OP_IMM;
  2446. ctxt->src.val = 0;
  2447. ctxt->src.bytes = 1;
  2448. fastop(ctxt, em_or);
  2449. return X86EMUL_CONTINUE;
  2450. }
  2451. static int em_call(struct x86_emulate_ctxt *ctxt)
  2452. {
  2453. long rel = ctxt->src.val;
  2454. ctxt->src.val = (unsigned long)ctxt->_eip;
  2455. jmp_rel(ctxt, rel);
  2456. return em_push(ctxt);
  2457. }
  2458. static int em_call_far(struct x86_emulate_ctxt *ctxt)
  2459. {
  2460. u16 sel, old_cs;
  2461. ulong old_eip;
  2462. int rc;
  2463. old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
  2464. old_eip = ctxt->_eip;
  2465. memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
  2466. if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
  2467. return X86EMUL_CONTINUE;
  2468. ctxt->_eip = 0;
  2469. memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
  2470. ctxt->src.val = old_cs;
  2471. rc = em_push(ctxt);
  2472. if (rc != X86EMUL_CONTINUE)
  2473. return rc;
  2474. ctxt->src.val = old_eip;
  2475. return em_push(ctxt);
  2476. }
  2477. static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
  2478. {
  2479. int rc;
  2480. ctxt->dst.type = OP_REG;
  2481. ctxt->dst.addr.reg = &ctxt->_eip;
  2482. ctxt->dst.bytes = ctxt->op_bytes;
  2483. rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
  2484. if (rc != X86EMUL_CONTINUE)
  2485. return rc;
  2486. rsp_increment(ctxt, ctxt->src.val);
  2487. return X86EMUL_CONTINUE;
  2488. }
  2489. static int em_xchg(struct x86_emulate_ctxt *ctxt)
  2490. {
  2491. /* Write back the register source. */
  2492. ctxt->src.val = ctxt->dst.val;
  2493. write_register_operand(&ctxt->src);
  2494. /* Write back the memory destination with implicit LOCK prefix. */
  2495. ctxt->dst.val = ctxt->src.orig_val;
  2496. ctxt->lock_prefix = 1;
  2497. return X86EMUL_CONTINUE;
  2498. }
  2499. static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
  2500. {
  2501. ctxt->dst.val = ctxt->src2.val;
  2502. return fastop(ctxt, em_imul);
  2503. }
  2504. static int em_cwd(struct x86_emulate_ctxt *ctxt)
  2505. {
  2506. ctxt->dst.type = OP_REG;
  2507. ctxt->dst.bytes = ctxt->src.bytes;
  2508. ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
  2509. ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
  2510. return X86EMUL_CONTINUE;
  2511. }
  2512. static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
  2513. {
  2514. u64 tsc = 0;
  2515. ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
  2516. *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
  2517. *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
  2518. return X86EMUL_CONTINUE;
  2519. }
  2520. static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
  2521. {
  2522. u64 pmc;
  2523. if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
  2524. return emulate_gp(ctxt, 0);
  2525. *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
  2526. *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
  2527. return X86EMUL_CONTINUE;
  2528. }
  2529. static int em_mov(struct x86_emulate_ctxt *ctxt)
  2530. {
  2531. memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
  2532. return X86EMUL_CONTINUE;
  2533. }
  2534. static int em_cr_write(struct x86_emulate_ctxt *ctxt)
  2535. {
  2536. if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
  2537. return emulate_gp(ctxt, 0);
  2538. /* Disable writeback. */
  2539. ctxt->dst.type = OP_NONE;
  2540. return X86EMUL_CONTINUE;
  2541. }
  2542. static int em_dr_write(struct x86_emulate_ctxt *ctxt)
  2543. {
  2544. unsigned long val;
  2545. if (ctxt->mode == X86EMUL_MODE_PROT64)
  2546. val = ctxt->src.val & ~0ULL;
  2547. else
  2548. val = ctxt->src.val & ~0U;
  2549. /* #UD condition is already handled. */
  2550. if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
  2551. return emulate_gp(ctxt, 0);
  2552. /* Disable writeback. */
  2553. ctxt->dst.type = OP_NONE;
  2554. return X86EMUL_CONTINUE;
  2555. }
  2556. static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
  2557. {
  2558. u64 msr_data;
  2559. msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
  2560. | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
  2561. if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
  2562. return emulate_gp(ctxt, 0);
  2563. return X86EMUL_CONTINUE;
  2564. }
  2565. static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
  2566. {
  2567. u64 msr_data;
  2568. if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
  2569. return emulate_gp(ctxt, 0);
  2570. *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
  2571. *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
  2572. return X86EMUL_CONTINUE;
  2573. }
  2574. static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
  2575. {
  2576. if (ctxt->modrm_reg > VCPU_SREG_GS)
  2577. return emulate_ud(ctxt);
  2578. ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
  2579. return X86EMUL_CONTINUE;
  2580. }
  2581. static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
  2582. {
  2583. u16 sel = ctxt->src.val;
  2584. if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
  2585. return emulate_ud(ctxt);
  2586. if (ctxt->modrm_reg == VCPU_SREG_SS)
  2587. ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
  2588. /* Disable writeback. */
  2589. ctxt->dst.type = OP_NONE;
  2590. return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
  2591. }
  2592. static int em_lldt(struct x86_emulate_ctxt *ctxt)
  2593. {
  2594. u16 sel = ctxt->src.val;
  2595. /* Disable writeback. */
  2596. ctxt->dst.type = OP_NONE;
  2597. return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
  2598. }
  2599. static int em_ltr(struct x86_emulate_ctxt *ctxt)
  2600. {
  2601. u16 sel = ctxt->src.val;
  2602. /* Disable writeback. */
  2603. ctxt->dst.type = OP_NONE;
  2604. return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
  2605. }
  2606. static int em_invlpg(struct x86_emulate_ctxt *ctxt)
  2607. {
  2608. int rc;
  2609. ulong linear;
  2610. rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
  2611. if (rc == X86EMUL_CONTINUE)
  2612. ctxt->ops->invlpg(ctxt, linear);
  2613. /* Disable writeback. */
  2614. ctxt->dst.type = OP_NONE;
  2615. return X86EMUL_CONTINUE;
  2616. }
  2617. static int em_clts(struct x86_emulate_ctxt *ctxt)
  2618. {
  2619. ulong cr0;
  2620. cr0 = ctxt->ops->get_cr(ctxt, 0);
  2621. cr0 &= ~X86_CR0_TS;
  2622. ctxt->ops->set_cr(ctxt, 0, cr0);
  2623. return X86EMUL_CONTINUE;
  2624. }
  2625. static int em_vmcall(struct x86_emulate_ctxt *ctxt)
  2626. {
  2627. int rc;
  2628. if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
  2629. return X86EMUL_UNHANDLEABLE;
  2630. rc = ctxt->ops->fix_hypercall(ctxt);
  2631. if (rc != X86EMUL_CONTINUE)
  2632. return rc;
  2633. /* Let the processor re-execute the fixed hypercall */
  2634. ctxt->_eip = ctxt->eip;
  2635. /* Disable writeback. */
  2636. ctxt->dst.type = OP_NONE;
  2637. return X86EMUL_CONTINUE;
  2638. }
  2639. static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
  2640. void (*get)(struct x86_emulate_ctxt *ctxt,
  2641. struct desc_ptr *ptr))
  2642. {
  2643. struct desc_ptr desc_ptr;
  2644. if (ctxt->mode == X86EMUL_MODE_PROT64)
  2645. ctxt->op_bytes = 8;
  2646. get(ctxt, &desc_ptr);
  2647. if (ctxt->op_bytes == 2) {
  2648. ctxt->op_bytes = 4;
  2649. desc_ptr.address &= 0x00ffffff;
  2650. }
  2651. /* Disable writeback. */
  2652. ctxt->dst.type = OP_NONE;
  2653. return segmented_write(ctxt, ctxt->dst.addr.mem,
  2654. &desc_ptr, 2 + ctxt->op_bytes);
  2655. }
  2656. static int em_sgdt(struct x86_emulate_ctxt *ctxt)
  2657. {
  2658. return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
  2659. }
  2660. static int em_sidt(struct x86_emulate_ctxt *ctxt)
  2661. {
  2662. return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
  2663. }
  2664. static int em_lgdt(struct x86_emulate_ctxt *ctxt)
  2665. {
  2666. struct desc_ptr desc_ptr;
  2667. int rc;
  2668. if (ctxt->mode == X86EMUL_MODE_PROT64)
  2669. ctxt->op_bytes = 8;
  2670. rc = read_descriptor(ctxt, ctxt->src.addr.mem,
  2671. &desc_ptr.size, &desc_ptr.address,
  2672. ctxt->op_bytes);
  2673. if (rc != X86EMUL_CONTINUE)
  2674. return rc;
  2675. ctxt->ops->set_gdt(ctxt, &desc_ptr);
  2676. /* Disable writeback. */
  2677. ctxt->dst.type = OP_NONE;
  2678. return X86EMUL_CONTINUE;
  2679. }
  2680. static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
  2681. {
  2682. int rc;
  2683. rc = ctxt->ops->fix_hypercall(ctxt);
  2684. /* Disable writeback. */
  2685. ctxt->dst.type = OP_NONE;
  2686. return rc;
  2687. }
  2688. static int em_lidt(struct x86_emulate_ctxt *ctxt)
  2689. {
  2690. struct desc_ptr desc_ptr;
  2691. int rc;
  2692. if (ctxt->mode == X86EMUL_MODE_PROT64)
  2693. ctxt->op_bytes = 8;
  2694. rc = read_descriptor(ctxt, ctxt->src.addr.mem,
  2695. &desc_ptr.size, &desc_ptr.address,
  2696. ctxt->op_bytes);
  2697. if (rc != X86EMUL_CONTINUE)
  2698. return rc;
  2699. ctxt->ops->set_idt(ctxt, &desc_ptr);
  2700. /* Disable writeback. */
  2701. ctxt->dst.type = OP_NONE;
  2702. return X86EMUL_CONTINUE;
  2703. }
  2704. static int em_smsw(struct x86_emulate_ctxt *ctxt)
  2705. {
  2706. ctxt->dst.bytes = 2;
  2707. ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
  2708. return X86EMUL_CONTINUE;
  2709. }
  2710. static int em_lmsw(struct x86_emulate_ctxt *ctxt)
  2711. {
  2712. ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
  2713. | (ctxt->src.val & 0x0f));
  2714. ctxt->dst.type = OP_NONE;
  2715. return X86EMUL_CONTINUE;
  2716. }
  2717. static int em_loop(struct x86_emulate_ctxt *ctxt)
  2718. {
  2719. register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
  2720. if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
  2721. (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
  2722. jmp_rel(ctxt, ctxt->src.val);
  2723. return X86EMUL_CONTINUE;
  2724. }
  2725. static int em_jcxz(struct x86_emulate_ctxt *ctxt)
  2726. {
  2727. if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
  2728. jmp_rel(ctxt, ctxt->src.val);
  2729. return X86EMUL_CONTINUE;
  2730. }
  2731. static int em_in(struct x86_emulate_ctxt *ctxt)
  2732. {
  2733. if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
  2734. &ctxt->dst.val))
  2735. return X86EMUL_IO_NEEDED;
  2736. return X86EMUL_CONTINUE;
  2737. }
  2738. static int em_out(struct x86_emulate_ctxt *ctxt)
  2739. {
  2740. ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
  2741. &ctxt->src.val, 1);
  2742. /* Disable writeback. */
  2743. ctxt->dst.type = OP_NONE;
  2744. return X86EMUL_CONTINUE;
  2745. }
  2746. static int em_cli(struct x86_emulate_ctxt *ctxt)
  2747. {
  2748. if (emulator_bad_iopl(ctxt))
  2749. return emulate_gp(ctxt, 0);
  2750. ctxt->eflags &= ~X86_EFLAGS_IF;
  2751. return X86EMUL_CONTINUE;
  2752. }
  2753. static int em_sti(struct x86_emulate_ctxt *ctxt)
  2754. {
  2755. if (emulator_bad_iopl(ctxt))
  2756. return emulate_gp(ctxt, 0);
  2757. ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
  2758. ctxt->eflags |= X86_EFLAGS_IF;
  2759. return X86EMUL_CONTINUE;
  2760. }
  2761. static int em_cpuid(struct x86_emulate_ctxt *ctxt)
  2762. {
  2763. u32 eax, ebx, ecx, edx;
  2764. eax = reg_read(ctxt, VCPU_REGS_RAX);
  2765. ecx = reg_read(ctxt, VCPU_REGS_RCX);
  2766. ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
  2767. *reg_write(ctxt, VCPU_REGS_RAX) = eax;
  2768. *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
  2769. *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
  2770. *reg_write(ctxt, VCPU_REGS_RDX) = edx;
  2771. return X86EMUL_CONTINUE;
  2772. }
  2773. static int em_lahf(struct x86_emulate_ctxt *ctxt)
  2774. {
  2775. *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
  2776. *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
  2777. return X86EMUL_CONTINUE;
  2778. }
  2779. static int em_bswap(struct x86_emulate_ctxt *ctxt)
  2780. {
  2781. switch (ctxt->op_bytes) {
  2782. #ifdef CONFIG_X86_64
  2783. case 8:
  2784. asm("bswap %0" : "+r"(ctxt->dst.val));
  2785. break;
  2786. #endif
  2787. default:
  2788. asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
  2789. break;
  2790. }
  2791. return X86EMUL_CONTINUE;
  2792. }
  2793. static bool valid_cr(int nr)
  2794. {
  2795. switch (nr) {
  2796. case 0:
  2797. case 2 ... 4:
  2798. case 8:
  2799. return true;
  2800. default:
  2801. return false;
  2802. }
  2803. }
  2804. static int check_cr_read(struct x86_emulate_ctxt *ctxt)
  2805. {
  2806. if (!valid_cr(ctxt->modrm_reg))
  2807. return emulate_ud(ctxt);
  2808. return X86EMUL_CONTINUE;
  2809. }
  2810. static int check_cr_write(struct x86_emulate_ctxt *ctxt)
  2811. {
  2812. u64 new_val = ctxt->src.val64;
  2813. int cr = ctxt->modrm_reg;
  2814. u64 efer = 0;
  2815. static u64 cr_reserved_bits[] = {
  2816. 0xffffffff00000000ULL,
  2817. 0, 0, 0, /* CR3 checked later */
  2818. CR4_RESERVED_BITS,
  2819. 0, 0, 0,
  2820. CR8_RESERVED_BITS,
  2821. };
  2822. if (!valid_cr(cr))
  2823. return emulate_ud(ctxt);
  2824. if (new_val & cr_reserved_bits[cr])
  2825. return emulate_gp(ctxt, 0);
  2826. switch (cr) {
  2827. case 0: {
  2828. u64 cr4;
  2829. if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
  2830. ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
  2831. return emulate_gp(ctxt, 0);
  2832. cr4 = ctxt->ops->get_cr(ctxt, 4);
  2833. ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
  2834. if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
  2835. !(cr4 & X86_CR4_PAE))
  2836. return emulate_gp(ctxt, 0);
  2837. break;
  2838. }
  2839. case 3: {
  2840. u64 rsvd = 0;
  2841. ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
  2842. if (efer & EFER_LMA)
  2843. rsvd = CR3_L_MODE_RESERVED_BITS;
  2844. else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
  2845. rsvd = CR3_PAE_RESERVED_BITS;
  2846. else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
  2847. rsvd = CR3_NONPAE_RESERVED_BITS;
  2848. if (new_val & rsvd)
  2849. return emulate_gp(ctxt, 0);
  2850. break;
  2851. }
  2852. case 4: {
  2853. ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
  2854. if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
  2855. return emulate_gp(ctxt, 0);
  2856. break;
  2857. }
  2858. }
  2859. return X86EMUL_CONTINUE;
  2860. }
  2861. static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
  2862. {
  2863. unsigned long dr7;
  2864. ctxt->ops->get_dr(ctxt, 7, &dr7);
  2865. /* Check if DR7.Global_Enable is set */
  2866. return dr7 & (1 << 13);
  2867. }
  2868. static int check_dr_read(struct x86_emulate_ctxt *ctxt)
  2869. {
  2870. int dr = ctxt->modrm_reg;
  2871. u64 cr4;
  2872. if (dr > 7)
  2873. return emulate_ud(ctxt);
  2874. cr4 = ctxt->ops->get_cr(ctxt, 4);
  2875. if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
  2876. return emulate_ud(ctxt);
  2877. if (check_dr7_gd(ctxt))
  2878. return emulate_db(ctxt);
  2879. return X86EMUL_CONTINUE;
  2880. }
  2881. static int check_dr_write(struct x86_emulate_ctxt *ctxt)
  2882. {
  2883. u64 new_val = ctxt->src.val64;
  2884. int dr = ctxt->modrm_reg;
  2885. if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
  2886. return emulate_gp(ctxt, 0);
  2887. return check_dr_read(ctxt);
  2888. }
  2889. static int check_svme(struct x86_emulate_ctxt *ctxt)
  2890. {
  2891. u64 efer;
  2892. ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
  2893. if (!(efer & EFER_SVME))
  2894. return emulate_ud(ctxt);
  2895. return X86EMUL_CONTINUE;
  2896. }
  2897. static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
  2898. {
  2899. u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
  2900. /* Valid physical address? */
  2901. if (rax & 0xffff000000000000ULL)
  2902. return emulate_gp(ctxt, 0);
  2903. return check_svme(ctxt);
  2904. }
  2905. static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
  2906. {
  2907. u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
  2908. if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
  2909. return emulate_ud(ctxt);
  2910. return X86EMUL_CONTINUE;
  2911. }
  2912. static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
  2913. {
  2914. u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
  2915. u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
  2916. if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
  2917. (rcx > 3))
  2918. return emulate_gp(ctxt, 0);
  2919. return X86EMUL_CONTINUE;
  2920. }
  2921. static int check_perm_in(struct x86_emulate_ctxt *ctxt)
  2922. {
  2923. ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
  2924. if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
  2925. return emulate_gp(ctxt, 0);
  2926. return X86EMUL_CONTINUE;
  2927. }
  2928. static int check_perm_out(struct x86_emulate_ctxt *ctxt)
  2929. {
  2930. ctxt->src.bytes = min(ctxt->src.bytes, 4u);
  2931. if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
  2932. return emulate_gp(ctxt, 0);
  2933. return X86EMUL_CONTINUE;
  2934. }
  2935. #define D(_y) { .flags = (_y) }
  2936. #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
  2937. #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
  2938. .check_perm = (_p) }
  2939. #define N D(NotImpl)
  2940. #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
  2941. #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
  2942. #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
  2943. #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
  2944. #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
  2945. #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
  2946. #define II(_f, _e, _i) \
  2947. { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
  2948. #define IIP(_f, _e, _i, _p) \
  2949. { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
  2950. .check_perm = (_p) }
  2951. #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
  2952. #define D2bv(_f) D((_f) | ByteOp), D(_f)
  2953. #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
  2954. #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
  2955. #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
  2956. #define I2bvIP(_f, _e, _i, _p) \
  2957. IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
  2958. #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
  2959. F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
  2960. F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
  2961. static const struct opcode group7_rm1[] = {
  2962. DI(SrcNone | Priv, monitor),
  2963. DI(SrcNone | Priv, mwait),
  2964. N, N, N, N, N, N,
  2965. };
  2966. static const struct opcode group7_rm3[] = {
  2967. DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
  2968. II(SrcNone | Prot | VendorSpecific, em_vmmcall, vmmcall),
  2969. DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
  2970. DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
  2971. DIP(SrcNone | Prot | Priv, stgi, check_svme),
  2972. DIP(SrcNone | Prot | Priv, clgi, check_svme),
  2973. DIP(SrcNone | Prot | Priv, skinit, check_svme),
  2974. DIP(SrcNone | Prot | Priv, invlpga, check_svme),
  2975. };
  2976. static const struct opcode group7_rm7[] = {
  2977. N,
  2978. DIP(SrcNone, rdtscp, check_rdtsc),
  2979. N, N, N, N, N, N,
  2980. };
  2981. static const struct opcode group1[] = {
  2982. F(Lock, em_add),
  2983. F(Lock | PageTable, em_or),
  2984. F(Lock, em_adc),
  2985. F(Lock, em_sbb),
  2986. F(Lock | PageTable, em_and),
  2987. F(Lock, em_sub),
  2988. F(Lock, em_xor),
  2989. F(NoWrite, em_cmp),
  2990. };
  2991. static const struct opcode group1A[] = {
  2992. I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
  2993. };
  2994. static const struct opcode group2[] = {
  2995. F(DstMem | ModRM, em_rol),
  2996. F(DstMem | ModRM, em_ror),
  2997. F(DstMem | ModRM, em_rcl),
  2998. F(DstMem | ModRM, em_rcr),
  2999. F(DstMem | ModRM, em_shl),
  3000. F(DstMem | ModRM, em_shr),
  3001. F(DstMem | ModRM, em_shl),
  3002. F(DstMem | ModRM, em_sar),
  3003. };
  3004. static const struct opcode group3[] = {
  3005. F(DstMem | SrcImm | NoWrite, em_test),
  3006. F(DstMem | SrcImm | NoWrite, em_test),
  3007. F(DstMem | SrcNone | Lock, em_not),
  3008. F(DstMem | SrcNone | Lock, em_neg),
  3009. F(DstXacc | Src2Mem, em_mul_ex),
  3010. F(DstXacc | Src2Mem, em_imul_ex),
  3011. F(DstXacc | Src2Mem, em_div_ex),
  3012. F(DstXacc | Src2Mem, em_idiv_ex),
  3013. };
  3014. static const struct opcode group4[] = {
  3015. F(ByteOp | DstMem | SrcNone | Lock, em_inc),
  3016. F(ByteOp | DstMem | SrcNone | Lock, em_dec),
  3017. N, N, N, N, N, N,
  3018. };
  3019. static const struct opcode group5[] = {
  3020. F(DstMem | SrcNone | Lock, em_inc),
  3021. F(DstMem | SrcNone | Lock, em_dec),
  3022. I(SrcMem | Stack, em_grp45),
  3023. I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
  3024. I(SrcMem | Stack, em_grp45),
  3025. I(SrcMemFAddr | ImplicitOps, em_grp45),
  3026. I(SrcMem | Stack, em_grp45), D(Undefined),
  3027. };
  3028. static const struct opcode group6[] = {
  3029. DI(Prot, sldt),
  3030. DI(Prot, str),
  3031. II(Prot | Priv | SrcMem16, em_lldt, lldt),
  3032. II(Prot | Priv | SrcMem16, em_ltr, ltr),
  3033. N, N, N, N,
  3034. };
  3035. static const struct group_dual group7 = { {
  3036. II(Mov | DstMem | Priv, em_sgdt, sgdt),
  3037. II(Mov | DstMem | Priv, em_sidt, sidt),
  3038. II(SrcMem | Priv, em_lgdt, lgdt),
  3039. II(SrcMem | Priv, em_lidt, lidt),
  3040. II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
  3041. II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
  3042. II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
  3043. }, {
  3044. I(SrcNone | Priv | VendorSpecific, em_vmcall),
  3045. EXT(0, group7_rm1),
  3046. N, EXT(0, group7_rm3),
  3047. II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
  3048. II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
  3049. EXT(0, group7_rm7),
  3050. } };
  3051. static const struct opcode group8[] = {
  3052. N, N, N, N,
  3053. F(DstMem | SrcImmByte | NoWrite, em_bt),
  3054. F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
  3055. F(DstMem | SrcImmByte | Lock, em_btr),
  3056. F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
  3057. };
  3058. static const struct group_dual group9 = { {
  3059. N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
  3060. }, {
  3061. N, N, N, N, N, N, N, N,
  3062. } };
  3063. static const struct opcode group11[] = {
  3064. I(DstMem | SrcImm | Mov | PageTable, em_mov),
  3065. X7(D(Undefined)),
  3066. };
  3067. static const struct gprefix pfx_0f_6f_0f_7f = {
  3068. I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
  3069. };
  3070. static const struct gprefix pfx_vmovntpx = {
  3071. I(0, em_mov), N, N, N,
  3072. };
  3073. static const struct escape escape_d9 = { {
  3074. N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
  3075. }, {
  3076. /* 0xC0 - 0xC7 */
  3077. N, N, N, N, N, N, N, N,
  3078. /* 0xC8 - 0xCF */
  3079. N, N, N, N, N, N, N, N,
  3080. /* 0xD0 - 0xC7 */
  3081. N, N, N, N, N, N, N, N,
  3082. /* 0xD8 - 0xDF */
  3083. N, N, N, N, N, N, N, N,
  3084. /* 0xE0 - 0xE7 */
  3085. N, N, N, N, N, N, N, N,
  3086. /* 0xE8 - 0xEF */
  3087. N, N, N, N, N, N, N, N,
  3088. /* 0xF0 - 0xF7 */
  3089. N, N, N, N, N, N, N, N,
  3090. /* 0xF8 - 0xFF */
  3091. N, N, N, N, N, N, N, N,
  3092. } };
  3093. static const struct escape escape_db = { {
  3094. N, N, N, N, N, N, N, N,
  3095. }, {
  3096. /* 0xC0 - 0xC7 */
  3097. N, N, N, N, N, N, N, N,
  3098. /* 0xC8 - 0xCF */
  3099. N, N, N, N, N, N, N, N,
  3100. /* 0xD0 - 0xC7 */
  3101. N, N, N, N, N, N, N, N,
  3102. /* 0xD8 - 0xDF */
  3103. N, N, N, N, N, N, N, N,
  3104. /* 0xE0 - 0xE7 */
  3105. N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
  3106. /* 0xE8 - 0xEF */
  3107. N, N, N, N, N, N, N, N,
  3108. /* 0xF0 - 0xF7 */
  3109. N, N, N, N, N, N, N, N,
  3110. /* 0xF8 - 0xFF */
  3111. N, N, N, N, N, N, N, N,
  3112. } };
  3113. static const struct escape escape_dd = { {
  3114. N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
  3115. }, {
  3116. /* 0xC0 - 0xC7 */
  3117. N, N, N, N, N, N, N, N,
  3118. /* 0xC8 - 0xCF */
  3119. N, N, N, N, N, N, N, N,
  3120. /* 0xD0 - 0xC7 */
  3121. N, N, N, N, N, N, N, N,
  3122. /* 0xD8 - 0xDF */
  3123. N, N, N, N, N, N, N, N,
  3124. /* 0xE0 - 0xE7 */
  3125. N, N, N, N, N, N, N, N,
  3126. /* 0xE8 - 0xEF */
  3127. N, N, N, N, N, N, N, N,
  3128. /* 0xF0 - 0xF7 */
  3129. N, N, N, N, N, N, N, N,
  3130. /* 0xF8 - 0xFF */
  3131. N, N, N, N, N, N, N, N,
  3132. } };
  3133. static const struct opcode opcode_table[256] = {
  3134. /* 0x00 - 0x07 */
  3135. F6ALU(Lock, em_add),
  3136. I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
  3137. I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
  3138. /* 0x08 - 0x0F */
  3139. F6ALU(Lock | PageTable, em_or),
  3140. I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
  3141. N,
  3142. /* 0x10 - 0x17 */
  3143. F6ALU(Lock, em_adc),
  3144. I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
  3145. I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
  3146. /* 0x18 - 0x1F */
  3147. F6ALU(Lock, em_sbb),
  3148. I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
  3149. I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
  3150. /* 0x20 - 0x27 */
  3151. F6ALU(Lock | PageTable, em_and), N, N,
  3152. /* 0x28 - 0x2F */
  3153. F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
  3154. /* 0x30 - 0x37 */
  3155. F6ALU(Lock, em_xor), N, N,
  3156. /* 0x38 - 0x3F */
  3157. F6ALU(NoWrite, em_cmp), N, N,
  3158. /* 0x40 - 0x4F */
  3159. X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
  3160. /* 0x50 - 0x57 */
  3161. X8(I(SrcReg | Stack, em_push)),
  3162. /* 0x58 - 0x5F */
  3163. X8(I(DstReg | Stack, em_pop)),
  3164. /* 0x60 - 0x67 */
  3165. I(ImplicitOps | Stack | No64, em_pusha),
  3166. I(ImplicitOps | Stack | No64, em_popa),
  3167. N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
  3168. N, N, N, N,
  3169. /* 0x68 - 0x6F */
  3170. I(SrcImm | Mov | Stack, em_push),
  3171. I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
  3172. I(SrcImmByte | Mov | Stack, em_push),
  3173. I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
  3174. I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
  3175. I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
  3176. /* 0x70 - 0x7F */
  3177. X16(D(SrcImmByte)),
  3178. /* 0x80 - 0x87 */
  3179. G(ByteOp | DstMem | SrcImm, group1),
  3180. G(DstMem | SrcImm, group1),
  3181. G(ByteOp | DstMem | SrcImm | No64, group1),
  3182. G(DstMem | SrcImmByte, group1),
  3183. F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
  3184. I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
  3185. /* 0x88 - 0x8F */
  3186. I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
  3187. I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
  3188. I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
  3189. D(ModRM | SrcMem | NoAccess | DstReg),
  3190. I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
  3191. G(0, group1A),
  3192. /* 0x90 - 0x97 */
  3193. DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
  3194. /* 0x98 - 0x9F */
  3195. D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
  3196. I(SrcImmFAddr | No64, em_call_far), N,
  3197. II(ImplicitOps | Stack, em_pushf, pushf),
  3198. II(ImplicitOps | Stack, em_popf, popf), N, I(ImplicitOps, em_lahf),
  3199. /* 0xA0 - 0xA7 */
  3200. I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
  3201. I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
  3202. I2bv(SrcSI | DstDI | Mov | String, em_mov),
  3203. F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
  3204. /* 0xA8 - 0xAF */
  3205. F2bv(DstAcc | SrcImm | NoWrite, em_test),
  3206. I2bv(SrcAcc | DstDI | Mov | String, em_mov),
  3207. I2bv(SrcSI | DstAcc | Mov | String, em_mov),
  3208. F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
  3209. /* 0xB0 - 0xB7 */
  3210. X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
  3211. /* 0xB8 - 0xBF */
  3212. X8(I(DstReg | SrcImm64 | Mov, em_mov)),
  3213. /* 0xC0 - 0xC7 */
  3214. G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
  3215. I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
  3216. I(ImplicitOps | Stack, em_ret),
  3217. I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
  3218. I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
  3219. G(ByteOp, group11), G(0, group11),
  3220. /* 0xC8 - 0xCF */
  3221. I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
  3222. N, I(ImplicitOps | Stack, em_ret_far),
  3223. D(ImplicitOps), DI(SrcImmByte, intn),
  3224. D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
  3225. /* 0xD0 - 0xD7 */
  3226. G(Src2One | ByteOp, group2), G(Src2One, group2),
  3227. G(Src2CL | ByteOp, group2), G(Src2CL, group2),
  3228. I(DstAcc | SrcImmUByte | No64, em_aam),
  3229. I(DstAcc | SrcImmUByte | No64, em_aad),
  3230. F(DstAcc | ByteOp | No64, em_salc),
  3231. I(DstAcc | SrcXLat | ByteOp, em_mov),
  3232. /* 0xD8 - 0xDF */
  3233. N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
  3234. /* 0xE0 - 0xE7 */
  3235. X3(I(SrcImmByte, em_loop)),
  3236. I(SrcImmByte, em_jcxz),
  3237. I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
  3238. I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
  3239. /* 0xE8 - 0xEF */
  3240. I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
  3241. I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
  3242. I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
  3243. I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
  3244. /* 0xF0 - 0xF7 */
  3245. N, DI(ImplicitOps, icebp), N, N,
  3246. DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
  3247. G(ByteOp, group3), G(0, group3),
  3248. /* 0xF8 - 0xFF */
  3249. D(ImplicitOps), D(ImplicitOps),
  3250. I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
  3251. D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
  3252. };
  3253. static const struct opcode twobyte_table[256] = {
  3254. /* 0x00 - 0x0F */
  3255. G(0, group6), GD(0, &group7), N, N,
  3256. N, I(ImplicitOps | VendorSpecific, em_syscall),
  3257. II(ImplicitOps | Priv, em_clts, clts), N,
  3258. DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
  3259. N, D(ImplicitOps | ModRM), N, N,
  3260. /* 0x10 - 0x1F */
  3261. N, N, N, N, N, N, N, N,
  3262. D(ImplicitOps | ModRM), N, N, N, N, N, N, D(ImplicitOps | ModRM),
  3263. /* 0x20 - 0x2F */
  3264. DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
  3265. DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
  3266. IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
  3267. IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
  3268. N, N, N, N,
  3269. N, N, N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
  3270. N, N, N, N,
  3271. /* 0x30 - 0x3F */
  3272. II(ImplicitOps | Priv, em_wrmsr, wrmsr),
  3273. IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
  3274. II(ImplicitOps | Priv, em_rdmsr, rdmsr),
  3275. IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
  3276. I(ImplicitOps | VendorSpecific, em_sysenter),
  3277. I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
  3278. N, N,
  3279. N, N, N, N, N, N, N, N,
  3280. /* 0x40 - 0x4F */
  3281. X16(D(DstReg | SrcMem | ModRM | Mov)),
  3282. /* 0x50 - 0x5F */
  3283. N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
  3284. /* 0x60 - 0x6F */
  3285. N, N, N, N,
  3286. N, N, N, N,
  3287. N, N, N, N,
  3288. N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
  3289. /* 0x70 - 0x7F */
  3290. N, N, N, N,
  3291. N, N, N, N,
  3292. N, N, N, N,
  3293. N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
  3294. /* 0x80 - 0x8F */
  3295. X16(D(SrcImm)),
  3296. /* 0x90 - 0x9F */
  3297. X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
  3298. /* 0xA0 - 0xA7 */
  3299. I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
  3300. II(ImplicitOps, em_cpuid, cpuid),
  3301. F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
  3302. F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
  3303. F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
  3304. /* 0xA8 - 0xAF */
  3305. I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
  3306. DI(ImplicitOps, rsm),
  3307. F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
  3308. F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
  3309. F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
  3310. D(ModRM), F(DstReg | SrcMem | ModRM, em_imul),
  3311. /* 0xB0 - 0xB7 */
  3312. I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
  3313. I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
  3314. F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
  3315. I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
  3316. I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
  3317. D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
  3318. /* 0xB8 - 0xBF */
  3319. N, N,
  3320. G(BitOp, group8),
  3321. F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
  3322. F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
  3323. D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
  3324. /* 0xC0 - 0xC7 */
  3325. F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
  3326. N, D(DstMem | SrcReg | ModRM | Mov),
  3327. N, N, N, GD(0, &group9),
  3328. /* 0xC8 - 0xCF */
  3329. X8(I(DstReg, em_bswap)),
  3330. /* 0xD0 - 0xDF */
  3331. N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
  3332. /* 0xE0 - 0xEF */
  3333. N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
  3334. /* 0xF0 - 0xFF */
  3335. N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
  3336. };
  3337. #undef D
  3338. #undef N
  3339. #undef G
  3340. #undef GD
  3341. #undef I
  3342. #undef GP
  3343. #undef EXT
  3344. #undef D2bv
  3345. #undef D2bvIP
  3346. #undef I2bv
  3347. #undef I2bvIP
  3348. #undef I6ALU
  3349. static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
  3350. {
  3351. unsigned size;
  3352. size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
  3353. if (size == 8)
  3354. size = 4;
  3355. return size;
  3356. }
  3357. static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
  3358. unsigned size, bool sign_extension)
  3359. {
  3360. int rc = X86EMUL_CONTINUE;
  3361. op->type = OP_IMM;
  3362. op->bytes = size;
  3363. op->addr.mem.ea = ctxt->_eip;
  3364. /* NB. Immediates are sign-extended as necessary. */
  3365. switch (op->bytes) {
  3366. case 1:
  3367. op->val = insn_fetch(s8, ctxt);
  3368. break;
  3369. case 2:
  3370. op->val = insn_fetch(s16, ctxt);
  3371. break;
  3372. case 4:
  3373. op->val = insn_fetch(s32, ctxt);
  3374. break;
  3375. case 8:
  3376. op->val = insn_fetch(s64, ctxt);
  3377. break;
  3378. }
  3379. if (!sign_extension) {
  3380. switch (op->bytes) {
  3381. case 1:
  3382. op->val &= 0xff;
  3383. break;
  3384. case 2:
  3385. op->val &= 0xffff;
  3386. break;
  3387. case 4:
  3388. op->val &= 0xffffffff;
  3389. break;
  3390. }
  3391. }
  3392. done:
  3393. return rc;
  3394. }
  3395. static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
  3396. unsigned d)
  3397. {
  3398. int rc = X86EMUL_CONTINUE;
  3399. switch (d) {
  3400. case OpReg:
  3401. decode_register_operand(ctxt, op);
  3402. break;
  3403. case OpImmUByte:
  3404. rc = decode_imm(ctxt, op, 1, false);
  3405. break;
  3406. case OpMem:
  3407. ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
  3408. mem_common:
  3409. *op = ctxt->memop;
  3410. ctxt->memopp = op;
  3411. if ((ctxt->d & BitOp) && op == &ctxt->dst)
  3412. fetch_bit_operand(ctxt);
  3413. op->orig_val = op->val;
  3414. break;
  3415. case OpMem64:
  3416. ctxt->memop.bytes = 8;
  3417. goto mem_common;
  3418. case OpAcc:
  3419. op->type = OP_REG;
  3420. op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
  3421. op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
  3422. fetch_register_operand(op);
  3423. op->orig_val = op->val;
  3424. break;
  3425. case OpAccLo:
  3426. op->type = OP_REG;
  3427. op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
  3428. op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
  3429. fetch_register_operand(op);
  3430. op->orig_val = op->val;
  3431. break;
  3432. case OpAccHi:
  3433. if (ctxt->d & ByteOp) {
  3434. op->type = OP_NONE;
  3435. break;
  3436. }
  3437. op->type = OP_REG;
  3438. op->bytes = ctxt->op_bytes;
  3439. op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
  3440. fetch_register_operand(op);
  3441. op->orig_val = op->val;
  3442. break;
  3443. case OpDI:
  3444. op->type = OP_MEM;
  3445. op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
  3446. op->addr.mem.ea =
  3447. register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
  3448. op->addr.mem.seg = VCPU_SREG_ES;
  3449. op->val = 0;
  3450. op->count = 1;
  3451. break;
  3452. case OpDX:
  3453. op->type = OP_REG;
  3454. op->bytes = 2;
  3455. op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
  3456. fetch_register_operand(op);
  3457. break;
  3458. case OpCL:
  3459. op->bytes = 1;
  3460. op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
  3461. break;
  3462. case OpImmByte:
  3463. rc = decode_imm(ctxt, op, 1, true);
  3464. break;
  3465. case OpOne:
  3466. op->bytes = 1;
  3467. op->val = 1;
  3468. break;
  3469. case OpImm:
  3470. rc = decode_imm(ctxt, op, imm_size(ctxt), true);
  3471. break;
  3472. case OpImm64:
  3473. rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
  3474. break;
  3475. case OpMem8:
  3476. ctxt->memop.bytes = 1;
  3477. if (ctxt->memop.type == OP_REG) {
  3478. ctxt->memop.addr.reg = decode_register(ctxt, ctxt->modrm_rm, 1);
  3479. fetch_register_operand(&ctxt->memop);
  3480. }
  3481. goto mem_common;
  3482. case OpMem16:
  3483. ctxt->memop.bytes = 2;
  3484. goto mem_common;
  3485. case OpMem32:
  3486. ctxt->memop.bytes = 4;
  3487. goto mem_common;
  3488. case OpImmU16:
  3489. rc = decode_imm(ctxt, op, 2, false);
  3490. break;
  3491. case OpImmU:
  3492. rc = decode_imm(ctxt, op, imm_size(ctxt), false);
  3493. break;
  3494. case OpSI:
  3495. op->type = OP_MEM;
  3496. op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
  3497. op->addr.mem.ea =
  3498. register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
  3499. op->addr.mem.seg = seg_override(ctxt);
  3500. op->val = 0;
  3501. op->count = 1;
  3502. break;
  3503. case OpXLat:
  3504. op->type = OP_MEM;
  3505. op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
  3506. op->addr.mem.ea =
  3507. register_address(ctxt,
  3508. reg_read(ctxt, VCPU_REGS_RBX) +
  3509. (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
  3510. op->addr.mem.seg = seg_override(ctxt);
  3511. op->val = 0;
  3512. break;
  3513. case OpImmFAddr:
  3514. op->type = OP_IMM;
  3515. op->addr.mem.ea = ctxt->_eip;
  3516. op->bytes = ctxt->op_bytes + 2;
  3517. insn_fetch_arr(op->valptr, op->bytes, ctxt);
  3518. break;
  3519. case OpMemFAddr:
  3520. ctxt->memop.bytes = ctxt->op_bytes + 2;
  3521. goto mem_common;
  3522. case OpES:
  3523. op->val = VCPU_SREG_ES;
  3524. break;
  3525. case OpCS:
  3526. op->val = VCPU_SREG_CS;
  3527. break;
  3528. case OpSS:
  3529. op->val = VCPU_SREG_SS;
  3530. break;
  3531. case OpDS:
  3532. op->val = VCPU_SREG_DS;
  3533. break;
  3534. case OpFS:
  3535. op->val = VCPU_SREG_FS;
  3536. break;
  3537. case OpGS:
  3538. op->val = VCPU_SREG_GS;
  3539. break;
  3540. case OpImplicit:
  3541. /* Special instructions do their own operand decoding. */
  3542. default:
  3543. op->type = OP_NONE; /* Disable writeback. */
  3544. break;
  3545. }
  3546. done:
  3547. return rc;
  3548. }
  3549. int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
  3550. {
  3551. int rc = X86EMUL_CONTINUE;
  3552. int mode = ctxt->mode;
  3553. int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
  3554. bool op_prefix = false;
  3555. struct opcode opcode;
  3556. ctxt->memop.type = OP_NONE;
  3557. ctxt->memopp = NULL;
  3558. ctxt->_eip = ctxt->eip;
  3559. ctxt->fetch.start = ctxt->_eip;
  3560. ctxt->fetch.end = ctxt->fetch.start + insn_len;
  3561. if (insn_len > 0)
  3562. memcpy(ctxt->fetch.data, insn, insn_len);
  3563. switch (mode) {
  3564. case X86EMUL_MODE_REAL:
  3565. case X86EMUL_MODE_VM86:
  3566. case X86EMUL_MODE_PROT16:
  3567. def_op_bytes = def_ad_bytes = 2;
  3568. break;
  3569. case X86EMUL_MODE_PROT32:
  3570. def_op_bytes = def_ad_bytes = 4;
  3571. break;
  3572. #ifdef CONFIG_X86_64
  3573. case X86EMUL_MODE_PROT64:
  3574. def_op_bytes = 4;
  3575. def_ad_bytes = 8;
  3576. break;
  3577. #endif
  3578. default:
  3579. return EMULATION_FAILED;
  3580. }
  3581. ctxt->op_bytes = def_op_bytes;
  3582. ctxt->ad_bytes = def_ad_bytes;
  3583. /* Legacy prefixes. */
  3584. for (;;) {
  3585. switch (ctxt->b = insn_fetch(u8, ctxt)) {
  3586. case 0x66: /* operand-size override */
  3587. op_prefix = true;
  3588. /* switch between 2/4 bytes */
  3589. ctxt->op_bytes = def_op_bytes ^ 6;
  3590. break;
  3591. case 0x67: /* address-size override */
  3592. if (mode == X86EMUL_MODE_PROT64)
  3593. /* switch between 4/8 bytes */
  3594. ctxt->ad_bytes = def_ad_bytes ^ 12;
  3595. else
  3596. /* switch between 2/4 bytes */
  3597. ctxt->ad_bytes = def_ad_bytes ^ 6;
  3598. break;
  3599. case 0x26: /* ES override */
  3600. case 0x2e: /* CS override */
  3601. case 0x36: /* SS override */
  3602. case 0x3e: /* DS override */
  3603. set_seg_override(ctxt, (ctxt->b >> 3) & 3);
  3604. break;
  3605. case 0x64: /* FS override */
  3606. case 0x65: /* GS override */
  3607. set_seg_override(ctxt, ctxt->b & 7);
  3608. break;
  3609. case 0x40 ... 0x4f: /* REX */
  3610. if (mode != X86EMUL_MODE_PROT64)
  3611. goto done_prefixes;
  3612. ctxt->rex_prefix = ctxt->b;
  3613. continue;
  3614. case 0xf0: /* LOCK */
  3615. ctxt->lock_prefix = 1;
  3616. break;
  3617. case 0xf2: /* REPNE/REPNZ */
  3618. case 0xf3: /* REP/REPE/REPZ */
  3619. ctxt->rep_prefix = ctxt->b;
  3620. break;
  3621. default:
  3622. goto done_prefixes;
  3623. }
  3624. /* Any legacy prefix after a REX prefix nullifies its effect. */
  3625. ctxt->rex_prefix = 0;
  3626. }
  3627. done_prefixes:
  3628. /* REX prefix. */
  3629. if (ctxt->rex_prefix & 8)
  3630. ctxt->op_bytes = 8; /* REX.W */
  3631. /* Opcode byte(s). */
  3632. opcode = opcode_table[ctxt->b];
  3633. /* Two-byte opcode? */
  3634. if (ctxt->b == 0x0f) {
  3635. ctxt->twobyte = 1;
  3636. ctxt->b = insn_fetch(u8, ctxt);
  3637. opcode = twobyte_table[ctxt->b];
  3638. }
  3639. ctxt->d = opcode.flags;
  3640. if (ctxt->d & ModRM)
  3641. ctxt->modrm = insn_fetch(u8, ctxt);
  3642. while (ctxt->d & GroupMask) {
  3643. switch (ctxt->d & GroupMask) {
  3644. case Group:
  3645. goffset = (ctxt->modrm >> 3) & 7;
  3646. opcode = opcode.u.group[goffset];
  3647. break;
  3648. case GroupDual:
  3649. goffset = (ctxt->modrm >> 3) & 7;
  3650. if ((ctxt->modrm >> 6) == 3)
  3651. opcode = opcode.u.gdual->mod3[goffset];
  3652. else
  3653. opcode = opcode.u.gdual->mod012[goffset];
  3654. break;
  3655. case RMExt:
  3656. goffset = ctxt->modrm & 7;
  3657. opcode = opcode.u.group[goffset];
  3658. break;
  3659. case Prefix:
  3660. if (ctxt->rep_prefix && op_prefix)
  3661. return EMULATION_FAILED;
  3662. simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
  3663. switch (simd_prefix) {
  3664. case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
  3665. case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
  3666. case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
  3667. case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
  3668. }
  3669. break;
  3670. case Escape:
  3671. if (ctxt->modrm > 0xbf)
  3672. opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
  3673. else
  3674. opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
  3675. break;
  3676. default:
  3677. return EMULATION_FAILED;
  3678. }
  3679. ctxt->d &= ~(u64)GroupMask;
  3680. ctxt->d |= opcode.flags;
  3681. }
  3682. ctxt->execute = opcode.u.execute;
  3683. ctxt->check_perm = opcode.check_perm;
  3684. ctxt->intercept = opcode.intercept;
  3685. /* Unrecognised? */
  3686. if (ctxt->d == 0 || (ctxt->d & NotImpl))
  3687. return EMULATION_FAILED;
  3688. if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
  3689. return EMULATION_FAILED;
  3690. if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
  3691. ctxt->op_bytes = 8;
  3692. if (ctxt->d & Op3264) {
  3693. if (mode == X86EMUL_MODE_PROT64)
  3694. ctxt->op_bytes = 8;
  3695. else
  3696. ctxt->op_bytes = 4;
  3697. }
  3698. if (ctxt->d & Sse)
  3699. ctxt->op_bytes = 16;
  3700. else if (ctxt->d & Mmx)
  3701. ctxt->op_bytes = 8;
  3702. /* ModRM and SIB bytes. */
  3703. if (ctxt->d & ModRM) {
  3704. rc = decode_modrm(ctxt, &ctxt->memop);
  3705. if (!ctxt->has_seg_override)
  3706. set_seg_override(ctxt, ctxt->modrm_seg);
  3707. } else if (ctxt->d & MemAbs)
  3708. rc = decode_abs(ctxt, &ctxt->memop);
  3709. if (rc != X86EMUL_CONTINUE)
  3710. goto done;
  3711. if (!ctxt->has_seg_override)
  3712. set_seg_override(ctxt, VCPU_SREG_DS);
  3713. ctxt->memop.addr.mem.seg = seg_override(ctxt);
  3714. if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
  3715. ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
  3716. /*
  3717. * Decode and fetch the source operand: register, memory
  3718. * or immediate.
  3719. */
  3720. rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
  3721. if (rc != X86EMUL_CONTINUE)
  3722. goto done;
  3723. /*
  3724. * Decode and fetch the second source operand: register, memory
  3725. * or immediate.
  3726. */
  3727. rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
  3728. if (rc != X86EMUL_CONTINUE)
  3729. goto done;
  3730. /* Decode and fetch the destination operand: register or memory. */
  3731. rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
  3732. done:
  3733. if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
  3734. ctxt->memopp->addr.mem.ea += ctxt->_eip;
  3735. return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
  3736. }
  3737. bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
  3738. {
  3739. return ctxt->d & PageTable;
  3740. }
  3741. static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
  3742. {
  3743. /* The second termination condition only applies for REPE
  3744. * and REPNE. Test if the repeat string operation prefix is
  3745. * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
  3746. * corresponding termination condition according to:
  3747. * - if REPE/REPZ and ZF = 0 then done
  3748. * - if REPNE/REPNZ and ZF = 1 then done
  3749. */
  3750. if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
  3751. (ctxt->b == 0xae) || (ctxt->b == 0xaf))
  3752. && (((ctxt->rep_prefix == REPE_PREFIX) &&
  3753. ((ctxt->eflags & EFLG_ZF) == 0))
  3754. || ((ctxt->rep_prefix == REPNE_PREFIX) &&
  3755. ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
  3756. return true;
  3757. return false;
  3758. }
  3759. static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
  3760. {
  3761. bool fault = false;
  3762. ctxt->ops->get_fpu(ctxt);
  3763. asm volatile("1: fwait \n\t"
  3764. "2: \n\t"
  3765. ".pushsection .fixup,\"ax\" \n\t"
  3766. "3: \n\t"
  3767. "movb $1, %[fault] \n\t"
  3768. "jmp 2b \n\t"
  3769. ".popsection \n\t"
  3770. _ASM_EXTABLE(1b, 3b)
  3771. : [fault]"+qm"(fault));
  3772. ctxt->ops->put_fpu(ctxt);
  3773. if (unlikely(fault))
  3774. return emulate_exception(ctxt, MF_VECTOR, 0, false);
  3775. return X86EMUL_CONTINUE;
  3776. }
  3777. static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
  3778. struct operand *op)
  3779. {
  3780. if (op->type == OP_MM)
  3781. read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
  3782. }
  3783. static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
  3784. {
  3785. ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
  3786. if (!(ctxt->d & ByteOp))
  3787. fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
  3788. asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
  3789. : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
  3790. [fastop]"+S"(fop)
  3791. : "c"(ctxt->src2.val));
  3792. ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
  3793. if (!fop) /* exception is returned in fop variable */
  3794. return emulate_de(ctxt);
  3795. return X86EMUL_CONTINUE;
  3796. }
  3797. int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
  3798. {
  3799. const struct x86_emulate_ops *ops = ctxt->ops;
  3800. int rc = X86EMUL_CONTINUE;
  3801. int saved_dst_type = ctxt->dst.type;
  3802. ctxt->mem_read.pos = 0;
  3803. if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
  3804. (ctxt->d & Undefined)) {
  3805. rc = emulate_ud(ctxt);
  3806. goto done;
  3807. }
  3808. /* LOCK prefix is allowed only with some instructions */
  3809. if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
  3810. rc = emulate_ud(ctxt);
  3811. goto done;
  3812. }
  3813. if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
  3814. rc = emulate_ud(ctxt);
  3815. goto done;
  3816. }
  3817. if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
  3818. || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
  3819. rc = emulate_ud(ctxt);
  3820. goto done;
  3821. }
  3822. if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
  3823. rc = emulate_nm(ctxt);
  3824. goto done;
  3825. }
  3826. if (ctxt->d & Mmx) {
  3827. rc = flush_pending_x87_faults(ctxt);
  3828. if (rc != X86EMUL_CONTINUE)
  3829. goto done;
  3830. /*
  3831. * Now that we know the fpu is exception safe, we can fetch
  3832. * operands from it.
  3833. */
  3834. fetch_possible_mmx_operand(ctxt, &ctxt->src);
  3835. fetch_possible_mmx_operand(ctxt, &ctxt->src2);
  3836. if (!(ctxt->d & Mov))
  3837. fetch_possible_mmx_operand(ctxt, &ctxt->dst);
  3838. }
  3839. if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
  3840. rc = emulator_check_intercept(ctxt, ctxt->intercept,
  3841. X86_ICPT_PRE_EXCEPT);
  3842. if (rc != X86EMUL_CONTINUE)
  3843. goto done;
  3844. }
  3845. /* Privileged instruction can be executed only in CPL=0 */
  3846. if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
  3847. rc = emulate_gp(ctxt, 0);
  3848. goto done;
  3849. }
  3850. /* Instruction can only be executed in protected mode */
  3851. if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
  3852. rc = emulate_ud(ctxt);
  3853. goto done;
  3854. }
  3855. /* Do instruction specific permission checks */
  3856. if (ctxt->check_perm) {
  3857. rc = ctxt->check_perm(ctxt);
  3858. if (rc != X86EMUL_CONTINUE)
  3859. goto done;
  3860. }
  3861. if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
  3862. rc = emulator_check_intercept(ctxt, ctxt->intercept,
  3863. X86_ICPT_POST_EXCEPT);
  3864. if (rc != X86EMUL_CONTINUE)
  3865. goto done;
  3866. }
  3867. if (ctxt->rep_prefix && (ctxt->d & String)) {
  3868. /* All REP prefixes have the same first termination condition */
  3869. if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
  3870. ctxt->eip = ctxt->_eip;
  3871. goto done;
  3872. }
  3873. }
  3874. if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
  3875. rc = segmented_read(ctxt, ctxt->src.addr.mem,
  3876. ctxt->src.valptr, ctxt->src.bytes);
  3877. if (rc != X86EMUL_CONTINUE)
  3878. goto done;
  3879. ctxt->src.orig_val64 = ctxt->src.val64;
  3880. }
  3881. if (ctxt->src2.type == OP_MEM) {
  3882. rc = segmented_read(ctxt, ctxt->src2.addr.mem,
  3883. &ctxt->src2.val, ctxt->src2.bytes);
  3884. if (rc != X86EMUL_CONTINUE)
  3885. goto done;
  3886. }
  3887. if ((ctxt->d & DstMask) == ImplicitOps)
  3888. goto special_insn;
  3889. if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
  3890. /* optimisation - avoid slow emulated read if Mov */
  3891. rc = segmented_read(ctxt, ctxt->dst.addr.mem,
  3892. &ctxt->dst.val, ctxt->dst.bytes);
  3893. if (rc != X86EMUL_CONTINUE)
  3894. goto done;
  3895. }
  3896. ctxt->dst.orig_val = ctxt->dst.val;
  3897. special_insn:
  3898. if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
  3899. rc = emulator_check_intercept(ctxt, ctxt->intercept,
  3900. X86_ICPT_POST_MEMACCESS);
  3901. if (rc != X86EMUL_CONTINUE)
  3902. goto done;
  3903. }
  3904. if (ctxt->execute) {
  3905. if (ctxt->d & Fastop) {
  3906. void (*fop)(struct fastop *) = (void *)ctxt->execute;
  3907. rc = fastop(ctxt, fop);
  3908. if (rc != X86EMUL_CONTINUE)
  3909. goto done;
  3910. goto writeback;
  3911. }
  3912. rc = ctxt->execute(ctxt);
  3913. if (rc != X86EMUL_CONTINUE)
  3914. goto done;
  3915. goto writeback;
  3916. }
  3917. if (ctxt->twobyte)
  3918. goto twobyte_insn;
  3919. switch (ctxt->b) {
  3920. case 0x63: /* movsxd */
  3921. if (ctxt->mode != X86EMUL_MODE_PROT64)
  3922. goto cannot_emulate;
  3923. ctxt->dst.val = (s32) ctxt->src.val;
  3924. break;
  3925. case 0x70 ... 0x7f: /* jcc (short) */
  3926. if (test_cc(ctxt->b, ctxt->eflags))
  3927. jmp_rel(ctxt, ctxt->src.val);
  3928. break;
  3929. case 0x8d: /* lea r16/r32, m */
  3930. ctxt->dst.val = ctxt->src.addr.mem.ea;
  3931. break;
  3932. case 0x90 ... 0x97: /* nop / xchg reg, rax */
  3933. if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
  3934. break;
  3935. rc = em_xchg(ctxt);
  3936. break;
  3937. case 0x98: /* cbw/cwde/cdqe */
  3938. switch (ctxt->op_bytes) {
  3939. case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
  3940. case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
  3941. case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
  3942. }
  3943. break;
  3944. case 0xcc: /* int3 */
  3945. rc = emulate_int(ctxt, 3);
  3946. break;
  3947. case 0xcd: /* int n */
  3948. rc = emulate_int(ctxt, ctxt->src.val);
  3949. break;
  3950. case 0xce: /* into */
  3951. if (ctxt->eflags & EFLG_OF)
  3952. rc = emulate_int(ctxt, 4);
  3953. break;
  3954. case 0xe9: /* jmp rel */
  3955. case 0xeb: /* jmp rel short */
  3956. jmp_rel(ctxt, ctxt->src.val);
  3957. ctxt->dst.type = OP_NONE; /* Disable writeback. */
  3958. break;
  3959. case 0xf4: /* hlt */
  3960. ctxt->ops->halt(ctxt);
  3961. break;
  3962. case 0xf5: /* cmc */
  3963. /* complement carry flag from eflags reg */
  3964. ctxt->eflags ^= EFLG_CF;
  3965. break;
  3966. case 0xf8: /* clc */
  3967. ctxt->eflags &= ~EFLG_CF;
  3968. break;
  3969. case 0xf9: /* stc */
  3970. ctxt->eflags |= EFLG_CF;
  3971. break;
  3972. case 0xfc: /* cld */
  3973. ctxt->eflags &= ~EFLG_DF;
  3974. break;
  3975. case 0xfd: /* std */
  3976. ctxt->eflags |= EFLG_DF;
  3977. break;
  3978. default:
  3979. goto cannot_emulate;
  3980. }
  3981. if (rc != X86EMUL_CONTINUE)
  3982. goto done;
  3983. writeback:
  3984. if (!(ctxt->d & NoWrite)) {
  3985. rc = writeback(ctxt, &ctxt->dst);
  3986. if (rc != X86EMUL_CONTINUE)
  3987. goto done;
  3988. }
  3989. if (ctxt->d & SrcWrite) {
  3990. BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
  3991. rc = writeback(ctxt, &ctxt->src);
  3992. if (rc != X86EMUL_CONTINUE)
  3993. goto done;
  3994. }
  3995. /*
  3996. * restore dst type in case the decoding will be reused
  3997. * (happens for string instruction )
  3998. */
  3999. ctxt->dst.type = saved_dst_type;
  4000. if ((ctxt->d & SrcMask) == SrcSI)
  4001. string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
  4002. if ((ctxt->d & DstMask) == DstDI)
  4003. string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
  4004. if (ctxt->rep_prefix && (ctxt->d & String)) {
  4005. unsigned int count;
  4006. struct read_cache *r = &ctxt->io_read;
  4007. if ((ctxt->d & SrcMask) == SrcSI)
  4008. count = ctxt->src.count;
  4009. else
  4010. count = ctxt->dst.count;
  4011. register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
  4012. -count);
  4013. if (!string_insn_completed(ctxt)) {
  4014. /*
  4015. * Re-enter guest when pio read ahead buffer is empty
  4016. * or, if it is not used, after each 1024 iteration.
  4017. */
  4018. if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
  4019. (r->end == 0 || r->end != r->pos)) {
  4020. /*
  4021. * Reset read cache. Usually happens before
  4022. * decode, but since instruction is restarted
  4023. * we have to do it here.
  4024. */
  4025. ctxt->mem_read.end = 0;
  4026. writeback_registers(ctxt);
  4027. return EMULATION_RESTART;
  4028. }
  4029. goto done; /* skip rip writeback */
  4030. }
  4031. }
  4032. ctxt->eip = ctxt->_eip;
  4033. done:
  4034. if (rc == X86EMUL_PROPAGATE_FAULT)
  4035. ctxt->have_exception = true;
  4036. if (rc == X86EMUL_INTERCEPTED)
  4037. return EMULATION_INTERCEPTED;
  4038. if (rc == X86EMUL_CONTINUE)
  4039. writeback_registers(ctxt);
  4040. return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
  4041. twobyte_insn:
  4042. switch (ctxt->b) {
  4043. case 0x09: /* wbinvd */
  4044. (ctxt->ops->wbinvd)(ctxt);
  4045. break;
  4046. case 0x08: /* invd */
  4047. case 0x0d: /* GrpP (prefetch) */
  4048. case 0x18: /* Grp16 (prefetch/nop) */
  4049. case 0x1f: /* nop */
  4050. break;
  4051. case 0x20: /* mov cr, reg */
  4052. ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
  4053. break;
  4054. case 0x21: /* mov from dr to reg */
  4055. ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
  4056. break;
  4057. case 0x40 ... 0x4f: /* cmov */
  4058. ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
  4059. if (!test_cc(ctxt->b, ctxt->eflags))
  4060. ctxt->dst.type = OP_NONE; /* no writeback */
  4061. break;
  4062. case 0x80 ... 0x8f: /* jnz rel, etc*/
  4063. if (test_cc(ctxt->b, ctxt->eflags))
  4064. jmp_rel(ctxt, ctxt->src.val);
  4065. break;
  4066. case 0x90 ... 0x9f: /* setcc r/m8 */
  4067. ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
  4068. break;
  4069. case 0xae: /* clflush */
  4070. break;
  4071. case 0xb6 ... 0xb7: /* movzx */
  4072. ctxt->dst.bytes = ctxt->op_bytes;
  4073. ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
  4074. : (u16) ctxt->src.val;
  4075. break;
  4076. case 0xbe ... 0xbf: /* movsx */
  4077. ctxt->dst.bytes = ctxt->op_bytes;
  4078. ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
  4079. (s16) ctxt->src.val;
  4080. break;
  4081. case 0xc3: /* movnti */
  4082. ctxt->dst.bytes = ctxt->op_bytes;
  4083. ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
  4084. (u64) ctxt->src.val;
  4085. break;
  4086. default:
  4087. goto cannot_emulate;
  4088. }
  4089. if (rc != X86EMUL_CONTINUE)
  4090. goto done;
  4091. goto writeback;
  4092. cannot_emulate:
  4093. return EMULATION_FAILED;
  4094. }
  4095. void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
  4096. {
  4097. invalidate_registers(ctxt);
  4098. }
  4099. void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
  4100. {
  4101. writeback_registers(ctxt);
  4102. }