lapic.c 27 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169
  1. /*
  2. * Local APIC virtualization
  3. *
  4. * Copyright (C) 2006 Qumranet, Inc.
  5. * Copyright (C) 2007 Novell
  6. * Copyright (C) 2007 Intel
  7. *
  8. * Authors:
  9. * Dor Laor <dor.laor@qumranet.com>
  10. * Gregory Haskins <ghaskins@novell.com>
  11. * Yaozu (Eddie) Dong <eddie.dong@intel.com>
  12. *
  13. * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation.
  14. *
  15. * This work is licensed under the terms of the GNU GPL, version 2. See
  16. * the COPYING file in the top-level directory.
  17. */
  18. #include <linux/kvm_host.h>
  19. #include <linux/kvm.h>
  20. #include <linux/mm.h>
  21. #include <linux/highmem.h>
  22. #include <linux/smp.h>
  23. #include <linux/hrtimer.h>
  24. #include <linux/io.h>
  25. #include <linux/module.h>
  26. #include <linux/math64.h>
  27. #include <asm/processor.h>
  28. #include <asm/msr.h>
  29. #include <asm/page.h>
  30. #include <asm/current.h>
  31. #include <asm/apicdef.h>
  32. #include <asm/atomic.h>
  33. #include "kvm_cache_regs.h"
  34. #include "irq.h"
  35. #define PRId64 "d"
  36. #define PRIx64 "llx"
  37. #define PRIu64 "u"
  38. #define PRIo64 "o"
  39. #define APIC_BUS_CYCLE_NS 1
  40. /* #define apic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) */
  41. #define apic_debug(fmt, arg...)
  42. #define APIC_LVT_NUM 6
  43. /* 14 is the version for Xeon and Pentium 8.4.8*/
  44. #define APIC_VERSION (0x14UL | ((APIC_LVT_NUM - 1) << 16))
  45. #define LAPIC_MMIO_LENGTH (1 << 12)
  46. /* followed define is not in apicdef.h */
  47. #define APIC_SHORT_MASK 0xc0000
  48. #define APIC_DEST_NOSHORT 0x0
  49. #define APIC_DEST_MASK 0x800
  50. #define MAX_APIC_VECTOR 256
  51. #define VEC_POS(v) ((v) & (32 - 1))
  52. #define REG_POS(v) (((v) >> 5) << 4)
  53. static inline u32 apic_get_reg(struct kvm_lapic *apic, int reg_off)
  54. {
  55. return *((u32 *) (apic->regs + reg_off));
  56. }
  57. static inline void apic_set_reg(struct kvm_lapic *apic, int reg_off, u32 val)
  58. {
  59. *((u32 *) (apic->regs + reg_off)) = val;
  60. }
  61. static inline int apic_test_and_set_vector(int vec, void *bitmap)
  62. {
  63. return test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
  64. }
  65. static inline int apic_test_and_clear_vector(int vec, void *bitmap)
  66. {
  67. return test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
  68. }
  69. static inline void apic_set_vector(int vec, void *bitmap)
  70. {
  71. set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
  72. }
  73. static inline void apic_clear_vector(int vec, void *bitmap)
  74. {
  75. clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
  76. }
  77. static inline int apic_hw_enabled(struct kvm_lapic *apic)
  78. {
  79. return (apic)->vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE;
  80. }
  81. static inline int apic_sw_enabled(struct kvm_lapic *apic)
  82. {
  83. return apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_APIC_ENABLED;
  84. }
  85. static inline int apic_enabled(struct kvm_lapic *apic)
  86. {
  87. return apic_sw_enabled(apic) && apic_hw_enabled(apic);
  88. }
  89. #define LVT_MASK \
  90. (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK)
  91. #define LINT_MASK \
  92. (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \
  93. APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER)
  94. static inline int kvm_apic_id(struct kvm_lapic *apic)
  95. {
  96. return (apic_get_reg(apic, APIC_ID) >> 24) & 0xff;
  97. }
  98. static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type)
  99. {
  100. return !(apic_get_reg(apic, lvt_type) & APIC_LVT_MASKED);
  101. }
  102. static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type)
  103. {
  104. return apic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK;
  105. }
  106. static inline int apic_lvtt_period(struct kvm_lapic *apic)
  107. {
  108. return apic_get_reg(apic, APIC_LVTT) & APIC_LVT_TIMER_PERIODIC;
  109. }
  110. static unsigned int apic_lvt_mask[APIC_LVT_NUM] = {
  111. LVT_MASK | APIC_LVT_TIMER_PERIODIC, /* LVTT */
  112. LVT_MASK | APIC_MODE_MASK, /* LVTTHMR */
  113. LVT_MASK | APIC_MODE_MASK, /* LVTPC */
  114. LINT_MASK, LINT_MASK, /* LVT0-1 */
  115. LVT_MASK /* LVTERR */
  116. };
  117. static int find_highest_vector(void *bitmap)
  118. {
  119. u32 *word = bitmap;
  120. int word_offset = MAX_APIC_VECTOR >> 5;
  121. while ((word_offset != 0) && (word[(--word_offset) << 2] == 0))
  122. continue;
  123. if (likely(!word_offset && !word[0]))
  124. return -1;
  125. else
  126. return fls(word[word_offset << 2]) - 1 + (word_offset << 5);
  127. }
  128. static inline int apic_test_and_set_irr(int vec, struct kvm_lapic *apic)
  129. {
  130. return apic_test_and_set_vector(vec, apic->regs + APIC_IRR);
  131. }
  132. static inline void apic_clear_irr(int vec, struct kvm_lapic *apic)
  133. {
  134. apic_clear_vector(vec, apic->regs + APIC_IRR);
  135. }
  136. static inline int apic_find_highest_irr(struct kvm_lapic *apic)
  137. {
  138. int result;
  139. result = find_highest_vector(apic->regs + APIC_IRR);
  140. ASSERT(result == -1 || result >= 16);
  141. return result;
  142. }
  143. int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu)
  144. {
  145. struct kvm_lapic *apic = vcpu->arch.apic;
  146. int highest_irr;
  147. if (!apic)
  148. return 0;
  149. highest_irr = apic_find_highest_irr(apic);
  150. return highest_irr;
  151. }
  152. EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr);
  153. int kvm_apic_set_irq(struct kvm_vcpu *vcpu, u8 vec, u8 trig)
  154. {
  155. struct kvm_lapic *apic = vcpu->arch.apic;
  156. if (!apic_test_and_set_irr(vec, apic)) {
  157. /* a new pending irq is set in IRR */
  158. if (trig)
  159. apic_set_vector(vec, apic->regs + APIC_TMR);
  160. else
  161. apic_clear_vector(vec, apic->regs + APIC_TMR);
  162. kvm_vcpu_kick(apic->vcpu);
  163. return 1;
  164. }
  165. return 0;
  166. }
  167. static inline int apic_find_highest_isr(struct kvm_lapic *apic)
  168. {
  169. int result;
  170. result = find_highest_vector(apic->regs + APIC_ISR);
  171. ASSERT(result == -1 || result >= 16);
  172. return result;
  173. }
  174. static void apic_update_ppr(struct kvm_lapic *apic)
  175. {
  176. u32 tpr, isrv, ppr;
  177. int isr;
  178. tpr = apic_get_reg(apic, APIC_TASKPRI);
  179. isr = apic_find_highest_isr(apic);
  180. isrv = (isr != -1) ? isr : 0;
  181. if ((tpr & 0xf0) >= (isrv & 0xf0))
  182. ppr = tpr & 0xff;
  183. else
  184. ppr = isrv & 0xf0;
  185. apic_debug("vlapic %p, ppr 0x%x, isr 0x%x, isrv 0x%x",
  186. apic, ppr, isr, isrv);
  187. apic_set_reg(apic, APIC_PROCPRI, ppr);
  188. }
  189. static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr)
  190. {
  191. apic_set_reg(apic, APIC_TASKPRI, tpr);
  192. apic_update_ppr(apic);
  193. }
  194. int kvm_apic_match_physical_addr(struct kvm_lapic *apic, u16 dest)
  195. {
  196. return kvm_apic_id(apic) == dest;
  197. }
  198. int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda)
  199. {
  200. int result = 0;
  201. u8 logical_id;
  202. logical_id = GET_APIC_LOGICAL_ID(apic_get_reg(apic, APIC_LDR));
  203. switch (apic_get_reg(apic, APIC_DFR)) {
  204. case APIC_DFR_FLAT:
  205. if (logical_id & mda)
  206. result = 1;
  207. break;
  208. case APIC_DFR_CLUSTER:
  209. if (((logical_id >> 4) == (mda >> 0x4))
  210. && (logical_id & mda & 0xf))
  211. result = 1;
  212. break;
  213. default:
  214. printk(KERN_WARNING "Bad DFR vcpu %d: %08x\n",
  215. apic->vcpu->vcpu_id, apic_get_reg(apic, APIC_DFR));
  216. break;
  217. }
  218. return result;
  219. }
  220. static int apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
  221. int short_hand, int dest, int dest_mode)
  222. {
  223. int result = 0;
  224. struct kvm_lapic *target = vcpu->arch.apic;
  225. apic_debug("target %p, source %p, dest 0x%x, "
  226. "dest_mode 0x%x, short_hand 0x%x",
  227. target, source, dest, dest_mode, short_hand);
  228. ASSERT(!target);
  229. switch (short_hand) {
  230. case APIC_DEST_NOSHORT:
  231. if (dest_mode == 0) {
  232. /* Physical mode. */
  233. if ((dest == 0xFF) || (dest == kvm_apic_id(target)))
  234. result = 1;
  235. } else
  236. /* Logical mode. */
  237. result = kvm_apic_match_logical_addr(target, dest);
  238. break;
  239. case APIC_DEST_SELF:
  240. if (target == source)
  241. result = 1;
  242. break;
  243. case APIC_DEST_ALLINC:
  244. result = 1;
  245. break;
  246. case APIC_DEST_ALLBUT:
  247. if (target != source)
  248. result = 1;
  249. break;
  250. default:
  251. printk(KERN_WARNING "Bad dest shorthand value %x\n",
  252. short_hand);
  253. break;
  254. }
  255. return result;
  256. }
  257. /*
  258. * Add a pending IRQ into lapic.
  259. * Return 1 if successfully added and 0 if discarded.
  260. */
  261. static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
  262. int vector, int level, int trig_mode)
  263. {
  264. int orig_irr, result = 0;
  265. struct kvm_vcpu *vcpu = apic->vcpu;
  266. switch (delivery_mode) {
  267. case APIC_DM_FIXED:
  268. case APIC_DM_LOWEST:
  269. /* FIXME add logic for vcpu on reset */
  270. if (unlikely(!apic_enabled(apic)))
  271. break;
  272. orig_irr = apic_test_and_set_irr(vector, apic);
  273. if (orig_irr && trig_mode) {
  274. apic_debug("level trig mode repeatedly for vector %d",
  275. vector);
  276. break;
  277. }
  278. if (trig_mode) {
  279. apic_debug("level trig mode for vector %d", vector);
  280. apic_set_vector(vector, apic->regs + APIC_TMR);
  281. } else
  282. apic_clear_vector(vector, apic->regs + APIC_TMR);
  283. kvm_vcpu_kick(vcpu);
  284. result = (orig_irr == 0);
  285. break;
  286. case APIC_DM_REMRD:
  287. printk(KERN_DEBUG "Ignoring delivery mode 3\n");
  288. break;
  289. case APIC_DM_SMI:
  290. printk(KERN_DEBUG "Ignoring guest SMI\n");
  291. break;
  292. case APIC_DM_NMI:
  293. kvm_inject_nmi(vcpu);
  294. break;
  295. case APIC_DM_INIT:
  296. if (level) {
  297. if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE)
  298. printk(KERN_DEBUG
  299. "INIT on a runnable vcpu %d\n",
  300. vcpu->vcpu_id);
  301. vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
  302. kvm_vcpu_kick(vcpu);
  303. } else {
  304. apic_debug("Ignoring de-assert INIT to vcpu %d\n",
  305. vcpu->vcpu_id);
  306. }
  307. break;
  308. case APIC_DM_STARTUP:
  309. apic_debug("SIPI to vcpu %d vector 0x%02x\n",
  310. vcpu->vcpu_id, vector);
  311. if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
  312. vcpu->arch.sipi_vector = vector;
  313. vcpu->arch.mp_state = KVM_MP_STATE_SIPI_RECEIVED;
  314. kvm_vcpu_kick(vcpu);
  315. }
  316. break;
  317. default:
  318. printk(KERN_ERR "TODO: unsupported delivery mode %x\n",
  319. delivery_mode);
  320. break;
  321. }
  322. return result;
  323. }
  324. static struct kvm_lapic *kvm_apic_round_robin(struct kvm *kvm, u8 vector,
  325. unsigned long bitmap)
  326. {
  327. int last;
  328. int next;
  329. struct kvm_lapic *apic = NULL;
  330. last = kvm->arch.round_robin_prev_vcpu;
  331. next = last;
  332. do {
  333. if (++next == KVM_MAX_VCPUS)
  334. next = 0;
  335. if (kvm->vcpus[next] == NULL || !test_bit(next, &bitmap))
  336. continue;
  337. apic = kvm->vcpus[next]->arch.apic;
  338. if (apic && apic_enabled(apic))
  339. break;
  340. apic = NULL;
  341. } while (next != last);
  342. kvm->arch.round_robin_prev_vcpu = next;
  343. if (!apic)
  344. printk(KERN_DEBUG "vcpu not ready for apic_round_robin\n");
  345. return apic;
  346. }
  347. struct kvm_vcpu *kvm_get_lowest_prio_vcpu(struct kvm *kvm, u8 vector,
  348. unsigned long bitmap)
  349. {
  350. struct kvm_lapic *apic;
  351. apic = kvm_apic_round_robin(kvm, vector, bitmap);
  352. if (apic)
  353. return apic->vcpu;
  354. return NULL;
  355. }
  356. static void apic_set_eoi(struct kvm_lapic *apic)
  357. {
  358. int vector = apic_find_highest_isr(apic);
  359. int trigger_mode;
  360. /*
  361. * Not every write EOI will has corresponding ISR,
  362. * one example is when Kernel check timer on setup_IO_APIC
  363. */
  364. if (vector == -1)
  365. return;
  366. apic_clear_vector(vector, apic->regs + APIC_ISR);
  367. apic_update_ppr(apic);
  368. if (apic_test_and_clear_vector(vector, apic->regs + APIC_TMR))
  369. trigger_mode = IOAPIC_LEVEL_TRIG;
  370. else
  371. trigger_mode = IOAPIC_EDGE_TRIG;
  372. kvm_ioapic_update_eoi(apic->vcpu->kvm, vector, trigger_mode);
  373. }
  374. static void apic_send_ipi(struct kvm_lapic *apic)
  375. {
  376. u32 icr_low = apic_get_reg(apic, APIC_ICR);
  377. u32 icr_high = apic_get_reg(apic, APIC_ICR2);
  378. unsigned int dest = GET_APIC_DEST_FIELD(icr_high);
  379. unsigned int short_hand = icr_low & APIC_SHORT_MASK;
  380. unsigned int trig_mode = icr_low & APIC_INT_LEVELTRIG;
  381. unsigned int level = icr_low & APIC_INT_ASSERT;
  382. unsigned int dest_mode = icr_low & APIC_DEST_MASK;
  383. unsigned int delivery_mode = icr_low & APIC_MODE_MASK;
  384. unsigned int vector = icr_low & APIC_VECTOR_MASK;
  385. struct kvm_vcpu *target;
  386. struct kvm_vcpu *vcpu;
  387. unsigned long lpr_map = 0;
  388. int i;
  389. apic_debug("icr_high 0x%x, icr_low 0x%x, "
  390. "short_hand 0x%x, dest 0x%x, trig_mode 0x%x, level 0x%x, "
  391. "dest_mode 0x%x, delivery_mode 0x%x, vector 0x%x\n",
  392. icr_high, icr_low, short_hand, dest,
  393. trig_mode, level, dest_mode, delivery_mode, vector);
  394. for (i = 0; i < KVM_MAX_VCPUS; i++) {
  395. vcpu = apic->vcpu->kvm->vcpus[i];
  396. if (!vcpu)
  397. continue;
  398. if (vcpu->arch.apic &&
  399. apic_match_dest(vcpu, apic, short_hand, dest, dest_mode)) {
  400. if (delivery_mode == APIC_DM_LOWEST)
  401. set_bit(vcpu->vcpu_id, &lpr_map);
  402. else
  403. __apic_accept_irq(vcpu->arch.apic, delivery_mode,
  404. vector, level, trig_mode);
  405. }
  406. }
  407. if (delivery_mode == APIC_DM_LOWEST) {
  408. target = kvm_get_lowest_prio_vcpu(vcpu->kvm, vector, lpr_map);
  409. if (target != NULL)
  410. __apic_accept_irq(target->arch.apic, delivery_mode,
  411. vector, level, trig_mode);
  412. }
  413. }
  414. static u32 apic_get_tmcct(struct kvm_lapic *apic)
  415. {
  416. u64 counter_passed;
  417. ktime_t passed, now;
  418. u32 tmcct;
  419. ASSERT(apic != NULL);
  420. now = apic->timer.dev.base->get_time();
  421. tmcct = apic_get_reg(apic, APIC_TMICT);
  422. /* if initial count is 0, current count should also be 0 */
  423. if (tmcct == 0)
  424. return 0;
  425. if (unlikely(ktime_to_ns(now) <=
  426. ktime_to_ns(apic->timer.last_update))) {
  427. /* Wrap around */
  428. passed = ktime_add(( {
  429. (ktime_t) {
  430. .tv64 = KTIME_MAX -
  431. (apic->timer.last_update).tv64}; }
  432. ), now);
  433. apic_debug("time elapsed\n");
  434. } else
  435. passed = ktime_sub(now, apic->timer.last_update);
  436. counter_passed = div64_u64(ktime_to_ns(passed),
  437. (APIC_BUS_CYCLE_NS * apic->timer.divide_count));
  438. if (counter_passed > tmcct) {
  439. if (unlikely(!apic_lvtt_period(apic))) {
  440. /* one-shot timers stick at 0 until reset */
  441. tmcct = 0;
  442. } else {
  443. /*
  444. * periodic timers reset to APIC_TMICT when they
  445. * hit 0. The while loop simulates this happening N
  446. * times. (counter_passed %= tmcct) would also work,
  447. * but might be slower or not work on 32-bit??
  448. */
  449. while (counter_passed > tmcct)
  450. counter_passed -= tmcct;
  451. tmcct -= counter_passed;
  452. }
  453. } else {
  454. tmcct -= counter_passed;
  455. }
  456. return tmcct;
  457. }
  458. static void __report_tpr_access(struct kvm_lapic *apic, bool write)
  459. {
  460. struct kvm_vcpu *vcpu = apic->vcpu;
  461. struct kvm_run *run = vcpu->run;
  462. set_bit(KVM_REQ_REPORT_TPR_ACCESS, &vcpu->requests);
  463. run->tpr_access.rip = kvm_rip_read(vcpu);
  464. run->tpr_access.is_write = write;
  465. }
  466. static inline void report_tpr_access(struct kvm_lapic *apic, bool write)
  467. {
  468. if (apic->vcpu->arch.tpr_access_reporting)
  469. __report_tpr_access(apic, write);
  470. }
  471. static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset)
  472. {
  473. u32 val = 0;
  474. KVMTRACE_1D(APIC_ACCESS, apic->vcpu, (u32)offset, handler);
  475. if (offset >= LAPIC_MMIO_LENGTH)
  476. return 0;
  477. switch (offset) {
  478. case APIC_ARBPRI:
  479. printk(KERN_WARNING "Access APIC ARBPRI register "
  480. "which is for P6\n");
  481. break;
  482. case APIC_TMCCT: /* Timer CCR */
  483. val = apic_get_tmcct(apic);
  484. break;
  485. case APIC_TASKPRI:
  486. report_tpr_access(apic, false);
  487. /* fall thru */
  488. default:
  489. apic_update_ppr(apic);
  490. val = apic_get_reg(apic, offset);
  491. break;
  492. }
  493. return val;
  494. }
  495. static void apic_mmio_read(struct kvm_io_device *this,
  496. gpa_t address, int len, void *data)
  497. {
  498. struct kvm_lapic *apic = (struct kvm_lapic *)this->private;
  499. unsigned int offset = address - apic->base_address;
  500. unsigned char alignment = offset & 0xf;
  501. u32 result;
  502. if ((alignment + len) > 4) {
  503. printk(KERN_ERR "KVM_APIC_READ: alignment error %lx %d",
  504. (unsigned long)address, len);
  505. return;
  506. }
  507. result = __apic_read(apic, offset & ~0xf);
  508. switch (len) {
  509. case 1:
  510. case 2:
  511. case 4:
  512. memcpy(data, (char *)&result + alignment, len);
  513. break;
  514. default:
  515. printk(KERN_ERR "Local APIC read with len = %x, "
  516. "should be 1,2, or 4 instead\n", len);
  517. break;
  518. }
  519. }
  520. static void update_divide_count(struct kvm_lapic *apic)
  521. {
  522. u32 tmp1, tmp2, tdcr;
  523. tdcr = apic_get_reg(apic, APIC_TDCR);
  524. tmp1 = tdcr & 0xf;
  525. tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1;
  526. apic->timer.divide_count = 0x1 << (tmp2 & 0x7);
  527. apic_debug("timer divide count is 0x%x\n",
  528. apic->timer.divide_count);
  529. }
  530. static void start_apic_timer(struct kvm_lapic *apic)
  531. {
  532. ktime_t now = apic->timer.dev.base->get_time();
  533. apic->timer.last_update = now;
  534. apic->timer.period = apic_get_reg(apic, APIC_TMICT) *
  535. APIC_BUS_CYCLE_NS * apic->timer.divide_count;
  536. atomic_set(&apic->timer.pending, 0);
  537. if (!apic->timer.period)
  538. return;
  539. hrtimer_start(&apic->timer.dev,
  540. ktime_add_ns(now, apic->timer.period),
  541. HRTIMER_MODE_ABS);
  542. apic_debug("%s: bus cycle is %" PRId64 "ns, now 0x%016"
  543. PRIx64 ", "
  544. "timer initial count 0x%x, period %lldns, "
  545. "expire @ 0x%016" PRIx64 ".\n", __func__,
  546. APIC_BUS_CYCLE_NS, ktime_to_ns(now),
  547. apic_get_reg(apic, APIC_TMICT),
  548. apic->timer.period,
  549. ktime_to_ns(ktime_add_ns(now,
  550. apic->timer.period)));
  551. }
  552. static void apic_mmio_write(struct kvm_io_device *this,
  553. gpa_t address, int len, const void *data)
  554. {
  555. struct kvm_lapic *apic = (struct kvm_lapic *)this->private;
  556. unsigned int offset = address - apic->base_address;
  557. unsigned char alignment = offset & 0xf;
  558. u32 val;
  559. /*
  560. * APIC register must be aligned on 128-bits boundary.
  561. * 32/64/128 bits registers must be accessed thru 32 bits.
  562. * Refer SDM 8.4.1
  563. */
  564. if (len != 4 || alignment) {
  565. /* Don't shout loud, $infamous_os would cause only noise. */
  566. apic_debug("apic write: bad size=%d %lx\n",
  567. len, (long)address);
  568. return;
  569. }
  570. val = *(u32 *) data;
  571. /* too common printing */
  572. if (offset != APIC_EOI)
  573. apic_debug("%s: offset 0x%x with length 0x%x, and value is "
  574. "0x%x\n", __func__, offset, len, val);
  575. offset &= 0xff0;
  576. KVMTRACE_1D(APIC_ACCESS, apic->vcpu, (u32)offset, handler);
  577. switch (offset) {
  578. case APIC_ID: /* Local APIC ID */
  579. apic_set_reg(apic, APIC_ID, val);
  580. break;
  581. case APIC_TASKPRI:
  582. report_tpr_access(apic, true);
  583. apic_set_tpr(apic, val & 0xff);
  584. break;
  585. case APIC_EOI:
  586. apic_set_eoi(apic);
  587. break;
  588. case APIC_LDR:
  589. apic_set_reg(apic, APIC_LDR, val & APIC_LDR_MASK);
  590. break;
  591. case APIC_DFR:
  592. apic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF);
  593. break;
  594. case APIC_SPIV:
  595. apic_set_reg(apic, APIC_SPIV, val & 0x3ff);
  596. if (!(val & APIC_SPIV_APIC_ENABLED)) {
  597. int i;
  598. u32 lvt_val;
  599. for (i = 0; i < APIC_LVT_NUM; i++) {
  600. lvt_val = apic_get_reg(apic,
  601. APIC_LVTT + 0x10 * i);
  602. apic_set_reg(apic, APIC_LVTT + 0x10 * i,
  603. lvt_val | APIC_LVT_MASKED);
  604. }
  605. atomic_set(&apic->timer.pending, 0);
  606. }
  607. break;
  608. case APIC_ICR:
  609. /* No delay here, so we always clear the pending bit */
  610. apic_set_reg(apic, APIC_ICR, val & ~(1 << 12));
  611. apic_send_ipi(apic);
  612. break;
  613. case APIC_ICR2:
  614. apic_set_reg(apic, APIC_ICR2, val & 0xff000000);
  615. break;
  616. case APIC_LVTT:
  617. case APIC_LVTTHMR:
  618. case APIC_LVTPC:
  619. case APIC_LVT0:
  620. case APIC_LVT1:
  621. case APIC_LVTERR:
  622. /* TODO: Check vector */
  623. if (!apic_sw_enabled(apic))
  624. val |= APIC_LVT_MASKED;
  625. val &= apic_lvt_mask[(offset - APIC_LVTT) >> 4];
  626. apic_set_reg(apic, offset, val);
  627. break;
  628. case APIC_TMICT:
  629. hrtimer_cancel(&apic->timer.dev);
  630. apic_set_reg(apic, APIC_TMICT, val);
  631. start_apic_timer(apic);
  632. return;
  633. case APIC_TDCR:
  634. if (val & 4)
  635. printk(KERN_ERR "KVM_WRITE:TDCR %x\n", val);
  636. apic_set_reg(apic, APIC_TDCR, val);
  637. update_divide_count(apic);
  638. break;
  639. default:
  640. apic_debug("Local APIC Write to read-only register %x\n",
  641. offset);
  642. break;
  643. }
  644. }
  645. static int apic_mmio_range(struct kvm_io_device *this, gpa_t addr,
  646. int len, int size)
  647. {
  648. struct kvm_lapic *apic = (struct kvm_lapic *)this->private;
  649. int ret = 0;
  650. if (apic_hw_enabled(apic) &&
  651. (addr >= apic->base_address) &&
  652. (addr < (apic->base_address + LAPIC_MMIO_LENGTH)))
  653. ret = 1;
  654. return ret;
  655. }
  656. void kvm_free_lapic(struct kvm_vcpu *vcpu)
  657. {
  658. if (!vcpu->arch.apic)
  659. return;
  660. hrtimer_cancel(&vcpu->arch.apic->timer.dev);
  661. if (vcpu->arch.apic->regs_page)
  662. __free_page(vcpu->arch.apic->regs_page);
  663. kfree(vcpu->arch.apic);
  664. }
  665. /*
  666. *----------------------------------------------------------------------
  667. * LAPIC interface
  668. *----------------------------------------------------------------------
  669. */
  670. void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8)
  671. {
  672. struct kvm_lapic *apic = vcpu->arch.apic;
  673. if (!apic)
  674. return;
  675. apic_set_tpr(apic, ((cr8 & 0x0f) << 4)
  676. | (apic_get_reg(apic, APIC_TASKPRI) & 4));
  677. }
  678. EXPORT_SYMBOL_GPL(kvm_lapic_set_tpr);
  679. u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu)
  680. {
  681. struct kvm_lapic *apic = vcpu->arch.apic;
  682. u64 tpr;
  683. if (!apic)
  684. return 0;
  685. tpr = (u64) apic_get_reg(apic, APIC_TASKPRI);
  686. return (tpr & 0xf0) >> 4;
  687. }
  688. EXPORT_SYMBOL_GPL(kvm_lapic_get_cr8);
  689. void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
  690. {
  691. struct kvm_lapic *apic = vcpu->arch.apic;
  692. if (!apic) {
  693. value |= MSR_IA32_APICBASE_BSP;
  694. vcpu->arch.apic_base = value;
  695. return;
  696. }
  697. if (apic->vcpu->vcpu_id)
  698. value &= ~MSR_IA32_APICBASE_BSP;
  699. vcpu->arch.apic_base = value;
  700. apic->base_address = apic->vcpu->arch.apic_base &
  701. MSR_IA32_APICBASE_BASE;
  702. /* with FSB delivery interrupt, we can restart APIC functionality */
  703. apic_debug("apic base msr is 0x%016" PRIx64 ", and base address is "
  704. "0x%lx.\n", apic->vcpu->arch.apic_base, apic->base_address);
  705. }
  706. u64 kvm_lapic_get_base(struct kvm_vcpu *vcpu)
  707. {
  708. return vcpu->arch.apic_base;
  709. }
  710. EXPORT_SYMBOL_GPL(kvm_lapic_get_base);
  711. void kvm_lapic_reset(struct kvm_vcpu *vcpu)
  712. {
  713. struct kvm_lapic *apic;
  714. int i;
  715. apic_debug("%s\n", __func__);
  716. ASSERT(vcpu);
  717. apic = vcpu->arch.apic;
  718. ASSERT(apic != NULL);
  719. /* Stop the timer in case it's a reset to an active apic */
  720. hrtimer_cancel(&apic->timer.dev);
  721. apic_set_reg(apic, APIC_ID, vcpu->vcpu_id << 24);
  722. apic_set_reg(apic, APIC_LVR, APIC_VERSION);
  723. for (i = 0; i < APIC_LVT_NUM; i++)
  724. apic_set_reg(apic, APIC_LVTT + 0x10 * i, APIC_LVT_MASKED);
  725. apic_set_reg(apic, APIC_LVT0,
  726. SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT));
  727. apic_set_reg(apic, APIC_DFR, 0xffffffffU);
  728. apic_set_reg(apic, APIC_SPIV, 0xff);
  729. apic_set_reg(apic, APIC_TASKPRI, 0);
  730. apic_set_reg(apic, APIC_LDR, 0);
  731. apic_set_reg(apic, APIC_ESR, 0);
  732. apic_set_reg(apic, APIC_ICR, 0);
  733. apic_set_reg(apic, APIC_ICR2, 0);
  734. apic_set_reg(apic, APIC_TDCR, 0);
  735. apic_set_reg(apic, APIC_TMICT, 0);
  736. for (i = 0; i < 8; i++) {
  737. apic_set_reg(apic, APIC_IRR + 0x10 * i, 0);
  738. apic_set_reg(apic, APIC_ISR + 0x10 * i, 0);
  739. apic_set_reg(apic, APIC_TMR + 0x10 * i, 0);
  740. }
  741. update_divide_count(apic);
  742. atomic_set(&apic->timer.pending, 0);
  743. if (vcpu->vcpu_id == 0)
  744. vcpu->arch.apic_base |= MSR_IA32_APICBASE_BSP;
  745. apic_update_ppr(apic);
  746. apic_debug(KERN_INFO "%s: vcpu=%p, id=%d, base_msr="
  747. "0x%016" PRIx64 ", base_address=0x%0lx.\n", __func__,
  748. vcpu, kvm_apic_id(apic),
  749. vcpu->arch.apic_base, apic->base_address);
  750. }
  751. EXPORT_SYMBOL_GPL(kvm_lapic_reset);
  752. int kvm_lapic_enabled(struct kvm_vcpu *vcpu)
  753. {
  754. struct kvm_lapic *apic = vcpu->arch.apic;
  755. int ret = 0;
  756. if (!apic)
  757. return 0;
  758. ret = apic_enabled(apic);
  759. return ret;
  760. }
  761. EXPORT_SYMBOL_GPL(kvm_lapic_enabled);
  762. /*
  763. *----------------------------------------------------------------------
  764. * timer interface
  765. *----------------------------------------------------------------------
  766. */
  767. /* TODO: make sure __apic_timer_fn runs in current pCPU */
  768. static int __apic_timer_fn(struct kvm_lapic *apic)
  769. {
  770. int result = 0;
  771. wait_queue_head_t *q = &apic->vcpu->wq;
  772. if(!atomic_inc_and_test(&apic->timer.pending))
  773. set_bit(KVM_REQ_PENDING_TIMER, &apic->vcpu->requests);
  774. if (waitqueue_active(q))
  775. wake_up_interruptible(q);
  776. if (apic_lvtt_period(apic)) {
  777. result = 1;
  778. apic->timer.dev.expires = ktime_add_ns(
  779. apic->timer.dev.expires,
  780. apic->timer.period);
  781. }
  782. return result;
  783. }
  784. int apic_has_pending_timer(struct kvm_vcpu *vcpu)
  785. {
  786. struct kvm_lapic *lapic = vcpu->arch.apic;
  787. if (lapic && apic_enabled(lapic) && apic_lvt_enabled(lapic, APIC_LVTT))
  788. return atomic_read(&lapic->timer.pending);
  789. return 0;
  790. }
  791. static int __inject_apic_timer_irq(struct kvm_lapic *apic)
  792. {
  793. int vector;
  794. vector = apic_lvt_vector(apic, APIC_LVTT);
  795. return __apic_accept_irq(apic, APIC_DM_FIXED, vector, 1, 0);
  796. }
  797. static enum hrtimer_restart apic_timer_fn(struct hrtimer *data)
  798. {
  799. struct kvm_lapic *apic;
  800. int restart_timer = 0;
  801. apic = container_of(data, struct kvm_lapic, timer.dev);
  802. restart_timer = __apic_timer_fn(apic);
  803. if (restart_timer)
  804. return HRTIMER_RESTART;
  805. else
  806. return HRTIMER_NORESTART;
  807. }
  808. int kvm_create_lapic(struct kvm_vcpu *vcpu)
  809. {
  810. struct kvm_lapic *apic;
  811. ASSERT(vcpu != NULL);
  812. apic_debug("apic_init %d\n", vcpu->vcpu_id);
  813. apic = kzalloc(sizeof(*apic), GFP_KERNEL);
  814. if (!apic)
  815. goto nomem;
  816. vcpu->arch.apic = apic;
  817. apic->regs_page = alloc_page(GFP_KERNEL);
  818. if (apic->regs_page == NULL) {
  819. printk(KERN_ERR "malloc apic regs error for vcpu %x\n",
  820. vcpu->vcpu_id);
  821. goto nomem_free_apic;
  822. }
  823. apic->regs = page_address(apic->regs_page);
  824. memset(apic->regs, 0, PAGE_SIZE);
  825. apic->vcpu = vcpu;
  826. hrtimer_init(&apic->timer.dev, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
  827. apic->timer.dev.function = apic_timer_fn;
  828. apic->base_address = APIC_DEFAULT_PHYS_BASE;
  829. vcpu->arch.apic_base = APIC_DEFAULT_PHYS_BASE;
  830. kvm_lapic_reset(vcpu);
  831. apic->dev.read = apic_mmio_read;
  832. apic->dev.write = apic_mmio_write;
  833. apic->dev.in_range = apic_mmio_range;
  834. apic->dev.private = apic;
  835. return 0;
  836. nomem_free_apic:
  837. kfree(apic);
  838. nomem:
  839. return -ENOMEM;
  840. }
  841. EXPORT_SYMBOL_GPL(kvm_create_lapic);
  842. int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu)
  843. {
  844. struct kvm_lapic *apic = vcpu->arch.apic;
  845. int highest_irr;
  846. if (!apic || !apic_enabled(apic))
  847. return -1;
  848. apic_update_ppr(apic);
  849. highest_irr = apic_find_highest_irr(apic);
  850. if ((highest_irr == -1) ||
  851. ((highest_irr & 0xF0) <= apic_get_reg(apic, APIC_PROCPRI)))
  852. return -1;
  853. return highest_irr;
  854. }
  855. int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu)
  856. {
  857. u32 lvt0 = apic_get_reg(vcpu->arch.apic, APIC_LVT0);
  858. int r = 0;
  859. if (vcpu->vcpu_id == 0) {
  860. if (!apic_hw_enabled(vcpu->arch.apic))
  861. r = 1;
  862. if ((lvt0 & APIC_LVT_MASKED) == 0 &&
  863. GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT)
  864. r = 1;
  865. }
  866. return r;
  867. }
  868. void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu)
  869. {
  870. struct kvm_lapic *apic = vcpu->arch.apic;
  871. if (apic && apic_lvt_enabled(apic, APIC_LVTT) &&
  872. atomic_read(&apic->timer.pending) > 0) {
  873. if (__inject_apic_timer_irq(apic))
  874. atomic_dec(&apic->timer.pending);
  875. }
  876. }
  877. void kvm_apic_timer_intr_post(struct kvm_vcpu *vcpu, int vec)
  878. {
  879. struct kvm_lapic *apic = vcpu->arch.apic;
  880. if (apic && apic_lvt_vector(apic, APIC_LVTT) == vec)
  881. apic->timer.last_update = ktime_add_ns(
  882. apic->timer.last_update,
  883. apic->timer.period);
  884. }
  885. int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu)
  886. {
  887. int vector = kvm_apic_has_interrupt(vcpu);
  888. struct kvm_lapic *apic = vcpu->arch.apic;
  889. if (vector == -1)
  890. return -1;
  891. apic_set_vector(vector, apic->regs + APIC_ISR);
  892. apic_update_ppr(apic);
  893. apic_clear_irr(vector, apic);
  894. return vector;
  895. }
  896. void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu)
  897. {
  898. struct kvm_lapic *apic = vcpu->arch.apic;
  899. apic->base_address = vcpu->arch.apic_base &
  900. MSR_IA32_APICBASE_BASE;
  901. apic_set_reg(apic, APIC_LVR, APIC_VERSION);
  902. apic_update_ppr(apic);
  903. hrtimer_cancel(&apic->timer.dev);
  904. update_divide_count(apic);
  905. start_apic_timer(apic);
  906. }
  907. void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)
  908. {
  909. struct kvm_lapic *apic = vcpu->arch.apic;
  910. struct hrtimer *timer;
  911. if (!apic)
  912. return;
  913. timer = &apic->timer.dev;
  914. if (hrtimer_cancel(timer))
  915. hrtimer_start(timer, timer->expires, HRTIMER_MODE_ABS);
  916. }
  917. void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu)
  918. {
  919. u32 data;
  920. void *vapic;
  921. if (!irqchip_in_kernel(vcpu->kvm) || !vcpu->arch.apic->vapic_addr)
  922. return;
  923. vapic = kmap_atomic(vcpu->arch.apic->vapic_page, KM_USER0);
  924. data = *(u32 *)(vapic + offset_in_page(vcpu->arch.apic->vapic_addr));
  925. kunmap_atomic(vapic, KM_USER0);
  926. apic_set_tpr(vcpu->arch.apic, data & 0xff);
  927. }
  928. void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu)
  929. {
  930. u32 data, tpr;
  931. int max_irr, max_isr;
  932. struct kvm_lapic *apic;
  933. void *vapic;
  934. if (!irqchip_in_kernel(vcpu->kvm) || !vcpu->arch.apic->vapic_addr)
  935. return;
  936. apic = vcpu->arch.apic;
  937. tpr = apic_get_reg(apic, APIC_TASKPRI) & 0xff;
  938. max_irr = apic_find_highest_irr(apic);
  939. if (max_irr < 0)
  940. max_irr = 0;
  941. max_isr = apic_find_highest_isr(apic);
  942. if (max_isr < 0)
  943. max_isr = 0;
  944. data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24);
  945. vapic = kmap_atomic(vcpu->arch.apic->vapic_page, KM_USER0);
  946. *(u32 *)(vapic + offset_in_page(vcpu->arch.apic->vapic_addr)) = data;
  947. kunmap_atomic(vapic, KM_USER0);
  948. }
  949. void kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr)
  950. {
  951. if (!irqchip_in_kernel(vcpu->kvm))
  952. return;
  953. vcpu->arch.apic->vapic_addr = vapic_addr;
  954. }