kvm_host.h 29 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100
  1. #ifndef __KVM_HOST_H
  2. #define __KVM_HOST_H
  3. #if IS_ENABLED(CONFIG_KVM)
  4. /*
  5. * This work is licensed under the terms of the GNU GPL, version 2. See
  6. * the COPYING file in the top-level directory.
  7. */
  8. #include <linux/types.h>
  9. #include <linux/hardirq.h>
  10. #include <linux/list.h>
  11. #include <linux/mutex.h>
  12. #include <linux/spinlock.h>
  13. #include <linux/signal.h>
  14. #include <linux/sched.h>
  15. #include <linux/bug.h>
  16. #include <linux/mm.h>
  17. #include <linux/mmu_notifier.h>
  18. #include <linux/preempt.h>
  19. #include <linux/msi.h>
  20. #include <linux/slab.h>
  21. #include <linux/rcupdate.h>
  22. #include <linux/ratelimit.h>
  23. #include <linux/err.h>
  24. #include <linux/irqflags.h>
  25. #include <asm/signal.h>
  26. #include <linux/kvm.h>
  27. #include <linux/kvm_para.h>
  28. #include <linux/kvm_types.h>
  29. #include <asm/kvm_host.h>
  30. #ifndef KVM_MMIO_SIZE
  31. #define KVM_MMIO_SIZE 8
  32. #endif
  33. /*
  34. * The bit 16 ~ bit 31 of kvm_memory_region::flags are internally used
  35. * in kvm, other bits are visible for userspace which are defined in
  36. * include/linux/kvm_h.
  37. */
  38. #define KVM_MEMSLOT_INVALID (1UL << 16)
  39. /* Two fragments for cross MMIO pages. */
  40. #define KVM_MAX_MMIO_FRAGMENTS 2
  41. /*
  42. * For the normal pfn, the highest 12 bits should be zero,
  43. * so we can mask bit 62 ~ bit 52 to indicate the error pfn,
  44. * mask bit 63 to indicate the noslot pfn.
  45. */
  46. #define KVM_PFN_ERR_MASK (0x7ffULL << 52)
  47. #define KVM_PFN_ERR_NOSLOT_MASK (0xfffULL << 52)
  48. #define KVM_PFN_NOSLOT (0x1ULL << 63)
  49. #define KVM_PFN_ERR_FAULT (KVM_PFN_ERR_MASK)
  50. #define KVM_PFN_ERR_HWPOISON (KVM_PFN_ERR_MASK + 1)
  51. #define KVM_PFN_ERR_RO_FAULT (KVM_PFN_ERR_MASK + 2)
  52. /*
  53. * error pfns indicate that the gfn is in slot but faild to
  54. * translate it to pfn on host.
  55. */
  56. static inline bool is_error_pfn(pfn_t pfn)
  57. {
  58. return !!(pfn & KVM_PFN_ERR_MASK);
  59. }
  60. /*
  61. * error_noslot pfns indicate that the gfn can not be
  62. * translated to pfn - it is not in slot or failed to
  63. * translate it to pfn.
  64. */
  65. static inline bool is_error_noslot_pfn(pfn_t pfn)
  66. {
  67. return !!(pfn & KVM_PFN_ERR_NOSLOT_MASK);
  68. }
  69. /* noslot pfn indicates that the gfn is not in slot. */
  70. static inline bool is_noslot_pfn(pfn_t pfn)
  71. {
  72. return pfn == KVM_PFN_NOSLOT;
  73. }
  74. #define KVM_HVA_ERR_BAD (PAGE_OFFSET)
  75. #define KVM_HVA_ERR_RO_BAD (PAGE_OFFSET + PAGE_SIZE)
  76. static inline bool kvm_is_error_hva(unsigned long addr)
  77. {
  78. return addr >= PAGE_OFFSET;
  79. }
  80. #define KVM_ERR_PTR_BAD_PAGE (ERR_PTR(-ENOENT))
  81. static inline bool is_error_page(struct page *page)
  82. {
  83. return IS_ERR(page);
  84. }
  85. /*
  86. * vcpu->requests bit members
  87. */
  88. #define KVM_REQ_TLB_FLUSH 0
  89. #define KVM_REQ_MIGRATE_TIMER 1
  90. #define KVM_REQ_REPORT_TPR_ACCESS 2
  91. #define KVM_REQ_MMU_RELOAD 3
  92. #define KVM_REQ_TRIPLE_FAULT 4
  93. #define KVM_REQ_PENDING_TIMER 5
  94. #define KVM_REQ_UNHALT 6
  95. #define KVM_REQ_MMU_SYNC 7
  96. #define KVM_REQ_CLOCK_UPDATE 8
  97. #define KVM_REQ_KICK 9
  98. #define KVM_REQ_DEACTIVATE_FPU 10
  99. #define KVM_REQ_EVENT 11
  100. #define KVM_REQ_APF_HALT 12
  101. #define KVM_REQ_STEAL_UPDATE 13
  102. #define KVM_REQ_NMI 14
  103. #define KVM_REQ_IMMEDIATE_EXIT 15
  104. #define KVM_REQ_PMU 16
  105. #define KVM_REQ_PMI 17
  106. #define KVM_REQ_WATCHDOG 18
  107. #define KVM_REQ_MASTERCLOCK_UPDATE 19
  108. #define KVM_REQ_MCLOCK_INPROGRESS 20
  109. #define KVM_REQ_EPR_EXIT 21
  110. #define KVM_REQ_SCAN_IOAPIC 22
  111. #define KVM_USERSPACE_IRQ_SOURCE_ID 0
  112. #define KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID 1
  113. struct kvm;
  114. struct kvm_vcpu;
  115. extern struct kmem_cache *kvm_vcpu_cache;
  116. extern raw_spinlock_t kvm_lock;
  117. extern struct list_head vm_list;
  118. struct kvm_io_range {
  119. gpa_t addr;
  120. int len;
  121. struct kvm_io_device *dev;
  122. };
  123. #define NR_IOBUS_DEVS 1000
  124. struct kvm_io_bus {
  125. int dev_count;
  126. struct kvm_io_range range[];
  127. };
  128. enum kvm_bus {
  129. KVM_MMIO_BUS,
  130. KVM_PIO_BUS,
  131. KVM_VIRTIO_CCW_NOTIFY_BUS,
  132. KVM_NR_BUSES
  133. };
  134. int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
  135. int len, const void *val);
  136. int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, int len,
  137. void *val);
  138. int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
  139. int len, struct kvm_io_device *dev);
  140. int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
  141. struct kvm_io_device *dev);
  142. #ifdef CONFIG_KVM_ASYNC_PF
  143. struct kvm_async_pf {
  144. struct work_struct work;
  145. struct list_head link;
  146. struct list_head queue;
  147. struct kvm_vcpu *vcpu;
  148. struct mm_struct *mm;
  149. gva_t gva;
  150. unsigned long addr;
  151. struct kvm_arch_async_pf arch;
  152. struct page *page;
  153. bool done;
  154. };
  155. void kvm_clear_async_pf_completion_queue(struct kvm_vcpu *vcpu);
  156. void kvm_check_async_pf_completion(struct kvm_vcpu *vcpu);
  157. int kvm_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn,
  158. struct kvm_arch_async_pf *arch);
  159. int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu);
  160. #endif
  161. enum {
  162. OUTSIDE_GUEST_MODE,
  163. IN_GUEST_MODE,
  164. EXITING_GUEST_MODE,
  165. READING_SHADOW_PAGE_TABLES,
  166. };
  167. /*
  168. * Sometimes a large or cross-page mmio needs to be broken up into separate
  169. * exits for userspace servicing.
  170. */
  171. struct kvm_mmio_fragment {
  172. gpa_t gpa;
  173. void *data;
  174. unsigned len;
  175. };
  176. struct kvm_vcpu {
  177. struct kvm *kvm;
  178. #ifdef CONFIG_PREEMPT_NOTIFIERS
  179. struct preempt_notifier preempt_notifier;
  180. #endif
  181. int cpu;
  182. int vcpu_id;
  183. int srcu_idx;
  184. int mode;
  185. unsigned long requests;
  186. unsigned long guest_debug;
  187. struct mutex mutex;
  188. struct kvm_run *run;
  189. int fpu_active;
  190. int guest_fpu_loaded, guest_xcr0_loaded;
  191. wait_queue_head_t wq;
  192. struct pid *pid;
  193. int sigset_active;
  194. sigset_t sigset;
  195. struct kvm_vcpu_stat stat;
  196. #ifdef CONFIG_HAS_IOMEM
  197. int mmio_needed;
  198. int mmio_read_completed;
  199. int mmio_is_write;
  200. int mmio_cur_fragment;
  201. int mmio_nr_fragments;
  202. struct kvm_mmio_fragment mmio_fragments[KVM_MAX_MMIO_FRAGMENTS];
  203. #endif
  204. #ifdef CONFIG_KVM_ASYNC_PF
  205. struct {
  206. u32 queued;
  207. struct list_head queue;
  208. struct list_head done;
  209. spinlock_t lock;
  210. } async_pf;
  211. #endif
  212. #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
  213. /*
  214. * Cpu relax intercept or pause loop exit optimization
  215. * in_spin_loop: set when a vcpu does a pause loop exit
  216. * or cpu relax intercepted.
  217. * dy_eligible: indicates whether vcpu is eligible for directed yield.
  218. */
  219. struct {
  220. bool in_spin_loop;
  221. bool dy_eligible;
  222. } spin_loop;
  223. #endif
  224. bool preempted;
  225. struct kvm_vcpu_arch arch;
  226. };
  227. static inline int kvm_vcpu_exiting_guest_mode(struct kvm_vcpu *vcpu)
  228. {
  229. return cmpxchg(&vcpu->mode, IN_GUEST_MODE, EXITING_GUEST_MODE);
  230. }
  231. /*
  232. * Some of the bitops functions do not support too long bitmaps.
  233. * This number must be determined not to exceed such limits.
  234. */
  235. #define KVM_MEM_MAX_NR_PAGES ((1UL << 31) - 1)
  236. struct kvm_memory_slot {
  237. gfn_t base_gfn;
  238. unsigned long npages;
  239. unsigned long *dirty_bitmap;
  240. struct kvm_arch_memory_slot arch;
  241. unsigned long userspace_addr;
  242. u32 flags;
  243. short id;
  244. };
  245. static inline unsigned long kvm_dirty_bitmap_bytes(struct kvm_memory_slot *memslot)
  246. {
  247. return ALIGN(memslot->npages, BITS_PER_LONG) / 8;
  248. }
  249. struct kvm_kernel_irq_routing_entry {
  250. u32 gsi;
  251. u32 type;
  252. int (*set)(struct kvm_kernel_irq_routing_entry *e,
  253. struct kvm *kvm, int irq_source_id, int level,
  254. bool line_status);
  255. union {
  256. struct {
  257. unsigned irqchip;
  258. unsigned pin;
  259. } irqchip;
  260. struct msi_msg msi;
  261. };
  262. struct hlist_node link;
  263. };
  264. #ifdef __KVM_HAVE_IOAPIC
  265. struct kvm_irq_routing_table {
  266. int chip[KVM_NR_IRQCHIPS][KVM_IOAPIC_NUM_PINS];
  267. struct kvm_kernel_irq_routing_entry *rt_entries;
  268. u32 nr_rt_entries;
  269. /*
  270. * Array indexed by gsi. Each entry contains list of irq chips
  271. * the gsi is connected to.
  272. */
  273. struct hlist_head map[0];
  274. };
  275. #else
  276. struct kvm_irq_routing_table {};
  277. #endif
  278. #ifndef KVM_PRIVATE_MEM_SLOTS
  279. #define KVM_PRIVATE_MEM_SLOTS 0
  280. #endif
  281. #ifndef KVM_MEM_SLOTS_NUM
  282. #define KVM_MEM_SLOTS_NUM (KVM_USER_MEM_SLOTS + KVM_PRIVATE_MEM_SLOTS)
  283. #endif
  284. /*
  285. * Note:
  286. * memslots are not sorted by id anymore, please use id_to_memslot()
  287. * to get the memslot by its id.
  288. */
  289. struct kvm_memslots {
  290. u64 generation;
  291. struct kvm_memory_slot memslots[KVM_MEM_SLOTS_NUM];
  292. /* The mapping table from slot id to the index in memslots[]. */
  293. short id_to_index[KVM_MEM_SLOTS_NUM];
  294. };
  295. struct kvm {
  296. spinlock_t mmu_lock;
  297. struct mutex slots_lock;
  298. struct mm_struct *mm; /* userspace tied to this vm */
  299. struct kvm_memslots *memslots;
  300. struct srcu_struct srcu;
  301. #ifdef CONFIG_KVM_APIC_ARCHITECTURE
  302. u32 bsp_vcpu_id;
  303. #endif
  304. struct kvm_vcpu *vcpus[KVM_MAX_VCPUS];
  305. atomic_t online_vcpus;
  306. int last_boosted_vcpu;
  307. struct list_head vm_list;
  308. struct mutex lock;
  309. struct kvm_io_bus *buses[KVM_NR_BUSES];
  310. #ifdef CONFIG_HAVE_KVM_EVENTFD
  311. struct {
  312. spinlock_t lock;
  313. struct list_head items;
  314. struct list_head resampler_list;
  315. struct mutex resampler_lock;
  316. } irqfds;
  317. struct list_head ioeventfds;
  318. #endif
  319. struct kvm_vm_stat stat;
  320. struct kvm_arch arch;
  321. atomic_t users_count;
  322. #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
  323. struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
  324. spinlock_t ring_lock;
  325. struct list_head coalesced_zones;
  326. #endif
  327. struct mutex irq_lock;
  328. #ifdef CONFIG_HAVE_KVM_IRQCHIP
  329. /*
  330. * Update side is protected by irq_lock and,
  331. * if configured, irqfds.lock.
  332. */
  333. struct kvm_irq_routing_table __rcu *irq_routing;
  334. struct hlist_head mask_notifier_list;
  335. struct hlist_head irq_ack_notifier_list;
  336. #endif
  337. #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
  338. struct mmu_notifier mmu_notifier;
  339. unsigned long mmu_notifier_seq;
  340. long mmu_notifier_count;
  341. #endif
  342. long tlbs_dirty;
  343. };
  344. #define kvm_err(fmt, ...) \
  345. pr_err("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__)
  346. #define kvm_info(fmt, ...) \
  347. pr_info("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__)
  348. #define kvm_debug(fmt, ...) \
  349. pr_debug("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__)
  350. #define kvm_pr_unimpl(fmt, ...) \
  351. pr_err_ratelimited("kvm [%i]: " fmt, \
  352. task_tgid_nr(current), ## __VA_ARGS__)
  353. /* The guest did something we don't support. */
  354. #define vcpu_unimpl(vcpu, fmt, ...) \
  355. kvm_pr_unimpl("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__)
  356. static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i)
  357. {
  358. smp_rmb();
  359. return kvm->vcpus[i];
  360. }
  361. #define kvm_for_each_vcpu(idx, vcpup, kvm) \
  362. for (idx = 0; \
  363. idx < atomic_read(&kvm->online_vcpus) && \
  364. (vcpup = kvm_get_vcpu(kvm, idx)) != NULL; \
  365. idx++)
  366. #define kvm_for_each_memslot(memslot, slots) \
  367. for (memslot = &slots->memslots[0]; \
  368. memslot < slots->memslots + KVM_MEM_SLOTS_NUM && memslot->npages;\
  369. memslot++)
  370. int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id);
  371. void kvm_vcpu_uninit(struct kvm_vcpu *vcpu);
  372. int __must_check vcpu_load(struct kvm_vcpu *vcpu);
  373. void vcpu_put(struct kvm_vcpu *vcpu);
  374. #ifdef __KVM_HAVE_IOAPIC
  375. int kvm_irqfd_init(void);
  376. void kvm_irqfd_exit(void);
  377. #else
  378. static inline int kvm_irqfd_init(void)
  379. {
  380. return 0;
  381. }
  382. static inline void kvm_irqfd_exit(void)
  383. {
  384. }
  385. #endif
  386. int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
  387. struct module *module);
  388. void kvm_exit(void);
  389. void kvm_get_kvm(struct kvm *kvm);
  390. void kvm_put_kvm(struct kvm *kvm);
  391. void update_memslots(struct kvm_memslots *slots, struct kvm_memory_slot *new,
  392. u64 last_generation);
  393. static inline struct kvm_memslots *kvm_memslots(struct kvm *kvm)
  394. {
  395. return rcu_dereference_check(kvm->memslots,
  396. srcu_read_lock_held(&kvm->srcu)
  397. || lockdep_is_held(&kvm->slots_lock));
  398. }
  399. static inline struct kvm_memory_slot *
  400. id_to_memslot(struct kvm_memslots *slots, int id)
  401. {
  402. int index = slots->id_to_index[id];
  403. struct kvm_memory_slot *slot;
  404. slot = &slots->memslots[index];
  405. WARN_ON(slot->id != id);
  406. return slot;
  407. }
  408. /*
  409. * KVM_SET_USER_MEMORY_REGION ioctl allows the following operations:
  410. * - create a new memory slot
  411. * - delete an existing memory slot
  412. * - modify an existing memory slot
  413. * -- move it in the guest physical memory space
  414. * -- just change its flags
  415. *
  416. * Since flags can be changed by some of these operations, the following
  417. * differentiation is the best we can do for __kvm_set_memory_region():
  418. */
  419. enum kvm_mr_change {
  420. KVM_MR_CREATE,
  421. KVM_MR_DELETE,
  422. KVM_MR_MOVE,
  423. KVM_MR_FLAGS_ONLY,
  424. };
  425. int kvm_set_memory_region(struct kvm *kvm,
  426. struct kvm_userspace_memory_region *mem);
  427. int __kvm_set_memory_region(struct kvm *kvm,
  428. struct kvm_userspace_memory_region *mem);
  429. void kvm_arch_free_memslot(struct kvm_memory_slot *free,
  430. struct kvm_memory_slot *dont);
  431. int kvm_arch_create_memslot(struct kvm_memory_slot *slot, unsigned long npages);
  432. int kvm_arch_prepare_memory_region(struct kvm *kvm,
  433. struct kvm_memory_slot *memslot,
  434. struct kvm_userspace_memory_region *mem,
  435. enum kvm_mr_change change);
  436. void kvm_arch_commit_memory_region(struct kvm *kvm,
  437. struct kvm_userspace_memory_region *mem,
  438. const struct kvm_memory_slot *old,
  439. enum kvm_mr_change change);
  440. bool kvm_largepages_enabled(void);
  441. void kvm_disable_largepages(void);
  442. /* flush all memory translations */
  443. void kvm_arch_flush_shadow_all(struct kvm *kvm);
  444. /* flush memory translations pointing to 'slot' */
  445. void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
  446. struct kvm_memory_slot *slot);
  447. int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages,
  448. int nr_pages);
  449. struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn);
  450. unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn);
  451. unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, gfn_t gfn);
  452. void kvm_release_page_clean(struct page *page);
  453. void kvm_release_page_dirty(struct page *page);
  454. void kvm_set_page_dirty(struct page *page);
  455. void kvm_set_page_accessed(struct page *page);
  456. pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn);
  457. pfn_t gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, bool *async,
  458. bool write_fault, bool *writable);
  459. pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn);
  460. pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
  461. bool *writable);
  462. pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn);
  463. pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn);
  464. void kvm_release_pfn_dirty(pfn_t pfn);
  465. void kvm_release_pfn_clean(pfn_t pfn);
  466. void kvm_set_pfn_dirty(pfn_t pfn);
  467. void kvm_set_pfn_accessed(pfn_t pfn);
  468. void kvm_get_pfn(pfn_t pfn);
  469. int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
  470. int len);
  471. int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
  472. unsigned long len);
  473. int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len);
  474. int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
  475. void *data, unsigned long len);
  476. int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
  477. int offset, int len);
  478. int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
  479. unsigned long len);
  480. int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
  481. void *data, unsigned long len);
  482. int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
  483. gpa_t gpa);
  484. int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len);
  485. int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len);
  486. struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn);
  487. int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn);
  488. unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn);
  489. void mark_page_dirty(struct kvm *kvm, gfn_t gfn);
  490. void mark_page_dirty_in_slot(struct kvm *kvm, struct kvm_memory_slot *memslot,
  491. gfn_t gfn);
  492. void kvm_vcpu_block(struct kvm_vcpu *vcpu);
  493. void kvm_vcpu_kick(struct kvm_vcpu *vcpu);
  494. bool kvm_vcpu_yield_to(struct kvm_vcpu *target);
  495. void kvm_vcpu_on_spin(struct kvm_vcpu *vcpu);
  496. void kvm_resched(struct kvm_vcpu *vcpu);
  497. void kvm_load_guest_fpu(struct kvm_vcpu *vcpu);
  498. void kvm_put_guest_fpu(struct kvm_vcpu *vcpu);
  499. void kvm_flush_remote_tlbs(struct kvm *kvm);
  500. void kvm_reload_remote_mmus(struct kvm *kvm);
  501. void kvm_make_mclock_inprogress_request(struct kvm *kvm);
  502. void kvm_make_scan_ioapic_request(struct kvm *kvm);
  503. long kvm_arch_dev_ioctl(struct file *filp,
  504. unsigned int ioctl, unsigned long arg);
  505. long kvm_arch_vcpu_ioctl(struct file *filp,
  506. unsigned int ioctl, unsigned long arg);
  507. int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf);
  508. int kvm_dev_ioctl_check_extension(long ext);
  509. int kvm_get_dirty_log(struct kvm *kvm,
  510. struct kvm_dirty_log *log, int *is_dirty);
  511. int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
  512. struct kvm_dirty_log *log);
  513. int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
  514. struct kvm_userspace_memory_region *mem);
  515. int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
  516. bool line_status);
  517. long kvm_arch_vm_ioctl(struct file *filp,
  518. unsigned int ioctl, unsigned long arg);
  519. int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu);
  520. int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu);
  521. int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
  522. struct kvm_translation *tr);
  523. int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs);
  524. int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs);
  525. int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
  526. struct kvm_sregs *sregs);
  527. int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
  528. struct kvm_sregs *sregs);
  529. int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
  530. struct kvm_mp_state *mp_state);
  531. int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
  532. struct kvm_mp_state *mp_state);
  533. int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
  534. struct kvm_guest_debug *dbg);
  535. int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run);
  536. int kvm_arch_init(void *opaque);
  537. void kvm_arch_exit(void);
  538. int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu);
  539. void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu);
  540. void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu);
  541. void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu);
  542. void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu);
  543. struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id);
  544. int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu);
  545. int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu);
  546. void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu);
  547. int kvm_arch_hardware_enable(void *garbage);
  548. void kvm_arch_hardware_disable(void *garbage);
  549. int kvm_arch_hardware_setup(void);
  550. void kvm_arch_hardware_unsetup(void);
  551. void kvm_arch_check_processor_compat(void *rtn);
  552. int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu);
  553. int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu);
  554. void kvm_free_physmem(struct kvm *kvm);
  555. void *kvm_kvzalloc(unsigned long size);
  556. void kvm_kvfree(const void *addr);
  557. #ifndef __KVM_HAVE_ARCH_VM_ALLOC
  558. static inline struct kvm *kvm_arch_alloc_vm(void)
  559. {
  560. return kzalloc(sizeof(struct kvm), GFP_KERNEL);
  561. }
  562. static inline void kvm_arch_free_vm(struct kvm *kvm)
  563. {
  564. kfree(kvm);
  565. }
  566. #endif
  567. static inline wait_queue_head_t *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu)
  568. {
  569. #ifdef __KVM_HAVE_ARCH_WQP
  570. return vcpu->arch.wqp;
  571. #else
  572. return &vcpu->wq;
  573. #endif
  574. }
  575. int kvm_arch_init_vm(struct kvm *kvm, unsigned long type);
  576. void kvm_arch_destroy_vm(struct kvm *kvm);
  577. void kvm_free_all_assigned_devices(struct kvm *kvm);
  578. void kvm_arch_sync_events(struct kvm *kvm);
  579. int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu);
  580. void kvm_vcpu_kick(struct kvm_vcpu *vcpu);
  581. bool kvm_is_mmio_pfn(pfn_t pfn);
  582. struct kvm_irq_ack_notifier {
  583. struct hlist_node link;
  584. unsigned gsi;
  585. void (*irq_acked)(struct kvm_irq_ack_notifier *kian);
  586. };
  587. struct kvm_assigned_dev_kernel {
  588. struct kvm_irq_ack_notifier ack_notifier;
  589. struct list_head list;
  590. int assigned_dev_id;
  591. int host_segnr;
  592. int host_busnr;
  593. int host_devfn;
  594. unsigned int entries_nr;
  595. int host_irq;
  596. bool host_irq_disabled;
  597. bool pci_2_3;
  598. struct msix_entry *host_msix_entries;
  599. int guest_irq;
  600. struct msix_entry *guest_msix_entries;
  601. unsigned long irq_requested_type;
  602. int irq_source_id;
  603. int flags;
  604. struct pci_dev *dev;
  605. struct kvm *kvm;
  606. spinlock_t intx_lock;
  607. spinlock_t intx_mask_lock;
  608. char irq_name[32];
  609. struct pci_saved_state *pci_saved_state;
  610. };
  611. struct kvm_irq_mask_notifier {
  612. void (*func)(struct kvm_irq_mask_notifier *kimn, bool masked);
  613. int irq;
  614. struct hlist_node link;
  615. };
  616. void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq,
  617. struct kvm_irq_mask_notifier *kimn);
  618. void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq,
  619. struct kvm_irq_mask_notifier *kimn);
  620. void kvm_fire_mask_notifiers(struct kvm *kvm, unsigned irqchip, unsigned pin,
  621. bool mask);
  622. #ifdef __KVM_HAVE_IOAPIC
  623. void kvm_get_intr_delivery_bitmask(struct kvm_ioapic *ioapic,
  624. union kvm_ioapic_redirect_entry *entry,
  625. unsigned long *deliver_bitmask);
  626. #endif
  627. int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level,
  628. bool line_status);
  629. int kvm_set_irq_inatomic(struct kvm *kvm, int irq_source_id, u32 irq, int level);
  630. int kvm_set_msi(struct kvm_kernel_irq_routing_entry *irq_entry, struct kvm *kvm,
  631. int irq_source_id, int level, bool line_status);
  632. bool kvm_irq_has_notifier(struct kvm *kvm, unsigned irqchip, unsigned pin);
  633. void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin);
  634. void kvm_register_irq_ack_notifier(struct kvm *kvm,
  635. struct kvm_irq_ack_notifier *kian);
  636. void kvm_unregister_irq_ack_notifier(struct kvm *kvm,
  637. struct kvm_irq_ack_notifier *kian);
  638. int kvm_request_irq_source_id(struct kvm *kvm);
  639. void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id);
  640. /* For vcpu->arch.iommu_flags */
  641. #define KVM_IOMMU_CACHE_COHERENCY 0x1
  642. #ifdef CONFIG_IOMMU_API
  643. int kvm_iommu_map_pages(struct kvm *kvm, struct kvm_memory_slot *slot);
  644. void kvm_iommu_unmap_pages(struct kvm *kvm, struct kvm_memory_slot *slot);
  645. int kvm_iommu_map_guest(struct kvm *kvm);
  646. int kvm_iommu_unmap_guest(struct kvm *kvm);
  647. int kvm_assign_device(struct kvm *kvm,
  648. struct kvm_assigned_dev_kernel *assigned_dev);
  649. int kvm_deassign_device(struct kvm *kvm,
  650. struct kvm_assigned_dev_kernel *assigned_dev);
  651. #else /* CONFIG_IOMMU_API */
  652. static inline int kvm_iommu_map_pages(struct kvm *kvm,
  653. struct kvm_memory_slot *slot)
  654. {
  655. return 0;
  656. }
  657. static inline void kvm_iommu_unmap_pages(struct kvm *kvm,
  658. struct kvm_memory_slot *slot)
  659. {
  660. }
  661. static inline int kvm_iommu_map_guest(struct kvm *kvm)
  662. {
  663. return -ENODEV;
  664. }
  665. static inline int kvm_iommu_unmap_guest(struct kvm *kvm)
  666. {
  667. return 0;
  668. }
  669. static inline int kvm_assign_device(struct kvm *kvm,
  670. struct kvm_assigned_dev_kernel *assigned_dev)
  671. {
  672. return 0;
  673. }
  674. static inline int kvm_deassign_device(struct kvm *kvm,
  675. struct kvm_assigned_dev_kernel *assigned_dev)
  676. {
  677. return 0;
  678. }
  679. #endif /* CONFIG_IOMMU_API */
  680. static inline void __guest_enter(void)
  681. {
  682. /*
  683. * This is running in ioctl context so we can avoid
  684. * the call to vtime_account() with its unnecessary idle check.
  685. */
  686. vtime_account_system(current);
  687. current->flags |= PF_VCPU;
  688. }
  689. static inline void __guest_exit(void)
  690. {
  691. /*
  692. * This is running in ioctl context so we can avoid
  693. * the call to vtime_account() with its unnecessary idle check.
  694. */
  695. vtime_account_system(current);
  696. current->flags &= ~PF_VCPU;
  697. }
  698. #ifdef CONFIG_CONTEXT_TRACKING
  699. extern void guest_enter(void);
  700. extern void guest_exit(void);
  701. #else /* !CONFIG_CONTEXT_TRACKING */
  702. static inline void guest_enter(void)
  703. {
  704. __guest_enter();
  705. }
  706. static inline void guest_exit(void)
  707. {
  708. __guest_exit();
  709. }
  710. #endif /* !CONFIG_CONTEXT_TRACKING */
  711. static inline void kvm_guest_enter(void)
  712. {
  713. unsigned long flags;
  714. BUG_ON(preemptible());
  715. local_irq_save(flags);
  716. guest_enter();
  717. local_irq_restore(flags);
  718. /* KVM does not hold any references to rcu protected data when it
  719. * switches CPU into a guest mode. In fact switching to a guest mode
  720. * is very similar to exiting to userspase from rcu point of view. In
  721. * addition CPU may stay in a guest mode for quite a long time (up to
  722. * one time slice). Lets treat guest mode as quiescent state, just like
  723. * we do with user-mode execution.
  724. */
  725. rcu_virt_note_context_switch(smp_processor_id());
  726. }
  727. static inline void kvm_guest_exit(void)
  728. {
  729. unsigned long flags;
  730. local_irq_save(flags);
  731. guest_exit();
  732. local_irq_restore(flags);
  733. }
  734. /*
  735. * search_memslots() and __gfn_to_memslot() are here because they are
  736. * used in non-modular code in arch/powerpc/kvm/book3s_hv_rm_mmu.c.
  737. * gfn_to_memslot() itself isn't here as an inline because that would
  738. * bloat other code too much.
  739. */
  740. static inline struct kvm_memory_slot *
  741. search_memslots(struct kvm_memslots *slots, gfn_t gfn)
  742. {
  743. struct kvm_memory_slot *memslot;
  744. kvm_for_each_memslot(memslot, slots)
  745. if (gfn >= memslot->base_gfn &&
  746. gfn < memslot->base_gfn + memslot->npages)
  747. return memslot;
  748. return NULL;
  749. }
  750. static inline struct kvm_memory_slot *
  751. __gfn_to_memslot(struct kvm_memslots *slots, gfn_t gfn)
  752. {
  753. return search_memslots(slots, gfn);
  754. }
  755. static inline unsigned long
  756. __gfn_to_hva_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
  757. {
  758. return slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE;
  759. }
  760. static inline int memslot_id(struct kvm *kvm, gfn_t gfn)
  761. {
  762. return gfn_to_memslot(kvm, gfn)->id;
  763. }
  764. static inline gfn_t gfn_to_index(gfn_t gfn, gfn_t base_gfn, int level)
  765. {
  766. /* KVM_HPAGE_GFN_SHIFT(PT_PAGE_TABLE_LEVEL) must be 0. */
  767. return (gfn >> KVM_HPAGE_GFN_SHIFT(level)) -
  768. (base_gfn >> KVM_HPAGE_GFN_SHIFT(level));
  769. }
  770. static inline gfn_t
  771. hva_to_gfn_memslot(unsigned long hva, struct kvm_memory_slot *slot)
  772. {
  773. gfn_t gfn_offset = (hva - slot->userspace_addr) >> PAGE_SHIFT;
  774. return slot->base_gfn + gfn_offset;
  775. }
  776. static inline gpa_t gfn_to_gpa(gfn_t gfn)
  777. {
  778. return (gpa_t)gfn << PAGE_SHIFT;
  779. }
  780. static inline gfn_t gpa_to_gfn(gpa_t gpa)
  781. {
  782. return (gfn_t)(gpa >> PAGE_SHIFT);
  783. }
  784. static inline hpa_t pfn_to_hpa(pfn_t pfn)
  785. {
  786. return (hpa_t)pfn << PAGE_SHIFT;
  787. }
  788. static inline void kvm_migrate_timers(struct kvm_vcpu *vcpu)
  789. {
  790. set_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests);
  791. }
  792. enum kvm_stat_kind {
  793. KVM_STAT_VM,
  794. KVM_STAT_VCPU,
  795. };
  796. struct kvm_stats_debugfs_item {
  797. const char *name;
  798. int offset;
  799. enum kvm_stat_kind kind;
  800. struct dentry *dentry;
  801. };
  802. extern struct kvm_stats_debugfs_item debugfs_entries[];
  803. extern struct dentry *kvm_debugfs_dir;
  804. #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
  805. static inline int mmu_notifier_retry(struct kvm *kvm, unsigned long mmu_seq)
  806. {
  807. if (unlikely(kvm->mmu_notifier_count))
  808. return 1;
  809. /*
  810. * Ensure the read of mmu_notifier_count happens before the read
  811. * of mmu_notifier_seq. This interacts with the smp_wmb() in
  812. * mmu_notifier_invalidate_range_end to make sure that the caller
  813. * either sees the old (non-zero) value of mmu_notifier_count or
  814. * the new (incremented) value of mmu_notifier_seq.
  815. * PowerPC Book3s HV KVM calls this under a per-page lock
  816. * rather than under kvm->mmu_lock, for scalability, so
  817. * can't rely on kvm->mmu_lock to keep things ordered.
  818. */
  819. smp_rmb();
  820. if (kvm->mmu_notifier_seq != mmu_seq)
  821. return 1;
  822. return 0;
  823. }
  824. #endif
  825. #ifdef KVM_CAP_IRQ_ROUTING
  826. #define KVM_MAX_IRQ_ROUTES 1024
  827. int kvm_setup_default_irq_routing(struct kvm *kvm);
  828. int kvm_set_irq_routing(struct kvm *kvm,
  829. const struct kvm_irq_routing_entry *entries,
  830. unsigned nr,
  831. unsigned flags);
  832. void kvm_free_irq_routing(struct kvm *kvm);
  833. int kvm_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi);
  834. #else
  835. static inline void kvm_free_irq_routing(struct kvm *kvm) {}
  836. #endif
  837. #ifdef CONFIG_HAVE_KVM_EVENTFD
  838. void kvm_eventfd_init(struct kvm *kvm);
  839. int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args);
  840. #ifdef CONFIG_HAVE_KVM_IRQCHIP
  841. int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args);
  842. void kvm_irqfd_release(struct kvm *kvm);
  843. void kvm_irq_routing_update(struct kvm *, struct kvm_irq_routing_table *);
  844. #else
  845. static inline int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args)
  846. {
  847. return -EINVAL;
  848. }
  849. static inline void kvm_irqfd_release(struct kvm *kvm) {}
  850. #endif
  851. #else
  852. static inline void kvm_eventfd_init(struct kvm *kvm) {}
  853. static inline int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args)
  854. {
  855. return -EINVAL;
  856. }
  857. static inline void kvm_irqfd_release(struct kvm *kvm) {}
  858. #ifdef CONFIG_HAVE_KVM_IRQCHIP
  859. static inline void kvm_irq_routing_update(struct kvm *kvm,
  860. struct kvm_irq_routing_table *irq_rt)
  861. {
  862. rcu_assign_pointer(kvm->irq_routing, irq_rt);
  863. }
  864. #endif
  865. static inline int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
  866. {
  867. return -ENOSYS;
  868. }
  869. #endif /* CONFIG_HAVE_KVM_EVENTFD */
  870. #ifdef CONFIG_KVM_APIC_ARCHITECTURE
  871. static inline bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)
  872. {
  873. return vcpu->kvm->bsp_vcpu_id == vcpu->vcpu_id;
  874. }
  875. bool kvm_vcpu_compatible(struct kvm_vcpu *vcpu);
  876. #else
  877. static inline bool kvm_vcpu_compatible(struct kvm_vcpu *vcpu) { return true; }
  878. #endif
  879. #ifdef __KVM_HAVE_DEVICE_ASSIGNMENT
  880. long kvm_vm_ioctl_assigned_device(struct kvm *kvm, unsigned ioctl,
  881. unsigned long arg);
  882. #else
  883. static inline long kvm_vm_ioctl_assigned_device(struct kvm *kvm, unsigned ioctl,
  884. unsigned long arg)
  885. {
  886. return -ENOTTY;
  887. }
  888. #endif
  889. static inline void kvm_make_request(int req, struct kvm_vcpu *vcpu)
  890. {
  891. set_bit(req, &vcpu->requests);
  892. }
  893. static inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu)
  894. {
  895. if (test_bit(req, &vcpu->requests)) {
  896. clear_bit(req, &vcpu->requests);
  897. return true;
  898. } else {
  899. return false;
  900. }
  901. }
  902. extern bool kvm_rebooting;
  903. #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
  904. static inline void kvm_vcpu_set_in_spin_loop(struct kvm_vcpu *vcpu, bool val)
  905. {
  906. vcpu->spin_loop.in_spin_loop = val;
  907. }
  908. static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val)
  909. {
  910. vcpu->spin_loop.dy_eligible = val;
  911. }
  912. #else /* !CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT */
  913. static inline void kvm_vcpu_set_in_spin_loop(struct kvm_vcpu *vcpu, bool val)
  914. {
  915. }
  916. static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val)
  917. {
  918. }
  919. static inline bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
  920. {
  921. return true;
  922. }
  923. #endif /* CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT */
  924. #else
  925. static inline void __guest_enter(void) { return; }
  926. static inline void __guest_exit(void) { return; }
  927. #endif /* IS_ENABLED(CONFIG_KVM) */
  928. #endif