arm.c 22 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004
  1. /*
  2. * Copyright (C) 2012 - Virtual Open Systems and Columbia University
  3. * Author: Christoffer Dall <c.dall@virtualopensystems.com>
  4. *
  5. * This program is free software; you can redistribute it and/or modify
  6. * it under the terms of the GNU General Public License, version 2, as
  7. * published by the Free Software Foundation.
  8. *
  9. * This program is distributed in the hope that it will be useful,
  10. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. * GNU General Public License for more details.
  13. *
  14. * You should have received a copy of the GNU General Public License
  15. * along with this program; if not, write to the Free Software
  16. * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  17. */
  18. #include <linux/cpu.h>
  19. #include <linux/errno.h>
  20. #include <linux/err.h>
  21. #include <linux/kvm_host.h>
  22. #include <linux/module.h>
  23. #include <linux/vmalloc.h>
  24. #include <linux/fs.h>
  25. #include <linux/mman.h>
  26. #include <linux/sched.h>
  27. #include <linux/kvm.h>
  28. #include <trace/events/kvm.h>
  29. #define CREATE_TRACE_POINTS
  30. #include "trace.h"
  31. #include <asm/uaccess.h>
  32. #include <asm/ptrace.h>
  33. #include <asm/mman.h>
  34. #include <asm/tlbflush.h>
  35. #include <asm/cacheflush.h>
  36. #include <asm/virt.h>
  37. #include <asm/kvm_arm.h>
  38. #include <asm/kvm_asm.h>
  39. #include <asm/kvm_mmu.h>
  40. #include <asm/kvm_emulate.h>
  41. #include <asm/kvm_coproc.h>
  42. #include <asm/kvm_psci.h>
  43. #ifdef REQUIRES_VIRT
  44. __asm__(".arch_extension virt");
  45. #endif
  46. static DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);
  47. static kvm_cpu_context_t __percpu *kvm_host_cpu_state;
  48. static unsigned long hyp_default_vectors;
  49. /* Per-CPU variable containing the currently running vcpu. */
  50. static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_arm_running_vcpu);
  51. /* The VMID used in the VTTBR */
  52. static atomic64_t kvm_vmid_gen = ATOMIC64_INIT(1);
  53. static u8 kvm_next_vmid;
  54. static DEFINE_SPINLOCK(kvm_vmid_lock);
  55. static bool vgic_present;
  56. static void kvm_arm_set_running_vcpu(struct kvm_vcpu *vcpu)
  57. {
  58. BUG_ON(preemptible());
  59. __get_cpu_var(kvm_arm_running_vcpu) = vcpu;
  60. }
  61. /**
  62. * kvm_arm_get_running_vcpu - get the vcpu running on the current CPU.
  63. * Must be called from non-preemptible context
  64. */
  65. struct kvm_vcpu *kvm_arm_get_running_vcpu(void)
  66. {
  67. BUG_ON(preemptible());
  68. return __get_cpu_var(kvm_arm_running_vcpu);
  69. }
  70. /**
  71. * kvm_arm_get_running_vcpus - get the per-CPU array of currently running vcpus.
  72. */
  73. struct kvm_vcpu __percpu **kvm_get_running_vcpus(void)
  74. {
  75. return &kvm_arm_running_vcpu;
  76. }
  77. int kvm_arch_hardware_enable(void *garbage)
  78. {
  79. return 0;
  80. }
  81. int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
  82. {
  83. return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
  84. }
  85. void kvm_arch_hardware_disable(void *garbage)
  86. {
  87. }
  88. int kvm_arch_hardware_setup(void)
  89. {
  90. return 0;
  91. }
  92. void kvm_arch_hardware_unsetup(void)
  93. {
  94. }
  95. void kvm_arch_check_processor_compat(void *rtn)
  96. {
  97. *(int *)rtn = 0;
  98. }
  99. void kvm_arch_sync_events(struct kvm *kvm)
  100. {
  101. }
  102. /**
  103. * kvm_arch_init_vm - initializes a VM data structure
  104. * @kvm: pointer to the KVM struct
  105. */
  106. int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
  107. {
  108. int ret = 0;
  109. if (type)
  110. return -EINVAL;
  111. ret = kvm_alloc_stage2_pgd(kvm);
  112. if (ret)
  113. goto out_fail_alloc;
  114. ret = create_hyp_mappings(kvm, kvm + 1);
  115. if (ret)
  116. goto out_free_stage2_pgd;
  117. /* Mark the initial VMID generation invalid */
  118. kvm->arch.vmid_gen = 0;
  119. return ret;
  120. out_free_stage2_pgd:
  121. kvm_free_stage2_pgd(kvm);
  122. out_fail_alloc:
  123. return ret;
  124. }
  125. int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
  126. {
  127. return VM_FAULT_SIGBUS;
  128. }
  129. void kvm_arch_free_memslot(struct kvm_memory_slot *free,
  130. struct kvm_memory_slot *dont)
  131. {
  132. }
  133. int kvm_arch_create_memslot(struct kvm_memory_slot *slot, unsigned long npages)
  134. {
  135. return 0;
  136. }
  137. /**
  138. * kvm_arch_destroy_vm - destroy the VM data structure
  139. * @kvm: pointer to the KVM struct
  140. */
  141. void kvm_arch_destroy_vm(struct kvm *kvm)
  142. {
  143. int i;
  144. kvm_free_stage2_pgd(kvm);
  145. for (i = 0; i < KVM_MAX_VCPUS; ++i) {
  146. if (kvm->vcpus[i]) {
  147. kvm_arch_vcpu_free(kvm->vcpus[i]);
  148. kvm->vcpus[i] = NULL;
  149. }
  150. }
  151. }
  152. int kvm_dev_ioctl_check_extension(long ext)
  153. {
  154. int r;
  155. switch (ext) {
  156. case KVM_CAP_IRQCHIP:
  157. r = vgic_present;
  158. break;
  159. case KVM_CAP_USER_MEMORY:
  160. case KVM_CAP_SYNC_MMU:
  161. case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
  162. case KVM_CAP_ONE_REG:
  163. case KVM_CAP_ARM_PSCI:
  164. r = 1;
  165. break;
  166. case KVM_CAP_COALESCED_MMIO:
  167. r = KVM_COALESCED_MMIO_PAGE_OFFSET;
  168. break;
  169. case KVM_CAP_ARM_SET_DEVICE_ADDR:
  170. r = 1;
  171. case KVM_CAP_NR_VCPUS:
  172. r = num_online_cpus();
  173. break;
  174. case KVM_CAP_MAX_VCPUS:
  175. r = KVM_MAX_VCPUS;
  176. break;
  177. default:
  178. r = kvm_arch_dev_ioctl_check_extension(ext);
  179. break;
  180. }
  181. return r;
  182. }
  183. long kvm_arch_dev_ioctl(struct file *filp,
  184. unsigned int ioctl, unsigned long arg)
  185. {
  186. return -EINVAL;
  187. }
  188. int kvm_arch_prepare_memory_region(struct kvm *kvm,
  189. struct kvm_memory_slot *memslot,
  190. struct kvm_userspace_memory_region *mem,
  191. enum kvm_mr_change change)
  192. {
  193. return 0;
  194. }
  195. void kvm_arch_commit_memory_region(struct kvm *kvm,
  196. struct kvm_userspace_memory_region *mem,
  197. const struct kvm_memory_slot *old,
  198. enum kvm_mr_change change)
  199. {
  200. }
  201. void kvm_arch_flush_shadow_all(struct kvm *kvm)
  202. {
  203. }
  204. void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
  205. struct kvm_memory_slot *slot)
  206. {
  207. }
  208. struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id)
  209. {
  210. int err;
  211. struct kvm_vcpu *vcpu;
  212. vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
  213. if (!vcpu) {
  214. err = -ENOMEM;
  215. goto out;
  216. }
  217. err = kvm_vcpu_init(vcpu, kvm, id);
  218. if (err)
  219. goto free_vcpu;
  220. err = create_hyp_mappings(vcpu, vcpu + 1);
  221. if (err)
  222. goto vcpu_uninit;
  223. return vcpu;
  224. vcpu_uninit:
  225. kvm_vcpu_uninit(vcpu);
  226. free_vcpu:
  227. kmem_cache_free(kvm_vcpu_cache, vcpu);
  228. out:
  229. return ERR_PTR(err);
  230. }
  231. int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
  232. {
  233. return 0;
  234. }
  235. void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
  236. {
  237. kvm_mmu_free_memory_caches(vcpu);
  238. kvm_timer_vcpu_terminate(vcpu);
  239. kmem_cache_free(kvm_vcpu_cache, vcpu);
  240. }
  241. void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
  242. {
  243. kvm_arch_vcpu_free(vcpu);
  244. }
  245. int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
  246. {
  247. return 0;
  248. }
  249. int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
  250. {
  251. int ret;
  252. /* Force users to call KVM_ARM_VCPU_INIT */
  253. vcpu->arch.target = -1;
  254. /* Set up VGIC */
  255. ret = kvm_vgic_vcpu_init(vcpu);
  256. if (ret)
  257. return ret;
  258. /* Set up the timer */
  259. kvm_timer_vcpu_init(vcpu);
  260. return 0;
  261. }
  262. void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
  263. {
  264. }
  265. void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
  266. {
  267. vcpu->cpu = cpu;
  268. vcpu->arch.host_cpu_context = this_cpu_ptr(kvm_host_cpu_state);
  269. /*
  270. * Check whether this vcpu requires the cache to be flushed on
  271. * this physical CPU. This is a consequence of doing dcache
  272. * operations by set/way on this vcpu. We do it here to be in
  273. * a non-preemptible section.
  274. */
  275. if (cpumask_test_and_clear_cpu(cpu, &vcpu->arch.require_dcache_flush))
  276. flush_cache_all(); /* We'd really want v7_flush_dcache_all() */
  277. kvm_arm_set_running_vcpu(vcpu);
  278. }
  279. void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
  280. {
  281. kvm_arm_set_running_vcpu(NULL);
  282. }
  283. int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
  284. struct kvm_guest_debug *dbg)
  285. {
  286. return -EINVAL;
  287. }
  288. int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
  289. struct kvm_mp_state *mp_state)
  290. {
  291. return -EINVAL;
  292. }
  293. int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
  294. struct kvm_mp_state *mp_state)
  295. {
  296. return -EINVAL;
  297. }
  298. /**
  299. * kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled
  300. * @v: The VCPU pointer
  301. *
  302. * If the guest CPU is not waiting for interrupts or an interrupt line is
  303. * asserted, the CPU is by definition runnable.
  304. */
  305. int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
  306. {
  307. return !!v->arch.irq_lines || kvm_vgic_vcpu_pending_irq(v);
  308. }
  309. /* Just ensure a guest exit from a particular CPU */
  310. static void exit_vm_noop(void *info)
  311. {
  312. }
  313. void force_vm_exit(const cpumask_t *mask)
  314. {
  315. smp_call_function_many(mask, exit_vm_noop, NULL, true);
  316. }
  317. /**
  318. * need_new_vmid_gen - check that the VMID is still valid
  319. * @kvm: The VM's VMID to checkt
  320. *
  321. * return true if there is a new generation of VMIDs being used
  322. *
  323. * The hardware supports only 256 values with the value zero reserved for the
  324. * host, so we check if an assigned value belongs to a previous generation,
  325. * which which requires us to assign a new value. If we're the first to use a
  326. * VMID for the new generation, we must flush necessary caches and TLBs on all
  327. * CPUs.
  328. */
  329. static bool need_new_vmid_gen(struct kvm *kvm)
  330. {
  331. return unlikely(kvm->arch.vmid_gen != atomic64_read(&kvm_vmid_gen));
  332. }
  333. /**
  334. * update_vttbr - Update the VTTBR with a valid VMID before the guest runs
  335. * @kvm The guest that we are about to run
  336. *
  337. * Called from kvm_arch_vcpu_ioctl_run before entering the guest to ensure the
  338. * VM has a valid VMID, otherwise assigns a new one and flushes corresponding
  339. * caches and TLBs.
  340. */
  341. static void update_vttbr(struct kvm *kvm)
  342. {
  343. phys_addr_t pgd_phys;
  344. u64 vmid;
  345. if (!need_new_vmid_gen(kvm))
  346. return;
  347. spin_lock(&kvm_vmid_lock);
  348. /*
  349. * We need to re-check the vmid_gen here to ensure that if another vcpu
  350. * already allocated a valid vmid for this vm, then this vcpu should
  351. * use the same vmid.
  352. */
  353. if (!need_new_vmid_gen(kvm)) {
  354. spin_unlock(&kvm_vmid_lock);
  355. return;
  356. }
  357. /* First user of a new VMID generation? */
  358. if (unlikely(kvm_next_vmid == 0)) {
  359. atomic64_inc(&kvm_vmid_gen);
  360. kvm_next_vmid = 1;
  361. /*
  362. * On SMP we know no other CPUs can use this CPU's or each
  363. * other's VMID after force_vm_exit returns since the
  364. * kvm_vmid_lock blocks them from reentry to the guest.
  365. */
  366. force_vm_exit(cpu_all_mask);
  367. /*
  368. * Now broadcast TLB + ICACHE invalidation over the inner
  369. * shareable domain to make sure all data structures are
  370. * clean.
  371. */
  372. kvm_call_hyp(__kvm_flush_vm_context);
  373. }
  374. kvm->arch.vmid_gen = atomic64_read(&kvm_vmid_gen);
  375. kvm->arch.vmid = kvm_next_vmid;
  376. kvm_next_vmid++;
  377. /* update vttbr to be used with the new vmid */
  378. pgd_phys = virt_to_phys(kvm->arch.pgd);
  379. vmid = ((u64)(kvm->arch.vmid) << VTTBR_VMID_SHIFT) & VTTBR_VMID_MASK;
  380. kvm->arch.vttbr = pgd_phys & VTTBR_BADDR_MASK;
  381. kvm->arch.vttbr |= vmid;
  382. spin_unlock(&kvm_vmid_lock);
  383. }
  384. static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu)
  385. {
  386. if (likely(vcpu->arch.has_run_once))
  387. return 0;
  388. vcpu->arch.has_run_once = true;
  389. /*
  390. * Initialize the VGIC before running a vcpu the first time on
  391. * this VM.
  392. */
  393. if (irqchip_in_kernel(vcpu->kvm) &&
  394. unlikely(!vgic_initialized(vcpu->kvm))) {
  395. int ret = kvm_vgic_init(vcpu->kvm);
  396. if (ret)
  397. return ret;
  398. }
  399. /*
  400. * Handle the "start in power-off" case by calling into the
  401. * PSCI code.
  402. */
  403. if (test_and_clear_bit(KVM_ARM_VCPU_POWER_OFF, vcpu->arch.features)) {
  404. *vcpu_reg(vcpu, 0) = KVM_PSCI_FN_CPU_OFF;
  405. kvm_psci_call(vcpu);
  406. }
  407. return 0;
  408. }
  409. static void vcpu_pause(struct kvm_vcpu *vcpu)
  410. {
  411. wait_queue_head_t *wq = kvm_arch_vcpu_wq(vcpu);
  412. wait_event_interruptible(*wq, !vcpu->arch.pause);
  413. }
  414. /**
  415. * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code
  416. * @vcpu: The VCPU pointer
  417. * @run: The kvm_run structure pointer used for userspace state exchange
  418. *
  419. * This function is called through the VCPU_RUN ioctl called from user space. It
  420. * will execute VM code in a loop until the time slice for the process is used
  421. * or some emulation is needed from user space in which case the function will
  422. * return with return value 0 and with the kvm_run structure filled in with the
  423. * required data for the requested emulation.
  424. */
  425. int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
  426. {
  427. int ret;
  428. sigset_t sigsaved;
  429. /* Make sure they initialize the vcpu with KVM_ARM_VCPU_INIT */
  430. if (unlikely(vcpu->arch.target < 0))
  431. return -ENOEXEC;
  432. ret = kvm_vcpu_first_run_init(vcpu);
  433. if (ret)
  434. return ret;
  435. if (run->exit_reason == KVM_EXIT_MMIO) {
  436. ret = kvm_handle_mmio_return(vcpu, vcpu->run);
  437. if (ret)
  438. return ret;
  439. }
  440. if (vcpu->sigset_active)
  441. sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
  442. ret = 1;
  443. run->exit_reason = KVM_EXIT_UNKNOWN;
  444. while (ret > 0) {
  445. /*
  446. * Check conditions before entering the guest
  447. */
  448. cond_resched();
  449. update_vttbr(vcpu->kvm);
  450. if (vcpu->arch.pause)
  451. vcpu_pause(vcpu);
  452. kvm_vgic_flush_hwstate(vcpu);
  453. kvm_timer_flush_hwstate(vcpu);
  454. local_irq_disable();
  455. /*
  456. * Re-check atomic conditions
  457. */
  458. if (signal_pending(current)) {
  459. ret = -EINTR;
  460. run->exit_reason = KVM_EXIT_INTR;
  461. }
  462. if (ret <= 0 || need_new_vmid_gen(vcpu->kvm)) {
  463. local_irq_enable();
  464. kvm_timer_sync_hwstate(vcpu);
  465. kvm_vgic_sync_hwstate(vcpu);
  466. continue;
  467. }
  468. /**************************************************************
  469. * Enter the guest
  470. */
  471. trace_kvm_entry(*vcpu_pc(vcpu));
  472. kvm_guest_enter();
  473. vcpu->mode = IN_GUEST_MODE;
  474. ret = kvm_call_hyp(__kvm_vcpu_run, vcpu);
  475. vcpu->mode = OUTSIDE_GUEST_MODE;
  476. vcpu->arch.last_pcpu = smp_processor_id();
  477. kvm_guest_exit();
  478. trace_kvm_exit(*vcpu_pc(vcpu));
  479. /*
  480. * We may have taken a host interrupt in HYP mode (ie
  481. * while executing the guest). This interrupt is still
  482. * pending, as we haven't serviced it yet!
  483. *
  484. * We're now back in SVC mode, with interrupts
  485. * disabled. Enabling the interrupts now will have
  486. * the effect of taking the interrupt again, in SVC
  487. * mode this time.
  488. */
  489. local_irq_enable();
  490. /*
  491. * Back from guest
  492. *************************************************************/
  493. kvm_timer_sync_hwstate(vcpu);
  494. kvm_vgic_sync_hwstate(vcpu);
  495. ret = handle_exit(vcpu, run, ret);
  496. }
  497. if (vcpu->sigset_active)
  498. sigprocmask(SIG_SETMASK, &sigsaved, NULL);
  499. return ret;
  500. }
  501. static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level)
  502. {
  503. int bit_index;
  504. bool set;
  505. unsigned long *ptr;
  506. if (number == KVM_ARM_IRQ_CPU_IRQ)
  507. bit_index = __ffs(HCR_VI);
  508. else /* KVM_ARM_IRQ_CPU_FIQ */
  509. bit_index = __ffs(HCR_VF);
  510. ptr = (unsigned long *)&vcpu->arch.irq_lines;
  511. if (level)
  512. set = test_and_set_bit(bit_index, ptr);
  513. else
  514. set = test_and_clear_bit(bit_index, ptr);
  515. /*
  516. * If we didn't change anything, no need to wake up or kick other CPUs
  517. */
  518. if (set == level)
  519. return 0;
  520. /*
  521. * The vcpu irq_lines field was updated, wake up sleeping VCPUs and
  522. * trigger a world-switch round on the running physical CPU to set the
  523. * virtual IRQ/FIQ fields in the HCR appropriately.
  524. */
  525. kvm_vcpu_kick(vcpu);
  526. return 0;
  527. }
  528. int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
  529. bool line_status)
  530. {
  531. u32 irq = irq_level->irq;
  532. unsigned int irq_type, vcpu_idx, irq_num;
  533. int nrcpus = atomic_read(&kvm->online_vcpus);
  534. struct kvm_vcpu *vcpu = NULL;
  535. bool level = irq_level->level;
  536. irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK;
  537. vcpu_idx = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;
  538. irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK;
  539. trace_kvm_irq_line(irq_type, vcpu_idx, irq_num, irq_level->level);
  540. switch (irq_type) {
  541. case KVM_ARM_IRQ_TYPE_CPU:
  542. if (irqchip_in_kernel(kvm))
  543. return -ENXIO;
  544. if (vcpu_idx >= nrcpus)
  545. return -EINVAL;
  546. vcpu = kvm_get_vcpu(kvm, vcpu_idx);
  547. if (!vcpu)
  548. return -EINVAL;
  549. if (irq_num > KVM_ARM_IRQ_CPU_FIQ)
  550. return -EINVAL;
  551. return vcpu_interrupt_line(vcpu, irq_num, level);
  552. case KVM_ARM_IRQ_TYPE_PPI:
  553. if (!irqchip_in_kernel(kvm))
  554. return -ENXIO;
  555. if (vcpu_idx >= nrcpus)
  556. return -EINVAL;
  557. vcpu = kvm_get_vcpu(kvm, vcpu_idx);
  558. if (!vcpu)
  559. return -EINVAL;
  560. if (irq_num < VGIC_NR_SGIS || irq_num >= VGIC_NR_PRIVATE_IRQS)
  561. return -EINVAL;
  562. return kvm_vgic_inject_irq(kvm, vcpu->vcpu_id, irq_num, level);
  563. case KVM_ARM_IRQ_TYPE_SPI:
  564. if (!irqchip_in_kernel(kvm))
  565. return -ENXIO;
  566. if (irq_num < VGIC_NR_PRIVATE_IRQS ||
  567. irq_num > KVM_ARM_IRQ_GIC_MAX)
  568. return -EINVAL;
  569. return kvm_vgic_inject_irq(kvm, 0, irq_num, level);
  570. }
  571. return -EINVAL;
  572. }
  573. long kvm_arch_vcpu_ioctl(struct file *filp,
  574. unsigned int ioctl, unsigned long arg)
  575. {
  576. struct kvm_vcpu *vcpu = filp->private_data;
  577. void __user *argp = (void __user *)arg;
  578. switch (ioctl) {
  579. case KVM_ARM_VCPU_INIT: {
  580. struct kvm_vcpu_init init;
  581. if (copy_from_user(&init, argp, sizeof(init)))
  582. return -EFAULT;
  583. return kvm_vcpu_set_target(vcpu, &init);
  584. }
  585. case KVM_SET_ONE_REG:
  586. case KVM_GET_ONE_REG: {
  587. struct kvm_one_reg reg;
  588. if (copy_from_user(&reg, argp, sizeof(reg)))
  589. return -EFAULT;
  590. if (ioctl == KVM_SET_ONE_REG)
  591. return kvm_arm_set_reg(vcpu, &reg);
  592. else
  593. return kvm_arm_get_reg(vcpu, &reg);
  594. }
  595. case KVM_GET_REG_LIST: {
  596. struct kvm_reg_list __user *user_list = argp;
  597. struct kvm_reg_list reg_list;
  598. unsigned n;
  599. if (copy_from_user(&reg_list, user_list, sizeof(reg_list)))
  600. return -EFAULT;
  601. n = reg_list.n;
  602. reg_list.n = kvm_arm_num_regs(vcpu);
  603. if (copy_to_user(user_list, &reg_list, sizeof(reg_list)))
  604. return -EFAULT;
  605. if (n < reg_list.n)
  606. return -E2BIG;
  607. return kvm_arm_copy_reg_indices(vcpu, user_list->reg);
  608. }
  609. default:
  610. return -EINVAL;
  611. }
  612. }
  613. int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
  614. {
  615. return -EINVAL;
  616. }
  617. static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm,
  618. struct kvm_arm_device_addr *dev_addr)
  619. {
  620. unsigned long dev_id, type;
  621. dev_id = (dev_addr->id & KVM_ARM_DEVICE_ID_MASK) >>
  622. KVM_ARM_DEVICE_ID_SHIFT;
  623. type = (dev_addr->id & KVM_ARM_DEVICE_TYPE_MASK) >>
  624. KVM_ARM_DEVICE_TYPE_SHIFT;
  625. switch (dev_id) {
  626. case KVM_ARM_DEVICE_VGIC_V2:
  627. if (!vgic_present)
  628. return -ENXIO;
  629. return kvm_vgic_set_addr(kvm, type, dev_addr->addr);
  630. default:
  631. return -ENODEV;
  632. }
  633. }
  634. long kvm_arch_vm_ioctl(struct file *filp,
  635. unsigned int ioctl, unsigned long arg)
  636. {
  637. struct kvm *kvm = filp->private_data;
  638. void __user *argp = (void __user *)arg;
  639. switch (ioctl) {
  640. case KVM_CREATE_IRQCHIP: {
  641. if (vgic_present)
  642. return kvm_vgic_create(kvm);
  643. else
  644. return -ENXIO;
  645. }
  646. case KVM_ARM_SET_DEVICE_ADDR: {
  647. struct kvm_arm_device_addr dev_addr;
  648. if (copy_from_user(&dev_addr, argp, sizeof(dev_addr)))
  649. return -EFAULT;
  650. return kvm_vm_ioctl_set_device_addr(kvm, &dev_addr);
  651. }
  652. default:
  653. return -EINVAL;
  654. }
  655. }
  656. static void cpu_init_hyp_mode(void *dummy)
  657. {
  658. unsigned long long boot_pgd_ptr;
  659. unsigned long long pgd_ptr;
  660. unsigned long hyp_stack_ptr;
  661. unsigned long stack_page;
  662. unsigned long vector_ptr;
  663. /* Switch from the HYP stub to our own HYP init vector */
  664. __hyp_set_vectors(kvm_get_idmap_vector());
  665. boot_pgd_ptr = (unsigned long long)kvm_mmu_get_boot_httbr();
  666. pgd_ptr = (unsigned long long)kvm_mmu_get_httbr();
  667. stack_page = __get_cpu_var(kvm_arm_hyp_stack_page);
  668. hyp_stack_ptr = stack_page + PAGE_SIZE;
  669. vector_ptr = (unsigned long)__kvm_hyp_vector;
  670. __cpu_init_hyp_mode(boot_pgd_ptr, pgd_ptr, hyp_stack_ptr, vector_ptr);
  671. }
  672. static int hyp_init_cpu_notify(struct notifier_block *self,
  673. unsigned long action, void *cpu)
  674. {
  675. switch (action) {
  676. case CPU_STARTING:
  677. case CPU_STARTING_FROZEN:
  678. cpu_init_hyp_mode(NULL);
  679. break;
  680. }
  681. return NOTIFY_OK;
  682. }
  683. static struct notifier_block hyp_init_cpu_nb = {
  684. .notifier_call = hyp_init_cpu_notify,
  685. };
  686. /**
  687. * Inits Hyp-mode on all online CPUs
  688. */
  689. static int init_hyp_mode(void)
  690. {
  691. int cpu;
  692. int err = 0;
  693. /*
  694. * Allocate Hyp PGD and setup Hyp identity mapping
  695. */
  696. err = kvm_mmu_init();
  697. if (err)
  698. goto out_err;
  699. /*
  700. * It is probably enough to obtain the default on one
  701. * CPU. It's unlikely to be different on the others.
  702. */
  703. hyp_default_vectors = __hyp_get_vectors();
  704. /*
  705. * Allocate stack pages for Hypervisor-mode
  706. */
  707. for_each_possible_cpu(cpu) {
  708. unsigned long stack_page;
  709. stack_page = __get_free_page(GFP_KERNEL);
  710. if (!stack_page) {
  711. err = -ENOMEM;
  712. goto out_free_stack_pages;
  713. }
  714. per_cpu(kvm_arm_hyp_stack_page, cpu) = stack_page;
  715. }
  716. /*
  717. * Map the Hyp-code called directly from the host
  718. */
  719. err = create_hyp_mappings(__kvm_hyp_code_start, __kvm_hyp_code_end);
  720. if (err) {
  721. kvm_err("Cannot map world-switch code\n");
  722. goto out_free_mappings;
  723. }
  724. /*
  725. * Map the Hyp stack pages
  726. */
  727. for_each_possible_cpu(cpu) {
  728. char *stack_page = (char *)per_cpu(kvm_arm_hyp_stack_page, cpu);
  729. err = create_hyp_mappings(stack_page, stack_page + PAGE_SIZE);
  730. if (err) {
  731. kvm_err("Cannot map hyp stack\n");
  732. goto out_free_mappings;
  733. }
  734. }
  735. /*
  736. * Map the host CPU structures
  737. */
  738. kvm_host_cpu_state = alloc_percpu(kvm_cpu_context_t);
  739. if (!kvm_host_cpu_state) {
  740. err = -ENOMEM;
  741. kvm_err("Cannot allocate host CPU state\n");
  742. goto out_free_mappings;
  743. }
  744. for_each_possible_cpu(cpu) {
  745. kvm_cpu_context_t *cpu_ctxt;
  746. cpu_ctxt = per_cpu_ptr(kvm_host_cpu_state, cpu);
  747. err = create_hyp_mappings(cpu_ctxt, cpu_ctxt + 1);
  748. if (err) {
  749. kvm_err("Cannot map host CPU state: %d\n", err);
  750. goto out_free_context;
  751. }
  752. }
  753. /*
  754. * Execute the init code on each CPU.
  755. */
  756. on_each_cpu(cpu_init_hyp_mode, NULL, 1);
  757. /*
  758. * Init HYP view of VGIC
  759. */
  760. err = kvm_vgic_hyp_init();
  761. if (err)
  762. goto out_free_context;
  763. #ifdef CONFIG_KVM_ARM_VGIC
  764. vgic_present = true;
  765. #endif
  766. /*
  767. * Init HYP architected timer support
  768. */
  769. err = kvm_timer_hyp_init();
  770. if (err)
  771. goto out_free_mappings;
  772. #ifndef CONFIG_HOTPLUG_CPU
  773. free_boot_hyp_pgd();
  774. #endif
  775. kvm_perf_init();
  776. kvm_info("Hyp mode initialized successfully\n");
  777. return 0;
  778. out_free_context:
  779. free_percpu(kvm_host_cpu_state);
  780. out_free_mappings:
  781. free_hyp_pgds();
  782. out_free_stack_pages:
  783. for_each_possible_cpu(cpu)
  784. free_page(per_cpu(kvm_arm_hyp_stack_page, cpu));
  785. out_err:
  786. kvm_err("error initializing Hyp mode: %d\n", err);
  787. return err;
  788. }
  789. static void check_kvm_target_cpu(void *ret)
  790. {
  791. *(int *)ret = kvm_target_cpu();
  792. }
  793. /**
  794. * Initialize Hyp-mode and memory mappings on all CPUs.
  795. */
  796. int kvm_arch_init(void *opaque)
  797. {
  798. int err;
  799. int ret, cpu;
  800. if (!is_hyp_mode_available()) {
  801. kvm_err("HYP mode not available\n");
  802. return -ENODEV;
  803. }
  804. for_each_online_cpu(cpu) {
  805. smp_call_function_single(cpu, check_kvm_target_cpu, &ret, 1);
  806. if (ret < 0) {
  807. kvm_err("Error, CPU %d not supported!\n", cpu);
  808. return -ENODEV;
  809. }
  810. }
  811. err = init_hyp_mode();
  812. if (err)
  813. goto out_err;
  814. err = register_cpu_notifier(&hyp_init_cpu_nb);
  815. if (err) {
  816. kvm_err("Cannot register HYP init CPU notifier (%d)\n", err);
  817. goto out_err;
  818. }
  819. kvm_coproc_table_init();
  820. return 0;
  821. out_err:
  822. return err;
  823. }
  824. /* NOP: Compiling as a module not supported */
  825. void kvm_arch_exit(void)
  826. {
  827. kvm_perf_teardown();
  828. }
  829. static int arm_init(void)
  830. {
  831. int rc = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
  832. return rc;
  833. }
  834. module_init(arm_init);