vhost.c 29 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130
  1. /* Copyright (C) 2009 Red Hat, Inc.
  2. * Copyright (C) 2006 Rusty Russell IBM Corporation
  3. *
  4. * Author: Michael S. Tsirkin <mst@redhat.com>
  5. *
  6. * Inspiration, some code, and most witty comments come from
  7. * Documentation/lguest/lguest.c, by Rusty Russell
  8. *
  9. * This work is licensed under the terms of the GNU GPL, version 2.
  10. *
  11. * Generic code for virtio server in host kernel.
  12. */
  13. #include <linux/eventfd.h>
  14. #include <linux/vhost.h>
  15. #include <linux/virtio_net.h>
  16. #include <linux/mm.h>
  17. #include <linux/miscdevice.h>
  18. #include <linux/mutex.h>
  19. #include <linux/workqueue.h>
  20. #include <linux/rcupdate.h>
  21. #include <linux/poll.h>
  22. #include <linux/file.h>
  23. #include <linux/highmem.h>
  24. #include <linux/slab.h>
  25. #include <linux/net.h>
  26. #include <linux/if_packet.h>
  27. #include <linux/if_arp.h>
  28. #include <net/sock.h>
  29. #include "vhost.h"
  30. enum {
  31. VHOST_MEMORY_MAX_NREGIONS = 64,
  32. VHOST_MEMORY_F_LOG = 0x1,
  33. };
  34. static struct workqueue_struct *vhost_workqueue;
  35. static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh,
  36. poll_table *pt)
  37. {
  38. struct vhost_poll *poll;
  39. poll = container_of(pt, struct vhost_poll, table);
  40. poll->wqh = wqh;
  41. add_wait_queue(wqh, &poll->wait);
  42. }
  43. static int vhost_poll_wakeup(wait_queue_t *wait, unsigned mode, int sync,
  44. void *key)
  45. {
  46. struct vhost_poll *poll;
  47. poll = container_of(wait, struct vhost_poll, wait);
  48. if (!((unsigned long)key & poll->mask))
  49. return 0;
  50. queue_work(vhost_workqueue, &poll->work);
  51. return 0;
  52. }
  53. /* Init poll structure */
  54. void vhost_poll_init(struct vhost_poll *poll, work_func_t func,
  55. unsigned long mask)
  56. {
  57. INIT_WORK(&poll->work, func);
  58. init_waitqueue_func_entry(&poll->wait, vhost_poll_wakeup);
  59. init_poll_funcptr(&poll->table, vhost_poll_func);
  60. poll->mask = mask;
  61. }
  62. /* Start polling a file. We add ourselves to file's wait queue. The caller must
  63. * keep a reference to a file until after vhost_poll_stop is called. */
  64. void vhost_poll_start(struct vhost_poll *poll, struct file *file)
  65. {
  66. unsigned long mask;
  67. mask = file->f_op->poll(file, &poll->table);
  68. if (mask)
  69. vhost_poll_wakeup(&poll->wait, 0, 0, (void *)mask);
  70. }
  71. /* Stop polling a file. After this function returns, it becomes safe to drop the
  72. * file reference. You must also flush afterwards. */
  73. void vhost_poll_stop(struct vhost_poll *poll)
  74. {
  75. remove_wait_queue(poll->wqh, &poll->wait);
  76. }
  77. /* Flush any work that has been scheduled. When calling this, don't hold any
  78. * locks that are also used by the callback. */
  79. void vhost_poll_flush(struct vhost_poll *poll)
  80. {
  81. flush_work(&poll->work);
  82. }
  83. void vhost_poll_queue(struct vhost_poll *poll)
  84. {
  85. queue_work(vhost_workqueue, &poll->work);
  86. }
  87. static void vhost_vq_reset(struct vhost_dev *dev,
  88. struct vhost_virtqueue *vq)
  89. {
  90. vq->num = 1;
  91. vq->desc = NULL;
  92. vq->avail = NULL;
  93. vq->used = NULL;
  94. vq->last_avail_idx = 0;
  95. vq->avail_idx = 0;
  96. vq->last_used_idx = 0;
  97. vq->used_flags = 0;
  98. vq->used_flags = 0;
  99. vq->log_used = false;
  100. vq->log_addr = -1ull;
  101. vq->hdr_size = 0;
  102. vq->private_data = NULL;
  103. vq->log_base = NULL;
  104. vq->error_ctx = NULL;
  105. vq->error = NULL;
  106. vq->kick = NULL;
  107. vq->call_ctx = NULL;
  108. vq->call = NULL;
  109. vq->log_ctx = NULL;
  110. }
  111. long vhost_dev_init(struct vhost_dev *dev,
  112. struct vhost_virtqueue *vqs, int nvqs)
  113. {
  114. int i;
  115. dev->vqs = vqs;
  116. dev->nvqs = nvqs;
  117. mutex_init(&dev->mutex);
  118. dev->log_ctx = NULL;
  119. dev->log_file = NULL;
  120. dev->memory = NULL;
  121. dev->mm = NULL;
  122. for (i = 0; i < dev->nvqs; ++i) {
  123. dev->vqs[i].dev = dev;
  124. mutex_init(&dev->vqs[i].mutex);
  125. vhost_vq_reset(dev, dev->vqs + i);
  126. if (dev->vqs[i].handle_kick)
  127. vhost_poll_init(&dev->vqs[i].poll,
  128. dev->vqs[i].handle_kick,
  129. POLLIN);
  130. }
  131. return 0;
  132. }
  133. /* Caller should have device mutex */
  134. long vhost_dev_check_owner(struct vhost_dev *dev)
  135. {
  136. /* Are you the owner? If not, I don't think you mean to do that */
  137. return dev->mm == current->mm ? 0 : -EPERM;
  138. }
  139. /* Caller should have device mutex */
  140. static long vhost_dev_set_owner(struct vhost_dev *dev)
  141. {
  142. /* Is there an owner already? */
  143. if (dev->mm)
  144. return -EBUSY;
  145. /* No owner, become one */
  146. dev->mm = get_task_mm(current);
  147. return 0;
  148. }
  149. /* Caller should have device mutex */
  150. long vhost_dev_reset_owner(struct vhost_dev *dev)
  151. {
  152. struct vhost_memory *memory;
  153. /* Restore memory to default empty mapping. */
  154. memory = kmalloc(offsetof(struct vhost_memory, regions), GFP_KERNEL);
  155. if (!memory)
  156. return -ENOMEM;
  157. vhost_dev_cleanup(dev);
  158. memory->nregions = 0;
  159. dev->memory = memory;
  160. return 0;
  161. }
  162. /* Caller should have device mutex */
  163. void vhost_dev_cleanup(struct vhost_dev *dev)
  164. {
  165. int i;
  166. for (i = 0; i < dev->nvqs; ++i) {
  167. if (dev->vqs[i].kick && dev->vqs[i].handle_kick) {
  168. vhost_poll_stop(&dev->vqs[i].poll);
  169. vhost_poll_flush(&dev->vqs[i].poll);
  170. }
  171. if (dev->vqs[i].error_ctx)
  172. eventfd_ctx_put(dev->vqs[i].error_ctx);
  173. if (dev->vqs[i].error)
  174. fput(dev->vqs[i].error);
  175. if (dev->vqs[i].kick)
  176. fput(dev->vqs[i].kick);
  177. if (dev->vqs[i].call_ctx)
  178. eventfd_ctx_put(dev->vqs[i].call_ctx);
  179. if (dev->vqs[i].call)
  180. fput(dev->vqs[i].call);
  181. vhost_vq_reset(dev, dev->vqs + i);
  182. }
  183. if (dev->log_ctx)
  184. eventfd_ctx_put(dev->log_ctx);
  185. dev->log_ctx = NULL;
  186. if (dev->log_file)
  187. fput(dev->log_file);
  188. dev->log_file = NULL;
  189. /* No one will access memory at this point */
  190. kfree(dev->memory);
  191. dev->memory = NULL;
  192. if (dev->mm)
  193. mmput(dev->mm);
  194. dev->mm = NULL;
  195. }
  196. static int log_access_ok(void __user *log_base, u64 addr, unsigned long sz)
  197. {
  198. u64 a = addr / VHOST_PAGE_SIZE / 8;
  199. /* Make sure 64 bit math will not overflow. */
  200. if (a > ULONG_MAX - (unsigned long)log_base ||
  201. a + (unsigned long)log_base > ULONG_MAX)
  202. return -EFAULT;
  203. return access_ok(VERIFY_WRITE, log_base + a,
  204. (sz + VHOST_PAGE_SIZE * 8 - 1) / VHOST_PAGE_SIZE / 8);
  205. }
  206. /* Caller should have vq mutex and device mutex. */
  207. static int vq_memory_access_ok(void __user *log_base, struct vhost_memory *mem,
  208. int log_all)
  209. {
  210. int i;
  211. if (!mem)
  212. return 0;
  213. for (i = 0; i < mem->nregions; ++i) {
  214. struct vhost_memory_region *m = mem->regions + i;
  215. unsigned long a = m->userspace_addr;
  216. if (m->memory_size > ULONG_MAX)
  217. return 0;
  218. else if (!access_ok(VERIFY_WRITE, (void __user *)a,
  219. m->memory_size))
  220. return 0;
  221. else if (log_all && !log_access_ok(log_base,
  222. m->guest_phys_addr,
  223. m->memory_size))
  224. return 0;
  225. }
  226. return 1;
  227. }
  228. /* Can we switch to this memory table? */
  229. /* Caller should have device mutex but not vq mutex */
  230. static int memory_access_ok(struct vhost_dev *d, struct vhost_memory *mem,
  231. int log_all)
  232. {
  233. int i;
  234. for (i = 0; i < d->nvqs; ++i) {
  235. int ok;
  236. mutex_lock(&d->vqs[i].mutex);
  237. /* If ring is inactive, will check when it's enabled. */
  238. if (d->vqs[i].private_data)
  239. ok = vq_memory_access_ok(d->vqs[i].log_base, mem,
  240. log_all);
  241. else
  242. ok = 1;
  243. mutex_unlock(&d->vqs[i].mutex);
  244. if (!ok)
  245. return 0;
  246. }
  247. return 1;
  248. }
  249. static int vq_access_ok(unsigned int num,
  250. struct vring_desc __user *desc,
  251. struct vring_avail __user *avail,
  252. struct vring_used __user *used)
  253. {
  254. return access_ok(VERIFY_READ, desc, num * sizeof *desc) &&
  255. access_ok(VERIFY_READ, avail,
  256. sizeof *avail + num * sizeof *avail->ring) &&
  257. access_ok(VERIFY_WRITE, used,
  258. sizeof *used + num * sizeof *used->ring);
  259. }
  260. /* Can we log writes? */
  261. /* Caller should have device mutex but not vq mutex */
  262. int vhost_log_access_ok(struct vhost_dev *dev)
  263. {
  264. return memory_access_ok(dev, dev->memory, 1);
  265. }
  266. /* Verify access for write logging. */
  267. /* Caller should have vq mutex and device mutex */
  268. static int vq_log_access_ok(struct vhost_virtqueue *vq, void __user *log_base)
  269. {
  270. return vq_memory_access_ok(log_base, vq->dev->memory,
  271. vhost_has_feature(vq->dev, VHOST_F_LOG_ALL)) &&
  272. (!vq->log_used || log_access_ok(log_base, vq->log_addr,
  273. sizeof *vq->used +
  274. vq->num * sizeof *vq->used->ring));
  275. }
  276. /* Can we start vq? */
  277. /* Caller should have vq mutex and device mutex */
  278. int vhost_vq_access_ok(struct vhost_virtqueue *vq)
  279. {
  280. return vq_access_ok(vq->num, vq->desc, vq->avail, vq->used) &&
  281. vq_log_access_ok(vq, vq->log_base);
  282. }
  283. static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m)
  284. {
  285. struct vhost_memory mem, *newmem, *oldmem;
  286. unsigned long size = offsetof(struct vhost_memory, regions);
  287. if (copy_from_user(&mem, m, size))
  288. return -EFAULT;
  289. if (mem.padding)
  290. return -EOPNOTSUPP;
  291. if (mem.nregions > VHOST_MEMORY_MAX_NREGIONS)
  292. return -E2BIG;
  293. newmem = kmalloc(size + mem.nregions * sizeof *m->regions, GFP_KERNEL);
  294. if (!newmem)
  295. return -ENOMEM;
  296. memcpy(newmem, &mem, size);
  297. if (copy_from_user(newmem->regions, m->regions,
  298. mem.nregions * sizeof *m->regions)) {
  299. kfree(newmem);
  300. return -EFAULT;
  301. }
  302. if (!memory_access_ok(d, newmem, vhost_has_feature(d, VHOST_F_LOG_ALL))) {
  303. kfree(newmem);
  304. return -EFAULT;
  305. }
  306. oldmem = d->memory;
  307. rcu_assign_pointer(d->memory, newmem);
  308. synchronize_rcu();
  309. kfree(oldmem);
  310. return 0;
  311. }
  312. static int init_used(struct vhost_virtqueue *vq,
  313. struct vring_used __user *used)
  314. {
  315. int r = put_user(vq->used_flags, &used->flags);
  316. if (r)
  317. return r;
  318. return get_user(vq->last_used_idx, &used->idx);
  319. }
  320. static long vhost_set_vring(struct vhost_dev *d, int ioctl, void __user *argp)
  321. {
  322. struct file *eventfp, *filep = NULL,
  323. *pollstart = NULL, *pollstop = NULL;
  324. struct eventfd_ctx *ctx = NULL;
  325. u32 __user *idxp = argp;
  326. struct vhost_virtqueue *vq;
  327. struct vhost_vring_state s;
  328. struct vhost_vring_file f;
  329. struct vhost_vring_addr a;
  330. u32 idx;
  331. long r;
  332. r = get_user(idx, idxp);
  333. if (r < 0)
  334. return r;
  335. if (idx >= d->nvqs)
  336. return -ENOBUFS;
  337. vq = d->vqs + idx;
  338. mutex_lock(&vq->mutex);
  339. switch (ioctl) {
  340. case VHOST_SET_VRING_NUM:
  341. /* Resizing ring with an active backend?
  342. * You don't want to do that. */
  343. if (vq->private_data) {
  344. r = -EBUSY;
  345. break;
  346. }
  347. if (copy_from_user(&s, argp, sizeof s)) {
  348. r = -EFAULT;
  349. break;
  350. }
  351. if (!s.num || s.num > 0xffff || (s.num & (s.num - 1))) {
  352. r = -EINVAL;
  353. break;
  354. }
  355. vq->num = s.num;
  356. break;
  357. case VHOST_SET_VRING_BASE:
  358. /* Moving base with an active backend?
  359. * You don't want to do that. */
  360. if (vq->private_data) {
  361. r = -EBUSY;
  362. break;
  363. }
  364. if (copy_from_user(&s, argp, sizeof s)) {
  365. r = -EFAULT;
  366. break;
  367. }
  368. if (s.num > 0xffff) {
  369. r = -EINVAL;
  370. break;
  371. }
  372. vq->last_avail_idx = s.num;
  373. /* Forget the cached index value. */
  374. vq->avail_idx = vq->last_avail_idx;
  375. break;
  376. case VHOST_GET_VRING_BASE:
  377. s.index = idx;
  378. s.num = vq->last_avail_idx;
  379. if (copy_to_user(argp, &s, sizeof s))
  380. r = -EFAULT;
  381. break;
  382. case VHOST_SET_VRING_ADDR:
  383. if (copy_from_user(&a, argp, sizeof a)) {
  384. r = -EFAULT;
  385. break;
  386. }
  387. if (a.flags & ~(0x1 << VHOST_VRING_F_LOG)) {
  388. r = -EOPNOTSUPP;
  389. break;
  390. }
  391. /* For 32bit, verify that the top 32bits of the user
  392. data are set to zero. */
  393. if ((u64)(unsigned long)a.desc_user_addr != a.desc_user_addr ||
  394. (u64)(unsigned long)a.used_user_addr != a.used_user_addr ||
  395. (u64)(unsigned long)a.avail_user_addr != a.avail_user_addr) {
  396. r = -EFAULT;
  397. break;
  398. }
  399. if ((a.avail_user_addr & (sizeof *vq->avail->ring - 1)) ||
  400. (a.used_user_addr & (sizeof *vq->used->ring - 1)) ||
  401. (a.log_guest_addr & (sizeof *vq->used->ring - 1))) {
  402. r = -EINVAL;
  403. break;
  404. }
  405. /* We only verify access here if backend is configured.
  406. * If it is not, we don't as size might not have been setup.
  407. * We will verify when backend is configured. */
  408. if (vq->private_data) {
  409. if (!vq_access_ok(vq->num,
  410. (void __user *)(unsigned long)a.desc_user_addr,
  411. (void __user *)(unsigned long)a.avail_user_addr,
  412. (void __user *)(unsigned long)a.used_user_addr)) {
  413. r = -EINVAL;
  414. break;
  415. }
  416. /* Also validate log access for used ring if enabled. */
  417. if ((a.flags & (0x1 << VHOST_VRING_F_LOG)) &&
  418. !log_access_ok(vq->log_base, a.log_guest_addr,
  419. sizeof *vq->used +
  420. vq->num * sizeof *vq->used->ring)) {
  421. r = -EINVAL;
  422. break;
  423. }
  424. }
  425. r = init_used(vq, (struct vring_used __user *)(unsigned long)
  426. a.used_user_addr);
  427. if (r)
  428. break;
  429. vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG));
  430. vq->desc = (void __user *)(unsigned long)a.desc_user_addr;
  431. vq->avail = (void __user *)(unsigned long)a.avail_user_addr;
  432. vq->log_addr = a.log_guest_addr;
  433. vq->used = (void __user *)(unsigned long)a.used_user_addr;
  434. break;
  435. case VHOST_SET_VRING_KICK:
  436. if (copy_from_user(&f, argp, sizeof f)) {
  437. r = -EFAULT;
  438. break;
  439. }
  440. eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd);
  441. if (IS_ERR(eventfp)) {
  442. r = PTR_ERR(eventfp);
  443. break;
  444. }
  445. if (eventfp != vq->kick) {
  446. pollstop = filep = vq->kick;
  447. pollstart = vq->kick = eventfp;
  448. } else
  449. filep = eventfp;
  450. break;
  451. case VHOST_SET_VRING_CALL:
  452. if (copy_from_user(&f, argp, sizeof f)) {
  453. r = -EFAULT;
  454. break;
  455. }
  456. eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd);
  457. if (IS_ERR(eventfp)) {
  458. r = PTR_ERR(eventfp);
  459. break;
  460. }
  461. if (eventfp != vq->call) {
  462. filep = vq->call;
  463. ctx = vq->call_ctx;
  464. vq->call = eventfp;
  465. vq->call_ctx = eventfp ?
  466. eventfd_ctx_fileget(eventfp) : NULL;
  467. } else
  468. filep = eventfp;
  469. break;
  470. case VHOST_SET_VRING_ERR:
  471. if (copy_from_user(&f, argp, sizeof f)) {
  472. r = -EFAULT;
  473. break;
  474. }
  475. eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd);
  476. if (IS_ERR(eventfp)) {
  477. r = PTR_ERR(eventfp);
  478. break;
  479. }
  480. if (eventfp != vq->error) {
  481. filep = vq->error;
  482. vq->error = eventfp;
  483. ctx = vq->error_ctx;
  484. vq->error_ctx = eventfp ?
  485. eventfd_ctx_fileget(eventfp) : NULL;
  486. } else
  487. filep = eventfp;
  488. break;
  489. default:
  490. r = -ENOIOCTLCMD;
  491. }
  492. if (pollstop && vq->handle_kick)
  493. vhost_poll_stop(&vq->poll);
  494. if (ctx)
  495. eventfd_ctx_put(ctx);
  496. if (filep)
  497. fput(filep);
  498. if (pollstart && vq->handle_kick)
  499. vhost_poll_start(&vq->poll, vq->kick);
  500. mutex_unlock(&vq->mutex);
  501. if (pollstop && vq->handle_kick)
  502. vhost_poll_flush(&vq->poll);
  503. return r;
  504. }
  505. /* Caller must have device mutex */
  506. long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, unsigned long arg)
  507. {
  508. void __user *argp = (void __user *)arg;
  509. struct file *eventfp, *filep = NULL;
  510. struct eventfd_ctx *ctx = NULL;
  511. u64 p;
  512. long r;
  513. int i, fd;
  514. /* If you are not the owner, you can become one */
  515. if (ioctl == VHOST_SET_OWNER) {
  516. r = vhost_dev_set_owner(d);
  517. goto done;
  518. }
  519. /* You must be the owner to do anything else */
  520. r = vhost_dev_check_owner(d);
  521. if (r)
  522. goto done;
  523. switch (ioctl) {
  524. case VHOST_SET_MEM_TABLE:
  525. r = vhost_set_memory(d, argp);
  526. break;
  527. case VHOST_SET_LOG_BASE:
  528. if (copy_from_user(&p, argp, sizeof p)) {
  529. r = -EFAULT;
  530. break;
  531. }
  532. if ((u64)(unsigned long)p != p) {
  533. r = -EFAULT;
  534. break;
  535. }
  536. for (i = 0; i < d->nvqs; ++i) {
  537. struct vhost_virtqueue *vq;
  538. void __user *base = (void __user *)(unsigned long)p;
  539. vq = d->vqs + i;
  540. mutex_lock(&vq->mutex);
  541. /* If ring is inactive, will check when it's enabled. */
  542. if (vq->private_data && !vq_log_access_ok(vq, base))
  543. r = -EFAULT;
  544. else
  545. vq->log_base = base;
  546. mutex_unlock(&vq->mutex);
  547. }
  548. break;
  549. case VHOST_SET_LOG_FD:
  550. r = get_user(fd, (int __user *)argp);
  551. if (r < 0)
  552. break;
  553. eventfp = fd == -1 ? NULL : eventfd_fget(fd);
  554. if (IS_ERR(eventfp)) {
  555. r = PTR_ERR(eventfp);
  556. break;
  557. }
  558. if (eventfp != d->log_file) {
  559. filep = d->log_file;
  560. ctx = d->log_ctx;
  561. d->log_ctx = eventfp ?
  562. eventfd_ctx_fileget(eventfp) : NULL;
  563. } else
  564. filep = eventfp;
  565. for (i = 0; i < d->nvqs; ++i) {
  566. mutex_lock(&d->vqs[i].mutex);
  567. d->vqs[i].log_ctx = d->log_ctx;
  568. mutex_unlock(&d->vqs[i].mutex);
  569. }
  570. if (ctx)
  571. eventfd_ctx_put(ctx);
  572. if (filep)
  573. fput(filep);
  574. break;
  575. default:
  576. r = vhost_set_vring(d, ioctl, argp);
  577. break;
  578. }
  579. done:
  580. return r;
  581. }
  582. static const struct vhost_memory_region *find_region(struct vhost_memory *mem,
  583. __u64 addr, __u32 len)
  584. {
  585. struct vhost_memory_region *reg;
  586. int i;
  587. /* linear search is not brilliant, but we really have on the order of 6
  588. * regions in practice */
  589. for (i = 0; i < mem->nregions; ++i) {
  590. reg = mem->regions + i;
  591. if (reg->guest_phys_addr <= addr &&
  592. reg->guest_phys_addr + reg->memory_size - 1 >= addr)
  593. return reg;
  594. }
  595. return NULL;
  596. }
  597. /* TODO: This is really inefficient. We need something like get_user()
  598. * (instruction directly accesses the data, with an exception table entry
  599. * returning -EFAULT). See Documentation/x86/exception-tables.txt.
  600. */
  601. static int set_bit_to_user(int nr, void __user *addr)
  602. {
  603. unsigned long log = (unsigned long)addr;
  604. struct page *page;
  605. void *base;
  606. int bit = nr + (log % PAGE_SIZE) * 8;
  607. int r;
  608. r = get_user_pages_fast(log, 1, 1, &page);
  609. if (r < 0)
  610. return r;
  611. BUG_ON(r != 1);
  612. base = kmap_atomic(page, KM_USER0);
  613. set_bit(bit, base);
  614. kunmap_atomic(base, KM_USER0);
  615. set_page_dirty_lock(page);
  616. put_page(page);
  617. return 0;
  618. }
  619. static int log_write(void __user *log_base,
  620. u64 write_address, u64 write_length)
  621. {
  622. int r;
  623. if (!write_length)
  624. return 0;
  625. write_address /= VHOST_PAGE_SIZE;
  626. for (;;) {
  627. u64 base = (u64)(unsigned long)log_base;
  628. u64 log = base + write_address / 8;
  629. int bit = write_address % 8;
  630. if ((u64)(unsigned long)log != log)
  631. return -EFAULT;
  632. r = set_bit_to_user(bit, (void __user *)(unsigned long)log);
  633. if (r < 0)
  634. return r;
  635. if (write_length <= VHOST_PAGE_SIZE)
  636. break;
  637. write_length -= VHOST_PAGE_SIZE;
  638. write_address += VHOST_PAGE_SIZE;
  639. }
  640. return r;
  641. }
  642. int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log,
  643. unsigned int log_num, u64 len)
  644. {
  645. int i, r;
  646. /* Make sure data written is seen before log. */
  647. smp_wmb();
  648. for (i = 0; i < log_num; ++i) {
  649. u64 l = min(log[i].len, len);
  650. r = log_write(vq->log_base, log[i].addr, l);
  651. if (r < 0)
  652. return r;
  653. len -= l;
  654. if (!len)
  655. return 0;
  656. }
  657. if (vq->log_ctx)
  658. eventfd_signal(vq->log_ctx, 1);
  659. /* Length written exceeds what we have stored. This is a bug. */
  660. BUG();
  661. return 0;
  662. }
  663. static int translate_desc(struct vhost_dev *dev, u64 addr, u32 len,
  664. struct iovec iov[], int iov_size)
  665. {
  666. const struct vhost_memory_region *reg;
  667. struct vhost_memory *mem;
  668. struct iovec *_iov;
  669. u64 s = 0;
  670. int ret = 0;
  671. rcu_read_lock();
  672. mem = rcu_dereference(dev->memory);
  673. while ((u64)len > s) {
  674. u64 size;
  675. if (unlikely(ret >= iov_size)) {
  676. ret = -ENOBUFS;
  677. break;
  678. }
  679. reg = find_region(mem, addr, len);
  680. if (unlikely(!reg)) {
  681. ret = -EFAULT;
  682. break;
  683. }
  684. _iov = iov + ret;
  685. size = reg->memory_size - addr + reg->guest_phys_addr;
  686. _iov->iov_len = min((u64)len, size);
  687. _iov->iov_base = (void __user *)(unsigned long)
  688. (reg->userspace_addr + addr - reg->guest_phys_addr);
  689. s += size;
  690. addr += size;
  691. ++ret;
  692. }
  693. rcu_read_unlock();
  694. return ret;
  695. }
  696. /* Each buffer in the virtqueues is actually a chain of descriptors. This
  697. * function returns the next descriptor in the chain,
  698. * or -1U if we're at the end. */
  699. static unsigned next_desc(struct vring_desc *desc)
  700. {
  701. unsigned int next;
  702. /* If this descriptor says it doesn't chain, we're done. */
  703. if (!(desc->flags & VRING_DESC_F_NEXT))
  704. return -1U;
  705. /* Check they're not leading us off end of descriptors. */
  706. next = desc->next;
  707. /* Make sure compiler knows to grab that: we don't want it changing! */
  708. /* We will use the result as an index in an array, so most
  709. * architectures only need a compiler barrier here. */
  710. read_barrier_depends();
  711. return next;
  712. }
  713. static int get_indirect(struct vhost_dev *dev, struct vhost_virtqueue *vq,
  714. struct iovec iov[], unsigned int iov_size,
  715. unsigned int *out_num, unsigned int *in_num,
  716. struct vhost_log *log, unsigned int *log_num,
  717. struct vring_desc *indirect)
  718. {
  719. struct vring_desc desc;
  720. unsigned int i = 0, count, found = 0;
  721. int ret;
  722. /* Sanity check */
  723. if (unlikely(indirect->len % sizeof desc)) {
  724. vq_err(vq, "Invalid length in indirect descriptor: "
  725. "len 0x%llx not multiple of 0x%zx\n",
  726. (unsigned long long)indirect->len,
  727. sizeof desc);
  728. return -EINVAL;
  729. }
  730. ret = translate_desc(dev, indirect->addr, indirect->len, vq->indirect,
  731. ARRAY_SIZE(vq->indirect));
  732. if (unlikely(ret < 0)) {
  733. vq_err(vq, "Translation failure %d in indirect.\n", ret);
  734. return ret;
  735. }
  736. /* We will use the result as an address to read from, so most
  737. * architectures only need a compiler barrier here. */
  738. read_barrier_depends();
  739. count = indirect->len / sizeof desc;
  740. /* Buffers are chained via a 16 bit next field, so
  741. * we can have at most 2^16 of these. */
  742. if (unlikely(count > USHRT_MAX + 1)) {
  743. vq_err(vq, "Indirect buffer length too big: %d\n",
  744. indirect->len);
  745. return -E2BIG;
  746. }
  747. do {
  748. unsigned iov_count = *in_num + *out_num;
  749. if (unlikely(++found > count)) {
  750. vq_err(vq, "Loop detected: last one at %u "
  751. "indirect size %u\n",
  752. i, count);
  753. return -EINVAL;
  754. }
  755. if (unlikely(memcpy_fromiovec((unsigned char *)&desc, vq->indirect,
  756. sizeof desc))) {
  757. vq_err(vq, "Failed indirect descriptor: idx %d, %zx\n",
  758. i, (size_t)indirect->addr + i * sizeof desc);
  759. return -EINVAL;
  760. }
  761. if (unlikely(desc.flags & VRING_DESC_F_INDIRECT)) {
  762. vq_err(vq, "Nested indirect descriptor: idx %d, %zx\n",
  763. i, (size_t)indirect->addr + i * sizeof desc);
  764. return -EINVAL;
  765. }
  766. ret = translate_desc(dev, desc.addr, desc.len, iov + iov_count,
  767. iov_size - iov_count);
  768. if (unlikely(ret < 0)) {
  769. vq_err(vq, "Translation failure %d indirect idx %d\n",
  770. ret, i);
  771. return ret;
  772. }
  773. /* If this is an input descriptor, increment that count. */
  774. if (desc.flags & VRING_DESC_F_WRITE) {
  775. *in_num += ret;
  776. if (unlikely(log)) {
  777. log[*log_num].addr = desc.addr;
  778. log[*log_num].len = desc.len;
  779. ++*log_num;
  780. }
  781. } else {
  782. /* If it's an output descriptor, they're all supposed
  783. * to come before any input descriptors. */
  784. if (unlikely(*in_num)) {
  785. vq_err(vq, "Indirect descriptor "
  786. "has out after in: idx %d\n", i);
  787. return -EINVAL;
  788. }
  789. *out_num += ret;
  790. }
  791. } while ((i = next_desc(&desc)) != -1);
  792. return 0;
  793. }
  794. /* This looks in the virtqueue and for the first available buffer, and converts
  795. * it to an iovec for convenient access. Since descriptors consist of some
  796. * number of output then some number of input descriptors, it's actually two
  797. * iovecs, but we pack them into one and note how many of each there were.
  798. *
  799. * This function returns the descriptor number found, or vq->num (which is
  800. * never a valid descriptor number) if none was found. A negative code is
  801. * returned on error. */
  802. int vhost_get_vq_desc(struct vhost_dev *dev, struct vhost_virtqueue *vq,
  803. struct iovec iov[], unsigned int iov_size,
  804. unsigned int *out_num, unsigned int *in_num,
  805. struct vhost_log *log, unsigned int *log_num)
  806. {
  807. struct vring_desc desc;
  808. unsigned int i, head, found = 0;
  809. u16 last_avail_idx;
  810. int ret;
  811. /* Check it isn't doing very strange things with descriptor numbers. */
  812. last_avail_idx = vq->last_avail_idx;
  813. if (unlikely(get_user(vq->avail_idx, &vq->avail->idx))) {
  814. vq_err(vq, "Failed to access avail idx at %p\n",
  815. &vq->avail->idx);
  816. return -EFAULT;
  817. }
  818. if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) {
  819. vq_err(vq, "Guest moved used index from %u to %u",
  820. last_avail_idx, vq->avail_idx);
  821. return -EFAULT;
  822. }
  823. /* If there's nothing new since last we looked, return invalid. */
  824. if (vq->avail_idx == last_avail_idx)
  825. return vq->num;
  826. /* Only get avail ring entries after they have been exposed by guest. */
  827. smp_rmb();
  828. /* Grab the next descriptor number they're advertising, and increment
  829. * the index we've seen. */
  830. if (unlikely(get_user(head,
  831. &vq->avail->ring[last_avail_idx % vq->num]))) {
  832. vq_err(vq, "Failed to read head: idx %d address %p\n",
  833. last_avail_idx,
  834. &vq->avail->ring[last_avail_idx % vq->num]);
  835. return -EFAULT;
  836. }
  837. /* If their number is silly, that's an error. */
  838. if (unlikely(head >= vq->num)) {
  839. vq_err(vq, "Guest says index %u > %u is available",
  840. head, vq->num);
  841. return -EINVAL;
  842. }
  843. /* When we start there are none of either input nor output. */
  844. *out_num = *in_num = 0;
  845. if (unlikely(log))
  846. *log_num = 0;
  847. i = head;
  848. do {
  849. unsigned iov_count = *in_num + *out_num;
  850. if (unlikely(i >= vq->num)) {
  851. vq_err(vq, "Desc index is %u > %u, head = %u",
  852. i, vq->num, head);
  853. return -EINVAL;
  854. }
  855. if (unlikely(++found > vq->num)) {
  856. vq_err(vq, "Loop detected: last one at %u "
  857. "vq size %u head %u\n",
  858. i, vq->num, head);
  859. return -EINVAL;
  860. }
  861. ret = copy_from_user(&desc, vq->desc + i, sizeof desc);
  862. if (unlikely(ret)) {
  863. vq_err(vq, "Failed to get descriptor: idx %d addr %p\n",
  864. i, vq->desc + i);
  865. return -EFAULT;
  866. }
  867. if (desc.flags & VRING_DESC_F_INDIRECT) {
  868. ret = get_indirect(dev, vq, iov, iov_size,
  869. out_num, in_num,
  870. log, log_num, &desc);
  871. if (unlikely(ret < 0)) {
  872. vq_err(vq, "Failure detected "
  873. "in indirect descriptor at idx %d\n", i);
  874. return ret;
  875. }
  876. continue;
  877. }
  878. ret = translate_desc(dev, desc.addr, desc.len, iov + iov_count,
  879. iov_size - iov_count);
  880. if (unlikely(ret < 0)) {
  881. vq_err(vq, "Translation failure %d descriptor idx %d\n",
  882. ret, i);
  883. return ret;
  884. }
  885. if (desc.flags & VRING_DESC_F_WRITE) {
  886. /* If this is an input descriptor,
  887. * increment that count. */
  888. *in_num += ret;
  889. if (unlikely(log)) {
  890. log[*log_num].addr = desc.addr;
  891. log[*log_num].len = desc.len;
  892. ++*log_num;
  893. }
  894. } else {
  895. /* If it's an output descriptor, they're all supposed
  896. * to come before any input descriptors. */
  897. if (unlikely(*in_num)) {
  898. vq_err(vq, "Descriptor has out after in: "
  899. "idx %d\n", i);
  900. return -EINVAL;
  901. }
  902. *out_num += ret;
  903. }
  904. } while ((i = next_desc(&desc)) != -1);
  905. /* On success, increment avail index. */
  906. vq->last_avail_idx++;
  907. return head;
  908. }
  909. /* Reverse the effect of vhost_get_vq_desc. Useful for error handling. */
  910. void vhost_discard_vq_desc(struct vhost_virtqueue *vq)
  911. {
  912. vq->last_avail_idx--;
  913. }
  914. /* After we've used one of their buffers, we tell them about it. We'll then
  915. * want to notify the guest, using eventfd. */
  916. int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len)
  917. {
  918. struct vring_used_elem __user *used;
  919. /* The virtqueue contains a ring of used buffers. Get a pointer to the
  920. * next entry in that used ring. */
  921. used = &vq->used->ring[vq->last_used_idx % vq->num];
  922. if (put_user(head, &used->id)) {
  923. vq_err(vq, "Failed to write used id");
  924. return -EFAULT;
  925. }
  926. if (put_user(len, &used->len)) {
  927. vq_err(vq, "Failed to write used len");
  928. return -EFAULT;
  929. }
  930. /* Make sure buffer is written before we update index. */
  931. smp_wmb();
  932. if (put_user(vq->last_used_idx + 1, &vq->used->idx)) {
  933. vq_err(vq, "Failed to increment used idx");
  934. return -EFAULT;
  935. }
  936. if (unlikely(vq->log_used)) {
  937. /* Make sure data is seen before log. */
  938. smp_wmb();
  939. /* Log used ring entry write. */
  940. log_write(vq->log_base,
  941. vq->log_addr +
  942. ((void __user *)used - (void __user *)vq->used),
  943. sizeof *used);
  944. /* Log used index update. */
  945. log_write(vq->log_base,
  946. vq->log_addr + offsetof(struct vring_used, idx),
  947. sizeof vq->used->idx);
  948. if (vq->log_ctx)
  949. eventfd_signal(vq->log_ctx, 1);
  950. }
  951. vq->last_used_idx++;
  952. return 0;
  953. }
  954. /* This actually signals the guest, using eventfd. */
  955. void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq)
  956. {
  957. __u16 flags;
  958. /* Flush out used index updates. This is paired
  959. * with the barrier that the Guest executes when enabling
  960. * interrupts. */
  961. smp_mb();
  962. if (get_user(flags, &vq->avail->flags)) {
  963. vq_err(vq, "Failed to get flags");
  964. return;
  965. }
  966. /* If they don't want an interrupt, don't signal, unless empty. */
  967. if ((flags & VRING_AVAIL_F_NO_INTERRUPT) &&
  968. (vq->avail_idx != vq->last_avail_idx ||
  969. !vhost_has_feature(dev, VIRTIO_F_NOTIFY_ON_EMPTY)))
  970. return;
  971. /* Signal the Guest tell them we used something up. */
  972. if (vq->call_ctx)
  973. eventfd_signal(vq->call_ctx, 1);
  974. }
  975. /* And here's the combo meal deal. Supersize me! */
  976. void vhost_add_used_and_signal(struct vhost_dev *dev,
  977. struct vhost_virtqueue *vq,
  978. unsigned int head, int len)
  979. {
  980. vhost_add_used(vq, head, len);
  981. vhost_signal(dev, vq);
  982. }
  983. /* OK, now we need to know about added descriptors. */
  984. bool vhost_enable_notify(struct vhost_virtqueue *vq)
  985. {
  986. u16 avail_idx;
  987. int r;
  988. if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY))
  989. return false;
  990. vq->used_flags &= ~VRING_USED_F_NO_NOTIFY;
  991. r = put_user(vq->used_flags, &vq->used->flags);
  992. if (r) {
  993. vq_err(vq, "Failed to enable notification at %p: %d\n",
  994. &vq->used->flags, r);
  995. return false;
  996. }
  997. /* They could have slipped one in as we were doing that: make
  998. * sure it's written, then check again. */
  999. smp_mb();
  1000. r = get_user(avail_idx, &vq->avail->idx);
  1001. if (r) {
  1002. vq_err(vq, "Failed to check avail idx at %p: %d\n",
  1003. &vq->avail->idx, r);
  1004. return false;
  1005. }
  1006. return avail_idx != vq->last_avail_idx;
  1007. }
  1008. /* We don't need to be notified again. */
  1009. void vhost_disable_notify(struct vhost_virtqueue *vq)
  1010. {
  1011. int r;
  1012. if (vq->used_flags & VRING_USED_F_NO_NOTIFY)
  1013. return;
  1014. vq->used_flags |= VRING_USED_F_NO_NOTIFY;
  1015. r = put_user(vq->used_flags, &vq->used->flags);
  1016. if (r)
  1017. vq_err(vq, "Failed to enable notification at %p: %d\n",
  1018. &vq->used->flags, r);
  1019. }
  1020. int vhost_init(void)
  1021. {
  1022. vhost_workqueue = create_singlethread_workqueue("vhost");
  1023. if (!vhost_workqueue)
  1024. return -ENOMEM;
  1025. return 0;
  1026. }
  1027. void vhost_cleanup(void)
  1028. {
  1029. destroy_workqueue(vhost_workqueue);
  1030. }