vhost.c 39 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573
  1. /* Copyright (C) 2009 Red Hat, Inc.
  2. * Copyright (C) 2006 Rusty Russell IBM Corporation
  3. *
  4. * Author: Michael S. Tsirkin <mst@redhat.com>
  5. *
  6. * Inspiration, some code, and most witty comments come from
  7. * Documentation/virtual/lguest/lguest.c, by Rusty Russell
  8. *
  9. * This work is licensed under the terms of the GNU GPL, version 2.
  10. *
  11. * Generic code for virtio server in host kernel.
  12. */
  13. #include <linux/eventfd.h>
  14. #include <linux/vhost.h>
  15. #include <linux/virtio_net.h>
  16. #include <linux/mm.h>
  17. #include <linux/mmu_context.h>
  18. #include <linux/miscdevice.h>
  19. #include <linux/mutex.h>
  20. #include <linux/rcupdate.h>
  21. #include <linux/poll.h>
  22. #include <linux/file.h>
  23. #include <linux/highmem.h>
  24. #include <linux/slab.h>
  25. #include <linux/kthread.h>
  26. #include <linux/cgroup.h>
  27. #include "vhost.h"
  28. enum {
  29. VHOST_MEMORY_MAX_NREGIONS = 64,
  30. VHOST_MEMORY_F_LOG = 0x1,
  31. };
  32. static unsigned vhost_zcopy_mask __read_mostly;
  33. #define vhost_used_event(vq) ((u16 __user *)&vq->avail->ring[vq->num])
  34. #define vhost_avail_event(vq) ((u16 __user *)&vq->used->ring[vq->num])
  35. static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh,
  36. poll_table *pt)
  37. {
  38. struct vhost_poll *poll;
  39. poll = container_of(pt, struct vhost_poll, table);
  40. poll->wqh = wqh;
  41. add_wait_queue(wqh, &poll->wait);
  42. }
  43. static int vhost_poll_wakeup(wait_queue_t *wait, unsigned mode, int sync,
  44. void *key)
  45. {
  46. struct vhost_poll *poll = container_of(wait, struct vhost_poll, wait);
  47. if (!((unsigned long)key & poll->mask))
  48. return 0;
  49. vhost_poll_queue(poll);
  50. return 0;
  51. }
  52. void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn)
  53. {
  54. INIT_LIST_HEAD(&work->node);
  55. work->fn = fn;
  56. init_waitqueue_head(&work->done);
  57. work->flushing = 0;
  58. work->queue_seq = work->done_seq = 0;
  59. }
  60. /* Init poll structure */
  61. void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn,
  62. unsigned long mask, struct vhost_dev *dev)
  63. {
  64. init_waitqueue_func_entry(&poll->wait, vhost_poll_wakeup);
  65. init_poll_funcptr(&poll->table, vhost_poll_func);
  66. poll->mask = mask;
  67. poll->dev = dev;
  68. vhost_work_init(&poll->work, fn);
  69. }
  70. /* Start polling a file. We add ourselves to file's wait queue. The caller must
  71. * keep a reference to a file until after vhost_poll_stop is called. */
  72. void vhost_poll_start(struct vhost_poll *poll, struct file *file)
  73. {
  74. unsigned long mask;
  75. mask = file->f_op->poll(file, &poll->table);
  76. if (mask)
  77. vhost_poll_wakeup(&poll->wait, 0, 0, (void *)mask);
  78. }
  79. /* Stop polling a file. After this function returns, it becomes safe to drop the
  80. * file reference. You must also flush afterwards. */
  81. void vhost_poll_stop(struct vhost_poll *poll)
  82. {
  83. remove_wait_queue(poll->wqh, &poll->wait);
  84. }
  85. static bool vhost_work_seq_done(struct vhost_dev *dev, struct vhost_work *work,
  86. unsigned seq)
  87. {
  88. int left;
  89. spin_lock_irq(&dev->work_lock);
  90. left = seq - work->done_seq;
  91. spin_unlock_irq(&dev->work_lock);
  92. return left <= 0;
  93. }
  94. static void vhost_work_flush(struct vhost_dev *dev, struct vhost_work *work)
  95. {
  96. unsigned seq;
  97. int flushing;
  98. spin_lock_irq(&dev->work_lock);
  99. seq = work->queue_seq;
  100. work->flushing++;
  101. spin_unlock_irq(&dev->work_lock);
  102. wait_event(work->done, vhost_work_seq_done(dev, work, seq));
  103. spin_lock_irq(&dev->work_lock);
  104. flushing = --work->flushing;
  105. spin_unlock_irq(&dev->work_lock);
  106. BUG_ON(flushing < 0);
  107. }
  108. /* Flush any work that has been scheduled. When calling this, don't hold any
  109. * locks that are also used by the callback. */
  110. void vhost_poll_flush(struct vhost_poll *poll)
  111. {
  112. vhost_work_flush(poll->dev, &poll->work);
  113. }
  114. void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work)
  115. {
  116. unsigned long flags;
  117. spin_lock_irqsave(&dev->work_lock, flags);
  118. if (list_empty(&work->node)) {
  119. list_add_tail(&work->node, &dev->work_list);
  120. work->queue_seq++;
  121. wake_up_process(dev->worker);
  122. }
  123. spin_unlock_irqrestore(&dev->work_lock, flags);
  124. }
  125. void vhost_poll_queue(struct vhost_poll *poll)
  126. {
  127. vhost_work_queue(poll->dev, &poll->work);
  128. }
  129. static void vhost_vq_reset(struct vhost_dev *dev,
  130. struct vhost_virtqueue *vq)
  131. {
  132. vq->num = 1;
  133. vq->desc = NULL;
  134. vq->avail = NULL;
  135. vq->used = NULL;
  136. vq->last_avail_idx = 0;
  137. vq->avail_idx = 0;
  138. vq->last_used_idx = 0;
  139. vq->signalled_used = 0;
  140. vq->signalled_used_valid = false;
  141. vq->used_flags = 0;
  142. vq->log_used = false;
  143. vq->log_addr = -1ull;
  144. vq->vhost_hlen = 0;
  145. vq->sock_hlen = 0;
  146. vq->private_data = NULL;
  147. vq->log_base = NULL;
  148. vq->error_ctx = NULL;
  149. vq->error = NULL;
  150. vq->kick = NULL;
  151. vq->call_ctx = NULL;
  152. vq->call = NULL;
  153. vq->log_ctx = NULL;
  154. vq->upend_idx = 0;
  155. vq->done_idx = 0;
  156. vq->ubufs = NULL;
  157. }
  158. static int vhost_worker(void *data)
  159. {
  160. struct vhost_dev *dev = data;
  161. struct vhost_work *work = NULL;
  162. unsigned uninitialized_var(seq);
  163. mm_segment_t oldfs = get_fs();
  164. set_fs(USER_DS);
  165. use_mm(dev->mm);
  166. for (;;) {
  167. /* mb paired w/ kthread_stop */
  168. set_current_state(TASK_INTERRUPTIBLE);
  169. spin_lock_irq(&dev->work_lock);
  170. if (work) {
  171. work->done_seq = seq;
  172. if (work->flushing)
  173. wake_up_all(&work->done);
  174. }
  175. if (kthread_should_stop()) {
  176. spin_unlock_irq(&dev->work_lock);
  177. __set_current_state(TASK_RUNNING);
  178. break;
  179. }
  180. if (!list_empty(&dev->work_list)) {
  181. work = list_first_entry(&dev->work_list,
  182. struct vhost_work, node);
  183. list_del_init(&work->node);
  184. seq = work->queue_seq;
  185. } else
  186. work = NULL;
  187. spin_unlock_irq(&dev->work_lock);
  188. if (work) {
  189. __set_current_state(TASK_RUNNING);
  190. work->fn(work);
  191. if (need_resched())
  192. schedule();
  193. } else
  194. schedule();
  195. }
  196. unuse_mm(dev->mm);
  197. set_fs(oldfs);
  198. return 0;
  199. }
  200. static void vhost_vq_free_iovecs(struct vhost_virtqueue *vq)
  201. {
  202. kfree(vq->indirect);
  203. vq->indirect = NULL;
  204. kfree(vq->log);
  205. vq->log = NULL;
  206. kfree(vq->heads);
  207. vq->heads = NULL;
  208. kfree(vq->ubuf_info);
  209. vq->ubuf_info = NULL;
  210. }
  211. void vhost_enable_zcopy(int vq)
  212. {
  213. vhost_zcopy_mask |= 0x1 << vq;
  214. }
  215. /* Helper to allocate iovec buffers for all vqs. */
  216. static long vhost_dev_alloc_iovecs(struct vhost_dev *dev)
  217. {
  218. int i;
  219. bool zcopy;
  220. for (i = 0; i < dev->nvqs; ++i) {
  221. dev->vqs[i].indirect = kmalloc(sizeof *dev->vqs[i].indirect *
  222. UIO_MAXIOV, GFP_KERNEL);
  223. dev->vqs[i].log = kmalloc(sizeof *dev->vqs[i].log * UIO_MAXIOV,
  224. GFP_KERNEL);
  225. dev->vqs[i].heads = kmalloc(sizeof *dev->vqs[i].heads *
  226. UIO_MAXIOV, GFP_KERNEL);
  227. zcopy = vhost_zcopy_mask & (0x1 << i);
  228. if (zcopy)
  229. dev->vqs[i].ubuf_info =
  230. kmalloc(sizeof *dev->vqs[i].ubuf_info *
  231. UIO_MAXIOV, GFP_KERNEL);
  232. if (!dev->vqs[i].indirect || !dev->vqs[i].log ||
  233. !dev->vqs[i].heads ||
  234. (zcopy && !dev->vqs[i].ubuf_info))
  235. goto err_nomem;
  236. }
  237. return 0;
  238. err_nomem:
  239. for (; i >= 0; --i)
  240. vhost_vq_free_iovecs(&dev->vqs[i]);
  241. return -ENOMEM;
  242. }
  243. static void vhost_dev_free_iovecs(struct vhost_dev *dev)
  244. {
  245. int i;
  246. for (i = 0; i < dev->nvqs; ++i)
  247. vhost_vq_free_iovecs(&dev->vqs[i]);
  248. }
  249. long vhost_dev_init(struct vhost_dev *dev,
  250. struct vhost_virtqueue *vqs, int nvqs)
  251. {
  252. int i;
  253. dev->vqs = vqs;
  254. dev->nvqs = nvqs;
  255. mutex_init(&dev->mutex);
  256. dev->log_ctx = NULL;
  257. dev->log_file = NULL;
  258. dev->memory = NULL;
  259. dev->mm = NULL;
  260. spin_lock_init(&dev->work_lock);
  261. INIT_LIST_HEAD(&dev->work_list);
  262. dev->worker = NULL;
  263. for (i = 0; i < dev->nvqs; ++i) {
  264. dev->vqs[i].log = NULL;
  265. dev->vqs[i].indirect = NULL;
  266. dev->vqs[i].heads = NULL;
  267. dev->vqs[i].ubuf_info = NULL;
  268. dev->vqs[i].dev = dev;
  269. mutex_init(&dev->vqs[i].mutex);
  270. vhost_vq_reset(dev, dev->vqs + i);
  271. if (dev->vqs[i].handle_kick)
  272. vhost_poll_init(&dev->vqs[i].poll,
  273. dev->vqs[i].handle_kick, POLLIN, dev);
  274. }
  275. return 0;
  276. }
  277. /* Caller should have device mutex */
  278. long vhost_dev_check_owner(struct vhost_dev *dev)
  279. {
  280. /* Are you the owner? If not, I don't think you mean to do that */
  281. return dev->mm == current->mm ? 0 : -EPERM;
  282. }
  283. struct vhost_attach_cgroups_struct {
  284. struct vhost_work work;
  285. struct task_struct *owner;
  286. int ret;
  287. };
  288. static void vhost_attach_cgroups_work(struct vhost_work *work)
  289. {
  290. struct vhost_attach_cgroups_struct *s;
  291. s = container_of(work, struct vhost_attach_cgroups_struct, work);
  292. s->ret = cgroup_attach_task_all(s->owner, current);
  293. }
  294. static int vhost_attach_cgroups(struct vhost_dev *dev)
  295. {
  296. struct vhost_attach_cgroups_struct attach;
  297. attach.owner = current;
  298. vhost_work_init(&attach.work, vhost_attach_cgroups_work);
  299. vhost_work_queue(dev, &attach.work);
  300. vhost_work_flush(dev, &attach.work);
  301. return attach.ret;
  302. }
  303. /* Caller should have device mutex */
  304. static long vhost_dev_set_owner(struct vhost_dev *dev)
  305. {
  306. struct task_struct *worker;
  307. int err;
  308. /* Is there an owner already? */
  309. if (dev->mm) {
  310. err = -EBUSY;
  311. goto err_mm;
  312. }
  313. /* No owner, become one */
  314. dev->mm = get_task_mm(current);
  315. worker = kthread_create(vhost_worker, dev, "vhost-%d", current->pid);
  316. if (IS_ERR(worker)) {
  317. err = PTR_ERR(worker);
  318. goto err_worker;
  319. }
  320. dev->worker = worker;
  321. wake_up_process(worker); /* avoid contributing to loadavg */
  322. err = vhost_attach_cgroups(dev);
  323. if (err)
  324. goto err_cgroup;
  325. err = vhost_dev_alloc_iovecs(dev);
  326. if (err)
  327. goto err_cgroup;
  328. return 0;
  329. err_cgroup:
  330. kthread_stop(worker);
  331. dev->worker = NULL;
  332. err_worker:
  333. if (dev->mm)
  334. mmput(dev->mm);
  335. dev->mm = NULL;
  336. err_mm:
  337. return err;
  338. }
  339. /* Caller should have device mutex */
  340. long vhost_dev_reset_owner(struct vhost_dev *dev)
  341. {
  342. struct vhost_memory *memory;
  343. /* Restore memory to default empty mapping. */
  344. memory = kmalloc(offsetof(struct vhost_memory, regions), GFP_KERNEL);
  345. if (!memory)
  346. return -ENOMEM;
  347. vhost_dev_cleanup(dev, true);
  348. memory->nregions = 0;
  349. RCU_INIT_POINTER(dev->memory, memory);
  350. return 0;
  351. }
  352. void vhost_dev_stop(struct vhost_dev *dev)
  353. {
  354. int i;
  355. for (i = 0; i < dev->nvqs; ++i) {
  356. if (dev->vqs[i].kick && dev->vqs[i].handle_kick) {
  357. vhost_poll_stop(&dev->vqs[i].poll);
  358. vhost_poll_flush(&dev->vqs[i].poll);
  359. }
  360. }
  361. }
  362. /* Caller should have device mutex if and only if locked is set */
  363. void vhost_dev_cleanup(struct vhost_dev *dev, bool locked)
  364. {
  365. int i;
  366. for (i = 0; i < dev->nvqs; ++i) {
  367. if (dev->vqs[i].error_ctx)
  368. eventfd_ctx_put(dev->vqs[i].error_ctx);
  369. if (dev->vqs[i].error)
  370. fput(dev->vqs[i].error);
  371. if (dev->vqs[i].kick)
  372. fput(dev->vqs[i].kick);
  373. if (dev->vqs[i].call_ctx)
  374. eventfd_ctx_put(dev->vqs[i].call_ctx);
  375. if (dev->vqs[i].call)
  376. fput(dev->vqs[i].call);
  377. vhost_vq_reset(dev, dev->vqs + i);
  378. }
  379. vhost_dev_free_iovecs(dev);
  380. if (dev->log_ctx)
  381. eventfd_ctx_put(dev->log_ctx);
  382. dev->log_ctx = NULL;
  383. if (dev->log_file)
  384. fput(dev->log_file);
  385. dev->log_file = NULL;
  386. /* No one will access memory at this point */
  387. kfree(rcu_dereference_protected(dev->memory,
  388. locked ==
  389. lockdep_is_held(&dev->mutex)));
  390. RCU_INIT_POINTER(dev->memory, NULL);
  391. WARN_ON(!list_empty(&dev->work_list));
  392. if (dev->worker) {
  393. kthread_stop(dev->worker);
  394. dev->worker = NULL;
  395. }
  396. if (dev->mm)
  397. mmput(dev->mm);
  398. dev->mm = NULL;
  399. }
  400. static int log_access_ok(void __user *log_base, u64 addr, unsigned long sz)
  401. {
  402. u64 a = addr / VHOST_PAGE_SIZE / 8;
  403. /* Make sure 64 bit math will not overflow. */
  404. if (a > ULONG_MAX - (unsigned long)log_base ||
  405. a + (unsigned long)log_base > ULONG_MAX)
  406. return 0;
  407. return access_ok(VERIFY_WRITE, log_base + a,
  408. (sz + VHOST_PAGE_SIZE * 8 - 1) / VHOST_PAGE_SIZE / 8);
  409. }
  410. /* Caller should have vq mutex and device mutex. */
  411. static int vq_memory_access_ok(void __user *log_base, struct vhost_memory *mem,
  412. int log_all)
  413. {
  414. int i;
  415. if (!mem)
  416. return 0;
  417. for (i = 0; i < mem->nregions; ++i) {
  418. struct vhost_memory_region *m = mem->regions + i;
  419. unsigned long a = m->userspace_addr;
  420. if (m->memory_size > ULONG_MAX)
  421. return 0;
  422. else if (!access_ok(VERIFY_WRITE, (void __user *)a,
  423. m->memory_size))
  424. return 0;
  425. else if (log_all && !log_access_ok(log_base,
  426. m->guest_phys_addr,
  427. m->memory_size))
  428. return 0;
  429. }
  430. return 1;
  431. }
  432. /* Can we switch to this memory table? */
  433. /* Caller should have device mutex but not vq mutex */
  434. static int memory_access_ok(struct vhost_dev *d, struct vhost_memory *mem,
  435. int log_all)
  436. {
  437. int i;
  438. for (i = 0; i < d->nvqs; ++i) {
  439. int ok;
  440. mutex_lock(&d->vqs[i].mutex);
  441. /* If ring is inactive, will check when it's enabled. */
  442. if (d->vqs[i].private_data)
  443. ok = vq_memory_access_ok(d->vqs[i].log_base, mem,
  444. log_all);
  445. else
  446. ok = 1;
  447. mutex_unlock(&d->vqs[i].mutex);
  448. if (!ok)
  449. return 0;
  450. }
  451. return 1;
  452. }
  453. static int vq_access_ok(struct vhost_dev *d, unsigned int num,
  454. struct vring_desc __user *desc,
  455. struct vring_avail __user *avail,
  456. struct vring_used __user *used)
  457. {
  458. size_t s = vhost_has_feature(d, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
  459. return access_ok(VERIFY_READ, desc, num * sizeof *desc) &&
  460. access_ok(VERIFY_READ, avail,
  461. sizeof *avail + num * sizeof *avail->ring + s) &&
  462. access_ok(VERIFY_WRITE, used,
  463. sizeof *used + num * sizeof *used->ring + s);
  464. }
  465. /* Can we log writes? */
  466. /* Caller should have device mutex but not vq mutex */
  467. int vhost_log_access_ok(struct vhost_dev *dev)
  468. {
  469. struct vhost_memory *mp;
  470. mp = rcu_dereference_protected(dev->memory,
  471. lockdep_is_held(&dev->mutex));
  472. return memory_access_ok(dev, mp, 1);
  473. }
  474. /* Verify access for write logging. */
  475. /* Caller should have vq mutex and device mutex */
  476. static int vq_log_access_ok(struct vhost_dev *d, struct vhost_virtqueue *vq,
  477. void __user *log_base)
  478. {
  479. struct vhost_memory *mp;
  480. size_t s = vhost_has_feature(d, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
  481. mp = rcu_dereference_protected(vq->dev->memory,
  482. lockdep_is_held(&vq->mutex));
  483. return vq_memory_access_ok(log_base, mp,
  484. vhost_has_feature(vq->dev, VHOST_F_LOG_ALL)) &&
  485. (!vq->log_used || log_access_ok(log_base, vq->log_addr,
  486. sizeof *vq->used +
  487. vq->num * sizeof *vq->used->ring + s));
  488. }
  489. /* Can we start vq? */
  490. /* Caller should have vq mutex and device mutex */
  491. int vhost_vq_access_ok(struct vhost_virtqueue *vq)
  492. {
  493. return vq_access_ok(vq->dev, vq->num, vq->desc, vq->avail, vq->used) &&
  494. vq_log_access_ok(vq->dev, vq, vq->log_base);
  495. }
  496. static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m)
  497. {
  498. struct vhost_memory mem, *newmem, *oldmem;
  499. unsigned long size = offsetof(struct vhost_memory, regions);
  500. if (copy_from_user(&mem, m, size))
  501. return -EFAULT;
  502. if (mem.padding)
  503. return -EOPNOTSUPP;
  504. if (mem.nregions > VHOST_MEMORY_MAX_NREGIONS)
  505. return -E2BIG;
  506. newmem = kmalloc(size + mem.nregions * sizeof *m->regions, GFP_KERNEL);
  507. if (!newmem)
  508. return -ENOMEM;
  509. memcpy(newmem, &mem, size);
  510. if (copy_from_user(newmem->regions, m->regions,
  511. mem.nregions * sizeof *m->regions)) {
  512. kfree(newmem);
  513. return -EFAULT;
  514. }
  515. if (!memory_access_ok(d, newmem,
  516. vhost_has_feature(d, VHOST_F_LOG_ALL))) {
  517. kfree(newmem);
  518. return -EFAULT;
  519. }
  520. oldmem = rcu_dereference_protected(d->memory,
  521. lockdep_is_held(&d->mutex));
  522. rcu_assign_pointer(d->memory, newmem);
  523. synchronize_rcu();
  524. kfree(oldmem);
  525. return 0;
  526. }
  527. long vhost_vring_ioctl(struct vhost_dev *d, int ioctl, void __user *argp)
  528. {
  529. struct file *eventfp, *filep = NULL;
  530. bool pollstart = false, pollstop = false;
  531. struct eventfd_ctx *ctx = NULL;
  532. u32 __user *idxp = argp;
  533. struct vhost_virtqueue *vq;
  534. struct vhost_vring_state s;
  535. struct vhost_vring_file f;
  536. struct vhost_vring_addr a;
  537. u32 idx;
  538. long r;
  539. r = get_user(idx, idxp);
  540. if (r < 0)
  541. return r;
  542. if (idx >= d->nvqs)
  543. return -ENOBUFS;
  544. vq = d->vqs + idx;
  545. mutex_lock(&vq->mutex);
  546. switch (ioctl) {
  547. case VHOST_SET_VRING_NUM:
  548. /* Resizing ring with an active backend?
  549. * You don't want to do that. */
  550. if (vq->private_data) {
  551. r = -EBUSY;
  552. break;
  553. }
  554. if (copy_from_user(&s, argp, sizeof s)) {
  555. r = -EFAULT;
  556. break;
  557. }
  558. if (!s.num || s.num > 0xffff || (s.num & (s.num - 1))) {
  559. r = -EINVAL;
  560. break;
  561. }
  562. vq->num = s.num;
  563. break;
  564. case VHOST_SET_VRING_BASE:
  565. /* Moving base with an active backend?
  566. * You don't want to do that. */
  567. if (vq->private_data) {
  568. r = -EBUSY;
  569. break;
  570. }
  571. if (copy_from_user(&s, argp, sizeof s)) {
  572. r = -EFAULT;
  573. break;
  574. }
  575. if (s.num > 0xffff) {
  576. r = -EINVAL;
  577. break;
  578. }
  579. vq->last_avail_idx = s.num;
  580. /* Forget the cached index value. */
  581. vq->avail_idx = vq->last_avail_idx;
  582. break;
  583. case VHOST_GET_VRING_BASE:
  584. s.index = idx;
  585. s.num = vq->last_avail_idx;
  586. if (copy_to_user(argp, &s, sizeof s))
  587. r = -EFAULT;
  588. break;
  589. case VHOST_SET_VRING_ADDR:
  590. if (copy_from_user(&a, argp, sizeof a)) {
  591. r = -EFAULT;
  592. break;
  593. }
  594. if (a.flags & ~(0x1 << VHOST_VRING_F_LOG)) {
  595. r = -EOPNOTSUPP;
  596. break;
  597. }
  598. /* For 32bit, verify that the top 32bits of the user
  599. data are set to zero. */
  600. if ((u64)(unsigned long)a.desc_user_addr != a.desc_user_addr ||
  601. (u64)(unsigned long)a.used_user_addr != a.used_user_addr ||
  602. (u64)(unsigned long)a.avail_user_addr != a.avail_user_addr) {
  603. r = -EFAULT;
  604. break;
  605. }
  606. if ((a.avail_user_addr & (sizeof *vq->avail->ring - 1)) ||
  607. (a.used_user_addr & (sizeof *vq->used->ring - 1)) ||
  608. (a.log_guest_addr & (sizeof *vq->used->ring - 1))) {
  609. r = -EINVAL;
  610. break;
  611. }
  612. /* We only verify access here if backend is configured.
  613. * If it is not, we don't as size might not have been setup.
  614. * We will verify when backend is configured. */
  615. if (vq->private_data) {
  616. if (!vq_access_ok(d, vq->num,
  617. (void __user *)(unsigned long)a.desc_user_addr,
  618. (void __user *)(unsigned long)a.avail_user_addr,
  619. (void __user *)(unsigned long)a.used_user_addr)) {
  620. r = -EINVAL;
  621. break;
  622. }
  623. /* Also validate log access for used ring if enabled. */
  624. if ((a.flags & (0x1 << VHOST_VRING_F_LOG)) &&
  625. !log_access_ok(vq->log_base, a.log_guest_addr,
  626. sizeof *vq->used +
  627. vq->num * sizeof *vq->used->ring)) {
  628. r = -EINVAL;
  629. break;
  630. }
  631. }
  632. vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG));
  633. vq->desc = (void __user *)(unsigned long)a.desc_user_addr;
  634. vq->avail = (void __user *)(unsigned long)a.avail_user_addr;
  635. vq->log_addr = a.log_guest_addr;
  636. vq->used = (void __user *)(unsigned long)a.used_user_addr;
  637. break;
  638. case VHOST_SET_VRING_KICK:
  639. if (copy_from_user(&f, argp, sizeof f)) {
  640. r = -EFAULT;
  641. break;
  642. }
  643. eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd);
  644. if (IS_ERR(eventfp)) {
  645. r = PTR_ERR(eventfp);
  646. break;
  647. }
  648. if (eventfp != vq->kick) {
  649. pollstop = (filep = vq->kick) != NULL;
  650. pollstart = (vq->kick = eventfp) != NULL;
  651. } else
  652. filep = eventfp;
  653. break;
  654. case VHOST_SET_VRING_CALL:
  655. if (copy_from_user(&f, argp, sizeof f)) {
  656. r = -EFAULT;
  657. break;
  658. }
  659. eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd);
  660. if (IS_ERR(eventfp)) {
  661. r = PTR_ERR(eventfp);
  662. break;
  663. }
  664. if (eventfp != vq->call) {
  665. filep = vq->call;
  666. ctx = vq->call_ctx;
  667. vq->call = eventfp;
  668. vq->call_ctx = eventfp ?
  669. eventfd_ctx_fileget(eventfp) : NULL;
  670. } else
  671. filep = eventfp;
  672. break;
  673. case VHOST_SET_VRING_ERR:
  674. if (copy_from_user(&f, argp, sizeof f)) {
  675. r = -EFAULT;
  676. break;
  677. }
  678. eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd);
  679. if (IS_ERR(eventfp)) {
  680. r = PTR_ERR(eventfp);
  681. break;
  682. }
  683. if (eventfp != vq->error) {
  684. filep = vq->error;
  685. vq->error = eventfp;
  686. ctx = vq->error_ctx;
  687. vq->error_ctx = eventfp ?
  688. eventfd_ctx_fileget(eventfp) : NULL;
  689. } else
  690. filep = eventfp;
  691. break;
  692. default:
  693. r = -ENOIOCTLCMD;
  694. }
  695. if (pollstop && vq->handle_kick)
  696. vhost_poll_stop(&vq->poll);
  697. if (ctx)
  698. eventfd_ctx_put(ctx);
  699. if (filep)
  700. fput(filep);
  701. if (pollstart && vq->handle_kick)
  702. vhost_poll_start(&vq->poll, vq->kick);
  703. mutex_unlock(&vq->mutex);
  704. if (pollstop && vq->handle_kick)
  705. vhost_poll_flush(&vq->poll);
  706. return r;
  707. }
  708. /* Caller must have device mutex */
  709. long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp)
  710. {
  711. struct file *eventfp, *filep = NULL;
  712. struct eventfd_ctx *ctx = NULL;
  713. u64 p;
  714. long r;
  715. int i, fd;
  716. /* If you are not the owner, you can become one */
  717. if (ioctl == VHOST_SET_OWNER) {
  718. r = vhost_dev_set_owner(d);
  719. goto done;
  720. }
  721. /* You must be the owner to do anything else */
  722. r = vhost_dev_check_owner(d);
  723. if (r)
  724. goto done;
  725. switch (ioctl) {
  726. case VHOST_SET_MEM_TABLE:
  727. r = vhost_set_memory(d, argp);
  728. break;
  729. case VHOST_SET_LOG_BASE:
  730. if (copy_from_user(&p, argp, sizeof p)) {
  731. r = -EFAULT;
  732. break;
  733. }
  734. if ((u64)(unsigned long)p != p) {
  735. r = -EFAULT;
  736. break;
  737. }
  738. for (i = 0; i < d->nvqs; ++i) {
  739. struct vhost_virtqueue *vq;
  740. void __user *base = (void __user *)(unsigned long)p;
  741. vq = d->vqs + i;
  742. mutex_lock(&vq->mutex);
  743. /* If ring is inactive, will check when it's enabled. */
  744. if (vq->private_data && !vq_log_access_ok(d, vq, base))
  745. r = -EFAULT;
  746. else
  747. vq->log_base = base;
  748. mutex_unlock(&vq->mutex);
  749. }
  750. break;
  751. case VHOST_SET_LOG_FD:
  752. r = get_user(fd, (int __user *)argp);
  753. if (r < 0)
  754. break;
  755. eventfp = fd == -1 ? NULL : eventfd_fget(fd);
  756. if (IS_ERR(eventfp)) {
  757. r = PTR_ERR(eventfp);
  758. break;
  759. }
  760. if (eventfp != d->log_file) {
  761. filep = d->log_file;
  762. ctx = d->log_ctx;
  763. d->log_ctx = eventfp ?
  764. eventfd_ctx_fileget(eventfp) : NULL;
  765. } else
  766. filep = eventfp;
  767. for (i = 0; i < d->nvqs; ++i) {
  768. mutex_lock(&d->vqs[i].mutex);
  769. d->vqs[i].log_ctx = d->log_ctx;
  770. mutex_unlock(&d->vqs[i].mutex);
  771. }
  772. if (ctx)
  773. eventfd_ctx_put(ctx);
  774. if (filep)
  775. fput(filep);
  776. break;
  777. default:
  778. r = -ENOIOCTLCMD;
  779. break;
  780. }
  781. done:
  782. return r;
  783. }
  784. static const struct vhost_memory_region *find_region(struct vhost_memory *mem,
  785. __u64 addr, __u32 len)
  786. {
  787. struct vhost_memory_region *reg;
  788. int i;
  789. /* linear search is not brilliant, but we really have on the order of 6
  790. * regions in practice */
  791. for (i = 0; i < mem->nregions; ++i) {
  792. reg = mem->regions + i;
  793. if (reg->guest_phys_addr <= addr &&
  794. reg->guest_phys_addr + reg->memory_size - 1 >= addr)
  795. return reg;
  796. }
  797. return NULL;
  798. }
  799. /* TODO: This is really inefficient. We need something like get_user()
  800. * (instruction directly accesses the data, with an exception table entry
  801. * returning -EFAULT). See Documentation/x86/exception-tables.txt.
  802. */
  803. static int set_bit_to_user(int nr, void __user *addr)
  804. {
  805. unsigned long log = (unsigned long)addr;
  806. struct page *page;
  807. void *base;
  808. int bit = nr + (log % PAGE_SIZE) * 8;
  809. int r;
  810. r = get_user_pages_fast(log, 1, 1, &page);
  811. if (r < 0)
  812. return r;
  813. BUG_ON(r != 1);
  814. base = kmap_atomic(page);
  815. set_bit(bit, base);
  816. kunmap_atomic(base);
  817. set_page_dirty_lock(page);
  818. put_page(page);
  819. return 0;
  820. }
  821. static int log_write(void __user *log_base,
  822. u64 write_address, u64 write_length)
  823. {
  824. u64 write_page = write_address / VHOST_PAGE_SIZE;
  825. int r;
  826. if (!write_length)
  827. return 0;
  828. write_length += write_address % VHOST_PAGE_SIZE;
  829. for (;;) {
  830. u64 base = (u64)(unsigned long)log_base;
  831. u64 log = base + write_page / 8;
  832. int bit = write_page % 8;
  833. if ((u64)(unsigned long)log != log)
  834. return -EFAULT;
  835. r = set_bit_to_user(bit, (void __user *)(unsigned long)log);
  836. if (r < 0)
  837. return r;
  838. if (write_length <= VHOST_PAGE_SIZE)
  839. break;
  840. write_length -= VHOST_PAGE_SIZE;
  841. write_page += 1;
  842. }
  843. return r;
  844. }
  845. int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log,
  846. unsigned int log_num, u64 len)
  847. {
  848. int i, r;
  849. /* Make sure data written is seen before log. */
  850. smp_wmb();
  851. for (i = 0; i < log_num; ++i) {
  852. u64 l = min(log[i].len, len);
  853. r = log_write(vq->log_base, log[i].addr, l);
  854. if (r < 0)
  855. return r;
  856. len -= l;
  857. if (!len) {
  858. if (vq->log_ctx)
  859. eventfd_signal(vq->log_ctx, 1);
  860. return 0;
  861. }
  862. }
  863. /* Length written exceeds what we have stored. This is a bug. */
  864. BUG();
  865. return 0;
  866. }
  867. static int vhost_update_used_flags(struct vhost_virtqueue *vq)
  868. {
  869. void __user *used;
  870. if (__put_user(vq->used_flags, &vq->used->flags) < 0)
  871. return -EFAULT;
  872. if (unlikely(vq->log_used)) {
  873. /* Make sure the flag is seen before log. */
  874. smp_wmb();
  875. /* Log used flag write. */
  876. used = &vq->used->flags;
  877. log_write(vq->log_base, vq->log_addr +
  878. (used - (void __user *)vq->used),
  879. sizeof vq->used->flags);
  880. if (vq->log_ctx)
  881. eventfd_signal(vq->log_ctx, 1);
  882. }
  883. return 0;
  884. }
  885. static int vhost_update_avail_event(struct vhost_virtqueue *vq, u16 avail_event)
  886. {
  887. if (__put_user(vq->avail_idx, vhost_avail_event(vq)))
  888. return -EFAULT;
  889. if (unlikely(vq->log_used)) {
  890. void __user *used;
  891. /* Make sure the event is seen before log. */
  892. smp_wmb();
  893. /* Log avail event write */
  894. used = vhost_avail_event(vq);
  895. log_write(vq->log_base, vq->log_addr +
  896. (used - (void __user *)vq->used),
  897. sizeof *vhost_avail_event(vq));
  898. if (vq->log_ctx)
  899. eventfd_signal(vq->log_ctx, 1);
  900. }
  901. return 0;
  902. }
  903. int vhost_init_used(struct vhost_virtqueue *vq)
  904. {
  905. int r;
  906. if (!vq->private_data)
  907. return 0;
  908. r = vhost_update_used_flags(vq);
  909. if (r)
  910. return r;
  911. vq->signalled_used_valid = false;
  912. return get_user(vq->last_used_idx, &vq->used->idx);
  913. }
  914. static int translate_desc(struct vhost_dev *dev, u64 addr, u32 len,
  915. struct iovec iov[], int iov_size)
  916. {
  917. const struct vhost_memory_region *reg;
  918. struct vhost_memory *mem;
  919. struct iovec *_iov;
  920. u64 s = 0;
  921. int ret = 0;
  922. rcu_read_lock();
  923. mem = rcu_dereference(dev->memory);
  924. while ((u64)len > s) {
  925. u64 size;
  926. if (unlikely(ret >= iov_size)) {
  927. ret = -ENOBUFS;
  928. break;
  929. }
  930. reg = find_region(mem, addr, len);
  931. if (unlikely(!reg)) {
  932. ret = -EFAULT;
  933. break;
  934. }
  935. _iov = iov + ret;
  936. size = reg->memory_size - addr + reg->guest_phys_addr;
  937. _iov->iov_len = min((u64)len - s, size);
  938. _iov->iov_base = (void __user *)(unsigned long)
  939. (reg->userspace_addr + addr - reg->guest_phys_addr);
  940. s += size;
  941. addr += size;
  942. ++ret;
  943. }
  944. rcu_read_unlock();
  945. return ret;
  946. }
  947. /* Each buffer in the virtqueues is actually a chain of descriptors. This
  948. * function returns the next descriptor in the chain,
  949. * or -1U if we're at the end. */
  950. static unsigned next_desc(struct vring_desc *desc)
  951. {
  952. unsigned int next;
  953. /* If this descriptor says it doesn't chain, we're done. */
  954. if (!(desc->flags & VRING_DESC_F_NEXT))
  955. return -1U;
  956. /* Check they're not leading us off end of descriptors. */
  957. next = desc->next;
  958. /* Make sure compiler knows to grab that: we don't want it changing! */
  959. /* We will use the result as an index in an array, so most
  960. * architectures only need a compiler barrier here. */
  961. read_barrier_depends();
  962. return next;
  963. }
  964. static int get_indirect(struct vhost_dev *dev, struct vhost_virtqueue *vq,
  965. struct iovec iov[], unsigned int iov_size,
  966. unsigned int *out_num, unsigned int *in_num,
  967. struct vhost_log *log, unsigned int *log_num,
  968. struct vring_desc *indirect)
  969. {
  970. struct vring_desc desc;
  971. unsigned int i = 0, count, found = 0;
  972. int ret;
  973. /* Sanity check */
  974. if (unlikely(indirect->len % sizeof desc)) {
  975. vq_err(vq, "Invalid length in indirect descriptor: "
  976. "len 0x%llx not multiple of 0x%zx\n",
  977. (unsigned long long)indirect->len,
  978. sizeof desc);
  979. return -EINVAL;
  980. }
  981. ret = translate_desc(dev, indirect->addr, indirect->len, vq->indirect,
  982. UIO_MAXIOV);
  983. if (unlikely(ret < 0)) {
  984. vq_err(vq, "Translation failure %d in indirect.\n", ret);
  985. return ret;
  986. }
  987. /* We will use the result as an address to read from, so most
  988. * architectures only need a compiler barrier here. */
  989. read_barrier_depends();
  990. count = indirect->len / sizeof desc;
  991. /* Buffers are chained via a 16 bit next field, so
  992. * we can have at most 2^16 of these. */
  993. if (unlikely(count > USHRT_MAX + 1)) {
  994. vq_err(vq, "Indirect buffer length too big: %d\n",
  995. indirect->len);
  996. return -E2BIG;
  997. }
  998. do {
  999. unsigned iov_count = *in_num + *out_num;
  1000. if (unlikely(++found > count)) {
  1001. vq_err(vq, "Loop detected: last one at %u "
  1002. "indirect size %u\n",
  1003. i, count);
  1004. return -EINVAL;
  1005. }
  1006. if (unlikely(memcpy_fromiovec((unsigned char *)&desc,
  1007. vq->indirect, sizeof desc))) {
  1008. vq_err(vq, "Failed indirect descriptor: idx %d, %zx\n",
  1009. i, (size_t)indirect->addr + i * sizeof desc);
  1010. return -EINVAL;
  1011. }
  1012. if (unlikely(desc.flags & VRING_DESC_F_INDIRECT)) {
  1013. vq_err(vq, "Nested indirect descriptor: idx %d, %zx\n",
  1014. i, (size_t)indirect->addr + i * sizeof desc);
  1015. return -EINVAL;
  1016. }
  1017. ret = translate_desc(dev, desc.addr, desc.len, iov + iov_count,
  1018. iov_size - iov_count);
  1019. if (unlikely(ret < 0)) {
  1020. vq_err(vq, "Translation failure %d indirect idx %d\n",
  1021. ret, i);
  1022. return ret;
  1023. }
  1024. /* If this is an input descriptor, increment that count. */
  1025. if (desc.flags & VRING_DESC_F_WRITE) {
  1026. *in_num += ret;
  1027. if (unlikely(log)) {
  1028. log[*log_num].addr = desc.addr;
  1029. log[*log_num].len = desc.len;
  1030. ++*log_num;
  1031. }
  1032. } else {
  1033. /* If it's an output descriptor, they're all supposed
  1034. * to come before any input descriptors. */
  1035. if (unlikely(*in_num)) {
  1036. vq_err(vq, "Indirect descriptor "
  1037. "has out after in: idx %d\n", i);
  1038. return -EINVAL;
  1039. }
  1040. *out_num += ret;
  1041. }
  1042. } while ((i = next_desc(&desc)) != -1);
  1043. return 0;
  1044. }
  1045. /* This looks in the virtqueue and for the first available buffer, and converts
  1046. * it to an iovec for convenient access. Since descriptors consist of some
  1047. * number of output then some number of input descriptors, it's actually two
  1048. * iovecs, but we pack them into one and note how many of each there were.
  1049. *
  1050. * This function returns the descriptor number found, or vq->num (which is
  1051. * never a valid descriptor number) if none was found. A negative code is
  1052. * returned on error. */
  1053. int vhost_get_vq_desc(struct vhost_dev *dev, struct vhost_virtqueue *vq,
  1054. struct iovec iov[], unsigned int iov_size,
  1055. unsigned int *out_num, unsigned int *in_num,
  1056. struct vhost_log *log, unsigned int *log_num)
  1057. {
  1058. struct vring_desc desc;
  1059. unsigned int i, head, found = 0;
  1060. u16 last_avail_idx;
  1061. int ret;
  1062. /* Check it isn't doing very strange things with descriptor numbers. */
  1063. last_avail_idx = vq->last_avail_idx;
  1064. if (unlikely(__get_user(vq->avail_idx, &vq->avail->idx))) {
  1065. vq_err(vq, "Failed to access avail idx at %p\n",
  1066. &vq->avail->idx);
  1067. return -EFAULT;
  1068. }
  1069. if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) {
  1070. vq_err(vq, "Guest moved used index from %u to %u",
  1071. last_avail_idx, vq->avail_idx);
  1072. return -EFAULT;
  1073. }
  1074. /* If there's nothing new since last we looked, return invalid. */
  1075. if (vq->avail_idx == last_avail_idx)
  1076. return vq->num;
  1077. /* Only get avail ring entries after they have been exposed by guest. */
  1078. smp_rmb();
  1079. /* Grab the next descriptor number they're advertising, and increment
  1080. * the index we've seen. */
  1081. if (unlikely(__get_user(head,
  1082. &vq->avail->ring[last_avail_idx % vq->num]))) {
  1083. vq_err(vq, "Failed to read head: idx %d address %p\n",
  1084. last_avail_idx,
  1085. &vq->avail->ring[last_avail_idx % vq->num]);
  1086. return -EFAULT;
  1087. }
  1088. /* If their number is silly, that's an error. */
  1089. if (unlikely(head >= vq->num)) {
  1090. vq_err(vq, "Guest says index %u > %u is available",
  1091. head, vq->num);
  1092. return -EINVAL;
  1093. }
  1094. /* When we start there are none of either input nor output. */
  1095. *out_num = *in_num = 0;
  1096. if (unlikely(log))
  1097. *log_num = 0;
  1098. i = head;
  1099. do {
  1100. unsigned iov_count = *in_num + *out_num;
  1101. if (unlikely(i >= vq->num)) {
  1102. vq_err(vq, "Desc index is %u > %u, head = %u",
  1103. i, vq->num, head);
  1104. return -EINVAL;
  1105. }
  1106. if (unlikely(++found > vq->num)) {
  1107. vq_err(vq, "Loop detected: last one at %u "
  1108. "vq size %u head %u\n",
  1109. i, vq->num, head);
  1110. return -EINVAL;
  1111. }
  1112. ret = __copy_from_user(&desc, vq->desc + i, sizeof desc);
  1113. if (unlikely(ret)) {
  1114. vq_err(vq, "Failed to get descriptor: idx %d addr %p\n",
  1115. i, vq->desc + i);
  1116. return -EFAULT;
  1117. }
  1118. if (desc.flags & VRING_DESC_F_INDIRECT) {
  1119. ret = get_indirect(dev, vq, iov, iov_size,
  1120. out_num, in_num,
  1121. log, log_num, &desc);
  1122. if (unlikely(ret < 0)) {
  1123. vq_err(vq, "Failure detected "
  1124. "in indirect descriptor at idx %d\n", i);
  1125. return ret;
  1126. }
  1127. continue;
  1128. }
  1129. ret = translate_desc(dev, desc.addr, desc.len, iov + iov_count,
  1130. iov_size - iov_count);
  1131. if (unlikely(ret < 0)) {
  1132. vq_err(vq, "Translation failure %d descriptor idx %d\n",
  1133. ret, i);
  1134. return ret;
  1135. }
  1136. if (desc.flags & VRING_DESC_F_WRITE) {
  1137. /* If this is an input descriptor,
  1138. * increment that count. */
  1139. *in_num += ret;
  1140. if (unlikely(log)) {
  1141. log[*log_num].addr = desc.addr;
  1142. log[*log_num].len = desc.len;
  1143. ++*log_num;
  1144. }
  1145. } else {
  1146. /* If it's an output descriptor, they're all supposed
  1147. * to come before any input descriptors. */
  1148. if (unlikely(*in_num)) {
  1149. vq_err(vq, "Descriptor has out after in: "
  1150. "idx %d\n", i);
  1151. return -EINVAL;
  1152. }
  1153. *out_num += ret;
  1154. }
  1155. } while ((i = next_desc(&desc)) != -1);
  1156. /* On success, increment avail index. */
  1157. vq->last_avail_idx++;
  1158. /* Assume notifications from guest are disabled at this point,
  1159. * if they aren't we would need to update avail_event index. */
  1160. BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY));
  1161. return head;
  1162. }
  1163. /* Reverse the effect of vhost_get_vq_desc. Useful for error handling. */
  1164. void vhost_discard_vq_desc(struct vhost_virtqueue *vq, int n)
  1165. {
  1166. vq->last_avail_idx -= n;
  1167. }
  1168. /* After we've used one of their buffers, we tell them about it. We'll then
  1169. * want to notify the guest, using eventfd. */
  1170. int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len)
  1171. {
  1172. struct vring_used_elem __user *used;
  1173. /* The virtqueue contains a ring of used buffers. Get a pointer to the
  1174. * next entry in that used ring. */
  1175. used = &vq->used->ring[vq->last_used_idx % vq->num];
  1176. if (__put_user(head, &used->id)) {
  1177. vq_err(vq, "Failed to write used id");
  1178. return -EFAULT;
  1179. }
  1180. if (__put_user(len, &used->len)) {
  1181. vq_err(vq, "Failed to write used len");
  1182. return -EFAULT;
  1183. }
  1184. /* Make sure buffer is written before we update index. */
  1185. smp_wmb();
  1186. if (__put_user(vq->last_used_idx + 1, &vq->used->idx)) {
  1187. vq_err(vq, "Failed to increment used idx");
  1188. return -EFAULT;
  1189. }
  1190. if (unlikely(vq->log_used)) {
  1191. /* Make sure data is seen before log. */
  1192. smp_wmb();
  1193. /* Log used ring entry write. */
  1194. log_write(vq->log_base,
  1195. vq->log_addr +
  1196. ((void __user *)used - (void __user *)vq->used),
  1197. sizeof *used);
  1198. /* Log used index update. */
  1199. log_write(vq->log_base,
  1200. vq->log_addr + offsetof(struct vring_used, idx),
  1201. sizeof vq->used->idx);
  1202. if (vq->log_ctx)
  1203. eventfd_signal(vq->log_ctx, 1);
  1204. }
  1205. vq->last_used_idx++;
  1206. /* If the driver never bothers to signal in a very long while,
  1207. * used index might wrap around. If that happens, invalidate
  1208. * signalled_used index we stored. TODO: make sure driver
  1209. * signals at least once in 2^16 and remove this. */
  1210. if (unlikely(vq->last_used_idx == vq->signalled_used))
  1211. vq->signalled_used_valid = false;
  1212. return 0;
  1213. }
  1214. static int __vhost_add_used_n(struct vhost_virtqueue *vq,
  1215. struct vring_used_elem *heads,
  1216. unsigned count)
  1217. {
  1218. struct vring_used_elem __user *used;
  1219. u16 old, new;
  1220. int start;
  1221. start = vq->last_used_idx % vq->num;
  1222. used = vq->used->ring + start;
  1223. if (__copy_to_user(used, heads, count * sizeof *used)) {
  1224. vq_err(vq, "Failed to write used");
  1225. return -EFAULT;
  1226. }
  1227. if (unlikely(vq->log_used)) {
  1228. /* Make sure data is seen before log. */
  1229. smp_wmb();
  1230. /* Log used ring entry write. */
  1231. log_write(vq->log_base,
  1232. vq->log_addr +
  1233. ((void __user *)used - (void __user *)vq->used),
  1234. count * sizeof *used);
  1235. }
  1236. old = vq->last_used_idx;
  1237. new = (vq->last_used_idx += count);
  1238. /* If the driver never bothers to signal in a very long while,
  1239. * used index might wrap around. If that happens, invalidate
  1240. * signalled_used index we stored. TODO: make sure driver
  1241. * signals at least once in 2^16 and remove this. */
  1242. if (unlikely((u16)(new - vq->signalled_used) < (u16)(new - old)))
  1243. vq->signalled_used_valid = false;
  1244. return 0;
  1245. }
  1246. /* After we've used one of their buffers, we tell them about it. We'll then
  1247. * want to notify the guest, using eventfd. */
  1248. int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads,
  1249. unsigned count)
  1250. {
  1251. int start, n, r;
  1252. start = vq->last_used_idx % vq->num;
  1253. n = vq->num - start;
  1254. if (n < count) {
  1255. r = __vhost_add_used_n(vq, heads, n);
  1256. if (r < 0)
  1257. return r;
  1258. heads += n;
  1259. count -= n;
  1260. }
  1261. r = __vhost_add_used_n(vq, heads, count);
  1262. /* Make sure buffer is written before we update index. */
  1263. smp_wmb();
  1264. if (put_user(vq->last_used_idx, &vq->used->idx)) {
  1265. vq_err(vq, "Failed to increment used idx");
  1266. return -EFAULT;
  1267. }
  1268. if (unlikely(vq->log_used)) {
  1269. /* Log used index update. */
  1270. log_write(vq->log_base,
  1271. vq->log_addr + offsetof(struct vring_used, idx),
  1272. sizeof vq->used->idx);
  1273. if (vq->log_ctx)
  1274. eventfd_signal(vq->log_ctx, 1);
  1275. }
  1276. return r;
  1277. }
  1278. static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
  1279. {
  1280. __u16 old, new, event;
  1281. bool v;
  1282. /* Flush out used index updates. This is paired
  1283. * with the barrier that the Guest executes when enabling
  1284. * interrupts. */
  1285. smp_mb();
  1286. if (vhost_has_feature(dev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
  1287. unlikely(vq->avail_idx == vq->last_avail_idx))
  1288. return true;
  1289. if (!vhost_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
  1290. __u16 flags;
  1291. if (__get_user(flags, &vq->avail->flags)) {
  1292. vq_err(vq, "Failed to get flags");
  1293. return true;
  1294. }
  1295. return !(flags & VRING_AVAIL_F_NO_INTERRUPT);
  1296. }
  1297. old = vq->signalled_used;
  1298. v = vq->signalled_used_valid;
  1299. new = vq->signalled_used = vq->last_used_idx;
  1300. vq->signalled_used_valid = true;
  1301. if (unlikely(!v))
  1302. return true;
  1303. if (get_user(event, vhost_used_event(vq))) {
  1304. vq_err(vq, "Failed to get used event idx");
  1305. return true;
  1306. }
  1307. return vring_need_event(event, new, old);
  1308. }
  1309. /* This actually signals the guest, using eventfd. */
  1310. void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq)
  1311. {
  1312. /* Signal the Guest tell them we used something up. */
  1313. if (vq->call_ctx && vhost_notify(dev, vq))
  1314. eventfd_signal(vq->call_ctx, 1);
  1315. }
  1316. /* And here's the combo meal deal. Supersize me! */
  1317. void vhost_add_used_and_signal(struct vhost_dev *dev,
  1318. struct vhost_virtqueue *vq,
  1319. unsigned int head, int len)
  1320. {
  1321. vhost_add_used(vq, head, len);
  1322. vhost_signal(dev, vq);
  1323. }
  1324. /* multi-buffer version of vhost_add_used_and_signal */
  1325. void vhost_add_used_and_signal_n(struct vhost_dev *dev,
  1326. struct vhost_virtqueue *vq,
  1327. struct vring_used_elem *heads, unsigned count)
  1328. {
  1329. vhost_add_used_n(vq, heads, count);
  1330. vhost_signal(dev, vq);
  1331. }
  1332. /* OK, now we need to know about added descriptors. */
  1333. bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
  1334. {
  1335. u16 avail_idx;
  1336. int r;
  1337. if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY))
  1338. return false;
  1339. vq->used_flags &= ~VRING_USED_F_NO_NOTIFY;
  1340. if (!vhost_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
  1341. r = vhost_update_used_flags(vq);
  1342. if (r) {
  1343. vq_err(vq, "Failed to enable notification at %p: %d\n",
  1344. &vq->used->flags, r);
  1345. return false;
  1346. }
  1347. } else {
  1348. r = vhost_update_avail_event(vq, vq->avail_idx);
  1349. if (r) {
  1350. vq_err(vq, "Failed to update avail event index at %p: %d\n",
  1351. vhost_avail_event(vq), r);
  1352. return false;
  1353. }
  1354. }
  1355. /* They could have slipped one in as we were doing that: make
  1356. * sure it's written, then check again. */
  1357. smp_mb();
  1358. r = __get_user(avail_idx, &vq->avail->idx);
  1359. if (r) {
  1360. vq_err(vq, "Failed to check avail idx at %p: %d\n",
  1361. &vq->avail->idx, r);
  1362. return false;
  1363. }
  1364. return avail_idx != vq->avail_idx;
  1365. }
  1366. /* We don't need to be notified again. */
  1367. void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
  1368. {
  1369. int r;
  1370. if (vq->used_flags & VRING_USED_F_NO_NOTIFY)
  1371. return;
  1372. vq->used_flags |= VRING_USED_F_NO_NOTIFY;
  1373. if (!vhost_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
  1374. r = vhost_update_used_flags(vq);
  1375. if (r)
  1376. vq_err(vq, "Failed to enable notification at %p: %d\n",
  1377. &vq->used->flags, r);
  1378. }
  1379. }
  1380. static void vhost_zerocopy_done_signal(struct kref *kref)
  1381. {
  1382. struct vhost_ubuf_ref *ubufs = container_of(kref, struct vhost_ubuf_ref,
  1383. kref);
  1384. wake_up(&ubufs->wait);
  1385. }
  1386. struct vhost_ubuf_ref *vhost_ubuf_alloc(struct vhost_virtqueue *vq,
  1387. bool zcopy)
  1388. {
  1389. struct vhost_ubuf_ref *ubufs;
  1390. /* No zero copy backend? Nothing to count. */
  1391. if (!zcopy)
  1392. return NULL;
  1393. ubufs = kmalloc(sizeof *ubufs, GFP_KERNEL);
  1394. if (!ubufs)
  1395. return ERR_PTR(-ENOMEM);
  1396. kref_init(&ubufs->kref);
  1397. init_waitqueue_head(&ubufs->wait);
  1398. ubufs->vq = vq;
  1399. return ubufs;
  1400. }
  1401. void vhost_ubuf_put(struct vhost_ubuf_ref *ubufs)
  1402. {
  1403. kref_put(&ubufs->kref, vhost_zerocopy_done_signal);
  1404. }
  1405. void vhost_ubuf_put_and_wait(struct vhost_ubuf_ref *ubufs)
  1406. {
  1407. kref_put(&ubufs->kref, vhost_zerocopy_done_signal);
  1408. wait_event(ubufs->wait, !atomic_read(&ubufs->kref.refcount));
  1409. kfree(ubufs);
  1410. }