vhost.c 41 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613
  1. /* Copyright (C) 2009 Red Hat, Inc.
  2. * Copyright (C) 2006 Rusty Russell IBM Corporation
  3. *
  4. * Author: Michael S. Tsirkin <mst@redhat.com>
  5. *
  6. * Inspiration, some code, and most witty comments come from
  7. * Documentation/virtual/lguest/lguest.c, by Rusty Russell
  8. *
  9. * This work is licensed under the terms of the GNU GPL, version 2.
  10. *
  11. * Generic code for virtio server in host kernel.
  12. */
  13. #include <linux/eventfd.h>
  14. #include <linux/vhost.h>
  15. #include <linux/virtio_net.h>
  16. #include <linux/mm.h>
  17. #include <linux/mmu_context.h>
  18. #include <linux/miscdevice.h>
  19. #include <linux/mutex.h>
  20. #include <linux/rcupdate.h>
  21. #include <linux/poll.h>
  22. #include <linux/file.h>
  23. #include <linux/highmem.h>
  24. #include <linux/slab.h>
  25. #include <linux/kthread.h>
  26. #include <linux/cgroup.h>
  27. #include <linux/net.h>
  28. #include <linux/if_packet.h>
  29. #include <linux/if_arp.h>
  30. #include "vhost.h"
  31. enum {
  32. VHOST_MEMORY_MAX_NREGIONS = 64,
  33. VHOST_MEMORY_F_LOG = 0x1,
  34. };
  35. static unsigned vhost_zcopy_mask __read_mostly;
  36. #define vhost_used_event(vq) ((u16 __user *)&vq->avail->ring[vq->num])
  37. #define vhost_avail_event(vq) ((u16 __user *)&vq->used->ring[vq->num])
  38. static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh,
  39. poll_table *pt)
  40. {
  41. struct vhost_poll *poll;
  42. poll = container_of(pt, struct vhost_poll, table);
  43. poll->wqh = wqh;
  44. add_wait_queue(wqh, &poll->wait);
  45. }
  46. static int vhost_poll_wakeup(wait_queue_t *wait, unsigned mode, int sync,
  47. void *key)
  48. {
  49. struct vhost_poll *poll = container_of(wait, struct vhost_poll, wait);
  50. if (!((unsigned long)key & poll->mask))
  51. return 0;
  52. vhost_poll_queue(poll);
  53. return 0;
  54. }
  55. static void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn)
  56. {
  57. INIT_LIST_HEAD(&work->node);
  58. work->fn = fn;
  59. init_waitqueue_head(&work->done);
  60. work->flushing = 0;
  61. work->queue_seq = work->done_seq = 0;
  62. }
  63. /* Init poll structure */
  64. void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn,
  65. unsigned long mask, struct vhost_dev *dev)
  66. {
  67. init_waitqueue_func_entry(&poll->wait, vhost_poll_wakeup);
  68. init_poll_funcptr(&poll->table, vhost_poll_func);
  69. poll->mask = mask;
  70. poll->dev = dev;
  71. vhost_work_init(&poll->work, fn);
  72. }
  73. /* Start polling a file. We add ourselves to file's wait queue. The caller must
  74. * keep a reference to a file until after vhost_poll_stop is called. */
  75. void vhost_poll_start(struct vhost_poll *poll, struct file *file)
  76. {
  77. unsigned long mask;
  78. mask = file->f_op->poll(file, &poll->table);
  79. if (mask)
  80. vhost_poll_wakeup(&poll->wait, 0, 0, (void *)mask);
  81. }
  82. /* Stop polling a file. After this function returns, it becomes safe to drop the
  83. * file reference. You must also flush afterwards. */
  84. void vhost_poll_stop(struct vhost_poll *poll)
  85. {
  86. remove_wait_queue(poll->wqh, &poll->wait);
  87. }
  88. static bool vhost_work_seq_done(struct vhost_dev *dev, struct vhost_work *work,
  89. unsigned seq)
  90. {
  91. int left;
  92. spin_lock_irq(&dev->work_lock);
  93. left = seq - work->done_seq;
  94. spin_unlock_irq(&dev->work_lock);
  95. return left <= 0;
  96. }
  97. static void vhost_work_flush(struct vhost_dev *dev, struct vhost_work *work)
  98. {
  99. unsigned seq;
  100. int flushing;
  101. spin_lock_irq(&dev->work_lock);
  102. seq = work->queue_seq;
  103. work->flushing++;
  104. spin_unlock_irq(&dev->work_lock);
  105. wait_event(work->done, vhost_work_seq_done(dev, work, seq));
  106. spin_lock_irq(&dev->work_lock);
  107. flushing = --work->flushing;
  108. spin_unlock_irq(&dev->work_lock);
  109. BUG_ON(flushing < 0);
  110. }
  111. /* Flush any work that has been scheduled. When calling this, don't hold any
  112. * locks that are also used by the callback. */
  113. void vhost_poll_flush(struct vhost_poll *poll)
  114. {
  115. vhost_work_flush(poll->dev, &poll->work);
  116. }
  117. static inline void vhost_work_queue(struct vhost_dev *dev,
  118. struct vhost_work *work)
  119. {
  120. unsigned long flags;
  121. spin_lock_irqsave(&dev->work_lock, flags);
  122. if (list_empty(&work->node)) {
  123. list_add_tail(&work->node, &dev->work_list);
  124. work->queue_seq++;
  125. wake_up_process(dev->worker);
  126. }
  127. spin_unlock_irqrestore(&dev->work_lock, flags);
  128. }
  129. void vhost_poll_queue(struct vhost_poll *poll)
  130. {
  131. vhost_work_queue(poll->dev, &poll->work);
  132. }
  133. static void vhost_vq_reset(struct vhost_dev *dev,
  134. struct vhost_virtqueue *vq)
  135. {
  136. vq->num = 1;
  137. vq->desc = NULL;
  138. vq->avail = NULL;
  139. vq->used = NULL;
  140. vq->last_avail_idx = 0;
  141. vq->avail_idx = 0;
  142. vq->last_used_idx = 0;
  143. vq->signalled_used = 0;
  144. vq->signalled_used_valid = false;
  145. vq->used_flags = 0;
  146. vq->log_used = false;
  147. vq->log_addr = -1ull;
  148. vq->vhost_hlen = 0;
  149. vq->sock_hlen = 0;
  150. vq->private_data = NULL;
  151. vq->log_base = NULL;
  152. vq->error_ctx = NULL;
  153. vq->error = NULL;
  154. vq->kick = NULL;
  155. vq->call_ctx = NULL;
  156. vq->call = NULL;
  157. vq->log_ctx = NULL;
  158. vq->upend_idx = 0;
  159. vq->done_idx = 0;
  160. vq->ubufs = NULL;
  161. }
  162. static int vhost_worker(void *data)
  163. {
  164. struct vhost_dev *dev = data;
  165. struct vhost_work *work = NULL;
  166. unsigned uninitialized_var(seq);
  167. mm_segment_t oldfs = get_fs();
  168. set_fs(USER_DS);
  169. use_mm(dev->mm);
  170. for (;;) {
  171. /* mb paired w/ kthread_stop */
  172. set_current_state(TASK_INTERRUPTIBLE);
  173. spin_lock_irq(&dev->work_lock);
  174. if (work) {
  175. work->done_seq = seq;
  176. if (work->flushing)
  177. wake_up_all(&work->done);
  178. }
  179. if (kthread_should_stop()) {
  180. spin_unlock_irq(&dev->work_lock);
  181. __set_current_state(TASK_RUNNING);
  182. break;
  183. }
  184. if (!list_empty(&dev->work_list)) {
  185. work = list_first_entry(&dev->work_list,
  186. struct vhost_work, node);
  187. list_del_init(&work->node);
  188. seq = work->queue_seq;
  189. } else
  190. work = NULL;
  191. spin_unlock_irq(&dev->work_lock);
  192. if (work) {
  193. __set_current_state(TASK_RUNNING);
  194. work->fn(work);
  195. if (need_resched())
  196. schedule();
  197. } else
  198. schedule();
  199. }
  200. unuse_mm(dev->mm);
  201. set_fs(oldfs);
  202. return 0;
  203. }
  204. static void vhost_vq_free_iovecs(struct vhost_virtqueue *vq)
  205. {
  206. kfree(vq->indirect);
  207. vq->indirect = NULL;
  208. kfree(vq->log);
  209. vq->log = NULL;
  210. kfree(vq->heads);
  211. vq->heads = NULL;
  212. kfree(vq->ubuf_info);
  213. vq->ubuf_info = NULL;
  214. }
  215. void vhost_enable_zcopy(int vq)
  216. {
  217. vhost_zcopy_mask |= 0x1 << vq;
  218. }
  219. /* Helper to allocate iovec buffers for all vqs. */
  220. static long vhost_dev_alloc_iovecs(struct vhost_dev *dev)
  221. {
  222. int i;
  223. bool zcopy;
  224. for (i = 0; i < dev->nvqs; ++i) {
  225. dev->vqs[i].indirect = kmalloc(sizeof *dev->vqs[i].indirect *
  226. UIO_MAXIOV, GFP_KERNEL);
  227. dev->vqs[i].log = kmalloc(sizeof *dev->vqs[i].log * UIO_MAXIOV,
  228. GFP_KERNEL);
  229. dev->vqs[i].heads = kmalloc(sizeof *dev->vqs[i].heads *
  230. UIO_MAXIOV, GFP_KERNEL);
  231. zcopy = vhost_zcopy_mask & (0x1 << i);
  232. if (zcopy)
  233. dev->vqs[i].ubuf_info =
  234. kmalloc(sizeof *dev->vqs[i].ubuf_info *
  235. UIO_MAXIOV, GFP_KERNEL);
  236. if (!dev->vqs[i].indirect || !dev->vqs[i].log ||
  237. !dev->vqs[i].heads ||
  238. (zcopy && !dev->vqs[i].ubuf_info))
  239. goto err_nomem;
  240. }
  241. return 0;
  242. err_nomem:
  243. for (; i >= 0; --i)
  244. vhost_vq_free_iovecs(&dev->vqs[i]);
  245. return -ENOMEM;
  246. }
  247. static void vhost_dev_free_iovecs(struct vhost_dev *dev)
  248. {
  249. int i;
  250. for (i = 0; i < dev->nvqs; ++i)
  251. vhost_vq_free_iovecs(&dev->vqs[i]);
  252. }
  253. long vhost_dev_init(struct vhost_dev *dev,
  254. struct vhost_virtqueue *vqs, int nvqs)
  255. {
  256. int i;
  257. dev->vqs = vqs;
  258. dev->nvqs = nvqs;
  259. mutex_init(&dev->mutex);
  260. dev->log_ctx = NULL;
  261. dev->log_file = NULL;
  262. dev->memory = NULL;
  263. dev->mm = NULL;
  264. spin_lock_init(&dev->work_lock);
  265. INIT_LIST_HEAD(&dev->work_list);
  266. dev->worker = NULL;
  267. for (i = 0; i < dev->nvqs; ++i) {
  268. dev->vqs[i].log = NULL;
  269. dev->vqs[i].indirect = NULL;
  270. dev->vqs[i].heads = NULL;
  271. dev->vqs[i].ubuf_info = NULL;
  272. dev->vqs[i].dev = dev;
  273. mutex_init(&dev->vqs[i].mutex);
  274. vhost_vq_reset(dev, dev->vqs + i);
  275. if (dev->vqs[i].handle_kick)
  276. vhost_poll_init(&dev->vqs[i].poll,
  277. dev->vqs[i].handle_kick, POLLIN, dev);
  278. }
  279. return 0;
  280. }
  281. /* Caller should have device mutex */
  282. long vhost_dev_check_owner(struct vhost_dev *dev)
  283. {
  284. /* Are you the owner? If not, I don't think you mean to do that */
  285. return dev->mm == current->mm ? 0 : -EPERM;
  286. }
  287. struct vhost_attach_cgroups_struct {
  288. struct vhost_work work;
  289. struct task_struct *owner;
  290. int ret;
  291. };
  292. static void vhost_attach_cgroups_work(struct vhost_work *work)
  293. {
  294. struct vhost_attach_cgroups_struct *s;
  295. s = container_of(work, struct vhost_attach_cgroups_struct, work);
  296. s->ret = cgroup_attach_task_all(s->owner, current);
  297. }
  298. static int vhost_attach_cgroups(struct vhost_dev *dev)
  299. {
  300. struct vhost_attach_cgroups_struct attach;
  301. attach.owner = current;
  302. vhost_work_init(&attach.work, vhost_attach_cgroups_work);
  303. vhost_work_queue(dev, &attach.work);
  304. vhost_work_flush(dev, &attach.work);
  305. return attach.ret;
  306. }
  307. /* Caller should have device mutex */
  308. static long vhost_dev_set_owner(struct vhost_dev *dev)
  309. {
  310. struct task_struct *worker;
  311. int err;
  312. /* Is there an owner already? */
  313. if (dev->mm) {
  314. err = -EBUSY;
  315. goto err_mm;
  316. }
  317. /* No owner, become one */
  318. dev->mm = get_task_mm(current);
  319. worker = kthread_create(vhost_worker, dev, "vhost-%d", current->pid);
  320. if (IS_ERR(worker)) {
  321. err = PTR_ERR(worker);
  322. goto err_worker;
  323. }
  324. dev->worker = worker;
  325. wake_up_process(worker); /* avoid contributing to loadavg */
  326. err = vhost_attach_cgroups(dev);
  327. if (err)
  328. goto err_cgroup;
  329. err = vhost_dev_alloc_iovecs(dev);
  330. if (err)
  331. goto err_cgroup;
  332. return 0;
  333. err_cgroup:
  334. kthread_stop(worker);
  335. dev->worker = NULL;
  336. err_worker:
  337. if (dev->mm)
  338. mmput(dev->mm);
  339. dev->mm = NULL;
  340. err_mm:
  341. return err;
  342. }
  343. /* Caller should have device mutex */
  344. long vhost_dev_reset_owner(struct vhost_dev *dev)
  345. {
  346. struct vhost_memory *memory;
  347. /* Restore memory to default empty mapping. */
  348. memory = kmalloc(offsetof(struct vhost_memory, regions), GFP_KERNEL);
  349. if (!memory)
  350. return -ENOMEM;
  351. vhost_dev_cleanup(dev, true);
  352. memory->nregions = 0;
  353. RCU_INIT_POINTER(dev->memory, memory);
  354. return 0;
  355. }
  356. /* In case of DMA done not in order in lower device driver for some reason.
  357. * upend_idx is used to track end of used idx, done_idx is used to track head
  358. * of used idx. Once lower device DMA done contiguously, we will signal KVM
  359. * guest used idx.
  360. */
  361. int vhost_zerocopy_signal_used(struct vhost_virtqueue *vq)
  362. {
  363. int i;
  364. int j = 0;
  365. for (i = vq->done_idx; i != vq->upend_idx; i = (i + 1) % UIO_MAXIOV) {
  366. if ((vq->heads[i].len == VHOST_DMA_DONE_LEN)) {
  367. vq->heads[i].len = VHOST_DMA_CLEAR_LEN;
  368. vhost_add_used_and_signal(vq->dev, vq,
  369. vq->heads[i].id, 0);
  370. ++j;
  371. } else
  372. break;
  373. }
  374. if (j)
  375. vq->done_idx = i;
  376. return j;
  377. }
  378. /* Caller should have device mutex if and only if locked is set */
  379. void vhost_dev_cleanup(struct vhost_dev *dev, bool locked)
  380. {
  381. int i;
  382. for (i = 0; i < dev->nvqs; ++i) {
  383. if (dev->vqs[i].kick && dev->vqs[i].handle_kick) {
  384. vhost_poll_stop(&dev->vqs[i].poll);
  385. vhost_poll_flush(&dev->vqs[i].poll);
  386. }
  387. /* Wait for all lower device DMAs done. */
  388. if (dev->vqs[i].ubufs)
  389. vhost_ubuf_put_and_wait(dev->vqs[i].ubufs);
  390. /* Signal guest as appropriate. */
  391. vhost_zerocopy_signal_used(&dev->vqs[i]);
  392. if (dev->vqs[i].error_ctx)
  393. eventfd_ctx_put(dev->vqs[i].error_ctx);
  394. if (dev->vqs[i].error)
  395. fput(dev->vqs[i].error);
  396. if (dev->vqs[i].kick)
  397. fput(dev->vqs[i].kick);
  398. if (dev->vqs[i].call_ctx)
  399. eventfd_ctx_put(dev->vqs[i].call_ctx);
  400. if (dev->vqs[i].call)
  401. fput(dev->vqs[i].call);
  402. vhost_vq_reset(dev, dev->vqs + i);
  403. }
  404. vhost_dev_free_iovecs(dev);
  405. if (dev->log_ctx)
  406. eventfd_ctx_put(dev->log_ctx);
  407. dev->log_ctx = NULL;
  408. if (dev->log_file)
  409. fput(dev->log_file);
  410. dev->log_file = NULL;
  411. /* No one will access memory at this point */
  412. kfree(rcu_dereference_protected(dev->memory,
  413. locked ==
  414. lockdep_is_held(&dev->mutex)));
  415. RCU_INIT_POINTER(dev->memory, NULL);
  416. WARN_ON(!list_empty(&dev->work_list));
  417. if (dev->worker) {
  418. kthread_stop(dev->worker);
  419. dev->worker = NULL;
  420. }
  421. if (dev->mm)
  422. mmput(dev->mm);
  423. dev->mm = NULL;
  424. }
  425. static int log_access_ok(void __user *log_base, u64 addr, unsigned long sz)
  426. {
  427. u64 a = addr / VHOST_PAGE_SIZE / 8;
  428. /* Make sure 64 bit math will not overflow. */
  429. if (a > ULONG_MAX - (unsigned long)log_base ||
  430. a + (unsigned long)log_base > ULONG_MAX)
  431. return 0;
  432. return access_ok(VERIFY_WRITE, log_base + a,
  433. (sz + VHOST_PAGE_SIZE * 8 - 1) / VHOST_PAGE_SIZE / 8);
  434. }
  435. /* Caller should have vq mutex and device mutex. */
  436. static int vq_memory_access_ok(void __user *log_base, struct vhost_memory *mem,
  437. int log_all)
  438. {
  439. int i;
  440. if (!mem)
  441. return 0;
  442. for (i = 0; i < mem->nregions; ++i) {
  443. struct vhost_memory_region *m = mem->regions + i;
  444. unsigned long a = m->userspace_addr;
  445. if (m->memory_size > ULONG_MAX)
  446. return 0;
  447. else if (!access_ok(VERIFY_WRITE, (void __user *)a,
  448. m->memory_size))
  449. return 0;
  450. else if (log_all && !log_access_ok(log_base,
  451. m->guest_phys_addr,
  452. m->memory_size))
  453. return 0;
  454. }
  455. return 1;
  456. }
  457. /* Can we switch to this memory table? */
  458. /* Caller should have device mutex but not vq mutex */
  459. static int memory_access_ok(struct vhost_dev *d, struct vhost_memory *mem,
  460. int log_all)
  461. {
  462. int i;
  463. for (i = 0; i < d->nvqs; ++i) {
  464. int ok;
  465. mutex_lock(&d->vqs[i].mutex);
  466. /* If ring is inactive, will check when it's enabled. */
  467. if (d->vqs[i].private_data)
  468. ok = vq_memory_access_ok(d->vqs[i].log_base, mem,
  469. log_all);
  470. else
  471. ok = 1;
  472. mutex_unlock(&d->vqs[i].mutex);
  473. if (!ok)
  474. return 0;
  475. }
  476. return 1;
  477. }
  478. static int vq_access_ok(struct vhost_dev *d, unsigned int num,
  479. struct vring_desc __user *desc,
  480. struct vring_avail __user *avail,
  481. struct vring_used __user *used)
  482. {
  483. size_t s = vhost_has_feature(d, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
  484. return access_ok(VERIFY_READ, desc, num * sizeof *desc) &&
  485. access_ok(VERIFY_READ, avail,
  486. sizeof *avail + num * sizeof *avail->ring + s) &&
  487. access_ok(VERIFY_WRITE, used,
  488. sizeof *used + num * sizeof *used->ring + s);
  489. }
  490. /* Can we log writes? */
  491. /* Caller should have device mutex but not vq mutex */
  492. int vhost_log_access_ok(struct vhost_dev *dev)
  493. {
  494. struct vhost_memory *mp;
  495. mp = rcu_dereference_protected(dev->memory,
  496. lockdep_is_held(&dev->mutex));
  497. return memory_access_ok(dev, mp, 1);
  498. }
  499. /* Verify access for write logging. */
  500. /* Caller should have vq mutex and device mutex */
  501. static int vq_log_access_ok(struct vhost_dev *d, struct vhost_virtqueue *vq,
  502. void __user *log_base)
  503. {
  504. struct vhost_memory *mp;
  505. size_t s = vhost_has_feature(d, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
  506. mp = rcu_dereference_protected(vq->dev->memory,
  507. lockdep_is_held(&vq->mutex));
  508. return vq_memory_access_ok(log_base, mp,
  509. vhost_has_feature(vq->dev, VHOST_F_LOG_ALL)) &&
  510. (!vq->log_used || log_access_ok(log_base, vq->log_addr,
  511. sizeof *vq->used +
  512. vq->num * sizeof *vq->used->ring + s));
  513. }
  514. /* Can we start vq? */
  515. /* Caller should have vq mutex and device mutex */
  516. int vhost_vq_access_ok(struct vhost_virtqueue *vq)
  517. {
  518. return vq_access_ok(vq->dev, vq->num, vq->desc, vq->avail, vq->used) &&
  519. vq_log_access_ok(vq->dev, vq, vq->log_base);
  520. }
  521. static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m)
  522. {
  523. struct vhost_memory mem, *newmem, *oldmem;
  524. unsigned long size = offsetof(struct vhost_memory, regions);
  525. if (copy_from_user(&mem, m, size))
  526. return -EFAULT;
  527. if (mem.padding)
  528. return -EOPNOTSUPP;
  529. if (mem.nregions > VHOST_MEMORY_MAX_NREGIONS)
  530. return -E2BIG;
  531. newmem = kmalloc(size + mem.nregions * sizeof *m->regions, GFP_KERNEL);
  532. if (!newmem)
  533. return -ENOMEM;
  534. memcpy(newmem, &mem, size);
  535. if (copy_from_user(newmem->regions, m->regions,
  536. mem.nregions * sizeof *m->regions)) {
  537. kfree(newmem);
  538. return -EFAULT;
  539. }
  540. if (!memory_access_ok(d, newmem,
  541. vhost_has_feature(d, VHOST_F_LOG_ALL))) {
  542. kfree(newmem);
  543. return -EFAULT;
  544. }
  545. oldmem = rcu_dereference_protected(d->memory,
  546. lockdep_is_held(&d->mutex));
  547. rcu_assign_pointer(d->memory, newmem);
  548. synchronize_rcu();
  549. kfree(oldmem);
  550. return 0;
  551. }
  552. static long vhost_set_vring(struct vhost_dev *d, int ioctl, void __user *argp)
  553. {
  554. struct file *eventfp, *filep = NULL,
  555. *pollstart = NULL, *pollstop = NULL;
  556. struct eventfd_ctx *ctx = NULL;
  557. u32 __user *idxp = argp;
  558. struct vhost_virtqueue *vq;
  559. struct vhost_vring_state s;
  560. struct vhost_vring_file f;
  561. struct vhost_vring_addr a;
  562. u32 idx;
  563. long r;
  564. r = get_user(idx, idxp);
  565. if (r < 0)
  566. return r;
  567. if (idx >= d->nvqs)
  568. return -ENOBUFS;
  569. vq = d->vqs + idx;
  570. mutex_lock(&vq->mutex);
  571. switch (ioctl) {
  572. case VHOST_SET_VRING_NUM:
  573. /* Resizing ring with an active backend?
  574. * You don't want to do that. */
  575. if (vq->private_data) {
  576. r = -EBUSY;
  577. break;
  578. }
  579. if (copy_from_user(&s, argp, sizeof s)) {
  580. r = -EFAULT;
  581. break;
  582. }
  583. if (!s.num || s.num > 0xffff || (s.num & (s.num - 1))) {
  584. r = -EINVAL;
  585. break;
  586. }
  587. vq->num = s.num;
  588. break;
  589. case VHOST_SET_VRING_BASE:
  590. /* Moving base with an active backend?
  591. * You don't want to do that. */
  592. if (vq->private_data) {
  593. r = -EBUSY;
  594. break;
  595. }
  596. if (copy_from_user(&s, argp, sizeof s)) {
  597. r = -EFAULT;
  598. break;
  599. }
  600. if (s.num > 0xffff) {
  601. r = -EINVAL;
  602. break;
  603. }
  604. vq->last_avail_idx = s.num;
  605. /* Forget the cached index value. */
  606. vq->avail_idx = vq->last_avail_idx;
  607. break;
  608. case VHOST_GET_VRING_BASE:
  609. s.index = idx;
  610. s.num = vq->last_avail_idx;
  611. if (copy_to_user(argp, &s, sizeof s))
  612. r = -EFAULT;
  613. break;
  614. case VHOST_SET_VRING_ADDR:
  615. if (copy_from_user(&a, argp, sizeof a)) {
  616. r = -EFAULT;
  617. break;
  618. }
  619. if (a.flags & ~(0x1 << VHOST_VRING_F_LOG)) {
  620. r = -EOPNOTSUPP;
  621. break;
  622. }
  623. /* For 32bit, verify that the top 32bits of the user
  624. data are set to zero. */
  625. if ((u64)(unsigned long)a.desc_user_addr != a.desc_user_addr ||
  626. (u64)(unsigned long)a.used_user_addr != a.used_user_addr ||
  627. (u64)(unsigned long)a.avail_user_addr != a.avail_user_addr) {
  628. r = -EFAULT;
  629. break;
  630. }
  631. if ((a.avail_user_addr & (sizeof *vq->avail->ring - 1)) ||
  632. (a.used_user_addr & (sizeof *vq->used->ring - 1)) ||
  633. (a.log_guest_addr & (sizeof *vq->used->ring - 1))) {
  634. r = -EINVAL;
  635. break;
  636. }
  637. /* We only verify access here if backend is configured.
  638. * If it is not, we don't as size might not have been setup.
  639. * We will verify when backend is configured. */
  640. if (vq->private_data) {
  641. if (!vq_access_ok(d, vq->num,
  642. (void __user *)(unsigned long)a.desc_user_addr,
  643. (void __user *)(unsigned long)a.avail_user_addr,
  644. (void __user *)(unsigned long)a.used_user_addr)) {
  645. r = -EINVAL;
  646. break;
  647. }
  648. /* Also validate log access for used ring if enabled. */
  649. if ((a.flags & (0x1 << VHOST_VRING_F_LOG)) &&
  650. !log_access_ok(vq->log_base, a.log_guest_addr,
  651. sizeof *vq->used +
  652. vq->num * sizeof *vq->used->ring)) {
  653. r = -EINVAL;
  654. break;
  655. }
  656. }
  657. vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG));
  658. vq->desc = (void __user *)(unsigned long)a.desc_user_addr;
  659. vq->avail = (void __user *)(unsigned long)a.avail_user_addr;
  660. vq->log_addr = a.log_guest_addr;
  661. vq->used = (void __user *)(unsigned long)a.used_user_addr;
  662. break;
  663. case VHOST_SET_VRING_KICK:
  664. if (copy_from_user(&f, argp, sizeof f)) {
  665. r = -EFAULT;
  666. break;
  667. }
  668. eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd);
  669. if (IS_ERR(eventfp)) {
  670. r = PTR_ERR(eventfp);
  671. break;
  672. }
  673. if (eventfp != vq->kick) {
  674. pollstop = filep = vq->kick;
  675. pollstart = vq->kick = eventfp;
  676. } else
  677. filep = eventfp;
  678. break;
  679. case VHOST_SET_VRING_CALL:
  680. if (copy_from_user(&f, argp, sizeof f)) {
  681. r = -EFAULT;
  682. break;
  683. }
  684. eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd);
  685. if (IS_ERR(eventfp)) {
  686. r = PTR_ERR(eventfp);
  687. break;
  688. }
  689. if (eventfp != vq->call) {
  690. filep = vq->call;
  691. ctx = vq->call_ctx;
  692. vq->call = eventfp;
  693. vq->call_ctx = eventfp ?
  694. eventfd_ctx_fileget(eventfp) : NULL;
  695. } else
  696. filep = eventfp;
  697. break;
  698. case VHOST_SET_VRING_ERR:
  699. if (copy_from_user(&f, argp, sizeof f)) {
  700. r = -EFAULT;
  701. break;
  702. }
  703. eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd);
  704. if (IS_ERR(eventfp)) {
  705. r = PTR_ERR(eventfp);
  706. break;
  707. }
  708. if (eventfp != vq->error) {
  709. filep = vq->error;
  710. vq->error = eventfp;
  711. ctx = vq->error_ctx;
  712. vq->error_ctx = eventfp ?
  713. eventfd_ctx_fileget(eventfp) : NULL;
  714. } else
  715. filep = eventfp;
  716. break;
  717. default:
  718. r = -ENOIOCTLCMD;
  719. }
  720. if (pollstop && vq->handle_kick)
  721. vhost_poll_stop(&vq->poll);
  722. if (ctx)
  723. eventfd_ctx_put(ctx);
  724. if (filep)
  725. fput(filep);
  726. if (pollstart && vq->handle_kick)
  727. vhost_poll_start(&vq->poll, vq->kick);
  728. mutex_unlock(&vq->mutex);
  729. if (pollstop && vq->handle_kick)
  730. vhost_poll_flush(&vq->poll);
  731. return r;
  732. }
  733. /* Caller must have device mutex */
  734. long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, unsigned long arg)
  735. {
  736. void __user *argp = (void __user *)arg;
  737. struct file *eventfp, *filep = NULL;
  738. struct eventfd_ctx *ctx = NULL;
  739. u64 p;
  740. long r;
  741. int i, fd;
  742. /* If you are not the owner, you can become one */
  743. if (ioctl == VHOST_SET_OWNER) {
  744. r = vhost_dev_set_owner(d);
  745. goto done;
  746. }
  747. /* You must be the owner to do anything else */
  748. r = vhost_dev_check_owner(d);
  749. if (r)
  750. goto done;
  751. switch (ioctl) {
  752. case VHOST_SET_MEM_TABLE:
  753. r = vhost_set_memory(d, argp);
  754. break;
  755. case VHOST_SET_LOG_BASE:
  756. if (copy_from_user(&p, argp, sizeof p)) {
  757. r = -EFAULT;
  758. break;
  759. }
  760. if ((u64)(unsigned long)p != p) {
  761. r = -EFAULT;
  762. break;
  763. }
  764. for (i = 0; i < d->nvqs; ++i) {
  765. struct vhost_virtqueue *vq;
  766. void __user *base = (void __user *)(unsigned long)p;
  767. vq = d->vqs + i;
  768. mutex_lock(&vq->mutex);
  769. /* If ring is inactive, will check when it's enabled. */
  770. if (vq->private_data && !vq_log_access_ok(d, vq, base))
  771. r = -EFAULT;
  772. else
  773. vq->log_base = base;
  774. mutex_unlock(&vq->mutex);
  775. }
  776. break;
  777. case VHOST_SET_LOG_FD:
  778. r = get_user(fd, (int __user *)argp);
  779. if (r < 0)
  780. break;
  781. eventfp = fd == -1 ? NULL : eventfd_fget(fd);
  782. if (IS_ERR(eventfp)) {
  783. r = PTR_ERR(eventfp);
  784. break;
  785. }
  786. if (eventfp != d->log_file) {
  787. filep = d->log_file;
  788. ctx = d->log_ctx;
  789. d->log_ctx = eventfp ?
  790. eventfd_ctx_fileget(eventfp) : NULL;
  791. } else
  792. filep = eventfp;
  793. for (i = 0; i < d->nvqs; ++i) {
  794. mutex_lock(&d->vqs[i].mutex);
  795. d->vqs[i].log_ctx = d->log_ctx;
  796. mutex_unlock(&d->vqs[i].mutex);
  797. }
  798. if (ctx)
  799. eventfd_ctx_put(ctx);
  800. if (filep)
  801. fput(filep);
  802. break;
  803. default:
  804. r = vhost_set_vring(d, ioctl, argp);
  805. break;
  806. }
  807. done:
  808. return r;
  809. }
  810. static const struct vhost_memory_region *find_region(struct vhost_memory *mem,
  811. __u64 addr, __u32 len)
  812. {
  813. struct vhost_memory_region *reg;
  814. int i;
  815. /* linear search is not brilliant, but we really have on the order of 6
  816. * regions in practice */
  817. for (i = 0; i < mem->nregions; ++i) {
  818. reg = mem->regions + i;
  819. if (reg->guest_phys_addr <= addr &&
  820. reg->guest_phys_addr + reg->memory_size - 1 >= addr)
  821. return reg;
  822. }
  823. return NULL;
  824. }
  825. /* TODO: This is really inefficient. We need something like get_user()
  826. * (instruction directly accesses the data, with an exception table entry
  827. * returning -EFAULT). See Documentation/x86/exception-tables.txt.
  828. */
  829. static int set_bit_to_user(int nr, void __user *addr)
  830. {
  831. unsigned long log = (unsigned long)addr;
  832. struct page *page;
  833. void *base;
  834. int bit = nr + (log % PAGE_SIZE) * 8;
  835. int r;
  836. r = get_user_pages_fast(log, 1, 1, &page);
  837. if (r < 0)
  838. return r;
  839. BUG_ON(r != 1);
  840. base = kmap_atomic(page);
  841. set_bit(bit, base);
  842. kunmap_atomic(base);
  843. set_page_dirty_lock(page);
  844. put_page(page);
  845. return 0;
  846. }
  847. static int log_write(void __user *log_base,
  848. u64 write_address, u64 write_length)
  849. {
  850. u64 write_page = write_address / VHOST_PAGE_SIZE;
  851. int r;
  852. if (!write_length)
  853. return 0;
  854. write_length += write_address % VHOST_PAGE_SIZE;
  855. for (;;) {
  856. u64 base = (u64)(unsigned long)log_base;
  857. u64 log = base + write_page / 8;
  858. int bit = write_page % 8;
  859. if ((u64)(unsigned long)log != log)
  860. return -EFAULT;
  861. r = set_bit_to_user(bit, (void __user *)(unsigned long)log);
  862. if (r < 0)
  863. return r;
  864. if (write_length <= VHOST_PAGE_SIZE)
  865. break;
  866. write_length -= VHOST_PAGE_SIZE;
  867. write_page += 1;
  868. }
  869. return r;
  870. }
  871. int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log,
  872. unsigned int log_num, u64 len)
  873. {
  874. int i, r;
  875. /* Make sure data written is seen before log. */
  876. smp_wmb();
  877. for (i = 0; i < log_num; ++i) {
  878. u64 l = min(log[i].len, len);
  879. r = log_write(vq->log_base, log[i].addr, l);
  880. if (r < 0)
  881. return r;
  882. len -= l;
  883. if (!len) {
  884. if (vq->log_ctx)
  885. eventfd_signal(vq->log_ctx, 1);
  886. return 0;
  887. }
  888. }
  889. /* Length written exceeds what we have stored. This is a bug. */
  890. BUG();
  891. return 0;
  892. }
  893. static int vhost_update_used_flags(struct vhost_virtqueue *vq)
  894. {
  895. void __user *used;
  896. if (__put_user(vq->used_flags, &vq->used->flags) < 0)
  897. return -EFAULT;
  898. if (unlikely(vq->log_used)) {
  899. /* Make sure the flag is seen before log. */
  900. smp_wmb();
  901. /* Log used flag write. */
  902. used = &vq->used->flags;
  903. log_write(vq->log_base, vq->log_addr +
  904. (used - (void __user *)vq->used),
  905. sizeof vq->used->flags);
  906. if (vq->log_ctx)
  907. eventfd_signal(vq->log_ctx, 1);
  908. }
  909. return 0;
  910. }
  911. static int vhost_update_avail_event(struct vhost_virtqueue *vq, u16 avail_event)
  912. {
  913. if (__put_user(vq->avail_idx, vhost_avail_event(vq)))
  914. return -EFAULT;
  915. if (unlikely(vq->log_used)) {
  916. void __user *used;
  917. /* Make sure the event is seen before log. */
  918. smp_wmb();
  919. /* Log avail event write */
  920. used = vhost_avail_event(vq);
  921. log_write(vq->log_base, vq->log_addr +
  922. (used - (void __user *)vq->used),
  923. sizeof *vhost_avail_event(vq));
  924. if (vq->log_ctx)
  925. eventfd_signal(vq->log_ctx, 1);
  926. }
  927. return 0;
  928. }
  929. int vhost_init_used(struct vhost_virtqueue *vq)
  930. {
  931. int r;
  932. if (!vq->private_data)
  933. return 0;
  934. r = vhost_update_used_flags(vq);
  935. if (r)
  936. return r;
  937. vq->signalled_used_valid = false;
  938. return get_user(vq->last_used_idx, &vq->used->idx);
  939. }
  940. static int translate_desc(struct vhost_dev *dev, u64 addr, u32 len,
  941. struct iovec iov[], int iov_size)
  942. {
  943. const struct vhost_memory_region *reg;
  944. struct vhost_memory *mem;
  945. struct iovec *_iov;
  946. u64 s = 0;
  947. int ret = 0;
  948. rcu_read_lock();
  949. mem = rcu_dereference(dev->memory);
  950. while ((u64)len > s) {
  951. u64 size;
  952. if (unlikely(ret >= iov_size)) {
  953. ret = -ENOBUFS;
  954. break;
  955. }
  956. reg = find_region(mem, addr, len);
  957. if (unlikely(!reg)) {
  958. ret = -EFAULT;
  959. break;
  960. }
  961. _iov = iov + ret;
  962. size = reg->memory_size - addr + reg->guest_phys_addr;
  963. _iov->iov_len = min((u64)len, size);
  964. _iov->iov_base = (void __user *)(unsigned long)
  965. (reg->userspace_addr + addr - reg->guest_phys_addr);
  966. s += size;
  967. addr += size;
  968. ++ret;
  969. }
  970. rcu_read_unlock();
  971. return ret;
  972. }
  973. /* Each buffer in the virtqueues is actually a chain of descriptors. This
  974. * function returns the next descriptor in the chain,
  975. * or -1U if we're at the end. */
  976. static unsigned next_desc(struct vring_desc *desc)
  977. {
  978. unsigned int next;
  979. /* If this descriptor says it doesn't chain, we're done. */
  980. if (!(desc->flags & VRING_DESC_F_NEXT))
  981. return -1U;
  982. /* Check they're not leading us off end of descriptors. */
  983. next = desc->next;
  984. /* Make sure compiler knows to grab that: we don't want it changing! */
  985. /* We will use the result as an index in an array, so most
  986. * architectures only need a compiler barrier here. */
  987. read_barrier_depends();
  988. return next;
  989. }
  990. static int get_indirect(struct vhost_dev *dev, struct vhost_virtqueue *vq,
  991. struct iovec iov[], unsigned int iov_size,
  992. unsigned int *out_num, unsigned int *in_num,
  993. struct vhost_log *log, unsigned int *log_num,
  994. struct vring_desc *indirect)
  995. {
  996. struct vring_desc desc;
  997. unsigned int i = 0, count, found = 0;
  998. int ret;
  999. /* Sanity check */
  1000. if (unlikely(indirect->len % sizeof desc)) {
  1001. vq_err(vq, "Invalid length in indirect descriptor: "
  1002. "len 0x%llx not multiple of 0x%zx\n",
  1003. (unsigned long long)indirect->len,
  1004. sizeof desc);
  1005. return -EINVAL;
  1006. }
  1007. ret = translate_desc(dev, indirect->addr, indirect->len, vq->indirect,
  1008. UIO_MAXIOV);
  1009. if (unlikely(ret < 0)) {
  1010. vq_err(vq, "Translation failure %d in indirect.\n", ret);
  1011. return ret;
  1012. }
  1013. /* We will use the result as an address to read from, so most
  1014. * architectures only need a compiler barrier here. */
  1015. read_barrier_depends();
  1016. count = indirect->len / sizeof desc;
  1017. /* Buffers are chained via a 16 bit next field, so
  1018. * we can have at most 2^16 of these. */
  1019. if (unlikely(count > USHRT_MAX + 1)) {
  1020. vq_err(vq, "Indirect buffer length too big: %d\n",
  1021. indirect->len);
  1022. return -E2BIG;
  1023. }
  1024. do {
  1025. unsigned iov_count = *in_num + *out_num;
  1026. if (unlikely(++found > count)) {
  1027. vq_err(vq, "Loop detected: last one at %u "
  1028. "indirect size %u\n",
  1029. i, count);
  1030. return -EINVAL;
  1031. }
  1032. if (unlikely(memcpy_fromiovec((unsigned char *)&desc,
  1033. vq->indirect, sizeof desc))) {
  1034. vq_err(vq, "Failed indirect descriptor: idx %d, %zx\n",
  1035. i, (size_t)indirect->addr + i * sizeof desc);
  1036. return -EINVAL;
  1037. }
  1038. if (unlikely(desc.flags & VRING_DESC_F_INDIRECT)) {
  1039. vq_err(vq, "Nested indirect descriptor: idx %d, %zx\n",
  1040. i, (size_t)indirect->addr + i * sizeof desc);
  1041. return -EINVAL;
  1042. }
  1043. ret = translate_desc(dev, desc.addr, desc.len, iov + iov_count,
  1044. iov_size - iov_count);
  1045. if (unlikely(ret < 0)) {
  1046. vq_err(vq, "Translation failure %d indirect idx %d\n",
  1047. ret, i);
  1048. return ret;
  1049. }
  1050. /* If this is an input descriptor, increment that count. */
  1051. if (desc.flags & VRING_DESC_F_WRITE) {
  1052. *in_num += ret;
  1053. if (unlikely(log)) {
  1054. log[*log_num].addr = desc.addr;
  1055. log[*log_num].len = desc.len;
  1056. ++*log_num;
  1057. }
  1058. } else {
  1059. /* If it's an output descriptor, they're all supposed
  1060. * to come before any input descriptors. */
  1061. if (unlikely(*in_num)) {
  1062. vq_err(vq, "Indirect descriptor "
  1063. "has out after in: idx %d\n", i);
  1064. return -EINVAL;
  1065. }
  1066. *out_num += ret;
  1067. }
  1068. } while ((i = next_desc(&desc)) != -1);
  1069. return 0;
  1070. }
  1071. /* This looks in the virtqueue and for the first available buffer, and converts
  1072. * it to an iovec for convenient access. Since descriptors consist of some
  1073. * number of output then some number of input descriptors, it's actually two
  1074. * iovecs, but we pack them into one and note how many of each there were.
  1075. *
  1076. * This function returns the descriptor number found, or vq->num (which is
  1077. * never a valid descriptor number) if none was found. A negative code is
  1078. * returned on error. */
  1079. int vhost_get_vq_desc(struct vhost_dev *dev, struct vhost_virtqueue *vq,
  1080. struct iovec iov[], unsigned int iov_size,
  1081. unsigned int *out_num, unsigned int *in_num,
  1082. struct vhost_log *log, unsigned int *log_num)
  1083. {
  1084. struct vring_desc desc;
  1085. unsigned int i, head, found = 0;
  1086. u16 last_avail_idx;
  1087. int ret;
  1088. /* Check it isn't doing very strange things with descriptor numbers. */
  1089. last_avail_idx = vq->last_avail_idx;
  1090. if (unlikely(__get_user(vq->avail_idx, &vq->avail->idx))) {
  1091. vq_err(vq, "Failed to access avail idx at %p\n",
  1092. &vq->avail->idx);
  1093. return -EFAULT;
  1094. }
  1095. if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) {
  1096. vq_err(vq, "Guest moved used index from %u to %u",
  1097. last_avail_idx, vq->avail_idx);
  1098. return -EFAULT;
  1099. }
  1100. /* If there's nothing new since last we looked, return invalid. */
  1101. if (vq->avail_idx == last_avail_idx)
  1102. return vq->num;
  1103. /* Only get avail ring entries after they have been exposed by guest. */
  1104. smp_rmb();
  1105. /* Grab the next descriptor number they're advertising, and increment
  1106. * the index we've seen. */
  1107. if (unlikely(__get_user(head,
  1108. &vq->avail->ring[last_avail_idx % vq->num]))) {
  1109. vq_err(vq, "Failed to read head: idx %d address %p\n",
  1110. last_avail_idx,
  1111. &vq->avail->ring[last_avail_idx % vq->num]);
  1112. return -EFAULT;
  1113. }
  1114. /* If their number is silly, that's an error. */
  1115. if (unlikely(head >= vq->num)) {
  1116. vq_err(vq, "Guest says index %u > %u is available",
  1117. head, vq->num);
  1118. return -EINVAL;
  1119. }
  1120. /* When we start there are none of either input nor output. */
  1121. *out_num = *in_num = 0;
  1122. if (unlikely(log))
  1123. *log_num = 0;
  1124. i = head;
  1125. do {
  1126. unsigned iov_count = *in_num + *out_num;
  1127. if (unlikely(i >= vq->num)) {
  1128. vq_err(vq, "Desc index is %u > %u, head = %u",
  1129. i, vq->num, head);
  1130. return -EINVAL;
  1131. }
  1132. if (unlikely(++found > vq->num)) {
  1133. vq_err(vq, "Loop detected: last one at %u "
  1134. "vq size %u head %u\n",
  1135. i, vq->num, head);
  1136. return -EINVAL;
  1137. }
  1138. ret = __copy_from_user(&desc, vq->desc + i, sizeof desc);
  1139. if (unlikely(ret)) {
  1140. vq_err(vq, "Failed to get descriptor: idx %d addr %p\n",
  1141. i, vq->desc + i);
  1142. return -EFAULT;
  1143. }
  1144. if (desc.flags & VRING_DESC_F_INDIRECT) {
  1145. ret = get_indirect(dev, vq, iov, iov_size,
  1146. out_num, in_num,
  1147. log, log_num, &desc);
  1148. if (unlikely(ret < 0)) {
  1149. vq_err(vq, "Failure detected "
  1150. "in indirect descriptor at idx %d\n", i);
  1151. return ret;
  1152. }
  1153. continue;
  1154. }
  1155. ret = translate_desc(dev, desc.addr, desc.len, iov + iov_count,
  1156. iov_size - iov_count);
  1157. if (unlikely(ret < 0)) {
  1158. vq_err(vq, "Translation failure %d descriptor idx %d\n",
  1159. ret, i);
  1160. return ret;
  1161. }
  1162. if (desc.flags & VRING_DESC_F_WRITE) {
  1163. /* If this is an input descriptor,
  1164. * increment that count. */
  1165. *in_num += ret;
  1166. if (unlikely(log)) {
  1167. log[*log_num].addr = desc.addr;
  1168. log[*log_num].len = desc.len;
  1169. ++*log_num;
  1170. }
  1171. } else {
  1172. /* If it's an output descriptor, they're all supposed
  1173. * to come before any input descriptors. */
  1174. if (unlikely(*in_num)) {
  1175. vq_err(vq, "Descriptor has out after in: "
  1176. "idx %d\n", i);
  1177. return -EINVAL;
  1178. }
  1179. *out_num += ret;
  1180. }
  1181. } while ((i = next_desc(&desc)) != -1);
  1182. /* On success, increment avail index. */
  1183. vq->last_avail_idx++;
  1184. /* Assume notifications from guest are disabled at this point,
  1185. * if they aren't we would need to update avail_event index. */
  1186. BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY));
  1187. return head;
  1188. }
  1189. /* Reverse the effect of vhost_get_vq_desc. Useful for error handling. */
  1190. void vhost_discard_vq_desc(struct vhost_virtqueue *vq, int n)
  1191. {
  1192. vq->last_avail_idx -= n;
  1193. }
  1194. /* After we've used one of their buffers, we tell them about it. We'll then
  1195. * want to notify the guest, using eventfd. */
  1196. int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len)
  1197. {
  1198. struct vring_used_elem __user *used;
  1199. /* The virtqueue contains a ring of used buffers. Get a pointer to the
  1200. * next entry in that used ring. */
  1201. used = &vq->used->ring[vq->last_used_idx % vq->num];
  1202. if (__put_user(head, &used->id)) {
  1203. vq_err(vq, "Failed to write used id");
  1204. return -EFAULT;
  1205. }
  1206. if (__put_user(len, &used->len)) {
  1207. vq_err(vq, "Failed to write used len");
  1208. return -EFAULT;
  1209. }
  1210. /* Make sure buffer is written before we update index. */
  1211. smp_wmb();
  1212. if (__put_user(vq->last_used_idx + 1, &vq->used->idx)) {
  1213. vq_err(vq, "Failed to increment used idx");
  1214. return -EFAULT;
  1215. }
  1216. if (unlikely(vq->log_used)) {
  1217. /* Make sure data is seen before log. */
  1218. smp_wmb();
  1219. /* Log used ring entry write. */
  1220. log_write(vq->log_base,
  1221. vq->log_addr +
  1222. ((void __user *)used - (void __user *)vq->used),
  1223. sizeof *used);
  1224. /* Log used index update. */
  1225. log_write(vq->log_base,
  1226. vq->log_addr + offsetof(struct vring_used, idx),
  1227. sizeof vq->used->idx);
  1228. if (vq->log_ctx)
  1229. eventfd_signal(vq->log_ctx, 1);
  1230. }
  1231. vq->last_used_idx++;
  1232. /* If the driver never bothers to signal in a very long while,
  1233. * used index might wrap around. If that happens, invalidate
  1234. * signalled_used index we stored. TODO: make sure driver
  1235. * signals at least once in 2^16 and remove this. */
  1236. if (unlikely(vq->last_used_idx == vq->signalled_used))
  1237. vq->signalled_used_valid = false;
  1238. return 0;
  1239. }
  1240. static int __vhost_add_used_n(struct vhost_virtqueue *vq,
  1241. struct vring_used_elem *heads,
  1242. unsigned count)
  1243. {
  1244. struct vring_used_elem __user *used;
  1245. u16 old, new;
  1246. int start;
  1247. start = vq->last_used_idx % vq->num;
  1248. used = vq->used->ring + start;
  1249. if (__copy_to_user(used, heads, count * sizeof *used)) {
  1250. vq_err(vq, "Failed to write used");
  1251. return -EFAULT;
  1252. }
  1253. if (unlikely(vq->log_used)) {
  1254. /* Make sure data is seen before log. */
  1255. smp_wmb();
  1256. /* Log used ring entry write. */
  1257. log_write(vq->log_base,
  1258. vq->log_addr +
  1259. ((void __user *)used - (void __user *)vq->used),
  1260. count * sizeof *used);
  1261. }
  1262. old = vq->last_used_idx;
  1263. new = (vq->last_used_idx += count);
  1264. /* If the driver never bothers to signal in a very long while,
  1265. * used index might wrap around. If that happens, invalidate
  1266. * signalled_used index we stored. TODO: make sure driver
  1267. * signals at least once in 2^16 and remove this. */
  1268. if (unlikely((u16)(new - vq->signalled_used) < (u16)(new - old)))
  1269. vq->signalled_used_valid = false;
  1270. return 0;
  1271. }
  1272. /* After we've used one of their buffers, we tell them about it. We'll then
  1273. * want to notify the guest, using eventfd. */
  1274. int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads,
  1275. unsigned count)
  1276. {
  1277. int start, n, r;
  1278. start = vq->last_used_idx % vq->num;
  1279. n = vq->num - start;
  1280. if (n < count) {
  1281. r = __vhost_add_used_n(vq, heads, n);
  1282. if (r < 0)
  1283. return r;
  1284. heads += n;
  1285. count -= n;
  1286. }
  1287. r = __vhost_add_used_n(vq, heads, count);
  1288. /* Make sure buffer is written before we update index. */
  1289. smp_wmb();
  1290. if (put_user(vq->last_used_idx, &vq->used->idx)) {
  1291. vq_err(vq, "Failed to increment used idx");
  1292. return -EFAULT;
  1293. }
  1294. if (unlikely(vq->log_used)) {
  1295. /* Log used index update. */
  1296. log_write(vq->log_base,
  1297. vq->log_addr + offsetof(struct vring_used, idx),
  1298. sizeof vq->used->idx);
  1299. if (vq->log_ctx)
  1300. eventfd_signal(vq->log_ctx, 1);
  1301. }
  1302. return r;
  1303. }
  1304. static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
  1305. {
  1306. __u16 old, new, event;
  1307. bool v;
  1308. /* Flush out used index updates. This is paired
  1309. * with the barrier that the Guest executes when enabling
  1310. * interrupts. */
  1311. smp_mb();
  1312. if (vhost_has_feature(dev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
  1313. unlikely(vq->avail_idx == vq->last_avail_idx))
  1314. return true;
  1315. if (!vhost_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
  1316. __u16 flags;
  1317. if (__get_user(flags, &vq->avail->flags)) {
  1318. vq_err(vq, "Failed to get flags");
  1319. return true;
  1320. }
  1321. return !(flags & VRING_AVAIL_F_NO_INTERRUPT);
  1322. }
  1323. old = vq->signalled_used;
  1324. v = vq->signalled_used_valid;
  1325. new = vq->signalled_used = vq->last_used_idx;
  1326. vq->signalled_used_valid = true;
  1327. if (unlikely(!v))
  1328. return true;
  1329. if (get_user(event, vhost_used_event(vq))) {
  1330. vq_err(vq, "Failed to get used event idx");
  1331. return true;
  1332. }
  1333. return vring_need_event(event, new, old);
  1334. }
  1335. /* This actually signals the guest, using eventfd. */
  1336. void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq)
  1337. {
  1338. /* Signal the Guest tell them we used something up. */
  1339. if (vq->call_ctx && vhost_notify(dev, vq))
  1340. eventfd_signal(vq->call_ctx, 1);
  1341. }
  1342. /* And here's the combo meal deal. Supersize me! */
  1343. void vhost_add_used_and_signal(struct vhost_dev *dev,
  1344. struct vhost_virtqueue *vq,
  1345. unsigned int head, int len)
  1346. {
  1347. vhost_add_used(vq, head, len);
  1348. vhost_signal(dev, vq);
  1349. }
  1350. /* multi-buffer version of vhost_add_used_and_signal */
  1351. void vhost_add_used_and_signal_n(struct vhost_dev *dev,
  1352. struct vhost_virtqueue *vq,
  1353. struct vring_used_elem *heads, unsigned count)
  1354. {
  1355. vhost_add_used_n(vq, heads, count);
  1356. vhost_signal(dev, vq);
  1357. }
  1358. /* OK, now we need to know about added descriptors. */
  1359. bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
  1360. {
  1361. u16 avail_idx;
  1362. int r;
  1363. if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY))
  1364. return false;
  1365. vq->used_flags &= ~VRING_USED_F_NO_NOTIFY;
  1366. if (!vhost_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
  1367. r = vhost_update_used_flags(vq);
  1368. if (r) {
  1369. vq_err(vq, "Failed to enable notification at %p: %d\n",
  1370. &vq->used->flags, r);
  1371. return false;
  1372. }
  1373. } else {
  1374. r = vhost_update_avail_event(vq, vq->avail_idx);
  1375. if (r) {
  1376. vq_err(vq, "Failed to update avail event index at %p: %d\n",
  1377. vhost_avail_event(vq), r);
  1378. return false;
  1379. }
  1380. }
  1381. /* They could have slipped one in as we were doing that: make
  1382. * sure it's written, then check again. */
  1383. smp_mb();
  1384. r = __get_user(avail_idx, &vq->avail->idx);
  1385. if (r) {
  1386. vq_err(vq, "Failed to check avail idx at %p: %d\n",
  1387. &vq->avail->idx, r);
  1388. return false;
  1389. }
  1390. return avail_idx != vq->avail_idx;
  1391. }
  1392. /* We don't need to be notified again. */
  1393. void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
  1394. {
  1395. int r;
  1396. if (vq->used_flags & VRING_USED_F_NO_NOTIFY)
  1397. return;
  1398. vq->used_flags |= VRING_USED_F_NO_NOTIFY;
  1399. if (!vhost_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
  1400. r = vhost_update_used_flags(vq);
  1401. if (r)
  1402. vq_err(vq, "Failed to enable notification at %p: %d\n",
  1403. &vq->used->flags, r);
  1404. }
  1405. }
  1406. static void vhost_zerocopy_done_signal(struct kref *kref)
  1407. {
  1408. struct vhost_ubuf_ref *ubufs = container_of(kref, struct vhost_ubuf_ref,
  1409. kref);
  1410. wake_up(&ubufs->wait);
  1411. }
  1412. struct vhost_ubuf_ref *vhost_ubuf_alloc(struct vhost_virtqueue *vq,
  1413. bool zcopy)
  1414. {
  1415. struct vhost_ubuf_ref *ubufs;
  1416. /* No zero copy backend? Nothing to count. */
  1417. if (!zcopy)
  1418. return NULL;
  1419. ubufs = kmalloc(sizeof *ubufs, GFP_KERNEL);
  1420. if (!ubufs)
  1421. return ERR_PTR(-ENOMEM);
  1422. kref_init(&ubufs->kref);
  1423. init_waitqueue_head(&ubufs->wait);
  1424. ubufs->vq = vq;
  1425. return ubufs;
  1426. }
  1427. void vhost_ubuf_put(struct vhost_ubuf_ref *ubufs)
  1428. {
  1429. kref_put(&ubufs->kref, vhost_zerocopy_done_signal);
  1430. }
  1431. void vhost_ubuf_put_and_wait(struct vhost_ubuf_ref *ubufs)
  1432. {
  1433. kref_put(&ubufs->kref, vhost_zerocopy_done_signal);
  1434. wait_event(ubufs->wait, !atomic_read(&ubufs->kref.refcount));
  1435. kfree(ubufs);
  1436. }
  1437. void vhost_zerocopy_callback(struct ubuf_info *ubuf)
  1438. {
  1439. struct vhost_ubuf_ref *ubufs = ubuf->ctx;
  1440. struct vhost_virtqueue *vq = ubufs->vq;
  1441. vhost_poll_queue(&vq->poll);
  1442. /* set len = 1 to mark this desc buffers done DMA */
  1443. vq->heads[ubuf->desc].len = VHOST_DMA_DONE_LEN;
  1444. kref_put(&ubufs->kref, vhost_zerocopy_done_signal);
  1445. }