virtio_pci.c 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833
  1. /*
  2. * Virtio PCI driver
  3. *
  4. * This module allows virtio devices to be used over a virtual PCI device.
  5. * This can be used with QEMU based VMMs like KVM or Xen.
  6. *
  7. * Copyright IBM Corp. 2007
  8. *
  9. * Authors:
  10. * Anthony Liguori <aliguori@us.ibm.com>
  11. *
  12. * This work is licensed under the terms of the GNU GPL, version 2 or later.
  13. * See the COPYING file in the top-level directory.
  14. *
  15. */
  16. #include <linux/module.h>
  17. #include <linux/list.h>
  18. #include <linux/pci.h>
  19. #include <linux/slab.h>
  20. #include <linux/interrupt.h>
  21. #include <linux/virtio.h>
  22. #include <linux/virtio_config.h>
  23. #include <linux/virtio_ring.h>
  24. #include <linux/virtio_pci.h>
  25. #include <linux/highmem.h>
  26. #include <linux/spinlock.h>
  27. MODULE_AUTHOR("Anthony Liguori <aliguori@us.ibm.com>");
  28. MODULE_DESCRIPTION("virtio-pci");
  29. MODULE_LICENSE("GPL");
  30. MODULE_VERSION("1");
  31. /* Our device structure */
  32. struct virtio_pci_device
  33. {
  34. struct virtio_device vdev;
  35. struct pci_dev *pci_dev;
  36. /* the IO mapping for the PCI config space */
  37. void __iomem *ioaddr;
  38. /* a list of queues so we can dispatch IRQs */
  39. spinlock_t lock;
  40. struct list_head virtqueues;
  41. /* MSI-X support */
  42. int msix_enabled;
  43. int intx_enabled;
  44. struct msix_entry *msix_entries;
  45. cpumask_var_t *msix_affinity_masks;
  46. /* Name strings for interrupts. This size should be enough,
  47. * and I'm too lazy to allocate each name separately. */
  48. char (*msix_names)[256];
  49. /* Number of available vectors */
  50. unsigned msix_vectors;
  51. /* Vectors allocated, excluding per-vq vectors if any */
  52. unsigned msix_used_vectors;
  53. /* Status saved during hibernate/restore */
  54. u8 saved_status;
  55. /* Whether we have vector per vq */
  56. bool per_vq_vectors;
  57. };
  58. /* Constants for MSI-X */
  59. /* Use first vector for configuration changes, second and the rest for
  60. * virtqueues Thus, we need at least 2 vectors for MSI. */
  61. enum {
  62. VP_MSIX_CONFIG_VECTOR = 0,
  63. VP_MSIX_VQ_VECTOR = 1,
  64. };
  65. struct virtio_pci_vq_info
  66. {
  67. /* the actual virtqueue */
  68. struct virtqueue *vq;
  69. /* the number of entries in the queue */
  70. int num;
  71. /* the virtual address of the ring queue */
  72. void *queue;
  73. /* the list node for the virtqueues list */
  74. struct list_head node;
  75. /* MSI-X vector (or none) */
  76. unsigned msix_vector;
  77. };
  78. /* Qumranet donated their vendor ID for devices 0x1000 thru 0x10FF. */
  79. static struct pci_device_id virtio_pci_id_table[] = {
  80. { 0x1af4, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
  81. { 0 },
  82. };
  83. MODULE_DEVICE_TABLE(pci, virtio_pci_id_table);
  84. /* Convert a generic virtio device to our structure */
  85. static struct virtio_pci_device *to_vp_device(struct virtio_device *vdev)
  86. {
  87. return container_of(vdev, struct virtio_pci_device, vdev);
  88. }
  89. /* virtio config->get_features() implementation */
  90. static u32 vp_get_features(struct virtio_device *vdev)
  91. {
  92. struct virtio_pci_device *vp_dev = to_vp_device(vdev);
  93. /* When someone needs more than 32 feature bits, we'll need to
  94. * steal a bit to indicate that the rest are somewhere else. */
  95. return ioread32(vp_dev->ioaddr + VIRTIO_PCI_HOST_FEATURES);
  96. }
  97. /* virtio config->finalize_features() implementation */
  98. static void vp_finalize_features(struct virtio_device *vdev)
  99. {
  100. struct virtio_pci_device *vp_dev = to_vp_device(vdev);
  101. /* Give virtio_ring a chance to accept features. */
  102. vring_transport_features(vdev);
  103. /* We only support 32 feature bits. */
  104. BUILD_BUG_ON(ARRAY_SIZE(vdev->features) != 1);
  105. iowrite32(vdev->features[0], vp_dev->ioaddr+VIRTIO_PCI_GUEST_FEATURES);
  106. }
  107. /* virtio config->get() implementation */
  108. static void vp_get(struct virtio_device *vdev, unsigned offset,
  109. void *buf, unsigned len)
  110. {
  111. struct virtio_pci_device *vp_dev = to_vp_device(vdev);
  112. void __iomem *ioaddr = vp_dev->ioaddr +
  113. VIRTIO_PCI_CONFIG(vp_dev) + offset;
  114. u8 *ptr = buf;
  115. int i;
  116. for (i = 0; i < len; i++)
  117. ptr[i] = ioread8(ioaddr + i);
  118. }
  119. /* the config->set() implementation. it's symmetric to the config->get()
  120. * implementation */
  121. static void vp_set(struct virtio_device *vdev, unsigned offset,
  122. const void *buf, unsigned len)
  123. {
  124. struct virtio_pci_device *vp_dev = to_vp_device(vdev);
  125. void __iomem *ioaddr = vp_dev->ioaddr +
  126. VIRTIO_PCI_CONFIG(vp_dev) + offset;
  127. const u8 *ptr = buf;
  128. int i;
  129. for (i = 0; i < len; i++)
  130. iowrite8(ptr[i], ioaddr + i);
  131. }
  132. /* config->{get,set}_status() implementations */
  133. static u8 vp_get_status(struct virtio_device *vdev)
  134. {
  135. struct virtio_pci_device *vp_dev = to_vp_device(vdev);
  136. return ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
  137. }
  138. static void vp_set_status(struct virtio_device *vdev, u8 status)
  139. {
  140. struct virtio_pci_device *vp_dev = to_vp_device(vdev);
  141. /* We should never be setting status to 0. */
  142. BUG_ON(status == 0);
  143. iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
  144. }
  145. /* wait for pending irq handlers */
  146. static void vp_synchronize_vectors(struct virtio_device *vdev)
  147. {
  148. struct virtio_pci_device *vp_dev = to_vp_device(vdev);
  149. int i;
  150. if (vp_dev->intx_enabled)
  151. synchronize_irq(vp_dev->pci_dev->irq);
  152. for (i = 0; i < vp_dev->msix_vectors; ++i)
  153. synchronize_irq(vp_dev->msix_entries[i].vector);
  154. }
  155. static void vp_reset(struct virtio_device *vdev)
  156. {
  157. struct virtio_pci_device *vp_dev = to_vp_device(vdev);
  158. /* 0 status means a reset. */
  159. iowrite8(0, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
  160. /* Flush out the status write, and flush in device writes,
  161. * including MSi-X interrupts, if any. */
  162. ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
  163. /* Flush pending VQ/configuration callbacks. */
  164. vp_synchronize_vectors(vdev);
  165. }
  166. /* the notify function used when creating a virt queue */
  167. static void vp_notify(struct virtqueue *vq)
  168. {
  169. struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
  170. /* we write the queue's selector into the notification register to
  171. * signal the other end */
  172. iowrite16(virtqueue_get_queue_index(vq),
  173. vp_dev->ioaddr + VIRTIO_PCI_QUEUE_NOTIFY);
  174. }
  175. /* Handle a configuration change: Tell driver if it wants to know. */
  176. static irqreturn_t vp_config_changed(int irq, void *opaque)
  177. {
  178. struct virtio_pci_device *vp_dev = opaque;
  179. struct virtio_driver *drv;
  180. drv = container_of(vp_dev->vdev.dev.driver,
  181. struct virtio_driver, driver);
  182. if (drv && drv->config_changed)
  183. drv->config_changed(&vp_dev->vdev);
  184. return IRQ_HANDLED;
  185. }
  186. /* Notify all virtqueues on an interrupt. */
  187. static irqreturn_t vp_vring_interrupt(int irq, void *opaque)
  188. {
  189. struct virtio_pci_device *vp_dev = opaque;
  190. struct virtio_pci_vq_info *info;
  191. irqreturn_t ret = IRQ_NONE;
  192. unsigned long flags;
  193. spin_lock_irqsave(&vp_dev->lock, flags);
  194. list_for_each_entry(info, &vp_dev->virtqueues, node) {
  195. if (vring_interrupt(irq, info->vq) == IRQ_HANDLED)
  196. ret = IRQ_HANDLED;
  197. }
  198. spin_unlock_irqrestore(&vp_dev->lock, flags);
  199. return ret;
  200. }
  201. /* A small wrapper to also acknowledge the interrupt when it's handled.
  202. * I really need an EIO hook for the vring so I can ack the interrupt once we
  203. * know that we'll be handling the IRQ but before we invoke the callback since
  204. * the callback may notify the host which results in the host attempting to
  205. * raise an interrupt that we would then mask once we acknowledged the
  206. * interrupt. */
  207. static irqreturn_t vp_interrupt(int irq, void *opaque)
  208. {
  209. struct virtio_pci_device *vp_dev = opaque;
  210. u8 isr;
  211. /* reading the ISR has the effect of also clearing it so it's very
  212. * important to save off the value. */
  213. isr = ioread8(vp_dev->ioaddr + VIRTIO_PCI_ISR);
  214. /* It's definitely not us if the ISR was not high */
  215. if (!isr)
  216. return IRQ_NONE;
  217. /* Configuration change? Tell driver if it wants to know. */
  218. if (isr & VIRTIO_PCI_ISR_CONFIG)
  219. vp_config_changed(irq, opaque);
  220. return vp_vring_interrupt(irq, opaque);
  221. }
  222. static void vp_free_vectors(struct virtio_device *vdev)
  223. {
  224. struct virtio_pci_device *vp_dev = to_vp_device(vdev);
  225. int i;
  226. if (vp_dev->intx_enabled) {
  227. free_irq(vp_dev->pci_dev->irq, vp_dev);
  228. vp_dev->intx_enabled = 0;
  229. }
  230. for (i = 0; i < vp_dev->msix_used_vectors; ++i)
  231. free_irq(vp_dev->msix_entries[i].vector, vp_dev);
  232. for (i = 0; i < vp_dev->msix_vectors; i++)
  233. if (vp_dev->msix_affinity_masks[i])
  234. free_cpumask_var(vp_dev->msix_affinity_masks[i]);
  235. if (vp_dev->msix_enabled) {
  236. /* Disable the vector used for configuration */
  237. iowrite16(VIRTIO_MSI_NO_VECTOR,
  238. vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
  239. /* Flush the write out to device */
  240. ioread16(vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
  241. pci_disable_msix(vp_dev->pci_dev);
  242. vp_dev->msix_enabled = 0;
  243. vp_dev->msix_vectors = 0;
  244. }
  245. vp_dev->msix_used_vectors = 0;
  246. kfree(vp_dev->msix_names);
  247. vp_dev->msix_names = NULL;
  248. kfree(vp_dev->msix_entries);
  249. vp_dev->msix_entries = NULL;
  250. kfree(vp_dev->msix_affinity_masks);
  251. vp_dev->msix_affinity_masks = NULL;
  252. }
  253. static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
  254. bool per_vq_vectors)
  255. {
  256. struct virtio_pci_device *vp_dev = to_vp_device(vdev);
  257. const char *name = dev_name(&vp_dev->vdev.dev);
  258. unsigned i, v;
  259. int err = -ENOMEM;
  260. vp_dev->msix_entries = kmalloc(nvectors * sizeof *vp_dev->msix_entries,
  261. GFP_KERNEL);
  262. if (!vp_dev->msix_entries)
  263. goto error;
  264. vp_dev->msix_names = kmalloc(nvectors * sizeof *vp_dev->msix_names,
  265. GFP_KERNEL);
  266. if (!vp_dev->msix_names)
  267. goto error;
  268. vp_dev->msix_affinity_masks
  269. = kzalloc(nvectors * sizeof *vp_dev->msix_affinity_masks,
  270. GFP_KERNEL);
  271. if (!vp_dev->msix_affinity_masks)
  272. goto error;
  273. for (i = 0; i < nvectors; ++i)
  274. if (!alloc_cpumask_var(&vp_dev->msix_affinity_masks[i],
  275. GFP_KERNEL))
  276. goto error;
  277. for (i = 0; i < nvectors; ++i)
  278. vp_dev->msix_entries[i].entry = i;
  279. /* pci_enable_msix returns positive if we can't get this many. */
  280. err = pci_enable_msix(vp_dev->pci_dev, vp_dev->msix_entries, nvectors);
  281. if (err > 0)
  282. err = -ENOSPC;
  283. if (err)
  284. goto error;
  285. vp_dev->msix_vectors = nvectors;
  286. vp_dev->msix_enabled = 1;
  287. /* Set the vector used for configuration */
  288. v = vp_dev->msix_used_vectors;
  289. snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
  290. "%s-config", name);
  291. err = request_irq(vp_dev->msix_entries[v].vector,
  292. vp_config_changed, 0, vp_dev->msix_names[v],
  293. vp_dev);
  294. if (err)
  295. goto error;
  296. ++vp_dev->msix_used_vectors;
  297. iowrite16(v, vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
  298. /* Verify we had enough resources to assign the vector */
  299. v = ioread16(vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
  300. if (v == VIRTIO_MSI_NO_VECTOR) {
  301. err = -EBUSY;
  302. goto error;
  303. }
  304. if (!per_vq_vectors) {
  305. /* Shared vector for all VQs */
  306. v = vp_dev->msix_used_vectors;
  307. snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
  308. "%s-virtqueues", name);
  309. err = request_irq(vp_dev->msix_entries[v].vector,
  310. vp_vring_interrupt, 0, vp_dev->msix_names[v],
  311. vp_dev);
  312. if (err)
  313. goto error;
  314. ++vp_dev->msix_used_vectors;
  315. }
  316. return 0;
  317. error:
  318. vp_free_vectors(vdev);
  319. return err;
  320. }
  321. static int vp_request_intx(struct virtio_device *vdev)
  322. {
  323. int err;
  324. struct virtio_pci_device *vp_dev = to_vp_device(vdev);
  325. err = request_irq(vp_dev->pci_dev->irq, vp_interrupt,
  326. IRQF_SHARED, dev_name(&vdev->dev), vp_dev);
  327. if (!err)
  328. vp_dev->intx_enabled = 1;
  329. return err;
  330. }
  331. static struct virtqueue *setup_vq(struct virtio_device *vdev, unsigned index,
  332. void (*callback)(struct virtqueue *vq),
  333. const char *name,
  334. u16 msix_vec)
  335. {
  336. struct virtio_pci_device *vp_dev = to_vp_device(vdev);
  337. struct virtio_pci_vq_info *info;
  338. struct virtqueue *vq;
  339. unsigned long flags, size;
  340. u16 num;
  341. int err;
  342. /* Select the queue we're interested in */
  343. iowrite16(index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
  344. /* Check if queue is either not available or already active. */
  345. num = ioread16(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_NUM);
  346. if (!num || ioread32(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN))
  347. return ERR_PTR(-ENOENT);
  348. /* allocate and fill out our structure the represents an active
  349. * queue */
  350. info = kmalloc(sizeof(struct virtio_pci_vq_info), GFP_KERNEL);
  351. if (!info)
  352. return ERR_PTR(-ENOMEM);
  353. info->num = num;
  354. info->msix_vector = msix_vec;
  355. size = PAGE_ALIGN(vring_size(num, VIRTIO_PCI_VRING_ALIGN));
  356. info->queue = alloc_pages_exact(size, GFP_KERNEL|__GFP_ZERO);
  357. if (info->queue == NULL) {
  358. err = -ENOMEM;
  359. goto out_info;
  360. }
  361. /* activate the queue */
  362. iowrite32(virt_to_phys(info->queue) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT,
  363. vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
  364. /* create the vring */
  365. vq = vring_new_virtqueue(index, info->num, VIRTIO_PCI_VRING_ALIGN, vdev,
  366. true, info->queue, vp_notify, callback, name);
  367. if (!vq) {
  368. err = -ENOMEM;
  369. goto out_activate_queue;
  370. }
  371. vq->priv = info;
  372. info->vq = vq;
  373. if (msix_vec != VIRTIO_MSI_NO_VECTOR) {
  374. iowrite16(msix_vec, vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
  375. msix_vec = ioread16(vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
  376. if (msix_vec == VIRTIO_MSI_NO_VECTOR) {
  377. err = -EBUSY;
  378. goto out_assign;
  379. }
  380. }
  381. if (callback) {
  382. spin_lock_irqsave(&vp_dev->lock, flags);
  383. list_add(&info->node, &vp_dev->virtqueues);
  384. spin_unlock_irqrestore(&vp_dev->lock, flags);
  385. } else {
  386. INIT_LIST_HEAD(&info->node);
  387. }
  388. return vq;
  389. out_assign:
  390. vring_del_virtqueue(vq);
  391. out_activate_queue:
  392. iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
  393. free_pages_exact(info->queue, size);
  394. out_info:
  395. kfree(info);
  396. return ERR_PTR(err);
  397. }
  398. static void vp_del_vq(struct virtqueue *vq)
  399. {
  400. struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
  401. struct virtio_pci_vq_info *info = vq->priv;
  402. unsigned long flags, size;
  403. spin_lock_irqsave(&vp_dev->lock, flags);
  404. list_del(&info->node);
  405. spin_unlock_irqrestore(&vp_dev->lock, flags);
  406. iowrite16(virtqueue_get_queue_index(vq),
  407. vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
  408. if (vp_dev->msix_enabled) {
  409. iowrite16(VIRTIO_MSI_NO_VECTOR,
  410. vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
  411. /* Flush the write out to device */
  412. ioread8(vp_dev->ioaddr + VIRTIO_PCI_ISR);
  413. }
  414. vring_del_virtqueue(vq);
  415. /* Select and deactivate the queue */
  416. iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
  417. size = PAGE_ALIGN(vring_size(info->num, VIRTIO_PCI_VRING_ALIGN));
  418. free_pages_exact(info->queue, size);
  419. kfree(info);
  420. }
  421. /* the config->del_vqs() implementation */
  422. static void vp_del_vqs(struct virtio_device *vdev)
  423. {
  424. struct virtio_pci_device *vp_dev = to_vp_device(vdev);
  425. struct virtqueue *vq, *n;
  426. struct virtio_pci_vq_info *info;
  427. list_for_each_entry_safe(vq, n, &vdev->vqs, list) {
  428. info = vq->priv;
  429. if (vp_dev->per_vq_vectors &&
  430. info->msix_vector != VIRTIO_MSI_NO_VECTOR)
  431. free_irq(vp_dev->msix_entries[info->msix_vector].vector,
  432. vq);
  433. vp_del_vq(vq);
  434. }
  435. vp_dev->per_vq_vectors = false;
  436. vp_free_vectors(vdev);
  437. }
  438. static int vp_try_to_find_vqs(struct virtio_device *vdev, unsigned nvqs,
  439. struct virtqueue *vqs[],
  440. vq_callback_t *callbacks[],
  441. const char *names[],
  442. bool use_msix,
  443. bool per_vq_vectors)
  444. {
  445. struct virtio_pci_device *vp_dev = to_vp_device(vdev);
  446. u16 msix_vec;
  447. int i, err, nvectors, allocated_vectors;
  448. if (!use_msix) {
  449. /* Old style: one normal interrupt for change and all vqs. */
  450. err = vp_request_intx(vdev);
  451. if (err)
  452. goto error_request;
  453. } else {
  454. if (per_vq_vectors) {
  455. /* Best option: one for change interrupt, one per vq. */
  456. nvectors = 1;
  457. for (i = 0; i < nvqs; ++i)
  458. if (callbacks[i])
  459. ++nvectors;
  460. } else {
  461. /* Second best: one for change, shared for all vqs. */
  462. nvectors = 2;
  463. }
  464. err = vp_request_msix_vectors(vdev, nvectors, per_vq_vectors);
  465. if (err)
  466. goto error_request;
  467. }
  468. vp_dev->per_vq_vectors = per_vq_vectors;
  469. allocated_vectors = vp_dev->msix_used_vectors;
  470. for (i = 0; i < nvqs; ++i) {
  471. if (!names[i]) {
  472. vqs[i] = NULL;
  473. continue;
  474. } else if (!callbacks[i] || !vp_dev->msix_enabled)
  475. msix_vec = VIRTIO_MSI_NO_VECTOR;
  476. else if (vp_dev->per_vq_vectors)
  477. msix_vec = allocated_vectors++;
  478. else
  479. msix_vec = VP_MSIX_VQ_VECTOR;
  480. vqs[i] = setup_vq(vdev, i, callbacks[i], names[i], msix_vec);
  481. if (IS_ERR(vqs[i])) {
  482. err = PTR_ERR(vqs[i]);
  483. goto error_find;
  484. }
  485. if (!vp_dev->per_vq_vectors || msix_vec == VIRTIO_MSI_NO_VECTOR)
  486. continue;
  487. /* allocate per-vq irq if available and necessary */
  488. snprintf(vp_dev->msix_names[msix_vec],
  489. sizeof *vp_dev->msix_names,
  490. "%s-%s",
  491. dev_name(&vp_dev->vdev.dev), names[i]);
  492. err = request_irq(vp_dev->msix_entries[msix_vec].vector,
  493. vring_interrupt, 0,
  494. vp_dev->msix_names[msix_vec],
  495. vqs[i]);
  496. if (err) {
  497. vp_del_vq(vqs[i]);
  498. goto error_find;
  499. }
  500. }
  501. return 0;
  502. error_find:
  503. vp_del_vqs(vdev);
  504. error_request:
  505. return err;
  506. }
  507. /* the config->find_vqs() implementation */
  508. static int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
  509. struct virtqueue *vqs[],
  510. vq_callback_t *callbacks[],
  511. const char *names[])
  512. {
  513. int err;
  514. /* Try MSI-X with one vector per queue. */
  515. err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names, true, true);
  516. if (!err)
  517. return 0;
  518. /* Fallback: MSI-X with one vector for config, one shared for queues. */
  519. err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
  520. true, false);
  521. if (!err)
  522. return 0;
  523. /* Finally fall back to regular interrupts. */
  524. return vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
  525. false, false);
  526. }
  527. static const char *vp_bus_name(struct virtio_device *vdev)
  528. {
  529. struct virtio_pci_device *vp_dev = to_vp_device(vdev);
  530. return pci_name(vp_dev->pci_dev);
  531. }
  532. /* Setup the affinity for a virtqueue:
  533. * - force the affinity for per vq vector
  534. * - OR over all affinities for shared MSI
  535. * - ignore the affinity request if we're using INTX
  536. */
  537. static int vp_set_vq_affinity(struct virtqueue *vq, int cpu)
  538. {
  539. struct virtio_device *vdev = vq->vdev;
  540. struct virtio_pci_device *vp_dev = to_vp_device(vdev);
  541. struct virtio_pci_vq_info *info = vq->priv;
  542. struct cpumask *mask;
  543. unsigned int irq;
  544. if (!vq->callback)
  545. return -EINVAL;
  546. if (vp_dev->msix_enabled) {
  547. mask = vp_dev->msix_affinity_masks[info->msix_vector];
  548. irq = vp_dev->msix_entries[info->msix_vector].vector;
  549. if (cpu == -1)
  550. irq_set_affinity_hint(irq, NULL);
  551. else {
  552. cpumask_set_cpu(cpu, mask);
  553. irq_set_affinity_hint(irq, mask);
  554. }
  555. }
  556. return 0;
  557. }
  558. static struct virtio_config_ops virtio_pci_config_ops = {
  559. .get = vp_get,
  560. .set = vp_set,
  561. .get_status = vp_get_status,
  562. .set_status = vp_set_status,
  563. .reset = vp_reset,
  564. .find_vqs = vp_find_vqs,
  565. .del_vqs = vp_del_vqs,
  566. .get_features = vp_get_features,
  567. .finalize_features = vp_finalize_features,
  568. .bus_name = vp_bus_name,
  569. .set_vq_affinity = vp_set_vq_affinity,
  570. };
  571. static void virtio_pci_release_dev(struct device *_d)
  572. {
  573. /*
  574. * No need for a release method as we allocate/free
  575. * all devices together with the pci devices.
  576. * Provide an empty one to avoid getting a warning from core.
  577. */
  578. }
  579. /* the PCI probing function */
  580. static int __devinit virtio_pci_probe(struct pci_dev *pci_dev,
  581. const struct pci_device_id *id)
  582. {
  583. struct virtio_pci_device *vp_dev;
  584. int err;
  585. /* We only own devices >= 0x1000 and <= 0x103f: leave the rest. */
  586. if (pci_dev->device < 0x1000 || pci_dev->device > 0x103f)
  587. return -ENODEV;
  588. if (pci_dev->revision != VIRTIO_PCI_ABI_VERSION) {
  589. printk(KERN_ERR "virtio_pci: expected ABI version %d, got %d\n",
  590. VIRTIO_PCI_ABI_VERSION, pci_dev->revision);
  591. return -ENODEV;
  592. }
  593. /* allocate our structure and fill it out */
  594. vp_dev = kzalloc(sizeof(struct virtio_pci_device), GFP_KERNEL);
  595. if (vp_dev == NULL)
  596. return -ENOMEM;
  597. vp_dev->vdev.dev.parent = &pci_dev->dev;
  598. vp_dev->vdev.dev.release = virtio_pci_release_dev;
  599. vp_dev->vdev.config = &virtio_pci_config_ops;
  600. vp_dev->pci_dev = pci_dev;
  601. INIT_LIST_HEAD(&vp_dev->virtqueues);
  602. spin_lock_init(&vp_dev->lock);
  603. /* Disable MSI/MSIX to bring device to a known good state. */
  604. pci_msi_off(pci_dev);
  605. /* enable the device */
  606. err = pci_enable_device(pci_dev);
  607. if (err)
  608. goto out;
  609. err = pci_request_regions(pci_dev, "virtio-pci");
  610. if (err)
  611. goto out_enable_device;
  612. vp_dev->ioaddr = pci_iomap(pci_dev, 0, 0);
  613. if (vp_dev->ioaddr == NULL) {
  614. err = -ENOMEM;
  615. goto out_req_regions;
  616. }
  617. pci_set_drvdata(pci_dev, vp_dev);
  618. pci_set_master(pci_dev);
  619. /* we use the subsystem vendor/device id as the virtio vendor/device
  620. * id. this allows us to use the same PCI vendor/device id for all
  621. * virtio devices and to identify the particular virtio driver by
  622. * the subsystem ids */
  623. vp_dev->vdev.id.vendor = pci_dev->subsystem_vendor;
  624. vp_dev->vdev.id.device = pci_dev->subsystem_device;
  625. /* finally register the virtio device */
  626. err = register_virtio_device(&vp_dev->vdev);
  627. if (err)
  628. goto out_set_drvdata;
  629. return 0;
  630. out_set_drvdata:
  631. pci_set_drvdata(pci_dev, NULL);
  632. pci_iounmap(pci_dev, vp_dev->ioaddr);
  633. out_req_regions:
  634. pci_release_regions(pci_dev);
  635. out_enable_device:
  636. pci_disable_device(pci_dev);
  637. out:
  638. kfree(vp_dev);
  639. return err;
  640. }
  641. static void __devexit virtio_pci_remove(struct pci_dev *pci_dev)
  642. {
  643. struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
  644. unregister_virtio_device(&vp_dev->vdev);
  645. vp_del_vqs(&vp_dev->vdev);
  646. pci_set_drvdata(pci_dev, NULL);
  647. pci_iounmap(pci_dev, vp_dev->ioaddr);
  648. pci_release_regions(pci_dev);
  649. pci_disable_device(pci_dev);
  650. kfree(vp_dev);
  651. }
  652. #ifdef CONFIG_PM
  653. static int virtio_pci_freeze(struct device *dev)
  654. {
  655. struct pci_dev *pci_dev = to_pci_dev(dev);
  656. struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
  657. struct virtio_driver *drv;
  658. int ret;
  659. drv = container_of(vp_dev->vdev.dev.driver,
  660. struct virtio_driver, driver);
  661. ret = 0;
  662. vp_dev->saved_status = vp_get_status(&vp_dev->vdev);
  663. if (drv && drv->freeze)
  664. ret = drv->freeze(&vp_dev->vdev);
  665. if (!ret)
  666. pci_disable_device(pci_dev);
  667. return ret;
  668. }
  669. static int virtio_pci_restore(struct device *dev)
  670. {
  671. struct pci_dev *pci_dev = to_pci_dev(dev);
  672. struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
  673. struct virtio_driver *drv;
  674. int ret;
  675. drv = container_of(vp_dev->vdev.dev.driver,
  676. struct virtio_driver, driver);
  677. ret = pci_enable_device(pci_dev);
  678. if (ret)
  679. return ret;
  680. pci_set_master(pci_dev);
  681. vp_finalize_features(&vp_dev->vdev);
  682. if (drv && drv->restore)
  683. ret = drv->restore(&vp_dev->vdev);
  684. /* Finally, tell the device we're all set */
  685. if (!ret)
  686. vp_set_status(&vp_dev->vdev, vp_dev->saved_status);
  687. return ret;
  688. }
  689. static const struct dev_pm_ops virtio_pci_pm_ops = {
  690. SET_SYSTEM_SLEEP_PM_OPS(virtio_pci_freeze, virtio_pci_restore)
  691. };
  692. #endif
  693. static struct pci_driver virtio_pci_driver = {
  694. .name = "virtio-pci",
  695. .id_table = virtio_pci_id_table,
  696. .probe = virtio_pci_probe,
  697. .remove = __devexit_p(virtio_pci_remove),
  698. #ifdef CONFIG_PM
  699. .driver.pm = &virtio_pci_pm_ops,
  700. #endif
  701. };
  702. module_pci_driver(virtio_pci_driver);