virtio_net.c 43 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756
  1. /* A network driver using virtio.
  2. *
  3. * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
  4. *
  5. * This program is free software; you can redistribute it and/or modify
  6. * it under the terms of the GNU General Public License as published by
  7. * the Free Software Foundation; either version 2 of the License, or
  8. * (at your option) any later version.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU General Public License
  16. * along with this program; if not, write to the Free Software
  17. * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  18. */
  19. //#define DEBUG
  20. #include <linux/netdevice.h>
  21. #include <linux/etherdevice.h>
  22. #include <linux/ethtool.h>
  23. #include <linux/module.h>
  24. #include <linux/virtio.h>
  25. #include <linux/virtio_net.h>
  26. #include <linux/scatterlist.h>
  27. #include <linux/if_vlan.h>
  28. #include <linux/slab.h>
  29. #include <linux/cpu.h>
  30. static int napi_weight = 128;
  31. module_param(napi_weight, int, 0444);
  32. static bool csum = true, gso = true;
  33. module_param(csum, bool, 0444);
  34. module_param(gso, bool, 0444);
  35. /* FIXME: MTU in config. */
  36. #define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
  37. #define GOOD_COPY_LEN 128
  38. #define VIRTNET_SEND_COMMAND_SG_MAX 2
  39. #define VIRTNET_DRIVER_VERSION "1.0.0"
  40. struct virtnet_stats {
  41. struct u64_stats_sync tx_syncp;
  42. struct u64_stats_sync rx_syncp;
  43. u64 tx_bytes;
  44. u64 tx_packets;
  45. u64 rx_bytes;
  46. u64 rx_packets;
  47. };
  48. /* Internal representation of a send virtqueue */
  49. struct send_queue {
  50. /* Virtqueue associated with this send _queue */
  51. struct virtqueue *vq;
  52. /* TX: fragments + linear part + virtio header */
  53. struct scatterlist sg[MAX_SKB_FRAGS + 2];
  54. /* Name of the send queue: output.$index */
  55. char name[40];
  56. };
  57. /* Internal representation of a receive virtqueue */
  58. struct receive_queue {
  59. /* Virtqueue associated with this receive_queue */
  60. struct virtqueue *vq;
  61. struct napi_struct napi;
  62. /* Number of input buffers, and max we've ever had. */
  63. unsigned int num, max;
  64. /* Chain pages by the private ptr. */
  65. struct page *pages;
  66. /* RX: fragments + linear part + virtio header */
  67. struct scatterlist sg[MAX_SKB_FRAGS + 2];
  68. /* Name of this receive queue: input.$index */
  69. char name[40];
  70. };
  71. struct virtnet_info {
  72. struct virtio_device *vdev;
  73. struct virtqueue *cvq;
  74. struct net_device *dev;
  75. struct send_queue *sq;
  76. struct receive_queue *rq;
  77. unsigned int status;
  78. /* Max # of queue pairs supported by the device */
  79. u16 max_queue_pairs;
  80. /* # of queue pairs currently used by the driver */
  81. u16 curr_queue_pairs;
  82. /* I like... big packets and I cannot lie! */
  83. bool big_packets;
  84. /* Host will merge rx buffers for big packets (shake it! shake it!) */
  85. bool mergeable_rx_bufs;
  86. /* Has control virtqueue */
  87. bool has_cvq;
  88. /* enable config space updates */
  89. bool config_enable;
  90. /* Active statistics */
  91. struct virtnet_stats __percpu *stats;
  92. /* Work struct for refilling if we run low on memory. */
  93. struct delayed_work refill;
  94. /* Work struct for config space updates */
  95. struct work_struct config_work;
  96. /* Lock for config space updates */
  97. struct mutex config_lock;
  98. /* Does the affinity hint is set for virtqueues? */
  99. bool affinity_hint_set;
  100. /* Per-cpu variable to show the mapping from CPU to virtqueue */
  101. int __percpu *vq_index;
  102. /* CPU hot plug notifier */
  103. struct notifier_block nb;
  104. };
  105. struct skb_vnet_hdr {
  106. union {
  107. struct virtio_net_hdr hdr;
  108. struct virtio_net_hdr_mrg_rxbuf mhdr;
  109. };
  110. };
  111. struct padded_vnet_hdr {
  112. struct virtio_net_hdr hdr;
  113. /*
  114. * virtio_net_hdr should be in a separated sg buffer because of a
  115. * QEMU bug, and data sg buffer shares same page with this header sg.
  116. * This padding makes next sg 16 byte aligned after virtio_net_hdr.
  117. */
  118. char padding[6];
  119. };
  120. /* Converting between virtqueue no. and kernel tx/rx queue no.
  121. * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq
  122. */
  123. static int vq2txq(struct virtqueue *vq)
  124. {
  125. return (virtqueue_get_queue_index(vq) - 1) / 2;
  126. }
  127. static int txq2vq(int txq)
  128. {
  129. return txq * 2 + 1;
  130. }
  131. static int vq2rxq(struct virtqueue *vq)
  132. {
  133. return virtqueue_get_queue_index(vq) / 2;
  134. }
  135. static int rxq2vq(int rxq)
  136. {
  137. return rxq * 2;
  138. }
  139. static inline struct skb_vnet_hdr *skb_vnet_hdr(struct sk_buff *skb)
  140. {
  141. return (struct skb_vnet_hdr *)skb->cb;
  142. }
  143. /*
  144. * private is used to chain pages for big packets, put the whole
  145. * most recent used list in the beginning for reuse
  146. */
  147. static void give_pages(struct receive_queue *rq, struct page *page)
  148. {
  149. struct page *end;
  150. /* Find end of list, sew whole thing into vi->rq.pages. */
  151. for (end = page; end->private; end = (struct page *)end->private);
  152. end->private = (unsigned long)rq->pages;
  153. rq->pages = page;
  154. }
  155. static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask)
  156. {
  157. struct page *p = rq->pages;
  158. if (p) {
  159. rq->pages = (struct page *)p->private;
  160. /* clear private here, it is used to chain pages */
  161. p->private = 0;
  162. } else
  163. p = alloc_page(gfp_mask);
  164. return p;
  165. }
  166. static void skb_xmit_done(struct virtqueue *vq)
  167. {
  168. struct virtnet_info *vi = vq->vdev->priv;
  169. /* Suppress further interrupts. */
  170. virtqueue_disable_cb(vq);
  171. /* We were probably waiting for more output buffers. */
  172. netif_wake_subqueue(vi->dev, vq2txq(vq));
  173. }
  174. static void set_skb_frag(struct sk_buff *skb, struct page *page,
  175. unsigned int offset, unsigned int *len)
  176. {
  177. int size = min((unsigned)PAGE_SIZE - offset, *len);
  178. int i = skb_shinfo(skb)->nr_frags;
  179. __skb_fill_page_desc(skb, i, page, offset, size);
  180. skb->data_len += size;
  181. skb->len += size;
  182. skb->truesize += PAGE_SIZE;
  183. skb_shinfo(skb)->nr_frags++;
  184. skb_shinfo(skb)->gso_type |= SKB_GSO_SHARED_FRAG;
  185. *len -= size;
  186. }
  187. /* Called from bottom half context */
  188. static struct sk_buff *page_to_skb(struct receive_queue *rq,
  189. struct page *page, unsigned int len)
  190. {
  191. struct virtnet_info *vi = rq->vq->vdev->priv;
  192. struct sk_buff *skb;
  193. struct skb_vnet_hdr *hdr;
  194. unsigned int copy, hdr_len, offset;
  195. char *p;
  196. p = page_address(page);
  197. /* copy small packet so we can reuse these pages for small data */
  198. skb = netdev_alloc_skb_ip_align(vi->dev, GOOD_COPY_LEN);
  199. if (unlikely(!skb))
  200. return NULL;
  201. hdr = skb_vnet_hdr(skb);
  202. if (vi->mergeable_rx_bufs) {
  203. hdr_len = sizeof hdr->mhdr;
  204. offset = hdr_len;
  205. } else {
  206. hdr_len = sizeof hdr->hdr;
  207. offset = sizeof(struct padded_vnet_hdr);
  208. }
  209. memcpy(hdr, p, hdr_len);
  210. len -= hdr_len;
  211. p += offset;
  212. copy = len;
  213. if (copy > skb_tailroom(skb))
  214. copy = skb_tailroom(skb);
  215. memcpy(skb_put(skb, copy), p, copy);
  216. len -= copy;
  217. offset += copy;
  218. /*
  219. * Verify that we can indeed put this data into a skb.
  220. * This is here to handle cases when the device erroneously
  221. * tries to receive more than is possible. This is usually
  222. * the case of a broken device.
  223. */
  224. if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) {
  225. net_dbg_ratelimited("%s: too much data\n", skb->dev->name);
  226. dev_kfree_skb(skb);
  227. return NULL;
  228. }
  229. while (len) {
  230. set_skb_frag(skb, page, offset, &len);
  231. page = (struct page *)page->private;
  232. offset = 0;
  233. }
  234. if (page)
  235. give_pages(rq, page);
  236. return skb;
  237. }
  238. static int receive_mergeable(struct receive_queue *rq, struct sk_buff *skb)
  239. {
  240. struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb);
  241. struct page *page;
  242. int num_buf, i, len;
  243. num_buf = hdr->mhdr.num_buffers;
  244. while (--num_buf) {
  245. i = skb_shinfo(skb)->nr_frags;
  246. if (i >= MAX_SKB_FRAGS) {
  247. pr_debug("%s: packet too long\n", skb->dev->name);
  248. skb->dev->stats.rx_length_errors++;
  249. return -EINVAL;
  250. }
  251. page = virtqueue_get_buf(rq->vq, &len);
  252. if (!page) {
  253. pr_debug("%s: rx error: %d buffers missing\n",
  254. skb->dev->name, hdr->mhdr.num_buffers);
  255. skb->dev->stats.rx_length_errors++;
  256. return -EINVAL;
  257. }
  258. if (len > PAGE_SIZE)
  259. len = PAGE_SIZE;
  260. set_skb_frag(skb, page, 0, &len);
  261. --rq->num;
  262. }
  263. return 0;
  264. }
  265. static void receive_buf(struct receive_queue *rq, void *buf, unsigned int len)
  266. {
  267. struct virtnet_info *vi = rq->vq->vdev->priv;
  268. struct net_device *dev = vi->dev;
  269. struct virtnet_stats *stats = this_cpu_ptr(vi->stats);
  270. struct sk_buff *skb;
  271. struct page *page;
  272. struct skb_vnet_hdr *hdr;
  273. if (unlikely(len < sizeof(struct virtio_net_hdr) + ETH_HLEN)) {
  274. pr_debug("%s: short packet %i\n", dev->name, len);
  275. dev->stats.rx_length_errors++;
  276. if (vi->mergeable_rx_bufs || vi->big_packets)
  277. give_pages(rq, buf);
  278. else
  279. dev_kfree_skb(buf);
  280. return;
  281. }
  282. if (!vi->mergeable_rx_bufs && !vi->big_packets) {
  283. skb = buf;
  284. len -= sizeof(struct virtio_net_hdr);
  285. skb_trim(skb, len);
  286. } else {
  287. page = buf;
  288. skb = page_to_skb(rq, page, len);
  289. if (unlikely(!skb)) {
  290. dev->stats.rx_dropped++;
  291. give_pages(rq, page);
  292. return;
  293. }
  294. if (vi->mergeable_rx_bufs)
  295. if (receive_mergeable(rq, skb)) {
  296. dev_kfree_skb(skb);
  297. return;
  298. }
  299. }
  300. hdr = skb_vnet_hdr(skb);
  301. u64_stats_update_begin(&stats->rx_syncp);
  302. stats->rx_bytes += skb->len;
  303. stats->rx_packets++;
  304. u64_stats_update_end(&stats->rx_syncp);
  305. if (hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
  306. pr_debug("Needs csum!\n");
  307. if (!skb_partial_csum_set(skb,
  308. hdr->hdr.csum_start,
  309. hdr->hdr.csum_offset))
  310. goto frame_err;
  311. } else if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID) {
  312. skb->ip_summed = CHECKSUM_UNNECESSARY;
  313. }
  314. skb->protocol = eth_type_trans(skb, dev);
  315. pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
  316. ntohs(skb->protocol), skb->len, skb->pkt_type);
  317. if (hdr->hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
  318. unsigned short gso_type = 0;
  319. pr_debug("GSO!\n");
  320. switch (hdr->hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
  321. case VIRTIO_NET_HDR_GSO_TCPV4:
  322. gso_type = SKB_GSO_TCPV4;
  323. break;
  324. case VIRTIO_NET_HDR_GSO_UDP:
  325. gso_type = SKB_GSO_UDP;
  326. break;
  327. case VIRTIO_NET_HDR_GSO_TCPV6:
  328. gso_type = SKB_GSO_TCPV6;
  329. break;
  330. default:
  331. net_warn_ratelimited("%s: bad gso type %u.\n",
  332. dev->name, hdr->hdr.gso_type);
  333. goto frame_err;
  334. }
  335. if (hdr->hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
  336. gso_type |= SKB_GSO_TCP_ECN;
  337. skb_shinfo(skb)->gso_size = hdr->hdr.gso_size;
  338. if (skb_shinfo(skb)->gso_size == 0) {
  339. net_warn_ratelimited("%s: zero gso size.\n", dev->name);
  340. goto frame_err;
  341. }
  342. skb_shinfo(skb)->gso_type |= gso_type;
  343. /* Header must be checked, and gso_segs computed. */
  344. skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
  345. skb_shinfo(skb)->gso_segs = 0;
  346. }
  347. netif_receive_skb(skb);
  348. return;
  349. frame_err:
  350. dev->stats.rx_frame_errors++;
  351. dev_kfree_skb(skb);
  352. }
  353. static int add_recvbuf_small(struct receive_queue *rq, gfp_t gfp)
  354. {
  355. struct virtnet_info *vi = rq->vq->vdev->priv;
  356. struct sk_buff *skb;
  357. struct skb_vnet_hdr *hdr;
  358. int err;
  359. skb = __netdev_alloc_skb_ip_align(vi->dev, MAX_PACKET_LEN, gfp);
  360. if (unlikely(!skb))
  361. return -ENOMEM;
  362. skb_put(skb, MAX_PACKET_LEN);
  363. hdr = skb_vnet_hdr(skb);
  364. sg_set_buf(rq->sg, &hdr->hdr, sizeof hdr->hdr);
  365. skb_to_sgvec(skb, rq->sg + 1, 0, skb->len);
  366. err = virtqueue_add_buf(rq->vq, rq->sg, 0, 2, skb, gfp);
  367. if (err < 0)
  368. dev_kfree_skb(skb);
  369. return err;
  370. }
  371. static int add_recvbuf_big(struct receive_queue *rq, gfp_t gfp)
  372. {
  373. struct page *first, *list = NULL;
  374. char *p;
  375. int i, err, offset;
  376. /* page in rq->sg[MAX_SKB_FRAGS + 1] is list tail */
  377. for (i = MAX_SKB_FRAGS + 1; i > 1; --i) {
  378. first = get_a_page(rq, gfp);
  379. if (!first) {
  380. if (list)
  381. give_pages(rq, list);
  382. return -ENOMEM;
  383. }
  384. sg_set_buf(&rq->sg[i], page_address(first), PAGE_SIZE);
  385. /* chain new page in list head to match sg */
  386. first->private = (unsigned long)list;
  387. list = first;
  388. }
  389. first = get_a_page(rq, gfp);
  390. if (!first) {
  391. give_pages(rq, list);
  392. return -ENOMEM;
  393. }
  394. p = page_address(first);
  395. /* rq->sg[0], rq->sg[1] share the same page */
  396. /* a separated rq->sg[0] for virtio_net_hdr only due to QEMU bug */
  397. sg_set_buf(&rq->sg[0], p, sizeof(struct virtio_net_hdr));
  398. /* rq->sg[1] for data packet, from offset */
  399. offset = sizeof(struct padded_vnet_hdr);
  400. sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset);
  401. /* chain first in list head */
  402. first->private = (unsigned long)list;
  403. err = virtqueue_add_buf(rq->vq, rq->sg, 0, MAX_SKB_FRAGS + 2,
  404. first, gfp);
  405. if (err < 0)
  406. give_pages(rq, first);
  407. return err;
  408. }
  409. static int add_recvbuf_mergeable(struct receive_queue *rq, gfp_t gfp)
  410. {
  411. struct page *page;
  412. int err;
  413. page = get_a_page(rq, gfp);
  414. if (!page)
  415. return -ENOMEM;
  416. sg_init_one(rq->sg, page_address(page), PAGE_SIZE);
  417. err = virtqueue_add_buf(rq->vq, rq->sg, 0, 1, page, gfp);
  418. if (err < 0)
  419. give_pages(rq, page);
  420. return err;
  421. }
  422. /*
  423. * Returns false if we couldn't fill entirely (OOM).
  424. *
  425. * Normally run in the receive path, but can also be run from ndo_open
  426. * before we're receiving packets, or from refill_work which is
  427. * careful to disable receiving (using napi_disable).
  428. */
  429. static bool try_fill_recv(struct receive_queue *rq, gfp_t gfp)
  430. {
  431. struct virtnet_info *vi = rq->vq->vdev->priv;
  432. int err;
  433. bool oom;
  434. do {
  435. if (vi->mergeable_rx_bufs)
  436. err = add_recvbuf_mergeable(rq, gfp);
  437. else if (vi->big_packets)
  438. err = add_recvbuf_big(rq, gfp);
  439. else
  440. err = add_recvbuf_small(rq, gfp);
  441. oom = err == -ENOMEM;
  442. if (err)
  443. break;
  444. ++rq->num;
  445. } while (rq->vq->num_free);
  446. if (unlikely(rq->num > rq->max))
  447. rq->max = rq->num;
  448. virtqueue_kick(rq->vq);
  449. return !oom;
  450. }
  451. static void skb_recv_done(struct virtqueue *rvq)
  452. {
  453. struct virtnet_info *vi = rvq->vdev->priv;
  454. struct receive_queue *rq = &vi->rq[vq2rxq(rvq)];
  455. /* Schedule NAPI, Suppress further interrupts if successful. */
  456. if (napi_schedule_prep(&rq->napi)) {
  457. virtqueue_disable_cb(rvq);
  458. __napi_schedule(&rq->napi);
  459. }
  460. }
  461. static void virtnet_napi_enable(struct receive_queue *rq)
  462. {
  463. napi_enable(&rq->napi);
  464. /* If all buffers were filled by other side before we napi_enabled, we
  465. * won't get another interrupt, so process any outstanding packets
  466. * now. virtnet_poll wants re-enable the queue, so we disable here.
  467. * We synchronize against interrupts via NAPI_STATE_SCHED */
  468. if (napi_schedule_prep(&rq->napi)) {
  469. virtqueue_disable_cb(rq->vq);
  470. local_bh_disable();
  471. __napi_schedule(&rq->napi);
  472. local_bh_enable();
  473. }
  474. }
  475. static void refill_work(struct work_struct *work)
  476. {
  477. struct virtnet_info *vi =
  478. container_of(work, struct virtnet_info, refill.work);
  479. bool still_empty;
  480. int i;
  481. for (i = 0; i < vi->max_queue_pairs; i++) {
  482. struct receive_queue *rq = &vi->rq[i];
  483. napi_disable(&rq->napi);
  484. still_empty = !try_fill_recv(rq, GFP_KERNEL);
  485. virtnet_napi_enable(rq);
  486. /* In theory, this can happen: if we don't get any buffers in
  487. * we will *never* try to fill again.
  488. */
  489. if (still_empty)
  490. schedule_delayed_work(&vi->refill, HZ/2);
  491. }
  492. }
  493. static int virtnet_poll(struct napi_struct *napi, int budget)
  494. {
  495. struct receive_queue *rq =
  496. container_of(napi, struct receive_queue, napi);
  497. struct virtnet_info *vi = rq->vq->vdev->priv;
  498. void *buf;
  499. unsigned int len, received = 0;
  500. again:
  501. while (received < budget &&
  502. (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) {
  503. receive_buf(rq, buf, len);
  504. --rq->num;
  505. received++;
  506. }
  507. if (rq->num < rq->max / 2) {
  508. if (!try_fill_recv(rq, GFP_ATOMIC))
  509. schedule_delayed_work(&vi->refill, 0);
  510. }
  511. /* Out of packets? */
  512. if (received < budget) {
  513. napi_complete(napi);
  514. if (unlikely(!virtqueue_enable_cb(rq->vq)) &&
  515. napi_schedule_prep(napi)) {
  516. virtqueue_disable_cb(rq->vq);
  517. __napi_schedule(napi);
  518. goto again;
  519. }
  520. }
  521. return received;
  522. }
  523. static int virtnet_open(struct net_device *dev)
  524. {
  525. struct virtnet_info *vi = netdev_priv(dev);
  526. int i;
  527. for (i = 0; i < vi->max_queue_pairs; i++) {
  528. /* Make sure we have some buffers: if oom use wq. */
  529. if (!try_fill_recv(&vi->rq[i], GFP_KERNEL))
  530. schedule_delayed_work(&vi->refill, 0);
  531. virtnet_napi_enable(&vi->rq[i]);
  532. }
  533. return 0;
  534. }
  535. static void free_old_xmit_skbs(struct send_queue *sq)
  536. {
  537. struct sk_buff *skb;
  538. unsigned int len;
  539. struct virtnet_info *vi = sq->vq->vdev->priv;
  540. struct virtnet_stats *stats = this_cpu_ptr(vi->stats);
  541. while ((skb = virtqueue_get_buf(sq->vq, &len)) != NULL) {
  542. pr_debug("Sent skb %p\n", skb);
  543. u64_stats_update_begin(&stats->tx_syncp);
  544. stats->tx_bytes += skb->len;
  545. stats->tx_packets++;
  546. u64_stats_update_end(&stats->tx_syncp);
  547. dev_kfree_skb_any(skb);
  548. }
  549. }
  550. static int xmit_skb(struct send_queue *sq, struct sk_buff *skb)
  551. {
  552. struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb);
  553. const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
  554. struct virtnet_info *vi = sq->vq->vdev->priv;
  555. unsigned num_sg;
  556. pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
  557. if (skb->ip_summed == CHECKSUM_PARTIAL) {
  558. hdr->hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
  559. hdr->hdr.csum_start = skb_checksum_start_offset(skb);
  560. hdr->hdr.csum_offset = skb->csum_offset;
  561. } else {
  562. hdr->hdr.flags = 0;
  563. hdr->hdr.csum_offset = hdr->hdr.csum_start = 0;
  564. }
  565. if (skb_is_gso(skb)) {
  566. hdr->hdr.hdr_len = skb_headlen(skb);
  567. hdr->hdr.gso_size = skb_shinfo(skb)->gso_size;
  568. if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
  569. hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
  570. else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
  571. hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
  572. else if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP)
  573. hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
  574. else
  575. BUG();
  576. if (skb_shinfo(skb)->gso_type & SKB_GSO_TCP_ECN)
  577. hdr->hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
  578. } else {
  579. hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
  580. hdr->hdr.gso_size = hdr->hdr.hdr_len = 0;
  581. }
  582. hdr->mhdr.num_buffers = 0;
  583. /* Encode metadata header at front. */
  584. if (vi->mergeable_rx_bufs)
  585. sg_set_buf(sq->sg, &hdr->mhdr, sizeof hdr->mhdr);
  586. else
  587. sg_set_buf(sq->sg, &hdr->hdr, sizeof hdr->hdr);
  588. num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len) + 1;
  589. return virtqueue_add_buf(sq->vq, sq->sg, num_sg,
  590. 0, skb, GFP_ATOMIC);
  591. }
  592. static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
  593. {
  594. struct virtnet_info *vi = netdev_priv(dev);
  595. int qnum = skb_get_queue_mapping(skb);
  596. struct send_queue *sq = &vi->sq[qnum];
  597. int err;
  598. /* Free up any pending old buffers before queueing new ones. */
  599. free_old_xmit_skbs(sq);
  600. /* Try to transmit */
  601. err = xmit_skb(sq, skb);
  602. /* This should not happen! */
  603. if (unlikely(err)) {
  604. dev->stats.tx_fifo_errors++;
  605. if (net_ratelimit())
  606. dev_warn(&dev->dev,
  607. "Unexpected TXQ (%d) queue failure: %d\n", qnum, err);
  608. dev->stats.tx_dropped++;
  609. kfree_skb(skb);
  610. return NETDEV_TX_OK;
  611. }
  612. virtqueue_kick(sq->vq);
  613. /* Don't wait up for transmitted skbs to be freed. */
  614. skb_orphan(skb);
  615. nf_reset(skb);
  616. /* Apparently nice girls don't return TX_BUSY; stop the queue
  617. * before it gets out of hand. Naturally, this wastes entries. */
  618. if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
  619. netif_stop_subqueue(dev, qnum);
  620. if (unlikely(!virtqueue_enable_cb_delayed(sq->vq))) {
  621. /* More just got used, free them then recheck. */
  622. free_old_xmit_skbs(sq);
  623. if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
  624. netif_start_subqueue(dev, qnum);
  625. virtqueue_disable_cb(sq->vq);
  626. }
  627. }
  628. }
  629. return NETDEV_TX_OK;
  630. }
  631. /*
  632. * Send command via the control virtqueue and check status. Commands
  633. * supported by the hypervisor, as indicated by feature bits, should
  634. * never fail unless improperly formated.
  635. */
  636. static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,
  637. struct scatterlist *data, int out, int in)
  638. {
  639. struct scatterlist *s, sg[VIRTNET_SEND_COMMAND_SG_MAX + 2];
  640. struct virtio_net_ctrl_hdr ctrl;
  641. virtio_net_ctrl_ack status = ~0;
  642. unsigned int tmp;
  643. int i;
  644. /* Caller should know better */
  645. BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ) ||
  646. (out + in > VIRTNET_SEND_COMMAND_SG_MAX));
  647. out++; /* Add header */
  648. in++; /* Add return status */
  649. ctrl.class = class;
  650. ctrl.cmd = cmd;
  651. sg_init_table(sg, out + in);
  652. sg_set_buf(&sg[0], &ctrl, sizeof(ctrl));
  653. for_each_sg(data, s, out + in - 2, i)
  654. sg_set_buf(&sg[i + 1], sg_virt(s), s->length);
  655. sg_set_buf(&sg[out + in - 1], &status, sizeof(status));
  656. BUG_ON(virtqueue_add_buf(vi->cvq, sg, out, in, vi, GFP_ATOMIC) < 0);
  657. virtqueue_kick(vi->cvq);
  658. /* Spin for a response, the kick causes an ioport write, trapping
  659. * into the hypervisor, so the request should be handled immediately.
  660. */
  661. while (!virtqueue_get_buf(vi->cvq, &tmp))
  662. cpu_relax();
  663. return status == VIRTIO_NET_OK;
  664. }
  665. static int virtnet_set_mac_address(struct net_device *dev, void *p)
  666. {
  667. struct virtnet_info *vi = netdev_priv(dev);
  668. struct virtio_device *vdev = vi->vdev;
  669. int ret;
  670. struct sockaddr *addr = p;
  671. struct scatterlist sg;
  672. ret = eth_prepare_mac_addr_change(dev, p);
  673. if (ret)
  674. return ret;
  675. if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
  676. sg_init_one(&sg, addr->sa_data, dev->addr_len);
  677. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
  678. VIRTIO_NET_CTRL_MAC_ADDR_SET,
  679. &sg, 1, 0)) {
  680. dev_warn(&vdev->dev,
  681. "Failed to set mac address by vq command.\n");
  682. return -EINVAL;
  683. }
  684. } else if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) {
  685. vdev->config->set(vdev, offsetof(struct virtio_net_config, mac),
  686. addr->sa_data, dev->addr_len);
  687. }
  688. eth_commit_mac_addr_change(dev, p);
  689. return 0;
  690. }
  691. static struct rtnl_link_stats64 *virtnet_stats(struct net_device *dev,
  692. struct rtnl_link_stats64 *tot)
  693. {
  694. struct virtnet_info *vi = netdev_priv(dev);
  695. int cpu;
  696. unsigned int start;
  697. for_each_possible_cpu(cpu) {
  698. struct virtnet_stats *stats = per_cpu_ptr(vi->stats, cpu);
  699. u64 tpackets, tbytes, rpackets, rbytes;
  700. do {
  701. start = u64_stats_fetch_begin_bh(&stats->tx_syncp);
  702. tpackets = stats->tx_packets;
  703. tbytes = stats->tx_bytes;
  704. } while (u64_stats_fetch_retry_bh(&stats->tx_syncp, start));
  705. do {
  706. start = u64_stats_fetch_begin_bh(&stats->rx_syncp);
  707. rpackets = stats->rx_packets;
  708. rbytes = stats->rx_bytes;
  709. } while (u64_stats_fetch_retry_bh(&stats->rx_syncp, start));
  710. tot->rx_packets += rpackets;
  711. tot->tx_packets += tpackets;
  712. tot->rx_bytes += rbytes;
  713. tot->tx_bytes += tbytes;
  714. }
  715. tot->tx_dropped = dev->stats.tx_dropped;
  716. tot->tx_fifo_errors = dev->stats.tx_fifo_errors;
  717. tot->rx_dropped = dev->stats.rx_dropped;
  718. tot->rx_length_errors = dev->stats.rx_length_errors;
  719. tot->rx_frame_errors = dev->stats.rx_frame_errors;
  720. return tot;
  721. }
  722. #ifdef CONFIG_NET_POLL_CONTROLLER
  723. static void virtnet_netpoll(struct net_device *dev)
  724. {
  725. struct virtnet_info *vi = netdev_priv(dev);
  726. int i;
  727. for (i = 0; i < vi->curr_queue_pairs; i++)
  728. napi_schedule(&vi->rq[i].napi);
  729. }
  730. #endif
  731. static void virtnet_ack_link_announce(struct virtnet_info *vi)
  732. {
  733. rtnl_lock();
  734. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE,
  735. VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL,
  736. 0, 0))
  737. dev_warn(&vi->dev->dev, "Failed to ack link announce.\n");
  738. rtnl_unlock();
  739. }
  740. static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
  741. {
  742. struct scatterlist sg;
  743. struct virtio_net_ctrl_mq s;
  744. struct net_device *dev = vi->dev;
  745. if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ))
  746. return 0;
  747. s.virtqueue_pairs = queue_pairs;
  748. sg_init_one(&sg, &s, sizeof(s));
  749. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,
  750. VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &sg, 1, 0)){
  751. dev_warn(&dev->dev, "Fail to set num of queue pairs to %d\n",
  752. queue_pairs);
  753. return -EINVAL;
  754. } else
  755. vi->curr_queue_pairs = queue_pairs;
  756. return 0;
  757. }
  758. static int virtnet_close(struct net_device *dev)
  759. {
  760. struct virtnet_info *vi = netdev_priv(dev);
  761. int i;
  762. /* Make sure refill_work doesn't re-enable napi! */
  763. cancel_delayed_work_sync(&vi->refill);
  764. for (i = 0; i < vi->max_queue_pairs; i++)
  765. napi_disable(&vi->rq[i].napi);
  766. return 0;
  767. }
  768. static void virtnet_set_rx_mode(struct net_device *dev)
  769. {
  770. struct virtnet_info *vi = netdev_priv(dev);
  771. struct scatterlist sg[2];
  772. u8 promisc, allmulti;
  773. struct virtio_net_ctrl_mac *mac_data;
  774. struct netdev_hw_addr *ha;
  775. int uc_count;
  776. int mc_count;
  777. void *buf;
  778. int i;
  779. /* We can't dynamicaly set ndo_set_rx_mode, so return gracefully */
  780. if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
  781. return;
  782. promisc = ((dev->flags & IFF_PROMISC) != 0);
  783. allmulti = ((dev->flags & IFF_ALLMULTI) != 0);
  784. sg_init_one(sg, &promisc, sizeof(promisc));
  785. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
  786. VIRTIO_NET_CTRL_RX_PROMISC,
  787. sg, 1, 0))
  788. dev_warn(&dev->dev, "Failed to %sable promisc mode.\n",
  789. promisc ? "en" : "dis");
  790. sg_init_one(sg, &allmulti, sizeof(allmulti));
  791. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
  792. VIRTIO_NET_CTRL_RX_ALLMULTI,
  793. sg, 1, 0))
  794. dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
  795. allmulti ? "en" : "dis");
  796. uc_count = netdev_uc_count(dev);
  797. mc_count = netdev_mc_count(dev);
  798. /* MAC filter - use one buffer for both lists */
  799. buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
  800. (2 * sizeof(mac_data->entries)), GFP_ATOMIC);
  801. mac_data = buf;
  802. if (!buf)
  803. return;
  804. sg_init_table(sg, 2);
  805. /* Store the unicast list and count in the front of the buffer */
  806. mac_data->entries = uc_count;
  807. i = 0;
  808. netdev_for_each_uc_addr(ha, dev)
  809. memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
  810. sg_set_buf(&sg[0], mac_data,
  811. sizeof(mac_data->entries) + (uc_count * ETH_ALEN));
  812. /* multicast list and count fill the end */
  813. mac_data = (void *)&mac_data->macs[uc_count][0];
  814. mac_data->entries = mc_count;
  815. i = 0;
  816. netdev_for_each_mc_addr(ha, dev)
  817. memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
  818. sg_set_buf(&sg[1], mac_data,
  819. sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
  820. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
  821. VIRTIO_NET_CTRL_MAC_TABLE_SET,
  822. sg, 2, 0))
  823. dev_warn(&dev->dev, "Failed to set MAC fitler table.\n");
  824. kfree(buf);
  825. }
  826. static int virtnet_vlan_rx_add_vid(struct net_device *dev, u16 vid)
  827. {
  828. struct virtnet_info *vi = netdev_priv(dev);
  829. struct scatterlist sg;
  830. sg_init_one(&sg, &vid, sizeof(vid));
  831. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
  832. VIRTIO_NET_CTRL_VLAN_ADD, &sg, 1, 0))
  833. dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid);
  834. return 0;
  835. }
  836. static int virtnet_vlan_rx_kill_vid(struct net_device *dev, u16 vid)
  837. {
  838. struct virtnet_info *vi = netdev_priv(dev);
  839. struct scatterlist sg;
  840. sg_init_one(&sg, &vid, sizeof(vid));
  841. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
  842. VIRTIO_NET_CTRL_VLAN_DEL, &sg, 1, 0))
  843. dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid);
  844. return 0;
  845. }
  846. static void virtnet_clean_affinity(struct virtnet_info *vi, long hcpu)
  847. {
  848. int i;
  849. int cpu;
  850. if (vi->affinity_hint_set) {
  851. for (i = 0; i < vi->max_queue_pairs; i++) {
  852. virtqueue_set_affinity(vi->rq[i].vq, -1);
  853. virtqueue_set_affinity(vi->sq[i].vq, -1);
  854. }
  855. vi->affinity_hint_set = false;
  856. }
  857. i = 0;
  858. for_each_online_cpu(cpu) {
  859. if (cpu == hcpu) {
  860. *per_cpu_ptr(vi->vq_index, cpu) = -1;
  861. } else {
  862. *per_cpu_ptr(vi->vq_index, cpu) =
  863. ++i % vi->curr_queue_pairs;
  864. }
  865. }
  866. }
  867. static void virtnet_set_affinity(struct virtnet_info *vi)
  868. {
  869. int i;
  870. int cpu;
  871. /* In multiqueue mode, when the number of cpu is equal to the number of
  872. * queue pairs, we let the queue pairs to be private to one cpu by
  873. * setting the affinity hint to eliminate the contention.
  874. */
  875. if (vi->curr_queue_pairs == 1 ||
  876. vi->max_queue_pairs != num_online_cpus()) {
  877. virtnet_clean_affinity(vi, -1);
  878. return;
  879. }
  880. i = 0;
  881. for_each_online_cpu(cpu) {
  882. virtqueue_set_affinity(vi->rq[i].vq, cpu);
  883. virtqueue_set_affinity(vi->sq[i].vq, cpu);
  884. *per_cpu_ptr(vi->vq_index, cpu) = i;
  885. i++;
  886. }
  887. vi->affinity_hint_set = true;
  888. }
  889. static int virtnet_cpu_callback(struct notifier_block *nfb,
  890. unsigned long action, void *hcpu)
  891. {
  892. struct virtnet_info *vi = container_of(nfb, struct virtnet_info, nb);
  893. switch(action & ~CPU_TASKS_FROZEN) {
  894. case CPU_ONLINE:
  895. case CPU_DOWN_FAILED:
  896. case CPU_DEAD:
  897. virtnet_set_affinity(vi);
  898. break;
  899. case CPU_DOWN_PREPARE:
  900. virtnet_clean_affinity(vi, (long)hcpu);
  901. break;
  902. default:
  903. break;
  904. }
  905. return NOTIFY_OK;
  906. }
  907. static void virtnet_get_ringparam(struct net_device *dev,
  908. struct ethtool_ringparam *ring)
  909. {
  910. struct virtnet_info *vi = netdev_priv(dev);
  911. ring->rx_max_pending = virtqueue_get_vring_size(vi->rq[0].vq);
  912. ring->tx_max_pending = virtqueue_get_vring_size(vi->sq[0].vq);
  913. ring->rx_pending = ring->rx_max_pending;
  914. ring->tx_pending = ring->tx_max_pending;
  915. }
  916. static void virtnet_get_drvinfo(struct net_device *dev,
  917. struct ethtool_drvinfo *info)
  918. {
  919. struct virtnet_info *vi = netdev_priv(dev);
  920. struct virtio_device *vdev = vi->vdev;
  921. strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
  922. strlcpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version));
  923. strlcpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info));
  924. }
  925. /* TODO: Eliminate OOO packets during switching */
  926. static int virtnet_set_channels(struct net_device *dev,
  927. struct ethtool_channels *channels)
  928. {
  929. struct virtnet_info *vi = netdev_priv(dev);
  930. u16 queue_pairs = channels->combined_count;
  931. int err;
  932. /* We don't support separate rx/tx channels.
  933. * We don't allow setting 'other' channels.
  934. */
  935. if (channels->rx_count || channels->tx_count || channels->other_count)
  936. return -EINVAL;
  937. if (queue_pairs > vi->max_queue_pairs)
  938. return -EINVAL;
  939. get_online_cpus();
  940. err = virtnet_set_queues(vi, queue_pairs);
  941. if (!err) {
  942. netif_set_real_num_tx_queues(dev, queue_pairs);
  943. netif_set_real_num_rx_queues(dev, queue_pairs);
  944. virtnet_set_affinity(vi);
  945. }
  946. put_online_cpus();
  947. return err;
  948. }
  949. static void virtnet_get_channels(struct net_device *dev,
  950. struct ethtool_channels *channels)
  951. {
  952. struct virtnet_info *vi = netdev_priv(dev);
  953. channels->combined_count = vi->curr_queue_pairs;
  954. channels->max_combined = vi->max_queue_pairs;
  955. channels->max_other = 0;
  956. channels->rx_count = 0;
  957. channels->tx_count = 0;
  958. channels->other_count = 0;
  959. }
  960. static const struct ethtool_ops virtnet_ethtool_ops = {
  961. .get_drvinfo = virtnet_get_drvinfo,
  962. .get_link = ethtool_op_get_link,
  963. .get_ringparam = virtnet_get_ringparam,
  964. .set_channels = virtnet_set_channels,
  965. .get_channels = virtnet_get_channels,
  966. };
  967. #define MIN_MTU 68
  968. #define MAX_MTU 65535
  969. static int virtnet_change_mtu(struct net_device *dev, int new_mtu)
  970. {
  971. if (new_mtu < MIN_MTU || new_mtu > MAX_MTU)
  972. return -EINVAL;
  973. dev->mtu = new_mtu;
  974. return 0;
  975. }
  976. /* To avoid contending a lock hold by a vcpu who would exit to host, select the
  977. * txq based on the processor id.
  978. */
  979. static u16 virtnet_select_queue(struct net_device *dev, struct sk_buff *skb)
  980. {
  981. int txq;
  982. struct virtnet_info *vi = netdev_priv(dev);
  983. if (skb_rx_queue_recorded(skb)) {
  984. txq = skb_get_rx_queue(skb);
  985. } else {
  986. txq = *__this_cpu_ptr(vi->vq_index);
  987. if (txq == -1)
  988. txq = 0;
  989. }
  990. while (unlikely(txq >= dev->real_num_tx_queues))
  991. txq -= dev->real_num_tx_queues;
  992. return txq;
  993. }
  994. static const struct net_device_ops virtnet_netdev = {
  995. .ndo_open = virtnet_open,
  996. .ndo_stop = virtnet_close,
  997. .ndo_start_xmit = start_xmit,
  998. .ndo_validate_addr = eth_validate_addr,
  999. .ndo_set_mac_address = virtnet_set_mac_address,
  1000. .ndo_set_rx_mode = virtnet_set_rx_mode,
  1001. .ndo_change_mtu = virtnet_change_mtu,
  1002. .ndo_get_stats64 = virtnet_stats,
  1003. .ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid,
  1004. .ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
  1005. .ndo_select_queue = virtnet_select_queue,
  1006. #ifdef CONFIG_NET_POLL_CONTROLLER
  1007. .ndo_poll_controller = virtnet_netpoll,
  1008. #endif
  1009. };
  1010. static void virtnet_config_changed_work(struct work_struct *work)
  1011. {
  1012. struct virtnet_info *vi =
  1013. container_of(work, struct virtnet_info, config_work);
  1014. u16 v;
  1015. mutex_lock(&vi->config_lock);
  1016. if (!vi->config_enable)
  1017. goto done;
  1018. if (virtio_config_val(vi->vdev, VIRTIO_NET_F_STATUS,
  1019. offsetof(struct virtio_net_config, status),
  1020. &v) < 0)
  1021. goto done;
  1022. if (v & VIRTIO_NET_S_ANNOUNCE) {
  1023. netdev_notify_peers(vi->dev);
  1024. virtnet_ack_link_announce(vi);
  1025. }
  1026. /* Ignore unknown (future) status bits */
  1027. v &= VIRTIO_NET_S_LINK_UP;
  1028. if (vi->status == v)
  1029. goto done;
  1030. vi->status = v;
  1031. if (vi->status & VIRTIO_NET_S_LINK_UP) {
  1032. netif_carrier_on(vi->dev);
  1033. netif_tx_wake_all_queues(vi->dev);
  1034. } else {
  1035. netif_carrier_off(vi->dev);
  1036. netif_tx_stop_all_queues(vi->dev);
  1037. }
  1038. done:
  1039. mutex_unlock(&vi->config_lock);
  1040. }
  1041. static void virtnet_config_changed(struct virtio_device *vdev)
  1042. {
  1043. struct virtnet_info *vi = vdev->priv;
  1044. schedule_work(&vi->config_work);
  1045. }
  1046. static void virtnet_free_queues(struct virtnet_info *vi)
  1047. {
  1048. kfree(vi->rq);
  1049. kfree(vi->sq);
  1050. }
  1051. static void free_receive_bufs(struct virtnet_info *vi)
  1052. {
  1053. int i;
  1054. for (i = 0; i < vi->max_queue_pairs; i++) {
  1055. while (vi->rq[i].pages)
  1056. __free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0);
  1057. }
  1058. }
  1059. static void free_unused_bufs(struct virtnet_info *vi)
  1060. {
  1061. void *buf;
  1062. int i;
  1063. for (i = 0; i < vi->max_queue_pairs; i++) {
  1064. struct virtqueue *vq = vi->sq[i].vq;
  1065. while ((buf = virtqueue_detach_unused_buf(vq)) != NULL)
  1066. dev_kfree_skb(buf);
  1067. }
  1068. for (i = 0; i < vi->max_queue_pairs; i++) {
  1069. struct virtqueue *vq = vi->rq[i].vq;
  1070. while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
  1071. if (vi->mergeable_rx_bufs || vi->big_packets)
  1072. give_pages(&vi->rq[i], buf);
  1073. else
  1074. dev_kfree_skb(buf);
  1075. --vi->rq[i].num;
  1076. }
  1077. BUG_ON(vi->rq[i].num != 0);
  1078. }
  1079. }
  1080. static void virtnet_del_vqs(struct virtnet_info *vi)
  1081. {
  1082. struct virtio_device *vdev = vi->vdev;
  1083. virtnet_clean_affinity(vi, -1);
  1084. vdev->config->del_vqs(vdev);
  1085. virtnet_free_queues(vi);
  1086. }
  1087. static int virtnet_find_vqs(struct virtnet_info *vi)
  1088. {
  1089. vq_callback_t **callbacks;
  1090. struct virtqueue **vqs;
  1091. int ret = -ENOMEM;
  1092. int i, total_vqs;
  1093. const char **names;
  1094. /* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by
  1095. * possible N-1 RX/TX queue pairs used in multiqueue mode, followed by
  1096. * possible control vq.
  1097. */
  1098. total_vqs = vi->max_queue_pairs * 2 +
  1099. virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ);
  1100. /* Allocate space for find_vqs parameters */
  1101. vqs = kzalloc(total_vqs * sizeof(*vqs), GFP_KERNEL);
  1102. if (!vqs)
  1103. goto err_vq;
  1104. callbacks = kmalloc(total_vqs * sizeof(*callbacks), GFP_KERNEL);
  1105. if (!callbacks)
  1106. goto err_callback;
  1107. names = kmalloc(total_vqs * sizeof(*names), GFP_KERNEL);
  1108. if (!names)
  1109. goto err_names;
  1110. /* Parameters for control virtqueue, if any */
  1111. if (vi->has_cvq) {
  1112. callbacks[total_vqs - 1] = NULL;
  1113. names[total_vqs - 1] = "control";
  1114. }
  1115. /* Allocate/initialize parameters for send/receive virtqueues */
  1116. for (i = 0; i < vi->max_queue_pairs; i++) {
  1117. callbacks[rxq2vq(i)] = skb_recv_done;
  1118. callbacks[txq2vq(i)] = skb_xmit_done;
  1119. sprintf(vi->rq[i].name, "input.%d", i);
  1120. sprintf(vi->sq[i].name, "output.%d", i);
  1121. names[rxq2vq(i)] = vi->rq[i].name;
  1122. names[txq2vq(i)] = vi->sq[i].name;
  1123. }
  1124. ret = vi->vdev->config->find_vqs(vi->vdev, total_vqs, vqs, callbacks,
  1125. names);
  1126. if (ret)
  1127. goto err_find;
  1128. if (vi->has_cvq) {
  1129. vi->cvq = vqs[total_vqs - 1];
  1130. if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
  1131. vi->dev->features |= NETIF_F_HW_VLAN_FILTER;
  1132. }
  1133. for (i = 0; i < vi->max_queue_pairs; i++) {
  1134. vi->rq[i].vq = vqs[rxq2vq(i)];
  1135. vi->sq[i].vq = vqs[txq2vq(i)];
  1136. }
  1137. kfree(names);
  1138. kfree(callbacks);
  1139. kfree(vqs);
  1140. return 0;
  1141. err_find:
  1142. kfree(names);
  1143. err_names:
  1144. kfree(callbacks);
  1145. err_callback:
  1146. kfree(vqs);
  1147. err_vq:
  1148. return ret;
  1149. }
  1150. static int virtnet_alloc_queues(struct virtnet_info *vi)
  1151. {
  1152. int i;
  1153. vi->sq = kzalloc(sizeof(*vi->sq) * vi->max_queue_pairs, GFP_KERNEL);
  1154. if (!vi->sq)
  1155. goto err_sq;
  1156. vi->rq = kzalloc(sizeof(*vi->rq) * vi->max_queue_pairs, GFP_KERNEL);
  1157. if (!vi->rq)
  1158. goto err_rq;
  1159. INIT_DELAYED_WORK(&vi->refill, refill_work);
  1160. for (i = 0; i < vi->max_queue_pairs; i++) {
  1161. vi->rq[i].pages = NULL;
  1162. netif_napi_add(vi->dev, &vi->rq[i].napi, virtnet_poll,
  1163. napi_weight);
  1164. sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
  1165. sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));
  1166. }
  1167. return 0;
  1168. err_rq:
  1169. kfree(vi->sq);
  1170. err_sq:
  1171. return -ENOMEM;
  1172. }
  1173. static int init_vqs(struct virtnet_info *vi)
  1174. {
  1175. int ret;
  1176. /* Allocate send & receive queues */
  1177. ret = virtnet_alloc_queues(vi);
  1178. if (ret)
  1179. goto err;
  1180. ret = virtnet_find_vqs(vi);
  1181. if (ret)
  1182. goto err_free;
  1183. get_online_cpus();
  1184. virtnet_set_affinity(vi);
  1185. put_online_cpus();
  1186. return 0;
  1187. err_free:
  1188. virtnet_free_queues(vi);
  1189. err:
  1190. return ret;
  1191. }
  1192. static int virtnet_probe(struct virtio_device *vdev)
  1193. {
  1194. int i, err;
  1195. struct net_device *dev;
  1196. struct virtnet_info *vi;
  1197. u16 max_queue_pairs;
  1198. /* Find if host supports multiqueue virtio_net device */
  1199. err = virtio_config_val(vdev, VIRTIO_NET_F_MQ,
  1200. offsetof(struct virtio_net_config,
  1201. max_virtqueue_pairs), &max_queue_pairs);
  1202. /* We need at least 2 queue's */
  1203. if (err || max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
  1204. max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX ||
  1205. !virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
  1206. max_queue_pairs = 1;
  1207. /* Allocate ourselves a network device with room for our info */
  1208. dev = alloc_etherdev_mq(sizeof(struct virtnet_info), max_queue_pairs);
  1209. if (!dev)
  1210. return -ENOMEM;
  1211. /* Set up network device as normal. */
  1212. dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE;
  1213. dev->netdev_ops = &virtnet_netdev;
  1214. dev->features = NETIF_F_HIGHDMA;
  1215. SET_ETHTOOL_OPS(dev, &virtnet_ethtool_ops);
  1216. SET_NETDEV_DEV(dev, &vdev->dev);
  1217. /* Do we support "hardware" checksums? */
  1218. if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) {
  1219. /* This opens up the world of extra features. */
  1220. dev->hw_features |= NETIF_F_HW_CSUM|NETIF_F_SG|NETIF_F_FRAGLIST;
  1221. if (csum)
  1222. dev->features |= NETIF_F_HW_CSUM|NETIF_F_SG|NETIF_F_FRAGLIST;
  1223. if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
  1224. dev->hw_features |= NETIF_F_TSO | NETIF_F_UFO
  1225. | NETIF_F_TSO_ECN | NETIF_F_TSO6;
  1226. }
  1227. /* Individual feature bits: what can host handle? */
  1228. if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4))
  1229. dev->hw_features |= NETIF_F_TSO;
  1230. if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6))
  1231. dev->hw_features |= NETIF_F_TSO6;
  1232. if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
  1233. dev->hw_features |= NETIF_F_TSO_ECN;
  1234. if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_UFO))
  1235. dev->hw_features |= NETIF_F_UFO;
  1236. if (gso)
  1237. dev->features |= dev->hw_features & (NETIF_F_ALL_TSO|NETIF_F_UFO);
  1238. /* (!csum && gso) case will be fixed by register_netdev() */
  1239. }
  1240. /* Configuration may specify what MAC to use. Otherwise random. */
  1241. if (virtio_config_val_len(vdev, VIRTIO_NET_F_MAC,
  1242. offsetof(struct virtio_net_config, mac),
  1243. dev->dev_addr, dev->addr_len) < 0)
  1244. eth_hw_addr_random(dev);
  1245. /* Set up our device-specific information */
  1246. vi = netdev_priv(dev);
  1247. vi->dev = dev;
  1248. vi->vdev = vdev;
  1249. vdev->priv = vi;
  1250. vi->stats = alloc_percpu(struct virtnet_stats);
  1251. err = -ENOMEM;
  1252. if (vi->stats == NULL)
  1253. goto free;
  1254. vi->vq_index = alloc_percpu(int);
  1255. if (vi->vq_index == NULL)
  1256. goto free_stats;
  1257. mutex_init(&vi->config_lock);
  1258. vi->config_enable = true;
  1259. INIT_WORK(&vi->config_work, virtnet_config_changed_work);
  1260. /* If we can receive ANY GSO packets, we must allocate large ones. */
  1261. if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
  1262. virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) ||
  1263. virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN))
  1264. vi->big_packets = true;
  1265. if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
  1266. vi->mergeable_rx_bufs = true;
  1267. if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
  1268. vi->has_cvq = true;
  1269. /* Use single tx/rx queue pair as default */
  1270. vi->curr_queue_pairs = 1;
  1271. vi->max_queue_pairs = max_queue_pairs;
  1272. /* Allocate/initialize the rx/tx queues, and invoke find_vqs */
  1273. err = init_vqs(vi);
  1274. if (err)
  1275. goto free_index;
  1276. netif_set_real_num_tx_queues(dev, 1);
  1277. netif_set_real_num_rx_queues(dev, 1);
  1278. err = register_netdev(dev);
  1279. if (err) {
  1280. pr_debug("virtio_net: registering device failed\n");
  1281. goto free_vqs;
  1282. }
  1283. /* Last of all, set up some receive buffers. */
  1284. for (i = 0; i < vi->max_queue_pairs; i++) {
  1285. try_fill_recv(&vi->rq[i], GFP_KERNEL);
  1286. /* If we didn't even get one input buffer, we're useless. */
  1287. if (vi->rq[i].num == 0) {
  1288. free_unused_bufs(vi);
  1289. err = -ENOMEM;
  1290. goto free_recv_bufs;
  1291. }
  1292. }
  1293. vi->nb.notifier_call = &virtnet_cpu_callback;
  1294. err = register_hotcpu_notifier(&vi->nb);
  1295. if (err) {
  1296. pr_debug("virtio_net: registering cpu notifier failed\n");
  1297. goto free_recv_bufs;
  1298. }
  1299. /* Assume link up if device can't report link status,
  1300. otherwise get link status from config. */
  1301. if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {
  1302. netif_carrier_off(dev);
  1303. schedule_work(&vi->config_work);
  1304. } else {
  1305. vi->status = VIRTIO_NET_S_LINK_UP;
  1306. netif_carrier_on(dev);
  1307. }
  1308. pr_debug("virtnet: registered device %s with %d RX and TX vq's\n",
  1309. dev->name, max_queue_pairs);
  1310. return 0;
  1311. free_recv_bufs:
  1312. free_receive_bufs(vi);
  1313. unregister_netdev(dev);
  1314. free_vqs:
  1315. cancel_delayed_work_sync(&vi->refill);
  1316. virtnet_del_vqs(vi);
  1317. free_index:
  1318. free_percpu(vi->vq_index);
  1319. free_stats:
  1320. free_percpu(vi->stats);
  1321. free:
  1322. free_netdev(dev);
  1323. return err;
  1324. }
  1325. static void remove_vq_common(struct virtnet_info *vi)
  1326. {
  1327. vi->vdev->config->reset(vi->vdev);
  1328. /* Free unused buffers in both send and recv, if any. */
  1329. free_unused_bufs(vi);
  1330. free_receive_bufs(vi);
  1331. virtnet_del_vqs(vi);
  1332. }
  1333. static void virtnet_remove(struct virtio_device *vdev)
  1334. {
  1335. struct virtnet_info *vi = vdev->priv;
  1336. unregister_hotcpu_notifier(&vi->nb);
  1337. /* Prevent config work handler from accessing the device. */
  1338. mutex_lock(&vi->config_lock);
  1339. vi->config_enable = false;
  1340. mutex_unlock(&vi->config_lock);
  1341. unregister_netdev(vi->dev);
  1342. remove_vq_common(vi);
  1343. flush_work(&vi->config_work);
  1344. free_percpu(vi->vq_index);
  1345. free_percpu(vi->stats);
  1346. free_netdev(vi->dev);
  1347. }
  1348. #ifdef CONFIG_PM
  1349. static int virtnet_freeze(struct virtio_device *vdev)
  1350. {
  1351. struct virtnet_info *vi = vdev->priv;
  1352. int i;
  1353. /* Prevent config work handler from accessing the device */
  1354. mutex_lock(&vi->config_lock);
  1355. vi->config_enable = false;
  1356. mutex_unlock(&vi->config_lock);
  1357. netif_device_detach(vi->dev);
  1358. cancel_delayed_work_sync(&vi->refill);
  1359. if (netif_running(vi->dev))
  1360. for (i = 0; i < vi->max_queue_pairs; i++) {
  1361. napi_disable(&vi->rq[i].napi);
  1362. netif_napi_del(&vi->rq[i].napi);
  1363. }
  1364. remove_vq_common(vi);
  1365. flush_work(&vi->config_work);
  1366. return 0;
  1367. }
  1368. static int virtnet_restore(struct virtio_device *vdev)
  1369. {
  1370. struct virtnet_info *vi = vdev->priv;
  1371. int err, i;
  1372. err = init_vqs(vi);
  1373. if (err)
  1374. return err;
  1375. if (netif_running(vi->dev))
  1376. for (i = 0; i < vi->max_queue_pairs; i++)
  1377. virtnet_napi_enable(&vi->rq[i]);
  1378. netif_device_attach(vi->dev);
  1379. for (i = 0; i < vi->max_queue_pairs; i++)
  1380. if (!try_fill_recv(&vi->rq[i], GFP_KERNEL))
  1381. schedule_delayed_work(&vi->refill, 0);
  1382. mutex_lock(&vi->config_lock);
  1383. vi->config_enable = true;
  1384. mutex_unlock(&vi->config_lock);
  1385. virtnet_set_queues(vi, vi->curr_queue_pairs);
  1386. return 0;
  1387. }
  1388. #endif
  1389. static struct virtio_device_id id_table[] = {
  1390. { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
  1391. { 0 },
  1392. };
  1393. static unsigned int features[] = {
  1394. VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM,
  1395. VIRTIO_NET_F_GSO, VIRTIO_NET_F_MAC,
  1396. VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6,
  1397. VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6,
  1398. VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO,
  1399. VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ,
  1400. VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN,
  1401. VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ,
  1402. VIRTIO_NET_F_CTRL_MAC_ADDR,
  1403. };
  1404. static struct virtio_driver virtio_net_driver = {
  1405. .feature_table = features,
  1406. .feature_table_size = ARRAY_SIZE(features),
  1407. .driver.name = KBUILD_MODNAME,
  1408. .driver.owner = THIS_MODULE,
  1409. .id_table = id_table,
  1410. .probe = virtnet_probe,
  1411. .remove = virtnet_remove,
  1412. .config_changed = virtnet_config_changed,
  1413. #ifdef CONFIG_PM
  1414. .freeze = virtnet_freeze,
  1415. .restore = virtnet_restore,
  1416. #endif
  1417. };
  1418. static int __init init(void)
  1419. {
  1420. return register_virtio_driver(&virtio_net_driver);
  1421. }
  1422. static void __exit fini(void)
  1423. {
  1424. unregister_virtio_driver(&virtio_net_driver);
  1425. }
  1426. module_init(init);
  1427. module_exit(fini);
  1428. MODULE_DEVICE_TABLE(virtio, id_table);
  1429. MODULE_DESCRIPTION("Virtio network driver");
  1430. MODULE_LICENSE("GPL");