virtio_net.c 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104
  1. /* A network driver using virtio.
  2. *
  3. * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
  4. *
  5. * This program is free software; you can redistribute it and/or modify
  6. * it under the terms of the GNU General Public License as published by
  7. * the Free Software Foundation; either version 2 of the License, or
  8. * (at your option) any later version.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU General Public License
  16. * along with this program; if not, write to the Free Software
  17. * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  18. */
  19. //#define DEBUG
  20. #include <linux/netdevice.h>
  21. #include <linux/etherdevice.h>
  22. #include <linux/ethtool.h>
  23. #include <linux/module.h>
  24. #include <linux/virtio.h>
  25. #include <linux/virtio_net.h>
  26. #include <linux/scatterlist.h>
  27. #include <linux/if_vlan.h>
  28. #include <linux/slab.h>
  29. static int napi_weight = 128;
  30. module_param(napi_weight, int, 0444);
  31. static int csum = 1, gso = 1;
  32. module_param(csum, bool, 0444);
  33. module_param(gso, bool, 0444);
  34. /* FIXME: MTU in config. */
  35. #define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
  36. #define GOOD_COPY_LEN 128
  37. #define VIRTNET_SEND_COMMAND_SG_MAX 2
  38. struct virtnet_info {
  39. struct virtio_device *vdev;
  40. struct virtqueue *rvq, *svq, *cvq;
  41. struct net_device *dev;
  42. struct napi_struct napi;
  43. unsigned int status;
  44. /* Number of input buffers, and max we've ever had. */
  45. unsigned int num, max;
  46. /* I like... big packets and I cannot lie! */
  47. bool big_packets;
  48. /* Host will merge rx buffers for big packets (shake it! shake it!) */
  49. bool mergeable_rx_bufs;
  50. /* Work struct for refilling if we run low on memory. */
  51. struct delayed_work refill;
  52. /* Chain pages by the private ptr. */
  53. struct page *pages;
  54. /* fragments + linear part + virtio header */
  55. struct scatterlist rx_sg[MAX_SKB_FRAGS + 2];
  56. struct scatterlist tx_sg[MAX_SKB_FRAGS + 2];
  57. };
  58. struct skb_vnet_hdr {
  59. union {
  60. struct virtio_net_hdr hdr;
  61. struct virtio_net_hdr_mrg_rxbuf mhdr;
  62. };
  63. unsigned int num_sg;
  64. };
  65. struct padded_vnet_hdr {
  66. struct virtio_net_hdr hdr;
  67. /*
  68. * virtio_net_hdr should be in a separated sg buffer because of a
  69. * QEMU bug, and data sg buffer shares same page with this header sg.
  70. * This padding makes next sg 16 byte aligned after virtio_net_hdr.
  71. */
  72. char padding[6];
  73. };
  74. static inline struct skb_vnet_hdr *skb_vnet_hdr(struct sk_buff *skb)
  75. {
  76. return (struct skb_vnet_hdr *)skb->cb;
  77. }
  78. /*
  79. * private is used to chain pages for big packets, put the whole
  80. * most recent used list in the beginning for reuse
  81. */
  82. static void give_pages(struct virtnet_info *vi, struct page *page)
  83. {
  84. struct page *end;
  85. /* Find end of list, sew whole thing into vi->pages. */
  86. for (end = page; end->private; end = (struct page *)end->private);
  87. end->private = (unsigned long)vi->pages;
  88. vi->pages = page;
  89. }
  90. static struct page *get_a_page(struct virtnet_info *vi, gfp_t gfp_mask)
  91. {
  92. struct page *p = vi->pages;
  93. if (p) {
  94. vi->pages = (struct page *)p->private;
  95. /* clear private here, it is used to chain pages */
  96. p->private = 0;
  97. } else
  98. p = alloc_page(gfp_mask);
  99. return p;
  100. }
  101. static void skb_xmit_done(struct virtqueue *svq)
  102. {
  103. struct virtnet_info *vi = svq->vdev->priv;
  104. /* Suppress further interrupts. */
  105. virtqueue_disable_cb(svq);
  106. /* We were probably waiting for more output buffers. */
  107. netif_wake_queue(vi->dev);
  108. }
  109. static void set_skb_frag(struct sk_buff *skb, struct page *page,
  110. unsigned int offset, unsigned int *len)
  111. {
  112. int i = skb_shinfo(skb)->nr_frags;
  113. skb_frag_t *f;
  114. f = &skb_shinfo(skb)->frags[i];
  115. f->size = min((unsigned)PAGE_SIZE - offset, *len);
  116. f->page_offset = offset;
  117. f->page = page;
  118. skb->data_len += f->size;
  119. skb->len += f->size;
  120. skb_shinfo(skb)->nr_frags++;
  121. *len -= f->size;
  122. }
  123. static struct sk_buff *page_to_skb(struct virtnet_info *vi,
  124. struct page *page, unsigned int len)
  125. {
  126. struct sk_buff *skb;
  127. struct skb_vnet_hdr *hdr;
  128. unsigned int copy, hdr_len, offset;
  129. char *p;
  130. p = page_address(page);
  131. /* copy small packet so we can reuse these pages for small data */
  132. skb = netdev_alloc_skb_ip_align(vi->dev, GOOD_COPY_LEN);
  133. if (unlikely(!skb))
  134. return NULL;
  135. hdr = skb_vnet_hdr(skb);
  136. if (vi->mergeable_rx_bufs) {
  137. hdr_len = sizeof hdr->mhdr;
  138. offset = hdr_len;
  139. } else {
  140. hdr_len = sizeof hdr->hdr;
  141. offset = sizeof(struct padded_vnet_hdr);
  142. }
  143. memcpy(hdr, p, hdr_len);
  144. len -= hdr_len;
  145. p += offset;
  146. copy = len;
  147. if (copy > skb_tailroom(skb))
  148. copy = skb_tailroom(skb);
  149. memcpy(skb_put(skb, copy), p, copy);
  150. len -= copy;
  151. offset += copy;
  152. while (len) {
  153. set_skb_frag(skb, page, offset, &len);
  154. page = (struct page *)page->private;
  155. offset = 0;
  156. }
  157. if (page)
  158. give_pages(vi, page);
  159. return skb;
  160. }
  161. static int receive_mergeable(struct virtnet_info *vi, struct sk_buff *skb)
  162. {
  163. struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb);
  164. struct page *page;
  165. int num_buf, i, len;
  166. num_buf = hdr->mhdr.num_buffers;
  167. while (--num_buf) {
  168. i = skb_shinfo(skb)->nr_frags;
  169. if (i >= MAX_SKB_FRAGS) {
  170. pr_debug("%s: packet too long\n", skb->dev->name);
  171. skb->dev->stats.rx_length_errors++;
  172. return -EINVAL;
  173. }
  174. page = virtqueue_get_buf(vi->rvq, &len);
  175. if (!page) {
  176. pr_debug("%s: rx error: %d buffers missing\n",
  177. skb->dev->name, hdr->mhdr.num_buffers);
  178. skb->dev->stats.rx_length_errors++;
  179. return -EINVAL;
  180. }
  181. if (len > PAGE_SIZE)
  182. len = PAGE_SIZE;
  183. set_skb_frag(skb, page, 0, &len);
  184. --vi->num;
  185. }
  186. return 0;
  187. }
  188. static void receive_buf(struct net_device *dev, void *buf, unsigned int len)
  189. {
  190. struct virtnet_info *vi = netdev_priv(dev);
  191. struct sk_buff *skb;
  192. struct page *page;
  193. struct skb_vnet_hdr *hdr;
  194. if (unlikely(len < sizeof(struct virtio_net_hdr) + ETH_HLEN)) {
  195. pr_debug("%s: short packet %i\n", dev->name, len);
  196. dev->stats.rx_length_errors++;
  197. if (vi->mergeable_rx_bufs || vi->big_packets)
  198. give_pages(vi, buf);
  199. else
  200. dev_kfree_skb(buf);
  201. return;
  202. }
  203. if (!vi->mergeable_rx_bufs && !vi->big_packets) {
  204. skb = buf;
  205. len -= sizeof(struct virtio_net_hdr);
  206. skb_trim(skb, len);
  207. } else {
  208. page = buf;
  209. skb = page_to_skb(vi, page, len);
  210. if (unlikely(!skb)) {
  211. dev->stats.rx_dropped++;
  212. give_pages(vi, page);
  213. return;
  214. }
  215. if (vi->mergeable_rx_bufs)
  216. if (receive_mergeable(vi, skb)) {
  217. dev_kfree_skb(skb);
  218. return;
  219. }
  220. }
  221. hdr = skb_vnet_hdr(skb);
  222. skb->truesize += skb->data_len;
  223. dev->stats.rx_bytes += skb->len;
  224. dev->stats.rx_packets++;
  225. if (hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
  226. pr_debug("Needs csum!\n");
  227. if (!skb_partial_csum_set(skb,
  228. hdr->hdr.csum_start,
  229. hdr->hdr.csum_offset))
  230. goto frame_err;
  231. }
  232. skb->protocol = eth_type_trans(skb, dev);
  233. pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
  234. ntohs(skb->protocol), skb->len, skb->pkt_type);
  235. if (hdr->hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
  236. pr_debug("GSO!\n");
  237. switch (hdr->hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
  238. case VIRTIO_NET_HDR_GSO_TCPV4:
  239. skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
  240. break;
  241. case VIRTIO_NET_HDR_GSO_UDP:
  242. skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
  243. break;
  244. case VIRTIO_NET_HDR_GSO_TCPV6:
  245. skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
  246. break;
  247. default:
  248. if (net_ratelimit())
  249. printk(KERN_WARNING "%s: bad gso type %u.\n",
  250. dev->name, hdr->hdr.gso_type);
  251. goto frame_err;
  252. }
  253. if (hdr->hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
  254. skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
  255. skb_shinfo(skb)->gso_size = hdr->hdr.gso_size;
  256. if (skb_shinfo(skb)->gso_size == 0) {
  257. if (net_ratelimit())
  258. printk(KERN_WARNING "%s: zero gso size.\n",
  259. dev->name);
  260. goto frame_err;
  261. }
  262. /* Header must be checked, and gso_segs computed. */
  263. skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
  264. skb_shinfo(skb)->gso_segs = 0;
  265. }
  266. netif_receive_skb(skb);
  267. return;
  268. frame_err:
  269. dev->stats.rx_frame_errors++;
  270. dev_kfree_skb(skb);
  271. }
  272. static int add_recvbuf_small(struct virtnet_info *vi, gfp_t gfp)
  273. {
  274. struct sk_buff *skb;
  275. struct skb_vnet_hdr *hdr;
  276. int err;
  277. skb = netdev_alloc_skb_ip_align(vi->dev, MAX_PACKET_LEN);
  278. if (unlikely(!skb))
  279. return -ENOMEM;
  280. skb_put(skb, MAX_PACKET_LEN);
  281. hdr = skb_vnet_hdr(skb);
  282. sg_set_buf(vi->rx_sg, &hdr->hdr, sizeof hdr->hdr);
  283. skb_to_sgvec(skb, vi->rx_sg + 1, 0, skb->len);
  284. err = virtqueue_add_buf_gfp(vi->rvq, vi->rx_sg, 0, 2, skb, gfp);
  285. if (err < 0)
  286. dev_kfree_skb(skb);
  287. return err;
  288. }
  289. static int add_recvbuf_big(struct virtnet_info *vi, gfp_t gfp)
  290. {
  291. struct page *first, *list = NULL;
  292. char *p;
  293. int i, err, offset;
  294. /* page in vi->rx_sg[MAX_SKB_FRAGS + 1] is list tail */
  295. for (i = MAX_SKB_FRAGS + 1; i > 1; --i) {
  296. first = get_a_page(vi, gfp);
  297. if (!first) {
  298. if (list)
  299. give_pages(vi, list);
  300. return -ENOMEM;
  301. }
  302. sg_set_buf(&vi->rx_sg[i], page_address(first), PAGE_SIZE);
  303. /* chain new page in list head to match sg */
  304. first->private = (unsigned long)list;
  305. list = first;
  306. }
  307. first = get_a_page(vi, gfp);
  308. if (!first) {
  309. give_pages(vi, list);
  310. return -ENOMEM;
  311. }
  312. p = page_address(first);
  313. /* vi->rx_sg[0], vi->rx_sg[1] share the same page */
  314. /* a separated vi->rx_sg[0] for virtio_net_hdr only due to QEMU bug */
  315. sg_set_buf(&vi->rx_sg[0], p, sizeof(struct virtio_net_hdr));
  316. /* vi->rx_sg[1] for data packet, from offset */
  317. offset = sizeof(struct padded_vnet_hdr);
  318. sg_set_buf(&vi->rx_sg[1], p + offset, PAGE_SIZE - offset);
  319. /* chain first in list head */
  320. first->private = (unsigned long)list;
  321. err = virtqueue_add_buf_gfp(vi->rvq, vi->rx_sg, 0, MAX_SKB_FRAGS + 2,
  322. first, gfp);
  323. if (err < 0)
  324. give_pages(vi, first);
  325. return err;
  326. }
  327. static int add_recvbuf_mergeable(struct virtnet_info *vi, gfp_t gfp)
  328. {
  329. struct page *page;
  330. int err;
  331. page = get_a_page(vi, gfp);
  332. if (!page)
  333. return -ENOMEM;
  334. sg_init_one(vi->rx_sg, page_address(page), PAGE_SIZE);
  335. err = virtqueue_add_buf_gfp(vi->rvq, vi->rx_sg, 0, 1, page, gfp);
  336. if (err < 0)
  337. give_pages(vi, page);
  338. return err;
  339. }
  340. /* Returns false if we couldn't fill entirely (OOM). */
  341. static bool try_fill_recv(struct virtnet_info *vi, gfp_t gfp)
  342. {
  343. int err;
  344. bool oom;
  345. do {
  346. if (vi->mergeable_rx_bufs)
  347. err = add_recvbuf_mergeable(vi, gfp);
  348. else if (vi->big_packets)
  349. err = add_recvbuf_big(vi, gfp);
  350. else
  351. err = add_recvbuf_small(vi, gfp);
  352. oom = err == -ENOMEM;
  353. if (err < 0)
  354. break;
  355. ++vi->num;
  356. } while (err > 0);
  357. if (unlikely(vi->num > vi->max))
  358. vi->max = vi->num;
  359. virtqueue_kick(vi->rvq);
  360. return !oom;
  361. }
  362. static void skb_recv_done(struct virtqueue *rvq)
  363. {
  364. struct virtnet_info *vi = rvq->vdev->priv;
  365. /* Schedule NAPI, Suppress further interrupts if successful. */
  366. if (napi_schedule_prep(&vi->napi)) {
  367. virtqueue_disable_cb(rvq);
  368. __napi_schedule(&vi->napi);
  369. }
  370. }
  371. static void refill_work(struct work_struct *work)
  372. {
  373. struct virtnet_info *vi;
  374. bool still_empty;
  375. vi = container_of(work, struct virtnet_info, refill.work);
  376. napi_disable(&vi->napi);
  377. still_empty = !try_fill_recv(vi, GFP_KERNEL);
  378. napi_enable(&vi->napi);
  379. /* In theory, this can happen: if we don't get any buffers in
  380. * we will *never* try to fill again. */
  381. if (still_empty)
  382. schedule_delayed_work(&vi->refill, HZ/2);
  383. }
  384. static int virtnet_poll(struct napi_struct *napi, int budget)
  385. {
  386. struct virtnet_info *vi = container_of(napi, struct virtnet_info, napi);
  387. void *buf;
  388. unsigned int len, received = 0;
  389. again:
  390. while (received < budget &&
  391. (buf = virtqueue_get_buf(vi->rvq, &len)) != NULL) {
  392. receive_buf(vi->dev, buf, len);
  393. --vi->num;
  394. received++;
  395. }
  396. if (vi->num < vi->max / 2) {
  397. if (!try_fill_recv(vi, GFP_ATOMIC))
  398. schedule_delayed_work(&vi->refill, 0);
  399. }
  400. /* Out of packets? */
  401. if (received < budget) {
  402. napi_complete(napi);
  403. if (unlikely(!virtqueue_enable_cb(vi->rvq)) &&
  404. napi_schedule_prep(napi)) {
  405. virtqueue_disable_cb(vi->rvq);
  406. __napi_schedule(napi);
  407. goto again;
  408. }
  409. }
  410. return received;
  411. }
  412. static unsigned int free_old_xmit_skbs(struct virtnet_info *vi)
  413. {
  414. struct sk_buff *skb;
  415. unsigned int len, tot_sgs = 0;
  416. while ((skb = virtqueue_get_buf(vi->svq, &len)) != NULL) {
  417. pr_debug("Sent skb %p\n", skb);
  418. vi->dev->stats.tx_bytes += skb->len;
  419. vi->dev->stats.tx_packets++;
  420. tot_sgs += skb_vnet_hdr(skb)->num_sg;
  421. dev_kfree_skb_any(skb);
  422. }
  423. return tot_sgs;
  424. }
  425. static int xmit_skb(struct virtnet_info *vi, struct sk_buff *skb)
  426. {
  427. struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb);
  428. const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
  429. pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
  430. if (skb->ip_summed == CHECKSUM_PARTIAL) {
  431. hdr->hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
  432. hdr->hdr.csum_start = skb->csum_start - skb_headroom(skb);
  433. hdr->hdr.csum_offset = skb->csum_offset;
  434. } else {
  435. hdr->hdr.flags = 0;
  436. hdr->hdr.csum_offset = hdr->hdr.csum_start = 0;
  437. }
  438. if (skb_is_gso(skb)) {
  439. hdr->hdr.hdr_len = skb_headlen(skb);
  440. hdr->hdr.gso_size = skb_shinfo(skb)->gso_size;
  441. if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
  442. hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
  443. else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
  444. hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
  445. else if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP)
  446. hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
  447. else
  448. BUG();
  449. if (skb_shinfo(skb)->gso_type & SKB_GSO_TCP_ECN)
  450. hdr->hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
  451. } else {
  452. hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
  453. hdr->hdr.gso_size = hdr->hdr.hdr_len = 0;
  454. }
  455. hdr->mhdr.num_buffers = 0;
  456. /* Encode metadata header at front. */
  457. if (vi->mergeable_rx_bufs)
  458. sg_set_buf(vi->tx_sg, &hdr->mhdr, sizeof hdr->mhdr);
  459. else
  460. sg_set_buf(vi->tx_sg, &hdr->hdr, sizeof hdr->hdr);
  461. hdr->num_sg = skb_to_sgvec(skb, vi->tx_sg + 1, 0, skb->len) + 1;
  462. return virtqueue_add_buf(vi->svq, vi->tx_sg, hdr->num_sg,
  463. 0, skb);
  464. }
  465. static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
  466. {
  467. struct virtnet_info *vi = netdev_priv(dev);
  468. int capacity;
  469. /* Free up any pending old buffers before queueing new ones. */
  470. free_old_xmit_skbs(vi);
  471. /* Try to transmit */
  472. capacity = xmit_skb(vi, skb);
  473. /* This can happen with OOM and indirect buffers. */
  474. if (unlikely(capacity < 0)) {
  475. if (net_ratelimit()) {
  476. if (likely(capacity == -ENOMEM)) {
  477. dev_warn(&dev->dev,
  478. "TX queue failure: out of memory\n");
  479. } else {
  480. dev->stats.tx_fifo_errors++;
  481. dev_warn(&dev->dev,
  482. "Unexpected TX queue failure: %d\n",
  483. capacity);
  484. }
  485. }
  486. dev->stats.tx_dropped++;
  487. kfree_skb(skb);
  488. return NETDEV_TX_OK;
  489. }
  490. virtqueue_kick(vi->svq);
  491. /* Don't wait up for transmitted skbs to be freed. */
  492. skb_orphan(skb);
  493. nf_reset(skb);
  494. /* Apparently nice girls don't return TX_BUSY; stop the queue
  495. * before it gets out of hand. Naturally, this wastes entries. */
  496. if (capacity < 2+MAX_SKB_FRAGS) {
  497. netif_stop_queue(dev);
  498. if (unlikely(!virtqueue_enable_cb(vi->svq))) {
  499. /* More just got used, free them then recheck. */
  500. capacity += free_old_xmit_skbs(vi);
  501. if (capacity >= 2+MAX_SKB_FRAGS) {
  502. netif_start_queue(dev);
  503. virtqueue_disable_cb(vi->svq);
  504. }
  505. }
  506. }
  507. return NETDEV_TX_OK;
  508. }
  509. static int virtnet_set_mac_address(struct net_device *dev, void *p)
  510. {
  511. struct virtnet_info *vi = netdev_priv(dev);
  512. struct virtio_device *vdev = vi->vdev;
  513. int ret;
  514. ret = eth_mac_addr(dev, p);
  515. if (ret)
  516. return ret;
  517. if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC))
  518. vdev->config->set(vdev, offsetof(struct virtio_net_config, mac),
  519. dev->dev_addr, dev->addr_len);
  520. return 0;
  521. }
  522. #ifdef CONFIG_NET_POLL_CONTROLLER
  523. static void virtnet_netpoll(struct net_device *dev)
  524. {
  525. struct virtnet_info *vi = netdev_priv(dev);
  526. napi_schedule(&vi->napi);
  527. }
  528. #endif
  529. static int virtnet_open(struct net_device *dev)
  530. {
  531. struct virtnet_info *vi = netdev_priv(dev);
  532. napi_enable(&vi->napi);
  533. /* If all buffers were filled by other side before we napi_enabled, we
  534. * won't get another interrupt, so process any outstanding packets
  535. * now. virtnet_poll wants re-enable the queue, so we disable here.
  536. * We synchronize against interrupts via NAPI_STATE_SCHED */
  537. if (napi_schedule_prep(&vi->napi)) {
  538. virtqueue_disable_cb(vi->rvq);
  539. __napi_schedule(&vi->napi);
  540. }
  541. return 0;
  542. }
  543. /*
  544. * Send command via the control virtqueue and check status. Commands
  545. * supported by the hypervisor, as indicated by feature bits, should
  546. * never fail unless improperly formated.
  547. */
  548. static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,
  549. struct scatterlist *data, int out, int in)
  550. {
  551. struct scatterlist *s, sg[VIRTNET_SEND_COMMAND_SG_MAX + 2];
  552. struct virtio_net_ctrl_hdr ctrl;
  553. virtio_net_ctrl_ack status = ~0;
  554. unsigned int tmp;
  555. int i;
  556. /* Caller should know better */
  557. BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ) ||
  558. (out + in > VIRTNET_SEND_COMMAND_SG_MAX));
  559. out++; /* Add header */
  560. in++; /* Add return status */
  561. ctrl.class = class;
  562. ctrl.cmd = cmd;
  563. sg_init_table(sg, out + in);
  564. sg_set_buf(&sg[0], &ctrl, sizeof(ctrl));
  565. for_each_sg(data, s, out + in - 2, i)
  566. sg_set_buf(&sg[i + 1], sg_virt(s), s->length);
  567. sg_set_buf(&sg[out + in - 1], &status, sizeof(status));
  568. BUG_ON(virtqueue_add_buf(vi->cvq, sg, out, in, vi) < 0);
  569. virtqueue_kick(vi->cvq);
  570. /*
  571. * Spin for a response, the kick causes an ioport write, trapping
  572. * into the hypervisor, so the request should be handled immediately.
  573. */
  574. while (!virtqueue_get_buf(vi->cvq, &tmp))
  575. cpu_relax();
  576. return status == VIRTIO_NET_OK;
  577. }
  578. static int virtnet_close(struct net_device *dev)
  579. {
  580. struct virtnet_info *vi = netdev_priv(dev);
  581. napi_disable(&vi->napi);
  582. return 0;
  583. }
  584. static void virtnet_get_drvinfo(struct net_device *dev,
  585. struct ethtool_drvinfo *drvinfo)
  586. {
  587. struct virtnet_info *vi = netdev_priv(dev);
  588. struct virtio_device *vdev = vi->vdev;
  589. strncpy(drvinfo->driver, KBUILD_MODNAME, ARRAY_SIZE(drvinfo->driver));
  590. strncpy(drvinfo->version, "N/A", ARRAY_SIZE(drvinfo->version));
  591. strncpy(drvinfo->fw_version, "N/A", ARRAY_SIZE(drvinfo->fw_version));
  592. strncpy(drvinfo->bus_info, dev_name(&vdev->dev),
  593. ARRAY_SIZE(drvinfo->bus_info));
  594. }
  595. static int virtnet_set_tx_csum(struct net_device *dev, u32 data)
  596. {
  597. struct virtnet_info *vi = netdev_priv(dev);
  598. struct virtio_device *vdev = vi->vdev;
  599. if (data && !virtio_has_feature(vdev, VIRTIO_NET_F_CSUM))
  600. return -ENOSYS;
  601. return ethtool_op_set_tx_hw_csum(dev, data);
  602. }
  603. static void virtnet_set_rx_mode(struct net_device *dev)
  604. {
  605. struct virtnet_info *vi = netdev_priv(dev);
  606. struct scatterlist sg[2];
  607. u8 promisc, allmulti;
  608. struct virtio_net_ctrl_mac *mac_data;
  609. struct netdev_hw_addr *ha;
  610. int uc_count;
  611. int mc_count;
  612. void *buf;
  613. int i;
  614. /* We can't dynamicaly set ndo_set_rx_mode, so return gracefully */
  615. if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
  616. return;
  617. promisc = ((dev->flags & IFF_PROMISC) != 0);
  618. allmulti = ((dev->flags & IFF_ALLMULTI) != 0);
  619. sg_init_one(sg, &promisc, sizeof(promisc));
  620. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
  621. VIRTIO_NET_CTRL_RX_PROMISC,
  622. sg, 1, 0))
  623. dev_warn(&dev->dev, "Failed to %sable promisc mode.\n",
  624. promisc ? "en" : "dis");
  625. sg_init_one(sg, &allmulti, sizeof(allmulti));
  626. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
  627. VIRTIO_NET_CTRL_RX_ALLMULTI,
  628. sg, 1, 0))
  629. dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
  630. allmulti ? "en" : "dis");
  631. uc_count = netdev_uc_count(dev);
  632. mc_count = netdev_mc_count(dev);
  633. /* MAC filter - use one buffer for both lists */
  634. buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
  635. (2 * sizeof(mac_data->entries)), GFP_ATOMIC);
  636. mac_data = buf;
  637. if (!buf) {
  638. dev_warn(&dev->dev, "No memory for MAC address buffer\n");
  639. return;
  640. }
  641. sg_init_table(sg, 2);
  642. /* Store the unicast list and count in the front of the buffer */
  643. mac_data->entries = uc_count;
  644. i = 0;
  645. netdev_for_each_uc_addr(ha, dev)
  646. memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
  647. sg_set_buf(&sg[0], mac_data,
  648. sizeof(mac_data->entries) + (uc_count * ETH_ALEN));
  649. /* multicast list and count fill the end */
  650. mac_data = (void *)&mac_data->macs[uc_count][0];
  651. mac_data->entries = mc_count;
  652. i = 0;
  653. netdev_for_each_mc_addr(ha, dev)
  654. memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
  655. sg_set_buf(&sg[1], mac_data,
  656. sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
  657. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
  658. VIRTIO_NET_CTRL_MAC_TABLE_SET,
  659. sg, 2, 0))
  660. dev_warn(&dev->dev, "Failed to set MAC fitler table.\n");
  661. kfree(buf);
  662. }
  663. static void virtnet_vlan_rx_add_vid(struct net_device *dev, u16 vid)
  664. {
  665. struct virtnet_info *vi = netdev_priv(dev);
  666. struct scatterlist sg;
  667. sg_init_one(&sg, &vid, sizeof(vid));
  668. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
  669. VIRTIO_NET_CTRL_VLAN_ADD, &sg, 1, 0))
  670. dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid);
  671. }
  672. static void virtnet_vlan_rx_kill_vid(struct net_device *dev, u16 vid)
  673. {
  674. struct virtnet_info *vi = netdev_priv(dev);
  675. struct scatterlist sg;
  676. sg_init_one(&sg, &vid, sizeof(vid));
  677. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
  678. VIRTIO_NET_CTRL_VLAN_DEL, &sg, 1, 0))
  679. dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid);
  680. }
  681. static const struct ethtool_ops virtnet_ethtool_ops = {
  682. .get_drvinfo = virtnet_get_drvinfo,
  683. .set_tx_csum = virtnet_set_tx_csum,
  684. .set_sg = ethtool_op_set_sg,
  685. .set_tso = ethtool_op_set_tso,
  686. .set_ufo = ethtool_op_set_ufo,
  687. .get_link = ethtool_op_get_link,
  688. };
  689. #define MIN_MTU 68
  690. #define MAX_MTU 65535
  691. static int virtnet_change_mtu(struct net_device *dev, int new_mtu)
  692. {
  693. if (new_mtu < MIN_MTU || new_mtu > MAX_MTU)
  694. return -EINVAL;
  695. dev->mtu = new_mtu;
  696. return 0;
  697. }
  698. static const struct net_device_ops virtnet_netdev = {
  699. .ndo_open = virtnet_open,
  700. .ndo_stop = virtnet_close,
  701. .ndo_start_xmit = start_xmit,
  702. .ndo_validate_addr = eth_validate_addr,
  703. .ndo_set_mac_address = virtnet_set_mac_address,
  704. .ndo_set_rx_mode = virtnet_set_rx_mode,
  705. .ndo_change_mtu = virtnet_change_mtu,
  706. .ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid,
  707. .ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
  708. #ifdef CONFIG_NET_POLL_CONTROLLER
  709. .ndo_poll_controller = virtnet_netpoll,
  710. #endif
  711. };
  712. static void virtnet_update_status(struct virtnet_info *vi)
  713. {
  714. u16 v;
  715. if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS))
  716. return;
  717. vi->vdev->config->get(vi->vdev,
  718. offsetof(struct virtio_net_config, status),
  719. &v, sizeof(v));
  720. /* Ignore unknown (future) status bits */
  721. v &= VIRTIO_NET_S_LINK_UP;
  722. if (vi->status == v)
  723. return;
  724. vi->status = v;
  725. if (vi->status & VIRTIO_NET_S_LINK_UP) {
  726. netif_carrier_on(vi->dev);
  727. netif_wake_queue(vi->dev);
  728. } else {
  729. netif_carrier_off(vi->dev);
  730. netif_stop_queue(vi->dev);
  731. }
  732. }
  733. static void virtnet_config_changed(struct virtio_device *vdev)
  734. {
  735. struct virtnet_info *vi = vdev->priv;
  736. virtnet_update_status(vi);
  737. }
  738. static int virtnet_probe(struct virtio_device *vdev)
  739. {
  740. int err;
  741. struct net_device *dev;
  742. struct virtnet_info *vi;
  743. struct virtqueue *vqs[3];
  744. vq_callback_t *callbacks[] = { skb_recv_done, skb_xmit_done, NULL};
  745. const char *names[] = { "input", "output", "control" };
  746. int nvqs;
  747. /* Allocate ourselves a network device with room for our info */
  748. dev = alloc_etherdev(sizeof(struct virtnet_info));
  749. if (!dev)
  750. return -ENOMEM;
  751. /* Set up network device as normal. */
  752. dev->netdev_ops = &virtnet_netdev;
  753. dev->features = NETIF_F_HIGHDMA;
  754. SET_ETHTOOL_OPS(dev, &virtnet_ethtool_ops);
  755. SET_NETDEV_DEV(dev, &vdev->dev);
  756. /* Do we support "hardware" checksums? */
  757. if (csum && virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) {
  758. /* This opens up the world of extra features. */
  759. dev->features |= NETIF_F_HW_CSUM|NETIF_F_SG|NETIF_F_FRAGLIST;
  760. if (gso && virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
  761. dev->features |= NETIF_F_TSO | NETIF_F_UFO
  762. | NETIF_F_TSO_ECN | NETIF_F_TSO6;
  763. }
  764. /* Individual feature bits: what can host handle? */
  765. if (gso && virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4))
  766. dev->features |= NETIF_F_TSO;
  767. if (gso && virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6))
  768. dev->features |= NETIF_F_TSO6;
  769. if (gso && virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
  770. dev->features |= NETIF_F_TSO_ECN;
  771. if (gso && virtio_has_feature(vdev, VIRTIO_NET_F_HOST_UFO))
  772. dev->features |= NETIF_F_UFO;
  773. }
  774. /* Configuration may specify what MAC to use. Otherwise random. */
  775. if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) {
  776. vdev->config->get(vdev,
  777. offsetof(struct virtio_net_config, mac),
  778. dev->dev_addr, dev->addr_len);
  779. } else
  780. random_ether_addr(dev->dev_addr);
  781. /* Set up our device-specific information */
  782. vi = netdev_priv(dev);
  783. netif_napi_add(dev, &vi->napi, virtnet_poll, napi_weight);
  784. vi->dev = dev;
  785. vi->vdev = vdev;
  786. vdev->priv = vi;
  787. vi->pages = NULL;
  788. INIT_DELAYED_WORK(&vi->refill, refill_work);
  789. sg_init_table(vi->rx_sg, ARRAY_SIZE(vi->rx_sg));
  790. sg_init_table(vi->tx_sg, ARRAY_SIZE(vi->tx_sg));
  791. /* If we can receive ANY GSO packets, we must allocate large ones. */
  792. if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
  793. virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) ||
  794. virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN))
  795. vi->big_packets = true;
  796. if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
  797. vi->mergeable_rx_bufs = true;
  798. /* We expect two virtqueues, receive then send,
  799. * and optionally control. */
  800. nvqs = virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ) ? 3 : 2;
  801. err = vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names);
  802. if (err)
  803. goto free;
  804. vi->rvq = vqs[0];
  805. vi->svq = vqs[1];
  806. if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ)) {
  807. vi->cvq = vqs[2];
  808. if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
  809. dev->features |= NETIF_F_HW_VLAN_FILTER;
  810. }
  811. err = register_netdev(dev);
  812. if (err) {
  813. pr_debug("virtio_net: registering device failed\n");
  814. goto free_vqs;
  815. }
  816. /* Last of all, set up some receive buffers. */
  817. try_fill_recv(vi, GFP_KERNEL);
  818. /* If we didn't even get one input buffer, we're useless. */
  819. if (vi->num == 0) {
  820. err = -ENOMEM;
  821. goto unregister;
  822. }
  823. vi->status = VIRTIO_NET_S_LINK_UP;
  824. virtnet_update_status(vi);
  825. netif_carrier_on(dev);
  826. pr_debug("virtnet: registered device %s\n", dev->name);
  827. return 0;
  828. unregister:
  829. unregister_netdev(dev);
  830. cancel_delayed_work_sync(&vi->refill);
  831. free_vqs:
  832. vdev->config->del_vqs(vdev);
  833. free:
  834. free_netdev(dev);
  835. return err;
  836. }
  837. static void free_unused_bufs(struct virtnet_info *vi)
  838. {
  839. void *buf;
  840. while (1) {
  841. buf = virtqueue_detach_unused_buf(vi->svq);
  842. if (!buf)
  843. break;
  844. dev_kfree_skb(buf);
  845. }
  846. while (1) {
  847. buf = virtqueue_detach_unused_buf(vi->rvq);
  848. if (!buf)
  849. break;
  850. if (vi->mergeable_rx_bufs || vi->big_packets)
  851. give_pages(vi, buf);
  852. else
  853. dev_kfree_skb(buf);
  854. --vi->num;
  855. }
  856. BUG_ON(vi->num != 0);
  857. }
  858. static void __devexit virtnet_remove(struct virtio_device *vdev)
  859. {
  860. struct virtnet_info *vi = vdev->priv;
  861. /* Stop all the virtqueues. */
  862. vdev->config->reset(vdev);
  863. unregister_netdev(vi->dev);
  864. cancel_delayed_work_sync(&vi->refill);
  865. /* Free unused buffers in both send and recv, if any. */
  866. free_unused_bufs(vi);
  867. vdev->config->del_vqs(vi->vdev);
  868. while (vi->pages)
  869. __free_pages(get_a_page(vi, GFP_KERNEL), 0);
  870. free_netdev(vi->dev);
  871. }
  872. static struct virtio_device_id id_table[] = {
  873. { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
  874. { 0 },
  875. };
  876. static unsigned int features[] = {
  877. VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM,
  878. VIRTIO_NET_F_GSO, VIRTIO_NET_F_MAC,
  879. VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6,
  880. VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6,
  881. VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO,
  882. VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ,
  883. VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN,
  884. };
  885. static struct virtio_driver virtio_net_driver = {
  886. .feature_table = features,
  887. .feature_table_size = ARRAY_SIZE(features),
  888. .driver.name = KBUILD_MODNAME,
  889. .driver.owner = THIS_MODULE,
  890. .id_table = id_table,
  891. .probe = virtnet_probe,
  892. .remove = __devexit_p(virtnet_remove),
  893. .config_changed = virtnet_config_changed,
  894. };
  895. static int __init init(void)
  896. {
  897. return register_virtio_driver(&virtio_net_driver);
  898. }
  899. static void __exit fini(void)
  900. {
  901. unregister_virtio_driver(&virtio_net_driver);
  902. }
  903. module_init(init);
  904. module_exit(fini);
  905. MODULE_DEVICE_TABLE(virtio, id_table);
  906. MODULE_DESCRIPTION("Virtio network driver");
  907. MODULE_LICENSE("GPL");