virtio_net.c 43 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754
  1. /* A network driver using virtio.
  2. *
  3. * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
  4. *
  5. * This program is free software; you can redistribute it and/or modify
  6. * it under the terms of the GNU General Public License as published by
  7. * the Free Software Foundation; either version 2 of the License, or
  8. * (at your option) any later version.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU General Public License
  16. * along with this program; if not, write to the Free Software
  17. * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  18. */
  19. //#define DEBUG
  20. #include <linux/netdevice.h>
  21. #include <linux/etherdevice.h>
  22. #include <linux/ethtool.h>
  23. #include <linux/module.h>
  24. #include <linux/virtio.h>
  25. #include <linux/virtio_net.h>
  26. #include <linux/scatterlist.h>
  27. #include <linux/if_vlan.h>
  28. #include <linux/slab.h>
  29. #include <linux/cpu.h>
  30. static int napi_weight = NAPI_POLL_WEIGHT;
  31. module_param(napi_weight, int, 0444);
  32. static bool csum = true, gso = true;
  33. module_param(csum, bool, 0444);
  34. module_param(gso, bool, 0444);
  35. /* FIXME: MTU in config. */
  36. #define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
  37. #define GOOD_COPY_LEN 128
  38. #define VIRTNET_DRIVER_VERSION "1.0.0"
  39. struct virtnet_stats {
  40. struct u64_stats_sync tx_syncp;
  41. struct u64_stats_sync rx_syncp;
  42. u64 tx_bytes;
  43. u64 tx_packets;
  44. u64 rx_bytes;
  45. u64 rx_packets;
  46. };
  47. /* Internal representation of a send virtqueue */
  48. struct send_queue {
  49. /* Virtqueue associated with this send _queue */
  50. struct virtqueue *vq;
  51. /* TX: fragments + linear part + virtio header */
  52. struct scatterlist sg[MAX_SKB_FRAGS + 2];
  53. /* Name of the send queue: output.$index */
  54. char name[40];
  55. };
  56. /* Internal representation of a receive virtqueue */
  57. struct receive_queue {
  58. /* Virtqueue associated with this receive_queue */
  59. struct virtqueue *vq;
  60. struct napi_struct napi;
  61. /* Number of input buffers, and max we've ever had. */
  62. unsigned int num, max;
  63. /* Chain pages by the private ptr. */
  64. struct page *pages;
  65. /* RX: fragments + linear part + virtio header */
  66. struct scatterlist sg[MAX_SKB_FRAGS + 2];
  67. /* Name of this receive queue: input.$index */
  68. char name[40];
  69. };
  70. struct virtnet_info {
  71. struct virtio_device *vdev;
  72. struct virtqueue *cvq;
  73. struct net_device *dev;
  74. struct send_queue *sq;
  75. struct receive_queue *rq;
  76. unsigned int status;
  77. /* Max # of queue pairs supported by the device */
  78. u16 max_queue_pairs;
  79. /* # of queue pairs currently used by the driver */
  80. u16 curr_queue_pairs;
  81. /* I like... big packets and I cannot lie! */
  82. bool big_packets;
  83. /* Host will merge rx buffers for big packets (shake it! shake it!) */
  84. bool mergeable_rx_bufs;
  85. /* Has control virtqueue */
  86. bool has_cvq;
  87. /* enable config space updates */
  88. bool config_enable;
  89. /* Active statistics */
  90. struct virtnet_stats __percpu *stats;
  91. /* Work struct for refilling if we run low on memory. */
  92. struct delayed_work refill;
  93. /* Work struct for config space updates */
  94. struct work_struct config_work;
  95. /* Lock for config space updates */
  96. struct mutex config_lock;
  97. /* Does the affinity hint is set for virtqueues? */
  98. bool affinity_hint_set;
  99. /* Per-cpu variable to show the mapping from CPU to virtqueue */
  100. int __percpu *vq_index;
  101. /* CPU hot plug notifier */
  102. struct notifier_block nb;
  103. };
  104. struct skb_vnet_hdr {
  105. union {
  106. struct virtio_net_hdr hdr;
  107. struct virtio_net_hdr_mrg_rxbuf mhdr;
  108. };
  109. };
  110. struct padded_vnet_hdr {
  111. struct virtio_net_hdr hdr;
  112. /*
  113. * virtio_net_hdr should be in a separated sg buffer because of a
  114. * QEMU bug, and data sg buffer shares same page with this header sg.
  115. * This padding makes next sg 16 byte aligned after virtio_net_hdr.
  116. */
  117. char padding[6];
  118. };
  119. /* Converting between virtqueue no. and kernel tx/rx queue no.
  120. * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq
  121. */
  122. static int vq2txq(struct virtqueue *vq)
  123. {
  124. return (vq->index - 1) / 2;
  125. }
  126. static int txq2vq(int txq)
  127. {
  128. return txq * 2 + 1;
  129. }
  130. static int vq2rxq(struct virtqueue *vq)
  131. {
  132. return vq->index / 2;
  133. }
  134. static int rxq2vq(int rxq)
  135. {
  136. return rxq * 2;
  137. }
  138. static inline struct skb_vnet_hdr *skb_vnet_hdr(struct sk_buff *skb)
  139. {
  140. return (struct skb_vnet_hdr *)skb->cb;
  141. }
  142. /*
  143. * private is used to chain pages for big packets, put the whole
  144. * most recent used list in the beginning for reuse
  145. */
  146. static void give_pages(struct receive_queue *rq, struct page *page)
  147. {
  148. struct page *end;
  149. /* Find end of list, sew whole thing into vi->rq.pages. */
  150. for (end = page; end->private; end = (struct page *)end->private);
  151. end->private = (unsigned long)rq->pages;
  152. rq->pages = page;
  153. }
  154. static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask)
  155. {
  156. struct page *p = rq->pages;
  157. if (p) {
  158. rq->pages = (struct page *)p->private;
  159. /* clear private here, it is used to chain pages */
  160. p->private = 0;
  161. } else
  162. p = alloc_page(gfp_mask);
  163. return p;
  164. }
  165. static void skb_xmit_done(struct virtqueue *vq)
  166. {
  167. struct virtnet_info *vi = vq->vdev->priv;
  168. /* Suppress further interrupts. */
  169. virtqueue_disable_cb(vq);
  170. /* We were probably waiting for more output buffers. */
  171. netif_wake_subqueue(vi->dev, vq2txq(vq));
  172. }
  173. static void set_skb_frag(struct sk_buff *skb, struct page *page,
  174. unsigned int offset, unsigned int *len)
  175. {
  176. int size = min((unsigned)PAGE_SIZE - offset, *len);
  177. int i = skb_shinfo(skb)->nr_frags;
  178. __skb_fill_page_desc(skb, i, page, offset, size);
  179. skb->data_len += size;
  180. skb->len += size;
  181. skb->truesize += PAGE_SIZE;
  182. skb_shinfo(skb)->nr_frags++;
  183. skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG;
  184. *len -= size;
  185. }
  186. /* Called from bottom half context */
  187. static struct sk_buff *page_to_skb(struct receive_queue *rq,
  188. struct page *page, unsigned int len)
  189. {
  190. struct virtnet_info *vi = rq->vq->vdev->priv;
  191. struct sk_buff *skb;
  192. struct skb_vnet_hdr *hdr;
  193. unsigned int copy, hdr_len, offset;
  194. char *p;
  195. p = page_address(page);
  196. /* copy small packet so we can reuse these pages for small data */
  197. skb = netdev_alloc_skb_ip_align(vi->dev, GOOD_COPY_LEN);
  198. if (unlikely(!skb))
  199. return NULL;
  200. hdr = skb_vnet_hdr(skb);
  201. if (vi->mergeable_rx_bufs) {
  202. hdr_len = sizeof hdr->mhdr;
  203. offset = hdr_len;
  204. } else {
  205. hdr_len = sizeof hdr->hdr;
  206. offset = sizeof(struct padded_vnet_hdr);
  207. }
  208. memcpy(hdr, p, hdr_len);
  209. len -= hdr_len;
  210. p += offset;
  211. copy = len;
  212. if (copy > skb_tailroom(skb))
  213. copy = skb_tailroom(skb);
  214. memcpy(skb_put(skb, copy), p, copy);
  215. len -= copy;
  216. offset += copy;
  217. /*
  218. * Verify that we can indeed put this data into a skb.
  219. * This is here to handle cases when the device erroneously
  220. * tries to receive more than is possible. This is usually
  221. * the case of a broken device.
  222. */
  223. if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) {
  224. net_dbg_ratelimited("%s: too much data\n", skb->dev->name);
  225. dev_kfree_skb(skb);
  226. return NULL;
  227. }
  228. while (len) {
  229. set_skb_frag(skb, page, offset, &len);
  230. page = (struct page *)page->private;
  231. offset = 0;
  232. }
  233. if (page)
  234. give_pages(rq, page);
  235. return skb;
  236. }
  237. static int receive_mergeable(struct receive_queue *rq, struct sk_buff *skb)
  238. {
  239. struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb);
  240. struct page *page;
  241. int num_buf, i, len;
  242. num_buf = hdr->mhdr.num_buffers;
  243. while (--num_buf) {
  244. i = skb_shinfo(skb)->nr_frags;
  245. if (i >= MAX_SKB_FRAGS) {
  246. pr_debug("%s: packet too long\n", skb->dev->name);
  247. skb->dev->stats.rx_length_errors++;
  248. return -EINVAL;
  249. }
  250. page = virtqueue_get_buf(rq->vq, &len);
  251. if (!page) {
  252. pr_debug("%s: rx error: %d buffers missing\n",
  253. skb->dev->name, hdr->mhdr.num_buffers);
  254. skb->dev->stats.rx_length_errors++;
  255. return -EINVAL;
  256. }
  257. if (len > PAGE_SIZE)
  258. len = PAGE_SIZE;
  259. set_skb_frag(skb, page, 0, &len);
  260. --rq->num;
  261. }
  262. return 0;
  263. }
  264. static void receive_buf(struct receive_queue *rq, void *buf, unsigned int len)
  265. {
  266. struct virtnet_info *vi = rq->vq->vdev->priv;
  267. struct net_device *dev = vi->dev;
  268. struct virtnet_stats *stats = this_cpu_ptr(vi->stats);
  269. struct sk_buff *skb;
  270. struct page *page;
  271. struct skb_vnet_hdr *hdr;
  272. if (unlikely(len < sizeof(struct virtio_net_hdr) + ETH_HLEN)) {
  273. pr_debug("%s: short packet %i\n", dev->name, len);
  274. dev->stats.rx_length_errors++;
  275. if (vi->mergeable_rx_bufs || vi->big_packets)
  276. give_pages(rq, buf);
  277. else
  278. dev_kfree_skb(buf);
  279. return;
  280. }
  281. if (!vi->mergeable_rx_bufs && !vi->big_packets) {
  282. skb = buf;
  283. len -= sizeof(struct virtio_net_hdr);
  284. skb_trim(skb, len);
  285. } else {
  286. page = buf;
  287. skb = page_to_skb(rq, page, len);
  288. if (unlikely(!skb)) {
  289. dev->stats.rx_dropped++;
  290. give_pages(rq, page);
  291. return;
  292. }
  293. if (vi->mergeable_rx_bufs)
  294. if (receive_mergeable(rq, skb)) {
  295. dev_kfree_skb(skb);
  296. return;
  297. }
  298. }
  299. hdr = skb_vnet_hdr(skb);
  300. u64_stats_update_begin(&stats->rx_syncp);
  301. stats->rx_bytes += skb->len;
  302. stats->rx_packets++;
  303. u64_stats_update_end(&stats->rx_syncp);
  304. if (hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
  305. pr_debug("Needs csum!\n");
  306. if (!skb_partial_csum_set(skb,
  307. hdr->hdr.csum_start,
  308. hdr->hdr.csum_offset))
  309. goto frame_err;
  310. } else if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID) {
  311. skb->ip_summed = CHECKSUM_UNNECESSARY;
  312. }
  313. skb->protocol = eth_type_trans(skb, dev);
  314. pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
  315. ntohs(skb->protocol), skb->len, skb->pkt_type);
  316. if (hdr->hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
  317. pr_debug("GSO!\n");
  318. switch (hdr->hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
  319. case VIRTIO_NET_HDR_GSO_TCPV4:
  320. skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
  321. break;
  322. case VIRTIO_NET_HDR_GSO_UDP:
  323. skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
  324. break;
  325. case VIRTIO_NET_HDR_GSO_TCPV6:
  326. skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
  327. break;
  328. default:
  329. net_warn_ratelimited("%s: bad gso type %u.\n",
  330. dev->name, hdr->hdr.gso_type);
  331. goto frame_err;
  332. }
  333. if (hdr->hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
  334. skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
  335. skb_shinfo(skb)->gso_size = hdr->hdr.gso_size;
  336. if (skb_shinfo(skb)->gso_size == 0) {
  337. net_warn_ratelimited("%s: zero gso size.\n", dev->name);
  338. goto frame_err;
  339. }
  340. /* Header must be checked, and gso_segs computed. */
  341. skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
  342. skb_shinfo(skb)->gso_segs = 0;
  343. }
  344. netif_receive_skb(skb);
  345. return;
  346. frame_err:
  347. dev->stats.rx_frame_errors++;
  348. dev_kfree_skb(skb);
  349. }
  350. static int add_recvbuf_small(struct receive_queue *rq, gfp_t gfp)
  351. {
  352. struct virtnet_info *vi = rq->vq->vdev->priv;
  353. struct sk_buff *skb;
  354. struct skb_vnet_hdr *hdr;
  355. int err;
  356. skb = __netdev_alloc_skb_ip_align(vi->dev, MAX_PACKET_LEN, gfp);
  357. if (unlikely(!skb))
  358. return -ENOMEM;
  359. skb_put(skb, MAX_PACKET_LEN);
  360. hdr = skb_vnet_hdr(skb);
  361. sg_set_buf(rq->sg, &hdr->hdr, sizeof hdr->hdr);
  362. skb_to_sgvec(skb, rq->sg + 1, 0, skb->len);
  363. err = virtqueue_add_inbuf(rq->vq, rq->sg, 2, skb, gfp);
  364. if (err < 0)
  365. dev_kfree_skb(skb);
  366. return err;
  367. }
  368. static int add_recvbuf_big(struct receive_queue *rq, gfp_t gfp)
  369. {
  370. struct page *first, *list = NULL;
  371. char *p;
  372. int i, err, offset;
  373. /* page in rq->sg[MAX_SKB_FRAGS + 1] is list tail */
  374. for (i = MAX_SKB_FRAGS + 1; i > 1; --i) {
  375. first = get_a_page(rq, gfp);
  376. if (!first) {
  377. if (list)
  378. give_pages(rq, list);
  379. return -ENOMEM;
  380. }
  381. sg_set_buf(&rq->sg[i], page_address(first), PAGE_SIZE);
  382. /* chain new page in list head to match sg */
  383. first->private = (unsigned long)list;
  384. list = first;
  385. }
  386. first = get_a_page(rq, gfp);
  387. if (!first) {
  388. give_pages(rq, list);
  389. return -ENOMEM;
  390. }
  391. p = page_address(first);
  392. /* rq->sg[0], rq->sg[1] share the same page */
  393. /* a separated rq->sg[0] for virtio_net_hdr only due to QEMU bug */
  394. sg_set_buf(&rq->sg[0], p, sizeof(struct virtio_net_hdr));
  395. /* rq->sg[1] for data packet, from offset */
  396. offset = sizeof(struct padded_vnet_hdr);
  397. sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset);
  398. /* chain first in list head */
  399. first->private = (unsigned long)list;
  400. err = virtqueue_add_inbuf(rq->vq, rq->sg, MAX_SKB_FRAGS + 2,
  401. first, gfp);
  402. if (err < 0)
  403. give_pages(rq, first);
  404. return err;
  405. }
  406. static int add_recvbuf_mergeable(struct receive_queue *rq, gfp_t gfp)
  407. {
  408. struct page *page;
  409. int err;
  410. page = get_a_page(rq, gfp);
  411. if (!page)
  412. return -ENOMEM;
  413. sg_init_one(rq->sg, page_address(page), PAGE_SIZE);
  414. err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, page, gfp);
  415. if (err < 0)
  416. give_pages(rq, page);
  417. return err;
  418. }
  419. /*
  420. * Returns false if we couldn't fill entirely (OOM).
  421. *
  422. * Normally run in the receive path, but can also be run from ndo_open
  423. * before we're receiving packets, or from refill_work which is
  424. * careful to disable receiving (using napi_disable).
  425. */
  426. static bool try_fill_recv(struct receive_queue *rq, gfp_t gfp)
  427. {
  428. struct virtnet_info *vi = rq->vq->vdev->priv;
  429. int err;
  430. bool oom;
  431. do {
  432. if (vi->mergeable_rx_bufs)
  433. err = add_recvbuf_mergeable(rq, gfp);
  434. else if (vi->big_packets)
  435. err = add_recvbuf_big(rq, gfp);
  436. else
  437. err = add_recvbuf_small(rq, gfp);
  438. oom = err == -ENOMEM;
  439. if (err)
  440. break;
  441. ++rq->num;
  442. } while (rq->vq->num_free);
  443. if (unlikely(rq->num > rq->max))
  444. rq->max = rq->num;
  445. virtqueue_kick(rq->vq);
  446. return !oom;
  447. }
  448. static void skb_recv_done(struct virtqueue *rvq)
  449. {
  450. struct virtnet_info *vi = rvq->vdev->priv;
  451. struct receive_queue *rq = &vi->rq[vq2rxq(rvq)];
  452. /* Schedule NAPI, Suppress further interrupts if successful. */
  453. if (napi_schedule_prep(&rq->napi)) {
  454. virtqueue_disable_cb(rvq);
  455. __napi_schedule(&rq->napi);
  456. }
  457. }
  458. static void virtnet_napi_enable(struct receive_queue *rq)
  459. {
  460. napi_enable(&rq->napi);
  461. /* If all buffers were filled by other side before we napi_enabled, we
  462. * won't get another interrupt, so process any outstanding packets
  463. * now. virtnet_poll wants re-enable the queue, so we disable here.
  464. * We synchronize against interrupts via NAPI_STATE_SCHED */
  465. if (napi_schedule_prep(&rq->napi)) {
  466. virtqueue_disable_cb(rq->vq);
  467. local_bh_disable();
  468. __napi_schedule(&rq->napi);
  469. local_bh_enable();
  470. }
  471. }
  472. static void refill_work(struct work_struct *work)
  473. {
  474. struct virtnet_info *vi =
  475. container_of(work, struct virtnet_info, refill.work);
  476. bool still_empty;
  477. int i;
  478. for (i = 0; i < vi->curr_queue_pairs; i++) {
  479. struct receive_queue *rq = &vi->rq[i];
  480. napi_disable(&rq->napi);
  481. still_empty = !try_fill_recv(rq, GFP_KERNEL);
  482. virtnet_napi_enable(rq);
  483. /* In theory, this can happen: if we don't get any buffers in
  484. * we will *never* try to fill again.
  485. */
  486. if (still_empty)
  487. schedule_delayed_work(&vi->refill, HZ/2);
  488. }
  489. }
  490. static int virtnet_poll(struct napi_struct *napi, int budget)
  491. {
  492. struct receive_queue *rq =
  493. container_of(napi, struct receive_queue, napi);
  494. struct virtnet_info *vi = rq->vq->vdev->priv;
  495. void *buf;
  496. unsigned int r, len, received = 0;
  497. again:
  498. while (received < budget &&
  499. (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) {
  500. receive_buf(rq, buf, len);
  501. --rq->num;
  502. received++;
  503. }
  504. if (rq->num < rq->max / 2) {
  505. if (!try_fill_recv(rq, GFP_ATOMIC))
  506. schedule_delayed_work(&vi->refill, 0);
  507. }
  508. /* Out of packets? */
  509. if (received < budget) {
  510. r = virtqueue_enable_cb_prepare(rq->vq);
  511. napi_complete(napi);
  512. if (unlikely(virtqueue_poll(rq->vq, r)) &&
  513. napi_schedule_prep(napi)) {
  514. virtqueue_disable_cb(rq->vq);
  515. __napi_schedule(napi);
  516. goto again;
  517. }
  518. }
  519. return received;
  520. }
  521. static int virtnet_open(struct net_device *dev)
  522. {
  523. struct virtnet_info *vi = netdev_priv(dev);
  524. int i;
  525. for (i = 0; i < vi->max_queue_pairs; i++) {
  526. if (i < vi->curr_queue_pairs)
  527. /* Make sure we have some buffers: if oom use wq. */
  528. if (!try_fill_recv(&vi->rq[i], GFP_KERNEL))
  529. schedule_delayed_work(&vi->refill, 0);
  530. virtnet_napi_enable(&vi->rq[i]);
  531. }
  532. return 0;
  533. }
  534. static void free_old_xmit_skbs(struct send_queue *sq)
  535. {
  536. struct sk_buff *skb;
  537. unsigned int len;
  538. struct virtnet_info *vi = sq->vq->vdev->priv;
  539. struct virtnet_stats *stats = this_cpu_ptr(vi->stats);
  540. while ((skb = virtqueue_get_buf(sq->vq, &len)) != NULL) {
  541. pr_debug("Sent skb %p\n", skb);
  542. u64_stats_update_begin(&stats->tx_syncp);
  543. stats->tx_bytes += skb->len;
  544. stats->tx_packets++;
  545. u64_stats_update_end(&stats->tx_syncp);
  546. dev_kfree_skb_any(skb);
  547. }
  548. }
  549. static int xmit_skb(struct send_queue *sq, struct sk_buff *skb)
  550. {
  551. struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb);
  552. const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
  553. struct virtnet_info *vi = sq->vq->vdev->priv;
  554. unsigned num_sg;
  555. pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
  556. if (skb->ip_summed == CHECKSUM_PARTIAL) {
  557. hdr->hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
  558. hdr->hdr.csum_start = skb_checksum_start_offset(skb);
  559. hdr->hdr.csum_offset = skb->csum_offset;
  560. } else {
  561. hdr->hdr.flags = 0;
  562. hdr->hdr.csum_offset = hdr->hdr.csum_start = 0;
  563. }
  564. if (skb_is_gso(skb)) {
  565. hdr->hdr.hdr_len = skb_headlen(skb);
  566. hdr->hdr.gso_size = skb_shinfo(skb)->gso_size;
  567. if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
  568. hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
  569. else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
  570. hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
  571. else if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP)
  572. hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
  573. else
  574. BUG();
  575. if (skb_shinfo(skb)->gso_type & SKB_GSO_TCP_ECN)
  576. hdr->hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
  577. } else {
  578. hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
  579. hdr->hdr.gso_size = hdr->hdr.hdr_len = 0;
  580. }
  581. hdr->mhdr.num_buffers = 0;
  582. /* Encode metadata header at front. */
  583. if (vi->mergeable_rx_bufs)
  584. sg_set_buf(sq->sg, &hdr->mhdr, sizeof hdr->mhdr);
  585. else
  586. sg_set_buf(sq->sg, &hdr->hdr, sizeof hdr->hdr);
  587. num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len) + 1;
  588. return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC);
  589. }
  590. static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
  591. {
  592. struct virtnet_info *vi = netdev_priv(dev);
  593. int qnum = skb_get_queue_mapping(skb);
  594. struct send_queue *sq = &vi->sq[qnum];
  595. int err;
  596. /* Free up any pending old buffers before queueing new ones. */
  597. free_old_xmit_skbs(sq);
  598. /* Try to transmit */
  599. err = xmit_skb(sq, skb);
  600. /* This should not happen! */
  601. if (unlikely(err)) {
  602. dev->stats.tx_fifo_errors++;
  603. if (net_ratelimit())
  604. dev_warn(&dev->dev,
  605. "Unexpected TXQ (%d) queue failure: %d\n", qnum, err);
  606. dev->stats.tx_dropped++;
  607. kfree_skb(skb);
  608. return NETDEV_TX_OK;
  609. }
  610. virtqueue_kick(sq->vq);
  611. /* Don't wait up for transmitted skbs to be freed. */
  612. skb_orphan(skb);
  613. nf_reset(skb);
  614. /* Apparently nice girls don't return TX_BUSY; stop the queue
  615. * before it gets out of hand. Naturally, this wastes entries. */
  616. if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
  617. netif_stop_subqueue(dev, qnum);
  618. if (unlikely(!virtqueue_enable_cb_delayed(sq->vq))) {
  619. /* More just got used, free them then recheck. */
  620. free_old_xmit_skbs(sq);
  621. if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
  622. netif_start_subqueue(dev, qnum);
  623. virtqueue_disable_cb(sq->vq);
  624. }
  625. }
  626. }
  627. return NETDEV_TX_OK;
  628. }
  629. /*
  630. * Send command via the control virtqueue and check status. Commands
  631. * supported by the hypervisor, as indicated by feature bits, should
  632. * never fail unless improperly formated.
  633. */
  634. static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,
  635. struct scatterlist *out,
  636. struct scatterlist *in)
  637. {
  638. struct scatterlist *sgs[4], hdr, stat;
  639. struct virtio_net_ctrl_hdr ctrl;
  640. virtio_net_ctrl_ack status = ~0;
  641. unsigned out_num = 0, in_num = 0, tmp;
  642. /* Caller should know better */
  643. BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ));
  644. ctrl.class = class;
  645. ctrl.cmd = cmd;
  646. /* Add header */
  647. sg_init_one(&hdr, &ctrl, sizeof(ctrl));
  648. sgs[out_num++] = &hdr;
  649. if (out)
  650. sgs[out_num++] = out;
  651. if (in)
  652. sgs[out_num + in_num++] = in;
  653. /* Add return status. */
  654. sg_init_one(&stat, &status, sizeof(status));
  655. sgs[out_num + in_num++] = &stat;
  656. BUG_ON(out_num + in_num > ARRAY_SIZE(sgs));
  657. BUG_ON(virtqueue_add_sgs(vi->cvq, sgs, out_num, in_num, vi, GFP_ATOMIC)
  658. < 0);
  659. virtqueue_kick(vi->cvq);
  660. /* Spin for a response, the kick causes an ioport write, trapping
  661. * into the hypervisor, so the request should be handled immediately.
  662. */
  663. while (!virtqueue_get_buf(vi->cvq, &tmp))
  664. cpu_relax();
  665. return status == VIRTIO_NET_OK;
  666. }
  667. static int virtnet_set_mac_address(struct net_device *dev, void *p)
  668. {
  669. struct virtnet_info *vi = netdev_priv(dev);
  670. struct virtio_device *vdev = vi->vdev;
  671. int ret;
  672. struct sockaddr *addr = p;
  673. struct scatterlist sg;
  674. ret = eth_prepare_mac_addr_change(dev, p);
  675. if (ret)
  676. return ret;
  677. if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
  678. sg_init_one(&sg, addr->sa_data, dev->addr_len);
  679. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
  680. VIRTIO_NET_CTRL_MAC_ADDR_SET,
  681. &sg, NULL)) {
  682. dev_warn(&vdev->dev,
  683. "Failed to set mac address by vq command.\n");
  684. return -EINVAL;
  685. }
  686. } else if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) {
  687. vdev->config->set(vdev, offsetof(struct virtio_net_config, mac),
  688. addr->sa_data, dev->addr_len);
  689. }
  690. eth_commit_mac_addr_change(dev, p);
  691. return 0;
  692. }
  693. static struct rtnl_link_stats64 *virtnet_stats(struct net_device *dev,
  694. struct rtnl_link_stats64 *tot)
  695. {
  696. struct virtnet_info *vi = netdev_priv(dev);
  697. int cpu;
  698. unsigned int start;
  699. for_each_possible_cpu(cpu) {
  700. struct virtnet_stats *stats = per_cpu_ptr(vi->stats, cpu);
  701. u64 tpackets, tbytes, rpackets, rbytes;
  702. do {
  703. start = u64_stats_fetch_begin_bh(&stats->tx_syncp);
  704. tpackets = stats->tx_packets;
  705. tbytes = stats->tx_bytes;
  706. } while (u64_stats_fetch_retry_bh(&stats->tx_syncp, start));
  707. do {
  708. start = u64_stats_fetch_begin_bh(&stats->rx_syncp);
  709. rpackets = stats->rx_packets;
  710. rbytes = stats->rx_bytes;
  711. } while (u64_stats_fetch_retry_bh(&stats->rx_syncp, start));
  712. tot->rx_packets += rpackets;
  713. tot->tx_packets += tpackets;
  714. tot->rx_bytes += rbytes;
  715. tot->tx_bytes += tbytes;
  716. }
  717. tot->tx_dropped = dev->stats.tx_dropped;
  718. tot->tx_fifo_errors = dev->stats.tx_fifo_errors;
  719. tot->rx_dropped = dev->stats.rx_dropped;
  720. tot->rx_length_errors = dev->stats.rx_length_errors;
  721. tot->rx_frame_errors = dev->stats.rx_frame_errors;
  722. return tot;
  723. }
  724. #ifdef CONFIG_NET_POLL_CONTROLLER
  725. static void virtnet_netpoll(struct net_device *dev)
  726. {
  727. struct virtnet_info *vi = netdev_priv(dev);
  728. int i;
  729. for (i = 0; i < vi->curr_queue_pairs; i++)
  730. napi_schedule(&vi->rq[i].napi);
  731. }
  732. #endif
  733. static void virtnet_ack_link_announce(struct virtnet_info *vi)
  734. {
  735. rtnl_lock();
  736. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE,
  737. VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL, NULL))
  738. dev_warn(&vi->dev->dev, "Failed to ack link announce.\n");
  739. rtnl_unlock();
  740. }
  741. static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
  742. {
  743. struct scatterlist sg;
  744. struct virtio_net_ctrl_mq s;
  745. struct net_device *dev = vi->dev;
  746. int i;
  747. if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ))
  748. return 0;
  749. s.virtqueue_pairs = queue_pairs;
  750. sg_init_one(&sg, &s, sizeof(s));
  751. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,
  752. VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &sg, NULL)) {
  753. dev_warn(&dev->dev, "Fail to set num of queue pairs to %d\n",
  754. queue_pairs);
  755. return -EINVAL;
  756. } else {
  757. for (i = vi->curr_queue_pairs; i < queue_pairs; i++)
  758. if (!try_fill_recv(&vi->rq[i], GFP_KERNEL))
  759. schedule_delayed_work(&vi->refill, 0);
  760. vi->curr_queue_pairs = queue_pairs;
  761. }
  762. return 0;
  763. }
  764. static int virtnet_close(struct net_device *dev)
  765. {
  766. struct virtnet_info *vi = netdev_priv(dev);
  767. int i;
  768. /* Make sure refill_work doesn't re-enable napi! */
  769. cancel_delayed_work_sync(&vi->refill);
  770. for (i = 0; i < vi->max_queue_pairs; i++)
  771. napi_disable(&vi->rq[i].napi);
  772. return 0;
  773. }
  774. static void virtnet_set_rx_mode(struct net_device *dev)
  775. {
  776. struct virtnet_info *vi = netdev_priv(dev);
  777. struct scatterlist sg[2];
  778. u8 promisc, allmulti;
  779. struct virtio_net_ctrl_mac *mac_data;
  780. struct netdev_hw_addr *ha;
  781. int uc_count;
  782. int mc_count;
  783. void *buf;
  784. int i;
  785. /* We can't dynamicaly set ndo_set_rx_mode, so return gracefully */
  786. if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
  787. return;
  788. promisc = ((dev->flags & IFF_PROMISC) != 0);
  789. allmulti = ((dev->flags & IFF_ALLMULTI) != 0);
  790. sg_init_one(sg, &promisc, sizeof(promisc));
  791. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
  792. VIRTIO_NET_CTRL_RX_PROMISC,
  793. sg, NULL))
  794. dev_warn(&dev->dev, "Failed to %sable promisc mode.\n",
  795. promisc ? "en" : "dis");
  796. sg_init_one(sg, &allmulti, sizeof(allmulti));
  797. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
  798. VIRTIO_NET_CTRL_RX_ALLMULTI,
  799. sg, NULL))
  800. dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
  801. allmulti ? "en" : "dis");
  802. uc_count = netdev_uc_count(dev);
  803. mc_count = netdev_mc_count(dev);
  804. /* MAC filter - use one buffer for both lists */
  805. buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
  806. (2 * sizeof(mac_data->entries)), GFP_ATOMIC);
  807. mac_data = buf;
  808. if (!buf)
  809. return;
  810. sg_init_table(sg, 2);
  811. /* Store the unicast list and count in the front of the buffer */
  812. mac_data->entries = uc_count;
  813. i = 0;
  814. netdev_for_each_uc_addr(ha, dev)
  815. memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
  816. sg_set_buf(&sg[0], mac_data,
  817. sizeof(mac_data->entries) + (uc_count * ETH_ALEN));
  818. /* multicast list and count fill the end */
  819. mac_data = (void *)&mac_data->macs[uc_count][0];
  820. mac_data->entries = mc_count;
  821. i = 0;
  822. netdev_for_each_mc_addr(ha, dev)
  823. memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
  824. sg_set_buf(&sg[1], mac_data,
  825. sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
  826. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
  827. VIRTIO_NET_CTRL_MAC_TABLE_SET,
  828. sg, NULL))
  829. dev_warn(&dev->dev, "Failed to set MAC fitler table.\n");
  830. kfree(buf);
  831. }
  832. static int virtnet_vlan_rx_add_vid(struct net_device *dev,
  833. __be16 proto, u16 vid)
  834. {
  835. struct virtnet_info *vi = netdev_priv(dev);
  836. struct scatterlist sg;
  837. sg_init_one(&sg, &vid, sizeof(vid));
  838. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
  839. VIRTIO_NET_CTRL_VLAN_ADD, &sg, NULL))
  840. dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid);
  841. return 0;
  842. }
  843. static int virtnet_vlan_rx_kill_vid(struct net_device *dev,
  844. __be16 proto, u16 vid)
  845. {
  846. struct virtnet_info *vi = netdev_priv(dev);
  847. struct scatterlist sg;
  848. sg_init_one(&sg, &vid, sizeof(vid));
  849. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
  850. VIRTIO_NET_CTRL_VLAN_DEL, &sg, NULL))
  851. dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid);
  852. return 0;
  853. }
  854. static void virtnet_clean_affinity(struct virtnet_info *vi, long hcpu)
  855. {
  856. int i;
  857. int cpu;
  858. if (vi->affinity_hint_set) {
  859. for (i = 0; i < vi->max_queue_pairs; i++) {
  860. virtqueue_set_affinity(vi->rq[i].vq, -1);
  861. virtqueue_set_affinity(vi->sq[i].vq, -1);
  862. }
  863. vi->affinity_hint_set = false;
  864. }
  865. i = 0;
  866. for_each_online_cpu(cpu) {
  867. if (cpu == hcpu) {
  868. *per_cpu_ptr(vi->vq_index, cpu) = -1;
  869. } else {
  870. *per_cpu_ptr(vi->vq_index, cpu) =
  871. ++i % vi->curr_queue_pairs;
  872. }
  873. }
  874. }
  875. static void virtnet_set_affinity(struct virtnet_info *vi)
  876. {
  877. int i;
  878. int cpu;
  879. /* In multiqueue mode, when the number of cpu is equal to the number of
  880. * queue pairs, we let the queue pairs to be private to one cpu by
  881. * setting the affinity hint to eliminate the contention.
  882. */
  883. if (vi->curr_queue_pairs == 1 ||
  884. vi->max_queue_pairs != num_online_cpus()) {
  885. virtnet_clean_affinity(vi, -1);
  886. return;
  887. }
  888. i = 0;
  889. for_each_online_cpu(cpu) {
  890. virtqueue_set_affinity(vi->rq[i].vq, cpu);
  891. virtqueue_set_affinity(vi->sq[i].vq, cpu);
  892. *per_cpu_ptr(vi->vq_index, cpu) = i;
  893. i++;
  894. }
  895. vi->affinity_hint_set = true;
  896. }
  897. static int virtnet_cpu_callback(struct notifier_block *nfb,
  898. unsigned long action, void *hcpu)
  899. {
  900. struct virtnet_info *vi = container_of(nfb, struct virtnet_info, nb);
  901. switch(action & ~CPU_TASKS_FROZEN) {
  902. case CPU_ONLINE:
  903. case CPU_DOWN_FAILED:
  904. case CPU_DEAD:
  905. virtnet_set_affinity(vi);
  906. break;
  907. case CPU_DOWN_PREPARE:
  908. virtnet_clean_affinity(vi, (long)hcpu);
  909. break;
  910. default:
  911. break;
  912. }
  913. return NOTIFY_OK;
  914. }
  915. static void virtnet_get_ringparam(struct net_device *dev,
  916. struct ethtool_ringparam *ring)
  917. {
  918. struct virtnet_info *vi = netdev_priv(dev);
  919. ring->rx_max_pending = virtqueue_get_vring_size(vi->rq[0].vq);
  920. ring->tx_max_pending = virtqueue_get_vring_size(vi->sq[0].vq);
  921. ring->rx_pending = ring->rx_max_pending;
  922. ring->tx_pending = ring->tx_max_pending;
  923. }
  924. static void virtnet_get_drvinfo(struct net_device *dev,
  925. struct ethtool_drvinfo *info)
  926. {
  927. struct virtnet_info *vi = netdev_priv(dev);
  928. struct virtio_device *vdev = vi->vdev;
  929. strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
  930. strlcpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version));
  931. strlcpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info));
  932. }
  933. /* TODO: Eliminate OOO packets during switching */
  934. static int virtnet_set_channels(struct net_device *dev,
  935. struct ethtool_channels *channels)
  936. {
  937. struct virtnet_info *vi = netdev_priv(dev);
  938. u16 queue_pairs = channels->combined_count;
  939. int err;
  940. /* We don't support separate rx/tx channels.
  941. * We don't allow setting 'other' channels.
  942. */
  943. if (channels->rx_count || channels->tx_count || channels->other_count)
  944. return -EINVAL;
  945. if (queue_pairs > vi->max_queue_pairs)
  946. return -EINVAL;
  947. get_online_cpus();
  948. err = virtnet_set_queues(vi, queue_pairs);
  949. if (!err) {
  950. netif_set_real_num_tx_queues(dev, queue_pairs);
  951. netif_set_real_num_rx_queues(dev, queue_pairs);
  952. virtnet_set_affinity(vi);
  953. }
  954. put_online_cpus();
  955. return err;
  956. }
  957. static void virtnet_get_channels(struct net_device *dev,
  958. struct ethtool_channels *channels)
  959. {
  960. struct virtnet_info *vi = netdev_priv(dev);
  961. channels->combined_count = vi->curr_queue_pairs;
  962. channels->max_combined = vi->max_queue_pairs;
  963. channels->max_other = 0;
  964. channels->rx_count = 0;
  965. channels->tx_count = 0;
  966. channels->other_count = 0;
  967. }
  968. static const struct ethtool_ops virtnet_ethtool_ops = {
  969. .get_drvinfo = virtnet_get_drvinfo,
  970. .get_link = ethtool_op_get_link,
  971. .get_ringparam = virtnet_get_ringparam,
  972. .set_channels = virtnet_set_channels,
  973. .get_channels = virtnet_get_channels,
  974. };
  975. #define MIN_MTU 68
  976. #define MAX_MTU 65535
  977. static int virtnet_change_mtu(struct net_device *dev, int new_mtu)
  978. {
  979. if (new_mtu < MIN_MTU || new_mtu > MAX_MTU)
  980. return -EINVAL;
  981. dev->mtu = new_mtu;
  982. return 0;
  983. }
  984. /* To avoid contending a lock hold by a vcpu who would exit to host, select the
  985. * txq based on the processor id.
  986. */
  987. static u16 virtnet_select_queue(struct net_device *dev, struct sk_buff *skb)
  988. {
  989. int txq;
  990. struct virtnet_info *vi = netdev_priv(dev);
  991. if (skb_rx_queue_recorded(skb)) {
  992. txq = skb_get_rx_queue(skb);
  993. } else {
  994. txq = *__this_cpu_ptr(vi->vq_index);
  995. if (txq == -1)
  996. txq = 0;
  997. }
  998. while (unlikely(txq >= dev->real_num_tx_queues))
  999. txq -= dev->real_num_tx_queues;
  1000. return txq;
  1001. }
  1002. static const struct net_device_ops virtnet_netdev = {
  1003. .ndo_open = virtnet_open,
  1004. .ndo_stop = virtnet_close,
  1005. .ndo_start_xmit = start_xmit,
  1006. .ndo_validate_addr = eth_validate_addr,
  1007. .ndo_set_mac_address = virtnet_set_mac_address,
  1008. .ndo_set_rx_mode = virtnet_set_rx_mode,
  1009. .ndo_change_mtu = virtnet_change_mtu,
  1010. .ndo_get_stats64 = virtnet_stats,
  1011. .ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid,
  1012. .ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
  1013. .ndo_select_queue = virtnet_select_queue,
  1014. #ifdef CONFIG_NET_POLL_CONTROLLER
  1015. .ndo_poll_controller = virtnet_netpoll,
  1016. #endif
  1017. };
  1018. static void virtnet_config_changed_work(struct work_struct *work)
  1019. {
  1020. struct virtnet_info *vi =
  1021. container_of(work, struct virtnet_info, config_work);
  1022. u16 v;
  1023. mutex_lock(&vi->config_lock);
  1024. if (!vi->config_enable)
  1025. goto done;
  1026. if (virtio_config_val(vi->vdev, VIRTIO_NET_F_STATUS,
  1027. offsetof(struct virtio_net_config, status),
  1028. &v) < 0)
  1029. goto done;
  1030. if (v & VIRTIO_NET_S_ANNOUNCE) {
  1031. netdev_notify_peers(vi->dev);
  1032. virtnet_ack_link_announce(vi);
  1033. }
  1034. /* Ignore unknown (future) status bits */
  1035. v &= VIRTIO_NET_S_LINK_UP;
  1036. if (vi->status == v)
  1037. goto done;
  1038. vi->status = v;
  1039. if (vi->status & VIRTIO_NET_S_LINK_UP) {
  1040. netif_carrier_on(vi->dev);
  1041. netif_tx_wake_all_queues(vi->dev);
  1042. } else {
  1043. netif_carrier_off(vi->dev);
  1044. netif_tx_stop_all_queues(vi->dev);
  1045. }
  1046. done:
  1047. mutex_unlock(&vi->config_lock);
  1048. }
  1049. static void virtnet_config_changed(struct virtio_device *vdev)
  1050. {
  1051. struct virtnet_info *vi = vdev->priv;
  1052. schedule_work(&vi->config_work);
  1053. }
  1054. static void virtnet_free_queues(struct virtnet_info *vi)
  1055. {
  1056. kfree(vi->rq);
  1057. kfree(vi->sq);
  1058. }
  1059. static void free_receive_bufs(struct virtnet_info *vi)
  1060. {
  1061. int i;
  1062. for (i = 0; i < vi->max_queue_pairs; i++) {
  1063. while (vi->rq[i].pages)
  1064. __free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0);
  1065. }
  1066. }
  1067. static void free_unused_bufs(struct virtnet_info *vi)
  1068. {
  1069. void *buf;
  1070. int i;
  1071. for (i = 0; i < vi->max_queue_pairs; i++) {
  1072. struct virtqueue *vq = vi->sq[i].vq;
  1073. while ((buf = virtqueue_detach_unused_buf(vq)) != NULL)
  1074. dev_kfree_skb(buf);
  1075. }
  1076. for (i = 0; i < vi->max_queue_pairs; i++) {
  1077. struct virtqueue *vq = vi->rq[i].vq;
  1078. while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
  1079. if (vi->mergeable_rx_bufs || vi->big_packets)
  1080. give_pages(&vi->rq[i], buf);
  1081. else
  1082. dev_kfree_skb(buf);
  1083. --vi->rq[i].num;
  1084. }
  1085. BUG_ON(vi->rq[i].num != 0);
  1086. }
  1087. }
  1088. static void virtnet_del_vqs(struct virtnet_info *vi)
  1089. {
  1090. struct virtio_device *vdev = vi->vdev;
  1091. virtnet_clean_affinity(vi, -1);
  1092. vdev->config->del_vqs(vdev);
  1093. virtnet_free_queues(vi);
  1094. }
  1095. static int virtnet_find_vqs(struct virtnet_info *vi)
  1096. {
  1097. vq_callback_t **callbacks;
  1098. struct virtqueue **vqs;
  1099. int ret = -ENOMEM;
  1100. int i, total_vqs;
  1101. const char **names;
  1102. /* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by
  1103. * possible N-1 RX/TX queue pairs used in multiqueue mode, followed by
  1104. * possible control vq.
  1105. */
  1106. total_vqs = vi->max_queue_pairs * 2 +
  1107. virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ);
  1108. /* Allocate space for find_vqs parameters */
  1109. vqs = kzalloc(total_vqs * sizeof(*vqs), GFP_KERNEL);
  1110. if (!vqs)
  1111. goto err_vq;
  1112. callbacks = kmalloc(total_vqs * sizeof(*callbacks), GFP_KERNEL);
  1113. if (!callbacks)
  1114. goto err_callback;
  1115. names = kmalloc(total_vqs * sizeof(*names), GFP_KERNEL);
  1116. if (!names)
  1117. goto err_names;
  1118. /* Parameters for control virtqueue, if any */
  1119. if (vi->has_cvq) {
  1120. callbacks[total_vqs - 1] = NULL;
  1121. names[total_vqs - 1] = "control";
  1122. }
  1123. /* Allocate/initialize parameters for send/receive virtqueues */
  1124. for (i = 0; i < vi->max_queue_pairs; i++) {
  1125. callbacks[rxq2vq(i)] = skb_recv_done;
  1126. callbacks[txq2vq(i)] = skb_xmit_done;
  1127. sprintf(vi->rq[i].name, "input.%d", i);
  1128. sprintf(vi->sq[i].name, "output.%d", i);
  1129. names[rxq2vq(i)] = vi->rq[i].name;
  1130. names[txq2vq(i)] = vi->sq[i].name;
  1131. }
  1132. ret = vi->vdev->config->find_vqs(vi->vdev, total_vqs, vqs, callbacks,
  1133. names);
  1134. if (ret)
  1135. goto err_find;
  1136. if (vi->has_cvq) {
  1137. vi->cvq = vqs[total_vqs - 1];
  1138. if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
  1139. vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
  1140. }
  1141. for (i = 0; i < vi->max_queue_pairs; i++) {
  1142. vi->rq[i].vq = vqs[rxq2vq(i)];
  1143. vi->sq[i].vq = vqs[txq2vq(i)];
  1144. }
  1145. kfree(names);
  1146. kfree(callbacks);
  1147. kfree(vqs);
  1148. return 0;
  1149. err_find:
  1150. kfree(names);
  1151. err_names:
  1152. kfree(callbacks);
  1153. err_callback:
  1154. kfree(vqs);
  1155. err_vq:
  1156. return ret;
  1157. }
  1158. static int virtnet_alloc_queues(struct virtnet_info *vi)
  1159. {
  1160. int i;
  1161. vi->sq = kzalloc(sizeof(*vi->sq) * vi->max_queue_pairs, GFP_KERNEL);
  1162. if (!vi->sq)
  1163. goto err_sq;
  1164. vi->rq = kzalloc(sizeof(*vi->rq) * vi->max_queue_pairs, GFP_KERNEL);
  1165. if (!vi->rq)
  1166. goto err_rq;
  1167. INIT_DELAYED_WORK(&vi->refill, refill_work);
  1168. for (i = 0; i < vi->max_queue_pairs; i++) {
  1169. vi->rq[i].pages = NULL;
  1170. netif_napi_add(vi->dev, &vi->rq[i].napi, virtnet_poll,
  1171. napi_weight);
  1172. sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
  1173. sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));
  1174. }
  1175. return 0;
  1176. err_rq:
  1177. kfree(vi->sq);
  1178. err_sq:
  1179. return -ENOMEM;
  1180. }
  1181. static int init_vqs(struct virtnet_info *vi)
  1182. {
  1183. int ret;
  1184. /* Allocate send & receive queues */
  1185. ret = virtnet_alloc_queues(vi);
  1186. if (ret)
  1187. goto err;
  1188. ret = virtnet_find_vqs(vi);
  1189. if (ret)
  1190. goto err_free;
  1191. get_online_cpus();
  1192. virtnet_set_affinity(vi);
  1193. put_online_cpus();
  1194. return 0;
  1195. err_free:
  1196. virtnet_free_queues(vi);
  1197. err:
  1198. return ret;
  1199. }
  1200. static int virtnet_probe(struct virtio_device *vdev)
  1201. {
  1202. int i, err;
  1203. struct net_device *dev;
  1204. struct virtnet_info *vi;
  1205. u16 max_queue_pairs;
  1206. /* Find if host supports multiqueue virtio_net device */
  1207. err = virtio_config_val(vdev, VIRTIO_NET_F_MQ,
  1208. offsetof(struct virtio_net_config,
  1209. max_virtqueue_pairs), &max_queue_pairs);
  1210. /* We need at least 2 queue's */
  1211. if (err || max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
  1212. max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX ||
  1213. !virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
  1214. max_queue_pairs = 1;
  1215. /* Allocate ourselves a network device with room for our info */
  1216. dev = alloc_etherdev_mq(sizeof(struct virtnet_info), max_queue_pairs);
  1217. if (!dev)
  1218. return -ENOMEM;
  1219. /* Set up network device as normal. */
  1220. dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE;
  1221. dev->netdev_ops = &virtnet_netdev;
  1222. dev->features = NETIF_F_HIGHDMA;
  1223. SET_ETHTOOL_OPS(dev, &virtnet_ethtool_ops);
  1224. SET_NETDEV_DEV(dev, &vdev->dev);
  1225. /* Do we support "hardware" checksums? */
  1226. if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) {
  1227. /* This opens up the world of extra features. */
  1228. dev->hw_features |= NETIF_F_HW_CSUM|NETIF_F_SG|NETIF_F_FRAGLIST;
  1229. if (csum)
  1230. dev->features |= NETIF_F_HW_CSUM|NETIF_F_SG|NETIF_F_FRAGLIST;
  1231. if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
  1232. dev->hw_features |= NETIF_F_TSO | NETIF_F_UFO
  1233. | NETIF_F_TSO_ECN | NETIF_F_TSO6;
  1234. }
  1235. /* Individual feature bits: what can host handle? */
  1236. if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4))
  1237. dev->hw_features |= NETIF_F_TSO;
  1238. if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6))
  1239. dev->hw_features |= NETIF_F_TSO6;
  1240. if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
  1241. dev->hw_features |= NETIF_F_TSO_ECN;
  1242. if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_UFO))
  1243. dev->hw_features |= NETIF_F_UFO;
  1244. if (gso)
  1245. dev->features |= dev->hw_features & (NETIF_F_ALL_TSO|NETIF_F_UFO);
  1246. /* (!csum && gso) case will be fixed by register_netdev() */
  1247. }
  1248. dev->vlan_features = dev->features;
  1249. /* Configuration may specify what MAC to use. Otherwise random. */
  1250. if (virtio_config_val_len(vdev, VIRTIO_NET_F_MAC,
  1251. offsetof(struct virtio_net_config, mac),
  1252. dev->dev_addr, dev->addr_len) < 0)
  1253. eth_hw_addr_random(dev);
  1254. /* Set up our device-specific information */
  1255. vi = netdev_priv(dev);
  1256. vi->dev = dev;
  1257. vi->vdev = vdev;
  1258. vdev->priv = vi;
  1259. vi->stats = alloc_percpu(struct virtnet_stats);
  1260. err = -ENOMEM;
  1261. if (vi->stats == NULL)
  1262. goto free;
  1263. vi->vq_index = alloc_percpu(int);
  1264. if (vi->vq_index == NULL)
  1265. goto free_stats;
  1266. mutex_init(&vi->config_lock);
  1267. vi->config_enable = true;
  1268. INIT_WORK(&vi->config_work, virtnet_config_changed_work);
  1269. /* If we can receive ANY GSO packets, we must allocate large ones. */
  1270. if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
  1271. virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) ||
  1272. virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN))
  1273. vi->big_packets = true;
  1274. if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
  1275. vi->mergeable_rx_bufs = true;
  1276. if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
  1277. vi->has_cvq = true;
  1278. /* Use single tx/rx queue pair as default */
  1279. vi->curr_queue_pairs = 1;
  1280. vi->max_queue_pairs = max_queue_pairs;
  1281. /* Allocate/initialize the rx/tx queues, and invoke find_vqs */
  1282. err = init_vqs(vi);
  1283. if (err)
  1284. goto free_index;
  1285. netif_set_real_num_tx_queues(dev, 1);
  1286. netif_set_real_num_rx_queues(dev, 1);
  1287. err = register_netdev(dev);
  1288. if (err) {
  1289. pr_debug("virtio_net: registering device failed\n");
  1290. goto free_vqs;
  1291. }
  1292. /* Last of all, set up some receive buffers. */
  1293. for (i = 0; i < vi->curr_queue_pairs; i++) {
  1294. try_fill_recv(&vi->rq[i], GFP_KERNEL);
  1295. /* If we didn't even get one input buffer, we're useless. */
  1296. if (vi->rq[i].num == 0) {
  1297. free_unused_bufs(vi);
  1298. err = -ENOMEM;
  1299. goto free_recv_bufs;
  1300. }
  1301. }
  1302. vi->nb.notifier_call = &virtnet_cpu_callback;
  1303. err = register_hotcpu_notifier(&vi->nb);
  1304. if (err) {
  1305. pr_debug("virtio_net: registering cpu notifier failed\n");
  1306. goto free_recv_bufs;
  1307. }
  1308. /* Assume link up if device can't report link status,
  1309. otherwise get link status from config. */
  1310. if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {
  1311. netif_carrier_off(dev);
  1312. schedule_work(&vi->config_work);
  1313. } else {
  1314. vi->status = VIRTIO_NET_S_LINK_UP;
  1315. netif_carrier_on(dev);
  1316. }
  1317. pr_debug("virtnet: registered device %s with %d RX and TX vq's\n",
  1318. dev->name, max_queue_pairs);
  1319. return 0;
  1320. free_recv_bufs:
  1321. free_receive_bufs(vi);
  1322. unregister_netdev(dev);
  1323. free_vqs:
  1324. cancel_delayed_work_sync(&vi->refill);
  1325. virtnet_del_vqs(vi);
  1326. free_index:
  1327. free_percpu(vi->vq_index);
  1328. free_stats:
  1329. free_percpu(vi->stats);
  1330. free:
  1331. free_netdev(dev);
  1332. return err;
  1333. }
  1334. static void remove_vq_common(struct virtnet_info *vi)
  1335. {
  1336. vi->vdev->config->reset(vi->vdev);
  1337. /* Free unused buffers in both send and recv, if any. */
  1338. free_unused_bufs(vi);
  1339. free_receive_bufs(vi);
  1340. virtnet_del_vqs(vi);
  1341. }
  1342. static void virtnet_remove(struct virtio_device *vdev)
  1343. {
  1344. struct virtnet_info *vi = vdev->priv;
  1345. unregister_hotcpu_notifier(&vi->nb);
  1346. /* Prevent config work handler from accessing the device. */
  1347. mutex_lock(&vi->config_lock);
  1348. vi->config_enable = false;
  1349. mutex_unlock(&vi->config_lock);
  1350. unregister_netdev(vi->dev);
  1351. remove_vq_common(vi);
  1352. flush_work(&vi->config_work);
  1353. free_percpu(vi->vq_index);
  1354. free_percpu(vi->stats);
  1355. free_netdev(vi->dev);
  1356. }
  1357. #ifdef CONFIG_PM
  1358. static int virtnet_freeze(struct virtio_device *vdev)
  1359. {
  1360. struct virtnet_info *vi = vdev->priv;
  1361. int i;
  1362. /* Prevent config work handler from accessing the device */
  1363. mutex_lock(&vi->config_lock);
  1364. vi->config_enable = false;
  1365. mutex_unlock(&vi->config_lock);
  1366. netif_device_detach(vi->dev);
  1367. cancel_delayed_work_sync(&vi->refill);
  1368. if (netif_running(vi->dev))
  1369. for (i = 0; i < vi->max_queue_pairs; i++) {
  1370. napi_disable(&vi->rq[i].napi);
  1371. netif_napi_del(&vi->rq[i].napi);
  1372. }
  1373. remove_vq_common(vi);
  1374. flush_work(&vi->config_work);
  1375. return 0;
  1376. }
  1377. static int virtnet_restore(struct virtio_device *vdev)
  1378. {
  1379. struct virtnet_info *vi = vdev->priv;
  1380. int err, i;
  1381. err = init_vqs(vi);
  1382. if (err)
  1383. return err;
  1384. if (netif_running(vi->dev))
  1385. for (i = 0; i < vi->max_queue_pairs; i++)
  1386. virtnet_napi_enable(&vi->rq[i]);
  1387. netif_device_attach(vi->dev);
  1388. for (i = 0; i < vi->curr_queue_pairs; i++)
  1389. if (!try_fill_recv(&vi->rq[i], GFP_KERNEL))
  1390. schedule_delayed_work(&vi->refill, 0);
  1391. mutex_lock(&vi->config_lock);
  1392. vi->config_enable = true;
  1393. mutex_unlock(&vi->config_lock);
  1394. virtnet_set_queues(vi, vi->curr_queue_pairs);
  1395. return 0;
  1396. }
  1397. #endif
  1398. static struct virtio_device_id id_table[] = {
  1399. { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
  1400. { 0 },
  1401. };
  1402. static unsigned int features[] = {
  1403. VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM,
  1404. VIRTIO_NET_F_GSO, VIRTIO_NET_F_MAC,
  1405. VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6,
  1406. VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6,
  1407. VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO,
  1408. VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ,
  1409. VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN,
  1410. VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ,
  1411. VIRTIO_NET_F_CTRL_MAC_ADDR,
  1412. };
  1413. static struct virtio_driver virtio_net_driver = {
  1414. .feature_table = features,
  1415. .feature_table_size = ARRAY_SIZE(features),
  1416. .driver.name = KBUILD_MODNAME,
  1417. .driver.owner = THIS_MODULE,
  1418. .id_table = id_table,
  1419. .probe = virtnet_probe,
  1420. .remove = virtnet_remove,
  1421. .config_changed = virtnet_config_changed,
  1422. #ifdef CONFIG_PM
  1423. .freeze = virtnet_freeze,
  1424. .restore = virtnet_restore,
  1425. #endif
  1426. };
  1427. module_virtio_driver(virtio_net_driver);
  1428. MODULE_DEVICE_TABLE(virtio, id_table);
  1429. MODULE_DESCRIPTION("Virtio network driver");
  1430. MODULE_LICENSE("GPL");