virtio_net.c 44 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787
  1. /* A network driver using virtio.
  2. *
  3. * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
  4. *
  5. * This program is free software; you can redistribute it and/or modify
  6. * it under the terms of the GNU General Public License as published by
  7. * the Free Software Foundation; either version 2 of the License, or
  8. * (at your option) any later version.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU General Public License
  16. * along with this program; if not, write to the Free Software
  17. * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  18. */
  19. //#define DEBUG
  20. #include <linux/netdevice.h>
  21. #include <linux/etherdevice.h>
  22. #include <linux/ethtool.h>
  23. #include <linux/module.h>
  24. #include <linux/virtio.h>
  25. #include <linux/virtio_net.h>
  26. #include <linux/scatterlist.h>
  27. #include <linux/if_vlan.h>
  28. #include <linux/slab.h>
  29. #include <linux/cpu.h>
  30. static int napi_weight = NAPI_POLL_WEIGHT;
  31. module_param(napi_weight, int, 0444);
  32. static bool csum = true, gso = true;
  33. module_param(csum, bool, 0444);
  34. module_param(gso, bool, 0444);
  35. /* FIXME: MTU in config. */
  36. #define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
  37. #define GOOD_COPY_LEN 128
  38. #define VIRTNET_DRIVER_VERSION "1.0.0"
  39. struct virtnet_stats {
  40. struct u64_stats_sync tx_syncp;
  41. struct u64_stats_sync rx_syncp;
  42. u64 tx_bytes;
  43. u64 tx_packets;
  44. u64 rx_bytes;
  45. u64 rx_packets;
  46. };
  47. /* Internal representation of a send virtqueue */
  48. struct send_queue {
  49. /* Virtqueue associated with this send _queue */
  50. struct virtqueue *vq;
  51. /* TX: fragments + linear part + virtio header */
  52. struct scatterlist sg[MAX_SKB_FRAGS + 2];
  53. /* Name of the send queue: output.$index */
  54. char name[40];
  55. };
  56. /* Internal representation of a receive virtqueue */
  57. struct receive_queue {
  58. /* Virtqueue associated with this receive_queue */
  59. struct virtqueue *vq;
  60. struct napi_struct napi;
  61. /* Number of input buffers, and max we've ever had. */
  62. unsigned int num, max;
  63. /* Chain pages by the private ptr. */
  64. struct page *pages;
  65. /* RX: fragments + linear part + virtio header */
  66. struct scatterlist sg[MAX_SKB_FRAGS + 2];
  67. /* Name of this receive queue: input.$index */
  68. char name[40];
  69. };
  70. struct virtnet_info {
  71. struct virtio_device *vdev;
  72. struct virtqueue *cvq;
  73. struct net_device *dev;
  74. struct send_queue *sq;
  75. struct receive_queue *rq;
  76. unsigned int status;
  77. /* Max # of queue pairs supported by the device */
  78. u16 max_queue_pairs;
  79. /* # of queue pairs currently used by the driver */
  80. u16 curr_queue_pairs;
  81. /* I like... big packets and I cannot lie! */
  82. bool big_packets;
  83. /* Host will merge rx buffers for big packets (shake it! shake it!) */
  84. bool mergeable_rx_bufs;
  85. /* Has control virtqueue */
  86. bool has_cvq;
  87. /* Host can handle any s/g split between our header and packet data */
  88. bool any_header_sg;
  89. /* enable config space updates */
  90. bool config_enable;
  91. /* Active statistics */
  92. struct virtnet_stats __percpu *stats;
  93. /* Work struct for refilling if we run low on memory. */
  94. struct delayed_work refill;
  95. /* Work struct for config space updates */
  96. struct work_struct config_work;
  97. /* Lock for config space updates */
  98. struct mutex config_lock;
  99. /* Does the affinity hint is set for virtqueues? */
  100. bool affinity_hint_set;
  101. /* Per-cpu variable to show the mapping from CPU to virtqueue */
  102. int __percpu *vq_index;
  103. /* CPU hot plug notifier */
  104. struct notifier_block nb;
  105. };
  106. struct skb_vnet_hdr {
  107. union {
  108. struct virtio_net_hdr hdr;
  109. struct virtio_net_hdr_mrg_rxbuf mhdr;
  110. };
  111. };
  112. struct padded_vnet_hdr {
  113. struct virtio_net_hdr hdr;
  114. /*
  115. * virtio_net_hdr should be in a separated sg buffer because of a
  116. * QEMU bug, and data sg buffer shares same page with this header sg.
  117. * This padding makes next sg 16 byte aligned after virtio_net_hdr.
  118. */
  119. char padding[6];
  120. };
  121. /* Converting between virtqueue no. and kernel tx/rx queue no.
  122. * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq
  123. */
  124. static int vq2txq(struct virtqueue *vq)
  125. {
  126. return (vq->index - 1) / 2;
  127. }
  128. static int txq2vq(int txq)
  129. {
  130. return txq * 2 + 1;
  131. }
  132. static int vq2rxq(struct virtqueue *vq)
  133. {
  134. return vq->index / 2;
  135. }
  136. static int rxq2vq(int rxq)
  137. {
  138. return rxq * 2;
  139. }
  140. static inline struct skb_vnet_hdr *skb_vnet_hdr(struct sk_buff *skb)
  141. {
  142. return (struct skb_vnet_hdr *)skb->cb;
  143. }
  144. /*
  145. * private is used to chain pages for big packets, put the whole
  146. * most recent used list in the beginning for reuse
  147. */
  148. static void give_pages(struct receive_queue *rq, struct page *page)
  149. {
  150. struct page *end;
  151. /* Find end of list, sew whole thing into vi->rq.pages. */
  152. for (end = page; end->private; end = (struct page *)end->private);
  153. end->private = (unsigned long)rq->pages;
  154. rq->pages = page;
  155. }
  156. static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask)
  157. {
  158. struct page *p = rq->pages;
  159. if (p) {
  160. rq->pages = (struct page *)p->private;
  161. /* clear private here, it is used to chain pages */
  162. p->private = 0;
  163. } else
  164. p = alloc_page(gfp_mask);
  165. return p;
  166. }
  167. static void skb_xmit_done(struct virtqueue *vq)
  168. {
  169. struct virtnet_info *vi = vq->vdev->priv;
  170. /* Suppress further interrupts. */
  171. virtqueue_disable_cb(vq);
  172. /* We were probably waiting for more output buffers. */
  173. netif_wake_subqueue(vi->dev, vq2txq(vq));
  174. }
  175. static void set_skb_frag(struct sk_buff *skb, struct page *page,
  176. unsigned int offset, unsigned int *len)
  177. {
  178. int size = min((unsigned)PAGE_SIZE - offset, *len);
  179. int i = skb_shinfo(skb)->nr_frags;
  180. __skb_fill_page_desc(skb, i, page, offset, size);
  181. skb->data_len += size;
  182. skb->len += size;
  183. skb->truesize += PAGE_SIZE;
  184. skb_shinfo(skb)->nr_frags++;
  185. skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG;
  186. *len -= size;
  187. }
  188. /* Called from bottom half context */
  189. static struct sk_buff *page_to_skb(struct receive_queue *rq,
  190. struct page *page, unsigned int len)
  191. {
  192. struct virtnet_info *vi = rq->vq->vdev->priv;
  193. struct sk_buff *skb;
  194. struct skb_vnet_hdr *hdr;
  195. unsigned int copy, hdr_len, offset;
  196. char *p;
  197. p = page_address(page);
  198. /* copy small packet so we can reuse these pages for small data */
  199. skb = netdev_alloc_skb_ip_align(vi->dev, GOOD_COPY_LEN);
  200. if (unlikely(!skb))
  201. return NULL;
  202. hdr = skb_vnet_hdr(skb);
  203. if (vi->mergeable_rx_bufs) {
  204. hdr_len = sizeof hdr->mhdr;
  205. offset = hdr_len;
  206. } else {
  207. hdr_len = sizeof hdr->hdr;
  208. offset = sizeof(struct padded_vnet_hdr);
  209. }
  210. memcpy(hdr, p, hdr_len);
  211. len -= hdr_len;
  212. p += offset;
  213. copy = len;
  214. if (copy > skb_tailroom(skb))
  215. copy = skb_tailroom(skb);
  216. memcpy(skb_put(skb, copy), p, copy);
  217. len -= copy;
  218. offset += copy;
  219. /*
  220. * Verify that we can indeed put this data into a skb.
  221. * This is here to handle cases when the device erroneously
  222. * tries to receive more than is possible. This is usually
  223. * the case of a broken device.
  224. */
  225. if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) {
  226. net_dbg_ratelimited("%s: too much data\n", skb->dev->name);
  227. dev_kfree_skb(skb);
  228. return NULL;
  229. }
  230. while (len) {
  231. set_skb_frag(skb, page, offset, &len);
  232. page = (struct page *)page->private;
  233. offset = 0;
  234. }
  235. if (page)
  236. give_pages(rq, page);
  237. return skb;
  238. }
  239. static int receive_mergeable(struct receive_queue *rq, struct sk_buff *skb)
  240. {
  241. struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb);
  242. struct page *page;
  243. int num_buf, i, len;
  244. num_buf = hdr->mhdr.num_buffers;
  245. while (--num_buf) {
  246. i = skb_shinfo(skb)->nr_frags;
  247. if (i >= MAX_SKB_FRAGS) {
  248. pr_debug("%s: packet too long\n", skb->dev->name);
  249. skb->dev->stats.rx_length_errors++;
  250. return -EINVAL;
  251. }
  252. page = virtqueue_get_buf(rq->vq, &len);
  253. if (!page) {
  254. pr_debug("%s: rx error: %d buffers missing\n",
  255. skb->dev->name, hdr->mhdr.num_buffers);
  256. skb->dev->stats.rx_length_errors++;
  257. return -EINVAL;
  258. }
  259. if (len > PAGE_SIZE)
  260. len = PAGE_SIZE;
  261. set_skb_frag(skb, page, 0, &len);
  262. --rq->num;
  263. }
  264. return 0;
  265. }
  266. static void receive_buf(struct receive_queue *rq, void *buf, unsigned int len)
  267. {
  268. struct virtnet_info *vi = rq->vq->vdev->priv;
  269. struct net_device *dev = vi->dev;
  270. struct virtnet_stats *stats = this_cpu_ptr(vi->stats);
  271. struct sk_buff *skb;
  272. struct page *page;
  273. struct skb_vnet_hdr *hdr;
  274. if (unlikely(len < sizeof(struct virtio_net_hdr) + ETH_HLEN)) {
  275. pr_debug("%s: short packet %i\n", dev->name, len);
  276. dev->stats.rx_length_errors++;
  277. if (vi->mergeable_rx_bufs || vi->big_packets)
  278. give_pages(rq, buf);
  279. else
  280. dev_kfree_skb(buf);
  281. return;
  282. }
  283. if (!vi->mergeable_rx_bufs && !vi->big_packets) {
  284. skb = buf;
  285. len -= sizeof(struct virtio_net_hdr);
  286. skb_trim(skb, len);
  287. } else {
  288. page = buf;
  289. skb = page_to_skb(rq, page, len);
  290. if (unlikely(!skb)) {
  291. dev->stats.rx_dropped++;
  292. give_pages(rq, page);
  293. return;
  294. }
  295. if (vi->mergeable_rx_bufs)
  296. if (receive_mergeable(rq, skb)) {
  297. dev_kfree_skb(skb);
  298. return;
  299. }
  300. }
  301. hdr = skb_vnet_hdr(skb);
  302. u64_stats_update_begin(&stats->rx_syncp);
  303. stats->rx_bytes += skb->len;
  304. stats->rx_packets++;
  305. u64_stats_update_end(&stats->rx_syncp);
  306. if (hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
  307. pr_debug("Needs csum!\n");
  308. if (!skb_partial_csum_set(skb,
  309. hdr->hdr.csum_start,
  310. hdr->hdr.csum_offset))
  311. goto frame_err;
  312. } else if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID) {
  313. skb->ip_summed = CHECKSUM_UNNECESSARY;
  314. }
  315. skb->protocol = eth_type_trans(skb, dev);
  316. pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
  317. ntohs(skb->protocol), skb->len, skb->pkt_type);
  318. if (hdr->hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
  319. pr_debug("GSO!\n");
  320. switch (hdr->hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
  321. case VIRTIO_NET_HDR_GSO_TCPV4:
  322. skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
  323. break;
  324. case VIRTIO_NET_HDR_GSO_UDP:
  325. skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
  326. break;
  327. case VIRTIO_NET_HDR_GSO_TCPV6:
  328. skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
  329. break;
  330. default:
  331. net_warn_ratelimited("%s: bad gso type %u.\n",
  332. dev->name, hdr->hdr.gso_type);
  333. goto frame_err;
  334. }
  335. if (hdr->hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
  336. skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
  337. skb_shinfo(skb)->gso_size = hdr->hdr.gso_size;
  338. if (skb_shinfo(skb)->gso_size == 0) {
  339. net_warn_ratelimited("%s: zero gso size.\n", dev->name);
  340. goto frame_err;
  341. }
  342. /* Header must be checked, and gso_segs computed. */
  343. skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
  344. skb_shinfo(skb)->gso_segs = 0;
  345. }
  346. netif_receive_skb(skb);
  347. return;
  348. frame_err:
  349. dev->stats.rx_frame_errors++;
  350. dev_kfree_skb(skb);
  351. }
  352. static int add_recvbuf_small(struct receive_queue *rq, gfp_t gfp)
  353. {
  354. struct virtnet_info *vi = rq->vq->vdev->priv;
  355. struct sk_buff *skb;
  356. struct skb_vnet_hdr *hdr;
  357. int err;
  358. skb = __netdev_alloc_skb_ip_align(vi->dev, MAX_PACKET_LEN, gfp);
  359. if (unlikely(!skb))
  360. return -ENOMEM;
  361. skb_put(skb, MAX_PACKET_LEN);
  362. hdr = skb_vnet_hdr(skb);
  363. sg_set_buf(rq->sg, &hdr->hdr, sizeof hdr->hdr);
  364. skb_to_sgvec(skb, rq->sg + 1, 0, skb->len);
  365. err = virtqueue_add_inbuf(rq->vq, rq->sg, 2, skb, gfp);
  366. if (err < 0)
  367. dev_kfree_skb(skb);
  368. return err;
  369. }
  370. static int add_recvbuf_big(struct receive_queue *rq, gfp_t gfp)
  371. {
  372. struct page *first, *list = NULL;
  373. char *p;
  374. int i, err, offset;
  375. /* page in rq->sg[MAX_SKB_FRAGS + 1] is list tail */
  376. for (i = MAX_SKB_FRAGS + 1; i > 1; --i) {
  377. first = get_a_page(rq, gfp);
  378. if (!first) {
  379. if (list)
  380. give_pages(rq, list);
  381. return -ENOMEM;
  382. }
  383. sg_set_buf(&rq->sg[i], page_address(first), PAGE_SIZE);
  384. /* chain new page in list head to match sg */
  385. first->private = (unsigned long)list;
  386. list = first;
  387. }
  388. first = get_a_page(rq, gfp);
  389. if (!first) {
  390. give_pages(rq, list);
  391. return -ENOMEM;
  392. }
  393. p = page_address(first);
  394. /* rq->sg[0], rq->sg[1] share the same page */
  395. /* a separated rq->sg[0] for virtio_net_hdr only due to QEMU bug */
  396. sg_set_buf(&rq->sg[0], p, sizeof(struct virtio_net_hdr));
  397. /* rq->sg[1] for data packet, from offset */
  398. offset = sizeof(struct padded_vnet_hdr);
  399. sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset);
  400. /* chain first in list head */
  401. first->private = (unsigned long)list;
  402. err = virtqueue_add_inbuf(rq->vq, rq->sg, MAX_SKB_FRAGS + 2,
  403. first, gfp);
  404. if (err < 0)
  405. give_pages(rq, first);
  406. return err;
  407. }
  408. static int add_recvbuf_mergeable(struct receive_queue *rq, gfp_t gfp)
  409. {
  410. struct page *page;
  411. int err;
  412. page = get_a_page(rq, gfp);
  413. if (!page)
  414. return -ENOMEM;
  415. sg_init_one(rq->sg, page_address(page), PAGE_SIZE);
  416. err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, page, gfp);
  417. if (err < 0)
  418. give_pages(rq, page);
  419. return err;
  420. }
  421. /*
  422. * Returns false if we couldn't fill entirely (OOM).
  423. *
  424. * Normally run in the receive path, but can also be run from ndo_open
  425. * before we're receiving packets, or from refill_work which is
  426. * careful to disable receiving (using napi_disable).
  427. */
  428. static bool try_fill_recv(struct receive_queue *rq, gfp_t gfp)
  429. {
  430. struct virtnet_info *vi = rq->vq->vdev->priv;
  431. int err;
  432. bool oom;
  433. do {
  434. if (vi->mergeable_rx_bufs)
  435. err = add_recvbuf_mergeable(rq, gfp);
  436. else if (vi->big_packets)
  437. err = add_recvbuf_big(rq, gfp);
  438. else
  439. err = add_recvbuf_small(rq, gfp);
  440. oom = err == -ENOMEM;
  441. if (err)
  442. break;
  443. ++rq->num;
  444. } while (rq->vq->num_free);
  445. if (unlikely(rq->num > rq->max))
  446. rq->max = rq->num;
  447. if (unlikely(!virtqueue_kick(rq->vq)))
  448. return false;
  449. return !oom;
  450. }
  451. static void skb_recv_done(struct virtqueue *rvq)
  452. {
  453. struct virtnet_info *vi = rvq->vdev->priv;
  454. struct receive_queue *rq = &vi->rq[vq2rxq(rvq)];
  455. /* Schedule NAPI, Suppress further interrupts if successful. */
  456. if (napi_schedule_prep(&rq->napi)) {
  457. virtqueue_disable_cb(rvq);
  458. __napi_schedule(&rq->napi);
  459. }
  460. }
  461. static void virtnet_napi_enable(struct receive_queue *rq)
  462. {
  463. napi_enable(&rq->napi);
  464. /* If all buffers were filled by other side before we napi_enabled, we
  465. * won't get another interrupt, so process any outstanding packets
  466. * now. virtnet_poll wants re-enable the queue, so we disable here.
  467. * We synchronize against interrupts via NAPI_STATE_SCHED */
  468. if (napi_schedule_prep(&rq->napi)) {
  469. virtqueue_disable_cb(rq->vq);
  470. local_bh_disable();
  471. __napi_schedule(&rq->napi);
  472. local_bh_enable();
  473. }
  474. }
  475. static void refill_work(struct work_struct *work)
  476. {
  477. struct virtnet_info *vi =
  478. container_of(work, struct virtnet_info, refill.work);
  479. bool still_empty;
  480. int i;
  481. for (i = 0; i < vi->curr_queue_pairs; i++) {
  482. struct receive_queue *rq = &vi->rq[i];
  483. napi_disable(&rq->napi);
  484. still_empty = !try_fill_recv(rq, GFP_KERNEL);
  485. virtnet_napi_enable(rq);
  486. /* In theory, this can happen: if we don't get any buffers in
  487. * we will *never* try to fill again.
  488. */
  489. if (still_empty)
  490. schedule_delayed_work(&vi->refill, HZ/2);
  491. }
  492. }
  493. static int virtnet_poll(struct napi_struct *napi, int budget)
  494. {
  495. struct receive_queue *rq =
  496. container_of(napi, struct receive_queue, napi);
  497. struct virtnet_info *vi = rq->vq->vdev->priv;
  498. void *buf;
  499. unsigned int r, len, received = 0;
  500. again:
  501. while (received < budget &&
  502. (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) {
  503. receive_buf(rq, buf, len);
  504. --rq->num;
  505. received++;
  506. }
  507. if (rq->num < rq->max / 2) {
  508. if (!try_fill_recv(rq, GFP_ATOMIC))
  509. schedule_delayed_work(&vi->refill, 0);
  510. }
  511. /* Out of packets? */
  512. if (received < budget) {
  513. r = virtqueue_enable_cb_prepare(rq->vq);
  514. napi_complete(napi);
  515. if (unlikely(virtqueue_poll(rq->vq, r)) &&
  516. napi_schedule_prep(napi)) {
  517. virtqueue_disable_cb(rq->vq);
  518. __napi_schedule(napi);
  519. goto again;
  520. }
  521. }
  522. return received;
  523. }
  524. static int virtnet_open(struct net_device *dev)
  525. {
  526. struct virtnet_info *vi = netdev_priv(dev);
  527. int i;
  528. for (i = 0; i < vi->max_queue_pairs; i++) {
  529. if (i < vi->curr_queue_pairs)
  530. /* Make sure we have some buffers: if oom use wq. */
  531. if (!try_fill_recv(&vi->rq[i], GFP_KERNEL))
  532. schedule_delayed_work(&vi->refill, 0);
  533. virtnet_napi_enable(&vi->rq[i]);
  534. }
  535. return 0;
  536. }
  537. static void free_old_xmit_skbs(struct send_queue *sq)
  538. {
  539. struct sk_buff *skb;
  540. unsigned int len;
  541. struct virtnet_info *vi = sq->vq->vdev->priv;
  542. struct virtnet_stats *stats = this_cpu_ptr(vi->stats);
  543. while ((skb = virtqueue_get_buf(sq->vq, &len)) != NULL) {
  544. pr_debug("Sent skb %p\n", skb);
  545. u64_stats_update_begin(&stats->tx_syncp);
  546. stats->tx_bytes += skb->len;
  547. stats->tx_packets++;
  548. u64_stats_update_end(&stats->tx_syncp);
  549. dev_kfree_skb_any(skb);
  550. }
  551. }
  552. static int xmit_skb(struct send_queue *sq, struct sk_buff *skb)
  553. {
  554. struct skb_vnet_hdr *hdr;
  555. const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
  556. struct virtnet_info *vi = sq->vq->vdev->priv;
  557. unsigned num_sg;
  558. unsigned hdr_len;
  559. bool can_push;
  560. pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
  561. if (vi->mergeable_rx_bufs)
  562. hdr_len = sizeof hdr->mhdr;
  563. else
  564. hdr_len = sizeof hdr->hdr;
  565. can_push = vi->any_header_sg &&
  566. !((unsigned long)skb->data & (__alignof__(*hdr) - 1)) &&
  567. !skb_header_cloned(skb) && skb_headroom(skb) >= hdr_len;
  568. /* Even if we can, don't push here yet as this would skew
  569. * csum_start offset below. */
  570. if (can_push)
  571. hdr = (struct skb_vnet_hdr *)(skb->data - hdr_len);
  572. else
  573. hdr = skb_vnet_hdr(skb);
  574. if (skb->ip_summed == CHECKSUM_PARTIAL) {
  575. hdr->hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
  576. hdr->hdr.csum_start = skb_checksum_start_offset(skb);
  577. hdr->hdr.csum_offset = skb->csum_offset;
  578. } else {
  579. hdr->hdr.flags = 0;
  580. hdr->hdr.csum_offset = hdr->hdr.csum_start = 0;
  581. }
  582. if (skb_is_gso(skb)) {
  583. hdr->hdr.hdr_len = skb_headlen(skb);
  584. hdr->hdr.gso_size = skb_shinfo(skb)->gso_size;
  585. if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
  586. hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
  587. else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
  588. hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
  589. else if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP)
  590. hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
  591. else
  592. BUG();
  593. if (skb_shinfo(skb)->gso_type & SKB_GSO_TCP_ECN)
  594. hdr->hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
  595. } else {
  596. hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
  597. hdr->hdr.gso_size = hdr->hdr.hdr_len = 0;
  598. }
  599. if (vi->mergeable_rx_bufs)
  600. hdr->mhdr.num_buffers = 0;
  601. if (can_push) {
  602. __skb_push(skb, hdr_len);
  603. num_sg = skb_to_sgvec(skb, sq->sg, 0, skb->len);
  604. /* Pull header back to avoid skew in tx bytes calculations. */
  605. __skb_pull(skb, hdr_len);
  606. } else {
  607. sg_set_buf(sq->sg, hdr, hdr_len);
  608. num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len) + 1;
  609. }
  610. return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC);
  611. }
  612. static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
  613. {
  614. struct virtnet_info *vi = netdev_priv(dev);
  615. int qnum = skb_get_queue_mapping(skb);
  616. struct send_queue *sq = &vi->sq[qnum];
  617. int err;
  618. /* Free up any pending old buffers before queueing new ones. */
  619. free_old_xmit_skbs(sq);
  620. /* Try to transmit */
  621. err = xmit_skb(sq, skb);
  622. /* This should not happen! */
  623. if (unlikely(err) || unlikely(!virtqueue_kick(sq->vq))) {
  624. dev->stats.tx_fifo_errors++;
  625. if (net_ratelimit())
  626. dev_warn(&dev->dev,
  627. "Unexpected TXQ (%d) queue failure: %d\n", qnum, err);
  628. dev->stats.tx_dropped++;
  629. kfree_skb(skb);
  630. return NETDEV_TX_OK;
  631. }
  632. /* Don't wait up for transmitted skbs to be freed. */
  633. skb_orphan(skb);
  634. nf_reset(skb);
  635. /* Apparently nice girls don't return TX_BUSY; stop the queue
  636. * before it gets out of hand. Naturally, this wastes entries. */
  637. if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
  638. netif_stop_subqueue(dev, qnum);
  639. if (unlikely(!virtqueue_enable_cb_delayed(sq->vq))) {
  640. /* More just got used, free them then recheck. */
  641. free_old_xmit_skbs(sq);
  642. if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
  643. netif_start_subqueue(dev, qnum);
  644. virtqueue_disable_cb(sq->vq);
  645. }
  646. }
  647. }
  648. return NETDEV_TX_OK;
  649. }
  650. /*
  651. * Send command via the control virtqueue and check status. Commands
  652. * supported by the hypervisor, as indicated by feature bits, should
  653. * never fail unless improperly formated.
  654. */
  655. static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,
  656. struct scatterlist *out,
  657. struct scatterlist *in)
  658. {
  659. struct scatterlist *sgs[4], hdr, stat;
  660. struct virtio_net_ctrl_hdr ctrl;
  661. virtio_net_ctrl_ack status = ~0;
  662. unsigned out_num = 0, in_num = 0, tmp;
  663. /* Caller should know better */
  664. BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ));
  665. ctrl.class = class;
  666. ctrl.cmd = cmd;
  667. /* Add header */
  668. sg_init_one(&hdr, &ctrl, sizeof(ctrl));
  669. sgs[out_num++] = &hdr;
  670. if (out)
  671. sgs[out_num++] = out;
  672. if (in)
  673. sgs[out_num + in_num++] = in;
  674. /* Add return status. */
  675. sg_init_one(&stat, &status, sizeof(status));
  676. sgs[out_num + in_num++] = &stat;
  677. BUG_ON(out_num + in_num > ARRAY_SIZE(sgs));
  678. BUG_ON(virtqueue_add_sgs(vi->cvq, sgs, out_num, in_num, vi, GFP_ATOMIC)
  679. < 0);
  680. if (unlikely(!virtqueue_kick(vi->cvq)))
  681. return status == VIRTIO_NET_OK;
  682. /* Spin for a response, the kick causes an ioport write, trapping
  683. * into the hypervisor, so the request should be handled immediately.
  684. */
  685. while (!virtqueue_get_buf(vi->cvq, &tmp) &&
  686. !virtqueue_is_broken(vi->cvq))
  687. cpu_relax();
  688. return status == VIRTIO_NET_OK;
  689. }
  690. static int virtnet_set_mac_address(struct net_device *dev, void *p)
  691. {
  692. struct virtnet_info *vi = netdev_priv(dev);
  693. struct virtio_device *vdev = vi->vdev;
  694. int ret;
  695. struct sockaddr *addr = p;
  696. struct scatterlist sg;
  697. ret = eth_prepare_mac_addr_change(dev, p);
  698. if (ret)
  699. return ret;
  700. if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
  701. sg_init_one(&sg, addr->sa_data, dev->addr_len);
  702. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
  703. VIRTIO_NET_CTRL_MAC_ADDR_SET,
  704. &sg, NULL)) {
  705. dev_warn(&vdev->dev,
  706. "Failed to set mac address by vq command.\n");
  707. return -EINVAL;
  708. }
  709. } else if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) {
  710. unsigned int i;
  711. /* Naturally, this has an atomicity problem. */
  712. for (i = 0; i < dev->addr_len; i++)
  713. virtio_cwrite8(vdev,
  714. offsetof(struct virtio_net_config, mac) +
  715. i, addr->sa_data[i]);
  716. }
  717. eth_commit_mac_addr_change(dev, p);
  718. return 0;
  719. }
  720. static struct rtnl_link_stats64 *virtnet_stats(struct net_device *dev,
  721. struct rtnl_link_stats64 *tot)
  722. {
  723. struct virtnet_info *vi = netdev_priv(dev);
  724. int cpu;
  725. unsigned int start;
  726. for_each_possible_cpu(cpu) {
  727. struct virtnet_stats *stats = per_cpu_ptr(vi->stats, cpu);
  728. u64 tpackets, tbytes, rpackets, rbytes;
  729. do {
  730. start = u64_stats_fetch_begin_bh(&stats->tx_syncp);
  731. tpackets = stats->tx_packets;
  732. tbytes = stats->tx_bytes;
  733. } while (u64_stats_fetch_retry_bh(&stats->tx_syncp, start));
  734. do {
  735. start = u64_stats_fetch_begin_bh(&stats->rx_syncp);
  736. rpackets = stats->rx_packets;
  737. rbytes = stats->rx_bytes;
  738. } while (u64_stats_fetch_retry_bh(&stats->rx_syncp, start));
  739. tot->rx_packets += rpackets;
  740. tot->tx_packets += tpackets;
  741. tot->rx_bytes += rbytes;
  742. tot->tx_bytes += tbytes;
  743. }
  744. tot->tx_dropped = dev->stats.tx_dropped;
  745. tot->tx_fifo_errors = dev->stats.tx_fifo_errors;
  746. tot->rx_dropped = dev->stats.rx_dropped;
  747. tot->rx_length_errors = dev->stats.rx_length_errors;
  748. tot->rx_frame_errors = dev->stats.rx_frame_errors;
  749. return tot;
  750. }
  751. #ifdef CONFIG_NET_POLL_CONTROLLER
  752. static void virtnet_netpoll(struct net_device *dev)
  753. {
  754. struct virtnet_info *vi = netdev_priv(dev);
  755. int i;
  756. for (i = 0; i < vi->curr_queue_pairs; i++)
  757. napi_schedule(&vi->rq[i].napi);
  758. }
  759. #endif
  760. static void virtnet_ack_link_announce(struct virtnet_info *vi)
  761. {
  762. rtnl_lock();
  763. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE,
  764. VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL, NULL))
  765. dev_warn(&vi->dev->dev, "Failed to ack link announce.\n");
  766. rtnl_unlock();
  767. }
  768. static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
  769. {
  770. struct scatterlist sg;
  771. struct virtio_net_ctrl_mq s;
  772. struct net_device *dev = vi->dev;
  773. if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ))
  774. return 0;
  775. s.virtqueue_pairs = queue_pairs;
  776. sg_init_one(&sg, &s, sizeof(s));
  777. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,
  778. VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &sg, NULL)) {
  779. dev_warn(&dev->dev, "Fail to set num of queue pairs to %d\n",
  780. queue_pairs);
  781. return -EINVAL;
  782. } else {
  783. vi->curr_queue_pairs = queue_pairs;
  784. schedule_delayed_work(&vi->refill, 0);
  785. }
  786. return 0;
  787. }
  788. static int virtnet_close(struct net_device *dev)
  789. {
  790. struct virtnet_info *vi = netdev_priv(dev);
  791. int i;
  792. /* Make sure refill_work doesn't re-enable napi! */
  793. cancel_delayed_work_sync(&vi->refill);
  794. for (i = 0; i < vi->max_queue_pairs; i++)
  795. napi_disable(&vi->rq[i].napi);
  796. return 0;
  797. }
  798. static void virtnet_set_rx_mode(struct net_device *dev)
  799. {
  800. struct virtnet_info *vi = netdev_priv(dev);
  801. struct scatterlist sg[2];
  802. u8 promisc, allmulti;
  803. struct virtio_net_ctrl_mac *mac_data;
  804. struct netdev_hw_addr *ha;
  805. int uc_count;
  806. int mc_count;
  807. void *buf;
  808. int i;
  809. /* We can't dynamicaly set ndo_set_rx_mode, so return gracefully */
  810. if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
  811. return;
  812. promisc = ((dev->flags & IFF_PROMISC) != 0);
  813. allmulti = ((dev->flags & IFF_ALLMULTI) != 0);
  814. sg_init_one(sg, &promisc, sizeof(promisc));
  815. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
  816. VIRTIO_NET_CTRL_RX_PROMISC,
  817. sg, NULL))
  818. dev_warn(&dev->dev, "Failed to %sable promisc mode.\n",
  819. promisc ? "en" : "dis");
  820. sg_init_one(sg, &allmulti, sizeof(allmulti));
  821. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
  822. VIRTIO_NET_CTRL_RX_ALLMULTI,
  823. sg, NULL))
  824. dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
  825. allmulti ? "en" : "dis");
  826. uc_count = netdev_uc_count(dev);
  827. mc_count = netdev_mc_count(dev);
  828. /* MAC filter - use one buffer for both lists */
  829. buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
  830. (2 * sizeof(mac_data->entries)), GFP_ATOMIC);
  831. mac_data = buf;
  832. if (!buf)
  833. return;
  834. sg_init_table(sg, 2);
  835. /* Store the unicast list and count in the front of the buffer */
  836. mac_data->entries = uc_count;
  837. i = 0;
  838. netdev_for_each_uc_addr(ha, dev)
  839. memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
  840. sg_set_buf(&sg[0], mac_data,
  841. sizeof(mac_data->entries) + (uc_count * ETH_ALEN));
  842. /* multicast list and count fill the end */
  843. mac_data = (void *)&mac_data->macs[uc_count][0];
  844. mac_data->entries = mc_count;
  845. i = 0;
  846. netdev_for_each_mc_addr(ha, dev)
  847. memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
  848. sg_set_buf(&sg[1], mac_data,
  849. sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
  850. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
  851. VIRTIO_NET_CTRL_MAC_TABLE_SET,
  852. sg, NULL))
  853. dev_warn(&dev->dev, "Failed to set MAC fitler table.\n");
  854. kfree(buf);
  855. }
  856. static int virtnet_vlan_rx_add_vid(struct net_device *dev,
  857. __be16 proto, u16 vid)
  858. {
  859. struct virtnet_info *vi = netdev_priv(dev);
  860. struct scatterlist sg;
  861. sg_init_one(&sg, &vid, sizeof(vid));
  862. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
  863. VIRTIO_NET_CTRL_VLAN_ADD, &sg, NULL))
  864. dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid);
  865. return 0;
  866. }
  867. static int virtnet_vlan_rx_kill_vid(struct net_device *dev,
  868. __be16 proto, u16 vid)
  869. {
  870. struct virtnet_info *vi = netdev_priv(dev);
  871. struct scatterlist sg;
  872. sg_init_one(&sg, &vid, sizeof(vid));
  873. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
  874. VIRTIO_NET_CTRL_VLAN_DEL, &sg, NULL))
  875. dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid);
  876. return 0;
  877. }
  878. static void virtnet_clean_affinity(struct virtnet_info *vi, long hcpu)
  879. {
  880. int i;
  881. int cpu;
  882. if (vi->affinity_hint_set) {
  883. for (i = 0; i < vi->max_queue_pairs; i++) {
  884. virtqueue_set_affinity(vi->rq[i].vq, -1);
  885. virtqueue_set_affinity(vi->sq[i].vq, -1);
  886. }
  887. vi->affinity_hint_set = false;
  888. }
  889. i = 0;
  890. for_each_online_cpu(cpu) {
  891. if (cpu == hcpu) {
  892. *per_cpu_ptr(vi->vq_index, cpu) = -1;
  893. } else {
  894. *per_cpu_ptr(vi->vq_index, cpu) =
  895. ++i % vi->curr_queue_pairs;
  896. }
  897. }
  898. }
  899. static void virtnet_set_affinity(struct virtnet_info *vi)
  900. {
  901. int i;
  902. int cpu;
  903. /* In multiqueue mode, when the number of cpu is equal to the number of
  904. * queue pairs, we let the queue pairs to be private to one cpu by
  905. * setting the affinity hint to eliminate the contention.
  906. */
  907. if (vi->curr_queue_pairs == 1 ||
  908. vi->max_queue_pairs != num_online_cpus()) {
  909. virtnet_clean_affinity(vi, -1);
  910. return;
  911. }
  912. i = 0;
  913. for_each_online_cpu(cpu) {
  914. virtqueue_set_affinity(vi->rq[i].vq, cpu);
  915. virtqueue_set_affinity(vi->sq[i].vq, cpu);
  916. *per_cpu_ptr(vi->vq_index, cpu) = i;
  917. i++;
  918. }
  919. vi->affinity_hint_set = true;
  920. }
  921. static int virtnet_cpu_callback(struct notifier_block *nfb,
  922. unsigned long action, void *hcpu)
  923. {
  924. struct virtnet_info *vi = container_of(nfb, struct virtnet_info, nb);
  925. switch(action & ~CPU_TASKS_FROZEN) {
  926. case CPU_ONLINE:
  927. case CPU_DOWN_FAILED:
  928. case CPU_DEAD:
  929. virtnet_set_affinity(vi);
  930. break;
  931. case CPU_DOWN_PREPARE:
  932. virtnet_clean_affinity(vi, (long)hcpu);
  933. break;
  934. default:
  935. break;
  936. }
  937. return NOTIFY_OK;
  938. }
  939. static void virtnet_get_ringparam(struct net_device *dev,
  940. struct ethtool_ringparam *ring)
  941. {
  942. struct virtnet_info *vi = netdev_priv(dev);
  943. ring->rx_max_pending = virtqueue_get_vring_size(vi->rq[0].vq);
  944. ring->tx_max_pending = virtqueue_get_vring_size(vi->sq[0].vq);
  945. ring->rx_pending = ring->rx_max_pending;
  946. ring->tx_pending = ring->tx_max_pending;
  947. }
  948. static void virtnet_get_drvinfo(struct net_device *dev,
  949. struct ethtool_drvinfo *info)
  950. {
  951. struct virtnet_info *vi = netdev_priv(dev);
  952. struct virtio_device *vdev = vi->vdev;
  953. strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
  954. strlcpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version));
  955. strlcpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info));
  956. }
  957. /* TODO: Eliminate OOO packets during switching */
  958. static int virtnet_set_channels(struct net_device *dev,
  959. struct ethtool_channels *channels)
  960. {
  961. struct virtnet_info *vi = netdev_priv(dev);
  962. u16 queue_pairs = channels->combined_count;
  963. int err;
  964. /* We don't support separate rx/tx channels.
  965. * We don't allow setting 'other' channels.
  966. */
  967. if (channels->rx_count || channels->tx_count || channels->other_count)
  968. return -EINVAL;
  969. if (queue_pairs > vi->max_queue_pairs)
  970. return -EINVAL;
  971. get_online_cpus();
  972. err = virtnet_set_queues(vi, queue_pairs);
  973. if (!err) {
  974. netif_set_real_num_tx_queues(dev, queue_pairs);
  975. netif_set_real_num_rx_queues(dev, queue_pairs);
  976. virtnet_set_affinity(vi);
  977. }
  978. put_online_cpus();
  979. return err;
  980. }
  981. static void virtnet_get_channels(struct net_device *dev,
  982. struct ethtool_channels *channels)
  983. {
  984. struct virtnet_info *vi = netdev_priv(dev);
  985. channels->combined_count = vi->curr_queue_pairs;
  986. channels->max_combined = vi->max_queue_pairs;
  987. channels->max_other = 0;
  988. channels->rx_count = 0;
  989. channels->tx_count = 0;
  990. channels->other_count = 0;
  991. }
  992. static const struct ethtool_ops virtnet_ethtool_ops = {
  993. .get_drvinfo = virtnet_get_drvinfo,
  994. .get_link = ethtool_op_get_link,
  995. .get_ringparam = virtnet_get_ringparam,
  996. .set_channels = virtnet_set_channels,
  997. .get_channels = virtnet_get_channels,
  998. };
  999. #define MIN_MTU 68
  1000. #define MAX_MTU 65535
  1001. static int virtnet_change_mtu(struct net_device *dev, int new_mtu)
  1002. {
  1003. if (new_mtu < MIN_MTU || new_mtu > MAX_MTU)
  1004. return -EINVAL;
  1005. dev->mtu = new_mtu;
  1006. return 0;
  1007. }
  1008. /* To avoid contending a lock hold by a vcpu who would exit to host, select the
  1009. * txq based on the processor id.
  1010. */
  1011. static u16 virtnet_select_queue(struct net_device *dev, struct sk_buff *skb)
  1012. {
  1013. int txq;
  1014. struct virtnet_info *vi = netdev_priv(dev);
  1015. if (skb_rx_queue_recorded(skb)) {
  1016. txq = skb_get_rx_queue(skb);
  1017. } else {
  1018. txq = *__this_cpu_ptr(vi->vq_index);
  1019. if (txq == -1)
  1020. txq = 0;
  1021. }
  1022. while (unlikely(txq >= dev->real_num_tx_queues))
  1023. txq -= dev->real_num_tx_queues;
  1024. return txq;
  1025. }
  1026. static const struct net_device_ops virtnet_netdev = {
  1027. .ndo_open = virtnet_open,
  1028. .ndo_stop = virtnet_close,
  1029. .ndo_start_xmit = start_xmit,
  1030. .ndo_validate_addr = eth_validate_addr,
  1031. .ndo_set_mac_address = virtnet_set_mac_address,
  1032. .ndo_set_rx_mode = virtnet_set_rx_mode,
  1033. .ndo_change_mtu = virtnet_change_mtu,
  1034. .ndo_get_stats64 = virtnet_stats,
  1035. .ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid,
  1036. .ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
  1037. .ndo_select_queue = virtnet_select_queue,
  1038. #ifdef CONFIG_NET_POLL_CONTROLLER
  1039. .ndo_poll_controller = virtnet_netpoll,
  1040. #endif
  1041. };
  1042. static void virtnet_config_changed_work(struct work_struct *work)
  1043. {
  1044. struct virtnet_info *vi =
  1045. container_of(work, struct virtnet_info, config_work);
  1046. u16 v;
  1047. mutex_lock(&vi->config_lock);
  1048. if (!vi->config_enable)
  1049. goto done;
  1050. if (virtio_cread_feature(vi->vdev, VIRTIO_NET_F_STATUS,
  1051. struct virtio_net_config, status, &v) < 0)
  1052. goto done;
  1053. if (v & VIRTIO_NET_S_ANNOUNCE) {
  1054. netdev_notify_peers(vi->dev);
  1055. virtnet_ack_link_announce(vi);
  1056. }
  1057. /* Ignore unknown (future) status bits */
  1058. v &= VIRTIO_NET_S_LINK_UP;
  1059. if (vi->status == v)
  1060. goto done;
  1061. vi->status = v;
  1062. if (vi->status & VIRTIO_NET_S_LINK_UP) {
  1063. netif_carrier_on(vi->dev);
  1064. netif_tx_wake_all_queues(vi->dev);
  1065. } else {
  1066. netif_carrier_off(vi->dev);
  1067. netif_tx_stop_all_queues(vi->dev);
  1068. }
  1069. done:
  1070. mutex_unlock(&vi->config_lock);
  1071. }
  1072. static void virtnet_config_changed(struct virtio_device *vdev)
  1073. {
  1074. struct virtnet_info *vi = vdev->priv;
  1075. schedule_work(&vi->config_work);
  1076. }
  1077. static void virtnet_free_queues(struct virtnet_info *vi)
  1078. {
  1079. kfree(vi->rq);
  1080. kfree(vi->sq);
  1081. }
  1082. static void free_receive_bufs(struct virtnet_info *vi)
  1083. {
  1084. int i;
  1085. for (i = 0; i < vi->max_queue_pairs; i++) {
  1086. while (vi->rq[i].pages)
  1087. __free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0);
  1088. }
  1089. }
  1090. static void free_unused_bufs(struct virtnet_info *vi)
  1091. {
  1092. void *buf;
  1093. int i;
  1094. for (i = 0; i < vi->max_queue_pairs; i++) {
  1095. struct virtqueue *vq = vi->sq[i].vq;
  1096. while ((buf = virtqueue_detach_unused_buf(vq)) != NULL)
  1097. dev_kfree_skb(buf);
  1098. }
  1099. for (i = 0; i < vi->max_queue_pairs; i++) {
  1100. struct virtqueue *vq = vi->rq[i].vq;
  1101. while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
  1102. if (vi->mergeable_rx_bufs || vi->big_packets)
  1103. give_pages(&vi->rq[i], buf);
  1104. else
  1105. dev_kfree_skb(buf);
  1106. --vi->rq[i].num;
  1107. }
  1108. BUG_ON(vi->rq[i].num != 0);
  1109. }
  1110. }
  1111. static void virtnet_del_vqs(struct virtnet_info *vi)
  1112. {
  1113. struct virtio_device *vdev = vi->vdev;
  1114. virtnet_clean_affinity(vi, -1);
  1115. vdev->config->del_vqs(vdev);
  1116. virtnet_free_queues(vi);
  1117. }
  1118. static int virtnet_find_vqs(struct virtnet_info *vi)
  1119. {
  1120. vq_callback_t **callbacks;
  1121. struct virtqueue **vqs;
  1122. int ret = -ENOMEM;
  1123. int i, total_vqs;
  1124. const char **names;
  1125. /* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by
  1126. * possible N-1 RX/TX queue pairs used in multiqueue mode, followed by
  1127. * possible control vq.
  1128. */
  1129. total_vqs = vi->max_queue_pairs * 2 +
  1130. virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ);
  1131. /* Allocate space for find_vqs parameters */
  1132. vqs = kzalloc(total_vqs * sizeof(*vqs), GFP_KERNEL);
  1133. if (!vqs)
  1134. goto err_vq;
  1135. callbacks = kmalloc(total_vqs * sizeof(*callbacks), GFP_KERNEL);
  1136. if (!callbacks)
  1137. goto err_callback;
  1138. names = kmalloc(total_vqs * sizeof(*names), GFP_KERNEL);
  1139. if (!names)
  1140. goto err_names;
  1141. /* Parameters for control virtqueue, if any */
  1142. if (vi->has_cvq) {
  1143. callbacks[total_vqs - 1] = NULL;
  1144. names[total_vqs - 1] = "control";
  1145. }
  1146. /* Allocate/initialize parameters for send/receive virtqueues */
  1147. for (i = 0; i < vi->max_queue_pairs; i++) {
  1148. callbacks[rxq2vq(i)] = skb_recv_done;
  1149. callbacks[txq2vq(i)] = skb_xmit_done;
  1150. sprintf(vi->rq[i].name, "input.%d", i);
  1151. sprintf(vi->sq[i].name, "output.%d", i);
  1152. names[rxq2vq(i)] = vi->rq[i].name;
  1153. names[txq2vq(i)] = vi->sq[i].name;
  1154. }
  1155. ret = vi->vdev->config->find_vqs(vi->vdev, total_vqs, vqs, callbacks,
  1156. names);
  1157. if (ret)
  1158. goto err_find;
  1159. if (vi->has_cvq) {
  1160. vi->cvq = vqs[total_vqs - 1];
  1161. if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
  1162. vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
  1163. }
  1164. for (i = 0; i < vi->max_queue_pairs; i++) {
  1165. vi->rq[i].vq = vqs[rxq2vq(i)];
  1166. vi->sq[i].vq = vqs[txq2vq(i)];
  1167. }
  1168. kfree(names);
  1169. kfree(callbacks);
  1170. kfree(vqs);
  1171. return 0;
  1172. err_find:
  1173. kfree(names);
  1174. err_names:
  1175. kfree(callbacks);
  1176. err_callback:
  1177. kfree(vqs);
  1178. err_vq:
  1179. return ret;
  1180. }
  1181. static int virtnet_alloc_queues(struct virtnet_info *vi)
  1182. {
  1183. int i;
  1184. vi->sq = kzalloc(sizeof(*vi->sq) * vi->max_queue_pairs, GFP_KERNEL);
  1185. if (!vi->sq)
  1186. goto err_sq;
  1187. vi->rq = kzalloc(sizeof(*vi->rq) * vi->max_queue_pairs, GFP_KERNEL);
  1188. if (!vi->rq)
  1189. goto err_rq;
  1190. INIT_DELAYED_WORK(&vi->refill, refill_work);
  1191. for (i = 0; i < vi->max_queue_pairs; i++) {
  1192. vi->rq[i].pages = NULL;
  1193. netif_napi_add(vi->dev, &vi->rq[i].napi, virtnet_poll,
  1194. napi_weight);
  1195. sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
  1196. sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));
  1197. }
  1198. return 0;
  1199. err_rq:
  1200. kfree(vi->sq);
  1201. err_sq:
  1202. return -ENOMEM;
  1203. }
  1204. static int init_vqs(struct virtnet_info *vi)
  1205. {
  1206. int ret;
  1207. /* Allocate send & receive queues */
  1208. ret = virtnet_alloc_queues(vi);
  1209. if (ret)
  1210. goto err;
  1211. ret = virtnet_find_vqs(vi);
  1212. if (ret)
  1213. goto err_free;
  1214. get_online_cpus();
  1215. virtnet_set_affinity(vi);
  1216. put_online_cpus();
  1217. return 0;
  1218. err_free:
  1219. virtnet_free_queues(vi);
  1220. err:
  1221. return ret;
  1222. }
  1223. static int virtnet_probe(struct virtio_device *vdev)
  1224. {
  1225. int i, err;
  1226. struct net_device *dev;
  1227. struct virtnet_info *vi;
  1228. u16 max_queue_pairs;
  1229. /* Find if host supports multiqueue virtio_net device */
  1230. err = virtio_cread_feature(vdev, VIRTIO_NET_F_MQ,
  1231. struct virtio_net_config,
  1232. max_virtqueue_pairs, &max_queue_pairs);
  1233. /* We need at least 2 queue's */
  1234. if (err || max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
  1235. max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX ||
  1236. !virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
  1237. max_queue_pairs = 1;
  1238. /* Allocate ourselves a network device with room for our info */
  1239. dev = alloc_etherdev_mq(sizeof(struct virtnet_info), max_queue_pairs);
  1240. if (!dev)
  1241. return -ENOMEM;
  1242. /* Set up network device as normal. */
  1243. dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE;
  1244. dev->netdev_ops = &virtnet_netdev;
  1245. dev->features = NETIF_F_HIGHDMA;
  1246. SET_ETHTOOL_OPS(dev, &virtnet_ethtool_ops);
  1247. SET_NETDEV_DEV(dev, &vdev->dev);
  1248. /* Do we support "hardware" checksums? */
  1249. if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) {
  1250. /* This opens up the world of extra features. */
  1251. dev->hw_features |= NETIF_F_HW_CSUM|NETIF_F_SG|NETIF_F_FRAGLIST;
  1252. if (csum)
  1253. dev->features |= NETIF_F_HW_CSUM|NETIF_F_SG|NETIF_F_FRAGLIST;
  1254. if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
  1255. dev->hw_features |= NETIF_F_TSO | NETIF_F_UFO
  1256. | NETIF_F_TSO_ECN | NETIF_F_TSO6;
  1257. }
  1258. /* Individual feature bits: what can host handle? */
  1259. if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4))
  1260. dev->hw_features |= NETIF_F_TSO;
  1261. if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6))
  1262. dev->hw_features |= NETIF_F_TSO6;
  1263. if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
  1264. dev->hw_features |= NETIF_F_TSO_ECN;
  1265. if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_UFO))
  1266. dev->hw_features |= NETIF_F_UFO;
  1267. if (gso)
  1268. dev->features |= dev->hw_features & (NETIF_F_ALL_TSO|NETIF_F_UFO);
  1269. /* (!csum && gso) case will be fixed by register_netdev() */
  1270. }
  1271. if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_CSUM))
  1272. dev->features |= NETIF_F_RXCSUM;
  1273. dev->vlan_features = dev->features;
  1274. /* Configuration may specify what MAC to use. Otherwise random. */
  1275. if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC))
  1276. virtio_cread_bytes(vdev,
  1277. offsetof(struct virtio_net_config, mac),
  1278. dev->dev_addr, dev->addr_len);
  1279. else
  1280. eth_hw_addr_random(dev);
  1281. /* Set up our device-specific information */
  1282. vi = netdev_priv(dev);
  1283. vi->dev = dev;
  1284. vi->vdev = vdev;
  1285. vdev->priv = vi;
  1286. vi->stats = alloc_percpu(struct virtnet_stats);
  1287. err = -ENOMEM;
  1288. if (vi->stats == NULL)
  1289. goto free;
  1290. vi->vq_index = alloc_percpu(int);
  1291. if (vi->vq_index == NULL)
  1292. goto free_stats;
  1293. mutex_init(&vi->config_lock);
  1294. vi->config_enable = true;
  1295. INIT_WORK(&vi->config_work, virtnet_config_changed_work);
  1296. /* If we can receive ANY GSO packets, we must allocate large ones. */
  1297. if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
  1298. virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) ||
  1299. virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN))
  1300. vi->big_packets = true;
  1301. if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
  1302. vi->mergeable_rx_bufs = true;
  1303. if (virtio_has_feature(vdev, VIRTIO_F_ANY_LAYOUT))
  1304. vi->any_header_sg = true;
  1305. if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
  1306. vi->has_cvq = true;
  1307. /* Use single tx/rx queue pair as default */
  1308. vi->curr_queue_pairs = 1;
  1309. vi->max_queue_pairs = max_queue_pairs;
  1310. /* Allocate/initialize the rx/tx queues, and invoke find_vqs */
  1311. err = init_vqs(vi);
  1312. if (err)
  1313. goto free_index;
  1314. netif_set_real_num_tx_queues(dev, 1);
  1315. netif_set_real_num_rx_queues(dev, 1);
  1316. err = register_netdev(dev);
  1317. if (err) {
  1318. pr_debug("virtio_net: registering device failed\n");
  1319. goto free_vqs;
  1320. }
  1321. /* Last of all, set up some receive buffers. */
  1322. for (i = 0; i < vi->curr_queue_pairs; i++) {
  1323. try_fill_recv(&vi->rq[i], GFP_KERNEL);
  1324. /* If we didn't even get one input buffer, we're useless. */
  1325. if (vi->rq[i].num == 0) {
  1326. free_unused_bufs(vi);
  1327. err = -ENOMEM;
  1328. goto free_recv_bufs;
  1329. }
  1330. }
  1331. vi->nb.notifier_call = &virtnet_cpu_callback;
  1332. err = register_hotcpu_notifier(&vi->nb);
  1333. if (err) {
  1334. pr_debug("virtio_net: registering cpu notifier failed\n");
  1335. goto free_recv_bufs;
  1336. }
  1337. /* Assume link up if device can't report link status,
  1338. otherwise get link status from config. */
  1339. if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {
  1340. netif_carrier_off(dev);
  1341. schedule_work(&vi->config_work);
  1342. } else {
  1343. vi->status = VIRTIO_NET_S_LINK_UP;
  1344. netif_carrier_on(dev);
  1345. }
  1346. pr_debug("virtnet: registered device %s with %d RX and TX vq's\n",
  1347. dev->name, max_queue_pairs);
  1348. return 0;
  1349. free_recv_bufs:
  1350. free_receive_bufs(vi);
  1351. unregister_netdev(dev);
  1352. free_vqs:
  1353. cancel_delayed_work_sync(&vi->refill);
  1354. virtnet_del_vqs(vi);
  1355. free_index:
  1356. free_percpu(vi->vq_index);
  1357. free_stats:
  1358. free_percpu(vi->stats);
  1359. free:
  1360. free_netdev(dev);
  1361. return err;
  1362. }
  1363. static void remove_vq_common(struct virtnet_info *vi)
  1364. {
  1365. vi->vdev->config->reset(vi->vdev);
  1366. /* Free unused buffers in both send and recv, if any. */
  1367. free_unused_bufs(vi);
  1368. free_receive_bufs(vi);
  1369. virtnet_del_vqs(vi);
  1370. }
  1371. static void virtnet_remove(struct virtio_device *vdev)
  1372. {
  1373. struct virtnet_info *vi = vdev->priv;
  1374. unregister_hotcpu_notifier(&vi->nb);
  1375. /* Prevent config work handler from accessing the device. */
  1376. mutex_lock(&vi->config_lock);
  1377. vi->config_enable = false;
  1378. mutex_unlock(&vi->config_lock);
  1379. unregister_netdev(vi->dev);
  1380. remove_vq_common(vi);
  1381. flush_work(&vi->config_work);
  1382. free_percpu(vi->vq_index);
  1383. free_percpu(vi->stats);
  1384. free_netdev(vi->dev);
  1385. }
  1386. #ifdef CONFIG_PM_SLEEP
  1387. static int virtnet_freeze(struct virtio_device *vdev)
  1388. {
  1389. struct virtnet_info *vi = vdev->priv;
  1390. int i;
  1391. /* Prevent config work handler from accessing the device */
  1392. mutex_lock(&vi->config_lock);
  1393. vi->config_enable = false;
  1394. mutex_unlock(&vi->config_lock);
  1395. netif_device_detach(vi->dev);
  1396. cancel_delayed_work_sync(&vi->refill);
  1397. if (netif_running(vi->dev))
  1398. for (i = 0; i < vi->max_queue_pairs; i++) {
  1399. napi_disable(&vi->rq[i].napi);
  1400. netif_napi_del(&vi->rq[i].napi);
  1401. }
  1402. remove_vq_common(vi);
  1403. flush_work(&vi->config_work);
  1404. return 0;
  1405. }
  1406. static int virtnet_restore(struct virtio_device *vdev)
  1407. {
  1408. struct virtnet_info *vi = vdev->priv;
  1409. int err, i;
  1410. err = init_vqs(vi);
  1411. if (err)
  1412. return err;
  1413. if (netif_running(vi->dev))
  1414. for (i = 0; i < vi->max_queue_pairs; i++)
  1415. virtnet_napi_enable(&vi->rq[i]);
  1416. netif_device_attach(vi->dev);
  1417. for (i = 0; i < vi->curr_queue_pairs; i++)
  1418. if (!try_fill_recv(&vi->rq[i], GFP_KERNEL))
  1419. schedule_delayed_work(&vi->refill, 0);
  1420. mutex_lock(&vi->config_lock);
  1421. vi->config_enable = true;
  1422. mutex_unlock(&vi->config_lock);
  1423. virtnet_set_queues(vi, vi->curr_queue_pairs);
  1424. return 0;
  1425. }
  1426. #endif
  1427. static struct virtio_device_id id_table[] = {
  1428. { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
  1429. { 0 },
  1430. };
  1431. static unsigned int features[] = {
  1432. VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM,
  1433. VIRTIO_NET_F_GSO, VIRTIO_NET_F_MAC,
  1434. VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6,
  1435. VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6,
  1436. VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO,
  1437. VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ,
  1438. VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN,
  1439. VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ,
  1440. VIRTIO_NET_F_CTRL_MAC_ADDR,
  1441. VIRTIO_F_ANY_LAYOUT,
  1442. };
  1443. static struct virtio_driver virtio_net_driver = {
  1444. .feature_table = features,
  1445. .feature_table_size = ARRAY_SIZE(features),
  1446. .driver.name = KBUILD_MODNAME,
  1447. .driver.owner = THIS_MODULE,
  1448. .id_table = id_table,
  1449. .probe = virtnet_probe,
  1450. .remove = virtnet_remove,
  1451. .config_changed = virtnet_config_changed,
  1452. #ifdef CONFIG_PM_SLEEP
  1453. .freeze = virtnet_freeze,
  1454. .restore = virtnet_restore,
  1455. #endif
  1456. };
  1457. module_virtio_driver(virtio_net_driver);
  1458. MODULE_DEVICE_TABLE(virtio, id_table);
  1459. MODULE_DESCRIPTION("Virtio network driver");
  1460. MODULE_LICENSE("GPL");