skbuff.c 73 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978
  1. /*
  2. * Routines having to do with the 'struct sk_buff' memory handlers.
  3. *
  4. * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk>
  5. * Florian La Roche <rzsfl@rz.uni-sb.de>
  6. *
  7. * Fixes:
  8. * Alan Cox : Fixed the worst of the load
  9. * balancer bugs.
  10. * Dave Platt : Interrupt stacking fix.
  11. * Richard Kooijman : Timestamp fixes.
  12. * Alan Cox : Changed buffer format.
  13. * Alan Cox : destructor hook for AF_UNIX etc.
  14. * Linus Torvalds : Better skb_clone.
  15. * Alan Cox : Added skb_copy.
  16. * Alan Cox : Added all the changed routines Linus
  17. * only put in the headers
  18. * Ray VanTassle : Fixed --skb->lock in free
  19. * Alan Cox : skb_copy copy arp field
  20. * Andi Kleen : slabified it.
  21. * Robert Olsson : Removed skb_head_pool
  22. *
  23. * NOTE:
  24. * The __skb_ routines should be called with interrupts
  25. * disabled, or you better be *real* sure that the operation is atomic
  26. * with respect to whatever list is being frobbed (e.g. via lock_sock()
  27. * or via disabling bottom half handlers, etc).
  28. *
  29. * This program is free software; you can redistribute it and/or
  30. * modify it under the terms of the GNU General Public License
  31. * as published by the Free Software Foundation; either version
  32. * 2 of the License, or (at your option) any later version.
  33. */
  34. /*
  35. * The functions in this file will not compile correctly with gcc 2.4.x
  36. */
  37. #include <linux/module.h>
  38. #include <linux/types.h>
  39. #include <linux/kernel.h>
  40. #include <linux/mm.h>
  41. #include <linux/interrupt.h>
  42. #include <linux/in.h>
  43. #include <linux/inet.h>
  44. #include <linux/slab.h>
  45. #include <linux/netdevice.h>
  46. #ifdef CONFIG_NET_CLS_ACT
  47. #include <net/pkt_sched.h>
  48. #endif
  49. #include <linux/string.h>
  50. #include <linux/skbuff.h>
  51. #include <linux/splice.h>
  52. #include <linux/cache.h>
  53. #include <linux/rtnetlink.h>
  54. #include <linux/init.h>
  55. #include <linux/scatterlist.h>
  56. #include <net/protocol.h>
  57. #include <net/dst.h>
  58. #include <net/sock.h>
  59. #include <net/checksum.h>
  60. #include <net/xfrm.h>
  61. #include <asm/uaccess.h>
  62. #include <asm/system.h>
  63. #include "kmap_skb.h"
  64. static struct kmem_cache *skbuff_head_cache __read_mostly;
  65. static struct kmem_cache *skbuff_fclone_cache __read_mostly;
  66. static void sock_pipe_buf_release(struct pipe_inode_info *pipe,
  67. struct pipe_buffer *buf)
  68. {
  69. put_page(buf->page);
  70. }
  71. static void sock_pipe_buf_get(struct pipe_inode_info *pipe,
  72. struct pipe_buffer *buf)
  73. {
  74. get_page(buf->page);
  75. }
  76. static int sock_pipe_buf_steal(struct pipe_inode_info *pipe,
  77. struct pipe_buffer *buf)
  78. {
  79. return 1;
  80. }
  81. /* Pipe buffer operations for a socket. */
  82. static struct pipe_buf_operations sock_pipe_buf_ops = {
  83. .can_merge = 0,
  84. .map = generic_pipe_buf_map,
  85. .unmap = generic_pipe_buf_unmap,
  86. .confirm = generic_pipe_buf_confirm,
  87. .release = sock_pipe_buf_release,
  88. .steal = sock_pipe_buf_steal,
  89. .get = sock_pipe_buf_get,
  90. };
  91. /*
  92. * Keep out-of-line to prevent kernel bloat.
  93. * __builtin_return_address is not used because it is not always
  94. * reliable.
  95. */
  96. /**
  97. * skb_over_panic - private function
  98. * @skb: buffer
  99. * @sz: size
  100. * @here: address
  101. *
  102. * Out of line support code for skb_put(). Not user callable.
  103. */
  104. void skb_over_panic(struct sk_buff *skb, int sz, void *here)
  105. {
  106. printk(KERN_EMERG "skb_over_panic: text:%p len:%d put:%d head:%p "
  107. "data:%p tail:%#lx end:%#lx dev:%s\n",
  108. here, skb->len, sz, skb->head, skb->data,
  109. (unsigned long)skb->tail, (unsigned long)skb->end,
  110. skb->dev ? skb->dev->name : "<NULL>");
  111. BUG();
  112. }
  113. EXPORT_SYMBOL(skb_over_panic);
  114. /**
  115. * skb_under_panic - private function
  116. * @skb: buffer
  117. * @sz: size
  118. * @here: address
  119. *
  120. * Out of line support code for skb_push(). Not user callable.
  121. */
  122. void skb_under_panic(struct sk_buff *skb, int sz, void *here)
  123. {
  124. printk(KERN_EMERG "skb_under_panic: text:%p len:%d put:%d head:%p "
  125. "data:%p tail:%#lx end:%#lx dev:%s\n",
  126. here, skb->len, sz, skb->head, skb->data,
  127. (unsigned long)skb->tail, (unsigned long)skb->end,
  128. skb->dev ? skb->dev->name : "<NULL>");
  129. BUG();
  130. }
  131. EXPORT_SYMBOL(skb_under_panic);
  132. void skb_truesize_bug(struct sk_buff *skb)
  133. {
  134. WARN(net_ratelimit(), KERN_ERR "SKB BUG: Invalid truesize (%u) "
  135. "len=%u, sizeof(sk_buff)=%Zd\n",
  136. skb->truesize, skb->len, sizeof(struct sk_buff));
  137. }
  138. EXPORT_SYMBOL(skb_truesize_bug);
  139. /* Allocate a new skbuff. We do this ourselves so we can fill in a few
  140. * 'private' fields and also do memory statistics to find all the
  141. * [BEEP] leaks.
  142. *
  143. */
  144. /**
  145. * __alloc_skb - allocate a network buffer
  146. * @size: size to allocate
  147. * @gfp_mask: allocation mask
  148. * @fclone: allocate from fclone cache instead of head cache
  149. * and allocate a cloned (child) skb
  150. * @node: numa node to allocate memory on
  151. *
  152. * Allocate a new &sk_buff. The returned buffer has no headroom and a
  153. * tail room of size bytes. The object has a reference count of one.
  154. * The return is the buffer. On a failure the return is %NULL.
  155. *
  156. * Buffers may only be allocated from interrupts using a @gfp_mask of
  157. * %GFP_ATOMIC.
  158. */
  159. struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
  160. int fclone, int node)
  161. {
  162. struct kmem_cache *cache;
  163. struct skb_shared_info *shinfo;
  164. struct sk_buff *skb;
  165. u8 *data;
  166. cache = fclone ? skbuff_fclone_cache : skbuff_head_cache;
  167. /* Get the HEAD */
  168. skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
  169. if (!skb)
  170. goto out;
  171. size = SKB_DATA_ALIGN(size);
  172. data = kmalloc_node_track_caller(size + sizeof(struct skb_shared_info),
  173. gfp_mask, node);
  174. if (!data)
  175. goto nodata;
  176. /*
  177. * Only clear those fields we need to clear, not those that we will
  178. * actually initialise below. Hence, don't put any more fields after
  179. * the tail pointer in struct sk_buff!
  180. */
  181. memset(skb, 0, offsetof(struct sk_buff, tail));
  182. skb->truesize = size + sizeof(struct sk_buff);
  183. atomic_set(&skb->users, 1);
  184. skb->head = data;
  185. skb->data = data;
  186. skb_reset_tail_pointer(skb);
  187. skb->end = skb->tail + size;
  188. /* make sure we initialize shinfo sequentially */
  189. shinfo = skb_shinfo(skb);
  190. atomic_set(&shinfo->dataref, 1);
  191. shinfo->nr_frags = 0;
  192. shinfo->gso_size = 0;
  193. shinfo->gso_segs = 0;
  194. shinfo->gso_type = 0;
  195. shinfo->ip6_frag_id = 0;
  196. shinfo->frag_list = NULL;
  197. if (fclone) {
  198. struct sk_buff *child = skb + 1;
  199. atomic_t *fclone_ref = (atomic_t *) (child + 1);
  200. skb->fclone = SKB_FCLONE_ORIG;
  201. atomic_set(fclone_ref, 1);
  202. child->fclone = SKB_FCLONE_UNAVAILABLE;
  203. }
  204. out:
  205. return skb;
  206. nodata:
  207. kmem_cache_free(cache, skb);
  208. skb = NULL;
  209. goto out;
  210. }
  211. EXPORT_SYMBOL(__alloc_skb);
  212. /**
  213. * __netdev_alloc_skb - allocate an skbuff for rx on a specific device
  214. * @dev: network device to receive on
  215. * @length: length to allocate
  216. * @gfp_mask: get_free_pages mask, passed to alloc_skb
  217. *
  218. * Allocate a new &sk_buff and assign it a usage count of one. The
  219. * buffer has unspecified headroom built in. Users should allocate
  220. * the headroom they think they need without accounting for the
  221. * built in space. The built in space is used for optimisations.
  222. *
  223. * %NULL is returned if there is no free memory.
  224. */
  225. struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
  226. unsigned int length, gfp_t gfp_mask)
  227. {
  228. int node = dev->dev.parent ? dev_to_node(dev->dev.parent) : -1;
  229. struct sk_buff *skb;
  230. skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask, 0, node);
  231. if (likely(skb)) {
  232. skb_reserve(skb, NET_SKB_PAD);
  233. skb->dev = dev;
  234. }
  235. return skb;
  236. }
  237. EXPORT_SYMBOL(__netdev_alloc_skb);
  238. struct page *__netdev_alloc_page(struct net_device *dev, gfp_t gfp_mask)
  239. {
  240. int node = dev->dev.parent ? dev_to_node(dev->dev.parent) : -1;
  241. struct page *page;
  242. page = alloc_pages_node(node, gfp_mask, 0);
  243. return page;
  244. }
  245. EXPORT_SYMBOL(__netdev_alloc_page);
  246. void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
  247. int size)
  248. {
  249. skb_fill_page_desc(skb, i, page, off, size);
  250. skb->len += size;
  251. skb->data_len += size;
  252. skb->truesize += size;
  253. }
  254. EXPORT_SYMBOL(skb_add_rx_frag);
  255. /**
  256. * dev_alloc_skb - allocate an skbuff for receiving
  257. * @length: length to allocate
  258. *
  259. * Allocate a new &sk_buff and assign it a usage count of one. The
  260. * buffer has unspecified headroom built in. Users should allocate
  261. * the headroom they think they need without accounting for the
  262. * built in space. The built in space is used for optimisations.
  263. *
  264. * %NULL is returned if there is no free memory. Although this function
  265. * allocates memory it can be called from an interrupt.
  266. */
  267. struct sk_buff *dev_alloc_skb(unsigned int length)
  268. {
  269. /*
  270. * There is more code here than it seems:
  271. * __dev_alloc_skb is an inline
  272. */
  273. return __dev_alloc_skb(length, GFP_ATOMIC);
  274. }
  275. EXPORT_SYMBOL(dev_alloc_skb);
  276. static void skb_drop_list(struct sk_buff **listp)
  277. {
  278. struct sk_buff *list = *listp;
  279. *listp = NULL;
  280. do {
  281. struct sk_buff *this = list;
  282. list = list->next;
  283. kfree_skb(this);
  284. } while (list);
  285. }
  286. static inline void skb_drop_fraglist(struct sk_buff *skb)
  287. {
  288. skb_drop_list(&skb_shinfo(skb)->frag_list);
  289. }
  290. static void skb_clone_fraglist(struct sk_buff *skb)
  291. {
  292. struct sk_buff *list;
  293. for (list = skb_shinfo(skb)->frag_list; list; list = list->next)
  294. skb_get(list);
  295. }
  296. static void skb_release_data(struct sk_buff *skb)
  297. {
  298. if (!skb->cloned ||
  299. !atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
  300. &skb_shinfo(skb)->dataref)) {
  301. if (skb_shinfo(skb)->nr_frags) {
  302. int i;
  303. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
  304. put_page(skb_shinfo(skb)->frags[i].page);
  305. }
  306. if (skb_shinfo(skb)->frag_list)
  307. skb_drop_fraglist(skb);
  308. kfree(skb->head);
  309. }
  310. }
  311. /*
  312. * Free an skbuff by memory without cleaning the state.
  313. */
  314. static void kfree_skbmem(struct sk_buff *skb)
  315. {
  316. struct sk_buff *other;
  317. atomic_t *fclone_ref;
  318. switch (skb->fclone) {
  319. case SKB_FCLONE_UNAVAILABLE:
  320. kmem_cache_free(skbuff_head_cache, skb);
  321. break;
  322. case SKB_FCLONE_ORIG:
  323. fclone_ref = (atomic_t *) (skb + 2);
  324. if (atomic_dec_and_test(fclone_ref))
  325. kmem_cache_free(skbuff_fclone_cache, skb);
  326. break;
  327. case SKB_FCLONE_CLONE:
  328. fclone_ref = (atomic_t *) (skb + 1);
  329. other = skb - 1;
  330. /* The clone portion is available for
  331. * fast-cloning again.
  332. */
  333. skb->fclone = SKB_FCLONE_UNAVAILABLE;
  334. if (atomic_dec_and_test(fclone_ref))
  335. kmem_cache_free(skbuff_fclone_cache, other);
  336. break;
  337. }
  338. }
  339. static void skb_release_head_state(struct sk_buff *skb)
  340. {
  341. dst_release(skb->dst);
  342. #ifdef CONFIG_XFRM
  343. secpath_put(skb->sp);
  344. #endif
  345. if (skb->destructor) {
  346. WARN_ON(in_irq());
  347. skb->destructor(skb);
  348. }
  349. #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
  350. nf_conntrack_put(skb->nfct);
  351. nf_conntrack_put_reasm(skb->nfct_reasm);
  352. #endif
  353. #ifdef CONFIG_BRIDGE_NETFILTER
  354. nf_bridge_put(skb->nf_bridge);
  355. #endif
  356. /* XXX: IS this still necessary? - JHS */
  357. #ifdef CONFIG_NET_SCHED
  358. skb->tc_index = 0;
  359. #ifdef CONFIG_NET_CLS_ACT
  360. skb->tc_verd = 0;
  361. #endif
  362. #endif
  363. }
  364. /* Free everything but the sk_buff shell. */
  365. static void skb_release_all(struct sk_buff *skb)
  366. {
  367. skb_release_head_state(skb);
  368. skb_release_data(skb);
  369. }
  370. /**
  371. * __kfree_skb - private function
  372. * @skb: buffer
  373. *
  374. * Free an sk_buff. Release anything attached to the buffer.
  375. * Clean the state. This is an internal helper function. Users should
  376. * always call kfree_skb
  377. */
  378. void __kfree_skb(struct sk_buff *skb)
  379. {
  380. skb_release_all(skb);
  381. kfree_skbmem(skb);
  382. }
  383. EXPORT_SYMBOL(__kfree_skb);
  384. /**
  385. * kfree_skb - free an sk_buff
  386. * @skb: buffer to free
  387. *
  388. * Drop a reference to the buffer and free it if the usage count has
  389. * hit zero.
  390. */
  391. void kfree_skb(struct sk_buff *skb)
  392. {
  393. if (unlikely(!skb))
  394. return;
  395. if (likely(atomic_read(&skb->users) == 1))
  396. smp_rmb();
  397. else if (likely(!atomic_dec_and_test(&skb->users)))
  398. return;
  399. __kfree_skb(skb);
  400. }
  401. EXPORT_SYMBOL(kfree_skb);
  402. /**
  403. * skb_recycle_check - check if skb can be reused for receive
  404. * @skb: buffer
  405. * @skb_size: minimum receive buffer size
  406. *
  407. * Checks that the skb passed in is not shared or cloned, and
  408. * that it is linear and its head portion at least as large as
  409. * skb_size so that it can be recycled as a receive buffer.
  410. * If these conditions are met, this function does any necessary
  411. * reference count dropping and cleans up the skbuff as if it
  412. * just came from __alloc_skb().
  413. */
  414. int skb_recycle_check(struct sk_buff *skb, int skb_size)
  415. {
  416. struct skb_shared_info *shinfo;
  417. if (skb_is_nonlinear(skb) || skb->fclone != SKB_FCLONE_UNAVAILABLE)
  418. return 0;
  419. skb_size = SKB_DATA_ALIGN(skb_size + NET_SKB_PAD);
  420. if (skb_end_pointer(skb) - skb->head < skb_size)
  421. return 0;
  422. if (skb_shared(skb) || skb_cloned(skb))
  423. return 0;
  424. skb_release_head_state(skb);
  425. shinfo = skb_shinfo(skb);
  426. atomic_set(&shinfo->dataref, 1);
  427. shinfo->nr_frags = 0;
  428. shinfo->gso_size = 0;
  429. shinfo->gso_segs = 0;
  430. shinfo->gso_type = 0;
  431. shinfo->ip6_frag_id = 0;
  432. shinfo->frag_list = NULL;
  433. memset(skb, 0, offsetof(struct sk_buff, tail));
  434. skb->data = skb->head + NET_SKB_PAD;
  435. skb_reset_tail_pointer(skb);
  436. return 1;
  437. }
  438. EXPORT_SYMBOL(skb_recycle_check);
  439. static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
  440. {
  441. new->tstamp = old->tstamp;
  442. new->dev = old->dev;
  443. new->transport_header = old->transport_header;
  444. new->network_header = old->network_header;
  445. new->mac_header = old->mac_header;
  446. new->dst = dst_clone(old->dst);
  447. #ifdef CONFIG_XFRM
  448. new->sp = secpath_get(old->sp);
  449. #endif
  450. memcpy(new->cb, old->cb, sizeof(old->cb));
  451. new->csum_start = old->csum_start;
  452. new->csum_offset = old->csum_offset;
  453. new->local_df = old->local_df;
  454. new->pkt_type = old->pkt_type;
  455. new->ip_summed = old->ip_summed;
  456. skb_copy_queue_mapping(new, old);
  457. new->priority = old->priority;
  458. #if defined(CONFIG_IP_VS) || defined(CONFIG_IP_VS_MODULE)
  459. new->ipvs_property = old->ipvs_property;
  460. #endif
  461. new->protocol = old->protocol;
  462. new->mark = old->mark;
  463. __nf_copy(new, old);
  464. #if defined(CONFIG_NETFILTER_XT_TARGET_TRACE) || \
  465. defined(CONFIG_NETFILTER_XT_TARGET_TRACE_MODULE)
  466. new->nf_trace = old->nf_trace;
  467. #endif
  468. #ifdef CONFIG_NET_SCHED
  469. new->tc_index = old->tc_index;
  470. #ifdef CONFIG_NET_CLS_ACT
  471. new->tc_verd = old->tc_verd;
  472. #endif
  473. #endif
  474. new->vlan_tci = old->vlan_tci;
  475. skb_copy_secmark(new, old);
  476. }
  477. static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
  478. {
  479. #define C(x) n->x = skb->x
  480. n->next = n->prev = NULL;
  481. n->sk = NULL;
  482. __copy_skb_header(n, skb);
  483. C(len);
  484. C(data_len);
  485. C(mac_len);
  486. n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
  487. n->cloned = 1;
  488. n->nohdr = 0;
  489. n->destructor = NULL;
  490. C(iif);
  491. C(tail);
  492. C(end);
  493. C(head);
  494. C(data);
  495. C(truesize);
  496. #if defined(CONFIG_MAC80211) || defined(CONFIG_MAC80211_MODULE)
  497. C(do_not_encrypt);
  498. C(requeue);
  499. #endif
  500. atomic_set(&n->users, 1);
  501. atomic_inc(&(skb_shinfo(skb)->dataref));
  502. skb->cloned = 1;
  503. return n;
  504. #undef C
  505. }
  506. /**
  507. * skb_morph - morph one skb into another
  508. * @dst: the skb to receive the contents
  509. * @src: the skb to supply the contents
  510. *
  511. * This is identical to skb_clone except that the target skb is
  512. * supplied by the user.
  513. *
  514. * The target skb is returned upon exit.
  515. */
  516. struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
  517. {
  518. skb_release_all(dst);
  519. return __skb_clone(dst, src);
  520. }
  521. EXPORT_SYMBOL_GPL(skb_morph);
  522. /**
  523. * skb_clone - duplicate an sk_buff
  524. * @skb: buffer to clone
  525. * @gfp_mask: allocation priority
  526. *
  527. * Duplicate an &sk_buff. The new one is not owned by a socket. Both
  528. * copies share the same packet data but not structure. The new
  529. * buffer has a reference count of 1. If the allocation fails the
  530. * function returns %NULL otherwise the new buffer is returned.
  531. *
  532. * If this function is called from an interrupt gfp_mask() must be
  533. * %GFP_ATOMIC.
  534. */
  535. struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
  536. {
  537. struct sk_buff *n;
  538. n = skb + 1;
  539. if (skb->fclone == SKB_FCLONE_ORIG &&
  540. n->fclone == SKB_FCLONE_UNAVAILABLE) {
  541. atomic_t *fclone_ref = (atomic_t *) (n + 1);
  542. n->fclone = SKB_FCLONE_CLONE;
  543. atomic_inc(fclone_ref);
  544. } else {
  545. n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
  546. if (!n)
  547. return NULL;
  548. n->fclone = SKB_FCLONE_UNAVAILABLE;
  549. }
  550. return __skb_clone(n, skb);
  551. }
  552. EXPORT_SYMBOL(skb_clone);
  553. static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
  554. {
  555. #ifndef NET_SKBUFF_DATA_USES_OFFSET
  556. /*
  557. * Shift between the two data areas in bytes
  558. */
  559. unsigned long offset = new->data - old->data;
  560. #endif
  561. __copy_skb_header(new, old);
  562. #ifndef NET_SKBUFF_DATA_USES_OFFSET
  563. /* {transport,network,mac}_header are relative to skb->head */
  564. new->transport_header += offset;
  565. new->network_header += offset;
  566. new->mac_header += offset;
  567. #endif
  568. skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
  569. skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
  570. skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
  571. }
  572. /**
  573. * skb_copy - create private copy of an sk_buff
  574. * @skb: buffer to copy
  575. * @gfp_mask: allocation priority
  576. *
  577. * Make a copy of both an &sk_buff and its data. This is used when the
  578. * caller wishes to modify the data and needs a private copy of the
  579. * data to alter. Returns %NULL on failure or the pointer to the buffer
  580. * on success. The returned buffer has a reference count of 1.
  581. *
  582. * As by-product this function converts non-linear &sk_buff to linear
  583. * one, so that &sk_buff becomes completely private and caller is allowed
  584. * to modify all the data of returned buffer. This means that this
  585. * function is not recommended for use in circumstances when only
  586. * header is going to be modified. Use pskb_copy() instead.
  587. */
  588. struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
  589. {
  590. int headerlen = skb->data - skb->head;
  591. /*
  592. * Allocate the copy buffer
  593. */
  594. struct sk_buff *n;
  595. #ifdef NET_SKBUFF_DATA_USES_OFFSET
  596. n = alloc_skb(skb->end + skb->data_len, gfp_mask);
  597. #else
  598. n = alloc_skb(skb->end - skb->head + skb->data_len, gfp_mask);
  599. #endif
  600. if (!n)
  601. return NULL;
  602. /* Set the data pointer */
  603. skb_reserve(n, headerlen);
  604. /* Set the tail pointer and length */
  605. skb_put(n, skb->len);
  606. if (skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len))
  607. BUG();
  608. copy_skb_header(n, skb);
  609. return n;
  610. }
  611. EXPORT_SYMBOL(skb_copy);
  612. /**
  613. * pskb_copy - create copy of an sk_buff with private head.
  614. * @skb: buffer to copy
  615. * @gfp_mask: allocation priority
  616. *
  617. * Make a copy of both an &sk_buff and part of its data, located
  618. * in header. Fragmented data remain shared. This is used when
  619. * the caller wishes to modify only header of &sk_buff and needs
  620. * private copy of the header to alter. Returns %NULL on failure
  621. * or the pointer to the buffer on success.
  622. * The returned buffer has a reference count of 1.
  623. */
  624. struct sk_buff *pskb_copy(struct sk_buff *skb, gfp_t gfp_mask)
  625. {
  626. /*
  627. * Allocate the copy buffer
  628. */
  629. struct sk_buff *n;
  630. #ifdef NET_SKBUFF_DATA_USES_OFFSET
  631. n = alloc_skb(skb->end, gfp_mask);
  632. #else
  633. n = alloc_skb(skb->end - skb->head, gfp_mask);
  634. #endif
  635. if (!n)
  636. goto out;
  637. /* Set the data pointer */
  638. skb_reserve(n, skb->data - skb->head);
  639. /* Set the tail pointer and length */
  640. skb_put(n, skb_headlen(skb));
  641. /* Copy the bytes */
  642. skb_copy_from_linear_data(skb, n->data, n->len);
  643. n->truesize += skb->data_len;
  644. n->data_len = skb->data_len;
  645. n->len = skb->len;
  646. if (skb_shinfo(skb)->nr_frags) {
  647. int i;
  648. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  649. skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
  650. get_page(skb_shinfo(n)->frags[i].page);
  651. }
  652. skb_shinfo(n)->nr_frags = i;
  653. }
  654. if (skb_shinfo(skb)->frag_list) {
  655. skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
  656. skb_clone_fraglist(n);
  657. }
  658. copy_skb_header(n, skb);
  659. out:
  660. return n;
  661. }
  662. EXPORT_SYMBOL(pskb_copy);
  663. /**
  664. * pskb_expand_head - reallocate header of &sk_buff
  665. * @skb: buffer to reallocate
  666. * @nhead: room to add at head
  667. * @ntail: room to add at tail
  668. * @gfp_mask: allocation priority
  669. *
  670. * Expands (or creates identical copy, if &nhead and &ntail are zero)
  671. * header of skb. &sk_buff itself is not changed. &sk_buff MUST have
  672. * reference count of 1. Returns zero in the case of success or error,
  673. * if expansion failed. In the last case, &sk_buff is not changed.
  674. *
  675. * All the pointers pointing into skb header may change and must be
  676. * reloaded after call to this function.
  677. */
  678. int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
  679. gfp_t gfp_mask)
  680. {
  681. int i;
  682. u8 *data;
  683. #ifdef NET_SKBUFF_DATA_USES_OFFSET
  684. int size = nhead + skb->end + ntail;
  685. #else
  686. int size = nhead + (skb->end - skb->head) + ntail;
  687. #endif
  688. long off;
  689. BUG_ON(nhead < 0);
  690. if (skb_shared(skb))
  691. BUG();
  692. size = SKB_DATA_ALIGN(size);
  693. data = kmalloc(size + sizeof(struct skb_shared_info), gfp_mask);
  694. if (!data)
  695. goto nodata;
  696. /* Copy only real data... and, alas, header. This should be
  697. * optimized for the cases when header is void. */
  698. #ifdef NET_SKBUFF_DATA_USES_OFFSET
  699. memcpy(data + nhead, skb->head, skb->tail);
  700. #else
  701. memcpy(data + nhead, skb->head, skb->tail - skb->head);
  702. #endif
  703. memcpy(data + size, skb_end_pointer(skb),
  704. sizeof(struct skb_shared_info));
  705. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
  706. get_page(skb_shinfo(skb)->frags[i].page);
  707. if (skb_shinfo(skb)->frag_list)
  708. skb_clone_fraglist(skb);
  709. skb_release_data(skb);
  710. off = (data + nhead) - skb->head;
  711. skb->head = data;
  712. skb->data += off;
  713. #ifdef NET_SKBUFF_DATA_USES_OFFSET
  714. skb->end = size;
  715. off = nhead;
  716. #else
  717. skb->end = skb->head + size;
  718. #endif
  719. /* {transport,network,mac}_header and tail are relative to skb->head */
  720. skb->tail += off;
  721. skb->transport_header += off;
  722. skb->network_header += off;
  723. skb->mac_header += off;
  724. skb->csum_start += nhead;
  725. skb->cloned = 0;
  726. skb->hdr_len = 0;
  727. skb->nohdr = 0;
  728. atomic_set(&skb_shinfo(skb)->dataref, 1);
  729. return 0;
  730. nodata:
  731. return -ENOMEM;
  732. }
  733. EXPORT_SYMBOL(pskb_expand_head);
  734. /* Make private copy of skb with writable head and some headroom */
  735. struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
  736. {
  737. struct sk_buff *skb2;
  738. int delta = headroom - skb_headroom(skb);
  739. if (delta <= 0)
  740. skb2 = pskb_copy(skb, GFP_ATOMIC);
  741. else {
  742. skb2 = skb_clone(skb, GFP_ATOMIC);
  743. if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
  744. GFP_ATOMIC)) {
  745. kfree_skb(skb2);
  746. skb2 = NULL;
  747. }
  748. }
  749. return skb2;
  750. }
  751. EXPORT_SYMBOL(skb_realloc_headroom);
  752. /**
  753. * skb_copy_expand - copy and expand sk_buff
  754. * @skb: buffer to copy
  755. * @newheadroom: new free bytes at head
  756. * @newtailroom: new free bytes at tail
  757. * @gfp_mask: allocation priority
  758. *
  759. * Make a copy of both an &sk_buff and its data and while doing so
  760. * allocate additional space.
  761. *
  762. * This is used when the caller wishes to modify the data and needs a
  763. * private copy of the data to alter as well as more space for new fields.
  764. * Returns %NULL on failure or the pointer to the buffer
  765. * on success. The returned buffer has a reference count of 1.
  766. *
  767. * You must pass %GFP_ATOMIC as the allocation priority if this function
  768. * is called from an interrupt.
  769. */
  770. struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
  771. int newheadroom, int newtailroom,
  772. gfp_t gfp_mask)
  773. {
  774. /*
  775. * Allocate the copy buffer
  776. */
  777. struct sk_buff *n = alloc_skb(newheadroom + skb->len + newtailroom,
  778. gfp_mask);
  779. int oldheadroom = skb_headroom(skb);
  780. int head_copy_len, head_copy_off;
  781. int off;
  782. if (!n)
  783. return NULL;
  784. skb_reserve(n, newheadroom);
  785. /* Set the tail pointer and length */
  786. skb_put(n, skb->len);
  787. head_copy_len = oldheadroom;
  788. head_copy_off = 0;
  789. if (newheadroom <= head_copy_len)
  790. head_copy_len = newheadroom;
  791. else
  792. head_copy_off = newheadroom - head_copy_len;
  793. /* Copy the linear header and data. */
  794. if (skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
  795. skb->len + head_copy_len))
  796. BUG();
  797. copy_skb_header(n, skb);
  798. off = newheadroom - oldheadroom;
  799. n->csum_start += off;
  800. #ifdef NET_SKBUFF_DATA_USES_OFFSET
  801. n->transport_header += off;
  802. n->network_header += off;
  803. n->mac_header += off;
  804. #endif
  805. return n;
  806. }
  807. EXPORT_SYMBOL(skb_copy_expand);
  808. /**
  809. * skb_pad - zero pad the tail of an skb
  810. * @skb: buffer to pad
  811. * @pad: space to pad
  812. *
  813. * Ensure that a buffer is followed by a padding area that is zero
  814. * filled. Used by network drivers which may DMA or transfer data
  815. * beyond the buffer end onto the wire.
  816. *
  817. * May return error in out of memory cases. The skb is freed on error.
  818. */
  819. int skb_pad(struct sk_buff *skb, int pad)
  820. {
  821. int err;
  822. int ntail;
  823. /* If the skbuff is non linear tailroom is always zero.. */
  824. if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
  825. memset(skb->data+skb->len, 0, pad);
  826. return 0;
  827. }
  828. ntail = skb->data_len + pad - (skb->end - skb->tail);
  829. if (likely(skb_cloned(skb) || ntail > 0)) {
  830. err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
  831. if (unlikely(err))
  832. goto free_skb;
  833. }
  834. /* FIXME: The use of this function with non-linear skb's really needs
  835. * to be audited.
  836. */
  837. err = skb_linearize(skb);
  838. if (unlikely(err))
  839. goto free_skb;
  840. memset(skb->data + skb->len, 0, pad);
  841. return 0;
  842. free_skb:
  843. kfree_skb(skb);
  844. return err;
  845. }
  846. EXPORT_SYMBOL(skb_pad);
  847. /**
  848. * skb_put - add data to a buffer
  849. * @skb: buffer to use
  850. * @len: amount of data to add
  851. *
  852. * This function extends the used data area of the buffer. If this would
  853. * exceed the total buffer size the kernel will panic. A pointer to the
  854. * first byte of the extra data is returned.
  855. */
  856. unsigned char *skb_put(struct sk_buff *skb, unsigned int len)
  857. {
  858. unsigned char *tmp = skb_tail_pointer(skb);
  859. SKB_LINEAR_ASSERT(skb);
  860. skb->tail += len;
  861. skb->len += len;
  862. if (unlikely(skb->tail > skb->end))
  863. skb_over_panic(skb, len, __builtin_return_address(0));
  864. return tmp;
  865. }
  866. EXPORT_SYMBOL(skb_put);
  867. /**
  868. * skb_push - add data to the start of a buffer
  869. * @skb: buffer to use
  870. * @len: amount of data to add
  871. *
  872. * This function extends the used data area of the buffer at the buffer
  873. * start. If this would exceed the total buffer headroom the kernel will
  874. * panic. A pointer to the first byte of the extra data is returned.
  875. */
  876. unsigned char *skb_push(struct sk_buff *skb, unsigned int len)
  877. {
  878. skb->data -= len;
  879. skb->len += len;
  880. if (unlikely(skb->data<skb->head))
  881. skb_under_panic(skb, len, __builtin_return_address(0));
  882. return skb->data;
  883. }
  884. EXPORT_SYMBOL(skb_push);
  885. /**
  886. * skb_pull - remove data from the start of a buffer
  887. * @skb: buffer to use
  888. * @len: amount of data to remove
  889. *
  890. * This function removes data from the start of a buffer, returning
  891. * the memory to the headroom. A pointer to the next data in the buffer
  892. * is returned. Once the data has been pulled future pushes will overwrite
  893. * the old data.
  894. */
  895. unsigned char *skb_pull(struct sk_buff *skb, unsigned int len)
  896. {
  897. return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len);
  898. }
  899. EXPORT_SYMBOL(skb_pull);
  900. /**
  901. * skb_trim - remove end from a buffer
  902. * @skb: buffer to alter
  903. * @len: new length
  904. *
  905. * Cut the length of a buffer down by removing data from the tail. If
  906. * the buffer is already under the length specified it is not modified.
  907. * The skb must be linear.
  908. */
  909. void skb_trim(struct sk_buff *skb, unsigned int len)
  910. {
  911. if (skb->len > len)
  912. __skb_trim(skb, len);
  913. }
  914. EXPORT_SYMBOL(skb_trim);
  915. /* Trims skb to length len. It can change skb pointers.
  916. */
  917. int ___pskb_trim(struct sk_buff *skb, unsigned int len)
  918. {
  919. struct sk_buff **fragp;
  920. struct sk_buff *frag;
  921. int offset = skb_headlen(skb);
  922. int nfrags = skb_shinfo(skb)->nr_frags;
  923. int i;
  924. int err;
  925. if (skb_cloned(skb) &&
  926. unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
  927. return err;
  928. i = 0;
  929. if (offset >= len)
  930. goto drop_pages;
  931. for (; i < nfrags; i++) {
  932. int end = offset + skb_shinfo(skb)->frags[i].size;
  933. if (end < len) {
  934. offset = end;
  935. continue;
  936. }
  937. skb_shinfo(skb)->frags[i++].size = len - offset;
  938. drop_pages:
  939. skb_shinfo(skb)->nr_frags = i;
  940. for (; i < nfrags; i++)
  941. put_page(skb_shinfo(skb)->frags[i].page);
  942. if (skb_shinfo(skb)->frag_list)
  943. skb_drop_fraglist(skb);
  944. goto done;
  945. }
  946. for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
  947. fragp = &frag->next) {
  948. int end = offset + frag->len;
  949. if (skb_shared(frag)) {
  950. struct sk_buff *nfrag;
  951. nfrag = skb_clone(frag, GFP_ATOMIC);
  952. if (unlikely(!nfrag))
  953. return -ENOMEM;
  954. nfrag->next = frag->next;
  955. kfree_skb(frag);
  956. frag = nfrag;
  957. *fragp = frag;
  958. }
  959. if (end < len) {
  960. offset = end;
  961. continue;
  962. }
  963. if (end > len &&
  964. unlikely((err = pskb_trim(frag, len - offset))))
  965. return err;
  966. if (frag->next)
  967. skb_drop_list(&frag->next);
  968. break;
  969. }
  970. done:
  971. if (len > skb_headlen(skb)) {
  972. skb->data_len -= skb->len - len;
  973. skb->len = len;
  974. } else {
  975. skb->len = len;
  976. skb->data_len = 0;
  977. skb_set_tail_pointer(skb, len);
  978. }
  979. return 0;
  980. }
  981. EXPORT_SYMBOL(___pskb_trim);
  982. /**
  983. * __pskb_pull_tail - advance tail of skb header
  984. * @skb: buffer to reallocate
  985. * @delta: number of bytes to advance tail
  986. *
  987. * The function makes a sense only on a fragmented &sk_buff,
  988. * it expands header moving its tail forward and copying necessary
  989. * data from fragmented part.
  990. *
  991. * &sk_buff MUST have reference count of 1.
  992. *
  993. * Returns %NULL (and &sk_buff does not change) if pull failed
  994. * or value of new tail of skb in the case of success.
  995. *
  996. * All the pointers pointing into skb header may change and must be
  997. * reloaded after call to this function.
  998. */
  999. /* Moves tail of skb head forward, copying data from fragmented part,
  1000. * when it is necessary.
  1001. * 1. It may fail due to malloc failure.
  1002. * 2. It may change skb pointers.
  1003. *
  1004. * It is pretty complicated. Luckily, it is called only in exceptional cases.
  1005. */
  1006. unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta)
  1007. {
  1008. /* If skb has not enough free space at tail, get new one
  1009. * plus 128 bytes for future expansions. If we have enough
  1010. * room at tail, reallocate without expansion only if skb is cloned.
  1011. */
  1012. int i, k, eat = (skb->tail + delta) - skb->end;
  1013. if (eat > 0 || skb_cloned(skb)) {
  1014. if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
  1015. GFP_ATOMIC))
  1016. return NULL;
  1017. }
  1018. if (skb_copy_bits(skb, skb_headlen(skb), skb_tail_pointer(skb), delta))
  1019. BUG();
  1020. /* Optimization: no fragments, no reasons to preestimate
  1021. * size of pulled pages. Superb.
  1022. */
  1023. if (!skb_shinfo(skb)->frag_list)
  1024. goto pull_pages;
  1025. /* Estimate size of pulled pages. */
  1026. eat = delta;
  1027. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  1028. if (skb_shinfo(skb)->frags[i].size >= eat)
  1029. goto pull_pages;
  1030. eat -= skb_shinfo(skb)->frags[i].size;
  1031. }
  1032. /* If we need update frag list, we are in troubles.
  1033. * Certainly, it possible to add an offset to skb data,
  1034. * but taking into account that pulling is expected to
  1035. * be very rare operation, it is worth to fight against
  1036. * further bloating skb head and crucify ourselves here instead.
  1037. * Pure masohism, indeed. 8)8)
  1038. */
  1039. if (eat) {
  1040. struct sk_buff *list = skb_shinfo(skb)->frag_list;
  1041. struct sk_buff *clone = NULL;
  1042. struct sk_buff *insp = NULL;
  1043. do {
  1044. BUG_ON(!list);
  1045. if (list->len <= eat) {
  1046. /* Eaten as whole. */
  1047. eat -= list->len;
  1048. list = list->next;
  1049. insp = list;
  1050. } else {
  1051. /* Eaten partially. */
  1052. if (skb_shared(list)) {
  1053. /* Sucks! We need to fork list. :-( */
  1054. clone = skb_clone(list, GFP_ATOMIC);
  1055. if (!clone)
  1056. return NULL;
  1057. insp = list->next;
  1058. list = clone;
  1059. } else {
  1060. /* This may be pulled without
  1061. * problems. */
  1062. insp = list;
  1063. }
  1064. if (!pskb_pull(list, eat)) {
  1065. if (clone)
  1066. kfree_skb(clone);
  1067. return NULL;
  1068. }
  1069. break;
  1070. }
  1071. } while (eat);
  1072. /* Free pulled out fragments. */
  1073. while ((list = skb_shinfo(skb)->frag_list) != insp) {
  1074. skb_shinfo(skb)->frag_list = list->next;
  1075. kfree_skb(list);
  1076. }
  1077. /* And insert new clone at head. */
  1078. if (clone) {
  1079. clone->next = list;
  1080. skb_shinfo(skb)->frag_list = clone;
  1081. }
  1082. }
  1083. /* Success! Now we may commit changes to skb data. */
  1084. pull_pages:
  1085. eat = delta;
  1086. k = 0;
  1087. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  1088. if (skb_shinfo(skb)->frags[i].size <= eat) {
  1089. put_page(skb_shinfo(skb)->frags[i].page);
  1090. eat -= skb_shinfo(skb)->frags[i].size;
  1091. } else {
  1092. skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];
  1093. if (eat) {
  1094. skb_shinfo(skb)->frags[k].page_offset += eat;
  1095. skb_shinfo(skb)->frags[k].size -= eat;
  1096. eat = 0;
  1097. }
  1098. k++;
  1099. }
  1100. }
  1101. skb_shinfo(skb)->nr_frags = k;
  1102. skb->tail += delta;
  1103. skb->data_len -= delta;
  1104. return skb_tail_pointer(skb);
  1105. }
  1106. EXPORT_SYMBOL(__pskb_pull_tail);
  1107. /* Copy some data bits from skb to kernel buffer. */
  1108. int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
  1109. {
  1110. int i, copy;
  1111. int start = skb_headlen(skb);
  1112. if (offset > (int)skb->len - len)
  1113. goto fault;
  1114. /* Copy header. */
  1115. if ((copy = start - offset) > 0) {
  1116. if (copy > len)
  1117. copy = len;
  1118. skb_copy_from_linear_data_offset(skb, offset, to, copy);
  1119. if ((len -= copy) == 0)
  1120. return 0;
  1121. offset += copy;
  1122. to += copy;
  1123. }
  1124. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  1125. int end;
  1126. WARN_ON(start > offset + len);
  1127. end = start + skb_shinfo(skb)->frags[i].size;
  1128. if ((copy = end - offset) > 0) {
  1129. u8 *vaddr;
  1130. if (copy > len)
  1131. copy = len;
  1132. vaddr = kmap_skb_frag(&skb_shinfo(skb)->frags[i]);
  1133. memcpy(to,
  1134. vaddr + skb_shinfo(skb)->frags[i].page_offset+
  1135. offset - start, copy);
  1136. kunmap_skb_frag(vaddr);
  1137. if ((len -= copy) == 0)
  1138. return 0;
  1139. offset += copy;
  1140. to += copy;
  1141. }
  1142. start = end;
  1143. }
  1144. if (skb_shinfo(skb)->frag_list) {
  1145. struct sk_buff *list = skb_shinfo(skb)->frag_list;
  1146. for (; list; list = list->next) {
  1147. int end;
  1148. WARN_ON(start > offset + len);
  1149. end = start + list->len;
  1150. if ((copy = end - offset) > 0) {
  1151. if (copy > len)
  1152. copy = len;
  1153. if (skb_copy_bits(list, offset - start,
  1154. to, copy))
  1155. goto fault;
  1156. if ((len -= copy) == 0)
  1157. return 0;
  1158. offset += copy;
  1159. to += copy;
  1160. }
  1161. start = end;
  1162. }
  1163. }
  1164. if (!len)
  1165. return 0;
  1166. fault:
  1167. return -EFAULT;
  1168. }
  1169. EXPORT_SYMBOL(skb_copy_bits);
  1170. /*
  1171. * Callback from splice_to_pipe(), if we need to release some pages
  1172. * at the end of the spd in case we error'ed out in filling the pipe.
  1173. */
  1174. static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
  1175. {
  1176. put_page(spd->pages[i]);
  1177. }
  1178. static inline struct page *linear_to_page(struct page *page, unsigned int *len,
  1179. unsigned int *offset,
  1180. struct sk_buff *skb)
  1181. {
  1182. struct sock *sk = skb->sk;
  1183. struct page *p = sk->sk_sndmsg_page;
  1184. unsigned int off;
  1185. if (!p) {
  1186. new_page:
  1187. p = sk->sk_sndmsg_page = alloc_pages(sk->sk_allocation, 0);
  1188. if (!p)
  1189. return NULL;
  1190. off = sk->sk_sndmsg_off = 0;
  1191. /* hold one ref to this page until it's full */
  1192. } else {
  1193. unsigned int mlen;
  1194. off = sk->sk_sndmsg_off;
  1195. mlen = PAGE_SIZE - off;
  1196. if (mlen < 64 && mlen < *len) {
  1197. put_page(p);
  1198. goto new_page;
  1199. }
  1200. *len = min_t(unsigned int, *len, mlen);
  1201. }
  1202. memcpy(page_address(p) + off, page_address(page) + *offset, *len);
  1203. sk->sk_sndmsg_off += *len;
  1204. *offset = off;
  1205. get_page(p);
  1206. return p;
  1207. }
  1208. /*
  1209. * Fill page/offset/length into spd, if it can hold more pages.
  1210. */
  1211. static inline int spd_fill_page(struct splice_pipe_desc *spd, struct page *page,
  1212. unsigned int *len, unsigned int offset,
  1213. struct sk_buff *skb, int linear)
  1214. {
  1215. if (unlikely(spd->nr_pages == PIPE_BUFFERS))
  1216. return 1;
  1217. if (linear) {
  1218. page = linear_to_page(page, len, &offset, skb);
  1219. if (!page)
  1220. return 1;
  1221. } else
  1222. get_page(page);
  1223. spd->pages[spd->nr_pages] = page;
  1224. spd->partial[spd->nr_pages].len = *len;
  1225. spd->partial[spd->nr_pages].offset = offset;
  1226. spd->nr_pages++;
  1227. return 0;
  1228. }
  1229. static inline void __segment_seek(struct page **page, unsigned int *poff,
  1230. unsigned int *plen, unsigned int off)
  1231. {
  1232. *poff += off;
  1233. *page += *poff / PAGE_SIZE;
  1234. *poff = *poff % PAGE_SIZE;
  1235. *plen -= off;
  1236. }
  1237. static inline int __splice_segment(struct page *page, unsigned int poff,
  1238. unsigned int plen, unsigned int *off,
  1239. unsigned int *len, struct sk_buff *skb,
  1240. struct splice_pipe_desc *spd, int linear)
  1241. {
  1242. if (!*len)
  1243. return 1;
  1244. /* skip this segment if already processed */
  1245. if (*off >= plen) {
  1246. *off -= plen;
  1247. return 0;
  1248. }
  1249. /* ignore any bits we already processed */
  1250. if (*off) {
  1251. __segment_seek(&page, &poff, &plen, *off);
  1252. *off = 0;
  1253. }
  1254. do {
  1255. unsigned int flen = min(*len, plen);
  1256. /* the linear region may spread across several pages */
  1257. flen = min_t(unsigned int, flen, PAGE_SIZE - poff);
  1258. if (spd_fill_page(spd, page, &flen, poff, skb, linear))
  1259. return 1;
  1260. __segment_seek(&page, &poff, &plen, flen);
  1261. *len -= flen;
  1262. } while (*len && plen);
  1263. return 0;
  1264. }
  1265. /*
  1266. * Map linear and fragment data from the skb to spd. It reports failure if the
  1267. * pipe is full or if we already spliced the requested length.
  1268. */
  1269. static int __skb_splice_bits(struct sk_buff *skb, unsigned int *offset,
  1270. unsigned int *len,
  1271. struct splice_pipe_desc *spd)
  1272. {
  1273. int seg;
  1274. /*
  1275. * map the linear part
  1276. */
  1277. if (__splice_segment(virt_to_page(skb->data),
  1278. (unsigned long) skb->data & (PAGE_SIZE - 1),
  1279. skb_headlen(skb),
  1280. offset, len, skb, spd, 1))
  1281. return 1;
  1282. /*
  1283. * then map the fragments
  1284. */
  1285. for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
  1286. const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
  1287. if (__splice_segment(f->page, f->page_offset, f->size,
  1288. offset, len, skb, spd, 0))
  1289. return 1;
  1290. }
  1291. return 0;
  1292. }
  1293. /*
  1294. * Map data from the skb to a pipe. Should handle both the linear part,
  1295. * the fragments, and the frag list. It does NOT handle frag lists within
  1296. * the frag list, if such a thing exists. We'd probably need to recurse to
  1297. * handle that cleanly.
  1298. */
  1299. int skb_splice_bits(struct sk_buff *skb, unsigned int offset,
  1300. struct pipe_inode_info *pipe, unsigned int tlen,
  1301. unsigned int flags)
  1302. {
  1303. struct partial_page partial[PIPE_BUFFERS];
  1304. struct page *pages[PIPE_BUFFERS];
  1305. struct splice_pipe_desc spd = {
  1306. .pages = pages,
  1307. .partial = partial,
  1308. .flags = flags,
  1309. .ops = &sock_pipe_buf_ops,
  1310. .spd_release = sock_spd_release,
  1311. };
  1312. /*
  1313. * __skb_splice_bits() only fails if the output has no room left,
  1314. * so no point in going over the frag_list for the error case.
  1315. */
  1316. if (__skb_splice_bits(skb, &offset, &tlen, &spd))
  1317. goto done;
  1318. else if (!tlen)
  1319. goto done;
  1320. /*
  1321. * now see if we have a frag_list to map
  1322. */
  1323. if (skb_shinfo(skb)->frag_list) {
  1324. struct sk_buff *list = skb_shinfo(skb)->frag_list;
  1325. for (; list && tlen; list = list->next) {
  1326. if (__skb_splice_bits(list, &offset, &tlen, &spd))
  1327. break;
  1328. }
  1329. }
  1330. done:
  1331. if (spd.nr_pages) {
  1332. struct sock *sk = skb->sk;
  1333. int ret;
  1334. /*
  1335. * Drop the socket lock, otherwise we have reverse
  1336. * locking dependencies between sk_lock and i_mutex
  1337. * here as compared to sendfile(). We enter here
  1338. * with the socket lock held, and splice_to_pipe() will
  1339. * grab the pipe inode lock. For sendfile() emulation,
  1340. * we call into ->sendpage() with the i_mutex lock held
  1341. * and networking will grab the socket lock.
  1342. */
  1343. release_sock(sk);
  1344. ret = splice_to_pipe(pipe, &spd);
  1345. lock_sock(sk);
  1346. return ret;
  1347. }
  1348. return 0;
  1349. }
  1350. /**
  1351. * skb_store_bits - store bits from kernel buffer to skb
  1352. * @skb: destination buffer
  1353. * @offset: offset in destination
  1354. * @from: source buffer
  1355. * @len: number of bytes to copy
  1356. *
  1357. * Copy the specified number of bytes from the source buffer to the
  1358. * destination skb. This function handles all the messy bits of
  1359. * traversing fragment lists and such.
  1360. */
  1361. int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
  1362. {
  1363. int i, copy;
  1364. int start = skb_headlen(skb);
  1365. if (offset > (int)skb->len - len)
  1366. goto fault;
  1367. if ((copy = start - offset) > 0) {
  1368. if (copy > len)
  1369. copy = len;
  1370. skb_copy_to_linear_data_offset(skb, offset, from, copy);
  1371. if ((len -= copy) == 0)
  1372. return 0;
  1373. offset += copy;
  1374. from += copy;
  1375. }
  1376. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  1377. skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
  1378. int end;
  1379. WARN_ON(start > offset + len);
  1380. end = start + frag->size;
  1381. if ((copy = end - offset) > 0) {
  1382. u8 *vaddr;
  1383. if (copy > len)
  1384. copy = len;
  1385. vaddr = kmap_skb_frag(frag);
  1386. memcpy(vaddr + frag->page_offset + offset - start,
  1387. from, copy);
  1388. kunmap_skb_frag(vaddr);
  1389. if ((len -= copy) == 0)
  1390. return 0;
  1391. offset += copy;
  1392. from += copy;
  1393. }
  1394. start = end;
  1395. }
  1396. if (skb_shinfo(skb)->frag_list) {
  1397. struct sk_buff *list = skb_shinfo(skb)->frag_list;
  1398. for (; list; list = list->next) {
  1399. int end;
  1400. WARN_ON(start > offset + len);
  1401. end = start + list->len;
  1402. if ((copy = end - offset) > 0) {
  1403. if (copy > len)
  1404. copy = len;
  1405. if (skb_store_bits(list, offset - start,
  1406. from, copy))
  1407. goto fault;
  1408. if ((len -= copy) == 0)
  1409. return 0;
  1410. offset += copy;
  1411. from += copy;
  1412. }
  1413. start = end;
  1414. }
  1415. }
  1416. if (!len)
  1417. return 0;
  1418. fault:
  1419. return -EFAULT;
  1420. }
  1421. EXPORT_SYMBOL(skb_store_bits);
  1422. /* Checksum skb data. */
  1423. __wsum skb_checksum(const struct sk_buff *skb, int offset,
  1424. int len, __wsum csum)
  1425. {
  1426. int start = skb_headlen(skb);
  1427. int i, copy = start - offset;
  1428. int pos = 0;
  1429. /* Checksum header. */
  1430. if (copy > 0) {
  1431. if (copy > len)
  1432. copy = len;
  1433. csum = csum_partial(skb->data + offset, copy, csum);
  1434. if ((len -= copy) == 0)
  1435. return csum;
  1436. offset += copy;
  1437. pos = copy;
  1438. }
  1439. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  1440. int end;
  1441. WARN_ON(start > offset + len);
  1442. end = start + skb_shinfo(skb)->frags[i].size;
  1443. if ((copy = end - offset) > 0) {
  1444. __wsum csum2;
  1445. u8 *vaddr;
  1446. skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
  1447. if (copy > len)
  1448. copy = len;
  1449. vaddr = kmap_skb_frag(frag);
  1450. csum2 = csum_partial(vaddr + frag->page_offset +
  1451. offset - start, copy, 0);
  1452. kunmap_skb_frag(vaddr);
  1453. csum = csum_block_add(csum, csum2, pos);
  1454. if (!(len -= copy))
  1455. return csum;
  1456. offset += copy;
  1457. pos += copy;
  1458. }
  1459. start = end;
  1460. }
  1461. if (skb_shinfo(skb)->frag_list) {
  1462. struct sk_buff *list = skb_shinfo(skb)->frag_list;
  1463. for (; list; list = list->next) {
  1464. int end;
  1465. WARN_ON(start > offset + len);
  1466. end = start + list->len;
  1467. if ((copy = end - offset) > 0) {
  1468. __wsum csum2;
  1469. if (copy > len)
  1470. copy = len;
  1471. csum2 = skb_checksum(list, offset - start,
  1472. copy, 0);
  1473. csum = csum_block_add(csum, csum2, pos);
  1474. if ((len -= copy) == 0)
  1475. return csum;
  1476. offset += copy;
  1477. pos += copy;
  1478. }
  1479. start = end;
  1480. }
  1481. }
  1482. BUG_ON(len);
  1483. return csum;
  1484. }
  1485. EXPORT_SYMBOL(skb_checksum);
  1486. /* Both of above in one bottle. */
  1487. __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
  1488. u8 *to, int len, __wsum csum)
  1489. {
  1490. int start = skb_headlen(skb);
  1491. int i, copy = start - offset;
  1492. int pos = 0;
  1493. /* Copy header. */
  1494. if (copy > 0) {
  1495. if (copy > len)
  1496. copy = len;
  1497. csum = csum_partial_copy_nocheck(skb->data + offset, to,
  1498. copy, csum);
  1499. if ((len -= copy) == 0)
  1500. return csum;
  1501. offset += copy;
  1502. to += copy;
  1503. pos = copy;
  1504. }
  1505. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  1506. int end;
  1507. WARN_ON(start > offset + len);
  1508. end = start + skb_shinfo(skb)->frags[i].size;
  1509. if ((copy = end - offset) > 0) {
  1510. __wsum csum2;
  1511. u8 *vaddr;
  1512. skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
  1513. if (copy > len)
  1514. copy = len;
  1515. vaddr = kmap_skb_frag(frag);
  1516. csum2 = csum_partial_copy_nocheck(vaddr +
  1517. frag->page_offset +
  1518. offset - start, to,
  1519. copy, 0);
  1520. kunmap_skb_frag(vaddr);
  1521. csum = csum_block_add(csum, csum2, pos);
  1522. if (!(len -= copy))
  1523. return csum;
  1524. offset += copy;
  1525. to += copy;
  1526. pos += copy;
  1527. }
  1528. start = end;
  1529. }
  1530. if (skb_shinfo(skb)->frag_list) {
  1531. struct sk_buff *list = skb_shinfo(skb)->frag_list;
  1532. for (; list; list = list->next) {
  1533. __wsum csum2;
  1534. int end;
  1535. WARN_ON(start > offset + len);
  1536. end = start + list->len;
  1537. if ((copy = end - offset) > 0) {
  1538. if (copy > len)
  1539. copy = len;
  1540. csum2 = skb_copy_and_csum_bits(list,
  1541. offset - start,
  1542. to, copy, 0);
  1543. csum = csum_block_add(csum, csum2, pos);
  1544. if ((len -= copy) == 0)
  1545. return csum;
  1546. offset += copy;
  1547. to += copy;
  1548. pos += copy;
  1549. }
  1550. start = end;
  1551. }
  1552. }
  1553. BUG_ON(len);
  1554. return csum;
  1555. }
  1556. EXPORT_SYMBOL(skb_copy_and_csum_bits);
  1557. void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
  1558. {
  1559. __wsum csum;
  1560. long csstart;
  1561. if (skb->ip_summed == CHECKSUM_PARTIAL)
  1562. csstart = skb->csum_start - skb_headroom(skb);
  1563. else
  1564. csstart = skb_headlen(skb);
  1565. BUG_ON(csstart > skb_headlen(skb));
  1566. skb_copy_from_linear_data(skb, to, csstart);
  1567. csum = 0;
  1568. if (csstart != skb->len)
  1569. csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
  1570. skb->len - csstart, 0);
  1571. if (skb->ip_summed == CHECKSUM_PARTIAL) {
  1572. long csstuff = csstart + skb->csum_offset;
  1573. *((__sum16 *)(to + csstuff)) = csum_fold(csum);
  1574. }
  1575. }
  1576. EXPORT_SYMBOL(skb_copy_and_csum_dev);
  1577. /**
  1578. * skb_dequeue - remove from the head of the queue
  1579. * @list: list to dequeue from
  1580. *
  1581. * Remove the head of the list. The list lock is taken so the function
  1582. * may be used safely with other locking list functions. The head item is
  1583. * returned or %NULL if the list is empty.
  1584. */
  1585. struct sk_buff *skb_dequeue(struct sk_buff_head *list)
  1586. {
  1587. unsigned long flags;
  1588. struct sk_buff *result;
  1589. spin_lock_irqsave(&list->lock, flags);
  1590. result = __skb_dequeue(list);
  1591. spin_unlock_irqrestore(&list->lock, flags);
  1592. return result;
  1593. }
  1594. EXPORT_SYMBOL(skb_dequeue);
  1595. /**
  1596. * skb_dequeue_tail - remove from the tail of the queue
  1597. * @list: list to dequeue from
  1598. *
  1599. * Remove the tail of the list. The list lock is taken so the function
  1600. * may be used safely with other locking list functions. The tail item is
  1601. * returned or %NULL if the list is empty.
  1602. */
  1603. struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
  1604. {
  1605. unsigned long flags;
  1606. struct sk_buff *result;
  1607. spin_lock_irqsave(&list->lock, flags);
  1608. result = __skb_dequeue_tail(list);
  1609. spin_unlock_irqrestore(&list->lock, flags);
  1610. return result;
  1611. }
  1612. EXPORT_SYMBOL(skb_dequeue_tail);
  1613. /**
  1614. * skb_queue_purge - empty a list
  1615. * @list: list to empty
  1616. *
  1617. * Delete all buffers on an &sk_buff list. Each buffer is removed from
  1618. * the list and one reference dropped. This function takes the list
  1619. * lock and is atomic with respect to other list locking functions.
  1620. */
  1621. void skb_queue_purge(struct sk_buff_head *list)
  1622. {
  1623. struct sk_buff *skb;
  1624. while ((skb = skb_dequeue(list)) != NULL)
  1625. kfree_skb(skb);
  1626. }
  1627. EXPORT_SYMBOL(skb_queue_purge);
  1628. /**
  1629. * skb_queue_head - queue a buffer at the list head
  1630. * @list: list to use
  1631. * @newsk: buffer to queue
  1632. *
  1633. * Queue a buffer at the start of the list. This function takes the
  1634. * list lock and can be used safely with other locking &sk_buff functions
  1635. * safely.
  1636. *
  1637. * A buffer cannot be placed on two lists at the same time.
  1638. */
  1639. void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
  1640. {
  1641. unsigned long flags;
  1642. spin_lock_irqsave(&list->lock, flags);
  1643. __skb_queue_head(list, newsk);
  1644. spin_unlock_irqrestore(&list->lock, flags);
  1645. }
  1646. EXPORT_SYMBOL(skb_queue_head);
  1647. /**
  1648. * skb_queue_tail - queue a buffer at the list tail
  1649. * @list: list to use
  1650. * @newsk: buffer to queue
  1651. *
  1652. * Queue a buffer at the tail of the list. This function takes the
  1653. * list lock and can be used safely with other locking &sk_buff functions
  1654. * safely.
  1655. *
  1656. * A buffer cannot be placed on two lists at the same time.
  1657. */
  1658. void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
  1659. {
  1660. unsigned long flags;
  1661. spin_lock_irqsave(&list->lock, flags);
  1662. __skb_queue_tail(list, newsk);
  1663. spin_unlock_irqrestore(&list->lock, flags);
  1664. }
  1665. EXPORT_SYMBOL(skb_queue_tail);
  1666. /**
  1667. * skb_unlink - remove a buffer from a list
  1668. * @skb: buffer to remove
  1669. * @list: list to use
  1670. *
  1671. * Remove a packet from a list. The list locks are taken and this
  1672. * function is atomic with respect to other list locked calls
  1673. *
  1674. * You must know what list the SKB is on.
  1675. */
  1676. void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
  1677. {
  1678. unsigned long flags;
  1679. spin_lock_irqsave(&list->lock, flags);
  1680. __skb_unlink(skb, list);
  1681. spin_unlock_irqrestore(&list->lock, flags);
  1682. }
  1683. EXPORT_SYMBOL(skb_unlink);
  1684. /**
  1685. * skb_append - append a buffer
  1686. * @old: buffer to insert after
  1687. * @newsk: buffer to insert
  1688. * @list: list to use
  1689. *
  1690. * Place a packet after a given packet in a list. The list locks are taken
  1691. * and this function is atomic with respect to other list locked calls.
  1692. * A buffer cannot be placed on two lists at the same time.
  1693. */
  1694. void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
  1695. {
  1696. unsigned long flags;
  1697. spin_lock_irqsave(&list->lock, flags);
  1698. __skb_queue_after(list, old, newsk);
  1699. spin_unlock_irqrestore(&list->lock, flags);
  1700. }
  1701. EXPORT_SYMBOL(skb_append);
  1702. /**
  1703. * skb_insert - insert a buffer
  1704. * @old: buffer to insert before
  1705. * @newsk: buffer to insert
  1706. * @list: list to use
  1707. *
  1708. * Place a packet before a given packet in a list. The list locks are
  1709. * taken and this function is atomic with respect to other list locked
  1710. * calls.
  1711. *
  1712. * A buffer cannot be placed on two lists at the same time.
  1713. */
  1714. void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
  1715. {
  1716. unsigned long flags;
  1717. spin_lock_irqsave(&list->lock, flags);
  1718. __skb_insert(newsk, old->prev, old, list);
  1719. spin_unlock_irqrestore(&list->lock, flags);
  1720. }
  1721. EXPORT_SYMBOL(skb_insert);
  1722. static inline void skb_split_inside_header(struct sk_buff *skb,
  1723. struct sk_buff* skb1,
  1724. const u32 len, const int pos)
  1725. {
  1726. int i;
  1727. skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
  1728. pos - len);
  1729. /* And move data appendix as is. */
  1730. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
  1731. skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
  1732. skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
  1733. skb_shinfo(skb)->nr_frags = 0;
  1734. skb1->data_len = skb->data_len;
  1735. skb1->len += skb1->data_len;
  1736. skb->data_len = 0;
  1737. skb->len = len;
  1738. skb_set_tail_pointer(skb, len);
  1739. }
  1740. static inline void skb_split_no_header(struct sk_buff *skb,
  1741. struct sk_buff* skb1,
  1742. const u32 len, int pos)
  1743. {
  1744. int i, k = 0;
  1745. const int nfrags = skb_shinfo(skb)->nr_frags;
  1746. skb_shinfo(skb)->nr_frags = 0;
  1747. skb1->len = skb1->data_len = skb->len - len;
  1748. skb->len = len;
  1749. skb->data_len = len - pos;
  1750. for (i = 0; i < nfrags; i++) {
  1751. int size = skb_shinfo(skb)->frags[i].size;
  1752. if (pos + size > len) {
  1753. skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
  1754. if (pos < len) {
  1755. /* Split frag.
  1756. * We have two variants in this case:
  1757. * 1. Move all the frag to the second
  1758. * part, if it is possible. F.e.
  1759. * this approach is mandatory for TUX,
  1760. * where splitting is expensive.
  1761. * 2. Split is accurately. We make this.
  1762. */
  1763. get_page(skb_shinfo(skb)->frags[i].page);
  1764. skb_shinfo(skb1)->frags[0].page_offset += len - pos;
  1765. skb_shinfo(skb1)->frags[0].size -= len - pos;
  1766. skb_shinfo(skb)->frags[i].size = len - pos;
  1767. skb_shinfo(skb)->nr_frags++;
  1768. }
  1769. k++;
  1770. } else
  1771. skb_shinfo(skb)->nr_frags++;
  1772. pos += size;
  1773. }
  1774. skb_shinfo(skb1)->nr_frags = k;
  1775. }
  1776. /**
  1777. * skb_split - Split fragmented skb to two parts at length len.
  1778. * @skb: the buffer to split
  1779. * @skb1: the buffer to receive the second part
  1780. * @len: new length for skb
  1781. */
  1782. void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
  1783. {
  1784. int pos = skb_headlen(skb);
  1785. if (len < pos) /* Split line is inside header. */
  1786. skb_split_inside_header(skb, skb1, len, pos);
  1787. else /* Second chunk has no header, nothing to copy. */
  1788. skb_split_no_header(skb, skb1, len, pos);
  1789. }
  1790. EXPORT_SYMBOL(skb_split);
  1791. /* Shifting from/to a cloned skb is a no-go.
  1792. *
  1793. * Caller cannot keep skb_shinfo related pointers past calling here!
  1794. */
  1795. static int skb_prepare_for_shift(struct sk_buff *skb)
  1796. {
  1797. return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
  1798. }
  1799. /**
  1800. * skb_shift - Shifts paged data partially from skb to another
  1801. * @tgt: buffer into which tail data gets added
  1802. * @skb: buffer from which the paged data comes from
  1803. * @shiftlen: shift up to this many bytes
  1804. *
  1805. * Attempts to shift up to shiftlen worth of bytes, which may be less than
  1806. * the length of the skb, from tgt to skb. Returns number bytes shifted.
  1807. * It's up to caller to free skb if everything was shifted.
  1808. *
  1809. * If @tgt runs out of frags, the whole operation is aborted.
  1810. *
  1811. * Skb cannot include anything else but paged data while tgt is allowed
  1812. * to have non-paged data as well.
  1813. *
  1814. * TODO: full sized shift could be optimized but that would need
  1815. * specialized skb free'er to handle frags without up-to-date nr_frags.
  1816. */
  1817. int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
  1818. {
  1819. int from, to, merge, todo;
  1820. struct skb_frag_struct *fragfrom, *fragto;
  1821. BUG_ON(shiftlen > skb->len);
  1822. BUG_ON(skb_headlen(skb)); /* Would corrupt stream */
  1823. todo = shiftlen;
  1824. from = 0;
  1825. to = skb_shinfo(tgt)->nr_frags;
  1826. fragfrom = &skb_shinfo(skb)->frags[from];
  1827. /* Actual merge is delayed until the point when we know we can
  1828. * commit all, so that we don't have to undo partial changes
  1829. */
  1830. if (!to ||
  1831. !skb_can_coalesce(tgt, to, fragfrom->page, fragfrom->page_offset)) {
  1832. merge = -1;
  1833. } else {
  1834. merge = to - 1;
  1835. todo -= fragfrom->size;
  1836. if (todo < 0) {
  1837. if (skb_prepare_for_shift(skb) ||
  1838. skb_prepare_for_shift(tgt))
  1839. return 0;
  1840. /* All previous frag pointers might be stale! */
  1841. fragfrom = &skb_shinfo(skb)->frags[from];
  1842. fragto = &skb_shinfo(tgt)->frags[merge];
  1843. fragto->size += shiftlen;
  1844. fragfrom->size -= shiftlen;
  1845. fragfrom->page_offset += shiftlen;
  1846. goto onlymerged;
  1847. }
  1848. from++;
  1849. }
  1850. /* Skip full, not-fitting skb to avoid expensive operations */
  1851. if ((shiftlen == skb->len) &&
  1852. (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
  1853. return 0;
  1854. if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
  1855. return 0;
  1856. while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
  1857. if (to == MAX_SKB_FRAGS)
  1858. return 0;
  1859. fragfrom = &skb_shinfo(skb)->frags[from];
  1860. fragto = &skb_shinfo(tgt)->frags[to];
  1861. if (todo >= fragfrom->size) {
  1862. *fragto = *fragfrom;
  1863. todo -= fragfrom->size;
  1864. from++;
  1865. to++;
  1866. } else {
  1867. get_page(fragfrom->page);
  1868. fragto->page = fragfrom->page;
  1869. fragto->page_offset = fragfrom->page_offset;
  1870. fragto->size = todo;
  1871. fragfrom->page_offset += todo;
  1872. fragfrom->size -= todo;
  1873. todo = 0;
  1874. to++;
  1875. break;
  1876. }
  1877. }
  1878. /* Ready to "commit" this state change to tgt */
  1879. skb_shinfo(tgt)->nr_frags = to;
  1880. if (merge >= 0) {
  1881. fragfrom = &skb_shinfo(skb)->frags[0];
  1882. fragto = &skb_shinfo(tgt)->frags[merge];
  1883. fragto->size += fragfrom->size;
  1884. put_page(fragfrom->page);
  1885. }
  1886. /* Reposition in the original skb */
  1887. to = 0;
  1888. while (from < skb_shinfo(skb)->nr_frags)
  1889. skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
  1890. skb_shinfo(skb)->nr_frags = to;
  1891. BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
  1892. onlymerged:
  1893. /* Most likely the tgt won't ever need its checksum anymore, skb on
  1894. * the other hand might need it if it needs to be resent
  1895. */
  1896. tgt->ip_summed = CHECKSUM_PARTIAL;
  1897. skb->ip_summed = CHECKSUM_PARTIAL;
  1898. /* Yak, is it really working this way? Some helper please? */
  1899. skb->len -= shiftlen;
  1900. skb->data_len -= shiftlen;
  1901. skb->truesize -= shiftlen;
  1902. tgt->len += shiftlen;
  1903. tgt->data_len += shiftlen;
  1904. tgt->truesize += shiftlen;
  1905. return shiftlen;
  1906. }
  1907. /**
  1908. * skb_prepare_seq_read - Prepare a sequential read of skb data
  1909. * @skb: the buffer to read
  1910. * @from: lower offset of data to be read
  1911. * @to: upper offset of data to be read
  1912. * @st: state variable
  1913. *
  1914. * Initializes the specified state variable. Must be called before
  1915. * invoking skb_seq_read() for the first time.
  1916. */
  1917. void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
  1918. unsigned int to, struct skb_seq_state *st)
  1919. {
  1920. st->lower_offset = from;
  1921. st->upper_offset = to;
  1922. st->root_skb = st->cur_skb = skb;
  1923. st->frag_idx = st->stepped_offset = 0;
  1924. st->frag_data = NULL;
  1925. }
  1926. EXPORT_SYMBOL(skb_prepare_seq_read);
  1927. /**
  1928. * skb_seq_read - Sequentially read skb data
  1929. * @consumed: number of bytes consumed by the caller so far
  1930. * @data: destination pointer for data to be returned
  1931. * @st: state variable
  1932. *
  1933. * Reads a block of skb data at &consumed relative to the
  1934. * lower offset specified to skb_prepare_seq_read(). Assigns
  1935. * the head of the data block to &data and returns the length
  1936. * of the block or 0 if the end of the skb data or the upper
  1937. * offset has been reached.
  1938. *
  1939. * The caller is not required to consume all of the data
  1940. * returned, i.e. &consumed is typically set to the number
  1941. * of bytes already consumed and the next call to
  1942. * skb_seq_read() will return the remaining part of the block.
  1943. *
  1944. * Note 1: The size of each block of data returned can be arbitary,
  1945. * this limitation is the cost for zerocopy seqeuental
  1946. * reads of potentially non linear data.
  1947. *
  1948. * Note 2: Fragment lists within fragments are not implemented
  1949. * at the moment, state->root_skb could be replaced with
  1950. * a stack for this purpose.
  1951. */
  1952. unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
  1953. struct skb_seq_state *st)
  1954. {
  1955. unsigned int block_limit, abs_offset = consumed + st->lower_offset;
  1956. skb_frag_t *frag;
  1957. if (unlikely(abs_offset >= st->upper_offset))
  1958. return 0;
  1959. next_skb:
  1960. block_limit = skb_headlen(st->cur_skb) + st->stepped_offset;
  1961. if (abs_offset < block_limit) {
  1962. *data = st->cur_skb->data + (abs_offset - st->stepped_offset);
  1963. return block_limit - abs_offset;
  1964. }
  1965. if (st->frag_idx == 0 && !st->frag_data)
  1966. st->stepped_offset += skb_headlen(st->cur_skb);
  1967. while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
  1968. frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
  1969. block_limit = frag->size + st->stepped_offset;
  1970. if (abs_offset < block_limit) {
  1971. if (!st->frag_data)
  1972. st->frag_data = kmap_skb_frag(frag);
  1973. *data = (u8 *) st->frag_data + frag->page_offset +
  1974. (abs_offset - st->stepped_offset);
  1975. return block_limit - abs_offset;
  1976. }
  1977. if (st->frag_data) {
  1978. kunmap_skb_frag(st->frag_data);
  1979. st->frag_data = NULL;
  1980. }
  1981. st->frag_idx++;
  1982. st->stepped_offset += frag->size;
  1983. }
  1984. if (st->frag_data) {
  1985. kunmap_skb_frag(st->frag_data);
  1986. st->frag_data = NULL;
  1987. }
  1988. if (st->root_skb == st->cur_skb &&
  1989. skb_shinfo(st->root_skb)->frag_list) {
  1990. st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
  1991. st->frag_idx = 0;
  1992. goto next_skb;
  1993. } else if (st->cur_skb->next) {
  1994. st->cur_skb = st->cur_skb->next;
  1995. st->frag_idx = 0;
  1996. goto next_skb;
  1997. }
  1998. return 0;
  1999. }
  2000. EXPORT_SYMBOL(skb_seq_read);
  2001. /**
  2002. * skb_abort_seq_read - Abort a sequential read of skb data
  2003. * @st: state variable
  2004. *
  2005. * Must be called if skb_seq_read() was not called until it
  2006. * returned 0.
  2007. */
  2008. void skb_abort_seq_read(struct skb_seq_state *st)
  2009. {
  2010. if (st->frag_data)
  2011. kunmap_skb_frag(st->frag_data);
  2012. }
  2013. EXPORT_SYMBOL(skb_abort_seq_read);
  2014. #define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb))
  2015. static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
  2016. struct ts_config *conf,
  2017. struct ts_state *state)
  2018. {
  2019. return skb_seq_read(offset, text, TS_SKB_CB(state));
  2020. }
  2021. static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
  2022. {
  2023. skb_abort_seq_read(TS_SKB_CB(state));
  2024. }
  2025. /**
  2026. * skb_find_text - Find a text pattern in skb data
  2027. * @skb: the buffer to look in
  2028. * @from: search offset
  2029. * @to: search limit
  2030. * @config: textsearch configuration
  2031. * @state: uninitialized textsearch state variable
  2032. *
  2033. * Finds a pattern in the skb data according to the specified
  2034. * textsearch configuration. Use textsearch_next() to retrieve
  2035. * subsequent occurrences of the pattern. Returns the offset
  2036. * to the first occurrence or UINT_MAX if no match was found.
  2037. */
  2038. unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
  2039. unsigned int to, struct ts_config *config,
  2040. struct ts_state *state)
  2041. {
  2042. unsigned int ret;
  2043. config->get_next_block = skb_ts_get_next_block;
  2044. config->finish = skb_ts_finish;
  2045. skb_prepare_seq_read(skb, from, to, TS_SKB_CB(state));
  2046. ret = textsearch_find(config, state);
  2047. return (ret <= to - from ? ret : UINT_MAX);
  2048. }
  2049. EXPORT_SYMBOL(skb_find_text);
  2050. /**
  2051. * skb_append_datato_frags: - append the user data to a skb
  2052. * @sk: sock structure
  2053. * @skb: skb structure to be appened with user data.
  2054. * @getfrag: call back function to be used for getting the user data
  2055. * @from: pointer to user message iov
  2056. * @length: length of the iov message
  2057. *
  2058. * Description: This procedure append the user data in the fragment part
  2059. * of the skb if any page alloc fails user this procedure returns -ENOMEM
  2060. */
  2061. int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
  2062. int (*getfrag)(void *from, char *to, int offset,
  2063. int len, int odd, struct sk_buff *skb),
  2064. void *from, int length)
  2065. {
  2066. int frg_cnt = 0;
  2067. skb_frag_t *frag = NULL;
  2068. struct page *page = NULL;
  2069. int copy, left;
  2070. int offset = 0;
  2071. int ret;
  2072. do {
  2073. /* Return error if we don't have space for new frag */
  2074. frg_cnt = skb_shinfo(skb)->nr_frags;
  2075. if (frg_cnt >= MAX_SKB_FRAGS)
  2076. return -EFAULT;
  2077. /* allocate a new page for next frag */
  2078. page = alloc_pages(sk->sk_allocation, 0);
  2079. /* If alloc_page fails just return failure and caller will
  2080. * free previous allocated pages by doing kfree_skb()
  2081. */
  2082. if (page == NULL)
  2083. return -ENOMEM;
  2084. /* initialize the next frag */
  2085. sk->sk_sndmsg_page = page;
  2086. sk->sk_sndmsg_off = 0;
  2087. skb_fill_page_desc(skb, frg_cnt, page, 0, 0);
  2088. skb->truesize += PAGE_SIZE;
  2089. atomic_add(PAGE_SIZE, &sk->sk_wmem_alloc);
  2090. /* get the new initialized frag */
  2091. frg_cnt = skb_shinfo(skb)->nr_frags;
  2092. frag = &skb_shinfo(skb)->frags[frg_cnt - 1];
  2093. /* copy the user data to page */
  2094. left = PAGE_SIZE - frag->page_offset;
  2095. copy = (length > left)? left : length;
  2096. ret = getfrag(from, (page_address(frag->page) +
  2097. frag->page_offset + frag->size),
  2098. offset, copy, 0, skb);
  2099. if (ret < 0)
  2100. return -EFAULT;
  2101. /* copy was successful so update the size parameters */
  2102. sk->sk_sndmsg_off += copy;
  2103. frag->size += copy;
  2104. skb->len += copy;
  2105. skb->data_len += copy;
  2106. offset += copy;
  2107. length -= copy;
  2108. } while (length > 0);
  2109. return 0;
  2110. }
  2111. EXPORT_SYMBOL(skb_append_datato_frags);
  2112. /**
  2113. * skb_pull_rcsum - pull skb and update receive checksum
  2114. * @skb: buffer to update
  2115. * @len: length of data pulled
  2116. *
  2117. * This function performs an skb_pull on the packet and updates
  2118. * the CHECKSUM_COMPLETE checksum. It should be used on
  2119. * receive path processing instead of skb_pull unless you know
  2120. * that the checksum difference is zero (e.g., a valid IP header)
  2121. * or you are setting ip_summed to CHECKSUM_NONE.
  2122. */
  2123. unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
  2124. {
  2125. BUG_ON(len > skb->len);
  2126. skb->len -= len;
  2127. BUG_ON(skb->len < skb->data_len);
  2128. skb_postpull_rcsum(skb, skb->data, len);
  2129. return skb->data += len;
  2130. }
  2131. EXPORT_SYMBOL_GPL(skb_pull_rcsum);
  2132. /**
  2133. * skb_segment - Perform protocol segmentation on skb.
  2134. * @skb: buffer to segment
  2135. * @features: features for the output path (see dev->features)
  2136. *
  2137. * This function performs segmentation on the given skb. It returns
  2138. * a pointer to the first in a list of new skbs for the segments.
  2139. * In case of error it returns ERR_PTR(err).
  2140. */
  2141. struct sk_buff *skb_segment(struct sk_buff *skb, int features)
  2142. {
  2143. struct sk_buff *segs = NULL;
  2144. struct sk_buff *tail = NULL;
  2145. struct sk_buff *fskb = skb_shinfo(skb)->frag_list;
  2146. unsigned int mss = skb_shinfo(skb)->gso_size;
  2147. unsigned int doffset = skb->data - skb_mac_header(skb);
  2148. unsigned int offset = doffset;
  2149. unsigned int headroom;
  2150. unsigned int len;
  2151. int sg = features & NETIF_F_SG;
  2152. int nfrags = skb_shinfo(skb)->nr_frags;
  2153. int err = -ENOMEM;
  2154. int i = 0;
  2155. int pos;
  2156. __skb_push(skb, doffset);
  2157. headroom = skb_headroom(skb);
  2158. pos = skb_headlen(skb);
  2159. do {
  2160. struct sk_buff *nskb;
  2161. skb_frag_t *frag;
  2162. int hsize;
  2163. int size;
  2164. len = skb->len - offset;
  2165. if (len > mss)
  2166. len = mss;
  2167. hsize = skb_headlen(skb) - offset;
  2168. if (hsize < 0)
  2169. hsize = 0;
  2170. if (hsize > len || !sg)
  2171. hsize = len;
  2172. if (!hsize && i >= nfrags) {
  2173. BUG_ON(fskb->len != len);
  2174. pos += len;
  2175. nskb = skb_clone(fskb, GFP_ATOMIC);
  2176. fskb = fskb->next;
  2177. if (unlikely(!nskb))
  2178. goto err;
  2179. hsize = skb_end_pointer(nskb) - nskb->head;
  2180. if (skb_cow_head(nskb, doffset + headroom)) {
  2181. kfree_skb(nskb);
  2182. goto err;
  2183. }
  2184. nskb->truesize += skb_end_pointer(nskb) - nskb->head -
  2185. hsize;
  2186. skb_release_head_state(nskb);
  2187. __skb_push(nskb, doffset);
  2188. } else {
  2189. nskb = alloc_skb(hsize + doffset + headroom,
  2190. GFP_ATOMIC);
  2191. if (unlikely(!nskb))
  2192. goto err;
  2193. skb_reserve(nskb, headroom);
  2194. __skb_put(nskb, doffset);
  2195. }
  2196. if (segs)
  2197. tail->next = nskb;
  2198. else
  2199. segs = nskb;
  2200. tail = nskb;
  2201. __copy_skb_header(nskb, skb);
  2202. nskb->mac_len = skb->mac_len;
  2203. skb_reset_mac_header(nskb);
  2204. skb_set_network_header(nskb, skb->mac_len);
  2205. nskb->transport_header = (nskb->network_header +
  2206. skb_network_header_len(skb));
  2207. skb_copy_from_linear_data(skb, nskb->data, doffset);
  2208. if (pos >= offset + len)
  2209. continue;
  2210. if (!sg) {
  2211. nskb->ip_summed = CHECKSUM_NONE;
  2212. nskb->csum = skb_copy_and_csum_bits(skb, offset,
  2213. skb_put(nskb, len),
  2214. len, 0);
  2215. continue;
  2216. }
  2217. frag = skb_shinfo(nskb)->frags;
  2218. skb_copy_from_linear_data_offset(skb, offset,
  2219. skb_put(nskb, hsize), hsize);
  2220. while (pos < offset + len && i < nfrags) {
  2221. *frag = skb_shinfo(skb)->frags[i];
  2222. get_page(frag->page);
  2223. size = frag->size;
  2224. if (pos < offset) {
  2225. frag->page_offset += offset - pos;
  2226. frag->size -= offset - pos;
  2227. }
  2228. skb_shinfo(nskb)->nr_frags++;
  2229. if (pos + size <= offset + len) {
  2230. i++;
  2231. pos += size;
  2232. } else {
  2233. frag->size -= pos + size - (offset + len);
  2234. goto skip_fraglist;
  2235. }
  2236. frag++;
  2237. }
  2238. if (pos < offset + len) {
  2239. struct sk_buff *fskb2 = fskb;
  2240. BUG_ON(pos + fskb->len != offset + len);
  2241. pos += fskb->len;
  2242. fskb = fskb->next;
  2243. if (fskb2->next) {
  2244. fskb2 = skb_clone(fskb2, GFP_ATOMIC);
  2245. if (!fskb2)
  2246. goto err;
  2247. } else
  2248. skb_get(fskb2);
  2249. BUG_ON(skb_shinfo(nskb)->frag_list);
  2250. skb_shinfo(nskb)->frag_list = fskb2;
  2251. }
  2252. skip_fraglist:
  2253. nskb->data_len = len - hsize;
  2254. nskb->len += nskb->data_len;
  2255. nskb->truesize += nskb->data_len;
  2256. } while ((offset += len) < skb->len);
  2257. return segs;
  2258. err:
  2259. while ((skb = segs)) {
  2260. segs = skb->next;
  2261. kfree_skb(skb);
  2262. }
  2263. return ERR_PTR(err);
  2264. }
  2265. EXPORT_SYMBOL_GPL(skb_segment);
  2266. int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb)
  2267. {
  2268. struct sk_buff *p = *head;
  2269. struct sk_buff *nskb;
  2270. unsigned int headroom;
  2271. unsigned int len = skb_gro_len(skb);
  2272. if (p->len + len >= 65536)
  2273. return -E2BIG;
  2274. if (skb_shinfo(p)->frag_list)
  2275. goto merge;
  2276. else if (skb_headlen(skb) <= skb_gro_offset(skb)) {
  2277. if (skb_shinfo(p)->nr_frags + skb_shinfo(skb)->nr_frags >
  2278. MAX_SKB_FRAGS)
  2279. return -E2BIG;
  2280. skb_shinfo(skb)->frags[0].page_offset +=
  2281. skb_gro_offset(skb) - skb_headlen(skb);
  2282. skb_shinfo(skb)->frags[0].size -=
  2283. skb_gro_offset(skb) - skb_headlen(skb);
  2284. memcpy(skb_shinfo(p)->frags + skb_shinfo(p)->nr_frags,
  2285. skb_shinfo(skb)->frags,
  2286. skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t));
  2287. skb_shinfo(p)->nr_frags += skb_shinfo(skb)->nr_frags;
  2288. skb_shinfo(skb)->nr_frags = 0;
  2289. skb->truesize -= skb->data_len;
  2290. skb->len -= skb->data_len;
  2291. skb->data_len = 0;
  2292. NAPI_GRO_CB(skb)->free = 1;
  2293. goto done;
  2294. }
  2295. headroom = skb_headroom(p);
  2296. nskb = netdev_alloc_skb(p->dev, headroom + skb_gro_offset(p));
  2297. if (unlikely(!nskb))
  2298. return -ENOMEM;
  2299. __copy_skb_header(nskb, p);
  2300. nskb->mac_len = p->mac_len;
  2301. skb_reserve(nskb, headroom);
  2302. __skb_put(nskb, skb_gro_offset(p));
  2303. skb_set_mac_header(nskb, skb_mac_header(p) - p->data);
  2304. skb_set_network_header(nskb, skb_network_offset(p));
  2305. skb_set_transport_header(nskb, skb_transport_offset(p));
  2306. __skb_pull(p, skb_gro_offset(p));
  2307. memcpy(skb_mac_header(nskb), skb_mac_header(p),
  2308. p->data - skb_mac_header(p));
  2309. *NAPI_GRO_CB(nskb) = *NAPI_GRO_CB(p);
  2310. skb_shinfo(nskb)->frag_list = p;
  2311. skb_shinfo(nskb)->gso_size = skb_shinfo(p)->gso_size;
  2312. skb_header_release(p);
  2313. nskb->prev = p;
  2314. nskb->data_len += p->len;
  2315. nskb->truesize += p->len;
  2316. nskb->len += p->len;
  2317. *head = nskb;
  2318. nskb->next = p->next;
  2319. p->next = NULL;
  2320. p = nskb;
  2321. merge:
  2322. if (skb_gro_offset(skb) > skb_headlen(skb)) {
  2323. skb_shinfo(skb)->frags[0].page_offset +=
  2324. skb_gro_offset(skb) - skb_headlen(skb);
  2325. skb_shinfo(skb)->frags[0].size -=
  2326. skb_gro_offset(skb) - skb_headlen(skb);
  2327. skb_gro_reset_offset(skb);
  2328. skb_gro_pull(skb, skb_headlen(skb));
  2329. }
  2330. __skb_pull(skb, skb_gro_offset(skb));
  2331. p->prev->next = skb;
  2332. p->prev = skb;
  2333. skb_header_release(skb);
  2334. done:
  2335. NAPI_GRO_CB(p)->count++;
  2336. p->data_len += len;
  2337. p->truesize += len;
  2338. p->len += len;
  2339. NAPI_GRO_CB(skb)->same_flow = 1;
  2340. return 0;
  2341. }
  2342. EXPORT_SYMBOL_GPL(skb_gro_receive);
  2343. void __init skb_init(void)
  2344. {
  2345. skbuff_head_cache = kmem_cache_create("skbuff_head_cache",
  2346. sizeof(struct sk_buff),
  2347. 0,
  2348. SLAB_HWCACHE_ALIGN|SLAB_PANIC,
  2349. NULL);
  2350. skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
  2351. (2*sizeof(struct sk_buff)) +
  2352. sizeof(atomic_t),
  2353. 0,
  2354. SLAB_HWCACHE_ALIGN|SLAB_PANIC,
  2355. NULL);
  2356. }
  2357. /**
  2358. * skb_to_sgvec - Fill a scatter-gather list from a socket buffer
  2359. * @skb: Socket buffer containing the buffers to be mapped
  2360. * @sg: The scatter-gather list to map into
  2361. * @offset: The offset into the buffer's contents to start mapping
  2362. * @len: Length of buffer space to be mapped
  2363. *
  2364. * Fill the specified scatter-gather list with mappings/pointers into a
  2365. * region of the buffer space attached to a socket buffer.
  2366. */
  2367. static int
  2368. __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
  2369. {
  2370. int start = skb_headlen(skb);
  2371. int i, copy = start - offset;
  2372. int elt = 0;
  2373. if (copy > 0) {
  2374. if (copy > len)
  2375. copy = len;
  2376. sg_set_buf(sg, skb->data + offset, copy);
  2377. elt++;
  2378. if ((len -= copy) == 0)
  2379. return elt;
  2380. offset += copy;
  2381. }
  2382. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  2383. int end;
  2384. WARN_ON(start > offset + len);
  2385. end = start + skb_shinfo(skb)->frags[i].size;
  2386. if ((copy = end - offset) > 0) {
  2387. skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
  2388. if (copy > len)
  2389. copy = len;
  2390. sg_set_page(&sg[elt], frag->page, copy,
  2391. frag->page_offset+offset-start);
  2392. elt++;
  2393. if (!(len -= copy))
  2394. return elt;
  2395. offset += copy;
  2396. }
  2397. start = end;
  2398. }
  2399. if (skb_shinfo(skb)->frag_list) {
  2400. struct sk_buff *list = skb_shinfo(skb)->frag_list;
  2401. for (; list; list = list->next) {
  2402. int end;
  2403. WARN_ON(start > offset + len);
  2404. end = start + list->len;
  2405. if ((copy = end - offset) > 0) {
  2406. if (copy > len)
  2407. copy = len;
  2408. elt += __skb_to_sgvec(list, sg+elt, offset - start,
  2409. copy);
  2410. if ((len -= copy) == 0)
  2411. return elt;
  2412. offset += copy;
  2413. }
  2414. start = end;
  2415. }
  2416. }
  2417. BUG_ON(len);
  2418. return elt;
  2419. }
  2420. int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
  2421. {
  2422. int nsg = __skb_to_sgvec(skb, sg, offset, len);
  2423. sg_mark_end(&sg[nsg - 1]);
  2424. return nsg;
  2425. }
  2426. EXPORT_SYMBOL_GPL(skb_to_sgvec);
  2427. /**
  2428. * skb_cow_data - Check that a socket buffer's data buffers are writable
  2429. * @skb: The socket buffer to check.
  2430. * @tailbits: Amount of trailing space to be added
  2431. * @trailer: Returned pointer to the skb where the @tailbits space begins
  2432. *
  2433. * Make sure that the data buffers attached to a socket buffer are
  2434. * writable. If they are not, private copies are made of the data buffers
  2435. * and the socket buffer is set to use these instead.
  2436. *
  2437. * If @tailbits is given, make sure that there is space to write @tailbits
  2438. * bytes of data beyond current end of socket buffer. @trailer will be
  2439. * set to point to the skb in which this space begins.
  2440. *
  2441. * The number of scatterlist elements required to completely map the
  2442. * COW'd and extended socket buffer will be returned.
  2443. */
  2444. int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
  2445. {
  2446. int copyflag;
  2447. int elt;
  2448. struct sk_buff *skb1, **skb_p;
  2449. /* If skb is cloned or its head is paged, reallocate
  2450. * head pulling out all the pages (pages are considered not writable
  2451. * at the moment even if they are anonymous).
  2452. */
  2453. if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
  2454. __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL)
  2455. return -ENOMEM;
  2456. /* Easy case. Most of packets will go this way. */
  2457. if (!skb_shinfo(skb)->frag_list) {
  2458. /* A little of trouble, not enough of space for trailer.
  2459. * This should not happen, when stack is tuned to generate
  2460. * good frames. OK, on miss we reallocate and reserve even more
  2461. * space, 128 bytes is fair. */
  2462. if (skb_tailroom(skb) < tailbits &&
  2463. pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
  2464. return -ENOMEM;
  2465. /* Voila! */
  2466. *trailer = skb;
  2467. return 1;
  2468. }
  2469. /* Misery. We are in troubles, going to mincer fragments... */
  2470. elt = 1;
  2471. skb_p = &skb_shinfo(skb)->frag_list;
  2472. copyflag = 0;
  2473. while ((skb1 = *skb_p) != NULL) {
  2474. int ntail = 0;
  2475. /* The fragment is partially pulled by someone,
  2476. * this can happen on input. Copy it and everything
  2477. * after it. */
  2478. if (skb_shared(skb1))
  2479. copyflag = 1;
  2480. /* If the skb is the last, worry about trailer. */
  2481. if (skb1->next == NULL && tailbits) {
  2482. if (skb_shinfo(skb1)->nr_frags ||
  2483. skb_shinfo(skb1)->frag_list ||
  2484. skb_tailroom(skb1) < tailbits)
  2485. ntail = tailbits + 128;
  2486. }
  2487. if (copyflag ||
  2488. skb_cloned(skb1) ||
  2489. ntail ||
  2490. skb_shinfo(skb1)->nr_frags ||
  2491. skb_shinfo(skb1)->frag_list) {
  2492. struct sk_buff *skb2;
  2493. /* Fuck, we are miserable poor guys... */
  2494. if (ntail == 0)
  2495. skb2 = skb_copy(skb1, GFP_ATOMIC);
  2496. else
  2497. skb2 = skb_copy_expand(skb1,
  2498. skb_headroom(skb1),
  2499. ntail,
  2500. GFP_ATOMIC);
  2501. if (unlikely(skb2 == NULL))
  2502. return -ENOMEM;
  2503. if (skb1->sk)
  2504. skb_set_owner_w(skb2, skb1->sk);
  2505. /* Looking around. Are we still alive?
  2506. * OK, link new skb, drop old one */
  2507. skb2->next = skb1->next;
  2508. *skb_p = skb2;
  2509. kfree_skb(skb1);
  2510. skb1 = skb2;
  2511. }
  2512. elt++;
  2513. *trailer = skb1;
  2514. skb_p = &skb1->next;
  2515. }
  2516. return elt;
  2517. }
  2518. EXPORT_SYMBOL_GPL(skb_cow_data);
  2519. /**
  2520. * skb_partial_csum_set - set up and verify partial csum values for packet
  2521. * @skb: the skb to set
  2522. * @start: the number of bytes after skb->data to start checksumming.
  2523. * @off: the offset from start to place the checksum.
  2524. *
  2525. * For untrusted partially-checksummed packets, we need to make sure the values
  2526. * for skb->csum_start and skb->csum_offset are valid so we don't oops.
  2527. *
  2528. * This function checks and sets those values and skb->ip_summed: if this
  2529. * returns false you should drop the packet.
  2530. */
  2531. bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
  2532. {
  2533. if (unlikely(start > skb->len - 2) ||
  2534. unlikely((int)start + off > skb->len - 2)) {
  2535. if (net_ratelimit())
  2536. printk(KERN_WARNING
  2537. "bad partial csum: csum=%u/%u len=%u\n",
  2538. start, off, skb->len);
  2539. return false;
  2540. }
  2541. skb->ip_summed = CHECKSUM_PARTIAL;
  2542. skb->csum_start = skb_headroom(skb) + start;
  2543. skb->csum_offset = off;
  2544. return true;
  2545. }
  2546. EXPORT_SYMBOL_GPL(skb_partial_csum_set);
  2547. void __skb_warn_lro_forwarding(const struct sk_buff *skb)
  2548. {
  2549. if (net_ratelimit())
  2550. pr_warning("%s: received packets cannot be forwarded"
  2551. " while LRO is enabled\n", skb->dev->name);
  2552. }
  2553. EXPORT_SYMBOL(__skb_warn_lro_forwarding);