skbuff.c 82 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305
  1. /*
  2. * Routines having to do with the 'struct sk_buff' memory handlers.
  3. *
  4. * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk>
  5. * Florian La Roche <rzsfl@rz.uni-sb.de>
  6. *
  7. * Fixes:
  8. * Alan Cox : Fixed the worst of the load
  9. * balancer bugs.
  10. * Dave Platt : Interrupt stacking fix.
  11. * Richard Kooijman : Timestamp fixes.
  12. * Alan Cox : Changed buffer format.
  13. * Alan Cox : destructor hook for AF_UNIX etc.
  14. * Linus Torvalds : Better skb_clone.
  15. * Alan Cox : Added skb_copy.
  16. * Alan Cox : Added all the changed routines Linus
  17. * only put in the headers
  18. * Ray VanTassle : Fixed --skb->lock in free
  19. * Alan Cox : skb_copy copy arp field
  20. * Andi Kleen : slabified it.
  21. * Robert Olsson : Removed skb_head_pool
  22. *
  23. * NOTE:
  24. * The __skb_ routines should be called with interrupts
  25. * disabled, or you better be *real* sure that the operation is atomic
  26. * with respect to whatever list is being frobbed (e.g. via lock_sock()
  27. * or via disabling bottom half handlers, etc).
  28. *
  29. * This program is free software; you can redistribute it and/or
  30. * modify it under the terms of the GNU General Public License
  31. * as published by the Free Software Foundation; either version
  32. * 2 of the License, or (at your option) any later version.
  33. */
  34. /*
  35. * The functions in this file will not compile correctly with gcc 2.4.x
  36. */
  37. #include <linux/module.h>
  38. #include <linux/types.h>
  39. #include <linux/kernel.h>
  40. #include <linux/kmemcheck.h>
  41. #include <linux/mm.h>
  42. #include <linux/interrupt.h>
  43. #include <linux/in.h>
  44. #include <linux/inet.h>
  45. #include <linux/slab.h>
  46. #include <linux/netdevice.h>
  47. #ifdef CONFIG_NET_CLS_ACT
  48. #include <net/pkt_sched.h>
  49. #endif
  50. #include <linux/string.h>
  51. #include <linux/skbuff.h>
  52. #include <linux/splice.h>
  53. #include <linux/cache.h>
  54. #include <linux/rtnetlink.h>
  55. #include <linux/init.h>
  56. #include <linux/scatterlist.h>
  57. #include <linux/errqueue.h>
  58. #include <linux/prefetch.h>
  59. #include <net/protocol.h>
  60. #include <net/dst.h>
  61. #include <net/sock.h>
  62. #include <net/checksum.h>
  63. #include <net/xfrm.h>
  64. #include <asm/uaccess.h>
  65. #include <trace/events/skb.h>
  66. #include <linux/highmem.h>
  67. static struct kmem_cache *skbuff_head_cache __read_mostly;
  68. static struct kmem_cache *skbuff_fclone_cache __read_mostly;
  69. static void sock_pipe_buf_release(struct pipe_inode_info *pipe,
  70. struct pipe_buffer *buf)
  71. {
  72. put_page(buf->page);
  73. }
  74. static void sock_pipe_buf_get(struct pipe_inode_info *pipe,
  75. struct pipe_buffer *buf)
  76. {
  77. get_page(buf->page);
  78. }
  79. static int sock_pipe_buf_steal(struct pipe_inode_info *pipe,
  80. struct pipe_buffer *buf)
  81. {
  82. return 1;
  83. }
  84. /* Pipe buffer operations for a socket. */
  85. static const struct pipe_buf_operations sock_pipe_buf_ops = {
  86. .can_merge = 0,
  87. .map = generic_pipe_buf_map,
  88. .unmap = generic_pipe_buf_unmap,
  89. .confirm = generic_pipe_buf_confirm,
  90. .release = sock_pipe_buf_release,
  91. .steal = sock_pipe_buf_steal,
  92. .get = sock_pipe_buf_get,
  93. };
  94. /*
  95. * Keep out-of-line to prevent kernel bloat.
  96. * __builtin_return_address is not used because it is not always
  97. * reliable.
  98. */
  99. /**
  100. * skb_over_panic - private function
  101. * @skb: buffer
  102. * @sz: size
  103. * @here: address
  104. *
  105. * Out of line support code for skb_put(). Not user callable.
  106. */
  107. static void skb_over_panic(struct sk_buff *skb, int sz, void *here)
  108. {
  109. printk(KERN_EMERG "skb_over_panic: text:%p len:%d put:%d head:%p "
  110. "data:%p tail:%#lx end:%#lx dev:%s\n",
  111. here, skb->len, sz, skb->head, skb->data,
  112. (unsigned long)skb->tail, (unsigned long)skb->end,
  113. skb->dev ? skb->dev->name : "<NULL>");
  114. BUG();
  115. }
  116. /**
  117. * skb_under_panic - private function
  118. * @skb: buffer
  119. * @sz: size
  120. * @here: address
  121. *
  122. * Out of line support code for skb_push(). Not user callable.
  123. */
  124. static void skb_under_panic(struct sk_buff *skb, int sz, void *here)
  125. {
  126. printk(KERN_EMERG "skb_under_panic: text:%p len:%d put:%d head:%p "
  127. "data:%p tail:%#lx end:%#lx dev:%s\n",
  128. here, skb->len, sz, skb->head, skb->data,
  129. (unsigned long)skb->tail, (unsigned long)skb->end,
  130. skb->dev ? skb->dev->name : "<NULL>");
  131. BUG();
  132. }
  133. /* Allocate a new skbuff. We do this ourselves so we can fill in a few
  134. * 'private' fields and also do memory statistics to find all the
  135. * [BEEP] leaks.
  136. *
  137. */
  138. /**
  139. * __alloc_skb - allocate a network buffer
  140. * @size: size to allocate
  141. * @gfp_mask: allocation mask
  142. * @fclone: allocate from fclone cache instead of head cache
  143. * and allocate a cloned (child) skb
  144. * @node: numa node to allocate memory on
  145. *
  146. * Allocate a new &sk_buff. The returned buffer has no headroom and a
  147. * tail room of size bytes. The object has a reference count of one.
  148. * The return is the buffer. On a failure the return is %NULL.
  149. *
  150. * Buffers may only be allocated from interrupts using a @gfp_mask of
  151. * %GFP_ATOMIC.
  152. */
  153. struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
  154. int fclone, int node)
  155. {
  156. struct kmem_cache *cache;
  157. struct skb_shared_info *shinfo;
  158. struct sk_buff *skb;
  159. u8 *data;
  160. cache = fclone ? skbuff_fclone_cache : skbuff_head_cache;
  161. /* Get the HEAD */
  162. skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
  163. if (!skb)
  164. goto out;
  165. prefetchw(skb);
  166. /* We do our best to align skb_shared_info on a separate cache
  167. * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives
  168. * aligned memory blocks, unless SLUB/SLAB debug is enabled.
  169. * Both skb->head and skb_shared_info are cache line aligned.
  170. */
  171. size = SKB_DATA_ALIGN(size);
  172. size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
  173. data = kmalloc_node_track_caller(size, gfp_mask, node);
  174. if (!data)
  175. goto nodata;
  176. /* kmalloc(size) might give us more room than requested.
  177. * Put skb_shared_info exactly at the end of allocated zone,
  178. * to allow max possible filling before reallocation.
  179. */
  180. size = SKB_WITH_OVERHEAD(ksize(data));
  181. prefetchw(data + size);
  182. /*
  183. * Only clear those fields we need to clear, not those that we will
  184. * actually initialise below. Hence, don't put any more fields after
  185. * the tail pointer in struct sk_buff!
  186. */
  187. memset(skb, 0, offsetof(struct sk_buff, tail));
  188. /* Account for allocated memory : skb + skb->head */
  189. skb->truesize = SKB_TRUESIZE(size);
  190. atomic_set(&skb->users, 1);
  191. skb->head = data;
  192. skb->data = data;
  193. skb_reset_tail_pointer(skb);
  194. skb->end = skb->tail + size;
  195. #ifdef NET_SKBUFF_DATA_USES_OFFSET
  196. skb->mac_header = ~0U;
  197. #endif
  198. /* make sure we initialize shinfo sequentially */
  199. shinfo = skb_shinfo(skb);
  200. memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
  201. atomic_set(&shinfo->dataref, 1);
  202. kmemcheck_annotate_variable(shinfo->destructor_arg);
  203. if (fclone) {
  204. struct sk_buff *child = skb + 1;
  205. atomic_t *fclone_ref = (atomic_t *) (child + 1);
  206. kmemcheck_annotate_bitfield(child, flags1);
  207. kmemcheck_annotate_bitfield(child, flags2);
  208. skb->fclone = SKB_FCLONE_ORIG;
  209. atomic_set(fclone_ref, 1);
  210. child->fclone = SKB_FCLONE_UNAVAILABLE;
  211. }
  212. out:
  213. return skb;
  214. nodata:
  215. kmem_cache_free(cache, skb);
  216. skb = NULL;
  217. goto out;
  218. }
  219. EXPORT_SYMBOL(__alloc_skb);
  220. /**
  221. * build_skb - build a network buffer
  222. * @data: data buffer provided by caller
  223. * @frag_size: size of fragment, or 0 if head was kmalloced
  224. *
  225. * Allocate a new &sk_buff. Caller provides space holding head and
  226. * skb_shared_info. @data must have been allocated by kmalloc()
  227. * The return is the new skb buffer.
  228. * On a failure the return is %NULL, and @data is not freed.
  229. * Notes :
  230. * Before IO, driver allocates only data buffer where NIC put incoming frame
  231. * Driver should add room at head (NET_SKB_PAD) and
  232. * MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info))
  233. * After IO, driver calls build_skb(), to allocate sk_buff and populate it
  234. * before giving packet to stack.
  235. * RX rings only contains data buffers, not full skbs.
  236. */
  237. struct sk_buff *build_skb(void *data, unsigned int frag_size)
  238. {
  239. struct skb_shared_info *shinfo;
  240. struct sk_buff *skb;
  241. unsigned int size = frag_size ? : ksize(data);
  242. skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
  243. if (!skb)
  244. return NULL;
  245. size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
  246. memset(skb, 0, offsetof(struct sk_buff, tail));
  247. skb->truesize = SKB_TRUESIZE(size);
  248. skb->head_frag = frag_size != 0;
  249. atomic_set(&skb->users, 1);
  250. skb->head = data;
  251. skb->data = data;
  252. skb_reset_tail_pointer(skb);
  253. skb->end = skb->tail + size;
  254. #ifdef NET_SKBUFF_DATA_USES_OFFSET
  255. skb->mac_header = ~0U;
  256. #endif
  257. /* make sure we initialize shinfo sequentially */
  258. shinfo = skb_shinfo(skb);
  259. memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
  260. atomic_set(&shinfo->dataref, 1);
  261. kmemcheck_annotate_variable(shinfo->destructor_arg);
  262. return skb;
  263. }
  264. EXPORT_SYMBOL(build_skb);
  265. /**
  266. * __netdev_alloc_skb - allocate an skbuff for rx on a specific device
  267. * @dev: network device to receive on
  268. * @length: length to allocate
  269. * @gfp_mask: get_free_pages mask, passed to alloc_skb
  270. *
  271. * Allocate a new &sk_buff and assign it a usage count of one. The
  272. * buffer has unspecified headroom built in. Users should allocate
  273. * the headroom they think they need without accounting for the
  274. * built in space. The built in space is used for optimisations.
  275. *
  276. * %NULL is returned if there is no free memory.
  277. */
  278. struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
  279. unsigned int length, gfp_t gfp_mask)
  280. {
  281. struct sk_buff *skb;
  282. skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask, 0, NUMA_NO_NODE);
  283. if (likely(skb)) {
  284. skb_reserve(skb, NET_SKB_PAD);
  285. skb->dev = dev;
  286. }
  287. return skb;
  288. }
  289. EXPORT_SYMBOL(__netdev_alloc_skb);
  290. void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
  291. int size, unsigned int truesize)
  292. {
  293. skb_fill_page_desc(skb, i, page, off, size);
  294. skb->len += size;
  295. skb->data_len += size;
  296. skb->truesize += truesize;
  297. }
  298. EXPORT_SYMBOL(skb_add_rx_frag);
  299. /**
  300. * dev_alloc_skb - allocate an skbuff for receiving
  301. * @length: length to allocate
  302. *
  303. * Allocate a new &sk_buff and assign it a usage count of one. The
  304. * buffer has unspecified headroom built in. Users should allocate
  305. * the headroom they think they need without accounting for the
  306. * built in space. The built in space is used for optimisations.
  307. *
  308. * %NULL is returned if there is no free memory. Although this function
  309. * allocates memory it can be called from an interrupt.
  310. */
  311. struct sk_buff *dev_alloc_skb(unsigned int length)
  312. {
  313. /*
  314. * There is more code here than it seems:
  315. * __dev_alloc_skb is an inline
  316. */
  317. return __dev_alloc_skb(length, GFP_ATOMIC);
  318. }
  319. EXPORT_SYMBOL(dev_alloc_skb);
  320. static void skb_drop_list(struct sk_buff **listp)
  321. {
  322. struct sk_buff *list = *listp;
  323. *listp = NULL;
  324. do {
  325. struct sk_buff *this = list;
  326. list = list->next;
  327. kfree_skb(this);
  328. } while (list);
  329. }
  330. static inline void skb_drop_fraglist(struct sk_buff *skb)
  331. {
  332. skb_drop_list(&skb_shinfo(skb)->frag_list);
  333. }
  334. static void skb_clone_fraglist(struct sk_buff *skb)
  335. {
  336. struct sk_buff *list;
  337. skb_walk_frags(skb, list)
  338. skb_get(list);
  339. }
  340. static void skb_free_head(struct sk_buff *skb)
  341. {
  342. if (skb->head_frag)
  343. put_page(virt_to_head_page(skb->head));
  344. else
  345. kfree(skb->head);
  346. }
  347. static void skb_release_data(struct sk_buff *skb)
  348. {
  349. if (!skb->cloned ||
  350. !atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
  351. &skb_shinfo(skb)->dataref)) {
  352. if (skb_shinfo(skb)->nr_frags) {
  353. int i;
  354. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
  355. skb_frag_unref(skb, i);
  356. }
  357. /*
  358. * If skb buf is from userspace, we need to notify the caller
  359. * the lower device DMA has done;
  360. */
  361. if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
  362. struct ubuf_info *uarg;
  363. uarg = skb_shinfo(skb)->destructor_arg;
  364. if (uarg->callback)
  365. uarg->callback(uarg);
  366. }
  367. if (skb_has_frag_list(skb))
  368. skb_drop_fraglist(skb);
  369. skb_free_head(skb);
  370. }
  371. }
  372. /*
  373. * Free an skbuff by memory without cleaning the state.
  374. */
  375. static void kfree_skbmem(struct sk_buff *skb)
  376. {
  377. struct sk_buff *other;
  378. atomic_t *fclone_ref;
  379. switch (skb->fclone) {
  380. case SKB_FCLONE_UNAVAILABLE:
  381. kmem_cache_free(skbuff_head_cache, skb);
  382. break;
  383. case SKB_FCLONE_ORIG:
  384. fclone_ref = (atomic_t *) (skb + 2);
  385. if (atomic_dec_and_test(fclone_ref))
  386. kmem_cache_free(skbuff_fclone_cache, skb);
  387. break;
  388. case SKB_FCLONE_CLONE:
  389. fclone_ref = (atomic_t *) (skb + 1);
  390. other = skb - 1;
  391. /* The clone portion is available for
  392. * fast-cloning again.
  393. */
  394. skb->fclone = SKB_FCLONE_UNAVAILABLE;
  395. if (atomic_dec_and_test(fclone_ref))
  396. kmem_cache_free(skbuff_fclone_cache, other);
  397. break;
  398. }
  399. }
  400. static void skb_release_head_state(struct sk_buff *skb)
  401. {
  402. skb_dst_drop(skb);
  403. #ifdef CONFIG_XFRM
  404. secpath_put(skb->sp);
  405. #endif
  406. if (skb->destructor) {
  407. WARN_ON(in_irq());
  408. skb->destructor(skb);
  409. }
  410. #if IS_ENABLED(CONFIG_NF_CONNTRACK)
  411. nf_conntrack_put(skb->nfct);
  412. #endif
  413. #ifdef NET_SKBUFF_NF_DEFRAG_NEEDED
  414. nf_conntrack_put_reasm(skb->nfct_reasm);
  415. #endif
  416. #ifdef CONFIG_BRIDGE_NETFILTER
  417. nf_bridge_put(skb->nf_bridge);
  418. #endif
  419. /* XXX: IS this still necessary? - JHS */
  420. #ifdef CONFIG_NET_SCHED
  421. skb->tc_index = 0;
  422. #ifdef CONFIG_NET_CLS_ACT
  423. skb->tc_verd = 0;
  424. #endif
  425. #endif
  426. }
  427. /* Free everything but the sk_buff shell. */
  428. static void skb_release_all(struct sk_buff *skb)
  429. {
  430. skb_release_head_state(skb);
  431. skb_release_data(skb);
  432. }
  433. /**
  434. * __kfree_skb - private function
  435. * @skb: buffer
  436. *
  437. * Free an sk_buff. Release anything attached to the buffer.
  438. * Clean the state. This is an internal helper function. Users should
  439. * always call kfree_skb
  440. */
  441. void __kfree_skb(struct sk_buff *skb)
  442. {
  443. skb_release_all(skb);
  444. kfree_skbmem(skb);
  445. }
  446. EXPORT_SYMBOL(__kfree_skb);
  447. /**
  448. * kfree_skb - free an sk_buff
  449. * @skb: buffer to free
  450. *
  451. * Drop a reference to the buffer and free it if the usage count has
  452. * hit zero.
  453. */
  454. void kfree_skb(struct sk_buff *skb)
  455. {
  456. if (unlikely(!skb))
  457. return;
  458. if (likely(atomic_read(&skb->users) == 1))
  459. smp_rmb();
  460. else if (likely(!atomic_dec_and_test(&skb->users)))
  461. return;
  462. trace_kfree_skb(skb, __builtin_return_address(0));
  463. __kfree_skb(skb);
  464. }
  465. EXPORT_SYMBOL(kfree_skb);
  466. /**
  467. * consume_skb - free an skbuff
  468. * @skb: buffer to free
  469. *
  470. * Drop a ref to the buffer and free it if the usage count has hit zero
  471. * Functions identically to kfree_skb, but kfree_skb assumes that the frame
  472. * is being dropped after a failure and notes that
  473. */
  474. void consume_skb(struct sk_buff *skb)
  475. {
  476. if (unlikely(!skb))
  477. return;
  478. if (likely(atomic_read(&skb->users) == 1))
  479. smp_rmb();
  480. else if (likely(!atomic_dec_and_test(&skb->users)))
  481. return;
  482. trace_consume_skb(skb);
  483. __kfree_skb(skb);
  484. }
  485. EXPORT_SYMBOL(consume_skb);
  486. /**
  487. * skb_recycle - clean up an skb for reuse
  488. * @skb: buffer
  489. *
  490. * Recycles the skb to be reused as a receive buffer. This
  491. * function does any necessary reference count dropping, and
  492. * cleans up the skbuff as if it just came from __alloc_skb().
  493. */
  494. void skb_recycle(struct sk_buff *skb)
  495. {
  496. struct skb_shared_info *shinfo;
  497. skb_release_head_state(skb);
  498. shinfo = skb_shinfo(skb);
  499. memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
  500. atomic_set(&shinfo->dataref, 1);
  501. memset(skb, 0, offsetof(struct sk_buff, tail));
  502. skb->data = skb->head + NET_SKB_PAD;
  503. skb_reset_tail_pointer(skb);
  504. }
  505. EXPORT_SYMBOL(skb_recycle);
  506. /**
  507. * skb_recycle_check - check if skb can be reused for receive
  508. * @skb: buffer
  509. * @skb_size: minimum receive buffer size
  510. *
  511. * Checks that the skb passed in is not shared or cloned, and
  512. * that it is linear and its head portion at least as large as
  513. * skb_size so that it can be recycled as a receive buffer.
  514. * If these conditions are met, this function does any necessary
  515. * reference count dropping and cleans up the skbuff as if it
  516. * just came from __alloc_skb().
  517. */
  518. bool skb_recycle_check(struct sk_buff *skb, int skb_size)
  519. {
  520. if (!skb_is_recycleable(skb, skb_size))
  521. return false;
  522. skb_recycle(skb);
  523. return true;
  524. }
  525. EXPORT_SYMBOL(skb_recycle_check);
  526. static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
  527. {
  528. new->tstamp = old->tstamp;
  529. new->dev = old->dev;
  530. new->transport_header = old->transport_header;
  531. new->network_header = old->network_header;
  532. new->mac_header = old->mac_header;
  533. skb_dst_copy(new, old);
  534. new->rxhash = old->rxhash;
  535. new->ooo_okay = old->ooo_okay;
  536. new->l4_rxhash = old->l4_rxhash;
  537. new->no_fcs = old->no_fcs;
  538. #ifdef CONFIG_XFRM
  539. new->sp = secpath_get(old->sp);
  540. #endif
  541. memcpy(new->cb, old->cb, sizeof(old->cb));
  542. new->csum = old->csum;
  543. new->local_df = old->local_df;
  544. new->pkt_type = old->pkt_type;
  545. new->ip_summed = old->ip_summed;
  546. skb_copy_queue_mapping(new, old);
  547. new->priority = old->priority;
  548. #if IS_ENABLED(CONFIG_IP_VS)
  549. new->ipvs_property = old->ipvs_property;
  550. #endif
  551. new->protocol = old->protocol;
  552. new->mark = old->mark;
  553. new->skb_iif = old->skb_iif;
  554. __nf_copy(new, old);
  555. #if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE)
  556. new->nf_trace = old->nf_trace;
  557. #endif
  558. #ifdef CONFIG_NET_SCHED
  559. new->tc_index = old->tc_index;
  560. #ifdef CONFIG_NET_CLS_ACT
  561. new->tc_verd = old->tc_verd;
  562. #endif
  563. #endif
  564. new->vlan_tci = old->vlan_tci;
  565. skb_copy_secmark(new, old);
  566. }
  567. /*
  568. * You should not add any new code to this function. Add it to
  569. * __copy_skb_header above instead.
  570. */
  571. static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
  572. {
  573. #define C(x) n->x = skb->x
  574. n->next = n->prev = NULL;
  575. n->sk = NULL;
  576. __copy_skb_header(n, skb);
  577. C(len);
  578. C(data_len);
  579. C(mac_len);
  580. n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
  581. n->cloned = 1;
  582. n->nohdr = 0;
  583. n->destructor = NULL;
  584. C(tail);
  585. C(end);
  586. C(head);
  587. C(head_frag);
  588. C(data);
  589. C(truesize);
  590. atomic_set(&n->users, 1);
  591. atomic_inc(&(skb_shinfo(skb)->dataref));
  592. skb->cloned = 1;
  593. return n;
  594. #undef C
  595. }
  596. /**
  597. * skb_morph - morph one skb into another
  598. * @dst: the skb to receive the contents
  599. * @src: the skb to supply the contents
  600. *
  601. * This is identical to skb_clone except that the target skb is
  602. * supplied by the user.
  603. *
  604. * The target skb is returned upon exit.
  605. */
  606. struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
  607. {
  608. skb_release_all(dst);
  609. return __skb_clone(dst, src);
  610. }
  611. EXPORT_SYMBOL_GPL(skb_morph);
  612. /* skb_copy_ubufs - copy userspace skb frags buffers to kernel
  613. * @skb: the skb to modify
  614. * @gfp_mask: allocation priority
  615. *
  616. * This must be called on SKBTX_DEV_ZEROCOPY skb.
  617. * It will copy all frags into kernel and drop the reference
  618. * to userspace pages.
  619. *
  620. * If this function is called from an interrupt gfp_mask() must be
  621. * %GFP_ATOMIC.
  622. *
  623. * Returns 0 on success or a negative error code on failure
  624. * to allocate kernel memory to copy to.
  625. */
  626. int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
  627. {
  628. int i;
  629. int num_frags = skb_shinfo(skb)->nr_frags;
  630. struct page *page, *head = NULL;
  631. struct ubuf_info *uarg = skb_shinfo(skb)->destructor_arg;
  632. for (i = 0; i < num_frags; i++) {
  633. u8 *vaddr;
  634. skb_frag_t *f = &skb_shinfo(skb)->frags[i];
  635. page = alloc_page(GFP_ATOMIC);
  636. if (!page) {
  637. while (head) {
  638. struct page *next = (struct page *)head->private;
  639. put_page(head);
  640. head = next;
  641. }
  642. return -ENOMEM;
  643. }
  644. vaddr = kmap_atomic(skb_frag_page(f));
  645. memcpy(page_address(page),
  646. vaddr + f->page_offset, skb_frag_size(f));
  647. kunmap_atomic(vaddr);
  648. page->private = (unsigned long)head;
  649. head = page;
  650. }
  651. /* skb frags release userspace buffers */
  652. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
  653. skb_frag_unref(skb, i);
  654. uarg->callback(uarg);
  655. /* skb frags point to kernel buffers */
  656. for (i = skb_shinfo(skb)->nr_frags; i > 0; i--) {
  657. __skb_fill_page_desc(skb, i-1, head, 0,
  658. skb_shinfo(skb)->frags[i - 1].size);
  659. head = (struct page *)head->private;
  660. }
  661. skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY;
  662. return 0;
  663. }
  664. /**
  665. * skb_clone - duplicate an sk_buff
  666. * @skb: buffer to clone
  667. * @gfp_mask: allocation priority
  668. *
  669. * Duplicate an &sk_buff. The new one is not owned by a socket. Both
  670. * copies share the same packet data but not structure. The new
  671. * buffer has a reference count of 1. If the allocation fails the
  672. * function returns %NULL otherwise the new buffer is returned.
  673. *
  674. * If this function is called from an interrupt gfp_mask() must be
  675. * %GFP_ATOMIC.
  676. */
  677. struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
  678. {
  679. struct sk_buff *n;
  680. if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
  681. if (skb_copy_ubufs(skb, gfp_mask))
  682. return NULL;
  683. }
  684. n = skb + 1;
  685. if (skb->fclone == SKB_FCLONE_ORIG &&
  686. n->fclone == SKB_FCLONE_UNAVAILABLE) {
  687. atomic_t *fclone_ref = (atomic_t *) (n + 1);
  688. n->fclone = SKB_FCLONE_CLONE;
  689. atomic_inc(fclone_ref);
  690. } else {
  691. n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
  692. if (!n)
  693. return NULL;
  694. kmemcheck_annotate_bitfield(n, flags1);
  695. kmemcheck_annotate_bitfield(n, flags2);
  696. n->fclone = SKB_FCLONE_UNAVAILABLE;
  697. }
  698. return __skb_clone(n, skb);
  699. }
  700. EXPORT_SYMBOL(skb_clone);
  701. static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
  702. {
  703. #ifndef NET_SKBUFF_DATA_USES_OFFSET
  704. /*
  705. * Shift between the two data areas in bytes
  706. */
  707. unsigned long offset = new->data - old->data;
  708. #endif
  709. __copy_skb_header(new, old);
  710. #ifndef NET_SKBUFF_DATA_USES_OFFSET
  711. /* {transport,network,mac}_header are relative to skb->head */
  712. new->transport_header += offset;
  713. new->network_header += offset;
  714. if (skb_mac_header_was_set(new))
  715. new->mac_header += offset;
  716. #endif
  717. skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
  718. skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
  719. skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
  720. }
  721. /**
  722. * skb_copy - create private copy of an sk_buff
  723. * @skb: buffer to copy
  724. * @gfp_mask: allocation priority
  725. *
  726. * Make a copy of both an &sk_buff and its data. This is used when the
  727. * caller wishes to modify the data and needs a private copy of the
  728. * data to alter. Returns %NULL on failure or the pointer to the buffer
  729. * on success. The returned buffer has a reference count of 1.
  730. *
  731. * As by-product this function converts non-linear &sk_buff to linear
  732. * one, so that &sk_buff becomes completely private and caller is allowed
  733. * to modify all the data of returned buffer. This means that this
  734. * function is not recommended for use in circumstances when only
  735. * header is going to be modified. Use pskb_copy() instead.
  736. */
  737. struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
  738. {
  739. int headerlen = skb_headroom(skb);
  740. unsigned int size = (skb_end_pointer(skb) - skb->head) + skb->data_len;
  741. struct sk_buff *n = alloc_skb(size, gfp_mask);
  742. if (!n)
  743. return NULL;
  744. /* Set the data pointer */
  745. skb_reserve(n, headerlen);
  746. /* Set the tail pointer and length */
  747. skb_put(n, skb->len);
  748. if (skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len))
  749. BUG();
  750. copy_skb_header(n, skb);
  751. return n;
  752. }
  753. EXPORT_SYMBOL(skb_copy);
  754. /**
  755. * __pskb_copy - create copy of an sk_buff with private head.
  756. * @skb: buffer to copy
  757. * @headroom: headroom of new skb
  758. * @gfp_mask: allocation priority
  759. *
  760. * Make a copy of both an &sk_buff and part of its data, located
  761. * in header. Fragmented data remain shared. This is used when
  762. * the caller wishes to modify only header of &sk_buff and needs
  763. * private copy of the header to alter. Returns %NULL on failure
  764. * or the pointer to the buffer on success.
  765. * The returned buffer has a reference count of 1.
  766. */
  767. struct sk_buff *__pskb_copy(struct sk_buff *skb, int headroom, gfp_t gfp_mask)
  768. {
  769. unsigned int size = skb_headlen(skb) + headroom;
  770. struct sk_buff *n = alloc_skb(size, gfp_mask);
  771. if (!n)
  772. goto out;
  773. /* Set the data pointer */
  774. skb_reserve(n, headroom);
  775. /* Set the tail pointer and length */
  776. skb_put(n, skb_headlen(skb));
  777. /* Copy the bytes */
  778. skb_copy_from_linear_data(skb, n->data, n->len);
  779. n->truesize += skb->data_len;
  780. n->data_len = skb->data_len;
  781. n->len = skb->len;
  782. if (skb_shinfo(skb)->nr_frags) {
  783. int i;
  784. if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
  785. if (skb_copy_ubufs(skb, gfp_mask)) {
  786. kfree_skb(n);
  787. n = NULL;
  788. goto out;
  789. }
  790. }
  791. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  792. skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
  793. skb_frag_ref(skb, i);
  794. }
  795. skb_shinfo(n)->nr_frags = i;
  796. }
  797. if (skb_has_frag_list(skb)) {
  798. skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
  799. skb_clone_fraglist(n);
  800. }
  801. copy_skb_header(n, skb);
  802. out:
  803. return n;
  804. }
  805. EXPORT_SYMBOL(__pskb_copy);
  806. /**
  807. * pskb_expand_head - reallocate header of &sk_buff
  808. * @skb: buffer to reallocate
  809. * @nhead: room to add at head
  810. * @ntail: room to add at tail
  811. * @gfp_mask: allocation priority
  812. *
  813. * Expands (or creates identical copy, if &nhead and &ntail are zero)
  814. * header of skb. &sk_buff itself is not changed. &sk_buff MUST have
  815. * reference count of 1. Returns zero in the case of success or error,
  816. * if expansion failed. In the last case, &sk_buff is not changed.
  817. *
  818. * All the pointers pointing into skb header may change and must be
  819. * reloaded after call to this function.
  820. */
  821. int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
  822. gfp_t gfp_mask)
  823. {
  824. int i;
  825. u8 *data;
  826. int size = nhead + (skb_end_pointer(skb) - skb->head) + ntail;
  827. long off;
  828. bool fastpath;
  829. BUG_ON(nhead < 0);
  830. if (skb_shared(skb))
  831. BUG();
  832. size = SKB_DATA_ALIGN(size);
  833. /* Check if we can avoid taking references on fragments if we own
  834. * the last reference on skb->head. (see skb_release_data())
  835. */
  836. if (!skb->cloned)
  837. fastpath = true;
  838. else {
  839. int delta = skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1;
  840. fastpath = atomic_read(&skb_shinfo(skb)->dataref) == delta;
  841. }
  842. if (fastpath && !skb->head_frag &&
  843. size + sizeof(struct skb_shared_info) <= ksize(skb->head)) {
  844. memmove(skb->head + size, skb_shinfo(skb),
  845. offsetof(struct skb_shared_info,
  846. frags[skb_shinfo(skb)->nr_frags]));
  847. memmove(skb->head + nhead, skb->head,
  848. skb_tail_pointer(skb) - skb->head);
  849. off = nhead;
  850. goto adjust_others;
  851. }
  852. data = kmalloc(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
  853. gfp_mask);
  854. if (!data)
  855. goto nodata;
  856. size = SKB_WITH_OVERHEAD(ksize(data));
  857. /* Copy only real data... and, alas, header. This should be
  858. * optimized for the cases when header is void.
  859. */
  860. memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head);
  861. memcpy((struct skb_shared_info *)(data + size),
  862. skb_shinfo(skb),
  863. offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags]));
  864. if (fastpath) {
  865. skb_free_head(skb);
  866. } else {
  867. /* copy this zero copy skb frags */
  868. if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
  869. if (skb_copy_ubufs(skb, gfp_mask))
  870. goto nofrags;
  871. }
  872. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
  873. skb_frag_ref(skb, i);
  874. if (skb_has_frag_list(skb))
  875. skb_clone_fraglist(skb);
  876. skb_release_data(skb);
  877. }
  878. off = (data + nhead) - skb->head;
  879. skb->head = data;
  880. skb->head_frag = 0;
  881. adjust_others:
  882. skb->data += off;
  883. #ifdef NET_SKBUFF_DATA_USES_OFFSET
  884. skb->end = size;
  885. off = nhead;
  886. #else
  887. skb->end = skb->head + size;
  888. #endif
  889. /* {transport,network,mac}_header and tail are relative to skb->head */
  890. skb->tail += off;
  891. skb->transport_header += off;
  892. skb->network_header += off;
  893. if (skb_mac_header_was_set(skb))
  894. skb->mac_header += off;
  895. /* Only adjust this if it actually is csum_start rather than csum */
  896. if (skb->ip_summed == CHECKSUM_PARTIAL)
  897. skb->csum_start += nhead;
  898. skb->cloned = 0;
  899. skb->hdr_len = 0;
  900. skb->nohdr = 0;
  901. atomic_set(&skb_shinfo(skb)->dataref, 1);
  902. return 0;
  903. nofrags:
  904. kfree(data);
  905. nodata:
  906. return -ENOMEM;
  907. }
  908. EXPORT_SYMBOL(pskb_expand_head);
  909. /* Make private copy of skb with writable head and some headroom */
  910. struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
  911. {
  912. struct sk_buff *skb2;
  913. int delta = headroom - skb_headroom(skb);
  914. if (delta <= 0)
  915. skb2 = pskb_copy(skb, GFP_ATOMIC);
  916. else {
  917. skb2 = skb_clone(skb, GFP_ATOMIC);
  918. if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
  919. GFP_ATOMIC)) {
  920. kfree_skb(skb2);
  921. skb2 = NULL;
  922. }
  923. }
  924. return skb2;
  925. }
  926. EXPORT_SYMBOL(skb_realloc_headroom);
  927. /**
  928. * skb_copy_expand - copy and expand sk_buff
  929. * @skb: buffer to copy
  930. * @newheadroom: new free bytes at head
  931. * @newtailroom: new free bytes at tail
  932. * @gfp_mask: allocation priority
  933. *
  934. * Make a copy of both an &sk_buff and its data and while doing so
  935. * allocate additional space.
  936. *
  937. * This is used when the caller wishes to modify the data and needs a
  938. * private copy of the data to alter as well as more space for new fields.
  939. * Returns %NULL on failure or the pointer to the buffer
  940. * on success. The returned buffer has a reference count of 1.
  941. *
  942. * You must pass %GFP_ATOMIC as the allocation priority if this function
  943. * is called from an interrupt.
  944. */
  945. struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
  946. int newheadroom, int newtailroom,
  947. gfp_t gfp_mask)
  948. {
  949. /*
  950. * Allocate the copy buffer
  951. */
  952. struct sk_buff *n = alloc_skb(newheadroom + skb->len + newtailroom,
  953. gfp_mask);
  954. int oldheadroom = skb_headroom(skb);
  955. int head_copy_len, head_copy_off;
  956. int off;
  957. if (!n)
  958. return NULL;
  959. skb_reserve(n, newheadroom);
  960. /* Set the tail pointer and length */
  961. skb_put(n, skb->len);
  962. head_copy_len = oldheadroom;
  963. head_copy_off = 0;
  964. if (newheadroom <= head_copy_len)
  965. head_copy_len = newheadroom;
  966. else
  967. head_copy_off = newheadroom - head_copy_len;
  968. /* Copy the linear header and data. */
  969. if (skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
  970. skb->len + head_copy_len))
  971. BUG();
  972. copy_skb_header(n, skb);
  973. off = newheadroom - oldheadroom;
  974. if (n->ip_summed == CHECKSUM_PARTIAL)
  975. n->csum_start += off;
  976. #ifdef NET_SKBUFF_DATA_USES_OFFSET
  977. n->transport_header += off;
  978. n->network_header += off;
  979. if (skb_mac_header_was_set(skb))
  980. n->mac_header += off;
  981. #endif
  982. return n;
  983. }
  984. EXPORT_SYMBOL(skb_copy_expand);
  985. /**
  986. * skb_pad - zero pad the tail of an skb
  987. * @skb: buffer to pad
  988. * @pad: space to pad
  989. *
  990. * Ensure that a buffer is followed by a padding area that is zero
  991. * filled. Used by network drivers which may DMA or transfer data
  992. * beyond the buffer end onto the wire.
  993. *
  994. * May return error in out of memory cases. The skb is freed on error.
  995. */
  996. int skb_pad(struct sk_buff *skb, int pad)
  997. {
  998. int err;
  999. int ntail;
  1000. /* If the skbuff is non linear tailroom is always zero.. */
  1001. if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
  1002. memset(skb->data+skb->len, 0, pad);
  1003. return 0;
  1004. }
  1005. ntail = skb->data_len + pad - (skb->end - skb->tail);
  1006. if (likely(skb_cloned(skb) || ntail > 0)) {
  1007. err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
  1008. if (unlikely(err))
  1009. goto free_skb;
  1010. }
  1011. /* FIXME: The use of this function with non-linear skb's really needs
  1012. * to be audited.
  1013. */
  1014. err = skb_linearize(skb);
  1015. if (unlikely(err))
  1016. goto free_skb;
  1017. memset(skb->data + skb->len, 0, pad);
  1018. return 0;
  1019. free_skb:
  1020. kfree_skb(skb);
  1021. return err;
  1022. }
  1023. EXPORT_SYMBOL(skb_pad);
  1024. /**
  1025. * skb_put - add data to a buffer
  1026. * @skb: buffer to use
  1027. * @len: amount of data to add
  1028. *
  1029. * This function extends the used data area of the buffer. If this would
  1030. * exceed the total buffer size the kernel will panic. A pointer to the
  1031. * first byte of the extra data is returned.
  1032. */
  1033. unsigned char *skb_put(struct sk_buff *skb, unsigned int len)
  1034. {
  1035. unsigned char *tmp = skb_tail_pointer(skb);
  1036. SKB_LINEAR_ASSERT(skb);
  1037. skb->tail += len;
  1038. skb->len += len;
  1039. if (unlikely(skb->tail > skb->end))
  1040. skb_over_panic(skb, len, __builtin_return_address(0));
  1041. return tmp;
  1042. }
  1043. EXPORT_SYMBOL(skb_put);
  1044. /**
  1045. * skb_push - add data to the start of a buffer
  1046. * @skb: buffer to use
  1047. * @len: amount of data to add
  1048. *
  1049. * This function extends the used data area of the buffer at the buffer
  1050. * start. If this would exceed the total buffer headroom the kernel will
  1051. * panic. A pointer to the first byte of the extra data is returned.
  1052. */
  1053. unsigned char *skb_push(struct sk_buff *skb, unsigned int len)
  1054. {
  1055. skb->data -= len;
  1056. skb->len += len;
  1057. if (unlikely(skb->data<skb->head))
  1058. skb_under_panic(skb, len, __builtin_return_address(0));
  1059. return skb->data;
  1060. }
  1061. EXPORT_SYMBOL(skb_push);
  1062. /**
  1063. * skb_pull - remove data from the start of a buffer
  1064. * @skb: buffer to use
  1065. * @len: amount of data to remove
  1066. *
  1067. * This function removes data from the start of a buffer, returning
  1068. * the memory to the headroom. A pointer to the next data in the buffer
  1069. * is returned. Once the data has been pulled future pushes will overwrite
  1070. * the old data.
  1071. */
  1072. unsigned char *skb_pull(struct sk_buff *skb, unsigned int len)
  1073. {
  1074. return skb_pull_inline(skb, len);
  1075. }
  1076. EXPORT_SYMBOL(skb_pull);
  1077. /**
  1078. * skb_trim - remove end from a buffer
  1079. * @skb: buffer to alter
  1080. * @len: new length
  1081. *
  1082. * Cut the length of a buffer down by removing data from the tail. If
  1083. * the buffer is already under the length specified it is not modified.
  1084. * The skb must be linear.
  1085. */
  1086. void skb_trim(struct sk_buff *skb, unsigned int len)
  1087. {
  1088. if (skb->len > len)
  1089. __skb_trim(skb, len);
  1090. }
  1091. EXPORT_SYMBOL(skb_trim);
  1092. /* Trims skb to length len. It can change skb pointers.
  1093. */
  1094. int ___pskb_trim(struct sk_buff *skb, unsigned int len)
  1095. {
  1096. struct sk_buff **fragp;
  1097. struct sk_buff *frag;
  1098. int offset = skb_headlen(skb);
  1099. int nfrags = skb_shinfo(skb)->nr_frags;
  1100. int i;
  1101. int err;
  1102. if (skb_cloned(skb) &&
  1103. unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
  1104. return err;
  1105. i = 0;
  1106. if (offset >= len)
  1107. goto drop_pages;
  1108. for (; i < nfrags; i++) {
  1109. int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]);
  1110. if (end < len) {
  1111. offset = end;
  1112. continue;
  1113. }
  1114. skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset);
  1115. drop_pages:
  1116. skb_shinfo(skb)->nr_frags = i;
  1117. for (; i < nfrags; i++)
  1118. skb_frag_unref(skb, i);
  1119. if (skb_has_frag_list(skb))
  1120. skb_drop_fraglist(skb);
  1121. goto done;
  1122. }
  1123. for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
  1124. fragp = &frag->next) {
  1125. int end = offset + frag->len;
  1126. if (skb_shared(frag)) {
  1127. struct sk_buff *nfrag;
  1128. nfrag = skb_clone(frag, GFP_ATOMIC);
  1129. if (unlikely(!nfrag))
  1130. return -ENOMEM;
  1131. nfrag->next = frag->next;
  1132. consume_skb(frag);
  1133. frag = nfrag;
  1134. *fragp = frag;
  1135. }
  1136. if (end < len) {
  1137. offset = end;
  1138. continue;
  1139. }
  1140. if (end > len &&
  1141. unlikely((err = pskb_trim(frag, len - offset))))
  1142. return err;
  1143. if (frag->next)
  1144. skb_drop_list(&frag->next);
  1145. break;
  1146. }
  1147. done:
  1148. if (len > skb_headlen(skb)) {
  1149. skb->data_len -= skb->len - len;
  1150. skb->len = len;
  1151. } else {
  1152. skb->len = len;
  1153. skb->data_len = 0;
  1154. skb_set_tail_pointer(skb, len);
  1155. }
  1156. return 0;
  1157. }
  1158. EXPORT_SYMBOL(___pskb_trim);
  1159. /**
  1160. * __pskb_pull_tail - advance tail of skb header
  1161. * @skb: buffer to reallocate
  1162. * @delta: number of bytes to advance tail
  1163. *
  1164. * The function makes a sense only on a fragmented &sk_buff,
  1165. * it expands header moving its tail forward and copying necessary
  1166. * data from fragmented part.
  1167. *
  1168. * &sk_buff MUST have reference count of 1.
  1169. *
  1170. * Returns %NULL (and &sk_buff does not change) if pull failed
  1171. * or value of new tail of skb in the case of success.
  1172. *
  1173. * All the pointers pointing into skb header may change and must be
  1174. * reloaded after call to this function.
  1175. */
  1176. /* Moves tail of skb head forward, copying data from fragmented part,
  1177. * when it is necessary.
  1178. * 1. It may fail due to malloc failure.
  1179. * 2. It may change skb pointers.
  1180. *
  1181. * It is pretty complicated. Luckily, it is called only in exceptional cases.
  1182. */
  1183. unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta)
  1184. {
  1185. /* If skb has not enough free space at tail, get new one
  1186. * plus 128 bytes for future expansions. If we have enough
  1187. * room at tail, reallocate without expansion only if skb is cloned.
  1188. */
  1189. int i, k, eat = (skb->tail + delta) - skb->end;
  1190. if (eat > 0 || skb_cloned(skb)) {
  1191. if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
  1192. GFP_ATOMIC))
  1193. return NULL;
  1194. }
  1195. if (skb_copy_bits(skb, skb_headlen(skb), skb_tail_pointer(skb), delta))
  1196. BUG();
  1197. /* Optimization: no fragments, no reasons to preestimate
  1198. * size of pulled pages. Superb.
  1199. */
  1200. if (!skb_has_frag_list(skb))
  1201. goto pull_pages;
  1202. /* Estimate size of pulled pages. */
  1203. eat = delta;
  1204. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  1205. int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
  1206. if (size >= eat)
  1207. goto pull_pages;
  1208. eat -= size;
  1209. }
  1210. /* If we need update frag list, we are in troubles.
  1211. * Certainly, it possible to add an offset to skb data,
  1212. * but taking into account that pulling is expected to
  1213. * be very rare operation, it is worth to fight against
  1214. * further bloating skb head and crucify ourselves here instead.
  1215. * Pure masohism, indeed. 8)8)
  1216. */
  1217. if (eat) {
  1218. struct sk_buff *list = skb_shinfo(skb)->frag_list;
  1219. struct sk_buff *clone = NULL;
  1220. struct sk_buff *insp = NULL;
  1221. do {
  1222. BUG_ON(!list);
  1223. if (list->len <= eat) {
  1224. /* Eaten as whole. */
  1225. eat -= list->len;
  1226. list = list->next;
  1227. insp = list;
  1228. } else {
  1229. /* Eaten partially. */
  1230. if (skb_shared(list)) {
  1231. /* Sucks! We need to fork list. :-( */
  1232. clone = skb_clone(list, GFP_ATOMIC);
  1233. if (!clone)
  1234. return NULL;
  1235. insp = list->next;
  1236. list = clone;
  1237. } else {
  1238. /* This may be pulled without
  1239. * problems. */
  1240. insp = list;
  1241. }
  1242. if (!pskb_pull(list, eat)) {
  1243. kfree_skb(clone);
  1244. return NULL;
  1245. }
  1246. break;
  1247. }
  1248. } while (eat);
  1249. /* Free pulled out fragments. */
  1250. while ((list = skb_shinfo(skb)->frag_list) != insp) {
  1251. skb_shinfo(skb)->frag_list = list->next;
  1252. kfree_skb(list);
  1253. }
  1254. /* And insert new clone at head. */
  1255. if (clone) {
  1256. clone->next = list;
  1257. skb_shinfo(skb)->frag_list = clone;
  1258. }
  1259. }
  1260. /* Success! Now we may commit changes to skb data. */
  1261. pull_pages:
  1262. eat = delta;
  1263. k = 0;
  1264. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  1265. int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
  1266. if (size <= eat) {
  1267. skb_frag_unref(skb, i);
  1268. eat -= size;
  1269. } else {
  1270. skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];
  1271. if (eat) {
  1272. skb_shinfo(skb)->frags[k].page_offset += eat;
  1273. skb_frag_size_sub(&skb_shinfo(skb)->frags[k], eat);
  1274. eat = 0;
  1275. }
  1276. k++;
  1277. }
  1278. }
  1279. skb_shinfo(skb)->nr_frags = k;
  1280. skb->tail += delta;
  1281. skb->data_len -= delta;
  1282. return skb_tail_pointer(skb);
  1283. }
  1284. EXPORT_SYMBOL(__pskb_pull_tail);
  1285. /**
  1286. * skb_copy_bits - copy bits from skb to kernel buffer
  1287. * @skb: source skb
  1288. * @offset: offset in source
  1289. * @to: destination buffer
  1290. * @len: number of bytes to copy
  1291. *
  1292. * Copy the specified number of bytes from the source skb to the
  1293. * destination buffer.
  1294. *
  1295. * CAUTION ! :
  1296. * If its prototype is ever changed,
  1297. * check arch/{*}/net/{*}.S files,
  1298. * since it is called from BPF assembly code.
  1299. */
  1300. int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
  1301. {
  1302. int start = skb_headlen(skb);
  1303. struct sk_buff *frag_iter;
  1304. int i, copy;
  1305. if (offset > (int)skb->len - len)
  1306. goto fault;
  1307. /* Copy header. */
  1308. if ((copy = start - offset) > 0) {
  1309. if (copy > len)
  1310. copy = len;
  1311. skb_copy_from_linear_data_offset(skb, offset, to, copy);
  1312. if ((len -= copy) == 0)
  1313. return 0;
  1314. offset += copy;
  1315. to += copy;
  1316. }
  1317. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  1318. int end;
  1319. skb_frag_t *f = &skb_shinfo(skb)->frags[i];
  1320. WARN_ON(start > offset + len);
  1321. end = start + skb_frag_size(f);
  1322. if ((copy = end - offset) > 0) {
  1323. u8 *vaddr;
  1324. if (copy > len)
  1325. copy = len;
  1326. vaddr = kmap_atomic(skb_frag_page(f));
  1327. memcpy(to,
  1328. vaddr + f->page_offset + offset - start,
  1329. copy);
  1330. kunmap_atomic(vaddr);
  1331. if ((len -= copy) == 0)
  1332. return 0;
  1333. offset += copy;
  1334. to += copy;
  1335. }
  1336. start = end;
  1337. }
  1338. skb_walk_frags(skb, frag_iter) {
  1339. int end;
  1340. WARN_ON(start > offset + len);
  1341. end = start + frag_iter->len;
  1342. if ((copy = end - offset) > 0) {
  1343. if (copy > len)
  1344. copy = len;
  1345. if (skb_copy_bits(frag_iter, offset - start, to, copy))
  1346. goto fault;
  1347. if ((len -= copy) == 0)
  1348. return 0;
  1349. offset += copy;
  1350. to += copy;
  1351. }
  1352. start = end;
  1353. }
  1354. if (!len)
  1355. return 0;
  1356. fault:
  1357. return -EFAULT;
  1358. }
  1359. EXPORT_SYMBOL(skb_copy_bits);
  1360. /*
  1361. * Callback from splice_to_pipe(), if we need to release some pages
  1362. * at the end of the spd in case we error'ed out in filling the pipe.
  1363. */
  1364. static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
  1365. {
  1366. put_page(spd->pages[i]);
  1367. }
  1368. static struct page *linear_to_page(struct page *page, unsigned int *len,
  1369. unsigned int *offset,
  1370. struct sk_buff *skb, struct sock *sk)
  1371. {
  1372. struct page *p = sk->sk_sndmsg_page;
  1373. unsigned int off;
  1374. if (!p) {
  1375. new_page:
  1376. p = sk->sk_sndmsg_page = alloc_pages(sk->sk_allocation, 0);
  1377. if (!p)
  1378. return NULL;
  1379. off = sk->sk_sndmsg_off = 0;
  1380. /* hold one ref to this page until it's full */
  1381. } else {
  1382. unsigned int mlen;
  1383. /* If we are the only user of the page, we can reset offset */
  1384. if (page_count(p) == 1)
  1385. sk->sk_sndmsg_off = 0;
  1386. off = sk->sk_sndmsg_off;
  1387. mlen = PAGE_SIZE - off;
  1388. if (mlen < 64 && mlen < *len) {
  1389. put_page(p);
  1390. goto new_page;
  1391. }
  1392. *len = min_t(unsigned int, *len, mlen);
  1393. }
  1394. memcpy(page_address(p) + off, page_address(page) + *offset, *len);
  1395. sk->sk_sndmsg_off += *len;
  1396. *offset = off;
  1397. return p;
  1398. }
  1399. static bool spd_can_coalesce(const struct splice_pipe_desc *spd,
  1400. struct page *page,
  1401. unsigned int offset)
  1402. {
  1403. return spd->nr_pages &&
  1404. spd->pages[spd->nr_pages - 1] == page &&
  1405. (spd->partial[spd->nr_pages - 1].offset +
  1406. spd->partial[spd->nr_pages - 1].len == offset);
  1407. }
  1408. /*
  1409. * Fill page/offset/length into spd, if it can hold more pages.
  1410. */
  1411. static bool spd_fill_page(struct splice_pipe_desc *spd,
  1412. struct pipe_inode_info *pipe, struct page *page,
  1413. unsigned int *len, unsigned int offset,
  1414. struct sk_buff *skb, bool linear,
  1415. struct sock *sk)
  1416. {
  1417. if (unlikely(spd->nr_pages == MAX_SKB_FRAGS))
  1418. return true;
  1419. if (linear) {
  1420. page = linear_to_page(page, len, &offset, skb, sk);
  1421. if (!page)
  1422. return true;
  1423. }
  1424. if (spd_can_coalesce(spd, page, offset)) {
  1425. spd->partial[spd->nr_pages - 1].len += *len;
  1426. return false;
  1427. }
  1428. get_page(page);
  1429. spd->pages[spd->nr_pages] = page;
  1430. spd->partial[spd->nr_pages].len = *len;
  1431. spd->partial[spd->nr_pages].offset = offset;
  1432. spd->nr_pages++;
  1433. return false;
  1434. }
  1435. static inline void __segment_seek(struct page **page, unsigned int *poff,
  1436. unsigned int *plen, unsigned int off)
  1437. {
  1438. unsigned long n;
  1439. *poff += off;
  1440. n = *poff / PAGE_SIZE;
  1441. if (n)
  1442. *page = nth_page(*page, n);
  1443. *poff = *poff % PAGE_SIZE;
  1444. *plen -= off;
  1445. }
  1446. static bool __splice_segment(struct page *page, unsigned int poff,
  1447. unsigned int plen, unsigned int *off,
  1448. unsigned int *len, struct sk_buff *skb,
  1449. struct splice_pipe_desc *spd, bool linear,
  1450. struct sock *sk,
  1451. struct pipe_inode_info *pipe)
  1452. {
  1453. if (!*len)
  1454. return true;
  1455. /* skip this segment if already processed */
  1456. if (*off >= plen) {
  1457. *off -= plen;
  1458. return false;
  1459. }
  1460. /* ignore any bits we already processed */
  1461. if (*off) {
  1462. __segment_seek(&page, &poff, &plen, *off);
  1463. *off = 0;
  1464. }
  1465. do {
  1466. unsigned int flen = min(*len, plen);
  1467. /* the linear region may spread across several pages */
  1468. flen = min_t(unsigned int, flen, PAGE_SIZE - poff);
  1469. if (spd_fill_page(spd, pipe, page, &flen, poff, skb, linear, sk))
  1470. return true;
  1471. __segment_seek(&page, &poff, &plen, flen);
  1472. *len -= flen;
  1473. } while (*len && plen);
  1474. return false;
  1475. }
  1476. /*
  1477. * Map linear and fragment data from the skb to spd. It reports true if the
  1478. * pipe is full or if we already spliced the requested length.
  1479. */
  1480. static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
  1481. unsigned int *offset, unsigned int *len,
  1482. struct splice_pipe_desc *spd, struct sock *sk)
  1483. {
  1484. int seg;
  1485. /*
  1486. * map the linear part
  1487. */
  1488. if (__splice_segment(virt_to_page(skb->data),
  1489. (unsigned long) skb->data & (PAGE_SIZE - 1),
  1490. skb_headlen(skb),
  1491. offset, len, skb, spd, true, sk, pipe))
  1492. return true;
  1493. /*
  1494. * then map the fragments
  1495. */
  1496. for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
  1497. const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
  1498. if (__splice_segment(skb_frag_page(f),
  1499. f->page_offset, skb_frag_size(f),
  1500. offset, len, skb, spd, false, sk, pipe))
  1501. return true;
  1502. }
  1503. return false;
  1504. }
  1505. /*
  1506. * Map data from the skb to a pipe. Should handle both the linear part,
  1507. * the fragments, and the frag list. It does NOT handle frag lists within
  1508. * the frag list, if such a thing exists. We'd probably need to recurse to
  1509. * handle that cleanly.
  1510. */
  1511. int skb_splice_bits(struct sk_buff *skb, unsigned int offset,
  1512. struct pipe_inode_info *pipe, unsigned int tlen,
  1513. unsigned int flags)
  1514. {
  1515. struct partial_page partial[MAX_SKB_FRAGS];
  1516. struct page *pages[MAX_SKB_FRAGS];
  1517. struct splice_pipe_desc spd = {
  1518. .pages = pages,
  1519. .partial = partial,
  1520. .flags = flags,
  1521. .ops = &sock_pipe_buf_ops,
  1522. .spd_release = sock_spd_release,
  1523. };
  1524. struct sk_buff *frag_iter;
  1525. struct sock *sk = skb->sk;
  1526. int ret = 0;
  1527. /*
  1528. * __skb_splice_bits() only fails if the output has no room left,
  1529. * so no point in going over the frag_list for the error case.
  1530. */
  1531. if (__skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk))
  1532. goto done;
  1533. else if (!tlen)
  1534. goto done;
  1535. /*
  1536. * now see if we have a frag_list to map
  1537. */
  1538. skb_walk_frags(skb, frag_iter) {
  1539. if (!tlen)
  1540. break;
  1541. if (__skb_splice_bits(frag_iter, pipe, &offset, &tlen, &spd, sk))
  1542. break;
  1543. }
  1544. done:
  1545. if (spd.nr_pages) {
  1546. /*
  1547. * Drop the socket lock, otherwise we have reverse
  1548. * locking dependencies between sk_lock and i_mutex
  1549. * here as compared to sendfile(). We enter here
  1550. * with the socket lock held, and splice_to_pipe() will
  1551. * grab the pipe inode lock. For sendfile() emulation,
  1552. * we call into ->sendpage() with the i_mutex lock held
  1553. * and networking will grab the socket lock.
  1554. */
  1555. release_sock(sk);
  1556. ret = splice_to_pipe(pipe, &spd);
  1557. lock_sock(sk);
  1558. }
  1559. return ret;
  1560. }
  1561. /**
  1562. * skb_store_bits - store bits from kernel buffer to skb
  1563. * @skb: destination buffer
  1564. * @offset: offset in destination
  1565. * @from: source buffer
  1566. * @len: number of bytes to copy
  1567. *
  1568. * Copy the specified number of bytes from the source buffer to the
  1569. * destination skb. This function handles all the messy bits of
  1570. * traversing fragment lists and such.
  1571. */
  1572. int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
  1573. {
  1574. int start = skb_headlen(skb);
  1575. struct sk_buff *frag_iter;
  1576. int i, copy;
  1577. if (offset > (int)skb->len - len)
  1578. goto fault;
  1579. if ((copy = start - offset) > 0) {
  1580. if (copy > len)
  1581. copy = len;
  1582. skb_copy_to_linear_data_offset(skb, offset, from, copy);
  1583. if ((len -= copy) == 0)
  1584. return 0;
  1585. offset += copy;
  1586. from += copy;
  1587. }
  1588. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  1589. skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
  1590. int end;
  1591. WARN_ON(start > offset + len);
  1592. end = start + skb_frag_size(frag);
  1593. if ((copy = end - offset) > 0) {
  1594. u8 *vaddr;
  1595. if (copy > len)
  1596. copy = len;
  1597. vaddr = kmap_atomic(skb_frag_page(frag));
  1598. memcpy(vaddr + frag->page_offset + offset - start,
  1599. from, copy);
  1600. kunmap_atomic(vaddr);
  1601. if ((len -= copy) == 0)
  1602. return 0;
  1603. offset += copy;
  1604. from += copy;
  1605. }
  1606. start = end;
  1607. }
  1608. skb_walk_frags(skb, frag_iter) {
  1609. int end;
  1610. WARN_ON(start > offset + len);
  1611. end = start + frag_iter->len;
  1612. if ((copy = end - offset) > 0) {
  1613. if (copy > len)
  1614. copy = len;
  1615. if (skb_store_bits(frag_iter, offset - start,
  1616. from, copy))
  1617. goto fault;
  1618. if ((len -= copy) == 0)
  1619. return 0;
  1620. offset += copy;
  1621. from += copy;
  1622. }
  1623. start = end;
  1624. }
  1625. if (!len)
  1626. return 0;
  1627. fault:
  1628. return -EFAULT;
  1629. }
  1630. EXPORT_SYMBOL(skb_store_bits);
  1631. /* Checksum skb data. */
  1632. __wsum skb_checksum(const struct sk_buff *skb, int offset,
  1633. int len, __wsum csum)
  1634. {
  1635. int start = skb_headlen(skb);
  1636. int i, copy = start - offset;
  1637. struct sk_buff *frag_iter;
  1638. int pos = 0;
  1639. /* Checksum header. */
  1640. if (copy > 0) {
  1641. if (copy > len)
  1642. copy = len;
  1643. csum = csum_partial(skb->data + offset, copy, csum);
  1644. if ((len -= copy) == 0)
  1645. return csum;
  1646. offset += copy;
  1647. pos = copy;
  1648. }
  1649. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  1650. int end;
  1651. skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
  1652. WARN_ON(start > offset + len);
  1653. end = start + skb_frag_size(frag);
  1654. if ((copy = end - offset) > 0) {
  1655. __wsum csum2;
  1656. u8 *vaddr;
  1657. if (copy > len)
  1658. copy = len;
  1659. vaddr = kmap_atomic(skb_frag_page(frag));
  1660. csum2 = csum_partial(vaddr + frag->page_offset +
  1661. offset - start, copy, 0);
  1662. kunmap_atomic(vaddr);
  1663. csum = csum_block_add(csum, csum2, pos);
  1664. if (!(len -= copy))
  1665. return csum;
  1666. offset += copy;
  1667. pos += copy;
  1668. }
  1669. start = end;
  1670. }
  1671. skb_walk_frags(skb, frag_iter) {
  1672. int end;
  1673. WARN_ON(start > offset + len);
  1674. end = start + frag_iter->len;
  1675. if ((copy = end - offset) > 0) {
  1676. __wsum csum2;
  1677. if (copy > len)
  1678. copy = len;
  1679. csum2 = skb_checksum(frag_iter, offset - start,
  1680. copy, 0);
  1681. csum = csum_block_add(csum, csum2, pos);
  1682. if ((len -= copy) == 0)
  1683. return csum;
  1684. offset += copy;
  1685. pos += copy;
  1686. }
  1687. start = end;
  1688. }
  1689. BUG_ON(len);
  1690. return csum;
  1691. }
  1692. EXPORT_SYMBOL(skb_checksum);
  1693. /* Both of above in one bottle. */
  1694. __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
  1695. u8 *to, int len, __wsum csum)
  1696. {
  1697. int start = skb_headlen(skb);
  1698. int i, copy = start - offset;
  1699. struct sk_buff *frag_iter;
  1700. int pos = 0;
  1701. /* Copy header. */
  1702. if (copy > 0) {
  1703. if (copy > len)
  1704. copy = len;
  1705. csum = csum_partial_copy_nocheck(skb->data + offset, to,
  1706. copy, csum);
  1707. if ((len -= copy) == 0)
  1708. return csum;
  1709. offset += copy;
  1710. to += copy;
  1711. pos = copy;
  1712. }
  1713. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  1714. int end;
  1715. WARN_ON(start > offset + len);
  1716. end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
  1717. if ((copy = end - offset) > 0) {
  1718. __wsum csum2;
  1719. u8 *vaddr;
  1720. skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
  1721. if (copy > len)
  1722. copy = len;
  1723. vaddr = kmap_atomic(skb_frag_page(frag));
  1724. csum2 = csum_partial_copy_nocheck(vaddr +
  1725. frag->page_offset +
  1726. offset - start, to,
  1727. copy, 0);
  1728. kunmap_atomic(vaddr);
  1729. csum = csum_block_add(csum, csum2, pos);
  1730. if (!(len -= copy))
  1731. return csum;
  1732. offset += copy;
  1733. to += copy;
  1734. pos += copy;
  1735. }
  1736. start = end;
  1737. }
  1738. skb_walk_frags(skb, frag_iter) {
  1739. __wsum csum2;
  1740. int end;
  1741. WARN_ON(start > offset + len);
  1742. end = start + frag_iter->len;
  1743. if ((copy = end - offset) > 0) {
  1744. if (copy > len)
  1745. copy = len;
  1746. csum2 = skb_copy_and_csum_bits(frag_iter,
  1747. offset - start,
  1748. to, copy, 0);
  1749. csum = csum_block_add(csum, csum2, pos);
  1750. if ((len -= copy) == 0)
  1751. return csum;
  1752. offset += copy;
  1753. to += copy;
  1754. pos += copy;
  1755. }
  1756. start = end;
  1757. }
  1758. BUG_ON(len);
  1759. return csum;
  1760. }
  1761. EXPORT_SYMBOL(skb_copy_and_csum_bits);
  1762. void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
  1763. {
  1764. __wsum csum;
  1765. long csstart;
  1766. if (skb->ip_summed == CHECKSUM_PARTIAL)
  1767. csstart = skb_checksum_start_offset(skb);
  1768. else
  1769. csstart = skb_headlen(skb);
  1770. BUG_ON(csstart > skb_headlen(skb));
  1771. skb_copy_from_linear_data(skb, to, csstart);
  1772. csum = 0;
  1773. if (csstart != skb->len)
  1774. csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
  1775. skb->len - csstart, 0);
  1776. if (skb->ip_summed == CHECKSUM_PARTIAL) {
  1777. long csstuff = csstart + skb->csum_offset;
  1778. *((__sum16 *)(to + csstuff)) = csum_fold(csum);
  1779. }
  1780. }
  1781. EXPORT_SYMBOL(skb_copy_and_csum_dev);
  1782. /**
  1783. * skb_dequeue - remove from the head of the queue
  1784. * @list: list to dequeue from
  1785. *
  1786. * Remove the head of the list. The list lock is taken so the function
  1787. * may be used safely with other locking list functions. The head item is
  1788. * returned or %NULL if the list is empty.
  1789. */
  1790. struct sk_buff *skb_dequeue(struct sk_buff_head *list)
  1791. {
  1792. unsigned long flags;
  1793. struct sk_buff *result;
  1794. spin_lock_irqsave(&list->lock, flags);
  1795. result = __skb_dequeue(list);
  1796. spin_unlock_irqrestore(&list->lock, flags);
  1797. return result;
  1798. }
  1799. EXPORT_SYMBOL(skb_dequeue);
  1800. /**
  1801. * skb_dequeue_tail - remove from the tail of the queue
  1802. * @list: list to dequeue from
  1803. *
  1804. * Remove the tail of the list. The list lock is taken so the function
  1805. * may be used safely with other locking list functions. The tail item is
  1806. * returned or %NULL if the list is empty.
  1807. */
  1808. struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
  1809. {
  1810. unsigned long flags;
  1811. struct sk_buff *result;
  1812. spin_lock_irqsave(&list->lock, flags);
  1813. result = __skb_dequeue_tail(list);
  1814. spin_unlock_irqrestore(&list->lock, flags);
  1815. return result;
  1816. }
  1817. EXPORT_SYMBOL(skb_dequeue_tail);
  1818. /**
  1819. * skb_queue_purge - empty a list
  1820. * @list: list to empty
  1821. *
  1822. * Delete all buffers on an &sk_buff list. Each buffer is removed from
  1823. * the list and one reference dropped. This function takes the list
  1824. * lock and is atomic with respect to other list locking functions.
  1825. */
  1826. void skb_queue_purge(struct sk_buff_head *list)
  1827. {
  1828. struct sk_buff *skb;
  1829. while ((skb = skb_dequeue(list)) != NULL)
  1830. kfree_skb(skb);
  1831. }
  1832. EXPORT_SYMBOL(skb_queue_purge);
  1833. /**
  1834. * skb_queue_head - queue a buffer at the list head
  1835. * @list: list to use
  1836. * @newsk: buffer to queue
  1837. *
  1838. * Queue a buffer at the start of the list. This function takes the
  1839. * list lock and can be used safely with other locking &sk_buff functions
  1840. * safely.
  1841. *
  1842. * A buffer cannot be placed on two lists at the same time.
  1843. */
  1844. void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
  1845. {
  1846. unsigned long flags;
  1847. spin_lock_irqsave(&list->lock, flags);
  1848. __skb_queue_head(list, newsk);
  1849. spin_unlock_irqrestore(&list->lock, flags);
  1850. }
  1851. EXPORT_SYMBOL(skb_queue_head);
  1852. /**
  1853. * skb_queue_tail - queue a buffer at the list tail
  1854. * @list: list to use
  1855. * @newsk: buffer to queue
  1856. *
  1857. * Queue a buffer at the tail of the list. This function takes the
  1858. * list lock and can be used safely with other locking &sk_buff functions
  1859. * safely.
  1860. *
  1861. * A buffer cannot be placed on two lists at the same time.
  1862. */
  1863. void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
  1864. {
  1865. unsigned long flags;
  1866. spin_lock_irqsave(&list->lock, flags);
  1867. __skb_queue_tail(list, newsk);
  1868. spin_unlock_irqrestore(&list->lock, flags);
  1869. }
  1870. EXPORT_SYMBOL(skb_queue_tail);
  1871. /**
  1872. * skb_unlink - remove a buffer from a list
  1873. * @skb: buffer to remove
  1874. * @list: list to use
  1875. *
  1876. * Remove a packet from a list. The list locks are taken and this
  1877. * function is atomic with respect to other list locked calls
  1878. *
  1879. * You must know what list the SKB is on.
  1880. */
  1881. void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
  1882. {
  1883. unsigned long flags;
  1884. spin_lock_irqsave(&list->lock, flags);
  1885. __skb_unlink(skb, list);
  1886. spin_unlock_irqrestore(&list->lock, flags);
  1887. }
  1888. EXPORT_SYMBOL(skb_unlink);
  1889. /**
  1890. * skb_append - append a buffer
  1891. * @old: buffer to insert after
  1892. * @newsk: buffer to insert
  1893. * @list: list to use
  1894. *
  1895. * Place a packet after a given packet in a list. The list locks are taken
  1896. * and this function is atomic with respect to other list locked calls.
  1897. * A buffer cannot be placed on two lists at the same time.
  1898. */
  1899. void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
  1900. {
  1901. unsigned long flags;
  1902. spin_lock_irqsave(&list->lock, flags);
  1903. __skb_queue_after(list, old, newsk);
  1904. spin_unlock_irqrestore(&list->lock, flags);
  1905. }
  1906. EXPORT_SYMBOL(skb_append);
  1907. /**
  1908. * skb_insert - insert a buffer
  1909. * @old: buffer to insert before
  1910. * @newsk: buffer to insert
  1911. * @list: list to use
  1912. *
  1913. * Place a packet before a given packet in a list. The list locks are
  1914. * taken and this function is atomic with respect to other list locked
  1915. * calls.
  1916. *
  1917. * A buffer cannot be placed on two lists at the same time.
  1918. */
  1919. void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
  1920. {
  1921. unsigned long flags;
  1922. spin_lock_irqsave(&list->lock, flags);
  1923. __skb_insert(newsk, old->prev, old, list);
  1924. spin_unlock_irqrestore(&list->lock, flags);
  1925. }
  1926. EXPORT_SYMBOL(skb_insert);
  1927. static inline void skb_split_inside_header(struct sk_buff *skb,
  1928. struct sk_buff* skb1,
  1929. const u32 len, const int pos)
  1930. {
  1931. int i;
  1932. skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
  1933. pos - len);
  1934. /* And move data appendix as is. */
  1935. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
  1936. skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
  1937. skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
  1938. skb_shinfo(skb)->nr_frags = 0;
  1939. skb1->data_len = skb->data_len;
  1940. skb1->len += skb1->data_len;
  1941. skb->data_len = 0;
  1942. skb->len = len;
  1943. skb_set_tail_pointer(skb, len);
  1944. }
  1945. static inline void skb_split_no_header(struct sk_buff *skb,
  1946. struct sk_buff* skb1,
  1947. const u32 len, int pos)
  1948. {
  1949. int i, k = 0;
  1950. const int nfrags = skb_shinfo(skb)->nr_frags;
  1951. skb_shinfo(skb)->nr_frags = 0;
  1952. skb1->len = skb1->data_len = skb->len - len;
  1953. skb->len = len;
  1954. skb->data_len = len - pos;
  1955. for (i = 0; i < nfrags; i++) {
  1956. int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
  1957. if (pos + size > len) {
  1958. skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
  1959. if (pos < len) {
  1960. /* Split frag.
  1961. * We have two variants in this case:
  1962. * 1. Move all the frag to the second
  1963. * part, if it is possible. F.e.
  1964. * this approach is mandatory for TUX,
  1965. * where splitting is expensive.
  1966. * 2. Split is accurately. We make this.
  1967. */
  1968. skb_frag_ref(skb, i);
  1969. skb_shinfo(skb1)->frags[0].page_offset += len - pos;
  1970. skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos);
  1971. skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos);
  1972. skb_shinfo(skb)->nr_frags++;
  1973. }
  1974. k++;
  1975. } else
  1976. skb_shinfo(skb)->nr_frags++;
  1977. pos += size;
  1978. }
  1979. skb_shinfo(skb1)->nr_frags = k;
  1980. }
  1981. /**
  1982. * skb_split - Split fragmented skb to two parts at length len.
  1983. * @skb: the buffer to split
  1984. * @skb1: the buffer to receive the second part
  1985. * @len: new length for skb
  1986. */
  1987. void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
  1988. {
  1989. int pos = skb_headlen(skb);
  1990. if (len < pos) /* Split line is inside header. */
  1991. skb_split_inside_header(skb, skb1, len, pos);
  1992. else /* Second chunk has no header, nothing to copy. */
  1993. skb_split_no_header(skb, skb1, len, pos);
  1994. }
  1995. EXPORT_SYMBOL(skb_split);
  1996. /* Shifting from/to a cloned skb is a no-go.
  1997. *
  1998. * Caller cannot keep skb_shinfo related pointers past calling here!
  1999. */
  2000. static int skb_prepare_for_shift(struct sk_buff *skb)
  2001. {
  2002. return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
  2003. }
  2004. /**
  2005. * skb_shift - Shifts paged data partially from skb to another
  2006. * @tgt: buffer into which tail data gets added
  2007. * @skb: buffer from which the paged data comes from
  2008. * @shiftlen: shift up to this many bytes
  2009. *
  2010. * Attempts to shift up to shiftlen worth of bytes, which may be less than
  2011. * the length of the skb, from skb to tgt. Returns number bytes shifted.
  2012. * It's up to caller to free skb if everything was shifted.
  2013. *
  2014. * If @tgt runs out of frags, the whole operation is aborted.
  2015. *
  2016. * Skb cannot include anything else but paged data while tgt is allowed
  2017. * to have non-paged data as well.
  2018. *
  2019. * TODO: full sized shift could be optimized but that would need
  2020. * specialized skb free'er to handle frags without up-to-date nr_frags.
  2021. */
  2022. int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
  2023. {
  2024. int from, to, merge, todo;
  2025. struct skb_frag_struct *fragfrom, *fragto;
  2026. BUG_ON(shiftlen > skb->len);
  2027. BUG_ON(skb_headlen(skb)); /* Would corrupt stream */
  2028. todo = shiftlen;
  2029. from = 0;
  2030. to = skb_shinfo(tgt)->nr_frags;
  2031. fragfrom = &skb_shinfo(skb)->frags[from];
  2032. /* Actual merge is delayed until the point when we know we can
  2033. * commit all, so that we don't have to undo partial changes
  2034. */
  2035. if (!to ||
  2036. !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom),
  2037. fragfrom->page_offset)) {
  2038. merge = -1;
  2039. } else {
  2040. merge = to - 1;
  2041. todo -= skb_frag_size(fragfrom);
  2042. if (todo < 0) {
  2043. if (skb_prepare_for_shift(skb) ||
  2044. skb_prepare_for_shift(tgt))
  2045. return 0;
  2046. /* All previous frag pointers might be stale! */
  2047. fragfrom = &skb_shinfo(skb)->frags[from];
  2048. fragto = &skb_shinfo(tgt)->frags[merge];
  2049. skb_frag_size_add(fragto, shiftlen);
  2050. skb_frag_size_sub(fragfrom, shiftlen);
  2051. fragfrom->page_offset += shiftlen;
  2052. goto onlymerged;
  2053. }
  2054. from++;
  2055. }
  2056. /* Skip full, not-fitting skb to avoid expensive operations */
  2057. if ((shiftlen == skb->len) &&
  2058. (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
  2059. return 0;
  2060. if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
  2061. return 0;
  2062. while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
  2063. if (to == MAX_SKB_FRAGS)
  2064. return 0;
  2065. fragfrom = &skb_shinfo(skb)->frags[from];
  2066. fragto = &skb_shinfo(tgt)->frags[to];
  2067. if (todo >= skb_frag_size(fragfrom)) {
  2068. *fragto = *fragfrom;
  2069. todo -= skb_frag_size(fragfrom);
  2070. from++;
  2071. to++;
  2072. } else {
  2073. __skb_frag_ref(fragfrom);
  2074. fragto->page = fragfrom->page;
  2075. fragto->page_offset = fragfrom->page_offset;
  2076. skb_frag_size_set(fragto, todo);
  2077. fragfrom->page_offset += todo;
  2078. skb_frag_size_sub(fragfrom, todo);
  2079. todo = 0;
  2080. to++;
  2081. break;
  2082. }
  2083. }
  2084. /* Ready to "commit" this state change to tgt */
  2085. skb_shinfo(tgt)->nr_frags = to;
  2086. if (merge >= 0) {
  2087. fragfrom = &skb_shinfo(skb)->frags[0];
  2088. fragto = &skb_shinfo(tgt)->frags[merge];
  2089. skb_frag_size_add(fragto, skb_frag_size(fragfrom));
  2090. __skb_frag_unref(fragfrom);
  2091. }
  2092. /* Reposition in the original skb */
  2093. to = 0;
  2094. while (from < skb_shinfo(skb)->nr_frags)
  2095. skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
  2096. skb_shinfo(skb)->nr_frags = to;
  2097. BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
  2098. onlymerged:
  2099. /* Most likely the tgt won't ever need its checksum anymore, skb on
  2100. * the other hand might need it if it needs to be resent
  2101. */
  2102. tgt->ip_summed = CHECKSUM_PARTIAL;
  2103. skb->ip_summed = CHECKSUM_PARTIAL;
  2104. /* Yak, is it really working this way? Some helper please? */
  2105. skb->len -= shiftlen;
  2106. skb->data_len -= shiftlen;
  2107. skb->truesize -= shiftlen;
  2108. tgt->len += shiftlen;
  2109. tgt->data_len += shiftlen;
  2110. tgt->truesize += shiftlen;
  2111. return shiftlen;
  2112. }
  2113. /**
  2114. * skb_prepare_seq_read - Prepare a sequential read of skb data
  2115. * @skb: the buffer to read
  2116. * @from: lower offset of data to be read
  2117. * @to: upper offset of data to be read
  2118. * @st: state variable
  2119. *
  2120. * Initializes the specified state variable. Must be called before
  2121. * invoking skb_seq_read() for the first time.
  2122. */
  2123. void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
  2124. unsigned int to, struct skb_seq_state *st)
  2125. {
  2126. st->lower_offset = from;
  2127. st->upper_offset = to;
  2128. st->root_skb = st->cur_skb = skb;
  2129. st->frag_idx = st->stepped_offset = 0;
  2130. st->frag_data = NULL;
  2131. }
  2132. EXPORT_SYMBOL(skb_prepare_seq_read);
  2133. /**
  2134. * skb_seq_read - Sequentially read skb data
  2135. * @consumed: number of bytes consumed by the caller so far
  2136. * @data: destination pointer for data to be returned
  2137. * @st: state variable
  2138. *
  2139. * Reads a block of skb data at &consumed relative to the
  2140. * lower offset specified to skb_prepare_seq_read(). Assigns
  2141. * the head of the data block to &data and returns the length
  2142. * of the block or 0 if the end of the skb data or the upper
  2143. * offset has been reached.
  2144. *
  2145. * The caller is not required to consume all of the data
  2146. * returned, i.e. &consumed is typically set to the number
  2147. * of bytes already consumed and the next call to
  2148. * skb_seq_read() will return the remaining part of the block.
  2149. *
  2150. * Note 1: The size of each block of data returned can be arbitrary,
  2151. * this limitation is the cost for zerocopy seqeuental
  2152. * reads of potentially non linear data.
  2153. *
  2154. * Note 2: Fragment lists within fragments are not implemented
  2155. * at the moment, state->root_skb could be replaced with
  2156. * a stack for this purpose.
  2157. */
  2158. unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
  2159. struct skb_seq_state *st)
  2160. {
  2161. unsigned int block_limit, abs_offset = consumed + st->lower_offset;
  2162. skb_frag_t *frag;
  2163. if (unlikely(abs_offset >= st->upper_offset))
  2164. return 0;
  2165. next_skb:
  2166. block_limit = skb_headlen(st->cur_skb) + st->stepped_offset;
  2167. if (abs_offset < block_limit && !st->frag_data) {
  2168. *data = st->cur_skb->data + (abs_offset - st->stepped_offset);
  2169. return block_limit - abs_offset;
  2170. }
  2171. if (st->frag_idx == 0 && !st->frag_data)
  2172. st->stepped_offset += skb_headlen(st->cur_skb);
  2173. while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
  2174. frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
  2175. block_limit = skb_frag_size(frag) + st->stepped_offset;
  2176. if (abs_offset < block_limit) {
  2177. if (!st->frag_data)
  2178. st->frag_data = kmap_atomic(skb_frag_page(frag));
  2179. *data = (u8 *) st->frag_data + frag->page_offset +
  2180. (abs_offset - st->stepped_offset);
  2181. return block_limit - abs_offset;
  2182. }
  2183. if (st->frag_data) {
  2184. kunmap_atomic(st->frag_data);
  2185. st->frag_data = NULL;
  2186. }
  2187. st->frag_idx++;
  2188. st->stepped_offset += skb_frag_size(frag);
  2189. }
  2190. if (st->frag_data) {
  2191. kunmap_atomic(st->frag_data);
  2192. st->frag_data = NULL;
  2193. }
  2194. if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) {
  2195. st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
  2196. st->frag_idx = 0;
  2197. goto next_skb;
  2198. } else if (st->cur_skb->next) {
  2199. st->cur_skb = st->cur_skb->next;
  2200. st->frag_idx = 0;
  2201. goto next_skb;
  2202. }
  2203. return 0;
  2204. }
  2205. EXPORT_SYMBOL(skb_seq_read);
  2206. /**
  2207. * skb_abort_seq_read - Abort a sequential read of skb data
  2208. * @st: state variable
  2209. *
  2210. * Must be called if skb_seq_read() was not called until it
  2211. * returned 0.
  2212. */
  2213. void skb_abort_seq_read(struct skb_seq_state *st)
  2214. {
  2215. if (st->frag_data)
  2216. kunmap_atomic(st->frag_data);
  2217. }
  2218. EXPORT_SYMBOL(skb_abort_seq_read);
  2219. #define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb))
  2220. static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
  2221. struct ts_config *conf,
  2222. struct ts_state *state)
  2223. {
  2224. return skb_seq_read(offset, text, TS_SKB_CB(state));
  2225. }
  2226. static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
  2227. {
  2228. skb_abort_seq_read(TS_SKB_CB(state));
  2229. }
  2230. /**
  2231. * skb_find_text - Find a text pattern in skb data
  2232. * @skb: the buffer to look in
  2233. * @from: search offset
  2234. * @to: search limit
  2235. * @config: textsearch configuration
  2236. * @state: uninitialized textsearch state variable
  2237. *
  2238. * Finds a pattern in the skb data according to the specified
  2239. * textsearch configuration. Use textsearch_next() to retrieve
  2240. * subsequent occurrences of the pattern. Returns the offset
  2241. * to the first occurrence or UINT_MAX if no match was found.
  2242. */
  2243. unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
  2244. unsigned int to, struct ts_config *config,
  2245. struct ts_state *state)
  2246. {
  2247. unsigned int ret;
  2248. config->get_next_block = skb_ts_get_next_block;
  2249. config->finish = skb_ts_finish;
  2250. skb_prepare_seq_read(skb, from, to, TS_SKB_CB(state));
  2251. ret = textsearch_find(config, state);
  2252. return (ret <= to - from ? ret : UINT_MAX);
  2253. }
  2254. EXPORT_SYMBOL(skb_find_text);
  2255. /**
  2256. * skb_append_datato_frags: - append the user data to a skb
  2257. * @sk: sock structure
  2258. * @skb: skb structure to be appened with user data.
  2259. * @getfrag: call back function to be used for getting the user data
  2260. * @from: pointer to user message iov
  2261. * @length: length of the iov message
  2262. *
  2263. * Description: This procedure append the user data in the fragment part
  2264. * of the skb if any page alloc fails user this procedure returns -ENOMEM
  2265. */
  2266. int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
  2267. int (*getfrag)(void *from, char *to, int offset,
  2268. int len, int odd, struct sk_buff *skb),
  2269. void *from, int length)
  2270. {
  2271. int frg_cnt = 0;
  2272. skb_frag_t *frag = NULL;
  2273. struct page *page = NULL;
  2274. int copy, left;
  2275. int offset = 0;
  2276. int ret;
  2277. do {
  2278. /* Return error if we don't have space for new frag */
  2279. frg_cnt = skb_shinfo(skb)->nr_frags;
  2280. if (frg_cnt >= MAX_SKB_FRAGS)
  2281. return -EFAULT;
  2282. /* allocate a new page for next frag */
  2283. page = alloc_pages(sk->sk_allocation, 0);
  2284. /* If alloc_page fails just return failure and caller will
  2285. * free previous allocated pages by doing kfree_skb()
  2286. */
  2287. if (page == NULL)
  2288. return -ENOMEM;
  2289. /* initialize the next frag */
  2290. skb_fill_page_desc(skb, frg_cnt, page, 0, 0);
  2291. skb->truesize += PAGE_SIZE;
  2292. atomic_add(PAGE_SIZE, &sk->sk_wmem_alloc);
  2293. /* get the new initialized frag */
  2294. frg_cnt = skb_shinfo(skb)->nr_frags;
  2295. frag = &skb_shinfo(skb)->frags[frg_cnt - 1];
  2296. /* copy the user data to page */
  2297. left = PAGE_SIZE - frag->page_offset;
  2298. copy = (length > left)? left : length;
  2299. ret = getfrag(from, skb_frag_address(frag) + skb_frag_size(frag),
  2300. offset, copy, 0, skb);
  2301. if (ret < 0)
  2302. return -EFAULT;
  2303. /* copy was successful so update the size parameters */
  2304. skb_frag_size_add(frag, copy);
  2305. skb->len += copy;
  2306. skb->data_len += copy;
  2307. offset += copy;
  2308. length -= copy;
  2309. } while (length > 0);
  2310. return 0;
  2311. }
  2312. EXPORT_SYMBOL(skb_append_datato_frags);
  2313. /**
  2314. * skb_pull_rcsum - pull skb and update receive checksum
  2315. * @skb: buffer to update
  2316. * @len: length of data pulled
  2317. *
  2318. * This function performs an skb_pull on the packet and updates
  2319. * the CHECKSUM_COMPLETE checksum. It should be used on
  2320. * receive path processing instead of skb_pull unless you know
  2321. * that the checksum difference is zero (e.g., a valid IP header)
  2322. * or you are setting ip_summed to CHECKSUM_NONE.
  2323. */
  2324. unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
  2325. {
  2326. BUG_ON(len > skb->len);
  2327. skb->len -= len;
  2328. BUG_ON(skb->len < skb->data_len);
  2329. skb_postpull_rcsum(skb, skb->data, len);
  2330. return skb->data += len;
  2331. }
  2332. EXPORT_SYMBOL_GPL(skb_pull_rcsum);
  2333. /**
  2334. * skb_segment - Perform protocol segmentation on skb.
  2335. * @skb: buffer to segment
  2336. * @features: features for the output path (see dev->features)
  2337. *
  2338. * This function performs segmentation on the given skb. It returns
  2339. * a pointer to the first in a list of new skbs for the segments.
  2340. * In case of error it returns ERR_PTR(err).
  2341. */
  2342. struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features)
  2343. {
  2344. struct sk_buff *segs = NULL;
  2345. struct sk_buff *tail = NULL;
  2346. struct sk_buff *fskb = skb_shinfo(skb)->frag_list;
  2347. unsigned int mss = skb_shinfo(skb)->gso_size;
  2348. unsigned int doffset = skb->data - skb_mac_header(skb);
  2349. unsigned int offset = doffset;
  2350. unsigned int headroom;
  2351. unsigned int len;
  2352. int sg = !!(features & NETIF_F_SG);
  2353. int nfrags = skb_shinfo(skb)->nr_frags;
  2354. int err = -ENOMEM;
  2355. int i = 0;
  2356. int pos;
  2357. __skb_push(skb, doffset);
  2358. headroom = skb_headroom(skb);
  2359. pos = skb_headlen(skb);
  2360. do {
  2361. struct sk_buff *nskb;
  2362. skb_frag_t *frag;
  2363. int hsize;
  2364. int size;
  2365. len = skb->len - offset;
  2366. if (len > mss)
  2367. len = mss;
  2368. hsize = skb_headlen(skb) - offset;
  2369. if (hsize < 0)
  2370. hsize = 0;
  2371. if (hsize > len || !sg)
  2372. hsize = len;
  2373. if (!hsize && i >= nfrags) {
  2374. BUG_ON(fskb->len != len);
  2375. pos += len;
  2376. nskb = skb_clone(fskb, GFP_ATOMIC);
  2377. fskb = fskb->next;
  2378. if (unlikely(!nskb))
  2379. goto err;
  2380. hsize = skb_end_pointer(nskb) - nskb->head;
  2381. if (skb_cow_head(nskb, doffset + headroom)) {
  2382. kfree_skb(nskb);
  2383. goto err;
  2384. }
  2385. nskb->truesize += skb_end_pointer(nskb) - nskb->head -
  2386. hsize;
  2387. skb_release_head_state(nskb);
  2388. __skb_push(nskb, doffset);
  2389. } else {
  2390. nskb = alloc_skb(hsize + doffset + headroom,
  2391. GFP_ATOMIC);
  2392. if (unlikely(!nskb))
  2393. goto err;
  2394. skb_reserve(nskb, headroom);
  2395. __skb_put(nskb, doffset);
  2396. }
  2397. if (segs)
  2398. tail->next = nskb;
  2399. else
  2400. segs = nskb;
  2401. tail = nskb;
  2402. __copy_skb_header(nskb, skb);
  2403. nskb->mac_len = skb->mac_len;
  2404. /* nskb and skb might have different headroom */
  2405. if (nskb->ip_summed == CHECKSUM_PARTIAL)
  2406. nskb->csum_start += skb_headroom(nskb) - headroom;
  2407. skb_reset_mac_header(nskb);
  2408. skb_set_network_header(nskb, skb->mac_len);
  2409. nskb->transport_header = (nskb->network_header +
  2410. skb_network_header_len(skb));
  2411. skb_copy_from_linear_data(skb, nskb->data, doffset);
  2412. if (fskb != skb_shinfo(skb)->frag_list)
  2413. continue;
  2414. if (!sg) {
  2415. nskb->ip_summed = CHECKSUM_NONE;
  2416. nskb->csum = skb_copy_and_csum_bits(skb, offset,
  2417. skb_put(nskb, len),
  2418. len, 0);
  2419. continue;
  2420. }
  2421. frag = skb_shinfo(nskb)->frags;
  2422. skb_copy_from_linear_data_offset(skb, offset,
  2423. skb_put(nskb, hsize), hsize);
  2424. while (pos < offset + len && i < nfrags) {
  2425. *frag = skb_shinfo(skb)->frags[i];
  2426. __skb_frag_ref(frag);
  2427. size = skb_frag_size(frag);
  2428. if (pos < offset) {
  2429. frag->page_offset += offset - pos;
  2430. skb_frag_size_sub(frag, offset - pos);
  2431. }
  2432. skb_shinfo(nskb)->nr_frags++;
  2433. if (pos + size <= offset + len) {
  2434. i++;
  2435. pos += size;
  2436. } else {
  2437. skb_frag_size_sub(frag, pos + size - (offset + len));
  2438. goto skip_fraglist;
  2439. }
  2440. frag++;
  2441. }
  2442. if (pos < offset + len) {
  2443. struct sk_buff *fskb2 = fskb;
  2444. BUG_ON(pos + fskb->len != offset + len);
  2445. pos += fskb->len;
  2446. fskb = fskb->next;
  2447. if (fskb2->next) {
  2448. fskb2 = skb_clone(fskb2, GFP_ATOMIC);
  2449. if (!fskb2)
  2450. goto err;
  2451. } else
  2452. skb_get(fskb2);
  2453. SKB_FRAG_ASSERT(nskb);
  2454. skb_shinfo(nskb)->frag_list = fskb2;
  2455. }
  2456. skip_fraglist:
  2457. nskb->data_len = len - hsize;
  2458. nskb->len += nskb->data_len;
  2459. nskb->truesize += nskb->data_len;
  2460. } while ((offset += len) < skb->len);
  2461. return segs;
  2462. err:
  2463. while ((skb = segs)) {
  2464. segs = skb->next;
  2465. kfree_skb(skb);
  2466. }
  2467. return ERR_PTR(err);
  2468. }
  2469. EXPORT_SYMBOL_GPL(skb_segment);
  2470. int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb)
  2471. {
  2472. struct sk_buff *p = *head;
  2473. struct sk_buff *nskb;
  2474. struct skb_shared_info *skbinfo = skb_shinfo(skb);
  2475. struct skb_shared_info *pinfo = skb_shinfo(p);
  2476. unsigned int headroom;
  2477. unsigned int len = skb_gro_len(skb);
  2478. unsigned int offset = skb_gro_offset(skb);
  2479. unsigned int headlen = skb_headlen(skb);
  2480. if (p->len + len >= 65536)
  2481. return -E2BIG;
  2482. if (pinfo->frag_list)
  2483. goto merge;
  2484. else if (headlen <= offset) {
  2485. skb_frag_t *frag;
  2486. skb_frag_t *frag2;
  2487. int i = skbinfo->nr_frags;
  2488. int nr_frags = pinfo->nr_frags + i;
  2489. offset -= headlen;
  2490. if (nr_frags > MAX_SKB_FRAGS)
  2491. return -E2BIG;
  2492. pinfo->nr_frags = nr_frags;
  2493. skbinfo->nr_frags = 0;
  2494. frag = pinfo->frags + nr_frags;
  2495. frag2 = skbinfo->frags + i;
  2496. do {
  2497. *--frag = *--frag2;
  2498. } while (--i);
  2499. frag->page_offset += offset;
  2500. skb_frag_size_sub(frag, offset);
  2501. skb->truesize -= skb->data_len;
  2502. skb->len -= skb->data_len;
  2503. skb->data_len = 0;
  2504. NAPI_GRO_CB(skb)->free = 1;
  2505. goto done;
  2506. } else if (skb_gro_len(p) != pinfo->gso_size)
  2507. return -E2BIG;
  2508. headroom = skb_headroom(p);
  2509. nskb = alloc_skb(headroom + skb_gro_offset(p), GFP_ATOMIC);
  2510. if (unlikely(!nskb))
  2511. return -ENOMEM;
  2512. __copy_skb_header(nskb, p);
  2513. nskb->mac_len = p->mac_len;
  2514. skb_reserve(nskb, headroom);
  2515. __skb_put(nskb, skb_gro_offset(p));
  2516. skb_set_mac_header(nskb, skb_mac_header(p) - p->data);
  2517. skb_set_network_header(nskb, skb_network_offset(p));
  2518. skb_set_transport_header(nskb, skb_transport_offset(p));
  2519. __skb_pull(p, skb_gro_offset(p));
  2520. memcpy(skb_mac_header(nskb), skb_mac_header(p),
  2521. p->data - skb_mac_header(p));
  2522. *NAPI_GRO_CB(nskb) = *NAPI_GRO_CB(p);
  2523. skb_shinfo(nskb)->frag_list = p;
  2524. skb_shinfo(nskb)->gso_size = pinfo->gso_size;
  2525. pinfo->gso_size = 0;
  2526. skb_header_release(p);
  2527. nskb->prev = p;
  2528. nskb->data_len += p->len;
  2529. nskb->truesize += p->truesize;
  2530. nskb->len += p->len;
  2531. *head = nskb;
  2532. nskb->next = p->next;
  2533. p->next = NULL;
  2534. p = nskb;
  2535. merge:
  2536. p->truesize += skb->truesize - len;
  2537. if (offset > headlen) {
  2538. unsigned int eat = offset - headlen;
  2539. skbinfo->frags[0].page_offset += eat;
  2540. skb_frag_size_sub(&skbinfo->frags[0], eat);
  2541. skb->data_len -= eat;
  2542. skb->len -= eat;
  2543. offset = headlen;
  2544. }
  2545. __skb_pull(skb, offset);
  2546. p->prev->next = skb;
  2547. p->prev = skb;
  2548. skb_header_release(skb);
  2549. done:
  2550. NAPI_GRO_CB(p)->count++;
  2551. p->data_len += len;
  2552. p->truesize += len;
  2553. p->len += len;
  2554. NAPI_GRO_CB(skb)->same_flow = 1;
  2555. return 0;
  2556. }
  2557. EXPORT_SYMBOL_GPL(skb_gro_receive);
  2558. void __init skb_init(void)
  2559. {
  2560. skbuff_head_cache = kmem_cache_create("skbuff_head_cache",
  2561. sizeof(struct sk_buff),
  2562. 0,
  2563. SLAB_HWCACHE_ALIGN|SLAB_PANIC,
  2564. NULL);
  2565. skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
  2566. (2*sizeof(struct sk_buff)) +
  2567. sizeof(atomic_t),
  2568. 0,
  2569. SLAB_HWCACHE_ALIGN|SLAB_PANIC,
  2570. NULL);
  2571. }
  2572. /**
  2573. * skb_to_sgvec - Fill a scatter-gather list from a socket buffer
  2574. * @skb: Socket buffer containing the buffers to be mapped
  2575. * @sg: The scatter-gather list to map into
  2576. * @offset: The offset into the buffer's contents to start mapping
  2577. * @len: Length of buffer space to be mapped
  2578. *
  2579. * Fill the specified scatter-gather list with mappings/pointers into a
  2580. * region of the buffer space attached to a socket buffer.
  2581. */
  2582. static int
  2583. __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
  2584. {
  2585. int start = skb_headlen(skb);
  2586. int i, copy = start - offset;
  2587. struct sk_buff *frag_iter;
  2588. int elt = 0;
  2589. if (copy > 0) {
  2590. if (copy > len)
  2591. copy = len;
  2592. sg_set_buf(sg, skb->data + offset, copy);
  2593. elt++;
  2594. if ((len -= copy) == 0)
  2595. return elt;
  2596. offset += copy;
  2597. }
  2598. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  2599. int end;
  2600. WARN_ON(start > offset + len);
  2601. end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
  2602. if ((copy = end - offset) > 0) {
  2603. skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
  2604. if (copy > len)
  2605. copy = len;
  2606. sg_set_page(&sg[elt], skb_frag_page(frag), copy,
  2607. frag->page_offset+offset-start);
  2608. elt++;
  2609. if (!(len -= copy))
  2610. return elt;
  2611. offset += copy;
  2612. }
  2613. start = end;
  2614. }
  2615. skb_walk_frags(skb, frag_iter) {
  2616. int end;
  2617. WARN_ON(start > offset + len);
  2618. end = start + frag_iter->len;
  2619. if ((copy = end - offset) > 0) {
  2620. if (copy > len)
  2621. copy = len;
  2622. elt += __skb_to_sgvec(frag_iter, sg+elt, offset - start,
  2623. copy);
  2624. if ((len -= copy) == 0)
  2625. return elt;
  2626. offset += copy;
  2627. }
  2628. start = end;
  2629. }
  2630. BUG_ON(len);
  2631. return elt;
  2632. }
  2633. int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
  2634. {
  2635. int nsg = __skb_to_sgvec(skb, sg, offset, len);
  2636. sg_mark_end(&sg[nsg - 1]);
  2637. return nsg;
  2638. }
  2639. EXPORT_SYMBOL_GPL(skb_to_sgvec);
  2640. /**
  2641. * skb_cow_data - Check that a socket buffer's data buffers are writable
  2642. * @skb: The socket buffer to check.
  2643. * @tailbits: Amount of trailing space to be added
  2644. * @trailer: Returned pointer to the skb where the @tailbits space begins
  2645. *
  2646. * Make sure that the data buffers attached to a socket buffer are
  2647. * writable. If they are not, private copies are made of the data buffers
  2648. * and the socket buffer is set to use these instead.
  2649. *
  2650. * If @tailbits is given, make sure that there is space to write @tailbits
  2651. * bytes of data beyond current end of socket buffer. @trailer will be
  2652. * set to point to the skb in which this space begins.
  2653. *
  2654. * The number of scatterlist elements required to completely map the
  2655. * COW'd and extended socket buffer will be returned.
  2656. */
  2657. int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
  2658. {
  2659. int copyflag;
  2660. int elt;
  2661. struct sk_buff *skb1, **skb_p;
  2662. /* If skb is cloned or its head is paged, reallocate
  2663. * head pulling out all the pages (pages are considered not writable
  2664. * at the moment even if they are anonymous).
  2665. */
  2666. if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
  2667. __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL)
  2668. return -ENOMEM;
  2669. /* Easy case. Most of packets will go this way. */
  2670. if (!skb_has_frag_list(skb)) {
  2671. /* A little of trouble, not enough of space for trailer.
  2672. * This should not happen, when stack is tuned to generate
  2673. * good frames. OK, on miss we reallocate and reserve even more
  2674. * space, 128 bytes is fair. */
  2675. if (skb_tailroom(skb) < tailbits &&
  2676. pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
  2677. return -ENOMEM;
  2678. /* Voila! */
  2679. *trailer = skb;
  2680. return 1;
  2681. }
  2682. /* Misery. We are in troubles, going to mincer fragments... */
  2683. elt = 1;
  2684. skb_p = &skb_shinfo(skb)->frag_list;
  2685. copyflag = 0;
  2686. while ((skb1 = *skb_p) != NULL) {
  2687. int ntail = 0;
  2688. /* The fragment is partially pulled by someone,
  2689. * this can happen on input. Copy it and everything
  2690. * after it. */
  2691. if (skb_shared(skb1))
  2692. copyflag = 1;
  2693. /* If the skb is the last, worry about trailer. */
  2694. if (skb1->next == NULL && tailbits) {
  2695. if (skb_shinfo(skb1)->nr_frags ||
  2696. skb_has_frag_list(skb1) ||
  2697. skb_tailroom(skb1) < tailbits)
  2698. ntail = tailbits + 128;
  2699. }
  2700. if (copyflag ||
  2701. skb_cloned(skb1) ||
  2702. ntail ||
  2703. skb_shinfo(skb1)->nr_frags ||
  2704. skb_has_frag_list(skb1)) {
  2705. struct sk_buff *skb2;
  2706. /* Fuck, we are miserable poor guys... */
  2707. if (ntail == 0)
  2708. skb2 = skb_copy(skb1, GFP_ATOMIC);
  2709. else
  2710. skb2 = skb_copy_expand(skb1,
  2711. skb_headroom(skb1),
  2712. ntail,
  2713. GFP_ATOMIC);
  2714. if (unlikely(skb2 == NULL))
  2715. return -ENOMEM;
  2716. if (skb1->sk)
  2717. skb_set_owner_w(skb2, skb1->sk);
  2718. /* Looking around. Are we still alive?
  2719. * OK, link new skb, drop old one */
  2720. skb2->next = skb1->next;
  2721. *skb_p = skb2;
  2722. kfree_skb(skb1);
  2723. skb1 = skb2;
  2724. }
  2725. elt++;
  2726. *trailer = skb1;
  2727. skb_p = &skb1->next;
  2728. }
  2729. return elt;
  2730. }
  2731. EXPORT_SYMBOL_GPL(skb_cow_data);
  2732. static void sock_rmem_free(struct sk_buff *skb)
  2733. {
  2734. struct sock *sk = skb->sk;
  2735. atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
  2736. }
  2737. /*
  2738. * Note: We dont mem charge error packets (no sk_forward_alloc changes)
  2739. */
  2740. int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb)
  2741. {
  2742. int len = skb->len;
  2743. if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
  2744. (unsigned int)sk->sk_rcvbuf)
  2745. return -ENOMEM;
  2746. skb_orphan(skb);
  2747. skb->sk = sk;
  2748. skb->destructor = sock_rmem_free;
  2749. atomic_add(skb->truesize, &sk->sk_rmem_alloc);
  2750. /* before exiting rcu section, make sure dst is refcounted */
  2751. skb_dst_force(skb);
  2752. skb_queue_tail(&sk->sk_error_queue, skb);
  2753. if (!sock_flag(sk, SOCK_DEAD))
  2754. sk->sk_data_ready(sk, len);
  2755. return 0;
  2756. }
  2757. EXPORT_SYMBOL(sock_queue_err_skb);
  2758. void skb_tstamp_tx(struct sk_buff *orig_skb,
  2759. struct skb_shared_hwtstamps *hwtstamps)
  2760. {
  2761. struct sock *sk = orig_skb->sk;
  2762. struct sock_exterr_skb *serr;
  2763. struct sk_buff *skb;
  2764. int err;
  2765. if (!sk)
  2766. return;
  2767. skb = skb_clone(orig_skb, GFP_ATOMIC);
  2768. if (!skb)
  2769. return;
  2770. if (hwtstamps) {
  2771. *skb_hwtstamps(skb) =
  2772. *hwtstamps;
  2773. } else {
  2774. /*
  2775. * no hardware time stamps available,
  2776. * so keep the shared tx_flags and only
  2777. * store software time stamp
  2778. */
  2779. skb->tstamp = ktime_get_real();
  2780. }
  2781. serr = SKB_EXT_ERR(skb);
  2782. memset(serr, 0, sizeof(*serr));
  2783. serr->ee.ee_errno = ENOMSG;
  2784. serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
  2785. err = sock_queue_err_skb(sk, skb);
  2786. if (err)
  2787. kfree_skb(skb);
  2788. }
  2789. EXPORT_SYMBOL_GPL(skb_tstamp_tx);
  2790. void skb_complete_wifi_ack(struct sk_buff *skb, bool acked)
  2791. {
  2792. struct sock *sk = skb->sk;
  2793. struct sock_exterr_skb *serr;
  2794. int err;
  2795. skb->wifi_acked_valid = 1;
  2796. skb->wifi_acked = acked;
  2797. serr = SKB_EXT_ERR(skb);
  2798. memset(serr, 0, sizeof(*serr));
  2799. serr->ee.ee_errno = ENOMSG;
  2800. serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS;
  2801. err = sock_queue_err_skb(sk, skb);
  2802. if (err)
  2803. kfree_skb(skb);
  2804. }
  2805. EXPORT_SYMBOL_GPL(skb_complete_wifi_ack);
  2806. /**
  2807. * skb_partial_csum_set - set up and verify partial csum values for packet
  2808. * @skb: the skb to set
  2809. * @start: the number of bytes after skb->data to start checksumming.
  2810. * @off: the offset from start to place the checksum.
  2811. *
  2812. * For untrusted partially-checksummed packets, we need to make sure the values
  2813. * for skb->csum_start and skb->csum_offset are valid so we don't oops.
  2814. *
  2815. * This function checks and sets those values and skb->ip_summed: if this
  2816. * returns false you should drop the packet.
  2817. */
  2818. bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
  2819. {
  2820. if (unlikely(start > skb_headlen(skb)) ||
  2821. unlikely((int)start + off > skb_headlen(skb) - 2)) {
  2822. if (net_ratelimit())
  2823. printk(KERN_WARNING
  2824. "bad partial csum: csum=%u/%u len=%u\n",
  2825. start, off, skb_headlen(skb));
  2826. return false;
  2827. }
  2828. skb->ip_summed = CHECKSUM_PARTIAL;
  2829. skb->csum_start = skb_headroom(skb) + start;
  2830. skb->csum_offset = off;
  2831. return true;
  2832. }
  2833. EXPORT_SYMBOL_GPL(skb_partial_csum_set);
  2834. void __skb_warn_lro_forwarding(const struct sk_buff *skb)
  2835. {
  2836. if (net_ratelimit())
  2837. pr_warning("%s: received packets cannot be forwarded"
  2838. " while LRO is enabled\n", skb->dev->name);
  2839. }
  2840. EXPORT_SYMBOL(__skb_warn_lro_forwarding);