skbuff.c 64 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617
  1. /*
  2. * Routines having to do with the 'struct sk_buff' memory handlers.
  3. *
  4. * Authors: Alan Cox <iiitac@pyr.swan.ac.uk>
  5. * Florian La Roche <rzsfl@rz.uni-sb.de>
  6. *
  7. * Version: $Id: skbuff.c,v 1.90 2001/11/07 05:56:19 davem Exp $
  8. *
  9. * Fixes:
  10. * Alan Cox : Fixed the worst of the load
  11. * balancer bugs.
  12. * Dave Platt : Interrupt stacking fix.
  13. * Richard Kooijman : Timestamp fixes.
  14. * Alan Cox : Changed buffer format.
  15. * Alan Cox : destructor hook for AF_UNIX etc.
  16. * Linus Torvalds : Better skb_clone.
  17. * Alan Cox : Added skb_copy.
  18. * Alan Cox : Added all the changed routines Linus
  19. * only put in the headers
  20. * Ray VanTassle : Fixed --skb->lock in free
  21. * Alan Cox : skb_copy copy arp field
  22. * Andi Kleen : slabified it.
  23. * Robert Olsson : Removed skb_head_pool
  24. *
  25. * NOTE:
  26. * The __skb_ routines should be called with interrupts
  27. * disabled, or you better be *real* sure that the operation is atomic
  28. * with respect to whatever list is being frobbed (e.g. via lock_sock()
  29. * or via disabling bottom half handlers, etc).
  30. *
  31. * This program is free software; you can redistribute it and/or
  32. * modify it under the terms of the GNU General Public License
  33. * as published by the Free Software Foundation; either version
  34. * 2 of the License, or (at your option) any later version.
  35. */
  36. /*
  37. * The functions in this file will not compile correctly with gcc 2.4.x
  38. */
  39. #include <linux/module.h>
  40. #include <linux/types.h>
  41. #include <linux/kernel.h>
  42. #include <linux/mm.h>
  43. #include <linux/interrupt.h>
  44. #include <linux/in.h>
  45. #include <linux/inet.h>
  46. #include <linux/slab.h>
  47. #include <linux/netdevice.h>
  48. #ifdef CONFIG_NET_CLS_ACT
  49. #include <net/pkt_sched.h>
  50. #endif
  51. #include <linux/string.h>
  52. #include <linux/skbuff.h>
  53. #include <linux/splice.h>
  54. #include <linux/cache.h>
  55. #include <linux/rtnetlink.h>
  56. #include <linux/init.h>
  57. #include <linux/scatterlist.h>
  58. #include <net/protocol.h>
  59. #include <net/dst.h>
  60. #include <net/sock.h>
  61. #include <net/checksum.h>
  62. #include <net/xfrm.h>
  63. #include <asm/uaccess.h>
  64. #include <asm/system.h>
  65. #include "kmap_skb.h"
  66. static struct kmem_cache *skbuff_head_cache __read_mostly;
  67. static struct kmem_cache *skbuff_fclone_cache __read_mostly;
  68. static void sock_pipe_buf_release(struct pipe_inode_info *pipe,
  69. struct pipe_buffer *buf)
  70. {
  71. struct sk_buff *skb = (struct sk_buff *) buf->private;
  72. kfree_skb(skb);
  73. }
  74. static void sock_pipe_buf_get(struct pipe_inode_info *pipe,
  75. struct pipe_buffer *buf)
  76. {
  77. struct sk_buff *skb = (struct sk_buff *) buf->private;
  78. skb_get(skb);
  79. }
  80. static int sock_pipe_buf_steal(struct pipe_inode_info *pipe,
  81. struct pipe_buffer *buf)
  82. {
  83. return 1;
  84. }
  85. /* Pipe buffer operations for a socket. */
  86. static struct pipe_buf_operations sock_pipe_buf_ops = {
  87. .can_merge = 0,
  88. .map = generic_pipe_buf_map,
  89. .unmap = generic_pipe_buf_unmap,
  90. .confirm = generic_pipe_buf_confirm,
  91. .release = sock_pipe_buf_release,
  92. .steal = sock_pipe_buf_steal,
  93. .get = sock_pipe_buf_get,
  94. };
  95. /*
  96. * Keep out-of-line to prevent kernel bloat.
  97. * __builtin_return_address is not used because it is not always
  98. * reliable.
  99. */
  100. /**
  101. * skb_over_panic - private function
  102. * @skb: buffer
  103. * @sz: size
  104. * @here: address
  105. *
  106. * Out of line support code for skb_put(). Not user callable.
  107. */
  108. void skb_over_panic(struct sk_buff *skb, int sz, void *here)
  109. {
  110. printk(KERN_EMERG "skb_over_panic: text:%p len:%d put:%d head:%p "
  111. "data:%p tail:%#lx end:%#lx dev:%s\n",
  112. here, skb->len, sz, skb->head, skb->data,
  113. (unsigned long)skb->tail, (unsigned long)skb->end,
  114. skb->dev ? skb->dev->name : "<NULL>");
  115. BUG();
  116. }
  117. /**
  118. * skb_under_panic - private function
  119. * @skb: buffer
  120. * @sz: size
  121. * @here: address
  122. *
  123. * Out of line support code for skb_push(). Not user callable.
  124. */
  125. void skb_under_panic(struct sk_buff *skb, int sz, void *here)
  126. {
  127. printk(KERN_EMERG "skb_under_panic: text:%p len:%d put:%d head:%p "
  128. "data:%p tail:%#lx end:%#lx dev:%s\n",
  129. here, skb->len, sz, skb->head, skb->data,
  130. (unsigned long)skb->tail, (unsigned long)skb->end,
  131. skb->dev ? skb->dev->name : "<NULL>");
  132. BUG();
  133. }
  134. void skb_truesize_bug(struct sk_buff *skb)
  135. {
  136. printk(KERN_ERR "SKB BUG: Invalid truesize (%u) "
  137. "len=%u, sizeof(sk_buff)=%Zd\n",
  138. skb->truesize, skb->len, sizeof(struct sk_buff));
  139. }
  140. EXPORT_SYMBOL(skb_truesize_bug);
  141. /* Allocate a new skbuff. We do this ourselves so we can fill in a few
  142. * 'private' fields and also do memory statistics to find all the
  143. * [BEEP] leaks.
  144. *
  145. */
  146. /**
  147. * __alloc_skb - allocate a network buffer
  148. * @size: size to allocate
  149. * @gfp_mask: allocation mask
  150. * @fclone: allocate from fclone cache instead of head cache
  151. * and allocate a cloned (child) skb
  152. * @node: numa node to allocate memory on
  153. *
  154. * Allocate a new &sk_buff. The returned buffer has no headroom and a
  155. * tail room of size bytes. The object has a reference count of one.
  156. * The return is the buffer. On a failure the return is %NULL.
  157. *
  158. * Buffers may only be allocated from interrupts using a @gfp_mask of
  159. * %GFP_ATOMIC.
  160. */
  161. struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
  162. int fclone, int node)
  163. {
  164. struct kmem_cache *cache;
  165. struct skb_shared_info *shinfo;
  166. struct sk_buff *skb;
  167. u8 *data;
  168. cache = fclone ? skbuff_fclone_cache : skbuff_head_cache;
  169. /* Get the HEAD */
  170. skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
  171. if (!skb)
  172. goto out;
  173. size = SKB_DATA_ALIGN(size);
  174. data = kmalloc_node_track_caller(size + sizeof(struct skb_shared_info),
  175. gfp_mask, node);
  176. if (!data)
  177. goto nodata;
  178. /*
  179. * See comment in sk_buff definition, just before the 'tail' member
  180. */
  181. memset(skb, 0, offsetof(struct sk_buff, tail));
  182. skb->truesize = size + sizeof(struct sk_buff);
  183. atomic_set(&skb->users, 1);
  184. skb->head = data;
  185. skb->data = data;
  186. skb_reset_tail_pointer(skb);
  187. skb->end = skb->tail + size;
  188. /* make sure we initialize shinfo sequentially */
  189. shinfo = skb_shinfo(skb);
  190. atomic_set(&shinfo->dataref, 1);
  191. shinfo->nr_frags = 0;
  192. shinfo->gso_size = 0;
  193. shinfo->gso_segs = 0;
  194. shinfo->gso_type = 0;
  195. shinfo->ip6_frag_id = 0;
  196. shinfo->frag_list = NULL;
  197. if (fclone) {
  198. struct sk_buff *child = skb + 1;
  199. atomic_t *fclone_ref = (atomic_t *) (child + 1);
  200. skb->fclone = SKB_FCLONE_ORIG;
  201. atomic_set(fclone_ref, 1);
  202. child->fclone = SKB_FCLONE_UNAVAILABLE;
  203. }
  204. out:
  205. return skb;
  206. nodata:
  207. kmem_cache_free(cache, skb);
  208. skb = NULL;
  209. goto out;
  210. }
  211. /**
  212. * __netdev_alloc_skb - allocate an skbuff for rx on a specific device
  213. * @dev: network device to receive on
  214. * @length: length to allocate
  215. * @gfp_mask: get_free_pages mask, passed to alloc_skb
  216. *
  217. * Allocate a new &sk_buff and assign it a usage count of one. The
  218. * buffer has unspecified headroom built in. Users should allocate
  219. * the headroom they think they need without accounting for the
  220. * built in space. The built in space is used for optimisations.
  221. *
  222. * %NULL is returned if there is no free memory.
  223. */
  224. struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
  225. unsigned int length, gfp_t gfp_mask)
  226. {
  227. int node = dev->dev.parent ? dev_to_node(dev->dev.parent) : -1;
  228. struct sk_buff *skb;
  229. skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask, 0, node);
  230. if (likely(skb)) {
  231. skb_reserve(skb, NET_SKB_PAD);
  232. skb->dev = dev;
  233. }
  234. return skb;
  235. }
  236. /**
  237. * dev_alloc_skb - allocate an skbuff for receiving
  238. * @length: length to allocate
  239. *
  240. * Allocate a new &sk_buff and assign it a usage count of one. The
  241. * buffer has unspecified headroom built in. Users should allocate
  242. * the headroom they think they need without accounting for the
  243. * built in space. The built in space is used for optimisations.
  244. *
  245. * %NULL is returned if there is no free memory. Although this function
  246. * allocates memory it can be called from an interrupt.
  247. */
  248. struct sk_buff *dev_alloc_skb(unsigned int length)
  249. {
  250. return __dev_alloc_skb(length, GFP_ATOMIC);
  251. }
  252. EXPORT_SYMBOL(dev_alloc_skb);
  253. static void skb_drop_list(struct sk_buff **listp)
  254. {
  255. struct sk_buff *list = *listp;
  256. *listp = NULL;
  257. do {
  258. struct sk_buff *this = list;
  259. list = list->next;
  260. kfree_skb(this);
  261. } while (list);
  262. }
  263. static inline void skb_drop_fraglist(struct sk_buff *skb)
  264. {
  265. skb_drop_list(&skb_shinfo(skb)->frag_list);
  266. }
  267. static void skb_clone_fraglist(struct sk_buff *skb)
  268. {
  269. struct sk_buff *list;
  270. for (list = skb_shinfo(skb)->frag_list; list; list = list->next)
  271. skb_get(list);
  272. }
  273. static void skb_release_data(struct sk_buff *skb)
  274. {
  275. if (!skb->cloned ||
  276. !atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
  277. &skb_shinfo(skb)->dataref)) {
  278. if (skb_shinfo(skb)->nr_frags) {
  279. int i;
  280. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
  281. put_page(skb_shinfo(skb)->frags[i].page);
  282. }
  283. if (skb_shinfo(skb)->frag_list)
  284. skb_drop_fraglist(skb);
  285. kfree(skb->head);
  286. }
  287. }
  288. /*
  289. * Free an skbuff by memory without cleaning the state.
  290. */
  291. static void kfree_skbmem(struct sk_buff *skb)
  292. {
  293. struct sk_buff *other;
  294. atomic_t *fclone_ref;
  295. switch (skb->fclone) {
  296. case SKB_FCLONE_UNAVAILABLE:
  297. kmem_cache_free(skbuff_head_cache, skb);
  298. break;
  299. case SKB_FCLONE_ORIG:
  300. fclone_ref = (atomic_t *) (skb + 2);
  301. if (atomic_dec_and_test(fclone_ref))
  302. kmem_cache_free(skbuff_fclone_cache, skb);
  303. break;
  304. case SKB_FCLONE_CLONE:
  305. fclone_ref = (atomic_t *) (skb + 1);
  306. other = skb - 1;
  307. /* The clone portion is available for
  308. * fast-cloning again.
  309. */
  310. skb->fclone = SKB_FCLONE_UNAVAILABLE;
  311. if (atomic_dec_and_test(fclone_ref))
  312. kmem_cache_free(skbuff_fclone_cache, other);
  313. break;
  314. }
  315. }
  316. /* Free everything but the sk_buff shell. */
  317. static void skb_release_all(struct sk_buff *skb)
  318. {
  319. dst_release(skb->dst);
  320. #ifdef CONFIG_XFRM
  321. secpath_put(skb->sp);
  322. #endif
  323. if (skb->destructor) {
  324. WARN_ON(in_irq());
  325. skb->destructor(skb);
  326. }
  327. #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
  328. nf_conntrack_put(skb->nfct);
  329. nf_conntrack_put_reasm(skb->nfct_reasm);
  330. #endif
  331. #ifdef CONFIG_BRIDGE_NETFILTER
  332. nf_bridge_put(skb->nf_bridge);
  333. #endif
  334. /* XXX: IS this still necessary? - JHS */
  335. #ifdef CONFIG_NET_SCHED
  336. skb->tc_index = 0;
  337. #ifdef CONFIG_NET_CLS_ACT
  338. skb->tc_verd = 0;
  339. #endif
  340. #endif
  341. skb_release_data(skb);
  342. }
  343. /**
  344. * __kfree_skb - private function
  345. * @skb: buffer
  346. *
  347. * Free an sk_buff. Release anything attached to the buffer.
  348. * Clean the state. This is an internal helper function. Users should
  349. * always call kfree_skb
  350. */
  351. void __kfree_skb(struct sk_buff *skb)
  352. {
  353. skb_release_all(skb);
  354. kfree_skbmem(skb);
  355. }
  356. /**
  357. * kfree_skb - free an sk_buff
  358. * @skb: buffer to free
  359. *
  360. * Drop a reference to the buffer and free it if the usage count has
  361. * hit zero.
  362. */
  363. void kfree_skb(struct sk_buff *skb)
  364. {
  365. if (unlikely(!skb))
  366. return;
  367. if (likely(atomic_read(&skb->users) == 1))
  368. smp_rmb();
  369. else if (likely(!atomic_dec_and_test(&skb->users)))
  370. return;
  371. __kfree_skb(skb);
  372. }
  373. static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
  374. {
  375. new->tstamp = old->tstamp;
  376. new->dev = old->dev;
  377. new->transport_header = old->transport_header;
  378. new->network_header = old->network_header;
  379. new->mac_header = old->mac_header;
  380. new->dst = dst_clone(old->dst);
  381. #ifdef CONFIG_INET
  382. new->sp = secpath_get(old->sp);
  383. #endif
  384. memcpy(new->cb, old->cb, sizeof(old->cb));
  385. new->csum_start = old->csum_start;
  386. new->csum_offset = old->csum_offset;
  387. new->local_df = old->local_df;
  388. new->pkt_type = old->pkt_type;
  389. new->ip_summed = old->ip_summed;
  390. skb_copy_queue_mapping(new, old);
  391. new->priority = old->priority;
  392. #if defined(CONFIG_IP_VS) || defined(CONFIG_IP_VS_MODULE)
  393. new->ipvs_property = old->ipvs_property;
  394. #endif
  395. new->protocol = old->protocol;
  396. new->mark = old->mark;
  397. __nf_copy(new, old);
  398. #if defined(CONFIG_NETFILTER_XT_TARGET_TRACE) || \
  399. defined(CONFIG_NETFILTER_XT_TARGET_TRACE_MODULE)
  400. new->nf_trace = old->nf_trace;
  401. #endif
  402. #ifdef CONFIG_NET_SCHED
  403. new->tc_index = old->tc_index;
  404. #ifdef CONFIG_NET_CLS_ACT
  405. new->tc_verd = old->tc_verd;
  406. #endif
  407. #endif
  408. skb_copy_secmark(new, old);
  409. }
  410. static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
  411. {
  412. #define C(x) n->x = skb->x
  413. n->next = n->prev = NULL;
  414. n->sk = NULL;
  415. __copy_skb_header(n, skb);
  416. C(len);
  417. C(data_len);
  418. C(mac_len);
  419. n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
  420. n->cloned = 1;
  421. n->nohdr = 0;
  422. n->destructor = NULL;
  423. C(iif);
  424. C(tail);
  425. C(end);
  426. C(head);
  427. C(data);
  428. C(truesize);
  429. atomic_set(&n->users, 1);
  430. atomic_inc(&(skb_shinfo(skb)->dataref));
  431. skb->cloned = 1;
  432. return n;
  433. #undef C
  434. }
  435. /**
  436. * skb_morph - morph one skb into another
  437. * @dst: the skb to receive the contents
  438. * @src: the skb to supply the contents
  439. *
  440. * This is identical to skb_clone except that the target skb is
  441. * supplied by the user.
  442. *
  443. * The target skb is returned upon exit.
  444. */
  445. struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
  446. {
  447. skb_release_all(dst);
  448. return __skb_clone(dst, src);
  449. }
  450. EXPORT_SYMBOL_GPL(skb_morph);
  451. /**
  452. * skb_clone - duplicate an sk_buff
  453. * @skb: buffer to clone
  454. * @gfp_mask: allocation priority
  455. *
  456. * Duplicate an &sk_buff. The new one is not owned by a socket. Both
  457. * copies share the same packet data but not structure. The new
  458. * buffer has a reference count of 1. If the allocation fails the
  459. * function returns %NULL otherwise the new buffer is returned.
  460. *
  461. * If this function is called from an interrupt gfp_mask() must be
  462. * %GFP_ATOMIC.
  463. */
  464. struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
  465. {
  466. struct sk_buff *n;
  467. n = skb + 1;
  468. if (skb->fclone == SKB_FCLONE_ORIG &&
  469. n->fclone == SKB_FCLONE_UNAVAILABLE) {
  470. atomic_t *fclone_ref = (atomic_t *) (n + 1);
  471. n->fclone = SKB_FCLONE_CLONE;
  472. atomic_inc(fclone_ref);
  473. } else {
  474. n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
  475. if (!n)
  476. return NULL;
  477. n->fclone = SKB_FCLONE_UNAVAILABLE;
  478. }
  479. return __skb_clone(n, skb);
  480. }
  481. static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
  482. {
  483. #ifndef NET_SKBUFF_DATA_USES_OFFSET
  484. /*
  485. * Shift between the two data areas in bytes
  486. */
  487. unsigned long offset = new->data - old->data;
  488. #endif
  489. __copy_skb_header(new, old);
  490. #ifndef NET_SKBUFF_DATA_USES_OFFSET
  491. /* {transport,network,mac}_header are relative to skb->head */
  492. new->transport_header += offset;
  493. new->network_header += offset;
  494. new->mac_header += offset;
  495. #endif
  496. skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
  497. skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
  498. skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
  499. }
  500. /**
  501. * skb_copy - create private copy of an sk_buff
  502. * @skb: buffer to copy
  503. * @gfp_mask: allocation priority
  504. *
  505. * Make a copy of both an &sk_buff and its data. This is used when the
  506. * caller wishes to modify the data and needs a private copy of the
  507. * data to alter. Returns %NULL on failure or the pointer to the buffer
  508. * on success. The returned buffer has a reference count of 1.
  509. *
  510. * As by-product this function converts non-linear &sk_buff to linear
  511. * one, so that &sk_buff becomes completely private and caller is allowed
  512. * to modify all the data of returned buffer. This means that this
  513. * function is not recommended for use in circumstances when only
  514. * header is going to be modified. Use pskb_copy() instead.
  515. */
  516. struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
  517. {
  518. int headerlen = skb->data - skb->head;
  519. /*
  520. * Allocate the copy buffer
  521. */
  522. struct sk_buff *n;
  523. #ifdef NET_SKBUFF_DATA_USES_OFFSET
  524. n = alloc_skb(skb->end + skb->data_len, gfp_mask);
  525. #else
  526. n = alloc_skb(skb->end - skb->head + skb->data_len, gfp_mask);
  527. #endif
  528. if (!n)
  529. return NULL;
  530. /* Set the data pointer */
  531. skb_reserve(n, headerlen);
  532. /* Set the tail pointer and length */
  533. skb_put(n, skb->len);
  534. if (skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len))
  535. BUG();
  536. copy_skb_header(n, skb);
  537. return n;
  538. }
  539. /**
  540. * pskb_copy - create copy of an sk_buff with private head.
  541. * @skb: buffer to copy
  542. * @gfp_mask: allocation priority
  543. *
  544. * Make a copy of both an &sk_buff and part of its data, located
  545. * in header. Fragmented data remain shared. This is used when
  546. * the caller wishes to modify only header of &sk_buff and needs
  547. * private copy of the header to alter. Returns %NULL on failure
  548. * or the pointer to the buffer on success.
  549. * The returned buffer has a reference count of 1.
  550. */
  551. struct sk_buff *pskb_copy(struct sk_buff *skb, gfp_t gfp_mask)
  552. {
  553. /*
  554. * Allocate the copy buffer
  555. */
  556. struct sk_buff *n;
  557. #ifdef NET_SKBUFF_DATA_USES_OFFSET
  558. n = alloc_skb(skb->end, gfp_mask);
  559. #else
  560. n = alloc_skb(skb->end - skb->head, gfp_mask);
  561. #endif
  562. if (!n)
  563. goto out;
  564. /* Set the data pointer */
  565. skb_reserve(n, skb->data - skb->head);
  566. /* Set the tail pointer and length */
  567. skb_put(n, skb_headlen(skb));
  568. /* Copy the bytes */
  569. skb_copy_from_linear_data(skb, n->data, n->len);
  570. n->truesize += skb->data_len;
  571. n->data_len = skb->data_len;
  572. n->len = skb->len;
  573. if (skb_shinfo(skb)->nr_frags) {
  574. int i;
  575. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  576. skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
  577. get_page(skb_shinfo(n)->frags[i].page);
  578. }
  579. skb_shinfo(n)->nr_frags = i;
  580. }
  581. if (skb_shinfo(skb)->frag_list) {
  582. skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
  583. skb_clone_fraglist(n);
  584. }
  585. copy_skb_header(n, skb);
  586. out:
  587. return n;
  588. }
  589. /**
  590. * pskb_expand_head - reallocate header of &sk_buff
  591. * @skb: buffer to reallocate
  592. * @nhead: room to add at head
  593. * @ntail: room to add at tail
  594. * @gfp_mask: allocation priority
  595. *
  596. * Expands (or creates identical copy, if &nhead and &ntail are zero)
  597. * header of skb. &sk_buff itself is not changed. &sk_buff MUST have
  598. * reference count of 1. Returns zero in the case of success or error,
  599. * if expansion failed. In the last case, &sk_buff is not changed.
  600. *
  601. * All the pointers pointing into skb header may change and must be
  602. * reloaded after call to this function.
  603. */
  604. int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
  605. gfp_t gfp_mask)
  606. {
  607. int i;
  608. u8 *data;
  609. #ifdef NET_SKBUFF_DATA_USES_OFFSET
  610. int size = nhead + skb->end + ntail;
  611. #else
  612. int size = nhead + (skb->end - skb->head) + ntail;
  613. #endif
  614. long off;
  615. if (skb_shared(skb))
  616. BUG();
  617. size = SKB_DATA_ALIGN(size);
  618. data = kmalloc(size + sizeof(struct skb_shared_info), gfp_mask);
  619. if (!data)
  620. goto nodata;
  621. /* Copy only real data... and, alas, header. This should be
  622. * optimized for the cases when header is void. */
  623. #ifdef NET_SKBUFF_DATA_USES_OFFSET
  624. memcpy(data + nhead, skb->head, skb->tail);
  625. #else
  626. memcpy(data + nhead, skb->head, skb->tail - skb->head);
  627. #endif
  628. memcpy(data + size, skb_end_pointer(skb),
  629. sizeof(struct skb_shared_info));
  630. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
  631. get_page(skb_shinfo(skb)->frags[i].page);
  632. if (skb_shinfo(skb)->frag_list)
  633. skb_clone_fraglist(skb);
  634. skb_release_data(skb);
  635. off = (data + nhead) - skb->head;
  636. skb->head = data;
  637. skb->data += off;
  638. #ifdef NET_SKBUFF_DATA_USES_OFFSET
  639. skb->end = size;
  640. off = nhead;
  641. #else
  642. skb->end = skb->head + size;
  643. #endif
  644. /* {transport,network,mac}_header and tail are relative to skb->head */
  645. skb->tail += off;
  646. skb->transport_header += off;
  647. skb->network_header += off;
  648. skb->mac_header += off;
  649. skb->csum_start += nhead;
  650. skb->cloned = 0;
  651. skb->hdr_len = 0;
  652. skb->nohdr = 0;
  653. atomic_set(&skb_shinfo(skb)->dataref, 1);
  654. return 0;
  655. nodata:
  656. return -ENOMEM;
  657. }
  658. /* Make private copy of skb with writable head and some headroom */
  659. struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
  660. {
  661. struct sk_buff *skb2;
  662. int delta = headroom - skb_headroom(skb);
  663. if (delta <= 0)
  664. skb2 = pskb_copy(skb, GFP_ATOMIC);
  665. else {
  666. skb2 = skb_clone(skb, GFP_ATOMIC);
  667. if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
  668. GFP_ATOMIC)) {
  669. kfree_skb(skb2);
  670. skb2 = NULL;
  671. }
  672. }
  673. return skb2;
  674. }
  675. /**
  676. * skb_copy_expand - copy and expand sk_buff
  677. * @skb: buffer to copy
  678. * @newheadroom: new free bytes at head
  679. * @newtailroom: new free bytes at tail
  680. * @gfp_mask: allocation priority
  681. *
  682. * Make a copy of both an &sk_buff and its data and while doing so
  683. * allocate additional space.
  684. *
  685. * This is used when the caller wishes to modify the data and needs a
  686. * private copy of the data to alter as well as more space for new fields.
  687. * Returns %NULL on failure or the pointer to the buffer
  688. * on success. The returned buffer has a reference count of 1.
  689. *
  690. * You must pass %GFP_ATOMIC as the allocation priority if this function
  691. * is called from an interrupt.
  692. */
  693. struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
  694. int newheadroom, int newtailroom,
  695. gfp_t gfp_mask)
  696. {
  697. /*
  698. * Allocate the copy buffer
  699. */
  700. struct sk_buff *n = alloc_skb(newheadroom + skb->len + newtailroom,
  701. gfp_mask);
  702. int oldheadroom = skb_headroom(skb);
  703. int head_copy_len, head_copy_off;
  704. int off;
  705. if (!n)
  706. return NULL;
  707. skb_reserve(n, newheadroom);
  708. /* Set the tail pointer and length */
  709. skb_put(n, skb->len);
  710. head_copy_len = oldheadroom;
  711. head_copy_off = 0;
  712. if (newheadroom <= head_copy_len)
  713. head_copy_len = newheadroom;
  714. else
  715. head_copy_off = newheadroom - head_copy_len;
  716. /* Copy the linear header and data. */
  717. if (skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
  718. skb->len + head_copy_len))
  719. BUG();
  720. copy_skb_header(n, skb);
  721. off = newheadroom - oldheadroom;
  722. n->csum_start += off;
  723. #ifdef NET_SKBUFF_DATA_USES_OFFSET
  724. n->transport_header += off;
  725. n->network_header += off;
  726. n->mac_header += off;
  727. #endif
  728. return n;
  729. }
  730. /**
  731. * skb_pad - zero pad the tail of an skb
  732. * @skb: buffer to pad
  733. * @pad: space to pad
  734. *
  735. * Ensure that a buffer is followed by a padding area that is zero
  736. * filled. Used by network drivers which may DMA or transfer data
  737. * beyond the buffer end onto the wire.
  738. *
  739. * May return error in out of memory cases. The skb is freed on error.
  740. */
  741. int skb_pad(struct sk_buff *skb, int pad)
  742. {
  743. int err;
  744. int ntail;
  745. /* If the skbuff is non linear tailroom is always zero.. */
  746. if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
  747. memset(skb->data+skb->len, 0, pad);
  748. return 0;
  749. }
  750. ntail = skb->data_len + pad - (skb->end - skb->tail);
  751. if (likely(skb_cloned(skb) || ntail > 0)) {
  752. err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
  753. if (unlikely(err))
  754. goto free_skb;
  755. }
  756. /* FIXME: The use of this function with non-linear skb's really needs
  757. * to be audited.
  758. */
  759. err = skb_linearize(skb);
  760. if (unlikely(err))
  761. goto free_skb;
  762. memset(skb->data + skb->len, 0, pad);
  763. return 0;
  764. free_skb:
  765. kfree_skb(skb);
  766. return err;
  767. }
  768. /**
  769. * skb_put - add data to a buffer
  770. * @skb: buffer to use
  771. * @len: amount of data to add
  772. *
  773. * This function extends the used data area of the buffer. If this would
  774. * exceed the total buffer size the kernel will panic. A pointer to the
  775. * first byte of the extra data is returned.
  776. */
  777. unsigned char *skb_put(struct sk_buff *skb, unsigned int len)
  778. {
  779. unsigned char *tmp = skb_tail_pointer(skb);
  780. SKB_LINEAR_ASSERT(skb);
  781. skb->tail += len;
  782. skb->len += len;
  783. if (unlikely(skb->tail > skb->end))
  784. skb_over_panic(skb, len, __builtin_return_address(0));
  785. return tmp;
  786. }
  787. EXPORT_SYMBOL(skb_put);
  788. /**
  789. * skb_push - add data to the start of a buffer
  790. * @skb: buffer to use
  791. * @len: amount of data to add
  792. *
  793. * This function extends the used data area of the buffer at the buffer
  794. * start. If this would exceed the total buffer headroom the kernel will
  795. * panic. A pointer to the first byte of the extra data is returned.
  796. */
  797. unsigned char *skb_push(struct sk_buff *skb, unsigned int len)
  798. {
  799. skb->data -= len;
  800. skb->len += len;
  801. if (unlikely(skb->data<skb->head))
  802. skb_under_panic(skb, len, __builtin_return_address(0));
  803. return skb->data;
  804. }
  805. EXPORT_SYMBOL(skb_push);
  806. /**
  807. * skb_pull - remove data from the start of a buffer
  808. * @skb: buffer to use
  809. * @len: amount of data to remove
  810. *
  811. * This function removes data from the start of a buffer, returning
  812. * the memory to the headroom. A pointer to the next data in the buffer
  813. * is returned. Once the data has been pulled future pushes will overwrite
  814. * the old data.
  815. */
  816. unsigned char *skb_pull(struct sk_buff *skb, unsigned int len)
  817. {
  818. return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len);
  819. }
  820. EXPORT_SYMBOL(skb_pull);
  821. /**
  822. * skb_trim - remove end from a buffer
  823. * @skb: buffer to alter
  824. * @len: new length
  825. *
  826. * Cut the length of a buffer down by removing data from the tail. If
  827. * the buffer is already under the length specified it is not modified.
  828. * The skb must be linear.
  829. */
  830. void skb_trim(struct sk_buff *skb, unsigned int len)
  831. {
  832. if (skb->len > len)
  833. __skb_trim(skb, len);
  834. }
  835. EXPORT_SYMBOL(skb_trim);
  836. /* Trims skb to length len. It can change skb pointers.
  837. */
  838. int ___pskb_trim(struct sk_buff *skb, unsigned int len)
  839. {
  840. struct sk_buff **fragp;
  841. struct sk_buff *frag;
  842. int offset = skb_headlen(skb);
  843. int nfrags = skb_shinfo(skb)->nr_frags;
  844. int i;
  845. int err;
  846. if (skb_cloned(skb) &&
  847. unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
  848. return err;
  849. i = 0;
  850. if (offset >= len)
  851. goto drop_pages;
  852. for (; i < nfrags; i++) {
  853. int end = offset + skb_shinfo(skb)->frags[i].size;
  854. if (end < len) {
  855. offset = end;
  856. continue;
  857. }
  858. skb_shinfo(skb)->frags[i++].size = len - offset;
  859. drop_pages:
  860. skb_shinfo(skb)->nr_frags = i;
  861. for (; i < nfrags; i++)
  862. put_page(skb_shinfo(skb)->frags[i].page);
  863. if (skb_shinfo(skb)->frag_list)
  864. skb_drop_fraglist(skb);
  865. goto done;
  866. }
  867. for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
  868. fragp = &frag->next) {
  869. int end = offset + frag->len;
  870. if (skb_shared(frag)) {
  871. struct sk_buff *nfrag;
  872. nfrag = skb_clone(frag, GFP_ATOMIC);
  873. if (unlikely(!nfrag))
  874. return -ENOMEM;
  875. nfrag->next = frag->next;
  876. kfree_skb(frag);
  877. frag = nfrag;
  878. *fragp = frag;
  879. }
  880. if (end < len) {
  881. offset = end;
  882. continue;
  883. }
  884. if (end > len &&
  885. unlikely((err = pskb_trim(frag, len - offset))))
  886. return err;
  887. if (frag->next)
  888. skb_drop_list(&frag->next);
  889. break;
  890. }
  891. done:
  892. if (len > skb_headlen(skb)) {
  893. skb->data_len -= skb->len - len;
  894. skb->len = len;
  895. } else {
  896. skb->len = len;
  897. skb->data_len = 0;
  898. skb_set_tail_pointer(skb, len);
  899. }
  900. return 0;
  901. }
  902. /**
  903. * __pskb_pull_tail - advance tail of skb header
  904. * @skb: buffer to reallocate
  905. * @delta: number of bytes to advance tail
  906. *
  907. * The function makes a sense only on a fragmented &sk_buff,
  908. * it expands header moving its tail forward and copying necessary
  909. * data from fragmented part.
  910. *
  911. * &sk_buff MUST have reference count of 1.
  912. *
  913. * Returns %NULL (and &sk_buff does not change) if pull failed
  914. * or value of new tail of skb in the case of success.
  915. *
  916. * All the pointers pointing into skb header may change and must be
  917. * reloaded after call to this function.
  918. */
  919. /* Moves tail of skb head forward, copying data from fragmented part,
  920. * when it is necessary.
  921. * 1. It may fail due to malloc failure.
  922. * 2. It may change skb pointers.
  923. *
  924. * It is pretty complicated. Luckily, it is called only in exceptional cases.
  925. */
  926. unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta)
  927. {
  928. /* If skb has not enough free space at tail, get new one
  929. * plus 128 bytes for future expansions. If we have enough
  930. * room at tail, reallocate without expansion only if skb is cloned.
  931. */
  932. int i, k, eat = (skb->tail + delta) - skb->end;
  933. if (eat > 0 || skb_cloned(skb)) {
  934. if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
  935. GFP_ATOMIC))
  936. return NULL;
  937. }
  938. if (skb_copy_bits(skb, skb_headlen(skb), skb_tail_pointer(skb), delta))
  939. BUG();
  940. /* Optimization: no fragments, no reasons to preestimate
  941. * size of pulled pages. Superb.
  942. */
  943. if (!skb_shinfo(skb)->frag_list)
  944. goto pull_pages;
  945. /* Estimate size of pulled pages. */
  946. eat = delta;
  947. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  948. if (skb_shinfo(skb)->frags[i].size >= eat)
  949. goto pull_pages;
  950. eat -= skb_shinfo(skb)->frags[i].size;
  951. }
  952. /* If we need update frag list, we are in troubles.
  953. * Certainly, it possible to add an offset to skb data,
  954. * but taking into account that pulling is expected to
  955. * be very rare operation, it is worth to fight against
  956. * further bloating skb head and crucify ourselves here instead.
  957. * Pure masohism, indeed. 8)8)
  958. */
  959. if (eat) {
  960. struct sk_buff *list = skb_shinfo(skb)->frag_list;
  961. struct sk_buff *clone = NULL;
  962. struct sk_buff *insp = NULL;
  963. do {
  964. BUG_ON(!list);
  965. if (list->len <= eat) {
  966. /* Eaten as whole. */
  967. eat -= list->len;
  968. list = list->next;
  969. insp = list;
  970. } else {
  971. /* Eaten partially. */
  972. if (skb_shared(list)) {
  973. /* Sucks! We need to fork list. :-( */
  974. clone = skb_clone(list, GFP_ATOMIC);
  975. if (!clone)
  976. return NULL;
  977. insp = list->next;
  978. list = clone;
  979. } else {
  980. /* This may be pulled without
  981. * problems. */
  982. insp = list;
  983. }
  984. if (!pskb_pull(list, eat)) {
  985. if (clone)
  986. kfree_skb(clone);
  987. return NULL;
  988. }
  989. break;
  990. }
  991. } while (eat);
  992. /* Free pulled out fragments. */
  993. while ((list = skb_shinfo(skb)->frag_list) != insp) {
  994. skb_shinfo(skb)->frag_list = list->next;
  995. kfree_skb(list);
  996. }
  997. /* And insert new clone at head. */
  998. if (clone) {
  999. clone->next = list;
  1000. skb_shinfo(skb)->frag_list = clone;
  1001. }
  1002. }
  1003. /* Success! Now we may commit changes to skb data. */
  1004. pull_pages:
  1005. eat = delta;
  1006. k = 0;
  1007. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  1008. if (skb_shinfo(skb)->frags[i].size <= eat) {
  1009. put_page(skb_shinfo(skb)->frags[i].page);
  1010. eat -= skb_shinfo(skb)->frags[i].size;
  1011. } else {
  1012. skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];
  1013. if (eat) {
  1014. skb_shinfo(skb)->frags[k].page_offset += eat;
  1015. skb_shinfo(skb)->frags[k].size -= eat;
  1016. eat = 0;
  1017. }
  1018. k++;
  1019. }
  1020. }
  1021. skb_shinfo(skb)->nr_frags = k;
  1022. skb->tail += delta;
  1023. skb->data_len -= delta;
  1024. return skb_tail_pointer(skb);
  1025. }
  1026. /* Copy some data bits from skb to kernel buffer. */
  1027. int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
  1028. {
  1029. int i, copy;
  1030. int start = skb_headlen(skb);
  1031. if (offset > (int)skb->len - len)
  1032. goto fault;
  1033. /* Copy header. */
  1034. if ((copy = start - offset) > 0) {
  1035. if (copy > len)
  1036. copy = len;
  1037. skb_copy_from_linear_data_offset(skb, offset, to, copy);
  1038. if ((len -= copy) == 0)
  1039. return 0;
  1040. offset += copy;
  1041. to += copy;
  1042. }
  1043. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  1044. int end;
  1045. BUG_TRAP(start <= offset + len);
  1046. end = start + skb_shinfo(skb)->frags[i].size;
  1047. if ((copy = end - offset) > 0) {
  1048. u8 *vaddr;
  1049. if (copy > len)
  1050. copy = len;
  1051. vaddr = kmap_skb_frag(&skb_shinfo(skb)->frags[i]);
  1052. memcpy(to,
  1053. vaddr + skb_shinfo(skb)->frags[i].page_offset+
  1054. offset - start, copy);
  1055. kunmap_skb_frag(vaddr);
  1056. if ((len -= copy) == 0)
  1057. return 0;
  1058. offset += copy;
  1059. to += copy;
  1060. }
  1061. start = end;
  1062. }
  1063. if (skb_shinfo(skb)->frag_list) {
  1064. struct sk_buff *list = skb_shinfo(skb)->frag_list;
  1065. for (; list; list = list->next) {
  1066. int end;
  1067. BUG_TRAP(start <= offset + len);
  1068. end = start + list->len;
  1069. if ((copy = end - offset) > 0) {
  1070. if (copy > len)
  1071. copy = len;
  1072. if (skb_copy_bits(list, offset - start,
  1073. to, copy))
  1074. goto fault;
  1075. if ((len -= copy) == 0)
  1076. return 0;
  1077. offset += copy;
  1078. to += copy;
  1079. }
  1080. start = end;
  1081. }
  1082. }
  1083. if (!len)
  1084. return 0;
  1085. fault:
  1086. return -EFAULT;
  1087. }
  1088. /*
  1089. * Callback from splice_to_pipe(), if we need to release some pages
  1090. * at the end of the spd in case we error'ed out in filling the pipe.
  1091. */
  1092. static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
  1093. {
  1094. struct sk_buff *skb = (struct sk_buff *) spd->partial[i].private;
  1095. kfree_skb(skb);
  1096. }
  1097. /*
  1098. * Fill page/offset/length into spd, if it can hold more pages.
  1099. */
  1100. static inline int spd_fill_page(struct splice_pipe_desc *spd, struct page *page,
  1101. unsigned int len, unsigned int offset,
  1102. struct sk_buff *skb)
  1103. {
  1104. if (unlikely(spd->nr_pages == PIPE_BUFFERS))
  1105. return 1;
  1106. spd->pages[spd->nr_pages] = page;
  1107. spd->partial[spd->nr_pages].len = len;
  1108. spd->partial[spd->nr_pages].offset = offset;
  1109. spd->partial[spd->nr_pages].private = (unsigned long) skb_get(skb);
  1110. spd->nr_pages++;
  1111. return 0;
  1112. }
  1113. /*
  1114. * Map linear and fragment data from the skb to spd. Returns number of
  1115. * pages mapped.
  1116. */
  1117. static int __skb_splice_bits(struct sk_buff *skb, unsigned int *offset,
  1118. unsigned int *total_len,
  1119. struct splice_pipe_desc *spd)
  1120. {
  1121. unsigned int nr_pages = spd->nr_pages;
  1122. unsigned int poff, plen, len, toff, tlen;
  1123. int headlen, seg;
  1124. toff = *offset;
  1125. tlen = *total_len;
  1126. if (!tlen)
  1127. goto err;
  1128. /*
  1129. * if the offset is greater than the linear part, go directly to
  1130. * the fragments.
  1131. */
  1132. headlen = skb_headlen(skb);
  1133. if (toff >= headlen) {
  1134. toff -= headlen;
  1135. goto map_frag;
  1136. }
  1137. /*
  1138. * first map the linear region into the pages/partial map, skipping
  1139. * any potential initial offset.
  1140. */
  1141. len = 0;
  1142. while (len < headlen) {
  1143. void *p = skb->data + len;
  1144. poff = (unsigned long) p & (PAGE_SIZE - 1);
  1145. plen = min_t(unsigned int, headlen - len, PAGE_SIZE - poff);
  1146. len += plen;
  1147. if (toff) {
  1148. if (plen <= toff) {
  1149. toff -= plen;
  1150. continue;
  1151. }
  1152. plen -= toff;
  1153. poff += toff;
  1154. toff = 0;
  1155. }
  1156. plen = min(plen, tlen);
  1157. if (!plen)
  1158. break;
  1159. /*
  1160. * just jump directly to update and return, no point
  1161. * in going over fragments when the output is full.
  1162. */
  1163. if (spd_fill_page(spd, virt_to_page(p), plen, poff, skb))
  1164. goto done;
  1165. tlen -= plen;
  1166. }
  1167. /*
  1168. * then map the fragments
  1169. */
  1170. map_frag:
  1171. for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
  1172. const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
  1173. plen = f->size;
  1174. poff = f->page_offset;
  1175. if (toff) {
  1176. if (plen <= toff) {
  1177. toff -= plen;
  1178. continue;
  1179. }
  1180. plen -= toff;
  1181. poff += toff;
  1182. toff = 0;
  1183. }
  1184. plen = min(plen, tlen);
  1185. if (!plen)
  1186. break;
  1187. if (spd_fill_page(spd, f->page, plen, poff, skb))
  1188. break;
  1189. tlen -= plen;
  1190. }
  1191. done:
  1192. if (spd->nr_pages - nr_pages) {
  1193. *offset = 0;
  1194. *total_len = tlen;
  1195. return 0;
  1196. }
  1197. err:
  1198. return 1;
  1199. }
  1200. /*
  1201. * Map data from the skb to a pipe. Should handle both the linear part,
  1202. * the fragments, and the frag list. It does NOT handle frag lists within
  1203. * the frag list, if such a thing exists. We'd probably need to recurse to
  1204. * handle that cleanly.
  1205. */
  1206. int skb_splice_bits(struct sk_buff *__skb, unsigned int offset,
  1207. struct pipe_inode_info *pipe, unsigned int tlen,
  1208. unsigned int flags)
  1209. {
  1210. struct partial_page partial[PIPE_BUFFERS];
  1211. struct page *pages[PIPE_BUFFERS];
  1212. struct splice_pipe_desc spd = {
  1213. .pages = pages,
  1214. .partial = partial,
  1215. .flags = flags,
  1216. .ops = &sock_pipe_buf_ops,
  1217. .spd_release = sock_spd_release,
  1218. };
  1219. struct sk_buff *skb;
  1220. /*
  1221. * I'd love to avoid the clone here, but tcp_read_sock()
  1222. * ignores reference counts and unconditonally kills the sk_buff
  1223. * on return from the actor.
  1224. */
  1225. skb = skb_clone(__skb, GFP_KERNEL);
  1226. if (unlikely(!skb))
  1227. return -ENOMEM;
  1228. /*
  1229. * __skb_splice_bits() only fails if the output has no room left,
  1230. * so no point in going over the frag_list for the error case.
  1231. */
  1232. if (__skb_splice_bits(skb, &offset, &tlen, &spd))
  1233. goto done;
  1234. else if (!tlen)
  1235. goto done;
  1236. /*
  1237. * now see if we have a frag_list to map
  1238. */
  1239. if (skb_shinfo(skb)->frag_list) {
  1240. struct sk_buff *list = skb_shinfo(skb)->frag_list;
  1241. for (; list && tlen; list = list->next) {
  1242. if (__skb_splice_bits(list, &offset, &tlen, &spd))
  1243. break;
  1244. }
  1245. }
  1246. done:
  1247. /*
  1248. * drop our reference to the clone, the pipe consumption will
  1249. * drop the rest.
  1250. */
  1251. kfree_skb(skb);
  1252. if (spd.nr_pages) {
  1253. int ret;
  1254. /*
  1255. * Drop the socket lock, otherwise we have reverse
  1256. * locking dependencies between sk_lock and i_mutex
  1257. * here as compared to sendfile(). We enter here
  1258. * with the socket lock held, and splice_to_pipe() will
  1259. * grab the pipe inode lock. For sendfile() emulation,
  1260. * we call into ->sendpage() with the i_mutex lock held
  1261. * and networking will grab the socket lock.
  1262. */
  1263. release_sock(__skb->sk);
  1264. ret = splice_to_pipe(pipe, &spd);
  1265. lock_sock(__skb->sk);
  1266. return ret;
  1267. }
  1268. return 0;
  1269. }
  1270. /**
  1271. * skb_store_bits - store bits from kernel buffer to skb
  1272. * @skb: destination buffer
  1273. * @offset: offset in destination
  1274. * @from: source buffer
  1275. * @len: number of bytes to copy
  1276. *
  1277. * Copy the specified number of bytes from the source buffer to the
  1278. * destination skb. This function handles all the messy bits of
  1279. * traversing fragment lists and such.
  1280. */
  1281. int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
  1282. {
  1283. int i, copy;
  1284. int start = skb_headlen(skb);
  1285. if (offset > (int)skb->len - len)
  1286. goto fault;
  1287. if ((copy = start - offset) > 0) {
  1288. if (copy > len)
  1289. copy = len;
  1290. skb_copy_to_linear_data_offset(skb, offset, from, copy);
  1291. if ((len -= copy) == 0)
  1292. return 0;
  1293. offset += copy;
  1294. from += copy;
  1295. }
  1296. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  1297. skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
  1298. int end;
  1299. BUG_TRAP(start <= offset + len);
  1300. end = start + frag->size;
  1301. if ((copy = end - offset) > 0) {
  1302. u8 *vaddr;
  1303. if (copy > len)
  1304. copy = len;
  1305. vaddr = kmap_skb_frag(frag);
  1306. memcpy(vaddr + frag->page_offset + offset - start,
  1307. from, copy);
  1308. kunmap_skb_frag(vaddr);
  1309. if ((len -= copy) == 0)
  1310. return 0;
  1311. offset += copy;
  1312. from += copy;
  1313. }
  1314. start = end;
  1315. }
  1316. if (skb_shinfo(skb)->frag_list) {
  1317. struct sk_buff *list = skb_shinfo(skb)->frag_list;
  1318. for (; list; list = list->next) {
  1319. int end;
  1320. BUG_TRAP(start <= offset + len);
  1321. end = start + list->len;
  1322. if ((copy = end - offset) > 0) {
  1323. if (copy > len)
  1324. copy = len;
  1325. if (skb_store_bits(list, offset - start,
  1326. from, copy))
  1327. goto fault;
  1328. if ((len -= copy) == 0)
  1329. return 0;
  1330. offset += copy;
  1331. from += copy;
  1332. }
  1333. start = end;
  1334. }
  1335. }
  1336. if (!len)
  1337. return 0;
  1338. fault:
  1339. return -EFAULT;
  1340. }
  1341. EXPORT_SYMBOL(skb_store_bits);
  1342. /* Checksum skb data. */
  1343. __wsum skb_checksum(const struct sk_buff *skb, int offset,
  1344. int len, __wsum csum)
  1345. {
  1346. int start = skb_headlen(skb);
  1347. int i, copy = start - offset;
  1348. int pos = 0;
  1349. /* Checksum header. */
  1350. if (copy > 0) {
  1351. if (copy > len)
  1352. copy = len;
  1353. csum = csum_partial(skb->data + offset, copy, csum);
  1354. if ((len -= copy) == 0)
  1355. return csum;
  1356. offset += copy;
  1357. pos = copy;
  1358. }
  1359. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  1360. int end;
  1361. BUG_TRAP(start <= offset + len);
  1362. end = start + skb_shinfo(skb)->frags[i].size;
  1363. if ((copy = end - offset) > 0) {
  1364. __wsum csum2;
  1365. u8 *vaddr;
  1366. skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
  1367. if (copy > len)
  1368. copy = len;
  1369. vaddr = kmap_skb_frag(frag);
  1370. csum2 = csum_partial(vaddr + frag->page_offset +
  1371. offset - start, copy, 0);
  1372. kunmap_skb_frag(vaddr);
  1373. csum = csum_block_add(csum, csum2, pos);
  1374. if (!(len -= copy))
  1375. return csum;
  1376. offset += copy;
  1377. pos += copy;
  1378. }
  1379. start = end;
  1380. }
  1381. if (skb_shinfo(skb)->frag_list) {
  1382. struct sk_buff *list = skb_shinfo(skb)->frag_list;
  1383. for (; list; list = list->next) {
  1384. int end;
  1385. BUG_TRAP(start <= offset + len);
  1386. end = start + list->len;
  1387. if ((copy = end - offset) > 0) {
  1388. __wsum csum2;
  1389. if (copy > len)
  1390. copy = len;
  1391. csum2 = skb_checksum(list, offset - start,
  1392. copy, 0);
  1393. csum = csum_block_add(csum, csum2, pos);
  1394. if ((len -= copy) == 0)
  1395. return csum;
  1396. offset += copy;
  1397. pos += copy;
  1398. }
  1399. start = end;
  1400. }
  1401. }
  1402. BUG_ON(len);
  1403. return csum;
  1404. }
  1405. /* Both of above in one bottle. */
  1406. __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
  1407. u8 *to, int len, __wsum csum)
  1408. {
  1409. int start = skb_headlen(skb);
  1410. int i, copy = start - offset;
  1411. int pos = 0;
  1412. /* Copy header. */
  1413. if (copy > 0) {
  1414. if (copy > len)
  1415. copy = len;
  1416. csum = csum_partial_copy_nocheck(skb->data + offset, to,
  1417. copy, csum);
  1418. if ((len -= copy) == 0)
  1419. return csum;
  1420. offset += copy;
  1421. to += copy;
  1422. pos = copy;
  1423. }
  1424. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  1425. int end;
  1426. BUG_TRAP(start <= offset + len);
  1427. end = start + skb_shinfo(skb)->frags[i].size;
  1428. if ((copy = end - offset) > 0) {
  1429. __wsum csum2;
  1430. u8 *vaddr;
  1431. skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
  1432. if (copy > len)
  1433. copy = len;
  1434. vaddr = kmap_skb_frag(frag);
  1435. csum2 = csum_partial_copy_nocheck(vaddr +
  1436. frag->page_offset +
  1437. offset - start, to,
  1438. copy, 0);
  1439. kunmap_skb_frag(vaddr);
  1440. csum = csum_block_add(csum, csum2, pos);
  1441. if (!(len -= copy))
  1442. return csum;
  1443. offset += copy;
  1444. to += copy;
  1445. pos += copy;
  1446. }
  1447. start = end;
  1448. }
  1449. if (skb_shinfo(skb)->frag_list) {
  1450. struct sk_buff *list = skb_shinfo(skb)->frag_list;
  1451. for (; list; list = list->next) {
  1452. __wsum csum2;
  1453. int end;
  1454. BUG_TRAP(start <= offset + len);
  1455. end = start + list->len;
  1456. if ((copy = end - offset) > 0) {
  1457. if (copy > len)
  1458. copy = len;
  1459. csum2 = skb_copy_and_csum_bits(list,
  1460. offset - start,
  1461. to, copy, 0);
  1462. csum = csum_block_add(csum, csum2, pos);
  1463. if ((len -= copy) == 0)
  1464. return csum;
  1465. offset += copy;
  1466. to += copy;
  1467. pos += copy;
  1468. }
  1469. start = end;
  1470. }
  1471. }
  1472. BUG_ON(len);
  1473. return csum;
  1474. }
  1475. void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
  1476. {
  1477. __wsum csum;
  1478. long csstart;
  1479. if (skb->ip_summed == CHECKSUM_PARTIAL)
  1480. csstart = skb->csum_start - skb_headroom(skb);
  1481. else
  1482. csstart = skb_headlen(skb);
  1483. BUG_ON(csstart > skb_headlen(skb));
  1484. skb_copy_from_linear_data(skb, to, csstart);
  1485. csum = 0;
  1486. if (csstart != skb->len)
  1487. csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
  1488. skb->len - csstart, 0);
  1489. if (skb->ip_summed == CHECKSUM_PARTIAL) {
  1490. long csstuff = csstart + skb->csum_offset;
  1491. *((__sum16 *)(to + csstuff)) = csum_fold(csum);
  1492. }
  1493. }
  1494. /**
  1495. * skb_dequeue - remove from the head of the queue
  1496. * @list: list to dequeue from
  1497. *
  1498. * Remove the head of the list. The list lock is taken so the function
  1499. * may be used safely with other locking list functions. The head item is
  1500. * returned or %NULL if the list is empty.
  1501. */
  1502. struct sk_buff *skb_dequeue(struct sk_buff_head *list)
  1503. {
  1504. unsigned long flags;
  1505. struct sk_buff *result;
  1506. spin_lock_irqsave(&list->lock, flags);
  1507. result = __skb_dequeue(list);
  1508. spin_unlock_irqrestore(&list->lock, flags);
  1509. return result;
  1510. }
  1511. /**
  1512. * skb_dequeue_tail - remove from the tail of the queue
  1513. * @list: list to dequeue from
  1514. *
  1515. * Remove the tail of the list. The list lock is taken so the function
  1516. * may be used safely with other locking list functions. The tail item is
  1517. * returned or %NULL if the list is empty.
  1518. */
  1519. struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
  1520. {
  1521. unsigned long flags;
  1522. struct sk_buff *result;
  1523. spin_lock_irqsave(&list->lock, flags);
  1524. result = __skb_dequeue_tail(list);
  1525. spin_unlock_irqrestore(&list->lock, flags);
  1526. return result;
  1527. }
  1528. /**
  1529. * skb_queue_purge - empty a list
  1530. * @list: list to empty
  1531. *
  1532. * Delete all buffers on an &sk_buff list. Each buffer is removed from
  1533. * the list and one reference dropped. This function takes the list
  1534. * lock and is atomic with respect to other list locking functions.
  1535. */
  1536. void skb_queue_purge(struct sk_buff_head *list)
  1537. {
  1538. struct sk_buff *skb;
  1539. while ((skb = skb_dequeue(list)) != NULL)
  1540. kfree_skb(skb);
  1541. }
  1542. /**
  1543. * skb_queue_head - queue a buffer at the list head
  1544. * @list: list to use
  1545. * @newsk: buffer to queue
  1546. *
  1547. * Queue a buffer at the start of the list. This function takes the
  1548. * list lock and can be used safely with other locking &sk_buff functions
  1549. * safely.
  1550. *
  1551. * A buffer cannot be placed on two lists at the same time.
  1552. */
  1553. void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
  1554. {
  1555. unsigned long flags;
  1556. spin_lock_irqsave(&list->lock, flags);
  1557. __skb_queue_head(list, newsk);
  1558. spin_unlock_irqrestore(&list->lock, flags);
  1559. }
  1560. /**
  1561. * skb_queue_tail - queue a buffer at the list tail
  1562. * @list: list to use
  1563. * @newsk: buffer to queue
  1564. *
  1565. * Queue a buffer at the tail of the list. This function takes the
  1566. * list lock and can be used safely with other locking &sk_buff functions
  1567. * safely.
  1568. *
  1569. * A buffer cannot be placed on two lists at the same time.
  1570. */
  1571. void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
  1572. {
  1573. unsigned long flags;
  1574. spin_lock_irqsave(&list->lock, flags);
  1575. __skb_queue_tail(list, newsk);
  1576. spin_unlock_irqrestore(&list->lock, flags);
  1577. }
  1578. /**
  1579. * skb_unlink - remove a buffer from a list
  1580. * @skb: buffer to remove
  1581. * @list: list to use
  1582. *
  1583. * Remove a packet from a list. The list locks are taken and this
  1584. * function is atomic with respect to other list locked calls
  1585. *
  1586. * You must know what list the SKB is on.
  1587. */
  1588. void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
  1589. {
  1590. unsigned long flags;
  1591. spin_lock_irqsave(&list->lock, flags);
  1592. __skb_unlink(skb, list);
  1593. spin_unlock_irqrestore(&list->lock, flags);
  1594. }
  1595. /**
  1596. * skb_append - append a buffer
  1597. * @old: buffer to insert after
  1598. * @newsk: buffer to insert
  1599. * @list: list to use
  1600. *
  1601. * Place a packet after a given packet in a list. The list locks are taken
  1602. * and this function is atomic with respect to other list locked calls.
  1603. * A buffer cannot be placed on two lists at the same time.
  1604. */
  1605. void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
  1606. {
  1607. unsigned long flags;
  1608. spin_lock_irqsave(&list->lock, flags);
  1609. __skb_append(old, newsk, list);
  1610. spin_unlock_irqrestore(&list->lock, flags);
  1611. }
  1612. /**
  1613. * skb_insert - insert a buffer
  1614. * @old: buffer to insert before
  1615. * @newsk: buffer to insert
  1616. * @list: list to use
  1617. *
  1618. * Place a packet before a given packet in a list. The list locks are
  1619. * taken and this function is atomic with respect to other list locked
  1620. * calls.
  1621. *
  1622. * A buffer cannot be placed on two lists at the same time.
  1623. */
  1624. void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
  1625. {
  1626. unsigned long flags;
  1627. spin_lock_irqsave(&list->lock, flags);
  1628. __skb_insert(newsk, old->prev, old, list);
  1629. spin_unlock_irqrestore(&list->lock, flags);
  1630. }
  1631. static inline void skb_split_inside_header(struct sk_buff *skb,
  1632. struct sk_buff* skb1,
  1633. const u32 len, const int pos)
  1634. {
  1635. int i;
  1636. skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
  1637. pos - len);
  1638. /* And move data appendix as is. */
  1639. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
  1640. skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
  1641. skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
  1642. skb_shinfo(skb)->nr_frags = 0;
  1643. skb1->data_len = skb->data_len;
  1644. skb1->len += skb1->data_len;
  1645. skb->data_len = 0;
  1646. skb->len = len;
  1647. skb_set_tail_pointer(skb, len);
  1648. }
  1649. static inline void skb_split_no_header(struct sk_buff *skb,
  1650. struct sk_buff* skb1,
  1651. const u32 len, int pos)
  1652. {
  1653. int i, k = 0;
  1654. const int nfrags = skb_shinfo(skb)->nr_frags;
  1655. skb_shinfo(skb)->nr_frags = 0;
  1656. skb1->len = skb1->data_len = skb->len - len;
  1657. skb->len = len;
  1658. skb->data_len = len - pos;
  1659. for (i = 0; i < nfrags; i++) {
  1660. int size = skb_shinfo(skb)->frags[i].size;
  1661. if (pos + size > len) {
  1662. skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
  1663. if (pos < len) {
  1664. /* Split frag.
  1665. * We have two variants in this case:
  1666. * 1. Move all the frag to the second
  1667. * part, if it is possible. F.e.
  1668. * this approach is mandatory for TUX,
  1669. * where splitting is expensive.
  1670. * 2. Split is accurately. We make this.
  1671. */
  1672. get_page(skb_shinfo(skb)->frags[i].page);
  1673. skb_shinfo(skb1)->frags[0].page_offset += len - pos;
  1674. skb_shinfo(skb1)->frags[0].size -= len - pos;
  1675. skb_shinfo(skb)->frags[i].size = len - pos;
  1676. skb_shinfo(skb)->nr_frags++;
  1677. }
  1678. k++;
  1679. } else
  1680. skb_shinfo(skb)->nr_frags++;
  1681. pos += size;
  1682. }
  1683. skb_shinfo(skb1)->nr_frags = k;
  1684. }
  1685. /**
  1686. * skb_split - Split fragmented skb to two parts at length len.
  1687. * @skb: the buffer to split
  1688. * @skb1: the buffer to receive the second part
  1689. * @len: new length for skb
  1690. */
  1691. void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
  1692. {
  1693. int pos = skb_headlen(skb);
  1694. if (len < pos) /* Split line is inside header. */
  1695. skb_split_inside_header(skb, skb1, len, pos);
  1696. else /* Second chunk has no header, nothing to copy. */
  1697. skb_split_no_header(skb, skb1, len, pos);
  1698. }
  1699. /**
  1700. * skb_prepare_seq_read - Prepare a sequential read of skb data
  1701. * @skb: the buffer to read
  1702. * @from: lower offset of data to be read
  1703. * @to: upper offset of data to be read
  1704. * @st: state variable
  1705. *
  1706. * Initializes the specified state variable. Must be called before
  1707. * invoking skb_seq_read() for the first time.
  1708. */
  1709. void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
  1710. unsigned int to, struct skb_seq_state *st)
  1711. {
  1712. st->lower_offset = from;
  1713. st->upper_offset = to;
  1714. st->root_skb = st->cur_skb = skb;
  1715. st->frag_idx = st->stepped_offset = 0;
  1716. st->frag_data = NULL;
  1717. }
  1718. /**
  1719. * skb_seq_read - Sequentially read skb data
  1720. * @consumed: number of bytes consumed by the caller so far
  1721. * @data: destination pointer for data to be returned
  1722. * @st: state variable
  1723. *
  1724. * Reads a block of skb data at &consumed relative to the
  1725. * lower offset specified to skb_prepare_seq_read(). Assigns
  1726. * the head of the data block to &data and returns the length
  1727. * of the block or 0 if the end of the skb data or the upper
  1728. * offset has been reached.
  1729. *
  1730. * The caller is not required to consume all of the data
  1731. * returned, i.e. &consumed is typically set to the number
  1732. * of bytes already consumed and the next call to
  1733. * skb_seq_read() will return the remaining part of the block.
  1734. *
  1735. * Note 1: The size of each block of data returned can be arbitary,
  1736. * this limitation is the cost for zerocopy seqeuental
  1737. * reads of potentially non linear data.
  1738. *
  1739. * Note 2: Fragment lists within fragments are not implemented
  1740. * at the moment, state->root_skb could be replaced with
  1741. * a stack for this purpose.
  1742. */
  1743. unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
  1744. struct skb_seq_state *st)
  1745. {
  1746. unsigned int block_limit, abs_offset = consumed + st->lower_offset;
  1747. skb_frag_t *frag;
  1748. if (unlikely(abs_offset >= st->upper_offset))
  1749. return 0;
  1750. next_skb:
  1751. block_limit = skb_headlen(st->cur_skb);
  1752. if (abs_offset < block_limit) {
  1753. *data = st->cur_skb->data + abs_offset;
  1754. return block_limit - abs_offset;
  1755. }
  1756. if (st->frag_idx == 0 && !st->frag_data)
  1757. st->stepped_offset += skb_headlen(st->cur_skb);
  1758. while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
  1759. frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
  1760. block_limit = frag->size + st->stepped_offset;
  1761. if (abs_offset < block_limit) {
  1762. if (!st->frag_data)
  1763. st->frag_data = kmap_skb_frag(frag);
  1764. *data = (u8 *) st->frag_data + frag->page_offset +
  1765. (abs_offset - st->stepped_offset);
  1766. return block_limit - abs_offset;
  1767. }
  1768. if (st->frag_data) {
  1769. kunmap_skb_frag(st->frag_data);
  1770. st->frag_data = NULL;
  1771. }
  1772. st->frag_idx++;
  1773. st->stepped_offset += frag->size;
  1774. }
  1775. if (st->frag_data) {
  1776. kunmap_skb_frag(st->frag_data);
  1777. st->frag_data = NULL;
  1778. }
  1779. if (st->cur_skb->next) {
  1780. st->cur_skb = st->cur_skb->next;
  1781. st->frag_idx = 0;
  1782. goto next_skb;
  1783. } else if (st->root_skb == st->cur_skb &&
  1784. skb_shinfo(st->root_skb)->frag_list) {
  1785. st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
  1786. goto next_skb;
  1787. }
  1788. return 0;
  1789. }
  1790. /**
  1791. * skb_abort_seq_read - Abort a sequential read of skb data
  1792. * @st: state variable
  1793. *
  1794. * Must be called if skb_seq_read() was not called until it
  1795. * returned 0.
  1796. */
  1797. void skb_abort_seq_read(struct skb_seq_state *st)
  1798. {
  1799. if (st->frag_data)
  1800. kunmap_skb_frag(st->frag_data);
  1801. }
  1802. #define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb))
  1803. static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
  1804. struct ts_config *conf,
  1805. struct ts_state *state)
  1806. {
  1807. return skb_seq_read(offset, text, TS_SKB_CB(state));
  1808. }
  1809. static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
  1810. {
  1811. skb_abort_seq_read(TS_SKB_CB(state));
  1812. }
  1813. /**
  1814. * skb_find_text - Find a text pattern in skb data
  1815. * @skb: the buffer to look in
  1816. * @from: search offset
  1817. * @to: search limit
  1818. * @config: textsearch configuration
  1819. * @state: uninitialized textsearch state variable
  1820. *
  1821. * Finds a pattern in the skb data according to the specified
  1822. * textsearch configuration. Use textsearch_next() to retrieve
  1823. * subsequent occurrences of the pattern. Returns the offset
  1824. * to the first occurrence or UINT_MAX if no match was found.
  1825. */
  1826. unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
  1827. unsigned int to, struct ts_config *config,
  1828. struct ts_state *state)
  1829. {
  1830. unsigned int ret;
  1831. config->get_next_block = skb_ts_get_next_block;
  1832. config->finish = skb_ts_finish;
  1833. skb_prepare_seq_read(skb, from, to, TS_SKB_CB(state));
  1834. ret = textsearch_find(config, state);
  1835. return (ret <= to - from ? ret : UINT_MAX);
  1836. }
  1837. /**
  1838. * skb_append_datato_frags: - append the user data to a skb
  1839. * @sk: sock structure
  1840. * @skb: skb structure to be appened with user data.
  1841. * @getfrag: call back function to be used for getting the user data
  1842. * @from: pointer to user message iov
  1843. * @length: length of the iov message
  1844. *
  1845. * Description: This procedure append the user data in the fragment part
  1846. * of the skb if any page alloc fails user this procedure returns -ENOMEM
  1847. */
  1848. int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
  1849. int (*getfrag)(void *from, char *to, int offset,
  1850. int len, int odd, struct sk_buff *skb),
  1851. void *from, int length)
  1852. {
  1853. int frg_cnt = 0;
  1854. skb_frag_t *frag = NULL;
  1855. struct page *page = NULL;
  1856. int copy, left;
  1857. int offset = 0;
  1858. int ret;
  1859. do {
  1860. /* Return error if we don't have space for new frag */
  1861. frg_cnt = skb_shinfo(skb)->nr_frags;
  1862. if (frg_cnt >= MAX_SKB_FRAGS)
  1863. return -EFAULT;
  1864. /* allocate a new page for next frag */
  1865. page = alloc_pages(sk->sk_allocation, 0);
  1866. /* If alloc_page fails just return failure and caller will
  1867. * free previous allocated pages by doing kfree_skb()
  1868. */
  1869. if (page == NULL)
  1870. return -ENOMEM;
  1871. /* initialize the next frag */
  1872. sk->sk_sndmsg_page = page;
  1873. sk->sk_sndmsg_off = 0;
  1874. skb_fill_page_desc(skb, frg_cnt, page, 0, 0);
  1875. skb->truesize += PAGE_SIZE;
  1876. atomic_add(PAGE_SIZE, &sk->sk_wmem_alloc);
  1877. /* get the new initialized frag */
  1878. frg_cnt = skb_shinfo(skb)->nr_frags;
  1879. frag = &skb_shinfo(skb)->frags[frg_cnt - 1];
  1880. /* copy the user data to page */
  1881. left = PAGE_SIZE - frag->page_offset;
  1882. copy = (length > left)? left : length;
  1883. ret = getfrag(from, (page_address(frag->page) +
  1884. frag->page_offset + frag->size),
  1885. offset, copy, 0, skb);
  1886. if (ret < 0)
  1887. return -EFAULT;
  1888. /* copy was successful so update the size parameters */
  1889. sk->sk_sndmsg_off += copy;
  1890. frag->size += copy;
  1891. skb->len += copy;
  1892. skb->data_len += copy;
  1893. offset += copy;
  1894. length -= copy;
  1895. } while (length > 0);
  1896. return 0;
  1897. }
  1898. /**
  1899. * skb_pull_rcsum - pull skb and update receive checksum
  1900. * @skb: buffer to update
  1901. * @len: length of data pulled
  1902. *
  1903. * This function performs an skb_pull on the packet and updates
  1904. * the CHECKSUM_COMPLETE checksum. It should be used on
  1905. * receive path processing instead of skb_pull unless you know
  1906. * that the checksum difference is zero (e.g., a valid IP header)
  1907. * or you are setting ip_summed to CHECKSUM_NONE.
  1908. */
  1909. unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
  1910. {
  1911. BUG_ON(len > skb->len);
  1912. skb->len -= len;
  1913. BUG_ON(skb->len < skb->data_len);
  1914. skb_postpull_rcsum(skb, skb->data, len);
  1915. return skb->data += len;
  1916. }
  1917. EXPORT_SYMBOL_GPL(skb_pull_rcsum);
  1918. /**
  1919. * skb_segment - Perform protocol segmentation on skb.
  1920. * @skb: buffer to segment
  1921. * @features: features for the output path (see dev->features)
  1922. *
  1923. * This function performs segmentation on the given skb. It returns
  1924. * the segment at the given position. It returns NULL if there are
  1925. * no more segments to generate, or when an error is encountered.
  1926. */
  1927. struct sk_buff *skb_segment(struct sk_buff *skb, int features)
  1928. {
  1929. struct sk_buff *segs = NULL;
  1930. struct sk_buff *tail = NULL;
  1931. unsigned int mss = skb_shinfo(skb)->gso_size;
  1932. unsigned int doffset = skb->data - skb_mac_header(skb);
  1933. unsigned int offset = doffset;
  1934. unsigned int headroom;
  1935. unsigned int len;
  1936. int sg = features & NETIF_F_SG;
  1937. int nfrags = skb_shinfo(skb)->nr_frags;
  1938. int err = -ENOMEM;
  1939. int i = 0;
  1940. int pos;
  1941. __skb_push(skb, doffset);
  1942. headroom = skb_headroom(skb);
  1943. pos = skb_headlen(skb);
  1944. do {
  1945. struct sk_buff *nskb;
  1946. skb_frag_t *frag;
  1947. int hsize;
  1948. int k;
  1949. int size;
  1950. len = skb->len - offset;
  1951. if (len > mss)
  1952. len = mss;
  1953. hsize = skb_headlen(skb) - offset;
  1954. if (hsize < 0)
  1955. hsize = 0;
  1956. if (hsize > len || !sg)
  1957. hsize = len;
  1958. nskb = alloc_skb(hsize + doffset + headroom, GFP_ATOMIC);
  1959. if (unlikely(!nskb))
  1960. goto err;
  1961. if (segs)
  1962. tail->next = nskb;
  1963. else
  1964. segs = nskb;
  1965. tail = nskb;
  1966. nskb->dev = skb->dev;
  1967. skb_copy_queue_mapping(nskb, skb);
  1968. nskb->priority = skb->priority;
  1969. nskb->protocol = skb->protocol;
  1970. nskb->dst = dst_clone(skb->dst);
  1971. memcpy(nskb->cb, skb->cb, sizeof(skb->cb));
  1972. nskb->pkt_type = skb->pkt_type;
  1973. nskb->mac_len = skb->mac_len;
  1974. skb_reserve(nskb, headroom);
  1975. skb_reset_mac_header(nskb);
  1976. skb_set_network_header(nskb, skb->mac_len);
  1977. nskb->transport_header = (nskb->network_header +
  1978. skb_network_header_len(skb));
  1979. skb_copy_from_linear_data(skb, skb_put(nskb, doffset),
  1980. doffset);
  1981. if (!sg) {
  1982. nskb->csum = skb_copy_and_csum_bits(skb, offset,
  1983. skb_put(nskb, len),
  1984. len, 0);
  1985. continue;
  1986. }
  1987. frag = skb_shinfo(nskb)->frags;
  1988. k = 0;
  1989. nskb->ip_summed = CHECKSUM_PARTIAL;
  1990. nskb->csum = skb->csum;
  1991. skb_copy_from_linear_data_offset(skb, offset,
  1992. skb_put(nskb, hsize), hsize);
  1993. while (pos < offset + len) {
  1994. BUG_ON(i >= nfrags);
  1995. *frag = skb_shinfo(skb)->frags[i];
  1996. get_page(frag->page);
  1997. size = frag->size;
  1998. if (pos < offset) {
  1999. frag->page_offset += offset - pos;
  2000. frag->size -= offset - pos;
  2001. }
  2002. k++;
  2003. if (pos + size <= offset + len) {
  2004. i++;
  2005. pos += size;
  2006. } else {
  2007. frag->size -= pos + size - (offset + len);
  2008. break;
  2009. }
  2010. frag++;
  2011. }
  2012. skb_shinfo(nskb)->nr_frags = k;
  2013. nskb->data_len = len - hsize;
  2014. nskb->len += nskb->data_len;
  2015. nskb->truesize += nskb->data_len;
  2016. } while ((offset += len) < skb->len);
  2017. return segs;
  2018. err:
  2019. while ((skb = segs)) {
  2020. segs = skb->next;
  2021. kfree_skb(skb);
  2022. }
  2023. return ERR_PTR(err);
  2024. }
  2025. EXPORT_SYMBOL_GPL(skb_segment);
  2026. void __init skb_init(void)
  2027. {
  2028. skbuff_head_cache = kmem_cache_create("skbuff_head_cache",
  2029. sizeof(struct sk_buff),
  2030. 0,
  2031. SLAB_HWCACHE_ALIGN|SLAB_PANIC,
  2032. NULL);
  2033. skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
  2034. (2*sizeof(struct sk_buff)) +
  2035. sizeof(atomic_t),
  2036. 0,
  2037. SLAB_HWCACHE_ALIGN|SLAB_PANIC,
  2038. NULL);
  2039. }
  2040. /**
  2041. * skb_to_sgvec - Fill a scatter-gather list from a socket buffer
  2042. * @skb: Socket buffer containing the buffers to be mapped
  2043. * @sg: The scatter-gather list to map into
  2044. * @offset: The offset into the buffer's contents to start mapping
  2045. * @len: Length of buffer space to be mapped
  2046. *
  2047. * Fill the specified scatter-gather list with mappings/pointers into a
  2048. * region of the buffer space attached to a socket buffer.
  2049. */
  2050. static int
  2051. __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
  2052. {
  2053. int start = skb_headlen(skb);
  2054. int i, copy = start - offset;
  2055. int elt = 0;
  2056. if (copy > 0) {
  2057. if (copy > len)
  2058. copy = len;
  2059. sg_set_buf(sg, skb->data + offset, copy);
  2060. elt++;
  2061. if ((len -= copy) == 0)
  2062. return elt;
  2063. offset += copy;
  2064. }
  2065. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  2066. int end;
  2067. BUG_TRAP(start <= offset + len);
  2068. end = start + skb_shinfo(skb)->frags[i].size;
  2069. if ((copy = end - offset) > 0) {
  2070. skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
  2071. if (copy > len)
  2072. copy = len;
  2073. sg_set_page(&sg[elt], frag->page, copy,
  2074. frag->page_offset+offset-start);
  2075. elt++;
  2076. if (!(len -= copy))
  2077. return elt;
  2078. offset += copy;
  2079. }
  2080. start = end;
  2081. }
  2082. if (skb_shinfo(skb)->frag_list) {
  2083. struct sk_buff *list = skb_shinfo(skb)->frag_list;
  2084. for (; list; list = list->next) {
  2085. int end;
  2086. BUG_TRAP(start <= offset + len);
  2087. end = start + list->len;
  2088. if ((copy = end - offset) > 0) {
  2089. if (copy > len)
  2090. copy = len;
  2091. elt += __skb_to_sgvec(list, sg+elt, offset - start,
  2092. copy);
  2093. if ((len -= copy) == 0)
  2094. return elt;
  2095. offset += copy;
  2096. }
  2097. start = end;
  2098. }
  2099. }
  2100. BUG_ON(len);
  2101. return elt;
  2102. }
  2103. int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
  2104. {
  2105. int nsg = __skb_to_sgvec(skb, sg, offset, len);
  2106. sg_mark_end(&sg[nsg - 1]);
  2107. return nsg;
  2108. }
  2109. /**
  2110. * skb_cow_data - Check that a socket buffer's data buffers are writable
  2111. * @skb: The socket buffer to check.
  2112. * @tailbits: Amount of trailing space to be added
  2113. * @trailer: Returned pointer to the skb where the @tailbits space begins
  2114. *
  2115. * Make sure that the data buffers attached to a socket buffer are
  2116. * writable. If they are not, private copies are made of the data buffers
  2117. * and the socket buffer is set to use these instead.
  2118. *
  2119. * If @tailbits is given, make sure that there is space to write @tailbits
  2120. * bytes of data beyond current end of socket buffer. @trailer will be
  2121. * set to point to the skb in which this space begins.
  2122. *
  2123. * The number of scatterlist elements required to completely map the
  2124. * COW'd and extended socket buffer will be returned.
  2125. */
  2126. int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
  2127. {
  2128. int copyflag;
  2129. int elt;
  2130. struct sk_buff *skb1, **skb_p;
  2131. /* If skb is cloned or its head is paged, reallocate
  2132. * head pulling out all the pages (pages are considered not writable
  2133. * at the moment even if they are anonymous).
  2134. */
  2135. if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
  2136. __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL)
  2137. return -ENOMEM;
  2138. /* Easy case. Most of packets will go this way. */
  2139. if (!skb_shinfo(skb)->frag_list) {
  2140. /* A little of trouble, not enough of space for trailer.
  2141. * This should not happen, when stack is tuned to generate
  2142. * good frames. OK, on miss we reallocate and reserve even more
  2143. * space, 128 bytes is fair. */
  2144. if (skb_tailroom(skb) < tailbits &&
  2145. pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
  2146. return -ENOMEM;
  2147. /* Voila! */
  2148. *trailer = skb;
  2149. return 1;
  2150. }
  2151. /* Misery. We are in troubles, going to mincer fragments... */
  2152. elt = 1;
  2153. skb_p = &skb_shinfo(skb)->frag_list;
  2154. copyflag = 0;
  2155. while ((skb1 = *skb_p) != NULL) {
  2156. int ntail = 0;
  2157. /* The fragment is partially pulled by someone,
  2158. * this can happen on input. Copy it and everything
  2159. * after it. */
  2160. if (skb_shared(skb1))
  2161. copyflag = 1;
  2162. /* If the skb is the last, worry about trailer. */
  2163. if (skb1->next == NULL && tailbits) {
  2164. if (skb_shinfo(skb1)->nr_frags ||
  2165. skb_shinfo(skb1)->frag_list ||
  2166. skb_tailroom(skb1) < tailbits)
  2167. ntail = tailbits + 128;
  2168. }
  2169. if (copyflag ||
  2170. skb_cloned(skb1) ||
  2171. ntail ||
  2172. skb_shinfo(skb1)->nr_frags ||
  2173. skb_shinfo(skb1)->frag_list) {
  2174. struct sk_buff *skb2;
  2175. /* Fuck, we are miserable poor guys... */
  2176. if (ntail == 0)
  2177. skb2 = skb_copy(skb1, GFP_ATOMIC);
  2178. else
  2179. skb2 = skb_copy_expand(skb1,
  2180. skb_headroom(skb1),
  2181. ntail,
  2182. GFP_ATOMIC);
  2183. if (unlikely(skb2 == NULL))
  2184. return -ENOMEM;
  2185. if (skb1->sk)
  2186. skb_set_owner_w(skb2, skb1->sk);
  2187. /* Looking around. Are we still alive?
  2188. * OK, link new skb, drop old one */
  2189. skb2->next = skb1->next;
  2190. *skb_p = skb2;
  2191. kfree_skb(skb1);
  2192. skb1 = skb2;
  2193. }
  2194. elt++;
  2195. *trailer = skb1;
  2196. skb_p = &skb1->next;
  2197. }
  2198. return elt;
  2199. }
  2200. /**
  2201. * skb_partial_csum_set - set up and verify partial csum values for packet
  2202. * @skb: the skb to set
  2203. * @start: the number of bytes after skb->data to start checksumming.
  2204. * @off: the offset from start to place the checksum.
  2205. *
  2206. * For untrusted partially-checksummed packets, we need to make sure the values
  2207. * for skb->csum_start and skb->csum_offset are valid so we don't oops.
  2208. *
  2209. * This function checks and sets those values and skb->ip_summed: if this
  2210. * returns false you should drop the packet.
  2211. */
  2212. bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
  2213. {
  2214. if (unlikely(start > skb->len - 2) ||
  2215. unlikely((int)start + off > skb->len - 2)) {
  2216. if (net_ratelimit())
  2217. printk(KERN_WARNING
  2218. "bad partial csum: csum=%u/%u len=%u\n",
  2219. start, off, skb->len);
  2220. return false;
  2221. }
  2222. skb->ip_summed = CHECKSUM_PARTIAL;
  2223. skb->csum_start = skb_headroom(skb) + start;
  2224. skb->csum_offset = off;
  2225. return true;
  2226. }
  2227. EXPORT_SYMBOL(___pskb_trim);
  2228. EXPORT_SYMBOL(__kfree_skb);
  2229. EXPORT_SYMBOL(kfree_skb);
  2230. EXPORT_SYMBOL(__pskb_pull_tail);
  2231. EXPORT_SYMBOL(__alloc_skb);
  2232. EXPORT_SYMBOL(__netdev_alloc_skb);
  2233. EXPORT_SYMBOL(pskb_copy);
  2234. EXPORT_SYMBOL(pskb_expand_head);
  2235. EXPORT_SYMBOL(skb_checksum);
  2236. EXPORT_SYMBOL(skb_clone);
  2237. EXPORT_SYMBOL(skb_copy);
  2238. EXPORT_SYMBOL(skb_copy_and_csum_bits);
  2239. EXPORT_SYMBOL(skb_copy_and_csum_dev);
  2240. EXPORT_SYMBOL(skb_copy_bits);
  2241. EXPORT_SYMBOL(skb_copy_expand);
  2242. EXPORT_SYMBOL(skb_over_panic);
  2243. EXPORT_SYMBOL(skb_pad);
  2244. EXPORT_SYMBOL(skb_realloc_headroom);
  2245. EXPORT_SYMBOL(skb_under_panic);
  2246. EXPORT_SYMBOL(skb_dequeue);
  2247. EXPORT_SYMBOL(skb_dequeue_tail);
  2248. EXPORT_SYMBOL(skb_insert);
  2249. EXPORT_SYMBOL(skb_queue_purge);
  2250. EXPORT_SYMBOL(skb_queue_head);
  2251. EXPORT_SYMBOL(skb_queue_tail);
  2252. EXPORT_SYMBOL(skb_unlink);
  2253. EXPORT_SYMBOL(skb_append);
  2254. EXPORT_SYMBOL(skb_split);
  2255. EXPORT_SYMBOL(skb_prepare_seq_read);
  2256. EXPORT_SYMBOL(skb_seq_read);
  2257. EXPORT_SYMBOL(skb_abort_seq_read);
  2258. EXPORT_SYMBOL(skb_find_text);
  2259. EXPORT_SYMBOL(skb_append_datato_frags);
  2260. EXPORT_SYMBOL_GPL(skb_to_sgvec);
  2261. EXPORT_SYMBOL_GPL(skb_cow_data);
  2262. EXPORT_SYMBOL_GPL(skb_partial_csum_set);