svc.c 31 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312
  1. /*
  2. * linux/net/sunrpc/svc.c
  3. *
  4. * High-level RPC service routines
  5. *
  6. * Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de>
  7. *
  8. * Multiple threads pools and NUMAisation
  9. * Copyright (c) 2006 Silicon Graphics, Inc.
  10. * by Greg Banks <gnb@melbourne.sgi.com>
  11. */
  12. #include <linux/linkage.h>
  13. #include <linux/sched.h>
  14. #include <linux/errno.h>
  15. #include <linux/net.h>
  16. #include <linux/in.h>
  17. #include <linux/mm.h>
  18. #include <linux/interrupt.h>
  19. #include <linux/module.h>
  20. #include <linux/kthread.h>
  21. #include <linux/sunrpc/types.h>
  22. #include <linux/sunrpc/xdr.h>
  23. #include <linux/sunrpc/stats.h>
  24. #include <linux/sunrpc/svcsock.h>
  25. #include <linux/sunrpc/clnt.h>
  26. #include <linux/sunrpc/bc_xprt.h>
  27. #define RPCDBG_FACILITY RPCDBG_SVCDSP
  28. static void svc_unregister(const struct svc_serv *serv);
  29. #define svc_serv_is_pooled(serv) ((serv)->sv_function)
  30. /*
  31. * Mode for mapping cpus to pools.
  32. */
  33. enum {
  34. SVC_POOL_AUTO = -1, /* choose one of the others */
  35. SVC_POOL_GLOBAL, /* no mapping, just a single global pool
  36. * (legacy & UP mode) */
  37. SVC_POOL_PERCPU, /* one pool per cpu */
  38. SVC_POOL_PERNODE /* one pool per numa node */
  39. };
  40. #define SVC_POOL_DEFAULT SVC_POOL_GLOBAL
  41. /*
  42. * Structure for mapping cpus to pools and vice versa.
  43. * Setup once during sunrpc initialisation.
  44. */
  45. static struct svc_pool_map {
  46. int count; /* How many svc_servs use us */
  47. int mode; /* Note: int not enum to avoid
  48. * warnings about "enumeration value
  49. * not handled in switch" */
  50. unsigned int npools;
  51. unsigned int *pool_to; /* maps pool id to cpu or node */
  52. unsigned int *to_pool; /* maps cpu or node to pool id */
  53. } svc_pool_map = {
  54. .count = 0,
  55. .mode = SVC_POOL_DEFAULT
  56. };
  57. static DEFINE_MUTEX(svc_pool_map_mutex);/* protects svc_pool_map.count only */
  58. static int
  59. param_set_pool_mode(const char *val, struct kernel_param *kp)
  60. {
  61. int *ip = (int *)kp->arg;
  62. struct svc_pool_map *m = &svc_pool_map;
  63. int err;
  64. mutex_lock(&svc_pool_map_mutex);
  65. err = -EBUSY;
  66. if (m->count)
  67. goto out;
  68. err = 0;
  69. if (!strncmp(val, "auto", 4))
  70. *ip = SVC_POOL_AUTO;
  71. else if (!strncmp(val, "global", 6))
  72. *ip = SVC_POOL_GLOBAL;
  73. else if (!strncmp(val, "percpu", 6))
  74. *ip = SVC_POOL_PERCPU;
  75. else if (!strncmp(val, "pernode", 7))
  76. *ip = SVC_POOL_PERNODE;
  77. else
  78. err = -EINVAL;
  79. out:
  80. mutex_unlock(&svc_pool_map_mutex);
  81. return err;
  82. }
  83. static int
  84. param_get_pool_mode(char *buf, struct kernel_param *kp)
  85. {
  86. int *ip = (int *)kp->arg;
  87. switch (*ip)
  88. {
  89. case SVC_POOL_AUTO:
  90. return strlcpy(buf, "auto", 20);
  91. case SVC_POOL_GLOBAL:
  92. return strlcpy(buf, "global", 20);
  93. case SVC_POOL_PERCPU:
  94. return strlcpy(buf, "percpu", 20);
  95. case SVC_POOL_PERNODE:
  96. return strlcpy(buf, "pernode", 20);
  97. default:
  98. return sprintf(buf, "%d", *ip);
  99. }
  100. }
  101. module_param_call(pool_mode, param_set_pool_mode, param_get_pool_mode,
  102. &svc_pool_map.mode, 0644);
  103. /*
  104. * Detect best pool mapping mode heuristically,
  105. * according to the machine's topology.
  106. */
  107. static int
  108. svc_pool_map_choose_mode(void)
  109. {
  110. unsigned int node;
  111. if (nr_online_nodes > 1) {
  112. /*
  113. * Actually have multiple NUMA nodes,
  114. * so split pools on NUMA node boundaries
  115. */
  116. return SVC_POOL_PERNODE;
  117. }
  118. node = first_online_node;
  119. if (nr_cpus_node(node) > 2) {
  120. /*
  121. * Non-trivial SMP, or CONFIG_NUMA on
  122. * non-NUMA hardware, e.g. with a generic
  123. * x86_64 kernel on Xeons. In this case we
  124. * want to divide the pools on cpu boundaries.
  125. */
  126. return SVC_POOL_PERCPU;
  127. }
  128. /* default: one global pool */
  129. return SVC_POOL_GLOBAL;
  130. }
  131. /*
  132. * Allocate the to_pool[] and pool_to[] arrays.
  133. * Returns 0 on success or an errno.
  134. */
  135. static int
  136. svc_pool_map_alloc_arrays(struct svc_pool_map *m, unsigned int maxpools)
  137. {
  138. m->to_pool = kcalloc(maxpools, sizeof(unsigned int), GFP_KERNEL);
  139. if (!m->to_pool)
  140. goto fail;
  141. m->pool_to = kcalloc(maxpools, sizeof(unsigned int), GFP_KERNEL);
  142. if (!m->pool_to)
  143. goto fail_free;
  144. return 0;
  145. fail_free:
  146. kfree(m->to_pool);
  147. fail:
  148. return -ENOMEM;
  149. }
  150. /*
  151. * Initialise the pool map for SVC_POOL_PERCPU mode.
  152. * Returns number of pools or <0 on error.
  153. */
  154. static int
  155. svc_pool_map_init_percpu(struct svc_pool_map *m)
  156. {
  157. unsigned int maxpools = nr_cpu_ids;
  158. unsigned int pidx = 0;
  159. unsigned int cpu;
  160. int err;
  161. err = svc_pool_map_alloc_arrays(m, maxpools);
  162. if (err)
  163. return err;
  164. for_each_online_cpu(cpu) {
  165. BUG_ON(pidx > maxpools);
  166. m->to_pool[cpu] = pidx;
  167. m->pool_to[pidx] = cpu;
  168. pidx++;
  169. }
  170. /* cpus brought online later all get mapped to pool0, sorry */
  171. return pidx;
  172. };
  173. /*
  174. * Initialise the pool map for SVC_POOL_PERNODE mode.
  175. * Returns number of pools or <0 on error.
  176. */
  177. static int
  178. svc_pool_map_init_pernode(struct svc_pool_map *m)
  179. {
  180. unsigned int maxpools = nr_node_ids;
  181. unsigned int pidx = 0;
  182. unsigned int node;
  183. int err;
  184. err = svc_pool_map_alloc_arrays(m, maxpools);
  185. if (err)
  186. return err;
  187. for_each_node_with_cpus(node) {
  188. /* some architectures (e.g. SN2) have cpuless nodes */
  189. BUG_ON(pidx > maxpools);
  190. m->to_pool[node] = pidx;
  191. m->pool_to[pidx] = node;
  192. pidx++;
  193. }
  194. /* nodes brought online later all get mapped to pool0, sorry */
  195. return pidx;
  196. }
  197. /*
  198. * Add a reference to the global map of cpus to pools (and
  199. * vice versa). Initialise the map if we're the first user.
  200. * Returns the number of pools.
  201. */
  202. static unsigned int
  203. svc_pool_map_get(void)
  204. {
  205. struct svc_pool_map *m = &svc_pool_map;
  206. int npools = -1;
  207. mutex_lock(&svc_pool_map_mutex);
  208. if (m->count++) {
  209. mutex_unlock(&svc_pool_map_mutex);
  210. return m->npools;
  211. }
  212. if (m->mode == SVC_POOL_AUTO)
  213. m->mode = svc_pool_map_choose_mode();
  214. switch (m->mode) {
  215. case SVC_POOL_PERCPU:
  216. npools = svc_pool_map_init_percpu(m);
  217. break;
  218. case SVC_POOL_PERNODE:
  219. npools = svc_pool_map_init_pernode(m);
  220. break;
  221. }
  222. if (npools < 0) {
  223. /* default, or memory allocation failure */
  224. npools = 1;
  225. m->mode = SVC_POOL_GLOBAL;
  226. }
  227. m->npools = npools;
  228. mutex_unlock(&svc_pool_map_mutex);
  229. return m->npools;
  230. }
  231. /*
  232. * Drop a reference to the global map of cpus to pools.
  233. * When the last reference is dropped, the map data is
  234. * freed; this allows the sysadmin to change the pool
  235. * mode using the pool_mode module option without
  236. * rebooting or re-loading sunrpc.ko.
  237. */
  238. static void
  239. svc_pool_map_put(void)
  240. {
  241. struct svc_pool_map *m = &svc_pool_map;
  242. mutex_lock(&svc_pool_map_mutex);
  243. if (!--m->count) {
  244. m->mode = SVC_POOL_DEFAULT;
  245. kfree(m->to_pool);
  246. kfree(m->pool_to);
  247. m->npools = 0;
  248. }
  249. mutex_unlock(&svc_pool_map_mutex);
  250. }
  251. /*
  252. * Set the given thread's cpus_allowed mask so that it
  253. * will only run on cpus in the given pool.
  254. */
  255. static inline void
  256. svc_pool_map_set_cpumask(struct task_struct *task, unsigned int pidx)
  257. {
  258. struct svc_pool_map *m = &svc_pool_map;
  259. unsigned int node = m->pool_to[pidx];
  260. /*
  261. * The caller checks for sv_nrpools > 1, which
  262. * implies that we've been initialized.
  263. */
  264. BUG_ON(m->count == 0);
  265. switch (m->mode) {
  266. case SVC_POOL_PERCPU:
  267. {
  268. set_cpus_allowed_ptr(task, cpumask_of(node));
  269. break;
  270. }
  271. case SVC_POOL_PERNODE:
  272. {
  273. set_cpus_allowed_ptr(task, cpumask_of_node(node));
  274. break;
  275. }
  276. }
  277. }
  278. /*
  279. * Use the mapping mode to choose a pool for a given CPU.
  280. * Used when enqueueing an incoming RPC. Always returns
  281. * a non-NULL pool pointer.
  282. */
  283. struct svc_pool *
  284. svc_pool_for_cpu(struct svc_serv *serv, int cpu)
  285. {
  286. struct svc_pool_map *m = &svc_pool_map;
  287. unsigned int pidx = 0;
  288. /*
  289. * An uninitialised map happens in a pure client when
  290. * lockd is brought up, so silently treat it the
  291. * same as SVC_POOL_GLOBAL.
  292. */
  293. if (svc_serv_is_pooled(serv)) {
  294. switch (m->mode) {
  295. case SVC_POOL_PERCPU:
  296. pidx = m->to_pool[cpu];
  297. break;
  298. case SVC_POOL_PERNODE:
  299. pidx = m->to_pool[cpu_to_node(cpu)];
  300. break;
  301. }
  302. }
  303. return &serv->sv_pools[pidx % serv->sv_nrpools];
  304. }
  305. /*
  306. * Create an RPC service
  307. */
  308. static struct svc_serv *
  309. __svc_create(struct svc_program *prog, unsigned int bufsize, int npools,
  310. void (*shutdown)(struct svc_serv *serv))
  311. {
  312. struct svc_serv *serv;
  313. unsigned int vers;
  314. unsigned int xdrsize;
  315. unsigned int i;
  316. if (!(serv = kzalloc(sizeof(*serv), GFP_KERNEL)))
  317. return NULL;
  318. serv->sv_name = prog->pg_name;
  319. serv->sv_program = prog;
  320. serv->sv_nrthreads = 1;
  321. serv->sv_stats = prog->pg_stats;
  322. if (bufsize > RPCSVC_MAXPAYLOAD)
  323. bufsize = RPCSVC_MAXPAYLOAD;
  324. serv->sv_max_payload = bufsize? bufsize : 4096;
  325. serv->sv_max_mesg = roundup(serv->sv_max_payload + PAGE_SIZE, PAGE_SIZE);
  326. serv->sv_shutdown = shutdown;
  327. xdrsize = 0;
  328. while (prog) {
  329. prog->pg_lovers = prog->pg_nvers-1;
  330. for (vers=0; vers<prog->pg_nvers ; vers++)
  331. if (prog->pg_vers[vers]) {
  332. prog->pg_hivers = vers;
  333. if (prog->pg_lovers > vers)
  334. prog->pg_lovers = vers;
  335. if (prog->pg_vers[vers]->vs_xdrsize > xdrsize)
  336. xdrsize = prog->pg_vers[vers]->vs_xdrsize;
  337. }
  338. prog = prog->pg_next;
  339. }
  340. serv->sv_xdrsize = xdrsize;
  341. INIT_LIST_HEAD(&serv->sv_tempsocks);
  342. INIT_LIST_HEAD(&serv->sv_permsocks);
  343. init_timer(&serv->sv_temptimer);
  344. spin_lock_init(&serv->sv_lock);
  345. serv->sv_nrpools = npools;
  346. serv->sv_pools =
  347. kcalloc(serv->sv_nrpools, sizeof(struct svc_pool),
  348. GFP_KERNEL);
  349. if (!serv->sv_pools) {
  350. kfree(serv);
  351. return NULL;
  352. }
  353. for (i = 0; i < serv->sv_nrpools; i++) {
  354. struct svc_pool *pool = &serv->sv_pools[i];
  355. dprintk("svc: initialising pool %u for %s\n",
  356. i, serv->sv_name);
  357. pool->sp_id = i;
  358. INIT_LIST_HEAD(&pool->sp_threads);
  359. INIT_LIST_HEAD(&pool->sp_sockets);
  360. INIT_LIST_HEAD(&pool->sp_all_threads);
  361. spin_lock_init(&pool->sp_lock);
  362. }
  363. /* Remove any stale portmap registrations */
  364. svc_unregister(serv);
  365. return serv;
  366. }
  367. struct svc_serv *
  368. svc_create(struct svc_program *prog, unsigned int bufsize,
  369. void (*shutdown)(struct svc_serv *serv))
  370. {
  371. return __svc_create(prog, bufsize, /*npools*/1, shutdown);
  372. }
  373. EXPORT_SYMBOL_GPL(svc_create);
  374. struct svc_serv *
  375. svc_create_pooled(struct svc_program *prog, unsigned int bufsize,
  376. void (*shutdown)(struct svc_serv *serv),
  377. svc_thread_fn func, struct module *mod)
  378. {
  379. struct svc_serv *serv;
  380. unsigned int npools = svc_pool_map_get();
  381. serv = __svc_create(prog, bufsize, npools, shutdown);
  382. if (serv != NULL) {
  383. serv->sv_function = func;
  384. serv->sv_module = mod;
  385. }
  386. return serv;
  387. }
  388. EXPORT_SYMBOL_GPL(svc_create_pooled);
  389. /*
  390. * Destroy an RPC service. Should be called with appropriate locking to
  391. * protect the sv_nrthreads, sv_permsocks and sv_tempsocks.
  392. */
  393. void
  394. svc_destroy(struct svc_serv *serv)
  395. {
  396. dprintk("svc: svc_destroy(%s, %d)\n",
  397. serv->sv_program->pg_name,
  398. serv->sv_nrthreads);
  399. if (serv->sv_nrthreads) {
  400. if (--(serv->sv_nrthreads) != 0) {
  401. svc_sock_update_bufs(serv);
  402. return;
  403. }
  404. } else
  405. printk("svc_destroy: no threads for serv=%p!\n", serv);
  406. del_timer_sync(&serv->sv_temptimer);
  407. svc_close_all(&serv->sv_tempsocks);
  408. if (serv->sv_shutdown)
  409. serv->sv_shutdown(serv);
  410. svc_close_all(&serv->sv_permsocks);
  411. BUG_ON(!list_empty(&serv->sv_permsocks));
  412. BUG_ON(!list_empty(&serv->sv_tempsocks));
  413. cache_clean_deferred(serv);
  414. if (svc_serv_is_pooled(serv))
  415. svc_pool_map_put();
  416. #if defined(CONFIG_NFS_V4_1)
  417. svc_sock_destroy(serv->bc_xprt);
  418. #endif /* CONFIG_NFS_V4_1 */
  419. svc_unregister(serv);
  420. kfree(serv->sv_pools);
  421. kfree(serv);
  422. }
  423. EXPORT_SYMBOL_GPL(svc_destroy);
  424. /*
  425. * Allocate an RPC server's buffer space.
  426. * We allocate pages and place them in rq_argpages.
  427. */
  428. static int
  429. svc_init_buffer(struct svc_rqst *rqstp, unsigned int size)
  430. {
  431. unsigned int pages, arghi;
  432. /* bc_xprt uses fore channel allocated buffers */
  433. if (svc_is_backchannel(rqstp))
  434. return 1;
  435. pages = size / PAGE_SIZE + 1; /* extra page as we hold both request and reply.
  436. * We assume one is at most one page
  437. */
  438. arghi = 0;
  439. BUG_ON(pages > RPCSVC_MAXPAGES);
  440. while (pages) {
  441. struct page *p = alloc_page(GFP_KERNEL);
  442. if (!p)
  443. break;
  444. rqstp->rq_pages[arghi++] = p;
  445. pages--;
  446. }
  447. return pages == 0;
  448. }
  449. /*
  450. * Release an RPC server buffer
  451. */
  452. static void
  453. svc_release_buffer(struct svc_rqst *rqstp)
  454. {
  455. unsigned int i;
  456. for (i = 0; i < ARRAY_SIZE(rqstp->rq_pages); i++)
  457. if (rqstp->rq_pages[i])
  458. put_page(rqstp->rq_pages[i]);
  459. }
  460. struct svc_rqst *
  461. svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool)
  462. {
  463. struct svc_rqst *rqstp;
  464. rqstp = kzalloc(sizeof(*rqstp), GFP_KERNEL);
  465. if (!rqstp)
  466. goto out_enomem;
  467. init_waitqueue_head(&rqstp->rq_wait);
  468. serv->sv_nrthreads++;
  469. spin_lock_bh(&pool->sp_lock);
  470. pool->sp_nrthreads++;
  471. list_add(&rqstp->rq_all, &pool->sp_all_threads);
  472. spin_unlock_bh(&pool->sp_lock);
  473. rqstp->rq_server = serv;
  474. rqstp->rq_pool = pool;
  475. rqstp->rq_argp = kmalloc(serv->sv_xdrsize, GFP_KERNEL);
  476. if (!rqstp->rq_argp)
  477. goto out_thread;
  478. rqstp->rq_resp = kmalloc(serv->sv_xdrsize, GFP_KERNEL);
  479. if (!rqstp->rq_resp)
  480. goto out_thread;
  481. if (!svc_init_buffer(rqstp, serv->sv_max_mesg))
  482. goto out_thread;
  483. return rqstp;
  484. out_thread:
  485. svc_exit_thread(rqstp);
  486. out_enomem:
  487. return ERR_PTR(-ENOMEM);
  488. }
  489. EXPORT_SYMBOL_GPL(svc_prepare_thread);
  490. /*
  491. * Choose a pool in which to create a new thread, for svc_set_num_threads
  492. */
  493. static inline struct svc_pool *
  494. choose_pool(struct svc_serv *serv, struct svc_pool *pool, unsigned int *state)
  495. {
  496. if (pool != NULL)
  497. return pool;
  498. return &serv->sv_pools[(*state)++ % serv->sv_nrpools];
  499. }
  500. /*
  501. * Choose a thread to kill, for svc_set_num_threads
  502. */
  503. static inline struct task_struct *
  504. choose_victim(struct svc_serv *serv, struct svc_pool *pool, unsigned int *state)
  505. {
  506. unsigned int i;
  507. struct task_struct *task = NULL;
  508. if (pool != NULL) {
  509. spin_lock_bh(&pool->sp_lock);
  510. } else {
  511. /* choose a pool in round-robin fashion */
  512. for (i = 0; i < serv->sv_nrpools; i++) {
  513. pool = &serv->sv_pools[--(*state) % serv->sv_nrpools];
  514. spin_lock_bh(&pool->sp_lock);
  515. if (!list_empty(&pool->sp_all_threads))
  516. goto found_pool;
  517. spin_unlock_bh(&pool->sp_lock);
  518. }
  519. return NULL;
  520. }
  521. found_pool:
  522. if (!list_empty(&pool->sp_all_threads)) {
  523. struct svc_rqst *rqstp;
  524. /*
  525. * Remove from the pool->sp_all_threads list
  526. * so we don't try to kill it again.
  527. */
  528. rqstp = list_entry(pool->sp_all_threads.next, struct svc_rqst, rq_all);
  529. list_del_init(&rqstp->rq_all);
  530. task = rqstp->rq_task;
  531. }
  532. spin_unlock_bh(&pool->sp_lock);
  533. return task;
  534. }
  535. /*
  536. * Create or destroy enough new threads to make the number
  537. * of threads the given number. If `pool' is non-NULL, applies
  538. * only to threads in that pool, otherwise round-robins between
  539. * all pools. Must be called with a svc_get() reference and
  540. * the BKL or another lock to protect access to svc_serv fields.
  541. *
  542. * Destroying threads relies on the service threads filling in
  543. * rqstp->rq_task, which only the nfs ones do. Assumes the serv
  544. * has been created using svc_create_pooled().
  545. *
  546. * Based on code that used to be in nfsd_svc() but tweaked
  547. * to be pool-aware.
  548. */
  549. int
  550. svc_set_num_threads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
  551. {
  552. struct svc_rqst *rqstp;
  553. struct task_struct *task;
  554. struct svc_pool *chosen_pool;
  555. int error = 0;
  556. unsigned int state = serv->sv_nrthreads-1;
  557. if (pool == NULL) {
  558. /* The -1 assumes caller has done a svc_get() */
  559. nrservs -= (serv->sv_nrthreads-1);
  560. } else {
  561. spin_lock_bh(&pool->sp_lock);
  562. nrservs -= pool->sp_nrthreads;
  563. spin_unlock_bh(&pool->sp_lock);
  564. }
  565. /* create new threads */
  566. while (nrservs > 0) {
  567. nrservs--;
  568. chosen_pool = choose_pool(serv, pool, &state);
  569. rqstp = svc_prepare_thread(serv, chosen_pool);
  570. if (IS_ERR(rqstp)) {
  571. error = PTR_ERR(rqstp);
  572. break;
  573. }
  574. __module_get(serv->sv_module);
  575. task = kthread_create(serv->sv_function, rqstp, serv->sv_name);
  576. if (IS_ERR(task)) {
  577. error = PTR_ERR(task);
  578. module_put(serv->sv_module);
  579. svc_exit_thread(rqstp);
  580. break;
  581. }
  582. rqstp->rq_task = task;
  583. if (serv->sv_nrpools > 1)
  584. svc_pool_map_set_cpumask(task, chosen_pool->sp_id);
  585. svc_sock_update_bufs(serv);
  586. wake_up_process(task);
  587. }
  588. /* destroy old threads */
  589. while (nrservs < 0 &&
  590. (task = choose_victim(serv, pool, &state)) != NULL) {
  591. send_sig(SIGINT, task, 1);
  592. nrservs++;
  593. }
  594. return error;
  595. }
  596. EXPORT_SYMBOL_GPL(svc_set_num_threads);
  597. /*
  598. * Called from a server thread as it's exiting. Caller must hold the BKL or
  599. * the "service mutex", whichever is appropriate for the service.
  600. */
  601. void
  602. svc_exit_thread(struct svc_rqst *rqstp)
  603. {
  604. struct svc_serv *serv = rqstp->rq_server;
  605. struct svc_pool *pool = rqstp->rq_pool;
  606. svc_release_buffer(rqstp);
  607. kfree(rqstp->rq_resp);
  608. kfree(rqstp->rq_argp);
  609. kfree(rqstp->rq_auth_data);
  610. spin_lock_bh(&pool->sp_lock);
  611. pool->sp_nrthreads--;
  612. list_del(&rqstp->rq_all);
  613. spin_unlock_bh(&pool->sp_lock);
  614. kfree(rqstp);
  615. /* Release the server */
  616. if (serv)
  617. svc_destroy(serv);
  618. }
  619. EXPORT_SYMBOL_GPL(svc_exit_thread);
  620. /*
  621. * Register an "inet" protocol family netid with the local
  622. * rpcbind daemon via an rpcbind v4 SET request.
  623. *
  624. * No netconfig infrastructure is available in the kernel, so
  625. * we map IP_ protocol numbers to netids by hand.
  626. *
  627. * Returns zero on success; a negative errno value is returned
  628. * if any error occurs.
  629. */
  630. static int __svc_rpcb_register4(const u32 program, const u32 version,
  631. const unsigned short protocol,
  632. const unsigned short port)
  633. {
  634. const struct sockaddr_in sin = {
  635. .sin_family = AF_INET,
  636. .sin_addr.s_addr = htonl(INADDR_ANY),
  637. .sin_port = htons(port),
  638. };
  639. const char *netid;
  640. int error;
  641. switch (protocol) {
  642. case IPPROTO_UDP:
  643. netid = RPCBIND_NETID_UDP;
  644. break;
  645. case IPPROTO_TCP:
  646. netid = RPCBIND_NETID_TCP;
  647. break;
  648. default:
  649. return -ENOPROTOOPT;
  650. }
  651. error = rpcb_v4_register(program, version,
  652. (const struct sockaddr *)&sin, netid);
  653. /*
  654. * User space didn't support rpcbind v4, so retry this
  655. * registration request with the legacy rpcbind v2 protocol.
  656. */
  657. if (error == -EPROTONOSUPPORT)
  658. error = rpcb_register(program, version, protocol, port);
  659. return error;
  660. }
  661. #if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
  662. /*
  663. * Register an "inet6" protocol family netid with the local
  664. * rpcbind daemon via an rpcbind v4 SET request.
  665. *
  666. * No netconfig infrastructure is available in the kernel, so
  667. * we map IP_ protocol numbers to netids by hand.
  668. *
  669. * Returns zero on success; a negative errno value is returned
  670. * if any error occurs.
  671. */
  672. static int __svc_rpcb_register6(const u32 program, const u32 version,
  673. const unsigned short protocol,
  674. const unsigned short port)
  675. {
  676. const struct sockaddr_in6 sin6 = {
  677. .sin6_family = AF_INET6,
  678. .sin6_addr = IN6ADDR_ANY_INIT,
  679. .sin6_port = htons(port),
  680. };
  681. const char *netid;
  682. int error;
  683. switch (protocol) {
  684. case IPPROTO_UDP:
  685. netid = RPCBIND_NETID_UDP6;
  686. break;
  687. case IPPROTO_TCP:
  688. netid = RPCBIND_NETID_TCP6;
  689. break;
  690. default:
  691. return -ENOPROTOOPT;
  692. }
  693. error = rpcb_v4_register(program, version,
  694. (const struct sockaddr *)&sin6, netid);
  695. /*
  696. * User space didn't support rpcbind version 4, so we won't
  697. * use a PF_INET6 listener.
  698. */
  699. if (error == -EPROTONOSUPPORT)
  700. error = -EAFNOSUPPORT;
  701. return error;
  702. }
  703. #endif /* defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE) */
  704. /*
  705. * Register a kernel RPC service via rpcbind version 4.
  706. *
  707. * Returns zero on success; a negative errno value is returned
  708. * if any error occurs.
  709. */
  710. static int __svc_register(const char *progname,
  711. const u32 program, const u32 version,
  712. const int family,
  713. const unsigned short protocol,
  714. const unsigned short port)
  715. {
  716. int error = -EAFNOSUPPORT;
  717. switch (family) {
  718. case PF_INET:
  719. error = __svc_rpcb_register4(program, version,
  720. protocol, port);
  721. break;
  722. #if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
  723. case PF_INET6:
  724. error = __svc_rpcb_register6(program, version,
  725. protocol, port);
  726. #endif /* defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE) */
  727. }
  728. if (error < 0)
  729. printk(KERN_WARNING "svc: failed to register %sv%u RPC "
  730. "service (errno %d).\n", progname, version, -error);
  731. return error;
  732. }
  733. /**
  734. * svc_register - register an RPC service with the local portmapper
  735. * @serv: svc_serv struct for the service to register
  736. * @family: protocol family of service's listener socket
  737. * @proto: transport protocol number to advertise
  738. * @port: port to advertise
  739. *
  740. * Service is registered for any address in the passed-in protocol family
  741. */
  742. int svc_register(const struct svc_serv *serv, const int family,
  743. const unsigned short proto, const unsigned short port)
  744. {
  745. struct svc_program *progp;
  746. unsigned int i;
  747. int error = 0;
  748. BUG_ON(proto == 0 && port == 0);
  749. for (progp = serv->sv_program; progp; progp = progp->pg_next) {
  750. for (i = 0; i < progp->pg_nvers; i++) {
  751. if (progp->pg_vers[i] == NULL)
  752. continue;
  753. dprintk("svc: svc_register(%sv%d, %s, %u, %u)%s\n",
  754. progp->pg_name,
  755. i,
  756. proto == IPPROTO_UDP? "udp" : "tcp",
  757. port,
  758. family,
  759. progp->pg_vers[i]->vs_hidden?
  760. " (but not telling portmap)" : "");
  761. if (progp->pg_vers[i]->vs_hidden)
  762. continue;
  763. error = __svc_register(progp->pg_name, progp->pg_prog,
  764. i, family, proto, port);
  765. if (error < 0)
  766. break;
  767. }
  768. }
  769. return error;
  770. }
  771. /*
  772. * If user space is running rpcbind, it should take the v4 UNSET
  773. * and clear everything for this [program, version]. If user space
  774. * is running portmap, it will reject the v4 UNSET, but won't have
  775. * any "inet6" entries anyway. So a PMAP_UNSET should be sufficient
  776. * in this case to clear all existing entries for [program, version].
  777. */
  778. static void __svc_unregister(const u32 program, const u32 version,
  779. const char *progname)
  780. {
  781. int error;
  782. error = rpcb_v4_register(program, version, NULL, "");
  783. /*
  784. * User space didn't support rpcbind v4, so retry this
  785. * request with the legacy rpcbind v2 protocol.
  786. */
  787. if (error == -EPROTONOSUPPORT)
  788. error = rpcb_register(program, version, 0, 0);
  789. dprintk("svc: %s(%sv%u), error %d\n",
  790. __func__, progname, version, error);
  791. }
  792. /*
  793. * All netids, bind addresses and ports registered for [program, version]
  794. * are removed from the local rpcbind database (if the service is not
  795. * hidden) to make way for a new instance of the service.
  796. *
  797. * The result of unregistration is reported via dprintk for those who want
  798. * verification of the result, but is otherwise not important.
  799. */
  800. static void svc_unregister(const struct svc_serv *serv)
  801. {
  802. struct svc_program *progp;
  803. unsigned long flags;
  804. unsigned int i;
  805. clear_thread_flag(TIF_SIGPENDING);
  806. for (progp = serv->sv_program; progp; progp = progp->pg_next) {
  807. for (i = 0; i < progp->pg_nvers; i++) {
  808. if (progp->pg_vers[i] == NULL)
  809. continue;
  810. if (progp->pg_vers[i]->vs_hidden)
  811. continue;
  812. __svc_unregister(progp->pg_prog, i, progp->pg_name);
  813. }
  814. }
  815. spin_lock_irqsave(&current->sighand->siglock, flags);
  816. recalc_sigpending();
  817. spin_unlock_irqrestore(&current->sighand->siglock, flags);
  818. }
  819. /*
  820. * Printk the given error with the address of the client that caused it.
  821. */
  822. static int
  823. __attribute__ ((format (printf, 2, 3)))
  824. svc_printk(struct svc_rqst *rqstp, const char *fmt, ...)
  825. {
  826. va_list args;
  827. int r;
  828. char buf[RPC_MAX_ADDRBUFLEN];
  829. if (!net_ratelimit())
  830. return 0;
  831. printk(KERN_WARNING "svc: %s: ",
  832. svc_print_addr(rqstp, buf, sizeof(buf)));
  833. va_start(args, fmt);
  834. r = vprintk(fmt, args);
  835. va_end(args);
  836. return r;
  837. }
  838. /*
  839. * Common routine for processing the RPC request.
  840. */
  841. static int
  842. svc_process_common(struct svc_rqst *rqstp, struct kvec *argv, struct kvec *resv)
  843. {
  844. struct svc_program *progp;
  845. struct svc_version *versp = NULL; /* compiler food */
  846. struct svc_procedure *procp = NULL;
  847. struct svc_serv *serv = rqstp->rq_server;
  848. kxdrproc_t xdr;
  849. __be32 *statp;
  850. u32 prog, vers, proc;
  851. __be32 auth_stat, rpc_stat;
  852. int auth_res;
  853. __be32 *reply_statp;
  854. rpc_stat = rpc_success;
  855. if (argv->iov_len < 6*4)
  856. goto err_short_len;
  857. /* Will be turned off only in gss privacy case: */
  858. rqstp->rq_splice_ok = 1;
  859. /* Will be turned off only when NFSv4 Sessions are used */
  860. rqstp->rq_usedeferral = 1;
  861. /* Setup reply header */
  862. rqstp->rq_xprt->xpt_ops->xpo_prep_reply_hdr(rqstp);
  863. svc_putu32(resv, rqstp->rq_xid);
  864. vers = svc_getnl(argv);
  865. /* First words of reply: */
  866. svc_putnl(resv, 1); /* REPLY */
  867. if (vers != 2) /* RPC version number */
  868. goto err_bad_rpc;
  869. /* Save position in case we later decide to reject: */
  870. reply_statp = resv->iov_base + resv->iov_len;
  871. svc_putnl(resv, 0); /* ACCEPT */
  872. rqstp->rq_prog = prog = svc_getnl(argv); /* program number */
  873. rqstp->rq_vers = vers = svc_getnl(argv); /* version number */
  874. rqstp->rq_proc = proc = svc_getnl(argv); /* procedure number */
  875. progp = serv->sv_program;
  876. for (progp = serv->sv_program; progp; progp = progp->pg_next)
  877. if (prog == progp->pg_prog)
  878. break;
  879. /*
  880. * Decode auth data, and add verifier to reply buffer.
  881. * We do this before anything else in order to get a decent
  882. * auth verifier.
  883. */
  884. auth_res = svc_authenticate(rqstp, &auth_stat);
  885. /* Also give the program a chance to reject this call: */
  886. if (auth_res == SVC_OK && progp) {
  887. auth_stat = rpc_autherr_badcred;
  888. auth_res = progp->pg_authenticate(rqstp);
  889. }
  890. switch (auth_res) {
  891. case SVC_OK:
  892. break;
  893. case SVC_GARBAGE:
  894. goto err_garbage;
  895. case SVC_SYSERR:
  896. rpc_stat = rpc_system_err;
  897. goto err_bad;
  898. case SVC_DENIED:
  899. goto err_bad_auth;
  900. case SVC_DROP:
  901. goto dropit;
  902. case SVC_COMPLETE:
  903. goto sendit;
  904. }
  905. if (progp == NULL)
  906. goto err_bad_prog;
  907. if (vers >= progp->pg_nvers ||
  908. !(versp = progp->pg_vers[vers]))
  909. goto err_bad_vers;
  910. procp = versp->vs_proc + proc;
  911. if (proc >= versp->vs_nproc || !procp->pc_func)
  912. goto err_bad_proc;
  913. rqstp->rq_procinfo = procp;
  914. /* Syntactic check complete */
  915. serv->sv_stats->rpccnt++;
  916. /* Build the reply header. */
  917. statp = resv->iov_base +resv->iov_len;
  918. svc_putnl(resv, RPC_SUCCESS);
  919. /* Bump per-procedure stats counter */
  920. procp->pc_count++;
  921. /* Initialize storage for argp and resp */
  922. memset(rqstp->rq_argp, 0, procp->pc_argsize);
  923. memset(rqstp->rq_resp, 0, procp->pc_ressize);
  924. /* un-reserve some of the out-queue now that we have a
  925. * better idea of reply size
  926. */
  927. if (procp->pc_xdrressize)
  928. svc_reserve_auth(rqstp, procp->pc_xdrressize<<2);
  929. /* Call the function that processes the request. */
  930. if (!versp->vs_dispatch) {
  931. /* Decode arguments */
  932. xdr = procp->pc_decode;
  933. if (xdr && !xdr(rqstp, argv->iov_base, rqstp->rq_argp))
  934. goto err_garbage;
  935. *statp = procp->pc_func(rqstp, rqstp->rq_argp, rqstp->rq_resp);
  936. /* Encode reply */
  937. if (*statp == rpc_drop_reply) {
  938. if (procp->pc_release)
  939. procp->pc_release(rqstp, NULL, rqstp->rq_resp);
  940. goto dropit;
  941. }
  942. if (*statp == rpc_success &&
  943. (xdr = procp->pc_encode) &&
  944. !xdr(rqstp, resv->iov_base+resv->iov_len, rqstp->rq_resp)) {
  945. dprintk("svc: failed to encode reply\n");
  946. /* serv->sv_stats->rpcsystemerr++; */
  947. *statp = rpc_system_err;
  948. }
  949. } else {
  950. dprintk("svc: calling dispatcher\n");
  951. if (!versp->vs_dispatch(rqstp, statp)) {
  952. /* Release reply info */
  953. if (procp->pc_release)
  954. procp->pc_release(rqstp, NULL, rqstp->rq_resp);
  955. goto dropit;
  956. }
  957. }
  958. /* Check RPC status result */
  959. if (*statp != rpc_success)
  960. resv->iov_len = ((void*)statp) - resv->iov_base + 4;
  961. /* Release reply info */
  962. if (procp->pc_release)
  963. procp->pc_release(rqstp, NULL, rqstp->rq_resp);
  964. if (procp->pc_encode == NULL)
  965. goto dropit;
  966. sendit:
  967. if (svc_authorise(rqstp))
  968. goto dropit;
  969. return 1; /* Caller can now send it */
  970. dropit:
  971. svc_authorise(rqstp); /* doesn't hurt to call this twice */
  972. dprintk("svc: svc_process dropit\n");
  973. svc_drop(rqstp);
  974. return 0;
  975. err_short_len:
  976. svc_printk(rqstp, "short len %Zd, dropping request\n",
  977. argv->iov_len);
  978. goto dropit; /* drop request */
  979. err_bad_rpc:
  980. serv->sv_stats->rpcbadfmt++;
  981. svc_putnl(resv, 1); /* REJECT */
  982. svc_putnl(resv, 0); /* RPC_MISMATCH */
  983. svc_putnl(resv, 2); /* Only RPCv2 supported */
  984. svc_putnl(resv, 2);
  985. goto sendit;
  986. err_bad_auth:
  987. dprintk("svc: authentication failed (%d)\n", ntohl(auth_stat));
  988. serv->sv_stats->rpcbadauth++;
  989. /* Restore write pointer to location of accept status: */
  990. xdr_ressize_check(rqstp, reply_statp);
  991. svc_putnl(resv, 1); /* REJECT */
  992. svc_putnl(resv, 1); /* AUTH_ERROR */
  993. svc_putnl(resv, ntohl(auth_stat)); /* status */
  994. goto sendit;
  995. err_bad_prog:
  996. dprintk("svc: unknown program %d\n", prog);
  997. serv->sv_stats->rpcbadfmt++;
  998. svc_putnl(resv, RPC_PROG_UNAVAIL);
  999. goto sendit;
  1000. err_bad_vers:
  1001. svc_printk(rqstp, "unknown version (%d for prog %d, %s)\n",
  1002. vers, prog, progp->pg_name);
  1003. serv->sv_stats->rpcbadfmt++;
  1004. svc_putnl(resv, RPC_PROG_MISMATCH);
  1005. svc_putnl(resv, progp->pg_lovers);
  1006. svc_putnl(resv, progp->pg_hivers);
  1007. goto sendit;
  1008. err_bad_proc:
  1009. svc_printk(rqstp, "unknown procedure (%d)\n", proc);
  1010. serv->sv_stats->rpcbadfmt++;
  1011. svc_putnl(resv, RPC_PROC_UNAVAIL);
  1012. goto sendit;
  1013. err_garbage:
  1014. svc_printk(rqstp, "failed to decode args\n");
  1015. rpc_stat = rpc_garbage_args;
  1016. err_bad:
  1017. serv->sv_stats->rpcbadfmt++;
  1018. svc_putnl(resv, ntohl(rpc_stat));
  1019. goto sendit;
  1020. }
  1021. EXPORT_SYMBOL_GPL(svc_process);
  1022. /*
  1023. * Process the RPC request.
  1024. */
  1025. int
  1026. svc_process(struct svc_rqst *rqstp)
  1027. {
  1028. struct kvec *argv = &rqstp->rq_arg.head[0];
  1029. struct kvec *resv = &rqstp->rq_res.head[0];
  1030. struct svc_serv *serv = rqstp->rq_server;
  1031. u32 dir;
  1032. int error;
  1033. /*
  1034. * Setup response xdr_buf.
  1035. * Initially it has just one page
  1036. */
  1037. rqstp->rq_resused = 1;
  1038. resv->iov_base = page_address(rqstp->rq_respages[0]);
  1039. resv->iov_len = 0;
  1040. rqstp->rq_res.pages = rqstp->rq_respages + 1;
  1041. rqstp->rq_res.len = 0;
  1042. rqstp->rq_res.page_base = 0;
  1043. rqstp->rq_res.page_len = 0;
  1044. rqstp->rq_res.buflen = PAGE_SIZE;
  1045. rqstp->rq_res.tail[0].iov_base = NULL;
  1046. rqstp->rq_res.tail[0].iov_len = 0;
  1047. rqstp->rq_xid = svc_getu32(argv);
  1048. dir = svc_getnl(argv);
  1049. if (dir != 0) {
  1050. /* direction != CALL */
  1051. svc_printk(rqstp, "bad direction %d, dropping request\n", dir);
  1052. serv->sv_stats->rpcbadfmt++;
  1053. svc_drop(rqstp);
  1054. return 0;
  1055. }
  1056. error = svc_process_common(rqstp, argv, resv);
  1057. if (error <= 0)
  1058. return error;
  1059. return svc_send(rqstp);
  1060. }
  1061. #if defined(CONFIG_NFS_V4_1)
  1062. /*
  1063. * Process a backchannel RPC request that arrived over an existing
  1064. * outbound connection
  1065. */
  1066. int
  1067. bc_svc_process(struct svc_serv *serv, struct rpc_rqst *req,
  1068. struct svc_rqst *rqstp)
  1069. {
  1070. struct kvec *argv = &rqstp->rq_arg.head[0];
  1071. struct kvec *resv = &rqstp->rq_res.head[0];
  1072. int error;
  1073. /* Build the svc_rqst used by the common processing routine */
  1074. rqstp->rq_xprt = serv->bc_xprt;
  1075. rqstp->rq_xid = req->rq_xid;
  1076. rqstp->rq_prot = req->rq_xprt->prot;
  1077. rqstp->rq_server = serv;
  1078. rqstp->rq_addrlen = sizeof(req->rq_xprt->addr);
  1079. memcpy(&rqstp->rq_addr, &req->rq_xprt->addr, rqstp->rq_addrlen);
  1080. memcpy(&rqstp->rq_arg, &req->rq_rcv_buf, sizeof(rqstp->rq_arg));
  1081. memcpy(&rqstp->rq_res, &req->rq_snd_buf, sizeof(rqstp->rq_res));
  1082. /* reset result send buffer "put" position */
  1083. resv->iov_len = 0;
  1084. if (rqstp->rq_prot != IPPROTO_TCP) {
  1085. printk(KERN_ERR "No support for Non-TCP transports!\n");
  1086. BUG();
  1087. }
  1088. /*
  1089. * Skip the next two words because they've already been
  1090. * processed in the trasport
  1091. */
  1092. svc_getu32(argv); /* XID */
  1093. svc_getnl(argv); /* CALLDIR */
  1094. error = svc_process_common(rqstp, argv, resv);
  1095. if (error <= 0)
  1096. return error;
  1097. memcpy(&req->rq_snd_buf, &rqstp->rq_res, sizeof(req->rq_snd_buf));
  1098. return bc_send(req);
  1099. }
  1100. EXPORT_SYMBOL(bc_svc_process);
  1101. #endif /* CONFIG_NFS_V4_1 */
  1102. /*
  1103. * Return (transport-specific) limit on the rpc payload.
  1104. */
  1105. u32 svc_max_payload(const struct svc_rqst *rqstp)
  1106. {
  1107. u32 max = rqstp->rq_xprt->xpt_class->xcl_max_payload;
  1108. if (rqstp->rq_server->sv_max_payload < max)
  1109. max = rqstp->rq_server->sv_max_payload;
  1110. return max;
  1111. }
  1112. EXPORT_SYMBOL_GPL(svc_max_payload);