mon_client.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869
  1. #include "ceph_debug.h"
  2. #include <linux/types.h>
  3. #include <linux/random.h>
  4. #include <linux/sched.h>
  5. #include "mon_client.h"
  6. #include "super.h"
  7. #include "auth.h"
  8. #include "decode.h"
  9. /*
  10. * Interact with Ceph monitor cluster. Handle requests for new map
  11. * versions, and periodically resend as needed. Also implement
  12. * statfs() and umount().
  13. *
  14. * A small cluster of Ceph "monitors" are responsible for managing critical
  15. * cluster configuration and state information. An odd number (e.g., 3, 5)
  16. * of cmon daemons use a modified version of the Paxos part-time parliament
  17. * algorithm to manage the MDS map (mds cluster membership), OSD map, and
  18. * list of clients who have mounted the file system.
  19. *
  20. * We maintain an open, active session with a monitor at all times in order to
  21. * receive timely MDSMap updates. We periodically send a keepalive byte on the
  22. * TCP socket to ensure we detect a failure. If the connection does break, we
  23. * randomly hunt for a new monitor. Once the connection is reestablished, we
  24. * resend any outstanding requests.
  25. */
  26. const static struct ceph_connection_operations mon_con_ops;
  27. /*
  28. * Decode a monmap blob (e.g., during mount).
  29. */
  30. struct ceph_monmap *ceph_monmap_decode(void *p, void *end)
  31. {
  32. struct ceph_monmap *m = NULL;
  33. int i, err = -EINVAL;
  34. struct ceph_fsid fsid;
  35. u32 epoch, num_mon;
  36. u16 version;
  37. u32 len;
  38. ceph_decode_32_safe(&p, end, len, bad);
  39. ceph_decode_need(&p, end, len, bad);
  40. dout("monmap_decode %p %p len %d\n", p, end, (int)(end-p));
  41. ceph_decode_16_safe(&p, end, version, bad);
  42. ceph_decode_need(&p, end, sizeof(fsid) + 2*sizeof(u32), bad);
  43. ceph_decode_copy(&p, &fsid, sizeof(fsid));
  44. epoch = ceph_decode_32(&p);
  45. num_mon = ceph_decode_32(&p);
  46. ceph_decode_need(&p, end, num_mon*sizeof(m->mon_inst[0]), bad);
  47. if (num_mon >= CEPH_MAX_MON)
  48. goto bad;
  49. m = kmalloc(sizeof(*m) + sizeof(m->mon_inst[0])*num_mon, GFP_NOFS);
  50. if (m == NULL)
  51. return ERR_PTR(-ENOMEM);
  52. m->fsid = fsid;
  53. m->epoch = epoch;
  54. m->num_mon = num_mon;
  55. ceph_decode_copy(&p, m->mon_inst, num_mon*sizeof(m->mon_inst[0]));
  56. for (i = 0; i < num_mon; i++)
  57. ceph_decode_addr(&m->mon_inst[i].addr);
  58. dout("monmap_decode epoch %d, num_mon %d\n", m->epoch,
  59. m->num_mon);
  60. for (i = 0; i < m->num_mon; i++)
  61. dout("monmap_decode mon%d is %s\n", i,
  62. pr_addr(&m->mon_inst[i].addr.in_addr));
  63. return m;
  64. bad:
  65. dout("monmap_decode failed with %d\n", err);
  66. kfree(m);
  67. return ERR_PTR(err);
  68. }
  69. /*
  70. * return true if *addr is included in the monmap.
  71. */
  72. int ceph_monmap_contains(struct ceph_monmap *m, struct ceph_entity_addr *addr)
  73. {
  74. int i;
  75. for (i = 0; i < m->num_mon; i++)
  76. if (ceph_entity_addr_equal(addr, &m->mon_inst[i].addr))
  77. return 1;
  78. return 0;
  79. }
  80. /*
  81. * Close monitor session, if any.
  82. */
  83. static void __close_session(struct ceph_mon_client *monc)
  84. {
  85. if (monc->con) {
  86. dout("__close_session closing mon%d\n", monc->cur_mon);
  87. ceph_con_revoke(monc->con, monc->m_auth);
  88. ceph_con_close(monc->con);
  89. monc->cur_mon = -1;
  90. ceph_auth_reset(monc->auth);
  91. }
  92. }
  93. /*
  94. * Open a session with a (new) monitor.
  95. */
  96. static int __open_session(struct ceph_mon_client *monc)
  97. {
  98. char r;
  99. int ret;
  100. if (monc->cur_mon < 0) {
  101. get_random_bytes(&r, 1);
  102. monc->cur_mon = r % monc->monmap->num_mon;
  103. dout("open_session num=%d r=%d -> mon%d\n",
  104. monc->monmap->num_mon, r, monc->cur_mon);
  105. monc->sub_sent = 0;
  106. monc->sub_renew_after = jiffies; /* i.e., expired */
  107. monc->want_next_osdmap = !!monc->want_next_osdmap;
  108. dout("open_session mon%d opening\n", monc->cur_mon);
  109. monc->con->peer_name.type = CEPH_ENTITY_TYPE_MON;
  110. monc->con->peer_name.num = cpu_to_le64(monc->cur_mon);
  111. ceph_con_open(monc->con,
  112. &monc->monmap->mon_inst[monc->cur_mon].addr);
  113. /* initiatiate authentication handshake */
  114. ret = ceph_auth_build_hello(monc->auth,
  115. monc->m_auth->front.iov_base,
  116. monc->m_auth->front_max);
  117. monc->m_auth->front.iov_len = ret;
  118. monc->m_auth->hdr.front_len = cpu_to_le32(ret);
  119. ceph_msg_get(monc->m_auth); /* keep our ref */
  120. ceph_con_send(monc->con, monc->m_auth);
  121. } else {
  122. dout("open_session mon%d already open\n", monc->cur_mon);
  123. }
  124. return 0;
  125. }
  126. static bool __sub_expired(struct ceph_mon_client *monc)
  127. {
  128. return time_after_eq(jiffies, monc->sub_renew_after);
  129. }
  130. /*
  131. * Reschedule delayed work timer.
  132. */
  133. static void __schedule_delayed(struct ceph_mon_client *monc)
  134. {
  135. unsigned delay;
  136. if (monc->cur_mon < 0 || __sub_expired(monc))
  137. delay = 10 * HZ;
  138. else
  139. delay = 20 * HZ;
  140. dout("__schedule_delayed after %u\n", delay);
  141. schedule_delayed_work(&monc->delayed_work, delay);
  142. }
  143. /*
  144. * Send subscribe request for mdsmap and/or osdmap.
  145. */
  146. static void __send_subscribe(struct ceph_mon_client *monc)
  147. {
  148. dout("__send_subscribe sub_sent=%u exp=%u want_osd=%d\n",
  149. (unsigned)monc->sub_sent, __sub_expired(monc),
  150. monc->want_next_osdmap);
  151. if ((__sub_expired(monc) && !monc->sub_sent) ||
  152. monc->want_next_osdmap == 1) {
  153. struct ceph_msg *msg;
  154. struct ceph_mon_subscribe_item *i;
  155. void *p, *end;
  156. msg = ceph_msg_new(CEPH_MSG_MON_SUBSCRIBE, 96, 0, 0, NULL);
  157. if (!msg)
  158. return;
  159. p = msg->front.iov_base;
  160. end = p + msg->front.iov_len;
  161. dout("__send_subscribe to 'mdsmap' %u+\n",
  162. (unsigned)monc->have_mdsmap);
  163. if (monc->want_next_osdmap) {
  164. dout("__send_subscribe to 'osdmap' %u\n",
  165. (unsigned)monc->have_osdmap);
  166. ceph_encode_32(&p, 3);
  167. ceph_encode_string(&p, end, "osdmap", 6);
  168. i = p;
  169. i->have = cpu_to_le64(monc->have_osdmap);
  170. i->onetime = 1;
  171. p += sizeof(*i);
  172. monc->want_next_osdmap = 2; /* requested */
  173. } else {
  174. ceph_encode_32(&p, 2);
  175. }
  176. ceph_encode_string(&p, end, "mdsmap", 6);
  177. i = p;
  178. i->have = cpu_to_le64(monc->have_mdsmap);
  179. i->onetime = 0;
  180. p += sizeof(*i);
  181. ceph_encode_string(&p, end, "monmap", 6);
  182. i = p;
  183. i->have = 0;
  184. i->onetime = 0;
  185. p += sizeof(*i);
  186. msg->front.iov_len = p - msg->front.iov_base;
  187. msg->hdr.front_len = cpu_to_le32(msg->front.iov_len);
  188. ceph_con_send(monc->con, msg);
  189. monc->sub_sent = jiffies | 1; /* never 0 */
  190. }
  191. }
  192. static void handle_subscribe_ack(struct ceph_mon_client *monc,
  193. struct ceph_msg *msg)
  194. {
  195. unsigned seconds;
  196. struct ceph_mon_subscribe_ack *h = msg->front.iov_base;
  197. if (msg->front.iov_len < sizeof(*h))
  198. goto bad;
  199. seconds = le32_to_cpu(h->duration);
  200. mutex_lock(&monc->mutex);
  201. if (monc->hunting) {
  202. pr_info("mon%d %s session established\n",
  203. monc->cur_mon, pr_addr(&monc->con->peer_addr.in_addr));
  204. monc->hunting = false;
  205. }
  206. dout("handle_subscribe_ack after %d seconds\n", seconds);
  207. monc->sub_renew_after = monc->sub_sent + (seconds >> 1)*HZ - 1;
  208. monc->sub_sent = 0;
  209. mutex_unlock(&monc->mutex);
  210. return;
  211. bad:
  212. pr_err("got corrupt subscribe-ack msg\n");
  213. }
  214. /*
  215. * Keep track of which maps we have
  216. */
  217. int ceph_monc_got_mdsmap(struct ceph_mon_client *monc, u32 got)
  218. {
  219. mutex_lock(&monc->mutex);
  220. monc->have_mdsmap = got;
  221. mutex_unlock(&monc->mutex);
  222. return 0;
  223. }
  224. int ceph_monc_got_osdmap(struct ceph_mon_client *monc, u32 got)
  225. {
  226. mutex_lock(&monc->mutex);
  227. monc->have_osdmap = got;
  228. monc->want_next_osdmap = 0;
  229. mutex_unlock(&monc->mutex);
  230. return 0;
  231. }
  232. /*
  233. * Register interest in the next osdmap
  234. */
  235. void ceph_monc_request_next_osdmap(struct ceph_mon_client *monc)
  236. {
  237. dout("request_next_osdmap have %u\n", monc->have_osdmap);
  238. mutex_lock(&monc->mutex);
  239. if (!monc->want_next_osdmap)
  240. monc->want_next_osdmap = 1;
  241. if (monc->want_next_osdmap < 2)
  242. __send_subscribe(monc);
  243. mutex_unlock(&monc->mutex);
  244. }
  245. #if 0
  246. /*
  247. * mount
  248. */
  249. static void __request_mount(struct ceph_mon_client *monc)
  250. {
  251. struct ceph_msg *msg;
  252. struct ceph_client_mount *h;
  253. dout("__request_mount\n");
  254. msg = ceph_msg_new(CEPH_MSG_CLIENT_MOUNT, sizeof(*h), 0, 0, NULL);
  255. if (IS_ERR(msg))
  256. return;
  257. h = msg->front.iov_base;
  258. h->monhdr.have_version = 0;
  259. h->monhdr.session_mon = cpu_to_le16(-1);
  260. h->monhdr.session_mon_tid = 0;
  261. ceph_con_send(monc->con, msg);
  262. }
  263. #endif
  264. /*
  265. *
  266. */
  267. int ceph_monc_open_session(struct ceph_mon_client *monc)
  268. {
  269. if (!monc->con) {
  270. monc->con = kmalloc(sizeof(*monc->con), GFP_KERNEL);
  271. if (!monc->con)
  272. return -ENOMEM;
  273. ceph_con_init(monc->client->msgr, monc->con);
  274. monc->con->private = monc;
  275. monc->con->ops = &mon_con_ops;
  276. }
  277. mutex_lock(&monc->mutex);
  278. __open_session(monc);
  279. __schedule_delayed(monc);
  280. mutex_unlock(&monc->mutex);
  281. return 0;
  282. }
  283. #if 0
  284. /*
  285. * The monitor responds with mount ack indicate mount success. The
  286. * included client ticket allows the client to talk to MDSs and OSDs.
  287. */
  288. static void handle_mount_ack(struct ceph_mon_client *monc, struct ceph_msg *msg)
  289. {
  290. struct ceph_client *client = monc->client;
  291. struct ceph_monmap *monmap = NULL, *old = monc->monmap;
  292. void *p, *end;
  293. s32 result;
  294. u32 len;
  295. s64 cnum;
  296. int err = -EINVAL;
  297. if (client->whoami >= 0) {
  298. dout("handle_mount_ack - already mounted\n");
  299. return;
  300. }
  301. mutex_lock(&monc->mutex);
  302. dout("handle_mount_ack\n");
  303. p = msg->front.iov_base;
  304. end = p + msg->front.iov_len;
  305. ceph_decode_64_safe(&p, end, cnum, bad);
  306. ceph_decode_32_safe(&p, end, result, bad);
  307. ceph_decode_32_safe(&p, end, len, bad);
  308. if (result) {
  309. pr_err("mount denied: %.*s (%d)\n", len, (char *)p,
  310. result);
  311. err = result;
  312. goto out;
  313. }
  314. p += len;
  315. ceph_decode_32_safe(&p, end, len, bad);
  316. ceph_decode_need(&p, end, len, bad);
  317. monmap = ceph_monmap_decode(p, p + len);
  318. if (IS_ERR(monmap)) {
  319. pr_err("problem decoding monmap, %d\n",
  320. (int)PTR_ERR(monmap));
  321. err = -EINVAL;
  322. goto out;
  323. }
  324. p += len;
  325. client->monc.monmap = monmap;
  326. kfree(old);
  327. client->signed_ticket = NULL;
  328. client->signed_ticket_len = 0;
  329. monc->want_mount = false;
  330. client->whoami = cnum;
  331. client->msgr->inst.name.type = CEPH_ENTITY_TYPE_CLIENT;
  332. client->msgr->inst.name.num = cpu_to_le64(cnum);
  333. pr_info("client%lld fsid " FSID_FORMAT "\n",
  334. client->whoami, PR_FSID(&client->monc.monmap->fsid));
  335. ceph_debugfs_client_init(client);
  336. __send_subscribe(monc);
  337. err = 0;
  338. goto out;
  339. bad:
  340. pr_err("error decoding mount_ack message\n");
  341. out:
  342. client->mount_err = err;
  343. mutex_unlock(&monc->mutex);
  344. wake_up(&client->mount_wq);
  345. }
  346. #endif
  347. /*
  348. * The monitor responds with mount ack indicate mount success. The
  349. * included client ticket allows the client to talk to MDSs and OSDs.
  350. */
  351. static void ceph_monc_handle_map(struct ceph_mon_client *monc, struct ceph_msg *msg)
  352. {
  353. struct ceph_client *client = monc->client;
  354. struct ceph_monmap *monmap = NULL, *old = monc->monmap;
  355. void *p, *end;
  356. mutex_lock(&monc->mutex);
  357. dout("handle_monmap\n");
  358. p = msg->front.iov_base;
  359. end = p + msg->front.iov_len;
  360. monmap = ceph_monmap_decode(p, end);
  361. if (IS_ERR(monmap)) {
  362. pr_err("problem decoding monmap, %d\n",
  363. (int)PTR_ERR(monmap));
  364. return;
  365. }
  366. if (monc->have_fsid &&
  367. ceph_fsid_compare(&monmap->fsid, &monc->monmap->fsid)) {
  368. print_hex_dump(KERN_ERR, "monmap->fsid: ", DUMP_PREFIX_NONE, 16, 1,
  369. (void *)&monmap->fsid, 16, 0);
  370. print_hex_dump(KERN_ERR, "monc->monmap->fsid: ", DUMP_PREFIX_NONE, 16, 1,
  371. (void *)&monc->monmap->fsid, 16, 0);
  372. pr_err("fsid mismatch, got a previous map with different fsid");
  373. kfree(monmap);
  374. return;
  375. }
  376. client->monc.monmap = monmap;
  377. client->monc.have_fsid = true;
  378. kfree(old);
  379. mutex_unlock(&monc->mutex);
  380. wake_up(&client->mount_wq);
  381. }
  382. /*
  383. * init client info after authentication
  384. */
  385. static void __init_authenticated_client(struct ceph_mon_client *monc)
  386. {
  387. struct ceph_client *client = monc->client;
  388. client->signed_ticket = NULL;
  389. client->signed_ticket_len = 0;
  390. client->msgr->inst.name.type = CEPH_ENTITY_TYPE_CLIENT;
  391. client->msgr->inst.name.num = monc->auth->global_id;
  392. ceph_debugfs_client_init(client);
  393. }
  394. /*
  395. * statfs
  396. */
  397. static void handle_statfs_reply(struct ceph_mon_client *monc,
  398. struct ceph_msg *msg)
  399. {
  400. struct ceph_mon_statfs_request *req;
  401. struct ceph_mon_statfs_reply *reply = msg->front.iov_base;
  402. u64 tid;
  403. if (msg->front.iov_len != sizeof(*reply))
  404. goto bad;
  405. tid = le64_to_cpu(reply->tid);
  406. dout("handle_statfs_reply %p tid %llu\n", msg, tid);
  407. mutex_lock(&monc->mutex);
  408. req = radix_tree_lookup(&monc->statfs_request_tree, tid);
  409. if (req) {
  410. *req->buf = reply->st;
  411. req->result = 0;
  412. }
  413. mutex_unlock(&monc->mutex);
  414. if (req)
  415. complete(&req->completion);
  416. return;
  417. bad:
  418. pr_err("corrupt statfs reply, no tid\n");
  419. }
  420. /*
  421. * (re)send a statfs request
  422. */
  423. static int send_statfs(struct ceph_mon_client *monc,
  424. struct ceph_mon_statfs_request *req)
  425. {
  426. struct ceph_msg *msg;
  427. struct ceph_mon_statfs *h;
  428. dout("send_statfs tid %llu\n", req->tid);
  429. msg = ceph_msg_new(CEPH_MSG_STATFS, sizeof(*h), 0, 0, NULL);
  430. if (IS_ERR(msg))
  431. return PTR_ERR(msg);
  432. req->request = msg;
  433. h = msg->front.iov_base;
  434. h->monhdr.have_version = 0;
  435. h->monhdr.session_mon = cpu_to_le16(-1);
  436. h->monhdr.session_mon_tid = 0;
  437. h->fsid = monc->monmap->fsid;
  438. h->tid = cpu_to_le64(req->tid);
  439. ceph_con_send(monc->con, msg);
  440. return 0;
  441. }
  442. /*
  443. * Do a synchronous statfs().
  444. */
  445. int ceph_monc_do_statfs(struct ceph_mon_client *monc, struct ceph_statfs *buf)
  446. {
  447. struct ceph_mon_statfs_request req;
  448. int err;
  449. req.buf = buf;
  450. init_completion(&req.completion);
  451. /* allocate memory for reply */
  452. err = ceph_msgpool_resv(&monc->msgpool_statfs_reply, 1);
  453. if (err)
  454. return err;
  455. /* register request */
  456. mutex_lock(&monc->mutex);
  457. req.tid = ++monc->last_tid;
  458. req.last_attempt = jiffies;
  459. req.delay = BASE_DELAY_INTERVAL;
  460. if (radix_tree_insert(&monc->statfs_request_tree, req.tid, &req) < 0) {
  461. mutex_unlock(&monc->mutex);
  462. pr_err("ENOMEM in do_statfs\n");
  463. return -ENOMEM;
  464. }
  465. monc->num_statfs_requests++;
  466. mutex_unlock(&monc->mutex);
  467. /* send request and wait */
  468. err = send_statfs(monc, &req);
  469. if (!err)
  470. err = wait_for_completion_interruptible(&req.completion);
  471. mutex_lock(&monc->mutex);
  472. radix_tree_delete(&monc->statfs_request_tree, req.tid);
  473. monc->num_statfs_requests--;
  474. ceph_msgpool_resv(&monc->msgpool_statfs_reply, -1);
  475. mutex_unlock(&monc->mutex);
  476. if (!err)
  477. err = req.result;
  478. return err;
  479. }
  480. /*
  481. * Resend pending statfs requests.
  482. */
  483. static void __resend_statfs(struct ceph_mon_client *monc)
  484. {
  485. u64 next_tid = 0;
  486. int got;
  487. int did = 0;
  488. struct ceph_mon_statfs_request *req;
  489. while (1) {
  490. got = radix_tree_gang_lookup(&monc->statfs_request_tree,
  491. (void **)&req,
  492. next_tid, 1);
  493. if (got == 0)
  494. break;
  495. did++;
  496. next_tid = req->tid + 1;
  497. send_statfs(monc, req);
  498. }
  499. }
  500. /*
  501. * Delayed work. If we haven't mounted yet, retry. Otherwise,
  502. * renew/retry subscription as needed (in case it is timing out, or we
  503. * got an ENOMEM). And keep the monitor connection alive.
  504. */
  505. static void delayed_work(struct work_struct *work)
  506. {
  507. struct ceph_mon_client *monc =
  508. container_of(work, struct ceph_mon_client, delayed_work.work);
  509. dout("monc delayed_work\n");
  510. mutex_lock(&monc->mutex);
  511. if (monc->hunting) {
  512. __close_session(monc);
  513. __open_session(monc); /* continue hunting */
  514. } else {
  515. ceph_con_keepalive(monc->con);
  516. if (monc->auth->ops->is_authenticated(monc->auth))
  517. __send_subscribe(monc);
  518. }
  519. __schedule_delayed(monc);
  520. mutex_unlock(&monc->mutex);
  521. }
  522. /*
  523. * On startup, we build a temporary monmap populated with the IPs
  524. * provided by mount(2).
  525. */
  526. static int build_initial_monmap(struct ceph_mon_client *monc)
  527. {
  528. struct ceph_mount_args *args = monc->client->mount_args;
  529. struct ceph_entity_addr *mon_addr = args->mon_addr;
  530. int num_mon = args->num_mon;
  531. int i;
  532. /* build initial monmap */
  533. monc->monmap = kzalloc(sizeof(*monc->monmap) +
  534. num_mon*sizeof(monc->monmap->mon_inst[0]),
  535. GFP_KERNEL);
  536. if (!monc->monmap)
  537. return -ENOMEM;
  538. for (i = 0; i < num_mon; i++) {
  539. monc->monmap->mon_inst[i].addr = mon_addr[i];
  540. monc->monmap->mon_inst[i].addr.erank = 0;
  541. monc->monmap->mon_inst[i].addr.nonce = 0;
  542. monc->monmap->mon_inst[i].name.type =
  543. CEPH_ENTITY_TYPE_MON;
  544. monc->monmap->mon_inst[i].name.num = cpu_to_le64(i);
  545. }
  546. monc->monmap->num_mon = num_mon;
  547. monc->have_fsid = false;
  548. /* release addr memory */
  549. kfree(args->mon_addr);
  550. args->mon_addr = NULL;
  551. args->num_mon = 0;
  552. return 0;
  553. }
  554. int ceph_monc_init(struct ceph_mon_client *monc, struct ceph_client *cl)
  555. {
  556. int err = 0;
  557. dout("init\n");
  558. memset(monc, 0, sizeof(*monc));
  559. monc->client = cl;
  560. monc->monmap = NULL;
  561. mutex_init(&monc->mutex);
  562. err = build_initial_monmap(monc);
  563. if (err)
  564. goto out;
  565. monc->con = NULL;
  566. /* authentication */
  567. monc->auth = ceph_auth_init(cl->mount_args->name,
  568. cl->mount_args->secret);
  569. if (IS_ERR(monc->auth))
  570. return PTR_ERR(monc->auth);
  571. monc->auth->want_keys =
  572. CEPH_ENTITY_TYPE_AUTH | CEPH_ENTITY_TYPE_MON |
  573. CEPH_ENTITY_TYPE_OSD | CEPH_ENTITY_TYPE_MDS;
  574. /* msg pools */
  575. err = ceph_msgpool_init(&monc->msgpool_subscribe_ack,
  576. sizeof(struct ceph_mon_subscribe_ack), 1, false);
  577. if (err < 0)
  578. goto out_monmap;
  579. err = ceph_msgpool_init(&monc->msgpool_statfs_reply,
  580. sizeof(struct ceph_mon_statfs_reply), 0, false);
  581. if (err < 0)
  582. goto out_pool1;
  583. err = ceph_msgpool_init(&monc->msgpool_auth_reply, 4096, 1, false);
  584. if (err < 0)
  585. goto out_pool2;
  586. monc->m_auth = ceph_msg_new(CEPH_MSG_AUTH, 4096, 0, 0, NULL);
  587. if (IS_ERR(monc->m_auth)) {
  588. err = PTR_ERR(monc->m_auth);
  589. monc->m_auth = NULL;
  590. goto out_pool3;
  591. }
  592. monc->cur_mon = -1;
  593. monc->hunting = true;
  594. monc->sub_renew_after = jiffies;
  595. monc->sub_sent = 0;
  596. INIT_DELAYED_WORK(&monc->delayed_work, delayed_work);
  597. INIT_RADIX_TREE(&monc->statfs_request_tree, GFP_NOFS);
  598. monc->num_statfs_requests = 0;
  599. monc->last_tid = 0;
  600. monc->have_mdsmap = 0;
  601. monc->have_osdmap = 0;
  602. monc->want_next_osdmap = 1;
  603. return 0;
  604. out_pool3:
  605. ceph_msgpool_destroy(&monc->msgpool_auth_reply);
  606. out_pool2:
  607. ceph_msgpool_destroy(&monc->msgpool_subscribe_ack);
  608. out_pool1:
  609. ceph_msgpool_destroy(&monc->msgpool_statfs_reply);
  610. out_monmap:
  611. kfree(monc->monmap);
  612. out:
  613. return err;
  614. }
  615. void ceph_monc_stop(struct ceph_mon_client *monc)
  616. {
  617. dout("stop\n");
  618. cancel_delayed_work_sync(&monc->delayed_work);
  619. mutex_lock(&monc->mutex);
  620. __close_session(monc);
  621. if (monc->con) {
  622. monc->con->private = NULL;
  623. monc->con->ops->put(monc->con);
  624. monc->con = NULL;
  625. }
  626. mutex_unlock(&monc->mutex);
  627. ceph_auth_destroy(monc->auth);
  628. ceph_msg_put(monc->m_auth);
  629. ceph_msgpool_destroy(&monc->msgpool_subscribe_ack);
  630. ceph_msgpool_destroy(&monc->msgpool_statfs_reply);
  631. ceph_msgpool_destroy(&monc->msgpool_auth_reply);
  632. kfree(monc->monmap);
  633. }
  634. static void handle_auth_reply(struct ceph_mon_client *monc,
  635. struct ceph_msg *msg)
  636. {
  637. int ret;
  638. mutex_lock(&monc->mutex);
  639. ret = ceph_handle_auth_reply(monc->auth, msg->front.iov_base,
  640. msg->front.iov_len,
  641. monc->m_auth->front.iov_base,
  642. monc->m_auth->front_max);
  643. if (ret < 0) {
  644. monc->client->mount_err = ret;
  645. wake_up(&monc->client->mount_wq);
  646. } else if (ret > 0) {
  647. monc->m_auth->front.iov_len = ret;
  648. monc->m_auth->hdr.front_len = cpu_to_le32(ret);
  649. ceph_msg_get(monc->m_auth); /* keep our ref */
  650. ceph_con_send(monc->con, monc->m_auth);
  651. } else if (monc->auth->ops->is_authenticated(monc->auth)) {
  652. dout("authenticated, starting session\n");
  653. __init_authenticated_client(monc);
  654. __send_subscribe(monc);
  655. __resend_statfs(monc);
  656. }
  657. mutex_unlock(&monc->mutex);
  658. }
  659. /*
  660. * handle incoming message
  661. */
  662. static void dispatch(struct ceph_connection *con, struct ceph_msg *msg)
  663. {
  664. struct ceph_mon_client *monc = con->private;
  665. int type = le16_to_cpu(msg->hdr.type);
  666. if (!monc)
  667. return;
  668. switch (type) {
  669. case CEPH_MSG_AUTH_REPLY:
  670. handle_auth_reply(monc, msg);
  671. break;
  672. case CEPH_MSG_MON_SUBSCRIBE_ACK:
  673. handle_subscribe_ack(monc, msg);
  674. break;
  675. case CEPH_MSG_STATFS_REPLY:
  676. handle_statfs_reply(monc, msg);
  677. break;
  678. case CEPH_MSG_MON_MAP:
  679. ceph_monc_handle_map(monc, msg);
  680. break;
  681. case CEPH_MSG_MDS_MAP:
  682. ceph_mdsc_handle_map(&monc->client->mdsc, msg);
  683. break;
  684. case CEPH_MSG_OSD_MAP:
  685. ceph_osdc_handle_map(&monc->client->osdc, msg);
  686. break;
  687. default:
  688. pr_err("received unknown message type %d %s\n", type,
  689. ceph_msg_type_name(type));
  690. }
  691. ceph_msg_put(msg);
  692. }
  693. /*
  694. * Allocate memory for incoming message
  695. */
  696. static struct ceph_msg *mon_alloc_msg(struct ceph_connection *con,
  697. struct ceph_msg_header *hdr)
  698. {
  699. struct ceph_mon_client *monc = con->private;
  700. int type = le16_to_cpu(hdr->type);
  701. int front = le32_to_cpu(hdr->front_len);
  702. switch (type) {
  703. case CEPH_MSG_MON_SUBSCRIBE_ACK:
  704. return ceph_msgpool_get(&monc->msgpool_subscribe_ack, front);
  705. case CEPH_MSG_STATFS_REPLY:
  706. return ceph_msgpool_get(&monc->msgpool_statfs_reply, front);
  707. case CEPH_MSG_AUTH_REPLY:
  708. return ceph_msgpool_get(&monc->msgpool_auth_reply, front);
  709. }
  710. return ceph_alloc_msg(con, hdr);
  711. }
  712. /*
  713. * If the monitor connection resets, pick a new monitor and resubmit
  714. * any pending requests.
  715. */
  716. static void mon_fault(struct ceph_connection *con)
  717. {
  718. struct ceph_mon_client *monc = con->private;
  719. if (!monc)
  720. return;
  721. dout("mon_fault\n");
  722. mutex_lock(&monc->mutex);
  723. if (!con->private)
  724. goto out;
  725. if (monc->con && !monc->hunting)
  726. pr_info("mon%d %s session lost, "
  727. "hunting for new mon\n", monc->cur_mon,
  728. pr_addr(&monc->con->peer_addr.in_addr));
  729. __close_session(monc);
  730. if (!monc->hunting) {
  731. /* start hunting */
  732. monc->hunting = true;
  733. __open_session(monc);
  734. } else {
  735. /* already hunting, let's wait a bit */
  736. __schedule_delayed(monc);
  737. }
  738. out:
  739. mutex_unlock(&monc->mutex);
  740. }
  741. const static struct ceph_connection_operations mon_con_ops = {
  742. .get = ceph_con_get,
  743. .put = ceph_con_put,
  744. .dispatch = dispatch,
  745. .fault = mon_fault,
  746. .alloc_msg = mon_alloc_msg,
  747. .alloc_middle = ceph_alloc_middle,
  748. };