mon_client.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879
  1. #include "ceph_debug.h"
  2. #include <linux/types.h>
  3. #include <linux/slab.h>
  4. #include <linux/random.h>
  5. #include <linux/sched.h>
  6. #include "mon_client.h"
  7. #include "super.h"
  8. #include "auth.h"
  9. #include "decode.h"
  10. /*
  11. * Interact with Ceph monitor cluster. Handle requests for new map
  12. * versions, and periodically resend as needed. Also implement
  13. * statfs() and umount().
  14. *
  15. * A small cluster of Ceph "monitors" are responsible for managing critical
  16. * cluster configuration and state information. An odd number (e.g., 3, 5)
  17. * of cmon daemons use a modified version of the Paxos part-time parliament
  18. * algorithm to manage the MDS map (mds cluster membership), OSD map, and
  19. * list of clients who have mounted the file system.
  20. *
  21. * We maintain an open, active session with a monitor at all times in order to
  22. * receive timely MDSMap updates. We periodically send a keepalive byte on the
  23. * TCP socket to ensure we detect a failure. If the connection does break, we
  24. * randomly hunt for a new monitor. Once the connection is reestablished, we
  25. * resend any outstanding requests.
  26. */
  27. static const struct ceph_connection_operations mon_con_ops;
  28. static int __validate_auth(struct ceph_mon_client *monc);
  29. /*
  30. * Decode a monmap blob (e.g., during mount).
  31. */
  32. struct ceph_monmap *ceph_monmap_decode(void *p, void *end)
  33. {
  34. struct ceph_monmap *m = NULL;
  35. int i, err = -EINVAL;
  36. struct ceph_fsid fsid;
  37. u32 epoch, num_mon;
  38. u16 version;
  39. u32 len;
  40. ceph_decode_32_safe(&p, end, len, bad);
  41. ceph_decode_need(&p, end, len, bad);
  42. dout("monmap_decode %p %p len %d\n", p, end, (int)(end-p));
  43. ceph_decode_16_safe(&p, end, version, bad);
  44. ceph_decode_need(&p, end, sizeof(fsid) + 2*sizeof(u32), bad);
  45. ceph_decode_copy(&p, &fsid, sizeof(fsid));
  46. epoch = ceph_decode_32(&p);
  47. num_mon = ceph_decode_32(&p);
  48. ceph_decode_need(&p, end, num_mon*sizeof(m->mon_inst[0]), bad);
  49. if (num_mon >= CEPH_MAX_MON)
  50. goto bad;
  51. m = kmalloc(sizeof(*m) + sizeof(m->mon_inst[0])*num_mon, GFP_NOFS);
  52. if (m == NULL)
  53. return ERR_PTR(-ENOMEM);
  54. m->fsid = fsid;
  55. m->epoch = epoch;
  56. m->num_mon = num_mon;
  57. ceph_decode_copy(&p, m->mon_inst, num_mon*sizeof(m->mon_inst[0]));
  58. for (i = 0; i < num_mon; i++)
  59. ceph_decode_addr(&m->mon_inst[i].addr);
  60. dout("monmap_decode epoch %d, num_mon %d\n", m->epoch,
  61. m->num_mon);
  62. for (i = 0; i < m->num_mon; i++)
  63. dout("monmap_decode mon%d is %s\n", i,
  64. pr_addr(&m->mon_inst[i].addr.in_addr));
  65. return m;
  66. bad:
  67. dout("monmap_decode failed with %d\n", err);
  68. kfree(m);
  69. return ERR_PTR(err);
  70. }
  71. /*
  72. * return true if *addr is included in the monmap.
  73. */
  74. int ceph_monmap_contains(struct ceph_monmap *m, struct ceph_entity_addr *addr)
  75. {
  76. int i;
  77. for (i = 0; i < m->num_mon; i++)
  78. if (memcmp(addr, &m->mon_inst[i].addr, sizeof(*addr)) == 0)
  79. return 1;
  80. return 0;
  81. }
  82. /*
  83. * Send an auth request.
  84. */
  85. static void __send_prepared_auth_request(struct ceph_mon_client *monc, int len)
  86. {
  87. monc->pending_auth = 1;
  88. monc->m_auth->front.iov_len = len;
  89. monc->m_auth->hdr.front_len = cpu_to_le32(len);
  90. ceph_con_revoke(monc->con, monc->m_auth);
  91. ceph_msg_get(monc->m_auth); /* keep our ref */
  92. ceph_con_send(monc->con, monc->m_auth);
  93. }
  94. /*
  95. * Close monitor session, if any.
  96. */
  97. static void __close_session(struct ceph_mon_client *monc)
  98. {
  99. if (monc->con) {
  100. dout("__close_session closing mon%d\n", monc->cur_mon);
  101. ceph_con_revoke(monc->con, monc->m_auth);
  102. ceph_con_close(monc->con);
  103. monc->cur_mon = -1;
  104. monc->pending_auth = 0;
  105. ceph_auth_reset(monc->auth);
  106. }
  107. }
  108. /*
  109. * Open a session with a (new) monitor.
  110. */
  111. static int __open_session(struct ceph_mon_client *monc)
  112. {
  113. char r;
  114. int ret;
  115. if (monc->cur_mon < 0) {
  116. get_random_bytes(&r, 1);
  117. monc->cur_mon = r % monc->monmap->num_mon;
  118. dout("open_session num=%d r=%d -> mon%d\n",
  119. monc->monmap->num_mon, r, monc->cur_mon);
  120. monc->sub_sent = 0;
  121. monc->sub_renew_after = jiffies; /* i.e., expired */
  122. monc->want_next_osdmap = !!monc->want_next_osdmap;
  123. dout("open_session mon%d opening\n", monc->cur_mon);
  124. monc->con->peer_name.type = CEPH_ENTITY_TYPE_MON;
  125. monc->con->peer_name.num = cpu_to_le64(monc->cur_mon);
  126. ceph_con_open(monc->con,
  127. &monc->monmap->mon_inst[monc->cur_mon].addr);
  128. /* initiatiate authentication handshake */
  129. ret = ceph_auth_build_hello(monc->auth,
  130. monc->m_auth->front.iov_base,
  131. monc->m_auth->front_max);
  132. __send_prepared_auth_request(monc, ret);
  133. } else {
  134. dout("open_session mon%d already open\n", monc->cur_mon);
  135. }
  136. return 0;
  137. }
  138. static bool __sub_expired(struct ceph_mon_client *monc)
  139. {
  140. return time_after_eq(jiffies, monc->sub_renew_after);
  141. }
  142. /*
  143. * Reschedule delayed work timer.
  144. */
  145. static void __schedule_delayed(struct ceph_mon_client *monc)
  146. {
  147. unsigned delay;
  148. if (monc->cur_mon < 0 || __sub_expired(monc))
  149. delay = 10 * HZ;
  150. else
  151. delay = 20 * HZ;
  152. dout("__schedule_delayed after %u\n", delay);
  153. schedule_delayed_work(&monc->delayed_work, delay);
  154. }
  155. /*
  156. * Send subscribe request for mdsmap and/or osdmap.
  157. */
  158. static void __send_subscribe(struct ceph_mon_client *monc)
  159. {
  160. dout("__send_subscribe sub_sent=%u exp=%u want_osd=%d\n",
  161. (unsigned)monc->sub_sent, __sub_expired(monc),
  162. monc->want_next_osdmap);
  163. if ((__sub_expired(monc) && !monc->sub_sent) ||
  164. monc->want_next_osdmap == 1) {
  165. struct ceph_msg *msg = monc->m_subscribe;
  166. struct ceph_mon_subscribe_item *i;
  167. void *p, *end;
  168. p = msg->front.iov_base;
  169. end = p + msg->front_max;
  170. dout("__send_subscribe to 'mdsmap' %u+\n",
  171. (unsigned)monc->have_mdsmap);
  172. if (monc->want_next_osdmap) {
  173. dout("__send_subscribe to 'osdmap' %u\n",
  174. (unsigned)monc->have_osdmap);
  175. ceph_encode_32(&p, 3);
  176. ceph_encode_string(&p, end, "osdmap", 6);
  177. i = p;
  178. i->have = cpu_to_le64(monc->have_osdmap);
  179. i->onetime = 1;
  180. p += sizeof(*i);
  181. monc->want_next_osdmap = 2; /* requested */
  182. } else {
  183. ceph_encode_32(&p, 2);
  184. }
  185. ceph_encode_string(&p, end, "mdsmap", 6);
  186. i = p;
  187. i->have = cpu_to_le64(monc->have_mdsmap);
  188. i->onetime = 0;
  189. p += sizeof(*i);
  190. ceph_encode_string(&p, end, "monmap", 6);
  191. i = p;
  192. i->have = 0;
  193. i->onetime = 0;
  194. p += sizeof(*i);
  195. msg->front.iov_len = p - msg->front.iov_base;
  196. msg->hdr.front_len = cpu_to_le32(msg->front.iov_len);
  197. ceph_con_revoke(monc->con, msg);
  198. ceph_con_send(monc->con, ceph_msg_get(msg));
  199. monc->sub_sent = jiffies | 1; /* never 0 */
  200. }
  201. }
  202. static void handle_subscribe_ack(struct ceph_mon_client *monc,
  203. struct ceph_msg *msg)
  204. {
  205. unsigned seconds;
  206. struct ceph_mon_subscribe_ack *h = msg->front.iov_base;
  207. if (msg->front.iov_len < sizeof(*h))
  208. goto bad;
  209. seconds = le32_to_cpu(h->duration);
  210. mutex_lock(&monc->mutex);
  211. if (monc->hunting) {
  212. pr_info("mon%d %s session established\n",
  213. monc->cur_mon, pr_addr(&monc->con->peer_addr.in_addr));
  214. monc->hunting = false;
  215. }
  216. dout("handle_subscribe_ack after %d seconds\n", seconds);
  217. monc->sub_renew_after = monc->sub_sent + (seconds >> 1)*HZ - 1;
  218. monc->sub_sent = 0;
  219. mutex_unlock(&monc->mutex);
  220. return;
  221. bad:
  222. pr_err("got corrupt subscribe-ack msg\n");
  223. ceph_msg_dump(msg);
  224. }
  225. /*
  226. * Keep track of which maps we have
  227. */
  228. int ceph_monc_got_mdsmap(struct ceph_mon_client *monc, u32 got)
  229. {
  230. mutex_lock(&monc->mutex);
  231. monc->have_mdsmap = got;
  232. mutex_unlock(&monc->mutex);
  233. return 0;
  234. }
  235. int ceph_monc_got_osdmap(struct ceph_mon_client *monc, u32 got)
  236. {
  237. mutex_lock(&monc->mutex);
  238. monc->have_osdmap = got;
  239. monc->want_next_osdmap = 0;
  240. mutex_unlock(&monc->mutex);
  241. return 0;
  242. }
  243. /*
  244. * Register interest in the next osdmap
  245. */
  246. void ceph_monc_request_next_osdmap(struct ceph_mon_client *monc)
  247. {
  248. dout("request_next_osdmap have %u\n", monc->have_osdmap);
  249. mutex_lock(&monc->mutex);
  250. if (!monc->want_next_osdmap)
  251. monc->want_next_osdmap = 1;
  252. if (monc->want_next_osdmap < 2)
  253. __send_subscribe(monc);
  254. mutex_unlock(&monc->mutex);
  255. }
  256. /*
  257. *
  258. */
  259. int ceph_monc_open_session(struct ceph_mon_client *monc)
  260. {
  261. if (!monc->con) {
  262. monc->con = kmalloc(sizeof(*monc->con), GFP_KERNEL);
  263. if (!monc->con)
  264. return -ENOMEM;
  265. ceph_con_init(monc->client->msgr, monc->con);
  266. monc->con->private = monc;
  267. monc->con->ops = &mon_con_ops;
  268. }
  269. mutex_lock(&monc->mutex);
  270. __open_session(monc);
  271. __schedule_delayed(monc);
  272. mutex_unlock(&monc->mutex);
  273. return 0;
  274. }
  275. /*
  276. * The monitor responds with mount ack indicate mount success. The
  277. * included client ticket allows the client to talk to MDSs and OSDs.
  278. */
  279. static void ceph_monc_handle_map(struct ceph_mon_client *monc,
  280. struct ceph_msg *msg)
  281. {
  282. struct ceph_client *client = monc->client;
  283. struct ceph_monmap *monmap = NULL, *old = monc->monmap;
  284. void *p, *end;
  285. mutex_lock(&monc->mutex);
  286. dout("handle_monmap\n");
  287. p = msg->front.iov_base;
  288. end = p + msg->front.iov_len;
  289. monmap = ceph_monmap_decode(p, end);
  290. if (IS_ERR(monmap)) {
  291. pr_err("problem decoding monmap, %d\n",
  292. (int)PTR_ERR(monmap));
  293. goto out;
  294. }
  295. if (ceph_check_fsid(monc->client, &monmap->fsid) < 0) {
  296. kfree(monmap);
  297. goto out;
  298. }
  299. client->monc.monmap = monmap;
  300. kfree(old);
  301. out:
  302. mutex_unlock(&monc->mutex);
  303. wake_up(&client->auth_wq);
  304. }
  305. /*
  306. * statfs
  307. */
  308. static struct ceph_mon_generic_request *__lookup_generic_req(
  309. struct ceph_mon_client *monc, u64 tid)
  310. {
  311. struct ceph_mon_generic_request *req;
  312. struct rb_node *n = monc->generic_request_tree.rb_node;
  313. while (n) {
  314. req = rb_entry(n, struct ceph_mon_generic_request, node);
  315. if (tid < req->tid)
  316. n = n->rb_left;
  317. else if (tid > req->tid)
  318. n = n->rb_right;
  319. else
  320. return req;
  321. }
  322. return NULL;
  323. }
  324. static void __insert_generic_request(struct ceph_mon_client *monc,
  325. struct ceph_mon_generic_request *new)
  326. {
  327. struct rb_node **p = &monc->generic_request_tree.rb_node;
  328. struct rb_node *parent = NULL;
  329. struct ceph_mon_generic_request *req = NULL;
  330. while (*p) {
  331. parent = *p;
  332. req = rb_entry(parent, struct ceph_mon_generic_request, node);
  333. if (new->tid < req->tid)
  334. p = &(*p)->rb_left;
  335. else if (new->tid > req->tid)
  336. p = &(*p)->rb_right;
  337. else
  338. BUG();
  339. }
  340. rb_link_node(&new->node, parent, p);
  341. rb_insert_color(&new->node, &monc->generic_request_tree);
  342. }
  343. static void release_generic_request(struct kref *kref)
  344. {
  345. struct ceph_mon_generic_request *req =
  346. container_of(kref, struct ceph_mon_generic_request, kref);
  347. if (req->reply)
  348. ceph_msg_put(req->reply);
  349. if (req->request)
  350. ceph_msg_put(req->request);
  351. }
  352. static void put_generic_request(struct ceph_mon_generic_request *req)
  353. {
  354. kref_put(&req->kref, release_generic_request);
  355. }
  356. static void get_generic_request(struct ceph_mon_generic_request *req)
  357. {
  358. kref_get(&req->kref);
  359. }
  360. static struct ceph_msg *get_generic_reply(struct ceph_connection *con,
  361. struct ceph_msg_header *hdr,
  362. int *skip)
  363. {
  364. struct ceph_mon_client *monc = con->private;
  365. struct ceph_mon_generic_request *req;
  366. u64 tid = le64_to_cpu(hdr->tid);
  367. struct ceph_msg *m;
  368. mutex_lock(&monc->mutex);
  369. req = __lookup_generic_req(monc, tid);
  370. if (!req) {
  371. dout("get_generic_reply %lld dne\n", tid);
  372. *skip = 1;
  373. m = NULL;
  374. } else {
  375. dout("get_generic_reply %lld got %p\n", tid, req->reply);
  376. m = ceph_msg_get(req->reply);
  377. /*
  378. * we don't need to track the connection reading into
  379. * this reply because we only have one open connection
  380. * at a time, ever.
  381. */
  382. }
  383. mutex_unlock(&monc->mutex);
  384. return m;
  385. }
  386. static void handle_statfs_reply(struct ceph_mon_client *monc,
  387. struct ceph_msg *msg)
  388. {
  389. struct ceph_mon_generic_request *req;
  390. struct ceph_mon_statfs_reply *reply = msg->front.iov_base;
  391. u64 tid = le64_to_cpu(msg->hdr.tid);
  392. if (msg->front.iov_len != sizeof(*reply))
  393. goto bad;
  394. dout("handle_statfs_reply %p tid %llu\n", msg, tid);
  395. mutex_lock(&monc->mutex);
  396. req = __lookup_generic_req(monc, tid);
  397. if (req) {
  398. *(struct ceph_statfs *)req->buf = reply->st;
  399. req->result = 0;
  400. get_generic_request(req);
  401. }
  402. mutex_unlock(&monc->mutex);
  403. if (req) {
  404. complete(&req->completion);
  405. put_generic_request(req);
  406. }
  407. return;
  408. bad:
  409. pr_err("corrupt generic reply, no tid\n");
  410. ceph_msg_dump(msg);
  411. }
  412. /*
  413. * Do a synchronous statfs().
  414. */
  415. int ceph_monc_do_statfs(struct ceph_mon_client *monc, struct ceph_statfs *buf)
  416. {
  417. struct ceph_mon_generic_request *req;
  418. struct ceph_mon_statfs *h;
  419. int err;
  420. req = kzalloc(sizeof(*req), GFP_NOFS);
  421. if (!req)
  422. return -ENOMEM;
  423. kref_init(&req->kref);
  424. req->buf = buf;
  425. init_completion(&req->completion);
  426. err = -ENOMEM;
  427. req->request = ceph_msg_new(CEPH_MSG_STATFS, sizeof(*h), GFP_NOFS);
  428. if (!req->request)
  429. goto out;
  430. req->reply = ceph_msg_new(CEPH_MSG_STATFS_REPLY, 1024, GFP_NOFS);
  431. if (!req->reply)
  432. goto out;
  433. /* fill out request */
  434. h = req->request->front.iov_base;
  435. h->monhdr.have_version = 0;
  436. h->monhdr.session_mon = cpu_to_le16(-1);
  437. h->monhdr.session_mon_tid = 0;
  438. h->fsid = monc->monmap->fsid;
  439. /* register request */
  440. mutex_lock(&monc->mutex);
  441. req->tid = ++monc->last_tid;
  442. req->request->hdr.tid = cpu_to_le64(req->tid);
  443. __insert_generic_request(monc, req);
  444. monc->num_generic_requests++;
  445. mutex_unlock(&monc->mutex);
  446. /* send request and wait */
  447. ceph_con_send(monc->con, ceph_msg_get(req->request));
  448. err = wait_for_completion_interruptible(&req->completion);
  449. mutex_lock(&monc->mutex);
  450. rb_erase(&req->node, &monc->generic_request_tree);
  451. monc->num_generic_requests--;
  452. mutex_unlock(&monc->mutex);
  453. if (!err)
  454. err = req->result;
  455. out:
  456. kref_put(&req->kref, release_generic_request);
  457. return err;
  458. }
  459. /*
  460. * Resend pending statfs requests.
  461. */
  462. static void __resend_generic_request(struct ceph_mon_client *monc)
  463. {
  464. struct ceph_mon_generic_request *req;
  465. struct rb_node *p;
  466. for (p = rb_first(&monc->generic_request_tree); p; p = rb_next(p)) {
  467. req = rb_entry(p, struct ceph_mon_generic_request, node);
  468. ceph_con_revoke(monc->con, req->request);
  469. ceph_con_send(monc->con, ceph_msg_get(req->request));
  470. }
  471. }
  472. /*
  473. * Delayed work. If we haven't mounted yet, retry. Otherwise,
  474. * renew/retry subscription as needed (in case it is timing out, or we
  475. * got an ENOMEM). And keep the monitor connection alive.
  476. */
  477. static void delayed_work(struct work_struct *work)
  478. {
  479. struct ceph_mon_client *monc =
  480. container_of(work, struct ceph_mon_client, delayed_work.work);
  481. dout("monc delayed_work\n");
  482. mutex_lock(&monc->mutex);
  483. if (monc->hunting) {
  484. __close_session(monc);
  485. __open_session(monc); /* continue hunting */
  486. } else {
  487. ceph_con_keepalive(monc->con);
  488. __validate_auth(monc);
  489. if (monc->auth->ops->is_authenticated(monc->auth))
  490. __send_subscribe(monc);
  491. }
  492. __schedule_delayed(monc);
  493. mutex_unlock(&monc->mutex);
  494. }
  495. /*
  496. * On startup, we build a temporary monmap populated with the IPs
  497. * provided by mount(2).
  498. */
  499. static int build_initial_monmap(struct ceph_mon_client *monc)
  500. {
  501. struct ceph_mount_args *args = monc->client->mount_args;
  502. struct ceph_entity_addr *mon_addr = args->mon_addr;
  503. int num_mon = args->num_mon;
  504. int i;
  505. /* build initial monmap */
  506. monc->monmap = kzalloc(sizeof(*monc->monmap) +
  507. num_mon*sizeof(monc->monmap->mon_inst[0]),
  508. GFP_KERNEL);
  509. if (!monc->monmap)
  510. return -ENOMEM;
  511. for (i = 0; i < num_mon; i++) {
  512. monc->monmap->mon_inst[i].addr = mon_addr[i];
  513. monc->monmap->mon_inst[i].addr.nonce = 0;
  514. monc->monmap->mon_inst[i].name.type =
  515. CEPH_ENTITY_TYPE_MON;
  516. monc->monmap->mon_inst[i].name.num = cpu_to_le64(i);
  517. }
  518. monc->monmap->num_mon = num_mon;
  519. monc->have_fsid = false;
  520. /* release addr memory */
  521. kfree(args->mon_addr);
  522. args->mon_addr = NULL;
  523. args->num_mon = 0;
  524. return 0;
  525. }
  526. int ceph_monc_init(struct ceph_mon_client *monc, struct ceph_client *cl)
  527. {
  528. int err = 0;
  529. dout("init\n");
  530. memset(monc, 0, sizeof(*monc));
  531. monc->client = cl;
  532. monc->monmap = NULL;
  533. mutex_init(&monc->mutex);
  534. err = build_initial_monmap(monc);
  535. if (err)
  536. goto out;
  537. monc->con = NULL;
  538. /* authentication */
  539. monc->auth = ceph_auth_init(cl->mount_args->name,
  540. cl->mount_args->secret);
  541. if (IS_ERR(monc->auth))
  542. return PTR_ERR(monc->auth);
  543. monc->auth->want_keys =
  544. CEPH_ENTITY_TYPE_AUTH | CEPH_ENTITY_TYPE_MON |
  545. CEPH_ENTITY_TYPE_OSD | CEPH_ENTITY_TYPE_MDS;
  546. /* msgs */
  547. err = -ENOMEM;
  548. monc->m_subscribe_ack = ceph_msg_new(CEPH_MSG_MON_SUBSCRIBE_ACK,
  549. sizeof(struct ceph_mon_subscribe_ack),
  550. GFP_NOFS);
  551. if (!monc->m_subscribe_ack)
  552. goto out_monmap;
  553. monc->m_subscribe = ceph_msg_new(CEPH_MSG_MON_SUBSCRIBE, 96, GFP_NOFS);
  554. if (!monc->m_subscribe)
  555. goto out_subscribe_ack;
  556. monc->m_auth_reply = ceph_msg_new(CEPH_MSG_AUTH_REPLY, 4096, GFP_NOFS);
  557. if (!monc->m_auth_reply)
  558. goto out_subscribe;
  559. monc->m_auth = ceph_msg_new(CEPH_MSG_AUTH, 4096, GFP_NOFS);
  560. monc->pending_auth = 0;
  561. if (!monc->m_auth)
  562. goto out_auth_reply;
  563. monc->cur_mon = -1;
  564. monc->hunting = true;
  565. monc->sub_renew_after = jiffies;
  566. monc->sub_sent = 0;
  567. INIT_DELAYED_WORK(&monc->delayed_work, delayed_work);
  568. monc->generic_request_tree = RB_ROOT;
  569. monc->num_generic_requests = 0;
  570. monc->last_tid = 0;
  571. monc->have_mdsmap = 0;
  572. monc->have_osdmap = 0;
  573. monc->want_next_osdmap = 1;
  574. return 0;
  575. out_auth_reply:
  576. ceph_msg_put(monc->m_auth_reply);
  577. out_subscribe:
  578. ceph_msg_put(monc->m_subscribe);
  579. out_subscribe_ack:
  580. ceph_msg_put(monc->m_subscribe_ack);
  581. out_monmap:
  582. kfree(monc->monmap);
  583. out:
  584. return err;
  585. }
  586. void ceph_monc_stop(struct ceph_mon_client *monc)
  587. {
  588. dout("stop\n");
  589. cancel_delayed_work_sync(&monc->delayed_work);
  590. mutex_lock(&monc->mutex);
  591. __close_session(monc);
  592. if (monc->con) {
  593. monc->con->private = NULL;
  594. monc->con->ops->put(monc->con);
  595. monc->con = NULL;
  596. }
  597. mutex_unlock(&monc->mutex);
  598. ceph_auth_destroy(monc->auth);
  599. ceph_msg_put(monc->m_auth);
  600. ceph_msg_put(monc->m_auth_reply);
  601. ceph_msg_put(monc->m_subscribe);
  602. ceph_msg_put(monc->m_subscribe_ack);
  603. kfree(monc->monmap);
  604. }
  605. static void handle_auth_reply(struct ceph_mon_client *monc,
  606. struct ceph_msg *msg)
  607. {
  608. int ret;
  609. int was_auth = 0;
  610. mutex_lock(&monc->mutex);
  611. if (monc->auth->ops)
  612. was_auth = monc->auth->ops->is_authenticated(monc->auth);
  613. monc->pending_auth = 0;
  614. ret = ceph_handle_auth_reply(monc->auth, msg->front.iov_base,
  615. msg->front.iov_len,
  616. monc->m_auth->front.iov_base,
  617. monc->m_auth->front_max);
  618. if (ret < 0) {
  619. monc->client->auth_err = ret;
  620. wake_up(&monc->client->auth_wq);
  621. } else if (ret > 0) {
  622. __send_prepared_auth_request(monc, ret);
  623. } else if (!was_auth && monc->auth->ops->is_authenticated(monc->auth)) {
  624. dout("authenticated, starting session\n");
  625. monc->client->msgr->inst.name.type = CEPH_ENTITY_TYPE_CLIENT;
  626. monc->client->msgr->inst.name.num = monc->auth->global_id;
  627. __send_subscribe(monc);
  628. __resend_generic_request(monc);
  629. }
  630. mutex_unlock(&monc->mutex);
  631. }
  632. static int __validate_auth(struct ceph_mon_client *monc)
  633. {
  634. int ret;
  635. if (monc->pending_auth)
  636. return 0;
  637. ret = ceph_build_auth(monc->auth, monc->m_auth->front.iov_base,
  638. monc->m_auth->front_max);
  639. if (ret <= 0)
  640. return ret; /* either an error, or no need to authenticate */
  641. __send_prepared_auth_request(monc, ret);
  642. return 0;
  643. }
  644. int ceph_monc_validate_auth(struct ceph_mon_client *monc)
  645. {
  646. int ret;
  647. mutex_lock(&monc->mutex);
  648. ret = __validate_auth(monc);
  649. mutex_unlock(&monc->mutex);
  650. return ret;
  651. }
  652. /*
  653. * handle incoming message
  654. */
  655. static void dispatch(struct ceph_connection *con, struct ceph_msg *msg)
  656. {
  657. struct ceph_mon_client *monc = con->private;
  658. int type = le16_to_cpu(msg->hdr.type);
  659. if (!monc)
  660. return;
  661. switch (type) {
  662. case CEPH_MSG_AUTH_REPLY:
  663. handle_auth_reply(monc, msg);
  664. break;
  665. case CEPH_MSG_MON_SUBSCRIBE_ACK:
  666. handle_subscribe_ack(monc, msg);
  667. break;
  668. case CEPH_MSG_STATFS_REPLY:
  669. handle_statfs_reply(monc, msg);
  670. break;
  671. case CEPH_MSG_MON_MAP:
  672. ceph_monc_handle_map(monc, msg);
  673. break;
  674. case CEPH_MSG_MDS_MAP:
  675. ceph_mdsc_handle_map(&monc->client->mdsc, msg);
  676. break;
  677. case CEPH_MSG_OSD_MAP:
  678. ceph_osdc_handle_map(&monc->client->osdc, msg);
  679. break;
  680. default:
  681. pr_err("received unknown message type %d %s\n", type,
  682. ceph_msg_type_name(type));
  683. }
  684. ceph_msg_put(msg);
  685. }
  686. /*
  687. * Allocate memory for incoming message
  688. */
  689. static struct ceph_msg *mon_alloc_msg(struct ceph_connection *con,
  690. struct ceph_msg_header *hdr,
  691. int *skip)
  692. {
  693. struct ceph_mon_client *monc = con->private;
  694. int type = le16_to_cpu(hdr->type);
  695. int front_len = le32_to_cpu(hdr->front_len);
  696. struct ceph_msg *m = NULL;
  697. *skip = 0;
  698. switch (type) {
  699. case CEPH_MSG_MON_SUBSCRIBE_ACK:
  700. m = ceph_msg_get(monc->m_subscribe_ack);
  701. break;
  702. case CEPH_MSG_STATFS_REPLY:
  703. return get_generic_reply(con, hdr, skip);
  704. case CEPH_MSG_AUTH_REPLY:
  705. m = ceph_msg_get(monc->m_auth_reply);
  706. break;
  707. case CEPH_MSG_MON_MAP:
  708. case CEPH_MSG_MDS_MAP:
  709. case CEPH_MSG_OSD_MAP:
  710. m = ceph_msg_new(type, front_len, GFP_NOFS);
  711. break;
  712. }
  713. if (!m) {
  714. pr_info("alloc_msg unknown type %d\n", type);
  715. *skip = 1;
  716. }
  717. return m;
  718. }
  719. /*
  720. * If the monitor connection resets, pick a new monitor and resubmit
  721. * any pending requests.
  722. */
  723. static void mon_fault(struct ceph_connection *con)
  724. {
  725. struct ceph_mon_client *monc = con->private;
  726. if (!monc)
  727. return;
  728. dout("mon_fault\n");
  729. mutex_lock(&monc->mutex);
  730. if (!con->private)
  731. goto out;
  732. if (monc->con && !monc->hunting)
  733. pr_info("mon%d %s session lost, "
  734. "hunting for new mon\n", monc->cur_mon,
  735. pr_addr(&monc->con->peer_addr.in_addr));
  736. __close_session(monc);
  737. if (!monc->hunting) {
  738. /* start hunting */
  739. monc->hunting = true;
  740. __open_session(monc);
  741. } else {
  742. /* already hunting, let's wait a bit */
  743. __schedule_delayed(monc);
  744. }
  745. out:
  746. mutex_unlock(&monc->mutex);
  747. }
  748. static const struct ceph_connection_operations mon_con_ops = {
  749. .get = ceph_con_get,
  750. .put = ceph_con_put,
  751. .dispatch = dispatch,
  752. .fault = mon_fault,
  753. .alloc_msg = mon_alloc_msg,
  754. };