libceph: set response data fields earlier

When an incoming message is destined for the osd client, the
messenger calls the osd client's alloc_msg method.  That function
looks up which request has the tid matching the incoming message,
and returns the request message that was preallocated to receive the
response.  The response message is therefore known before the
request is even started.

Between the start of the request and the receipt of the response,
the request and its data fields will not change, so there's no
reason we need to hold off setting them.  In fact it's preferable
to set them just once because it's more obvious that they're
unchanging.

So set up the fields describing where incoming data is to land in a
response message at the beginning of ceph_osdc_start_request().
Define a helper function that sets these fields, and use it to
set the fields for both outgoing data in the request message and
incoming data in the response.

This resolves:
    http://tracker.ceph.com/issues/4284

Signed-off-by: Alex Elder <elder@inktank.com>
Reviewed-by: Josh Durgin <josh.durgin@inktank.com>

net/ceph/osd_client.c

@@ -1744,32 +1744,36 @@ bad:
 	return;
 }
 
-/*
- * Register request, send initial attempt.
- */
-int ceph_osdc_start_request(struct ceph_osd_client *osdc,
-			    struct ceph_osd_request *req,
-			    bool nofail)
+static void ceph_osdc_msg_data_set(struct ceph_msg *msg,
+				struct ceph_osd_data *osd_data)
 {
-	int rc = 0;
-	struct ceph_osd_data *osd_data;
-
-	/* Set up outgoing data */
-
-	osd_data = &req->r_data_out;
 	if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGES) {
 		BUG_ON(osd_data->length > (u64) SIZE_MAX);
 		if (osd_data->length)
-			ceph_msg_data_set_pages(req->r_request,
-				osd_data->pages, osd_data->length,
-				osd_data->alignment);
+			ceph_msg_data_set_pages(msg, osd_data->pages,
+				osd_data->length, osd_data->alignment);
 #ifdef CONFIG_BLOCK
 	} else if (osd_data->type == CEPH_OSD_DATA_TYPE_BIO) {
-		ceph_msg_data_set_bio(req->r_request, osd_data->bio);
+		ceph_msg_data_set_bio(msg, osd_data->bio);
 #endif
 	} else {
 		BUG_ON(osd_data->type != CEPH_OSD_DATA_TYPE_NONE);
 	}
+}
+
+/*
+ * Register request, send initial attempt.
+ */
+int ceph_osdc_start_request(struct ceph_osd_client *osdc,
+			    struct ceph_osd_request *req,
+			    bool nofail)
+{
+	int rc = 0;
+
+	/* Set up response incoming data and request outgoing data fields */
+
+	ceph_osdc_msg_data_set(req->r_reply, &req->r_data_in);
+	ceph_osdc_msg_data_set(req->r_request, &req->r_data_out);
 	if (req->r_trail.length)
 		ceph_msg_data_set_trail(req->r_request, &req->r_trail);
 
@@ -2130,13 +2134,6 @@ static struct ceph_msg *get_reply(struct ceph_connection *con,
 				m = NULL;
 				goto out;
 			}
-			BUG_ON(osd_data->length > (u64) SIZE_MAX);
-			ceph_msg_data_set_pages(m, osd_data->pages,
-				osd_data->length, osd_data->alignment);
-#ifdef CONFIG_BLOCK
-		} else if (osd_data->type == CEPH_OSD_DATA_TYPE_BIO) {
-			ceph_msg_data_set_bio(m, osd_data->bio);
-#endif
 		}
 	}
 	*skip = 0;