drbd: Fix completion of requests while the device is suspended

In various places (E.g. CONNECTION_LOST_WHILE_PENDING) the
RQ_COMPLETION_SUSP mask is passed in the clear set to mod_rq_state().

The issue was that it tried to clear the RQ_COMPLETION_SUSP bit
out of the state mask first, and eventuelly set it afterwards,
in the drbd_req_put_completion_ref() function.

Fixed that by moving the reference getting out of
drbd_req_put_completion_ref() into the mod_rq_state(), before the place
where the extra reference might be put.

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>

drivers/block/drbd/drbd_req.c

@@ -304,18 +304,6 @@ static int drbd_req_put_completion_ref(struct drbd_request *req, struct bio_and_
 	if (!atomic_sub_and_test(put, &req->completion_ref))
 		return 0;
 
-	if (drbd_suspended(mdev)) {
-		/* We do not allow completion while suspended.  Re-get a
-		 * reference, so whatever happens when this is resumed
-		 * may put and complete. */
-
-		D_ASSERT(!(req->rq_state & RQ_COMPLETION_SUSP));
-		req->rq_state |= RQ_COMPLETION_SUSP;
-		atomic_inc(&req->completion_ref);
-		return 0;
-	}
-
-	/* else */
 	drbd_req_complete(req, m);
 
 	if (req->rq_state & RQ_POSTPONED) {
@@ -338,6 +326,9 @@ static void mod_rq_state(struct drbd_request *req, struct bio_and_error *m,
 	int c_put = 0;
 	int k_put = 0;
 
+	if (drbd_suspended(mdev) && !((s | clear) & RQ_COMPLETION_SUSP))
+		set |= RQ_COMPLETION_SUSP;
+
 	/* apply */
 
 	req->rq_state &= ~clear;
@@ -366,6 +357,9 @@ static void mod_rq_state(struct drbd_request *req, struct bio_and_error *m,
 	if (!(s & RQ_NET_SENT) && (set & RQ_NET_SENT))
 		atomic_add(req->i.size >> 9, &mdev->ap_in_flight);
 
+	if (!(s & RQ_COMPLETION_SUSP) && (set & RQ_COMPLETION_SUSP))
+		atomic_inc(&req->completion_ref);
+
 	/* progress: put references */
 
 	if ((s & RQ_COMPLETION_SUSP) && (clear & RQ_COMPLETION_SUSP))