File srp_daemon-fix-CQ-handling.patch of Package rdma-core

Overview Repositories Revisions Requests Users Attributes Meta

File srp_daemon-fix-CQ-handling.patch of Package rdma-core (Revision 148f00c20227bf65acc77b4db09b07be)

Currently displaying revision 148f00c20227bf65acc77b4db09b07be , Show latest

commit c1c584c34d249987d7c36ff061bc5f2eedec38fe
Author: Nicolas Morey-Chaisemartin <NMoreyChaisemartin@suse.com>
Date:   Mon Dec 11 15:37:28 2017 +0100

srp_daemon: fix CQ handling
    
    SM traps are polled through poll_cq which waited for a CQ event
    before polling the CQ itself.
    However it may happens that multiple completions are attached
    to a single event. As stated by the ibv_get_cq_event man page,
    it is required to poll the the CQ to get those completions
    after the call to ibv_req_notify_cq.
    
    As completions need to be handled one by one in an outer function,
    start by polling the CQ and return the completion (if any) before
    waiting for the next completion event.
    This will allow emptying all pending completions, through multiple calls
    to poll_cq,  before waiting for a new event.
    
    The buggy use case seems to appear when the master SM is switched multiple
    times between two nodes. As the number of ping-pong between the SMs increases,
    the number of traps sent to notify that the SM just became master increases
    too. This causes burst of completions linked to a single event.
    Note that the race condition is also possible in other scenario.
    
    Signed-off-by: Nicolas Morey-Chaisemartin <NMoreyChaisemartin@suse.com>
    Cc: stable@linux-rdma.org # v14, v15, v16

diff --git srp_daemon/srp_handle_traps.c srp_daemon/srp_handle_traps.c
index 25f2b9ab..77a47db3 100644
--- srp_daemon/srp_handle_traps.c
+++ srp_daemon/srp_handle_traps.c
@@ -496,6 +496,34 @@ static int stop_threads(struct sync_resources *sync_res)
 	return result;
 }
 
+/*****************************************************************************
+* Function: poll_cq_once
+* Poll a CQ once.
+* Returns the number of completion polled (0 or 1).
+* Returns a negative value on error.
+*****************************************************************************/
+static int poll_cq_once(struct sync_resources *sync_res, struct ibv_cq *cq,
+			struct ibv_wc *wc)
+{
+	int ret;
+
+	ret = ibv_poll_cq(cq, 1, wc);
+	if (ret < 0) {
+		pr_err("poll CQ failed\n");
+		return ret;
+	}
+
+	if (ret > 0 && wc->status != IBV_WC_SUCCESS) {
+		if (!stop_threads(sync_res))
+			pr_err("got bad completion with status: 0x%x\n",
+			       wc->status);
+		return -ret;
+	}
+
+	return ret;
+}
+
+
 static int poll_cq(struct sync_resources *sync_res, struct ibv_cq *cq,
 		   struct ibv_wc *wc, struct ibv_comp_channel *channel)
 {
@@ -504,6 +532,16 @@ static int poll_cq(struct sync_resources *sync_res, struct ibv_cq *cq,
 	void          *ev_ctx;
 
 	if (channel) {
+		/* There may be extra completions that
+		 * were associated to the previous event.
+		 * Only poll for the first one. If there are more than one,
+		 * they will be handled by later call to poll_cq */
+		ret = poll_cq_once(sync_res, cq, wc);
+		/* return directly if there was an error or
+		 * 1 completion polled */
+		if (ret)
+			return ret;
+
 		if (ibv_get_cq_event(channel, &ev_cq, &ev_ctx)) {
 			pr_err("Failed to get cq_event\n");
 			return -1;
@@ -524,18 +562,9 @@ static int poll_cq(struct sync_resources *sync_res, struct ibv_cq *cq,
 	}
 
 	do {
-		ret = ibv_poll_cq(cq, 1, wc);
-		if (ret < 0) {
-			pr_err("poll CQ failed\n");
+		ret = poll_cq_once(sync_res, cq, wc);
+		if (ret < 0)
 			return ret;
-		}
-
-		if (ret > 0 && wc->status != IBV_WC_SUCCESS) {
-			if (!stop_threads(sync_res))
-				pr_err("got bad completion with status: 0x%x\n",
-				       wc->status);
-			return -ret;
-		}
 
 		if (ret == 0 && channel) {
 			pr_err("Weird poll returned no cqe after CQ event\n");

Places

File srp_daemon-fix-CQ-handling.patch of Package rdma-core (Revision 148f00c20227bf65acc77b4db09b07be)

Places