C++ rdma_destroy_qp函数代码示例

/*
 * rpcrdma_ep_destroy
 *
 * Disconnect and destroy endpoint. After this, the only
 * valid operations on the ep are to free it (if dynamically
 * allocated) or re-create it.
 *
 * The caller's error handling must be sure to not leak the endpoint
 * if this function fails.
 */
int
rpcrdma_ep_destroy(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)
{
	int rc;

	dprintk("RPC:       %s: entering, connected is %d\n",
		__func__, ep->rep_connected);

	if (ia->ri_id->qp) {
		rc = rpcrdma_ep_disconnect(ep, ia);
		if (rc)
			dprintk("RPC:       %s: rpcrdma_ep_disconnect"
				" returned %i\n", __func__, rc);
		rdma_destroy_qp(ia->ri_id);
		ia->ri_id->qp = NULL;
	}

	/* padding - could be done in rpcrdma_buffer_destroy... */
	if (ep->rep_pad_mr) {
		rpcrdma_deregister_internal(ia, ep->rep_pad_mr, &ep->rep_pad);
		ep->rep_pad_mr = NULL;
	}

	rpcrdma_clean_cq(ep->rep_cq);
	rc = ib_destroy_cq(ep->rep_cq);
	if (rc)
		dprintk("RPC:       %s: ib_destroy_cq returned %i\n",
			__func__, rc);

	return rc;
}

/**
 * releases the FMR pool, QP and CMA ID objects, returns 0 on success,
 * -1 on failure
 */
static int iser_free_ib_conn_res(struct iser_conn *ib_conn, int can_destroy_id)
{
	BUG_ON(ib_conn == NULL);

	iser_err("freeing conn %p cma_id %p fmr pool %p qp %p\n",
		 ib_conn, ib_conn->cma_id,
		 ib_conn->fmr_pool, ib_conn->qp);

	/* qp is created only once both addr & route are resolved */
	if (ib_conn->fmr_pool != NULL)
		ib_destroy_fmr_pool(ib_conn->fmr_pool);

	if (ib_conn->qp != NULL)
		rdma_destroy_qp(ib_conn->cma_id);

	/* if cma handler context, the caller acts s.t the cma destroy the id */
	if (ib_conn->cma_id != NULL && can_destroy_id)
		rdma_destroy_id(ib_conn->cma_id);

	ib_conn->fmr_pool = NULL;
	ib_conn->qp	  = NULL;
	ib_conn->cma_id   = NULL;
	kfree(ib_conn->page_vec);

	return 0;
}

/**
 * Cleanup a conn_t struct.
 *
 * @param[in] conn
 */
void conn_fini(void *arg)
{
    conn_t *conn = arg;

#if WITH_TRANSPORT_IB
    if (conn->transport.type == CONN_TYPE_RDMA) {
        if (conn->rdma.cm_id) {
            if (conn->rdma.cm_id->qp)
                rdma_destroy_qp(conn->rdma.cm_id);
            rdma_destroy_id(conn->rdma.cm_id);
            conn->rdma.cm_id = NULL;
        }
    }
#endif

#if WITH_TRANSPORT_UDP && WITH_RUDP
    if (conn->transport.type == CONN_TYPE_UDP) {
        atomic_set(&conn->udp.send_seq_num, 0);
        atomic_set(&conn->udp.recv_seq_num, 0);
    }
#endif

    pthread_mutex_destroy(&conn->mutex);
#if WITH_TRANSPORT_IB || WITH_TRANSPORT_UDP
    pthread_cond_destroy(&conn->move_wait);
#endif
}

void IBConnection::on_rejected(struct rdma_cm_event* /* event */)
{
    L_(debug) << "[" << index_ << "] "
              << "connection rejected";

    rdma_destroy_qp(cm_id_);
}

static void destroy_node(struct cmatest_node *node)
{
	if (!node->cma_id)
		return;

	if (node->cma_id->qp)
		rdma_destroy_qp(node->cma_id);

	if (node->cq[SEND_CQ_INDEX])
		ibv_destroy_cq(node->cq[SEND_CQ_INDEX]);

	if (node->cq[RECV_CQ_INDEX])
		ibv_destroy_cq(node->cq[RECV_CQ_INDEX]);

	if (node->mem) {
		ibv_dereg_mr(node->mr);
		free(node->mem);
	}

	if (node->pd)
		ibv_dealloc_pd(node->pd);

	/* Destroy the RDMA ID after all device resources */
	rdma_destroy_id(node->cma_id);
}

void on_disconnect(struct rdma_cm_id *id)
{
  struct timeval start, end, dt;
  gettimeofday(&start, NULL);
  struct connection *conn = (struct connection *)id->context;

  printf("disconnected.\n");

  rdma_destroy_qp(id);

  ibv_dereg_mr(conn->send_region_mr);
  ibv_dereg_mr(conn->recv_region_mr);

  ibv_dereg_mr(conn->send_msg_mr);
  ibv_dereg_mr(conn->recv_msg_mr);

  free(conn->send_region);
  free(conn->recv_region);

  free(conn->recv_msg);
  free(conn->send_msg);

  rdma_destroy_id(id);

  free(conn);
  gettimeofday(&end, NULL);
  timersub(&end, &start, &dt);
  long usec = dt.tv_usec + 1000000 * dt.tv_sec;
  printf("[Derigester] takes %ld micro_secs.\n", usec);


  return; /* exit event loop */
}

/*
 * rpcrdma_ep_destroy
 *
 * Disconnect and destroy endpoint. After this, the only
 * valid operations on the ep are to free it (if dynamically
 * allocated) or re-create it.
 */
void
rpcrdma_ep_destroy(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)
{
	int rc;

	dprintk("RPC:       %s: entering, connected is %d\n",
		__func__, ep->rep_connected);

	cancel_delayed_work_sync(&ep->rep_connect_worker);

	if (ia->ri_id->qp) {
		rpcrdma_ep_disconnect(ep, ia);
		rdma_destroy_qp(ia->ri_id);
		ia->ri_id->qp = NULL;
	}

	ib_free_cq(ep->rep_attr.recv_cq);
	ib_free_cq(ep->rep_attr.send_cq);

	if (ia->ri_dma_mr) {
		rc = ib_dereg_mr(ia->ri_dma_mr);
		dprintk("RPC:       %s: ib_dereg_mr returned %i\n",
			__func__, rc);
	}
}

void destroy_connection(rdma_conn_t *conn)
{
    rdma_destroy_qp(conn->id);

    ibv_dereg_mr(conn->send_mr);
    ibv_dereg_mr(conn->recv_mr);
    ibv_dereg_mr(conn->data_mr);
    ibv_dereg_mr(conn->addr_mr);

    if (conn->send_msg) {
        free(conn->send_msg);
        conn->send_msg = NULL;
    }
    if (conn->recv_msg) {
        free(conn->recv_msg);
        conn->recv_msg = NULL;
    }

    rdma_destroy_id(conn->id);

    if (conn) {
        free(conn);
        conn = NULL;
    }
}

static void process_connect_reject(struct rdma_cm_event *event, conn_t *conn)
{
    pthread_mutex_lock(&conn->mutex);

    if (event->status == 28) {
        /* 28 = Consumer Reject. The remote side called rdma_reject,
         * so there is a payload. */
        const struct cm_priv_reject *rej = event->param.conn.private_data;

        if (rej->reason == REJECT_REASON_CONNECTED ||
            rej->reason == REJECT_REASON_CONNECTING) {
            /* Both sides tried to connect at the same time. This is
             * good. */
            pthread_mutex_unlock(&conn->mutex);
            return;
        }
    }

    /* That's bad, and that should not happen. */
    conn->state = CONN_STATE_DISCONNECTED;
    pthread_cond_broadcast(&conn->move_wait);

    rdma_destroy_qp(conn->rdma.cm_id);

    pthread_mutex_unlock(&conn->mutex);

    conn_put(conn);
}

static ssize_t
fi_ibv_rdm_process_addr_resolved(struct rdma_cm_id *id,
				 struct fi_ibv_rdm_ep *ep)
{
	ssize_t ret = FI_SUCCESS;
	struct ibv_qp_init_attr qp_attr;
	struct fi_ibv_rdm_tagged_conn *conn = id->context;

	VERBS_INFO(FI_LOG_AV, "ADDR_RESOLVED conn %p, addr %s:%u\n",
		   conn, inet_ntoa(conn->addr.sin_addr),
		   ntohs(conn->addr.sin_port));

	assert(id->verbs == ep->domain->verbs);

	do {
		fi_ibv_rdm_tagged_init_qp_attributes(&qp_attr, ep);
		if (rdma_create_qp(id, ep->domain->pd, &qp_attr)) {
			VERBS_INFO_ERRNO(FI_LOG_AV,
					 "rdma_create_qp failed\n", errno);
			return -errno;
		}

		if (conn->cm_role == FI_VERBS_CM_PASSIVE) {
			break;
		}

		conn->qp[0] = id->qp;
		assert(conn->id[0] == id);
		if (conn->cm_role == FI_VERBS_CM_SELF) {
			break;
		}

		ret = fi_ibv_rdm_prepare_conn_memory(ep, conn);
		if (ret != FI_SUCCESS) {
			goto err;
		}

		ret = fi_ibv_rdm_repost_receives(conn, ep, ep->rq_wr_depth);
		if (ret < 0) {
			VERBS_INFO(FI_LOG_AV, "repost receives failed\n");
			goto err;
		} else {
			ret = FI_SUCCESS;
		}
	} while (0);

	if (rdma_resolve_route(id, FI_IBV_RDM_CM_RESOLVEADDR_TIMEOUT)) {
		VERBS_INFO(FI_LOG_AV, "rdma_resolve_route failed\n");
		ret = -FI_EHOSTUNREACH;
		goto err;
	}

	return ret;
err:
	rdma_destroy_qp(id);
	return ret;
}

void xfer_rdma_destroy_ctx(struct xfer_context *ctx) {
	rdma_destroy_qp(ctx->cm_id);
	ibv_destroy_cq(ctx->cq);
	ibv_destroy_comp_channel(ctx->ch);
	ibv_dereg_mr(ctx->send_mr);
	ibv_dereg_mr(ctx->recv_mr);
	ibv_dealloc_pd(ctx->pd);
	free(ctx);
}

void network_release() {
    ibv_dereg_mr(mr_data);
    
    rdma_destroy_qp(cm_id);
    ibv_destroy_cq(cq);
    ibv_destroy_comp_channel(comp_chan);
    rdma_destroy_id(cm_id);
    rdma_destroy_event_channel(cm_channel);
}

/*
 * Connect unconnected endpoint.
 */
int
rpcrdma_ep_connect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)
{
	struct rdma_cm_id *id, *old;
	int rc = 0;
	int retry_count = 0;

	if (ep->rep_connected != 0) {
		struct rpcrdma_xprt *xprt;
retry:
		dprintk("RPC:       %s: reconnecting...\n", __func__);

		rpcrdma_ep_disconnect(ep, ia);
		rpcrdma_flush_cqs(ep);

		xprt = container_of(ia, struct rpcrdma_xprt, rx_ia);
		ia->ri_ops->ro_reset(xprt);

		id = rpcrdma_create_id(xprt, ia,
				(struct sockaddr *)&xprt->rx_data.addr);
		if (IS_ERR(id)) {
			rc = -EHOSTUNREACH;
			goto out;
		}
		/* TEMP TEMP TEMP - fail if new device:
		 * Deregister/remarshal *all* requests!
		 * Close and recreate adapter, pd, etc!
		 * Re-determine all attributes still sane!
		 * More stuff I haven't thought of!
		 * Rrrgh!
		 */
		if (ia->ri_id->device != id->device) {
			printk("RPC:       %s: can't reconnect on "
				"different device!\n", __func__);
			rdma_destroy_id(id);
			rc = -ENETUNREACH;
			goto out;
		}
		/* END TEMP */
		rc = rdma_create_qp(id, ia->ri_pd, &ep->rep_attr);
		if (rc) {
			dprintk("RPC:       %s: rdma_create_qp failed %i\n",
				__func__, rc);
			rdma_destroy_id(id);
			rc = -ENETUNREACH;
			goto out;
		}

		write_lock(&ia->ri_qplock);
		old = ia->ri_id;
		ia->ri_id = id;
		write_unlock(&ia->ri_qplock);

		rdma_destroy_qp(old);
		rdma_destroy_id(old);
	} else {

/**
 * @param[in] ni
 * @param[in] conn
 * @param[in] event
 *
 * @return status
 *
 * conn is locked
 */
static int accept_connection_request(ni_t *ni, conn_t *conn,
                                     struct rdma_cm_event *event)
{
    struct rdma_conn_param conn_param;
    struct ibv_qp_init_attr init_attr;
    struct cm_priv_accept priv;

    conn->state = CONN_STATE_CONNECTING;

    memset(&init_attr, 0, sizeof(init_attr));

    init_attr.qp_type = IBV_QPT_RC;
    init_attr.cap.max_send_wr = ni->iface->cap.max_send_wr;
    init_attr.send_cq = ni->rdma.cq;
    init_attr.recv_cq = ni->rdma.cq;
    init_attr.srq = ni->rdma.srq;
    init_attr.cap.max_send_sge = ni->iface->cap.max_send_sge;

    if (rdma_create_qp(event->id, ni->iface->pd, &init_attr)) {
        conn->state = CONN_STATE_DISCONNECTED;
        pthread_cond_broadcast(&conn->move_wait);

        return PTL_FAIL;
    }

    /* If we were already trying to connect ourselves, cancel it. */
    if (conn->rdma.cm_id != NULL) {
        assert(conn->rdma.cm_id->context == conn);
        conn->rdma.cm_id->context = NULL;
    }

    event->id->context = conn;
    conn->rdma.cm_id = event->id;

    memset(&conn_param, 0, sizeof conn_param);
    conn_param.responder_resources = 1;
    conn_param.initiator_depth = 1;
    conn_param.retry_count = 7;
    conn_param.rnr_retry_count = 7;

    if (ni->options & PTL_NI_LOGICAL) {
        conn_param.private_data = &priv;
        conn_param.private_data_len = sizeof(priv);
    }

    if (rdma_accept(event->id, &conn_param)) {
        rdma_destroy_qp(event->id);
        conn->rdma.cm_id = NULL;
        conn->state = CONN_STATE_DISCONNECTED;
        pthread_cond_broadcast(&conn->move_wait);

        return PTL_FAIL;
    }

    return PTL_OK;
}

void CompletionContext::removeConnection(struct rdma_cm_id* id) {
    if (id->qp == nullptr) {
        // Queue Pair already destroyed
        return;
    }

    LOG_TRACE("%1%: Destroying queue pair", formatRemoteAddress(id));
    mSocketMap.erase(id->qp->qp_num);
    rdma_destroy_qp(id);
}