C++ rdma_create_qp函数代码示例

static int rping_create_qp(struct rping_cb *cb)
{
	struct ibv_qp_init_attr init_attr;
	int ret;

	memset(&init_attr, 0, sizeof(init_attr));
	init_attr.cap.max_send_wr = RPING_SQ_DEPTH;
	init_attr.cap.max_recv_wr = 2;
	init_attr.cap.max_recv_sge = 1;
	init_attr.cap.max_send_sge = 1;
	init_attr.qp_type = IBV_QPT_RC;
	init_attr.send_cq = cb->cq;
	init_attr.recv_cq = cb->cq;

	if (cb->server) {
		ret = rdma_create_qp(cb->child_cm_id, cb->pd, &init_attr);
		if (!ret)
			cb->qp = cb->child_cm_id->qp;
	} else {
		ret = rdma_create_qp(cb->cm_id, cb->pd, &init_attr);
		if (!ret)
			cb->qp = cb->cm_id->qp;
	}

	return ret;
}

int on_addr_resolved(struct rdma_cm_id *id)
{
  struct ibv_qp_init_attr qp_attr;
  struct connection *conn;

  printf("address resolved.\n");

  build_context(id->verbs);
  build_qp_attr(&qp_attr);

  TEST_NZ(rdma_create_qp(id, s_ctx->pd, &qp_attr));

  id->context = conn = (struct connection *)malloc(sizeof(struct connection));

  conn->id = id;
  conn->qp = id->qp;
  conn->num_completions = 0;

  register_memory(conn);
  post_receives(conn);

  TEST_NZ(rdma_resolve_route(id, TIMEOUT_IN_MS));

  return 0;
}

static int init_node(struct cmatest_node *node)
{
	struct ibv_qp_init_attr init_qp_attr;
	int cqe, ret;
	int i;
	struct ibv_cq **cqs[] = {&node->cq[SEND_CQ_INDEX],
				 &node->cq[RECV_CQ_INDEX]};

	node->pd = ibv_alloc_pd(node->cma_id->verbs);
	if (!node->pd) {
		ret = -ENOMEM;
		printf("cmatose: unable to allocate PD\n");
		goto out;
	}

	cqe = message_count ? message_count : 1;
	for (i = 0; i < sizeof(cqs)/sizeof(cqs[0]); i++) {
		if (set_ts) {
			struct ibv_exp_cq_init_attr cq_init_attr;
			memset(&cq_init_attr, 0, sizeof(cq_init_attr));
			cq_init_attr.flags = IBV_EXP_CQ_TIMESTAMP;
			cq_init_attr.comp_mask = IBV_EXP_CQ_INIT_ATTR_FLAGS;
			*cqs[i] = (struct ibv_cq *)ibv_exp_create_cq(
					node->cma_id->verbs, cqe, node,
					NULL, 0, &cq_init_attr);
		} else {
			*cqs[i] = ibv_create_cq(node->cma_id->verbs, cqe, node,
					       0, 0);
		}
	}
	if (!node->cq[SEND_CQ_INDEX] || !node->cq[RECV_CQ_INDEX]) {
		ret = -ENOMEM;
		printf("cmatose: unable to create CQ\n");
		goto out;
	}

	memset(&init_qp_attr, 0, sizeof init_qp_attr);
	init_qp_attr.cap.max_send_wr = cqe;
	init_qp_attr.cap.max_recv_wr = cqe;
	init_qp_attr.cap.max_send_sge = 1;
	init_qp_attr.cap.max_recv_sge = 1;
	init_qp_attr.qp_context = node;
	init_qp_attr.sq_sig_all = 1;
	init_qp_attr.qp_type = IBV_QPT_RC;
	init_qp_attr.send_cq = node->cq[SEND_CQ_INDEX];
	init_qp_attr.recv_cq = node->cq[RECV_CQ_INDEX];
	ret = rdma_create_qp(node->cma_id, node->pd, &init_qp_attr);
	if (ret) {
		perror("cmatose: unable to create QP");
		goto out;
	}

	ret = create_message(node);
	if (ret) {
		printf("cmatose: failed to create messages: %d\n", ret);
		goto out;
	}
out:
	return ret;
}

void build_connection(struct rdma_cm_id *id)
{
  struct connection *conn;
  struct ibv_qp_init_attr qp_attr;

  //init semaphores
  sem_init(&read_ops, 0, 0);
  sem_init(&done_ops, 0, 0);
  sem_init(&write_ops, 0, 1);

  build_context(id->verbs);
  build_qp_attr(&qp_attr);

  TEST_NZ(rdma_create_qp(id, s_ctx->pd, &qp_attr));

  id->context = conn = (struct connection *)malloc(sizeof(struct connection));

  conn->id = id;
  conn->qp = id->qp;

  conn->send_state = SS_INIT;
  conn->recv_state = RS_INIT;

  conn->connected = 0;

  register_memory(conn);
  post_receives(conn);
}

void build_connection(struct rdma_cm_id *id)
{
    rdma_conn_t *conn;
    struct ibv_qp_init_attr qp_attr;

    build_context(id->verbs);
    build_qp_attr(&qp_attr);

    TEST_NZ(rdma_create_qp(id, s_ctx->pd, &qp_attr));

    conn = malloc(sizeof(rdma_conn_t));
    id->context = conn; 
    rdma_conn = conn;

    conn->id = id;
    conn->qp = id->qp;

    conn->send_state = SS_INIT;
    conn->recv_state = RS_INIT;

    conn->connected = 0;

    register_memory(conn);

    post_receives(conn);
}

void RDMAServerSocket::accept(client_t client_id) const {
  ibv_qp_init_attr qp_attr = {};
  qp_attr.qp_type = IBV_QPT_RC;
  qp_attr.cap.max_send_wr = 256;
  qp_attr.cap.max_recv_wr = 0;
  qp_attr.cap.max_send_sge = 1;
  qp_attr.cap.max_recv_sge = 0;
  qp_attr.cap.max_inline_data = 72;
  qp_attr.recv_cq = cq;
  qp_attr.send_cq = cq;
  qp_attr.srq = id->srq;
  qp_attr.sq_sig_all = 1;

  check_zero(rdma_create_qp(client_id.get(), NULL, &qp_attr));

  check_zero(rdma_accept(client_id.get(), nullptr));

  clients([client_id = std::move(client_id)](auto && clients) mutable {
    auto pos = std::lower_bound(std::begin(clients), std::end(clients),
                                client_id->qp->qp_num,
                                [](const auto &client, const qp_t &qp_num) {
      return client->qp->qp_num < qp_num;
    });
    clients.insert(pos, std::move(client_id));
  });
}

static int isert_conn_qp_create(struct isert_connection *isert_conn)
{
	struct rdma_cm_id *cm_id = isert_conn->cm_id;
	struct isert_device *isert_dev = isert_conn->isert_dev;
	struct ib_qp_init_attr qp_attr;
	int err;
	int cq_idx;
	int max_wr = ISER_MAX_WCE;

	TRACE_ENTRY();

	cq_idx = isert_get_cq_idx(isert_dev);

	memset(&qp_attr, 0, sizeof(qp_attr));

	qp_attr.event_handler = isert_async_evt_handler;
	qp_attr.qp_context = isert_conn;
	qp_attr.send_cq = isert_dev->cq_desc[cq_idx].cq;
	qp_attr.recv_cq = isert_dev->cq_desc[cq_idx].cq;

	isert_conn->cq_desc = &isert_dev->cq_desc[cq_idx];

	qp_attr.cap.max_send_sge = isert_conn->max_sge;
	qp_attr.cap.max_recv_sge = 3;
	qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
	qp_attr.qp_type = IB_QPT_RC;

	do {
		if (max_wr < ISER_MIN_SQ_SIZE) {
			pr_err("Failed to create qp, not enough memory\n");
			goto fail_create_qp;
		}

		qp_attr.cap.max_send_wr = max_wr;
		qp_attr.cap.max_recv_wr = max_wr;

		err = rdma_create_qp(cm_id, isert_dev->pd, &qp_attr);
		if (err && err != -ENOMEM) {
			pr_err("Failed to create qp, err:%d\n", err);
			goto fail_create_qp;
		}

		max_wr /= 2;
	} while (err == -ENOMEM);

	isert_conn->qp = cm_id->qp;

	pr_info("iser created cm_id:%p qp:0x%X\n", cm_id, cm_id->qp->qp_num);

out:
	TRACE_EXIT_RES(err);
	return err;

fail_create_qp:
	mutex_lock(&dev_list_mutex);
	isert_dev->cq_qps[cq_idx]--;
	mutex_unlock(&dev_list_mutex);
	goto out;
}

void Connector::build_conn(struct rdma_cm_id* id_)
{
  struct ibv_qp_init_attr qp_attr;

  build_context(id_->verbs);
  build_qp_attr(&qp_attr);

  TEST_NZ(rdma_create_qp(id_, s_ctx_->pd_, &qp_attr) )
}

static ssize_t
fi_ibv_rdm_process_addr_resolved(struct rdma_cm_id *id,
				 struct fi_ibv_rdm_ep *ep)
{
	ssize_t ret = FI_SUCCESS;
	struct ibv_qp_init_attr qp_attr;
	struct fi_ibv_rdm_tagged_conn *conn = id->context;

	VERBS_INFO(FI_LOG_AV, "ADDR_RESOLVED conn %p, addr %s:%u\n",
		   conn, inet_ntoa(conn->addr.sin_addr),
		   ntohs(conn->addr.sin_port));

	assert(id->verbs == ep->domain->verbs);

	do {
		fi_ibv_rdm_tagged_init_qp_attributes(&qp_attr, ep);
		if (rdma_create_qp(id, ep->domain->pd, &qp_attr)) {
			VERBS_INFO_ERRNO(FI_LOG_AV,
					 "rdma_create_qp failed\n", errno);
			return -errno;
		}

		if (conn->cm_role == FI_VERBS_CM_PASSIVE) {
			break;
		}

		conn->qp[0] = id->qp;
		assert(conn->id[0] == id);
		if (conn->cm_role == FI_VERBS_CM_SELF) {
			break;
		}

		ret = fi_ibv_rdm_prepare_conn_memory(ep, conn);
		if (ret != FI_SUCCESS) {
			goto err;
		}

		ret = fi_ibv_rdm_repost_receives(conn, ep, ep->rq_wr_depth);
		if (ret < 0) {
			VERBS_INFO(FI_LOG_AV, "repost receives failed\n");
			goto err;
		} else {
			ret = FI_SUCCESS;
		}
	} while (0);

	if (rdma_resolve_route(id, FI_IBV_RDM_CM_RESOLVEADDR_TIMEOUT)) {
		VERBS_INFO(FI_LOG_AV, "rdma_resolve_route failed\n");
		ret = -FI_EHOSTUNREACH;
		goto err;
	}

	return ret;
err:
	rdma_destroy_qp(id);
	return ret;
}

/**
 * @param[in] ni
 * @param[in] conn
 * @param[in] event
 *
 * @return status
 *
 * conn is locked
 */
static int accept_connection_request(ni_t *ni, conn_t *conn,
                                     struct rdma_cm_event *event)
{
    struct rdma_conn_param conn_param;
    struct ibv_qp_init_attr init_attr;
    struct cm_priv_accept priv;

    conn->state = CONN_STATE_CONNECTING;

    memset(&init_attr, 0, sizeof(init_attr));

    init_attr.qp_type = IBV_QPT_RC;
    init_attr.cap.max_send_wr = ni->iface->cap.max_send_wr;
    init_attr.send_cq = ni->rdma.cq;
    init_attr.recv_cq = ni->rdma.cq;
    init_attr.srq = ni->rdma.srq;
    init_attr.cap.max_send_sge = ni->iface->cap.max_send_sge;

    if (rdma_create_qp(event->id, ni->iface->pd, &init_attr)) {
        conn->state = CONN_STATE_DISCONNECTED;
        pthread_cond_broadcast(&conn->move_wait);

        return PTL_FAIL;
    }

    /* If we were already trying to connect ourselves, cancel it. */
    if (conn->rdma.cm_id != NULL) {
        assert(conn->rdma.cm_id->context == conn);
        conn->rdma.cm_id->context = NULL;
    }

    event->id->context = conn;
    conn->rdma.cm_id = event->id;

    memset(&conn_param, 0, sizeof conn_param);
    conn_param.responder_resources = 1;
    conn_param.initiator_depth = 1;
    conn_param.retry_count = 7;
    conn_param.rnr_retry_count = 7;

    if (ni->options & PTL_NI_LOGICAL) {
        conn_param.private_data = &priv;
        conn_param.private_data_len = sizeof(priv);
    }

    if (rdma_accept(event->id, &conn_param)) {
        rdma_destroy_qp(event->id);
        conn->rdma.cm_id = NULL;
        conn->state = CONN_STATE_DISCONNECTED;
        pthread_cond_broadcast(&conn->move_wait);

        return PTL_FAIL;
    }

    return PTL_OK;
}

/*
 * Connect unconnected endpoint.
 */
int
rpcrdma_ep_connect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)
{
	struct rdma_cm_id *id, *old;
	int rc = 0;
	int retry_count = 0;

	if (ep->rep_connected != 0) {
		struct rpcrdma_xprt *xprt;
retry:
		dprintk("RPC:       %s: reconnecting...\n", __func__);

		rpcrdma_ep_disconnect(ep, ia);
		rpcrdma_flush_cqs(ep);

		xprt = container_of(ia, struct rpcrdma_xprt, rx_ia);
		ia->ri_ops->ro_reset(xprt);

		id = rpcrdma_create_id(xprt, ia,
				(struct sockaddr *)&xprt->rx_data.addr);
		if (IS_ERR(id)) {
			rc = -EHOSTUNREACH;
			goto out;
		}
		/* TEMP TEMP TEMP - fail if new device:
		 * Deregister/remarshal *all* requests!
		 * Close and recreate adapter, pd, etc!
		 * Re-determine all attributes still sane!
		 * More stuff I haven't thought of!
		 * Rrrgh!
		 */
		if (ia->ri_id->device != id->device) {
			printk("RPC:       %s: can't reconnect on "
				"different device!\n", __func__);
			rdma_destroy_id(id);
			rc = -ENETUNREACH;
			goto out;
		}
		/* END TEMP */
		rc = rdma_create_qp(id, ia->ri_pd, &ep->rep_attr);
		if (rc) {
			dprintk("RPC:       %s: rdma_create_qp failed %i\n",
				__func__, rc);
			rdma_destroy_id(id);
			rc = -ENETUNREACH;
			goto out;
		}

		write_lock(&ia->ri_qplock);
		old = ia->ri_id;
		ia->ri_id = id;
		write_unlock(&ia->ri_qplock);

		rdma_destroy_qp(old);
		rdma_destroy_id(old);
	} else {

void IBConnection::create_qp(struct ibv_pd* pd, struct ibv_cq* cq)
{
    struct ibv_qp_init_attr qp_attr;
    memset(&qp_attr, 0, sizeof qp_attr);
    qp_attr.cap = qp_cap_;
    qp_attr.send_cq = cq;
    qp_attr.recv_cq = cq;
    qp_attr.qp_type = IBV_QPT_RC;
    int err = rdma_create_qp(cm_id_, pd, &qp_attr);
    if (err)
        throw InfinibandException("creation of QP failed");
}

void build_connection(struct rdma_cm_id *id){

  struct ibv_qp_init_attr qp_attr;
  struct connection *conn;
  build_qp_attr(&qp_attr);

  TEST_NZ(rdma_create_qp(id, s_ctx->pd, &qp_attr));

  id->context = conn = (struct connection *)malloc(sizeof(struct connection));

  conn->id = id;
  conn->qp = id->qp;

}

static int init_node(struct cmatest_node *node)
{
	struct ibv_qp_init_attr init_qp_attr;
	int cqe, ret;

	node->pd = ibv_alloc_pd(node->cma_id->verbs);
	if (!node->pd) {
		ret = -ENOMEM;
		printf("cmatose: unable to allocate PD\n");
		goto out;
	}

	cqe = message_count ? message_count : 1;
	node->cq[SEND_CQ_INDEX] = ibv_create_cq(node->cma_id->verbs, cqe, node, NULL, 0);
	node->cq[RECV_CQ_INDEX] = ibv_create_cq(node->cma_id->verbs, cqe, node, NULL, 0);
	if (!node->cq[SEND_CQ_INDEX] || !node->cq[RECV_CQ_INDEX]) {
		ret = -ENOMEM;
		printf("cmatose: unable to create CQ\n");
		goto out;
	}

	memset(&init_qp_attr, 0, sizeof init_qp_attr);
	init_qp_attr.cap.max_send_wr = cqe;
	init_qp_attr.cap.max_recv_wr = cqe;
	init_qp_attr.cap.max_send_sge = 1;
	init_qp_attr.cap.max_recv_sge = 1;
	init_qp_attr.qp_context = node;
	init_qp_attr.sq_sig_all = 1;
	init_qp_attr.qp_type = IBV_QPT_RC;
	init_qp_attr.send_cq = node->cq[SEND_CQ_INDEX];
	init_qp_attr.recv_cq = node->cq[RECV_CQ_INDEX];
	ret = rdma_create_qp(node->cma_id, node->pd, &init_qp_attr);
	if (ret) {
		perror("cmatose: unable to create QP");
		goto out;
	}

	ret = create_message(node);
	if (ret) {
		printf("cmatose: failed to create messages: %d\n", ret);
		goto out;
	}
out:
	return ret;
}

/**
 * Accept an RC connection request to self.
 *
 * called while holding connect->mutex
 * only used for physical NIs
 *
 * @param[in] ni
 * @param[in] conn
 * @param[in] event
 *
 * @return status
 */
static int accept_connection_self(ni_t *ni, conn_t *conn,
                                  struct rdma_cm_event *event)
{
    struct rdma_conn_param conn_param;
    struct ibv_qp_init_attr init_attr;

    conn->state = CONN_STATE_CONNECTING;

    memset(&init_attr, 0, sizeof(init_attr));
    init_attr.qp_type = IBV_QPT_RC;
    init_attr.send_cq = ni->rdma.cq;
    init_attr.recv_cq = ni->rdma.cq;
    init_attr.srq = ni->rdma.srq;
    init_attr.cap.max_send_wr = ni->iface->cap.max_send_wr;
    init_attr.cap.max_send_sge = ni->iface->cap.max_send_sge;

    if (rdma_create_qp(event->id, ni->iface->pd, &init_attr)) {
        conn->state = CONN_STATE_DISCONNECTED;
        pthread_cond_broadcast(&conn->move_wait);

        return PTL_FAIL;
    }

    ni->rdma.self_cm_id = event->id;

    /* The lower 2 bits (on 32 bits hosts), or 3 bits (on 64 bits
     * hosts) of a pointer is always 0. Use it to store the type of
     * context. 0=conn; 1=NI. */
    event->id->context = (void *)((uintptr_t) ni | 1);

    memset(&conn_param, 0, sizeof conn_param);
    conn_param.responder_resources = 1;
    conn_param.initiator_depth = 1;
    conn_param.rnr_retry_count = 7;

    if (rdma_accept(event->id, &conn_param)) {
        rdma_destroy_qp(event->id);
        conn->state = CONN_STATE_DISCONNECTED;
        pthread_cond_broadcast(&conn->move_wait);

        return PTL_FAIL;
    }

    return PTL_OK;
}