C++ ibv_post_send函数代码示例

static void
kickoff_rdma_with_offset(uintptr_t offset, IbvConnection *conn, int length)
{
    struct ibv_send_wr wr, *bad_wr = NULL;
    struct ibv_sge sge;

    memset(&wr, 0, sizeof(wr));

    wr.wr_id = (uintptr_t)conn;
    wr.opcode = IBV_WR_RDMA_WRITE;
    wr.sg_list = &sge;
    wr.num_sge = 1;
    wr.send_flags = IBV_SEND_SIGNALED;
    wr.wr.rdma.remote_addr = (uintptr_t)conn->peer_mr.addr + offset;
    wr.wr.rdma.rkey = conn->peer_mr.rkey;

    sge.addr = (uintptr_t)conn->rdma_local_region + offset;
    sge.length = length;
    sge.lkey = conn->rdma_local_mr->lkey;

    if (RDMA_BUFFER_SIZE < (offset + length)) {
        WARN(0, "kickoff_rdma_with_offset: offset + length (=%d) exceeds RDMA_BUFFER_SIZE (=%d).\n",
             offset + length, RDMA_BUFFER_SIZE);
        exit(1);
    }

    TEST_NZ(ibv_post_send(conn->qp, &wr, &bad_wr));
}

void _rdma_write_offset(void *context, void* buf, uint64_t offset)
{
  struct connection *conn = (struct connection *)context;
  struct ibv_send_wr wr, *bad_wr = NULL;
  struct ibv_sge sge;
  //printf("7\n");
  memset(&wr, 0, sizeof(wr));

  wr.wr_id = (uintptr_t)conn;
  wr.opcode = IBV_WR_RDMA_WRITE;
  wr.sg_list = &sge;
  wr.num_sge = 1;
  wr.send_flags = IBV_SEND_SIGNALED;
  wr.wr.rdma.remote_addr = (uintptr_t)conn->peer_mr.addr + offset;
  wr.wr.rdma.rkey = conn->peer_mr.rkey;

  sge.addr = (uintptr_t)conn->rdma_remote_region;
  sge.length = RDMA_BUFFER_SIZE;
  sge.lkey = conn->rdma_remote_mr->lkey;

  time_stamp(2);
  sem_wait(&write_ops);
  memcpy(conn->rdma_remote_region, (char*)buf, RDMA_BUFFER_SIZE);
  TEST_NZ(ibv_post_send(conn->qp, &wr, &bad_wr));

}

static int post_sends(struct cmatest_node *node, int signal_flag)
{
	struct ibv_send_wr send_wr, *bad_send_wr;
	struct ibv_sge sge;
	int i, ret = 0;

	if (!node->connected || !message_count)
		return 0;

	send_wr.next = NULL;
	send_wr.sg_list = &sge;
	send_wr.num_sge = 1;
	send_wr.opcode = IBV_WR_SEND_WITH_IMM;
	send_wr.send_flags = signal_flag;
	send_wr.wr_id = (unsigned long)node;
	send_wr.imm_data = htonl(node->cma_id->qp->qp_num);

	send_wr.wr.ud.ah = node->ah;
	send_wr.wr.ud.remote_qpn = node->remote_qpn;
	send_wr.wr.ud.remote_qkey = node->remote_qkey;

	sge.length = message_size;
	sge.lkey = node->mr->lkey;
	sge.addr = (uintptr_t) node->mem;

	for (i = 0; i < message_count && !ret; i++) {
		ret = ibv_post_send(node->cma_id->qp, &send_wr, &bad_send_wr);
		if (ret)
			printf("failed to post sends: %d\n", ret);
	}
	return ret;
}

static int post_sends(struct cmatest_node *node)
{
	struct ibv_send_wr send_wr, *bad_send_wr;
	struct ibv_sge sge;
	int i, ret = 0;

	if (!node->connected || !message_count)
		return 0;

	send_wr.next = NULL;
	send_wr.sg_list = &sge;
	send_wr.num_sge = 1;
	send_wr.opcode = IBV_WR_SEND;
	send_wr.send_flags = 0;
	send_wr.wr_id = (unsigned long)node;

	sge.length = message_size;
	sge.lkey = node->mr->lkey;
	sge.addr = (uintptr_t) node->mem;

	for (i = 0; i < message_count && !ret; i++) {
		ret = ibv_post_send(node->cma_id->qp, &send_wr, &bad_send_wr);
		if (ret) 
			printf("failed to post sends: %d\n", ret);
	}
	return ret;
}

int mca_btl_openib_get_internal (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *ep,
                                 mca_btl_openib_get_frag_t *frag)
{
    int qp = to_base_frag(frag)->base.order;
    struct ibv_send_wr *bad_wr;

    /* check for a send wqe */
    if (qp_get_wqe(ep, qp) < 0) {
        qp_put_wqe(ep, qp);
        return OPAL_ERR_OUT_OF_RESOURCE;
    }

    /* check for a get token */
    if (OPAL_THREAD_ADD32(&ep->get_tokens,-1) < 0) {
        qp_put_wqe(ep, qp);
        OPAL_THREAD_ADD32(&ep->get_tokens,1);
        return OPAL_ERR_OUT_OF_RESOURCE;
    }

    qp_inflight_wqe_to_frag(ep, qp, to_com_frag(frag));
    qp_reset_signal_count(ep, qp);

    if (ibv_post_send(ep->qps[qp].qp->lcl_qp, &frag->sr_desc, &bad_wr)) {
        qp_put_wqe(ep, qp);
        OPAL_THREAD_ADD32(&ep->get_tokens,1);
        return OPAL_ERROR;
    }

    return OPAL_SUCCESS;
}

void write_remote(struct connection * conn, uint32_t len){

    uint32_t size =len&(~(1U<<31));
    snprintf(conn->send_region, send_buffer_size, "message from active/client side with pid %d", getpid());
    struct ibv_send_wr wr, *bad_wr = NULL; 
    struct ibv_sge sge;

    memset(&wr,0,sizeof(wr));

    wr.wr_id = (uintptr_t)conn;
    wr.opcode = IBV_WR_RDMA_WRITE_WITH_IMM;
    wr.send_flags = IBV_SEND_SIGNALED;

    wr.imm_data = htonl(len);
    wr.wr.rdma.remote_addr = (uintptr_t)conn->peer_addr;
    wr.wr.rdma.rkey = conn->peer_rkey;

    if (size>0){
        wr.sg_list = &sge;
        wr.num_sge = 1;
    sge.addr = (uintptr_t)conn->send_region;
    sge.length = size;
    sge.lkey = conn->send_region_mr->lkey;
    }
    TEST_NZ(ibv_post_send(conn->qp, &wr, &bad_wr));


}

/**
 * Sends a buffer's memory region so that it can be mapped to it's remote end.
 */
void RDMAChannel::SendMR(ibv_mr* mr, int id) {
  // Map the memory region itself so that it can be sent
  ibv_mr* init = ibv_reg_mr(adapter_.pd_, mr, sizeof(ibv_mr),
                            IBV_ACCESS_LOCAL_WRITE);

  struct ibv_sge list;
  list.addr = (uint64_t) mr;
  list.length = sizeof(ibv_mr);
  list.lkey = init->lkey;

  struct ibv_send_wr wr;
  caffe_memset(sizeof(wr), 0, &wr);
  wr.wr_id = (uint64_t) init;
  wr.sg_list = &list;
  wr.num_sge = 1;
  wr.opcode = IBV_WR_SEND_WITH_IMM;
  wr.send_flags = IBV_SEND_SIGNALED;
  wr.imm_data = id;

  struct ibv_send_wr *bad_wr;
  CHECK(!ibv_post_send(qp_, &wr, &bad_wr));

  for (;;) {
    ibv_wc wc;
    int ne = ibv_poll_cq(write_cq_, 1, &wc);
    CHECK_GE(ne, 0);
    if (ne && wc.wr_id == (uint64_t) init) {
      break;
    }
  }
  CHECK(!ibv_dereg_mr(init));
}

int Process::post_send(void *context){
	
	Connection *conn = (Connection *) context;
	std::cout<<"SEND LOCATION CONN -> ID"<<conn->identifier<<"\n";
	std::cout<<"SEND POINTER ID"<<listener<<std::endl<<std::flush;

	assert(conn != nullptr);
	assert(conn->identifier != nullptr);

	struct ibv_send_wr wr, *bad_wr = nullptr;
	struct ibv_sge sge;
	assert(&message != nullptr);
	calc_message_numerical(&message);
	assert(&message != nullptr);
	assert(conn->send_region != nullptr);
	memcpy(conn->send_region, message.x, message.size*sizeof(char));
	printf("connected. posting send...\n");
	
	memsetzero(&wr);

	wr.wr_id = (uintptr_t)conn;
	wr.opcode = IBV_WR_SEND;
	wr.sg_list = &sge;
	wr.num_sge = 1;
	wr.send_flags = IBV_SEND_SIGNALED;
	
	sge.addr = (uintptr_t)conn->send_region;
	sge.length = message.size;
	sge.lkey = conn->send_memory_region->lkey;
	
	TEST_NZ(ibv_post_send(conn->queue_pair, &wr, &bad_wr));
	
	return 0;
}

void send_ack() {
    /* Send ack */
    ack_buffer = client_pdata.index;
    sge_send.addr = (uintptr_t)&ack_buffer;
    sge_send.length = sizeof(ack_buffer);
    sge_send.lkey = mr_ack_buffer->lkey;

    send_wr.wr_id = 1;
    send_wr.opcode = IBV_WR_SEND;
    send_wr.send_flags = IBV_SEND_SIGNALED;
    send_wr.sg_list = &sge_send;
    send_wr.num_sge = 1;

    err = ibv_post_send(cm_id->qp, &send_wr, &bad_send_wr);
    assert(err == 0);

    /* Wait send completion */
    err = ibv_get_cq_event(comp_chan, &evt_cq, &cq_context);
    assert(err == 0);

    ibv_ack_cq_events(evt_cq, 1);

    err = ibv_req_notify_cq(cq, 0);
    assert(err == 0);
    
    n = ibv_poll_cq(cq, 1, &wc);
    assert(n >= 1); 
    if (wc.status != IBV_WC_SUCCESS) 
        printf("Warning: Client %d send ack failed\n", client_pdata.index);
}

int on_connection(void *context)
{
  struct connection *conn = (struct connection *)context;
  struct ibv_send_wr wr, *bad_wr = NULL;
  struct ibv_sge sge;

  snprintf(conn->send_region, BUFFER_SIZE, "message from active/client side with pid %d", getpid());

  printf("connected. posting send...\n");

  memset(&wr, 0, sizeof(wr));

  wr.wr_id = (uintptr_t)conn;
  wr.opcode = IBV_WR_SEND;
  wr.sg_list = &sge;
  wr.num_sge = 1;
  wr.send_flags = IBV_SEND_SIGNALED;

  sge.addr = (uintptr_t)conn->send_region;
  sge.length = BUFFER_SIZE;
  sge.lkey = conn->send_mr->lkey;

  TEST_NZ(ibv_post_send(conn->qp, &wr, &bad_wr));

  return 0;
}

void cfio_rdma_client_write_data(
        int remote_offset, 
        int length, 
        int local_offset)
{
    // rdma_debug("write data ...");
    if (remote_offset < 0 || remote_offset + length > DATA_REGION_SIZE) {
        die("RDMA out of region");
    }

    struct ibv_send_wr wr, *bad_wr = NULL;
    struct ibv_sge sge;

    memset(&wr, 0, sizeof(wr));

    rdma_conn_t *conn = rdma_conn;
    wr.wr_id = (uintptr_t)(conn);
    wr.opcode = IBV_WR_RDMA_WRITE;
    wr.sg_list = &sge;
    wr.num_sge = 1;
    wr.send_flags = IBV_SEND_SIGNALED;
    wr.wr.rdma.remote_addr = (uintptr_t)((char *)conn->peer_data_mr.addr + remote_offset); 
    wr.wr.rdma.rkey = conn->peer_data_mr.rkey;

    sge.addr = (uintptr_t)(conn->data_region + local_offset);
    sge.length = length;
    sge.lkey = conn->data_mr->lkey;

    ++ request_stack_size;
    TEST_NZ(ibv_post_send(conn->qp, &wr, &bad_wr));
}

static int
_gaspi_event_send(gaspi_cuda_event *event, int queue)
{
  struct ibv_send_wr swr;
  struct ibv_sge slist;
  struct ibv_send_wr *bad_wr;

  swr.wr.rdma.rkey = glb_gaspi_ctx.rrmd[event->segment_remote][event->rank].rkey;
  swr.sg_list    = &slist;
  swr.num_sge    = 1;
  swr.wr_id      = event->rank;
  swr.opcode     = IBV_WR_RDMA_WRITE;
  swr.send_flags = IBV_SEND_SIGNALED;
  swr.next       = NULL;

  slist.addr = (uintptr_t) (char*)(glb_gaspi_ctx.rrmd[event->segment_local][event->rank].host_ptr + NOTIFY_OFFSET + event->offset_local);

  slist.length = event->size;
  slist.lkey = ((struct ibv_mr *)glb_gaspi_ctx.rrmd[event->segment_local][glb_gaspi_ctx.rank].host_mr)->lkey;

  if(glb_gaspi_ctx.rrmd[event->segment_remote][event->rank].cudaDevId >= 0)
    swr.wr.rdma.remote_addr = (glb_gaspi_ctx.rrmd[event->segment_remote][event->rank].addr + event->offset_remote);
  else
    swr.wr.rdma.remote_addr = (glb_gaspi_ctx.rrmd[event->segment_remote][event->rank].addr + NOTIFY_OFFSET + event->offset_remote);

  if(ibv_post_send(glb_gaspi_ctx_ib.qpC[queue][event->rank], &swr, &bad_wr))
  {
    glb_gaspi_ctx.qp_state_vec[queue][event->rank] = GASPI_STATE_CORRUPT;
    return -1;
  }

  event->ib_use = 1;

  return 0;
}

static UCS_F_ALWAYS_INLINE void
uct_rc_verbs_ep_post_send(uct_rc_verbs_iface_t* iface, uct_rc_verbs_ep_t* ep,
                          struct ibv_send_wr *wr, int send_flags, int max_log_sge)
{
    struct ibv_send_wr *bad_wr;
    int ret;

    uct_rc_txqp_check(&ep->super.txqp);

    if (!(send_flags & IBV_SEND_SIGNALED)) {
        send_flags |= uct_rc_iface_tx_moderation(&iface->super, &ep->super.txqp,
                                                 IBV_SEND_SIGNALED);
    }
    if (wr->opcode == IBV_WR_RDMA_READ) {
        send_flags |= uct_rc_ep_atomic_fence(&iface->super, &ep->fi,
                                             IBV_SEND_FENCE);
    }

    wr->send_flags = send_flags;
    wr->wr_id      = uct_rc_txqp_unsignaled(&ep->super.txqp);

    uct_ib_log_post_send(&iface->super.super, ep->super.txqp.qp, wr, max_log_sge,
                         (wr->opcode == IBV_WR_SEND) ? uct_rc_ep_packet_dump : NULL);

    ret = ibv_post_send(ep->super.txqp.qp, wr, &bad_wr);
    if (ret != 0) {
        ucs_fatal("ibv_post_send() returned %d (%m)", ret);
    }

    uct_rc_verbs_txqp_posted(&ep->super.txqp, &ep->txcnt, &iface->super, send_flags & IBV_SEND_SIGNALED);
}

static UCS_F_ALWAYS_INLINE void
uct_rc_verbs_ep_post_send(uct_rc_verbs_iface_t* iface, uct_rc_verbs_ep_t* ep,
                          struct ibv_send_wr *wr, int send_flags)
{
    struct ibv_send_wr *bad_wr;
    int ret;

    uct_rc_txqp_check(&ep->super.txqp);

    if (!(send_flags & IBV_SEND_SIGNALED)) {
        send_flags |= uct_rc_iface_tx_moderation(&iface->super, &ep->super.txqp,
                                                 IBV_SEND_SIGNALED);
    }
    wr->send_flags = send_flags;
    wr->wr_id      = uct_rc_txqp_unsignaled(&ep->super.txqp);

    uct_ib_log_post_send(&iface->super.super, ep->super.txqp.qp, wr,
                         (wr->opcode == IBV_WR_SEND) ? uct_rc_ep_am_packet_dump : NULL);

    UCT_IB_INSTRUMENT_RECORD_SEND_WR_LEN("uct_rc_verbs_ep_post_send", wr);

    ret = ibv_post_send(ep->super.txqp.qp, wr, &bad_wr);
    if (ret != 0) {
        ucs_fatal("ibv_post_send() returned %d (%m)", ret);
    }

    uct_rc_verbs_txqp_posted(&ep->super.txqp, &ep->txcnt, &iface->super, send_flags & IBV_SEND_SIGNALED);
}

static int rdma_write_keys(struct pingpong_dest *my_dest,
		struct perftest_comm *comm)
{
	struct ibv_send_wr wr;
	struct ibv_send_wr *bad_wr;
	struct ibv_sge list;
	struct ibv_wc wc;
	int ne;

	#ifdef HAVE_ENDIAN
	int i;
	struct pingpong_dest m_my_dest;

	m_my_dest.lid 		= htobe32(my_dest->lid);
	m_my_dest.out_reads 	= htobe32(my_dest->out_reads);
	m_my_dest.qpn 		= htobe32(my_dest->qpn);
	m_my_dest.psn 		= htobe32(my_dest->psn);
	m_my_dest.rkey 		= htobe32(my_dest->rkey);
	m_my_dest.srqn		= htobe32(my_dest->srqn);
	m_my_dest.gid_index	= htobe32(my_dest->gid_index);
	m_my_dest.vaddr		= htobe64(my_dest->vaddr);

	for(i=0; i<16; i++) {
		m_my_dest.gid.raw[i] = my_dest->gid.raw[i];
	}

	memcpy(comm->rdma_ctx->buf, &m_my_dest, sizeof(struct pingpong_dest));
	#else
	memcpy(comm->rdma_ctx->buf, &my_dest, sizeof(struct pingpong_dest));
	#endif
	list.addr   = (uintptr_t)comm->rdma_ctx->buf;
	list.length = sizeof(struct pingpong_dest);
	list.lkey   = comm->rdma_ctx->mr->lkey;


	wr.wr_id      = SYNC_SPEC_ID;
	wr.sg_list    = &list;
	wr.num_sge    = 1;
	wr.opcode     = IBV_WR_SEND;
	wr.send_flags = IBV_SEND_SIGNALED;
	wr.next       = NULL;

	if (ibv_post_send(comm->rdma_ctx->qp[0],&wr,&bad_wr)) {
		fprintf(stderr, "Function ibv_post_send failed\n");
		return 1;
	}

	do {
		ne = ibv_poll_cq(comm->rdma_ctx->send_cq, 1,&wc);
	} while (ne == 0);

	if (wc.status || wc.opcode != IBV_WC_SEND || wc.wr_id != SYNC_SPEC_ID) {
		fprintf(stderr, " Bad wc status %d\n",(int)wc.status);
		return 1;
	}

	return 0;
}