C++ ibv_reg_mr函数代码示例

void register_memory(rdma_conn_t *conn)
{
    conn->send_msg = malloc(sizeof(rdma_msg_t));
    conn->recv_msg = malloc(sizeof(rdma_msg_t));

    conn->data_region = data_region;
    conn->addr_region = addr_region;

    TEST_Z(conn->send_mr = ibv_reg_mr(
                s_ctx->pd, 
                conn->send_msg, 
                sizeof(rdma_msg_t), 
                IBV_ACCESS_LOCAL_WRITE));

    TEST_Z(conn->recv_mr = ibv_reg_mr(
                s_ctx->pd, 
                conn->recv_msg, 
                sizeof(rdma_msg_t), 
                IBV_ACCESS_LOCAL_WRITE));

    TEST_Z(conn->addr_mr = ibv_reg_mr(
                s_ctx->pd, 
                conn->addr_region, 
                ADDR_REGION_SIZE, 
                IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE));

    TEST_Z(conn->data_mr = ibv_reg_mr(
                s_ctx->pd, 
                conn->data_region, 
                DATA_REGION_SIZE, 
                IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ));
}

void register_memory(struct connection *conn)
{
  conn->send_region = malloc(send_buffer_size);
  conn->recv_region = malloc(recv_buffer_size);
  memset(conn->recv_region,0, recv_buffer_size);

  conn->send_msg = malloc(sizeof(struct message));
  conn->recv_msg = malloc(sizeof(struct message));

  TEST_Z(conn->send_region_mr = ibv_reg_mr(
    s_ctx->pd, 
    conn->send_region, 
    send_buffer_size, 
    IBV_ACCESS_LOCAL_WRITE));

  TEST_Z(conn->recv_region_mr = ibv_reg_mr(
    s_ctx->pd, 
    conn->recv_region, 
    recv_buffer_size, 
    IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE));

  TEST_Z(conn->send_msg_mr = ibv_reg_mr(
    s_ctx->pd, 
    conn->send_msg, 
    sizeof(struct message), 
    IBV_ACCESS_LOCAL_WRITE));

  TEST_Z(conn->recv_msg_mr = ibv_reg_mr(
    s_ctx->pd, 
    conn->recv_msg, 
    sizeof(struct message), 
    IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE));
}

void InputChannelSender::on_addr_resolved(struct rdma_cm_id* id)
{
    IBConnectionGroup<InputChannelConnection>::on_addr_resolved(id);

    if (!mr_data_) {
        // Register memory regions.
        mr_data_ = ibv_reg_mr(
            pd_, const_cast<uint8_t*>(data_source_.data_send_buffer().ptr()),
            data_source_.data_send_buffer().bytes(), IBV_ACCESS_LOCAL_WRITE);
        if (!mr_data_) {
            L_(error) << "ibv_reg_mr failed for mr_data: " << strerror(errno);
            throw InfinibandException("registration of memory region failed");
        }

        mr_desc_ = ibv_reg_mr(pd_, const_cast<fles::MicrosliceDescriptor*>(
                                       data_source_.desc_send_buffer().ptr()),
                              data_source_.desc_send_buffer().bytes(),
                              IBV_ACCESS_LOCAL_WRITE);
        if (!mr_desc_) {
            L_(error) << "ibv_reg_mr failed for mr_desc: " << strerror(errno);
            throw InfinibandException("registration of memory region failed");
        }

        if (true) {
            dump_mr(mr_desc_);
            dump_mr(mr_data_);
        }
    }
}

void register_memory(struct connection *conn)
{
  conn->send_msg = malloc(sizeof(struct message));
  conn->recv_msg = malloc(sizeof(struct message));

  conn->rdma_local_region = malloc(RDMA_BUFFER_SIZE);
  conn->rdma_remote_region = malloc(RDMA_BUFFER_SIZE);

  TEST_Z(conn->send_mr = ibv_reg_mr(
    s_ctx->pd,
    conn->send_msg,
    sizeof(struct message),
    IBV_ACCESS_LOCAL_WRITE));

  TEST_Z(conn->recv_mr = ibv_reg_mr(
    s_ctx->pd,
    conn->recv_msg,
    sizeof(struct message),
    IBV_ACCESS_LOCAL_WRITE | ((s_mode == M_WRITE) ? IBV_ACCESS_REMOTE_WRITE : IBV_ACCESS_REMOTE_READ)));

  TEST_Z(conn->rdma_local_mr = ibv_reg_mr(
    s_ctx->pd,
    conn->rdma_local_region,
    RDMA_BUFFER_SIZE,
    IBV_ACCESS_LOCAL_WRITE));

  TEST_Z(conn->rdma_remote_mr = ibv_reg_mr(
    s_ctx->pd,
    conn->rdma_remote_region,
    RDMA_BUFFER_SIZE,
    IBV_ACCESS_LOCAL_WRITE | ((s_mode == M_WRITE) ? IBV_ACCESS_REMOTE_WRITE : IBV_ACCESS_REMOTE_READ)));
}

void register_memory(IbvConnection *conn)
{
    conn->send_msg = (message *)malloc(sizeof(struct message));
    conn->recv_msg = (message *)malloc(sizeof(struct message));

    conn->rdma_local_region = (char *)calloc(RDMA_BUFFER_SIZE, 1);
    conn->rdma_remote_region = (char *)calloc(RDMA_BUFFER_SIZE, 1);

    TEST_Z(conn->send_mr = ibv_reg_mr(
                                      conn->pd, 
                                      conn->send_msg, 
                                      sizeof(struct message), 
                                      IBV_ACCESS_LOCAL_WRITE));

    TEST_Z(conn->recv_mr = ibv_reg_mr(
                                      conn->pd, 
                                      conn->recv_msg, 
                                      sizeof(struct message), 
                                      IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE ));

    TEST_Z(conn->rdma_local_mr = ibv_reg_mr(
                                            conn->pd, 
                                            conn->rdma_local_region, 
                                            RDMA_BUFFER_SIZE, 
                                            IBV_ACCESS_LOCAL_WRITE));

    TEST_Z(conn->rdma_remote_mr = ibv_reg_mr(
                                             conn->pd, 
                                             conn->rdma_remote_region, 
                                             RDMA_BUFFER_SIZE, 
                                             IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE));
}

        void on_preconnect(rdma_cm_id * id, ibv_pd * pd, boost::system::error_code &ec)
        {
            close();
            {
                util::spinlock::scoped_lock lk(mtx_);
                HPX_ASSERT(buffer_);
                buffer_mr_ = ibv_reg_mr(
                    pd
                  , buffer_
                  , buffer_size_
                  , IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE
                );
                if(!buffer_mr_)
                {
                    int verrno = errno;
                    boost::system::error_code err(verrno, boost::system::system_category());
                    HPX_IBVERBS_THROWS_IF(
                        ec
                      , err
                    );
                }
                server_msg_mr_ = ibv_reg_mr(
                    pd
                  , server_msg_
                  , sizeof(hpx::parcelset::policies::ibverbs::message)
                  , IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE
                );
                if(!server_msg_mr_)
                {
                    int verrno = errno;
                    boost::system::error_code err(verrno, boost::system::system_category());
                    HPX_IBVERBS_THROWS_IF(
                        ec
                      , err
                    );
                }
                client_msg_mr_ = ibv_reg_mr(
                    pd
                  , client_msg_
                  , sizeof(hpx::parcelset::policies::ibverbs::message)
                  , IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE
                );
                if(!client_msg_mr_)
                {
                    int verrno = errno;
                    boost::system::error_code err(verrno, boost::system::system_category());
                    HPX_IBVERBS_THROWS_IF(
                        ec
                      , err
                    );
                }

                id_ = id;
            }

            //post_receive();
        }

int BClientContext::register_memory() {
	int mr_flags = 
		IBV_ACCESS_LOCAL_WRITE
			| IBV_ACCESS_REMOTE_READ
				| IBV_ACCESS_REMOTE_WRITE;
	
	TEST_Z(recv_memory_mr		= ibv_reg_mr(pd, &recv_memory_msg, sizeof(struct BMemoryKeys), mr_flags));
	TEST_Z(lock_result_mr			= ibv_reg_mr(pd, &lock_result, sizeof(uint64_t), mr_flags));
	
	return 0;
}

void vbuf_reregister_all()
{
    int i = 0;
    vbuf_region *vr = vbuf_region_head;

    MPIDI_STATE_DECL(MPID_STATE_VBUF_REREGISTER_ALL);
    MPIDI_FUNC_ENTER(MPID_STATE_VBUF_REREGISTER_ALL);

    for (; i < rdma_num_hcas; ++i)
    {
        ptag_save[i] = MPIDI_CH3I_RDMA_Process.ptag[i];
    }

    while (vr)
    {
        for (i = 0; i < rdma_num_hcas; ++i)
        {
            vr->mem_handle[i] = ibv_reg_mr(
                ptag_save[i],
                vr->malloc_buf_start,
                vr->count * rdma_vbuf_total_size,
                IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);

            if (!vr->mem_handle[i])
            {
                ibv_error_abort(IBV_RETURN_ERR,"Cannot reregister vbuf region\n");
            }
        }

        vr = vr->next;
    }

    MPIDI_FUNC_EXIT(MPID_STATE_VBUF_REREGISTER_ALL);
}

/**
 * Sends a buffer's memory region so that it can be mapped to it's remote end.
 */
void RDMAChannel::SendMR(ibv_mr* mr, int id) {
  // Map the memory region itself so that it can be sent
  ibv_mr* init = ibv_reg_mr(adapter_.pd_, mr, sizeof(ibv_mr),
                            IBV_ACCESS_LOCAL_WRITE);

  struct ibv_sge list;
  list.addr = (uint64_t) mr;
  list.length = sizeof(ibv_mr);
  list.lkey = init->lkey;

  struct ibv_send_wr wr;
  caffe_memset(sizeof(wr), 0, &wr);
  wr.wr_id = (uint64_t) init;
  wr.sg_list = &list;
  wr.num_sge = 1;
  wr.opcode = IBV_WR_SEND_WITH_IMM;
  wr.send_flags = IBV_SEND_SIGNALED;
  wr.imm_data = id;

  struct ibv_send_wr *bad_wr;
  CHECK(!ibv_post_send(qp_, &wr, &bad_wr));

  for (;;) {
    ibv_wc wc;
    int ne = ibv_poll_cq(write_cq_, 1, &wc);
    CHECK_GE(ne, 0);
    if (ne && wc.wr_id == (uint64_t) init) {
      break;
    }
  }
  CHECK(!ibv_dereg_mr(init));
}

int m_pi_create_wc_q(struct mcm_qp *m_qp, int entries)
{
	/* RDMA proxy WC pool, register with SCIF and IB, set pool and segm size with parameters */
	m_qp->wrc.wc_sz = ALIGN_64(sizeof(struct mcm_wc_rx));
	m_qp->wrc.wc_len = m_qp->wrc.wc_sz * entries; /* 64 byte aligned for signal_fence */
	m_qp->wrc.wc_end = entries - 1;
	m_qp->wc_hd_rem = 0;
	m_qp->wc_tl_rem = 0;

	if (posix_memalign((void **)&m_qp->wrc.wc_addr, 4096, ALIGN_PAGE(m_qp->wrc.wc_len))) {
		mlog(0, "failed to allocate wc_rbuf, m_qp=%p, wc_len=%d, entries=%d\n",
			m_qp, m_qp->wrc.wc_len, entries);
		return -1;
	}
	memset((void*)m_qp->wrc.wc_addr, 0, ALIGN_PAGE(m_qp->wrc.wc_len));

	mlog(4, " WC rbuf pool %p, LEN req=%d, act=%d\n",
		m_qp->wrc.wc_addr, m_qp->wrc.wc_len, ALIGN_PAGE(m_qp->wrc.wc_len));

	m_qp->wc_rbuf_mr = ibv_reg_mr(m_qp->smd->md->pd, (void*)m_qp->wrc.wc_addr, m_qp->wrc.wc_len,
				      IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
	if (!m_qp->wc_rbuf_mr) {
		mlog(0, " IB_register addr=%p,%d failed %s\n",
		        m_qp->wrc.wc_addr, ALIGN_PAGE(m_qp->wrc.wc_len), strerror(errno));
		return -1;
	}
	m_qp->wrc.wc_addr = (uint64_t)(uintptr_t)m_qp->wc_rbuf_mr->addr;
	m_qp->wrc.wc_rkey = m_qp->wc_rbuf_mr->rkey;

	mlog(4, " IB_mr for wc_buf addr %p, mr 0x%llx, len %d, entries %d rkey %x lkey %x\n",
		m_qp->wrc.wc_addr, m_qp->wc_rbuf_mr->addr, ALIGN_PAGE(m_qp->wrc.wc_len),
		entries, m_qp->wc_rbuf_mr->rkey, m_qp->wc_rbuf_mr->lkey);

	return 0;
}

static int create_message(struct cmatest_node *node)
{
	if (!message_size)
		message_count = 0;

	if (!message_count)
		return 0;

	node->mem = malloc(message_size + sizeof(struct ibv_grh));
	if (!node->mem) {
		printf("failed message allocation\n");
		return -1;
	}
	node->mr = ibv_reg_mr(node->pd, node->mem,
			      message_size + sizeof(struct ibv_grh),
			      IBV_ACCESS_LOCAL_WRITE);
	if (!node->mr) {
		printf("failed to reg MR\n");
		goto err;
	}
	return 0;
err:
	free(node->mem);
	return -1;
}

static
int psoib_vapi_alloc(hca_info_t *hca_info, int size, enum ibv_access_flags access_perm, mem_info_t *mem_info)
{
    mem_info->mr = NULL;

    /* Region for buffers */
    mem_info->ptr = valloc(size);
    if (!mem_info->ptr) goto err_malloc;

//    printf("ibv_reg_mr(pd = %p, ptr = %p, size = %d, access_perm = 0x%x)\n",
//	   hca_info->pd, mem_info->ptr, size, access_perm);

    mem_info->mr = ibv_reg_mr(hca_info->pd, mem_info->ptr, size, access_perm);
    if (!mem_info->mr) goto err_reg_mr;

    return 0;
    /* --- */
 err_reg_mr:
    free(mem_info->ptr);
    mem_info->ptr = NULL;
    psoib_err_errno("ibv_reg_mr() failed", errno);
    if (errno == ENOMEM) print_mlock_help(size);
    return -1;
 err_malloc:
    psoib_err_errno("malloc() failed!", errno);
    return -1;
}

static void on_pre_conn(struct rdma_cm_id *id)
{
  struct conn_context *ctx = (struct conn_context *)malloc(sizeof(struct conn_context));

  id->context = ctx;

  ctx->file_name[0] = '\0'; // take this to mean we don't have the file name

  posix_memalign((void **)&ctx->buffer, sysconf(_SC_PAGESIZE), BUFFER_SIZE);
  TEST_Z(ctx->buffer_mr = ibv_reg_mr(rc_get_pd(), ctx->buffer, BUFFER_SIZE, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE));

  posix_memalign((void **)&ctx->msg, sysconf(_SC_PAGESIZE), sizeof(*ctx->msg));
  TEST_Z(ctx->msg_mr = ibv_reg_mr(rc_get_pd(), ctx->msg, sizeof(*ctx->msg), IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE));

  post_receive(id);
}

/**
 * Register mempool as a memory region.
 *
 * @param pd
 *   Pointer to protection domain.
 * @param mp
 *   Pointer to memory pool.
 *
 * @return
 *   Memory region pointer, NULL in case of error.
 */
struct ibv_mr *
mlx5_mp2mr(struct ibv_pd *pd, const struct rte_mempool *mp)
{
	const struct rte_memseg *ms = rte_eal_get_physmem_layout();
	uintptr_t start = mp->elt_va_start;
	uintptr_t end = mp->elt_va_end;
	unsigned int i;

	DEBUG("mempool %p area start=%p end=%p size=%zu",
	      (const void *)mp, (void *)start, (void *)end,
	      (size_t)(end - start));
	/* Round start and end to page boundary if found in memory segments. */
	for (i = 0; (i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL); ++i) {
		uintptr_t addr = (uintptr_t)ms[i].addr;
		size_t len = ms[i].len;
		unsigned int align = ms[i].hugepage_sz;

		if ((start > addr) && (start < addr + len))
			start = RTE_ALIGN_FLOOR(start, align);
		if ((end > addr) && (end < addr + len))
			end = RTE_ALIGN_CEIL(end, align);
	}
	DEBUG("mempool %p using start=%p end=%p size=%zu for MR",
	      (const void *)mp, (void *)start, (void *)end,
	      (size_t)(end - start));
	return ibv_reg_mr(pd,
			  (void *)start,
			  end - start,
			  IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
}

int xfer_rdma_register_buffer(struct xfer_context *ctx, struct xfer_rdma_buf_handle_t *handle)
{
	/* We dont really want IBV_ACCESS_LOCAL_WRITE, but IB spec says:
	 * The Consumer is not allowed to assign Remote Write or Remote Atomic to
	 * a Memory Region that has not been assigned Local Write. */
	handle->local_mr = ibv_reg_mr(ctx->pd, handle->buf, handle->local_size,
				      IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE |
				      IBV_ACCESS_REMOTE_READ);
	if (!handle->local_mr) {
		fprintf(stderr, "%d:%s: Couldn't allocate MR\n", pid, __func__);
		return -1;
	}

	handle->id = (uintptr_t) handle->buf;
	handle->got_done = 0;
	handle->ctx = ctx;

	handle->remote_mr = malloc(sizeof(struct ibv_mr));
	if (!handle->remote_mr) {
		fprintf(stderr, "%d:%s: could not malloc remote_mr\n", pid, __func__);
		return -1;
	}

	return 0;
}