C++ rdma_create_id函数代码示例

/* create a portal, after listening starts all events
 * are received in isert_cm_evt_handler()
 */
struct isert_portal *isert_portal_create(void)
{
	struct isert_portal *portal;
	struct rdma_cm_id *cm_id;
	int err;

	if (unlikely(!try_module_get(THIS_MODULE))) {
		pr_err("Unable increment module reference\n");
		portal = ERR_PTR(-EINVAL);
		goto out;
	}

	portal = kzalloc(sizeof(*portal), GFP_KERNEL);
	if (unlikely(!portal)) {
		pr_err("Unable to allocate struct portal\n");
		portal = ERR_PTR(-ENOMEM);
		goto err_alloc;
	}

#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0) && \
	(!defined(RHEL_MAJOR) || RHEL_MAJOR -0 <= 5)
	cm_id = rdma_create_id(isert_cm_evt_handler, portal, RDMA_PS_TCP);
#else
	cm_id = rdma_create_id(isert_cm_evt_handler, portal, RDMA_PS_TCP,
			       IB_QPT_RC);
#endif
	if (unlikely(IS_ERR(cm_id))) {
		err = PTR_ERR(cm_id);
		pr_err("Failed to create rdma id, err:%d\n", err);
		goto create_id_err;
	}
	portal->cm_id = cm_id;

	INIT_LIST_HEAD(&portal->conn_list);
	isert_portal_list_add(portal);

#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0)
	rdma_set_afonly(cm_id, 1);
#endif

	pr_info("Created iser portal cm_id:%p\n", cm_id);
out:
	return portal;

create_id_err:
	kfree(portal);
	portal = ERR_PTR(err);
err_alloc:
	module_put(THIS_MODULE);
	goto out;
}

int ibrdma_transfer(struct transfer_info *tfi, int num_tfi) {
  struct addrinfo *addr;
  struct rdma_cm_id *cmid= NULL;
  struct rdma_event_channel *ec = NULL;
  struct rdma_conn_param cm_params;
  TEST_NZ(getaddrinfo(host, port, NULL, &addr));
  TEST_Z(ec = rdma_create_event_channel());
  TEST_NZ(rdma_create_id(ec, &cmid, NULL, RDMA_PS_TCP));
  TEST_NZ(rdma_resolve_addr(cmid, NULL, addr->ai_addr, TIMEOUT_IN_MS));
  TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ADDR_RESOLVED));
  freeaddrinfo(addr);
  build_connection(cmid);
  TEST_NZ(rdma_resolve_route(cmid, TIMEOUT_IN_MS));
  TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ROUTE_RESOLVED));
  build_params(&cm_params);
  TEST_NZ(rdma_connect(cmid, &cm_params));
  TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ESTABLISHED));
  on_connect(cmid->context);
  


  TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_DISCONNECTED));
  rdma_destroy_id(&cmid);
  rdma_destroy_event_channel(&ec);

  return 0;
}

int main(int argc, char **argv)
{
	struct sockaddr_in addr;
	struct rdma_cm_event *event = NULL;
	struct rdma_cm_id *listener = NULL;
	struct rdma_event_channel *ec = NULL;
	uint16_t port = 0;

	memset(&addr, 0, sizeof(addr));
	addr.sin_family = AF_INET;

	TEST_Z(ec = rdma_create_event_channel());
	TEST_NZ(rdma_create_id(ec, &listener, NULL, RDMA_PS_TCP));
	TEST_NZ(rdma_bind_addr(listener, (struct sockaddr *)&addr));
	TEST_NZ(rdma_listen(listener, 10)); /* backlog=10 is arbitrary */

	port = ntohs(rdma_get_src_port(listener));

	printf("listening on port %d.\n", port);

	while (rdma_get_cm_event(ec, &event) == 0) {
		struct rdma_cm_event event_copy;

		memcpy(&event_copy, event, sizeof(*event));
		rdma_ack_cm_event(event);

		if (on_event(&event_copy))
			break;
	}

	rdma_destroy_id(listener);
	rdma_destroy_event_channel(ec);

	return 0;
}

int xfer_rdma_init(struct xfer_data *data)
{
        int duplex = 0;
	
	/* Get the PID and prepend it to every output on stdout/stderr
         * This helps to parse output when multiple client/server are
         * run from single host
         */
        pid = getpid();

        printf("%d: | port=%d | ib_port=%d | tx_depth=%d | sl=%d | duplex=%d | cma=%d |\n",
	       pid, data->port, data->ib_port, data->tx_depth, sl, duplex, data->use_cma);

        srand48(pid * time(NULL));
        page_size = sysconf(_SC_PAGESIZE);

        if (data->use_cma) {
		data->cm_channel = rdma_create_event_channel();
                if (!data->cm_channel) {
                        fprintf(stderr, "%d:%s: rdma_create_event_channel failed\n",
				pid, __func__);
                        return -1;
                }
                if (rdma_create_id(data->cm_channel, &data->cm_id, NULL, RDMA_PS_TCP)) {
                        fprintf(stderr, "%d:%s: rdma_create_id failed\n",
				pid, __func__);
                        return -1;
                }
        } else {
		// use an alternative to CMA here
	}
	
	return 0;
}

//static int run(int argc, char **argv)
int ibrdma_send(char* host, char* port, void* data, uint64_t size)
{
  
  struct addrinfo *addr;
  struct rdma_cm_id *cmid= NULL;
  struct rdma_event_channel *ec = NULL;
  struct rdma_conn_param cm_params;
  TEST_NZ(getaddrinfo(host, port, NULL, &addr));
  TEST_Z(ec = rdma_create_event_channel());
  TEST_NZ(rdma_create_id(ec, &cmid, NULL, RDMA_PS_TCP));
  TEST_NZ(rdma_resolve_addr(cmid, NULL, addr->ai_addr, TIMEOUT_IN_MS));
  TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ADDR_RESOLVED));
  freeaddrinfo(addr);
  build_connection(cmid);
  TEST_NZ(rdma_resolve_route(cmid, TIMEOUT_IN_MS));
  TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ROUTE_RESOLVED));
  build_params(&cm_params);
  TEST_NZ(rdma_connect(cmid, &cm_params));
  TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ESTABLISHED));
  on_connect(cmid->context);

  /* Init MSG send to start RDMA*/
  init_tfile(data,  size);
  send_init(cmid->context);
  /*----------------------------*/

  TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_DISCONNECTED));
  rdma_destroy_id(cmid);
  rdma_destroy_event_channel(ec);

  return 0;
}

//static int run(int argc, char **argv)
//int RDMA_Connect(struct RDMA_communicator *comm, struct RDMA_param *param)
int RDMA_Active_Init(struct RDMA_communicator *comm, struct RDMA_param *param)
{
  struct addrinfo *addr;
  //  struct rdma_cm_id *cm_id= NULL;
  //  struct rdma_event_channel *ec = NULL;
  struct rdma_conn_param cm_params;
  char port[8];
  //  int i,j;

  sprintf(port, "%d", RDMA_PORT);

  TEST_NZ(getaddrinfo(param->host, port, NULL, &addr));

  TEST_Z(comm->ec = rdma_create_event_channel());

  TEST_NZ(rdma_create_id(comm->ec, &(comm->cm_id), NULL, RDMA_PS_TCP));
  TEST_NZ(rdma_resolve_addr(comm->cm_id, NULL, addr->ai_addr, TIMEOUT_IN_MS));
  TEST_NZ(wait_for_event(comm->ec, RDMA_CM_EVENT_ADDR_RESOLVED));
  freeaddrinfo(addr);

  build_connection(comm->cm_id);
  TEST_NZ(rdma_resolve_route(comm->cm_id, TIMEOUT_IN_MS));
  TEST_NZ(wait_for_event(comm->ec, RDMA_CM_EVENT_ROUTE_RESOLVED));
  build_params(&cm_params);
  TEST_NZ(rdma_connect(comm->cm_id, &cm_params));
  TEST_NZ(wait_for_event(comm->ec, RDMA_CM_EVENT_ESTABLISHED));
  //  on_connect(cm_id->context);

  return 0;
}

int main(int argc, char **argv)
{
  struct addrinfo *addr;
  struct rdma_cm_event *event = NULL;
  struct rdma_cm_id *conn= NULL;
  struct rdma_event_channel *ec = NULL;

  if (argc != 3)
    die("usage: client <server-address> <server-port>");

  TEST_NZ(getaddrinfo(argv[1], argv[2], NULL, &addr));

  TEST_Z(ec = rdma_create_event_channel());
  TEST_NZ(rdma_create_id(ec, &conn, NULL, RDMA_PS_TCP));
  TEST_NZ(rdma_resolve_addr(conn, NULL, addr->ai_addr, TIMEOUT_IN_MS));

  freeaddrinfo(addr);

  while (rdma_get_cm_event(ec, &event) == 0) {
    struct rdma_cm_event event_copy;

    memcpy(&event_copy, event, sizeof(*event));
    rdma_ack_cm_event(event);

    if (on_event(&event_copy))
      break;
  }

  rdma_destroy_event_channel(ec);

  return 0;
}

 /**
 * starts the process of connecting to the target
 * sleeps untill the connection is established or rejected
 */
int iser_connect(struct iser_conn   *ib_conn,
		 struct sockaddr_in *src_addr,
		 struct sockaddr_in *dst_addr,
		 int                 non_blocking)
{
	struct sockaddr *src, *dst;
	int err = 0;

	sprintf(ib_conn->name, "%pI4:%d",
		&dst_addr->sin_addr.s_addr, dst_addr->sin_port);

	/* the device is known only --after-- address resolution */
	ib_conn->device = NULL;

	iser_err("connecting to: %pI4, port 0x%x\n",
		 &dst_addr->sin_addr, dst_addr->sin_port);

	ib_conn->state = ISER_CONN_PENDING;

	iser_conn_get(ib_conn); /* ref ib conn's cma id */
	ib_conn->cma_id = rdma_create_id(iser_cma_handler,
					     (void *)ib_conn,
					     RDMA_PS_TCP);
	if (IS_ERR(ib_conn->cma_id)) {
		err = PTR_ERR(ib_conn->cma_id);
		iser_err("rdma_create_id failed: %d\n", err);
		goto id_failure;
	}

	src = (struct sockaddr *)src_addr;
	dst = (struct sockaddr *)dst_addr;
	err = rdma_resolve_addr(ib_conn->cma_id, src, dst, 1000);
	if (err) {
		iser_err("rdma_resolve_addr failed: %d\n", err);
		goto addr_failure;
	}

	if (!non_blocking) {
		wait_event_interruptible(ib_conn->wait,
					 (ib_conn->state != ISER_CONN_PENDING));

		if (ib_conn->state != ISER_CONN_UP) {
			err =  -EIO;
			goto connect_failure;
		}
	}

	mutex_lock(&ig.connlist_mutex);
	list_add(&ib_conn->conn_list, &ig.connlist);
	mutex_unlock(&ig.connlist_mutex);
	return 0;

id_failure:
	ib_conn->cma_id = NULL;
addr_failure:
	ib_conn->state = ISER_CONN_DOWN;
connect_failure:
	iser_conn_release(ib_conn, 1);
	return err;
}

static int alloc_nodes(void)
{
	int ret, i;

	test.nodes = malloc(sizeof *test.nodes * connections);
	if (!test.nodes) {
		printf("cmatose: unable to allocate memory for test nodes\n");
		return -ENOMEM;
	}
	memset(test.nodes, 0, sizeof *test.nodes * connections);

	for (i = 0; i < connections; i++) {
		test.nodes[i].id = i;
		if (dst_addr) {
			ret = rdma_create_id(test.channel,
					     &test.nodes[i].cma_id,
					     &test.nodes[i], hints.ai_port_space);
			if (ret)
				goto err;
		}
	}
	return 0;
err:
	while (--i >= 0)
		rdma_destroy_id(test.nodes[i].cma_id);
	free(test.nodes);
	return ret;
}

static int fi_ibv_pep_setname(fid_t pep_fid, void *addr, size_t addrlen)
{
	struct fi_ibv_pep *pep;
	int ret;

	pep = container_of(pep_fid, struct fi_ibv_pep, pep_fid);

	if (pep->src_addrlen && (addrlen != pep->src_addrlen)) {
		FI_INFO(&fi_ibv_prov, FI_LOG_FABRIC, "addrlen expected: %d, got: %d.\n",
				pep->src_addrlen, addrlen);
		return -FI_EINVAL;
	}

	/* Re-create id if already bound */
	if (pep->bound) {
		ret = rdma_destroy_id(pep->id);
		if (ret) {
			FI_INFO(&fi_ibv_prov, FI_LOG_FABRIC, "Unable to destroy previous rdma_cm_id\n");
			return -errno;
		}
		ret = rdma_create_id(NULL, &pep->id, NULL, RDMA_PS_TCP);
		if (ret) {
			FI_INFO(&fi_ibv_prov, FI_LOG_FABRIC, "Unable to create rdma_cm_id\n");
			return -errno;
		}
	}

	ret = rdma_bind_addr(pep->id, (struct sockaddr *)addr);
	if (ret) {
		FI_INFO(&fi_ibv_prov, FI_LOG_FABRIC, "Unable to bind addres to rdma_cm_id\n");
		return -errno;
	}

	return 0;
}

int ibw_connect(struct ibw_conn *conn, struct sockaddr_in *serv_addr, void *conn_userdata)
{
	struct ibw_ctx_priv *pctx = talloc_get_type(conn->ctx->internal, struct ibw_ctx_priv);
	struct ibw_conn_priv *pconn = NULL;
	int	rc;

	assert(conn!=NULL);

	conn->conn_userdata = conn_userdata;
	pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
	DEBUG(DEBUG_DEBUG, ("ibw_connect: addr=%s, port=%u\n", inet_ntoa(serv_addr->sin_addr),
		ntohs(serv_addr->sin_port)));

	/* clean previous - probably half - initialization */
	if (ibw_conn_priv_destruct(pconn)) {
		DEBUG(DEBUG_ERR, ("ibw_connect/ibw_pconn_destruct failed for cm_id=%p\n", pconn->cm_id));
		return -1;
	}

	/* init cm */
#if RDMA_USER_CM_MAX_ABI_VERSION >= 2
	rc = rdma_create_id(pctx->cm_channel, &pconn->cm_id, conn, RDMA_PS_TCP);
#else
	rc = rdma_create_id(pctx->cm_channel, &pconn->cm_id, conn);
#endif
	if (rc) {
		rc = errno;
		sprintf(ibw_lasterr, "ibw_connect/rdma_create_id error %d\n", rc);
		talloc_free(conn);
		return -1;
	}
	DEBUG(DEBUG_DEBUG, ("ibw_connect: rdma_create_id succeeded, cm_id=%p\n", pconn->cm_id));

	rc = rdma_resolve_addr(pconn->cm_id, NULL, (struct sockaddr *) serv_addr, 2000);
	if (rc) {
		sprintf(ibw_lasterr, "rdma_resolve_addr error %d\n", rc);
		DEBUG(DEBUG_ERR, (ibw_lasterr));
		talloc_free(conn);
		return -1;
	}

	/* continued at RDMA_CM_EVENT_ADDR_RESOLVED */

	return 0;
}

/***************************************************************************//**
 * Connection server
 *
 ******************************************************************************/
static struct rdma_conn *
build_connection(struct thread_context *ctx) {
    struct rdma_conn *c = calloc(1, sizeof(struct rdma_conn));

    if (0 != rdma_create_id(NULL, &c->id, c, RDMA_PS_TCP)) {
        perror("rdma_create_id()");
        return NULL;
    }
    struct rdma_addrinfo    hints = { .ai_port_space = RDMA_PS_TCP },

/*
 * Create a listening RDMA service endpoint.
 */
static struct svc_xprt *svc_rdma_create(struct svc_serv *serv,
					struct net *net,
					struct sockaddr *sa, int salen,
					int flags)
{
	struct rdma_cm_id *listen_id;
	struct svcxprt_rdma *cma_xprt;
	struct svc_xprt *xprt;
	int ret;

	dprintk("svcrdma: Creating RDMA socket\n");
	if (sa->sa_family != AF_INET) {
		dprintk("svcrdma: Address family %d is not supported.\n", sa->sa_family);
		return ERR_PTR(-EAFNOSUPPORT);
	}
	cma_xprt = rdma_create_xprt(serv, 1);
	if (!cma_xprt)
		return ERR_PTR(-ENOMEM);
	xprt = &cma_xprt->sc_xprt;

	listen_id = rdma_create_id(rdma_listen_handler, cma_xprt, RDMA_PS_TCP,
				   IB_QPT_RC);
	if (IS_ERR(listen_id)) {
		ret = PTR_ERR(listen_id);
		dprintk("svcrdma: rdma_create_id failed = %d\n", ret);
		goto err0;
	}

	ret = rdma_bind_addr(listen_id, sa);
	if (ret) {
		dprintk("svcrdma: rdma_bind_addr failed = %d\n", ret);
		goto err1;
	}
	cma_xprt->sc_cm_id = listen_id;

	ret = rdma_listen(listen_id, RPCRDMA_LISTEN_BACKLOG);
	if (ret) {
		dprintk("svcrdma: rdma_listen failed = %d\n", ret);
		goto err1;
	}

	/*
	 * We need to use the address from the cm_id in case the
	 * caller specified 0 for the port number.
	 */
	sa = (struct sockaddr *)&cma_xprt->sc_cm_id->route.addr.src_addr;
	svc_xprt_set_local(&cma_xprt->sc_xprt, sa, salen);

	return &cma_xprt->sc_xprt;

 err1:
	rdma_destroy_id(listen_id);
 err0:
	kfree(cma_xprt);
	return ERR_PTR(ret);
}

static int fi_ibv_rdm_cm_init(struct fi_ibv_rdm_cm* cm,
			      const struct rdma_addrinfo* rai)
{
	struct sockaddr_in* src_addr = (struct sockaddr_in*)rai->ai_src_addr;
	cm->ec = rdma_create_event_channel();

	if (!cm->ec) {
		VERBS_INFO(FI_LOG_EP_CTRL,
			"Failed to create listener event channel: %s\n",
			strerror(errno));
		return -FI_EOTHER;
	}

	if (fi_fd_nonblock(cm->ec->fd) != 0) {
		VERBS_INFO_ERRNO(FI_LOG_EP_CTRL, "fcntl", errno);
		return -FI_EOTHER;
	}

	if (rdma_create_id(cm->ec, &cm->listener, NULL, RDMA_PS_TCP)) {
		VERBS_INFO(FI_LOG_EP_CTRL, "Failed to create cm listener: %s\n",
			     strerror(errno));
		return -FI_EOTHER;
	}

	if (fi_ibv_rdm_find_ipoib_addr(src_addr, cm)) {
		VERBS_INFO(FI_LOG_EP_CTRL, 
			   "Failed to find correct IPoIB address\n");
		return -FI_ENODEV;
	}

	cm->my_addr.sin_port = src_addr->sin_port;

	char my_ipoib_addr_str[INET6_ADDRSTRLEN];
	inet_ntop(cm->my_addr.sin_family,
		  &cm->my_addr.sin_addr.s_addr,
		  my_ipoib_addr_str, INET_ADDRSTRLEN);

	VERBS_INFO(FI_LOG_EP_CTRL, "My IPoIB: %s\n", my_ipoib_addr_str);

	if (rdma_bind_addr(cm->listener, (struct sockaddr *)&cm->my_addr)) {
		VERBS_INFO(FI_LOG_EP_CTRL,
			"Failed to bind cm listener to my IPoIB addr %s: %s\n",
			my_ipoib_addr_str, strerror(errno));
		return -FI_EOTHER;
	}

	if (!cm->my_addr.sin_port) {
		cm->my_addr.sin_port = rdma_get_src_port(cm->listener);
	}
	assert(cm->my_addr.sin_family == AF_INET);

	VERBS_INFO(FI_LOG_EP_CTRL, "My ep_addr: %s:%u\n",
		inet_ntoa(cm->my_addr.sin_addr), ntohs(cm->my_addr.sin_port));

	return FI_SUCCESS;
}

static int create_rdma(void *ctx)
{

  //  struct addrinfo *addr;
  struct rdma_cm_event *event = NULL;
  struct rdma_cm_id *id = NULL;
  rdma_cm_event_handler event_handler = NULL;

  /*
  if (strcmp(argv[1], "write") == 0)
    set_mode(M_WRITE);
  else if (strcmp(argv[1], "read") == 0)
    set_mode(M_READ);
  else
    usage(argv[0]);
  */


 // TEST_NZ(getaddrinfo(s_ip, s_port, NULL, &addr));
  struct sockaddr_in addr = { 0 };
  addr.sin_family = AF_INET;
  addr.sin_addr.s_addr = htonl(ip);
  addr.sin_port = htons(port);


  //TEST_Z(ec = rdma_create_event_channel()); 
  TEST_NZ( id = rdma_create_id( event_handler, NULL, RDMA_PS_TCP, IB_QPT_RC ) ); //WHAT QP?
  TEST_NZ( rdma_resolve_addr(id, NULL, (struct sockaddr*) &addr, TIMEOUT_IN_MS) );

//  kfree(addr);


  time_stamp(0);


  while ( event_handler(id, event) == 0) {
    struct rdma_cm_event event_copy;

    memcpy(&event_copy, event, sizeof(*event));
//    rdma_ack_cm_event(event);


    if (on_event(id, &event_copy))
      break;
  }

  rdma_destroy_id(id);

  time_stamp(9);
  time_calculate();
  return 0;
}