C++ rdma_disconnect函数代码示例

RDMAServerSocket::~RDMAServerSocket() {
  running = false;
  for (auto &&id : ids) {
    rdma_disconnect(id.get());
  }
  rdma_disconnect(id.get());
  // should cause an event to be fired, but doesn't.
  // use epoll w/ timeouts instead
  id.reset();
}

static int fio_rdmaio_close_file(struct thread_data *td, struct fio_file *f)
{
	struct rdmaio_data *rd = td->io_ops->data;
	struct ibv_send_wr *bad_wr;

	/* unregister rdma buffer */

	/*
	 * Client sends notification to the server side
	 */
	/* refer to: http://linux.die.net/man/7/rdma_cm */
	if ((rd->is_client == 1) && ((rd->rdma_protocol == FIO_RDMA_MEM_WRITE)
				     || (rd->rdma_protocol ==
					 FIO_RDMA_MEM_READ))) {
		if (ibv_post_send(rd->qp, &rd->sq_wr, &bad_wr) != 0) {
			log_err("fio: ibv_post_send fail");
			return 1;
		}

		dprint(FD_IO, "fio: close information sent success\n");
		rdma_poll_wait(td, IBV_WC_SEND);
	}

	if (rd->is_client == 1)
		rdma_disconnect(rd->cm_id);
	else {
		rdma_disconnect(rd->child_cm_id);
#if 0
		rdma_disconnect(rd->cm_id);
#endif
	}

#if 0
	if (get_next_channel_event(td, rd->cm_channel, RDMA_CM_EVENT_DISCONNECTED) != 0) {
		log_err("fio: wait for RDMA_CM_EVENT_DISCONNECTED\n");
		return 1;
	}
#endif

	ibv_destroy_cq(rd->cq);
	ibv_destroy_qp(rd->qp);

	if (rd->is_client == 1)
		rdma_destroy_id(rd->cm_id);
	else {
		rdma_destroy_id(rd->child_cm_id);
		rdma_destroy_id(rd->cm_id);
	}

	ibv_destroy_comp_channel(rd->channel);
	ibv_dealloc_pd(rd->pd);

	return 0;
}

void client_disconnect(void){

    struct rdma_event_channel *ec = s_ctx->ec;
    struct rdma_cm_id *id = s_ctx->id;
    struct rdma_cm_event *event = NULL;
    struct timeval start, end, dt;

        gettimeofday(&start, NULL);
        printf("ready to disconnect\n");
        rdma_disconnect(id);
        printf("send disconnect\n");
        gettimeofday(&end, NULL);
        timersub(&end, &start, &dt); 
        long usec = dt.tv_usec + 1000000 * dt.tv_sec;
        printf("[rdma_disconnect] takes %ld micro_secs.\n", usec);

    while(1){
        if(rdma_get_cm_event(ec, &event) == 0) {
            struct rdma_cm_event event_copy;
            memcpy(&event_copy, event, sizeof(*event));
            rdma_ack_cm_event(event);
    
            if (event_copy.event == RDMA_CM_EVENT_DISCONNECTED){
                on_disconnect(event_copy.id);
                rdma_destroy_event_channel(ec);
                break;
            }
        }
    
    }

  return;

}

void isert_conn_disconnect(struct isert_connection *isert_conn)
{
	int err = rdma_disconnect(isert_conn->cm_id);

	if (unlikely(err))
		pr_err("Failed to rdma disconnect, err:%d\n", err);
}

static int rping_run_server(struct rping_cb *cb)
{
	struct ibv_recv_wr *bad_wr;
	int ret;

	ret = rping_bind_server(cb);
	if (ret)
		return ret;

	sem_wait(&cb->sem);
	if (cb->state != CONNECT_REQUEST) {
		fprintf(stderr, "wait for CONNECT_REQUEST state %d\n",
			cb->state);
		return -1;
	}

	ret = rping_setup_qp(cb, cb->child_cm_id);
	if (ret) {
		fprintf(stderr, "setup_qp failed: %d\n", ret);
		return ret;
	}

	ret = rping_setup_buffers(cb);
	if (ret) {
		fprintf(stderr, "rping_setup_buffers failed: %d\n", ret);
		goto err1;
	}

	ret = ibv_post_recv(cb->qp, &cb->rq_wr, &bad_wr);
	if (ret) {
		fprintf(stderr, "ibv_post_recv failed: %d\n", ret);
		goto err2;
	}

	pthread_create(&cb->cqthread, NULL, cq_thread, cb);

	ret = rping_accept(cb);
	if (ret) {
		fprintf(stderr, "connect error %d\n", ret);
		goto err2;
	}

	ret = rping_test_server(cb);
	if (ret) {
		fprintf(stderr, "rping server failed: %d\n", ret);
		goto err3;
	}

	ret = 0;
err3:
	rdma_disconnect(cb->child_cm_id);
	pthread_join(cb->cqthread, NULL);
	rdma_destroy_id(cb->child_cm_id);
err2:
	rping_free_buffers(cb);
err1:
	rping_free_qp(cb);

	return ret;
}

void disconnect_conn_locked(conn_t *conn)
{
    if (conn->transport.type == CONN_TYPE_RDMA) {
        switch (conn->state) {
            case CONN_STATE_CONNECTING:
            case CONN_STATE_CONNECTED:
            case CONN_STATE_RESOLVING_ROUTE:
                conn->state = CONN_STATE_DISCONNECTING;
                if (conn->rdma.cm_id)
                    rdma_disconnect(conn->rdma.cm_id);
                break;

            case CONN_STATE_RESOLVING_ADDR:
                conn->state = CONN_STATE_DISCONNECTING;
                break;

            case CONN_STATE_DISCONNECTING:
                /* That case should not be possible because it would mean
                 * this function got called twice. */
                abort();
                break;

            case CONN_STATE_DISCONNECTED:
                break;
        }
    }
}

int ibw_disconnect(struct ibw_conn *conn)
{
	int	rc;
	struct ibw_conn_priv *pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);

	DEBUG(DEBUG_DEBUG, ("ibw_disconnect: cmid=%p\n", pconn->cm_id));

	assert(pconn!=NULL);

	switch(conn->state) {
	case IBWC_ERROR:
		ibw_conn_priv_destruct(pconn); /* do this here right now */
		break;
	case IBWC_CONNECTED:
		rc = rdma_disconnect(pconn->cm_id);
		if (rc) {
			sprintf(ibw_lasterr, "ibw_disconnect failed with %d\n", rc);
			DEBUG(DEBUG_ERR, (ibw_lasterr));
			return rc;
		}
		break;
	default:
		DEBUG(DEBUG_DEBUG, ("invalid state for disconnect: %d\n", conn->state));
		break;
	}

	return 0;
}

        void close()
        {
            util::spinlock::scoped_lock lk(mtx_);
            if(buffer_mr_)
            {
                ibv_dereg_mr(buffer_mr_);
                buffer_mr_ = 0;
            }
            if(server_msg_mr_)
            {
                ibv_dereg_mr(server_msg_mr_);
                server_msg_mr_ = 0;
            }
            if(client_msg_mr_)
            {
                ibv_dereg_mr(client_msg_mr_);
                client_msg_mr_ = 0;
            }

            if(id_)
            {
                rdma_disconnect(id_);
                id_ = 0;
            }

            size_ = 0;
        }

	// closes all connections
	inline int closeConnections() {

		for (int i=0; i<threadCounter; i++) {
			while (!cqPollerThreadEnded[i])
			 	usleep(1000);
		}

		waitSendThread();
		for (unsigned int i=0; i<connections.size(); i++)
			rdma_disconnect(connections[i]->id);

		for (unsigned int i=0; i<connections.size(); i++)
			waitForEvent(connections[i]->id, RDMA_CM_EVENT_DISCONNECTED, connections[i]->getContext());

		for(unsigned i=0;i<connections.size();++i) {
			connections[i]->destroyConnection();

			delete connections[i];
			connections[i]=NULL;
		}

		for (int i=0; i<threadCounter; i++)
			free(contexts[i]);

		return 0;
	}

static void
rdma_trans_destroy(void *a)
{
	Rdmatrans *rdma;
	struct ibv_qp_attr attr;

	rdma = a;
	if (rdma->connected)
		rdma_disconnect(rdma->cm_id);
	if (rdma->qp) {
		attr.qp_state = IBV_QPS_ERR;
		ibv_modify_qp(rdma->qp, &attr, IBV_QP_STATE);
		ibv_destroy_qp(rdma->qp);
	}
	if (rdma->cq)
		ibv_destroy_cq(rdma->cq);
	if (rdma->ch)
		ibv_destroy_comp_channel(rdma->ch);
	if (rdma->snd_mr)
		ibv_dereg_mr(rdma->snd_mr);
	if (rdma->snd_buf)
		free(rdma->snd_buf);
	if (rdma->rcv_mr)
		ibv_dereg_mr(rdma->rcv_mr);
	if (rdma->rcv_buf)
		free(rdma->rcv_buf);
	if (rdma->pd)
		ibv_dealloc_pd(rdma->pd);
	if (rdma->cm_id)
		rdma_destroy_id(rdma->cm_id);
}

RDMACMSocket::~RDMACMSocket() {
    
    rdma_disconnect(this->client_id);
    rdma_dereg_mr(this->verbs_mr);
    rdma_destroy_ep(this->client_id);
    this->verbs_buf.free();
}

int xfer_rdma_finalize(struct xfer_data *data)
{
        struct rdma_cm_event *event;
        int rc;

        if (data->servername) {
                rc = rdma_disconnect(data->cm_id);
                if (rc) {
                        perror("rdma_disconnect");
                        fprintf(stderr, "%d:%s: rdma disconnect error\n", pid,
				__func__);
                        return -1;
                }
        }

        rdma_get_cm_event(data->cm_channel, &event);
        if (event->event != RDMA_CM_EVENT_DISCONNECTED)
                fprintf(stderr, "%d:%s: unexpected event during disconnect %d\n",
                        pid, __func__, event->event);

        rdma_ack_cm_event(event);
        rdma_destroy_id(data->cm_id);
        rdma_destroy_event_channel(data->cm_channel);

	return 0;
}

void client_disconnect(void){

    struct rdma_event_channel *ec = s_ctx->ec;
    struct rdma_cm_id *id = s_ctx->id;
    struct rdma_cm_event *event = NULL;

        printf("ready to disconnect\n");
        rdma_disconnect(id);
        printf("send disconnect\n");

while(1){
    if(rdma_get_cm_event(ec, &event) == 0) {
        struct rdma_cm_event event_copy;
        memcpy(&event_copy, event, sizeof(*event));
        rdma_ack_cm_event(event);

        if (event_copy.event == RDMA_CM_EVENT_DISCONNECTED){
            on_disconnect(event_copy.id);
            rdma_destroy_event_channel(ec);
            break;
        }
    }

}

  return;

}

void RDMAServerSocket::cm_events() const {
  eventThread.send([=]() {
    hydra::util::epoll poll;
    epoll_event poll_event;
    poll_event.events = EPOLLIN;
    poll_event.data.fd = ec->fd;

    poll.add(ec->fd, &poll_event);

    while (running) {
      rdma_cm_event *cm_event = nullptr;

      int ret = poll.wait(&poll_event, 1, 2000);
      if (ret) {
        if (poll_event.data.fd == ec->fd) {
          check_zero(rdma_get_cm_event(ec.get(), &cm_event));
          check_zero(cm_event->status);
          if (cm_event->event == RDMA_CM_EVENT_CONNECT_REQUEST) {
            accept(client_t(cm_event->id));
          } else if (cm_event->event == RDMA_CM_EVENT_DISCONNECTED) {
            rdma_disconnect(cm_event->id);
          }
          check_zero(rdma_ack_cm_event(cm_event));
        } else {
          log_err() << "Unkown fd " << poll_event.data.fd << " set. Expected "
                    << id->channel->fd;
          std::terminate();
        }
      }
    }
  });
}

void on_completion(struct ibv_wc *wc)
{
  struct connection *conn = (struct connection *)(uintptr_t)wc->wr_id;

  if (wc->status != IBV_WC_SUCCESS)
    die("on_completion: status is not IBV_WC_SUCCESS.");

  if (wc->opcode & IBV_WC_RECV) {
    conn->recv_state++;
    printf("RECV: Recieved: TYPE=%d\n", conn->recv_msg->type);
    if (conn->recv_msg->type == MSG_MR) {
      memcpy(&conn->peer_mr, &conn->recv_msg->data.mr, sizeof(conn->peer_mr));
      post_receives(conn); /* only rearm for MSG_MR */
      if (conn->send_state == SS_INIT) /* received peer's MR before sending ours, so send ours back */
        send_mr(conn);
    }

  } else {
    conn->send_state++;
    printf("SEND: Sent out: TYPE=%d\n", conn->send_msg->type);
  }

  if (conn->send_state == SS_MR_SENT && conn->recv_state == RS_MR_RECV) {
    /*
    struct ibv_send_wr wr, *bad_wr = NULL;
    struct ibv_sge sge;

    if (s_mode == M_WRITE)
      printf(" -> received MSG_MR. writing message to remote memory...\n");
    else
      printf(" -> received MSG_MR. reading message from remote memory...\n");

    memset(&wr, 0, sizeof(wr));

    wr.wr_id = (uintptr_t)conn;
    wr.opcode = (s_mode == M_WRITE) ? IBV_WR_RDMA_WRITE : IBV_WR_RDMA_READ;
    wr.sg_list = &sge;
    wr.num_sge = 1;
    wr.send_flags = IBV_SEND_SIGNALED;
    wr.wr.rdma.remote_addr = (uintptr_t)conn->peer_mr.addr;
    wr.wr.rdma.rkey = conn->peer_mr.rkey;

    sge.addr = (uintptr_t)conn->rdma_local_region;
    sge.length = RDMA_BUFFER_SIZE;
    sge.lkey = conn->rdma_local_mr->lkey;

    TEST_NZ(ibv_post_send(conn->qp, &wr, &bad_wr));
    printf("PSEND: Posted send request: MSG=%s\n", conn->rdma_local_region);
    */

    conn->send_msg->type = MSG_DONE;
    send_message(conn);

  } else if (conn->send_state == SS_DONE_SENT && conn->recv_state == RS_DONE_RECV) {
    printf(" -> remote buffer: %s\n", get_peer_message_region(conn));
    rdma_disconnect(conn->id);
  }
  printf("== STATE: send=%d / recv=%d ==\n", conn->send_state, conn->recv_state);
}