Java HdfsConstants.WRITE_TIMEOUT_EXTENSION属性代码示例

/**
 * Return the timeout that clients should use when writing to datanodes.
 * @param numNodes the number of nodes in the pipeline.
 */
int getDatanodeWriteTimeout(int numNodes) {
  int confTime =
      conf.getInt("dfs.datanode.socket.write.timeout",
                  HdfsConstants.WRITE_TIMEOUT);

  return (confTime > 0) ?
    (confTime + HdfsConstants.WRITE_TIMEOUT_EXTENSION * numNodes) : 0;
}
 

/**
 * Do the deed, write the bytes
 */
public void run() {
  xmitsInProgress.getAndIncrement();
  Socket sock = null;
  DataOutputStream out = null;
  BlockSender blockSender = null;
  
  try {
    InetSocketAddress curTarget = 
      NetUtils.createSocketAddr(targets[0].getName());
    sock = newSocket();
    NetUtils.connect(sock, curTarget, socketTimeout);
    sock.setSoTimeout(targets.length * socketTimeout);

    long writeTimeout = socketWriteTimeout + 
                        HdfsConstants.WRITE_TIMEOUT_EXTENSION * (targets.length-1);
    OutputStream baseStream = NetUtils.getOutputStream(sock, writeTimeout);
    out = new DataOutputStream(new BufferedOutputStream(baseStream, 
                                                        SMALL_BUFFER_SIZE));

    blockSender = new BlockSender(b, 0, b.getNumBytes(), false, false, false, 
        datanode);
    DatanodeInfo srcNode = new DatanodeInfo(dnRegistration);

    //
    // Header info
    //
    out.writeShort(DataTransferProtocol.DATA_TRANSFER_VERSION);
    out.writeByte(DataTransferProtocol.OP_WRITE_BLOCK);
    out.writeLong(b.getBlockId());
    out.writeLong(b.getGenerationStamp());
    out.writeInt(0);           // no pipelining
    out.writeBoolean(false);   // not part of recovery
    Text.writeString(out, ""); // client
    out.writeBoolean(true); // sending src node information
    srcNode.write(out); // Write src node DatanodeInfo
    // write targets
    out.writeInt(targets.length - 1);
    for (int i = 1; i < targets.length; i++) {
      targets[i].write(out);
    }
    Token<BlockTokenIdentifier> accessToken = BlockTokenSecretManager.DUMMY_TOKEN;
    if (isBlockTokenEnabled) {
      accessToken = blockTokenSecretManager.generateToken(null, b,
          EnumSet.of(BlockTokenSecretManager.AccessMode.WRITE));
    }
    accessToken.write(out);
    // send data & checksum
    blockSender.sendBlock(out, baseStream, null);

    // no response necessary
    LOG.info(dnRegistration + ":Transmitted block " + b + " to " + curTarget);

  } catch (IOException ie) {
    LOG.warn(dnRegistration + ":Failed to transfer " + b + " to " + targets[0].getName()
        + " got " + StringUtils.stringifyException(ie));
    // check if there are any disk problem
    datanode.checkDiskError();
    
  } finally {
    xmitsInProgress.getAndDecrement();
    IOUtils.closeStream(blockSender);
    IOUtils.closeStream(out);
    IOUtils.closeSocket(sock);
  }
}
 

/**
 * Do the deed, write the bytes
 */
public void run() {
  xmitsInProgress.getAndIncrement();
  Socket sock = null;
  DataOutputStream out = null;
  BlockSender blockSender = null;
  
  try {
    InetSocketAddress curTarget = 
      NetUtils.createSocketAddr(targets[0].getName());
    sock = newSocket();
    NetUtils.connect(sock, curTarget, socketTimeout);
    sock.setSoTimeout(targets.length * socketTimeout);

    long writeTimeout = socketWriteTimeout + 
                        HdfsConstants.WRITE_TIMEOUT_EXTENSION * (targets.length-1);
    OutputStream baseStream = NetUtils.getOutputStream(sock, writeTimeout);
    out = new DataOutputStream(new BufferedOutputStream(baseStream, 
                                                        SMALL_BUFFER_SIZE));

    blockSender = new BlockSender(b, 0, b.getNumBytes(), 
        false, false, false, datanode);
    DatanodeInfo srcNode = new DatanodeInfo(dnRegistration);

    //
    // Header info
    //
    Token<BlockTokenIdentifier> accessToken = BlockTokenSecretManager.DUMMY_TOKEN;
    if (isBlockTokenEnabled) {
      accessToken = blockTokenSecretManager.generateToken(null, b,
      EnumSet.of(BlockTokenSecretManager.AccessMode.WRITE));
    }
    DataTransferProtocol.Sender.opWriteBlock(out,
        b, 0, BlockConstructionStage.PIPELINE_SETUP_CREATE, 0, 0, 0, "",
        srcNode, targets, accessToken);

    // send data & checksum
    blockSender.sendBlock(out, baseStream, null);

    // no response necessary
    LOG.info(dnRegistration + ":Transmitted block " + b + " to " + curTarget);

  } catch (IOException ie) {
    LOG.warn(dnRegistration + ":Failed to transfer " + b + " to " + targets[0].getName()
        + " got " + StringUtils.stringifyException(ie));
    // check if there are any disk problem
    datanode.checkDiskError();
    
  } finally {
    xmitsInProgress.getAndDecrement();
    IOUtils.closeStream(blockSender);
    IOUtils.closeStream(out);
    IOUtils.closeSocket(sock);
  }
}
 

/**
 * submit a tree-structured block recovery job to the root node,
 * and wait for it to complete.
 * 
 * @param treeRoot
 *        the newcomer
 * @param header
 *        recovery job header
 * @return
 *        true if succeeds, otherwise false
 */
boolean executeRepairTask(RecoverTreeNode treeRoot, 
		MergeBlockHeader header) {

	String rootHost = treeRoot.getHostName();
	Socket rootSocket = null;
	DataOutputStream rootOutput = null;
	DataInputStream rootInput = null;
	boolean returnvalue = false;
	
	try{
		InetSocketAddress rootAddr = NetUtils.createSocketAddr(rootHost);
		rootSocket = new Socket();

		/**
		 * To Do: We should multiply timeout according to children
		 * number of subnodes. I'm not sure the current way is OK.
		 */
		int timeoutValue = HdfsConstants.READ_TIMEOUT 
				+ HdfsConstants.READ_TIMEOUT_EXTENSION * 10;
		int writeTimeout = HdfsConstants.WRITE_TIMEOUT 
				+ HdfsConstants.WRITE_TIMEOUT_EXTENSION;
		NetUtils.connect(rootSocket, rootAddr, timeoutValue);
		rootSocket.setSoTimeout(timeoutValue);
		rootSocket.setSendBufferSize(FSConstants.DEFAULT_DATA_SOCKET_SIZE);
		rootOutput = new DataOutputStream(new BufferedOutputStream(
				NetUtils.getOutputStream(rootSocket, writeTimeout),
				FSConstants.SMALL_BUFFER_SIZE));
		rootInput = new DataInputStream(NetUtils.getInputStream(rootSocket));
		
		header.writeVersionAndOpCode(rootOutput);
		header.write(rootOutput);
		treeRoot.write(rootOutput);
		rootOutput.flush();
				
		// wait for the reconstruction to complete
		int status = rootInput.readInt();
		if(status < 0) {
			throw new IOException("Root node: " + rootHost 
					+ " return error status  during reconstructing block.");
		}
			
		returnvalue = true;
	} catch (IOException ioe) {
		LOG.error("NTar: executeTreeReconstructJob: error occurred during reconstructing block: " + ioe);
		returnvalue = false;
	} finally {
		IOUtils.closeStream(rootOutput);
		IOUtils.closeStream(rootInput);
		IOUtils.closeSocket(rootSocket);
	}
	return returnvalue;
}
 

private boolean createBlockOutputStream(DatanodeInfo[] nodes, String client,
                boolean recoveryFlag) {
  String firstBadLink = "";
  if (LOG.isDebugEnabled()) {
    for (int i = 0; i < nodes.length; i++) {
      LOG.debug("pipeline = " + nodes[i].getName());
    }
  }

  // persist blocks on namenode on next flush
  persistBlocks = true;

  try {
    LOG.debug("Connecting to " + nodes[0].getName());
    InetSocketAddress target = NetUtils.createSocketAddr(nodes[0].getName());
    s = socketFactory.createSocket();
    int timeoutValue = 3000 * nodes.length + socketTimeout;
    NetUtils.connect(s, target, timeoutValue);
    s.setSoTimeout(timeoutValue);
    s.setSendBufferSize(DEFAULT_DATA_SOCKET_SIZE);
    LOG.debug("Send buf size " + s.getSendBufferSize());
    long writeTimeout = HdfsConstants.WRITE_TIMEOUT_EXTENSION * nodes.length +
                        datanodeWriteTimeout;

    //
    // Xmit header info to datanode
    //
    DataOutputStream out = new DataOutputStream(
        new BufferedOutputStream(NetUtils.getOutputStream(s, writeTimeout), 
                                 DataNode.SMALL_BUFFER_SIZE));
    blockReplyStream = new DataInputStream(NetUtils.getInputStream(s));

    out.writeShort( DataTransferProtocol.DATA_TRANSFER_VERSION );
    out.write( DataTransferProtocol.OP_WRITE_BLOCK );
    out.writeLong( block.getBlockId() );
    out.writeLong( block.getGenerationStamp() );
    out.writeInt( nodes.length );
    out.writeBoolean( recoveryFlag );       // recovery flag
    Text.writeString( out, client );
    out.writeBoolean(false); // Not sending src node information
    out.writeInt( nodes.length - 1 );
    for (int i = 1; i < nodes.length; i++) {
      nodes[i].write(out);
    }
    checksum.writeHeader( out );
    out.flush();

    // receive ack for connect
    firstBadLink = Text.readString(blockReplyStream);
    if (firstBadLink.length() != 0) {
      throw new IOException("Bad connect ack with firstBadLink " + firstBadLink);
    }

    blockStream = out;
    return true;     // success

  } catch (IOException ie) {

    LOG.info("Exception in createBlockOutputStream " + ie);

    // find the datanode that matches
    if (firstBadLink.length() != 0) {
      for (int i = 0; i < nodes.length; i++) {
        if (nodes[i].getName().equals(firstBadLink)) {
          errorIndex = i;
          break;
        }
      }
    }
    hasError = true;
    setLastException(ie);
    blockReplyStream = null;
    return false;  // error
  }
}
 

/**
 * Do the deed, write the bytes
 */
public void run() {
  xmitsInProgress.getAndIncrement();
  Socket sock = null;
  DataOutputStream out = null;
  BlockSender blockSender = null;
  
  try {
    InetSocketAddress curTarget = 
      NetUtils.createSocketAddr(targets[0].getName());
    sock = newSocket();
    NetUtils.connect(sock, curTarget, socketTimeout);
    sock.setSoTimeout(targets.length * socketTimeout);

    long writeTimeout = socketWriteTimeout + 
                        HdfsConstants.WRITE_TIMEOUT_EXTENSION * (targets.length-1);
    OutputStream baseStream = NetUtils.getOutputStream(sock, writeTimeout);
    out = new DataOutputStream(new BufferedOutputStream(baseStream, 
                                                        SMALL_BUFFER_SIZE));

    blockSender = new BlockSender(b, 0, b.getNumBytes(), false, false, false, 
        datanode);
    DatanodeInfo srcNode = new DatanodeInfo(dnRegistration);

    //
    // Header info
    //
    out.writeShort(DataTransferProtocol.DATA_TRANSFER_VERSION);
    out.writeByte(DataTransferProtocol.OP_WRITE_BLOCK);
    out.writeLong(b.getBlockId());
    out.writeLong(b.getGenerationStamp());
    out.writeInt(0);           // no pipelining
    out.writeBoolean(false);   // not part of recovery
    Text.writeString(out, ""); // client
    out.writeBoolean(true); // sending src node information
    srcNode.write(out); // Write src node DatanodeInfo
    // write targets
    out.writeInt(targets.length - 1);
    for (int i = 1; i < targets.length; i++) {
      targets[i].write(out);
    }
    // send data & checksum
    blockSender.sendBlock(out, baseStream, null);

    // no response necessary
    LOG.info(dnRegistration + ":Transmitted block " + b + " to " + curTarget);

  } catch (IOException ie) {
    LOG.warn(dnRegistration + ":Failed to transfer " + b + " to " + targets[0].getName()
        + " got " + StringUtils.stringifyException(ie));
  } finally {
    xmitsInProgress.getAndDecrement();
    IOUtils.closeStream(blockSender);
    IOUtils.closeStream(out);
    IOUtils.closeSocket(sock);
  }
}