Java ThreadUtil类代码示例

import org.apache.hadoop.util.ThreadUtil; //导入依赖的package包/类
/**
 * The execute() method invokes doExecute() until either: 1. doExecute() succeeds, or 2. the command may no longer be
 * retried (e.g. runs out of retry-attempts).
 *
 * @param arguments The list of arguments for the command.
 * @return Generic "Object" from doExecute(), on success.
 * @throws IOException, IOException, on complete failure.
 */
public T execute(Object... arguments) throws Exception {
  Exception latestException;
  int counter = 0;
  while (true) {
    try {
      return doExecute(arguments);
    } catch (Exception exception) {
      LOG.error("Failure in Retriable command: {}", description, exception);
      latestException = exception;
    }
    counter++;
    RetryAction action = retryPolicy.shouldRetry(latestException, counter, 0, true);
    if (action.action == RetryPolicy.RetryAction.RetryDecision.RETRY) {
      ThreadUtil.sleepAtLeastIgnoreInterrupts(action.delayMillis);
    } else {
      break;
    }
  }

  throw new IOException("Couldn't run retriable-command: " + description, latestException);
}
 

import org.apache.hadoop.util.ThreadUtil; //导入依赖的package包/类
/**
 * The execute() method invokes doExecute() until either:
 *  1. doExecute() succeeds, or
 *  2. the command may no longer be retried (e.g. runs out of retry-attempts).
 * @param arguments The list of arguments for the command.
 * @return Generic "Object" from doExecute(), on success.
 * @throws Exception
 */
public Object execute(Object... arguments) throws Exception {
  Exception latestException;
  int counter = 0;
  while (true) {
    try {
      return doExecute(arguments);
    } catch(Exception exception) {
      LOG.error("Failure in Retriable command: " + description, exception);
      latestException = exception;
    }
    counter++;
    RetryAction action = retryPolicy.shouldRetry(latestException, counter, 0, true);
    if (action.action == RetryPolicy.RetryAction.RetryDecision.RETRY) {
      ThreadUtil.sleepAtLeastIgnoreInterrupts(action.delayMillis);
    } else {
      break;
    }
  }

  throw new IOException("Couldn't run retriable-command: " + description,
                        latestException);
}
 

import org.apache.hadoop.util.ThreadUtil; //导入依赖的package包/类
/**
 * The execute() method invokes doExecute() until either:
 *  1. doExecute() succeeds, or
 *  2. the command may no longer be retried (e.g. runs out of retry-attempts).
 * @param arguments The list of arguments for the command.
 * @return Generic "Object" from doExecute(), on success.
 * @throws IOException, IOException, on complete failure.
 */
public Object execute(Object... arguments) throws Exception {
  Exception latestException;
  int counter = 0;
  while (true) {
    try {
      return doExecute(arguments);
    } catch(Exception exception) {
      LOG.error("Failure in Retriable command: " + description, exception);
      latestException = exception;
    }
    counter++;
    RetryAction action = retryPolicy.shouldRetry(latestException, counter, 0, true);
    if (action.action == RetryPolicy.RetryAction.RetryDecision.RETRY) {
      ThreadUtil.sleepAtLeastIgnoreInterrupts(action.delayMillis);
    } else {
      break;
    }
  }

  throw new IOException("Couldn't run retriable-command: " + description,
                        latestException);
}
 

import org.apache.hadoop.util.ThreadUtil; //导入依赖的package包/类
/**
 * Ensure that when all configured services are throwing StandbyException
 * that we fail over back and forth between them until one is no longer
 * throwing StandbyException.
 */
@Test
public void testFailoverBetweenMultipleStandbys()
    throws UnreliableException, StandbyException, IOException {
  
  final long millisToSleep = 10000;
  
  final UnreliableImplementation impl1 = new UnreliableImplementation("impl1",
      TypeOfExceptionToFailWith.STANDBY_EXCEPTION);
  FlipFlopProxyProvider<UnreliableInterface> proxyProvider
      = new FlipFlopProxyProvider<UnreliableInterface>(
      UnreliableInterface.class,
      impl1,
      new UnreliableImplementation("impl2",
          TypeOfExceptionToFailWith.STANDBY_EXCEPTION));
  
  final UnreliableInterface unreliable = (UnreliableInterface)RetryProxy
    .create(UnreliableInterface.class, proxyProvider,
        RetryPolicies.failoverOnNetworkException(
            RetryPolicies.TRY_ONCE_THEN_FAIL, 10, 1000, 10000));
  
  new Thread() {
    @Override
    public void run() {
      ThreadUtil.sleepAtLeastIgnoreInterrupts(millisToSleep);
      impl1.setIdentifier("renamed-impl1");
    }
  }.start();
  
  String result = unreliable.failsIfIdentifierDoesntMatch("renamed-impl1");
  assertEquals("renamed-impl1", result);
}
 

import org.apache.hadoop.util.ThreadUtil; //导入依赖的package包/类
@Before
public void setUp() throws IOException {
  originalOut = System.err;
  memOut = new ByteArrayOutputStream();
  filterOut = new FilterOut(memOut);
  capturedOut = new PrintStream(filterOut);
  System.setErr(capturedOut);
  InputStream is =
      ThreadUtil.getResourceAsStream("log4j-kmsaudit.properties");
  PropertyConfigurator.configure(is);
  IOUtils.closeStream(is);

  this.kmsAudit = new KMSAudit(1000);
}
 

import org.apache.hadoop.util.ThreadUtil; //导入依赖的package包/类
/**
 * Regression test for HDFS-3864 - NN does not update internal file mtime for
 * OP_CLOSE when reading from the edit log.
 */
@Test
public void testModTimePersistsAfterRestart() throws IOException {
  final long sleepTime = 10; // 10 milliseconds
  MiniDFSCluster cluster = null;
  FileSystem fs = null;
  Configuration conf = new HdfsConfiguration();
  try {
    cluster = new MiniDFSCluster.Builder(conf).build();
    fs = cluster.getFileSystem();
    Path testPath = new Path("/test");
    
    // Open a file, and get its initial modification time.
    OutputStream out = fs.create(testPath);
    long initialModTime = fs.getFileStatus(testPath).getModificationTime();
    assertTrue(initialModTime > 0);
    
    // Wait and then close the file. Ensure that the mod time goes up.
    ThreadUtil.sleepAtLeastIgnoreInterrupts(sleepTime);
    out.close();
    long modTimeAfterClose = fs.getFileStatus(testPath).getModificationTime();
    assertTrue(modTimeAfterClose >= initialModTime + sleepTime);
    
    // Restart the NN, and make sure that the later mod time is still used.
    cluster.restartNameNode();
    long modTimeAfterRestart = fs.getFileStatus(testPath).getModificationTime();
    assertEquals(modTimeAfterClose, modTimeAfterRestart);
  } finally {
    if (fs != null) {
      fs.close();
    }
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

import org.apache.hadoop.util.ThreadUtil; //导入依赖的package包/类
private void copyResource(String inputResourceName, File outputFile) throws
    IOException {
  InputStream is = null;
  OutputStream os = null;
  try {
    is = ThreadUtil.getResourceAsStream(inputResourceName);
    os = new FileOutputStream(outputFile);
    IOUtils.copy(is, os);
  } finally {
    IOUtils.closeQuietly(is);
    IOUtils.closeQuietly(os);
  }
}
 

import org.apache.hadoop.util.ThreadUtil; //导入依赖的package包/类
/**
 * Make sure that clients will receive StandbyExceptions even when a
 * checkpoint is in progress on the SBN, and therefore the StandbyCheckpointer
 * thread will have FSNS lock. Regression test for HDFS-4591.
 */
@Test(timeout=300000)
public void testStandbyExceptionThrownDuringCheckpoint() throws Exception {
  
  // Set it up so that we know when the SBN checkpoint starts and ends.
  FSImage spyImage1 = NameNodeAdapter.spyOnFsImage(nn1);
  DelayAnswer answerer = new DelayAnswer(LOG);
  Mockito.doAnswer(answerer).when(spyImage1)
      .saveNamespace(Mockito.any(FSNamesystem.class),
          Mockito.any(Canceler.class));
  
  // Perform some edits and wait for a checkpoint to start on the SBN.
  doEdits(0, 1000);
  nn0.getRpcServer().rollEditLog();
  answerer.waitForCall();
  answerer.proceed();
  assertTrue("SBN is not performing checkpoint but it should be.",
      answerer.getFireCount() == 1 && answerer.getResultCount() == 0);
  
  // Make sure that the lock has actually been taken by the checkpointing
  // thread.
  ThreadUtil.sleepAtLeastIgnoreInterrupts(1000);
  try {
    // Perform an RPC to the SBN and make sure it throws a StandbyException.
    nn1.getRpcServer().getFileInfo("/");
    fail("Should have thrown StandbyException, but instead succeeded.");
  } catch (StandbyException se) {
    GenericTestUtils.assertExceptionContains("is not supported", se);
  }
  
  // Make sure that the checkpoint is still going on, implying that the client
  // RPC to the SBN happened during the checkpoint.
  assertTrue("SBN should have still been checkpointing.",
      answerer.getFireCount() == 1 && answerer.getResultCount() == 0);
  answerer.waitForResult();
  assertTrue("SBN should have finished checkpointing.",
      answerer.getFireCount() == 1 && answerer.getResultCount() == 1);
}
 

import org.apache.hadoop.util.ThreadUtil; //导入依赖的package包/类
/**
 * Ensure that when all configured services are throwing StandbyException
 * that we fail over back and forth between them until one is no longer
 * throwing StandbyException.
 */
@Test
public void testFailoverBetweenMultipleStandbys()
    throws UnreliableException, StandbyException, IOException {
  
  final long millisToSleep = 10000;
  
  final UnreliableImplementation impl1 = new UnreliableImplementation("impl1",
      TypeOfExceptionToFailWith.STANDBY_EXCEPTION);
  FlipFlopProxyProvider proxyProvider = new FlipFlopProxyProvider(
      UnreliableInterface.class,
      impl1,
      new UnreliableImplementation("impl2",
          TypeOfExceptionToFailWith.STANDBY_EXCEPTION));
  
  final UnreliableInterface unreliable = (UnreliableInterface)RetryProxy
    .create(UnreliableInterface.class, proxyProvider,
        RetryPolicies.failoverOnNetworkException(
            RetryPolicies.TRY_ONCE_THEN_FAIL, 10, 1000, 10000));
  
  new Thread() {
    @Override
    public void run() {
      ThreadUtil.sleepAtLeastIgnoreInterrupts(millisToSleep);
      impl1.setIdentifier("renamed-impl1");
    }
  }.start();
  
  String result = unreliable.failsIfIdentifierDoesntMatch("renamed-impl1");
  assertEquals("renamed-impl1", result);
}
 

import org.apache.hadoop.util.ThreadUtil; //导入依赖的package包/类
/**
 * Make sure that clients will receive StandbyExceptions even when a
 * checkpoint is in progress on the SBN, and therefore the StandbyCheckpointer
 * thread will have FSNS lock. Regression test for HDFS-4591.
 */
@Test(timeout=300000)
public void testStandbyExceptionThrownDuringCheckpoint() throws Exception {
  
  // Set it up so that we know when the SBN checkpoint starts and ends.
  FSImage spyImage1 = NameNodeAdapter.spyOnFsImage(nn1);
  DelayAnswer answerer = new DelayAnswer(LOG);
  Mockito.doAnswer(answerer).when(spyImage1)
      .saveNamespace(Mockito.any(FSNamesystem.class),
          Mockito.eq(NameNodeFile.IMAGE), Mockito.any(Canceler.class));

  // Perform some edits and wait for a checkpoint to start on the SBN.
  doEdits(0, 1000);
  nn0.getRpcServer().rollEditLog();
  answerer.waitForCall();
  assertTrue("SBN is not performing checkpoint but it should be.",
      answerer.getFireCount() == 1 && answerer.getResultCount() == 0);
  
  // Make sure that the lock has actually been taken by the checkpointing
  // thread.
  ThreadUtil.sleepAtLeastIgnoreInterrupts(1000);
  try {
    // Perform an RPC to the SBN and make sure it throws a StandbyException.
    nn1.getRpcServer().getFileInfo("/");
    fail("Should have thrown StandbyException, but instead succeeded.");
  } catch (StandbyException se) {
    GenericTestUtils.assertExceptionContains("is not supported", se);
  }
  
  // Make sure that the checkpoint is still going on, implying that the client
  // RPC to the SBN happened during the checkpoint.
  assertTrue("SBN should have still been checkpointing.",
      answerer.getFireCount() == 1 && answerer.getResultCount() == 0);
  answerer.proceed();
  answerer.waitForResult();
  assertTrue("SBN should have finished checkpointing.",
      answerer.getFireCount() == 1 && answerer.getResultCount() == 1);
}
 

import org.apache.hadoop.util.ThreadUtil; //导入依赖的package包/类
/**
 * Regression test for HDFS-3864 - NN does not update internal file mtime for
 * OP_CLOSE when reading from the edit log.
 */
@Test
public void testModTimePersistsAfterRestart() throws IOException {
  final long sleepTime = 10; // 10 milliseconds
  MiniDFSCluster cluster = null;
  FileSystem fs = null;
  Configuration conf = new HdfsConfiguration();
  try {
    cluster = new MiniDFSCluster.Builder(conf).build();
    fs = cluster.getFileSystem();
    Path testPath = new Path("/test");
    
    // Open a file, and get its initial modification time.
    OutputStream out = fs.create(testPath);
    long initialModTime = fs.getFileStatus(testPath).getModificationTime();
    assertTrue(initialModTime > 0);
    
    // Wait and then close the file. Ensure that the mod time goes up.
    ThreadUtil.sleepAtLeastIgnoreInterrupts(sleepTime);
    out.close();
    long modTimeAfterClose = fs.getFileStatus(testPath).getModificationTime();
    assertTrue(modTimeAfterClose >= initialModTime + sleepTime);
    
    // Restart the NN, and make sure that the later mod time is still used.
    cluster.restartNameNode();
    long modTimeAfterRestart =
        fs.getFileStatus(testPath).getModificationTime();
    assertEquals(modTimeAfterClose, modTimeAfterRestart);
  } finally {
    if (fs != null) {
      fs.close();
    }
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

import org.apache.hadoop.util.ThreadUtil; //导入依赖的package包/类
@Test(timeout=60000)
public void testExcludedNodesForgiveness() throws IOException {
  // Forgive nodes in under 2.5s for this test case.
  conf.setLong(
      DFSConfigKeys.DFS_CLIENT_WRITE_EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL,
      2500);
  // We'll be using a 512 bytes block size just for tests
  // so making sure the checksum bytes too match it.
  conf.setInt("io.bytes.per.checksum", 512);
  cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
  List<DataNodeProperties> props = cluster.dataNodes;
  FileSystem fs = cluster.getFileSystem();
  Path filePath = new Path("/testForgivingExcludedNodes");

  // 256 bytes data chunk for writes
  byte[] bytes = new byte[256];
  for (int index=0; index<bytes.length; index++) {
    bytes[index] = '0';
  }

  // File with a 512 bytes block size
  FSDataOutputStream out = fs.create(filePath, true, 4096, (short) 3, 512);

  // Write a block to all 3 DNs (2x256bytes).
  out.write(bytes);
  out.write(bytes);
  out.hflush();

  // Remove two DNs, to put them into the exclude list.
  DataNodeProperties two = cluster.stopDataNode(2);
  DataNodeProperties one = cluster.stopDataNode(1);

  // Write another block.
  // At this point, we have two nodes already in excluded list.
  out.write(bytes);
  out.write(bytes);
  out.hflush();

  // Bring back the older DNs, since they are gonna be forgiven only
  // afterwards of this previous block write.
  Assert.assertEquals(true, cluster.restartDataNode(one, true));
  Assert.assertEquals(true, cluster.restartDataNode(two, true));
  cluster.waitActive();

  // Sleep for 5s, to let the excluded nodes be expired
  // from the excludes list (i.e. forgiven after the configured wait period).
  // [Sleeping just in case the restart of the DNs completed < 5s cause
  // otherwise, we'll end up quickly excluding those again.]
  ThreadUtil.sleepAtLeastIgnoreInterrupts(5000);

  // Terminate the last good DN, to assert that there's no
  // single-DN-available scenario, caused by not forgiving the other
  // two by now.
  cluster.stopDataNode(0);

  try {
    // Attempt writing another block, which should still pass
    // cause the previous two should have been forgiven by now,
    // while the last good DN added to excludes this time.
    out.write(bytes);
    out.hflush();
    out.close();
  } catch (Exception e) {
    fail("Excluded DataNodes should be forgiven after a while and " +
         "not cause file writing exception of: '" + e.getMessage() + "'");
  }
}
 

import org.apache.hadoop.util.ThreadUtil; //导入依赖的package包/类
/**
 * Make sure that clients will receive StandbyExceptions even when a
 * checkpoint is in progress on the SBN, and therefore the StandbyCheckpointer
 * thread will have FSNS lock. Regression test for HDFS-4591.
 */
@Test(timeout=300000)
public void testStandbyExceptionThrownDuringCheckpoint() throws Exception {
  
  // Set it up so that we know when the SBN checkpoint starts and ends.
  FSImage spyImage1 = NameNodeAdapter.spyOnFsImage(nn1);
  DelayAnswer answerer = new DelayAnswer(LOG);
  Mockito.doAnswer(answerer).when(spyImage1)
      .saveNamespace(Mockito.any(FSNamesystem.class),
          Mockito.eq(NameNodeFile.IMAGE), Mockito.any(Canceler.class));

  // Perform some edits and wait for a checkpoint to start on the SBN.
  doEdits(0, 1000);
  nn0.getRpcServer().rollEditLog();
  answerer.waitForCall();
  assertTrue("SBN is not performing checkpoint but it should be.",
      answerer.getFireCount() == 1 && answerer.getResultCount() == 0);
  
  // Make sure that the lock has actually been taken by the checkpointing
  // thread.
  ThreadUtil.sleepAtLeastIgnoreInterrupts(1000);
  try {
    // Perform an RPC to the SBN and make sure it throws a StandbyException.
    nn1.getRpcServer().getFileInfo("/");
    fail("Should have thrown StandbyException, but instead succeeded.");
  } catch (StandbyException se) {
    GenericTestUtils.assertExceptionContains("is not supported", se);
  }

  // Make sure new incremental block reports are processed during
  // checkpointing on the SBN.
  assertEquals(0, cluster.getNamesystem(1).getPendingDataNodeMessageCount());
  doCreate();
  Thread.sleep(1000);
  assertTrue(cluster.getNamesystem(1).getPendingDataNodeMessageCount() > 0);
  
  // Make sure that the checkpoint is still going on, implying that the client
  // RPC to the SBN happened during the checkpoint.
  assertTrue("SBN should have still been checkpointing.",
      answerer.getFireCount() == 1 && answerer.getResultCount() == 0);
  answerer.proceed();
  answerer.waitForResult();
  assertTrue("SBN should have finished checkpointing.",
      answerer.getFireCount() == 1 && answerer.getResultCount() == 1);
}