Java SplitLogTask.Owned方法代码示例

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
@Test (timeout=180000)
public void testOrphanTaskAcquisition() throws Exception {
  LOG.info("TestOrphanTaskAcquisition");

  String tasknode = ZKSplitLog.getEncodedNodeName(zkw, "orphan/test/slash");
  SplitLogTask slt = new SplitLogTask.Owned(DUMMY_MASTER, this.mode);
  zkw.getRecoverableZooKeeper().create(tasknode, slt.toByteArray(), Ids.OPEN_ACL_UNSAFE,
      CreateMode.PERSISTENT);

  slm = new SplitLogManager(ds, conf, stopper, master, DUMMY_MASTER);
  waitForCounter(tot_mgr_orphan_task_acquired, 0, 1, to/2);
  Task task = slm.findOrCreateOrphanTask(tasknode);
  assertTrue(task.isOrphan());
  waitForCounter(tot_mgr_heartbeat, 0, 1, to/2);
  assertFalse(task.isUnassigned());
  long curt = System.currentTimeMillis();
  assertTrue((task.last_update <= curt) &&
      (task.last_update > (curt - 1000)));
  LOG.info("waiting for manager to resubmit the orphan task");
  waitForCounter(tot_mgr_resubmit, 0, 1, to + to/2);
  assertTrue(task.isUnassigned());
  waitForCounter(tot_mgr_rescan, 0, 1, to + to/2);
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
@Test (timeout=180000)
public void testWorkerCrash() throws Exception {
  slm = new SplitLogManager(ds, conf, stopper, master, DUMMY_MASTER);
  TaskBatch batch = new TaskBatch();

  String tasknode = submitTaskAndWait(batch, "foo/1");
  final ServerName worker1 = ServerName.valueOf("worker1,1,1");

  SplitLogTask slt = new SplitLogTask.Owned(worker1, this.mode);
  ZKUtil.setData(zkw, tasknode, slt.toByteArray());
  if (tot_mgr_heartbeat.get() == 0) waitForCounter(tot_mgr_heartbeat, 0, 1, to/2);

  // Not yet resubmitted.
  Assert.assertEquals(0, tot_mgr_resubmit.get());

  // This server becomes dead
  Mockito.when(sm.isServerOnline(worker1)).thenReturn(false);

  Thread.sleep(1300); // The timeout checker is done every 1000 ms (hardcoded).

  // It has been resubmitted
  Assert.assertEquals(1, tot_mgr_resubmit.get());
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
@Test
public void testOrphanTaskAcquisition() throws Exception {
  LOG.info("TestOrphanTaskAcquisition");

  String tasknode = ZKSplitLog.getEncodedNodeName(zkw, "orphan/test/slash");
  SplitLogTask slt = new SplitLogTask.Owned(DUMMY_MASTER);
  zkw.getRecoverableZooKeeper().create(tasknode, slt.toByteArray(), Ids.OPEN_ACL_UNSAFE,
      CreateMode.PERSISTENT);

  slm = new SplitLogManager(zkw, conf, stopper, master, DUMMY_MASTER);
  waitForCounter(tot_mgr_orphan_task_acquired, 0, 1, to/2);
  Task task = slm.findOrCreateOrphanTask(tasknode);
  assertTrue(task.isOrphan());
  waitForCounter(tot_mgr_heartbeat, 0, 1, to/2);
  assertFalse(task.isUnassigned());
  long curt = System.currentTimeMillis();
  assertTrue((task.last_update <= curt) &&
      (task.last_update > (curt - 1000)));
  LOG.info("waiting for manager to resubmit the orphan task");
  waitForCounter(tot_mgr_resubmit, 0, 1, to + to/2);
  assertTrue(task.isUnassigned());
  waitForCounter(tot_mgr_rescan, 0, 1, to + to/2);
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
@Test
public void testWorkerCrash() throws Exception {
  slm = new SplitLogManager(zkw, conf, stopper, master, DUMMY_MASTER);
  TaskBatch batch = new TaskBatch();

  String tasknode = submitTaskAndWait(batch, "foo/1");
  final ServerName worker1 = ServerName.valueOf("worker1,1,1");

  SplitLogTask slt = new SplitLogTask.Owned(worker1);
  ZKUtil.setData(zkw, tasknode, slt.toByteArray());
  if (tot_mgr_heartbeat.get() == 0) waitForCounter(tot_mgr_heartbeat, 0, 1, to/2);

  // Not yet resubmitted.
  Assert.assertEquals(0, tot_mgr_resubmit.get());

  // This server becomes dead
  Mockito.when(sm.isServerOnline(worker1)).thenReturn(false);

  Thread.sleep(1300); // The timeout checker is done every 1000 ms (hardcoded).

  // It has been resubmitted
  Assert.assertEquals(1, tot_mgr_resubmit.get());
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
@Test (timeout=180000)
public void testOrphanTaskAcquisition() throws Exception {
  LOG.info("TestOrphanTaskAcquisition");

  String tasknode = ZKSplitLog.getEncodedNodeName(zkw, "orphan/test/slash");
  SplitLogTask slt = new SplitLogTask.Owned(master.getServerName());
  zkw.getRecoverableZooKeeper().create(tasknode, slt.toByteArray(), Ids.OPEN_ACL_UNSAFE,
      CreateMode.PERSISTENT);

  slm = new SplitLogManager(master, conf);
  waitForCounter(tot_mgr_orphan_task_acquired, 0, 1, to/2);
  Task task = findOrCreateOrphanTask(tasknode);
  assertTrue(task.isOrphan());
  waitForCounter(tot_mgr_heartbeat, 0, 1, to/2);
  assertFalse(task.isUnassigned());
  long curt = System.currentTimeMillis();
  assertTrue((task.last_update <= curt) &&
      (task.last_update > (curt - 1000)));
  LOG.info("waiting for manager to resubmit the orphan task");
  waitForCounter(tot_mgr_resubmit, 0, 1, to + to/2);
  assertTrue(task.isUnassigned());
  waitForCounter(tot_mgr_rescan, 0, 1, to + to/2);
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
@Test (timeout=180000)
public void testWorkerCrash() throws Exception {
  slm = new SplitLogManager(master, conf);
  TaskBatch batch = new TaskBatch();

  String tasknode = submitTaskAndWait(batch, "foo/1");
  final ServerName worker1 = ServerName.valueOf("worker1,1,1");

  SplitLogTask slt = new SplitLogTask.Owned(worker1);
  ZKUtil.setData(zkw, tasknode, slt.toByteArray());
  if (tot_mgr_heartbeat.sum() == 0) waitForCounter(tot_mgr_heartbeat, 0, 1, to/2);

  // Not yet resubmitted.
  Assert.assertEquals(0, tot_mgr_resubmit.sum());

  // This server becomes dead
  Mockito.when(sm.isServerOnline(worker1)).thenReturn(false);

  Thread.sleep(1300); // The timeout checker is done every 1000 ms (hardcoded).

  // It has been resubmitted
  Assert.assertEquals(1, tot_mgr_resubmit.sum());
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
@Test
public void testOrphanTaskAcquisition() throws Exception {
  LOG.info("TestOrphanTaskAcquisition");

  String tasknode = ZKSplitLog.getEncodedNodeName(zkw, "orphan/test/slash");
  SplitLogTask slt = new SplitLogTask.Owned(DUMMY_MASTER);
  zkw.getRecoverableZooKeeper().create(tasknode, slt.toByteArray(), Ids.OPEN_ACL_UNSAFE,
      CreateMode.PERSISTENT);

  slm = new SplitLogManager(zkw, conf, stopper, master, DUMMY_MASTER, null);
  slm.finishInitialization();
  waitForCounter(tot_mgr_orphan_task_acquired, 0, 1, to/2);
  Task task = slm.findOrCreateOrphanTask(tasknode);
  assertTrue(task.isOrphan());
  waitForCounter(tot_mgr_heartbeat, 0, 1, to/2);
  assertFalse(task.isUnassigned());
  long curt = System.currentTimeMillis();
  assertTrue((task.last_update <= curt) &&
      (task.last_update > (curt - 1000)));
  LOG.info("waiting for manager to resubmit the orphan task");
  waitForCounter(tot_mgr_resubmit, 0, 1, to + to/2);
  assertTrue(task.isUnassigned());
  waitForCounter(tot_mgr_rescan, 0, 1, to + to/2);
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
@Test
public void testWorkerCrash() throws Exception {
  slm = new SplitLogManager(zkw, conf, stopper, master, DUMMY_MASTER, null);
  slm.finishInitialization();
  TaskBatch batch = new TaskBatch();

  String tasknode = submitTaskAndWait(batch, "foo/1");
  final ServerName worker1 = new ServerName("worker1,1,1");

  SplitLogTask slt = new SplitLogTask.Owned(worker1);
  ZKUtil.setData(zkw, tasknode, slt.toByteArray());
  if (tot_mgr_heartbeat.get() == 0) waitForCounter(tot_mgr_heartbeat, 0, 1, to/2);

  // Not yet resubmitted.
  Assert.assertEquals(0, tot_mgr_resubmit.get());

  // This server becomes dead
  Mockito.when(sm.isServerOnline(worker1)).thenReturn(false);

  Thread.sleep(1300); // The timeout checker is done every 1000 ms (hardcoded).

  // It has been resubmitted
  Assert.assertEquals(1, tot_mgr_resubmit.get());
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
/**
 * Try to own the task by transitioning the zk node data from UNASSIGNED to OWNED.
 * <p>
 * This method is also used to periodically heartbeat the task progress by transitioning the node
 * from OWNED to OWNED.
 * <p>
 * @param isFirstTime shows whther it's the first attempt.
 * @param zkw zk wathcer
 * @param server name
 * @param task to own
 * @param taskZKVersion version of the task in zk
 * @return non-negative integer value when task can be owned by current region server otherwise -1
 */
protected static int attemptToOwnTask(boolean isFirstTime, ZooKeeperWatcher zkw,
    ServerName server, String task, RecoveryMode mode, int taskZKVersion) {
  int latestZKVersion = FAILED_TO_OWN_TASK;
  try {
    SplitLogTask slt = new SplitLogTask.Owned(server, mode);
    Stat stat = zkw.getRecoverableZooKeeper().setData(task, slt.toByteArray(), taskZKVersion);
    if (stat == null) {
      LOG.warn("zk.setData() returned null for path " + task);
      SplitLogCounters.tot_wkr_task_heartbeat_failed.incrementAndGet();
      return FAILED_TO_OWN_TASK;
    }
    latestZKVersion = stat.getVersion();
    SplitLogCounters.tot_wkr_task_heartbeat.incrementAndGet();
    return latestZKVersion;
  } catch (KeeperException e) {
    if (!isFirstTime) {
      if (e.code().equals(KeeperException.Code.NONODE)) {
        LOG.warn("NONODE failed to assert ownership for " + task, e);
      } else if (e.code().equals(KeeperException.Code.BADVERSION)) {
        LOG.warn("BADVERSION failed to assert ownership for " + task, e);
      } else {
        LOG.warn("failed to assert ownership for " + task, e);
      }
    }
  } catch (InterruptedException e1) {
    LOG.warn("Interrupted while trying to assert ownership of " + task + " "
        + StringUtils.stringifyException(e1));
    Thread.currentThread().interrupt();
  }
  SplitLogCounters.tot_wkr_task_heartbeat_failed.incrementAndGet();
  return FAILED_TO_OWN_TASK;
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
@Test (timeout=180000)
public void testMultipleResubmits() throws Exception {
  LOG.info("TestMultipleResbmits - no indefinite resubmissions");
  conf.setInt("hbase.splitlog.max.resubmit", 2);
  slm = new SplitLogManager(ds, conf, stopper, master, DUMMY_MASTER);
  TaskBatch batch = new TaskBatch();

  String tasknode = submitTaskAndWait(batch, "foo/1");
  int version = ZKUtil.checkExists(zkw, tasknode);
  final ServerName worker1 = ServerName.valueOf("worker1,1,1");
  final ServerName worker2 = ServerName.valueOf("worker2,1,1");
  final ServerName worker3 = ServerName.valueOf("worker3,1,1");
  SplitLogTask slt = new SplitLogTask.Owned(worker1, this.mode);
  ZKUtil.setData(zkw, tasknode, slt.toByteArray());
  waitForCounter(tot_mgr_heartbeat, 0, 1, to/2);
  waitForCounter(tot_mgr_resubmit, 0, 1, to + to/2);
  int version1 = ZKUtil.checkExists(zkw, tasknode);
  assertTrue(version1 > version);
  slt = new SplitLogTask.Owned(worker2, this.mode);
  ZKUtil.setData(zkw, tasknode, slt.toByteArray());
  waitForCounter(tot_mgr_heartbeat, 1, 2, to/2);
  waitForCounter(tot_mgr_resubmit, 1, 2, to + to/2);
  int version2 = ZKUtil.checkExists(zkw, tasknode);
  assertTrue(version2 > version1);
  slt = new SplitLogTask.Owned(worker3, this.mode);
  ZKUtil.setData(zkw, tasknode, slt.toByteArray());
  waitForCounter(tot_mgr_heartbeat, 2, 3, to/2);
  waitForCounter(tot_mgr_resubmit_threshold_reached, 0, 1, to + to/2);
  Thread.sleep(to + to/2);
  assertEquals(2L, tot_mgr_resubmit.get() - tot_mgr_resubmit_force.get());
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
@Test (timeout=180000)
public void testRescanCleanup() throws Exception {
  LOG.info("TestRescanCleanup - ensure RESCAN nodes are cleaned up");

  slm = new SplitLogManager(ds, conf, stopper, master, DUMMY_MASTER);
  TaskBatch batch = new TaskBatch();

  String tasknode = submitTaskAndWait(batch, "foo/1");
  int version = ZKUtil.checkExists(zkw, tasknode);
  final ServerName worker1 = ServerName.valueOf("worker1,1,1");
  SplitLogTask slt = new SplitLogTask.Owned(worker1, this.mode);
  ZKUtil.setData(zkw, tasknode, slt.toByteArray());
  waitForCounter(tot_mgr_heartbeat, 0, 1, to/2);
  waitForCounter(new Expr() {
    @Override
    public long eval() {
      return (tot_mgr_resubmit.get() + tot_mgr_resubmit_failed.get());
    }
  }, 0, 1, 5*60000); // wait long enough
  Assert.assertEquals("Could not run test. Lost ZK connection?", 0, tot_mgr_resubmit_failed.get());
  int version1 = ZKUtil.checkExists(zkw, tasknode);
  assertTrue(version1 > version);
  byte[] taskstate = ZKUtil.getData(zkw, tasknode);
  slt = SplitLogTask.parseFrom(taskstate);
  assertTrue(slt.isUnassigned(DUMMY_MASTER));

  waitForCounter(tot_mgr_rescan_deleted, 0, 1, to/2);
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
@Test (timeout=180000)
public void testUnassignedTimeout() throws Exception {
  LOG.info("TestUnassignedTimeout - iff all tasks are unassigned then" +
      " resubmit");

  // create an orphan task in OWNED state
  String tasknode1 = ZKSplitLog.getEncodedNodeName(zkw, "orphan/1");
  final ServerName worker1 = ServerName.valueOf("worker1,1,1");
  SplitLogTask slt = new SplitLogTask.Owned(worker1, this.mode);
  zkw.getRecoverableZooKeeper().create(tasknode1, slt.toByteArray(), Ids.OPEN_ACL_UNSAFE,
      CreateMode.PERSISTENT);

  slm = new SplitLogManager(ds, conf, stopper, master, DUMMY_MASTER);
  waitForCounter(tot_mgr_orphan_task_acquired, 0, 1, to/2);

  // submit another task which will stay in unassigned mode
  TaskBatch batch = new TaskBatch();
  submitTaskAndWait(batch, "foo/1");

  // keep updating the orphan owned node every to/2 seconds
  for (int i = 0; i < (3 * to)/100; i++) {
    Thread.sleep(100);
    final ServerName worker2 = ServerName.valueOf("worker1,1,1");
    slt = new SplitLogTask.Owned(worker2, this.mode);
    ZKUtil.setData(zkw, tasknode1, slt.toByteArray());
  }

  // since we have stopped heartbeating the owned node therefore it should
  // get resubmitted
  LOG.info("waiting for manager to resubmit the orphan task");
  waitForCounter(tot_mgr_resubmit, 0, 1, to + to/2);

  // now all the nodes are unassigned. manager should post another rescan
  waitForCounter(tot_mgr_resubmit_unassigned, 0, 1, 2 * to + to/2);
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
@Test (timeout=180000)
public void testDeadWorker() throws Exception {
  LOG.info("testDeadWorker");

  conf.setLong("hbase.splitlog.max.resubmit", 0);
  slm = new SplitLogManager(ds, conf, stopper, master, DUMMY_MASTER);
  TaskBatch batch = new TaskBatch();

  String tasknode = submitTaskAndWait(batch, "foo/1");
  int version = ZKUtil.checkExists(zkw, tasknode);
  final ServerName worker1 = ServerName.valueOf("worker1,1,1");
  SplitLogTask slt = new SplitLogTask.Owned(worker1, this.mode);
  ZKUtil.setData(zkw, tasknode, slt.toByteArray());
  if (tot_mgr_heartbeat.get() == 0) waitForCounter(tot_mgr_heartbeat, 0, 1, to/2);
  slm.handleDeadWorker(worker1);
  if (tot_mgr_resubmit.get() == 0) waitForCounter(tot_mgr_resubmit, 0, 1, to+to/2);
  if (tot_mgr_resubmit_dead_server_task.get() == 0) {
    waitForCounter(tot_mgr_resubmit_dead_server_task, 0, 1, to + to/2);
  }

  int version1 = ZKUtil.checkExists(zkw, tasknode);
  assertTrue(version1 > version);
  byte[] taskstate = ZKUtil.getData(zkw, tasknode);
  slt = SplitLogTask.parseFrom(taskstate);
  assertTrue(slt.isUnassigned(DUMMY_MASTER));
  return;
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
@Test(timeout=60000)
public void testPreemptTask() throws Exception {
  LOG.info("testPreemptTask");
  SplitLogCounters.resetCounters();
  final ServerName SRV = ServerName.valueOf("tpt_svr,1,1");
  final String PATH = ZKSplitLog.getEncodedNodeName(zkw, "tpt_task");
  RegionServerServices mockedRS = getRegionServer(SRV);
  SplitLogWorker slw =
      new SplitLogWorker(ds, TEST_UTIL.getConfiguration(), mockedRS, neverEndingTask);
  slw.start();
  try {
    Thread.yield(); // let the worker start
    Thread.sleep(1000);
    waitForCounter(SplitLogCounters.tot_wkr_task_grabing, 0, 1, WAIT_TIME);

    // this time create a task node after starting the splitLogWorker
    zkw.getRecoverableZooKeeper().create(PATH,
      new SplitLogTask.Unassigned(MANAGER, this.mode).toByteArray(),
      Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);

    waitForCounter(SplitLogCounters.tot_wkr_task_acquired, 0, 1, WAIT_TIME);
    assertEquals(1, slw.getTaskReadySeq());
    byte [] bytes = ZKUtil.getData(zkw, PATH);
    SplitLogTask slt = SplitLogTask.parseFrom(bytes);
    assertTrue(slt.isOwned(SRV));
    slt = new SplitLogTask.Owned(MANAGER, this.mode);
    ZKUtil.setData(zkw, PATH, slt.toByteArray());
    waitForCounter(SplitLogCounters.tot_wkr_preempt_task, 0, 1, WAIT_TIME);
  } finally {
    stopSplitLogWorker(slw);
  }
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
/**
 * Try to own the task by transitioning the zk node data from UNASSIGNED to OWNED.
 * <p>
 * This method is also used to periodically heartbeat the task progress by transitioning the node
 * from OWNED to OWNED.
 * <p>
 * @param isFirstTime
 * @param zkw
 * @param server
 * @param task
 * @param taskZKVersion
 * @return non-negative integer value when task can be owned by current region server otherwise -1
 */
protected static int attemptToOwnTask(boolean isFirstTime, ZooKeeperWatcher zkw,
    ServerName server, String task, int taskZKVersion) {
  int latestZKVersion = FAILED_TO_OWN_TASK;
  try {
    SplitLogTask slt = new SplitLogTask.Owned(server);
    Stat stat = zkw.getRecoverableZooKeeper().setData(task, slt.toByteArray(), taskZKVersion);
    if (stat == null) {
      LOG.warn("zk.setData() returned null for path " + task);
      SplitLogCounters.tot_wkr_task_heartbeat_failed.incrementAndGet();
      return FAILED_TO_OWN_TASK;
    }
    latestZKVersion = stat.getVersion();
    SplitLogCounters.tot_wkr_task_heartbeat.incrementAndGet();
    return latestZKVersion;
  } catch (KeeperException e) {
    if (!isFirstTime) {
      if (e.code().equals(KeeperException.Code.NONODE)) {
        LOG.warn("NONODE failed to assert ownership for " + task, e);
      } else if (e.code().equals(KeeperException.Code.BADVERSION)) {
        LOG.warn("BADVERSION failed to assert ownership for " + task, e);
      } else {
        LOG.warn("failed to assert ownership for " + task, e);
      }
    }
  } catch (InterruptedException e1) {
    LOG.warn("Interrupted while trying to assert ownership of " +
        task + " " + StringUtils.stringifyException(e1));
    Thread.currentThread().interrupt();
  }
  SplitLogCounters.tot_wkr_task_heartbeat_failed.incrementAndGet();
  return FAILED_TO_OWN_TASK;
}