Java SplitLogTask.parseFrom方法代码示例

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
@Test (timeout=180000)
public void testTaskResigned() throws Exception {
  LOG.info("TestTaskResigned - resubmit task node once in RESIGNED state");
  assertEquals(tot_mgr_resubmit.get(), 0);
  slm = new SplitLogManager(ds, conf, stopper, master, DUMMY_MASTER);
  assertEquals(tot_mgr_resubmit.get(), 0);
  TaskBatch batch = new TaskBatch();
  String tasknode = submitTaskAndWait(batch, "foo/1");
  assertEquals(tot_mgr_resubmit.get(), 0);
  final ServerName worker1 = ServerName.valueOf("worker1,1,1");
  assertEquals(tot_mgr_resubmit.get(), 0);
  SplitLogTask slt = new SplitLogTask.Resigned(worker1, this.mode);
  assertEquals(tot_mgr_resubmit.get(), 0);
  ZKUtil.setData(zkw, tasknode, slt.toByteArray());
  int version = ZKUtil.checkExists(zkw, tasknode);
  // Could be small race here.
  if (tot_mgr_resubmit.get() == 0) {
    waitForCounter(tot_mgr_resubmit, 0, 1, to/2);
  }
  assertEquals(tot_mgr_resubmit.get(), 1);

  byte[] taskstate = ZKUtil.getData(zkw, tasknode);
  slt = SplitLogTask.parseFrom(taskstate);
  assertTrue(slt.isUnassigned(DUMMY_MASTER));
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
@Test(timeout=60000)
public void testAcquireTaskAtStartup() throws Exception {
  LOG.info("testAcquireTaskAtStartup");
  SplitLogCounters.resetCounters();
  final String TATAS = "tatas";
  final ServerName RS = ServerName.valueOf("rs,1,1");
  RegionServerServices mockedRS = getRegionServer(RS);
  zkw.getRecoverableZooKeeper().create(ZKSplitLog.getEncodedNodeName(zkw, TATAS),
    new SplitLogTask.Unassigned(ServerName.valueOf("mgr,1,1"), this.mode).toByteArray(),
      Ids.OPEN_ACL_UNSAFE,
      CreateMode.PERSISTENT);

  SplitLogWorker slw =
      new SplitLogWorker(ds, TEST_UTIL.getConfiguration(), mockedRS, neverEndingTask);
  slw.start();
  try {
    waitForCounter(SplitLogCounters.tot_wkr_task_acquired, 0, 1, WAIT_TIME);
    byte [] bytes = ZKUtil.getData(zkw, ZKSplitLog.getEncodedNodeName(zkw, TATAS));
    SplitLogTask slt = SplitLogTask.parseFrom(bytes);
    assertTrue(slt.isOwned(RS));
  } finally {
   stopSplitLogWorker(slw);
  }
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
@Test(timeout=60000)
public void testAcquireTaskAtStartup() throws Exception {
  LOG.info("testAcquireTaskAtStartup");
  SplitLogCounters.resetCounters();
  final String TATAS = "tatas";
  final ServerName RS = ServerName.valueOf("rs,1,1");
  RegionServerServices mockedRS = getRegionServer(RS);
  zkw.getRecoverableZooKeeper().create(ZKSplitLog.getEncodedNodeName(zkw, TATAS),
    new SplitLogTask.Unassigned(ServerName.valueOf("mgr,1,1")).toByteArray(), Ids.OPEN_ACL_UNSAFE,
      CreateMode.PERSISTENT);

  SplitLogWorker slw =
      new SplitLogWorker(zkw, TEST_UTIL.getConfiguration(), mockedRS, neverEndingTask);
  slw.start();
  try {
    waitForCounter(SplitLogCounters.tot_wkr_task_acquired, 0, 1, WAIT_TIME);
    byte [] bytes = ZKUtil.getData(zkw, ZKSplitLog.getEncodedNodeName(zkw, TATAS));
    SplitLogTask slt = SplitLogTask.parseFrom(bytes);
    assertTrue(slt.isOwned(RS));
  } finally {
    stopSplitLogWorker(slw);
  }
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
@Test
public void testTaskResigned() throws Exception {
  LOG.info("TestTaskResigned - resubmit task node once in RESIGNED state");
  assertEquals(tot_mgr_resubmit.get(), 0);
  slm = new SplitLogManager(zkw, conf, stopper, master, DUMMY_MASTER);
  assertEquals(tot_mgr_resubmit.get(), 0);
  TaskBatch batch = new TaskBatch();
  String tasknode = submitTaskAndWait(batch, "foo/1");
  assertEquals(tot_mgr_resubmit.get(), 0);
  final ServerName worker1 = ServerName.valueOf("worker1,1,1");
  assertEquals(tot_mgr_resubmit.get(), 0);
  SplitLogTask slt = new SplitLogTask.Resigned(worker1);
  assertEquals(tot_mgr_resubmit.get(), 0);
  ZKUtil.setData(zkw, tasknode, slt.toByteArray());
  int version = ZKUtil.checkExists(zkw, tasknode);
  // Could be small race here.
  if (tot_mgr_resubmit.get() == 0) {
    waitForCounter(tot_mgr_resubmit, 0, 1, to/2);
  }
  assertEquals(tot_mgr_resubmit.get(), 1);

  byte[] taskstate = ZKUtil.getData(zkw, tasknode);
  slt = SplitLogTask.parseFrom(taskstate);
  assertTrue(slt.isUnassigned(DUMMY_MASTER));
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
@Test (timeout=180000)
public void testTaskResigned() throws Exception {
  LOG.info("TestTaskResigned - resubmit task node once in RESIGNED state");
  assertEquals(0, tot_mgr_resubmit.sum());
  slm = new SplitLogManager(master, conf);
  assertEquals(0, tot_mgr_resubmit.sum());
  TaskBatch batch = new TaskBatch();
  String tasknode = submitTaskAndWait(batch, "foo/1");
  assertEquals(0, tot_mgr_resubmit.sum());
  final ServerName worker1 = ServerName.valueOf("worker1,1,1");
  assertEquals(0, tot_mgr_resubmit.sum());
  SplitLogTask slt = new SplitLogTask.Resigned(worker1);
  assertEquals(0, tot_mgr_resubmit.sum());
  ZKUtil.setData(zkw, tasknode, slt.toByteArray());
  ZKUtil.checkExists(zkw, tasknode);
  // Could be small race here.
  if (tot_mgr_resubmit.sum() == 0) {
    waitForCounter(tot_mgr_resubmit, 0, 1, to/2);
  }
  assertEquals(1, tot_mgr_resubmit.sum());

  byte[] taskstate = ZKUtil.getData(zkw, tasknode);
  slt = SplitLogTask.parseFrom(taskstate);
  assertTrue(slt.isUnassigned(master.getServerName()));
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
@Test
public void testTaskResigned() throws Exception {
  LOG.info("TestTaskResigned - resubmit task node once in RESIGNED state");
  assertEquals(tot_mgr_resubmit.get(), 0);
  slm = new SplitLogManager(zkw, conf, stopper, master, DUMMY_MASTER);
  assertEquals(tot_mgr_resubmit.get(), 0);
  TaskBatch batch = new TaskBatch();
  String tasknode = submitTaskAndWait(batch, "foo/1");
  assertEquals(tot_mgr_resubmit.get(), 0);
  final ServerName worker1 = ServerName.valueOf("worker1,1,1");
  assertEquals(tot_mgr_resubmit.get(), 0);
  SplitLogTask slt = new SplitLogTask.Resigned(worker1);
  assertEquals(tot_mgr_resubmit.get(), 0);
  ZKUtil.setData(zkw, tasknode, slt.toByteArray());
  int version = ZKUtil.checkExists(zkw, tasknode);
  // Could be small race here.
  if (tot_mgr_resubmit.get() == 0) waitForCounter(tot_mgr_resubmit, 0, 1, to/2);
  assertEquals(tot_mgr_resubmit.get(), 1);
  int version1 = ZKUtil.checkExists(zkw, tasknode);
  assertTrue("version1=" + version1 + ", version=" + version, version1 > version);

  byte[] taskstate = ZKUtil.getData(zkw, tasknode);
  slt = SplitLogTask.parseFrom(taskstate);
  assertTrue(slt.isUnassigned(DUMMY_MASTER));
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
/**
 * Test whether the splitlog correctly creates a task in zookeeper
 * @throws Exception
 */
@Test
public void testTaskCreation() throws Exception {

  LOG.info("TestTaskCreation - test the creation of a task in zk");
  slm = new SplitLogManager(zkw, conf, stopper, master, DUMMY_MASTER, null);
  slm.finishInitialization();
  TaskBatch batch = new TaskBatch();

  String tasknode = submitTaskAndWait(batch, "foo/1");

  byte[] data = ZKUtil.getData(zkw, tasknode);
  SplitLogTask slt = SplitLogTask.parseFrom(data);
  LOG.info("Task node created " + slt.toString());
  assertTrue(slt.isUnassigned(DUMMY_MASTER));
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
@Test
public void testTaskResigned() throws Exception {
  LOG.info("TestTaskResigned - resubmit task node once in RESIGNED state");
  assertEquals(tot_mgr_resubmit.get(), 0);
  slm = new SplitLogManager(zkw, conf, stopper, master, DUMMY_MASTER, null);
  slm.finishInitialization();
  assertEquals(tot_mgr_resubmit.get(), 0);
  TaskBatch batch = new TaskBatch();
  String tasknode = submitTaskAndWait(batch, "foo/1");
  assertEquals(tot_mgr_resubmit.get(), 0);
  final ServerName worker1 = new ServerName("worker1,1,1");
  assertEquals(tot_mgr_resubmit.get(), 0);
  SplitLogTask slt = new SplitLogTask.Resigned(worker1);
  assertEquals(tot_mgr_resubmit.get(), 0);
  ZKUtil.setData(zkw, tasknode, slt.toByteArray());
  int version = ZKUtil.checkExists(zkw, tasknode);
  // Could be small race here.
  if (tot_mgr_resubmit.get() == 0) waitForCounter(tot_mgr_resubmit, 0, 1, to/2);
  assertEquals(tot_mgr_resubmit.get(), 1);
  int version1 = ZKUtil.checkExists(zkw, tasknode);
  assertTrue("version1=" + version1 + ", version=" + version, version1 > version);

  byte[] taskstate = ZKUtil.getData(zkw, tasknode);
  slt = SplitLogTask.parseFrom(taskstate);
  assertTrue(slt.isUnassigned(DUMMY_MASTER));
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
@Test
public void testAcquireTaskAtStartup() throws Exception {
  LOG.info("testAcquireTaskAtStartup");
  SplitLogCounters.resetCounters();
  final String TATAS = "tatas";
  final ServerName RS = new ServerName("rs,1,1");
  zkw.getRecoverableZooKeeper().create(ZKSplitLog.getEncodedNodeName(zkw, TATAS),
    new SplitLogTask.Unassigned(new ServerName("mgr,1,1")).toByteArray(), Ids.OPEN_ACL_UNSAFE,
      CreateMode.PERSISTENT);

  SplitLogWorker slw = new SplitLogWorker(zkw, TEST_UTIL.getConfiguration(), RS, neverEndingTask);
  slw.start();
  try {
    waitForCounter(SplitLogCounters.tot_wkr_task_acquired, 0, 1, 1000);
    byte [] bytes = ZKUtil.getData(zkw, ZKSplitLog.getEncodedNodeName(zkw, TATAS));
    SplitLogTask slt = SplitLogTask.parseFrom(bytes);
    assertTrue(slt.isOwned(RS));
  } finally {
    stopSplitLogWorker(slw);
  }
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
void getDataSetWatchSuccess(String path, byte[] data) {
  SplitLogTask slt;
  try {
    slt = SplitLogTask.parseFrom(data);
  } catch (DeserializationException e) {
    LOG.warn("Failed parse", e);
    return;
  }
  synchronized (grabTaskLock) {
    if (workerInGrabTask) {
      // currentTask can change but that's ok
      String taskpath = currentTask;
      if (taskpath != null && taskpath.equals(path)) {
        ServerName serverName = manager.getServer().getServerName();
        // have to compare data. cannot compare version because then there
        // will be race with attemptToOwnTask()
        // cannot just check whether the node has been transitioned to
        // UNASSIGNED because by the time this worker sets the data watch
        // the node might have made two transitions - from owned by this
        // worker to unassigned to owned by another worker
        if (!slt.isOwned(serverName) && !slt.isDone(serverName) && !slt.isErr(serverName)
            && !slt.isResigned(serverName)) {
          LOG.info("task " + taskpath + " preempted from " + serverName
              + ", current task state and owner=" + slt.toString());
          worker.stopTask();
        }
      }
    }
  }
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
/**
 * Test whether the splitlog correctly creates a task in zookeeper
 * @throws Exception
 */
@Test (timeout=180000)
public void testTaskCreation() throws Exception {

  LOG.info("TestTaskCreation - test the creation of a task in zk");
  slm = new SplitLogManager(ds, conf, stopper, master, DUMMY_MASTER);
  TaskBatch batch = new TaskBatch();

  String tasknode = submitTaskAndWait(batch, "foo/1");

  byte[] data = ZKUtil.getData(zkw, tasknode);
  SplitLogTask slt = SplitLogTask.parseFrom(data);
  LOG.info("Task node created " + slt.toString());
  assertTrue(slt.isUnassigned(DUMMY_MASTER));
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
@Test (timeout=180000)
public void testRescanCleanup() throws Exception {
  LOG.info("TestRescanCleanup - ensure RESCAN nodes are cleaned up");

  slm = new SplitLogManager(ds, conf, stopper, master, DUMMY_MASTER);
  TaskBatch batch = new TaskBatch();

  String tasknode = submitTaskAndWait(batch, "foo/1");
  int version = ZKUtil.checkExists(zkw, tasknode);
  final ServerName worker1 = ServerName.valueOf("worker1,1,1");
  SplitLogTask slt = new SplitLogTask.Owned(worker1, this.mode);
  ZKUtil.setData(zkw, tasknode, slt.toByteArray());
  waitForCounter(tot_mgr_heartbeat, 0, 1, to/2);
  waitForCounter(new Expr() {
    @Override
    public long eval() {
      return (tot_mgr_resubmit.get() + tot_mgr_resubmit_failed.get());
    }
  }, 0, 1, 5*60000); // wait long enough
  Assert.assertEquals("Could not run test. Lost ZK connection?", 0, tot_mgr_resubmit_failed.get());
  int version1 = ZKUtil.checkExists(zkw, tasknode);
  assertTrue(version1 > version);
  byte[] taskstate = ZKUtil.getData(zkw, tasknode);
  slt = SplitLogTask.parseFrom(taskstate);
  assertTrue(slt.isUnassigned(DUMMY_MASTER));

  waitForCounter(tot_mgr_rescan_deleted, 0, 1, to/2);
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
@Test (timeout=180000)
public void testDeadWorker() throws Exception {
  LOG.info("testDeadWorker");

  conf.setLong("hbase.splitlog.max.resubmit", 0);
  slm = new SplitLogManager(ds, conf, stopper, master, DUMMY_MASTER);
  TaskBatch batch = new TaskBatch();

  String tasknode = submitTaskAndWait(batch, "foo/1");
  int version = ZKUtil.checkExists(zkw, tasknode);
  final ServerName worker1 = ServerName.valueOf("worker1,1,1");
  SplitLogTask slt = new SplitLogTask.Owned(worker1, this.mode);
  ZKUtil.setData(zkw, tasknode, slt.toByteArray());
  if (tot_mgr_heartbeat.get() == 0) waitForCounter(tot_mgr_heartbeat, 0, 1, to/2);
  slm.handleDeadWorker(worker1);
  if (tot_mgr_resubmit.get() == 0) waitForCounter(tot_mgr_resubmit, 0, 1, to+to/2);
  if (tot_mgr_resubmit_dead_server_task.get() == 0) {
    waitForCounter(tot_mgr_resubmit_dead_server_task, 0, 1, to + to/2);
  }

  int version1 = ZKUtil.checkExists(zkw, tasknode);
  assertTrue(version1 > version);
  byte[] taskstate = ZKUtil.getData(zkw, tasknode);
  slt = SplitLogTask.parseFrom(taskstate);
  assertTrue(slt.isUnassigned(DUMMY_MASTER));
  return;
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
@Test(timeout=60000)
public void testRaceForTask() throws Exception {
  LOG.info("testRaceForTask");
  SplitLogCounters.resetCounters();
  final String TRFT = "trft";
  final ServerName SVR1 = ServerName.valueOf("svr1,1,1");
  final ServerName SVR2 = ServerName.valueOf("svr2,1,1");
  zkw.getRecoverableZooKeeper().create(ZKSplitLog.getEncodedNodeName(zkw, TRFT),
    new SplitLogTask.Unassigned(MANAGER, this.mode).toByteArray(),
      Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
  RegionServerServices mockedRS1 = getRegionServer(SVR1);
  RegionServerServices mockedRS2 = getRegionServer(SVR2);
  SplitLogWorker slw1 =
      new SplitLogWorker(ds, TEST_UTIL.getConfiguration(), mockedRS1, neverEndingTask);
  SplitLogWorker slw2 =
      new SplitLogWorker(ds, TEST_UTIL.getConfiguration(), mockedRS2, neverEndingTask);
  slw1.start();
  slw2.start();
  try {
    waitForCounter(SplitLogCounters.tot_wkr_task_acquired, 0, 1, WAIT_TIME);
    // Assert that either the tot_wkr_failed_to_grab_task_owned count was set of if
    // not it, that we fell through to the next counter in line and it was set.
    assertTrue(waitForCounterBoolean(SplitLogCounters.tot_wkr_failed_to_grab_task_owned, 0, 1,
        WAIT_TIME, false) ||
      SplitLogCounters.tot_wkr_failed_to_grab_task_lost_race.get() == 1);
    byte [] bytes = ZKUtil.getData(zkw, ZKSplitLog.getEncodedNodeName(zkw, TRFT));
    SplitLogTask slt = SplitLogTask.parseFrom(bytes);
    assertTrue(slt.isOwned(SVR1) || slt.isOwned(SVR2));
  } finally {
    stopSplitLogWorker(slw1);
    stopSplitLogWorker(slw2);
  }
}
 

import org.apache.hadoop.hbase.SplitLogTask; //导入方法依赖的package包/类
@Test(timeout=60000)
public void testPreemptTask() throws Exception {
  LOG.info("testPreemptTask");
  SplitLogCounters.resetCounters();
  final ServerName SRV = ServerName.valueOf("tpt_svr,1,1");
  final String PATH = ZKSplitLog.getEncodedNodeName(zkw, "tpt_task");
  RegionServerServices mockedRS = getRegionServer(SRV);
  SplitLogWorker slw =
      new SplitLogWorker(ds, TEST_UTIL.getConfiguration(), mockedRS, neverEndingTask);
  slw.start();
  try {
    Thread.yield(); // let the worker start
    Thread.sleep(1000);
    waitForCounter(SplitLogCounters.tot_wkr_task_grabing, 0, 1, WAIT_TIME);

    // this time create a task node after starting the splitLogWorker
    zkw.getRecoverableZooKeeper().create(PATH,
      new SplitLogTask.Unassigned(MANAGER, this.mode).toByteArray(),
      Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);

    waitForCounter(SplitLogCounters.tot_wkr_task_acquired, 0, 1, WAIT_TIME);
    assertEquals(1, slw.getTaskReadySeq());
    byte [] bytes = ZKUtil.getData(zkw, PATH);
    SplitLogTask slt = SplitLogTask.parseFrom(bytes);
    assertTrue(slt.isOwned(SRV));
    slt = new SplitLogTask.Owned(MANAGER, this.mode);
    ZKUtil.setData(zkw, PATH, slt.toByteArray());
    waitForCounter(SplitLogCounters.tot_wkr_preempt_task, 0, 1, WAIT_TIME);
  } finally {
    stopSplitLogWorker(slw);
  }
}