Java ServerCallable类代码示例

import org.apache.hadoop.hbase.client.ServerCallable; //导入依赖的package包/类
@SuppressWarnings({ "deprecation" })
public static void putAndCommit(HTable table, final Put put) throws IOException {
  try {
    table.getConnection().getRegionServerWithRetries(
      new ServerCallable<Boolean>(table.getConnection(), table.getTableName(), put.getRow()) {
        public Boolean call() throws Exception {
          server.put(location.getRegionInfo().getRegionName(), put);
          return true;
        }
      });
  } catch (Exception e) {
    throw new IOException(e);
  }
}
 

import org.apache.hadoop.hbase.client.ServerCallable; //导入依赖的package包/类
/**
 * Attempts to do an atomic load of many hfiles into a region.  If it fails,
 * it returns a list of hfiles that need to be retried.  If it is successful
 * it will return an empty list.
 * 
 * NOTE: To maintain row atomicity guarantees, region server callable should
 * succeed atomically and fails atomically.
 * 
 * Protected for testing.
 * 
 * @return empty list if success, list of items to retry on recoverable
 * failure
 */
protected List<LoadQueueItem> tryAtomicRegionLoad(final HConnection conn,
    byte[] tableName, final byte[] first, Collection<LoadQueueItem> lqis) throws IOException {

  final List<Pair<byte[], String>> famPaths =
    new ArrayList<Pair<byte[], String>>(lqis.size());
  for (LoadQueueItem lqi : lqis) {
    famPaths.add(Pair.newPair(lqi.family, lqi.hfilePath.toString()));
  }

  final ServerCallable<Boolean> svrCallable = new ServerCallable<Boolean>(conn,
      tableName, first) {
    @Override
    public Boolean call() throws Exception {
      LOG.debug("Going to connect to server " + location + " for row "
          + Bytes.toStringBinary(row));
      byte[] regionName = location.getRegionInfo().getRegionName();
      return server.bulkLoadHFiles(famPaths, regionName);
    }
  };

  try {
    List<LoadQueueItem> toRetry = new ArrayList<LoadQueueItem>();
    boolean success = conn.getRegionServerWithRetries(svrCallable);
    if (!success) {
      LOG.warn("Attempt to bulk load region containing "
          + Bytes.toStringBinary(first) + " into table "
          + Bytes.toStringBinary(tableName)  + " with files " + lqis
          + " failed.  This is recoverable and they will be retried.");
      toRetry.addAll(lqis); // return lqi's to retry
    }
    // success
    return toRetry;
  } catch (IOException e) {
    LOG.error("Encountered unrecoverable error from region server", e);
    throw e;
  }
}
 

import org.apache.hadoop.hbase.client.ServerCallable; //导入依赖的package包/类
/**
 * Attempts to do an atomic load of many hfiles into a region.  If it fails,
 * it returns a list of hfiles that need to be retried.  If it is successful
 * it will return an empty list.
 * 
 * NOTE: To maintain row atomicity guarantees, region server callable should
 * succeed atomically and fails atomically.
 * 
 * Protected for testing.
 * 
 * @return empty list if success, list of items to retry on recoverable
 * failure
 */
protected List<LoadQueueItem> tryAtomicRegionLoad(final HConnection conn,
    byte[] tableName, final byte[] first, Collection<LoadQueueItem> lqis) throws IOException {

  final List<Pair<byte[], String>> famPaths =
    new ArrayList<Pair<byte[], String>>(lqis.size());
  for (LoadQueueItem lqi : lqis) {
    famPaths.add(Pair.newPair(lqi.family, lqi.hfilePath.toString()));
  }

  final ServerCallable<Boolean> svrCallable = new ServerCallable<Boolean>(conn,
      tableName, first) {
    @Override
    public Boolean call() throws Exception {
      LOG.debug("Going to connect to server " + location + " for row "
          + Bytes.toStringBinary(row));
      byte[] regionName = location.getRegionInfo().getRegionName();
      return ProtobufUtil.bulkLoadHFile(server, famPaths, regionName,
          assignSeqIds);
    }
  };

  try {
    List<LoadQueueItem> toRetry = new ArrayList<LoadQueueItem>();
    boolean success = svrCallable.withRetries();
    if (!success) {
      LOG.warn("Attempt to bulk load region containing "
          + Bytes.toStringBinary(first) + " into table "
          + Bytes.toStringBinary(tableName)  + " with files " + lqis
          + " failed.  This is recoverable and they will be retried.");
      toRetry.addAll(lqis); // return lqi's to retry
    }
    // success
    return toRetry;
  } catch (IOException e) {
    LOG.error("Encountered unrecoverable error from region server", e);
    throw e;
  }
}
 

import org.apache.hadoop.hbase.client.ServerCallable; //导入依赖的package包/类
@Override
protected Message callExecService(Descriptors.MethodDescriptor method,
                                Message request, Message responsePrototype)
    throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("Call: "+method.getName()+", "+request.toString());
  }

  if (row == null) {
    throw new IllegalArgumentException("Missing row property for remote region location");
  }

  final ClientProtos.CoprocessorServiceCall call =
      ClientProtos.CoprocessorServiceCall.newBuilder()
          .setRow(ByteString.copyFrom(row))
          .setServiceName(method.getService().getFullName())
          .setMethodName(method.getName())
          .setRequest(request.toByteString()).build();
  ServerCallable<CoprocessorServiceResponse> callable =
      new ServerCallable<CoprocessorServiceResponse>(connection, table, row) {
        public CoprocessorServiceResponse call() throws Exception {
          byte[] regionName = location.getRegionInfo().getRegionName();
          return ProtobufUtil.execService(server, call, regionName);
        }
      };
  CoprocessorServiceResponse result = callable.withRetries();
  Message response = null;
  if (result.getValue().hasValue()) {
    response = responsePrototype.newBuilderForType()
        .mergeFrom(result.getValue().getValue()).build();
  } else {
    response = responsePrototype.getDefaultInstanceForType();
  }
  lastRegion = result.getRegion().getValue().toByteArray();
  if (LOG.isTraceEnabled()) {
    LOG.trace("Result is region=" + Bytes.toStringBinary(lastRegion) + ", value=" + response);
  }
  return response;
}
 

import org.apache.hadoop.hbase.client.ServerCallable; //导入依赖的package包/类
/**
 * Attempts to do an atomic load of many hfiles into a region. If it fails, it returns a list of
 * hfiles that need to be retried. If it is successful it will return an empty list. NOTE: To
 * maintain row atomicity guarantees, region server callable should succeed atomically and fails
 * atomically. Protected for testing.
 * @return empty list if success, list of items to retry on recoverable failure
 */
protected List<LoadQueueItem> tryAtomicRegionLoad(final HConnection conn, byte[] tableName,
    final byte[] first, Collection<LoadQueueItem> lqis) throws IOException {

  final List<Pair<byte[], String>> famPaths = new ArrayList<Pair<byte[], String>>(lqis.size());
  for (LoadQueueItem lqi : lqis) {
    famPaths.add(Pair.newPair(lqi.family, lqi.hfilePath.toString()));
  }

  final ServerCallable<Boolean> svrCallable =
      new ServerCallable<Boolean>(conn, tableName, first) {
        @Override
        public Boolean call() throws Exception {
          if (LOG.isDebugEnabled()) {
            LOG.debug("Going to connect to server " + location + " for row "
                + Bytes.toStringBinary(row));
          }
          byte[] regionName = location.getRegionInfo().getRegionName();
          return server.bulkLoadHFiles(famPaths, regionName);
        }
      };

  try {
    List<LoadQueueItem> toRetry = new ArrayList<LoadQueueItem>();
    boolean success = svrCallable.withRetries();
    if (!success) {
      LOG.warn("Attempt to bulk load region containing " + Bytes.toStringBinary(first)
          + " into table " + Bytes.toStringBinary(tableName) + " with files " + lqis
          + " failed.  This is recoverable and they will be retried.");
      toRetry.addAll(lqis); // return lqi's to retry
    }
    // success
    return toRetry;
  } catch (IOException e) {
    LOG.error("Encountered unrecoverable error from region server", e);
    throw e;
  }
}
 

import org.apache.hadoop.hbase.client.ServerCallable; //导入依赖的package包/类
/**
 * Test for HBASE-4288.  Throw an IOE when trying to verify meta region and
 * prove it doesn't cause master shutdown.
 * @see <a href="https://issues.apache.org/jira/browse/HBASE-4288">HBASE-4288</a>
 * @throws IOException
 * @throws InterruptedException
 * @throws KeeperException
 */
@Test
public void testServerNotRunningIOException()
throws IOException, InterruptedException, KeeperException {
  // Mock an HRegionInterface.
  final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
  HConnection connection = mockConnection(implementation);

  // If a 'getRegionInfo' is called on mocked HRegionInterface, throw IOE
  // the first time.  'Succeed' the second time we are called.
  Mockito.when(implementation.getRegionInfo((byte[]) Mockito.any())).
    thenThrow(new IOException("Server not running, aborting")).
    thenReturn(new HRegionInfo());

  // After we encounter the above 'Server not running', we should catch the
  // IOE and go into retrying for the meta mode.  We'll do gets on -ROOT- to
  // get new meta location.  Return something so this 'get' succeeds
  // (here we mock up getRegionServerWithRetries, the wrapper around
  // the actual get).

  // TODO: Refactor.  This method has been moved out of HConnection.
  // It works for now but has been deprecated.
  Mockito.when(connection.getRegionServerWithRetries((ServerCallable<Result>)Mockito.any())).
    thenReturn(getMetaTableRowResult());

  // Now start up the catalogtracker with our doctored Connection.
  final CatalogTracker ct = constructAndStartCatalogTracker(connection);
  try {
    // Set a location for root and meta.
    RootLocationEditor.setRootLocation(this.watcher, SN);
    ct.setMetaLocation(SN);
    // Call the method that HBASE-4288 calls.  It will try and verify the
    // meta location and will fail on first attempt then go into a long wait.
    // So, do this in a thread and then reset meta location to break it out
    // of its wait after a bit of time.
    final AtomicBoolean metaSet = new AtomicBoolean(false);
    final CountDownLatch latch = new CountDownLatch(1);
    Thread t = new Thread() {
      @Override
      public void run() {
        try {
          latch.countDown();
          metaSet.set(ct.waitForMeta(100000) !=  null);
        } catch (Exception e) {
          throw new RuntimeException(e);
        }
      }
    };
    t.start();
    latch.await();
    Threads.sleep(1);
    // Now reset the meta as though it were redeployed.
    ct.setMetaLocation(SN);
    t.join();
    Assert.assertTrue(metaSet.get());
  } finally {
    // Clean out root and meta locations or later tests will be confused...
    // they presume start fresh in zk.
    ct.resetMetaLocation();
  }
}
 

import org.apache.hadoop.hbase.client.ServerCallable; //导入依赖的package包/类
/**
 * Test waiting on meta w/ no timeout specified.
 * @throws Exception 
 */
@Ignore // Can't make it work reliably on all platforms; mockito gets confused
// Throwing: org.mockito.exceptions.misusing.WrongTypeOfReturnValue:
// Result cannot be returned by locateRegion()
// If you plug locateRegion, it then throws for incCounter, and if you plug
// that ... and so one.
@Test public void testNoTimeoutWaitForMeta()
throws Exception {
  // Mock an HConnection and a HRegionInterface implementation.  Have the
  // HConnection return the HRI.  Have the HRI return a few mocked up responses
  // to make our test work.
  // Mock an HRegionInterface.
  final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
  HConnection connection = mockConnection(implementation);

  // Now the ct is up... set into the mocks some answers that make it look
  // like things have been getting assigned. Make it so we'll return a
  // location (no matter what the Get is). Same for getHRegionInfo -- always
  // just return the meta region.
  final Result result = getMetaTableRowResult();

  // TODO: Refactor.  This method has been moved out of HConnection.
  // It works for now but has been deprecated.
  Mockito.when(connection.getRegionServerWithRetries((ServerCallable<Result>)Mockito.any())).
    thenReturn(result);
  Mockito.when(implementation.getRegionInfo((byte[]) Mockito.any())).
    thenReturn(HRegionInfo.FIRST_META_REGIONINFO);
  final CatalogTracker ct = constructAndStartCatalogTracker(connection);
  ServerName hsa = ct.getMetaLocation();
  Assert.assertNull(hsa);

  // Now test waiting on meta location getting set.
  Thread t = new WaitOnMetaThread(ct) {
    @Override
    void doWaiting() throws InterruptedException {
      this.ct.waitForMeta();
    }
  };
  startWaitAliveThenWaitItLives(t, 1000);

  // This should trigger wake up of meta wait (Its the removal of the meta
  // region unassigned node that triggers catalogtrackers that a meta has
  // been assigned).
  String node = ct.getMetaNodeTracker().getNode();
  ZKUtil.createAndFailSilent(this.watcher, node);
  MetaEditor.updateMetaLocation(ct, HRegionInfo.FIRST_META_REGIONINFO, SN);
  ZKUtil.deleteNode(this.watcher, node);
  // Go get the new meta location. waitForMeta gets and verifies meta.
  Assert.assertTrue(ct.waitForMeta(10000).equals(SN));
  // Join the thread... should exit shortly.
  t.join();
  // Now meta is available.
  Assert.assertTrue(ct.waitForMeta(10000).equals(SN));
}
 

import org.apache.hadoop.hbase.client.ServerCallable; //导入依赖的package包/类
public void doAnAction() throws Exception {
  long iteration = numBulkLoads.getAndIncrement();
  Path dir =  UTIL.getDataTestDir(String.format("bulkLoad_%08d",
      iteration));

  // create HFiles for different column families
  FileSystem fs = UTIL.getTestFileSystem();
  byte[] val = Bytes.toBytes(String.format("%010d", iteration));
  final List<Pair<byte[], String>> famPaths = new ArrayList<Pair<byte[], String>>(
      NUM_CFS);
  for (int i = 0; i < NUM_CFS; i++) {
    Path hfile = new Path(dir, family(i));
    byte[] fam = Bytes.toBytes(family(i));
    createHFile(fs, hfile, fam, QUAL, val, 1000);
    famPaths.add(new Pair<byte[], String>(fam, hfile.toString()));
  }

  // bulk load HFiles
  HConnection conn = UTIL.getHBaseAdmin().getConnection();
  byte[] tbl = Bytes.toBytes(tableName);
  new ServerCallable<Void>(conn, tbl, Bytes
      .toBytes("aaa")) {
    @Override
    public Void call() throws Exception {
      LOG.debug("Going to connect to server " + location + " for row "
          + Bytes.toStringBinary(row));
      byte[] regionName = location.getRegionInfo().getRegionName();
      server.bulkLoadHFiles(famPaths, regionName, true);
      return null;
    }
  }.withRetries();

  // Periodically do compaction to reduce the number of open file handles.
  if (numBulkLoads.get() % 10 == 0) {
    // 10 * 50 = 500 open file handles!
    new ServerCallable<Void>(conn, tbl,
        Bytes.toBytes("aaa")) {
      @Override
      public Void call() throws Exception {
        LOG.debug("compacting " + location + " for row "
            + Bytes.toStringBinary(row));
        server.compactRegion(location.getRegionInfo(), true);
        numCompactions.incrementAndGet();
        return null;
      }
    }.withRetries();
  }
}
 

import org.apache.hadoop.hbase.client.ServerCallable; //导入依赖的package包/类
/**
 * Test for HBASE-4288.  Throw an IOE when trying to verify meta region and
 * prove it doesn't cause master shutdown.
 * @see <a href="https://issues.apache.org/jira/browse/HBASE-4288">HBASE-4288</a>
 * @throws IOException
 * @throws InterruptedException
 * @throws KeeperException
 */
@Test
public void testServerNotRunningIOException()
throws IOException, InterruptedException, KeeperException {
  // Mock an HRegionInterface.
  final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
  HConnection connection = mockConnection(implementation);
  try {
    // If a 'getRegionInfo' is called on mocked HRegionInterface, throw IOE
    // the first time.  'Succeed' the second time we are called.
    Mockito.when(implementation.getRegionInfo((byte[]) Mockito.any())).
      thenThrow(new IOException("Server not running, aborting")).
      thenReturn(new HRegionInfo());

    // After we encounter the above 'Server not running', we should catch the
    // IOE and go into retrying for the meta mode.  We'll do gets on -ROOT- to
    // get new meta location.  Return something so this 'get' succeeds
    // (here we mock up getRegionServerWithRetries, the wrapper around
    // the actual get).

    // TODO: Refactor.  This method has been moved out of HConnection.
    // It works for now but has been deprecated.
    Mockito.when(connection.getRegionServerWithRetries((ServerCallable<Result>)Mockito.any())).
      thenReturn(getMetaTableRowResult());

    // Now start up the catalogtracker with our doctored Connection.
    final CatalogTracker ct = constructAndStartCatalogTracker(connection);
    try {
      // Set a location for root and meta.
      RootLocationEditor.setRootLocation(this.watcher, SN);
      ct.setMetaLocation(SN);
      // Call the method that HBASE-4288 calls.  It will try and verify the
      // meta location and will fail on first attempt then go into a long wait.
      // So, do this in a thread and then reset meta location to break it out
      // of its wait after a bit of time.
      final AtomicBoolean metaSet = new AtomicBoolean(false);
      final CountDownLatch latch = new CountDownLatch(1);
      Thread t = new Thread() {
        @Override
        public void run() {
          try {
            latch.countDown();
            metaSet.set(ct.waitForMeta(100000) !=  null);
          } catch (Exception e) {
            throw new RuntimeException(e);
          }
        }
      };
      t.start();
      latch.await();
      Threads.sleep(1);
      // Now reset the meta as though it were redeployed.
      ct.setMetaLocation(SN);
      t.join();
      Assert.assertTrue(metaSet.get());
    } finally {
      // Clean out root and meta locations or later tests will be confused...
      // they presume start fresh in zk.
      ct.resetMetaLocation();
      RootLocationEditor.deleteRootLocation(this.watcher);
    }
  } finally {
    // Clear out our doctored connection or could mess up subsequent tests.
    HConnectionManager.deleteConnection(UTIL.getConfiguration());
  }
}
 

import org.apache.hadoop.hbase.client.ServerCallable; //导入依赖的package包/类
/**
 * Test waiting on meta w/ no timeout specified.
 * @throws Exception 
 */
@Ignore // Can't make it work reliably on all platforms; mockito gets confused
// Throwing: org.mockito.exceptions.misusing.WrongTypeOfReturnValue:
// Result cannot be returned by locateRegion()
// If you plug locateRegion, it then throws for incCounter, and if you plug
// that ... and so one.
@Test public void testNoTimeoutWaitForMeta()
throws Exception {
  // Mock an HConnection and a HRegionInterface implementation.  Have the
  // HConnection return the HRI.  Have the HRI return a few mocked up responses
  // to make our test work.
  // Mock an HRegionInterface.
  final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
  HConnection connection = mockConnection(implementation);
  try {
    // Now the ct is up... set into the mocks some answers that make it look
    // like things have been getting assigned. Make it so we'll return a
    // location (no matter what the Get is). Same for getHRegionInfo -- always
    // just return the meta region.
    final Result result = getMetaTableRowResult();

    // TODO: Refactor.  This method has been moved out of HConnection.
    // It works for now but has been deprecated.
    Mockito.when(connection.getRegionServerWithRetries((ServerCallable<Result>)Mockito.any())).
      thenReturn(result);
    Mockito.when(implementation.getRegionInfo((byte[]) Mockito.any())).
      thenReturn(HRegionInfo.FIRST_META_REGIONINFO);
    final CatalogTracker ct = constructAndStartCatalogTracker(connection);
    ServerName hsa = ct.getMetaLocation();
    Assert.assertNull(hsa);

    // Now test waiting on meta location getting set.
    Thread t = new WaitOnMetaThread(ct) {
      @Override
      void doWaiting() throws InterruptedException {
        this.ct.waitForMeta();
      }
    };
    startWaitAliveThenWaitItLives(t, 1000);

    // This should trigger wake up of meta wait (Its the removal of the meta
    // region unassigned node that triggers catalogtrackers that a meta has
    // been assigned).
    String node = ct.getMetaNodeTracker().getNode();
    ZKUtil.createAndFailSilent(this.watcher, node);
    MetaEditor.updateMetaLocation(ct, HRegionInfo.FIRST_META_REGIONINFO, SN);
    ZKUtil.deleteNode(this.watcher, node);
    // Go get the new meta location. waitForMeta gets and verifies meta.
    Assert.assertTrue(ct.waitForMeta(10000).equals(SN));
    // Join the thread... should exit shortly.
    t.join();
    // Now meta is available.
    Assert.assertTrue(ct.waitForMeta(10000).equals(SN));
  } finally {
    HConnectionManager.deleteConnection(UTIL.getConfiguration());
  }
}
 

import org.apache.hadoop.hbase.client.ServerCallable; //导入依赖的package包/类
public void doAnAction() throws Exception {
  long iteration = numBulkLoads.getAndIncrement();
  Path dir =  UTIL.getDataTestDir(String.format("bulkLoad_%08d",
      iteration));

  // create HFiles for different column families
  FileSystem fs = UTIL.getTestFileSystem();
  byte[] val = Bytes.toBytes(String.format("%010d", iteration));
  final List<Pair<byte[], String>> famPaths = new ArrayList<Pair<byte[], String>>(
      NUM_CFS);
  for (int i = 0; i < NUM_CFS; i++) {
    Path hfile = new Path(dir, family(i));
    byte[] fam = Bytes.toBytes(family(i));
    createHFile(fs, hfile, fam, QUAL, val, 1000);
    famPaths.add(new Pair<byte[], String>(fam, hfile.toString()));
  }

  // bulk load HFiles
  HConnection conn = UTIL.getHBaseAdmin().getConnection();
  byte[] tbl = Bytes.toBytes(tableName);
  new ServerCallable<Void>(conn, tbl, Bytes
      .toBytes("aaa")) {
    @Override
    public Void call() throws Exception {
      LOG.debug("Going to connect to server " + location + " for row "
          + Bytes.toStringBinary(row));
      byte[] regionName = location.getRegionInfo().getRegionName();
      server.bulkLoadHFiles(famPaths, regionName);
      return null;
    }
  }.withRetries();

  // Periodically do compaction to reduce the number of open file handles.
  if (numBulkLoads.get() % 10 == 0) {
    // 10 * 50 = 500 open file handles!
    new ServerCallable<Void>(conn, tbl,
        Bytes.toBytes("aaa")) {
      @Override
      public Void call() throws Exception {
        LOG.debug("compacting " + location + " for row "
            + Bytes.toStringBinary(row));
        server.compactRegion(location.getRegionInfo(), true);
        numCompactions.incrementAndGet();
        return null;
      }
    }.withRetries();
  }
}
 

import org.apache.hadoop.hbase.client.ServerCallable; //导入依赖的package包/类
/**
 * Test waiting on meta w/ no timeout specified.
 * @throws Exception
 */
@Ignore // Can't make it work reliably on all platforms; mockito gets confused
// Throwing: org.mockito.exceptions.misusing.WrongTypeOfReturnValue:
// Result cannot be returned by locateRegion()
// If you plug locateRegion, it then throws for incCounter, and if you plug
// that ... and so one.
@Test public void testNoTimeoutWaitForMeta()
throws Exception {
  // Mock an HConnection and a AdminProtocol implementation.  Have the
  // HConnection return the HRI.  Have the HRI return a few mocked up responses
  // to make our test work.
  // Mock an AdminProtocol.
  final AdminProtocol implementation = Mockito.mock(AdminProtocol.class);
  HConnection connection = mockConnection(implementation, null);
  try {
    // Now the ct is up... set into the mocks some answers that make it look
    // like things have been getting assigned. Make it so we'll return a
    // location (no matter what the Get is). Same for getHRegionInfo -- always
    // just return the meta region.
    final Result result = getMetaTableRowResult();

    // TODO: Refactor.  This method has been moved out of HConnection.
    // It works for now but has been deprecated.
    Mockito.when(connection.getRegionServerWithRetries((ServerCallable<Result>)Mockito.any())).
      thenReturn(result);
    GetRegionInfoResponse.Builder builder = GetRegionInfoResponse.newBuilder();
    builder.setRegionInfo(HRegionInfo.convert(HRegionInfo.FIRST_META_REGIONINFO));
    Mockito.when(implementation.getRegionInfo((RpcController)Mockito.any(),
      (GetRegionInfoRequest)Mockito.any())).thenReturn(builder.build());
    final CatalogTracker ct = constructAndStartCatalogTracker(connection);
    ServerName hsa = ct.getMetaLocation();
    Assert.assertNull(hsa);

    // Now test waiting on meta location getting set.
    Thread t = new WaitOnMetaThread(ct) {
      @Override
      void doWaiting() throws InterruptedException {
        this.ct.waitForMeta();
      }
    };
    startWaitAliveThenWaitItLives(t, 1000);

    // This should trigger wake up of meta wait (Its the removal of the meta
    // region unassigned node that triggers catalogtrackers that a meta has
    // been assigned).
    String node = ct.getMetaNodeTracker().getNode();
    ZKUtil.createAndFailSilent(this.watcher, node);
    MetaEditor.updateMetaLocation(ct, HRegionInfo.FIRST_META_REGIONINFO, SN);
    ZKUtil.deleteNode(this.watcher, node);
    // Go get the new meta location. waitForMeta gets and verifies meta.
    Assert.assertTrue(ct.waitForMeta(10000).equals(SN));
    // Join the thread... should exit shortly.
    t.join();
    // Now meta is available.
    Assert.assertTrue(ct.waitForMeta(10000).equals(SN));
  } finally {
    HConnectionManager.deleteConnection(UTIL.getConfiguration(), true);
  }
}