Java DefaultWALProvider.getWALDirectoryName方法代码示例

import org.apache.hadoop.hbase.wal.DefaultWALProvider; //导入方法依赖的package包/类
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value="UL_UNRELEASED_LOCK", justification=
    "We only release this lock when we set it. Updates to code that uses it should verify use " +
    "of the guard boolean.")
private List<Path> getLogDirs(final Set<ServerName> serverNames) throws IOException {
  List<Path> logDirs = new ArrayList<Path>();
  boolean needReleaseLock = false;
  if (!this.services.isInitialized()) {
    // during master initialization, we could have multiple places splitting a same wal
    this.splitLogLock.lock();
    needReleaseLock = true;
  }
  try {
    for (ServerName serverName : serverNames) {
      Path logDir = new Path(this.rootdir,
          DefaultWALProvider.getWALDirectoryName(serverName.toString()));
      Path splitDir = logDir.suffix(DefaultWALProvider.SPLITTING_EXT);
      // Rename the directory so a rogue RS doesn't create more WALs
      if (fs.exists(logDir)) {
        if (!this.fs.rename(logDir, splitDir)) {
          throw new IOException("Failed fs.rename for log split: " + logDir);
        }
        logDir = splitDir;
        LOG.debug("Renamed region directory: " + splitDir);
      } else if (!fs.exists(splitDir)) {
        LOG.info("Log dir for server " + serverName + " does not exist");
        continue;
      }
      logDirs.add(splitDir);
    }
  } finally {
    if (needReleaseLock) {
      this.splitLogLock.unlock();
    }
  }
  return logDirs;
}
 

import org.apache.hadoop.hbase.wal.DefaultWALProvider; //导入方法依赖的package包/类
@Before
public void setUp() throws Exception {
  this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
  // this.cluster = TEST_UTIL.getDFSCluster();
  this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
  this.hbaseRootDir = FSUtils.getRootDir(conf);
  this.dir = new Path(this.hbaseRootDir, TestWALObserver.class.getName());
  this.oldLogDir = new Path(this.hbaseRootDir,
      HConstants.HREGION_OLDLOGDIR_NAME);
  this.logDir = new Path(this.hbaseRootDir,
      DefaultWALProvider.getWALDirectoryName(currentTest.getMethodName()));
  this.logName = HConstants.HREGION_LOGDIR_NAME;

  if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
    TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
  }
  this.wals = new WALFactory(conf, null, currentTest.getMethodName());
}
 

import org.apache.hadoop.hbase.wal.DefaultWALProvider; //导入方法依赖的package包/类
/**
 *
 * @param serverName the 'unique-over-restarts' name, i.e. hostname with start code suffix
 * @param hlogName name of HLog
 */
private long getLogFileSize(String serverName, String hlogName) throws IOException {
    Path hbaseLogDir = new Path(hbaseRootDir, DefaultWALProvider.getWALDirectoryName(serverName));
    Path path = new Path(hbaseLogDir, hlogName);
    try {
        FileStatus status = fileSystem.getFileStatus(path);
        return status.getLen();
    } catch (FileNotFoundException e) {
        Path oldLogPath = new Path(hbaseOldLogDir, hlogName);
        try {
            return fileSystem.getFileStatus(oldLogPath).getLen();
        } catch (FileNotFoundException e2) {
            // TODO there is still another place to look for log files, cfr dead region servers, see openReader in replicationsource
            System.err.println("HLog not found at : " + path + " or " + oldLogPath);
            return -1;
        }
    }
}
 

import org.apache.hadoop.hbase.wal.DefaultWALProvider; //导入方法依赖的package包/类
/**
 * Setup WAL log and replication if enabled.
 * Replication setup is done in here because it wants to be hooked up to WAL.
 *
 * @return A WAL instance.
 * @throws IOException
 */
private WALFactory setupWALAndReplication() throws IOException {
  // TODO Replication make assumptions here based on the default filesystem impl
  final Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
  final String logName = DefaultWALProvider.getWALDirectoryName(this.serverName.toString());

  Path logdir = new Path(rootDir, logName);
  if (LOG.isDebugEnabled()) LOG.debug("logdir=" + logdir);
  if (this.fs.exists(logdir)) {
    throw new RegionServerRunningException(
        "Region server has already " + "created directory at " + this.serverName.toString());
  }

  // Instantiate replication manager if replication enabled.  Pass it the
  // log directories.
  createNewReplicationInstance(conf, this, this.fs, logdir, oldLogDir);

  // listeners the wal factory will add to wals it creates.
  final List<WALActionsListener> listeners = new ArrayList<WALActionsListener>();
  listeners.add(new MetricsWAL());
  if (this.replicationSourceHandler != null
      && this.replicationSourceHandler.getWALActionsListener() != null) {
    // Replication handler is an implementation of WALActionsListener.
    listeners.add(this.replicationSourceHandler.getWALActionsListener());
  }

  return new WALFactory(conf, listeners, serverName.toString());
}
 

import org.apache.hadoop.hbase.wal.DefaultWALProvider; //导入方法依赖的package包/类
@Before
public void setUp() throws Exception {
  this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
  this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
  this.hbaseRootDir = FSUtils.getRootDir(this.conf);
  this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
  this.logName = DefaultWALProvider.getWALDirectoryName(currentTest.getMethodName() + "-manual");
  this.logDir = new Path(this.hbaseRootDir, logName);
  if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
    TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
  }
  this.mode = (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false) ?
      RecoveryMode.LOG_REPLAY : RecoveryMode.LOG_SPLITTING);
  this.wals = new WALFactory(conf, null, currentTest.getMethodName());
}
 

import org.apache.hadoop.hbase.wal.DefaultWALProvider; //导入方法依赖的package包/类
/**
 * Tests the case where a RegionServer enters a GC pause,
 * comes back online after the master declared it dead and started to split.
 * Want log rolling after a master split to fail. See HBASE-2312.
 */
@Test (timeout=300000)
public void testLogRollAfterSplitStart() throws IOException {
  LOG.info("Verify wal roll after split starts will fail.");
  String logName = "testLogRollAfterSplitStart";
  Path thisTestsDir = new Path(HBASEDIR, DefaultWALProvider.getWALDirectoryName(logName));
  final WALFactory wals = new WALFactory(conf, null, logName);

  try {
    // put some entries in an WAL
    TableName tableName =
        TableName.valueOf(this.getClass().getName());
    HRegionInfo regioninfo = new HRegionInfo(tableName,
        HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
    final WAL log = wals.getWAL(regioninfo.getEncodedNameAsBytes());
    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);

    final int total = 20;
    for (int i = 0; i < total; i++) {
      WALEdit kvs = new WALEdit();
      kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));
      HTableDescriptor htd = new HTableDescriptor(tableName);
      htd.addFamily(new HColumnDescriptor("column"));
      log.append(htd, regioninfo, new WALKey(regioninfo.getEncodedNameAsBytes(), tableName,
          System.currentTimeMillis(), mvcc), kvs, true);
    }
    // Send the data to HDFS datanodes and close the HDFS writer
    log.sync();
    ((FSHLog) log).replaceWriter(((FSHLog)log).getOldPath(), null, null, null);

    /* code taken from MasterFileSystem.getLogDirs(), which is called from MasterFileSystem.splitLog()
     * handles RS shutdowns (as observed by the splitting process)
     */
    // rename the directory so a rogue RS doesn't create more WALs
    Path rsSplitDir = thisTestsDir.suffix(DefaultWALProvider.SPLITTING_EXT);
    if (!fs.rename(thisTestsDir, rsSplitDir)) {
      throw new IOException("Failed fs.rename for log split: " + thisTestsDir);
    }
    LOG.debug("Renamed region directory: " + rsSplitDir);

    LOG.debug("Processing the old log files.");
    WALSplitter.split(HBASEDIR, rsSplitDir, OLDLOGDIR, fs, conf, wals);

    LOG.debug("Trying to roll the WAL.");
    try {
      log.rollWriter();
      Assert.fail("rollWriter() did not throw any exception.");
    } catch (IOException ioe) {
      if (ioe.getCause() instanceof FileNotFoundException) {
        LOG.info("Got the expected exception: ", ioe.getCause());
      } else {
        Assert.fail("Unexpected exception: " + ioe);
      }
    }
  } finally {
    wals.close();
    if (fs.exists(thisTestsDir)) {
      fs.delete(thisTestsDir, true);
    }
  }
}
 

import org.apache.hadoop.hbase.wal.DefaultWALProvider; //导入方法依赖的package包/类
@Override
public void setUp() throws Exception {
  // setup config values necessary for store
  this.conf = TEST_UTIL.getConfiguration();
  this.conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0);
  this.conf.setInt("hbase.hstore.compaction.min", minFiles);
  this.conf.setInt("hbase.hstore.compaction.max", maxFiles);
  this.conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, minSize);
  this.conf.setLong("hbase.hstore.compaction.max.size", maxSize);
  this.conf.setFloat("hbase.hstore.compaction.ratio", 1.0F);

  //Setting up a Store
  final String id = TestDefaultCompactSelection.class.getName();
  Path basedir = new Path(DIR);
  final Path logdir = new Path(basedir, DefaultWALProvider.getWALDirectoryName(id));
  HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes("family"));
  FileSystem fs = FileSystem.get(conf);

  fs.delete(logdir, true);

  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(Bytes.toBytes("table")));
  htd.addFamily(hcd);
  HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);

  final Configuration walConf = new Configuration(conf);
  FSUtils.setRootDir(walConf, basedir);
  wals = new WALFactory(walConf, null, id);
  region = HRegion.createHRegion(info, basedir, conf, htd);
  HRegion.closeHRegion(region);
  Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName());
  region = new HRegion(tableDir, wals.getWAL(info.getEncodedNameAsBytes()), fs, conf, info, htd,
      null);

  store = new HStore(region, hcd, conf);

  TEST_FILE = region.getRegionFileSystem().createTempName();
  fs.createNewFile(TEST_FILE);
}
 

import org.apache.hadoop.hbase.wal.DefaultWALProvider; //导入方法依赖的package包/类
@SuppressWarnings("deprecation")
private Store init(String methodName, Configuration conf, HTableDescriptor htd,
    HColumnDescriptor hcd) throws IOException {
  //Setting up a Store
  Path basedir = new Path(DIR+methodName);
  Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName());
  final Path logdir = new Path(basedir, DefaultWALProvider.getWALDirectoryName(methodName));

  FileSystem fs = FileSystem.get(conf);

  fs.delete(logdir, true);

  if (htd.hasFamily(hcd.getName())) {
    htd.modifyFamily(hcd);
  } else {
    htd.addFamily(hcd);
  }
  HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
  final Configuration walConf = new Configuration(conf);
  FSUtils.setRootDir(walConf, basedir);
  final WALFactory wals = new WALFactory(walConf, null, methodName);
  HRegion region = new HRegion(tableDir, wals.getWAL(info.getEncodedNameAsBytes()), fs, conf,
      info, htd, null);

  store = new HStore(region, hcd, conf);
  return store;
}
 

import org.apache.hadoop.hbase.wal.DefaultWALProvider; //导入方法依赖的package包/类
@Before
public void setUp() throws IOException {
  // parameterized tests add [#] suffix get rid of [ and ].
  table = Bytes.toBytes(name.getMethodName().replaceAll("[\\[\\]]", "_"));

  conf = TEST_UTIL.getConfiguration();
  conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MAX_FORMAT_VERSION);
  conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, false);
  conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, false);
  conf.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, false);

  fs = HFileSystem.get(conf);

  // Create the schema
  HColumnDescriptor hcd = new HColumnDescriptor(family);
  hcd.setBloomFilterType(BloomType.ROWCOL);
  cowType.modifyFamilySchema(hcd);
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
  htd.addFamily(hcd);

  // Create a store based on the schema
  final String id = TestCacheOnWriteInSchema.class.getName();
  final Path logdir = new Path(FSUtils.getRootDir(conf),
      DefaultWALProvider.getWALDirectoryName(id));
  fs.delete(logdir, true);

  HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
  walFactory = new WALFactory(conf, null, id);

  region = TEST_UTIL.createLocalHRegion(info, htd,
      walFactory.getWAL(info.getEncodedNameAsBytes()));
  store = new HStore(region, hcd, conf);
}
 

import org.apache.hadoop.hbase.wal.DefaultWALProvider; //导入方法依赖的package包/类
/**
 * Setup WAL log and replication if enabled.
 * Replication setup is done in here because it wants to be hooked up to WAL.
 *
 * @return A WAL instance.
 * @throws IOException
 */
private WALFactory setupWALAndReplication() throws IOException {
    // TODO Replication make assumptions here based on the default filesystem impl
    final Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
    final String logName = DefaultWALProvider.getWALDirectoryName(this.serverName.toString());

    Path logdir = new Path(rootDir, logName);
    if (LOG.isDebugEnabled()) LOG.debug("logdir=" + logdir);
    if (this.fs.exists(logdir)) {
        throw new RegionServerRunningException("Region server has already " +
                "created directory at " + this.serverName.toString());
    }

    // Instantiate replication manager if replication enabled.  Pass it the
    // log directories.
    createNewReplicationInstance(conf, this, this.fs, logdir, oldLogDir);

    // listeners the wal factory will add to wals it creates.
    final List<WALActionsListener> listeners = new ArrayList<WALActionsListener>();
    listeners.add(new MetricsWAL());
    if (this.replicationSourceHandler != null &&
            this.replicationSourceHandler.getWALActionsListener() != null) {
        // Replication handler is an implementation of WALActionsListener.
        listeners.add(this.replicationSourceHandler.getWALActionsListener());
    }

    return new WALFactory(conf, listeners, serverName.toString());
}
 

import org.apache.hadoop.hbase.wal.DefaultWALProvider; //导入方法依赖的package包/类
/**
 * The original intention of this test was to force an abort of a region
 * server and to make sure that the failure path in the region servers is
 * properly evaluated. But it is difficult to ensure that the region server
 * doesn't finish the log splitting before it aborts. Also now, there is
 * this code path where the master will preempt the region server when master
 * detects that the region server has aborted.
 * @throws Exception
 */
@Test (timeout=300000)
public void testWorkerAbort() throws Exception {
  LOG.info("testWorkerAbort");
  startCluster(3);
  final int NUM_LOG_LINES = 10000;
  final SplitLogManager slm = master.getMasterFileSystem().splitLogManager;
  FileSystem fs = master.getMasterFileSystem().getFileSystem();

  final List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
  HRegionServer hrs = findRSToKill(false, "table");
  Path rootdir = FSUtils.getRootDir(conf);
  final Path logDir = new Path(rootdir,
      DefaultWALProvider.getWALDirectoryName(hrs.getServerName().toString()));

  installTable(new ZooKeeperWatcher(conf, "table-creation", null),
      "table", "family", 40);

  makeWAL(hrs, ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()),
    "table", "family", NUM_LOG_LINES, 100);

  new Thread() {
    @Override
    public void run() {
      waitForCounter(tot_wkr_task_acquired, 0, 1, 1000);
      for (RegionServerThread rst : rsts) {
        rst.getRegionServer().abort("testing");
        break;
      }
    }
  }.start();
  // slm.splitLogDistributed(logDir);
  FileStatus[] logfiles = fs.listStatus(logDir);
  TaskBatch batch = new TaskBatch();
  slm.enqueueSplitTask(logfiles[0].getPath().toString(), batch);
  //waitForCounter but for one of the 2 counters
  long curt = System.currentTimeMillis();
  long waitTime = 80000;
  long endt = curt + waitTime;
  while (curt < endt) {
    if ((tot_wkr_task_resigned.get() + tot_wkr_task_err.get() +
        tot_wkr_final_transition_failed.get() + tot_wkr_task_done.get() +
        tot_wkr_preempt_task.get()) == 0) {
      Thread.yield();
      curt = System.currentTimeMillis();
    } else {
      assertTrue(1 <= (tot_wkr_task_resigned.get() + tot_wkr_task_err.get() +
          tot_wkr_final_transition_failed.get() + tot_wkr_task_done.get() +
          tot_wkr_preempt_task.get()));
      return;
    }
  }
  fail("none of the following counters went up in " + waitTime +
      " milliseconds - " +
      "tot_wkr_task_resigned, tot_wkr_task_err, " +
      "tot_wkr_final_transition_failed, tot_wkr_task_done, " +
      "tot_wkr_preempt_task");
}
 

import org.apache.hadoop.hbase.wal.DefaultWALProvider; //导入方法依赖的package包/类
/**
 * Tests the case where a RegionServer enters a GC pause,
 * comes back online after the master declared it dead and started to split.
 * Want log rolling after a master split to fail. See HBASE-2312.
 */
@Test (timeout=300000)
public void testLogRollAfterSplitStart() throws IOException {
  LOG.info("Verify wal roll after split starts will fail.");
  String logName = "testLogRollAfterSplitStart";
  Path thisTestsDir = new Path(HBASEDIR, DefaultWALProvider.getWALDirectoryName(logName));
  final WALFactory wals = new WALFactory(conf, null, logName);

  try {
    // put some entries in an WAL
    TableName tableName =
        TableName.valueOf(this.getClass().getName());
    HRegionInfo regioninfo = new HRegionInfo(tableName,
        HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
    final WAL log = wals.getWAL(regioninfo.getEncodedNameAsBytes());
  
    final AtomicLong sequenceId = new AtomicLong(1);

    final int total = 20;
    for (int i = 0; i < total; i++) {
      WALEdit kvs = new WALEdit();
      kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));
      HTableDescriptor htd = new HTableDescriptor(tableName);
      htd.addFamily(new HColumnDescriptor("column"));
      log.append(htd, regioninfo, new WALKey(regioninfo.getEncodedNameAsBytes(), tableName,
          System.currentTimeMillis()), kvs, sequenceId, true, null);
    }
    // Send the data to HDFS datanodes and close the HDFS writer
    log.sync();
    ((FSHLog) log).replaceWriter(((FSHLog)log).getOldPath(), null, null, null);

    /* code taken from MasterFileSystem.getLogDirs(), which is called from MasterFileSystem.splitLog()
     * handles RS shutdowns (as observed by the splitting process)
     */
    // rename the directory so a rogue RS doesn't create more WALs
    Path rsSplitDir = thisTestsDir.suffix(DefaultWALProvider.SPLITTING_EXT);
    if (!fs.rename(thisTestsDir, rsSplitDir)) {
      throw new IOException("Failed fs.rename for log split: " + thisTestsDir);
    }
    LOG.debug("Renamed region directory: " + rsSplitDir);

    LOG.debug("Processing the old log files.");
    WALSplitter.split(HBASEDIR, rsSplitDir, OLDLOGDIR, fs, conf, wals);

    LOG.debug("Trying to roll the WAL.");
    try {
      log.rollWriter();
      Assert.fail("rollWriter() did not throw any exception.");
    } catch (IOException ioe) {
      if (ioe.getCause() instanceof FileNotFoundException) {
        LOG.info("Got the expected exception: ", ioe.getCause());
      } else {
        Assert.fail("Unexpected exception: " + ioe);
      }
    }
  } finally {
    wals.close();
    if (fs.exists(thisTestsDir)) {
      fs.delete(thisTestsDir, true);
    }
  }
}
 

import org.apache.hadoop.hbase.wal.DefaultWALProvider; //导入方法依赖的package包/类
/**
 * Get log directory for a server in a snapshot.
 *
 * @param snapshotDir directory where the specific snapshot is stored
 * @param serverName name of the parent regionserver for the log files
 * @return path to the log home directory for the archive files.
 */
public static Path getLogsDir(Path snapshotDir, String serverName) {
  return new Path(snapshotDir, DefaultWALProvider.getWALDirectoryName(serverName));
}