Java HConstants.HREGION_LOGDIR_NAME屬性代碼示例

/**
 * Performs log splitting for all regionserver directories.
 * @throws Exception
 */
private void doOfflineLogSplitting() throws Exception {
  LOG.info("Starting Log splitting");
  final Path rootDir = FSUtils.getRootDir(getConf());
  final Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
  // since this is the singleton, we needn't close it.
  final WALFactory factory = WALFactory.getInstance(getConf());
  FileSystem fs = FSUtils.getCurrentFileSystem(getConf());
  Path logDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
  FileStatus[] regionServerLogDirs = FSUtils.listStatus(fs, logDir);
  if (regionServerLogDirs == null || regionServerLogDirs.length == 0) {
    LOG.info("No log directories to split, returning");
    return;
  }
  try {
    for (FileStatus regionServerLogDir : regionServerLogDirs) {
      // split its log dir, if exists
      WALSplitter.split(rootDir, regionServerLogDir.getPath(), oldLogDir, fs, getConf(), factory);
    }
    LOG.info("Successfully completed Log splitting");
  } catch (Exception e) {
    LOG.error("Got exception while doing Log splitting ", e);
    throw e;
  }
}
 

protected Merger(Configuration conf, FileSystem fs, final TableName tableName)
throws IOException {
  this.conf = conf;
  this.fs = fs;
  this.maxFilesize = conf.getLong(HConstants.HREGION_MAX_FILESIZE,
      HConstants.DEFAULT_MAX_FILE_SIZE);

  this.rootDir = FSUtils.getRootDir(conf);
  Path tabledir = FSUtils.getTableDir(this.rootDir, tableName);
  this.htd = FSTableDescriptors.getTableDescriptorFromFs(this.fs, tabledir);
  String logname = "merge_" + System.currentTimeMillis() + HConstants.HREGION_LOGDIR_NAME;

  final Configuration walConf = new Configuration(conf);
  FSUtils.setRootDir(walConf, tabledir);
  this.walFactory = new WALFactory(walConf, null, logname);
}
 

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  // Make block sizes small.
  conf = TEST_UTIL.getConfiguration();
  conf.setInt("dfs.blocksize", 1024 * 1024);
  conf.setInt("dfs.replication", 1);
  TEST_UTIL.startMiniDFSCluster(1);

  conf = TEST_UTIL.getConfiguration();
  fs = TEST_UTIL.getDFSCluster().getFileSystem();

  hbaseDir = TEST_UTIL.createRootDir();
  
  logDir = new Path(hbaseDir, HConstants.HREGION_LOGDIR_NAME);

  htd = new HTableDescriptor(tableName);
  htd.addFamily(new HColumnDescriptor(family));
}
 

@Before
public void setUp() throws Exception {
  this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
  // this.cluster = TEST_UTIL.getDFSCluster();
  this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
  this.hbaseRootDir = FSUtils.getRootDir(conf);
  this.dir = new Path(this.hbaseRootDir, TestWALObserver.class.getName());
  this.oldLogDir = new Path(this.hbaseRootDir,
      HConstants.HREGION_OLDLOGDIR_NAME);
  this.logDir = new Path(this.hbaseRootDir,
      DefaultWALProvider.getWALDirectoryName(currentTest.getMethodName()));
  this.logName = HConstants.HREGION_LOGDIR_NAME;

  if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
    TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
  }
  this.wals = new WALFactory(conf, null, currentTest.getMethodName());
}
 

@Before
public void setUp() throws Exception {
  fs = FileSystem.get(TEST_UTIL.getConfiguration());
  rootDir = TEST_UTIL.getDataTestDir("hbase");
  logsDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);

  tableFamilies = new HashSet<String>();
  tableRegions = new HashSet<String>();
  recoveredEdits = new HashSet<String>();
  tableHFiles = new HashSet<String>();
  regionServers = new HashSet<String>();
  serverLogs = new HashSet<String>();
  tableDir = createTableFiles(rootDir, TABLE_NAME, tableRegions, tableFamilies, tableHFiles);
  createRecoverEdits(tableDir, tableRegions, recoveredEdits);
  createLogs(logsDir, regionServers, serverLogs);
  FSUtils.logFileSystemState(fs, rootDir, LOG);
}
 

public int run(Path inputDir, int numMappers) throws Exception {
  getConf().set(SEARCHER_INPUTDIR_KEY, inputDir.toString());
  SortedSet<byte []> keys = readKeysToSearch(getConf());
  if (keys.isEmpty()) throw new RuntimeException("No keys to find");
  LOG.info("Count of keys to find: " + keys.size());
  for(byte [] key: keys)  LOG.info("Key: " + Bytes.toStringBinary(key));
  Path hbaseDir = new Path(getConf().get(HConstants.HBASE_DIR));
  // Now read all WALs. In two dirs. Presumes certain layout.
  Path walsDir = new Path(hbaseDir, HConstants.HREGION_LOGDIR_NAME);
  Path oldWalsDir = new Path(hbaseDir, HConstants.HREGION_OLDLOGDIR_NAME);
  LOG.info("Running Search with keys inputDir=" + inputDir +", numMappers=" + numMappers +
    " against " + getConf().get(HConstants.HBASE_DIR));
  int ret = ToolRunner.run(new WALSearcher(getConf()), new String [] {walsDir.toString(), ""});
  if (ret != 0) return ret;
  return ToolRunner.run(new WALSearcher(getConf()), new String [] {oldWalsDir.toString(), ""});
}
 

private int doSearch(Configuration conf, String keysDir) throws Exception {
  Path inputDir = new Path(keysDir);

  getConf().set(SEARCHER_INPUTDIR_KEY, inputDir.toString());
  SortedSet<byte []> keys = readKeysToSearch(getConf());
  if (keys.isEmpty()) throw new RuntimeException("No keys to find");
  LOG.info("Count of keys to find: " + keys.size());
  for(byte [] key: keys)  LOG.info("Key: " + Bytes.toStringBinary(key));
  Path hbaseDir = new Path(getConf().get(HConstants.HBASE_DIR));
  // Now read all WALs. In two dirs. Presumes certain layout.
  Path walsDir = new Path(hbaseDir, HConstants.HREGION_LOGDIR_NAME);
  Path oldWalsDir = new Path(hbaseDir, HConstants.HREGION_OLDLOGDIR_NAME);
  LOG.info("Running Search with keys inputDir=" + inputDir +
    " against " + getConf().get(HConstants.HBASE_DIR));
  int ret = ToolRunner.run(new WALSearcher(getConf()), new String [] {walsDir.toString(), ""});
  if (ret != 0) return ret;
  return ToolRunner.run(new WALSearcher(getConf()), new String [] {oldWalsDir.toString(), ""});
}
 

/**
 * Rename all the dot dirs -- .data, .archive, etc. -- as data, archive, etc.; i.e. minus the dot.
 * @throws IOException
 */
public void migrateDotDirs() throws IOException {
  // Dot dirs to rename.  Leave the tmp dir named '.tmp' and snapshots as .hbase-snapshot.
  final Path archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
  Path [][] dirs = new Path[][] {
    new Path [] {new Path(rootDir, DOT_CORRUPT), new Path(rootDir, HConstants.CORRUPT_DIR_NAME)},
    new Path [] {new Path(rootDir, DOT_LOGS), new Path(rootDir, HConstants.HREGION_LOGDIR_NAME)},
    new Path [] {new Path(rootDir, DOT_OLD_LOGS),
      new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME)},
    new Path [] {new Path(rootDir, TMP_DATA_DIR),
      new Path(rootDir, HConstants.BASE_NAMESPACE_DIR)},
    new Path[] { new Path(rootDir, DOT_LIB_DIR),
      new Path(rootDir, HConstants.LIB_DIR)}};
  for (Path [] dir: dirs) {
    Path src = dir[0];
    Path tgt = dir[1];
    if (!this.fs.exists(src)) {
      LOG.info("Does not exist: " + src);
      continue;
    }
    rename(src, tgt);
  }
  // Do the .archive dir.  Need to move its subdirs to the default ns dir under data dir... so
  // from '.archive/foo', to 'archive/data/default/foo'.
  Path oldArchiveDir = new Path(rootDir, DOT_ARCHIVE);
  if (this.fs.exists(oldArchiveDir)) {
    // This is a pain doing two nn calls but portable over h1 and h2.
    mkdirs(archiveDir);
    Path archiveDataDir = new Path(archiveDir, HConstants.BASE_NAMESPACE_DIR);
    mkdirs(archiveDataDir);
    rename(oldArchiveDir, new Path(archiveDataDir,
      NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR));
  }
  // Update the system and user namespace dirs removing the dot in front of .data.
  Path dataDir = new Path(rootDir, HConstants.BASE_NAMESPACE_DIR);
  sysNsDir = new Path(dataDir, NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR);
  defNsDir = new Path(dataDir, NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR);
}
 

/**
 * @return the WAL associated with the given region
 * @throws IOException e
 */
public synchronized WAL getLog(HRegionInfo info) throws IOException {
  if (this.walFactory == null) {
    String logName = 
        HConstants.HREGION_LOGDIR_NAME + "_" + System.currentTimeMillis();
    final Configuration walConf = new Configuration(this.conf);
    FSUtils.setRootDir(walConf, fs.getHomeDirectory());
    this.walFactory = new WALFactory(walConf, null, logName);
  }
  final byte[] region = info.getEncodedNameAsBytes();
  return info.isMetaRegion() ? walFactory.getMetaWAL(region) : walFactory.getWAL(region);
}
 

/**
 * Iterate over hbase log files
 *
 * @param fs {@link FileSystem}
 * @param rootDir {@link Path} to the HBase root folder
 * @param visitor callback object to get the log files
 * @throws IOException if an error occurred while scanning the directory
 */
public static void visitLogFiles(final FileSystem fs, final Path rootDir,
    final LogFileVisitor visitor) throws IOException {
  Path logsDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
  FileStatus[] logServerDirs = FSUtils.listStatus(fs, logsDir);
  if (logServerDirs == null) {
    if (LOG.isTraceEnabled()) {
      LOG.trace("No logs under directory:" + logsDir);
    }
    return;
  }

  for (FileStatus serverLogs: logServerDirs) {
    String serverName = serverLogs.getPath().getName();

    FileStatus[] wals = FSUtils.listStatus(fs, serverLogs.getPath());
    if (wals == null) {
      if (LOG.isTraceEnabled()) {
        LOG.trace("No wals found for server: " + serverName + ", skipping.");
      }
      continue;
    }

    for (FileStatus walRef: wals) {
      visitor.logFile(serverName, walRef.getPath().getName());
    }
  }
}
 

/**
 * @throws java.lang.Exception
 */
@BeforeClass
public static void setUpBeforeClass() throws Exception {
  TEST_UTIL.startMiniDFSCluster(1);
  FS = TEST_UTIL.getDFSCluster().getFileSystem();
  Path rootDir = TEST_UTIL.createRootDir();
  oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
  if (FS.exists(oldLogDir)) FS.delete(oldLogDir, true);
  logDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
  if (FS.exists(logDir)) FS.delete(logDir, true);
}
 

@Test (timeout=300000)
public void testNewDirsArePresentPostMigration() throws IOException {
  FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
  // Below list does not include 'corrupt' because there is no 'corrupt' in the tgz
  String [] newdirs = new String [] {HConstants.BASE_NAMESPACE_DIR,
    HConstants.HREGION_LOGDIR_NAME};
  Path hbaseRootDir = TEST_UTIL.getDefaultRootDirPath();
  for (String dir: newdirs) {
    assertTrue(dir, fs.exists(new Path(hbaseRootDir, dir)));
  }
}
 

@Test
public void testFindsSnapshotFilesWhenCleaning() throws IOException {
  Configuration conf = TEST_UTIL.getConfiguration();
  FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir());
  Path rootDir = FSUtils.getRootDir(conf);
  FileSystem fs = FileSystem.get(conf);
  SnapshotLogCleaner cleaner = new SnapshotLogCleaner();
  cleaner.setConf(conf);

  // write an hfile to the snapshot directory
  String snapshotName = "snapshot";
  byte[] snapshot = Bytes.toBytes(snapshotName);
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
  Path snapshotLogDir = new Path(snapshotDir, HConstants.HREGION_LOGDIR_NAME);
  String timestamp = "1339643343027";
  String hostFromMaster = "localhost%2C59648%2C1339643336601";

  Path hostSnapshotLogDir = new Path(snapshotLogDir, hostFromMaster);
  String snapshotlogfile = hostFromMaster + "." + timestamp + ".hbase";

  // add the reference to log in the snapshot
  fs.create(new Path(hostSnapshotLogDir, snapshotlogfile));

  // now check to see if that log file would get deleted.
  Path oldlogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
  Path logFile = new Path(oldlogDir, snapshotlogfile);
  fs.create(logFile);

  // make sure that the file isn't deletable
  assertFalse(cleaner.isFileDeletable(fs.getFileStatus(logFile)));
}
 

/**
 * Inspect the log directory to find dead servers which need recovery work
 * @return A set of ServerNames which aren't running but still have WAL files left in file system
 */
Set<ServerName> getFailedServersFromLogFolders() {
  boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
      WALSplitter.SPLIT_SKIP_ERRORS_DEFAULT);

  Set<ServerName> serverNames = new HashSet<ServerName>();
  Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);

  do {
    if (master.isStopped()) {
      LOG.warn("Master stopped while trying to get failed servers.");
      break;
    }
    try {
      if (!this.fs.exists(logsDirPath)) return serverNames;
      FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
      // Get online servers after getting log folders to avoid log folder deletion of newly
      // checked in region servers . see HBASE-5916
      Set<ServerName> onlineServers = ((HMaster) master).getServerManager().getOnlineServers()
          .keySet();

      if (logFolders == null || logFolders.length == 0) {
        LOG.debug("No log files to split, proceeding...");
        return serverNames;
      }
      for (FileStatus status : logFolders) {
        FileStatus[] curLogFiles = FSUtils.listStatus(this.fs, status.getPath(), null);
        if (curLogFiles == null || curLogFiles.length == 0) {
          // Empty log folder. No recovery needed
          continue;
        }
        final ServerName serverName = DefaultWALProvider.getServerNameFromWALDirectoryName(
            status.getPath());
        if (null == serverName) {
          LOG.warn("Log folder " + status.getPath() + " doesn't look like its name includes a " +
              "region server name; leaving in place. If you see later errors about missing " +
              "write ahead logs they may be saved in this location.");
        } else if (!onlineServers.contains(serverName)) {
          LOG.info("Log folder " + status.getPath() + " doesn't belong "
              + "to a known region server, splitting");
          serverNames.add(serverName);
        } else {
          LOG.info("Log folder " + status.getPath() + " belongs to an existing region server");
        }
      }
      retrySplitting = false;
    } catch (IOException ioe) {
      LOG.warn("Failed getting failed servers to be recovered.", ioe);
      if (!checkFileSystem()) {
        LOG.warn("Bad Filesystem, exiting");
        Runtime.getRuntime().halt(1);
      }
      try {
        if (retrySplitting) {
          Thread.sleep(conf.getInt("hbase.hlog.split.failure.retry.interval", 30 * 1000));
        }
      } catch (InterruptedException e) {
        LOG.warn("Interrupted, aborting since cannot return w/o splitting");
        Thread.currentThread().interrupt();
        retrySplitting = false;
        Runtime.getRuntime().halt(1);
      }
    }
  } while (retrySplitting);

  return serverNames;
}
 

/**
 * @param rootDir Path to the root directory where hbase files are stored
 * @param serverName Region Server owner of the log
 * @param logName WAL file name
 */
public WALLink(final Path rootDir, final String serverName, final String logName) {
  final Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
  final Path logDir = new Path(new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), serverName);
  setLocations(new Path(logDir, logName), new Path(oldLogDir, logName));
}