Java LeveldbIterator类代码示例

import org.apache.hadoop.yarn.server.utils.LeveldbIterator; //导入依赖的package包/类
private LocalResourceTrackerState loadResourceTrackerState(
    LeveldbIterator iter, String keyPrefix) throws IOException {
  final String completedPrefix = keyPrefix + LOCALIZATION_COMPLETED_SUFFIX;
  final String startedPrefix = keyPrefix + LOCALIZATION_STARTED_SUFFIX;
  LocalResourceTrackerState state = new LocalResourceTrackerState();
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }

    if (key.startsWith(completedPrefix)) {
      state.localizedResources = loadCompletedResources(iter,
          completedPrefix);
    } else if (key.startsWith(startedPrefix)) {
      state.inProgressResources = loadStartedResources(iter, startedPrefix);
    } else {
      throw new IOException("Unexpected key in resource tracker state: "
          + key);
    }
  }

  return state;
}
 

import org.apache.hadoop.yarn.server.utils.LeveldbIterator; //导入依赖的package包/类
private List<LocalizedResourceProto> loadCompletedResources(
    LeveldbIterator iter, String keyPrefix) throws IOException {
  List<LocalizedResourceProto> rsrcs =
      new ArrayList<LocalizedResourceProto>();
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }

    if (LOG.isDebugEnabled()) {
      LOG.debug("Loading completed resource from " + key);
    }
    rsrcs.add(LocalizedResourceProto.parseFrom(entry.getValue()));
    iter.next();
  }

  return rsrcs;
}
 

import org.apache.hadoop.yarn.server.utils.LeveldbIterator; //导入依赖的package包/类
private Map<LocalResourceProto, Path> loadStartedResources(
    LeveldbIterator iter, String keyPrefix) throws IOException {
  Map<LocalResourceProto, Path> rsrcs =
      new HashMap<LocalResourceProto, Path>();
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }

    Path localPath = new Path(key.substring(keyPrefix.length()));
    if (LOG.isDebugEnabled()) {
      LOG.debug("Loading in-progress resource at " + localPath);
    }
    rsrcs.put(LocalResourceProto.parseFrom(entry.getValue()), localPath);
    iter.next();
  }

  return rsrcs;
}
 

import org.apache.hadoop.yarn.server.utils.LeveldbIterator; //导入依赖的package包/类
private int loadTokenMasterKeys(TimelineServiceState state)
    throws IOException {
  byte[] base = KeyBuilder.newInstance().add(TOKEN_MASTER_KEY_ENTRY_PREFIX)
      .getBytesForLookup();
  int numKeys = 0;
  LeveldbIterator iterator = null;
  try {
    for (iterator = new LeveldbIterator(db), iterator.seek(base);
        iterator.hasNext(); iterator.next()) {
      byte[] k = iterator.peekNext().getKey();
      if (!prefixMatches(base, base.length, k)) {
        break;
      }
      byte[] v = iterator.peekNext().getValue();
      loadTokenMasterKeyData(state, v);
      ++numKeys;
    }
  } finally {
    IOUtils.cleanup(LOG, iterator);
  }
  return numKeys;
}
 

import org.apache.hadoop.yarn.server.utils.LeveldbIterator; //导入依赖的package包/类
private int loadTokens(TimelineServiceState state) throws IOException {
  byte[] base = KeyBuilder.newInstance().add(TOKEN_ENTRY_PREFIX)
      .getBytesForLookup();
  int numTokens = 0;
  LeveldbIterator iterator = null;
  try {
    for (iterator = new LeveldbIterator(db), iterator.seek(base);
        iterator.hasNext(); iterator.next()) {
      byte[] k = iterator.peekNext().getKey();
      if (!prefixMatches(base, base.length, k)) {
        break;
      }
      byte[] v = iterator.peekNext().getValue();
      loadTokenData(state, v);
      ++numTokens;
    }
  } catch (DBException e) {
    throw new IOException(e);
  } finally {
    IOUtils.cleanup(LOG, iterator);
  }
  return numTokens;
}
 

import org.apache.hadoop.yarn.server.utils.LeveldbIterator; //导入依赖的package包/类
private RecoveredContainerState loadContainerState(ContainerId containerId,
    LeveldbIterator iter, String keyPrefix) throws IOException {
  RecoveredContainerState rcs = new RecoveredContainerState();
  rcs.status = RecoveredContainerStatus.REQUESTED;
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }
    iter.next();

    String suffix = key.substring(keyPrefix.length()-1);  // start with '/'
    if (suffix.equals(CONTAINER_REQUEST_KEY_SUFFIX)) {
      rcs.startRequest = new StartContainerRequestPBImpl(
          StartContainerRequestProto.parseFrom(entry.getValue()));
    } else if (suffix.equals(CONTAINER_DIAGS_KEY_SUFFIX)) {
      rcs.diagnostics = asString(entry.getValue());
    } else if (suffix.equals(CONTAINER_LAUNCHED_KEY_SUFFIX)) {
      if (rcs.status == RecoveredContainerStatus.REQUESTED) {
        rcs.status = RecoveredContainerStatus.LAUNCHED;
      }
    } else if (suffix.equals(CONTAINER_KILLED_KEY_SUFFIX)) {
      rcs.killed = true;
    } else if (suffix.equals(CONTAINER_EXIT_CODE_KEY_SUFFIX)) {
      rcs.status = RecoveredContainerStatus.COMPLETED;
      rcs.exitCode = Integer.parseInt(asString(entry.getValue()));
    } else {
      throw new IOException("Unexpected container state key: " + key);
    }
  }
  return rcs;
}
 

import org.apache.hadoop.yarn.server.utils.LeveldbIterator; //导入依赖的package包/类
private RecoveredUserResources loadUserLocalizedResources(
    LeveldbIterator iter, String keyPrefix) throws IOException {
  RecoveredUserResources userResources = new RecoveredUserResources();
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }

    if (key.startsWith(LOCALIZATION_FILECACHE_SUFFIX, keyPrefix.length())) {
      userResources.privateTrackerState = loadResourceTrackerState(iter,
          keyPrefix + LOCALIZATION_FILECACHE_SUFFIX);
    } else if (key.startsWith(LOCALIZATION_APPCACHE_SUFFIX,
        keyPrefix.length())) {
      int appIdStartPos = keyPrefix.length() +
          LOCALIZATION_APPCACHE_SUFFIX.length();
      int appIdEndPos = key.indexOf('/', appIdStartPos);
      if (appIdEndPos < 0) {
        throw new IOException("Unable to determine appID in resource key: "
            + key);
      }
      ApplicationId appId = ConverterUtils.toApplicationId(
          key.substring(appIdStartPos, appIdEndPos));
      userResources.appTrackerStates.put(appId,
          loadResourceTrackerState(iter, key.substring(0, appIdEndPos+1)));
    } else {
      throw new IOException("Unexpected user resource key " + key);
    }
  }
  return userResources;
}
 

import org.apache.hadoop.yarn.server.utils.LeveldbIterator; //导入依赖的package包/类
private int loadRMApp(RMState rmState, LeveldbIterator iter, String appIdStr,
    byte[] appData) throws IOException {
  ApplicationStateData appState = createApplicationState(appIdStr, appData);
  ApplicationId appId =
      appState.getApplicationSubmissionContext().getApplicationId();
  rmState.appState.put(appId, appState);
  String attemptNodePrefix = getApplicationNodeKey(appId) + SEPARATOR;
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(attemptNodePrefix)) {
      break;
    }

    String attemptId = key.substring(attemptNodePrefix.length());
    if (attemptId.startsWith(ApplicationAttemptId.appAttemptIdStrPrefix)) {
      ApplicationAttemptStateData attemptState =
          createAttemptState(attemptId, entry.getValue());
      appState.attempts.put(attemptState.getAttemptId(), attemptState);
    } else {
      LOG.warn("Ignoring unknown application key: " + key);
    }
    iter.next();
  }
  int numAttempts = appState.attempts.size();
  if (LOG.isDebugEnabled()) {
    LOG.debug("Loaded application " + appId + " with " + numAttempts
        + " attempts");
  }
  return numAttempts;
}
 

import org.apache.hadoop.yarn.server.utils.LeveldbIterator; //导入依赖的package包/类
@VisibleForTesting
List<String> getEntityTypes() throws IOException {
  LeveldbIterator iterator = null;
  try {
    iterator = getDbIterator(false);
    List<String> entityTypes = new ArrayList<String>();
    iterator.seek(ENTITY_ENTRY_PREFIX);
    while (iterator.hasNext()) {
      byte[] key = iterator.peekNext().getKey();
      if (key[0] != ENTITY_ENTRY_PREFIX[0]) {
        break;
      }
      KeyParser kp = new KeyParser(key,
          ENTITY_ENTRY_PREFIX.length);
      String entityType = kp.getNextString();
      entityTypes.add(entityType);
      byte[] lookupKey = KeyBuilder.newInstance().add(ENTITY_ENTRY_PREFIX)
          .add(entityType).getBytesForLookup();
      if (lookupKey[lookupKey.length - 1] != 0x0) {
        throw new IOException("Found unexpected end byte in lookup key");
      }
      lookupKey[lookupKey.length - 1] = 0x1;
      iterator.seek(lookupKey);
    }
    return entityTypes;
  } catch(DBException e) {
    throw new IOException(e);            	
  } finally {
    IOUtils.cleanup(LOG, iterator);
  }
}
 

import org.apache.hadoop.yarn.server.utils.LeveldbIterator; //导入依赖的package包/类
/**
 * Finds all keys in the db that have a given prefix and deletes them on
 * the given write batch.
 */
private void deleteKeysWithPrefix(WriteBatch writeBatch, byte[] prefix,
    LeveldbIterator iterator) {
  for (iterator.seek(prefix); iterator.hasNext(); iterator.next()) {
    byte[] key = iterator.peekNext().getKey();
    if (!prefixMatches(prefix, prefix.length, key)) {
      break;
    }
    writeBatch.delete(key);
  }
}
 

import org.apache.hadoop.yarn.server.utils.LeveldbIterator; //导入依赖的package包/类
private static TimelineDomain getTimelineDomain(
    LeveldbIterator iterator, String domainId, byte[] prefix) throws IOException {
  // Iterate over all the rows whose key starts with prefix to retrieve the
  // domain information.
  TimelineDomain domain = new TimelineDomain();
  domain.setId(domainId);
  boolean noRows = true;
  for (; iterator.hasNext(); iterator.next()) {
    byte[] key = iterator.peekNext().getKey();
    if (!prefixMatches(prefix, prefix.length, key)) {
      break;
    }
    if (noRows) {
      noRows = false;
    }
    byte[] value = iterator.peekNext().getValue();
    if (value != null && value.length > 0) {
      if (key[prefix.length] == DESCRIPTION_COLUMN[0]) {
        domain.setDescription(new String(value, Charset.forName("UTF-8")));
      } else if (key[prefix.length] == OWNER_COLUMN[0]) {
        domain.setOwner(new String(value, Charset.forName("UTF-8")));
      } else if (key[prefix.length] == READER_COLUMN[0]) {
        domain.setReaders(new String(value, Charset.forName("UTF-8")));
      } else if (key[prefix.length] == WRITER_COLUMN[0]) {
        domain.setWriters(new String(value, Charset.forName("UTF-8")));
      } else if (key[prefix.length] == TIMESTAMP_COLUMN[0]) {
        domain.setCreatedTime(readReverseOrderedLong(value, 0));
        domain.setModifiedTime(readReverseOrderedLong(value, 8));
      } else {
        LOG.error("Unrecognized domain column: " + key[prefix.length]);
      }
    }
  }
  if (noRows) {
    return null;
  } else {
    return domain;
  }
}
 

import org.apache.hadoop.yarn.server.utils.LeveldbIterator; //导入依赖的package包/类
private boolean deleteNextEntity(String entityType, byte[] ts)
    throws IOException, InterruptedException {
  LeveldbIterator iterator = null;
  LeveldbIterator pfIterator = null;
  try {
    iterator = ((LeveldbTimelineStore)store).getDbIterator(false);
    pfIterator = ((LeveldbTimelineStore)store).getDbIterator(false);
    return ((LeveldbTimelineStore)store).deleteNextEntity(entityType, ts,
        iterator, pfIterator, false);
  } catch(DBException e) {
    throw new IOException(e);   	
  } finally {
    IOUtils.cleanup(null, iterator, pfIterator);
  }
}
 

import org.apache.hadoop.yarn.server.utils.LeveldbIterator; //导入依赖的package包/类
private RecoveredContainerState loadContainerState(ContainerId containerId,
    LeveldbIterator iter, String keyPrefix) throws IOException {
  RecoveredContainerState rcs = new RecoveredContainerState();
  rcs.status = RecoveredContainerStatus.REQUESTED;
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }
    iter.next();

    String suffix = key.substring(keyPrefix.length()-1);  // start with '/'
    if (suffix.equals(CONTAINER_REQUEST_KEY_SUFFIX)) {
      rcs.startRequest = new StartContainerRequestPBImpl(
          StartContainerRequestProto.parseFrom(entry.getValue()));
    } else if (suffix.equals(CONTAINER_DIAGS_KEY_SUFFIX)) {
      rcs.diagnostics = asString(entry.getValue());
    } else if (suffix.equals(CONTAINER_LAUNCHED_KEY_SUFFIX)) {
      if (rcs.status == RecoveredContainerStatus.REQUESTED) {
        rcs.status = RecoveredContainerStatus.LAUNCHED;
      }
    } else if (suffix.equals(CONTAINER_KILLED_KEY_SUFFIX)) {
      rcs.killed = true;
    } else if (suffix.equals(CONTAINER_EXIT_CODE_KEY_SUFFIX)) {
      rcs.status = RecoveredContainerStatus.COMPLETED;
      rcs.exitCode = Integer.parseInt(asString(entry.getValue()));
    } else if (suffix.equals(CONTAINER_RESOURCE_CHANGED_KEY_SUFFIX)) {
      rcs.capability = new ResourcePBImpl(
          ResourceProto.parseFrom(entry.getValue()));
    } else {
      throw new IOException("Unexpected container state key: " + key);
    }
  }
  return rcs;
}
 

import org.apache.hadoop.yarn.server.utils.LeveldbIterator; //导入依赖的package包/类
private boolean deleteNextEntity(String entityType, byte[] ts)
    throws IOException, InterruptedException {
  LeveldbIterator iterator = null;
  LeveldbIterator pfIterator = null;
  try {
    iterator = ((LeveldbTimelineStore)store).getDbIterator(false);
    pfIterator = ((LeveldbTimelineStore)store).getDbIterator(false);
    return ((LeveldbTimelineStore)store).deleteNextEntity(entityType, ts,
        iterator, pfIterator, false);
  } catch(DBException e) {
    throw new IOException(e);
  } finally {
    IOUtils.cleanup(null, iterator, pfIterator);
  }
}