Java HelixPropertyStore类代码示例

import org.apache.helix.store.HelixPropertyStore; //导入依赖的package包/类
/**
 * Remove a set of jobs from a workflow. This removes the config, context, IS and EV associated
 * with each individual job, and removes all the jobs from the WorkflowConfig, and job states from
 * WorkflowContext.
 *
 * @param dataAccessor
 * @param propertyStore
 * @param jobs
 * @param workflow
 * @param maintainDependency
 *
 * @return True if remove success, otherwise false
 */
protected static boolean removeJobsFromWorkflow(final HelixDataAccessor dataAccessor,
    final HelixPropertyStore propertyStore, final String workflow, final Set<String> jobs,
    boolean maintainDependency) {
  boolean success = true;
  if (!removeJobsFromDag(dataAccessor, workflow, jobs, maintainDependency)) {
    LOG.warn("Error occurred while trying to remove jobs + " + jobs + " from the workflow "
        + workflow);
    success = false;
  }
  if (!removeJobsState(propertyStore, workflow, jobs)) {
    LOG.warn(
        "Error occurred while trying to remove jobs states from workflow + " + workflow + " jobs "
            + jobs);
    success = false;
  }
  for (String job : jobs) {
    if (!removeJob(dataAccessor, propertyStore, job)) {
      success = false;
    }
  }

  return success;
}
 

import org.apache.helix.store.HelixPropertyStore; //导入依赖的package包/类
/**
 * Return all jobs that are COMPLETED and passes its expiry time.
 *
 * @param dataAccessor
 * @param propertyStore
 * @param workflowConfig
 * @param workflowContext
 *
 * @return
 */
protected static Set<String> getExpiredJobs(HelixDataAccessor dataAccessor,
    HelixPropertyStore propertyStore, WorkflowConfig workflowConfig,
    WorkflowContext workflowContext) {
  Set<String> expiredJobs = new HashSet<String>();

  if (workflowContext != null) {
    Map<String, TaskState> jobStates = workflowContext.getJobStates();
    for (String job : workflowConfig.getJobDag().getAllNodes()) {
      JobConfig jobConfig = TaskUtil.getJobConfig(dataAccessor, job);
      JobContext jobContext = TaskUtil.getJobContext(propertyStore, job);
      long expiry = jobConfig.getExpiry();
      if (expiry == workflowConfig.DEFAULT_EXPIRY || expiry < 0) {
        expiry = workflowConfig.getExpiry();
      }
      if (jobContext != null && jobStates.get(job) == TaskState.COMPLETED) {
        if (System.currentTimeMillis() >= jobContext.getFinishTime() + expiry) {
          expiredJobs.add(job);
        }
      }
    }
  }
  return expiredJobs;
}
 

import org.apache.helix.store.HelixPropertyStore; //导入依赖的package包/类
protected static boolean removeJob(HelixDataAccessor accessor, HelixPropertyStore propertyStore,
    String job) {
  boolean success = true;
  if (!cleanupJobIdealStateExtView(accessor, job)) {
    LOG.warn(String
        .format("Error occurred while trying to remove job idealstate/externalview for %s.",
            job));
    success = false;
  }
  if (!removeJobConfig(accessor, job)) {
    LOG.warn(String.format("Error occurred while trying to remove job config for %s.", job));
    success = false;
  }
  if (!removeJobContext(propertyStore, job)) {
    LOG.warn(String.format("Error occurred while trying to remove job context for %s.", job));
    success = false;
  }

  return success;
}
 

import org.apache.helix.store.HelixPropertyStore; //导入依赖的package包/类
/**
 * A constructor that gets a {@link HelixNotifier} based on {@link HelixPropertyStoreConfig}.
 * @param storeConfig A {@link HelixPropertyStore} used to instantiate a {@link HelixNotifier}. Cannot be {@code null}.
 */
public HelixNotifier(HelixPropertyStoreConfig storeConfig) {
  if (storeConfig == null) {
    throw new IllegalArgumentException("storeConfig cannot be null");
  }
  long startTimeMs = System.currentTimeMillis();
  logger.info("Starting a HelixNotifier");
  ZkClient zkClient = new ZkClient(storeConfig.zkClientConnectString, storeConfig.zkClientSessionTimeoutMs,
      storeConfig.zkClientConnectionTimeoutMs, new ZNRecordSerializer());
  List<String> subscribedPaths = Collections.singletonList(storeConfig.rootPath + HelixNotifier.TOPIC_PATH);
  HelixPropertyStore<ZNRecord> helixStore =
      new ZkHelixPropertyStore<>(new ZkBaseDataAccessor<>(zkClient), storeConfig.rootPath, subscribedPaths);
  logger.info("HelixPropertyStore started with zkClientConnectString={}, zkClientSessionTimeoutMs={}, "
          + "zkClientConnectionTimeoutMs={}, rootPath={}, subscribedPaths={}", storeConfig.zkClientConnectString,
      storeConfig.zkClientSessionTimeoutMs, storeConfig.zkClientConnectionTimeoutMs, storeConfig.rootPath,
      subscribedPaths);
  this.helixStore = helixStore;
  long startUpTimeInMs = System.currentTimeMillis() - startTimeMs;
  logger.info("HelixNotifier started, took {} ms", startUpTimeInMs);
}
 

import org.apache.helix.store.HelixPropertyStore; //导入依赖的package包/类
/**
 * A constructor that gets a {@link HelixStoreOperator} based on the {@link HelixPropertyStoreConfig}.
 * @param storeConfig A {@link HelixPropertyStore} used to instantiate a {@link HelixStoreOperator}.
 */
public HelixStoreOperator(HelixPropertyStoreConfig storeConfig) {
  if (storeConfig == null) {
    throw new IllegalArgumentException("storeConfig cannot be null");
  }
  long startTimeMs = System.currentTimeMillis();
  logger.info("Starting a HelixStoreOperator");
  ZkClient zkClient = new ZkClient(storeConfig.zkClientConnectString, storeConfig.zkClientSessionTimeoutMs,
      storeConfig.zkClientConnectionTimeoutMs, new ZNRecordSerializer());
  List<String> subscribedPaths = Collections.singletonList(storeConfig.rootPath);
  HelixPropertyStore<ZNRecord> helixStore =
      new ZkHelixPropertyStore<>(new ZkBaseDataAccessor<>(zkClient), storeConfig.rootPath, subscribedPaths);
  logger.info("HelixPropertyStore started with zkClientConnectString={}, zkClientSessionTimeoutMs={}, "
          + "zkClientConnectionTimeoutMs={}, rootPath={}, subscribedPaths={}", storeConfig.zkClientConnectString,
      storeConfig.zkClientSessionTimeoutMs, storeConfig.zkClientConnectionTimeoutMs, storeConfig.rootPath,
      subscribedPaths);
  this.helixStore = helixStore;
  logger.info("HelixStoreOperator started, took {}ms", System.currentTimeMillis() - startTimeMs);
}
 

import org.apache.helix.store.HelixPropertyStore; //导入依赖的package包/类
@Override
public AccountService getAccountService() {
  try {
    long startTimeMs = System.currentTimeMillis();
    logger.info("Starting a HelixAccountService");
    ZkClient zkClient = new ZkClient(storeConfig.zkClientConnectString, storeConfig.zkClientSessionTimeoutMs,
        storeConfig.zkClientConnectionTimeoutMs, new ZNRecordSerializer());
    HelixPropertyStore<ZNRecord> helixStore =
        new ZkHelixPropertyStore<>(new ZkBaseDataAccessor<>(zkClient), storeConfig.rootPath, null);
    logger.info("HelixPropertyStore started with zkClientConnectString={}, zkClientSessionTimeoutMs={}, "
            + "zkClientConnectionTimeoutMs={}, rootPath={}", storeConfig.zkClientConnectString,
        storeConfig.zkClientSessionTimeoutMs, storeConfig.zkClientConnectionTimeoutMs, storeConfig.rootPath);
    ScheduledExecutorService scheduler =
        accountServiceConfig.updaterPollingIntervalMs > 0 ? Utils.newScheduler(1, HELIX_ACCOUNT_UPDATER_PREFIX, false)
            : null;
    HelixAccountService helixAccountService =
        new HelixAccountService(helixStore, accountServiceMetrics, notifier, scheduler, accountServiceConfig);
    long spentTimeMs = System.currentTimeMillis() - startTimeMs;
    logger.info("HelixAccountService started, took {} ms", spentTimeMs);
    accountServiceMetrics.startupTimeInMs.update(spentTimeMs);
    return helixAccountService;
  } catch (Exception e) {
    throw new IllegalStateException("Could not instantiate HelixAccountService", e);
  }
}
 

import org.apache.helix.store.HelixPropertyStore; //导入依赖的package包/类
public TaskDriver(HelixAdmin admin, HelixDataAccessor accessor,
    HelixPropertyStore<ZNRecord> propertyStore, String clusterName) {
  _admin = admin;
  _accessor = accessor;
  _propertyStore = propertyStore;
  _clusterName = clusterName;
}
 

import org.apache.helix.store.HelixPropertyStore; //导入依赖的package包/类
/**
 * Intialize the user content store znode setup
 * @param propertyStore       zookeeper property store
 * @param workflowJobResource the name of workflow or job
 * @param record              the initial data
 */
protected static void createUserContent(HelixPropertyStore propertyStore, String workflowJobResource,
    ZNRecord record) {
  propertyStore.create(Joiner.on("/")
      .join(TaskConstants.REBALANCER_CONTEXT_ROOT, workflowJobResource,
          TaskUtil.USER_CONTENT_NODE), record, AccessOption.PERSISTENT);
}
 

import org.apache.helix.store.HelixPropertyStore; //导入依赖的package包/类
/**
 * Remove a workflow and all jobs for the workflow. This removes the workflow config, idealstate,
 * externalview and workflow contexts associated with this workflow, and all jobs information,
 * including their configs, context, IS and EV.
 *
 * @param accessor
 * @param propertyStore
 * @param workflow the workflow name.
 * @param jobs     all job names in this workflow.
 *
 * @return  True if remove success, otherwise false
 */
protected static boolean removeWorkflow(final HelixDataAccessor accessor, final HelixPropertyStore propertyStore,
    String workflow, Set<String> jobs) {
  boolean success = true;

  // clean up all jobs
  for (String job : jobs) {
    if (!removeJob(accessor, propertyStore, job)) {
      success = false;
    }
  }

  if (!cleanupWorkflowIdealStateExtView(accessor, workflow)) {
    LOG.warn(String
        .format("Error occurred while trying to remove workflow idealstate/externalview for %s.",
            workflow));
    success = false;
  }
  if (!removeWorkflowConfig(accessor, workflow)) {
    LOG.warn(
        String.format("Error occurred while trying to remove workflow config for %s.", workflow));
    success = false;
  }
  if (!removeWorkflowContext(propertyStore, workflow)) {
    LOG.warn(String
        .format("Error occurred while trying to remove workflow context for %s.", workflow));
    success = false;
  }

  return success;
}
 

import org.apache.helix.store.HelixPropertyStore; //导入依赖的package包/类
/**
 * update workflow's property to remove jobs from JOB_STATES if there are already started.
 */
protected static boolean removeJobsState(final HelixPropertyStore propertyStore,
    final String workflow, final Set<String> jobs) {
  String contextPath =
      Joiner.on("/").join(TaskConstants.REBALANCER_CONTEXT_ROOT, workflow, TaskUtil.CONTEXT_NODE);

  // If the queue is not started, there is no JobState need to be removed.
  if (!propertyStore.exists(contextPath, 0)) {
    return true;
  }

  DataUpdater<ZNRecord> updater = new DataUpdater<ZNRecord>() {
    @Override public ZNRecord update(ZNRecord currentData) {
      if (currentData != null) {
        WorkflowContext workflowContext = new WorkflowContext(currentData);
        workflowContext.removeJobStates(jobs);
        workflowContext.removeJobStartTime(jobs);
        currentData = workflowContext.getRecord();
      }
      return currentData;
    }
  };
  if (!propertyStore.update(contextPath, updater, AccessOption.PERSISTENT)) {
    LOG.warn("Fail to remove job state for jobs " + jobs + " from workflow " + workflow);
    return false;
  }
  return true;
}
 

import org.apache.helix.store.HelixPropertyStore; //导入依赖的package包/类
private static boolean removeWorkflowJobContext(HelixPropertyStore<ZNRecord> propertyStore,
    String workflowJobResource) {
  String path = Joiner.on("/").join(TaskConstants.REBALANCER_CONTEXT_ROOT, workflowJobResource);
  if (propertyStore.exists(path, AccessOption.PERSISTENT)) {
    if (!propertyStore.remove(path, AccessOption.PERSISTENT)) {
      LOG.warn(String.format(
          "Error occurred while trying to remove workflow/jobcontext for %s. Failed to remove node %s.",
          workflowJobResource, path));
      return false;
    }
  }
  return true;
}
 

import org.apache.helix.store.HelixPropertyStore; //导入依赖的package包/类
/**
 * <p>
 *   Constructor. It fetches the remote account data in {@code ZooKeeper} and caches locally during initialization,
 *   and updates the remote account data during updating accounts. If a non-null {@link Notifier} is provided, it
 *   will listen to the changes of the remote accounts, and reactively updates its local cache. It als sends message
 *   to notify other listeners after each update made to the remote accounts.
 * </p>
 * <p>
 *   This call is blocking until it fetches all the {@link Account} metadata from {@link HelixPropertyStore}.
 * </p>
 * @param helixStore A {@link HelixPropertyStore} used by the {@code HelixAccountService}. Cannot be {@code null}.
 * @param accountServiceMetrics {@link AccountServiceMetrics} to report metrics. Cannot be {@code null}.
 * @param notifier A {@link Notifier} that will be used to publish message after updating {@link Account}s, and
 *                 listen to {@link Account} change messages. Can be {@code null}.
 * @param scheduler A {@link ScheduledExecutorService} that will run thread to update accounts in background.
 *                  {@code null} to disable background updating.
 * @param config The configs for {@code HelixAccountService}.
 * @throws IOException if backup directory creation was needed but failed.
 */
HelixAccountService(HelixPropertyStore<ZNRecord> helixStore, AccountServiceMetrics accountServiceMetrics,
    Notifier<String> notifier, ScheduledExecutorService scheduler, HelixAccountServiceConfig config)
    throws IOException {
  this.helixStore = Objects.requireNonNull(helixStore, "helixStore cannot be null");
  this.accountServiceMetrics = Objects.requireNonNull(accountServiceMetrics, "accountServiceMetrics cannot be null");
  this.notifier = notifier;
  this.scheduler = scheduler;
  this.config = config;

  backupDirPath = config.backupDir.isEmpty() ? null : Files.createDirectories(Paths.get(config.backupDir));
  if (notifier != null) {
    notifier.subscribe(ACCOUNT_METADATA_CHANGE_TOPIC, this::onAccountChangeMessage);
  } else {
    logger.warn("Notifier is null. Account updates cannot be notified to other entities. Local account cache may not "
        + "be in sync with remote account data.");
    accountServiceMetrics.nullNotifierCount.inc();
  }
  Runnable updater = () -> {
    try {
      readFullAccountAndUpdateCache(FULL_ACCOUNT_METADATA_PATH, false);
    } catch (Exception e) {
      logger.error("Exception occurred when fetching remote account data", e);
      accountServiceMetrics.fetchRemoteAccountErrorCount.inc();
    }
  };
  updater.run();
  if (scheduler != null) {
    int initialDelay = new Random().nextInt(config.updaterPollingIntervalMs + 1);
    scheduler.scheduleAtFixedRate(updater, initialDelay, config.updaterPollingIntervalMs, TimeUnit.MILLISECONDS);
    logger.info(
        "Background account updater will fetch accounts from remote starting {} ms from now and repeat with interval={} ms",
        initialDelay, config.updaterPollingIntervalMs);
  }
}
 

import org.apache.helix.store.HelixPropertyStore; //导入依赖的package包/类
public GobblinHelixTaskDriver(HelixAdmin admin, HelixDataAccessor accessor, ConfigAccessor cfgAccessor,
    HelixPropertyStore<ZNRecord> propertyStore, String clusterName) {
  _taskDriver = new TaskDriver(admin, accessor, cfgAccessor, propertyStore, clusterName);
}
 

import org.apache.helix.store.HelixPropertyStore; //导入依赖的package包/类
/**
 * Get the runtime context of a single job.
 * This method is internal API, please use TaskDriver.getJobContext();
 *
 * @param propertyStore Property store for the cluster
 * @param jobResource   The name of the job
 * @return the {@link JobContext}, or null if none is available
 */
protected static JobContext getJobContext(HelixPropertyStore<ZNRecord> propertyStore,
    String jobResource) {
  ZNRecord r = propertyStore
      .get(Joiner.on("/").join(TaskConstants.REBALANCER_CONTEXT_ROOT, jobResource, CONTEXT_NODE),
          null, AccessOption.PERSISTENT);
  return r != null ? new JobContext(r) : null;
}
 

import org.apache.helix.store.HelixPropertyStore; //导入依赖的package包/类
/**
 * Get the runtime context of a single workflow.
 * This method is internal API, please use the corresponding one in TaskDriver.getWorkflowContext();
 *
 * @param propertyStore    Property store of the cluster
 * @param workflow The name of the workflow
 * @return the {@link WorkflowContext}, or null if none is available
 */
protected static WorkflowContext getWorkflowContext(HelixPropertyStore<ZNRecord> propertyStore,
    String workflow) {
  ZNRecord r = propertyStore.get(
      Joiner.on("/").join(TaskConstants.REBALANCER_CONTEXT_ROOT, workflow, CONTEXT_NODE),
      null, AccessOption.PERSISTENT);
  return r != null ? new WorkflowContext(r) : null;
}