Java QueryUnitRequest类代码示例

import org.apache.tajo.engine.query.QueryUnitRequest; //导入依赖的package包/类
private void assignTask(QueryUnitAttemptScheduleContext attemptContext, QueryUnitAttempt taskAttempt) {
  QueryUnitAttemptId attemptId = taskAttempt.getId();
  ContainerProxy containerProxy = context.getMasterContext().getResourceAllocator().
      getContainer(attemptContext.getContainerId());
  QueryUnitRequest taskAssign = new QueryUnitRequestImpl(
      attemptId,
      new ArrayList<FragmentProto>(taskAttempt.getQueryUnit().getAllFragments()),
      "",
      false,
      taskAttempt.getQueryUnit().getLogicalPlan().toJson(),
      context.getMasterContext().getQueryContext(),
      subQuery.getDataChannel(), subQuery.getBlock().getEnforcer());
  if (checkIfInterQuery(subQuery.getMasterPlan(), subQuery.getBlock())) {
    taskAssign.setInterQuery();
  }

  if (!context.isLeafQuery()) {
    Map<String, List<URI>> fetch = scheduledFetches.getNextFetch();
    scheduledFetches.popNextFetch();

    for (Entry<String, List<URI>> fetchEntry : fetch.entrySet()) {
      for (URI eachValue : fetchEntry.getValue()) {
        taskAssign.addFetch(fetchEntry.getKey(), eachValue);
      }
    }
  }

  context.getMasterContext().getEventHandler().handle(new TaskAttemptAssignedEvent(attemptId,
      attemptContext.getContainerId(), attemptContext.getHost(), containerProxy.getTaskPort()));

  totalAssigned++;
  attemptContext.getCallback().run(taskAssign.getProto());

  if (context.isLeafQuery()) {
    LOG.debug("DiskLocalAssigned / Total: " + diskLocalAssigned + " / " + totalAssigned);
    LOG.debug("HostLocalAssigned / Total: " + hostLocalAssigned + " / " + totalAssigned);
    LOG.debug("RackLocalAssigned: " + rackLocalAssigned + " / " + totalAssigned);
  }
}
 

import org.apache.tajo.engine.query.QueryUnitRequest; //导入依赖的package包/类
public void assignToNonLeafTasks(LinkedList<TaskRequestEvent> taskRequests) {
  Collections.shuffle(taskRequests);

  TaskRequestEvent taskRequest;
  while (!taskRequests.isEmpty()) {
    taskRequest = taskRequests.pollFirst();
    LOG.debug("assignToNonLeafTasks: " + taskRequest.getExecutionBlockId());

    QueryUnitAttemptId attemptId;
    // random allocation
    if (nonLeafTasks.size() > 0) {
      synchronized (nonLeafTasks){
        attemptId = nonLeafTasks.iterator().next();
        nonLeafTasks.remove(attemptId);
      }
      LOG.debug("Assigned based on * match");

      QueryUnit task;
      task = subQuery.getQueryUnit(attemptId.getQueryUnitId());
      QueryUnitRequest taskAssign = new QueryUnitRequestImpl(
          attemptId,
          Lists.newArrayList(task.getAllFragments()),
          "",
          false,
          task.getLogicalPlan().toJson(),
          context.getMasterContext().getQueryContext(),
          subQuery.getDataChannel(),
          subQuery.getBlock().getEnforcer());
      if (checkIfInterQuery(subQuery.getMasterPlan(), subQuery.getBlock())) {
        taskAssign.setInterQuery();
      }
      for (ScanNode scan : task.getScanNodes()) {
        Collection<URI> fetches = task.getFetch(scan);
        if (fetches != null) {
          for (URI fetch : fetches) {
            taskAssign.addFetch(scan.getTableName(), fetch);
          }
        }
      }

      ContainerProxy container = context.getMasterContext().getResourceAllocator().getContainer(
          taskRequest.getContainerId());
      context.getMasterContext().getEventHandler().handle(new TaskAttemptAssignedEvent(attemptId,
          taskRequest.getContainerId(), container.getTaskHostName(), container.getTaskPort()));
      taskRequest.getCallback().run(taskAssign.getProto());
      totalAssigned++;
      scheduledObjectNum--;
    }
  }
}
 

import org.apache.tajo.engine.query.QueryUnitRequest; //导入依赖的package包/类
public Task(QueryUnitAttemptId taskId,
            final TaskRunner.TaskRunnerContext worker,
            final QueryMasterProtocolService.Interface masterProxy,
            final QueryUnitRequest request) throws IOException {
  this.request = request;
  this.taskId = taskId;

  this.systemConf = worker.getConf();
  this.queryContext = request.getQueryContext();
  this.taskRunnerContext = worker;
  this.masterProxy = masterProxy;
  this.localFS = worker.getLocalFS();
  this.lDirAllocator = worker.getLocalDirAllocator();
  this.taskDir = StorageUtil.concatPath(taskRunnerContext.getBaseDir(),
      taskId.getQueryUnitId().getId() + "_" + taskId.getId());

  this.context = new TaskAttemptContext(systemConf, taskId,
      request.getFragments().toArray(new FragmentProto[request.getFragments().size()]), taskDir);
  this.context.setDataChannel(request.getDataChannel());
  this.context.setEnforcer(request.getEnforcer());
  this.inputStats = new TableStats();

  this.reporter = new Reporter(taskId, masterProxy);
  this.reporter.startCommunicationThread();

  plan = CoreGsonHelper.fromJson(request.getSerializedData(), LogicalNode.class);
  LogicalNode [] scanNode = PlannerUtil.findAllNodes(plan, NodeType.SCAN);
  for (LogicalNode node : scanNode) {
    ScanNode scan = (ScanNode)node;
    descs.put(scan.getCanonicalName(), scan.getTableDesc());
  }

  interQuery = request.getProto().getInterQuery();
  if (interQuery) {
    context.setInterQuery();
    this.shuffleType = context.getDataChannel().getShuffleType();

    if (shuffleType == ShuffleType.RANGE_SHUFFLE) {
      SortNode sortNode = PlannerUtil.findTopNode(plan, NodeType.SORT);
      this.finalSchema = PlannerUtil.sortSpecsToSchema(sortNode.getSortKeys());
      this.sortComp = new TupleComparator(finalSchema, sortNode.getSortKeys());
    }
  } else {
    // The final result of a task will be written in a file named part-ss-nnnnnnn,
    // where ss is the subquery id associated with this task, and nnnnnn is the task id.
    Path outFilePath = StorageUtil.concatPath(queryContext.getStagingDir(), TajoConstants.RESULT_DIR_NAME,
        OUTPUT_FILE_PREFIX +
        OUTPUT_FILE_FORMAT_SUBQUERY.get().format(taskId.getQueryUnitId().getExecutionBlockId().getId()) + "-" +
        OUTPUT_FILE_FORMAT_TASK.get().format(taskId.getQueryUnitId().getId()));
    LOG.info("Output File Path: " + outFilePath);
    context.setOutputPath(outFilePath);
  }

  context.setState(TaskAttemptState.TA_PENDING);
  LOG.info("==================================");
  LOG.info("* Subquery " + request.getId() + " is initialized");
  LOG.info("* InterQuery: " + interQuery
      + (interQuery ? ", Use " + this.shuffleType + " shuffle":""));

  LOG.info("* Fragments (num: " + request.getFragments().size() + ")");
  LOG.info("* Fetches (total:" + request.getFetches().size() + ") :");
  for (Fetch f : request.getFetches()) {
    LOG.info("Table Id: " + f.getName() + ", url: " + f.getUrls());
  }
  LOG.info("* Local task dir: " + taskDir);
  if(LOG.isDebugEnabled()) {
    LOG.debug("* plan:\n");
    LOG.debug(plan.toString());
  }
  LOG.info("==================================");
}
 

import org.apache.tajo.engine.query.QueryUnitRequest; //导入依赖的package包/类
public void localize(QueryUnitRequest request) throws IOException {
  fetcherRunners = getFetchRunners(context, request.getFetches());
}