Java QueryUnitAttemptId类代码示例

import org.apache.tajo.QueryUnitAttemptId; //导入依赖的package包/类
public TaskAttemptAssignedEvent(QueryUnitAttemptId id, ContainerId cId,
                                String hostname, int pullServerPort) {
  super(id, TaskAttemptEventType.TA_ASSIGNED);
  this.cId = cId;
  this.hostName = hostname;
  this.pullServerPort = pullServerPort;
}
 

import org.apache.tajo.QueryUnitAttemptId; //导入依赖的package包/类
static void fatalError(QueryMasterProtocolService.Interface qmClientService,
                       QueryUnitAttemptId taskAttemptId, String message) {
  if (message == null) {
     message = "No error message";
  }
  TaskFatalErrorReport.Builder builder = TaskFatalErrorReport.newBuilder()
      .setId(taskAttemptId.getProto())
      .setErrorMessage(message);

  qmClientService.fatalError(null, builder.build(), NullCallback.get());
}
 

import org.apache.tajo.QueryUnitAttemptId; //导入依赖的package包/类
public synchronized QueryUnitAttemptId getQueryUnitAttemptIdByRack(String rack) {
  QueryUnitAttemptId queryUnitAttemptId = null;

  if (unassignedTaskForEachVolume.size() > 0 && this.rack.equals(rack)) {
    int retry = unassignedTaskForEachVolume.size();
    do {
      //clean and get a remaining task
      int volumeId = getLowestVolumeId();
      queryUnitAttemptId = getAndRemove(volumeId);
      if (queryUnitAttemptId == null) {
        if (volumeId > REMOTE) {
          diskVolumeLoads.remove(volumeId);
        }
        retry--;
      } else {
        break;
      }
    } while (retry > 0);
  }
  return queryUnitAttemptId;
}
 

import org.apache.tajo.QueryUnitAttemptId; //导入依赖的package包/类
private QueryUnitAttemptId allocateLocalTask(String host, ContainerId containerId){
  HostVolumeMapping hostVolumeMapping = leafTaskHostMapping.get(host);

  if (hostVolumeMapping != null) { //tajo host is located in hadoop datanode
    while (hostVolumeMapping.getRemainingLocalTaskSize() > 0) {
      QueryUnitAttemptId attemptId = hostVolumeMapping.getLocalTask(containerId);
      //find remaining local task
      if (leafTasks.contains(attemptId)) {
        leafTasks.remove(attemptId);
        //LOG.info(attemptId + " Assigned based on host match " + hostName);
        hostLocalAssigned++;
        totalAssigned++;
        return attemptId;
      }
    }
  }
  return null;
}
 

import org.apache.tajo.QueryUnitAttemptId; //导入依赖的package包/类
public TaskAttemptContext(TajoConf conf, final QueryUnitAttemptId queryId,
                          final FragmentProto[] fragments,
                          final Path workDir) {
  this.conf = conf;
  this.queryId = queryId;
  
  for(FragmentProto t : fragments) {
    if (fragmentMap.containsKey(t.getId())) {
      fragmentMap.get(t.getId()).add(t);
    } else {
      List<FragmentProto> frags = new ArrayList<FragmentProto>();
      frags.add(t);
      fragmentMap.put(t.getId(), frags);
    }
  }

  this.workDir = workDir;
  this.shuffleFileOutputs = Maps.newHashMap();

  state = TaskAttemptState.TA_PENDING;
}
 

import org.apache.tajo.QueryUnitAttemptId; //导入依赖的package包/类
public void set(QueryUnitAttemptId id, List<FragmentProto> fragments,
		String outputTable, boolean clusteredOutput,
		String serializedData, QueryContext queryContext, DataChannel dataChannel, Enforcer enforcer) {
	this.id = id;
	this.fragments = fragments;
	this.outputTable = outputTable;
	this.clusteredOutput = clusteredOutput;
	this.serializedData = serializedData;
	this.isUpdated = true;
   this.queryContext = queryContext;
   this.queryContext = queryContext;
   this.dataChannel = dataChannel;
   this.enforcer = enforcer;
}
 

import org.apache.tajo.QueryUnitAttemptId; //导入依赖的package包/类
@Override
public QueryUnitAttemptId getId() {
	QueryUnitRequestProtoOrBuilder p = viaProto ? proto : builder;
	if (id != null) {
		return this.id;
	}
	if (!p.hasId()) {
		return null;
	}
	this.id = new QueryUnitAttemptId(p.getId());
	return this.id;
}
 

import org.apache.tajo.QueryUnitAttemptId; //导入依赖的package包/类
public QueryUnitAttempt(final QueryUnitAttemptScheduleContext scheduleContext,
                        final QueryUnitAttemptId id, final QueryUnit queryUnit,
                        final EventHandler eventHandler) {
  this.scheduleContext = scheduleContext;
  this.id = id;
  this.expire = QueryUnitAttempt.EXPIRE_TIME;
  this.queryUnit = queryUnit;
  this.eventHandler = eventHandler;

  ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
  this.readLock = readWriteLock.readLock();
  this.writeLock = readWriteLock.writeLock();

  stateMachine = stateMachineFactory.make(this);
}
 

import org.apache.tajo.QueryUnitAttemptId; //导入依赖的package包/类
private void addAndScheduleAttempt() {
  // Create new task attempt
  QueryUnitAttempt attempt = newAttempt();
  if (LOG.isDebugEnabled()) {
    LOG.debug("Created attempt " + attempt.getId());
  }
  switch (attempts.size()) {
    case 0:
      attempts = Collections.singletonMap(attempt.getId(), attempt);
      break;

    case 1:
      Map<QueryUnitAttemptId, QueryUnitAttempt> newAttempts
          = new LinkedHashMap<QueryUnitAttemptId, QueryUnitAttempt>(3);
      newAttempts.putAll(attempts);
      attempts = newAttempts;
      attempts.put(attempt.getId(), attempt);
      break;

    default:
      attempts.put(attempt.getId(), attempt);
      break;
  }

  if (failedAttempts > 0) {
    eventHandler.handle(new TaskAttemptScheduleEvent(systemConf, attempt.getId(),
        TaskAttemptEventType.TA_RESCHEDULE));
  } else {
    eventHandler.handle(new TaskAttemptScheduleEvent(systemConf, attempt.getId(),
        TaskAttemptEventType.TA_SCHEDULE));
  }
}
 

import org.apache.tajo.QueryUnitAttemptId; //导入依赖的package包/类
/**
 * It sends a kill RPC request to a corresponding worker.
 *
 * @param taskAttemptId The TaskAttemptId to be killed.
 */
public void killTaskAttempt(QueryUnitAttemptId taskAttemptId) {
  NettyClientBase tajoWorkerRpc = null;
  try {
    InetSocketAddress addr = new InetSocketAddress(container.getNodeId().getHost(), container.getNodeId().getPort());
    tajoWorkerRpc = RpcConnectionPool.getPool(context.getConf()).getConnection(addr, TajoWorkerProtocol.class, true);
    TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub();
    tajoWorkerRpcClient.killTaskAttempt(null, taskAttemptId.getProto(), NullCallback.get());
  } catch (Exception e) {
    LOG.error(e.getMessage(), e);
  } finally {
    RpcConnectionPool.getPool(context.getConf()).releaseConnection(tajoWorkerRpc);
  }
}
 

import org.apache.tajo.QueryUnitAttemptId; //导入依赖的package包/类
/**
 *  Priorities
 *  1. a task list in a volume of host
 *  2. unknown block or Non-splittable task in host
 *  3. remote tasks. unassignedTaskForEachVolume is only contained local task. so it will be null
 */
public synchronized QueryUnitAttemptId getLocalTask(ContainerId containerId) {
  int volumeId;
  QueryUnitAttemptId queryUnitAttemptId = null;

  if (!lastAssignedVolumeId.containsKey(containerId)) {
    volumeId = getLowestVolumeId();
    increaseConcurrency(containerId, volumeId);
  } else {
    volumeId = lastAssignedVolumeId.get(containerId);
  }

  if (unassignedTaskForEachVolume.size() >  0) {
    int retry = unassignedTaskForEachVolume.size();
    do {
      //clean and get a remaining local task
      queryUnitAttemptId = getAndRemove(volumeId);
      if(!unassignedTaskForEachVolume.containsKey(volumeId)) {
        decreaseConcurrency(containerId);
        if (volumeId > REMOTE) {
          diskVolumeLoads.remove(volumeId);
        }
      }

      if (queryUnitAttemptId == null) {
        //reassign next volume
        volumeId = getLowestVolumeId();
        increaseConcurrency(containerId, volumeId);
        retry--;
      } else {
        break;
      }
    } while (retry > 0);
  } else {
    this.remainTasksNum.set(0);
  }
  return queryUnitAttemptId;
}
 

import org.apache.tajo.QueryUnitAttemptId; //导入依赖的package包/类
private synchronized QueryUnitAttemptId getAndRemove(int volumeId){
  QueryUnitAttemptId queryUnitAttemptId = null;
  if(!unassignedTaskForEachVolume.containsKey(volumeId)) return queryUnitAttemptId;

  LinkedHashSet<QueryUnitAttempt> list = unassignedTaskForEachVolume.get(volumeId);
  if(list != null && list.size() > 0){
    QueryUnitAttempt queryUnitAttempt;
    synchronized (unassignedTaskForEachVolume) {
      Iterator<QueryUnitAttempt> iterator = list.iterator();
      queryUnitAttempt = iterator.next();
      iterator.remove();
    }

    this.remainTasksNum.getAndDecrement();
    queryUnitAttemptId = queryUnitAttempt.getId();
    for (DataLocation location : queryUnitAttempt.getQueryUnit().getDataLocations()) {
      if (!this.getHost().equals(location.getHost())) {
        HostVolumeMapping volumeMapping = scheduledRequests.leafTaskHostMapping.get(location.getHost());
        volumeMapping.removeQueryUnitAttempt(location.getVolumeId(), queryUnitAttempt);
      }
    }
  }

  if(list == null || list.isEmpty()) {
    unassignedTaskForEachVolume.remove(volumeId);
  }
  return queryUnitAttemptId;
}
 

import org.apache.tajo.QueryUnitAttemptId; //导入依赖的package包/类
private void addLeafTask(QueryUnitAttemptScheduleEvent event) {
  QueryUnitAttempt queryUnitAttempt = event.getQueryUnitAttempt();
  List<DataLocation> locations = queryUnitAttempt.getQueryUnit().getDataLocations();

  for (DataLocation location : locations) {
    String host = location.getHost();

    HostVolumeMapping hostVolumeMapping = leafTaskHostMapping.get(host);
    if (hostVolumeMapping == null) {
      String rack = RackResolver.resolve(host).getNetworkLocation();
      hostVolumeMapping = new HostVolumeMapping(host, rack);
      leafTaskHostMapping.put(host, hostVolumeMapping);
    }
    hostVolumeMapping.addQueryUnitAttempt(location.getVolumeId(), queryUnitAttempt);

    if (LOG.isDebugEnabled()) {
      LOG.debug("Added attempt req to host " + host);
    }

    LinkedList<QueryUnitAttemptId> list = leafTasksRackMapping.get(hostVolumeMapping.getRack());
    if (list == null) {
      list = new LinkedList<QueryUnitAttemptId>();
      leafTasksRackMapping.put(hostVolumeMapping.getRack(), list);
    }

    if(!list.contains(queryUnitAttempt.getId())) list.add(queryUnitAttempt.getId());

    if (LOG.isDebugEnabled()) {
      LOG.debug("Added attempt req to rack " + hostVolumeMapping.getRack());
    }
  }

  leafTasks.add(queryUnitAttempt.getId());
}
 

import org.apache.tajo.QueryUnitAttemptId; //导入依赖的package包/类
private void assignTask(QueryUnitAttemptScheduleContext attemptContext, QueryUnitAttempt taskAttempt) {
  QueryUnitAttemptId attemptId = taskAttempt.getId();
  ContainerProxy containerProxy = context.getMasterContext().getResourceAllocator().
      getContainer(attemptContext.getContainerId());
  QueryUnitRequest taskAssign = new QueryUnitRequestImpl(
      attemptId,
      new ArrayList<FragmentProto>(taskAttempt.getQueryUnit().getAllFragments()),
      "",
      false,
      taskAttempt.getQueryUnit().getLogicalPlan().toJson(),
      context.getMasterContext().getQueryContext(),
      subQuery.getDataChannel(), subQuery.getBlock().getEnforcer());
  if (checkIfInterQuery(subQuery.getMasterPlan(), subQuery.getBlock())) {
    taskAssign.setInterQuery();
  }

  if (!context.isLeafQuery()) {
    Map<String, List<URI>> fetch = scheduledFetches.getNextFetch();
    scheduledFetches.popNextFetch();

    for (Entry<String, List<URI>> fetchEntry : fetch.entrySet()) {
      for (URI eachValue : fetchEntry.getValue()) {
        taskAssign.addFetch(fetchEntry.getKey(), eachValue);
      }
    }
  }

  context.getMasterContext().getEventHandler().handle(new TaskAttemptAssignedEvent(attemptId,
      attemptContext.getContainerId(), attemptContext.getHost(), containerProxy.getTaskPort()));

  totalAssigned++;
  attemptContext.getCallback().run(taskAssign.getProto());

  if (context.isLeafQuery()) {
    LOG.debug("DiskLocalAssigned / Total: " + diskLocalAssigned + " / " + totalAssigned);
    LOG.debug("HostLocalAssigned / Total: " + hostLocalAssigned + " / " + totalAssigned);
    LOG.debug("RackLocalAssigned: " + rackLocalAssigned + " / " + totalAssigned);
  }
}
 

import org.apache.tajo.QueryUnitAttemptId; //导入依赖的package包/类
public void register(QueryUnitAttemptId id, RetrieverHandler handler) {
  synchronized (handlerMap) {
    if (!handlerMap.containsKey(id.toString())) {
      handlerMap.put(id.toString(), handler);
    }
  } 
}