Java PhysicalPlan类代码示例

import org.apache.drill.exec.physical.PhysicalPlan; //导入依赖的package包/类
public PhysicalPlanReader(DrillConfig config, ObjectMapper mapper, final DrillbitEndpoint endpoint,
                          final StoragePluginRegistry pluginRegistry) {

  // Endpoint serializer/deserializer.
  SimpleModule deserModule = new SimpleModule("PhysicalOperatorModule") //
      .addSerializer(DrillbitEndpoint.class, new DrillbitEndpointSerDe.Se()) //
      .addDeserializer(DrillbitEndpoint.class, new DrillbitEndpointSerDe.De()) //
      .addSerializer(MajorType.class, new MajorTypeSerDe.Se())
      .addDeserializer(MajorType.class, new MajorTypeSerDe.De());


  mapper.registerModule(deserModule);
  mapper.registerSubtypes(PhysicalOperatorUtil.getSubTypes(config));
  InjectableValues injectables = new InjectableValues.Std() //
          .addValue(StoragePluginRegistry.class, pluginRegistry) //
      .addValue(DrillbitEndpoint.class, endpoint); //

  this.mapper = mapper;
  this.physicalPlanReader = mapper.reader(PhysicalPlan.class).with(injectables);
  this.operatorReader = mapper.reader(PhysicalOperator.class).with(injectables);
  this.logicalPlanReader = mapper.reader(LogicalPlan.class).with(injectables);
}
 

import org.apache.drill.exec.physical.PhysicalPlan; //导入依赖的package包/类
/**
 * Function resolves the schema and invokes the drop method. Raises an exception if the schema is
 * immutable.
 * @param sqlNode - Table name identifier
 * @return - Single row indicating drop succeeded, raise exception otherwise
 * @throws ValidationException
 * @throws RelConversionException
 * @throws IOException
 */
@Override
public PhysicalPlan getPlan(SqlNode sqlNode) throws ValidationException, RelConversionException, IOException {

  SqlDropTable dropTableNode = ((SqlDropTable) sqlNode);
  SqlIdentifier tableIdentifier = dropTableNode.getTableIdentifier();

  SchemaPlus defaultSchema = context.getNewDefaultSchema();
  AbstractSchema drillSchema = null;

  if (tableIdentifier != null) {
    drillSchema = SchemaUtilites.resolveToMutableDrillSchema(defaultSchema, dropTableNode.getSchema());
  }

  String tableName = ((SqlDropTable) sqlNode).getName();
  if (drillSchema == null) {
    throw UserException.validationError()
        .message("Invalid table_name [%s]", tableName)
        .build(logger);
  }

  drillSchema.dropTable(tableName);

  return DirectPlan.createDirectPlan(context, true,
      String.format("Table [%s] %s", tableName, "dropped"));
}
 

import org.apache.drill.exec.physical.PhysicalPlan; //导入依赖的package包/类
@Override
public PhysicalPlan getPlan(SqlNode sqlNode) throws ValidationException, RelConversionException, IOException, ForemanSetupException {
  final ConvertedRelNode convertedRelNode = validateAndConvert(sqlNode);
  final RelDataType validatedRowType = convertedRelNode.getValidatedRowType();
  final RelNode queryRelNode = convertedRelNode.getConvertedNode();

  log("Optiq Logical", queryRelNode, logger);
  DrillRel drel = convertToDrel(queryRelNode, validatedRowType);

  log("Drill Logical", drel, logger);
  Prel prel = convertToPrel(drel);
  log("Drill Physical", prel, logger);
  PhysicalOperator pop = convertToPop(prel);
  PhysicalPlan plan = convertToPlan(pop);
  log("Drill Plan", plan, logger);
  return plan;
}
 

import org.apache.drill.exec.physical.PhysicalPlan; //导入依赖的package包/类
@Override
public PhysicalPlan getPlan(SqlNode sqlNode) throws ValidationException, RelConversionException, IOException, ForemanSetupException {
  SqlDropView dropView = unwrap(sqlNode, SqlDropView.class);
  final String viewToDrop = dropView.getName();
  final AbstractSchema drillSchema =
      SchemaUtilites.resolveToMutableDrillSchema(context.getNewDefaultSchema(), dropView.getSchemaPath());

  final String schemaPath = drillSchema.getFullSchemaName();

  final Table existingTable = SqlHandlerUtil.getTableFromSchema(drillSchema, viewToDrop);
  if (existingTable != null && existingTable.getJdbcTableType() != Schema.TableType.VIEW) {
    throw UserException.validationError()
        .message("[%s] is not a VIEW in schema [%s]", viewToDrop, schemaPath)
        .build(logger);
  } else if (existingTable == null) {
    throw UserException.validationError()
        .message("Unknown view [%s] in schema [%s].", viewToDrop, schemaPath)
        .build(logger);
  }

  drillSchema.dropView(viewToDrop);

  return DirectPlan.createDirectPlan(context, true,
      String.format("View [%s] deleted successfully from schema [%s].", viewToDrop, schemaPath));
}
 

import org.apache.drill.exec.physical.PhysicalPlan; //导入依赖的package包/类
@Override
public PhysicalPlan getPlan(SqlNode sqlNode) throws ValidationException, RelConversionException, IOException, ForemanSetupException {
  final ConvertedRelNode convertedRelNode = validateAndConvert(sqlNode);
  final RelDataType validatedRowType = convertedRelNode.getValidatedRowType();
  final RelNode queryRelNode = convertedRelNode.getConvertedNode();

  log("Optiq Logical", queryRelNode, logger);
  DrillRel drel = convertToDrel(queryRelNode, validatedRowType);
  log("Drill Logical", drel, logger);

  if (mode == ResultMode.LOGICAL) {
    LogicalExplain logicalResult = new LogicalExplain(drel, level, context);
    return DirectPlan.createDirectPlan(context, logicalResult);
  }

  Prel prel = convertToPrel(drel);
  log("Drill Physical", prel, logger);
  PhysicalOperator pop = convertToPop(prel);
  PhysicalPlan plan = convertToPlan(pop);
  log("Drill Plan", plan, logger);
  PhysicalExplain physicalResult = new PhysicalExplain(prel, plan, level, context);
  return DirectPlan.createDirectPlan(context, physicalResult);
}
 

import org.apache.drill.exec.physical.PhysicalPlan; //导入依赖的package包/类
private void parseAndRunLogicalPlan(final String json) throws ExecutionSetupException {
  LogicalPlan logicalPlan;
  try {
    logicalPlan = drillbitContext.getPlanReader().readLogicalPlan(json);
  } catch (final IOException e) {
    throw new ForemanException("Failure parsing logical plan.", e);
  }

  if (logicalPlan.getProperties().resultMode == ResultMode.LOGICAL) {
    throw new ForemanException(
        "Failure running plan.  You requested a result mode of LOGICAL and submitted a logical plan.  In this case you're output mode must be PHYSICAL or EXEC.");
  }

  log(logicalPlan);

  final PhysicalPlan physicalPlan = convert(logicalPlan);

  if (logicalPlan.getProperties().resultMode == ResultMode.PHYSICAL) {
    returnPhysical(physicalPlan);
    return;
  }

  log(physicalPlan);
  runPhysicalPlan(physicalPlan);
}
 

import org.apache.drill.exec.physical.PhysicalPlan; //导入依赖的package包/类
private void runPhysicalPlan(final PhysicalPlan plan) throws ExecutionSetupException {
  validatePlan(plan);
  setupSortMemoryAllocations(plan);
  acquireQuerySemaphore(plan);

  final QueryWorkUnit work = getQueryWorkUnit(plan);
  final List<PlanFragment> planFragments = work.getFragments();
  final PlanFragment rootPlanFragment = work.getRootFragment();
  assert queryId == rootPlanFragment.getHandle().getQueryId();

  drillbitContext.getWorkBus().addFragmentStatusListener(queryId, queryManager.getFragmentStatusListener());
  drillbitContext.getClusterCoordinator().addDrillbitStatusListener(queryManager.getDrillbitStatusListener());

  logger.debug("Submitting fragments to run.");

  // set up the root fragment first so we'll have incoming buffers available.
  setupRootFragment(rootPlanFragment, work.getRootOperator());

  setupNonRootFragments(planFragments);
  drillbitContext.getAllocator().resetFragmentLimits(); // TODO a global effect for this query?!?

  moveToState(QueryState.RUNNING, null);
  logger.debug("Fragments running.");
}
 

import org.apache.drill.exec.physical.PhysicalPlan; //导入依赖的package包/类
private void setupSortMemoryAllocations(final PhysicalPlan plan) {
  // look for external sorts
  final List<ExternalSort> sortList = new LinkedList<>();
  for (final PhysicalOperator op : plan.getSortedOperators()) {
    if (op instanceof ExternalSort) {
      sortList.add((ExternalSort) op);
    }
  }

  // if there are any sorts, compute the maximum allocation, and set it on them
  if (sortList.size() > 0) {
    final OptionManager optionManager = queryContext.getOptions();
    final long maxWidthPerNode = optionManager.getOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY).num_val;
    long maxAllocPerNode = Math.min(DrillConfig.getMaxDirectMemory(),
        queryContext.getConfig().getLong(ExecConstants.TOP_LEVEL_MAX_ALLOC));
    maxAllocPerNode = Math.min(maxAllocPerNode,
        optionManager.getOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY).num_val);
    final long maxSortAlloc = maxAllocPerNode / (sortList.size() * maxWidthPerNode);
    logger.debug("Max sort alloc: {}", maxSortAlloc);

    for(final ExternalSort externalSort : sortList) {
      externalSort.setMaxAllocation(maxSortAlloc);
    }
  }
}
 

import org.apache.drill.exec.physical.PhysicalPlan; //导入依赖的package包/类
@Override
public PhysicalPlan optimize(final OptimizationContext context, final LogicalPlan plan)
    throws OptimizerException {
  final Object obj = new Object();
  final Collection<SinkOperator> roots = plan.getGraph().getRoots();
  final List<PhysicalOperator> physOps = new ArrayList<>(roots.size());
  final LogicalConverter converter = new LogicalConverter(plan);

  for (SinkOperator op : roots) {
    final PhysicalOperator pop  = op.accept(converter, obj);
    physOps.add(pop);
  }

  final PlanProperties logicalProperties = plan.getProperties();
  final PlanProperties props = PlanProperties.builder()
      .type(PlanProperties.PlanType.APACHE_DRILL_PHYSICAL)
      .version(logicalProperties.version)
      .generator(logicalProperties.generator)
      .options(new JSONOptions(context.getOptions().getOptionList())).build();
  final PhysicalPlan p = new PhysicalPlan(props, physOps);
  return p;
}
 

import org.apache.drill.exec.physical.PhysicalPlan; //导入依赖的package包/类
private void verifyLimitCount(DrillbitContext bitContext, UserServer.UserClientConnection connection, String testPlan, int expectedCount) throws Throwable {
  final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
  final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/limit/" + testPlan), Charsets.UTF_8));
  final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
  final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
  final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
  int recordCount = 0;
  while(exec.next()) {
    recordCount += exec.getRecordCount();
  }

  assertEquals(expectedCount, recordCount);

  if(context.getFailureCause() != null) {
    throw context.getFailureCause();
  }

  assertTrue(!context.isFailed());
}
 

import org.apache.drill.exec.physical.PhysicalPlan; //导入依赖的package包/类
private void verifySum(DrillbitContext bitContext, UserServer.UserClientConnection connection, String testPlan, int expectedCount, long expectedSum) throws Throwable {
  final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
  final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/limit/" + testPlan), Charsets.UTF_8));
  final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
  final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
  final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
  int recordCount = 0;
  long sum = 0;
  while(exec.next()) {
    recordCount += exec.getRecordCount();
    final BigIntVector v = (BigIntVector) exec.iterator().next();
    for (int i = 0; i < v.getAccessor().getValueCount(); i++) {
      sum += v.getAccessor().get(i);
    }
  }

  assertEquals(expectedCount, recordCount);
  assertEquals(expectedSum, sum);

  if(context.getFailureCause() != null) {
    throw context.getFailureCause();
  }
  assertTrue(!context.isFailed());
}
 

import org.apache.drill.exec.physical.PhysicalPlan; //导入依赖的package包/类
private SimpleRootExec doTest(final DrillbitContext bitContext, UserClientConnection connection, String file) throws Exception {
  new NonStrictExpectations() {{
    bitContext.getMetrics(); result = new MetricRegistry();
    bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c);
    bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
    bitContext.getConfig(); result = c;
    bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
  }};

  final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
  final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile(file), Charsets.UTF_8));
  final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
  final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
  final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
  return exec;
}
 

import org.apache.drill.exec.physical.PhysicalPlan; //导入依赖的package包/类
private SimpleRootExec doPhysicalTest(final DrillbitContext bitContext, UserClientConnection connection, String file)
    throws Exception {
  new NonStrictExpectations() {
    {
      bitContext.getMetrics();
      result = new MetricRegistry();
      bitContext.getAllocator();
      result = RootAllocatorFactory.newRoot(config);
      bitContext.getConfig();
      result = config;
    }
  };

  final StoragePluginRegistry reg = new StoragePluginRegistry(bitContext);

  final PhysicalPlanReader reader = new PhysicalPlanReader(config, config.getMapper(),
      CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), reg);
  final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile(file), Charsets.UTF_8));
  final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(config);
  final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
  final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false)
      .iterator().next()));
  return exec;
}
 

import org.apache.drill.exec.physical.PhysicalPlan; //导入依赖的package包/类
@Test
public void testJoinBatchSize(@Injectable final DrillbitContext bitContext, @Injectable UserClientConnection connection) throws Throwable{
  new NonStrictExpectations() {{
    bitContext.getMetrics(); result = new MetricRegistry();
    bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c);
    bitContext.getConfig(); result = c;
    bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
    bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
  }};

  final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
  final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/join/join_batchsize.json"), Charsets.UTF_8));
  final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
  final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
  final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
  exec.next(); // skip schema batch
  while (exec.next()) {
    assertEquals(100, exec.getRecordCount());
  }

  if (context.getFailureCause() != null) {
    throw context.getFailureCause();
  }
  assertTrue(!context.isFailed());
}
 

import org.apache.drill.exec.physical.PhysicalPlan; //导入依赖的package包/类
@SuppressWarnings("deprecation")
private SimpleRootExec doTest(final DrillbitContext bitContext, UserClientConnection connection, String plan_path) throws Exception{
    new NonStrictExpectations() {{
      bitContext.getMetrics(); result = new MetricRegistry();
      bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c);
      bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
      bitContext.getConfig(); result = c;
      bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
    }};

    final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
    final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile(plan_path), Charsets.UTF_8));
    final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
    final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
    final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
    return exec;
  }