Java FragmentContext.getHandle方法代码示例

import org.apache.drill.exec.ops.FragmentContext; //导入方法依赖的package包/类
private void removeFragmentContext(FragmentContext c) {
  if (parent != null){
    if (parent.parent==null){
      // only fragment level allocators will have the fragment context saved
      parent.removeFragmentContext(c);
    }
  }else{
    if(logger.isDebugEnabled()) {
      FragmentHandle hndle;
      String fragStr;
      if (c != null) {
        hndle = c.getHandle();
        fragStr = (hndle != null) ? (hndle.getMajorFragmentId() + ":" + hndle.getMinorFragmentId()) : "[Null Fragment Handle]";
      } else {
        fragStr = "[Null Context]";
      }
      fragStr += " (Object Id: " + System.identityHashCode(c) + ")";
      logger.trace("Fragment " + fragStr + " removed from root accountor");
    }
    synchronized(this) {
      fragmentContexts.remove(c);
    }
  }
}
 

import org.apache.drill.exec.ops.FragmentContext; //导入方法依赖的package包/类
public ExternalSortBatch(ExternalSort popConfig, FragmentContext context, RecordBatch incoming) throws OutOfMemoryException {
  super(popConfig, context, true);
  this.incoming = incoming;
  DrillConfig config = context.getConfig();
  Configuration conf = new Configuration();
  conf.set("fs.default.name", config.getString(ExecConstants.EXTERNAL_SORT_SPILL_FILESYSTEM));
  try {
    this.fs = FileSystem.get(conf);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
  SPILL_BATCH_GROUP_SIZE = config.getInt(ExecConstants.EXTERNAL_SORT_SPILL_GROUP_SIZE);
  SPILL_THRESHOLD = config.getInt(ExecConstants.EXTERNAL_SORT_SPILL_THRESHOLD);
  dirs = Iterators.cycle(config.getStringList(ExecConstants.EXTERNAL_SORT_SPILL_DIRS));
  copierAllocator = oContext.getAllocator().getChildAllocator(
      context, PriorityQueueCopier.INITIAL_ALLOCATION, PriorityQueueCopier.MAX_ALLOCATION, true);
  FragmentHandle handle = context.getHandle();
  fileName = String.format("%s/major_fragment_%s/minor_fragment_%s/operator_%s", QueryIdHelper.getQueryId(handle.getQueryId()),
      handle.getMajorFragmentId(), handle.getMinorFragmentId(), popConfig.getOperatorId());
}
 

import org.apache.drill.exec.ops.FragmentContext; //导入方法依赖的package包/类
public static String getFileNameForQueryFragment(FragmentContext context, String location, String tag) {
   /*
   * From the context, get the query id, major fragment id, minor fragment id. This will be used as the file name to
   * which we will dump the incoming buffer data
   */
  ExecProtos.FragmentHandle handle = context.getHandle();

  String qid = QueryIdHelper.getQueryId(handle.getQueryId());

  int majorFragmentId = handle.getMajorFragmentId();
  int minorFragmentId = handle.getMinorFragmentId();

  String fileName = String.format("%s//%s_%s_%s_%s", location, qid, majorFragmentId, minorFragmentId, tag);

  return fileName;
}
 

import org.apache.drill.exec.ops.FragmentContext; //导入方法依赖的package包/类
@Override
public RecordWriter getRecordWriter(FragmentContext context, EasyWriter writer) throws IOException {
  Map<String, String> options = Maps.newHashMap();

  options.put("location", writer.getLocation());

  FragmentHandle handle = context.getHandle();
  String fragmentId = String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId());
  options.put("prefix", fragmentId);

  options.put("separator", " ");
  options.put(FileSystem.FS_DEFAULT_NAME_KEY, ((FileSystemConfig)writer.getStorageConfig()).connection);

  options.put("extension", "json");
  options.put("extended", Boolean.toString(context.getOptions().getOption(ExecConstants.JSON_EXTENDED_TYPES)));

  RecordWriter recordWriter = new JsonRecordWriter();
  recordWriter.init(options);

  return recordWriter;
}
 

import org.apache.drill.exec.ops.FragmentContext; //导入方法依赖的package包/类
@Override
public RecordWriter getRecordWriter(FragmentContext context, EasyWriter writer) throws IOException {
  Map<String, String> options = Maps.newHashMap();

  options.put("location", writer.getLocation());

  FragmentHandle handle = context.getHandle();
  String fragmentId = String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId());
  options.put("prefix", fragmentId);

  options.put("separator", ((ModelFormatConfig)getConfig()).getDelimiter());
  options.put(FileSystem.FS_DEFAULT_NAME_KEY, ((FileSystemConfig)writer.getStorageConfig()).connection);

  options.put("extension", ((ModelFormatConfig)getConfig()).getExtensions().get(0));

  RecordWriter recordWriter = new DrillModelWriter(/*context.getAllocator()*/);
  recordWriter.init(options);

  return recordWriter;
}
 

import org.apache.drill.exec.ops.FragmentContext; //导入方法依赖的package包/类
@Override
public RecordWriter getRecordWriter(final FragmentContext context, final EasyWriter writer) throws IOException {
  final Map<String, String> options = Maps.newHashMap();

  options.put("location", writer.getLocation());

  FragmentHandle handle = context.getHandle();
  String fragmentId = String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId());
  options.put("prefix", fragmentId);

  options.put("separator", ((TextFormatConfig)getConfig()).getFieldDelimiterAsString());
  options.put(FileSystem.FS_DEFAULT_NAME_KEY, ((FileSystemConfig)writer.getStorageConfig()).connection);

  options.put("extension", ((TextFormatConfig)getConfig()).getExtensions().get(0));

  RecordWriter recordWriter = new DrillTextRecordWriter(context.getAllocator());
  recordWriter.init(options);

  return recordWriter;
}
 

import org.apache.drill.exec.ops.FragmentContext; //导入方法依赖的package包/类
public RecordWriter getRecordWriter(FragmentContext context, ParquetWriter writer) throws IOException, OutOfMemoryException {
  Map<String, String> options = Maps.newHashMap();

  options.put("location", writer.getLocation());

  FragmentHandle handle = context.getHandle();
  String fragmentId = String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId());
  options.put("prefix", fragmentId);

  options.put(FileSystem.FS_DEFAULT_NAME_KEY, ((FileSystemConfig)writer.getStorageConfig()).connection);

  options.put(ExecConstants.PARQUET_BLOCK_SIZE, context.getOptions().getOption(ExecConstants.PARQUET_BLOCK_SIZE).num_val.toString());
  options.put(ExecConstants.PARQUET_PAGE_SIZE, context.getOptions().getOption(ExecConstants.PARQUET_PAGE_SIZE).num_val.toString());
  options.put(ExecConstants.PARQUET_DICT_PAGE_SIZE, context.getOptions().getOption(ExecConstants.PARQUET_DICT_PAGE_SIZE).num_val.toString());

  options.put(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE,
      context.getOptions().getOption(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE).string_val);

  options.put(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING,
      context.getOptions().getOption(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING).bool_val.toString());

  RecordWriter recordWriter = new ParquetRecordWriter(context, writer);
  recordWriter.init(options);

  return recordWriter;
}
 

import org.apache.drill.exec.ops.FragmentContext; //导入方法依赖的package包/类
public Accountor(DrillConfig config, boolean errorOnLeak, FragmentContext context, Accountor parent, long max, long preAllocated, boolean applyFragLimit) {
  // TODO: fix preallocation stuff
  this.errorOnLeak = errorOnLeak;
  AtomicRemainder parentRemainder = parent != null ? parent.remainder : null;
  this.parent = parent;

  boolean enableFragmentLimit;
  double  fragmentMemOvercommitFactor;

  try {
    enableFragmentLimit = config.getBoolean(ExecConstants.ENABLE_FRAGMENT_MEMORY_LIMIT);
    fragmentMemOvercommitFactor = config.getDouble(ExecConstants.FRAGMENT_MEM_OVERCOMMIT_FACTOR);
  }catch(Exception e){
    enableFragmentLimit = DEFAULT_ENABLE_FRAGMENT_LIMIT;
    fragmentMemOvercommitFactor = DEFAULT_FRAGMENT_MEM_OVERCOMMIT_FACTOR;
  }
  this.enableFragmentLimit = enableFragmentLimit;
  this.fragmentMemOvercommitFactor = fragmentMemOvercommitFactor;


  this.applyFragmentLimit=applyFragLimit;

  this.remainder = new AtomicRemainder(errorOnLeak, parentRemainder, max, preAllocated, applyFragmentLimit);
  this.total = max;
  this.fragmentContext=context;
  this.handle = (context!=null) ? context.getHandle() : null;
  this.fragmentStr= (handle!=null) ? ( handle.getMajorFragmentId()+":"+handle.getMinorFragmentId() ) : "0:0";
  this.fragmentLimit=this.total; // Allow as much as possible to start with;
  if (ENABLE_ACCOUNTING) {
    buffers = Maps.newConcurrentMap();
  } else {
    buffers = null;
  }
  this.fragmentContexts = new ArrayList<FragmentContext>();
  if(parent!=null && parent.parent==null){ // Only add the fragment context to the fragment level accountor
    synchronized(this) {
      addFragmentContext(this.fragmentContext);
    }
  }
}
 

import org.apache.drill.exec.ops.FragmentContext; //导入方法依赖的package包/类
private void addFragmentContext(FragmentContext c) {
  if (parent != null){
    parent.addFragmentContext(c);
  }else {
    if(logger.isTraceEnabled()) {
      FragmentHandle hndle;
      String fragStr;
      if(c!=null) {
        hndle = c.getHandle();
        fragStr = (hndle != null) ? (hndle.getMajorFragmentId() + ":" + hndle.getMinorFragmentId()) : "[Null Fragment Handle]";
      }else{
        fragStr = "[Null Context]";
      }
      fragStr+=" (Object Id: "+System.identityHashCode(c)+")";
      StackTraceElement[] ste = (new Throwable()).getStackTrace();
      StringBuffer sb = new StringBuffer();
      for (StackTraceElement s : ste) {
        sb.append(s.toString());
        sb.append("\n");
      }

      logger.trace("Fragment " + fragStr + " added to root accountor.\n"+sb.toString());
    }
    synchronized(this) {
      fragmentContexts.add(c);
    }
  }
}
 

import org.apache.drill.exec.ops.FragmentContext; //导入方法依赖的package包/类
public WriterRecordBatch(Writer writer, RecordBatch incoming, FragmentContext context, RecordWriter recordWriter) throws OutOfMemoryException {
  super(writer, context, false);
  this.incoming = incoming;

  final FragmentHandle handle = context.getHandle();
  fragmentUniqueId = String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId());
  this.recordWriter = recordWriter;
}
 

import org.apache.drill.exec.ops.FragmentContext; //导入方法依赖的package包/类
public BroadcastSenderRootExec(FragmentContext context,
                               RecordBatch incoming,
                               BroadcastSender config) throws OutOfMemoryException {
  super(context, context.newOperatorContext(config, null, false), config);
  this.ok = true;
  this.incoming = incoming;
  this.config = config;
  this.handle = context.getHandle();
  List<MinorFragmentEndpoint> destinations = config.getDestinations();
  ArrayListMultimap<DrillbitEndpoint, Integer> dests = ArrayListMultimap.create();

  for(MinorFragmentEndpoint destination : destinations) {
    dests.put(destination.getEndpoint(), destination.getId());
  }

  int destCount = dests.keySet().size();
  int i = 0;

  this.tunnels = new AccountingDataTunnel[destCount];
  this.receivingMinorFragments = new int[destCount][];
  for(final DrillbitEndpoint ep : dests.keySet()){
    List<Integer> minorsList= dests.get(ep);
    int[] minorsArray = new int[minorsList.size()];
    int x = 0;
    for(Integer m : minorsList){
      minorsArray[x++] = m;
    }
    receivingMinorFragments[i] = minorsArray;
    tunnels[i] = context.getDataTunnel(ep);
    i++;
  }
}
 

import org.apache.drill.exec.ops.FragmentContext; //导入方法依赖的package包/类
@Test
public void testDumpCat(@Injectable final DrillbitContext bitContext, @Injectable UserClientConnection connection) throws Throwable
{

    new NonStrictExpectations(){{
        bitContext.getMetrics(); result = new MetricRegistry();
        bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c);
        bitContext.getConfig(); result = c;
        bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
        bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
    }};

    final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
    final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/trace/simple_trace.json"), Charsets.UTF_8));
    final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
    final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
    final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));

    while(exec.next()) {
    }

    if(context.getFailureCause() != null) {
        throw context.getFailureCause();
    }
    assertTrue(!context.isFailed());

    exec.close();

    FragmentHandle handle = context.getHandle();

    /* Form the file name to which the trace output will dump the record batches */
    String qid = QueryIdHelper.getQueryId(handle.getQueryId());

    final int majorFragmentId = handle.getMajorFragmentId();
    final int minorFragmentId = handle.getMinorFragmentId();

    final String logLocation = c.getString(ExecConstants.TRACE_DUMP_DIRECTORY);

    System.out.println("Found log location: " + logLocation);

    final String filename = String.format("%s//%s_%d_%d_mock-scan", logLocation, qid, majorFragmentId, minorFragmentId);

    System.out.println("File Name: " + filename);

    final Configuration conf = new Configuration();
    conf.set(FileSystem.FS_DEFAULT_NAME_KEY, c.getString(ExecConstants.TRACE_DUMP_FILESYSTEM));

    final FileSystem fs = FileSystem.get(conf);
    final Path path = new Path(filename);
    assertTrue("Trace file does not exist", fs.exists(path));

    final DumpCat dumpCat = new DumpCat();

    //Test Query mode
    try (final FileInputStream input = new FileInputStream(filename)) {
      dumpCat.doQuery(input);
    }

    //Test Batch mode
    try(final FileInputStream input = new FileInputStream(filename)) {
      dumpCat.doBatch(input, 0, true);
    }
}
 

import org.apache.drill.exec.ops.FragmentContext; //导入方法依赖的package包/类
@Test
public void testFilter(@Injectable final DrillbitContext bitContext, @Injectable UserClientConnection connection) throws Throwable {
  new NonStrictExpectations() {{
    bitContext.getMetrics(); result = new MetricRegistry();
    bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c);
    bitContext.getConfig(); result = c;
    bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
    bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
  }};

  final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
  final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/trace/simple_trace.json"), Charsets.UTF_8));
  final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
  final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
  final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));

  while(exec.next()) {
  }

  exec.close();

  if(context.getFailureCause() != null) {
    throw context.getFailureCause();
  }
  assertTrue(!context.isFailed());

  final FragmentHandle handle = context.getHandle();

  /* Form the file name to which the trace output will dump the record batches */
  final String qid = QueryIdHelper.getQueryId(handle.getQueryId());
  final int majorFragmentId = handle.getMajorFragmentId();
  final int minorFragmentId = handle.getMinorFragmentId();

  final String logLocation = c.getString(ExecConstants.TRACE_DUMP_DIRECTORY);
  System.out.println("Found log location: " + logLocation);

  final String filename = String.format("%s//%s_%d_%d_mock-scan", logLocation, qid, majorFragmentId, minorFragmentId);
  System.out.println("File Name: " + filename);

  final Configuration conf = new Configuration();
  conf.set(FileSystem.FS_DEFAULT_NAME_KEY, c.getString(ExecConstants.TRACE_DUMP_FILESYSTEM));

  final FileSystem fs = FileSystem.get(conf);
  final Path path = new Path(filename);
  assertTrue("Trace file does not exist", fs.exists(path));
  final FSDataInputStream in = fs.open(path);

  final VectorAccessibleSerializable wrap = new VectorAccessibleSerializable(context.getAllocator());
  wrap.readFromStream(in);
  final VectorAccessible container = wrap.get();

  /* Assert there are no selection vectors */
  assertTrue(wrap.getSv2() == null);

  /* Assert there is only one record */
  assertTrue(container.getRecordCount() == 1);

  /* Read the Integer value and ASSERT its Integer.MIN_VALUE */
  final int value = (int) container.iterator().next().getValueVector().getAccessor().getObject(0);
  assertTrue(value == Integer.MIN_VALUE);
}