Java Tap.openForRead方法代码示例

import cascading.tap.Tap; //导入方法依赖的package包/类
/**
 * Tests the content of an output path against the given expected path.
 */
@SuppressWarnings("unchecked")
private void testPaths(String actual, String expected) throws Exception {

  Tap outputTest = new Hfs(new TextLine(), actual);
  Tap expectedTest = new Hfs(new TextLine(), expected);

  FlowProcess outputProcess = new HadoopFlowProcess(new JobConf(new Configuration()));
  FlowProcess expectedProcess = new HadoopFlowProcess(new JobConf(new Configuration()));

  TupleEntryIterator outputIterator = outputTest.openForRead(outputProcess);
  TupleEntryIterator expectedIterator = expectedTest.openForRead(expectedProcess);

  List<String> outputList = new ArrayList<>();
  while (outputIterator.hasNext()) {
    outputList.add(outputIterator.next().getTuple().getString(1));
  }

  List<String> expectedList = new ArrayList<>();
  while (expectedIterator.hasNext()) {
    expectedList.add(expectedIterator.next().getTuple().getString(1));
  }

  assertTrue(outputList.equals(expectedList));

}
 

import cascading.tap.Tap; //导入方法依赖的package包/类
public static boolean compareTaps(final Tap source1, final Tap source2, final Configuration conf) throws IOException {
    final FlowProcess flowProcess1 = new HadoopFlowProcess(new JobConf(conf));
    source1.getScheme().retrieveSourceFields(flowProcess1, source1);
    final TupleEntryIterator iter1 = source1.openForRead(new HadoopFlowProcess(new JobConf(conf)));
    final FlowProcess flowProcess2 = new HadoopFlowProcess(new JobConf(conf));
    source2.getScheme().retrieveSourceFields(flowProcess2, source2);
    final TupleEntryIterator iter2 = source2.openForRead(new HadoopFlowProcess(new JobConf(conf)));
    if (!iter1.getFields().equals(iter2.getFields()))
        return false;
    List<Tuple> list1 = new ArrayList<Tuple>();
    while (iter1.hasNext())
        list1.add(new Tuple(iter1.next().getTuple()));
    iter1.close();
    Collections.sort(list1);
    List<Tuple> list2 = new ArrayList<Tuple>();
    while (iter2.hasNext())
        list2.add(new Tuple(iter2.next().getTuple()));
    iter2.close();
    Collections.sort(list2);
    return list1.equals(list2);
}
 

import cascading.tap.Tap; //导入方法依赖的package包/类
/**
 * Reads the header record from the source file.
 */
@SuppressWarnings("unchecked")
private CSVRecord getHeaderRecord(FlowProcess<JobConf> flowProcess, Tap tap) {
  Tap textLine = new Hfs(new TextLine(new Fields("line")), tap.getFullIdentifier(flowProcess.getConfigCopy()));

  try (TupleEntryIterator iterator = textLine.openForRead(flowProcess)) {
    String line = iterator.next().getTuple().getString(0);
    boolean skipHeaderRecord = format.getSkipHeaderRecord();
    CSVRecord headerRecord = CSVParser.parse(line, format.withSkipHeaderRecord(false)).iterator().next();
    format.withSkipHeaderRecord(skipHeaderRecord);
    return headerRecord;
  } catch (IOException e) {
    throw new TapException(e);
  }
}
 

import cascading.tap.Tap; //导入方法依赖的package包/类
private TupleEntryIterator getHadoopTupleEntryIterator() throws IOException {
  @SuppressWarnings("unchecked")
  Tap<JobConf, ?, ?> hadoopTap = (Tap<JobConf, ?, ?>) source;
  JobConf conf = new JobConf();
  FlowProcess<JobConf> flowProcess = new HadoopFlowProcess(conf);
  hadoopTap.sourceConfInit(flowProcess, conf);
  return hadoopTap.openForRead(flowProcess);
}
 

import cascading.tap.Tap; //导入方法依赖的package包/类
private TupleEntryIterator getLocalTupleEntryIterator() throws IOException {
  @SuppressWarnings("unchecked")
  Tap<Properties, ?, ?> localTap = (Tap<Properties, ?, ?>) source;
  Properties properties = new Properties();
  FlowProcess<Properties> flowProcess = new LocalFlowProcess(properties);
  localTap.sourceConfInit(flowProcess, properties);
  return localTap.openForRead(flowProcess);
}
 

import cascading.tap.Tap; //导入方法依赖的package包/类
@Override
public TupleEntryIterator openTapForRead(Tap tap) throws IOException {
	return tap.openForRead( this );
}