Java Flow类代码示例

import cascading.flow.Flow; //导入依赖的package包/类
public static void main(String [] args) {

        Properties properties = new Properties();

        properties.put(SplunkConf.SPLUNK_USERNAME, "admin");
        properties.put(SplunkConf.SPLUNK_PASSWORD, "changeIt");
        properties.put(SplunkConf.SPLUNK_HOST, "localhost");
        properties.put(SplunkConf.SPLUNK_PORT, "9050");

        SplunkDataQuery splunkSearch = new SplunkDataQuery();
        SplunkScheme inputScheme = new SplunkScheme(splunkSearch);
        SplunkTap input = new SplunkTap(properties,inputScheme);

        TextLine outputScheme = new TextLine();
        Hfs output = new Hfs( outputScheme, PATH_TO_OUTPUT, SinkMode.REPLACE );

        Pipe pipe = new Pipe( "test" );
        Flow flow = new HadoopFlowConnector().connect( input, output, pipe );

        flow.complete();
    }
 

import cascading.flow.Flow; //导入依赖的package包/类
public Flow runCascadingJob( Path inputPath, Path outputPath) throws IOException
{
    Properties properties = new Properties();

    ShuttlCsv inputScheme = new ShuttlCsv(new SplunkDataQuery());
    TextLine outputScheme = new TextLine();

    Hfs input = new Hfs(inputScheme,inputPath.toString());
    Hfs output = new Hfs(outputScheme,outputPath.toString(),SinkMode.REPLACE);

    Pipe pipe = new Pipe( "test" );
    Flow flow = new HadoopFlowConnector( properties ).connect( input, output, pipe );

    flow.complete();

    return flow;
}
 

import cascading.flow.Flow; //导入依赖的package包/类
public void runSplunkScheme(String path, String inputData) throws IOException
{
    Properties properties = TestConfigurations.getSplunkLoginAsProperties();
    SplunkScheme inputScheme = new SplunkScheme(TestConfigurations.getSplunkSearch());
    TextLine outputScheme = new TextLine();

    SplunkTap input = new SplunkTap(properties,inputScheme);
    Hfs output = new Hfs( outputScheme, outputPath + "/quoted/" + path, SinkMode.REPLACE );

    Pipe pipe = new Pipe( "test" );
    Flow flow = new HadoopFlowConnector().connect( input, output, pipe );

    flow.complete();

    validateLength( flow, 10, 2 );

    TupleEntryIterator iterator = flow.openSource();

    // TODO: Header information not used in SplunkScheme yet
    // verifyHeader(iterator.getFields());

    verifyContent(iterator);
}
 

import cascading.flow.Flow; //导入依赖的package包/类
private static void printStats(Flow flow) {
    CascadingStats stats = flow.getStats();
    System.out.println("Cascading stats for " + Counter.class.getName());
    for (String counter : stats.getCountersFor(Counter.class)) {
        System.out.println(String.format("%s\t%s", counter, stats.getCounterValue(Counter.class.getName(), counter)));
    }
}
 

import cascading.flow.Flow; //导入依赖的package包/类
private void printStats(Flow flow) {
    CascadingStats stats = flow.getStats();
    System.out.println("Cascading stats for " + Counter.class.getName());
    for (String counter : stats.getCountersFor(Counter.class)) {
        System.out.println(String.format("%s\t%s", counter, stats.getCounterValue(Counter.class.getName(), counter)));
    }
}
 

import cascading.flow.Flow; //导入依赖的package包/类
@Test
public void testCascadeConnector() {
    Pipe copy = new Pipe("copy");
    Properties cfg = new TestSettings().getProperties();

    FlowDef flow = new FlowDef().addSource(copy, sourceTap()).addTailSink(copy,
            new EsTap("cascading-local/cascade-connector"));

    FlowConnector connector = new LocalFlowConnector(cfg);
    Flow[] flows = new Flow[] { connector.connect(flow) };

    CascadeConnector cascadeConnector = new CascadeConnector(cfg);
    cascadeConnector.connect(flows).complete();
}
 

import cascading.flow.Flow; //导入依赖的package包/类
@Test
public void testCascadeConnector() {
    Pipe copy = new Pipe("copy");
    Properties cfg = HdpBootstrap.asProperties(CascadingHadoopSuite.configuration);

    FlowDef flow = new FlowDef().addSource(copy, sourceTap())
            .addTailSink(copy, new EsTap("cascading-hadoop/cascade-connector"));

    FlowConnector connector = new HadoopFlowConnector(cfg);
    Flow[] flows = new Flow[] { connector.connect(flow) };

    CascadeConnector cascadeConnector = new CascadeConnector(cfg);
    cascadeConnector.connect(flows).complete();
}
 

import cascading.flow.Flow; //导入依赖的package包/类
@SuppressWarnings("rawtypes")
public static void countFacts() throws Exception {
	// Erzeuge Tap aus Master Pail
	Tap masterDataset = dataTap(FileUtils.prepareMasterFactsPath(false,false));

	// Die Query erzeugt nur einen Zählerwert
	query = new Subquery("?count")
			// Master PailTap generiert Tupel nur mit Data Element
			.predicate(masterDataset, "_", "?raw")
			// Aggregation aller Tupel als Zählerwert
			.predicate(new Count(), "?count");

	// Query ausführen; Ergebnis wird an Stdout gesendet
	Api.execute(new StdoutTap(), query);

	///////////////////////////////////////////////////////////////
	// Optional:
	// Die Querry kann auch als Cascading Flow kompiliert werden
	Flow flow = Api.compileFlow(new StdoutTap(), query);

	// Eine Grafik des Cascading Flows erzeugen (ähnlich Pipe Diagramm)
	flow.writeDOT("/tmp/count_fact_flow.dot");

	// Alternative Möglichkeit, die Query auszuführen
	// flow.complete();
	///////////////////////////////////////////////////////////////
}
 

import cascading.flow.Flow; //导入依赖的package包/类
public static void main(String[] args) {

		if (args.length < 2) {
			throw new IllegalArgumentException("Please specify input and ouput paths as arguments.");
		}

		Fields token = new Fields( "token", String.class );
		Fields text = new Fields( "text" );
		RegexSplitGenerator splitter = new RegexSplitGenerator( token, "\\s+" );
		// only returns "token"
		Pipe docPipe = new Each( "token", text, splitter, Fields.RESULTS );

		Pipe wcPipe = new Pipe( "wc", docPipe );
		wcPipe = new AggregateBy( wcPipe, token, new CountBy(new Fields("count")));

		Tap inTap = new Hfs(new TextDelimited(text, "\n" ), args[0]);
		Tap outTap = new Hfs(new TextDelimited(false, "\n"), args[1], SinkMode.REPLACE);

		FlowDef flowDef = FlowDef.flowDef().setName( "wc" )
				.addSource( docPipe, inTap )
				.addTailSink( wcPipe, outTap );

		FlowConnector flowConnector = new FlinkConnector();

		Flow wcFlow = flowConnector.connect( flowDef );

		wcFlow.complete();
	}
 

import cascading.flow.Flow; //导入依赖的package包/类
@Test
public void readWriteInFlowMR1() throws IOException {
  try (OrcWriter writer = new OrcWriter.Builder(conf, new Path(path, "part-00000"))
      .addField("a", TypeInfoFactory.stringTypeInfo)
      .addField("b", TypeInfoFactory.stringTypeInfo)
      .build()) {
    writer.addRow("A1", "B1");
    writer.addRow("A2", "B2");
  }

  String output = new File(temporaryFolder.getRoot(), "output").getCanonicalPath();

  Pipe pipe = new Pipe(UUID.randomUUID().toString());
  FlowDef flowDef = FlowDef
      .flowDef()
      .setName(UUID.randomUUID().toString())
      .addSource(pipe, new Hfs(OrcFile.source().declaredFields(FIELDS_AB).schemaFromFile().build(), path))
      .addTailSink(pipe, new Hfs(OrcFile.sink().schema(FIELDS_AB).build(), output));

  Flow<?> flow = new Hadoop2MR1FlowConnector(HadoopUtil.createProperties(conf)).connect(flowDef);
  flow.complete();
  flow.cleanup();

  try (OrcReader reader = new OrcReader(conf, new Path(output, "part-00000"))) {
    assertThat(reader.hasNext(), is(true));
    List<Object> list = reader.next();
    assertThat(list.size(), is(2));
    assertThat(list.get(0), is((Object) "A1"));
    assertThat(list.get(1), is((Object) "B1"));

    assertThat(reader.hasNext(), is(true));
    list = reader.next();
    assertThat(list.size(), is(2));
    assertThat(list.get(0), is((Object) "A2"));
    assertThat(list.get(1), is((Object) "B2"));

    assertThat(reader.hasNext(), is(false));
  }
}
 

import cascading.flow.Flow; //导入依赖的package包/类
@Test
public void readWriteInFlowTez() throws IOException {
  try (OrcWriter writer = new OrcWriter.Builder(conf, new Path(path, "part-00000"))
      .addField("a", TypeInfoFactory.stringTypeInfo)
      .addField("b", TypeInfoFactory.stringTypeInfo)
      .build()) {
    writer.addRow("A1", "B1");
    writer.addRow("A2", "B2");
  }

  String output = new File(temporaryFolder.getRoot(), "output").getCanonicalPath();

  Pipe pipe = new Pipe(UUID.randomUUID().toString());
  FlowDef flowDef = FlowDef
      .flowDef()
      .setName(UUID.randomUUID().toString())
      .addSource(pipe, new Hfs(OrcFile.source().declaredFields(FIELDS_AB).schemaFromFile().build(), path))
      .addTailSink(pipe, new Hfs(OrcFile.sink().schema(FIELDS_AB).build(), output));

  Flow<?> flow = new Hadoop2TezFlowConnector(HadoopUtil.createProperties(conf)).connect(flowDef);
  flow.complete();
  flow.cleanup();

  try (OrcReader reader = new OrcReader(conf, new Path(output, "part-v000-o000-00000"))) {
    assertThat(reader.hasNext(), is(true));
    List<Object> list = reader.next();
    assertThat(list.size(), is(2));
    assertThat(list.get(0), is((Object) "A1"));
    assertThat(list.get(1), is((Object) "B1"));

    assertThat(reader.hasNext(), is(true));
    list = reader.next();
    assertThat(list.size(), is(2));
    assertThat(list.get(0), is((Object) "A2"));
    assertThat(list.get(1), is((Object) "B2"));

    assertThat(reader.hasNext(), is(false));
  }
}
 

import cascading.flow.Flow; //导入依赖的package包/类
public static void main(String[] args) {
  String salesPath = args[0];
  String storePath = args[1];
  String outPath = args[2];
  String date = "2452229";

  Properties properties = new Properties();
  AppProps.setApplicationJarClass(properties, Main.class);
  HadoopFlowConnector flowConnector = new HadoopFlowConnector(properties);

  Tap salesTap = new Hfs(new ORCFile(null, "0,7"), salesPath);
  Tap storeTap = new Hfs(new AvroScheme(), storePath);
  Tap outTap = new Hfs(new TextDelimited(true, "\t"), outPath);

  Pipe salesPipe = new Each("sales", new Fields("solddatesk"), new DateFilter(Integer.valueOf(date)));
  Pipe storePipe = new Pipe("store");
  Pipe joinPipe = new HashJoin(salesPipe, new Fields("storesk"), storePipe, new Fields("storeSk"));

  // _col24 is state_name
  Pipe countPipe = new CountBy(joinPipe, new Fields("state"),
      new Fields("item_count"));

  FlowDef flowDef = FlowDef.flowDef().setName("count")
      .addSource(salesPipe, salesTap)
      .addSource(storePipe, storeTap)
      .addTailSink(countPipe, outTap);
      //.addTailSink(joinPipe, outTap);

  Flow countFlow = flowConnector.connect(flowDef);
  countFlow.complete();
}
 

import cascading.flow.Flow; //导入依赖的package包/类
public static void main(String [] args) {

        ShuttlCsv csv = new ShuttlCsv();
        Hfs input = new Hfs(csv,PATH_TO_SHUTTL_ARCHIVE);

        TextLine outputScheme = new TextLine();
        Hfs output = new Hfs(outputScheme, PATH_TO_OUTPUT);
        Pipe pipe = new Pipe( "test" );

        Properties properties = new Properties();
        Flow flow = new HadoopFlowConnector( properties ).connect( input, output, pipe );

        flow.complete();
    }
 

import cascading.flow.Flow; //导入依赖的package包/类
public void runCSVLine( String path, String inputData) throws IOException
{
    Properties properties = new Properties();

    CSVLine inputScheme = new CSVLine();
    TextLine outputScheme = new TextLine();

    Hfs input = new Hfs( inputScheme, inputData );
    Hfs output = new Hfs( outputScheme, outputPath + "/quoted/" + path, SinkMode.REPLACE );

    Pipe pipe = new Pipe( "test" );
    Flow flow = new HadoopFlowConnector( properties ).connect( input, output, pipe );

    flow.complete();

    validateLength( flow, 4, 2 ); // The file contains 4 rows, however there are only 3 CSV rows (inc the header row)


    TupleEntryIterator iterator = flow.openSource();

    ArrayListTextWritable expected = new ArrayListTextWritable();

    expected.add(new Text("header1"));
    expected.add(new Text("header2"));
    assertEquals(expected, iterator.next().getTuple().getObject(1));

    expected.clear();
    expected.add(new Text("Column1"));
    expected.add(new Text("Column 2 using\ntwo rows"));
    assertEquals(expected, iterator.next().getTuple().getObject(1));

    expected.clear();
    expected.add(new Text("c1"));
    expected.add(new Text("c2"));
    assertEquals(expected, iterator.next().getTuple().getObject(1));
}
 

import cascading.flow.Flow; //导入依赖的package包/类
@Test
public void testCascadeConnector() {
    Pipe copy = new Pipe("copy");
    Properties cfg = new TestSettings().getProperties();

    FlowDef flow = new FlowDef().addSource(copy, sourceTap()).addTailSink(copy,
            new EsTap("cascading-local-cascade-connector/data"));

    FlowConnector connector = new LocalFlowConnector(cfg);
    Flow[] flows = new Flow[] { connector.connect(flow) };

    CascadeConnector cascadeConnector = new CascadeConnector(cfg);
    cascadeConnector.connect(flows).complete();
}