Java Duration類代碼示例

import org.apache.spark.streaming.Duration; //導入依賴的package包/類
public static void main(String[] args) {

        SparkConf conf = new SparkConf()
                .setAppName("kafka-sandbox")
                .setMaster("local[*]");
        JavaSparkContext sc = new JavaSparkContext(conf);
        JavaStreamingContext ssc = new JavaStreamingContext(sc, new Duration(2000));

        Set<String> topics = Collections.singleton("mytopic");
        Map<String, String> kafkaParams = new HashMap<>();
        kafkaParams.put("metadata.broker.list", "localhost:9092");

        JavaPairInputDStream<String, String> directKafkaStream = KafkaUtils.createDirectStream(ssc,
                String.class, String.class, StringDecoder.class, StringDecoder.class, kafkaParams, topics);

        directKafkaStream.foreachRDD(rdd -> {
            System.out.println("--- New RDD with " + rdd.partitions().size()
                    + " partitions and " + rdd.count() + " records");
            rdd.foreach(record -> System.out.println(record._2));
        });

        ssc.start();
        ssc.awaitTermination();
    }
 

import org.apache.spark.streaming.Duration; //導入依賴的package包/類
public static void main(String[] args) throws InterruptedException {
  SparkConf sc = new SparkConf().setAppName("POC-Kafka-New");
  
  try(JavaStreamingContext jsc = new JavaStreamingContext(sc, new Duration(2000))) {
    
    JavaPairInputDStream<String, String> stream = KafkaUtils.createDirectStream(
        jsc, String.class, String.class, StringDecoder.class, StringDecoder.class,
        Collections.singletonMap("metadata.broker.list", KAFKA_HOST_PORT),
        Collections.singleton(EXAMPLE_TOPIC));

    JavaDStream<ExampleXML> records = stream.map(t -> t._2()).map(new ParseXML());
    records.foreachRDD(rdd -> System.out.printf("Amount of XMLs: %d\n", rdd.count()));

    jsc.start();
    jsc.awaitTermination();
  }
}
 

import org.apache.spark.streaming.Duration; //導入依賴的package包/類
public static void main(String[] args) throws InterruptedException {
  SparkConf sc = new SparkConf().setAppName("POC-Streaming");
  try(JavaStreamingContext jsc = new JavaStreamingContext(sc, new Duration(2000))) {
    //JavaDStream<SampleXML> records = jsc.textFileStream("input/").map(new ParseXML());
    //textFileStream process lines of files, so xml has to be 1 line to work //alternative below

    JavaRDD<String> files = jsc.sparkContext().wholeTextFiles("input/").map(tuple -> tuple._2());
    Queue<JavaRDD<String>> rddQueue = new LinkedList<>();
    rddQueue.add(files);
    JavaDStream<String> records = jsc.queueStream(rddQueue);

    records.foreachRDD(rdd -> System.out.printf("Amount of XMLs: %d\n", rdd.count()));

    jsc.start();
    jsc.awaitTermination();
  }
}
 

import org.apache.spark.streaming.Duration; //導入依賴的package包/類
public static void main(String[] args) throws InterruptedException, IOException {
  SparkConf sc = new SparkConf().setAppName("POC-BigQuery");
  
  try(JavaStreamingContext jsc = new JavaStreamingContext(sc, new Duration(60000))) {
    JavaPairInputDStream<String, String> stream = KafkaUtils.createDirectStream(
        jsc, String.class, String.class, StringDecoder.class, StringDecoder.class,
        Collections.singletonMap("metadata.broker.list", KAFKA_HOST_PORT), Collections.singleton(EXAMPLE_TOPIC));

    Configuration conf = new Configuration();
    BigQueryConfiguration.configureBigQueryOutput(conf, BQ_EXAMPLE_TABLE, BQ_EXAMPLE_SCHEMA);
    conf.set("mapreduce.job.outputformat.class", BigQueryOutputFormat.class.getName());

    JavaDStream<ExampleXML> records = stream.map(t -> t._2()).map(new ParseXML());
    records.foreachRDD(rdd -> {
      System.out.printf("Amount of XMLs: %d\n", rdd.count());
      long time = System.currentTimeMillis();
      rdd.mapToPair(new PrepToBQ()).saveAsNewAPIHadoopDataset(conf);
      System.out.printf("Sent to BQ in %fs\n", (System.currentTimeMillis()-time)/1000f);
    });
    
    jsc.start();
    jsc.awaitTermination();
  }
}
 

import org.apache.spark.streaming.Duration; //導入依賴的package包/類
public static void main(String[] args) throws InterruptedException, IOException, JAXBException {
  SparkConf sc = new SparkConf().setAppName("Receiving-KafkaToBQ");

  try (JavaStreamingContext jsc = new JavaStreamingContext(sc, new Duration(60000))) {

    JavaPairDStream<String, String> stream = new KafkaInputWithOffsets(
        KAFKA_HOST_PORT, EXAMPLE_TOPIC, ZOOKEEPER_HOST, ZK_PATH).createResumableStream(jsc);

    stream.foreachRDD(IdleStop.create(jsc, 2, "XMLs count: %d\n"));

    stream
        .mapToPair(parseXml())
        .filter(t -> t != null)
        .mapToPair(prepToBq())
        .foreachRDD(BigQueryHelper.outputTo(BQ_EXAMPLE_TABLE, BQ_EXAMPLE_SCHEMA));

    jsc.start();
    jsc.awaitTermination();
  }
}
 

import org.apache.spark.streaming.Duration; //導入依賴的package包/類
public static void setFromCommandLineArgs(Options options, String[] args) {
	CommandLineParser parser = new PosixParser();
	try {
		CommandLine cl = parser.parse(options, args);
		// 參數默認值
		THE_INSTANCE.windowLength = new Duration(
				Integer.parseInt(cl.getOptionValue(AppMain.WINDOW_LENGTH, "30")) * 1000);
		THE_INSTANCE.slideInterval = new Duration(
				Integer.parseInt(cl.getOptionValue(AppMain.SLIDE_INTERVAL, "5")) * 1000);
		THE_INSTANCE.kafka_broker = cl.getOptionValue(AppMain.KAFKA_BROKER, "kafka:9092");
		THE_INSTANCE.kafka_topic = cl.getOptionValue(AppMain.KAFKA_TOPIC, "apache");
		THE_INSTANCE.parquet_file = cl.getOptionValue(AppMain.PARQUET_FILE, "/user/spark/");
		THE_INSTANCE.initialized = true;
	} catch (ParseException e) {
		THE_INSTANCE.initialized = false;
		System.err.println("Parsing failed.  Reason: " + e.getMessage());
	}
}
 

import org.apache.spark.streaming.Duration; //導入依賴的package包/類
@Test(timeout=6000)
public void testNatsToSparkConnectorWithAdditionalPropertiesAndSubjects() throws InterruptedException {
	
	JavaStreamingContext ssc = new JavaStreamingContext(sc, new Duration(200));

	final Properties properties = new Properties();
	properties.setProperty(PROP_URL, NATS_SERVER_URL);
	final JavaReceiverInputDStream<String> messages =  
			NatsToSparkConnector
				.receiveFromNats(String.class, StorageLevel.MEMORY_ONLY())
				.withProperties(properties)
				.withSubjects(DEFAULT_SUBJECT)
				.asStreamOf(ssc);

	validateTheReceptionOfMessages(ssc, messages);
}
 

import org.apache.spark.streaming.Duration; //導入依賴的package包/類
@Test(timeout=6000)
public void testNatsToSparkConnectorWithAdditionalPropertiesAndMultipleSubjects() throws InterruptedException {
	
	JavaStreamingContext ssc = new JavaStreamingContext(sc, new Duration(200));

	final Properties properties = new Properties();
	final JavaReceiverInputDStream<String> messages = 
			NatsToSparkConnector
				.receiveFromNats(String.class, StorageLevel.MEMORY_ONLY())
				.withNatsURL(NATS_SERVER_URL)
				.withProperties(properties)
				.withSubjects(DEFAULT_SUBJECT, "EXTRA_SUBJECT")
				.asStreamOf(ssc);

	validateTheReceptionOfMessages(ssc, messages);
}
 

import org.apache.spark.streaming.Duration; //導入依賴的package包/類
@Test(timeout=6000)
public void testNatsToSparkConnectorWithAdditionalProperties() throws InterruptedException {
	
	JavaStreamingContext ssc = new JavaStreamingContext(sc, new Duration(200));

	final Properties properties = new Properties();
	properties.setProperty(PROP_SUBJECTS, "sub1,"+DEFAULT_SUBJECT+" , sub2");
	properties.setProperty(PROP_URL, NATS_SERVER_URL);
	final JavaReceiverInputDStream<String> messages = 
			NatsToSparkConnector
				.receiveFromNats(String.class, StorageLevel.MEMORY_ONLY())
				.withProperties(properties)
				.asStreamOf(ssc);

	validateTheReceptionOfMessages(ssc, messages);
}
 

import org.apache.spark.streaming.Duration; //導入依賴的package包/類
@Test(timeout=6000)
public void testNatsToSparkConnectorWithAdditionalPropertiesAndSubjects() throws InterruptedException {
	
	JavaStreamingContext ssc = new JavaStreamingContext(sc, new Duration(200));

	final Properties properties = new Properties();
	properties.setProperty(PROP_URL, NATS_SERVER_URL);

	final JavaPairDStream<String, String> messages = 
			NatsToSparkConnector
				.receiveFromNats(String.class, StorageLevel.MEMORY_ONLY())
				.withProperties(properties)
				.withSubjects(DEFAULT_SUBJECT)
				.asStreamOfKeyValue(ssc);

	validateTheReceptionOfMessages(ssc, messages);
}
 

import org.apache.spark.streaming.Duration; //導入依賴的package包/類
@Test(timeout=6000)
public void testNatsToSparkConnectorWithAdditionalPropertiesAndMultipleSubjects() throws InterruptedException {
	
	JavaStreamingContext ssc = new JavaStreamingContext(sc, new Duration(200));

	final Properties properties = new Properties();
	final JavaPairDStream<String, String> messages = 
			NatsToSparkConnector
				.receiveFromNats(String.class, StorageLevel.MEMORY_ONLY())
				.withNatsURL(NATS_SERVER_URL)
				.withProperties(properties)
				.withSubjects(DEFAULT_SUBJECT, "EXTRA_SUBJECT")
				.asStreamOfKeyValue(ssc);

	validateTheReceptionOfMessages(ssc, messages);
}
 

import org.apache.spark.streaming.Duration; //導入依賴的package包/類
@Test(timeout=6000)
public void testNatsToSparkConnectorWithAdditionalProperties() throws InterruptedException {
	
	JavaStreamingContext ssc = new JavaStreamingContext(sc, new Duration(200));

	final Properties properties = new Properties();
	properties.setProperty(PROP_SUBJECTS, "sub1,"+DEFAULT_SUBJECT+" , sub2");
	properties.setProperty(PROP_URL, NATS_SERVER_URL);
	final JavaPairDStream<String, String> messages = 
			NatsToSparkConnector
				.receiveFromNats(String.class, StorageLevel.MEMORY_ONLY())
				.withProperties(properties)
				.asStreamOfKeyValue(ssc);

	validateTheReceptionOfMessages(ssc, messages);
}
 

import org.apache.spark.streaming.Duration; //導入依賴的package包/類
@Test(timeout=6000)
public void testNatsToSparkConnectorWithAdditionalPropertiesAndSubjects() throws InterruptedException {
	
	JavaStreamingContext ssc = new JavaStreamingContext(sc, new Duration(200));

	final Properties properties = new Properties();
	properties.setProperty(PROP_URL, NATS_SERVER_URL);
	final JavaReceiverInputDStream<Integer> messages =  
			NatsToSparkConnector
				.receiveFromNats(Integer.class, StorageLevel.MEMORY_ONLY())
				.withProperties(properties)
				.withSubjects(DEFAULT_SUBJECT)
				.asStreamOf(ssc);

	validateTheReceptionOfIntegerMessages(ssc, messages);
}
 

import org.apache.spark.streaming.Duration; //導入依賴的package包/類
/**
 * Join two streams base on processing time
 *
 * @param componentId
 * @param joinStream         the other stream<K,R>
 * @param windowDuration     window length of this stream
 * @param joinWindowDuration window length of joinStream
 * @param <R>
 * @return
 * @throws WorkloadException
 */
@Override
public <R> PairWorkloadOperator<K, Tuple2<V, R>> join(String componentId,
                                                      PairWorkloadOperator<K, R> joinStream,
                                                      TimeDurations windowDuration,
                                                      TimeDurations joinWindowDuration) throws WorkloadException {
    if (windowDuration.toMilliSeconds() % windowDuration.toMilliSeconds() != 0) {
        throw new WorkloadException("WindowDuration should be multi times of joinWindowDuration");
    }
    Duration windowDurations = Utils.timeDurationsToSparkDuration(windowDuration);
    Duration windowDurations2 = Utils.timeDurationsToSparkDuration(joinWindowDuration);

    if (joinStream instanceof SparkPairWorkloadOperator) {
        SparkPairWorkloadOperator<K, R> joinSparkStream = ((SparkPairWorkloadOperator<K, R>) joinStream);
        JavaPairDStream<K, Tuple2<V, R>> joinedStream = pairDStream
                .window(windowDurations.plus(windowDurations2), windowDurations2)
                .join(joinSparkStream.pairDStream.window(windowDurations2, windowDurations2));
        // filter illegal joined data

        return new SparkPairWorkloadOperator<>(joinedStream, parallelism);
    }
    throw new WorkloadException("Cast joinStream to SparkPairWorkloadOperator failed");
}
 

import org.apache.spark.streaming.Duration; //導入依賴的package包/類
public JKinesisReceiver(String applicationName, String streamName,
                        String endpointUrl, String regionName,
                        Duration checkpoint, InitialPositionInStream position) {
    super(StorageLevel.MEMORY_ONLY_SER());

    this.workerId = getHostname() + ":" + String.valueOf(UUID.randomUUID());
    this.checkpointInterval = checkpoint;
    this.initialPosition = position;

    Region region = RegionUtils.getRegion(regionName);

    try {
        this.kclConfig = new KinesisClientLibConfiguration(applicationName, streamName,
                                                  getCredsProvider(),
                                                  workerId)
                        .withCommonClientConfig(CLIENT_CONF)
                        .withRegionName(region.getName())
                        .withKinesisEndpoint(endpointUrl)
                        .withInitialPositionInStream(InitialPositionInStream.LATEST)
                        .withTaskBackoffTimeMillis(500);
    } catch (Exception ex) {
        // do absolutely nothing - and feel good about it!
        // but ...
        // we'd do something meaningful in a PROD context
    }
}