当前位置: 首页>>代码示例>>Java>>正文

Java RichFlatMapFunction类代码示例

本文整理汇总了Java中org.apache.flink.api.common.functions.RichFlatMapFunction类的典型用法代码示例。如果您正苦于以下问题：Java RichFlatMapFunction类的具体用法？Java RichFlatMapFunction怎么用？Java RichFlatMapFunction使用的例子？那么, 这里精选的类代码示例或许可以为您提供帮助。


RichFlatMapFunction类属于org.apache.flink.api.common.functions包，在下文中一共展示了RichFlatMapFunction类的11个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: testTuple0

import org.apache.flink.api.common.functions.RichFlatMapFunction; //导入依赖的package包/类
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testTuple0() {
	// use getFlatMapReturnTypes()
	RichFlatMapFunction<?, ?> function = new RichFlatMapFunction<Tuple0, Tuple0>() {
		private static final long serialVersionUID = 1L;

		@Override
		public void flatMap(Tuple0 value, Collector<Tuple0> out) throws Exception {
			// nothing to do
		}
	};

	TypeInformation<?> ti = TypeExtractor.getFlatMapReturnTypes(function,
			(TypeInformation) TypeInfoParser.parse("Tuple0"));

	Assert.assertTrue(ti.isTupleType());
	Assert.assertEquals(0, ti.getArity());
	Assert.assertTrue(ti instanceof TupleTypeInfo);
}
开发者ID:axbaretto，项目名称:flink，代码行数:21，代码来源:TypeExtractorTest.java

示例2: runKeyValueTest

import org.apache.flink.api.common.functions.RichFlatMapFunction; //导入依赖的package包/类
public void runKeyValueTest() throws Exception {
	final String topic = "keyvaluetest";
	createTestTopic(topic, 1, 1);
	final int elementCount = 5000;

	// ----------- Write some data into Kafka -------------------

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);
	env.setRestartStrategy(RestartStrategies.noRestart());
	env.getConfig().disableSysoutLogging();

	DataStream<Tuple2<Long, PojoValue>> kvStream = env.addSource(new SourceFunction<Tuple2<Long, PojoValue>>() {
		@Override
		public void run(SourceContext<Tuple2<Long, PojoValue>> ctx) throws Exception {
			Random rnd = new Random(1337);
			for (long i = 0; i < elementCount; i++) {
				PojoValue pojo = new PojoValue();
				pojo.when = new Date(rnd.nextLong());
				pojo.lon = rnd.nextLong();
				pojo.lat = i;
				// make every second key null to ensure proper "null" serialization
				Long key = (i % 2 == 0) ? null : i;
				ctx.collect(new Tuple2<>(key, pojo));
			}
		}

		@Override
		public void cancel() {
		}
	});

	KeyedSerializationSchema<Tuple2<Long, PojoValue>> schema = new TypeInformationKeyValueSerializationSchema<>(Long.class, PojoValue.class, env.getConfig());
	Properties producerProperties = FlinkKafkaProducerBase.getPropertiesFromBrokerList(brokerConnectionStrings);
	producerProperties.setProperty("retries", "3");
	kafkaServer.produceIntoKafka(kvStream, topic, schema, producerProperties, null);
	env.execute("Write KV to Kafka");

	// ----------- Read the data again -------------------

	env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);
	env.setRestartStrategy(RestartStrategies.noRestart());
	env.getConfig().disableSysoutLogging();

	KeyedDeserializationSchema<Tuple2<Long, PojoValue>> readSchema = new TypeInformationKeyValueSerializationSchema<>(Long.class, PojoValue.class, env.getConfig());

	Properties props = new Properties();
	props.putAll(standardProps);
	props.putAll(secureProps);
	DataStream<Tuple2<Long, PojoValue>> fromKafka = env.addSource(kafkaServer.getConsumer(topic, readSchema, props));
	fromKafka.flatMap(new RichFlatMapFunction<Tuple2<Long, PojoValue>, Object>() {
		long counter = 0;
		@Override
		public void flatMap(Tuple2<Long, PojoValue> value, Collector<Object> out) throws Exception {
			// the elements should be in order.
			Assert.assertTrue("Wrong value " + value.f1.lat, value.f1.lat == counter);
			if (value.f1.lat % 2 == 0) {
				assertNull("key was not null", value.f0);
			} else {
				Assert.assertTrue("Wrong value " + value.f0, value.f0 == counter);
			}
			counter++;
			if (counter == elementCount) {
				// we got the right number of elements
				throw new SuccessException();
			}
		}
	});

	tryExecute(env, "Read KV from Kafka");

	deleteTestTopic(topic);
}
开发者ID:axbaretto，项目名称:flink，代码行数:75，代码来源:KafkaConsumerTestBase.java

示例3: testAccumulatorsAfterNoOp

import org.apache.flink.api.common.functions.RichFlatMapFunction; //导入依赖的package包/类
@Test
public void testAccumulatorsAfterNoOp() {

	final String accName = "test_accumulator";

	try {
		env.setParallelism(6);
		env.getConfig().disableSysoutLogging();

		env.generateSequence(1, 1000000)
				.rebalance()
				.flatMap(new RichFlatMapFunction<Long, Long>() {

					private LongCounter counter;

					@Override
					public void open(Configuration parameters) {
						counter = getRuntimeContext().getLongCounter(accName);
					}

					@Override
					public void flatMap(Long value, Collector<Long> out) {
						counter.add(1L);
					}
				})
				.output(new DiscardingOutputFormat<Long>());

		JobExecutionResult result = env.execute();

		assertEquals(1000000L, result.getAllAccumulatorResults().get(accName));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
开发者ID:axbaretto，项目名称:flink，代码行数:37，代码来源:MiscellaneousIssuesITCase.java

示例4: runAllDeletesTest

import org.apache.flink.api.common.functions.RichFlatMapFunction; //导入依赖的package包/类
/**
 * Test delete behavior and metrics for producer.
 * @throws Exception
 */
public void runAllDeletesTest() throws Exception {
	final String topic = "alldeletestest";
	createTestTopic(topic, 1, 1);
	final int elementCount = 300;

	// ----------- Write some data into Kafka -------------------

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);
	env.getConfig().setRestartStrategy(RestartStrategies.noRestart());
	env.getConfig().disableSysoutLogging();

	DataStream<Tuple2<byte[], PojoValue>> kvStream = env.addSource(new SourceFunction<Tuple2<byte[], PojoValue>>() {
		@Override
		public void run(SourceContext<Tuple2<byte[], PojoValue>> ctx) throws Exception {
			Random rnd = new Random(1337);
			for (long i = 0; i < elementCount; i++) {
				final byte[] key = new byte[200];
				rnd.nextBytes(key);
				ctx.collect(new Tuple2<>(key, (PojoValue) null));
			}
		}

		@Override
		public void cancel() {
		}
	});

	TypeInformationKeyValueSerializationSchema<byte[], PojoValue> schema = new TypeInformationKeyValueSerializationSchema<>(byte[].class, PojoValue.class, env.getConfig());

	Properties producerProperties = FlinkKafkaProducerBase.getPropertiesFromBrokerList(brokerConnectionStrings);
	producerProperties.setProperty("retries", "3");
	producerProperties.putAll(secureProps);
	kafkaServer.produceIntoKafka(kvStream, topic, schema, producerProperties, null);

	env.execute("Write deletes to Kafka");

	// ----------- Read the data again -------------------

	env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);
	env.getConfig().setRestartStrategy(RestartStrategies.noRestart());
	env.getConfig().disableSysoutLogging();

	Properties props = new Properties();
	props.putAll(standardProps);
	props.putAll(secureProps);
	DataStream<Tuple2<byte[], PojoValue>> fromKafka = env.addSource(kafkaServer.getConsumer(topic, schema, props));

	fromKafka.flatMap(new RichFlatMapFunction<Tuple2<byte[], PojoValue>, Object>() {
		long counter = 0;
		@Override
		public void flatMap(Tuple2<byte[], PojoValue> value, Collector<Object> out) throws Exception {
			// ensure that deleted messages are passed as nulls
			assertNull(value.f1);
			counter++;
			if (counter == elementCount) {
				// we got the right number of elements
				throw new SuccessException();
			}
		}
	});

	tryExecute(env, "Read deletes from Kafka");

	deleteTestTopic(topic);
}
开发者ID:axbaretto，项目名称:flink，代码行数:72，代码来源:KafkaConsumerTestBase.java

示例5: testTupleWithTuples

import org.apache.flink.api.common.functions.RichFlatMapFunction; //导入依赖的package包/类
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testTupleWithTuples() {
	// use getFlatMapReturnTypes()
	RichFlatMapFunction<?, ?> function = new RichFlatMapFunction<Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>, Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>>() {
		private static final long serialVersionUID = 1L;

		@Override
		public void flatMap(Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>> value,
				Collector<Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>> out) throws Exception {
			// nothing to do
		}
	};

	TypeInformation<?> ti = TypeExtractor.getFlatMapReturnTypes(function, (TypeInformation) TypeInfoParser.parse("Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>"));
	Assert.assertTrue(ti.isTupleType());
	Assert.assertEquals(3, ti.getArity());
	Assert.assertTrue(ti instanceof TupleTypeInfo);
	List<FlatFieldDescriptor> ffd = new ArrayList<FlatFieldDescriptor>();
	
	((TupleTypeInfo) ti).getFlatFields("f0.f0", 0, ffd);
	Assert.assertEquals(0, ffd.get(0).getPosition() );
	ffd.clear();
	
	((TupleTypeInfo) ti).getFlatFields("f0.f0", 0, ffd);
	Assert.assertTrue( ffd.get(0).getType() instanceof BasicTypeInfo );
	Assert.assertTrue( ffd.get(0).getType().getTypeClass().equals(String.class) );
	ffd.clear();
	
	((TupleTypeInfo) ti).getFlatFields("f1.f0", 0, ffd);
	Assert.assertEquals(1, ffd.get(0).getPosition() );
	ffd.clear();

	TupleTypeInfo<?> tti = (TupleTypeInfo<?>) ti;
	Assert.assertEquals(Tuple3.class, tti.getTypeClass());

	Assert.assertTrue(tti.getTypeAt(0).isTupleType());
	Assert.assertTrue(tti.getTypeAt(1).isTupleType());
	Assert.assertTrue(tti.getTypeAt(2).isTupleType());
	
	Assert.assertEquals(Tuple1.class, tti.getTypeAt(0).getTypeClass());
	Assert.assertEquals(Tuple1.class, tti.getTypeAt(1).getTypeClass());
	Assert.assertEquals(Tuple2.class, tti.getTypeAt(2).getTypeClass());

	Assert.assertEquals(1, tti.getTypeAt(0).getArity());
	Assert.assertEquals(1, tti.getTypeAt(1).getArity());
	Assert.assertEquals(2, tti.getTypeAt(2).getArity());

	Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, ((TupleTypeInfo<?>) tti.getTypeAt(0)).getTypeAt(0));
	Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, ((TupleTypeInfo<?>) tti.getTypeAt(1)).getTypeAt(0));
	Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, ((TupleTypeInfo<?>) tti.getTypeAt(2)).getTypeAt(0));
	Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, ((TupleTypeInfo<?>) tti.getTypeAt(2)).getTypeAt(1));

	// use getForObject()
	Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>> t = new Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>(
			new Tuple1<String>("hello"), new Tuple1<Integer>(1), new Tuple2<Long, Long>(2L, 3L));
	Assert.assertTrue(TypeExtractor.getForObject(t) instanceof TupleTypeInfo);
	TupleTypeInfo<?> tti2 = (TupleTypeInfo<?>) TypeExtractor.getForObject(t);

	Assert.assertEquals(1, tti2.getTypeAt(0).getArity());
	Assert.assertEquals(1, tti2.getTypeAt(1).getArity());
	Assert.assertEquals(2, tti2.getTypeAt(2).getArity());

	Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, ((TupleTypeInfo<?>) tti2.getTypeAt(0)).getTypeAt(0));
	Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, ((TupleTypeInfo<?>) tti2.getTypeAt(1)).getTypeAt(0));
	Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, ((TupleTypeInfo<?>) tti2.getTypeAt(2)).getTypeAt(0));
	Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, ((TupleTypeInfo<?>) tti2.getTypeAt(2)).getTypeAt(1));
}
开发者ID:axbaretto，项目名称:flink，代码行数:69，代码来源:TypeExtractorTest.java

示例6: runKeyValueTest

import org.apache.flink.api.common.functions.RichFlatMapFunction; //导入依赖的package包/类
public void runKeyValueTest() throws Exception {
	final String topic = "keyvaluetest";
	createTestTopic(topic, 1, 1);
	final int ELEMENT_COUNT = 5000;

	// ----------- Write some data into Kafka -------------------

	StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
	env.setParallelism(1);
	env.setRestartStrategy(RestartStrategies.noRestart());
	env.getConfig().disableSysoutLogging();

	DataStream<Tuple2<Long, PojoValue>> kvStream = env.addSource(new SourceFunction<Tuple2<Long, PojoValue>>() {
		@Override
		public void run(SourceContext<Tuple2<Long, PojoValue>> ctx) throws Exception {
			Random rnd = new Random(1337);
			for (long i = 0; i < ELEMENT_COUNT; i++) {
				PojoValue pojo = new PojoValue();
				pojo.when = new Date(rnd.nextLong());
				pojo.lon = rnd.nextLong();
				pojo.lat = i;
				// make every second key null to ensure proper "null" serialization
				Long key = (i % 2 == 0) ? null : i;
				ctx.collect(new Tuple2<>(key, pojo));
			}
		}
		@Override
		public void cancel() {
		}
	});

	KeyedSerializationSchema<Tuple2<Long, PojoValue>> schema = new TypeInformationKeyValueSerializationSchema<>(Long.class, PojoValue.class, env.getConfig());
	Properties producerProperties = FlinkKafkaProducerBase.getPropertiesFromBrokerList(brokerConnectionStrings);
	producerProperties.setProperty("retries", "3");
	kafkaServer.produceIntoKafka(kvStream, topic, schema, producerProperties, null);
	env.execute("Write KV to Kafka");

	// ----------- Read the data again -------------------

	env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
	env.setParallelism(1);
	env.setRestartStrategy(RestartStrategies.noRestart());
	env.getConfig().disableSysoutLogging();


	KeyedDeserializationSchema<Tuple2<Long, PojoValue>> readSchema = new TypeInformationKeyValueSerializationSchema<>(Long.class, PojoValue.class, env.getConfig());

	Properties props = new Properties();
	props.putAll(standardProps);
	props.putAll(secureProps);
	DataStream<Tuple2<Long, PojoValue>> fromKafka = env.addSource(kafkaServer.getConsumer(topic, readSchema, props));
	fromKafka.flatMap(new RichFlatMapFunction<Tuple2<Long,PojoValue>, Object>() {
		long counter = 0;
		@Override
		public void flatMap(Tuple2<Long, PojoValue> value, Collector<Object> out) throws Exception {
			// the elements should be in order.
			Assert.assertTrue("Wrong value " + value.f1.lat, value.f1.lat == counter );
			if (value.f1.lat % 2 == 0) {
				assertNull("key was not null", value.f0);
			} else {
				Assert.assertTrue("Wrong value " + value.f0, value.f0 == counter);
			}
			counter++;
			if (counter == ELEMENT_COUNT) {
				// we got the right number of elements
				throw new SuccessException();
			}
		}
	});

	tryExecute(env, "Read KV from Kafka");

	deleteTestTopic(topic);
}
开发者ID:axbaretto，项目名称:flink，代码行数:75，代码来源:KafkaConsumerTestBase.java

示例7: runAllDeletesTest

import org.apache.flink.api.common.functions.RichFlatMapFunction; //导入依赖的package包/类
/**
 * Test delete behavior and metrics for producer
 * @throws Exception
 */
public void runAllDeletesTest() throws Exception {
	final String topic = "alldeletestest";
	createTestTopic(topic, 1, 1);
	final int ELEMENT_COUNT = 300;

	// ----------- Write some data into Kafka -------------------

	StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
	env.setParallelism(1);
	env.getConfig().setRestartStrategy(RestartStrategies.noRestart());
	env.getConfig().disableSysoutLogging();

	DataStream<Tuple2<byte[], PojoValue>> kvStream = env.addSource(new SourceFunction<Tuple2<byte[], PojoValue>>() {
		@Override
		public void run(SourceContext<Tuple2<byte[], PojoValue>> ctx) throws Exception {
			Random rnd = new Random(1337);
			for (long i = 0; i < ELEMENT_COUNT; i++) {
				final byte[] key = new byte[200];
				rnd.nextBytes(key);
				ctx.collect(new Tuple2<>(key, (PojoValue) null));
			}
		}
		@Override
		public void cancel() {
		}
	});

	TypeInformationKeyValueSerializationSchema<byte[], PojoValue> schema = new TypeInformationKeyValueSerializationSchema<>(byte[].class, PojoValue.class, env.getConfig());

	Properties producerProperties = FlinkKafkaProducerBase.getPropertiesFromBrokerList(brokerConnectionStrings);
	producerProperties.setProperty("retries", "3");
	producerProperties.putAll(secureProps);
	kafkaServer.produceIntoKafka(kvStream, topic, schema, producerProperties, null);

	env.execute("Write deletes to Kafka");

	// ----------- Read the data again -------------------

	env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
	env.setParallelism(1);
	env.getConfig().setRestartStrategy(RestartStrategies.noRestart());
	env.getConfig().disableSysoutLogging();

	Properties props = new Properties();
	props.putAll(standardProps);
	props.putAll(secureProps);
	DataStream<Tuple2<byte[], PojoValue>> fromKafka = env.addSource(kafkaServer.getConsumer(topic, schema, props));

	fromKafka.flatMap(new RichFlatMapFunction<Tuple2<byte[], PojoValue>, Object>() {
		long counter = 0;
		@Override
		public void flatMap(Tuple2<byte[], PojoValue> value, Collector<Object> out) throws Exception {
			// ensure that deleted messages are passed as nulls
			assertNull(value.f1);
			counter++;
			if (counter == ELEMENT_COUNT) {
				// we got the right number of elements
				throw new SuccessException();
			}
		}
	});

	tryExecute(env, "Read deletes from Kafka");

	deleteTestTopic(topic);
}
开发者ID:axbaretto，项目名称:flink，代码行数:71，代码来源:KafkaConsumerTestBase.java

示例8: plan

import org.apache.flink.api.common.functions.RichFlatMapFunction; //导入依赖的package包/类
public void plan() {
    enableSingleOutputFile();
    ParameterTool params = ParameterTool.fromArgs(args);

    String inputWikiFilename = params.getRequired("wikidump");
    String inputLinkTuplesFilename = params.getRequired("links");

    outputFilename = params.getRequired("output");

    DataSet<RedirectMapping> redirects = WikiSimRedirects.getRedirectsDataSet(env, params.getRequired("redirects"));

    DataSet<Tuple2<String, String>> linkTupleList = env.readCsvFile(inputLinkTuplesFilename)
            .fieldDelimiter(WikiSimConfiguration.csvFieldDelimiter)
            .types(String.class, String.class)
            .coGroup(redirects)
            .where(1) // link B (Redirect target)
            .equalTo(1) // redirect target
            .with(new ReplaceLinkTuples(1))
            .coGroup(redirects)
            .where(0) // link A (Redirect target)
            .equalTo(1) // redirect target
            .with(new ReplaceLinkTuples(0));

    DataSource<String> text = env.readFile(new WikiDocumentDelimitedInputFormat(), inputWikiFilename);

    result = text.flatMap(new RichFlatMapFunction<String, Tuple4<String, String, String, Integer>>() {
        Collection<Tuple2<String, String>> linkTupleList;

        @Override
        public void open(Configuration parameters) throws Exception {
            linkTupleList = getRuntimeContext().getBroadcastVariable("linkTupleList");
        }

        @Override
        public void flatMap(String content, Collector<Tuple4<String, String, String, Integer>> out) throws Exception {
            LinkPair linkPair = new LinkPair();

            WikiDocument doc = new DocumentProcessor().processDoc(content);
            if (doc == null) return;

            // Get links & wordmap
            List<Map.Entry<String, Integer>> outLinks = doc.getOutLinks();
            TreeMap<Integer, Integer> wordMap = doc.getWordMap();

            // Loop all link pairs
            for (Map.Entry<String, Integer> outLink1 : outLinks) {
                for (Map.Entry<String, Integer> outLink2 : outLinks) {
                    int order = outLink1.getKey().compareTo(outLink2.getKey());
                    if (order > 0) {
                        int w1 = wordMap.floorEntry(outLink1.getValue()).getValue();
                        int w2 = wordMap.floorEntry(outLink2.getValue()).getValue();
                        int d = max(abs(w1 - w2), 1);
                        //recDistance.setValue(1 / (pow(d, α)));

                        linkPair.setFirst(outLink1.getKey());
                        linkPair.setSecond(outLink2.getKey());

                        // Add result to collector
                        if (linkPair.isValid() && (linkTupleList.contains(linkPair) || linkTupleList.contains(linkPair.getTwin()))) {
                            out.collect(new Tuple4<>(
                                            doc.getTitle(),
                                            linkPair.getFirst(),
                                            linkPair.getSecond(),
                                            d)
                            );
                        }
                    }
                }
            }

        }
    }).withBroadcastSet(linkTupleList, "linkTupleList");
}
开发者ID:wikimedia，项目名称:citolytics，代码行数:74，代码来源:LinkGraph.java

示例9: plan

import org.apache.flink.api.common.functions.RichFlatMapFunction; //导入依赖的package包/类
@Override
public void plan() throws Exception {
    String wikiSimInputFilename = getParams().get("wikisim"); // not in use

    String wikiDumpInputFilename = getParams().get("wikidump");
    String articleStatsFilename = getParams().get("article-stats");
    String cpiExpr = getParams().get("cpi");

    outputFilename = getParams().getRequired("output");
    String redirectsFilename = getParams().get("redirects", null);
    int topK = getParams().getInt("topk", WikiSimConfiguration.DEFAULT_TOP_K);
    int fieldScore = getParams().getInt("score", RecommendationPair.CPI_LIST_KEY);
    int fieldPageA = getParams().getInt("page-a", RecommendationPair.PAGE_A_KEY);
    int fieldPageB = getParams().getInt("page-b", RecommendationPair.PAGE_B_KEY);
    int fieldPageIdA = getParams().getInt("page-id-a", RecommendationPair.PAGE_A_ID_KEY);
    int fieldPageIdB = getParams().getInt("page-id-b", RecommendationPair.PAGE_B_ID_KEY);

    boolean disableScores = getParams().has("disable-scores");
    boolean elasticBulkSyntax = getParams().has("enable-elastic");
    boolean ignoreMissingIds = getParams().has("ignore-missing-ids");
    boolean resolveRedirects = getParams().has("resolve-redirects");
    boolean includeIds = getParams().has("include-ids");
    boolean relativeProximity = getParams().has("relative-proximity");
    boolean backupRecommendations = getParams().has("backup-recommendations");
    double alpha = getParams().getDouble("alpha", 0.855); // From JCDL paper: a1=0.81, a2=0.9 -> a_mean = 0.855

    setJobName("CirrusSearch PrepareOutput");

    // Prepare results
    DataSet<Recommendation>  recommendations;
    if(wikiSimInputFilename != null) {
        // Use existing result list;
        recommendations = WikiSimReader.readWikiSimOutput(env, wikiSimInputFilename, fieldPageA, fieldPageB, fieldScore, fieldPageIdA, fieldPageIdB);
    } else if(wikiDumpInputFilename != null) {

        // Build new result list
        WikiSim wikiSimJob = new WikiSim();
        wikiSimJob.alpha = String.valueOf(alpha);
        wikiSimJob.inputFilename = wikiDumpInputFilename;
        wikiSimJob.redirectsFilename = redirectsFilename;
        wikiSimJob.ignoreMissingIds = ignoreMissingIds; // Ensures that page ids exist
        wikiSimJob.resolveRedirects = resolveRedirects;
        wikiSimJob.relativeProximity = relativeProximity;
        wikiSimJob.setEnvironment(env);
        wikiSimJob.plan();

        recommendations = wikiSimJob.result
                .flatMap(new RichFlatMapFunction<RecommendationPair, Recommendation>() {
                    @Override
                    public void flatMap(RecommendationPair pair, Collector<Recommendation> out) throws Exception {
                        WikiSimReader.collectRecommendationsFromPair(pair, out, 0);
                    }
                });

    } else {
        throw new Exception("Either --wikidump or --wikisim parameter must be provided.");
    }

    // Compute recommendation sets
    DataSet<RecommendationSet> recommendationSets = WikiSimReader.buildRecommendationSets(env, recommendations,
            topK, cpiExpr, articleStatsFilename);

    // Backup recommendations
    if(backupRecommendations) {
        if(wikiDumpInputFilename == null) {
            throw new Exception("To use backup recommendations the --wikidump parameter must be provided.");
        }

        // Merge original and backup recommendation sets
        recommendationSets = recommendationSets
                .fullOuterJoin(BackupRecommendationSetExtractor.getBackupRecommendations(env, wikiDumpInputFilename))
                .where(RecommendationSet.SOURCE_TITLE_KEY)
                .equalTo(RecommendationSet.SOURCE_TITLE_KEY)
                .with(new BackupRecommendationSetMerger(topK));
    }

    // Transform result list to JSON
    result = recommendationSets.flatMap(new JSONMapper(disableScores, elasticBulkSyntax, ignoreMissingIds, includeIds));
}
开发者ID:wikimedia，项目名称:citolytics，代码行数:80，代码来源:PrepareOutput.java

示例10: main

import org.apache.flink.api.common.functions.RichFlatMapFunction; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	final ParameterTool pt = ParameterTool.fromArgs(args);
	see.getConfig().setGlobalJobParameters(pt);
	see.getConfig().enableObjectReuse();

	// see.setParallelism(1);

	DataStreamSource<Integer> src = see.addSource(new RichParallelSourceFunction<Integer>() {

		boolean running = true;
		@Override
		public void run(SourceContext<Integer> ctx) throws Exception {
			int i = 0;
			while (running) {
				ctx.collect(i++);
			}
		}

		@Override
		public void cancel() {
			running = false;
		}
	});

	src/*.map(new MapFunction<Integer, Integer>() {
		@Override
		public Integer map(Integer s) throws Exception {
			return s;
		}
	}).*/.map(new MapFunction<Integer, Integer>() {
		@Override
		public Integer map(Integer s) throws Exception {
			return s;
		}
	}).flatMap(new RichFlatMapFunction<Integer, Integer>() {
		long received = 0;
		long logfreq = pt.getInt("logfreq");
		long lastLog = -1;
		long lastElements = 0;
		long matches = 0;
		private final Pattern threeDigitAbbr = Pattern.compile("[A-Z]{3}\\.");

		@Override
		public void open(Configuration parameters) throws Exception {
			super.open(parameters);
		}

		@Override
		public void flatMap(Integer in, Collector<Integer> collector) throws Exception {

			received++;
			if (received % logfreq == 0) {
				// throughput over entire time
				long now = System.currentTimeMillis();

				// throughput for the last "logfreq" elements
				if (lastLog == -1) {
					// init (the first)
					lastLog = now;
					lastElements = received;
				} else {
					long timeDiff = now - lastLog;
					long elementDiff = received - lastElements;
					double ex = (1000 / (double) timeDiff);
					LOG.info("During the last {} ms, we received {} elements. That's {} elements/second/core", timeDiff, elementDiff, Double.valueOf(elementDiff * ex).longValue());
					// reinit
					lastLog = now;
					lastElements = received;
				}
			}
		}
	});

	see.execute();
}
开发者ID:project-flink，项目名称:flink-perf，代码行数:77，代码来源:ChainingSpeed.java

示例11: testTupleWithTuples

import org.apache.flink.api.common.functions.RichFlatMapFunction; //导入依赖的package包/类
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testTupleWithTuples() {
	// use getFlatMapReturnTypes()
	RichFlatMapFunction<?, ?> function = new RichFlatMapFunction<Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>, Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>>() {
		private static final long serialVersionUID = 1L;

		@Override
		public void flatMap(Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>> value,
				Collector<Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>> out) throws Exception {
			// nothing to do
		}
	};

	TypeInformation<?> ti = TypeExtractor.getFlatMapReturnTypes(function, (TypeInformation) TypeInfoParser.parse("Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>"));
	Assert.assertTrue(ti.isTupleType());
	Assert.assertEquals(3, ti.getArity());
	Assert.assertTrue(ti instanceof TupleTypeInfo);
	List<FlatFieldDescriptor> ffd = new ArrayList<FlatFieldDescriptor>();
	
	((TupleTypeInfo) ti).getKey("f0.f0", 0, ffd);
	Assert.assertEquals(0, ffd.get(0).getPosition() );
	ffd.clear();
	
	((TupleTypeInfo) ti).getKey("f0.f0", 0, ffd);
	Assert.assertTrue( ffd.get(0).getType() instanceof BasicTypeInfo );
	Assert.assertTrue( ffd.get(0).getType().getTypeClass().equals(String.class) );
	ffd.clear();
	
	((TupleTypeInfo) ti).getKey("f1.f0", 0, ffd);
	Assert.assertEquals(1, ffd.get(0).getPosition() );
	ffd.clear();

	TupleTypeInfo<?> tti = (TupleTypeInfo<?>) ti;
	Assert.assertEquals(Tuple3.class, tti.getTypeClass());

	Assert.assertTrue(tti.getTypeAt(0).isTupleType());
	Assert.assertTrue(tti.getTypeAt(1).isTupleType());
	Assert.assertTrue(tti.getTypeAt(2).isTupleType());
	
	Assert.assertEquals(Tuple1.class, tti.getTypeAt(0).getTypeClass());
	Assert.assertEquals(Tuple1.class, tti.getTypeAt(1).getTypeClass());
	Assert.assertEquals(Tuple2.class, tti.getTypeAt(2).getTypeClass());

	Assert.assertEquals(1, tti.getTypeAt(0).getArity());
	Assert.assertEquals(1, tti.getTypeAt(1).getArity());
	Assert.assertEquals(2, tti.getTypeAt(2).getArity());

	Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, ((TupleTypeInfo<?>) tti.getTypeAt(0)).getTypeAt(0));
	Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, ((TupleTypeInfo<?>) tti.getTypeAt(1)).getTypeAt(0));
	Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, ((TupleTypeInfo<?>) tti.getTypeAt(2)).getTypeAt(0));
	Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, ((TupleTypeInfo<?>) tti.getTypeAt(2)).getTypeAt(1));

	// use getForObject()
	Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>> t = new Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>(
			new Tuple1<String>("hello"), new Tuple1<Integer>(1), new Tuple2<Long, Long>(2L, 3L));
	Assert.assertTrue(TypeExtractor.getForObject(t) instanceof TupleTypeInfo);
	TupleTypeInfo<?> tti2 = (TupleTypeInfo<?>) TypeExtractor.getForObject(t);

	Assert.assertEquals(1, tti2.getTypeAt(0).getArity());
	Assert.assertEquals(1, tti2.getTypeAt(1).getArity());
	Assert.assertEquals(2, tti2.getTypeAt(2).getArity());

	Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, ((TupleTypeInfo<?>) tti2.getTypeAt(0)).getTypeAt(0));
	Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, ((TupleTypeInfo<?>) tti2.getTypeAt(1)).getTypeAt(0));
	Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, ((TupleTypeInfo<?>) tti2.getTypeAt(2)).getTypeAt(0));
	Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, ((TupleTypeInfo<?>) tti2.getTypeAt(2)).getTypeAt(1));
}
开发者ID:citlab，项目名称:vs.msc.ws14，代码行数:69，代码来源:TypeExtractorTest.java


注：本文中的org.apache.flink.api.common.functions.RichFlatMapFunction类示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。