本文整理汇总了Java中org.apache.spark.sql.Encoders.bean方法的典型用法代码示例。如果您正苦于以下问题：Java Encoders.bean方法的具体用法？Java Encoders.bean怎么用？Java Encoders.bean使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.spark.sql.Encoders的用法示例。
在下文中一共展示了Encoders.bean方法的8个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: main
import org.apache.spark.sql.Encoders; //导入方法依赖的package包/类
public static void main(String[] args) {
SparkSession spark = SparkSession
.builder()
.appName("Dataset-JavaBean")
.master("local[4]")
.getOrCreate();
//
// The Java API requires you to explicitly instantiate an encoder for
// any JavaBean you want to use for schema inference
//
Encoder<Number> numberEncoder = Encoders.bean(Number.class);
//
// Create a container of the JavaBean instances
//
List<Number> data = Arrays.asList(
new Number(1, "one", "un"),
new Number(2, "two", "deux"),
new Number(3, "three", "trois"));
//
// Use the encoder and the container of JavaBean instances to create a
// Dataset
//
Dataset<Number> ds = spark.createDataset(data, numberEncoder);
System.out.println("*** here is the schema inferred from the bean");
ds.printSchema();
System.out.println("*** here is the data");
ds.show();
// Use the convenient bean-inferred column names to query
System.out.println("*** filter by one column and fetch others");
ds.where(col("i").gt(2)).select(col("english"), col("french")).show();
spark.stop();
}
示例2: test0FailOnIndexCreationDisabled
import org.apache.spark.sql.Encoders; //导入方法依赖的package包/类
@Test(expected = EsHadoopIllegalArgumentException.class)
public void test0FailOnIndexCreationDisabled() throws Exception {
String target = wrapIndex("test-nonexisting/data");
JavaStreamingQueryTestHarness<RecordBean> test = new JavaStreamingQueryTestHarness<>(spark, Encoders.bean(RecordBean.class));
RecordBean doc1 = new RecordBean();
doc1.setId(1);
doc1.setName("Spark");
RecordBean doc2 = new RecordBean();
doc2.setId(2);
doc2.setName("Hadoop");
RecordBean doc3 = new RecordBean();
doc3.setId(3);
doc3.setName("YARN");
Dataset<RecordBean> dataset = test
.withInput(doc1)
.withInput(doc2)
.withInput(doc3)
.expectingToThrow(EsHadoopIllegalArgumentException.class)
.stream();
test.run(
dataset.writeStream()
.option("checkpointLocation", checkpoint(target))
.option(ES_INDEX_AUTO_CREATE, "no")
.format("es"),
target
);
assertTrue(!RestUtils.exists(target));
}
开发者ID:elastic，项目名称:elasticsearch-hadoop，代码行数:35，代码来源:AbstractJavaEsSparkStructuredStreamingTest.java
示例3: test1BasicWrite
import org.apache.spark.sql.Encoders; //导入方法依赖的package包/类
@Test
public void test1BasicWrite() throws Exception {
String target = wrapIndex("test-write/data");
JavaStreamingQueryTestHarness<RecordBean> test = new JavaStreamingQueryTestHarness<>(spark, Encoders.bean(RecordBean.class));
RecordBean doc1 = new RecordBean();
doc1.setId(1);
doc1.setName("Spark");
RecordBean doc2 = new RecordBean();
doc2.setId(2);
doc2.setName("Hadoop");
RecordBean doc3 = new RecordBean();
doc3.setId(3);
doc3.setName("YARN");
Dataset<RecordBean> dataset = test
.withInput(doc1)
.withInput(doc2)
.withInput(doc3)
.stream();
test.run(
dataset.writeStream()
.option("checkpointLocation", checkpoint(target))
.format("es"),
target
);
assertTrue(RestUtils.exists(target));
assertThat(RestUtils.get(target + "/_search?"), containsString("Spark"));
assertThat(RestUtils.get(target + "/_search?"), containsString("Hadoop"));
assertThat(RestUtils.get(target + "/_search?"), containsString("YARN"));
}
开发者ID:elastic，项目名称:elasticsearch-hadoop，代码行数:36，代码来源:AbstractJavaEsSparkStructuredStreamingTest.java
示例4: test1WriteWithMappingId
import org.apache.spark.sql.Encoders; //导入方法依赖的package包/类
@Test
public void test1WriteWithMappingId() throws Exception {
String target = wrapIndex("test-write-id/data");
JavaStreamingQueryTestHarness<RecordBean> test = new JavaStreamingQueryTestHarness<>(spark, Encoders.bean(RecordBean.class));
RecordBean doc1 = new RecordBean();
doc1.setId(1);
doc1.setName("Spark");
RecordBean doc2 = new RecordBean();
doc2.setId(2);
doc2.setName("Hadoop");
RecordBean doc3 = new RecordBean();
doc3.setId(3);
doc3.setName("YARN");
Dataset<RecordBean> dataset = test
.withInput(doc1)
.withInput(doc2)
.withInput(doc3)
.stream();
test.run(
dataset.writeStream()
.option("checkpointLocation", checkpoint(target))
.option("es.mapping.id", "id")
.format("es"),
target
);
assertEquals(3, JavaEsSpark.esRDD(new JavaSparkContext(spark.sparkContext()), target).count());
assertTrue(RestUtils.exists(target + "/1"));
assertTrue(RestUtils.exists(target + "/2"));
assertTrue(RestUtils.exists(target + "/3"));
assertThat(RestUtils.get(target + "/_search?"), containsString("Spark"));
}
开发者ID:elastic，项目名称:elasticsearch-hadoop，代码行数:39，代码来源:AbstractJavaEsSparkStructuredStreamingTest.java
示例5: test1WriteWithMappingExclude
import org.apache.spark.sql.Encoders; //导入方法依赖的package包/类
@Test
public void test1WriteWithMappingExclude() throws Exception {
String target = wrapIndex("test-mapping-exclude/data");
JavaStreamingQueryTestHarness<RecordBean> test = new JavaStreamingQueryTestHarness<>(spark, Encoders.bean(RecordBean.class));
RecordBean doc1 = new RecordBean();
doc1.setId(1);
doc1.setName("Spark");
RecordBean doc2 = new RecordBean();
doc2.setId(2);
doc2.setName("Hadoop");
RecordBean doc3 = new RecordBean();
doc3.setId(3);
doc3.setName("YARN");
Dataset<RecordBean> dataset = test
.withInput(doc1)
.withInput(doc2)
.withInput(doc3)
.stream();
test.run(
dataset.writeStream()
.option("checkpointLocation", checkpoint(target))
.option(ES_MAPPING_EXCLUDE, "name")
.format("es"),
target
);
assertTrue(RestUtils.exists(target));
assertThat(RestUtils.get(target + "/_search?"), not(containsString("Spark")));
assertThat(RestUtils.get(target + "/_search?"), not(containsString("Hadoop")));
assertThat(RestUtils.get(target + "/_search?"), not(containsString("YARN")));
}
开发者ID:elastic，项目名称:elasticsearch-hadoop，代码行数:37，代码来源:AbstractJavaEsSparkStructuredStreamingTest.java
示例6: test1MultiIndexWrite
import org.apache.spark.sql.Encoders; //导入方法依赖的package包/类
@Test
public void test1MultiIndexWrite() throws Exception {
String target = wrapIndex("test-write-tech-{name}/data");
JavaStreamingQueryTestHarness<RecordBean> test = new JavaStreamingQueryTestHarness<>(spark, Encoders.bean(RecordBean.class));
RecordBean doc1 = new RecordBean();
doc1.setId(1);
doc1.setName("spark");
RecordBean doc2 = new RecordBean();
doc2.setId(2);
doc2.setName("hadoop");
Dataset<RecordBean> dataset = test
.withInput(doc1)
.withInput(doc2)
.stream();
test.run(
dataset.writeStream()
.option("checkpointLocation", checkpoint(target))
.format("es"),
target
);
assertTrue(RestUtils.exists(wrapIndex("test-write-tech-spark/data")));
assertTrue(RestUtils.exists(wrapIndex("test-write-tech-hadoop/data")));
assertThat(RestUtils.get(wrapIndex("test-write-tech-spark/data/_search?")), containsString("\"name\":\"spark\""));
assertThat(RestUtils.get(wrapIndex("test-write-tech-hadoop/data/_search?")), containsString("\"name\":\"hadoop\""));
}
开发者ID:elastic，项目名称:elasticsearch-hadoop，代码行数:32，代码来源:AbstractJavaEsSparkStructuredStreamingTest.java
示例7: bufferEncoder
import org.apache.spark.sql.Encoders; //导入方法依赖的package包/类
public Encoder<Average> bufferEncoder() {
return Encoders.bean(Average.class);
}
示例8: test2WriteToIngestPipeline
import org.apache.spark.sql.Encoders; //导入方法依赖的package包/类
@Test
public void test2WriteToIngestPipeline() throws Exception {
EsAssume.versionOnOrAfter(EsMajorVersion.V_5_X, "Ingest Supported in 5.x and above only");
String pipelineName = prefix + "-pipeline";
String pipeline = "{\"description\":\"Test Pipeline\",\"processors\":[{\"set\":{\"field\":\"pipeTEST\",\"value\":true,\"override\":true}}]}";
RestUtils.put("/_ingest/pipeline/" + pipelineName, StringUtils.toUTF(pipeline));
String target = wrapIndex("test-write-ingest/data");
JavaStreamingQueryTestHarness<RecordBean> test = new JavaStreamingQueryTestHarness<>(spark, Encoders.bean(RecordBean.class));
RecordBean doc1 = new RecordBean();
doc1.setId(1);
doc1.setName("Spark");
RecordBean doc2 = new RecordBean();
doc2.setId(2);
doc2.setName("Hadoop");
RecordBean doc3 = new RecordBean();
doc3.setId(3);
doc3.setName("YARN");
Dataset<RecordBean> dataset = test
.withInput(doc1)
.withInput(doc2)
.withInput(doc3)
.stream();
test.run(
dataset.writeStream()
.option("checkpointLocation", checkpoint(target))
.option(ES_INGEST_PIPELINE, pipelineName)
.option(ES_NODES_INGEST_ONLY, "true")
.format("es"),
target
);
assertTrue(RestUtils.exists(target));
assertThat(RestUtils.get(target+"/_search?"), containsString("\"pipeTEST\":true"));
}
开发者ID:elastic，项目名称:elasticsearch-hadoop，代码行数:42，代码来源:AbstractJavaEsSparkStructuredStreamingTest.java