当前位置: 首页>>代码示例>>Java>>正文

Java JavaEsSpark类代码示例

本文整理汇总了Java中org.elasticsearch.spark.rdd.api.java.JavaEsSpark类的典型用法代码示例。如果您正苦于以下问题：Java JavaEsSpark类的具体用法？Java JavaEsSpark怎么用？Java JavaEsSpark使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。


JavaEsSpark类属于org.elasticsearch.spark.rdd.api.java包，在下文中一共展示了JavaEsSpark类的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: testEsRDDWriteWithDynamicMapping

import org.elasticsearch.spark.rdd.api.java.JavaEsSpark; //导入依赖的package包/类
public void testEsRDDWriteWithDynamicMapping() throws Exception {
    Map<String, ?> doc1 = ImmutableMap.of("one", 1, "two", 2, "number", 1);
    Map<String, ?> doc2 = ImmutableMap.of("OTP", "Otopeni", "SFO", "San Fran", "number", 2);

    String target = "spark-test/java-dyn-id-write";
    JavaPairRDD<?, ?> pairRdd = sc.parallelizePairs(ImmutableList.of(new Tuple2<Object,Object>(1, doc1),
            new Tuple2<Object, Object>(2, doc2)));
    //JavaPairRDD pairRDD = JavaPairRDD.fromJavaRDD(tupleRdd);
    // eliminate with static import
    JavaEsSpark.saveToEsWithMeta(pairRdd, target);

    assertEquals(2, JavaEsSpark.esRDD(sc, target).count());
    assertTrue(RestUtils.exists(target + "/1"));
    assertTrue(RestUtils.exists(target + "/2"));
    String results = RestUtils.get(target + "/_search?");
    assertThat(results, containsString("SFO"));
}
开发者ID:xushjie1987，项目名称:es-hadoop-v2.2.0，代码行数:18，代码来源:AbstractJavaEsSparkTest.java

示例2: testEsRDDWrite

import org.elasticsearch.spark.rdd.api.java.JavaEsSpark; //导入依赖的package包/类
public void testEsRDDWrite() throws Exception {
    Map<String, ?> doc1 = ImmutableMap.of("one", 1, "two", 2);
    Map<String, ?> doc2 = ImmutableMap.of("OTP", "Otopeni", "SFO", "San Fran");

    String target = "spark-test/java-basic-write";
    JavaRDD<Map<String, ?>> javaRDD = sc.parallelize(ImmutableList.of(doc1, doc2));
    // eliminate with static import
    JavaEsSpark.saveToEs(javaRDD, target);
    JavaEsSpark.saveToEs(javaRDD, ImmutableMap.of(ES_RESOURCE, target + "1"));

    assertEquals(2, JavaEsSpark.esRDD(sc, target).count());
    assertTrue(RestUtils.exists(target));
    String results = RestUtils.get(target + "/_search?");
    assertThat(results, containsString("SFO"));
}
开发者ID:xushjie1987，项目名称:es-hadoop-v2.2.0，代码行数:16，代码来源:AbstractJavaEsSparkTest.java

示例3: testEsRDDWriteWithDynamicMappingBasedOnMaps

import org.elasticsearch.spark.rdd.api.java.JavaEsSpark; //导入依赖的package包/类
public void testEsRDDWriteWithDynamicMappingBasedOnMaps() throws Exception {
    Map<String, ?> doc1 = ImmutableMap.of("one", 1, "two", 2, "number", 1);
    Map<String, ?> doc2 = ImmutableMap.of("OTP", "Otopeni", "SFO", "San Fran", "number", 2);

    String target = "spark-test/java-dyn-map-id-write";
    Map<Metadata, Object> header1 = ImmutableMap.<Metadata, Object> of(ID, 1, TTL, "1d");
    Map<Metadata, Object> header2 = ImmutableMap.<Metadata, Object> of(ID, "2", TTL, "2d");
    JavaRDD<Tuple2<Object, Object>> tupleRdd = sc.parallelize(ImmutableList.<Tuple2<Object, Object>> of(new Tuple2(header1, doc1), new Tuple2(header2, doc2)));
    JavaPairRDD pairRDD = JavaPairRDD.fromJavaRDD(tupleRdd);
    // eliminate with static import
    JavaEsSpark.saveToEsWithMeta(pairRDD, target);

    assertEquals(2, JavaEsSpark.esRDD(sc, target).count());
    assertTrue(RestUtils.exists(target + "/1"));
    assertTrue(RestUtils.exists(target + "/2"));
    String results = RestUtils.get(target + "/_search?");
    assertThat(results, containsString("SFO"));
}
开发者ID:xushjie1987，项目名称:es-hadoop-v2.2.0，代码行数:19，代码来源:AbstractJavaEsSparkTest.java

示例4: testEsRDDWriteAsJsonMultiWrite

import org.elasticsearch.spark.rdd.api.java.JavaEsSpark; //导入依赖的package包/类
public void testEsRDDWriteAsJsonMultiWrite() throws Exception {
  String json1 = "{\"reason\" : \"business\",\"airport\" : \"SFO\"}";
  String json2 = "{\"participants\" : 5,\"airport\" : \"OTP\"}";

  JavaRDD<String> stringRDD = sc.parallelize(ImmutableList.of(json1, json2));
  JavaEsSpark.saveJsonToEs(stringRDD, "spark-test/json-{airport}");
  JavaEsSpark.saveJsonToEs(stringRDD, "spark-test/json1-{airport}", Collections.<String, String> emptyMap());
  JavaEsSpark.saveJsonToEs(stringRDD, ImmutableMap.of(ES_RESOURCE, "spark-test/json2-{airport}"));

  byte[] json1BA = json1.getBytes();
  byte[] json2BA = json2.getBytes();

  JavaRDD<byte[]> byteRDD = sc.parallelize(ImmutableList.of(json1BA, json2BA));
  JavaEsSpark.saveJsonByteArrayToEs(byteRDD, "spark-test/json-ba-{airport}");
  JavaEsSpark.saveJsonByteArrayToEs(byteRDD, "spark-test/json-ba1-{airport}", Collections.<String, String> emptyMap());
  JavaEsSpark.saveJsonByteArrayToEs(byteRDD, ImmutableMap.of(ES_RESOURCE, "spark-test/json-ba2-{airport}"));

  assertTrue(RestUtils.exists("spark-test/json-SFO"));
  assertTrue(RestUtils.exists("spark-test/json-OTP"));

  assertTrue(RestUtils.exists("spark-test/json1-SFO"));
  assertTrue(RestUtils.exists("spark-test/json1-OTP"));

  assertTrue(RestUtils.exists("spark-test/json2-SFO"));
  assertTrue(RestUtils.exists("spark-test/json2-OTP"));

  assertTrue(RestUtils.exists("spark-test/json-ba-SFO"));
  assertTrue(RestUtils.exists("spark-test/json-ba-OTP"));

  assertTrue(RestUtils.exists("spark-test/json-ba1-SFO"));
  assertTrue(RestUtils.exists("spark-test/json-ba1-OTP"));

  assertTrue(RestUtils.exists("spark-test/json-ba2-SFO"));
  assertTrue(RestUtils.exists("spark-test/json-ba2-OTP"));

  assertThat(RestUtils.get("spark-test/json-SFO/_search?"), containsString("business"));
  assertThat(RestUtils.get("spark-test/json-OTP/_search?"), containsString("participants"));
}
开发者ID:xushjie1987，项目名称:es-hadoop-v2.2.0，代码行数:39，代码来源:AbstractJavaEsSparkTest.java

示例5: testEsRDDZRead

import org.elasticsearch.spark.rdd.api.java.JavaEsSpark; //导入依赖的package包/类
public void testEsRDDZRead() throws Exception {
        String target = "spark-test/java-basic-read";

        RestUtils.touch("spark-test");
        RestUtils.postData(target, "{\"message\" : \"Hello World\",\"message_date\" : \"2014-05-25\"}".getBytes());
        RestUtils.postData(target, "{\"message\" : \"Goodbye World\",\"message_date\" : \"2014-05-25\"}".getBytes());
        RestUtils.refresh("spark-test");

//        JavaRDD<scala.collection.Map<String, Object>> esRDD = JavaEsSpark.esRDD(sc, target);
//        JavaRDD messages = esRDD.filter(new Function<scala.collection.Map<String, Object>, Boolean>() {
//            public Boolean call(scala.collection.Map<String, Object> map) {
//                for (Entry<String, Object> entry: JavaConversions.asJavaMap(map).entrySet()) {
//                    if (entry.getValue().toString().contains("message")) {
//                        return Boolean.TRUE;
//                    }
//                }
//                return Boolean.FALSE;
//            }
//        });

        JavaRDD<Map<String, Object>> esRDD = JavaEsSpark.esRDD(sc, target).values();
        System.out.println(esRDD.collect());
        JavaRDD<Map<String, Object>> messages = esRDD.filter(new Function<Map<String, Object>, Boolean>() {
            @Override
            public Boolean call(Map<String, Object> map) throws Exception {
                return map.containsKey("message");
            }
        });

        // jdk8
        //esRDD.filter(m -> m.stream().filter(v -> v.contains("message")));

        assertThat((int) messages.count(), is(2));
        System.out.println(messages.take(10));
        System.out.println(messages);
    }
开发者ID:xushjie1987，项目名称:es-hadoop-v2.2.0，代码行数:37，代码来源:AbstractJavaEsSparkTest.java

示例6: testEsRDDZReadJson

import org.elasticsearch.spark.rdd.api.java.JavaEsSpark; //导入依赖的package包/类
public void testEsRDDZReadJson() throws Exception {
    String target = "spark-test/java-basic-json-read";

    RestUtils.touch("spark-test");
    RestUtils.postData(target, "{\"message\" : \"Hello World\",\"message_date\" : \"2014-05-25\"}".getBytes());
    RestUtils.postData(target, "{\"message\" : \"Goodbye World\",\"message_date\" : \"2014-05-25\"}".getBytes());
    RestUtils.refresh("spark-test");

    JavaRDD<String> esRDD = JavaEsSpark.esJsonRDD(sc, target).values();
    System.out.println(esRDD.collect());
    JavaRDD<String> messages = esRDD.filter(new Function<String, Boolean>() {
        @Override
        public Boolean call(String string) throws Exception {
            return string.contains("message");
        }
    });

    // jdk8
    //esRDD.filter(m -> m.contains("message")));

    assertThat((int) messages.count(), is(2));
    System.out.println(messages.take(10));
    System.out.println(messages);
}
开发者ID:xushjie1987，项目名称:es-hadoop-v2.2.0，代码行数:25，代码来源:AbstractJavaEsSparkTest.java

示例7: testEsRDDZReadMultiIndex

import org.elasticsearch.spark.rdd.api.java.JavaEsSpark; //导入依赖的package包/类
@Test
public void testEsRDDZReadMultiIndex() throws Exception {
    String index = "spark-test";

    for (int i = 0; i < 10; i++) {
        RestUtils.postData(index + "/foo", ("{\"message\" : \"Hello World\", \"counter\":" + i +", \"message_date\" : \"2014-05-25\"}").getBytes());
        RestUtils.postData(index + "/bar", ("{\"message\" : \"Goodbye World\", \"counter\":" + i +", \"message_date\" : \"2014-05-25\"}").getBytes());
    }

    RestUtils.refresh(index);

    //JavaRDD<Map<String, Object>> wildRDD = JavaEsSpark.esRDD(sc, ImmutableMap.of(ES_RESOURCE, "spark*/foo")).values();
    JavaRDD<Map<String, Object>> typeRDD = JavaEsSpark.esRDD(sc, ImmutableMap.of(ES_RESOURCE, "spark*")).values();

    JavaRDD<Map<String, Object>> allRDD = JavaEsSpark.esRDD(sc, "_all/foo", "").values();
    //assertEquals(wildRDD.count(), allRDD.count());
    System.out.println(typeRDD.count());
    // assertEquals(500, allRDD.count());
}
开发者ID:xushjie1987，项目名称:es-hadoop-v2.2.0，代码行数:20，代码来源:AbstractJavaEsSparkTest.java

示例8: run

import org.elasticsearch.spark.rdd.api.java.JavaEsSpark; //导入依赖的package包/类
void run(String spanResource, String dependencyLinkResource, SpanAcceptor decoder) {
  log.info("Processing spans from {}", spanResource);
  JavaSparkContext sc = new JavaSparkContext(conf);
  try {
    JavaRDD<Map<String, Object>> links = JavaEsSpark.esJsonRDD(sc, spanResource)
        .groupBy(pair -> traceId(pair._2))
        .flatMapValues(new TraceIdAndJsonToDependencyLinks(logInitializer, decoder))
        .values()
        .mapToPair(link -> new Tuple2<>(new Tuple2<>(link.parent, link.child), link))
        .reduceByKey((l, r) -> DependencyLink.builder()
            .parent(l.parent)
            .child(l.child)
            .callCount(l.callCount + r.callCount)
            .errorCount(l.errorCount + r.errorCount).build())
        .values()
        .map(ElasticsearchDependenciesJob::dependencyLinkJson);

    if (links.isEmpty()) {
      log.info("No spans found at {}", spanResource);
    } else {
      log.info("Saving dependency links to {}", dependencyLinkResource);
      JavaEsSpark.saveToEs(links, dependencyLinkResource,
          Collections.singletonMap("es.mapping.id", "id")); // allows overwriting the link
    }
  } finally {
    sc.stop();
  }
}
开发者ID:openzipkin，项目名称:zipkin-dependencies，代码行数:29，代码来源:ElasticsearchDependenciesJob.java

示例9: testEsRDDWriteWIthMappingId

import org.elasticsearch.spark.rdd.api.java.JavaEsSpark; //导入依赖的package包/类
@Test
public void testEsRDDWriteWIthMappingId() throws Exception {
    Map<String, Object> doc1 = new HashMap<>();
    doc1.put("number", 1);
    doc1.put("one", null);
    Set<String> values = new HashSet<>();
    values.add("2");
    doc1.put("two", values);
    doc1.put("three", ".");

    Map<String, Object> doc2 = new HashMap<>();
    doc2.put("number", 2);
    doc2.put("OTP", "Otopeni");
    doc2.put("SFO", "San Fran");

    List<Map<String, Object>> docs = new ArrayList<>();
    docs.add(doc1);
    docs.add(doc2);

    Map<String, String> localConf = new HashMap<>(cfg);
    localConf.put("es.mapping.id", "number");

    String target = wrapIndex("spark-streaming-test-scala-id-write/data");

    JavaRDD<Map<String,Object>> batch = sc.parallelize(docs);
    Queue<JavaRDD<Map<String, Object>>> rddQueue = new LinkedList<>();
    rddQueue.add(batch);
    JavaDStream<Map<String, Object>> dstream = ssc.queueStream(rddQueue);
    JavaEsSparkStreaming.saveToEs(dstream, target, localConf);
    ssc.start();
    TimeUnit.SECONDS.sleep(2);
    ssc.stop(false, true);

    assertEquals(2, JavaEsSpark.esRDD(sc, target).count());
    assertTrue(RestUtils.exists(target + "/1"));
    assertTrue(RestUtils.exists(target + "/2"));

    assertThat(RestUtils.get(target + "/_search?"), containsString("SFO"));
}
开发者ID:elastic，项目名称:elasticsearch-hadoop，代码行数:40，代码来源:AbstractJavaEsSparkStreamingTest.java

示例10: testEsRDDWriteWithDynamicMapping

import org.elasticsearch.spark.rdd.api.java.JavaEsSpark; //导入依赖的package包/类
@Test
public void testEsRDDWriteWithDynamicMapping() throws Exception {
    Map<String, Object> doc1 = new HashMap<>();
    doc1.put("number", 3);
    doc1.put("one", null);
    Set<String> values = new HashSet<>();
    values.add("2");
    doc1.put("two", values);
    doc1.put("three", ".");

    Map<String, Object> doc2 = new HashMap<>();
    doc2.put("number", 4);
    doc2.put("OTP", "Otopeni");
    doc2.put("SFO", "San Fran");

    List<Map<String, Object>> docs = new ArrayList<>();
    docs.add(doc1);
    docs.add(doc2);

    String target = wrapIndex("spark-streaming-test-scala-dyn-id-write/data");

    JavaRDD<Map<String,Object>> batch = sc.parallelize(docs);
    Queue<JavaRDD<Map<String, Object>>> rddQueue = new LinkedList<>();
    rddQueue.add(batch);
    JavaDStream<Map<String, Object>> dstream = ssc.queueStream(rddQueue);

    JavaPairDStream<Integer, Map<String, Object>> metaDstream = dstream.mapToPair(new ExtractIDFunction());

    JavaEsSparkStreaming.saveToEsWithMeta(metaDstream, target, cfg);
    ssc.start();
    TimeUnit.SECONDS.sleep(2);
    ssc.stop(false, true);

    assertEquals(2, JavaEsSpark.esRDD(sc, target).count());
    assertTrue(RestUtils.exists(target + "/3"));
    assertTrue(RestUtils.exists(target + "/4"));

    assertThat(RestUtils.get(target + "/_search?"), containsString("SFO"));
}
开发者ID:elastic，项目名称:elasticsearch-hadoop，代码行数:40，代码来源:AbstractJavaEsSparkStreamingTest.java

示例11: testEsRDDWriteWithDynamicMapping

import org.elasticsearch.spark.rdd.api.java.JavaEsSpark; //导入依赖的package包/类
public void testEsRDDWriteWithDynamicMapping() throws Exception {
    Map<String, ?> doc1 = ImmutableMap.of("one", 1, "two", 2, "number", 1);
    Map<String, ?> doc2 = ImmutableMap.of("OTP", "Otopeni", "SFO", "San Fran", "number", 2);

    String target = "spark-test-java-dyn-id-write/data";
    JavaPairRDD<?, ?> pairRdd = sc.parallelizePairs(ImmutableList.of(new Tuple2<Object,Object>(1, doc1),
            new Tuple2<Object, Object>(2, doc2)));
    //JavaPairRDD pairRDD = JavaPairRDD.fromJavaRDD(tupleRdd);
    // eliminate with static import
    JavaEsSpark.saveToEsWithMeta(pairRdd, target);

    assertEquals(2, JavaEsSpark.esRDD(sc, target).count());
    assertTrue(RestUtils.exists(target + "/1"));
    assertTrue(RestUtils.exists(target + "/2"));
    String results = RestUtils.get(target + "/_search?");
    assertThat(results, containsString("SFO"));
}
开发者ID:elastic，项目名称:elasticsearch-hadoop，代码行数:18，代码来源:AbstractJavaEsSparkTest.java

示例12: testEsRDDWriteWithDynamicMappingBasedOnMaps

import org.elasticsearch.spark.rdd.api.java.JavaEsSpark; //导入依赖的package包/类
public void testEsRDDWriteWithDynamicMappingBasedOnMaps() throws Exception {
    Map<String, ?> doc1 = ImmutableMap.of("one", 1, "two", 2, "number", 1);
    Map<String, ?> doc2 = ImmutableMap.of("OTP", "Otopeni", "SFO", "San Fran", "number", 2);

    String target = "spark-test-java-dyn-map-id-write/data";
    Map<Metadata, Object> header1 = ImmutableMap.<Metadata, Object> of(ID, 1, TTL, "1d");
    Map<Metadata, Object> header2 = ImmutableMap.<Metadata, Object> of(ID, "2", TTL, "2d");
    JavaRDD<Tuple2<Object, Object>> tupleRdd = sc.parallelize(ImmutableList.<Tuple2<Object, Object>> of(new Tuple2(header1, doc1), new Tuple2(header2, doc2)));
    JavaPairRDD pairRDD = JavaPairRDD.fromJavaRDD(tupleRdd);
    // eliminate with static import
    JavaEsSpark.saveToEsWithMeta(pairRDD, target);

    assertEquals(2, JavaEsSpark.esRDD(sc, target).count());
    assertTrue(RestUtils.exists(target + "/1"));
    assertTrue(RestUtils.exists(target + "/2"));
    String results = RestUtils.get(target + "/_search?");
    assertThat(results, containsString("SFO"));
}
开发者ID:elastic，项目名称:elasticsearch-hadoop，代码行数:19，代码来源:AbstractJavaEsSparkTest.java

示例13: testEsRDDZReadJson

import org.elasticsearch.spark.rdd.api.java.JavaEsSpark; //导入依赖的package包/类
public void testEsRDDZReadJson() throws Exception {
    String target = "spark-test-java-basic-json-read/data";

    RestUtils.touch("spark-test-java-basic-json-read");
    RestUtils.postData(target, "{\"message\" : \"Hello World\",\"message_date\" : \"2014-05-25\"}".getBytes());
    RestUtils.postData(target, "{\"message\" : \"Goodbye World\",\"message_date\" : \"2014-05-25\"}".getBytes());
    RestUtils.refresh("spark-test*");

    JavaRDD<String> esRDD = JavaEsSpark.esJsonRDD(sc, target).values();
    System.out.println(esRDD.collect());
    JavaRDD<String> messages = esRDD.filter(new Function<String, Boolean>() {
        @Override
        public Boolean call(String string) throws Exception {
            return string.contains("message");
        }
    });

    // jdk8
    //esRDD.filter(m -> m.contains("message")));

    assertThat((int) messages.count(), is(2));
    System.out.println(messages.take(10));
    System.out.println(messages);
}
开发者ID:elastic，项目名称:elasticsearch-hadoop，代码行数:25，代码来源:AbstractJavaEsSparkTest.java

示例14: testEsRDDZReadMultiIndex

import org.elasticsearch.spark.rdd.api.java.JavaEsSpark; //导入依赖的package包/类
@Test
public void testEsRDDZReadMultiIndex() throws Exception {
    EsAssume.versionOnOrBefore(EsMajorVersion.V_5_X, "Multiple Types Disabled in 6.0");
    String index = "spark-test";

    RestUtils.createMultiTypeIndex(index);

    int count = 10;

    for (int i = 0; i < count; i++) {
        RestUtils.postData(index + "/foo", ("{\"message\" : \"Hello World\", \"counter\":" + i +", \"message_date\" : \"2014-05-25\"}").getBytes());
        RestUtils.postData(index + "/bar", ("{\"message\" : \"Goodbye World\", \"counter\":" + i +", \"message_date\" : \"2014-05-25\"}").getBytes());
    }

    RestUtils.refresh(index+"*");

    //JavaRDD<Map<String, Object>> wildRDD = JavaEsSpark.esRDD(sc, ImmutableMap.of(ES_RESOURCE, "spark*/foo")).values();
    JavaRDD<Map<String, Object>> typeRDD = JavaEsSpark.esRDD(sc, ImmutableMap.of(ES_RESOURCE, "spark*")).values();

    JavaRDD<Map<String, Object>> allRDD = JavaEsSpark.esRDD(sc, "_all/foo", "").values();
    // assertEquals(wildRDD.count(), allRDD.count());
    // System.out.println(typeRDD.count());
    assertEquals(count, allRDD.count());
}
开发者ID:elastic，项目名称:elasticsearch-hadoop，代码行数:25，代码来源:AbstractJavaEsSparkTest.java

示例15: run

import org.elasticsearch.spark.rdd.api.java.JavaEsSpark; //导入依赖的package包/类
void run(String spanResource, String depResource) {
  log.info("Running Dependencies job for {}, reading from {} index, result storing to {}", day, spanResource ,depResource);
  JavaSparkContext sc = new JavaSparkContext(conf);
  try {
    JavaPairRDD<String, Iterable<Span>> traces = JavaEsSpark.esJsonRDD(sc, spanResource)
        .map(new ElasticTupleToSpan())
        .groupBy(Span::getTraceId);

    List<Dependency> dependencyLinks = DependenciesSparkHelper.derive(traces);
    store(sc, dependencyLinks, depResource);
    log.info("Done, {} dependency objects created", dependencyLinks.size());
  } finally {
    sc.stop();
  }
}
开发者ID:jaegertracing，项目名称:spark-dependencies，代码行数:16，代码来源:ElasticsearchDependenciesJob.java


注：本文中的org.elasticsearch.spark.rdd.api.java.JavaEsSpark类示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。