Java SparkConf.setMaster方法代码示例

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
public static void main(String[] args) 
{
 SparkConf conf = new SparkConf();
 conf.setAppName("Wordcount Background");
 conf.setMaster("local");
  
 
 JavaStreamingContext ssc = new JavaStreamingContext(conf, Durations.seconds(15));
 
 
 JavaDStream<String> lines = ssc.textFileStream("/home/rahul/DATASET");
 JavaDStream<String> words = lines.flatMap(WORDS_EXTRACTOR);
 JavaPairDStream<String, Integer> pairs = words.mapToPair(WORDS_MAPPER);
 JavaPairDStream<String, Integer> counter = pairs.reduceByKey(WORDS_REDUCER);
 
 counter.print();
 
 ssc.start();
 
 ssc.awaitTermination();
 

 /*JavaRDD<String> file = context.textFile("/home/rahul/Desktop/palestine.txt");
 JavaRDD<String> words = file.flatMap(WORDS_EXTRACTOR);
 JavaPairRDD<String, Integer> pairs = words.mapToPair(WORDS_MAPPER);
 JavaPairRDD<String, Integer> counter = pairs.reduceByKey(WORDS_REDUCER);
 counter.saveAsTextFile("/home/rahul/Desktop/wc"); 
 context.close();*/
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
/**
 * Provide a {@link JavaSparkContext} based on default settings
 *
 * @return a {@link JavaSparkContext} based on default settings
 */
public static JavaSparkContext provide() {
    SparkConf config = new SparkConf()
            .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
            .registerKryoClasses(getSerializableClasses());

    if (!config.contains("spark.app.name")) {
        config.setAppName("RDF2X");
    }
    if (!config.contains("spark.master")) {
        config.setMaster("local");
    }

    // set serialization registration required if you want to make sure you registered all your classes
    // some spark internal classes will need to be registered as well
    // config.set("spark.kryo.registrationRequired", "true");


    log.info("Getting Spark Context for config: \n{}", config.toDebugString());
    return new JavaSparkContext(config);
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
private void configureSparkContext(Properties properties) {
    SparkConf sparkConf = new SparkConf();
    sparkConf.setAppName("Write pipeline");
    sparkConf.set("spark.driver.allowMultipleContexts", "true");

    sparkConf.setMaster(properties.getProperty("spark.master"));
    sparkConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer");

    sparkConf.set("spark.cassandra.connection.host", properties.getProperty("cassandra.nodes"));
    sparkConf.set("spark.cassandra.output.batch.size.bytes", properties.getProperty("cassandra.batch.size.bytes"));
    sparkConf.set("spark.cassandra.connection.port", properties.getProperty("cassandra.port"));

    sparkConf.set("es.nodes", properties.getProperty("elasticsearch.nodes") + ":" + properties.getProperty("elasticsearch.port.rest"));
    sparkConf.set("es.batch.size.entries", properties.getProperty("elasticsearch.batch.size.entries"));
    sparkConf.set("es.batch.size.bytes", properties.getProperty("elasticsearch.batch.size.bytes"));
    sparkConf.set("es.nodes.discovery", properties.getProperty("elasticsearch.nodes.dicovery"));

    sparkContext = new JavaSparkContext(sparkConf);
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
@BeforeClass
public static void createContext() throws IOException {

	Configuration hdfsConfig = HDFSUtils.getConfiguration();
	SparkConf config = new SparkConf();
	config.setMaster("local[*]");
	config.setAppName("my JUnit running Spark");
	sc = new JavaSparkContext(config);
	fileSystem = FileSystem.get(hdfsConfig);
	sqlContext = new SQLContext(sc);
	engine = new ParquetRepartEngine(fileSystem, sqlContext);
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
protected final JavaStreamingContext buildStreamingContext() {
  log.info("Starting SparkContext with interval {} seconds", generationIntervalSec);

  SparkConf sparkConf = new SparkConf();

  // Only for tests, really
  if (sparkConf.getOption("spark.master").isEmpty()) {
    log.info("Overriding master to {} for tests", streamingMaster);
    sparkConf.setMaster(streamingMaster);
  }
  // Only for tests, really
  if (sparkConf.getOption("spark.app.name").isEmpty()) {
    String appName = "Oryx" + getLayerName();
    if (id != null) {
      appName = appName + "-" + id;
    }
    log.info("Overriding app name to {} for tests", appName);
    sparkConf.setAppName(appName);
  }
  extraSparkConfig.forEach((key, value) -> sparkConf.setIfMissing(key, value.toString()));

  // Turn this down to prevent long blocking at shutdown
  sparkConf.setIfMissing(
      "spark.streaming.gracefulStopTimeout",
      Long.toString(TimeUnit.MILLISECONDS.convert(generationIntervalSec, TimeUnit.SECONDS)));
  sparkConf.setIfMissing("spark.cleaner.ttl", Integer.toString(20 * generationIntervalSec));
  long generationIntervalMS =
      TimeUnit.MILLISECONDS.convert(generationIntervalSec, TimeUnit.SECONDS);

  JavaSparkContext jsc = JavaSparkContext.fromSparkContext(SparkContext.getOrCreate(sparkConf));
  return new JavaStreamingContext(jsc, new Duration(generationIntervalMS));
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
public static JavaSparkContext getSparkContext(){
    if(jscSingleton == null){
        SparkConf sparkConf = new SparkConf().setAppName(appName);
        sparkConf.setMaster("local[4]");
        jscSingleton = new JavaSparkContext(sparkConf);
    }
    return jscSingleton;
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
public static JavaSparkContext getSparkContext(String appName){
    if(jscSingleton == null){
        SparkConf sparkConf = new SparkConf().setAppName(appName);
        sparkConf.setMaster("local[*]");
        sparkConf.set("spark.driver.maxResultSize", "16g");
        jscSingleton = new JavaSparkContext(sparkConf);
    }
    return jscSingleton;
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
private static JavaSparkContext getSparkContext(){
    if(jscSingleton == null){
        SparkConf sparkConf = new SparkConf().setAppName(appName);
        sparkConf.setMaster("local[4]");
        jscSingleton = new JavaSparkContext(sparkConf);
    }
    return jscSingleton;
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
public JavaSparkContext getSparkContext(){
    if(jscSingleton == null){
        SparkConf sparkConf = new SparkConf().setAppName(appName);
        sparkConf.setMaster(CLUSTER_CONF_LOCAL_4);
        jscSingleton = new JavaSparkContext(sparkConf);
    }
    return jscSingleton;
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
public SparkConf getSparkConf() {
    SparkConf sparkConf = new SparkConf();
    sparkConf.set("spark.streaming.kafka.maxRatePerPartition",
            config.getSparkStreamingKafkaMaxRatePerPartition()); // rate limiting
    sparkConf.setAppName("StreamingEngine-" + config.getTopicSet().toString() + "-" + config.getNamespace());

    if (config.getLocalMode()) {
        sparkConf.setMaster("local[4]");
    }
    return sparkConf;
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
public static void main(String... args) {
  SparkConf conf = new SparkConf();
  conf.setMaster("local[2]");
  conf.setAppName("Spark Streaming Test Java");

  JavaSparkContext sc = new JavaSparkContext(conf);
  JavaStreamingContext ssc = new JavaStreamingContext(sc, Durations.seconds(10));

  processStream(ssc, sc);

  ssc.start();
  ssc.awaitTermination();
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
private static JavaSparkContext createSparkContext(SparkContextOptions contextOptions) {
  if (usesProvidedSparkContext) {
    LOG.info("Using a provided Spark Context");
    JavaSparkContext jsc = contextOptions.getProvidedSparkContext();
    if (jsc == null || jsc.sc().isStopped()){
      LOG.error("The provided Spark context " + jsc + " was not created or was stopped");
      throw new RuntimeException("The provided Spark context was not created or was stopped");
    }
    return jsc;
  } else {
    LOG.info("Creating a brand new Spark Context.");
    SparkConf conf = new SparkConf();
    if (!conf.contains("spark.master")) {
      // set master if not set.
      conf.setMaster(contextOptions.getSparkMaster());
    }

    if (contextOptions.getFilesToStage() != null && !contextOptions.getFilesToStage().isEmpty()) {
      conf.setJars(contextOptions.getFilesToStage().toArray(new String[0]));
    }

    conf.setAppName(contextOptions.getAppName());
    // register immutable collections serializers because the SDK uses them.
    conf.set("spark.kryo.registrator", BeamSparkRunnerRegistrator.class.getName());
    return new JavaSparkContext(conf);
  }
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
public static void main(String args[]) {
    SparkConf sparkConf = new SparkConf();
    sparkConf.setAppName("spark-phoenix-df");
    sparkConf.setMaster("local[*]");
    JavaSparkContext sc = new JavaSparkContext(sparkConf);
    SQLContext sqlContext = new org.apache.spark.sql.SQLContext(sc);

    DataFrame df = sqlContext.read()
            .format("org.apache.phoenix.spark")
            .option("table", "ORDERS")
            .option("zkUrl", "localhost:2181")
            .load();
    df.count();

}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
@Before
public void setup() {
    SparkConf sparkConf = new SparkConf();
    String master = "local[2]";
    sparkConf.setMaster(master);
    sparkConf.setAppName("Local Spark Unit Test");
    sc = new JavaSparkContext(new SparkContext(sparkConf));
    sqlContext = new SQLContext(sc);
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
public void run() throws IOException {
    FileSystem fs = DistributedFileSystem.get(new Configuration());
    Path inpath = new Path(input);
    Path outpath = new Path(output);
    if (!fs.exists(inpath)) {
        throw new IllegalArgumentException("Input file not found: " + inpath);
    }
    if (fs.exists(outpath)) {
        throw new IllegalArgumentException("Output file exists, Not overwriting it: " + inpath);
    }

    SparkConf conf = new SparkConf();
    conf.setMaster(sparkMaster);
    conf.setAppName(getClass().getSimpleName() + "::" + System.currentTimeMillis());
    JavaSparkContext ctx = new JavaSparkContext(conf);

    //STEP1: READ
    JavaPairRDD<Text, BytesWritable> rdd = ctx.sequenceFile(input, Text.class, BytesWritable.class);
            //.mapToPair(rec -> new Tuple2<>(new Text(rec._1()), new BytesWritable(rec._2().getBytes())));
    //STEP2: PARSE
    JavaPairRDD<Text, Metadata> parsedRDD = rdd.mapToPair(
            (PairFunction<Tuple2<Text, BytesWritable>, Text, Metadata>) rec -> {
                Metadata md = new Metadata();
                try (ByteArrayInputStream stream = new ByteArrayInputStream(rec._2().getBytes())) {
                    String content = TikaHolder.tika.parseToString(stream, md);
                    md.add("CONTENT", content);
                }
                return new Tuple2<>(rec._1(), md);
            });
    //STEP3: FORMAT
    JavaRDD<String> outRDD = parsedRDD.map((Function<Tuple2<Text, Metadata>, String>) rec -> {
        String key = rec._1().toString();
        Metadata metadata = rec._2();
        JSONObject object = new JSONObject();
        for (String name : metadata.names()) {
            if (metadata.isMultiValued(name)) {
                JSONArray arr = new JSONArray();
                for (String val : metadata.getValues(name)) {
                    arr.add(val);
                }
                object.put(name, arr);
            } else {
                object.put(name, metadata.get(name));
            }
        }
        return key + "\t\t" + object.toJSONString();
    });
    //STEP4: SAVE
    LOG.info("Saving at " + outpath);
    outRDD.saveAsTextFile(output);
    LOG.info("Stopping");
    ctx.stop();
}