Java SparkConf.set方法代码示例

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
public SparkDriver(Properties props) {
  SparkConf conf = new SparkConf().setAppName(props.getProperty(MudrodConstants.SPARK_APP_NAME, "MudrodSparkApp")).setIfMissing("spark.master", props.getProperty(MudrodConstants.SPARK_MASTER))
      .set("spark.hadoop.validateOutputSpecs", "false").set("spark.files.overwrite", "true");

  String esHost = props.getProperty(MudrodConstants.ES_UNICAST_HOSTS);
  String esPort = props.getProperty(MudrodConstants.ES_HTTP_PORT);

  if (!"".equals(esHost)) {
    conf.set("es.nodes", esHost);
  }

  if (!"".equals(esPort)) {
    conf.set("es.port", esPort);
  }

  conf.set("spark.serializer", KryoSerializer.class.getName());
  conf.set("es.batch.size.entries", "1500");

  sc = new JavaSparkContext(conf);
  sqlContext = new SQLContext(sc);
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
public SparkTrainWorker(
                        SparkConf conf,
                        String modelName,
                        String configPath,
                        String configFile,
                        String pyTransformScript,
                        boolean needPyTransform,
                        String loginName,
                        String hostName,
                        int hostPort,
                        int slaveNum,
                        int threadNum) throws Exception {
    super(modelName, configPath, configFile, pyTransformScript, needPyTransform,
            loginName, hostName, hostPort, threadNum);
    this.slaveNum = slaveNum;

    conf.set("spark.files.fetchTimeout", "3200");
    conf.set("spark.network.timeout", "3200");
    conf.set("spark.dynamicAllocation.executorIdleTimeout", "3200");
    conf.set("spark.dynamicAllocation.schedulerBacklogTimeout", "300");
    conf.set("spark.core.connection.auth.wait.timeout", "3200");
    conf.set("spark.memory.fraction", "0.01");
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
private void configureSparkContext(Properties properties) {
    SparkConf sparkConf = new SparkConf();
    sparkConf.setAppName("Write pipeline");
    sparkConf.set("spark.driver.allowMultipleContexts", "true");

    sparkConf.setMaster(properties.getProperty("spark.master"));
    sparkConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer");

    sparkConf.set("spark.cassandra.connection.host", properties.getProperty("cassandra.nodes"));
    sparkConf.set("spark.cassandra.output.batch.size.bytes", properties.getProperty("cassandra.batch.size.bytes"));
    sparkConf.set("spark.cassandra.connection.port", properties.getProperty("cassandra.port"));

    sparkConf.set("es.nodes", properties.getProperty("elasticsearch.nodes") + ":" + properties.getProperty("elasticsearch.port.rest"));
    sparkConf.set("es.batch.size.entries", properties.getProperty("elasticsearch.batch.size.entries"));
    sparkConf.set("es.batch.size.bytes", properties.getProperty("elasticsearch.batch.size.bytes"));
    sparkConf.set("es.nodes.discovery", properties.getProperty("elasticsearch.nodes.dicovery"));

    sparkContext = new JavaSparkContext(sparkConf);
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
public synchronized JavaSparkContext getContext() {
    if (sc != null)
        return sc;

    SparkConf sparkConf = new SparkConf().setMaster("local[*]").set("spark.driver.host", "localhost")
                    .set("spark.driverEnv.SPARK_LOCAL_IP", "127.0.0.1")
                    .set("spark.executorEnv.SPARK_LOCAL_IP", "127.0.0.1").setAppName("sparktest");
    if (useKryo()) {
        sparkConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
    }


    sc = new JavaSparkContext(sparkConf);

    return sc;
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
@Memoized
JavaStreamingContext jsc() {
  SparkConf conf = new SparkConf(true)
      .setMaster(master())
      .setAppName(getClass().getName());
  if (!jars().isEmpty()) conf.setJars(jars().toArray(new String[0]));
  for (Map.Entry<String, String> entry : conf().entrySet()) {
    conf.set(entry.getKey(), entry.getValue());
  }
  return new JavaStreamingContext(conf, new Duration(batchDuration()));
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
public static JavaSparkContext getSparkContext(String appName){
    if(jscSingleton == null){
        SparkConf sparkConf = new SparkConf().setAppName(appName);
        sparkConf.setMaster("local[*]");
        sparkConf.set("spark.driver.maxResultSize", "16g");
        jscSingleton = new JavaSparkContext(sparkConf);
    }
    return jscSingleton;
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
public SparkConf getSparkConf() {
    SparkConf sparkConf = new SparkConf();
    sparkConf.set("spark.streaming.kafka.maxRatePerPartition",
            config.getSparkStreamingKafkaMaxRatePerPartition()); // rate limiting
    sparkConf.setAppName("StreamingEngine-" + config.getTopicSet().toString() + "-" + config.getNamespace());

    if (config.getLocalMode()) {
        sparkConf.setMaster("local[4]");
    }
    return sparkConf;
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
@Override
public void run(String... args) throws Exception {
    SparkConf sparkConf = new SparkConf();
    sparkConf.set("spark.yarn.jar", config.getAssemblyJar());

    List<String> submitArgs = new ArrayList<String>();
    if (StringUtils.hasText(config.getAppName())) {
        submitArgs.add("--name");
        submitArgs.add(config.getAppName());
    }
    submitArgs.add("--jar");
    submitArgs.add(config.getAppJar());
    submitArgs.add("--class");
    submitArgs.add(config.getAppClass());
    if (StringUtils.hasText(config.getResourceFiles())) {
        submitArgs.add("--files");
        submitArgs.add(config.getResourceFiles());
    }
    if (StringUtils.hasText(config.getResourceArchives())) {
        submitArgs.add("--archives");
        submitArgs.add(config.getResourceArchives());
    }
    submitArgs.add("--executor-memory");
    submitArgs.add(config.getExecutorMemory());
    submitArgs.add("--num-executors");
    submitArgs.add("" + config.getNumExecutors());
    for (String arg : config.getAppArgs()) {
        submitArgs.add("--arg");
        submitArgs.add(arg);
    }
    logger.info("Submit App with args: " + Arrays.asList(submitArgs));
    ClientArguments clientArguments =
            new ClientArguments(submitArgs.toArray(new String[submitArgs.size()]), sparkConf);
    clientArguments.isClusterMode();
    Client client = new Client(clientArguments, hadoopConfiguration, sparkConf);
    System.setProperty("SPARK_YARN_MODE", "true");
    try {
        client.run();
    } catch (Throwable t) {
        logger.error("Spark Application failed: " + t.getMessage(), t);
        throw new RuntimeException("Spark Application failed", t);
    }
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
@After
@Before
public void setupTest() {
    SparkConf sparkConfiguration = new SparkConf();
    sparkConfiguration.setAppName(this.getClass().getCanonicalName() + "-setupTest");
    sparkConfiguration.set("spark.master", "local[4]");
    JavaSparkContext sparkContext = new JavaSparkContext(SparkContext.getOrCreate(sparkConfiguration));
    sparkContext.close();
    Spark.create(sparkContext.sc());
    Spark.close();
    logger.info("SparkContext has been closed for " + this.getClass().getCanonicalName() + "-setupTest");
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
private static JavaSparkContext createSparkContext(SparkContextOptions contextOptions) {
  if (usesProvidedSparkContext) {
    LOG.info("Using a provided Spark Context");
    JavaSparkContext jsc = contextOptions.getProvidedSparkContext();
    if (jsc == null || jsc.sc().isStopped()){
      LOG.error("The provided Spark context " + jsc + " was not created or was stopped");
      throw new RuntimeException("The provided Spark context was not created or was stopped");
    }
    return jsc;
  } else {
    LOG.info("Creating a brand new Spark Context.");
    SparkConf conf = new SparkConf();
    if (!conf.contains("spark.master")) {
      // set master if not set.
      conf.setMaster(contextOptions.getSparkMaster());
    }

    if (contextOptions.getFilesToStage() != null && !contextOptions.getFilesToStage().isEmpty()) {
      conf.setJars(contextOptions.getFilesToStage().toArray(new String[0]));
    }

    conf.setAppName(contextOptions.getAppName());
    // register immutable collections serializers because the SDK uses them.
    conf.set("spark.kryo.registrator", BeamSparkRunnerRegistrator.class.getName());
    return new JavaSparkContext(conf);
  }
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
public static void main(String[] args) {

        SparkConf sparkConf = new SparkConf();//.setAppName("BLASpark - Example CG");

        sparkConf.set("spark.shuffle.reduceLocality.enabled","false");
        //sparkConf.set("spark.memory.useLegacyMode","true");


        //The ctx is created from the previous config
        JavaSparkContext ctx = new JavaSparkContext(sparkConf);
        //ctx.hadoopConfiguration().set("parquet.enable.summary-metadata", "false");

        GeneralOptions BLASparkOptions = new GeneralOptions(args);

        if(BLASparkOptions.getMode() == GeneralOptions.Mode.DMXV) {

            LOG.warn("Starting dense matrix dot vector multiplication example...");
            DmXV exDmXV = new DmXV(BLASparkOptions, ctx);

            exDmXV.calculate();
        }
        else if(BLASparkOptions.getMode() == GeneralOptions.Mode.CG) {

            LOG.warn("Starting conjugate gradient example...");
            ConjugateGradientExample exCG = new ConjugateGradientExample(BLASparkOptions, ctx);

            exCG.calculate();
        }
        else if(BLASparkOptions.getMode() == GeneralOptions.Mode.JACOBI) {

            LOG.warn("Starting Jacobi example...");
            JacobiExample exJacobi = new JacobiExample(BLASparkOptions, ctx);

            exJacobi.calculate();
        }
        else if(BLASparkOptions.getMode() == GeneralOptions.Mode.DMXDM) {

            LOG.warn("Starting dense matrix dot dense matrix example...");
            DmXDm exDmXDm = new DmXDm(BLASparkOptions, ctx);

            exDmXDm.calculate();
        }
        else {
            LOG.warn("No execution mode selected...");
            BLASparkOptions.printHelp();
        }

    }
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
private LinkedBlockingQueue<Kryo> initialize(final Configuration conf) {
    // DCL is safe in this case due to volatility
    if (!initialized) {
        synchronized (UnshadedKryoShimService.class) {
            if (!initialized) {
                final SparkConf sparkConf = new SparkConf();

                // Copy the user's IoRegistry from the param conf to the SparkConf we just created
                final String regStr = conf.getString(GryoPool.CONFIG_IO_REGISTRY);
                if (null != regStr) { // SparkConf rejects null values with NPE, so this has to be checked before set(...)
                    sparkConf.set(GryoPool.CONFIG_IO_REGISTRY, regStr);
                }
                // Setting spark.serializer here almost certainly isn't necessary, but it doesn't hurt
                sparkConf.set("spark.serializer", IoRegistryAwareKryoSerializer.class.getCanonicalName());

                final String registrator = conf.getString("spark.kryo.registrator");
                if (null != registrator) {
                    sparkConf.set("spark.kryo.registrator", registrator);
                    log.info("Copied spark.kryo.registrator: {}", registrator);
                } else {
                    log.info("Not copying spark.kryo.registrator");
                }

                // Reuse Gryo poolsize for Kryo poolsize (no need to copy this to SparkConf)
                final int poolSize = conf.getInt(GryoPool.CONFIG_IO_GRYO_POOL_SIZE,
                        GryoPool.CONFIG_IO_GRYO_POOL_SIZE_DEFAULT);
                // Instantiate the spark.serializer
                final IoRegistryAwareKryoSerializer ioReg = new IoRegistryAwareKryoSerializer(sparkConf);
                // Setup a pool backed by our spark.serializer instance

                for (int i = 0; i < poolSize; i++) {
                    KRYOS.add(ioReg.newKryo());
                }

                initialized = true;
            }
        }
    }

    return KRYOS;
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
public ComputeResponse(FileSystem fileSys) throws PIRException
{
  fs = fileSys;
  storage = new HadoopFileSystemStore(fs);

  dataInputFormat = SystemConfiguration.getProperty("pir.dataInputFormat");
  if (!InputFormatConst.ALLOWED_FORMATS.contains(dataInputFormat))
  {
    throw new IllegalArgumentException("inputFormat = " + dataInputFormat + " is of an unknown form");
  }
  logger.info("inputFormat = " + dataInputFormat);
  if (dataInputFormat.equals(InputFormatConst.BASE_FORMAT))
  {
    inputData = SystemConfiguration.getProperty("pir.inputData", "none");
    if (inputData.equals("none"))
    {
      throw new IllegalArgumentException("For inputFormat = " + dataInputFormat + " an inputFile must be specified");
    }
    logger.info("inputFile = " + inputData);
  }
  else if (dataInputFormat.equals(InputFormatConst.ES))
  {
    esQuery = SystemConfiguration.getProperty("pir.esQuery", "none");
    esResource = SystemConfiguration.getProperty("pir.esResource", "none");
    if (esQuery.equals("none"))
    {
      throw new IllegalArgumentException("esQuery must be specified");
    }
    if (esResource.equals("none"))
    {
      throw new IllegalArgumentException("esResource must be specified");
    }
    logger.info("esQuery = " + esQuery + " esResource = " + esResource);
  }
  outputFile = SystemConfiguration.getProperty("pir.outputFile");
  outputDirExp = outputFile + "_exp";

  queryInput = SystemConfiguration.getProperty("pir.queryInput");
  String stopListFile = SystemConfiguration.getProperty("pir.stopListFile");
  useModExpJoin = SystemConfiguration.getBooleanProperty("pir.useModExpJoin", false);

  logger.info("outputFile = " + outputFile + " queryInputDir = " + queryInput + " stopListFile = " + stopListFile + " esQuery = " + esQuery + " esResource = "
      + esResource);

  // Set the necessary configurations
  SparkConf conf = new SparkConf().setAppName("SparkPIR").setMaster("yarn-cluster");
  conf.set("es.nodes", SystemConfiguration.getProperty("es.nodes", "none"));
  conf.set("es.port", SystemConfiguration.getProperty("es.port", "none"));
  conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
  conf.set("spark.memory.storageFraction", "0.10");
  conf.set("spark.memory.fraction", "0.25");
  // conf.set("spark.memory.fraction", "0.25");
  // conf.set("spark.executor.extraJavaOptions", "-XX:+UseCompressedOops");
  sc = new JavaSparkContext(conf);

  // Setup, run query, teardown
  logger.info("Setting up for query run");
  try
  {
    setup();
  } catch (IOException e)
  {
    throw new PIRException("An error occurred setting up the Spark responder.", e);
  }
  logger.info("Setup complete");
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
public ComputeStreamingResponse(FileSystem fileSys) throws PIRException
{
  fs = fileSys;
  storage = new HadoopFileSystemStore(fs);

  dataInputFormat = SystemConfiguration.getProperty("pir.dataInputFormat");
  if (!InputFormatConst.ALLOWED_FORMATS.contains(dataInputFormat))
  {
    throw new IllegalArgumentException("inputFormat = " + dataInputFormat + " is of an unknown form");
  }
  logger.info("inputFormat = " + dataInputFormat);
  if (dataInputFormat.equals(InputFormatConst.BASE_FORMAT))
  {
    inputData = SystemConfiguration.getProperty("pir.inputData", "none");
    if (inputData.equals("none"))
    {
      throw new IllegalArgumentException("For inputFormat = " + dataInputFormat + " an inputFile must be specified");
    }
    logger.info("inputFile = " + inputData);
  }
  else if (dataInputFormat.equals(InputFormatConst.ES))
  {
    esQuery = SystemConfiguration.getProperty("pir.esQuery", "none");
    esResource = SystemConfiguration.getProperty("pir.esResource", "none");
    if (esQuery.equals("none"))
    {
      throw new IllegalArgumentException("esQuery must be specified");
    }
    if (esResource.equals("none"))
    {
      throw new IllegalArgumentException("esResource must be specified");
    }
    logger.info("esQuery = " + esQuery + " esResource = " + esResource);
  }
  outputFile = SystemConfiguration.getProperty("pir.outputFile");
  outputDirExp = outputFile + "_exp";

  queryInput = SystemConfiguration.getProperty("pir.queryInput");
  String stopListFile = SystemConfiguration.getProperty("pir.stopListFile");

  logger.info("outputFile = " + outputFile + " queryInputDir = " + queryInput + " stopListFile = " + stopListFile + " esQuery = " + esQuery + " esResource = "
      + esResource);

  // Pull the batchSeconds and windowLength parameters
  long batchSeconds = SystemConfiguration.getLongProperty("pir.sparkstreaming.batchSeconds", 30);
  windowLength = SystemConfiguration.getLongProperty("pir.sparkstreaming.windowLength", 60);
  if (windowLength % batchSeconds != 0)
  {
    throw new IllegalArgumentException("batchSeconds = " + batchSeconds + " must divide windowLength = " + windowLength);
  }
  useQueueStream = SystemConfiguration.getBooleanProperty("pir.sparkstreaming.useQueueStream", false);
  logger.info("useQueueStream = " + useQueueStream);

  // Set the necessary configurations
  SparkConf conf = new SparkConf().setAppName("SparkPIR").setMaster("yarn-cluster");
  conf.set("es.nodes", SystemConfiguration.getProperty("es.nodes", "none"));
  conf.set("es.port", SystemConfiguration.getProperty("es.port", "none"));
  conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
  conf.set("spark.streaming.stopGracefullyOnShutdown", SystemConfiguration.getProperty("spark.streaming.stopGracefullyOnShutdown", "false"));

  JavaSparkContext sc = new JavaSparkContext(conf);
  jssc = new JavaStreamingContext(sc, Durations.seconds(batchSeconds));

  // Setup, run query, teardown
  logger.info("Setting up for query run");
  try
  {
    setup();
  } catch (IOException e)
  {
    throw new PIRException("An error occurred setting up the streaming responder.", e);
  }
  logger.info("Setup complete");
}
 

import org.apache.spark.SparkConf; //导入方法依赖的package包/类
@SuppressWarnings("deprecation")
public static void main(String[] args) {
	System.setProperty("hadoop.home.dir", "C:\\softwares\\Winutils");
	SparkConf conf =new SparkConf().setMaster("local").setAppName("Cassandra Example");
	conf.set("spark.cassandra.connection.host", "127.0.0.1");
	//conf.set("spark.sql.warehouse.dir", "file:////C:/Users/sgulati/spark-warehouse");
	
	JavaSparkContext jsc=new JavaSparkContext(conf);
	JavaRDD<Employee> cassandraTable = CassandraJavaUtil.javaFunctions(jsc).cassandraTable("my_keyspace", "emp",CassandraJavaUtil.mapRowTo(Employee.class));
	
    JavaRDD<String> selectEmpDept = CassandraJavaUtil.javaFunctions(jsc).cassandraTable("my_keyspace", "emp",CassandraJavaUtil.mapColumnTo(String.class)).select("emp_dept","emp_name");
	
	 
    cassandraTable.collect().forEach(System.out::println);
    //selectEmpDept.collect().forEach(System.out::println);
	
    
    CassandraJavaUtil.javaFunctions(cassandraTable)
       .writerBuilder("my_keyspace", "emp1", CassandraJavaUtil.mapToRow(Employee.class)).saveToCassandra();
    
	/*SQLContext sqlContext = new SQLContext(jsc);
	
	Map<String,String> map =new HashMap<>();
	map.put("table" , "emp");
	map.put("keyspace", "my_keyspace");
	
	Dataset<Row> df = sqlContext.read().format("org.apache.spark.sql.cassandra")
	  .options(map)
	  .load();
	
	df.show();*/
	
	
	
}