Java GiraphConfiguration.setWorkerConfiguration方法代码示例

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
@Override
public int run(String[] args) throws Exception {
    if (args.length != 3) {
        throw new IllegalArgumentException(
            "Syntax error: Must have 3 arguments <numbersOfWorkers> <inputLocaiton> <outputLocation>");
    }

    int numberOfWorkers = Integer.parseInt(args[0]);
    String inputLocation = args[1];
    String outputLocation = args[2];

    GiraphJob job = new GiraphJob(getConf(), getClass().getName());
    GiraphConfiguration gconf = job.getConfiguration();
    gconf.setWorkerConfiguration(numberOfWorkers, numberOfWorkers, 100.0f);

    GiraphFileInputFormat.addVertexInputPath(gconf, new Path(inputLocation));
    FileOutputFormat.setOutputPath(job.getInternalJob(), new Path(outputLocation));

    gconf.setComputationClass(ZombieComputation.class);
    gconf.setMasterComputeClass(ZombieMasterCompute.class);
    gconf.setVertexInputFormatClass(ZombieTextVertexInputFormat.class);
    gconf.setVertexOutputFormatClass(ZombieTextVertexOutputFormat.class);
    gconf.setWorkerContextClass(ZombieWorkerContext.class);

    boolean verbose = true;
    if (job.run(verbose)) {
        return 0;
    } else {
        return -1;
    }
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Set up the GiraphConfiguration settings we need to run a no-op Giraph
 * job on a MiniYARNCluster as an integration test. Some YARN-specific
 * flags are set inside GiraphYarnClient and won't need to be set here.
 */
private void setupYarnConfiguration() throws IOException {
  conf = new GiraphConfiguration();
  conf.setWorkerConfiguration(1, 1, 100.0f);
  conf.setMaxMasterSuperstepWaitMsecs(30 * 1000);
  conf.setEventWaitMsecs(3 * 1000);
  conf.setYarnLibJars(""); // no need
  conf.setYarnTaskHeapMb(256); // small since no work to be done
  conf.setComputationClass(DummyYarnComputation.class);
  conf.setVertexInputFormatClass(IntIntNullTextInputFormat.class);
  conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
  conf.setNumComputeThreads(1);
  conf.setMaxTaskAttempts(1);
  conf.setNumInputSplitsThreads(1);
  // Giraph on YARN only ever things its running in "non-local" mode
  conf.setLocalTestMode(false);
  // this has to happen here before we populate the conf with the temp dirs
  setupTempDirectories();
  conf.set(OUTDIR, new Path(outputDir.getAbsolutePath()).toString());
  GiraphFileInputFormat.addVertexInputPath(conf, new Path(inputDir.getAbsolutePath()));
  // hand off the ZK info we just created to our no-op job
  GiraphConstants.ZOOKEEPER_SERVERLIST_POLL_MSECS.set(conf, 500);
  conf.setZooKeeperConfiguration(zkList);
  conf.set(GiraphConstants.ZOOKEEPER_DIR, zkDir.getAbsolutePath());
  GiraphConstants.ZOOKEEPER_MANAGER_DIRECTORY.set(conf, zkMgrDir.getAbsolutePath());
  // without this, our "real" client won't connect w/"fake" YARN cluster
  conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Set up the GiraphConfiguration settings we need to run a no-op Giraph
 * job on a MiniYARNCluster as an integration test. Some YARN-specific
 * flags are set inside GiraphYarnClient and won't need to be set here.
 */
private void setupYarnConfiguration() throws IOException {
  conf = new GiraphConfiguration();
  conf.setWorkerConfiguration(1, 1, 100.0f);
  conf.setMaxMasterSuperstepWaitMsecs(30 * 1000);
  conf.setEventWaitMsecs(3 * 1000);
  conf.setYarnLibJars(""); // no need
  conf.setYarnTaskHeapMb(256); // small since no work to be done
  conf.setVertexClass(DummyYarnVertex.class);
  conf.setVertexInputFormatClass(IntIntNullTextInputFormat.class);
  conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
  conf.setNumComputeThreads(1);
  conf.setMaxTaskAttempts(1);
  conf.setNumInputSplitsThreads(1);
  // Giraph on YARN only ever things its running in "non-local" mode
  conf.setLocalTestMode(false);
  // this has to happen here before we populate the conf with the temp dirs
  setupTempDirectories();
  conf.set(OUTDIR, new Path(outputDir.getAbsolutePath()).toString());
  GiraphFileInputFormat.addVertexInputPath(conf, new Path(inputDir.getAbsolutePath()));
  // hand off the ZK info we just created to our no-op job
  GiraphConstants.ZOOKEEPER_SERVERLIST_POLL_MSECS.set(conf, 500);
  conf.setZooKeeperConfiguration(zkList);
  conf.set(GiraphConstants.ZOOKEEPER_DIR, zkDir.getAbsolutePath());
  GiraphConstants.ZOOKEEPER_MANAGER_DIRECTORY.set(conf, zkMgrDir.getAbsolutePath());
  // without this, our "real" client won't connect w/"fake" YARN cluster
  conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Attempts to run the vertex internally in the current JVM, reading from and
 * writing to a temporary folder on local disk. Will start its own zookeeper
 * instance.
 *
 * @param conf           GiraphClasses specifying which types to use
 * @param checkpointsDir if set, will use this folder
 *                       for storing checkpoints.
 * @param tmpDir         file path for storing temporary files.
 * @return linewise output data, or null if job fails
 * @throws Exception if anything goes wrong
 */
public static Iterable<String> run(
        GiraphConfiguration conf,
        String checkpointsDir,
        File tmpDir) throws Exception {

    String ns = conf.get(HBaseGraphConfiguration.Keys.GRAPH_NAMESPACE);
    String prefix = conf.get(HBaseGraphConfiguration.Keys.GRAPH_TABLE_PREFIX);
    String tablePrefix = (ns != null ? ns + TableName.NAMESPACE_DELIM : "") + (prefix != null ? prefix : "");
    conf.set(Constants.EDGE_INPUT_TABLE, tablePrefix + Constants.EDGES);
    conf.set(Constants.VERTEX_INPUT_TABLE, tablePrefix + Constants.VERTICES);

    File outputDir = FileUtils.createTempDir(tmpDir, "output");
    File zkDir = FileUtils.createTempDir(tmpDir, "_bspZooKeeper");
    File zkMgrDir = FileUtils.createTempDir(tmpDir, "_defaultZkManagerDir");

    conf.setWorkerConfiguration(1, 1, 100.0f);
    GiraphConstants.SPLIT_MASTER_WORKER.set(conf, false);
    GiraphConstants.LOCAL_TEST_MODE.set(conf, true);

    conf.set(GiraphConstants.ZOOKEEPER_DIR, zkDir.toString());
    GiraphConstants.ZOOKEEPER_MANAGER_DIRECTORY.set(conf,
            zkMgrDir.toString());

    if (checkpointsDir == null) {
        checkpointsDir = FileUtils.createTempDir(
                tmpDir, "_checkpoints").toString();
    }
    GiraphConstants.CHECKPOINT_DIRECTORY.set(conf, checkpointsDir);

    // Create and configure the job to run the vertex
    GiraphJob job = new GiraphJob(conf, conf.getComputationName());

    Job internalJob = job.getInternalJob();
    //FileOutputFormatUtil.setOutputPath(job.getInternalJob(),
    //        new Path(outputDir.toString()));
    internalJob.getConfiguration().set("mapred.output.dir", outputDir.toString());

    // Configure a local zookeeper instance
    ZookeeperConfig qpConfig = configLocalZooKeeper(zkDir);

    boolean success = runZooKeeperAndJob(qpConfig, job);
    if (!success) {
        return null;
    }

    File outFile = new File(outputDir, "part-m-00000");
    if (conf.hasVertexOutputFormat() && outFile.canRead()) {
        return Files.readLines(outFile, Charsets.UTF_8);
    } else {
        return ImmutableList.of();
    }

}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Attempts to run the vertex internally in the current JVM, reading and
 * writing to an in-memory graph. Will start its own zookeeper
 * instance.
 *
 * @param <I> Vertex ID
 * @param <V> Vertex Value
 * @param <E> Edge Value
 * @param conf GiraphClasses specifying which types to use
 * @param graph input graph
 * @return iterable output data
 * @throws Exception if anything goes wrong
 */
public static <I extends WritableComparable,
  V extends Writable,
  E extends Writable> TestGraph<I, V, E> run(
    GiraphConfiguration conf,
    TestGraph<I, V, E> graph) throws Exception {
  File tmpDir = null;
  try {
    // Prepare temporary folders
    tmpDir = FileUtils.createTestDir(conf.getComputationName());

    File zkDir = FileUtils.createTempDir(tmpDir, "_bspZooKeeper");
    File zkMgrDir = FileUtils.createTempDir(tmpDir, "_defaultZkManagerDir");
    File checkpointsDir = FileUtils.createTempDir(tmpDir, "_checkpoints");

    conf.setVertexInputFormatClass(InMemoryVertexInputFormat.class);

    // Create and configure the job to run the vertex
    GiraphJob job = new GiraphJob(conf, conf.getComputationName());

    InMemoryVertexInputFormat.setGraph(graph);

    conf.setWorkerConfiguration(1, 1, 100.0f);
    GiraphConstants.SPLIT_MASTER_WORKER.set(conf, false);
    GiraphConstants.LOCAL_TEST_MODE.set(conf, true);
    conf.set(GiraphConstants.ZOOKEEPER_LIST, "localhost:" +
        String.valueOf(LOCAL_ZOOKEEPER_PORT));

    conf.set(GiraphConstants.ZOOKEEPER_DIR, zkDir.toString());
    GiraphConstants.ZOOKEEPER_MANAGER_DIRECTORY.set(conf,
        zkMgrDir.toString());
    GiraphConstants.CHECKPOINT_DIRECTORY.set(conf, checkpointsDir.toString());

    // Configure a local zookeeper instance
    Properties zkProperties = configLocalZooKeeper(zkDir);

    QuorumPeerConfig qpConfig = new QuorumPeerConfig();
    qpConfig.parseProperties(zkProperties);

    // Create and run the zookeeper instance
    final InternalZooKeeper zookeeper = new InternalZooKeeper();
    final ServerConfig zkConfig = new ServerConfig();
    zkConfig.readFrom(qpConfig);

    ExecutorService executorService = Executors.newSingleThreadExecutor();
    executorService.execute(new Runnable() {
      @Override
      public void run() {
        try {
          zookeeper.runFromConfig(zkConfig);
        } catch (IOException e) {
          throw new RuntimeException(e);
        }
      }
    });
    try {
      job.run(true);
    } finally {
      executorService.shutdown();
      zookeeper.end();
    }
    return graph;
  } finally {
    FileUtils.delete(tmpDir);
  }
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Adjust the configuration to the basic test case
 */
public final Configuration setupConfiguration(GiraphJob job)
    throws IOException {
  GiraphConfiguration conf = job.getConfiguration();
  conf.set("mapred.jar", getJarLocation());

  // Allow this test to be run on a real Hadoop setup
  if (runningInDistributedMode()) {
    System.out.println("setupConfiguration: Sending job to job tracker " +
        jobTracker + " with jar path " + getJarLocation()
        + " for " + getName());
    conf.set("mapred.job.tracker", jobTracker);
    conf.setWorkerConfiguration(getNumWorkers(), getNumWorkers(), 100.0f);
  }
  else {
    System.out.println("setupConfiguration: Using local job runner with " +
        "location " + getJarLocation() + " for " + getName());
    conf.setWorkerConfiguration(1, 1, 100.0f);
    // Single node testing
    GiraphConstants.SPLIT_MASTER_WORKER.set(conf, false);
    GiraphConstants.LOCAL_TEST_MODE.set(conf, true);
  }
  conf.setMaxMasterSuperstepWaitMsecs(30 * 1000);
  conf.setEventWaitMsecs(3 * 1000);
  GiraphConstants.ZOOKEEPER_SERVERLIST_POLL_MSECS.set(conf, 500);
  if (getZooKeeperList() != null) {
    conf.setZooKeeperConfiguration(getZooKeeperList());
  }
  // GeneratedInputSplit will generate 5 vertices
  conf.setLong(READER_VERTICES_OPT, 5);

  // Setup pathes for temporary files
  Path zookeeperDir = getTempPath("_bspZooKeeper");
  Path zkManagerDir = getTempPath("_defaultZkManagerDir");
  Path checkPointDir = getTempPath("_checkpoints");

  // We might start several jobs per test, so we need to clean up here
  FileUtils.deletePath(conf, zookeeperDir);
  FileUtils.deletePath(conf, zkManagerDir);
  FileUtils.deletePath(conf, checkPointDir);

  conf.set(GiraphConstants.ZOOKEEPER_DIR, zookeeperDir.toString());
  GiraphConstants.ZOOKEEPER_MANAGER_DIRECTORY.set(conf,
      zkManagerDir.toString());
  GiraphConstants.CHECKPOINT_DIRECTORY.set(conf, checkPointDir.toString());

  return conf;
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Attempts to run the vertex internally in the current JVM, reading and
 * writing to an in-memory graph. Will start its own zookeeper
 * instance.
 *
 * @param <I> Vertex ID
 * @param <V> Vertex Value
 * @param <E> Edge Value
 * @param <M> Message Value
 * @param conf GiraphClasses specifying which types to use
 * @param graph input graph
 * @return iterable output data
 * @throws Exception if anything goes wrong
 */
public static <I extends WritableComparable,
  V extends Writable,
  E extends Writable,
  M extends Writable> TestGraph<I, V, E, M> run(
    GiraphConfiguration conf,
    TestGraph<I, V, E, M> graph) throws Exception {
  File tmpDir = null;
  try {
    // Prepare temporary folders
    tmpDir = FileUtils.createTestDir(conf.getVertexClass());

    File zkDir = FileUtils.createTempDir(tmpDir, "_bspZooKeeper");
    File zkMgrDir = FileUtils.createTempDir(tmpDir, "_defaultZkManagerDir");
    File checkpointsDir = FileUtils.createTempDir(tmpDir, "_checkpoints");

    conf.setVertexInputFormatClass(InMemoryVertexInputFormat.class);

    // Create and configure the job to run the vertex
    GiraphJob job = new GiraphJob(conf, conf.getVertexClass().getName());

    InMemoryVertexInputFormat.setGraph(graph);

    conf.setWorkerConfiguration(1, 1, 100.0f);
    GiraphConstants.SPLIT_MASTER_WORKER.set(conf, false);
    GiraphConstants.LOCAL_TEST_MODE.set(conf, true);
    conf.set(GiraphConstants.ZOOKEEPER_LIST, "localhost:" +
        String.valueOf(LOCAL_ZOOKEEPER_PORT));

    conf.set(GiraphConstants.ZOOKEEPER_DIR, zkDir.toString());
    GiraphConstants.ZOOKEEPER_MANAGER_DIRECTORY.set(conf,
        zkMgrDir.toString());
    GiraphConstants.CHECKPOINT_DIRECTORY.set(conf, checkpointsDir.toString());

    // Configure a local zookeeper instance
    Properties zkProperties = configLocalZooKeeper(zkDir);

    QuorumPeerConfig qpConfig = new QuorumPeerConfig();
    qpConfig.parseProperties(zkProperties);

    // Create and run the zookeeper instance
    final InternalZooKeeper zookeeper = new InternalZooKeeper();
    final ServerConfig zkConfig = new ServerConfig();
    zkConfig.readFrom(qpConfig);

    ExecutorService executorService = Executors.newSingleThreadExecutor();
    executorService.execute(new Runnable() {
      @Override
      public void run() {
        try {
          zookeeper.runFromConfig(zkConfig);
        } catch (IOException e) {
          throw new RuntimeException(e);
        }
      }
    });
    try {
      job.run(true);
    } finally {
      executorService.shutdown();
      zookeeper.end();
    }
    return graph;
  } finally {
    FileUtils.delete(tmpDir);
  }
}
 

import org.apache.giraph.conf.GiraphConfiguration; //导入方法依赖的package包/类
/**
 * Adjust the configuration to the basic test case
 */
public final Configuration setupConfiguration(GiraphJob job)
    throws IOException {
  GiraphConfiguration conf = job.getConfiguration();
  conf.set("mapred.jar", getJarLocation());

  // Allow this test to be run on a real Hadoop setup
  if (runningInDistributedMode()) {
    System.out.println("setupConfiguration: Sending job to job tracker " +
        jobTracker + " with jar path " + getJarLocation()
        + " for " + getName());
    conf.set("mapred.job.tracker", jobTracker);
    conf.setWorkerConfiguration(getNumWorkers(), getNumWorkers(), 100.0f);
  }
  else {
    System.out.println("setupConfiguration: Using local job runner with " +
        "location " + getJarLocation() + " for " + getName());
    conf.setWorkerConfiguration(1, 1, 100.0f);
    // Single node testing
    GiraphConstants.SPLIT_MASTER_WORKER.set(conf, false);
  }
  conf.setMaxMasterSuperstepWaitMsecs(30 * 1000);
  conf.setEventWaitMsecs(3 * 1000);
  GiraphConstants.ZOOKEEPER_SERVERLIST_POLL_MSECS.set(conf, 500);
  if (getZooKeeperList() != null) {
    conf.setZooKeeperConfiguration(getZooKeeperList());
  }
  // GeneratedInputSplit will generate 5 vertices
  conf.setLong(READER_VERTICES, 5);

  // Setup pathes for temporary files
  Path zookeeperDir = getTempPath("_bspZooKeeper");
  Path zkManagerDir = getTempPath("_defaultZkManagerDir");
  Path checkPointDir = getTempPath("_checkpoints");

  // We might start several jobs per test, so we need to clean up here
  FileUtils.deletePath(conf, zookeeperDir);
  FileUtils.deletePath(conf, zkManagerDir);
  FileUtils.deletePath(conf, checkPointDir);

  conf.set(GiraphConstants.ZOOKEEPER_DIR, zookeeperDir.toString());
  GiraphConstants.ZOOKEEPER_MANAGER_DIRECTORY.set(conf,
      zkManagerDir.toString());
  GiraphConstants.CHECKPOINT_DIRECTORY.set(conf, checkPointDir.toString());

  return conf;
}