Java ModifiableHadoopConfiguration.set方法代码示例

import com.thinkaurelius.titan.hadoop.config.ModifiableHadoopConfiguration; //导入方法依赖的package包/类
public void testRecordReaderWithVertexQueryFilterDirection() throws Exception {
    Configuration config = new Configuration();
    ModifiableHadoopConfiguration faunusConf = ModifiableHadoopConfiguration.of(config);
    faunusConf.set(TitanHadoopConfiguration.INPUT_VERTEX_QUERY_FILTER, "v.query().direction(OUT)");
    GraphSONRecordReader reader = new GraphSONRecordReader(VertexQueryFilter.create(config));
    reader.initialize(new FileSplit(new Path(GraphSONRecordReaderTest.class.getResource("graph-of-the-gods.json").toURI()), 0, Long.MAX_VALUE, new String[]{}),
            HadoopCompatLoader.getCompat().newTask(new Configuration(), new TaskAttemptID()));
    int counter = 0;
    while (reader.nextKeyValue()) {
        counter++;
        assertEquals(reader.getCurrentKey(), NullWritable.get());
        FaunusVertex vertex = reader.getCurrentValue();
        assertEquals(Iterables.size(vertex.getEdges(Direction.IN)), 0);
    }
    assertEquals(counter, 12);
    reader.close();
}
 

import com.thinkaurelius.titan.hadoop.config.ModifiableHadoopConfiguration; //导入方法依赖的package包/类
public static ScanMetrics runScanJob(ScanJob scanJob, Configuration conf, String confRootField,
                                 org.apache.hadoop.conf.Configuration hadoopConf,
                                 Class<? extends InputFormat> inputFormat)
        throws IOException, InterruptedException, ClassNotFoundException {

    ModifiableHadoopConfiguration scanConf =
            ModifiableHadoopConfiguration.of(TitanHadoopConfiguration.MAPRED_NS, hadoopConf);

    tryToLoadClassByName(scanJob);

    // Set the ScanJob class
    scanConf.set(TitanHadoopConfiguration.SCAN_JOB_CLASS, scanJob.getClass().getName());

    String jobName = HadoopScanMapper.class.getSimpleName() + "[" + scanJob + "]";

    return runJob(conf, confRootField, hadoopConf, inputFormat, jobName, HadoopScanMapper.class);
}
 

import com.thinkaurelius.titan.hadoop.config.ModifiableHadoopConfiguration; //导入方法依赖的package包/类
public static ScanMetrics runVertexScanJob(VertexScanJob vertexScanJob, Configuration conf, String confRootField,
                                     org.apache.hadoop.conf.Configuration hadoopConf,
                                     Class<? extends InputFormat> inputFormat)
        throws IOException, InterruptedException, ClassNotFoundException {

    ModifiableHadoopConfiguration scanConf =
            ModifiableHadoopConfiguration.of(TitanHadoopConfiguration.MAPRED_NS, hadoopConf);

    tryToLoadClassByName(vertexScanJob);

    // Set the VertexScanJob class
    scanConf.set(TitanHadoopConfiguration.SCAN_JOB_CLASS, vertexScanJob.getClass().getName());

    String jobName = HadoopScanMapper.class.getSimpleName() + "[" + vertexScanJob + "]";

    return runJob(conf, confRootField, hadoopConf, inputFormat, jobName, HadoopVertexScanMapper.class);
}
 

import com.thinkaurelius.titan.hadoop.config.ModifiableHadoopConfiguration; //导入方法依赖的package包/类
public static org.apache.hadoop.conf.Configuration createConfiguration(final Direction direction, final String label, final int step, final String mergeWeightKey) {
    ModifiableHadoopConfiguration c = ModifiableHadoopConfiguration.withoutResources();

    c.set(LINK_STEP, step);
    c.set(LINK_DIRECTION, direction);
    c.set(LINK_LABEL, label);

    if (null == mergeWeightKey) {
        c.set(LINK_MERGE_DUPLICATES, false);
        c.set(LINK_MERGE_WEIGHT_KEY, NO_WEIGHT_KEY);
    } else {
        c.set(LINK_MERGE_DUPLICATES, true);
        c.set(LINK_MERGE_WEIGHT_KEY, mergeWeightKey);
    }

    c.set(PIPELINE_TRACK_PATHS, true);

    return c.getHadoopConfiguration();
}
 

import com.thinkaurelius.titan.hadoop.config.ModifiableHadoopConfiguration; //导入方法依赖的package包/类
/**
 * Return an unfinished configuration that writes to Titan and
 * includes custom and common faunus options.  This config has
 * no input settings (which should be added by the caller after
 * this method returns).
 *
 * @return an incomplete configuration that outputs to titan
 */
private ModifiableHadoopConfiguration getTitanOutputConfig() {
    ModifiableHadoopConfiguration faunusConf = new ModifiableHadoopConfiguration();
    ModifiableConfiguration titanConf = getTitanConfiguration();

    // Output
    faunusConf.set(OUTPUT_FORMAT, getTitanOutputFormatClass().getCanonicalName());
    faunusConf.setAllOutput(titanConf.getAll());

    setCommonFaunusOptions(faunusConf);
    setCustomFaunusOptions(faunusConf);

    return faunusConf;
}
 

import com.thinkaurelius.titan.hadoop.config.ModifiableHadoopConfiguration; //导入方法依赖的package包/类
public static void cassandraRepair(Properties titanProperties, String indexName, String indexType, String partitioner) throws Exception {
    Configuration hadoopConfig = new Configuration();
    ConfigHelper.setInputPartitioner(hadoopConfig, partitioner);
    ModifiableHadoopConfiguration titanConf = ModifiableHadoopConfiguration.of(hadoopConfig);
    titanConf.set(TitanHadoopConfiguration.INPUT_FORMAT, TitanCassandraInputFormat.class.getCanonicalName());

    setCommonRepairOptions(titanConf, indexName, indexType);
    copyPropertiesToInputAndOutputConf(hadoopConfig, titanProperties);

    HadoopGraph hg = new HadoopGraph(hadoopConfig);
    repairIndex(hg);

}
 

import com.thinkaurelius.titan.hadoop.config.ModifiableHadoopConfiguration; //导入方法依赖的package包/类
public static void hbaseRepair(Properties titanProperties, String indexName, String indexType) throws Exception {
    Configuration hadoopConfig = new Configuration();
    ModifiableHadoopConfiguration titanConf = ModifiableHadoopConfiguration.of(hadoopConfig);
    titanConf.set(TitanHadoopConfiguration.INPUT_FORMAT, TitanHBaseInputFormat.class.getCanonicalName());

    setCommonRepairOptions(titanConf, indexName, indexType);
    copyPropertiesToInputAndOutputConf(hadoopConfig, titanProperties);

    HadoopGraph hg = new HadoopGraph(hadoopConfig);
    repairIndex(hg);
}
 

import com.thinkaurelius.titan.hadoop.config.ModifiableHadoopConfiguration; //导入方法依赖的package包/类
private static void setCommonRepairOptions(ModifiableHadoopConfiguration titanConf, String indexName, String indexType) {
    titanConf.set(TitanHadoopConfiguration.INDEX_NAME, indexName);
    titanConf.set(TitanHadoopConfiguration.INDEX_TYPE, indexType);
    log.info("Set input format {}", titanConf.get(TitanHadoopConfiguration.INPUT_FORMAT));
    titanConf.set(TitanHadoopConfiguration.OUTPUT_FORMAT, NullOutputFormat.class.getCanonicalName());
    log.info("Set output format {}", titanConf.get(TitanHadoopConfiguration.OUTPUT_FORMAT));
    titanConf.set(TitanHadoopConfiguration.SIDE_EFFECT_FORMAT, TextOutputFormat.class.getCanonicalName());
    titanConf.set(TitanHadoopConfiguration.JOBDIR_LOCATION, "jobs");
    titanConf.set(TitanHadoopConfiguration.JOBDIR_OVERWRITE, true);
}
 

import com.thinkaurelius.titan.hadoop.config.ModifiableHadoopConfiguration; //导入方法依赖的package包/类
private void setCommonFaunusOptions(ModifiableHadoopConfiguration faunusConf) {
    // Side effect and misc
    faunusConf.set(OUTPUT_INFER_SCHEMA, true);
    faunusConf.set(PIPELINE_TRACK_PATHS, true);
    faunusConf.set(PIPELINE_TRACK_STATE, true);
    faunusConf.set(SIDE_EFFECT_FORMAT, TextOutputFormat.class.getCanonicalName());
    faunusConf.set(JOBDIR_LOCATION, Joiner.on(File.separator).join("target", "test-data", "output"));
    faunusConf.set(JOBDIR_OVERWRITE, true);
}
 

import com.thinkaurelius.titan.hadoop.config.ModifiableHadoopConfiguration; //导入方法依赖的package包/类
public static org.apache.hadoop.conf.Configuration createConfiguration(final Class<? extends Element> klass, final int step) {
    ModifiableHadoopConfiguration c = ModifiableHadoopConfiguration.withoutResources();
    c.set(BACK_FILTER_STEP, step);
    c.set(BACK_FILTER_CLASS, klass.getCanonicalName());
    c.set(PIPELINE_TRACK_PATHS, true);
    return c.getHadoopConfiguration();
}
 

import com.thinkaurelius.titan.hadoop.config.ModifiableHadoopConfiguration; //导入方法依赖的package包/类
public void testVertexQueryConstruction() {
    Configuration config = new Configuration();
    ModifiableHadoopConfiguration faunusConf = ModifiableHadoopConfiguration.of(config);
    faunusConf.set(TitanHadoopConfiguration.INPUT_VERTEX_QUERY_FILTER, "v.query().limit(0).direction(IN).labels('knows')");
    VertexQueryFilter query = VertexQueryFilter.create(config);
    assertTrue(query.doesFilter());
    assertEquals(query.limit, 0);
    assertEquals(query.hasContainers.size(), 0);
    assertEquals(query.direction, Direction.IN);
    assertEquals(query.labels.length, 1);
    assertEquals(query.labels[0], "knows");
}
 

import com.thinkaurelius.titan.hadoop.config.ModifiableHadoopConfiguration; //导入方法依赖的package包/类
private HadoopGraph getCustomIncrementalCerberusLoad() {
    ModifiableHadoopConfiguration faunusConf = getTitanOutputConfig();

    // Input
    faunusConf.set(INPUT_FORMAT, GraphSONInputFormat.class.getCanonicalName());
    faunusConf.set(INPUT_LOCATION, "target/test-classes/com/thinkaurelius/titan/hadoop/formats/graphson/extra-cerberus.json");

    // Output (incremental loading)
    faunusConf.set(OUTPUT_LOADER_SCRIPT_FILE, "target/test-classes/com/thinkaurelius/titan/hadoop/formats/graphson/incremental-custom-cerberus-load.groovy");

    return new HadoopGraph(faunusConf.getHadoopConfiguration());
}
 

import com.thinkaurelius.titan.hadoop.config.ModifiableHadoopConfiguration; //导入方法依赖的package包/类
private HadoopGraph getNaiveIncrementalCerberusLoad() {
    ModifiableHadoopConfiguration faunusConf = getTitanOutputConfig();

    // Input
    faunusConf.set(INPUT_FORMAT, GraphSONInputFormat.class.getCanonicalName());
    faunusConf.set(INPUT_LOCATION, "target/test-classes/com/thinkaurelius/titan/hadoop/formats/graphson/extra-cerberus.json");

    // Output (incremental loading)
    faunusConf.set(OUTPUT_LOADER_SCRIPT_FILE, "target/test-classes/com/thinkaurelius/titan/hadoop/formats/graphson/incremental-naive-cerberus-load.groovy");

    return new HadoopGraph(faunusConf.getHadoopConfiguration());
}
 

import com.thinkaurelius.titan.hadoop.config.ModifiableHadoopConfiguration; //导入方法依赖的package包/类
private HadoopGraph getIncrementalGraphSONToTitan() {
    ModifiableHadoopConfiguration faunusConf = getTitanOutputConfig();

    // Input
    faunusConf.set(INPUT_FORMAT, GraphSONInputFormat.class.getCanonicalName());
    faunusConf.set(INPUT_LOCATION, "target/test-classes/com/thinkaurelius/titan/hadoop/formats/graphson/graph-of-the-gods.json");

    // Output (incremental loading)
    faunusConf.set(OUTPUT_LOADER_SCRIPT_FILE, "target/test-classes/com/thinkaurelius/titan/hadoop/formats/graphson/incremental-load.groovy");

    return new HadoopGraph(faunusConf.getHadoopConfiguration());
}
 

import com.thinkaurelius.titan.hadoop.config.ModifiableHadoopConfiguration; //导入方法依赖的package包/类
private HadoopGraph getGraphSONToTitan() {
    ModifiableHadoopConfiguration faunusConf = getTitanOutputConfig();

    // Input
    faunusConf.set(INPUT_FORMAT, GraphSONInputFormat.class.getCanonicalName());
    faunusConf.set(INPUT_LOCATION, "target/test-classes/com/thinkaurelius/titan/hadoop/formats/graphson/graph-of-the-gods.json");

    return new HadoopGraph(faunusConf.getHadoopConfiguration());
}