Java AccumuloInputFormat.addIterator方法代码示例

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入方法依赖的package包/类
/**
 * Sets up Accumulo input for a job: the job receives
 * ({@link org.apache.accumulo.core.data.Key},
 * {@link org.apache.accumulo.core.data.Value}) pairs from the table
 * specified by the configuration (using
 * {@link MRUtils#TABLE_PREFIX_PROPERTY} and
 * {@link MRUtils#TABLE_LAYOUT_PROP}).
 * @param   job     MapReduce Job to configure
 * @throws  AccumuloSecurityException if connecting to Accumulo with the
 *          given username and password fails.
 */
protected void setupAccumuloInput(Job job) throws AccumuloSecurityException {
    // set up accumulo input
    if (!hdfsInput) {
        job.setInputFormatClass(AccumuloInputFormat.class);
    } else {
        job.setInputFormatClass(AccumuloHDFSFileInputFormat.class);
    }
    AccumuloInputFormat.setConnectorInfo(job, userName, new PasswordToken(pwd));
    String tableName = RdfCloudTripleStoreUtils.layoutPrefixToTable(rdfTableLayout, tablePrefix);
    AccumuloInputFormat.setInputTableName(job, tableName);
    AccumuloInputFormat.setScanAuthorizations(job, authorizations);
    if (mock) {
        AccumuloInputFormat.setMockInstance(job, instance);
    } else {
        ClientConfiguration clientConfig = ClientConfiguration.loadDefault()
                .withInstance(instance).withZkHosts(zk);
        AccumuloInputFormat.setZooKeeperInstance(job, clientConfig);
    }
    if (ttl != null) {
        IteratorSetting setting = new IteratorSetting(1, "fi", AgeOffFilter.class.getName());
        AgeOffFilter.setTTL(setting, Long.valueOf(ttl));
        AccumuloInputFormat.addIterator(job, setting);
    }
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入方法依赖的package包/类
@Override
public int run(String[] args) throws Exception {
  Opts opts = new Opts();
  opts.parseArgs(getClass().getName(), args);

  Job job = Job.getInstance(getConf());
  job.setJobName(getClass().getSimpleName());
  job.setJarByClass(getClass());

  job.setInputFormatClass(AccumuloInputFormat.class);
  opts.setAccumuloConfigs(job);

  IteratorSetting regex = new IteratorSetting(50, "regex", RegExFilter.class);
  RegExFilter.setRegexs(regex, opts.rowRegex, opts.columnFamilyRegex, opts.columnQualifierRegex, opts.valueRegex, false);
  AccumuloInputFormat.addIterator(job, regex);

  job.setMapperClass(RegexMapper.class);
  job.setMapOutputKeyClass(Key.class);
  job.setMapOutputValueClass(Value.class);

  job.setNumReduceTasks(0);

  job.setOutputFormatClass(TextOutputFormat.class);
  TextOutputFormat.setOutputPath(job, new Path(opts.destination));

  System.out.println("setRowRegex: " + opts.rowRegex);
  System.out.println("setColumnFamilyRegex: " + opts.columnFamilyRegex);
  System.out.println("setColumnQualifierRegex: " + opts.columnQualifierRegex);
  System.out.println("setValueRegex: " + opts.valueRegex);

  job.waitForCompletion(true);
  return job.isSuccessful() ? 0 : 1;
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入方法依赖的package包/类
@Test
public void testRFileReaderRDDAppliesIteratorCorrectly() throws IOException,
        InterruptedException, AccumuloSecurityException, AccumuloException, TableNotFoundException,
        TableExistsException, StoreException {
    // Given
    final String table = "table2";
    final MiniAccumuloCluster cluster = MiniAccumuloClusterProvider.getMiniAccumuloCluster();
    // Add some data
    final Connector connector = cluster.getConnector(MiniAccumuloClusterProvider.USER,
            MiniAccumuloClusterProvider.PASSWORD);
    connector.tableOperations().create(table);
    final BatchWriter bw = connector.createBatchWriter(table, new BatchWriterConfig());
    final Mutation m1 = new Mutation("row");
    m1.put("CF", "CQ", "value");
    bw.addMutation(m1);
    final Mutation m2 = new Mutation("row2");
    m2.put("CF", "CQ", "not");
    bw.addMutation(m2);
    bw.close();
    // Compact to ensure an RFile is created, sleep to give it a little time to do it
    connector.tableOperations().compact(table, new CompactionConfig());
    Thread.sleep(1000L);
    // Create an iterator and an option to grep for "val"
    final Map<String, String> options = new HashMap<>();
    options.put("term", "val");
    final Configuration conf = new Configuration();
    final Job job = Job.getInstance(conf);
    AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "NAME", GrepIterator.class.getName(), options));
    InputConfigurator.fetchColumns(AccumuloInputFormat.class, job.getConfiguration(),
            Sets.newHashSet(new Pair<>(new Text("CF"), new Text("CQ"))));

    // When
    final SparkSession sparkSession = SparkSessionProvider.getSparkSession();
    final RFileReaderRDD rdd = new RFileReaderRDD(sparkSession.sparkContext(),
            cluster.getInstanceName(), cluster.getZooKeepers(), MiniAccumuloClusterProvider.USER,
            MiniAccumuloClusterProvider.PASSWORD, table, new HashSet<>(),
            serialiseConfiguration(job.getConfiguration()));
    final long count = rdd.count();

    // Then
    assertEquals(1L, count);
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入方法依赖的package包/类
@Override
protected void configureInputFormat(Job job) {
    AccumuloInputFormat.addIterator(job, new IteratorSetting(100,
            WholeRowIterator.class));
}