Java AccumuloInputFormat类代码示例

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入依赖的package包/类
private RDD<Element> doOperation(final GetRDDOfElements operation,
                                 final Context context,
                                 final AccumuloStore accumuloStore)
        throws OperationException {
    final Configuration conf = getConfiguration(operation);
    final SparkContext sparkContext = SparkContextUtil.getSparkSession(context, accumuloStore.getProperties()).sparkContext();
    sparkContext.hadoopConfiguration().addResource(conf);
    // Use batch scan option when performing seeded operation
    InputConfigurator.setBatchScan(AccumuloInputFormat.class, conf, true);
    addIterators(accumuloStore, conf, context.getUser(), operation);
    addRanges(accumuloStore, conf, operation);
    final RDD<Tuple2<Element, NullWritable>> pairRDD = sparkContext.newAPIHadoopRDD(conf,
            ElementInputFormat.class,
            Element.class,
            NullWritable.class);
    return pairRDD.map(new FirstElement(), ClassTagConstants.ELEMENT_CLASS_TAG);
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入依赖的package包/类
@Override
public void setAccumuloConfigs(Job job) throws AccumuloSecurityException {
  super.setAccumuloConfigs(job);

  final String principal = getPrincipal(), tableName = getTableName();

  if (tokenFile.isEmpty()) {
    AuthenticationToken token = getToken();
    AccumuloInputFormat.setConnectorInfo(job, principal, token);
    AccumuloOutputFormat.setConnectorInfo(job, principal, token);
  } else {
    AccumuloInputFormat.setConnectorInfo(job, principal, tokenFile);
    AccumuloOutputFormat.setConnectorInfo(job, principal, tokenFile);
  }
  AccumuloInputFormat.setInputTableName(job, tableName);
  AccumuloInputFormat.setScanAuthorizations(job, auths);
  AccumuloOutputFormat.setCreateTables(job, true);
  AccumuloOutputFormat.setDefaultTableName(job, tableName);
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入依赖的package包/类
/**
 * Sets up Accumulo input for a job: the job receives
 * ({@link org.apache.accumulo.core.data.Key},
 * {@link org.apache.accumulo.core.data.Value}) pairs from the table
 * specified by the configuration (using
 * {@link MRUtils#TABLE_PREFIX_PROPERTY} and
 * {@link MRUtils#TABLE_LAYOUT_PROP}).
 * @param   job     MapReduce Job to configure
 * @throws  AccumuloSecurityException if connecting to Accumulo with the
 *          given username and password fails.
 */
protected void setupAccumuloInput(Job job) throws AccumuloSecurityException {
    // set up accumulo input
    if (!hdfsInput) {
        job.setInputFormatClass(AccumuloInputFormat.class);
    } else {
        job.setInputFormatClass(AccumuloHDFSFileInputFormat.class);
    }
    AccumuloInputFormat.setConnectorInfo(job, userName, new PasswordToken(pwd));
    String tableName = RdfCloudTripleStoreUtils.layoutPrefixToTable(rdfTableLayout, tablePrefix);
    AccumuloInputFormat.setInputTableName(job, tableName);
    AccumuloInputFormat.setScanAuthorizations(job, authorizations);
    if (mock) {
        AccumuloInputFormat.setMockInstance(job, instance);
    } else {
        ClientConfiguration clientConfig = ClientConfiguration.loadDefault()
                .withInstance(instance).withZkHosts(zk);
        AccumuloInputFormat.setZooKeeperInstance(job, clientConfig);
    }
    if (ttl != null) {
        IteratorSetting setting = new IteratorSetting(1, "fi", AgeOffFilter.class.getName());
        AgeOffFilter.setTTL(setting, Long.valueOf(ttl));
        AccumuloInputFormat.addIterator(job, setting);
    }
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入依赖的package包/类
@Override
protected void setupAccumuloInput(final Job job) throws AccumuloSecurityException {
    // set up accumulo input
    if (!hdfsInput) {
        job.setInputFormatClass(AccumuloInputFormat.class);
    } else {
        job.setInputFormatClass(AccumuloHDFSFileInputFormat.class);
    }
    AbstractInputFormat.setConnectorInfo(job, userName, new PasswordToken(pwd));
    InputFormatBase.setInputTableName(job, RdfCloudTripleStoreUtils.layoutPrefixToTable(rdfTableLayout, tablePrefix));
    AbstractInputFormat.setScanAuthorizations(job, authorizations);
    if (!mock) {
        AbstractInputFormat.setZooKeeperInstance(job, new ClientConfiguration().withInstance(instance).withZkHosts(zk));
    } else {
        AbstractInputFormat.setMockInstance(job, instance);
    }
    if (ttl != null) {
        final IteratorSetting setting = new IteratorSetting(1, "fi", AgeOffFilter.class);
        AgeOffFilter.setTTL(setting, Long.valueOf(ttl));
        InputFormatBase.addIterator(job, setting);
    }
    for (final IteratorSetting iteratorSetting : AccumuloRyaUtils.COMMON_REG_EX_FILTER_SETTINGS) {
        InputFormatBase.addIterator(job, iteratorSetting);
    }
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入依赖的package包/类
private RDD<Element> doOperationUsingElementInputFormat(final GetRDDOfAllElements operation,
                                                        final Context context,
                                                        final AccumuloStore accumuloStore)
        throws OperationException {
    final Configuration conf = getConfiguration(operation);
    addIterators(accumuloStore, conf, context.getUser(), operation);
    final String useBatchScannerRDD = operation.getOption(USE_BATCH_SCANNER_RDD);
    if (Boolean.parseBoolean(useBatchScannerRDD)) {
        InputConfigurator.setBatchScan(AccumuloInputFormat.class, conf, true);
    }
    final RDD<Tuple2<Element, NullWritable>> pairRDD = SparkContextUtil.getSparkSession(context, accumuloStore.getProperties()).sparkContext().newAPIHadoopRDD(conf,
            ElementInputFormat.class,
            Element.class,
            NullWritable.class);
    return pairRDD.map(new FirstElement(), ELEMENT_CLASS_TAG);
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入依赖的package包/类
private JavaRDD<Element> doOperation(final GetJavaRDDOfElements operation,
                                     final Context context,
                                     final AccumuloStore accumuloStore) throws OperationException {


    final JavaSparkContext sparkContext = JavaSparkContext.fromSparkContext(SparkContextUtil.getSparkSession(context, accumuloStore.getProperties()).sparkContext());
    final Configuration conf = getConfiguration(operation);
    // Use batch scan option when performing seeded operation
    InputConfigurator.setBatchScan(AccumuloInputFormat.class, conf, true);
    addIterators(accumuloStore, conf, context.getUser(), operation);
    addRanges(accumuloStore, conf, operation);
    final JavaPairRDD<Element, NullWritable> pairRDD = sparkContext.newAPIHadoopRDD(conf,
            ElementInputFormat.class,
            Element.class,
            NullWritable.class);
    final JavaRDD<Element> rdd = pairRDD.map(new FirstElement());
    return rdd;
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入依赖的package包/类
public <INPUT_OP extends Operation & GraphFilters & Input<Iterable<? extends ElementId>>>
void addRanges(final AccumuloStore accumuloStore,
               final Configuration conf,
               final INPUT_OP operation)
        throws OperationException {
    final List<Range> ranges = new ArrayList<>();
    for (final ElementId entityId : operation.getInput()) {
        try {
            ranges.addAll(accumuloStore.getKeyPackage()
                    .getRangeFactory()
                    .getRange(entityId, operation));
        } catch (final RangeFactoryException e) {
            throw new OperationException("Failed to add ranges to configuration", e);
        }
    }
    InputConfigurator.setRanges(AccumuloInputFormat.class, conf, ranges);
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入依赖的package包/类
public Job getExpectedLoadJob(String inst, String zookeepers, String user,
        String password, String table, String start, String end,
        Authorizations authorizations,
        List<Pair<Text, Text>> columnFamilyColumnQualifierPairs)
        throws IOException {
    Collection<Range> ranges = new LinkedList<Range>();
    ranges.add(new Range(start, end));

    Job expected = new Job(new Configuration());

    try {
        AccumuloInputFormat.setConnectorInfo(expected, user,
                new PasswordToken(password));
    } catch (AccumuloSecurityException e) {
        Assert.fail(e.getMessage());
    }
    AccumuloInputFormat.setInputTableName(expected, table);
    AccumuloInputFormat.setScanAuthorizations(expected, authorizations);
    AccumuloInputFormat.setZooKeeperInstance(expected, inst, zookeepers);
    AccumuloInputFormat.fetchColumns(expected,
            columnFamilyColumnQualifierPairs);
    AccumuloInputFormat.setRanges(expected, ranges);

    return expected;
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入依赖的package包/类
/**
 * Configure properties needed to connect to a Fluo application
 *
 * @param conf Job configuration
 * @param config use {@link org.apache.fluo.api.config.FluoConfiguration} to configure
 *        programmatically
 */
@SuppressWarnings("deprecation")
public static void configure(Job conf, SimpleConfiguration config) {
  try {
    FluoConfiguration fconfig = new FluoConfiguration(config);
    try (Environment env = new Environment(fconfig)) {
      long ts =
          env.getSharedResources().getTimestampTracker().allocateTimestamp().getTxTimestamp();
      conf.getConfiguration().setLong(TIMESTAMP_CONF_KEY, ts);

      ByteArrayOutputStream baos = new ByteArrayOutputStream();
      config.save(baos);
      conf.getConfiguration().set(PROPS_CONF_KEY,
          new String(baos.toByteArray(), StandardCharsets.UTF_8));

      AccumuloInputFormat.setZooKeeperInstance(conf, fconfig.getAccumuloInstance(),
          fconfig.getAccumuloZookeepers());
      AccumuloInputFormat.setConnectorInfo(conf, fconfig.getAccumuloUser(),
          new PasswordToken(fconfig.getAccumuloPassword()));
      AccumuloInputFormat.setInputTableName(conf, env.getTable());
      AccumuloInputFormat.setScanAuthorizations(conf, env.getAuthorizations());
    }
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入依赖的package包/类
/**
 * Configure properties needed to connect to a Fluo application
 *
 * @param conf Job configuration
 * @param config use {@link FluoConfiguration} to configure programmatically
 */
@SuppressWarnings("deprecation")
public static void configure(Job conf, SimpleConfiguration config) {
  try {
    FluoConfiguration fconfig = new FluoConfiguration(config);
    try (Environment env = new Environment(fconfig)) {
      long ts =
          env.getSharedResources().getTimestampTracker().allocateTimestamp().getTxTimestamp();
      conf.getConfiguration().setLong(TIMESTAMP_CONF_KEY, ts);

      ByteArrayOutputStream baos = new ByteArrayOutputStream();
      config.save(baos);
      conf.getConfiguration().set(PROPS_CONF_KEY,
          new String(baos.toByteArray(), StandardCharsets.UTF_8));

      AccumuloInputFormat.setZooKeeperInstance(conf, fconfig.getAccumuloInstance(),
          fconfig.getAccumuloZookeepers());
      AccumuloInputFormat.setConnectorInfo(conf, fconfig.getAccumuloUser(),
          new PasswordToken(fconfig.getAccumuloPassword()));
      AccumuloInputFormat.setInputTableName(conf, env.getTable());
      AccumuloInputFormat.setScanAuthorizations(conf, env.getAuthorizations());
    }
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入依赖的package包/类
public DataSet<Tuple2<Key,Value>> getDataFromAccumulo(ExecutionEnvironment env) throws IOException, AccumuloSecurityException {
    job = Job.getInstance(new Configuration(), "HighScoreJob");
    AccumuloInputFormat.setConnectorInfo(job, accumuloUser, new PasswordToken(accumuloPassword));
    AccumuloInputFormat.setScanAuthorizations(job, new Authorizations("standard"));
    ClientConfiguration clientConfig = new ClientConfiguration();
    clientConfig.withInstance(accumuloInstanceName);
    clientConfig.withZkHosts(accumuloZookeeper);
    AccumuloInputFormat.setZooKeeperInstance(job, clientConfig);
    AccumuloInputFormat.setInputTableName(job, inTable);
    return env.createHadoopInput(new AccumuloInputFormat(),Key.class,Value.class, job);
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入依赖的package包/类
/**
 * makes accumulo input accessible by flink DataSet api
 * @param env
 * @return
 * @throws IOException
 * @throws AccumuloSecurityException
 */
public DataSet<Tuple2<Key,Value>> getDataFromAccumulo(ExecutionEnvironment env) throws IOException, AccumuloSecurityException {
    job = Job.getInstance(new Configuration(), jobName);
    AccumuloInputFormat.setConnectorInfo(job, accumuloUser, new PasswordToken(accumuloPassword));
    AccumuloInputFormat.setScanAuthorizations(job, new Authorizations(AccumuloIdentifiers.AUTHORIZATION.toString()));
    ClientConfiguration clientConfig = new ClientConfiguration();
    clientConfig.withInstance(accumuloInstanceName);
    clientConfig.withZkHosts(accumuloZookeeper);
    AccumuloInputFormat.setZooKeeperInstance(job, clientConfig);
    AccumuloInputFormat.setInputTableName(job, inTable);
    return env.createHadoopInput(new AccumuloInputFormat(),Key.class,Value.class, job);
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入依赖的package包/类
/**
 * makes accumulo input accessible by flink DataSet api
 * @param env
 * @return
 * @throws IOException
 * @throws AccumuloSecurityException
 */
// TODO make private after testing
public DataSet<Tuple2<Key,Value>> getDataFromAccumulo(ExecutionEnvironment env) throws IOException, AccumuloSecurityException {
    job = Job.getInstance(new Configuration(), "areaCalculationJob");
    AccumuloInputFormat.setConnectorInfo(job, accumuloUser, new PasswordToken(accumuloPassword));
    AccumuloInputFormat.setScanAuthorizations(job, new Authorizations("standard"));
    ClientConfiguration clientConfig = new ClientConfiguration();
    clientConfig.withInstance(accumuloInstanceName);
    clientConfig.withZkHosts(accumuloZookeeper);
    AccumuloInputFormat.setZooKeeperInstance(job, clientConfig);
    AccumuloInputFormat.setInputTableName(job, inTable);
    return env.createHadoopInput(new AccumuloInputFormat(),Key.class,Value.class, job);
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入依赖的package包/类
/**
 * makes accumulo input accessible by flink DataSet api
 * @param env
 * @return
 * @throws IOException
 * @throws AccumuloSecurityException
 */
// TODO make private after testing
public DataSet<Tuple2<Key,Value>> getDataFromAccumulo(ExecutionEnvironment env) throws IOException, AccumuloSecurityException {
    job = Job.getInstance(new Configuration(), "pathCalculationJob");
    AccumuloInputFormat.setConnectorInfo(job, accumuloUser, new PasswordToken(accumuloPassword));
    AccumuloInputFormat.setScanAuthorizations(job, new Authorizations("standard"));
    ClientConfiguration clientConfig = new ClientConfiguration();
    clientConfig.withInstance(accumuloInstanceName);
    clientConfig.withZkHosts(accumuloZookeeper);
    AccumuloInputFormat.setZooKeeperInstance(job, clientConfig);
    AccumuloInputFormat.setInputTableName(job, inTable);
    return env.createHadoopInput(new AccumuloInputFormat(),Key.class,Value.class, job);
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入依赖的package包/类
@Override
public int run(String[] args) throws Exception {
  Job job = Job.getInstance(getConf());
  job.setJobName(this.getClass().getName());
  job.setJarByClass(this.getClass());
  Opts opts = new Opts();
  opts.parseArgs(RowHash.class.getName(), args);
  job.setInputFormatClass(AccumuloInputFormat.class);
  opts.setAccumuloConfigs(job);

  String col = opts.column;
  int idx = col.indexOf(":");
  Text cf = new Text(idx < 0 ? col : col.substring(0, idx));
  Text cq = idx < 0 ? null : new Text(col.substring(idx + 1));
  if (cf.getLength() > 0)
    AccumuloInputFormat.fetchColumns(job, Collections.singleton(new Pair<>(cf, cq)));

  job.setMapperClass(HashDataMapper.class);
  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(Mutation.class);

  job.setNumReduceTasks(0);

  job.setOutputFormatClass(AccumuloOutputFormat.class);

  job.waitForCompletion(true);
  return job.isSuccessful() ? 0 : 1;
}