Java AccumuloInputFormat.setZooKeeperInstance方法代码示例

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入方法依赖的package包/类
/**
 * Sets up Accumulo input for a job: the job receives
 * ({@link org.apache.accumulo.core.data.Key},
 * {@link org.apache.accumulo.core.data.Value}) pairs from the table
 * specified by the configuration (using
 * {@link MRUtils#TABLE_PREFIX_PROPERTY} and
 * {@link MRUtils#TABLE_LAYOUT_PROP}).
 * @param   job     MapReduce Job to configure
 * @throws  AccumuloSecurityException if connecting to Accumulo with the
 *          given username and password fails.
 */
protected void setupAccumuloInput(Job job) throws AccumuloSecurityException {
    // set up accumulo input
    if (!hdfsInput) {
        job.setInputFormatClass(AccumuloInputFormat.class);
    } else {
        job.setInputFormatClass(AccumuloHDFSFileInputFormat.class);
    }
    AccumuloInputFormat.setConnectorInfo(job, userName, new PasswordToken(pwd));
    String tableName = RdfCloudTripleStoreUtils.layoutPrefixToTable(rdfTableLayout, tablePrefix);
    AccumuloInputFormat.setInputTableName(job, tableName);
    AccumuloInputFormat.setScanAuthorizations(job, authorizations);
    if (mock) {
        AccumuloInputFormat.setMockInstance(job, instance);
    } else {
        ClientConfiguration clientConfig = ClientConfiguration.loadDefault()
                .withInstance(instance).withZkHosts(zk);
        AccumuloInputFormat.setZooKeeperInstance(job, clientConfig);
    }
    if (ttl != null) {
        IteratorSetting setting = new IteratorSetting(1, "fi", AgeOffFilter.class.getName());
        AgeOffFilter.setTTL(setting, Long.valueOf(ttl));
        AccumuloInputFormat.addIterator(job, setting);
    }
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入方法依赖的package包/类
public Job getExpectedLoadJob(String inst, String zookeepers, String user,
        String password, String table, String start, String end,
        Authorizations authorizations,
        List<Pair<Text, Text>> columnFamilyColumnQualifierPairs)
        throws IOException {
    Collection<Range> ranges = new LinkedList<Range>();
    ranges.add(new Range(start, end));

    Job expected = new Job(new Configuration());

    try {
        AccumuloInputFormat.setConnectorInfo(expected, user,
                new PasswordToken(password));
    } catch (AccumuloSecurityException e) {
        Assert.fail(e.getMessage());
    }
    AccumuloInputFormat.setInputTableName(expected, table);
    AccumuloInputFormat.setScanAuthorizations(expected, authorizations);
    AccumuloInputFormat.setZooKeeperInstance(expected, inst, zookeepers);
    AccumuloInputFormat.fetchColumns(expected,
            columnFamilyColumnQualifierPairs);
    AccumuloInputFormat.setRanges(expected, ranges);

    return expected;
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入方法依赖的package包/类
/**
 * Configure properties needed to connect to a Fluo application
 *
 * @param conf Job configuration
 * @param config use {@link org.apache.fluo.api.config.FluoConfiguration} to configure
 *        programmatically
 */
@SuppressWarnings("deprecation")
public static void configure(Job conf, SimpleConfiguration config) {
  try {
    FluoConfiguration fconfig = new FluoConfiguration(config);
    try (Environment env = new Environment(fconfig)) {
      long ts =
          env.getSharedResources().getTimestampTracker().allocateTimestamp().getTxTimestamp();
      conf.getConfiguration().setLong(TIMESTAMP_CONF_KEY, ts);

      ByteArrayOutputStream baos = new ByteArrayOutputStream();
      config.save(baos);
      conf.getConfiguration().set(PROPS_CONF_KEY,
          new String(baos.toByteArray(), StandardCharsets.UTF_8));

      AccumuloInputFormat.setZooKeeperInstance(conf, fconfig.getAccumuloInstance(),
          fconfig.getAccumuloZookeepers());
      AccumuloInputFormat.setConnectorInfo(conf, fconfig.getAccumuloUser(),
          new PasswordToken(fconfig.getAccumuloPassword()));
      AccumuloInputFormat.setInputTableName(conf, env.getTable());
      AccumuloInputFormat.setScanAuthorizations(conf, env.getAuthorizations());
    }
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入方法依赖的package包/类
/**
 * Configure properties needed to connect to a Fluo application
 *
 * @param conf Job configuration
 * @param config use {@link FluoConfiguration} to configure programmatically
 */
@SuppressWarnings("deprecation")
public static void configure(Job conf, SimpleConfiguration config) {
  try {
    FluoConfiguration fconfig = new FluoConfiguration(config);
    try (Environment env = new Environment(fconfig)) {
      long ts =
          env.getSharedResources().getTimestampTracker().allocateTimestamp().getTxTimestamp();
      conf.getConfiguration().setLong(TIMESTAMP_CONF_KEY, ts);

      ByteArrayOutputStream baos = new ByteArrayOutputStream();
      config.save(baos);
      conf.getConfiguration().set(PROPS_CONF_KEY,
          new String(baos.toByteArray(), StandardCharsets.UTF_8));

      AccumuloInputFormat.setZooKeeperInstance(conf, fconfig.getAccumuloInstance(),
          fconfig.getAccumuloZookeepers());
      AccumuloInputFormat.setConnectorInfo(conf, fconfig.getAccumuloUser(),
          new PasswordToken(fconfig.getAccumuloPassword()));
      AccumuloInputFormat.setInputTableName(conf, env.getTable());
      AccumuloInputFormat.setScanAuthorizations(conf, env.getAuthorizations());
    }
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入方法依赖的package包/类
public DataSet<Tuple2<Key,Value>> getDataFromAccumulo(ExecutionEnvironment env) throws IOException, AccumuloSecurityException {
    job = Job.getInstance(new Configuration(), "HighScoreJob");
    AccumuloInputFormat.setConnectorInfo(job, accumuloUser, new PasswordToken(accumuloPassword));
    AccumuloInputFormat.setScanAuthorizations(job, new Authorizations("standard"));
    ClientConfiguration clientConfig = new ClientConfiguration();
    clientConfig.withInstance(accumuloInstanceName);
    clientConfig.withZkHosts(accumuloZookeeper);
    AccumuloInputFormat.setZooKeeperInstance(job, clientConfig);
    AccumuloInputFormat.setInputTableName(job, inTable);
    return env.createHadoopInput(new AccumuloInputFormat(),Key.class,Value.class, job);
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入方法依赖的package包/类
/**
 * makes accumulo input accessible by flink DataSet api
 * @param env
 * @return
 * @throws IOException
 * @throws AccumuloSecurityException
 */
public DataSet<Tuple2<Key,Value>> getDataFromAccumulo(ExecutionEnvironment env) throws IOException, AccumuloSecurityException {
    job = Job.getInstance(new Configuration(), jobName);
    AccumuloInputFormat.setConnectorInfo(job, accumuloUser, new PasswordToken(accumuloPassword));
    AccumuloInputFormat.setScanAuthorizations(job, new Authorizations(AccumuloIdentifiers.AUTHORIZATION.toString()));
    ClientConfiguration clientConfig = new ClientConfiguration();
    clientConfig.withInstance(accumuloInstanceName);
    clientConfig.withZkHosts(accumuloZookeeper);
    AccumuloInputFormat.setZooKeeperInstance(job, clientConfig);
    AccumuloInputFormat.setInputTableName(job, inTable);
    return env.createHadoopInput(new AccumuloInputFormat(),Key.class,Value.class, job);
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入方法依赖的package包/类
/**
 * makes accumulo input accessible by flink DataSet api
 * @param env
 * @return
 * @throws IOException
 * @throws AccumuloSecurityException
 */
// TODO make private after testing
public DataSet<Tuple2<Key,Value>> getDataFromAccumulo(ExecutionEnvironment env) throws IOException, AccumuloSecurityException {
    job = Job.getInstance(new Configuration(), "areaCalculationJob");
    AccumuloInputFormat.setConnectorInfo(job, accumuloUser, new PasswordToken(accumuloPassword));
    AccumuloInputFormat.setScanAuthorizations(job, new Authorizations("standard"));
    ClientConfiguration clientConfig = new ClientConfiguration();
    clientConfig.withInstance(accumuloInstanceName);
    clientConfig.withZkHosts(accumuloZookeeper);
    AccumuloInputFormat.setZooKeeperInstance(job, clientConfig);
    AccumuloInputFormat.setInputTableName(job, inTable);
    return env.createHadoopInput(new AccumuloInputFormat(),Key.class,Value.class, job);
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入方法依赖的package包/类
/**
 * makes accumulo input accessible by flink DataSet api
 * @param env
 * @return
 * @throws IOException
 * @throws AccumuloSecurityException
 */
// TODO make private after testing
public DataSet<Tuple2<Key,Value>> getDataFromAccumulo(ExecutionEnvironment env) throws IOException, AccumuloSecurityException {
    job = Job.getInstance(new Configuration(), "pathCalculationJob");
    AccumuloInputFormat.setConnectorInfo(job, accumuloUser, new PasswordToken(accumuloPassword));
    AccumuloInputFormat.setScanAuthorizations(job, new Authorizations("standard"));
    ClientConfiguration clientConfig = new ClientConfiguration();
    clientConfig.withInstance(accumuloInstanceName);
    clientConfig.withZkHosts(accumuloZookeeper);
    AccumuloInputFormat.setZooKeeperInstance(job, clientConfig);
    AccumuloInputFormat.setInputTableName(job, inTable);
    return env.createHadoopInput(new AccumuloInputFormat(),Key.class,Value.class, job);
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入方法依赖的package包/类
@Override
 public void setLocation(final String location, final Job job) throws IOException {
     if (logger.isDebugEnabled()) {
         logger.debug("Set Location[" + location + "] for job[" + job.getJobName() + "]");
     }
     conf = job.getConfiguration();
     setLocationFromUri(location, job);

     if (!ConfiguratorBase.isConnectorInfoSet(AccumuloInputFormat.class, conf)) {
         try {
	AccumuloInputFormat.setConnectorInfo(job, user, new PasswordToken(userP.getBytes(StandardCharsets.UTF_8)));
} catch (final AccumuloSecurityException e) {
	throw new RuntimeException(e);
}
         AccumuloInputFormat.setInputTableName(job, table);
 		AccumuloInputFormat.setScanAuthorizations(job, authorizations);
         if (!mock) {
             AccumuloInputFormat.setZooKeeperInstance(job, inst, zookeepers);
         } else {
             AccumuloInputFormat.setMockInstance(job, inst);
         }
     }
     if (columnFamilyColumnQualifierPairs.size() > 0) {
         AccumuloInputFormat.fetchColumns(job, columnFamilyColumnQualifierPairs);
     }
     logger.info("Set ranges[" + ranges + "] for job[" + job.getJobName() + "] on table[" + table + "] " +
             "for columns[" + columnFamilyColumnQualifierPairs + "] with authorizations[" + authorizations + "]");

     if (ranges.size() == 0) {
         throw new IOException("Accumulo Range must be specified");
     }
     AccumuloInputFormat.setRanges(job, ranges);
 }
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入方法依赖的package包/类
public static void initTableMRJob(Job job, String intable, String outtable, String auths) throws AccumuloSecurityException {
  Configuration conf = job.getConfiguration();
  String username = conf.get(USERNAME);
  String password = conf.get(PASSWORD);
  String instance = conf.get(INSTANCE);
  String zookeepers = conf.get(ZOOKEEPERS);

  System.out.println("Zookeepers are " + auths);

  if (zookeepers != null) {
    AccumuloInputFormat.setZooKeeperInstance(job, instance, zookeepers);
    AccumuloOutputFormat.setZooKeeperInstance(job, instance, zookeepers);
  } else {
    throw new IllegalArgumentException("Must specify either mock or zookeepers");
  }

  AccumuloInputFormat.setConnectorInfo(job, username, new PasswordToken(password));
  AccumuloInputFormat.setScanAuthorizations(job, new Authorizations(auths));
  AccumuloInputFormat.setInputTableName(job, intable);
  job.setInputFormatClass(AccumuloInputFormat.class);
  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(IntWritable.class);

  // OUTPUT
  AccumuloOutputFormat.setConnectorInfo(job, username, new PasswordToken(password));
  AccumuloOutputFormat.setDefaultTableName(job, outtable);
  job.setOutputFormatClass(AccumuloOutputFormat.class);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Mutation.class);

}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入方法依赖的package包/类
public static void initTabToSeqFileJob(Job job, String intable, String outpath, String auths) throws AccumuloSecurityException {

    Configuration conf = job.getConfiguration();
    String username = conf.get(USERNAME);
    String password = conf.get(PASSWORD);
    String instance = conf.get(INSTANCE);
    String zookeepers = conf.get(ZOOKEEPERS);

    System.out.println("Zookeepers are " + auths);

    if (zookeepers != null) {
      AccumuloInputFormat.setZooKeeperInstance(job, instance, zookeepers);
    } else {
      throw new IllegalArgumentException("Must specify either mock or zookeepers");
    }

    AccumuloInputFormat.setConnectorInfo(job, username, new PasswordToken(password));
    AccumuloInputFormat.setScanAuthorizations(job, new Authorizations(auths));
    AccumuloInputFormat.setInputTableName(job, intable);
    job.setInputFormatClass(AccumuloInputFormat.class);
    job.setMapOutputKeyClass(CompositeType.class);
    job.setMapOutputValueClass(TripleCard.class);

    // OUTPUT
    SequenceFileOutputFormat.setOutputPath(job, new Path(outpath));
    job.setOutputFormatClass(SequenceFileOutputFormat.class);
    job.setOutputKeyClass(CompositeType.class);
    job.setOutputValueClass(TripleCard.class);

  }
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入方法依赖的package包/类
public static void initMRJob(final Job job, final String table, final String outtable, final String[] auths) throws AccumuloSecurityException {
    final Configuration conf = job.getConfiguration();
    final String username = conf.get(USERNAME);
    final String password = conf.get(PASSWORD);
    final String instance = conf.get(INSTANCE);
    final String zookeepers = conf.get(ZOOKEEPERS);
    final String mock = conf.get(MOCK);

    //input
    if (Boolean.parseBoolean(mock)) {
        AccumuloInputFormat.setMockInstance(job, instance);
        AccumuloOutputFormat.setMockInstance(job, instance);
    } else if (zookeepers != null) {
        AccumuloInputFormat.setZooKeeperInstance(job, instance, zookeepers);
        AccumuloOutputFormat.setZooKeeperInstance(job, instance, zookeepers);
    } else {
        throw new IllegalArgumentException("Must specify either mock or zookeepers");
    }

    AccumuloInputFormat.setConnectorInfo(job, username, new PasswordToken(password.getBytes(StandardCharsets.UTF_8)));
    AccumuloInputFormat.setInputTableName(job, table);
    job.setInputFormatClass(AccumuloInputFormat.class);
    AccumuloInputFormat.setScanAuthorizations(job, new Authorizations(auths));

    // OUTPUT
    job.setOutputFormatClass(AccumuloOutputFormat.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Mutation.class);
    AccumuloOutputFormat.setConnectorInfo(job, username, new PasswordToken(password.getBytes(StandardCharsets.UTF_8)));
    AccumuloOutputFormat.setDefaultTableName(job, outtable);
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入方法依赖的package包/类
public void setAccumuloConfigs(Job job) throws AccumuloSecurityException {
  AccumuloInputFormat.setZooKeeperInstance(job, this.getClientConfiguration());
  AccumuloOutputFormat.setZooKeeperInstance(job, this.getClientConfiguration());
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入方法依赖的package包/类
@Override
public void setLocation(String location, Job job) throws IOException {
    setLocationFromUri(location);

    loadDependentJars(job.getConfiguration());

    Map<String, String> entries = getInputFormatEntries(job
            .getConfiguration());
    unsetEntriesFromConfiguration(job.getConfiguration(), entries);

    try {
        AccumuloInputFormat.setConnectorInfo(job, user, new PasswordToken(
                password));
    } catch (AccumuloSecurityException e) {
        throw new IOException(e);
    }

    AccumuloInputFormat.setInputTableName(job, table);
    AccumuloInputFormat.setScanAuthorizations(job, authorizations);
    AccumuloInputFormat.setZooKeeperInstance(job, inst, zookeepers);

    List<Pair<Text, Text>> inputFormatColumns = new LinkedList<Pair<Text, Text>>();
    int colfamPrefix = 0;

    for (Column c : columns) {
        switch (c.getType()) {
        case LITERAL:
            // Pull the colf[:colq] individually
            inputFormatColumns.add(makePair(c.getColumnFamily(),
                    c.getColumnQualifier()));
            break;
        case COLFAM_PREFIX:
            // Some colfams
            colfamPrefix++;
            break;
        case COLQUAL_PREFIX:
            // Some colquals in a given colfam
            inputFormatColumns.add(makePair(c.getColumnFamily(), null));
            break;
        default:
            log.info("Ignoring unhandled column type");
            break;
        }
    }

    // If we have colfam prefixes, we have to pull all columns and filter on
    // client-side
    // TODO Create an iterator that lets us push-down *all* of the filter
    // logic
    if (0 == colfamPrefix && !inputFormatColumns.isEmpty()) {
        AccumuloInputFormat.fetchColumns(job, inputFormatColumns);
    }

    Collection<Range> ranges = Collections.singleton(new Range(start, end));

    log.info("Scanning Accumulo for " + ranges + " for table " + table);

    AccumuloInputFormat.setRanges(job, ranges);

    configureInputFormat(job);
}
 

import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; //导入方法依赖的package包/类
/**
 * The run method which sets the configuration and starts the MapReduce job
 */
public int run(String[] args) throws Exception {
  
  if(USE_MINI_ACCUMULO){
    Connector connector = LocalEnvUtil.getConnector(userPass);
    userName = "root";
    instanceName = connector.getInstance().getInstanceName();
    zookeepers = connector.getInstance().getZooKeepers();
  }
	
	// Create and initialize a MapReduce Job
	Job job = Job.getInstance(getConf(), "tweetIndexer");
	job.setJarByClass(IndexedDocIndexer.class);
	
	
	// Set the AccumuloInputFormat so the mapper can read from Accumulo
	
	AccumuloInputFormat.setConnectorInfo(job,
	    userName, 
	    new PasswordToken(userPass));
	
	AccumuloInputFormat.setInputTableName(job, twitterDataTable);
	
	AccumuloInputFormat.setScanAuthorizations(job, new Authorizations());
	
	ClientConfiguration clientConfig = new ClientConfiguration();
	clientConfig.withInstance(instanceName);
	clientConfig.withZkHosts(zookeepers);
	
	AccumuloInputFormat.setZooKeeperInstance(job, clientConfig);
	
	
	AccumuloOutputFormat.setConnectorInfo(job, 
	    userName,
	    new PasswordToken(userPass));
	
	AccumuloOutputFormat.setCreateTables(job, createTables);
	
	AccumuloOutputFormat.setDefaultTableName(job, tweetDocIndex);
	
	AccumuloOutputFormat.setZooKeeperInstance(job, clientConfig);
	
	
	// Set the map and reduce classes
	job.setMapperClass(TweetMapper.class);
	job.setReducerClass(TweetReducer.class);
	
	// Set the output key and value class for the mapper
	job.setMapOutputKeyClass(Text.class);
	job.setMapOutputValueClass(Text.class);
	
	// Set the output key and value class for the reducer
	job.setOutputKeyClass(Text.class);
	job.setOutputValueClass(Mutation.class);
	
	// Set the InputFormat and OutputFormat for the job
	job.setInputFormatClass(AccumuloInputFormat.class);
	job.setOutputFormatClass(AccumuloOutputFormat.class);
	
	// Run the MapReduce job and return 0 for success, 1 otherwise
	return job.waitForCompletion(true) ? 0 : 1;
}