Java DataStoreFactory.getDataStore方法代码示例

import org.apache.gora.store.DataStoreFactory; //导入方法依赖的package包/类
@Override
public int run(String[] args) throws Exception {
  
  DataStore<String,WebPage> inStore;
  DataStore<String, TokenDatum> outStore;
  Configuration conf = new Configuration();
  if(args.length > 0) {
    String dataStoreClass = args[0];
    inStore = DataStoreFactory.getDataStore(dataStoreClass, 
        String.class, WebPage.class, conf);
    if(args.length > 1) {
      dataStoreClass = args[1];
    }
    outStore = DataStoreFactory.getDataStore(dataStoreClass, 
        String.class, TokenDatum.class, conf);
  } else {
    inStore = DataStoreFactory.getDataStore(String.class, WebPage.class, conf);
    outStore = DataStoreFactory.getDataStore(String.class, TokenDatum.class, conf);
  }
  
  return wordCount(inStore, outStore);
}
 

import org.apache.gora.store.DataStoreFactory; //导入方法依赖的package包/类
/**
 * Creates an WebPage object in-memory setting several fields to dirty. 
 * Run a query over the persistent data.
 * Asserts that the results can be serialized and 
 * deserialzed without loosing data. We do this by asserting
 * what we get 'before' and 'after' (de)serialization processes.
 * Also simple assertion for equal number of URL's in WebPage 
 * and results.
 * @throws Exception
 */
@SuppressWarnings("unchecked")
@Test
public void testSerdeWebPage() throws Exception {

  MemStore<String, WebPage> store = DataStoreFactory.getDataStore(
      MemStore.class, String.class, WebPage.class, new Configuration());
  WebPageDataCreator.createWebPageData(store);

  Result<String, WebPage> result = store.newQuery().execute();

  int i = 0;
  while (result.next()) {
    WebPage page = result.get();
    TestIOUtils.testSerializeDeserialize(page);
    i++;
  }
  assertEquals(WebPageDataCreator.URLS.length, i);
}
 

import org.apache.gora.store.DataStoreFactory; //导入方法依赖的package包/类
public int run(String[] args) throws Exception {

    DataStore<String,WebPage> inStore;
    DataStore<String, TokenDatum> outStore;
    Configuration hadoopConf = new Configuration();
    if(args.length > 0) {
      String dataStoreClass = args[0];
      inStore = DataStoreFactory.getDataStore(dataStoreClass, String.class, WebPage.class, hadoopConf);
      if(args.length > 1) {
        dataStoreClass = args[1];
      }
      outStore = DataStoreFactory.getDataStore(dataStoreClass,
        String.class, TokenDatum.class, hadoopConf);
      } else {
        inStore = DataStoreFactory.getDataStore(String.class, WebPage.class, hadoopConf);
        outStore = DataStoreFactory.getDataStore(String.class, TokenDatum.class, hadoopConf);
      }

      return wordCount(inStore, outStore);
  }
 

import org.apache.gora.store.DataStoreFactory; //导入方法依赖的package包/类
@Override
public int run(String[] args) throws Exception {

  DataStore<String, WebPage> inStore;
  DataStore<String, WebPage> outStore;
  Configuration conf = new Configuration();
  if (args.length > 0) {
    String dataStoreClass = args[0];
    inStore = DataStoreFactory.getDataStore(dataStoreClass,
            String.class, WebPage.class, conf);
    if (args.length > 1) {
      dataStoreClass = args[1];
    }
    outStore = DataStoreFactory.getDataStore(dataStoreClass,
            String.class, WebPage.class, conf);
  } else {
    inStore = DataStoreFactory.getDataStore(String.class, WebPage.class, conf);
    outStore = DataStoreFactory.getDataStore(String.class, WebPage.class, conf);
  }

  return mapReduceSerialization(inStore, outStore);
}
 

import org.apache.gora.store.DataStoreFactory; //导入方法依赖的package包/类
/**
 * Creates an WebPage object in-memory setting several fields to dirty.
 * Run a query over the persistent data.
 * Asserts that the results can be serialized and
 * deserialzed without loosing data. We do this by asserting
 * what we get 'before' and 'after' (de)serialization processes.
 * Also simple assertion for equal number of URL's in WebPage
 * and results.
 * @throws Exception
 */
@SuppressWarnings("unchecked")
@Test
public void testSerdeWebPage() throws Exception {

  MemStore<String, WebPage> store = DataStoreFactory.getDataStore(
          MemStore.class, String.class, WebPage.class, new Configuration());
  WebPageDataCreator.createWebPageData(store);

  Result<String, WebPage> result = store.newQuery().execute();

  int i = 0;
  while (result.next()) {
    WebPage page = result.get();
    TestIOUtils.testSerializeDeserialize(page);
    i++;
  }
  assertEquals(WebPageDataCreator.URLS.length, i);
}
 

import org.apache.gora.store.DataStoreFactory; //导入方法依赖的package包/类
@SuppressWarnings("unchecked")
@Test
public void testSerdeWebPage() throws Exception {

  MemStore<String, WebPage> store = DataStoreFactory.getDataStore(
      MemStore.class, String.class, WebPage.class, new Configuration());
  WebPageDataCreator.createWebPageData(store);

  Result<String, WebPage> result = store.newQuery().execute();

  int i=0;
  while(result.next()) {
    WebPage page = result.get();
    TestIOUtils.testSerializeDeserialize(page);
    i++;
  }
  assertEquals(WebPageDataCreator.URLS.length, i);
}
 

import org.apache.gora.store.DataStoreFactory; //导入方法依赖的package包/类
/**
 *
 * Initialize class and create DataStore instance
 *
 * @param config component configuration
 * @throws IOException
 */
private void init(final GoraConfiguration config) throws IOException {
    this.goraProperties = DataStoreFactory.createProps();
    this.dataStore = DataStoreFactory.getDataStore(goraProperties.getProperty(GORA_DEFAULT_DATASTORE_KEY,
                                                                              config.getDataStoreClass()),
                                                    config.getKeyClass(),
                                                    config.getValueClass(),
                                                    this.configuration);
}
 

import org.apache.gora.store.DataStoreFactory; //导入方法依赖的package包/类
public void start(Path outputDir, int numReducers, boolean concurrent) throws Exception {
  LOG.info("Running Verify with outputDir=" + outputDir +", numReducers=" + numReducers);
  
  DataStore<Long,CINode> store = DataStoreFactory.getDataStore(Long.class, CINode.class, new Configuration());

  job = new Job(getConf());
  
  if (!job.getConfiguration().get("io.serializations").contains("org.apache.hadoop.io.serializer.JavaSerialization")) {
    job.getConfiguration().set("io.serializations", job.getConfiguration().get("io.serializations") + ",org.apache.hadoop.io.serializer.JavaSerialization");
  }

  job.setJobName("Link Verifier");
  job.setNumReduceTasks(numReducers);
  job.setJarByClass(getClass());
  
  Query<Long,CINode> query = store.newQuery();
  if (!concurrent) {
    // no concurrency filtering, only need prev field
    query.setFields("prev");
  } else {
    readFlushed(job.getConfiguration());
  }

  GoraMapper.initMapperJob(job, query, store, LongWritable.class, VLongWritable.class, VerifyMapper.class, true);

  job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", false);
  
  job.setReducerClass(VerifyReducer.class);
  job.setOutputFormatClass(TextOutputFormat.class);
  TextOutputFormat.setOutputPath(job, outputDir);

  store.close();
  
  job.submit();
}
 

import org.apache.gora.store.DataStoreFactory; //导入方法依赖的package包/类
private void readFlushed(Configuration conf) throws Exception {
  DataStore<Utf8,Flushed> flushedTable = DataStoreFactory.getDataStore(Utf8.class, Flushed.class, conf);
  
  Query<Utf8,Flushed> query = flushedTable.newQuery();
  Result<Utf8,Flushed> result = flushedTable.execute(query);
  
  ArrayList<String> flushedEntries = new ArrayList<>();
  while (result.next()) {
    flushedEntries.add(result.getKey() + ":" + result.get().getCount());
  }
  
  conf.setStrings("org.apache.gora.goraci.verify.flushed", flushedEntries.toArray(new String[] {}));
  
  flushedTable.close();
}
 

import org.apache.gora.store.DataStoreFactory; //导入方法依赖的package包/类
/**
 * Creates an Employee object in-memory setting several fields to dirty. 
 * Asserts that it can be serialized and 
 * deserialzed without loosing data. We do this by asserting
 * what we get 'before' and 'after' (de)serialization processes.
 * @throws Exception
 */
@SuppressWarnings("unchecked")
@Test
public void testSerdeEmployee() throws Exception {

  MemStore<String, Employee> store = DataStoreFactory.getDataStore(
      MemStore.class, String.class, Employee.class, new Configuration());

  Employee employee = DataStoreTestUtil.createEmployee(store);

  TestIOUtils.testSerializeDeserialize(employee);
}
 

import org.apache.gora.store.DataStoreFactory; //导入方法依赖的package包/类
@Override
public int run(String[] args) throws Exception {
  
  DataStore<Long, Pageview> inStore;
  DataStore<String, MetricDatum> outStore;
  Configuration conf = new Configuration();

  if(args.length > 0) {
    String dataStoreClass = args[0];
    inStore = DataStoreFactory.
        getDataStore(dataStoreClass, Long.class, Pageview.class, conf);
    if(args.length > 1) {
      dataStoreClass = args[1];
    }
    outStore = DataStoreFactory.
        getDataStore(dataStoreClass, String.class, MetricDatum.class, conf);
  } else {
   inStore = DataStoreFactory.getDataStore(Long.class, Pageview.class, conf);
   outStore = DataStoreFactory.getDataStore(String.class, MetricDatum.class, conf);
  }
  
  Job job = createJob(inStore, outStore, 3);
  boolean success = job.waitForCompletion(true);
  
  inStore.close();
  outStore.close();
  
  log.info("Log completed with {}", (success ? "success" : "failure"));
  
  return success ? 0 : 1;
}
 

import org.apache.gora.store.DataStoreFactory; //导入方法依赖的package包/类
public int run(String[] args) throws Exception {
  String dataStoreClass = "org.apache.gora.hbase.store.HBaseStore";
  if(args.length > 0) {
    dataStoreClass = args[0];
  }
  
  DataStore<String,WebPage> store 
    = DataStoreFactory.getDataStore(dataStoreClass, String.class, WebPage.class, new Configuration());
  createWebPageData(store);
  
  return 0;
}
 

import org.apache.gora.store.DataStoreFactory; //导入方法依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public int run(String[] args) throws Exception {

  if(args.length < 2) {
    LOG.info("Usage QueryCounter <keyClass> <persistentClass> [dataStoreClass]");
    return 1;
  }

  Class<K> keyClass = (Class<K>) ClassLoadingUtils.loadClass(args[0]);
  Class<T> persistentClass = (Class<T>) ClassLoadingUtils.loadClass(args[1]);

  DataStore<K,T> dataStore;
  Configuration conf = new Configuration();

  if(args.length > 2) {
    Class<? extends DataStore<K,T>> dataStoreClass
        = (Class<? extends DataStore<K, T>>) Class.forName(args[2]);
    dataStore = DataStoreFactory.getDataStore(dataStoreClass, keyClass, persistentClass, conf);
  }
  else {
    dataStore = DataStoreFactory.getDataStore(keyClass, persistentClass, conf);
  }

  long results = countQuery(dataStore);

  LOG.info("Number of result to the query:" + results);

  return 0;
}
 

import org.apache.gora.store.DataStoreFactory; //导入方法依赖的package包/类
public static MockDataStore get() {
  MockDataStore dataStore;
  try {
    dataStore = DataStoreFactory.getDataStore(MockDataStore.class
        , String.class, MockPersistent.class, new Configuration());
    return dataStore;
  } catch (GoraException ex) {
    throw new RuntimeException(ex);
  }
}
 

import org.apache.gora.store.DataStoreFactory; //导入方法依赖的package包/类
/**
 * Tests and asserts that an in-memory representation of the 
 * Employee object is Equal to a clone of the same object.
 * @throws IOException
 * @throws Exception
 */
@Test
public void testClone() throws Exception {
  //more tests for clone are in TestPersistentDatumReader
  @SuppressWarnings("unchecked")
  MemStore<String, Employee> store = DataStoreFactory.getDataStore(
      MemStore.class, String.class, Employee.class, new Configuration());

  Employee employee = DataStoreTestUtil.createEmployee(store);
  
  assertEquals(employee, Employee.newBuilder(employee).build());
}