Java StoreType类代码示例

import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; //导入依赖的package包/类
private ExternalSortExec(final TaskAttemptContext context, final AbstractStorageManager sm, final SortNode plan)
    throws PhysicalPlanningException {
  super(context, plan.getInSchema(), plan.getOutSchema(), null, plan.getSortKeys());

  this.plan = plan;
  this.meta = CatalogUtil.newTableMeta(StoreType.ROWFILE);

  this.defaultFanout = context.getConf().getIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT);
  if (defaultFanout < 2) {
    throw new PhysicalPlanningException(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT.varname + " cannot be lower than 2");
  }
  // TODO - sort buffer and core num should be changed to use the allocated container resource.
  this.sortBufferBytesNum = context.getConf().getLongVar(ConfVars.EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE) * 1048576L;
  this.allocatedCoreNum = context.getConf().getIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_THREAD_NUM);
  this.executorService = Executors.newFixedThreadPool(this.allocatedCoreNum);
  this.inMemoryTable = new ArrayList<Tuple>(100000);

  this.sortTmpDir = getExecutorTmpDir();
  localDirAllocator = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname);
  localFS = new RawLocalFileSystem();
}
 

import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; //导入依赖的package包/类
@Test
public final void testCreateTableDef() throws PlanningException {
  Expr expr = sqlAnalyzer.parse(CREATE_TABLE[0]);
  LogicalNode plan = planner.createPlan(expr).getRootBlock().getRoot();
  LogicalRootNode root = (LogicalRootNode) plan;
  testJsonSerDerObject(root);
  assertEquals(NodeType.CREATE_TABLE, root.getChild().getType());
  CreateTableNode createTable = root.getChild();

  Schema def = createTable.getTableSchema();
  assertEquals("name", def.getColumn(0).getSimpleName());
  assertEquals(Type.TEXT, def.getColumn(0).getDataType().getType());
  assertEquals("age", def.getColumn(1).getSimpleName());
  assertEquals(Type.INT4, def.getColumn(1).getDataType().getType());
  assertEquals("earn", def.getColumn(2).getSimpleName());
  assertEquals(Type.INT8, def.getColumn(2).getDataType().getType());
  assertEquals("score", def.getColumn(3).getSimpleName());
  assertEquals(Type.FLOAT4, def.getColumn(3).getDataType().getType());
  assertEquals(StoreType.CSV, createTable.getStorageType());
  assertEquals("/tmp/data", createTable.getPath().toString());
  assertTrue(createTable.hasOptions());
  assertEquals("|", createTable.getOptions().get("csv.delimiter"));
}
 

import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; //导入依赖的package包/类
@Test
public void test() throws CloneNotSupportedException, IOException {
  Schema schema = new Schema();
  schema.addColumn("name", Type.BLOB);
  schema.addColumn("addr", Type.TEXT);
  TableMeta info = CatalogUtil.newTableMeta(StoreType.CSV);
  testClone(info);

  Path path = new Path(CommonTestingUtil.getTestDir(), "tajo");

  TableDesc desc = new TableDesc("table1", schema, info, path);
  assertEquals("table1", desc.getName());
  
  assertEquals(path, desc.getPath());
  assertEquals(info, desc.getMeta());
  testClone(desc);
}
 

import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; //导入依赖的package包/类
@Test
public void testGetTable() throws Exception {
	schema1 = new Schema();
	schema1.addColumn(FieldName1, Type.BLOB);
	schema1.addColumn(FieldName2, Type.INT4);
	schema1.addColumn(FieldName3, Type.INT8);
   Path path = new Path(CommonTestingUtil.getTestDir(), "table1");
   TableDesc meta = CatalogUtil.newTableDesc(
       "getTable",
       schema1,
       StoreType.CSV,
       new Options(),
       path);

	assertFalse(catalog.existsTable("getTable"));
   catalog.addTable(meta);
   assertTrue(catalog.existsTable("getTable"));

   catalog.deleteTable("getTable");
   assertFalse(catalog.existsTable("getTable"));
}
 

import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; //导入依赖的package包/类
@Test
public final void testAddAndDeleteTable() throws Exception {
  Schema schema = new Schema();
  schema.addColumn("id", Type.INT4)
  .addColumn("name", Type.TEXT)
  .addColumn("age", Type.INT4)
  .addColumn("score", Type.FLOAT8);
  
  String tableName = "addedtable";
  Options opts = new Options();
  opts.put("file.delimiter", ",");
  TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts);
  TableDesc desc = new TableDesc(tableName, schema, meta, new Path(CommonTestingUtil.getTestDir(), "addedtable"));
  assertFalse(store.existTable(tableName));
  store.addTable(desc.getProto());
  assertTrue(store.existTable(tableName));

  TableDesc retrieved = new TableDesc(store.getTable(tableName));
  // Schema order check
  assertSchemaOrder(desc.getSchema(), retrieved.getSchema());
  store.deleteTable(tableName);
  assertFalse(store.existTable(tableName));
}
 

import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; //导入依赖的package包/类
@Test
public final void testGetAllTableNames() throws Exception {
  Schema schema = new Schema();
  schema.addColumn("id", Type.INT4)
  .addColumn("name", Type.TEXT)
  .addColumn("age", Type.INT4)
  .addColumn("score", Type.FLOAT8);
  
  int numTables = 5;
  for (int i = 0; i < numTables; i++) {
    String tableName = "tableA_" + i;
    TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
    TableDesc desc = new TableDesc(tableName, schema, meta,
        new Path(CommonTestingUtil.getTestDir(), "tableA_" + i));
    store.addTable(desc.getProto());
  }
  
  assertEquals(numTables, store.getAllTableNames().size());
}
 

import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; //导入依赖的package包/类
public TestStorages(StoreType type, boolean splitable, boolean statsable) throws IOException {
  this.storeType = type;
  this.splitable = splitable;
  this.statsable = statsable;

  conf = new TajoConf();
  conf.setBoolVar(ConfVars.STORAGE_MANAGER_VERSION_2, true);

  if (storeType == StoreType.RCFILE) {
    conf.setInt(RCFile.RECORD_INTERVAL_CONF_STR, 100);
  }


  testDir = CommonTestingUtil.getTestDir(TEST_PATH);
  fs = testDir.getFileSystem(conf);
}
 

import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; //导入依赖的package包/类
@Test
public final void testCreateTableDef() throws PlanningException {
  Expr expr = sqlAnalyzer.parse(CREATE_TABLE[0]);
  LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
  LogicalRootNode root = (LogicalRootNode) plan;
  testJsonSerDerObject(root);
  assertEquals(NodeType.CREATE_TABLE, root.getChild().getType());
  CreateTableNode createTable = root.getChild();

  Schema def = createTable.getTableSchema();
  assertEquals("name", def.getColumn(0).getSimpleName());
  assertEquals(Type.TEXT, def.getColumn(0).getDataType().getType());
  assertEquals("age", def.getColumn(1).getSimpleName());
  assertEquals(Type.INT4, def.getColumn(1).getDataType().getType());
  assertEquals("earn", def.getColumn(2).getSimpleName());
  assertEquals(Type.INT8, def.getColumn(2).getDataType().getType());
  assertEquals("score", def.getColumn(3).getSimpleName());
  assertEquals(Type.FLOAT4, def.getColumn(3).getDataType().getType());
  assertEquals(StoreType.CSV, createTable.getStorageType());
  assertEquals("/tmp/data", createTable.getPath().toString());
  assertTrue(createTable.hasOptions());
  assertEquals("|", createTable.getOptions().get("csv.delimiter"));
}
 

import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; //导入依赖的package包/类
@Before
public void setup() throws IOException {
  schema = new Schema();
   schema.addColumn("name", Type.BLOB);
   schema.addColumn("addr", Type.TEXT);
   info = CatalogUtil.newTableMeta(StoreType.CSV);
   path = new Path(CommonTestingUtil.getTestDir(), "table1");
   desc = new TableDesc("table1", schema, info, path);
   stats = new TableStats();
   stats.setNumRows(957685);
   stats.setNumBytes(1023234);
   stats.setNumBlocks(3123);
   stats.setNumShuffleOutputs(5);
   stats.setAvgRows(80000);

   int numCols = 2;
   ColumnStats[] cols = new ColumnStats[numCols];
   for (int i = 0; i < numCols; i++) {
     cols[i] = new ColumnStats(schema.getColumn(i));
     cols[i].setNumDistVals(1024 * i);
     cols[i].setNumNulls(100 * i);
     stats.addColumnStat(cols[i]);
   }
   desc.setStats(stats);
}
 

import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; //导入依赖的package包/类
@Test
public void testGetTable() throws Exception {
	schema1 = new Schema();
	schema1.addColumn(FieldName1, Type.BLOB);
	schema1.addColumn(FieldName2, Type.INT4);
	schema1.addColumn(FieldName3, Type.INT8);
   Path path = new Path(CommonTestingUtil.getTestDir(), "table1");
   TableDesc meta = new TableDesc(
       CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "getTable"),
       schema1,
       StoreType.CSV,
       new Options(),
       path);

	assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, "getTable"));
   catalog.createTable(meta);
   assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, "getTable"));

   catalog.dropTable(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "getTable"));
   assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, "getTable"));
}
 

import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; //导入依赖的package包/类
public TestBSTIndex(StoreType type) {
  this.storeType = type;
  conf = new TajoConf();
  conf.setVar(TajoConf.ConfVars.ROOT_DIR, TEST_PATH);
  schema = new Schema();
  schema.addColumn(new Column("int", Type.INT4));
  schema.addColumn(new Column("long", Type.INT8));
  schema.addColumn(new Column("double", Type.FLOAT8));
  schema.addColumn(new Column("float", Type.FLOAT4));
  schema.addColumn(new Column("string", Type.TEXT));
}
 

import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; //导入依赖的package包/类
/**
 * Sort a tuple block and store them into a chunk file
 */
private Path sortAndStoreChunk(int chunkId, List<Tuple> tupleBlock)
    throws IOException {
  TableMeta meta = CatalogUtil.newTableMeta(StoreType.RAW);
  int rowNum = tupleBlock.size();

  long sortStart = System.currentTimeMillis();
  Collections.sort(tupleBlock, getComparator());
  long sortEnd = System.currentTimeMillis();

  long chunkWriteStart = System.currentTimeMillis();
  Path outputPath = getChunkPathForWrite(0, chunkId);
  final RawFileAppender appender = new RawFileAppender(context.getConf(), inSchema, meta, outputPath);
  appender.init();
  for (Tuple t : tupleBlock) {
    appender.addTuple(t);
  }
  appender.close();
  tupleBlock.clear();
  long chunkWriteEnd = System.currentTimeMillis();


  info(LOG, "Chunk #" + chunkId + " sort and written (" +
      FileUtil.humanReadableByteCount(appender.getOffset(), false) + " bytes, " + rowNum + " rows, " +
      ", sort time: " + (sortEnd - sortStart) + " msec, " +
      "write time: " + (chunkWriteEnd - chunkWriteStart) + " msec)");
  return outputPath;
}
 

import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; //导入依赖的package包/类
@BeforeClass
public static void setup() throws Exception {
  util = TpchTestBase.getInstance().getTestingCluster();
  conf = util.getConfiguration();
  sm = StorageManagerFactory.getStorageManager(conf);

  scoreSchema = new Schema();
  scoreSchema.addColumn("deptname", Type.TEXT);
  scoreSchema.addColumn("score", Type.INT4);
  scoreMeta = CatalogUtil.newTableMeta(StoreType.CSV);
  TableStats stats = new TableStats();

  Path p = sm.getTablePath("score");
  sm.getFileSystem().mkdirs(p);
  Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(scoreMeta, scoreSchema,
      new Path(p, "score"));
  appender.init();
  int deptSize = 100;
  int tupleNum = 10000;
  Tuple tuple;
  long written = 0;
  for (int i = 0; i < tupleNum; i++) {
    tuple = new VTuple(2);
    String key = "test" + (i % deptSize);
    tuple.put(0, DatumFactory.createText(key));
    tuple.put(1, DatumFactory.createInt4(i + 1));
    written += key.length() + Integer.SIZE;
    appender.addTuple(tuple);
  }
  appender.close();
  stats.setNumRows(tupleNum);
  stats.setNumBytes(written);
  stats.setAvgRows(tupleNum);
  stats.setNumBlocks(1000);
  stats.setNumShuffleOutputs(100);
  desc = new TableDesc("score", scoreSchema, scoreMeta, p);
  desc.setStats(stats);
}
 

import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; //导入依赖的package包/类
@BeforeClass
public static void setUp() throws Exception {
  util = new TajoTestingCluster();
  util.startCatalogCluster();
  catalog = util.getMiniCatalogCluster().getCatalog();

  Schema schema = new Schema();
  schema.addColumn("name", Type.TEXT);
  schema.addColumn("empId", CatalogUtil.newSimpleDataType(Type.INT4));
  schema.addColumn("deptName", Type.TEXT);

  Schema schema2 = new Schema();
  schema2.addColumn("deptName", Type.TEXT);
  schema2.addColumn("manager", Type.TEXT);

  Schema schema3 = new Schema();
  schema3.addColumn("deptName", Type.TEXT);
  schema3.addColumn("score", CatalogUtil.newSimpleDataType(Type.INT4));

  TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
  TableDesc people = new TableDesc("employee", schema, meta, CommonTestingUtil.getTestDir());
  catalog.addTable(people);

  TableDesc student = new TableDesc("dept", schema2, StoreType.CSV, new Options(), CommonTestingUtil.getTestDir());
  catalog.addTable(student);

  TableDesc score = new TableDesc("score", schema3, StoreType.CSV, new Options(), CommonTestingUtil.getTestDir());
  catalog.addTable(score);

  FunctionDesc funcDesc = new FunctionDesc("sumtest", SumInt.class, FunctionType.AGGREGATION,
      CatalogUtil.newSimpleDataType(Type.INT4),
      CatalogUtil.newSimpleDataTypeArray(Type.INT4));

  catalog.createFunction(funcDesc);
  analyzer = new SQLAnalyzer();
  planner = new LogicalPlanner(catalog);
}
 

import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; //导入依赖的package包/类
@Test
public void testEquals() {
  Schema schema = new Schema();
  schema.addColumn("id", Type.INT4);
  schema.addColumn("name", Type.TEXT);
  schema.addColumn("age", Type.INT2);
  GroupbyNode groupbyNode = new GroupbyNode(0);
  groupbyNode.setGroupingColumns(new Column[]{schema.getColumn(1), schema.getColumn(2)});
  ScanNode scanNode = new ScanNode(0);
  scanNode.init(CatalogUtil.newTableDesc("in", schema, CatalogUtil.newTableMeta(StoreType.CSV), new Path("in")));

  GroupbyNode groupbyNode2 = new GroupbyNode(0);
  groupbyNode2.setGroupingColumns(new Column[]{schema.getColumn(1), schema.getColumn(2)});
  JoinNode joinNode = new JoinNode(0);
  ScanNode scanNode2 = new ScanNode(0);
  scanNode2.init(CatalogUtil.newTableDesc("in2", schema, CatalogUtil.newTableMeta(StoreType.CSV), new Path("in2")));

  groupbyNode.setChild(scanNode);
  groupbyNode2.setChild(joinNode);
  joinNode.setLeftChild(scanNode);
  joinNode.setRightChild(scanNode2);

  assertTrue(groupbyNode.equals(groupbyNode2));
  assertFalse(groupbyNode.deepEquals(groupbyNode2));

  ScanNode scanNode3 = new ScanNode(0);
  scanNode3.init(CatalogUtil.newTableDesc("in", schema, CatalogUtil.newTableMeta(StoreType.CSV), new Path("in")));
  groupbyNode2.setChild(scanNode3);

  assertTrue(groupbyNode.equals(groupbyNode2));
  assertTrue(groupbyNode.deepEquals(groupbyNode2));
}