Java CQLSSTableWriter类代码示例

import org.apache.cassandra.io.sstable.CQLSSTableWriter; //导入依赖的package包/类
/**
 * The column values must correspond to the order in which
 * they appear in the insert stored procedure. 
 * 
 * Key is not used, so it can be null or any object.
 * </p>
 *
 * @param key
 *            any object or null.
 * @param values
 *            the values to write.
 * @throws IOException
 */
@Override
public void write(Object key, List<ByteBuffer> values) throws IOException
{
    prepareWriter();
    try
    {
        ((CQLSSTableWriter) writer).rawAddRow(values);
        
        if (null != progress)
            progress.progress();
        if (null != context)
            HadoopCompat.progress(context);
    } 
    catch (InvalidRequestException e)
    {
        throw new IOException("Error adding row with key: " + key, e);
    }
}
 

import org.apache.cassandra.io.sstable.CQLSSTableWriter; //导入依赖的package包/类
/**
 * <p>
 * The column values must correspond to the order in which
 * they appear in the insert stored procedure. 
 * 
 * Key is not used, so it can be null or any object.
 * </p>
 *
 * @param key
 *            any object or null.
 * @param values
 *            the values to write.
 * @throws IOException
 */
@Override
public void write(Object key, List<ByteBuffer> values) throws IOException
{
    prepareWriter();
    try
    {
        ((CQLSSTableWriter) writer).rawAddRow(values);
        
        if (null != progress)
            progress.progress();
        if (null != context)
            HadoopCompat.progress(context);
    } 
    catch (InvalidRequestException e)
    {
        throw new IOException("Error adding row with key: " + key, e);
    }
}
 

import org.apache.cassandra.io.sstable.CQLSSTableWriter; //导入依赖的package包/类
private void createSSTable(ColumnFamilyStore cfs, int numPartitions) throws IOException
{
    cfs.truncateBlocking();

    String schema = "CREATE TABLE \"%s\".\"%s\" (key ascii, name ascii, val ascii, val1 ascii, PRIMARY KEY (key, name))";
    String query = "INSERT INTO \"%s\".\"%s\" (key, name, val) VALUES (?, ?, ?)";

    try (CQLSSTableWriter writer = CQLSSTableWriter.builder()
                                                   .inDirectory(cfs.getDirectories().getDirectoryForNewSSTables())
                                                   .forTable(String.format(schema, cfs.keyspace.getName(), cfs.name))
                                                   .using(String.format(query, cfs.keyspace.getName(), cfs.name))
                                                   .build())
    {
        for (int j = 0; j < numPartitions; j ++)
            writer.addRow(String.format("key%d", j), "col1", "0");
    }

    cfs.loadNewSSTables();
}
 

import org.apache.cassandra.io.sstable.CQLSSTableWriter; //导入依赖的package包/类
/**
 * Builds an SSTableWriter for a static table
 * @param sorted Defines if the SSTableWriters created by this should be set as sorted, which improves performance if
 *               rows are inserted in SSTable sort order, but throws exceptions if they are inserted in the wrong order.
 * @return A CQLSSTableWriter for this static table
 * @throws CQLGenerationException
 * @throws IOException
 */
private CQLSSTableWriter buildSSTableWriterForShardIndexTable(boolean sorted) throws CQLGenerationException, IOException {
	// Generate CQL create syntax
	String createCQL = this.cqlGenerator.makeCQLforShardIndexTableCreate().getQuery();

	// Generate CQL insert syntax
	String tableName = CObjectShardList.SHARD_INDEX_TABLE_NAME;
	String insertCQL = this.cqlGenerator.makeCQLforInsertNoValuesforShardIndex(tableName).getQuery();

	String SSTablePath = this.defaultSSTableOutputPath + "/" + keyspaceDefinition.getName() + "/" + tableName;
	if (!new File(SSTablePath).mkdir()) {
		throw new IOException("Failed to create new directory for SSTable writing at path: " + SSTablePath);
	}

	CQLSSTableWriter.Builder builder =
			CQLSSTableWriter.builder()
					.inDirectory(SSTablePath)
					.forTable(createCQL)
					.using(insertCQL);
	if (sorted) { builder = builder.sorted(); }
	return builder.build();
}
 

import org.apache.cassandra.io.sstable.CQLSSTableWriter; //导入依赖的package包/类
/**
 * Builds an SSTableWriter for a static table
 * @param definition Definition of object to build table for
 * @param sorted Defines if the SSTableWriters created by this should be set as sorted, which improves performance if
 *               rows are inserted in SSTable sort order, but throws exceptions if they are inserted in the wrong order.
 * @return A CQLSSTableWriter for this static table
 * @throws CQLGenerationException
 * @throws IOException
 */
private CQLSSTableWriter buildSSTableWriterForStaticTable(CDefinition definition, boolean sorted) throws CQLGenerationException, IOException {
	// Generate CQL create syntax
	String tableName = definition.getName();
	String createCQL = this.cqlGenerator.makeStaticTableCreate(definition).getQuery();

	// Generate CQL insert syntax
	String insertCQL = this.cqlGenerator.makeCQLforInsertNoValuesforStaticTable(tableName).getQuery();

	String SSTablePath = this.defaultSSTableOutputPath + "/" + keyspaceDefinition.getName() + "/" + tableName;
	if (!new File(SSTablePath).mkdir()) {
		throw new IOException("Failed to create new directory for SSTable writing at path: " + SSTablePath);
	}

	CQLSSTableWriter.Builder builder =
		CQLSSTableWriter.builder()
			.inDirectory(SSTablePath)
			.forTable(createCQL)
			.using(insertCQL);
	if (sorted) { builder = builder.sorted(); }
	return builder.build();
}
 

import org.apache.cassandra.io.sstable.CQLSSTableWriter; //导入依赖的package包/类
/**
 * Builds an SSTableWriter for a wide/index table
 * @param definition The definition this index/wide table is on
 * @param index The index/wide table to create an CQLSSTableWriter for
 * @param sorted Defines if the SSTableWriters created by this should be set as sorted, which improves performance if
 *               rows are inserted in SSTable sort order, but throws exceptions if they are inserted in the wrong order.
 * @return An CQLSSTableWriter for this wide table
 * @throws CQLGenerationException
 */
private CQLSSTableWriter buildSSTableWriterForWideTable(CDefinition definition, CIndex index, boolean sorted) throws CQLGenerationException, IOException {
	String indexTableName = CObjectCQLGenerator.makeTableName(definition, index);
	// Generate CQL create syntax
	String createCQL = this.cqlGenerator.makeWideTableCreate(definition, index).getQuery();

	// Generate CQL insert syntax
	// Just use 1 as the value for shardId, doesn't matter since we're not actually using values here
	String insertCQL = this.cqlGenerator.makeCQLforInsertNoValuesforWideTable(definition, indexTableName, 1L).getQuery();

	String SSTablePath = this.defaultSSTableOutputPath + "/" + keyspaceDefinition.getName() + "/" + indexTableName;
	if (!new File(SSTablePath).mkdir()) {
		throw new IOException("Failed to create new directory for SSTable writing at path: " + SSTablePath);
	}

	CQLSSTableWriter.Builder builder =
			CQLSSTableWriter.builder()
					.inDirectory(SSTablePath)
					.forTable(createCQL)
					.using(insertCQL);
	if (sorted) { builder = builder.sorted(); }
	return builder.build();
}
 

import org.apache.cassandra.io.sstable.CQLSSTableWriter; //导入依赖的package包/类
/**
 * Completes writes to SSTables and cleans up empty table directories. Must be called after writing to SSTables if
 * you actually want to use the SSTables for anything.
 * @throws IOException
 */
public void completeSSTableWrites() throws IOException {
	Map<String, CDefinition> definitions = this.keyspaceDefinition.getDefinitions();
	for (String tableName : this.SSTableWriters.keySet()) {
		// Close all SSTableWriters
		this.SSTableWriters.get(tableName).left.close();
		this.clearSSTableDirectoryIfEmpty(tableName);

		Map<CIndex, CQLSSTableWriter> indexWriters = this.SSTableWriters.get(tableName).right;
		if (indexWriters != null) {
			for (CQLSSTableWriter indexWriter : indexWriters.values()) {
				indexWriter.close();
			}
			// Clear out empty SSTable directories that haven't been written to
			CDefinition def = definitions.get(tableName);
			List<CIndex> indexes = def.getIndexesAsList();
			for (CIndex index : indexes) {
				this.clearSSTableDirectoryIfEmpty(CObjectCQLGenerator.makeTableName(def, index));
			}
		}
	}
}
 

import org.apache.cassandra.io.sstable.CQLSSTableWriter; //导入依赖的package包/类
private void prepareWriter()  {
  try {
    if (writer == null) {
      writer = CQLSSTableWriter.builder()
          .forTable(schema)
          .using(insertStatement)
          .withPartitioner(ConfigHelper.getOutputPartitioner(conf))
          .inDirectory(outputDir)
          .sorted()
          .build();
    }
    if (loader == null) {
      CrunchExternalClient externalClient = new CrunchExternalClient(conf);
      externalClient.addKnownCfs(keyspace, schema);
      this.loader = new SSTableLoader(outputDir, externalClient,
          new BulkRecordWriter.NullOutputHandler());
    }
  } catch (Exception e) {
    throw new CrunchRuntimeException(e);
  }
}
 

import org.apache.cassandra.io.sstable.CQLSSTableWriter; //导入依赖的package包/类
@Override
public void write(final ByteBuffer ignoredKey, final CQLRecord record)  {
  prepareWriter();
  // To ensure Crunch doesn't reuse CQLSSTableWriter's objects
  List<ByteBuffer> bb = Lists.newArrayList();
  for (ByteBuffer v : record.getValues()) {
    bb.add(ByteBufferUtil.clone(v));
  }
  try {
    ((CQLSSTableWriter) writer).rawAddRow(bb);
    if (null != progress)
      progress.progress();
    if (null != context)
      HadoopCompat.progress(context);
  } catch (InvalidRequestException | IOException e) {
    LOG.error(e.getMessage());
    throw new CrunchRuntimeException("Error adding row : " + e.getMessage());
  }
}
 

import org.apache.cassandra.io.sstable.CQLSSTableWriter; //导入依赖的package包/类
private void prepareWriter() throws IOException
{
    try
    {
        if (writer == null)
        {
            writer = CQLSSTableWriter.builder()
                .forTable(schema)
                .using(insertStatement)
                .withPartitioner(ConfigHelper.getOutputPartitioner(conf))
                .inDirectory(outputDir)
                .withBufferSizeInMB(Integer.parseInt(conf.get(BUFFER_SIZE_IN_MB, "64")))
                .build();
        }
        if (loader == null)
        {
            ExternalClient externalClient = new ExternalClient(conf);
            
            externalClient.addKnownCfs(keyspace, schema);

            this.loader = new SSTableLoader(outputDir, externalClient, new BulkRecordWriter.NullOutputHandler()) {
                @Override
                public void onSuccess(StreamState finalState)
                {
                    if (deleteSrc)
                        FileUtils.deleteRecursive(outputDir);
                }
            };
        }
    }
    catch (Exception e)
    {
        throw new IOException(e);
    }      
}
 

import org.apache.cassandra.io.sstable.CQLSSTableWriter; //导入依赖的package包/类
private void prepareWriter() throws IOException
{
    try
    {
        if (writer == null)
        {
            writer = CQLSSTableWriter.builder()
                .forTable(schema)
                .using(insertStatement)
                .withPartitioner(ConfigHelper.getOutputPartitioner(conf))
                .inDirectory(outputDir)
                .withBufferSizeInMB(Integer.parseInt(conf.get(BUFFER_SIZE_IN_MB, "64")))
                .build();
        }
        if (loader == null)
        {
            ExternalClient externalClient = new ExternalClient(conf);
            
            externalClient.addKnownCfs(keyspace, schema);

            this.loader = new SSTableLoader(outputDir, externalClient, new BulkRecordWriter.NullOutputHandler());
        }
    }
    catch (Exception e)
    {
        throw new IOException(e);
    }      
}
 

import org.apache.cassandra.io.sstable.CQLSSTableWriter; //导入依赖的package包/类
private CQLSSTableWriter createWriter() {
        String schema = "CREATE TABLE " + keyspace +  ".data ( " +
                "tenant_id text, " +
                "type tinyint, " +
                "metric text, " +
                "dpart bigint, " +
                "time timeuuid, " +
                "data_retention int static, " +
                "n_value double, " +
                "availability blob, " +
                "l_value bigint, " +
                // Commenting out the aggregates column because there appears to be a bug in the C*
                // code that breaks CQLSSTableWriter when the schema includes a collection. Fortunately,
                // we are not using the column so it can safely be ignored.
//                "aggregates set<frozen <aggregate_data>>, " +
                "PRIMARY KEY ((tenant_id, type, metric, dpart), time) " +
                ") WITH CLUSTERING ORDER BY (time DESC)";

        String insertGauge = "INSERT INTO " + keyspace + ".data (tenant_id, type, metric, dpart, time, n_value) " +
                "VALUES (?, ?, ?, ?, ?, ?)";


        return CQLSSTableWriter.builder()
                .inDirectory(dataDir)
                .forTable(schema)
                .using(insertGauge)
                .withBufferSizeInMB(bufferSize)
                .build();
    }
 

import org.apache.cassandra.io.sstable.CQLSSTableWriter; //导入依赖的package包/类
/**
 * Writes a sample SSTable that can be used for running the example job {@link SSTableMRExample}
 *
 * @param args
 *            Args to be parsed
 * @throws Exception
 */
public static void main(String[] args) throws Exception {
    buildParametersFromArgs(args);

    IPartitioner partitioner = StorageService.getPartitioner();
    String schema = String.format("CREATE TABLE %s.%s (studentid 'LongType', "
                                                    + "eventid 'LongType',"
                                                    + "data 'BytesType', "
                                                    + "PRIMARY KEY (studentid, eventid))"
                     + " WITH COMPACT STORAGE", KEYSPACE_NAME, COLUMN_FAMILY_NAME);

    String insertStatement = String.format("INSERT INTO %s.%s (studentid, eventid, data) "
        + "VALUES (?, ?, ?)", KEYSPACE_NAME, COLUMN_FAMILY_NAME);

    CQLSSTableWriter tableWriter = CQLSSTableWriter.builder()
                                                    .inDirectory(tableDirectory)
                                                    .withPartitioner(partitioner)
                                                    .forTable(schema)
                                                    .using(insertStatement)
                                                    .build();

    for (int i = 0; i < numberOfStudents; i++) {
        for (int j = 0; j < eventsPerStudent; j++) {
            StudentEvent studentEvent =
                    RandomStudentEventGenerator.getRandomStudentEvent();

            ByteBuffer columnValue = ByteBuffer.wrap(
                    RandomStudentEventGenerator.serializeStudentEventData(
                            studentEvent.getData()));

            tableWriter.addRow(RandomStudentEventGenerator.getRandomId(),
                               studentEvent.getId(),
                               columnValue);
        }
    }

    tableWriter.close();
}
 

import org.apache.cassandra.io.sstable.CQLSSTableWriter; //导入依赖的package包/类
/**
 * Creates an SSTable keyspace output directory at defaultSSTableOutputPath and table output directories for each SSTable,
 * and initializes each SSTableWriter for each static and index table in this keyspace.
 * @param sorted Defines if the SSTableWriters created by this should be set as sorted, which improves performance if
 *               rows are inserted in SSTable sort order, but throws exceptions if they are inserted in the wrong order.
 * @throws CQLGenerationException
 * @throws IOException
 */
public void initializeSSTableWriters(boolean sorted) throws CQLGenerationException, IOException {
	Map<String, CDefinition> definitions = this.keyspaceDefinition.getDefinitions();

	// Make sure the SSTableOutput directory exists and is clear
	String keyspacePath = this.defaultSSTableOutputPath + "/" + this.keyspaceDefinition.getName();
	File keyspaceDir = new File(keyspacePath);
	if (keyspaceDir.exists()) {
		FileUtils.deleteRecursive(new File(keyspacePath));
	}
	if (!new File(keyspacePath).mkdir()) {
		throw new IOException("Failed to create SSTable keyspace output directory at " + keyspacePath);
	}

	this.SSTableWriters.put(CObjectShardList.SHARD_INDEX_TABLE_NAME, Pair.create(this.buildSSTableWriterForShardIndexTable(sorted), (Map<CIndex, CQLSSTableWriter>) null));

	for (String defName : definitions.keySet()) {
		// Build the CQLSSTableWriter for the static table
		CQLSSTableWriter staticWriter = buildSSTableWriterForStaticTable(definitions.get(defName), sorted);

		// Build the CQLSSTableWriter for all the index tables
		List<CIndex> indexes = definitions.get(defName).getIndexesAsList();
		Map<CIndex, CQLSSTableWriter> indexWriters = Maps.newHashMap();
		for (CIndex index : indexes) {
			CQLSSTableWriter writer = buildSSTableWriterForWideTable(definitions.get(defName), index, sorted);
			indexWriters.put(index, writer);
		}
		this.SSTableWriters.put(defName, Pair.create(staticWriter, indexWriters));
	}
}
 

import org.apache.cassandra.io.sstable.CQLSSTableWriter; //导入依赖的package包/类
public void run(CommandLine cmdLine) throws Exception {
    Stopwatch stopwatch = Stopwatch.createStarted();

    if (cmdLine.hasOption("h")) {
        printUsage();
        return;
    }

    keyspace = cmdLine.getOptionValue("keyspace", "hawkular_metrics");
    dataDir = new File(cmdLine.getOptionValue("data-dir", "./data"));
    dataDir.mkdirs();
    tenants = Integer.parseInt(cmdLine.getOptionValue("tenants", "100"));
    metricsPerTenant = Integer.parseInt(cmdLine.getOptionValue("metrics-per-tenant", "100"));

    ValueServer valueServer = new ValueServer();
    valueServer.setMu(100);
    valueServer.setMode(ValueServer.UNIFORM_MODE);

    String endValue = cmdLine.getOptionValue("end");
    if (endValue == null) {
        endTime = System.currentTimeMillis();
    } else {
        endTime = getDuration("end", endValue, startEndRegexp);
    }

    String startValue = cmdLine.getOptionValue("start");
    if (startValue == null) {
        startTime = endTime - TimeUnit.MILLISECONDS.convert(1, TimeUnit.HOURS);
    } else {
        startTime = endTime - getDuration("start", startValue, startEndRegexp);
    }

    String intervalValue = cmdLine.getOptionValue("interval");
    if (intervalValue == null) {
        interval = TimeUnit.MILLISECONDS.convert(1, TimeUnit.MINUTES);
    } else {
        interval = getDuration("interval", intervalValue, intervalRegexp);
    }

    bufferSize = Integer.parseInt(cmdLine.getOptionValue("buffer-size", "128"));

    CQLSSTableWriter writer = createWriter();
    long totalDataPoints = 0;

    long currentTime = startTime;
    while (currentTime <= endTime) {
        for (int i = 0; i < tenants; ++i) {
            for (int j = 0; j < metricsPerTenant; ++j) {
                UUID timeUUID = TimeUUIDUtils.getTimeUUID(currentTime);
                writer.addRow("TENANT-" + i, GAUGE.getCode(), "GAUGE-" + j, 0L, timeUUID, valueServer.getNext());
                ++totalDataPoints;
            }
        }
        currentTime += interval;
    }

    writer.close();
    stopwatch.stop();

    System.out.println("\n\nStart time: " + startTime);
    System.out.println("End time: " + endTime);
    System.out.println("Total duration: " + (endTime - startTime) + " ms");
    System.out.println("Interval: " + interval);
    System.out.println("Tenants: " + tenants);
    System.out.println("Metrics per tenant: " + metricsPerTenant);
    System.out.println("Total data points: " + totalDataPoints);
    System.out.println("Execution time: " + stopwatch.elapsed(TimeUnit.MILLISECONDS) + " ms");
}