本文整理汇总了Java中org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy.Units.MB属性的典型用法代码示例。如果您正苦于以下问题:Java Units.MB属性的具体用法?Java Units.MB怎么用?Java Units.MB使用的例子?那么恭喜您, 这里精选的属性代码示例或许可以为您提供帮助。您也可以进一步了解该属性所在类org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy.Units
的用法示例。
在下文中一共展示了Units.MB属性的3个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: createHdfsBolt
/**
* Create bolt which will persist ticks to HDFS.
*/
private static HdfsBolt createHdfsBolt() {
// Use "|" instead of "," for field delimiter:
RecordFormat format = new DelimitedRecordFormat()
.withFieldDelimiter("|");
// sync the filesystem after every 1k tuples:
SyncPolicy syncPolicy = new CountSyncPolicy(100);
// Rotate files when they reach 5MB:
FileRotationPolicy rotationPolicy =
new FileSizeRotationPolicy(5.0f, Units.MB);
// Write records to <user>/stock-ticks/ directory in HDFS:
FileNameFormat fileNameFormat = new DefaultFileNameFormat()
.withPath("stock-ticks/");
HdfsBolt hdfsBolt = new HdfsBolt()
.withFsUrl("hdfs://localhost:8020")
.withFileNameFormat(fileNameFormat)
.withRecordFormat(format)
.withRotationPolicy(rotationPolicy)
.withSyncPolicy(syncPolicy);
return hdfsBolt;
}
示例2: initializeHDFSBolt
private boolean initializeHDFSBolt(String topology_name, String name) {
try {
String messageUpstreamComponent = messageComponents
.get(messageComponents.size() - 1);
System.out.println("[OpenSOC] ------" + name
+ " is initializing from " + messageUpstreamComponent);
RecordFormat format = new DelimitedRecordFormat()
.withFieldDelimiter(
config.getString("bolt.hdfs.field.delimiter")
.toString()).withFields(
new Fields("message"));
// sync the file system after every x number of tuples
SyncPolicy syncPolicy = new CountSyncPolicy(Integer.valueOf(config
.getString("bolt.hdfs.batch.size").toString()));
// rotate files when they reach certain size
FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(
Float.valueOf(config.getString(
"bolt.hdfs.file.rotation.size.in.mb").toString()),
Units.MB);
FileNameFormat fileNameFormat = new DefaultFileNameFormat()
.withPath(config.getString("bolt.hdfs.wip.file.path")
.toString());
// Post rotate action
MoveFileAction moveFileAction = (new MoveFileAction())
.toDestination(config.getString(
"bolt.hdfs.finished.file.path").toString());
HdfsBolt hdfsBolt = new HdfsBolt()
.withFsUrl(
config.getString("bolt.hdfs.file.system.url")
.toString())
.withFileNameFormat(fileNameFormat)
.withRecordFormat(format)
.withRotationPolicy(rotationPolicy)
.withSyncPolicy(syncPolicy)
.addRotationAction(moveFileAction);
if (config.getString("bolt.hdfs.compression.codec.class") != null) {
hdfsBolt.withCompressionCodec(config.getString(
"bolt.hdfs.compression.codec.class").toString());
}
builder.setBolt(name, hdfsBolt,
config.getInt("bolt.hdfs.parallelism.hint"))
.shuffleGrouping(messageUpstreamComponent, "message")
.setNumTasks(config.getInt("bolt.hdfs.num.tasks"));
} catch (Exception e) {
e.printStackTrace();
System.exit(0);
}
return true;
}
示例3: main
public static void main(String[] args) throws Exception {
Config config = new Config();
config.setNumWorkers(1);
SentenceSpout spout = new SentenceSpout();
// sync the filesystem after every 1k tuples
SyncPolicy syncPolicy = new CountSyncPolicy(1000);
// rotate files when they reach 5MB
FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, Units.MB);
FileNameFormat fileNameFormat = new DefaultFileNameFormat()
.withPath("/source/")
.withExtension(".seq");
// create sequence format instance.
DefaultSequenceFormat format = new DefaultSequenceFormat("timestamp", "sentence");
SequenceFileBolt bolt = new SequenceFileBolt()
.withFsUrl(args[0])
.withFileNameFormat(fileNameFormat)
.withSequenceFormat(format)
.withRotationPolicy(rotationPolicy)
.withSyncPolicy(syncPolicy)
.withCompressionType(SequenceFile.CompressionType.RECORD)
.withCompressionCodec("deflate")
.addRotationAction(new MoveFileAction().toDestination("/dest/"));
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout(SENTENCE_SPOUT_ID, spout, 1);
// SentenceSpout --> MyBolt
builder.setBolt(BOLT_ID, bolt, 4)
.shuffleGrouping(SENTENCE_SPOUT_ID);
if (args.length == 1) {
LocalCluster cluster = new LocalCluster();
cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology());
waitForSeconds(120);
cluster.killTopology(TOPOLOGY_NAME);
cluster.shutdown();
System.exit(0);
} else if(args.length == 2) {
StormSubmitter.submitTopology(args[1], config, builder.createTopology());
}
}