Java FileNameFormat类代码示例

import org.apache.storm.hdfs.bolt.format.FileNameFormat; //导入依赖的package包/类
/**
 * Create bolt which will persist ticks to HDFS.
 */
private static HdfsBolt createHdfsBolt() {

  // Use "|" instead of "," for field delimiter:
  RecordFormat format = new DelimitedRecordFormat()
    .withFieldDelimiter("|");
  // sync the filesystem after every 1k tuples:
  SyncPolicy syncPolicy = new CountSyncPolicy(100);

  // Rotate files when they reach 5MB:
  FileRotationPolicy rotationPolicy = 
    new FileSizeRotationPolicy(5.0f, Units.MB);

  // Write records to <user>/stock-ticks/ directory in HDFS:
  FileNameFormat fileNameFormat = new DefaultFileNameFormat()
    .withPath("stock-ticks/");

  HdfsBolt hdfsBolt = new HdfsBolt()
    .withFsUrl("hdfs://localhost:8020")
    .withFileNameFormat(fileNameFormat)
    .withRecordFormat(format)
    .withRotationPolicy(rotationPolicy)
    .withSyncPolicy(syncPolicy);

  return hdfsBolt;
}
 

import org.apache.storm.hdfs.bolt.format.FileNameFormat; //导入依赖的package包/类
private static HdfsBolt createHdfsBolt() {
    // use "|" instead of "," for field delimiter
    RecordFormat format = new DelimitedRecordFormat()
            .withFieldDelimiter("|");

    // sync the filesystem after every 1k tuples
    SyncPolicy syncPolicy = new CountSyncPolicy(1000);

    // rotate files when they reach 5MB
    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB);

    FileNameFormat fileNameFormat = new DefaultFileNameFormat()
            .withPath(Properties.getString("sa.storm.hdfs_output_file"));

    return new HdfsBolt()
            .withFsUrl(Properties.getString("sa.storm.hdfs_url"))
            .withFileNameFormat(fileNameFormat)
            .withRecordFormat(format)
            .withRotationPolicy(rotationPolicy)
            .withSyncPolicy(syncPolicy);
}
 

import org.apache.storm.hdfs.bolt.format.FileNameFormat; //导入依赖的package包/类
public HdfsBolt getHdfsBolt() {

        LOG.info("HDFSBOLT: Configuring the HdfsBolt");

        // Define the RecordFormat, SyncPolicy, and FileNameFormat
        RecordFormat format = new DelimitedRecordFormat().withFieldDelimiter(fieldDelimiter);
        SyncPolicy syncPolicy = new CountSyncPolicy(syncCount);
        FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(outputLocation);

        // Configure the Bolt
        return new HdfsBolt()
                .withFsUrl(hdfsDefaultFs)
                .withFileNameFormat(fileNameFormat)
                .withRecordFormat(format)
                .withRotationPolicy(fileRotationPolicy)
                .withSyncPolicy(syncPolicy);

    }
 

import org.apache.storm.hdfs.bolt.format.FileNameFormat; //导入依赖的package包/类
public static HdfsBolt getHdfsBolt(String fsUrl, String srcDir, String rotationDir) {
    // sync the filesystem after every tuple
    SyncPolicy syncPolicy = new CountSyncPolicy(1);

    FileNameFormat fileNameFormat = new DefaultFileNameFormat()
            .withPath(srcDir)
            .withExtension(".txt");

    RecordFormat format = new DelimitedRecordFormat().withFieldDelimiter(",");
    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(1f, FileSizeRotationPolicy.Units.KB);

    HdfsBolt bolt = new HdfsBolt()
            .withFsUrl(fsUrl)
            .withFileNameFormat(fileNameFormat)
            .withRecordFormat(format)
            .withSyncPolicy(syncPolicy)
            .withRotationPolicy(rotationPolicy)
            .addRotationAction(new MoveFileAction().toDestination(rotationDir));

    return bolt;
}
 

import org.apache.storm.hdfs.bolt.format.FileNameFormat; //导入依赖的package包/类
public static void configureHdfsBolt(TopologyBuilder builder, 
                                     String delimiter, 
                                     String outputPath, 
                                     String hdfsUri,
                                     String hdfsBoltName, 
                                     String spoutName,
                                     int parallelismHint,
                                     FileRotationPolicy rotationPolicy,
                                     int syncCount) {
    
    LOG.info("HDFSBOLT: Configuring the HdfsBolt");
    
    // Define the RecordFormat, SyncPolicy, and FileNameFormat
    RecordFormat format = new DelimitedRecordFormat().withFieldDelimiter(delimiter);
    SyncPolicy syncPolicy = new CountSyncPolicy(syncCount);
    FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(outputPath);
    
    // Configure the Bolt
    HdfsBolt bolt = new HdfsBolt()
            .withFsUrl(hdfsUri)
            .withFileNameFormat(fileNameFormat)
            .withRecordFormat(format)
            .withRotationPolicy(rotationPolicy)
            .withSyncPolicy(syncPolicy);
    
    // Set the Bolt
    builder.setBolt(hdfsBoltName, bolt, parallelismHint).shuffleGrouping(spoutName);

}
 

import org.apache.storm.hdfs.bolt.format.FileNameFormat; //导入依赖的package包/类
public SourceHandler(List<RotationAction> rotationActions
                    , FileRotationPolicy rotationPolicy
                    , SyncPolicy syncPolicy
                    , FileNameFormat fileNameFormat
                    , SourceHandlerCallback cleanupCallback) throws IOException {
  this.rotationActions = rotationActions;
  this.rotationPolicy = rotationPolicy;
  this.syncPolicy = syncPolicy;
  this.fileNameFormat = fileNameFormat;
  this.cleanupCallback = cleanupCallback;
  initialize();
}
 

import org.apache.storm.hdfs.bolt.format.FileNameFormat; //导入依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void testWriteSingleFileWithNull() throws Exception {
  String function = "FORMAT('test-%s/%s', test.key, test.key)";
  WriterConfiguration config = buildWriterConfiguration(function);
  FileNameFormat format = new DefaultFileNameFormat()
          .withPath(folder.toString())
          .withExtension(".json")
          .withPrefix("prefix-");
  HdfsWriter writer = new HdfsWriter().withFileNameFormat(format);
  writer.init(new HashMap<String, String>(), createTopologyContext(), config);

  // These two messages will be routed to the same folder, because test.key is the same
  JSONObject message = new JSONObject();
  message.put("test.key2", "test.value2");
  ArrayList<JSONObject> messages = new ArrayList<>();
  messages.add(message);

  ArrayList<Tuple> tuples = new ArrayList<>();

  writer.write(SENSOR_NAME, config, tuples, messages);
  writer.close();

  ArrayList<String> expected = new ArrayList<>();
  expected.add(message.toJSONString());
  Collections.sort(expected);

  File outputFolder = new File(folder.getAbsolutePath() + "/test-null/null/");
  Assert.assertTrue(outputFolder.exists() && outputFolder.isDirectory());
  Assert.assertEquals(1, outputFolder.listFiles().length);

  for(File file : outputFolder.listFiles()) {
    List<String> lines = Files.readAllLines(file.toPath());
    Collections.sort(lines);
    Assert.assertEquals(expected, lines);
  }
}
 

import org.apache.storm.hdfs.bolt.format.FileNameFormat; //导入依赖的package包/类
@Test
public void testGetPath() {
  FileNameFormat delegate = new DefaultFileNameFormat().withExtension(EXTENSION).withPath(PATH);
  FileNameFormat sourceFormat = new PathExtensionFileNameFormat(PATH_EXTENSION, delegate);
  String actual = sourceFormat.getPath();
  String expected = PATH + "/" + PATH_EXTENSION;
  Assert.assertEquals(expected, actual);
}
 

import org.apache.storm.hdfs.bolt.format.FileNameFormat; //导入依赖的package包/类
@Test
public void testGetPathEmptyPathExtension() {
  FileNameFormat delegate = new DefaultFileNameFormat().withExtension(EXTENSION).withPath(PATH);
  FileNameFormat sourceFormat = new PathExtensionFileNameFormat("", delegate);
  String actual = sourceFormat.getPath();
  Assert.assertEquals(PATH + "/", actual);
}
 

import org.apache.storm.hdfs.bolt.format.FileNameFormat; //导入依赖的package包/类
public static void configureHdfsBolt(TopologyBuilder builder, String delimiter, String outputPath, String hdfsUri) {
    RecordFormat format = new DelimitedRecordFormat().withFieldDelimiter(delimiter);
    SyncPolicy syncPolicy = new CountSyncPolicy(1000);
    //FileRotationPolicy rotationPolicy = new TimedRotationPolicy(300, TimedRotationPolicy.TimeUnit.SECONDS);
    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(1, FileSizeRotationPolicy.Units.KB);
    FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(outputPath);
    HdfsBolt bolt = new HdfsBolt()
            .withFsUrl(hdfsUri)
            .withFileNameFormat(fileNameFormat)
            .withRecordFormat(format)
            .withRotationPolicy(rotationPolicy)
            .withSyncPolicy(syncPolicy);
    builder.setBolt("hdfsbolt", bolt, 1).shuffleGrouping("kafkaspout");

}
 

import org.apache.storm.hdfs.bolt.format.FileNameFormat; //导入依赖的package包/类
public static void main(String[] args) {
    try{
        String zkhost = "wxb-1:2181,wxb-2:2181,wxb-3:2181";
        String topic = "order";
        String groupId = "id";
        int spoutNum = 3;
        int boltNum = 1;
        ZkHosts zkHosts = new ZkHosts(zkhost);//kafaka所在的zookeeper
        SpoutConfig spoutConfig = new SpoutConfig(zkHosts, topic, "/order", groupId);  // create /order /id
        spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
        KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig);

        // HDFS bolt
        // use "|" instead of "," for field delimiter
        RecordFormat format = new DelimitedRecordFormat()
                .withFieldDelimiter("|");

        // sync the filesystem after every 1k tuples
        SyncPolicy syncPolicy = new CountSyncPolicy(1000);

        // rotate files when they reach 5MB
        FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB);
        // FileRotationPolicy rotationPolicy = new TimedRotationPolicy(1.0f, TimedRotationPolicy.TimeUnit.MINUTES);

        FileNameFormat fileNameFormat = new DefaultFileNameFormat()
                .withPath("/tmp/").withPrefix("order_").withExtension(".log");

        HdfsBolt hdfsBolt = new HdfsBolt()
                .withFsUrl("hdfs://wxb-1:8020")
                .withFileNameFormat(fileNameFormat)
                .withRecordFormat(format)
                .withRotationPolicy(rotationPolicy)
                .withSyncPolicy(syncPolicy);

        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("spout", kafkaSpout, spoutNum);
        builder.setBolt("check", new CheckOrderBolt(), boltNum).shuffleGrouping("spout");
        builder.setBolt("counter", new CounterBolt(),boltNum).shuffleGrouping("check");
        builder.setBolt("hdfs", hdfsBolt,boltNum).shuffleGrouping("counter");

        Config config = new Config();
        config.setDebug(true);

        if(args!=null && args.length > 0) {
            config.setNumWorkers(2);
            StormSubmitter.submitTopology(args[0], config, builder.createTopology());
        } else {
            config.setMaxTaskParallelism(2);

            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology("Wordcount-Topology", config, builder.createTopology());

            Thread.sleep(500000);

            cluster.shutdown();
        }
    }catch (Exception e) {
        e.printStackTrace();
    }
}
 

import org.apache.storm.hdfs.bolt.format.FileNameFormat; //导入依赖的package包/类
public void configureHDFSBolt(TopologyBuilder builder) {
  // Use pipe as record boundary

  String rootPath = topologyConfig.getProperty("hdfs.path");
  String prefix = topologyConfig.getProperty("hdfs.file.prefix");
  String fsUrl = topologyConfig.getProperty("hdfs.url");
  String sourceMetastoreUrl = topologyConfig.getProperty("hive.metastore.url");
  String hiveStagingTableName = topologyConfig.getProperty("hive.staging.table.name");
  String databaseName = topologyConfig.getProperty("hive.database.name");
  Float rotationTimeInMinutes = Float.valueOf(topologyConfig.getProperty("hdfs.file.rotation.time.minutes"));

  RecordFormat format = new DelimitedRecordFormat().withFieldDelimiter(",");

  //Synchronize data buffer with the filesystem every 1000 tuples
  SyncPolicy syncPolicy = new CountSyncPolicy(1000);

  // Rotate data files when they reach five MB
  //FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, Units.MB);

  //Rotate every X minutes
  FileTimeRotationPolicy rotationPolicy = new FileTimeRotationPolicy(rotationTimeInMinutes, FileTimeRotationPolicy
      .Units.MINUTES);

  //Hive Partition Action
  HiveTablePartitionAction hivePartitionAction = new HiveTablePartitionAction(sourceMetastoreUrl,
      hiveStagingTableName, databaseName, fsUrl);

  //MoveFileAction moveFileAction = new MoveFileAction().toDestination(rootPath + "/working");


  FileNameFormat fileNameFormat = new DefaultFileNameFormat()
      .withPath(rootPath + "/staging")
      .withPrefix(prefix);

  // Instantiate the HdfsBolt
  HdfsBolt hdfsBolt = new HdfsBolt()
      .withFsUrl(fsUrl)
      .withFileNameFormat(fileNameFormat)
      .withRecordFormat(format)
      .withRotationPolicy(rotationPolicy)
      .withSyncPolicy(syncPolicy)
      .addRotationAction(hivePartitionAction);

  int hdfsBoltCount = Integer.valueOf(topologyConfig.getProperty("hdfsbolt.thread.count"));
  builder.setBolt("hdfs_bolt", hdfsBolt, hdfsBoltCount).shuffleGrouping("kafkaSpout");
}
 

import org.apache.storm.hdfs.bolt.format.FileNameFormat; //导入依赖的package包/类
public PathExtensionFileNameFormat(String pathExtension, FileNameFormat delegate) {
  this.delegate = delegate;
  this.pathExtension = pathExtension;
}
 

import org.apache.storm.hdfs.bolt.format.FileNameFormat; //导入依赖的package包/类
public HdfsWriter withFileNameFormat(FileNameFormat fileNameFormat){
  this.fileNameFormat = fileNameFormat;
  return this;
}
 

import org.apache.storm.hdfs.bolt.format.FileNameFormat; //导入依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void testWriteNoOutputFunction() throws Exception {
  FileNameFormat format = new DefaultFileNameFormat()
          .withPath(folder.toString())
          .withExtension(".json")
          .withPrefix("prefix-");
  HdfsWriter writer = new HdfsWriter().withFileNameFormat(format);
  IndexingConfigurations indexingConfig = new IndexingConfigurations();
  WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, indexingConfig);
  writer.init(new HashMap<String, String>(), createTopologyContext(), config);

  JSONObject message = new JSONObject();
  message.put("test.key", "test.value");
  message.put("test.key2", "test.value2");
  JSONObject message2 = new JSONObject();
  message2.put("test.key", "test.value3");
  message2.put("test.key2", "test.value2");
  ArrayList<JSONObject> messages = new ArrayList<>();
  messages.add(message);
  messages.add(message2);

  ArrayList<Tuple> tuples = new ArrayList<>();

  writer.write(SENSOR_NAME, config, tuples, messages);
  writer.close();

  ArrayList<String> expected = new ArrayList<>();
  expected.add(message.toJSONString());
  expected.add(message2.toJSONString());
  Collections.sort(expected);

  // Default to just putting it in the base folder + the sensor name
  File outputFolder = new File(folder.getAbsolutePath() + "/" + SENSOR_NAME);
  Assert.assertTrue(outputFolder.exists() && outputFolder.isDirectory());
  Assert.assertEquals(1, outputFolder.listFiles().length);

  for(File file : outputFolder.listFiles()) {
    List<String> lines = Files.readAllLines(file.toPath());
    Collections.sort(lines);
    Assert.assertEquals(expected, lines);
  }
}