Java Storage类代码示例

import io.confluent.connect.hdfs.storage.Storage; //导入依赖的package包/类
private static ArrayList<FileStatus> traverseImpl(Storage storage, Path path, PathFilter filter)
    throws IOException {
  if (!storage.exists(path.toString())) {
    return new ArrayList<>();
  }
  ArrayList<FileStatus> result = new ArrayList<>();
  FileStatus[] statuses = storage.listStatus(path.toString());
  for (FileStatus status : statuses) {
    if (status.isDirectory()) {
      result.addAll(traverseImpl(storage, status.getPath(), filter));
    } else {
      if (filter.accept(status.getPath())) {
        result.add(status);
      }
    }
  }
  return result;
}
 

import io.confluent.connect.hdfs.storage.Storage; //导入依赖的package包/类
private static ArrayList<FileStatus> getDirectoriesImpl(Storage storage, Path path)
    throws IOException {
  FileStatus[] statuses = storage.listStatus(path.toString());
  ArrayList<FileStatus> result = new ArrayList<>();
  for (FileStatus status : statuses) {
    if (status.isDirectory()) {
      int count = 0;
      FileStatus[] fileStatuses = storage.listStatus(status.getPath().toString());
      for (FileStatus fileStatus : fileStatuses) {
        if (fileStatus.isDirectory()) {
          result.addAll(getDirectoriesImpl(storage, fileStatus.getPath()));
        } else {
          count++;
        }
      }
      if (count == fileStatuses.length) {
        result.add(status);
      }
    }
  }
  return result;
}
 

import io.confluent.connect.hdfs.storage.Storage; //导入依赖的package包/类
@Test
public void testTruncate() throws Exception {
  Storage storage = new HdfsStorage(conf, url);
  TopicPartition tp = new TopicPartition("mytopic", 123);
  FSWAL wal = new FSWAL("/logs", tp, storage);
  wal.append("a", "b");
  assertTrue("WAL file should exist after append",
          storage.exists("/logs/mytopic/123/log"));
  wal.truncate();
  assertFalse("WAL file should not exist after truncate",
          storage.exists("/logs/mytopic/123/log"));
  assertTrue("Rotated WAL file should exist after truncate",
          storage.exists("/logs/mytopic/123/log.1"));
  wal.append("c", "d");
  assertTrue("WAL file should be recreated after truncate + append",
          storage.exists("/logs/mytopic/123/log"));
  assertTrue("Rotated WAL file should exist after truncate + append",
          storage.exists("/logs/mytopic/123/log.1"));
}
 

import io.confluent.connect.hdfs.storage.Storage; //导入依赖的package包/类
@Before
public void setUp() throws Exception {
  super.setUp();

  @SuppressWarnings("unchecked")
  Format format = ((Class<Format>) Class.forName(connectorConfig.getString(HdfsSinkConnectorConfig.FORMAT_CLASS_CONFIG))).newInstance();
  writerProvider = format.getRecordWriterProvider();
  schemaFileReader = format.getSchemaFileReader(avroData);
  extension = writerProvider.getExtension();
  @SuppressWarnings("unchecked")
  Class<? extends Storage> storageClass = (Class<? extends Storage>) Class
          .forName(connectorConfig.getString(HdfsSinkConnectorConfig.STORAGE_CLASS_CONFIG));
  storage = StorageFactory.createStorage(storageClass, conf, url);
  createTopicDir(url, topicsDir, TOPIC);
  createLogsDir(url, logsDir);
}
 

import io.confluent.connect.hdfs.storage.Storage; //导入依赖的package包/类
private void createWALs(Map<TopicPartition, List<String>> tempfiles,
                        Map<TopicPartition, List<String>> committedFiles) throws Exception {
  @SuppressWarnings("unchecked")
  Class<? extends Storage> storageClass = (Class<? extends Storage>)
      Class.forName(connectorConfig.getString(HdfsSinkConnectorConfig.STORAGE_CLASS_CONFIG));
  Storage storage = StorageFactory.createStorage(storageClass, conf, url);

  for (TopicPartition tp: tempfiles.keySet()) {
    WAL wal = storage.wal(logsDir, tp);
    List<String> tempList = tempfiles.get(tp);
    List<String> committedList = committedFiles.get(tp);
    wal.append(WAL.beginMarker, "");
    for (int i = 0; i < tempList.size(); ++i) {
      wal.append(tempList.get(i), committedList.get(i));
    }
    wal.append(WAL.endMarker, "");
    wal.close();
  }
}
 

import io.confluent.connect.hdfs.storage.Storage; //导入依赖的package包/类
@Override
public WAL wal(String topicsDir, TopicPartition topicPart) {
  try {
    Class<? extends WAL> walClass = (Class<? extends WAL>) Class
        .forName(config.getString(S3SinkConnectorConfig.WAL_CLASS_CONFIG));
    if (walClass.equals(DummyWAL.class)) {
      return new DummyWAL();
    }
    else {
      Constructor<? extends WAL> ctor = walClass.getConstructor(String.class, TopicPartition.class, Storage.class, HdfsSinkConnectorConfig.class);
      return ctor.newInstance(topicsDir, topicPart, this, config);
    }
  } catch (ClassNotFoundException | NoSuchMethodException | InvocationTargetException | MethodInvocationException | InstantiationException | IllegalAccessException e) {
    throw new ConnectException(e);
  }
}
 

import io.confluent.connect.hdfs.storage.Storage; //导入依赖的package包/类
@Before
public void setUp() throws Exception {
  super.setUp();

  @SuppressWarnings("unchecked")
  Format format = ((Class<Format>) Class.forName(connectorConfig.getString(HdfsSinkConnectorConfig.FORMAT_CLASS_CONFIG))).newInstance();
  writerProvider = format.getRecordWriterProvider();
  schemaFileReader = format.getSchemaFileReader(avroData);
  extension = writerProvider.getExtension();
  @SuppressWarnings("unchecked")
  Class<? extends Storage> storageClass = (Class<? extends Storage>) Class
          .forName(connectorConfig.getString(HdfsSinkConnectorConfig.STORAGE_CLASS_CONFIG));
  storage = StorageFactory.createStorage(storageClass, connectorConfig, conf, url);
  createTopicDir(url, topicsDir, TOPIC);
  createLogsDir(url, logsDir);
}
 

import io.confluent.connect.hdfs.storage.Storage; //导入依赖的package包/类
private void createWALs(Map<TopicPartition, List<String>> tempfiles,
                        Map<TopicPartition, List<String>> committedFiles) throws Exception {
  @SuppressWarnings("unchecked")
  Class<? extends Storage> storageClass = (Class<? extends Storage>)
      Class.forName(connectorConfig.getString(HdfsSinkConnectorConfig.STORAGE_CLASS_CONFIG));
  Storage storage = StorageFactory.createStorage(storageClass, connectorConfig, conf, url);

  for (TopicPartition tp: tempfiles.keySet()) {
    WAL wal = storage.wal(logsDir, tp);
    List<String> tempList = tempfiles.get(tp);
    List<String> committedList = committedFiles.get(tp);
    wal.append(WAL.beginMarker, "");
    for (int i = 0; i < tempList.size(); ++i) {
      wal.append(tempList.get(i), committedList.get(i));
    }
    wal.append(WAL.endMarker, "");
    wal.close();
  }
}
 

import io.confluent.connect.hdfs.storage.Storage; //导入依赖的package包/类
public FSWAL(String logsDir, TopicPartition topicPart, Storage storage)
    throws ConnectException {
  this.storage = storage;
  this.conf = storage.conf();
  String url = storage.url();
  logFile = FileUtils.logFileName(url, logsDir, topicPart);
}
 

import io.confluent.connect.hdfs.storage.Storage; //导入依赖的package包/类
public TopicPartitionWriter(
    TopicPartition tp,
    Storage storage,
    RecordWriterProvider writerProvider,
    Partitioner partitioner,
    HdfsSinkConnectorConfig connectorConfig,
    SinkTaskContext context,
    AvroData avroData) {
  this(tp, storage, writerProvider, partitioner, connectorConfig, context, avroData, null, null, null, null, null);
}
 

import io.confluent.connect.hdfs.storage.Storage; //导入依赖的package包/类
public Storage getStorage() {
  return storage;
}
 

import io.confluent.connect.hdfs.storage.Storage; //导入依赖的package包/类
public static FileStatus[] traverse(Storage storage, Path path, PathFilter filter)
    throws IOException {
  ArrayList<FileStatus> result = traverseImpl(storage, path, filter);
  return result.toArray(new FileStatus[result.size()]);
}
 

import io.confluent.connect.hdfs.storage.Storage; //导入依赖的package包/类
public static FileStatus[] getDirectories(Storage storage, Path path) throws IOException {
  ArrayList<FileStatus> result = getDirectoriesImpl(storage, path);
  return result.toArray(new FileStatus[result.size()]);
}
 

import io.confluent.connect.hdfs.storage.Storage; //导入依赖的package包/类
@Test
public void testWALMultiClient() throws Exception {
  fs.delete(new Path(FileUtils.directoryName(url, topicsDir, TOPIC_PARTITION)), true);

  @SuppressWarnings("unchecked")
  Class<? extends Storage> storageClass = (Class<? extends Storage>)
      Class.forName(connectorConfig.getString(HdfsSinkConnectorConfig.STORAGE_CLASS_CONFIG));
  Storage storage = StorageFactory.createStorage(storageClass, conf, url);

  final WAL wal1 = storage.wal(topicsDir, TOPIC_PARTITION);
  final WAL wal2 = storage.wal(topicsDir, TOPIC_PARTITION);

  String directory = TOPIC + "/" + String.valueOf(PARTITION);
  final String tempfile = FileUtils.tempFileName(url, topicsDir, directory, extension);
  final String commitedFile = FileUtils.committedFileName(url, topicsDir, directory,
                                                          TOPIC_PARTITION, 0, 10, extension,
                                                          ZERO_PAD_FMT);

  fs.createNewFile(new Path(tempfile));

  wal1.acquireLease();
  wal1.append(WAL.beginMarker, "");
  wal1.append(tempfile, commitedFile);
  wal1.append(WAL.endMarker, "");

  Thread thread = new Thread(new Runnable() {
    @Override
    public void run() {
      try {
        // holding the lease for awhile
        Thread.sleep(3000);
        closed = true;
        wal1.close();
      } catch (ConnectException | InterruptedException e) {
        // Ignored
      }
    }
  });
  thread.start();

  wal2.acquireLease();
  assertTrue(closed);
  wal2.apply();
  wal2.close();

  assertTrue(fs.exists(new Path(commitedFile)));
  assertFalse(fs.exists(new Path(tempfile)));
  storage.close();
}
 

import io.confluent.connect.hdfs.storage.Storage; //导入依赖的package包/类
@Test
public void testRecovery() throws Exception {
  fs.delete(new Path(FileUtils.directoryName(url, topicsDir, TOPIC_PARTITION)), true);

  @SuppressWarnings("unchecked")
  Class<? extends Storage> storageClass = (Class<? extends Storage>)
      Class.forName(connectorConfig.getString(HdfsSinkConnectorConfig.STORAGE_CLASS_CONFIG));
  Storage storage = StorageFactory.createStorage(storageClass, conf, url);

  WAL wal = storage.wal(logsDir, TOPIC_PARTITION);

  wal.append(WAL.beginMarker, "");
  Set<String> committedFiles = new HashSet<>();

  String directory = TOPIC + "/" + "partition=" + String.valueOf(PARTITION);

  for (int i = 0; i < 5; ++i) {
    long startOffset = i * 10;
    long endOffset = (i + 1) * 10 - 1;
    String tempfile = FileUtils.tempFileName(url, topicsDir, directory, extension);
    fs.createNewFile(new Path(tempfile));
    String committedFile = FileUtils.committedFileName(url, topicsDir, directory, TOPIC_PARTITION, startOffset,
                                                       endOffset, extension, ZERO_PAD_FMT);
    committedFiles.add(committedFile);
    wal.append(tempfile, committedFile);
  }
  wal.append(WAL.endMarker, "");
  wal.close();

  DataWriter hdfsWriter = new DataWriter(connectorConfig, context, avroData);
  hdfsWriter.recover(TOPIC_PARTITION);
  Map<TopicPartition, Long> offsets = context.offsets();
  assertTrue(offsets.containsKey(TOPIC_PARTITION));
  assertEquals(50L, (long) offsets.get(TOPIC_PARTITION));

  String key = "key";
  Schema schema = createSchema();
  Struct record = createRecord(schema);

  // Need enough records to trigger file rotation
  ArrayList<SinkRecord> sinkRecords = new ArrayList<>();
  for (int i = 0; i < 3; i++)
    sinkRecords.add(
        new SinkRecord(TOPIC, PARTITION, Schema.STRING_SCHEMA, key, schema, record, 50 + i));

  hdfsWriter.write(sinkRecords);
  hdfsWriter.close(assignment);
  hdfsWriter.stop();

  committedFiles.add(FileUtils.committedFileName(url, topicsDir, directory, TOPIC_PARTITION,
                                                 50, 52, extension, ZERO_PAD_FMT));
  FileStatus[] statuses = fs.listStatus(new Path(FileUtils.directoryName(url, topicsDir, directory)),
                    new TopicPartitionCommittedFileFilter(TOPIC_PARTITION));
  assertEquals(committedFiles.size(), statuses.length);
  for (FileStatus status : statuses) {
    assertTrue(committedFiles.contains(status.getPath().toString()));
  }
}