Java DistributedLogConfiguration类代码示例

import org.apache.distributedlog.DistributedLogConfiguration; //导入依赖的package包/类
Namespace initializeNamespace(URI uri) throws IOException {
  DistributedLogConfiguration conf = new DistributedLogConfiguration()
      .setWriteLockEnabled(false)
      .setOutputBufferSize(256 * 1024)                  // 256k
      .setPeriodicFlushFrequencyMilliSeconds(0)         // disable periodical flush
      .setImmediateFlushEnabled(false)                  // disable immediate flush
      .setLogSegmentRollingIntervalMinutes(0)           // disable time-based rolling
      .setMaxLogSegmentBytes(Long.MAX_VALUE)            // disable size-based rolling
      .setExplicitTruncationByApplication(true)         // no auto-truncation
      .setRetentionPeriodHours(Integer.MAX_VALUE)       // long retention
      .setEnsembleSize(numReplicas)                     // replica settings
      .setWriteQuorumSize(numReplicas)
      .setAckQuorumSize(numReplicas)
      .setUseDaemonThread(true)                         // use daemon thread
      .setNumWorkerThreads(1)                           // use 1 worker thread
      .setBKClientNumberIOThreads(1);

  return this.nsBuilderSupplier.get()
      .clientId("heron-stateful-storage")
      .conf(conf)
      .uri(uri)
      .build();
}
 

import org.apache.distributedlog.DistributedLogConfiguration; //导入依赖的package包/类
@Before
public void before() throws Exception {
  Map<String, Object> config = new HashMap<>();
  config.put(DlogStorage.NS_URI_KEY, ROOT_URI);

  mockNamespace = mock(Namespace.class);
  mockNsBuilder = mock(NamespaceBuilder.class);
  when(mockNsBuilder.clientId(anyString())).thenReturn(mockNsBuilder);
  when(mockNsBuilder.conf(any(DistributedLogConfiguration.class))).thenReturn(mockNsBuilder);
  when(mockNsBuilder.uri(any(URI.class))).thenReturn(mockNsBuilder);
  when(mockNsBuilder.build()).thenReturn(mockNamespace);

  dlogStorage = new DlogStorage(() -> mockNsBuilder);
  dlogStorage.init(config);
  dlogStorage = spy(dlogStorage);

  instance = StatefulStorageTestContext.getInstance();
  instanceStateCheckpoint = StatefulStorageTestContext.getInstanceStateCheckpoint();
}
 

import org.apache.distributedlog.DistributedLogConfiguration; //导入依赖的package包/类
public static synchronized DistributedLogNamespace getNamespace(Settings settings, String localNodeId) throws IllegalArgumentException, NullPointerException, IOException {
    if (logNamespace == null) {
        String logServiceUrl = settings.get(LOG_SERVICE_ENDPOINT);
        URI uri = URI.create(logServiceUrl);
        DistributedLogConfiguration conf = new DistributedLogConfiguration();
        conf.setOutputBufferSize(settings.getAsInt(DL_MERGE_BUFFER_SIZE, 4 * 1024));
        // immediate flush means write the user record and write a control record immediately, so that current client could get the record immediately
        // but this means write two record into bookkeeper
        // in our case we do not need that because replica replay it and not need read it immediately
        // if primary failed, if it recovering, it will write a control record into bk and could read it again
        conf.setImmediateFlushEnabled(false);
        // set write enabled == false, because lease already confirmed there is only one writer
        conf.setWriteLockEnabled(false);
        // this enables move lac after 10 seconds so that other node could see the latest records
        conf.setPeriodicFlushFrequencyMilliSeconds(2);
        // batch write to bookkeeper is disabled
        conf.setMinDelayBetweenImmediateFlushMs(0);
        conf.setZKSessionTimeoutSeconds(settings.getAsInt(ZK_SESSION_TIMEOUT, 10));
        conf.setLockTimeout(DistributedLogConstants.LOCK_IMMEDIATE);
        conf.setLogSegmentRollingIntervalMinutes(0); // has to set to 0 to disable time based rolling policy and enable size based rolling policy
        conf.setMaxLogSegmentBytes(1 << 20 << settings.getAsInt(DL_SEGMENT_SIZE_MB, 8)); // set it to 256MB
        conf.setEnsembleSize(settings.getAsInt(DL_ENSEMBLE_SIZE, 3));
        conf.setAckQuorumSize(settings.getAsInt(DL_ACK_QUORUM_SIZE, 2));
        conf.setWriteQuorumSize(settings.getAsInt(DL_REPLICA_NUM, 3));
        conf.setRowAwareEnsemblePlacementEnabled(false);
        conf.setReadAheadMaxRecords(100);
        conf.setReadAheadBatchSize(3);
        conf.setExplicitTruncationByApplication(true); // set it to true to disable auto truncate
        conf.setRetentionPeriodHours(1); // dl will purge truncated log segments after 1 hour
        logNamespace = DistributedLogNamespaceBuilder.newBuilder()
                .conf(conf)
                .uri(uri)
                .regionId(DistributedLogConstants.LOCAL_REGION_ID)
                .clientId(localNodeId)
                .build();
    }
    return logNamespace;
}
 

import org.apache.distributedlog.DistributedLogConfiguration; //导入依赖的package包/类
private void initializeNamespace(Config upConfig) throws IOException {
  int numReplicas = DLContext.dlTopologiesNumReplicas(upConfig);

  DistributedLogConfiguration conf = new DistributedLogConfiguration()
      .setWriteLockEnabled(false)
      .setOutputBufferSize(256 * 1024)                  // 256k
      .setPeriodicFlushFrequencyMilliSeconds(0)         // disable periodical flush
      .setImmediateFlushEnabled(false)                  // disable immediate flush
      .setLogSegmentRollingIntervalMinutes(0)           // disable time-based rolling
      .setMaxLogSegmentBytes(Long.MAX_VALUE)            // disable size-based rolling
      .setExplicitTruncationByApplication(true)         // no auto-truncation
      .setRetentionPeriodHours(Integer.MAX_VALUE)       // long retention
      .setEnsembleSize(numReplicas)                     // replica settings
      .setWriteQuorumSize(numReplicas)
      .setAckQuorumSize(numReplicas)
      .setUseDaemonThread(true);                        // use daemon thread

  URI uri = URI.create(DLContext.dlTopologiesNamespaceURI(this.config));
  LOG.info(String.format(
      "Initializing distributedlog namespace for uploading topologies : %s",
      uri));

  this.namespace = this.nsBuilderSupplier.get()
      .clientId("heron-uploader")
      .conf(conf)
      .uri(uri)
      .build();
}
 

import org.apache.distributedlog.DistributedLogConfiguration; //导入依赖的package包/类
@Before
public void setUp() throws Exception {
  config = mock(Config.class);
  when(config.getStringValue(eq(DLContext.DL_TOPOLOGIES_NS_URI)))
      .thenReturn(DL_URI);
  nsBuilder = mock(NamespaceBuilder.class);
  when(nsBuilder.clientId(anyString())).thenReturn(nsBuilder);
  when(nsBuilder.conf(any(DistributedLogConfiguration.class))).thenReturn(nsBuilder);
  when(nsBuilder.uri(any(URI.class))).thenReturn(nsBuilder);

  copier = mock(Copier.class);
  uploader = new DLUploader(() -> nsBuilder, copier);
}
 

import org.apache.distributedlog.DistributedLogConfiguration; //导入依赖的package包/类
private static Namespace openNamespace(URI uri) throws IOException {
  return NamespaceBuilder.newBuilder()
      .uri(uri)
      .clientId("dlog-util")
      .conf(new DistributedLogConfiguration())
      .build();
}
 

import org.apache.distributedlog.DistributedLogConfiguration; //导入依赖的package包/类
@Test
public void testDownload() throws Exception {
  String logName = "test-download";
  URI uri = URI.create("distributedlog://127.0.0.1/test/distributedlog/" + logName);

  File tempFile = File.createTempFile("test", "download");
  // make sure it is deleted when the test completes
  tempFile.deleteOnExit();
  Path path = Paths.get(tempFile.toURI());

  Namespace ns = mock(Namespace.class);
  DistributedLogManager dlm = mock(DistributedLogManager.class);
  LogReader reader = mock(LogReader.class);
  when(ns.openLog(anyString())).thenReturn(dlm);
  when(dlm.getInputStream(eq(DLSN.InitialDLSN))).thenReturn(reader);
  when(reader.readNext(anyBoolean())).thenThrow(new EndOfStreamException("eos"));

  NamespaceBuilder nsBuilder = mock(NamespaceBuilder.class);
  when(nsBuilder.clientId(anyString())).thenReturn(nsBuilder);
  when(nsBuilder.conf(any(DistributedLogConfiguration.class))).thenReturn(nsBuilder);
  when(nsBuilder.uri(any(URI.class))).thenReturn(nsBuilder);
  when(nsBuilder.build()).thenReturn(ns);

  PowerMockito.mockStatic(Extractor.class);
  PowerMockito.doNothing()
      .when(Extractor.class, "extract", any(InputStream.class), any(Path.class));

  DLDownloader downloader = new DLDownloader(() -> nsBuilder);
  downloader.download(uri, path);

  URI parentUri = URI.create("distributedlog://127.0.0.1/test/distributedlog");
  verify(nsBuilder, times(1)).clientId(eq("heron-downloader"));
  verify(nsBuilder, times(1)).conf(eq(CONF));
  verify(nsBuilder, times(1)).uri(parentUri);

  PowerMockito.verifyStatic(times(1));
  Extractor.extract(any(InputStream.class), eq(path));

  verify(ns, times(1)).openLog(eq(logName));
  verify(ns, times(1)).close();
}