Java JetInstance类代码示例

import com.hazelcast.jet.JetInstance; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
    if (args.length == 0) {
        System.out.println("Usage: hdfs-to-map <name> <input path> <parallelism>");
        return;
    }

    String name = args[0];
    String inputPath = args[1];
    int parallelism = Integer.parseInt(args[2]);

    JetInstance client = Jet.newJetClient();
    IStreamMap<Long, String> map = client.getMap(name);
    map.clear();

    try {
        long begin = System.currentTimeMillis();
        fillMap(client, name, inputPath, parallelism);
        long elapsed = System.currentTimeMillis() - begin;
        System.out.println("Time=" + elapsed);
    } finally {
        client.shutdown();
    }
}
 

import com.hazelcast.jet.JetInstance; //导入依赖的package包/类
private static void fillMap(JetInstance client, String name, String inputPath, int parallelism) throws Exception {
    DAG dag = new DAG();
    JobConf conf = new JobConf();
    conf.setInputFormat(TextInputFormat.class);
    TextInputFormat.addInputPath(conf, new Path(inputPath));


    Vertex reader = dag.newVertex("reader", readHdfsP(conf, Util::entry));
    Vertex mapper = dag.newVertex("mapper",
            mapP((Map.Entry<LongWritable, Text> e) -> entry(e.getKey().get(), e.getValue().toString())));
    Vertex writer = dag.newVertex("writer", writeMapP(name));

    reader.localParallelism(parallelism);
    mapper.localParallelism(parallelism);
    writer.localParallelism(parallelism);

    dag.edge(between(reader, mapper));
    dag.edge(between(mapper, writer));


    JobConfig jobConfig = new JobConfig();
    jobConfig.addClass(HdfsToMap.class);

    client.newJob(dag, jobConfig).join();
}
 

import com.hazelcast.jet.JetInstance; //导入依赖的package包/类
@Test
public void test_Jar_Distribution() throws Throwable {
    createCluster();

    DAG dag = new DAG();
    dag.newVertex("create and print person", LoadPersonIsolated::new);


    JetInstance jetInstance = getJetInstance();
    JobConfig jobConfig = new JobConfig();
    jobConfig.addJar(this.getClass().getResource("/sample-pojo-1.0-person.jar"));
    jobConfig.addJar(this.getClass().getResource("/sample-pojo-1.0-deployment.jar"));
    jobConfig.addClass(AbstractDeploymentTest.class);

    executeAndPeel(jetInstance.newJob(dag, jobConfig));
}
 

import com.hazelcast.jet.JetInstance; //导入依赖的package包/类
@Before
public void setUp() {
    JetConfig config = new JetConfig();

    EventJournalConfig journalConfig = new EventJournalConfig()
            .setMapName("*")
            .setCapacity(JOURNAL_CAPACITY)
            .setEnabled(true);

    config.getHazelcastConfig().setProperty(PARTITION_COUNT.getName(), String.valueOf(NUM_PARTITIONS));
    config.getHazelcastConfig().addEventJournalConfig(journalConfig);
    JetInstance instance = this.createJetMember(config);

    map = (MapProxyImpl<Integer, Integer>) instance.getHazelcastInstance().<Integer, Integer>getMap("test");
    List<Integer> allPartitions = IntStream.range(0, NUM_PARTITIONS).boxed().collect(toList());

    supplier = () -> new StreamEventJournalP<>(map, allPartitions, e -> true,
            EventJournalMapEvent::getNewValue, START_FROM_OLDEST, false,
            wmGenParams(Integer::intValue, withFixedLag(0), suppressAll(), -1));
}
 

import com.hazelcast.jet.JetInstance; //导入依赖的package包/类
@Test
public void when_writeBufferedJobFailed_then_bufferDisposed() throws Exception {
    JetInstance instance = createJetMember();
    try {
        DAG dag = new DAG();
        Vertex source = dag.newVertex("source", StuckForeverSourceP::new);
        Vertex sink = dag.newVertex("sink", getLoggingBufferedWriter()).localParallelism(1);

        dag.edge(Edge.between(source, sink));

        Job job = instance.newJob(dag);
        // wait for the job to initialize
        Thread.sleep(5000);
        job.cancel();

        assertTrueEventually(() -> assertTrue("No \"dispose\", only: " + events, events.contains("dispose")), 60);
        System.out.println(events);
    } finally {
        instance.shutdown();
    }
}
 

import com.hazelcast.jet.JetInstance; //导入依赖的package包/类
@Test
public void test_serializationFromNodeToClient() {
    // create one member and one client
    createJetMember();
    JetInstance client = createJetClient();

    RuntimeException exc = new RuntimeException("myException");
    try {
        DAG dag = new DAG();
        dag.newVertex("source", () -> new ProcessorThatFailsInComplete(exc)).localParallelism(1);
        client.newJob(dag).join();
    } catch (Exception caught) {
        assertThat(caught.toString(), containsString(exc.toString()));
        TestUtil.assertExceptionInCauses(exc, caught);
    } finally {
        shutdownFactory();
    }
}
 

import com.hazelcast.jet.JetInstance; //导入依赖的package包/类
@Test
public void test_serializationOnNode() {
    JetTestInstanceFactory factory = new JetTestInstanceFactory();
    // create one member and one client
    JetInstance client = factory.newMember();

    RuntimeException exc = new RuntimeException("myException");
    try {
        DAG dag = new DAG();
        dag.newVertex("source", () -> new ProcessorThatFailsInComplete(exc)).localParallelism(1);
        client.newJob(dag).join();
    } catch (Exception caught) {
        assertThat(caught.toString(), containsString(exc.toString()));
        TestUtil.assertExceptionInCauses(exc, caught);
    } finally {
        factory.terminateAll();
    }
}
 

import com.hazelcast.jet.JetInstance; //导入依赖的package包/类
private void testJobSubmissionTimeWhenJobIsRunning(JetInstance instance) throws InterruptedException {
    // Given
    DAG dag = new DAG().vertex(new Vertex("test", new MockPS(StuckProcessor::new, NODE_COUNT)));
    JobConfig config = new JobConfig();
    String jobName = "job1";
    config.setName(jobName);

    // When
    Job job = instance1.newJob(dag, config);
    StuckProcessor.executionStarted.await();
    Job trackedJob = instance.getJob("job1");

    // Then
    assertNotNull(trackedJob);
    assertNotEquals(0, job.getSubmissionTime());
    assertNotEquals(0, trackedJob.getSubmissionTime());
    StuckProcessor.proceedLatch.countDown();
}
 

import com.hazelcast.jet.JetInstance; //导入依赖的package包/类
@Before
public void setup() {
    factory = new JetTestInstanceFactory();

    JetConfig config = new JetConfig();
    config.getInstanceConfig().setCooperativeThreadCount(LOCAL_PARALLELISM);

    // force snapshots to fail by adding a failing map store configuration for snapshot data maps
    MapConfig mapConfig = new MapConfig(SnapshotRepository.SNAPSHOT_DATA_NAME_PREFIX + '*');
    MapStoreConfig mapStoreConfig = mapConfig.getMapStoreConfig();
    mapStoreConfig.setEnabled(true);
    mapStoreConfig.setImplementation(new FailingMapStore());
    config.getHazelcastConfig().addMapConfig(mapConfig);

    JetInstance[] instances = factory.newMembers(config, 2);
    instance1 = instances[0];
}
 

import com.hazelcast.jet.JetInstance; //导入依赖的package包/类
@Test
public void when_jobCancelledOnSingleNode_then_terminatedEventually() {
    // Given
    JetInstance instance = newInstance();

    DAG dag = new DAG();
    dag.newVertex("slow", StuckSource::new);

    Job job = instance.newJob(dag);
    assertExecutionStarted();

    // When
    job.cancel();

    // Then
    assertExecutionTerminated();
    expectedException.expect(CancellationException.class);
    job.join();
}
 

import com.hazelcast.jet.JetInstance; //导入依赖的package包/类
@Test
public void when_jobCancelledOnMultipleNodes_then_terminatedEventually() {
    // Given
    newInstance();
    JetInstance instance = newInstance();

    DAG dag = new DAG();
    dag.newVertex("slow", StuckSource::new);

    Job job = instance.newJob(dag);
    assertExecutionStarted();

    // When
    job.cancel();

    // Then
    assertExecutionTerminated();
    expectedException.expect(CancellationException.class);
    job.join();
}
 

import com.hazelcast.jet.JetInstance; //导入依赖的package包/类
@Test
public void when_jobCancelled_then_jobStatusIsSetEventually() {
    // Given
    JetInstance instance = newInstance();

    DAG dag = new DAG();
    dag.newVertex("slow", StuckSource::new);

    Job job = instance.newJob(dag);
    assertExecutionStarted();

    // When
    job.cancel();

    // Then
    assertTrueEventually(() -> assertEquals(JobStatus.COMPLETED, job.getStatus()), 3);
}
 

import com.hazelcast.jet.JetInstance; //导入依赖的package包/类
@Test
public void when_jobCancelledFromClient_then_terminatedEventually() {
    // Given
    newInstance();
    newInstance();
    JetInstance client = factory.newClient();

    DAG dag = new DAG();
    dag.newVertex("slow", StuckSource::new);

    Job job = client.newJob(dag);
    assertExecutionStarted();

    // When
    job.cancel();

    // Then
    assertExecutionTerminated();
    expectedException.expect(CancellationException.class);
    job.join();
}
 

import com.hazelcast.jet.JetInstance; //导入依赖的package包/类
@Test
public void when_jobCancelledFromClient_then_jobStatusIsSetEventually() {
    // Given
    newInstance();
    newInstance();
    JetInstance client = factory.newClient();

    DAG dag = new DAG();
    dag.newVertex("slow", StuckSource::new);

    Job job = client.newJob(dag);
    assertExecutionStarted();

    // When
    job.cancel();

    // Then
    assertTrueEventually(() -> assertEquals(JobStatus.COMPLETED, job.getStatus()), 3);
}
 

import com.hazelcast.jet.JetInstance; //导入依赖的package包/类
@Test
public void when_jobCancelled_then_trackedJobsGetNotified() {
    // Given
    JetInstance instance1 = newInstance();
    JetInstance instance2 = newInstance();

    DAG dag = new DAG();
    dag.newVertex("slow", StuckSource::new);

    Job job = instance1.newJob(dag);
    assertExecutionStarted();

    // When
    job.cancel();

    // Then
    assertExecutionTerminated();
    expectedException.expect(CancellationException.class);
    Job tracked = instance2.getJobs().iterator().next();
    tracked.join();
}