本文整理汇总了Java中org.apache.beam.runners.dataflow.options.DataflowPipelineOptions.setGcpTempLocation方法的典型用法代码示例。如果您正苦于以下问题：Java DataflowPipelineOptions.setGcpTempLocation方法的具体用法？Java DataflowPipelineOptions.setGcpTempLocation怎么用？Java DataflowPipelineOptions.setGcpTempLocation使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.beam.runners.dataflow.options.DataflowPipelineOptions的用法示例。
在下文中一共展示了DataflowPipelineOptions.setGcpTempLocation方法的11个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: testInvalidNumberOfWorkerHarnessThreads
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testInvalidNumberOfWorkerHarnessThreads() throws IOException {
DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
FileSystems.setDefaultPipelineOptions(options);
options.setRunner(DataflowRunner.class);
options.setProject("foo-12345");
options.setGcpTempLocation(VALID_TEMP_BUCKET);
options.setGcsUtil(mockGcsUtil);
options.as(DataflowPipelineDebugOptions.class).setNumberOfWorkerHarnessThreads(-1);
thrown.expect(IllegalArgumentException.class);
thrown.expectMessage("Number of worker harness threads");
thrown.expectMessage("Please make sure the value is non-negative.");
DataflowRunner.fromOptions(options);
}
示例2: createTestServiceRunner
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
/**
* Create a test pipeline that uses the {@link DataflowRunner} so that {@link GroupByKey}
* is not expanded. This is used for verifying that even without expansion the proper errors show
* up.
*/
private Pipeline createTestServiceRunner() {
DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
options.setRunner(DataflowRunner.class);
options.setProject("someproject");
options.setGcpTempLocation("gs://staging");
options.setPathValidatorClass(NoopPathValidator.class);
options.setDataflowClient(dataflow);
return Pipeline.create(options);
}
示例3: createTestBatchRunner
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
private Pipeline createTestBatchRunner() {
DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
options.setRunner(DataflowRunner.class);
options.setProject("someproject");
options.setGcpTempLocation("gs://staging");
options.setPathValidatorClass(NoopPathValidator.class);
options.setDataflowClient(dataflow);
return Pipeline.create(options);
}
示例4: createTestStreamingRunner
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
private Pipeline createTestStreamingRunner() {
DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
options.setRunner(DataflowRunner.class);
options.setStreaming(true);
options.setProject("someproject");
options.setGcpTempLocation("gs://staging");
options.setPathValidatorClass(NoopPathValidator.class);
options.setDataflowClient(dataflow);
return Pipeline.create(options);
}
示例5: testInvalidGcpTempLocation
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testInvalidGcpTempLocation() throws IOException {
DataflowPipelineOptions options = buildPipelineOptions();
options.setGcpTempLocation("file://temp/location");
thrown.expect(IllegalArgumentException.class);
thrown.expectMessage(containsString("Expected a valid 'gs://' path but was given"));
DataflowRunner.fromOptions(options);
ArgumentCaptor<Job> jobCaptor = ArgumentCaptor.forClass(Job.class);
Mockito.verify(mockJobs).create(eq(PROJECT_ID), eq(REGION_ID), jobCaptor.capture());
assertValidJob(jobCaptor.getValue());
}
示例6: testNonExistentTempLocation
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testNonExistentTempLocation() throws IOException {
DataflowPipelineOptions options = buildPipelineOptions();
options.setGcpTempLocation(NON_EXISTENT_BUCKET);
thrown.expect(IllegalArgumentException.class);
thrown.expectMessage(containsString(
"Output path does not exist or is not writeable: " + NON_EXISTENT_BUCKET));
DataflowRunner.fromOptions(options);
ArgumentCaptor<Job> jobCaptor = ArgumentCaptor.forClass(Job.class);
Mockito.verify(mockJobs).create(eq(PROJECT_ID), eq(REGION_ID), jobCaptor.capture());
assertValidJob(jobCaptor.getValue());
}
示例7: testProjectId
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testProjectId() throws IOException {
DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
options.setRunner(DataflowRunner.class);
options.setProject("foo-12345");
options.setGcpTempLocation(VALID_TEMP_BUCKET);
options.setGcsUtil(mockGcsUtil);
options.setGcpCredential(new TestCredential());
DataflowRunner.fromOptions(options);
}
示例8: testProjectPrefix
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testProjectPrefix() throws IOException {
DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
options.setRunner(DataflowRunner.class);
options.setProject("google.com:some-project-12345");
options.setGcpTempLocation(VALID_TEMP_BUCKET);
options.setGcsUtil(mockGcsUtil);
options.setGcpCredential(new TestCredential());
DataflowRunner.fromOptions(options);
}
示例9: testProjectNumber
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testProjectNumber() throws IOException {
DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
options.setRunner(DataflowRunner.class);
options.setProject("12345");
options.setGcpTempLocation(VALID_TEMP_BUCKET);
options.setGcsUtil(mockGcsUtil);
thrown.expect(IllegalArgumentException.class);
thrown.expectMessage("Project ID");
thrown.expectMessage("project number");
DataflowRunner.fromOptions(options);
}
示例10: testProjectDescription
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testProjectDescription() throws IOException {
DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
options.setRunner(DataflowRunner.class);
options.setProject("some project");
options.setGcpTempLocation(VALID_TEMP_BUCKET);
options.setGcsUtil(mockGcsUtil);
thrown.expect(IllegalArgumentException.class);
thrown.expectMessage("Project ID");
thrown.expectMessage("project description");
DataflowRunner.fromOptions(options);
}
示例11: testGcpTempAndNoTempLocationSucceeds
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testGcpTempAndNoTempLocationSucceeds() throws Exception {
DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
options.setRunner(DataflowRunner.class);
options.setGcpCredential(new TestCredential());
options.setProject("foo-project");
options.setGcpTempLocation(VALID_TEMP_BUCKET);
options.setGcsUtil(mockGcsUtil);
DataflowRunner.fromOptions(options);
}