本文整理汇总了Java中org.apache.beam.runners.dataflow.options.DataflowPipelineOptions.setGcpCredential方法的典型用法代码示例。如果您正苦于以下问题:Java DataflowPipelineOptions.setGcpCredential方法的具体用法?Java DataflowPipelineOptions.setGcpCredential怎么用?Java DataflowPipelineOptions.setGcpCredential使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.beam.runners.dataflow.options.DataflowPipelineOptions
的用法示例。
在下文中一共展示了DataflowPipelineOptions.setGcpCredential方法的14个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: buildPipelineOptions
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
private static DataflowPipelineOptions buildPipelineOptions() throws IOException {
GcsUtil mockGcsUtil = mock(GcsUtil.class);
when(mockGcsUtil.expand(any(GcsPath.class))).then(new Answer<List<GcsPath>>() {
@Override
public List<GcsPath> answer(InvocationOnMock invocation) throws Throwable {
return ImmutableList.of((GcsPath) invocation.getArguments()[0]);
}
});
when(mockGcsUtil.bucketAccessible(any(GcsPath.class))).thenReturn(true);
DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
options.setRunner(DataflowRunner.class);
options.setGcpCredential(new TestCredential());
options.setJobName("some-job-name");
options.setProject("some-project");
options.setRegion("some-region");
options.setTempLocation(GcsPath.fromComponents("somebucket", "some/path").toString());
options.setFilesToStage(new LinkedList<String>());
options.setDataflowClient(buildMockDataflow(new IsValidCreateRequest()));
options.setGcsUtil(mockGcsUtil);
return options;
}
示例2: buildPipelineOptions
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
private static DataflowPipelineOptions buildPipelineOptions(String ... args) throws IOException {
GcsUtil mockGcsUtil = mock(GcsUtil.class);
when(mockGcsUtil.expand(any(GcsPath.class))).then(new Answer<List<GcsPath>>() {
@Override
public List<GcsPath> answer(InvocationOnMock invocation) throws Throwable {
return ImmutableList.of((GcsPath) invocation.getArguments()[0]);
}
});
when(mockGcsUtil.bucketAccessible(any(GcsPath.class))).thenReturn(true);
DataflowPipelineOptions options =
PipelineOptionsFactory.fromArgs(args).as(DataflowPipelineOptions.class);
options.setRunner(DataflowRunner.class);
options.setGcpCredential(new TestCredential());
options.setJobName("some-job-name");
options.setProject("some-project");
options.setRegion("some-region");
options.setTempLocation(GcsPath.fromComponents("somebucket", "some/path").toString());
options.setFilesToStage(new LinkedList<String>());
options.setGcsUtil(mockGcsUtil);
return options;
}
示例3: buildPipelineOptions
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
private DataflowPipelineOptions buildPipelineOptions() throws IOException {
DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
options.setRunner(DataflowRunner.class);
options.setProject(PROJECT_ID);
options.setTempLocation(VALID_TEMP_BUCKET);
options.setRegion(REGION_ID);
// Set FILES_PROPERTY to empty to prevent a default value calculated from classpath.
options.setFilesToStage(new LinkedList<String>());
options.setDataflowClient(buildMockDataflow());
options.setGcsUtil(mockGcsUtil);
options.setGcpCredential(new TestCredential());
// Configure the FileSystem registrar to use these options.
FileSystems.setDefaultPipelineOptions(options);
return options;
}
示例4: testTemplateRunnerFullCompletion
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
/**
* Tests that the {@link DataflowRunner} with {@code --templateLocation} returns normally when the
* runner is successfully run.
*/
@Test
public void testTemplateRunnerFullCompletion() throws Exception {
File existingFile = tmpFolder.newFile();
DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
options.setJobName("TestJobName");
options.setGcpCredential(new TestCredential());
options.setPathValidatorClass(NoopPathValidator.class);
options.setProject("test-project");
options.setRunner(DataflowRunner.class);
options.setTemplateLocation(existingFile.getPath());
options.setTempLocation(tmpFolder.getRoot().getPath());
Pipeline p = Pipeline.create(options);
p.run();
expectedLogs.verifyInfo("Template successfully created");
}
示例5: testTemplateRunnerLoggedErrorForFile
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
/**
* Tests that the {@link DataflowRunner} with {@code --templateLocation} throws the appropriate
* exception when an output file is not writable.
*/
@Test
public void testTemplateRunnerLoggedErrorForFile() throws Exception {
DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
options.setJobName("TestJobName");
options.setRunner(DataflowRunner.class);
options.setTemplateLocation("//bad/path");
options.setProject("test-project");
options.setTempLocation(tmpFolder.getRoot().getPath());
options.setGcpCredential(new TestCredential());
options.setPathValidatorClass(NoopPathValidator.class);
Pipeline p = Pipeline.create(options);
thrown.expectMessage("Cannot create output file at");
thrown.expect(RuntimeException.class);
p.run();
}
示例6: testDontOverrideEndpointWithDefaultApi
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testDontOverrideEndpointWithDefaultApi() {
DataflowPipelineOptions options =
PipelineOptionsFactory.create().as(DataflowPipelineOptions.class);
options.setProject(PROJECT_ID);
options.setGcpCredential(new TestCredential());
String cancelCommand = MonitoringUtil.getGcloudCancelCommand(options, JOB_ID);
assertEquals("gcloud dataflow jobs --project=someProject cancel 1234", cancelCommand);
}
示例7: testOverridesEndpointWithStagedDataflowEndpoint
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testOverridesEndpointWithStagedDataflowEndpoint() {
DataflowPipelineOptions options =
PipelineOptionsFactory.create().as(DataflowPipelineOptions.class);
options.setProject(PROJECT_ID);
options.setGcpCredential(new TestCredential());
String stagingDataflowEndpoint = "v0neverExisted";
options.setDataflowEndpoint(stagingDataflowEndpoint);
String cancelCommand = MonitoringUtil.getGcloudCancelCommand(options, JOB_ID);
assertEquals(
"CLOUDSDK_API_ENDPOINT_OVERRIDES_DATAFLOW=https://dataflow.googleapis.com/v0neverExisted/ "
+ "gcloud dataflow jobs --project=someProject cancel 1234",
cancelCommand);
}
示例8: testGcsStagingLocationInitialization
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testGcsStagingLocationInitialization() throws Exception {
// Set temp location (required), and check that staging location is set.
DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
options.setTempLocation(VALID_TEMP_BUCKET);
options.setProject(PROJECT_ID);
options.setGcpCredential(new TestCredential());
options.setGcsUtil(mockGcsUtil);
options.setRunner(DataflowRunner.class);
DataflowRunner.fromOptions(options);
assertNotNull(options.getStagingLocation());
}
示例9: testProjectId
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testProjectId() throws IOException {
DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
options.setRunner(DataflowRunner.class);
options.setProject("foo-12345");
options.setGcpTempLocation(VALID_TEMP_BUCKET);
options.setGcsUtil(mockGcsUtil);
options.setGcpCredential(new TestCredential());
DataflowRunner.fromOptions(options);
}
示例10: testProjectPrefix
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testProjectPrefix() throws IOException {
DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
options.setRunner(DataflowRunner.class);
options.setProject("google.com:some-project-12345");
options.setGcpTempLocation(VALID_TEMP_BUCKET);
options.setGcsUtil(mockGcsUtil);
options.setGcpCredential(new TestCredential());
DataflowRunner.fromOptions(options);
}
示例11: testGcpTempAndNoTempLocationSucceeds
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testGcpTempAndNoTempLocationSucceeds() throws Exception {
DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
options.setRunner(DataflowRunner.class);
options.setGcpCredential(new TestCredential());
options.setProject("foo-project");
options.setGcpTempLocation(VALID_TEMP_BUCKET);
options.setGcsUtil(mockGcsUtil);
DataflowRunner.fromOptions(options);
}
示例12: testTempLocationAndNoGcpTempLocationSucceeds
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testTempLocationAndNoGcpTempLocationSucceeds() throws Exception {
DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
options.setRunner(DataflowRunner.class);
options.setGcpCredential(new TestCredential());
options.setProject("foo-project");
options.setTempLocation(VALID_TEMP_BUCKET);
options.setGcsUtil(mockGcsUtil);
DataflowRunner.fromOptions(options);
}
示例13: testToString
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testToString() {
DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
options.setJobName("TestJobName");
options.setProject("test-project");
options.setTempLocation("gs://test/temp/location");
options.setGcpCredential(new TestCredential());
options.setPathValidatorClass(NoopPathValidator.class);
options.setRunner(DataflowRunner.class);
assertEquals(
"DataflowRunner#testjobname",
DataflowRunner.fromOptions(options).toString());
}
示例14: testRunWithFiles
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testRunWithFiles() throws IOException {
// Test that the function DataflowRunner.stageFiles works as expected.
final String cloudDataflowDataset = "somedataset";
// Create some temporary files.
File temp1 = File.createTempFile("DataflowRunnerTest", "txt");
temp1.deleteOnExit();
File temp2 = File.createTempFile("DataflowRunnerTest2", "txt");
temp2.deleteOnExit();
String overridePackageName = "alias.txt";
when(mockGcsUtil.getObjects(anyListOf(GcsPath.class)))
.thenReturn(ImmutableList.of(GcsUtil.StorageObjectOrIOException.create(
new FileNotFoundException("some/path"))));
DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
options.setFilesToStage(ImmutableList.of(
temp1.getAbsolutePath(),
overridePackageName + "=" + temp2.getAbsolutePath()));
options.setStagingLocation(VALID_STAGING_BUCKET);
options.setTempLocation(VALID_TEMP_BUCKET);
options.setTempDatasetId(cloudDataflowDataset);
options.setProject(PROJECT_ID);
options.setRegion(REGION_ID);
options.setJobName("job");
options.setDataflowClient(buildMockDataflow());
options.setGcsUtil(mockGcsUtil);
options.setGcpCredential(new TestCredential());
when(mockGcsUtil.create(any(GcsPath.class), anyString(), anyInt()))
.then(
new Answer<SeekableByteChannel>() {
@Override
public SeekableByteChannel answer(InvocationOnMock invocation) throws Throwable {
return FileChannel.open(
Files.createTempFile("channel-", ".tmp"),
StandardOpenOption.CREATE,
StandardOpenOption.WRITE,
StandardOpenOption.DELETE_ON_CLOSE);
}
});
Pipeline p = buildDataflowPipeline(options);
DataflowPipelineJob job = (DataflowPipelineJob) p.run();
assertEquals("newid", job.getJobId());
ArgumentCaptor<Job> jobCaptor = ArgumentCaptor.forClass(Job.class);
Mockito.verify(mockJobs).create(eq(PROJECT_ID), eq(REGION_ID), jobCaptor.capture());
Job workflowJob = jobCaptor.getValue();
assertValidJob(workflowJob);
assertEquals(
2,
workflowJob.getEnvironment().getWorkerPools().get(0).getPackages().size());
DataflowPackage workflowPackage1 =
workflowJob.getEnvironment().getWorkerPools().get(0).getPackages().get(0);
assertThat(workflowPackage1.getName(), startsWith(temp1.getName()));
DataflowPackage workflowPackage2 =
workflowJob.getEnvironment().getWorkerPools().get(0).getPackages().get(1);
assertEquals(overridePackageName, workflowPackage2.getName());
assertEquals(
GcsPath.fromUri(VALID_TEMP_BUCKET).toResourceName(),
workflowJob.getEnvironment().getTempStoragePrefix());
assertEquals(
cloudDataflowDataset,
workflowJob.getEnvironment().getDataset());
assertEquals(
DataflowRunnerInfo.getDataflowRunnerInfo().getName(),
workflowJob.getEnvironment().getUserAgent().get("name"));
assertEquals(
DataflowRunnerInfo.getDataflowRunnerInfo().getVersion(),
workflowJob.getEnvironment().getUserAgent().get("version"));
}