本文整理汇总了Java中com.google.api.services.bigquery.model.TableDataInsertAllResponse.InsertErrors方法的典型用法代码示例。如果您正苦于以下问题:Java TableDataInsertAllResponse.InsertErrors方法的具体用法?Java TableDataInsertAllResponse.InsertErrors怎么用?Java TableDataInsertAllResponse.InsertErrors使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类com.google.api.services.bigquery.model.TableDataInsertAllResponse
的用法示例。
在下文中一共展示了TableDataInsertAllResponse.InsertErrors方法的9个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: getInsertErrors
import com.google.api.services.bigquery.model.TableDataInsertAllResponse; //导入方法依赖的package包/类
Map<TableRow, List<TableDataInsertAllResponse.InsertErrors>> getInsertErrors() {
Map<TableRow, List<TableDataInsertAllResponse.InsertErrors>> parsedInsertErrors =
Maps.newHashMap();
synchronized (tables) {
for (Map.Entry<String, List<String>> entry : this.insertErrors.entrySet()) {
TableRow tableRow = BigQueryHelpers.fromJsonString(entry.getKey(), TableRow.class);
List<TableDataInsertAllResponse.InsertErrors> allErrors = Lists.newArrayList();
for (String errorsString : entry.getValue()) {
allErrors.add(BigQueryHelpers.fromJsonString(
errorsString, TableDataInsertAllResponse.InsertErrors.class));
}
parsedInsertErrors.put(tableRow, allErrors);
}
}
return parsedInsertErrors;
}
示例2: run
import com.google.api.services.bigquery.model.TableDataInsertAllResponse; //导入方法依赖的package包/类
public void run() {
try {
// Prepare target table
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
Date now = new Date();
String date = sdf.format(now);
String targetTable = filterId + "_results_" + date + "_v" + parent.TABLE_STRUCTURE_VERSION;
targetTable = targetTable.replace('-', '_');
parent.prepareTable(targetTable);
// Execute
TableDataInsertAllRequest ir = new TableDataInsertAllRequest().setRows(rows);
TableDataInsertAllResponse response = parent.bigquery.tabledata().insertAll(parent.projectId, parent.datasetId, targetTable, ir).execute();
List<TableDataInsertAllResponse.InsertErrors> errors = response.getInsertErrors();
if (errors != null) {
LOG.error(errors.size() + " error(s) while writing " + filterId + " to BigQuery");
} else {
// Log lines for debug
LOG.info(rows.size() + " lines written for " + filterId);
}
} catch (Exception e) {
LOG.error("Failed to write to BigQuery", e);
}
}
示例3: generateErrorAmongMany
import com.google.api.services.bigquery.model.TableDataInsertAllResponse; //导入方法依赖的package包/类
private TableDataInsertAllResponse.InsertErrors generateErrorAmongMany(
int numErrors, String baseReason, String exceptionalReason) {
// The retry policies are expected to search through the entire list of ErrorProtos to determine
// whether to retry. Stick the exceptionalReason in a random position to exercise this.
List<ErrorProto> errorProtos = Lists.newArrayListWithExpectedSize(numErrors);
int exceptionalPosition = ThreadLocalRandom.current().nextInt(numErrors);
for (int i = 0; i < numErrors; ++i) {
ErrorProto error = new ErrorProto();
error.setReason((i == exceptionalPosition) ? exceptionalReason : baseReason);
errorProtos.add(error);
}
TableDataInsertAllResponse.InsertErrors errors = new TableDataInsertAllResponse.InsertErrors();
errors.setErrors(errorProtos);
return errors;
}
示例4: onInsertAll
import com.google.api.services.bigquery.model.TableDataInsertAllResponse; //导入方法依赖的package包/类
private void onInsertAll(List<List<Long>> errorIndicesSequence) throws Exception {
when(mockClient.tabledata())
.thenReturn(mockTabledata);
final List<TableDataInsertAllResponse> responses = new ArrayList<>();
for (List<Long> errorIndices : errorIndicesSequence) {
List<TableDataInsertAllResponse.InsertErrors> errors = new ArrayList<>();
for (long i : errorIndices) {
TableDataInsertAllResponse.InsertErrors error =
new TableDataInsertAllResponse.InsertErrors();
error.setIndex(i);
}
TableDataInsertAllResponse response = new TableDataInsertAllResponse();
response.setInsertErrors(errors);
responses.add(response);
}
doAnswer(
new Answer<Bigquery.Tabledata.InsertAll>() {
@Override
public Bigquery.Tabledata.InsertAll answer(InvocationOnMock invocation) throws Throwable {
Bigquery.Tabledata.InsertAll mockInsertAll = mock(Bigquery.Tabledata.InsertAll.class);
when(mockInsertAll.execute())
.thenReturn(responses.get(0),
responses.subList(1, responses.size()).toArray(
new TableDataInsertAllResponse[responses.size() - 1]));
return mockInsertAll;
}
})
.when(mockTabledata)
.insertAll(anyString(), anyString(), anyString(), any(TableDataInsertAllRequest.class));
}
示例5: failOnInsert
import com.google.api.services.bigquery.model.TableDataInsertAllResponse; //导入方法依赖的package包/类
/**
* Cause a given {@link TableRow} object to fail when it's inserted. The errors link the list
* will be returned on subsequent retries, and the insert will succeed when the errors run out.
*/
public void failOnInsert(
Map<TableRow, List<TableDataInsertAllResponse.InsertErrors>> insertErrors) {
synchronized (tables) {
for (Map.Entry<TableRow, List<TableDataInsertAllResponse.InsertErrors>> entry
: insertErrors.entrySet()) {
List<String> errorStrings = Lists.newArrayList();
for (TableDataInsertAllResponse.InsertErrors errors : entry.getValue()) {
errorStrings.add(BigQueryHelpers.toJsonString(errors));
}
this.insertErrors.put(BigQueryHelpers.toJsonString(entry.getKey()), errorStrings);
}
}
}
示例6: testFailuresNoRetryPolicy
import com.google.api.services.bigquery.model.TableDataInsertAllResponse; //导入方法依赖的package包/类
@Test
public void testFailuresNoRetryPolicy() throws Exception {
TableRow row1 = new TableRow().set("name", "a").set("number", "1");
TableRow row2 = new TableRow().set("name", "b").set("number", "2");
TableRow row3 = new TableRow().set("name", "c").set("number", "3");
TableDataInsertAllResponse.InsertErrors ephemeralError =
new TableDataInsertAllResponse.InsertErrors().setErrors(
ImmutableList.of(new ErrorProto().setReason("timeout")));
fakeDatasetService.failOnInsert(
ImmutableMap.<TableRow, List<TableDataInsertAllResponse.InsertErrors>>of(
row1, ImmutableList.of(ephemeralError, ephemeralError),
row2, ImmutableList.of(ephemeralError, ephemeralError)));
p.apply(Create.of(row1, row2, row3))
.apply(
BigQueryIO.writeTableRows()
.to("project-id:dataset-id.table-id")
.withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED)
.withMethod(BigQueryIO.Write.Method.STREAMING_INSERTS)
.withSchema(
new TableSchema()
.setFields(
ImmutableList.of(
new TableFieldSchema().setName("name").setType("STRING"),
new TableFieldSchema().setName("number").setType("INTEGER"))))
.withTestServices(fakeBqServices)
.withoutValidation());
p.run();
assertThat(
fakeDatasetService.getAllRows("project-id", "dataset-id", "table-id"),
containsInAnyOrder(row1, row2, row3));
}
示例7: testRetryPolicy
import com.google.api.services.bigquery.model.TableDataInsertAllResponse; //导入方法依赖的package包/类
@Test
public void testRetryPolicy() throws Exception {
TableRow row1 = new TableRow().set("name", "a").set("number", "1");
TableRow row2 = new TableRow().set("name", "b").set("number", "2");
TableRow row3 = new TableRow().set("name", "c").set("number", "3");
TableDataInsertAllResponse.InsertErrors ephemeralError =
new TableDataInsertAllResponse.InsertErrors().setErrors(
ImmutableList.of(new ErrorProto().setReason("timeout")));
TableDataInsertAllResponse.InsertErrors persistentError =
new TableDataInsertAllResponse.InsertErrors().setErrors(
ImmutableList.of(new ErrorProto().setReason("invalidQuery")));
fakeDatasetService.failOnInsert(
ImmutableMap.<TableRow, List<TableDataInsertAllResponse.InsertErrors>>of(
row1, ImmutableList.of(ephemeralError, ephemeralError),
row2, ImmutableList.of(ephemeralError, ephemeralError, persistentError)));
PCollection<TableRow> failedRows =
p.apply(Create.of(row1, row2, row3))
.apply(BigQueryIO.writeTableRows().to("project-id:dataset-id.table-id")
.withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED)
.withMethod(BigQueryIO.Write.Method.STREAMING_INSERTS)
.withSchema(new TableSchema().setFields(
ImmutableList.of(
new TableFieldSchema().setName("name").setType("STRING"),
new TableFieldSchema().setName("number").setType("INTEGER"))))
.withFailedInsertRetryPolicy(InsertRetryPolicy.retryTransientErrors())
.withTestServices(fakeBqServices)
.withoutValidation()).getFailedInserts();
// row2 finally fails with a non-retryable error, so we expect to see it in the collection of
// failed rows.
PAssert.that(failedRows).containsInAnyOrder(row2);
p.run();
// Only row1 and row3 were successfully inserted.
assertThat(
fakeDatasetService.getAllRows("project-id", "dataset-id", "table-id"),
containsInAnyOrder(row1, row3));
}
示例8: Context
import com.google.api.services.bigquery.model.TableDataInsertAllResponse; //导入方法依赖的package包/类
public Context(TableDataInsertAllResponse.InsertErrors errors) {
this.errors = errors;
}
示例9: insertAll
import com.google.api.services.bigquery.model.TableDataInsertAllResponse; //导入方法依赖的package包/类
@Override
public long insertAll(
TableReference ref, List<ValueInSingleWindow<TableRow>> rowList,
@Nullable List<String> insertIdList,
InsertRetryPolicy retryPolicy, List<ValueInSingleWindow<TableRow>> failedInserts)
throws IOException, InterruptedException {
Map<TableRow, List<TableDataInsertAllResponse.InsertErrors>> insertErrors = getInsertErrors();
synchronized (tables) {
if (insertIdList != null) {
assertEquals(rowList.size(), insertIdList.size());
} else {
insertIdList = Lists.newArrayListWithExpectedSize(rowList.size());
for (int i = 0; i < rowList.size(); ++i) {
insertIdList.add(Integer.toString(ThreadLocalRandom.current().nextInt()));
}
}
long dataSize = 0;
TableContainer tableContainer = getTableContainer(
ref.getProjectId(),
ref.getDatasetId(),
BigQueryHelpers.stripPartitionDecorator(ref.getTableId()));
for (int i = 0; i < rowList.size(); ++i) {
TableRow row = rowList.get(i).getValue();
List<TableDataInsertAllResponse.InsertErrors> allErrors = insertErrors.get(row);
boolean shouldInsert = true;
if (allErrors != null) {
for (TableDataInsertAllResponse.InsertErrors errors : allErrors) {
if (!retryPolicy.shouldRetry(new Context(errors))) {
shouldInsert = false;
}
}
}
if (shouldInsert) {
dataSize += tableContainer.addRow(row, insertIdList.get(i));
} else {
failedInserts.add(rowList.get(i));
}
}
return dataSize;
}
}