Java IndexedRecord类代码示例

import org.apache.avro.generic.IndexedRecord; //导入依赖的package包/类
private void recurse(String context, TypeDescriptor<?> type, Schema schema) {
  if (type.getRawType().isAnnotationPresent(AvroSchema.class)) {
    reportError(context, "Custom schemas are not supported -- remove @AvroSchema.");
    return;
  }

  if (!activeTypes.add(type)) {
    reportError(context, "%s appears recursively", type);
    return;
  }

  // If the the record isn't a true class, but rather a GenericRecord, SpecificRecord, etc.
  // with a specified schema, then we need to make the decision based on the generated
  // implementations.
  if (isSubtypeOf(type, IndexedRecord.class)) {
    checkIndexedRecord(context, schema, null);
  } else {
    doCheck(context, type, schema);
  }

  activeTypes.remove(type);
}
 

import org.apache.avro.generic.IndexedRecord; //导入依赖的package包/类
private void checkRecord(TypeDescriptor<?> type, Schema schema) {
  // For a record, we want to make sure that all the fields are deterministic.
  Class<?> clazz = type.getRawType();
  for (org.apache.avro.Schema.Field fieldSchema : schema.getFields()) {
    Field field = getField(clazz, fieldSchema.name());
    String fieldContext = field.getDeclaringClass().getName() + "#" + field.getName();

    if (field.isAnnotationPresent(AvroEncode.class)) {
      reportError(fieldContext,
          "Custom encoders may be non-deterministic -- remove @AvroEncode");
      continue;
    }

    if (!IndexedRecord.class.isAssignableFrom(field.getType())
        && field.isAnnotationPresent(AvroSchema.class)) {
      // TODO: We should be able to support custom schemas on POJO fields, but we shouldn't
      // need to, so we just allow it in the case of IndexedRecords.
      reportError(fieldContext,
          "Custom schemas are only supported for subtypes of IndexedRecord.");
      continue;
    }

    TypeDescriptor<?> fieldType = type.resolveType(field.getGenericType());
    recurse(fieldContext, fieldType, fieldSchema.schema());
  }
}
 

import org.apache.avro.generic.IndexedRecord; //导入依赖的package包/类
@Test
public void testHierarchical_TFD2119_ERR4_NullArray() throws Exception {
    DoFnTester<Object, IndexedRecord> fnTester = DoFnTester.of( //
            new FilterRowDoFn().withProperties(addCriteria(null, //
                    ".b1[0].id", //
                    null, //
                    ConditionsRowConstant.Operator.EQUAL, //
                    "1") //

            ).withOutputSchema(true));

    // Looks like this is not an exception -- it just doesn't match.
    IndexedRecord[] input = copyAndReplaceSubrecordArray(inputB, 10, true);
    List<IndexedRecord> output = fnTester.processBundle(input);
    for (IndexedRecord main : output) {
        List<IndexedRecord> subrecords = getSubrecords(main);
        assertThat(main.toString(), subrecords.get(0).get(0), is((Object) 1));
    }
    assertThat(output, hasSize(102));
}
 

import org.apache.avro.generic.IndexedRecord; //导入依赖的package包/类
@SuppressWarnings("rawtypes")
@Test
public void testListQueues() throws Throwable {
    TAzureStorageQueueListProperties properties = new TAzureStorageQueueListProperties("tests");
    properties.setupProperties();
    properties = (TAzureStorageQueueListProperties) setupConnectionProperties(
            (AzureStorageProvideConnectionProperties) properties);
    BoundedReader reader = createBoundedReader(properties);
    assertTrue(reader.start());
    do {
        IndexedRecord current = (IndexedRecord) reader.getCurrent();
        assertNotNull(current);
        assertTrue(current.get(0) instanceof String);
    } while (reader.advance());
    assertTrue((int) reader.getReturnValues().get(AzureStorageDefinition.RETURN_TOTAL_RECORD_COUNT) > 0);
    reader.close();
}
 

import org.apache.avro.generic.IndexedRecord; //导入依赖的package包/类
@SuppressWarnings("rawtypes")
@Test
public void testSystemReader() throws Throwable {
    String ctable = tbl_test + "InputSys";
    createSampleDataset(ctable);

    properties.tableName.setValue(ctable);
    properties.schema.schema.setValue(getSystemSchema());
    properties.useFilterExpression.setValue(false);
    BoundedReader reader = createBoundedReader(properties);
    assertTrue(reader.start());
    while (reader.advance()) {
        IndexedRecord current = (IndexedRecord) reader.getCurrent();
        assertNotNull(current);
        assertEquals(getSystemSchema(), current.getSchema());
        assertEquals(3, current.getSchema().getFields().size());
    }
    reader.close();
}
 

import org.apache.avro.generic.IndexedRecord; //导入依赖的package包/类
/**
 * Test aggregate query field not case sensitive
 */
@Test
public void testAggregateQueryColumnNameCaseSensitive() throws Throwable {
    TSalesforceInputProperties props = createTSalesforceInputProperties(true, false);
    props.manualQuery.setValue(true);
    props.query.setValue("SELECT MIN(CreatedDate) value FROM Contact GROUP BY FirstName, LastName LIMIT 1");
    props.module.main.schema.setValue(SCHEMA_DATE);
    List<IndexedRecord> outputRows = readRows(props);
    if (outputRows.isEmpty()) {
        return;
    }
    IndexedRecord record = outputRows.get(0);
    assertNotNull(record.getSchema());
    Object value = record.get(0);
    Assert.assertTrue(value != null && value instanceof Long);
}
 

import org.apache.avro.generic.IndexedRecord; //导入依赖的package包/类
@Test
public void test_OneOutputRow() throws Exception {
    String inputAsString = generateInputJSON(inputSchema, inputIndexedRecord1);

    FixedFlowInputProperties properties = new FixedFlowInputProperties("test");
    properties.init();
    properties.schemaFlow.schema.setValue(inputSchema);
    properties.values.setValue(inputAsString);
    properties.nbRows.setValue(1);

    FixedFlowInputRuntime runtime = new FixedFlowInputRuntime();
    runtime.initialize(null, properties);

    PCollection<IndexedRecord> indexRecords = pipeline.apply(runtime);
    try (DirectCollector<IndexedRecord> collector = DirectCollector.of()) {
        indexRecords.apply(collector);

        // Run the pipeline to fill the collectors.
        pipeline.run().waitUntilFinish();;

        // Validate the contents of the collected outputs.
        List<IndexedRecord> outputs = collector.getRecords();
        assertEquals(1, outputs.size());
        assertEquals(inputIndexedRecord1.toString(), outputs.get(0).toString());
    }
}
 

import org.apache.avro.generic.IndexedRecord; //导入依赖的package包/类
@Test
public void testSyncMultipleLeads() throws Exception {
    doReturn(getSyncMultipleLeadsResult()).when(port).syncMultipleLeads(any(ParamsSyncMultipleLeads.class),
            any(AuthenticationHeader.class));
    oprops.afterOutputOperation();
    oprops.beforeMappingInput();
    mktoSR = client.syncLead(oprops, null);
    assertNotNull(mktoSR);
    assertFalse(mktoSR.isSuccess());
    IndexedRecord record = new Record(MarketoConstants.getSOAPOutputSchemaForSyncLead());
    record.put(0, 12345);
    record.put(1, "[email protected]");
    mktoSR = client.syncMultipleLeads(oprops, Arrays.asList(record));
    assertNotNull(mktoSR);
    assertTrue(mktoSR.isSuccess());
    //
    doThrow(new RuntimeException("error")).when(port).syncMultipleLeads(any(ParamsSyncMultipleLeads.class),
            any(AuthenticationHeader.class));
    mktoSR = client.syncMultipleLeads(oprops, Arrays.asList(record));
    assertNotNull(mktoSR);
    assertFalse(mktoSR.isSuccess());
}
 

import org.apache.avro.generic.IndexedRecord; //导入依赖的package包/类
public void createSampleDataset(String table) throws Throwable {
    tableClient.getTableReference(table).createIfNotExists();
    TAzureStorageOutputTableProperties props = new TAzureStorageOutputTableProperties("tests");
    props = (TAzureStorageOutputTableProperties) setupConnectionProperties(props);
    props.setupProperties();
    props.schema.schema.setValue(getDynamicSchema());
    props.actionOnTable.setValue(ActionOnTable.Default);
    props.actionOnData.setValue(ActionOnData.Insert);
    props.schemaListener.afterSchema();
    props.tableName.setValue(table);
    Writer<?> writer = createWriter(props);
    writer.open("test-uid");
    for (String p : partitions) {
        for (String r : rows) {
            IndexedRecord entity = new GenericData.Record(getWriteSchema());
            entity.put(0, p);
            entity.put(1, r);
            entity.put(2, RandomStringUtils.random(50));
            entity.put(3, RandomStringUtils.randomNumeric(10));
            writer.write(entity);
        }
    }
    writer.close();
}
 

import org.apache.avro.generic.IndexedRecord; //导入依赖的package包/类
@Test
public void testHierarchical_TFD2119_B1_AtLeastOneSubRecordHasValueGt10() throws Exception {
    DoFnTester<Object, IndexedRecord> fnTester = DoFnTester.of( //
            new FilterRowDoFn().withProperties(addCriteria(null, //
                    ".b1{.value > 10}", //
                    ConditionsRowConstant.Function.COUNT, //
                    ConditionsRowConstant.Operator.GREATER, //
                    "0") //

            ).withOutputSchema(true));

    List<IndexedRecord> output = fnTester.processBundle(inputB);
    for (IndexedRecord main : output) {
        boolean atLeastOne = false;
        for (IndexedRecord subrecord : getSubrecords(main)) {
            if ((double) subrecord.get(2) > 10)
                atLeastOne = true;
        }
        assertThat(main.toString(), atLeastOne, is(true));
    }
    assertThat(output, hasSize(274));
}
 

import org.apache.avro.generic.IndexedRecord; //导入依赖的package包/类
private void getSampleAction(JDBCDatasetProperties dataset) {
    JDBCDatasetRuntime runtime = new JDBCDatasetRuntime();
    runtime.initialize(null, dataset);
    final IndexedRecord[] record = new IndexedRecord[1];
    Consumer<IndexedRecord> storeTheRecords = new Consumer<IndexedRecord>() {

        @Override
        public void accept(IndexedRecord data) {
            record[0] = data;

        }
    };

    runtime.getSample(1, storeTheRecords);
    Assert.assertEquals("1", record[0].get(0));
    Assert.assertEquals("wangwei", record[0].get(1));
}
 

import org.apache.avro.generic.IndexedRecord; //导入依赖的package包/类
@Test
public void testGetCustomObjectWithCompoundKey() throws Exception {
    irProps.customObjectAction.setValue(CustomObjectAction.get);
    irProps.customObjectName.setValue(TEST_CO_NAME_CAR);
    irProps.validateFetchCustomObjectSchema();
    irProps.useCompoundKey.setValue(true);
    // "searchableFields": "[[\"customerId\",\"VIN\"],[\"marketoGUID\"],[\"customerId\"]]"
    irProps.compoundKey.keyName.setValue(Arrays.asList("customerId", "VIN"));
    irProps.compoundKey.keyValue.setValue(Arrays.asList("4137181", "WBA4R7C30HK896061"));// WBA4R7C55HK895912
    MarketoSource source = new MarketoSource();
    source.initialize(null, irProps);
    MarketoRESTClient client = (MarketoRESTClient) source.getClientService(null);
    MarketoRecordResult result = client.getCustomObjects(irProps, null);
    LOG.debug("result = {}.", result);
    assertNotNull(result.getRecords());
    assertEquals(1, result.getRecords().size());
    IndexedRecord record = result.getRecords().get(0);
    Schema s = record.getSchema();
    assertEquals(4137181, record.get(s.getField("customerId").pos()));
    assertEquals("WBA4R7C30HK896061", record.get(s.getField("VIN").pos()));
    assertEquals("FIT", record.get(s.getField("brand").pos()));
}
 

import org.apache.avro.generic.IndexedRecord; //导入依赖的package包/类
private void map(IndexedRecord input, ProcessContext context) throws IOException {
    // Prepare Python environment
    interpretor.set("inputJSON", new PyString(input.toString()));

    // Add user command
    interpretor.exec(pythonFunction);

    // Retrieve results
    interpretor.exec("outputJSON = userFunction(inputJSON)");
    PyObject output = interpretor.get("outputJSON");

    if (jsonGenericRecordConverter == null) {
        JsonSchemaInferrer jsonSchemaInferrer = new JsonSchemaInferrer(new ObjectMapper());
        Schema jsonSchema = jsonSchemaInferrer.inferSchema(output.toString());
        jsonGenericRecordConverter = new JsonGenericRecordConverter(jsonSchema);
    }

    GenericRecord outputRecord = jsonGenericRecordConverter.convertToAvro(output.toString());
    context.output(outputRecord);
}
 

import org.apache.avro.generic.IndexedRecord; //导入依赖的package包/类
@Test (expected = UnmodifiableAdapterException.class)
public void testConvertToAvroFailedToChangeRecord() {
    ByteBuf buffer = Mockito.mock(ByteBuf.class);

    // Mocking key object
    Mockito.when(buffer.getByte(0)).thenReturn(MessageUtil.MAGIC_REQ);
    Mockito.when(buffer.getByte(1)).thenReturn(MessageUtil.DCP_EXPIRATION_OPCODE);
    Mockito.when(buffer.getByte(4)).thenReturn(OFFSET);
    Mockito.when(buffer.getShort(2)).thenReturn(LENGTH);
    ByteBuf key = Mockito.mock(ByteBuf.class);
    Mockito.when(key.readableBytes()).thenReturn(4);
    Mockito.when(buffer.slice(29, 10)).thenReturn(key);

    IndexedRecord record = converter.convertToAvro(buffer);

    record.put(0, "Update is not supported");
}
 

import org.apache.avro.generic.IndexedRecord; //导入依赖的package包/类
@Test
public void testArrayOfArrayOfSimpleType() throws IOException {
    RecordingVisitor visitor = new RecordingVisitor();
    IndexedRecord record = loadRecord("arrayOfArrayOfSimpleType");

    VisitableRecord wrapper = new VisitableRecord(record);
    wrapper.accept(visitor);

    List<List<Integer>> array = createArrayOfArray();

    visitor.verifyRoot();
    visitor.verifyField("/intField", 123);
    visitor.verifyField("/arrayOfArrayOfSimpleType", array);
    visitor.verifyField("/arrayOfArrayOfSimpleType[0]", array.get(0));
    visitor.verifyField("/arrayOfArrayOfSimpleType[0][0]", array.get(0).get(0));
    visitor.verifyField("/arrayOfArrayOfSimpleType[0][1]", array.get(0).get(1));
    visitor.verifyField("/arrayOfArrayOfSimpleType[0][2]", array.get(0).get(2));
    visitor.verifyField("/arrayOfArrayOfSimpleType[1]", array.get(1));
    visitor.verifyField("/arrayOfArrayOfSimpleType[1][0]", array.get(1).get(0));
    visitor.verifyField("/arrayOfArrayOfSimpleType[1][1]", array.get(1).get(1));
    visitor.verifyField("/arrayOfArrayOfSimpleType[1][2]", array.get(1).get(2));
    Assert.assertTrue("Visitor not verified", visitor.isVerified());
}