Java Field.addProp方法代码示例

import org.apache.avro.Schema.Field; //导入方法依赖的package包/类
public static Schema getSchemaForThisFields(String schemaName, FieldDescription[] fields, String[] keys) {
    Schema schema = Schema.createRecord(schemaName, "", "", false);
    List<Field> fieldList = new ArrayList<>();
    if (fields == null) {
        return null;
    }
    for (FieldDescription field : fields) {
        Field f = field.toAvroField();
        for (String key : keys) {
            if (field.getName().equals(key)) {
                f.addProp(SchemaConstants.TALEND_COLUMN_IS_KEY, "true");
            }
        }
        fieldList.add(f);
    }
    schema.setFields(fieldList);

    return schema;
}
 

import org.apache.avro.Schema.Field; //导入方法依赖的package包/类
public Schema generate(String schemaNameOverride) throws IOException {
  ClassWriter classWriter = new ClassWriter(options, connManager,
      tableName, null);
  Map<String, Integer> columnTypes = classWriter.getColumnTypes();
  String[] columnNames = classWriter.getColumnNames(columnTypes);

  List<Field> fields = new ArrayList<Field>();
  for (String columnName : columnNames) {
    String cleanedCol = AvroUtil.toAvroIdentifier(ClassWriter.toJavaIdentifier(columnName));
    int sqlType = columnTypes.get(columnName);
    Schema avroSchema = toAvroSchema(sqlType, columnName);
    Field field = new Field(cleanedCol, avroSchema, null,  NullNode.getInstance());
    field.addProp("columnName", columnName);
    field.addProp("sqlType", Integer.toString(sqlType));
    fields.add(field);
  }

  TableClassName tableClassName = new TableClassName(options);
  String shortClassName = tableClassName.getShortClassForTable(tableName);
  String avroTableName = (tableName == null ? TableClassName.QUERY_RESULT : tableName);
  String avroName = schemaNameOverride != null ? schemaNameOverride :
      (shortClassName == null ? avroTableName : shortClassName);
  String avroNamespace = tableClassName.getPackageForTable();

  String doc = "Sqoop import of " + avroTableName;
  Schema schema = Schema.createRecord(avroName, doc, avroNamespace, false);
  schema.setFields(fields);
  schema.addProp("tableName", avroTableName);
  return schema;
}
 

import org.apache.avro.Schema.Field; //导入方法依赖的package包/类
@Override
public Schema convertSchema(JsonArray schema, WorkUnitState workUnit) throws SchemaConversionException {
  List<Schema.Field> fields = new ArrayList<Schema.Field>();

  for (JsonElement elem : schema) {
    JsonObject map = (JsonObject) elem;

    String columnName = map.get("columnName").getAsString();
    String comment = map.get("comment").getAsString();
    boolean nullable = map.has("isNullable") ? map.get("isNullable").getAsBoolean() : false;
    Schema fldSchema;

    try {
      JsonElementConversionFactory.JsonElementConverter converter =
          JsonElementConversionFactory.getConvertor(columnName, map.get("dataType").getAsJsonObject().get("type")
              .getAsString(), map, workUnit, nullable);
      converters.put(columnName, converter);
      fldSchema = converter.getSchema();
    } catch (UnsupportedDateTypeException e) {
      throw new SchemaConversionException(e);
    }

    Field fld = new Field(columnName, fldSchema, comment, nullable ? JsonNodeFactory.instance.nullNode() : null);
    fld.addProp("source.type", map.get("dataType").getAsJsonObject().get("type").getAsString());
    fields.add(fld);
  }

  Schema avroSchema =
      Schema.createRecord(workUnit.getExtract().getTable(), "", workUnit.getExtract().getNamespace(), false);
  avroSchema.setFields(fields);

  if (workUnit.getPropAsBoolean(CONVERTER_AVRO_NULLIFY_FIELDS_ENABLED, DEFAULT_CONVERTER_AVRO_NULLIFY_FIELDS_ENABLED)) {
    return this.generateSchemaWithNullifiedField(workUnit, avroSchema);
  }

  return avroSchema;
}
 

import org.apache.avro.Schema.Field; //导入方法依赖的package包/类
protected Schema getSchema(RuntimeContainer container, Connection connection, String tableName) throws IOException {
    Schema tableSchema = null;

    SnowflakeConnectionProperties connProps = getEffectiveConnectionProperties(container);
    try {
        DatabaseMetaData metaData = connection.getMetaData();

        ResultSet resultSet = metaData.getColumns(getCatalog(connProps), getDbSchema(connProps), tableName, null);
        tableSchema = getSnowflakeAvroRegistry().inferSchema(resultSet);
        if (tableSchema == null) {
            throw new IOException(i18nMessages.getMessage("error.tableNotFound", tableName));
        }

        // Update the schema with Primary Key details
        // FIXME - move this into the inferSchema stuff
        ResultSet keysIter = metaData.getPrimaryKeys(getCatalog(connProps), getDbSchema(connProps), tableName);

        List<String> pkColumns = new ArrayList<>(); // List of Primary Key columns for this table
        while (keysIter.next()) {
            pkColumns.add(keysIter.getString("COLUMN_NAME"));
        }

        for (Field f : tableSchema.getFields()) {
            if (pkColumns.contains(f.name())) {
                f.addProp(SchemaConstants.TALEND_COLUMN_IS_KEY, "true");
            }
        }

    } catch (SQLException se) {
        throw new IOException(se);
    }

    return tableSchema;

}
 

import org.apache.avro.Schema.Field; //导入方法依赖的package包/类
public static List<Field> getSchemaFieldsList(Schema schema) {
    List<Field> result = new ArrayList<>();
    for (Field f : schema.getFields()) {
        Field nf = new Field(f.name(), f.schema(), f.doc(), f.defaultVal());
        nf.getObjectProps().putAll(f.getObjectProps());
        for (Map.Entry<String, Object> entry : f.getObjectProps().entrySet()) {
            nf.addProp(entry.getKey(), entry.getValue());
        }
        result.add(nf);
    }
    return result;
}
 

import org.apache.avro.Schema.Field; //导入方法依赖的package包/类
private Field getMigratedField(Field origin, Schema expectedSchema, String expectedDIType) {
    Field expectedField = new Schema.Field(origin.name(), expectedSchema, origin.doc(), origin.defaultVal(), origin.order());
    for (Map.Entry<String, Object> entry : origin.getObjectProps().entrySet()) {
        if ("di.column.talendType".equals(entry.getKey())) {
            expectedField.addProp("di.column.talendType", expectedDIType);
        } else {
            expectedField.addProp(entry.getKey(), entry.getValue());
        }
    }
    return expectedField;
}
 

import org.apache.avro.Schema.Field; //导入方法依赖的package包/类
@Test
public void testGenerateNewField() throws Exception {
    Field in = new Schema.Field("email", AvroUtils._string(), "doc", null, Order.ASCENDING);
    in.addProp("test", "testvalue");
    Field out = MarketoUtils.generateNewField(in);
    assertEquals("email", out.name());
    assertEquals("string", out.schema().getType().getName());
    assertEquals("doc", out.doc());
    assertEquals(Order.ASCENDING, out.order());
    assertNotNull(out.getProp("test"));
    assertEquals("testvalue", out.getProp("test"));
}
 

import org.apache.avro.Schema.Field; //导入方法依赖的package包/类
@Override
protected Field sqlType2Avro(int size, int scale, int dbtype, boolean nullable, String name, String dbColumnName,
        Object defaultValue) {
    Field field = null;
    Schema schema = null;

    switch (dbtype) {
    case java.sql.Types.VARCHAR:
    case java.sql.Types.LONGVARCHAR:
    case java.sql.Types.CHAR:
        schema = AvroUtils._string();
        field = wrap(name, schema, nullable, defaultValue);
        field.addProp(SchemaConstants.TALEND_COLUMN_DB_LENGTH, size);
        break;
    case java.sql.Types.INTEGER:
    case java.sql.Types.DECIMAL:
    case java.sql.Types.BIGINT:
    case java.sql.Types.NUMERIC:
    case java.sql.Types.TINYINT:
    case java.sql.Types.SMALLINT:
        schema = AvroUtils._decimal();
        field = wrap(name, schema, nullable, defaultValue);
        field.addProp(SchemaConstants.TALEND_COLUMN_PRECISION, size);
        field.addProp(SchemaConstants.TALEND_COLUMN_SCALE, scale);
        break;
    case java.sql.Types.DOUBLE:
    case java.sql.Types.FLOAT:
    case java.sql.Types.REAL:
        schema = AvroUtils._double();
        field = wrap(name, schema, nullable, defaultValue);
        break;
    case java.sql.Types.DATE:
        schema = AvroUtils._int();
        LogicalTypes.date().addToSchema(schema);
        field = wrap(name, schema, nullable, defaultValue);
        field.addProp(SchemaConstants.TALEND_COLUMN_PATTERN, SnowflakeConstants.TALEND_DEFAULT_DATE_PATTERN);
        break;
    case java.sql.Types.TIME:
        schema = AvroUtils._int();
        LogicalTypes.timeMillis().addToSchema(schema);
        field = wrap(name, schema, nullable, defaultValue);
        break;
    case java.sql.Types.TIMESTAMP:
        schema = AvroUtils._long();
        LogicalTypes.timestampMillis().addToSchema(schema);
        field = wrap(name, schema, nullable, defaultValue);
        field.addProp(SchemaConstants.TALEND_COLUMN_PATTERN, SnowflakeConstants.TALEND_DAFEULT_TIMESTAMP_PATTERN);
        break;
    case java.sql.Types.BOOLEAN:
        schema = AvroUtils._boolean();
        field = wrap(name, schema, nullable, defaultValue);
        break;
    default:
        schema = AvroUtils._string();
        field = wrap(name, schema, nullable, defaultValue);
        break;
    }

    field.addProp(SchemaConstants.TALEND_COLUMN_DB_TYPE, dbtype);
    field.addProp(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, dbColumnName);

    if (defaultValue != null) {
        field.addProp(SchemaConstants.TALEND_COLUMN_DEFAULT, defaultValue);
    }

    return field;
}
 

import org.apache.avro.Schema.Field; //导入方法依赖的package包/类
public Field toAvroField() {
    Schema fs = null;
    String fname = getName().replaceAll("-", "_");
    //
    fs = AvroUtils._string();
    switch (getDataType()) {
    case ("string"):
    case ("text"):
    case ("phone"):
    case ("email"):
    case ("url"):
    case ("lead_function"):
    case ("reference"):
        fs = AvroUtils._string();
        break;
    case ("integer"):
        fs = AvroUtils._int();
        break;
    case ("boolean"):
        fs = AvroUtils._boolean();
        break;
    case ("float"):
    case ("currency"):
        fs = AvroUtils._float();
        break;
    case ("date"):
    case ("datetime"):
        fs = AvroUtils._date();
        break;
    default:
        LOG.warn("Non managed type : {}. for {}. Defaulting to String.", getDataType(), this);
    }
    Field f = new Field(fname, fs, getDisplayName(), (Object) null);
    f.addProp(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, getName());
    if (getLength() != null) {
        f.addProp(SchemaConstants.TALEND_COLUMN_DB_LENGTH, getLength().toString());
    }
    if (fs.equals(AvroUtils._date())) {
        f.addProp(SchemaConstants.TALEND_COLUMN_PATTERN, MarketoConstants.DATETIME_PATTERN_REST);
        f.addProp(SchemaConstants.JAVA_CLASS_FLAG, "java.util.Date");
    }
    if (updateable != null && !updateable) {
        f.addProp(SchemaConstants.TALEND_IS_LOCKED, "true");
    }
    //
    if (getId() != null) {
        f.addProp("mktoId", getId().toString());
    }
    f.addProp("mktoType", getDataType());
    return f;
}