Java DataDictionary类代码示例

import org.dmg.pmml.DataDictionary; //导入依赖的package包/类
/**
 * Validates that the encoded PMML model received matches expected schema.
 *
 * @param pmml {@link PMML} encoding of KMeans Clustering
 * @param schema expected schema attributes of KMeans Clustering
 */
public static void validatePMMLVsSchema(PMML pmml, InputSchema schema) {
  List<Model> models = pmml.getModels();
  Preconditions.checkArgument(models.size() == 1,
      "Should have exactly one model, but had %s", models.size());

  Model model = models.get(0);
  Preconditions.checkArgument(model instanceof ClusteringModel);
  Preconditions.checkArgument(model.getMiningFunction() == MiningFunction.CLUSTERING);

  DataDictionary dictionary = pmml.getDataDictionary();
  Preconditions.checkArgument(
      schema.getFeatureNames().equals(AppPMMLUtils.getFeatureNames(dictionary)),
      "Feature names in schema don't match names in PMML");

  MiningSchema miningSchema = model.getMiningSchema();
  Preconditions.checkArgument(schema.getFeatureNames().equals(
      AppPMMLUtils.getFeatureNames(miningSchema)));

}
 

import org.dmg.pmml.DataDictionary; //导入依赖的package包/类
@Test
public void testBuildDataDictionary() {
  Map<Integer,Collection<String>> distinctValues = new HashMap<>();
  distinctValues.put(1, Arrays.asList("one", "two", "three", "four", "five"));
  CategoricalValueEncodings categoricalValueEncodings =
      new CategoricalValueEncodings(distinctValues);

  DataDictionary dictionary =
      AppPMMLUtils.buildDataDictionary(buildTestSchema(), categoricalValueEncodings);
  assertEquals(4, dictionary.getNumberOfFields().intValue());
  checkDataField(dictionary.getDataFields().get(0), "foo", null);
  checkDataField(dictionary.getDataFields().get(1), "bar", true);
  checkDataField(dictionary.getDataFields().get(2), "baz", null);
  checkDataField(dictionary.getDataFields().get(3), "bing", false);

  List<Value> dfValues = dictionary.getDataFields().get(1).getValues();
  assertEquals(5, dfValues.size());
  String[] categoricalValues = { "one", "two", "three", "four", "five" };
  for (int i = 0; i < categoricalValues.length; i++) {
    assertEquals(categoricalValues[i], dfValues.get(i).getValue());
  }
}
 

import org.dmg.pmml.DataDictionary; //导入依赖的package包/类
@Test
public void testBuildCategoricalEncoding() {
  List<DataField> dataFields = new ArrayList<>();
  dataFields.add(new DataField(FieldName.create("foo"), OpType.CONTINUOUS, DataType.DOUBLE));
  DataField barField =
      new DataField(FieldName.create("bar"), OpType.CATEGORICAL, DataType.STRING);
  barField.addValues(new Value("b"), new Value("a"));
  dataFields.add(barField);
  DataDictionary dictionary = new DataDictionary(dataFields).setNumberOfFields(dataFields.size());
  CategoricalValueEncodings encodings = AppPMMLUtils.buildCategoricalValueEncodings(dictionary);
  assertEquals(2, encodings.getValueCount(1));
  assertEquals(0, encodings.getValueEncodingMap(1).get("b").intValue());
  assertEquals(1, encodings.getValueEncodingMap(1).get("a").intValue());
  assertEquals("b", encodings.getEncodingValueMap(1).get(0));
  assertEquals("a", encodings.getEncodingValueMap(1).get(1));
  assertEquals(Collections.singletonMap(1, 2), encodings.getCategoryCounts());
}
 

import org.dmg.pmml.DataDictionary; //导入依赖的package包/类
public static DataField getRawDataField(DataDictionary dataDictionary, String rawFieldName) {
    // now find the actual dataField
    DataField rawField = null;
    for (DataField dataField : dataDictionary.getDataFields()) {
        String rawDataFieldName = dataField.getName().getValue();
        if (rawDataFieldName.equals(rawFieldName)) {
            rawField = dataField;
            break;
        }
    }
    if (rawField == null) {
        throw new UnsupportedOperationException("Could not trace back {} to a raw input field. Maybe saomething is not implemented " +
                "yet or the PMML file is faulty.");
    }
    return rawField;
}
 

import org.dmg.pmml.DataDictionary; //导入依赖的package包/类
@Override
public ModelAndModelInputEvaluator<MapModelInput, String> buildFromPMML(TreeModel treeModel, DataDictionary dataDictionary,
                                                                TransformationDictionary transformationDictionary) {
    if (treeModel.getFunctionName().value().equals("classification")
            && treeModel.getSplitCharacteristic().value().equals("binarySplit")
            && treeModel.getMissingValueStrategy().value().equals("defaultChild")
            && treeModel.getNoTrueChildStrategy().value().equals("returnLastPrediction")) {

        List<VectorRange> fields = getFieldValuesList(treeModel, dataDictionary, transformationDictionary);
        VectorRangesToVectorPMML.VectorRangesToVectorPMMLTreeModel fieldsToVector =
                new VectorRangesToVectorPMML.VectorRangesToVectorPMMLTreeModel(fields);
        Map<String, String> fieldToTypeMap = getFieldToTypeMap(fields);
        EsTreeModel esTreeModel = getEsTreeModel(treeModel, fieldToTypeMap);
        return new ModelAndModelInputEvaluator<>(fieldsToVector, esTreeModel);
    } else {
        throw new UnsupportedOperationException("TreeModel does not support the following parameters yet: "
                + " splitCharacteristic:" + treeModel.getSplitCharacteristic().value()
                + " missingValueStrategy:" + treeModel.getMissingValueStrategy().value()
                + " noTrueChildStrategy:" + treeModel.getNoTrueChildStrategy().value());
    }
}
 

import org.dmg.pmml.DataDictionary; //导入依赖的package包/类
protected static List<VectorRange> getFieldValuesList(TreeModel treeModel, DataDictionary dataDictionary,
                                                      TransformationDictionary transformationDictionary) {
    // walk the tree model and gather all the field name
    Set<String> fieldNames = new HashSet<>();
    Node startNode = treeModel.getNode();
    getFieldNamesFromNode(fieldNames, startNode);
    // create the actual VectorRange objects, copy paste much from GLMHelper
    List<VectorRange> fieldsToValues = new ArrayList<>();
    List<DerivedField> allDerivedFields = ProcessPMMLHelper.getAllDerivedFields(treeModel, transformationDictionary);
    for(String fieldName : fieldNames) {
        List<DerivedField> derivedFields = new ArrayList<>();
        String rawFieldName = ProcessPMMLHelper.getDerivedFields(fieldName, allDerivedFields, derivedFields);
        DataField rawField = ProcessPMMLHelper.getRawDataField(dataDictionary, rawFieldName);
        MiningField miningField = ProcessPMMLHelper.getMiningField(treeModel, rawFieldName);
        fieldsToValues.add(new PMMLVectorRange.FieldToValue(rawField, miningField, derivedFields.toArray(new
                DerivedField[derivedFields.size()])));
    }
    return fieldsToValues;
}
 

import org.dmg.pmml.DataDictionary; //导入依赖的package包/类
private List<VectorRange> convertToFeatureEntries(GeneralRegressionModel grModel,
                                                  TransformationDictionary transformationDictionary,
                                                  DataDictionary dataDictionary, TreeMap<String, List<PPCell>> fieldToPPCellMap,
                                                  List<String> orderedParameterList) {
    // for each predictor: get vector entries?
    List<VectorRange> vectorRangeList = new ArrayList<>();
    int indexCounter = 0;
    // for each of the fields create the feature entries
    for (String fieldname : fieldToPPCellMap.keySet()) {
        PMMLVectorRange featureEntries = ProcessPMMLHelper.extractVectorRange(grModel, dataDictionary,
                transformationDictionary, fieldname, () -> {
                    // sort values first
                    return fieldToPPCellMap.get(fieldname).stream().map(PPCell::getValue).collect(Collectors.toList());
                }, indexCounter, null);

        for (PPCell cell : fieldToPPCellMap.get(fieldname)) {
            orderedParameterList.add(cell.getParameterName());
        }
        indexCounter += featureEntries.size();
        vectorRangeList.add(featureEntries);

    }
    return vectorRangeList;
}
 

import org.dmg.pmml.DataDictionary; //导入依赖的package包/类
private String[] findTargetCategories(DataDictionary dataDictionary, Map<String, List<PCell>> targetClassPCellMap,
                                      String targetVariable) {
    String[] targetCategories = new String[2];
    String class1 = targetClassPCellMap.keySet().iterator().next();
    targetCategories[0] = class1;
    // find it in the datafields
    for (DataField dataField : dataDictionary.getDataFields()) {
        if (dataField.getName().toString().equals(targetVariable)) {
            for (Value value : dataField.getValues()) {
                String valueString = value.getValue();
                if (valueString.equals(class1) == false) {
                    targetCategories[1] = valueString;
                }
            }
            if (targetCategories[1] == null) {
                throw new ElasticsearchParseException("could not find target class");
            }
            break;
        }
    }
    return targetCategories;
}
 

import org.dmg.pmml.DataDictionary; //导入依赖的package包/类
private ModelAndModelInputEvaluator<VectorModelInput, String> initModel(RegressionModel model,
                                                                        DataDictionary dataDictionary,
                                                                        TransformationDictionary transformationDictionary,
                                                                        RegressionModelConstructor constructor) {
    List<VectorRange> vectorRanges = new ArrayList<>();
    int indexCounter = 0;
    Map<String, OpType> types = new HashMap<>();
    // TODO: add
    RegressionTable regressionTable = model.getRegressionTables().get(0);
    for (NumericPredictor predictor : regressionTable.getNumericPredictors()) {
        PMMLVectorRange vectorRange = ProcessPMMLHelper.extractVectorRange(model, dataDictionary,
                transformationDictionary, predictor.getName().getValue(), () -> {
                    throw new IllegalArgumentException("Categorical fields are not supported yet");
                }, indexCounter, types);
        vectorRanges.add(vectorRange);
        indexCounter += vectorRange.size();
    }
    VectorModelInputEvaluator vectorPMML = new VectorModelInputEvaluator(vectorRanges);
    EsModelEvaluator<VectorModelInput, String> modelEvaluator = buildLinerModel(model, constructor);
    return new ModelAndModelInputEvaluator<>(vectorPMML, modelEvaluator);
}
 

import org.dmg.pmml.DataDictionary; //导入依赖的package包/类
@Override
public PMMLObject popParent(){
	PMMLObject parent = super.popParent();

	if(parent instanceof Model){
		Model model = (Model)parent;

		processModel(model);
	} else

	if(parent instanceof PMML){
		PMML pmml = (PMML)parent;

		DataDictionary dataDictionary = pmml.getDataDictionary();
		if(dataDictionary != null){
			processDataDictionary(dataDictionary);
		}
	}

	return parent;
}
 

import org.dmg.pmml.DataDictionary; //导入依赖的package包/类
@Test
public void marshal() throws Exception {
	RegressionModel regressionModel = new RegressionModel()
		.addRegressionTables(new RegressionTable());

	PMML pmml = new PMML(Version.PMML_4_3.getVersion(), new Header(), new DataDictionary())
		.addModels(regressionModel);

	JAXBContext context = JAXBContextFactory.createContext(new Class[]{org.dmg.pmml.ObjectFactory.class, org.dmg.pmml.regression.ObjectFactory.class}, null);

	Marshaller marshaller = context.createMarshaller();

	ByteArrayOutputStream os = new ByteArrayOutputStream();

	marshaller.marshal(pmml, os);

	String string = os.toString("UTF-8");

	assertTrue(string.contains("<PMML xmlns=\"http://www.dmg.org/PMML-4_3\" version=\"4.3\">"));
	assertTrue(string.contains("<RegressionModel>"));
	assertTrue(string.contains("</RegressionModel>"));
	assertTrue(string.contains("</PMML>"));
}
 

import org.dmg.pmml.DataDictionary; //导入依赖的package包/类
@Override
public VisitorAction visit(final DataDictionary dataDictionary){
	check(new CollectionSize(dataDictionary){

		@Override
		public Integer getSize(){
			return dataDictionary.getNumberOfFields();
		}

		@Override
		public Collection<?> getCollection(){
			return dataDictionary.getDataFields();
		}
	});

	return super.visit(dataDictionary);
}
 

import org.dmg.pmml.DataDictionary; //导入依赖的package包/类
static
private TreeModelEvaluator createTreeModelEvaluator(MiningFunction miningFunction, MathContext mathContext, Target target){
	Node root = new Node()
		.setPredicate(new False());

	Targets targets = new Targets()
		.addTargets(target);

	TreeModel treeModel = new TreeModel(miningFunction, new MiningSchema(), root)
		.setSplitCharacteristic(TreeModel.SplitCharacteristic.BINARY_SPLIT)
		.setMathContext(mathContext)
		.setTargets(targets);

	PMML pmml = new PMML(Version.PMML_4_3.getVersion(), new Header(), new DataDictionary())
		.addModels(treeModel);

	return new TreeModelEvaluator(pmml);
}
 

import org.dmg.pmml.DataDictionary; //导入依赖的package包/类
public static Map<Integer, BiMap<String, Integer>> buildColumnCategoryMapping(DataDictionary dictionary) {
  Preconditions.checkNotNull(dictionary);
  InboundSettings settings = InboundSettings.create(ConfigUtils.getDefaultConfig());
  List<String> columnNames = settings.getColumnNames();

  Preconditions.checkNotNull(dictionary);
  Map<Integer,BiMap<String,Integer>> columnToCategoryNameToIDMapping = Maps.newHashMap();
  for (TypeDefinitionField field : dictionary.getDataFields()) {
    Collection<Value> values = field.getValues();
    if (values != null && !values.isEmpty()) {
      String columnName = field.getName().getValue();
      int columnNumber = columnNames.indexOf(columnName);
      BiMap<String,Integer> categoryNameToID = columnToCategoryNameToIDMapping.get(columnNumber);
      if (categoryNameToID == null) {
        categoryNameToID = HashBiMap.create();
        columnToCategoryNameToIDMapping.put(columnNumber, categoryNameToID);
      }
      for (Value value : values) {
        categoryNameToID.put(value.getValue(), categoryNameToID.size());
      }
    }
  }
  return columnToCategoryNameToIDMapping;
}
 

import org.dmg.pmml.DataDictionary; //导入依赖的package包/类
public static DataDictionary buildDataDictionary(
    InputSchema schema,
    CategoricalValueEncodings categoricalValueEncodings) {
  List<String> featureNames = schema.getFeatureNames();

  List<DataField> dataFields = new ArrayList<>();
  for (int featureIndex = 0; featureIndex < featureNames.size(); featureIndex++) {
    String featureName = featureNames.get(featureIndex);
    OpType opType;
    DataType dataType;
    if (schema.isNumeric(featureName)) {
      opType = OpType.CONTINUOUS;
      dataType = DataType.DOUBLE;
    } else if (schema.isCategorical(featureName)) {
      opType = OpType.CATEGORICAL;
      dataType = DataType.STRING;
    } else {
      // Don't know
      opType = null;
      dataType = null;
    }
    DataField field = new DataField(FieldName.create(featureName), opType, dataType);
    if (schema.isCategorical(featureName)) {
      categoricalValueEncodings.getEncodingValueMap(featureIndex).entrySet().stream().
          sorted(Comparator.comparing(Map.Entry::getKey)).
          map(Map.Entry::getValue).
          forEach(value -> field.addValues(new Value(value)));
    }
    dataFields.add(field);
  }

  return new DataDictionary(dataFields).setNumberOfFields(dataFields.size());
}