Java Cleaner类代码示例

import no.priv.garshol.duke.Cleaner; //导入依赖的package包/类
public void addValue(Column col, String value) {
  if (value == null || value.equals(""))
    return;
  
  String prop = col.getProperty();
  Cleaner cleaner = col.getCleaner();
  if (col.isSplit()) {
    for (String v : col.split(value)) {
      if (cleaner != null)
        v = cleaner.clean(v);
      if (v != null && !v.equals(""))
        record.addValue(prop, v);
    }
  } else {
    if (cleaner != null)
      value = cleaner.clean(value);
    if (value != null && !value.equals(""))
      record.addValue(prop, value);
  }
}
 

import no.priv.garshol.duke.Cleaner; //导入依赖的package包/类
private static double computeProb(Collection<String> vs1, Collection<String> vs2, Comparator comp, List<Cleaner> cleanersList, Double max, Double min) {
    double high = 0.0;
    for (String v1 : vs1) {
        if (v1.equals("")) {
            continue;
        }

        v2fieldloop:
        for (String v2 : vs2) {
            if (v2.equals("")) {
                continue;
            }

            for (Cleaner cl : cleanersList) {
                v2 = cl.clean(v2);
                if ((v2 == null) || v2.equals("")) {
                    continue v2fieldloop;
                }
            }
            double p = compare(v1, v2, max, min, comp);
            high = Math.max(high, p);
        }
    }
    return high;
}
 

import no.priv.garshol.duke.Cleaner; //导入依赖的package包/类
/**
 * . Reads & instantiates cleaners
 *
 * @param cleanersList array of cleaners from JSON
 * @return the list of instantiated cleaners
 */
private static List<Cleaner> getCleaners(
        final List<Map<String, String>> cleanersList) {
    List<Cleaner> cleanList = new ArrayList<>();

    for (Map aCleaner : cleanersList) {
        Cleaner cleaner = (Cleaner) instantiate((String) aCleaner.get(NAME));
        setParams(cleaner, aCleaner.get(PARAMS));
        cleanList.add(cleaner);
    }
    return cleanList;
}
 

import no.priv.garshol.duke.Cleaner; //导入依赖的package包/类
private void readFields(List<Map<String, Object>> fields, Map<String, Collection<String>> props) {
    for (Map<String, Object> value : fields) {
        String field = (String) value.get("field");
        String fieldValue = (String) value.get("value");
        for (Cleaner cl : (ArrayList<Cleaner>) entityParams.get(field).get(CLEANERS)) {
            fieldValue = cl.clean(fieldValue);
        }
        props.put(field, Collections.singleton(fieldValue));
    }
}
 

import no.priv.garshol.duke.Cleaner; //导入依赖的package包/类
/**
 * . Configures with data from JSON payload only
 *
 * @param fieldsParams fields parameters from JSON
 * @return the record for comparison
 */
private Record configureWithFieldsOnly(
        final List<Map<String, Object>> fieldsParams) {

    Map<String, Collection<String>> props =
            new HashMap<>();
    entityParams = new HashMap<>();

    for (Map<String, Object> value : fieldsParams) {
        HashMap<String, Object> map = new HashMap<>();

        String field = (String) value.get("field");

        List<Cleaner> cleanList =
                getCleaners((ArrayList<Map<String, String>>) value.get(CLEANERS));

        map.put(CLEANERS, cleanList);

        Double maxValue = 0.0;
        if (value.get(HIGH) != null) {
            maxValue = (Double) value.get(HIGH);
        }

        Double minValue = 0.0;
        if (value.get(LOW) != null) {
            minValue = (Double) value.get(LOW);
        }

        Comparator comp = getComparator(value);

        map.put(HIGH, maxValue);
        map.put(LOW, minValue);
        map.put(COMPARATOR, comp);

        entityParams.put(field, map);
    }

    readFields(fieldsParams, props);
    return new RecordImpl(props);
}
 

import no.priv.garshol.duke.Cleaner; //导入依赖的package包/类
public Column(String name, String property, String prefix, Cleaner cleaner) {
  this.name = name;
  this.property = property;
  this.prefix = prefix;
  this.cleaner = cleaner;
}
 

import no.priv.garshol.duke.Cleaner; //导入依赖的package包/类
public Cleaner getCleaner() {
  return cleaner;
}
 

import no.priv.garshol.duke.Cleaner; //导入依赖的package包/类
public ChainedCleaner(Cleaner[] cleaners) {
  this.cleaners = cleaners;
}
 

import no.priv.garshol.duke.Cleaner; //导入依赖的package包/类
/**
 * Compares two records and returns the probability that they represent the
 * same real-world entity.
 *
 * @param r1     1st Record
 * @param r2     2nd Record
 * @param params Parameters for comparison
 * @return Bayesian probability
 */

private static double compare(
        final Record r1,
        final Record r2,
        final Map<String, HashMap<String, Object>> params) {
    double prob = AVERAGE_SCORE;

    for (String propname : r1.getProperties()) {
        Collection<String> vs1 = r1.getValues(propname);
        Collection<String> vs2 = r2.getValues(propname);

        Boolean v1empty = true;
        for (String v1 : vs1) {
            if (!v1.equals("")) {
                v1empty = false;
                break;
            }
        }

        Boolean v2empty = true;
        for (String v2 : vs2) {
            if (!v2.equals("")) {
                v2empty = false;
                break;
            }
        }


        if (vs1.isEmpty() || vs2.isEmpty() || v1empty || v2empty) {
            continue; // no values to compare, so skip
        }

        Comparator comp =
                (Comparator) params.get(propname).get(COMPARATOR);
        ArrayList<Cleaner> cleanersList =
                (ArrayList<Cleaner>) params.get(propname).get(CLEANERS);

        Double max = (Double) params.get(propname).get(HIGH);
        Double min = (Double) params.get(propname).get(LOW);

        double high = computeProb(vs1, vs2, comp, cleanersList, max, min);
        prob = computeBayes(prob, high);
    }
    return prob;
}