Java Vectors.dense方法代码示例

import org.apache.spark.mllib.linalg.Vectors; //导入方法依赖的package包/类
/**
 * Method of compare two search resutls
 *
 * @param o1 search result 1
 * @param o2 search result 2
 * @return 1 if o1 is greater than o2, 0 otherwise
 */
public int comp(SResult o1, SResult o2) {
  List<Double> instList = new ArrayList<>();
  for (int i = 0; i < SResult.rlist.length; i++) {
    double o2Score = SResult.get(o2, SResult.rlist[i]);
    double o1Score = SResult.get(o1, SResult.rlist[i]);
    instList.add(o2Score - o1Score);
  }

  double[] ins = instList.stream().mapToDouble(i -> i).toArray();
  LabeledPoint insPoint = new LabeledPoint(99.0, Vectors.dense(ins));
  double prediction = le.classify(insPoint);
  if (equalComp(prediction, 1)) { //different from weka where the return value is 1 or 2
    return 0;
  } else {
    return 1;
  }
}
 

import org.apache.spark.mllib.linalg.Vectors; //导入方法依赖的package包/类
/**
 * Create a distributed matrix given an Apache Commons RealMatrix.
 *
 * @param sc Never {@code null}
 * @param realMat Apache Commons RealMatrix.  Never {@code null}
 * @return A distributed Spark matrix
 */
public static RowMatrix convertRealMatrixToSparkRowMatrix(JavaSparkContext sc, RealMatrix realMat, int numSlices) {
    logger.info("Converting matrix to distributed Spark matrix...");
    final double [][] dataArray = realMat.getData();
    final LinkedList<Vector> rowsList = new LinkedList<>();
    for (final double [] i : dataArray) {
        final Vector currentRow = Vectors.dense(i);
        rowsList.add(currentRow);
    }

    // We may want to swap out this static value for something dynamic (as shown below), but this seems to slow it down.
    // final int totalSpace = realMat.getColumnDimension() * realMat.getRowDimension() * Double.BYTES;
    // // Want the partitions to be ~100KB of space
    // final int slices = totalSpace/100000;
    final JavaRDD<Vector> rows = sc.parallelize(rowsList, numSlices);

    // Create a RowMatrix from JavaRDD<Vector>.
    final RowMatrix mat = new RowMatrix(rows.rdd());
    logger.info("Done converting matrix to distributed Spark matrix...");
    return mat;
}
 

import org.apache.spark.mllib.linalg.Vectors; //导入方法依赖的package包/类
@Override
public Vector call(String[] tokens) throws MLModelBuilderException {
    try {
        double[] features = new double[indices.size()];
        int i = 0;
        for (int j : indices) {
            if (NumberUtils.isNumber(tokens[j])) {
                features[i] = Double.parseDouble(tokens[j]);
            }
            i++;
        }
        return Vectors.dense(features);
    } catch (Exception e) {
        throw new MLModelBuilderException(
                "An error occurred while converting tokens to vectors: " + e.getMessage(), e);
    }
}
 

import org.apache.spark.mllib.linalg.Vectors; //导入方法依赖的package包/类
@Override
public LabeledPoint call(Tuple2<WritableComparable, HCatRecord> tuple) throws Exception {
  HCatRecord record = tuple._2();

  if (record == null) {
    log.info("@@@ Null record");
    return defaultLabeledPoint;
  }

  double[] features = new double[numFeatures];

  for (int i = 0; i < numFeatures; i++) {
    int featurePos = featurePositions[i];
    features[i] = featureValueMappers[i].call(record.get(featurePos));
  }

  double label = featureValueMappers[labelColumnPos].call(record.get(labelColumnPos));
  return new LabeledPoint(label, Vectors.dense(features));
}
 

import org.apache.spark.mllib.linalg.Vectors; //导入方法依赖的package包/类
/**
 * Returns a labeled point of the writables
 * where the final item is the point and the rest of the items are
 * features
 * @param writables the writables
 * @return the labeled point
 */
public static LabeledPoint pointOf(Collection<Writable> writables) {
    double[] ret = new double[writables.size() - 1];
    int count = 0;
    double target = 0;
    for (Writable w : writables) {
        if (count < writables.size() - 1)
            ret[count++] = Float.parseFloat(w.toString());
        else
            target = Float.parseFloat(w.toString());
    }

    if (target < 0)
        throw new IllegalStateException("Target must be >= 0");
    return new LabeledPoint(target, Vectors.dense(ret));
}
 

import org.apache.spark.mllib.linalg.Vectors; //导入方法依赖的package包/类
Vector postProcessing(HashMap<String, Object> value) {
    org.apache.spark.mllib.linalg.Vector normedForVal;

    double[] values = new double[numberOfTargetValue];
    for (int j = 0; j < numberOfTargetValue; j++) {
        values[j] = 0;
        HashMap<String, Object> features = (HashMap<String, Object>) value.get(AthenaFeatureField.FEATURE);

        if (features.containsKey(listOfTargetFeatures.get(j).getValue())) {
            Object obj = features.get(listOfTargetFeatures.get(j).getValue());
            if (obj instanceof Long) {
                values[j] = (Long) obj;
            } else if (obj instanceof Double) {
                values[j] = (Double) obj;
            } else if (obj instanceof Boolean) {
                values[j] = (Boolean) obj ? 1 : 0;
            } else {
                return null;
            }

            //check weight
            if (weight.containsKey(listOfTargetFeatures.get(j))) {
                values[j] *= weight.get(listOfTargetFeatures.get(j));
            }
            //check absolute
            if (isAbsolute) {
                values[j] = Math.abs(values[j]);
            }
        }
    }


    if (isNormalization) {
        normedForVal = normalizer.transform(Vectors.dense(values));
    } else {
        normedForVal = Vectors.dense(values);
    }

    return normedForVal;
}
 

import org.apache.spark.mllib.linalg.Vectors; //导入方法依赖的package包/类
@Override
public Vector call(String line) {
  String[] tok = SPACE.split(line);
  double[] point = new double[tok.length];
  for (int i = 0; i < tok.length; ++i) {
    point[i] = Double.parseDouble(tok[i]);
  }
  return Vectors.dense(point);
}
 

import org.apache.spark.mllib.linalg.Vectors; //导入方法依赖的package包/类
private LogisticRegressionModel instantiateSparkModel() {
  Configuration conf = new Configuration();
  conf.set("fs.defaultFS", topologyConfig.getProperty("hdfs.url"));

  double[] sparkModelInfo = null;

  try {
    sparkModelInfo = getSparkModelInfoFromHDFS(new Path(topologyConfig.getProperty("hdfs.url") +
        "/tmp/sparkML_weights"), conf);
  } catch (Exception e) {
    LOG.error("Couldn't instantiate Spark model in prediction bolt: " + e.getMessage());
    e.printStackTrace();

    throw new RuntimeException(e);
  }

  // all numbers besides the last value are the weights
  double[] weights = Arrays.copyOfRange(sparkModelInfo, 0, sparkModelInfo.length - 1);

  // the last number in the array is the intercept
  double intercept = sparkModelInfo[sparkModelInfo.length - 1];

  org.apache.spark.mllib.linalg.Vector weightsV = (Vectors.dense(weights));
  return new LogisticRegressionModel(weightsV, intercept);


}
 

import org.apache.spark.mllib.linalg.Vectors; //导入方法依赖的package包/类
@Override
        public Vector call(String str) {
            String[] list = str.split(TimeSeparatorRegex);
//            long time = System.currentTimeMillis();
//            if(list.length == 2) {
//                time = Long.parseLong(list[1]);
//            }
            String[] tok = SPACE.split(list[0]);
            double[] point = new double[tok.length];
            for (int i = 0; i < tok.length; ++i) {
                point[i] = Double.parseDouble(tok[i]);
            }
            return Vectors.dense(point);
        }
 

import org.apache.spark.mllib.linalg.Vectors; //导入方法依赖的package包/类
public Vector call(String line) {
    String[] tok = SPACE.split(line);
    double[] point = new double[tok.length - 1];
    for (int i = 1; i < tok.length; ++i) {
        point[i - 1] = Double.parseDouble(tok[i]);
    }
    return Vectors.dense(point);
}
 

import org.apache.spark.mllib.linalg.Vectors; //导入方法依赖的package包/类
/**
 * Function to transform double array into labeled point
 *
 * @param tokens Double array of tokens
 * @return Labeled point
 */
@Override
public LabeledPoint call(double[] tokens) {
    // last index is the response value after the upstream transformations
    double response = tokens[tokens.length - 1];
    // new feature vector does not contain response variable value
    double[] features = new double[tokens.length - 1];
    for (int i = 0; i < tokens.length - 1; i++) {
        features[i] = tokens[i];
    }
    return new LabeledPoint(response, Vectors.dense(features));
}
 

import org.apache.spark.mllib.linalg.Vectors; //导入方法依赖的package包/类
/**
 * Transforms each line into a LabeledPoint
 *
 * @param line Row in the dataset
 * @return an instance of LabeledPoint
 * @throws DecompositionException
 */
@Override
public LabeledPoint call(String line) throws DecompositionException {
    try {
        double[] features = new double[featureIndices.size()];
        String[] tokens = tokenSeparator.split(line);

        int index = 0;
        for (int i = 0; i < tokens.length; i++) {
            if (featureIndices.contains(i)) {
                String token = tokens[i];
                if (token.equalsIgnoreCase(DecompositionConstants.EMPTY)
                        || token.equalsIgnoreCase(DecompositionConstants.NA)) {
                    features[index] = 0.0;
                } else {

                    features[index] = Double.parseDouble(token.trim());
                }
                index++;
            }
        }
        return new LabeledPoint(Double.parseDouble(tokens[labelIndex]), Vectors.dense(features));

    } catch (Exception e) {
        throw new DecompositionException("An error occurred while transforming lines to tokens: " + e.getMessage(),
                e);
    }
}
 

import org.apache.spark.mllib.linalg.Vectors; //导入方法依赖的package包/类
protected Vector getVector(List<Double> list) {
	double[] values = new double[list.size()];
	for(int i = 0;i < values.length; i++) {
		values[i] = list.get(i);
	}
	Vector features =  Vectors.dense(values);
	return features;
}
 

import org.apache.spark.mllib.linalg.Vectors; //导入方法依赖的package包/类
/**
 * Feature positions and value mappers are parallel arrays. featurePositions[i] gives the position of ith feature in
 * the HCatRecord, and valueMappers[i] gives the value mapper used to map that feature to a Double value
 *
 * @param featurePositions position number of feature column in the HCatRecord
 * @param valueMappers     mapper for each column position
 * @param labelColumnPos   position of the label column
 * @param numFeatures      number of features in the feature vector
 * @param defaultLabel     default lable to be used for null records
 */
public ColumnFeatureFunction(int[] featurePositions, FeatureValueMapper[] valueMappers, int labelColumnPos,
  int numFeatures, double defaultLabel) {
  Preconditions.checkNotNull(valueMappers, "Value mappers argument is required");
  Preconditions.checkNotNull(featurePositions, "Feature positions are required");
  Preconditions.checkArgument(valueMappers.length == featurePositions.length,
    "Mismatch between number of value mappers and feature positions");

  this.featurePositions = featurePositions;
  this.featureValueMappers = valueMappers;
  this.labelColumnPos = labelColumnPos;
  this.numFeatures = numFeatures;
  defaultLabeledPoint = new LabeledPoint(defaultLabel, Vectors.dense(new double[numFeatures]));
}
 

import org.apache.spark.mllib.linalg.Vectors; //导入方法依赖的package包/类
/**
 * Convert an ndarray to a vector
 * @param arr the array
 * @return an mllib vector
 */
public static Vector toVector(INDArray arr) {
    if (!arr.isVector()) {
        throw new IllegalArgumentException("passed in array must be a vector");
    }
    double[] ret = new double[arr.length()];
    for (int i = 0; i < arr.length(); i++) {
        ret[i] = arr.getDouble(i);
    }

    return Vectors.dense(ret);
}