Java DenseVector类代码示例

import org.apache.spark.mllib.linalg.DenseVector; //导入依赖的package包/类
public static DistributedMatrix DGER(double alpha, DenseVector x, DenseVector y, DistributedMatrix A, JavaSparkContext jsc) {

        // Case of IndexedRowMatrix
        if( A.getClass() == IndexedRowMatrix.class) {
            return L2.DGER_IRW((IndexedRowMatrix) A, alpha, x, y, jsc);
        }
        else if (A.getClass() == CoordinateMatrix.class) {
            return L2.DGER_COORD((CoordinateMatrix) A, alpha, x, y, jsc);
        }
        else if (A.getClass() == BlockMatrix.class){
            //return L2.DGER_BCK((BlockMatrix) A, alpha, x, y, jsc);
            return null;
        }
        else {
            return null;
        }

    }
 

import org.apache.spark.mllib.linalg.DenseVector; //导入依赖的package包/类
private List<Vector> getVectors(List<String[]> data) {
    List<Vector> vectors = new ArrayList<Vector>();
    List<Map<String, Integer>> encodings = model.getEncodings();
    BasicEncoder encoder = new BasicEncoder.Builder().encodings(encodings).build();
    for (String[] dataEntry : data) {
        String[] encodedEntry;
        try {
            encodedEntry = encoder.call(dataEntry);
        } catch (Exception e) {
            log.warn("Data encoding failed. Cause: " + e.getMessage());
            encodedEntry = dataEntry;
        }
        double[] doubleValues = MLUtils.toDoubleArray(encodedEntry);
        Vector vector = new DenseVector(doubleValues);
        vectors.add(vector);
    }
    return vectors;
}
 

import org.apache.spark.mllib.linalg.DenseVector; //导入依赖的package包/类
public static DenseVector DGEMV(double alpha, DistributedMatrix A, DenseVector x, double beta, DenseVector y, JavaSparkContext jsc){

        // First form  y := beta*y.
        if (beta != 1.0) {
            if (beta == 0.0) {
                y = Vectors.zeros(y.size()).toDense();
            }
            else {
                BLAS.scal(beta, y);
            }
        }

        if (alpha == 0.0) {
            return y;
        }

        DenseVector tmpVector = Vectors.zeros(y.size()).toDense();

        // Form  y := alpha*A*x + y.
        // Case of IndexedRowMatrix
        if( A.getClass() == IndexedRowMatrix.class) {
            tmpVector = L2.DGEMV_IRW((IndexedRowMatrix) A, alpha, x, jsc);
        }
        else if (A.getClass() == CoordinateMatrix.class) {
            tmpVector = L2.DGEMV_COORD((CoordinateMatrix) A, alpha, x, jsc);
        }
        else if (A.getClass() == BlockMatrix.class){
            tmpVector = L2.DGEMV_BCK((BlockMatrix) A, alpha, x, jsc);
        }
        else {
            tmpVector = null;
        }

        BLAS.axpy(1.0, tmpVector, y);


        return y;

    }
 

import org.apache.spark.mllib.linalg.DenseVector; //导入依赖的package包/类
private static DenseVector DGEMV_COORD(CoordinateMatrix matrix, double alpha, DenseVector vector, JavaSparkContext jsc) {

        JavaRDD<MatrixEntry> items = matrix.entries().toJavaRDD();
        DenseVector result = items.mapPartitions(new MatrixEntriesMultiplication(vector, alpha))
                .reduce(new MatrixEntriesMultiplicationReducer());

        return result;
    }
 

import org.apache.spark.mllib.linalg.DenseVector; //导入依赖的package包/类
@Test
public void testVectorBinarizerDense() {
    // prepare data

    JavaRDD<Row> jrdd = sc.parallelize(Arrays.asList(
            RowFactory.create(0d, 1d, new DenseVector(new double[]{-2d, -3d, -4d, -1d, 6d, -7d, 8d, 0d, 0d, 0d, 0d, 0d})),
            RowFactory.create(1d, 2d, new DenseVector(new double[]{4d, -5d, 6d, 7d, -8d, 9d, -10d, 0d, 0d, 0d, 0d, 0d})),
            RowFactory.create(2d, 3d, new DenseVector(new double[]{-5d, 6d, -8d, 9d, 10d, 11d, 12d, 0d, 0d, 0d, 0d, 0d}))
    ));

    StructType schema = new StructType(new StructField[]{
            new StructField("id", DataTypes.DoubleType, false, Metadata.empty()),
            new StructField("value1", DataTypes.DoubleType, false, Metadata.empty()),
            new StructField("vector1", new VectorUDT(), false, Metadata.empty())
    });

    DataFrame df = sqlContext.createDataFrame(jrdd, schema);
    VectorBinarizer vectorBinarizer = new VectorBinarizer()
            .setInputCol("vector1")
            .setOutputCol("binarized")
            .setThreshold(2d);


    //Export this model
    byte[] exportedModel = ModelExporter.export(vectorBinarizer, df);

    //Import and get Transformer
    Transformer transformer = ModelImporter.importAndGetTransformer(exportedModel);
    //compare predictions
    Row[] sparkOutput = vectorBinarizer.transform(df).orderBy("id").select("id", "value1", "vector1", "binarized").collect();
    for (Row row : sparkOutput) {

        Map<String, Object> data = new HashMap<>();
        data.put(vectorBinarizer.getInputCol(), ((DenseVector) row.get(2)).toArray());
        transformer.transform(data);
        double[] output = (double[]) data.get(vectorBinarizer.getOutputCol());
        assertArrayEquals(output, ((DenseVector) row.get(3)).toArray(), 0d);
    }
}
 

import org.apache.spark.mllib.linalg.DenseVector; //导入依赖的package包/类
public Status predict(double lat, double lon, double speedKnots, double courseMinusHeading,
        double preEffectiveSpeedKnots, double preError, double postEffectiveSpeedKnots,
        double postError) {
    Vector features = new DenseVector(new double[] { lat, lon, speedKnots, courseMinusHeading,
            preEffectiveSpeedKnots, preError, postEffectiveSpeedKnots, postError });
    double prediction = model.predict(features);

    if (is(prediction, 1))
        return Status.MOORED;
    else if (is(prediction, 2))
        return Status.ANCHORED;
    else
        return Status.OTHER;
}
 

import org.apache.spark.mllib.linalg.DenseVector; //导入依赖的package包/类
/**
 *
 * ExecRow is one-based as far as the elements
 *
 * @param execRow
 * @return
 */
public static Vector convertExecRowToVector(ExecRow execRow) throws StandardException {
    int length = execRow.nColumns();
    double[] vectorValues = new double[length];
    for (int i=1;i<=length;i++) {
        vectorValues[i] = execRow.getColumn(i).getDouble();
    }
    return new DenseVector(vectorValues);
}
 

import org.apache.spark.mllib.linalg.DenseVector; //导入依赖的package包/类
@Override
public IndexedRow call(Tuple2<Long, double[]> longTuple2) throws Exception {
    return new IndexedRow(longTuple2._1(), new DenseVector(longTuple2._2()));
}
 

import org.apache.spark.mllib.linalg.DenseVector; //导入依赖的package包/类
private static IndexedRowMatrix GetD_IRW(IndexedRowMatrix A, boolean inverseValues, JavaSparkContext jsc) {

        JavaRDD<IndexedRow> rows = A.rows().toJavaRDD().cache();

        final Broadcast<Boolean> inverseValuesBC = jsc.broadcast(inverseValues);
        JavaRDD<IndexedRow> LURows = rows.map(new Function<IndexedRow, IndexedRow>() {

            @Override
            public IndexedRow call(IndexedRow indexedRow) throws Exception {
                long index = indexedRow.index();
                DenseVector vect = indexedRow.vector().toDense();

                boolean inverseValuesValue = inverseValuesBC.getValue().booleanValue();

                double newValues[] = new double[vect.size()];


                for(int i = 0; i< vect.size(); i++) {

                    if( i == index) {
                        if(inverseValuesValue) {
                            newValues[i] = 1.0/vect.apply(i);
                        }
                        else {
                            newValues[i] = vect.apply(i);
                        }
                    }
                    else {
                        newValues[i] = 0.0;
                    }

                }

                DenseVector newVector = new DenseVector(newValues);

                return new IndexedRow(index, newVector);

            }
        });

        IndexedRowMatrix newMatrix = new IndexedRowMatrix(LURows.rdd());

        return newMatrix;
    }
 

import org.apache.spark.mllib.linalg.DenseVector; //导入依赖的package包/类
@Override
public DenseVector call(DenseVector vector, DenseVector vector2) throws Exception {
    BLAS.axpy(1.0, vector, vector2);

    return vector2;
}
 

import org.apache.spark.mllib.linalg.DenseVector; //导入依赖的package包/类
public MatrixEntriesMultiplication(DenseVector vector, double alpha) {

        this.vector = vector;
        this.alpha = alpha;
    }
 

import org.apache.spark.mllib.linalg.DenseVector; //导入依赖的package包/类
public static DenseVector readVectorFromFileInHDFS(String file, Configuration conf){

		try {
			FileSystem fs = FileSystem.get(conf);

			Path pt = new Path(file);

			//FileSystem fileSystem = FileSystem.get(context.getConfiguration());
			BufferedReader br=new BufferedReader(new InputStreamReader(fs.open(pt)));
			String line;
			line=br.readLine();

			double vector[] = null;

			boolean arrayInfo = true;

			int i = 0;

			while (line != null){

				if((arrayInfo == true) && (line.charAt(0) == '%')){
					arrayInfo = true;
					//LOG.info("JMAbuin:: Skipping line with %");
				}
				else if((arrayInfo == true) && !(line.charAt(0) == '%')){
					arrayInfo = false;
					String[] matrixInfo = line.split(" ");
					//LOG.info("JMAbuin:: Creating vector after line with %");
					vector = new double[Integer.parseInt(matrixInfo[0])];

				}
				else{
					vector[i] = Double.parseDouble(line);
					i++;
				}

				line=br.readLine();
			}

			br.close();

			return new DenseVector(vector);

		} catch (IOException e) {
			LOG.error("Error in " + IO.class.getName() + ": " + e.getMessage());
			e.printStackTrace();
			System.exit(1);
		}

		return null;
	}
 

import org.apache.spark.mllib.linalg.DenseVector; //导入依赖的package包/类
@Override
public Vector format(Object value){
	List<String> labels = getLabels();

	HasProbability hasProbability = (HasProbability)value;

	double[] probabilities = new double[labels.size()];

	for(int i = 0; i < labels.size(); i++){
		String label = labels.get(i);

		probabilities[i] = hasProbability.getProbability(label);
	}

	return new DenseVector(probabilities);
}
 

import org.apache.spark.mllib.linalg.DenseVector; //导入依赖的package包/类
@Test
public void testVectorAssembler() {
    // prepare data

    JavaRDD<Row> jrdd = sc.parallelize(Arrays.asList(
            RowFactory.create(0d, 1d, new DenseVector(new double[]{2d, 3d})),
            RowFactory.create(1d, 2d, new DenseVector(new double[]{3d, 4d})),
            RowFactory.create(2d, 3d, new DenseVector(new double[]{4d, 5d})),
            RowFactory.create(3d, 4d, new DenseVector(new double[]{5d, 6d})),
            RowFactory.create(4d, 5d, new DenseVector(new double[]{6d, 7d}))
    ));

    StructType schema = new StructType(new StructField[]{
            new StructField("id", DataTypes.DoubleType, false, Metadata.empty()),
            new StructField("value1", DataTypes.DoubleType, false, Metadata.empty()),
            new StructField("vector1", new VectorUDT(), false, Metadata.empty())
    });

    DataFrame df = sqlContext.createDataFrame(jrdd, schema);
    VectorAssembler vectorAssembler = new VectorAssembler()
            .setInputCols(new String[]{"value1", "vector1"})
            .setOutputCol("feature");


    //Export this model
    byte[] exportedModel = ModelExporter.export(vectorAssembler, null);

    String exportedModelJson = new String(exportedModel);
    //Import and get Transformer
    Transformer transformer = ModelImporter.importAndGetTransformer(exportedModel);
    //compare predictions
    Row[] sparkOutput = vectorAssembler.transform(df).orderBy("id").select("id", "value1", "vector1", "feature").collect();
    for (Row row : sparkOutput) {

        Map<String, Object> data = new HashMap<>();
        data.put(vectorAssembler.getInputCols()[0], row.get(1));
        data.put(vectorAssembler.getInputCols()[1], ((DenseVector) row.get(2)).toArray());
        transformer.transform(data);
        double[] output = (double[]) data.get(vectorAssembler.getOutputCol());
        assertArrayEquals(output, ((DenseVector) row.get(3)).toArray(), 0d);
    }
}
 

import org.apache.spark.mllib.linalg.DenseVector; //导入依赖的package包/类
@Test
public void testChiSqSelector() {
    // prepare data

    JavaRDD<Row> jrdd = sc.parallelize(Arrays.asList(
            RowFactory.create(0d, 0d, new DenseVector(new double[]{8d, 7d, 0d})),
            RowFactory.create(1d, 1d, new DenseVector(new double[]{0d, 9d, 6d})),
            RowFactory.create(2d, 1d, new DenseVector(new double[]{0.0d, 9.0d, 8.0d})),
            RowFactory.create(3d, 2d, new DenseVector(new double[]{8.0d, 9.0d, 5.0d}))
    ));

    double[] preFilteredData = {0.0d, 6.0d, 8.0d, 5.0d};

    StructType schema = new StructType(new StructField[]{
            new StructField("id", DataTypes.DoubleType, false, Metadata.empty()),
            new StructField("label", DataTypes.DoubleType, false, Metadata.empty()),
            new StructField("features", new VectorUDT(), false, Metadata.empty())
    });

    DataFrame df = sqlContext.createDataFrame(jrdd, schema);
    ChiSqSelector chiSqSelector = new ChiSqSelector();
    chiSqSelector.setNumTopFeatures(1);
    chiSqSelector.setFeaturesCol("features");
    chiSqSelector.setLabelCol("label");
    chiSqSelector.setOutputCol("output");

    ChiSqSelectorModel chiSqSelectorModel = chiSqSelector.fit(df);

    //Export this model
    byte[] exportedModel = ModelExporter.export(chiSqSelectorModel, null);

    String exportedModelJson = new String(exportedModel);

    //Import and get Transformer
    Transformer transformer = ModelImporter.importAndGetTransformer(exportedModel);

    //compare predictions
    Row[] sparkOutput = chiSqSelectorModel.transform(df).orderBy("id").select("id", "label", "features", "output").collect();
    for (Row row : sparkOutput) {
        Map<String, Object> data = new HashMap<>();
        data.put(chiSqSelectorModel.getFeaturesCol(), ((DenseVector) row.get(2)).toArray());
        transformer.transform(data);
        double[] output = (double[]) data.get(chiSqSelectorModel.getOutputCol());
        System.out.println(Arrays.toString(output));
        assertArrayEquals(output, ((DenseVector) row.get(3)).toArray(), 0d);
    }
}