Java VectorWritable.set方法代码示例

import org.apache.mahout.math.VectorWritable; //导入方法依赖的package包/类
static void writeMatrix(Matrix origMatrix,
     Path outPath, Path tmpPath, String label) throws IOException {
   Configuration conf = new Configuration();
   Path outputDir = new Path(outPath, label + origMatrix.numRows() + "x"
       + origMatrix.numCols());
   FileSystem fs = FileSystem.get(outputDir.toUri(), conf);
   if (!fs.exists(outputDir)) {
     Path outputFile = new Path(outputDir, "singleSliceMatrix");
     SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf,
         outputFile, IntWritable.class, VectorWritable.class);
     VectorWritable vectorWritable = new VectorWritable();
     try {
       for (int r = 0; r < origMatrix.numRows(); r++) {
         Vector vector = origMatrix.viewRow(r);
         vectorWritable.set(vector);
         writer.append(new IntWritable(r), vectorWritable);
       }
     } finally {
       writer.close();
     }
   } else {
     log.warn("----------- Skip matrix " + outputDir + " - already exists");
   }
}
 

import org.apache.mahout.math.VectorWritable; //导入方法依赖的package包/类
static Path toDistributedVector(Vector vector, Path outputDir, String label,
    Configuration conf) throws IOException {
  Path outputFile = new Path(outputDir, "Vector-" + label);
  FileSystem fs = FileSystem.get(outputDir.toUri(), conf);
  if (fs.exists(outputFile)) {
    log.warn("----------- OVERWRITE " + outputFile + " already exists");
    fs.delete(outputFile, false);
  }
  SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, outputFile,
      IntWritable.class, VectorWritable.class);
  VectorWritable vectorWritable = new VectorWritable();
  vectorWritable.set(vector);
  writer.append(new IntWritable(0), vectorWritable);
  writer.close();
  return outputFile;
}
 

import org.apache.mahout.math.VectorWritable; //导入方法依赖的package包/类
@Override
public void cleanup(Context context) throws InterruptedException,
    IOException {
  // 2. Y' * X ----combiner part
  AtxBCombiner(ytxMatrix, ym, xm, sumxi, totalRows);
  // 3. X' * X ----combiner part
  AtxBCombiner(xtxMatrix, xm, xm, sumxi, totalRows);

  VectorWritable outVector = new VectorWritable();
  CompositeWritable ytxCompositeKey = new CompositeWritable(
      CompositeWritable.YTX_TYPE);
  for (int i = 0; i < ytxMatrix.numRows(); i++) {
    ytxCompositeKey.set(i);
    outVector.set(ytxMatrix.viewRow(i));
    context.write(ytxCompositeKey, outVector);
  }
  // for all XtX rows the key is the same
  // the last column of the value vector determines the row id
  CompositeWritable xtxSingleKey = new CompositeWritable(
      CompositeWritable.XTX_TYPE);
  int idCol = xtxMatrix.numCols() - 1;// last is id column
  for (int i = 0; i < xtxMatrix.numRows(); i++) {
    xtxMatrix.setQuick(i, idCol, i);
    outVector.set(xtxMatrix.viewRow(i));
    context.write(xtxSingleKey, outVector);
  }
}
 

import org.apache.mahout.math.VectorWritable; //导入方法依赖的package包/类
/**
 * Convert an in-memory representation of a matrix to a distributed version It
 * then can be used in distributed jobs
 * 
 * @param oriMatrix
 * @return path that contains the matrix files
 * @throws IOException
 */
static DistributedRowMatrix toDistributedRowMatrix(Matrix origMatrix,
    Path outPath, Path tmpPath, String label) throws IOException {
  Configuration conf = new Configuration();
  Path outputDir = new Path(outPath, label + origMatrix.numRows() + "x"
      + origMatrix.numCols());
  FileSystem fs = FileSystem.get(outputDir.toUri(), conf);
  if (!fs.exists(outputDir)) {
    Path outputFile = new Path(outputDir, "singleSliceMatrix");
    SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf,
        outputFile, IntWritable.class, VectorWritable.class);
    VectorWritable vectorWritable = new VectorWritable();
    try {
      for (int r = 0; r < origMatrix.numRows(); r++) {
        Vector vector = origMatrix.viewRow(r);
        vectorWritable.set(vector);
        writer.append(new IntWritable(r), vectorWritable);
      }
    } finally {
      writer.close();
    }
  } else {
    log.warn("----------- Skip matrix " + outputDir + " - already exists");
  }
  DistributedRowMatrix dMatrix = new DistributedRowMatrix(outputDir, tmpPath,
      origMatrix.numRows(), origMatrix.numCols());
  dMatrix.setConf(conf);
  return dMatrix;
}
 

import org.apache.mahout.math.VectorWritable; //导入方法依赖的package包/类
private static void textToSequnceFile(String inputStr) throws IOException {
	BufferedReader inputReader = new BufferedReader(
			new FileReader(inputStr));
	Configuration conf = new Configuration();
	Path inputPath = new Path(inputStr);
	Path outputPath = new Path(inputPath.getParent(), inputPath.getName()
			+ ".formatted");
	FileSystem fs = FileSystem.get(inputPath.toUri(), conf);
	SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf,
			outputPath, IntWritable.class, VectorWritable.class);
	VectorWritable vectorWritable = new VectorWritable();
	String line;
	int index = 0;
	try {
		while ((line = inputReader.readLine()) != null) {
			String[] columns = line.split(" ");
			int shift = 0;
			if (columns[0].isEmpty())
				shift++;
			double[] columnsDouble = new double[columns.length-shift];
			for (int i = 0; i < columnsDouble.length; i++) {
				columnsDouble[i] = Double.valueOf(columns[i+shift]);
			}
			Vector vector = new DenseVector(columnsDouble, true);
			vectorWritable.set(vector);
			writer.append(new IntWritable(index), vectorWritable);
			index++;
		}
	} finally {
		writer.close();
	}
	inputReader.close();
	System.out.println("Finish writing to " + outputPath);
}
 

import org.apache.mahout.math.VectorWritable; //导入方法依赖的package包/类
public static void main(String args[]) throws FileNotFoundException, IOException {

        String input = args[0];
        String output = args[1];
        Configuration conf;
        conf = new Configuration();
        FileSystem fs = FileSystem.get(conf);
        InputStream is = null;
        
        if (args.length>2 && args[2].equals("localInput")){
            System.out.println("Using local file "+input);
            is=(new FileInputStream( new File(input)));
        }            
        else{
            System.out.println("Using HDFS file "+input);
            is=fs.open(new Path(input));
        }

        try (
                BufferedReader reader =  new BufferedReader(new InputStreamReader(is));
                SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, new Path(output), IntWritable.class, VectorWritable.class)) {
            String line;
            int counter = 0;
            while ((line = reader.readLine()) != null) {
                String[] c = line.split(",");
                if (c.length > 1) {
                    double[] d = new double[c.length];
                    for (int i = 0; i < c.length; i++) {
                        d[i] = Double.parseDouble(c[i]);
                    }
                    Vector vec = new RandomAccessSparseVector(c.length);
                    vec.assign(d);

                    VectorWritable writable = new VectorWritable();
                    writable.set(vec);
                    writer.append(new IntWritable(counter++), writable);
                }
            }
            writer.close();
        }
    }