Java VectorWritable.get方法代码示例

import org.apache.mahout.math.VectorWritable; //导入方法依赖的package包/类
@Override
public void configure(Configuration jobConf) {
  if (parameters == null) {
    ParameteredGeneralizations.configureParameters(this, jobConf);
  }
  try {
    if (weightsFile.get() != null) {
      FileSystem fs = FileSystem.get(weightsFile.get().toUri(), jobConf);
      VectorWritable weights =
          ClassUtils.instantiateAs((Class<? extends VectorWritable>) vectorClass.get(), VectorWritable.class);
      if (!fs.exists(weightsFile.get())) {
        throw new FileNotFoundException(weightsFile.get().toString());
      }
      DataInputStream in = fs.open(weightsFile.get());
      try {
        weights.readFields(in);
      } finally {
        Closeables.closeQuietly(in);
      }
      this.weights = weights.get();
    }
  } catch (IOException e) {
    throw new IllegalStateException(e);
  }
}
 

import org.apache.mahout.math.VectorWritable; //导入方法依赖的package包/类
private static void printSequenceFile(String inputStr, int printRow) throws IOException {
  Configuration conf = new Configuration();
  Path finalNumberFile = new Path(inputStr);
  SequenceFile.Reader reader = new SequenceFile.Reader(FileSystem.get(conf),
      finalNumberFile, conf);
  IntWritable key = new IntWritable();
  VectorWritable value = new VectorWritable();
  Vector printVector = null;
  while (reader.next(key, value)) {
    if (key.get() == printRow)
      printVector = value.get();
    int cnt = 0;
    Iterator<Element> iter = value.get().nonZeroes().iterator();
    for (; iter.hasNext(); iter.next())
      cnt++;
    System.out.println("# "+ key + " " + cnt + " " + value.get().zSum());
  }
  reader.close();
  if (printVector != null)
    System.out.println("##### "+ printRow + " " + printVector);
  else
    System.out.println("##### "+ key + " " + value.get());
}
 

import org.apache.mahout.math.VectorWritable; //导入方法依赖的package包/类
@Override
protected void reduce(Text arg0, Iterable<VectorWritable> values,
    Context context) throws IOException, InterruptedException {
  for (VectorWritable value : values) {
    Vector point = value.get();
    canopyClusterer.addPointToCanopies(point, canopies);
  }
  for (Canopy canopy : canopies) {
    ClusterWritable clusterWritable = new ClusterWritable();
    canopy.computeParameters();
    if (canopy.getNumObservations() > clusterFilter) {
  	clusterWritable.setValue(canopy);
      context.write(new Text(canopy.getIdentifier()), clusterWritable);
    }
  }
}
 

import org.apache.mahout.math.VectorWritable; //导入方法依赖的package包/类
private static void doTestVectorWritableEquals(Vector v) throws IOException {
  Writable vectorWritable = new VectorWritable(v);
  VectorWritable vectorWritable2 = new VectorWritable();
  writeAndRead(vectorWritable, vectorWritable2);
  Vector v2 = vectorWritable2.get();
  if (v instanceof NamedVector) {
    assertTrue(v2 instanceof NamedVector);
    NamedVector nv = (NamedVector) v;
    NamedVector nv2 = (NamedVector) v2;
    assertEquals(nv.getName(), nv2.getName());
    assertEquals("Victor", nv.getName());
  }
  assertEquals(v, v2);
}
 

import org.apache.mahout.math.VectorWritable; //导入方法依赖的package包/类
/**
 * compute the distance of the contribution with the corresponding 'user' id and return the num of suspected users
 *
 * @param conf
 * @param path
 * @param ranks
 * @param rowIndex2userId
 * @param suspectedUserNum
 * @throws IOException
 */
private static int[] computeDistance(Configuration conf, String path, String ranks,
                                     HashMap<Integer, Integer> rowIndex2userId,
                                     int suspectedUserNum) throws IOException {
    FileSystem fs = FileSystem.get(conf);
    FileStatus[] statuses = fs.listStatus(new Path(path), hiddenFileFilter);
    HashMap<Integer, Double> userCoefficients = new HashMap<Integer, Double>();
    int count = 0;
    for (int i = 0; i < statuses.length; i++) {
        SequenceFile.Reader reader = new SequenceFile.Reader(conf,
                SequenceFile.Reader.file(statuses[i].getPath()));
        IntWritable key = new IntWritable();
        VectorWritable value = new VectorWritable();
        while (reader.next(key, value)) {
            count++;
            DenseVector denseVector = (DenseVector) value.get();
            double sum = 0;
            for (int j = 0; j < Integer.parseInt(ranks); j++) {
                sum += Math.pow(denseVector.get(j) * 100, 2);
            }
            userCoefficients.put(key.get(), sum);
        }
        reader.close();
    }
    ValueComparator vc = new ValueComparator(userCoefficients);
    TreeMap<Integer, Double> sortedMap = new TreeMap<Integer, Double>(vc);
    sortedMap.putAll(userCoefficients);
    System.out.println("user counts: " + count);
    System.out.println("user map size: " + userCoefficients.size());

    int[] suspectedUsers = new int[suspectedUserNum];
    Iterator<Map.Entry<Integer, Double>> iterator = sortedMap.entrySet().iterator();
    for (int i = 0; i < suspectedUserNum; i++) {
        suspectedUsers[i] = rowIndex2userId.get(iterator.next().getKey());
    }
    return suspectedUsers;
}
 

import org.apache.mahout.math.VectorWritable; //导入方法依赖的package包/类
@Override
protected void map(IntWritable key, VectorWritable row, Context ctx) 
  throws IOException, InterruptedException {
  
  for (Vector.Element e : row.get()) {
    double dii = Functions.SQRT.apply(diagonal.get(key.get()));
    double djj = Functions.SQRT.apply(diagonal.get(e.index()));
    double mij = e.get();
    e.set(dii * mij * djj);
  }
  ctx.write(key, row);
}
 

import org.apache.mahout.math.VectorWritable; //导入方法依赖的package包/类
private static void classifyAndWrite(List<Cluster> clusterModels, Double clusterClassificationThreshold,
    boolean emitMostLikely, SequenceFile.Writer writer, VectorWritable vw, Vector pdfPerCluster) throws IOException {
  if (emitMostLikely) {
    int maxValueIndex = pdfPerCluster.maxValueIndex();
    WeightedVectorWritable wvw = new WeightedVectorWritable(pdfPerCluster.maxValue(), vw.get());
    write(clusterModels, writer, wvw, maxValueIndex);
  } else {
    writeAllAboveThreshold(clusterModels, clusterClassificationThreshold, writer, vw, pdfPerCluster);
  }
}
 

import org.apache.mahout.math.VectorWritable; //导入方法依赖的package包/类
/**
 * Iterate over data using a prior-trained ClusterClassifier, for a number of iterations using a sequential
 * implementation
 * 
 * @param conf
 *          the Configuration
 * @param inPath
 *          a Path to input VectorWritables
 * @param priorPath
 *          a Path to the prior classifier
 * @param outPath
 *          a Path of output directory
 * @param numIterations
 *          the int number of iterations to perform
 */
public static void iterateSeq(Configuration conf, Path inPath, Path priorPath, Path outPath, int numIterations)
  throws IOException {
  ClusterClassifier classifier = new ClusterClassifier();
  classifier.readFromSeqFiles(conf, priorPath);
  Path clustersOut = null;
  int iteration = 1;
  while (iteration <= numIterations) {
    for (VectorWritable vw : new SequenceFileDirValueIterable<VectorWritable>(inPath, PathType.LIST,
        PathFilters.logsCRCFilter(), conf)) {
      Vector vector = vw.get();
      // classification yields probabilities
      Vector probabilities = classifier.classify(vector);
      // policy selects weights for models given those probabilities
      Vector weights = classifier.getPolicy().select(probabilities);
      // training causes all models to observe data
      for (Iterator<Vector.Element> it = weights.iterateNonZero(); it.hasNext();) {
        int index = it.next().index();
        classifier.train(index, vector, weights.get(index));
      }
    }
    // compute the posterior models
    classifier.close();
    // update the policy
    classifier.getPolicy().update(classifier);
    // output the classifier
    clustersOut = new Path(outPath, Cluster.CLUSTERS_DIR + iteration);
    classifier.writeToSeqFiles(clustersOut);
    FileSystem fs = FileSystem.get(outPath.toUri(), conf);
    iteration++;
    if (isConverged(clustersOut, conf, fs)) {
      break;
    }
  }
  Path finalClustersIn = new Path(outPath, Cluster.CLUSTERS_DIR + (iteration - 1) + Cluster.FINAL_ITERATION_SUFFIX);
  FileSystem.get(clustersOut.toUri(), conf).rename(clustersOut, finalClustersIn);
}
 

import org.apache.mahout.math.VectorWritable; //导入方法依赖的package包/类
@Override
protected void map(WritableComparable<?> key, VectorWritable value, Context context)
  throws IOException, InterruptedException {
  Vector vector = value.get();
  Iterator<Vector.Element> it = vector.iterateNonZero();

  while (it.hasNext()) {
    Vector.Element e = it.next();
    context.write(new IntWritable(e.index()), ONE);
  }
  context.write(TOTAL_COUNT, ONE);
}
 

import org.apache.mahout.math.VectorWritable; //导入方法依赖的package包/类
@Override
protected void reduce(WritableComparable<?> key, Iterable<VectorWritable> values, Context ctx)
  throws IOException, InterruptedException {
  Vector vector = null;
  for (VectorWritable v : values) {
    if (vector == null) {
      vector = v.get();
    } else {
      vector.assign(v.get(), Functions.PLUS);
    }
  }
  ctx.write(key, new VectorWritable(vector));
}
 

import org.apache.mahout.math.VectorWritable; //导入方法依赖的package包/类
private static void writeAllAboveThreshold(List<Cluster> clusterModels, Double clusterClassificationThreshold,
    SequenceFile.Writer writer, VectorWritable vw, Vector pdfPerCluster) throws IOException {
  Iterator<Element> iterateNonZero = pdfPerCluster.iterateNonZero();
  while (iterateNonZero.hasNext()) {
    Element pdf = iterateNonZero.next();
    if (pdf.get() >= clusterClassificationThreshold) {
      WeightedVectorWritable wvw = new WeightedVectorWritable(pdf.get(), vw.get());
      int clusterIndex = pdf.index();
      write(clusterModels, writer, wvw, clusterIndex);
    }
  }
}
 

import org.apache.mahout.math.VectorWritable; //导入方法依赖的package包/类
@Override
protected void map(IntWritable row, VectorWritable vw, Context context) 
  throws IOException, InterruptedException {
  
  // first, does this particular eigenvector even pass the required threshold?
  double eigenvalue = Math.abs(eigenvalues.get(row.get()));
  double betak = -Functions.LOGARITHM.apply(2) / Functions.LOGARITHM.apply(eigenvalue);
  if (eigenvalue >= 1.0 || betak <= epsilon * beta0) {
    // doesn't pass the threshold! quit
    return;
  }
  
  // go through the vector, performing the calculations
  // sadly, no way to get around n^2 computations      
  Map<Integer, EigencutsSensitivityNode> columns = Maps.newHashMap();
  Vector ev = vw.get();
  for (int i = 0; i < ev.size(); i++) {
    double minsij = Double.MAX_VALUE;
    int minInd = -1;
    for (int j = 0; j < ev.size(); j++) {          
      double sij = performSensitivityCalculation(eigenvalue, ev.get(i),
          ev.get(j), diagonal.get(i), diagonal.get(j));
      
      // perform non-maximal suppression
      // is this the smallest value in the row?
      if (sij < minsij) {
        minsij = sij;
        minInd = j;
      }
    }
    
    // is this the smallest value in the column?
    Integer column = minInd;
    EigencutsSensitivityNode value = new EigencutsSensitivityNode(i, minInd, minsij);
    if (!columns.containsKey(column)) {
      columns.put(column, value);
    } else if (columns.get(column).getSensitivity() > minsij) {
      columns.remove(column);
      columns.put(column, value);
    }
  }
  
  // write whatever values made it through
  
  for (EigencutsSensitivityNode e : columns.values()) {
    context.write(new IntWritable(e.getRow()), e);
  }
}