Java Frame.vecs方法代码示例

import water.fvec.Frame; //导入方法依赖的package包/类
private static void runKMeans() {
  File f = new File("/Users/apple/Downloads/whiskey.csv");
  NFSFileVec nfs = NFSFileVec.make(f);
  Frame frame = water.parser.ParseDataset.parse(Key.make(), nfs._key);
  KMeansModel.KMeansParameters params = new KMeansModel.KMeansParameters();
  params._train = frame._key;
  params._k = 5;
  params._max_iterations = 1000;
  KMeans job = new KMeans(params);
  KMeansModel kmm = job.trainModel().get();
  Frame output = kmm.score(frame);
  Vec[] vecs = output.vecs();
  System.out.print(Arrays.toString(vecs));
  for (Vec v : vecs) {
    System.out.println(v + ", " + v.length());
    for (int i = 0; i < v.length(); i++) {
      System.out.println(i + ": " + v.at(i));
    }
  }
  System.out.println();
}
 

import water.fvec.Frame; //导入方法依赖的package包/类
void doIt( Frame f, String msg, boolean expensive ) {
  boolean any=false;
  for( int i = 0; i < f.vecs().length - _specialVecs; i++ ) {
    if( filter(f.vecs()[i]) ) {
      if( any ) msg += ", "; // Log dropped cols
      any = true;
      msg += f._names[i];
      f.remove(i);
      i--; // Re-run at same iteration after dropping a col
    }
  }
  if( any ) {
    warn("_train", msg);
    if (expensive) Log.info(msg);
  }
}
 

import water.fvec.Frame; //导入方法依赖的package包/类
/**
 * Fetch all Frames from the KV store.
 */
protected static Frame[] fetchAll() {
  // Get all the frames.
  final Key[] frameKeys = KeySnapshot.globalKeysOfClass(Frame.class);
  List<Frame> frames = new ArrayList<Frame>(frameKeys.length);
  for (int i = 0; i < frameKeys.length; i++) {
    Frame frame = getFromDKV("(none)", frameKeys[i]);
    // Weed out frames with vecs that are no longer in DKV
    Vec[] vs = frame.vecs();
    boolean skip = false;
    for (int j=0; j < vs.length; j++) {
      if (DKV.get(vs[j]._key) == null) {
        Log.warn("Leaked frame: Frame "+frame._key+" points to one or more deleted vecs.");
        skip = true;
        break;
      }
    }
    if (!skip) frames.add(frame);
  }
  return frames.toArray(new Frame[frames.size()]);
}
 

import water.fvec.Frame; //导入方法依赖的package包/类
static public DHistogram[] initialHist(Frame fr, int ncols, int nbins, int nbins_cats, DHistogram hs[]) {
  Vec vecs[] = fr.vecs();
  for( int c=0; c<ncols; c++ ) {
    Vec v = vecs[c];
    final float minIn = (float)Math.max(v.min(),-Float.MAX_VALUE); // inclusive vector min
    final float maxIn = (float)Math.min(v.max(), Float.MAX_VALUE); // inclusive vector max
    final float maxEx = find_maxEx(maxIn,v.isInt()?1:0); // smallest exclusive max
    final long vlen = v.length();
    hs[c] = v.naCnt()==vlen || v.min()==v.max() ? null :
      make(fr._names[c],nbins, nbins_cats, (byte)(v.isCategorical() ? 2 : (v.isInt()?1:0)), minIn, maxEx);
    assert (hs[c] == null || vlen > 0);
  }
  return hs;
}
 

import water.fvec.Frame; //导入方法依赖的package包/类
@Test
public void DCT_2D() {
  Frame frame = null;
  Frame frameDCT = null;
  Frame frameRec = null;
  try {
    CreateFrame cf = new CreateFrame();
    cf.rows = 100;
    int height = 47;
    int width = 29;
    int depth = 1;
    cf.cols = height*width*depth;
    cf.categorical_fraction = 0.0;
    cf.integer_fraction = 0;
    cf.binary_fraction = 0;
    cf.missing_fraction = 0;
    cf.factors = 0;
    cf.seed = 1234;
    cf.execImpl();
    cf.get();
    frame = DKV.getGet(cf.dest());
    long now = System.currentTimeMillis();
    frameDCT = MathUtils.DCT.transform2D(frame, height, width, false);
    Log.info("Computed 2D DCT of " + cf.rows + " rows of size " + cf.cols + " in" + PrettyPrint.msecs(System.currentTimeMillis() - now, true));

    now = System.currentTimeMillis();
    frameRec = MathUtils.DCT.transform2D(frameDCT, height, width, true);
    Log.info("Computed inverse 2D DCT of " + cf.rows + " rows of size " + cf.cols + " in" + PrettyPrint.msecs(System.currentTimeMillis() - now, true));

    for (int i=0; i<frame.vecs().length; ++i)
      TestUtil.assertVecEquals(frame.vecs()[i], frameRec.vecs()[i], 1e-5);
    Log.info("Identity test passed: DCT^-1(DCT(frame)) == frame");
  } finally {
    if (frame!=null) frame.delete();
    if (frameDCT!=null) frameDCT.delete();
    if (frameRec!=null) frameRec.delete();
  }
}
 

import water.fvec.Frame; //导入方法依赖的package包/类
/** Make either a prediction or a reconstruction.
 * @param orig Test dataset
 * @param adaptedFr Test dataset, adapted to the model
 * @return A frame containing the prediction or reconstruction
 */
@Override protected Frame predictScoreImpl(Frame orig, Frame adaptedFr, String destination_key) {
  if (!get_params()._autoencoder) {
    return super.predictScoreImpl(orig, adaptedFr, destination_key);
  } else {
    // Reconstruction
    final int len = model_info().data_info().fullN();
    assert(model_info().data_info()._responses == 0);
    String[] coefnames = model_info().data_info().coefNames();
    assert(len == coefnames.length);
    String[] names = new String[len];
    for(int i = 0; i < names.length; ++i) {
      names[i] = "reconstr_" + coefnames[i];
    }
    Frame f = new MRTask() {
      @Override public void map( Chunk chks[], NewChunk recon[] ) {
        double tmp [] = new double[_output._names.length];
        double preds[] = new double [len];
        final Neurons[] neurons = DeepLearningTask.makeNeuronsForTesting(model_info);
        for( int row=0; row<chks[0]._len; row++ ) {
          double p[] = score_autoencoder(chks, row, tmp, preds, neurons, true /*reconstruction*/, false /*reconstruction_error_per_feature*/);
          for( int c=0; c<len; c++ )
            recon[c].addNum(p[c]);
        }
      }
    }.doAll_numericResult(len,adaptedFr).outputFrame();

    Frame of = new Frame((null == destination_key ? Key.make() : Key.make(destination_key)), names, f.vecs());
    DKV.put(of);
    makeMetricBuilder(null).makeModelMetrics(this, orig);
    return of;
  }
}
 

import water.fvec.Frame; //导入方法依赖的package包/类
@Test
public void run() {
  // Put chunks into KV store
  Frame f = new TestUtil().parse_test_file("smalldata/junit/syn_2659x1049.csv.gz");
  // Create two lockable frames in KV store
  Frame fr1 = new Frame(Key.make(), f.names(), f.vecs());
  Frame fr2 = new Frame(Key.make(), f.names(), f.vecs());
  // Lock the frames against writes
  fr1.delete_and_lock(null);
  fr2.delete_and_lock(null);
  int i = 0;
  try {
    // try to delete the write-locked frames -> will throw an exception
    fr1.delete();
    fr2.delete(); // won't be reached
  } catch (Throwable t) {
    Log.info("Correctly unable to delete (was locked): " + t.getClass()); //either AssertionError if local or DistributedException if remote
    i++;
  } finally {
    // second attempt: will unlock and delete properly
    new UnlockTask().doAllNodes(); // without this line, there will be a leak (and assertion won't be shown)
    fr1.delete();
    fr2.delete();
    f.delete();
    Log.info("Able to delete after unlocking.");
  }
  Assert.assertTrue(i == 1);
}
 

import water.fvec.Frame; //导入方法依赖的package包/类
public static Val execute(AST ast) {
  // Execute
  Env env = new Env();
  Val val = ast.exec(env);
  // Results.  Deep copy returned Vecs.  Always return a key-less Frame
  if( val.isFrame() ) {
    Frame fr = val.getFrame();
    if( fr._key != null ) val=new ValFrame(fr = new Frame(null,fr.names(),fr.vecs()));
    Vec vecs[] = fr.vecs();
    for( int i=0; i<vecs.length; i++ )
      if( env.isPreExistingGlobal(vecs[i]) )
        fr.replace(i,vecs[i].makeCopy());
  }
  return val;
}
 

import water.fvec.Frame; //导入方法依赖的package包/类
@Test
public void DCT_1D() {
  Frame frame = null;
  Frame frameDCT = null;
  Frame frameRec = null;
  try {
    CreateFrame cf = new CreateFrame();
    cf.rows = 100;
    int height = 513;
    int width = 1;
    int depth = 1;
    cf.cols = height*width*depth;
    cf.categorical_fraction = 0.0;
    cf.integer_fraction = 0;
    cf.binary_fraction = 0;
    cf.missing_fraction = 0;
    cf.factors = 0;
    cf.seed = 1234;
    cf.execImpl();
    cf.get();
    frame = DKV.getGet(cf.dest());
    long now = System.currentTimeMillis();
    frameDCT = MathUtils.DCT.transform1D(frame, height, false);
    Log.info("Computed 1D DCT of " + cf.rows + " rows of size " + cf.cols + " in" + PrettyPrint.msecs(System.currentTimeMillis() - now, true));

    now = System.currentTimeMillis();
    frameRec = MathUtils.DCT.transform1D(frameDCT, height, true);
    Log.info("Computed inverse 1D DCT of " + cf.rows + " rows of size " + cf.cols + " in" + PrettyPrint.msecs(System.currentTimeMillis() - now, true));

    for (int i=0; i<frame.vecs().length; ++i)
      TestUtil.assertVecEquals(frame.vecs()[i], frameRec.vecs()[i], 1e-5);
    Log.info("Identity test passed: DCT^-1(DCT(frame)) == frame");
  } finally {
    if (frame!=null) frame.delete();
    if (frameDCT!=null) frameDCT.delete();
    if (frameRec!=null) frameRec.delete();
  }
}
 

import water.fvec.Frame; //导入方法依赖的package包/类
public static double sparseRatio(Frame fr) {
  double reg = 1.0/fr.numCols();
  double res = 0;
  for(Vec v:fr.vecs())
    res += v.sparseRatio();
  return res * reg;
}
 

import water.fvec.Frame; //导入方法依赖的package包/类
public static void initCheck(Frame input, int width, int height, int depth) {
  ConcurrencyUtils.setNumberOfThreads(1);
  if (width < 1 || height < 1 || depth < 1)
    throw new H2OIllegalArgumentException("dimensions must be >= 1");
  if (width*height*depth != input.numCols())
    throw new H2OIllegalArgumentException("dimensions HxWxD must match the # columns of the frame");
  for (Vec v : input.vecs()) {
    if (v.naCnt() > 0)
      throw new H2OIllegalArgumentException("DCT can not be computed on rows with missing values");
    if (!v.isNumeric())
      throw new H2OIllegalArgumentException("DCT can only be computed on numeric columns");
  }
}
 

import water.fvec.Frame; //导入方法依赖的package包/类
@Override public void close() {
  Futures fs = null;
  int sp = sp();
  while( sp > _sp ) {
    Frame fr = _refcnt.remove(--sp);
    for( Vec vec : fr.vecs() )
      if( !inUse(vec) ) {
        if( fs == null ) fs = new Futures();
        vec.remove(fs);
      }
  }
  if( fs != null ) fs.blockForPending();
}
 

import water.fvec.Frame; //导入方法依赖的package包/类
private Val array( Frame frx, Frame fry, Mode mode ) {
  Vec[] vecxs = frx.vecs();  int ncolx = vecxs.length;
  Vec[] vecys = fry.vecs();  int ncoly = vecys.length;
  double[] ymeans = new double[ncoly];
  for( int y=0; y<ncoly; y++ ) // All the Y means
    ymeans[y] = vecys[y].mean();

  // Launch tasks; each does all Ys vs one X
  CoVarTask[] cvts = new CoVarTask[ncolx];
  for( int x=0; x<ncolx; x++ )
    cvts[x] = new CoVarTask(vecxs[x].mean(),ymeans).dfork(new Frame(vecxs[x]).add(fry));
  // Short cut for the 1-row-1-col result: return a scalar
  if( ncolx==1 && ncoly==1 )
    return new ValNum(cvts[0].getResult()._covs[0]/(frx.numRows()-1));

  // Gather all the Ys-vs-X covariance arrays; divide by rows
  Vec[] res = new Vec[ncolx];
  Key<Vec>[] keys = Vec.VectorGroup.VG_LEN1.addVecs(ncolx);
  for( int x=0; x<ncolx; x++ )
    res[x] = Vec.makeVec(ArrayUtils.div(cvts[x].getResult()._covs, (frx.numRows()-1)), keys[x]);

  // CNC - For fun, uncomment this code to scale all values by their
  // respective std-devs, basically standardizing the results.  This gives
  // something similar to a r^2 correlation where 1.0 (max possible value)
  // indicates perfect linearity (maybe someting weaker: perfect equality?),
  // and -1 perfectly anti-linear (90% twist), and zero is remains
  // uncorrelated (with actual covariance, zero is also uncorrelated but
  // non-zero values are scaled by the columns' numeric range).
  //
  //for( int x=0; x<ncolx; x++ ) {
  //  double ds[] = ArrayUtils.div(cvts[x].getResult()._covs, (frx.numRows()-1));
  //  ArrayUtils.div(cvts[x].getResult()._covs, vecxs[x].sigma());
  //  for( int y=0; y<ncoly; y++ )
  //    ds[y] /= vecys[y].sigma();
  //  res[x] = Vec.makeVec(ds, keys[x]);
  //}
  return new ValFrame(new Frame(frx._names,res));
}
 

import water.fvec.Frame; //导入方法依赖的package包/类
/**
 * Transpose the Frame as if it was a matrix (rows <-> columns).
 * Must be all numeric, will fail if there are too many rows ( >= ~.5M).
 *
 * Result is made to be compatible (i.e. the same vector group and chunking) with the target frame.
 *
 * @param src
 * @return
 */
public static Frame transpose(Frame src, Frame tgt){
  if(src.numRows() != tgt.numCols() || src.numCols() != tgt.numRows())
    throw new IllegalArgumentException("dimension do not match!");
  for(Vec v:src.vecs()) {
    if (v.isCategorical())
      throw new IllegalArgumentException("transpose can only be applied to all-numeric frames (representing a matrix)");
    if(v.length() > 1000000)
      throw new IllegalArgumentException("too many rows, transpose only works for frames with < 1M rows.");
  }
  new TransposeTsk(tgt).doAll(src);
  return tgt;
}
 

import water.fvec.Frame; //导入方法依赖的package包/类
@Test
public void DCT_3D() {
  Frame frame = null;
  Frame frameDCT = null;
  Frame frameRec = null;
  try {
    CreateFrame cf = new CreateFrame();
    cf.rows = 100;
    int height = 35;
    int width = 11;
    int depth = 14;
    cf.cols = height*width*depth;
    cf.categorical_fraction = 0.0;
    cf.integer_fraction = 0;
    cf.binary_fraction = 0;
    cf.missing_fraction = 0;
    cf.factors = 0;
    cf.seed = 1234;
    cf.execImpl();
    cf.get();
    frame = DKV.getGet(cf.dest());
    long now = System.currentTimeMillis();
    frameDCT = MathUtils.DCT.transform3D(frame, height, width, depth, false);
    Log.info("Computed 3D DCT of " + cf.rows + " rows of size " + cf.cols + " in" + PrettyPrint.msecs(System.currentTimeMillis() - now, true));

    now = System.currentTimeMillis();
    frameRec = MathUtils.DCT.transform3D(frameDCT, height, width, depth, true);
    Log.info("Computed inverse 3D DCT of " + cf.rows + " rows of size " + cf.cols + " in" + PrettyPrint.msecs(System.currentTimeMillis() - now, true));

    for (int i=0; i<frame.vecs().length; ++i)
      TestUtil.assertVecEquals(frame.vecs()[i], frameRec.vecs()[i], 1e-5);
    Log.info("Identity test passed: DCT^-1(DCT(frame)) == frame");
  } finally {
    if (frame!=null) frame.delete();
    if (frameDCT!=null) frameDCT.delete();
    if (frameRec!=null) frameRec.delete();
  }
}