Java Frame.numRows方法代码示例

import water.fvec.Frame; //导入方法依赖的package包/类
/** Initialize the ModelBuilder, validating all arguments and preparing the
 *  training frame.  This call is expected to be overridden in the subclasses
 *  and each subclass will start with "super.init();".
 *
 *  Validate K, max_iterations and the number of rows. */
@Override public void init(boolean expensive) {
  super.init(expensive);
  if( _parms._max_iterations < 0 || _parms._max_iterations > 1e6) error("_max_iterations", " max_iterations must be between 0 and 1e6");
  if( _train == null ) return;
  if( _parms._init == Initialization.User && _parms._user_points == null )
    error("_user_y","Must specify initial cluster centers");
  if( null != _parms._user_points ){ // Check dimensions of user-specified centers
    Frame user_points = _parms._user_points.get();
    if( user_points.numCols() != _train.numCols() - numSpecialCols()) {
      error("_user_y","The user-specified points must have the same number of columns (" + (_train.numCols() - numSpecialCols()) + ") as the training observations");
    } else if( user_points.numRows() != _parms._k)
      error("_user_y","The number of rows in the user-specified points is not equal to k = " + _parms._k);
  }
  if (expensive && error_count() == 0) checkMemoryFootPrint();
}
 

import water.fvec.Frame; //导入方法依赖的package包/类
@Override Val apply( Env env, Env.StackHelp stk, AST asts[] ) {
  Frame frx = stk.track(asts[1].exec(env)).getFrame();
  Frame fry = stk.track(asts[2].exec(env)).getFrame();
  if( frx.numRows() != fry.numRows() )
    throw new IllegalArgumentException("Frames must have the same number of rows, found "+frx.numRows()+" and "+fry.numRows());
  if( frx.numCols() != fry.numCols() )
    throw new IllegalArgumentException("Frames must have the same number of columns, found "+frx.numCols()+" and "+fry.numCols());

  String use = stk.track(asts[3].exec(env)).getStr();
  Mode mode;
  switch( use ) {
  case "everything":            mode = Mode.Everything; break;
  case "all.obs":               mode = Mode.AllObs; break;
  case "complete.obs":          mode = Mode.CompleteObs; break;
  default: throw new IllegalArgumentException("unknown use mode, found: "+use);
  }

  return frx.numRows() == 1 ? scalar(frx,fry,mode) : array(frx,fry,mode);
}
 

import water.fvec.Frame; //导入方法依赖的package包/类
/**
 * Transpose the Frame as if it was a matrix (i.e. rows become coumns).
 * Must be all numeric, currently will fail if there are too many rows ( >= ~.5M).
 * Result will be put into a new Vectro Group and will be balanced so that each vec will have
 * (4*num cpus in the cluster) chunks.
 *
 * @param src
 * @return
 */
public static Frame transpose(Frame src){
  if(src.numRows() != (int)src.numRows())
    throw H2O.unimpl();
  int nchunks = Math.max(1,src.numCols()/10000);
  long [] espc = new long[nchunks+1];
  int rpc = (src.numCols() / nchunks);
  int rem = (src.numCols() % nchunks);
  Arrays.fill(espc, rpc);
  for (int i = 0; i < rem; ++i) ++espc[i];
  long sum = 0;
  for (int i = 0; i < espc.length; ++i) {
    long s = espc[i];
    espc[i] = sum;
    sum += s;
  }
  Key key = Vec.newKey();
  int rowLayout = Vec.ESPC.rowLayout(key,espc);
  return transpose(src, new Frame(new Vec(key,rowLayout).makeZeros((int)src.numRows())));
}
 

import water.fvec.Frame; //导入方法依赖的package包/类
@Test public void test() {
  // Load data
  Frame f = parse_test_file(Key.make("iris.csv"), "smalldata/iris/iris.csv");
  long numRows = f.numRows();
  Assert.assertEquals(150, numRows);
  // Perform frame split via API
  try {
    SplitFrame sf = new SplitFrame();
    sf.dataset = f;
    sf.ratios = new double[] { 0.5, 0.5 };
    sf.destination_frames = new Key[] { Key.make("train.hex"), Key.make("test.hex")};
    // Invoke the job
    sf.exec().get();
    Assert.assertTrue("The job is not in DONE state, but in " + sf._state, sf.isDone());
    Key[] ksplits = sf.destination_frames;
    Frame[] fsplits = new Frame[ksplits.length];
    for (int i=0; i<ksplits.length; i++) fsplits[i] = DKV.get(ksplits[i]).get();
    Assert.assertEquals("Number of splits", 2, ksplits.length);
    Assert.assertEquals("1. split 75rows", 75, fsplits[0].numRows());
    Assert.assertEquals("2. split 75rows", 75, fsplits[1].numRows());
    fsplits[0].delete();
    fsplits[1].delete();
  } finally {
    f.delete();
  }
}
 

import water.fvec.Frame; //导入方法依赖的package包/类
public double[] predict(Frame inputs) {
    Frame predsFrame = dlModel.score(inputs);
    int numRows = (int) inputs.numRows();
    Vec predsVector = predsFrame.vec(0);
    double[] predVals = new double[numRows];
    for (int i = 0; i < numRows; i++) {
        predVals[i] = predsVector.at(i);
    }
    return predVals;
}
 

import water.fvec.Frame; //导入方法依赖的package包/类
/** Load simple dataset, rebalance to a number of chunks > number of rows, and run deep learning */
@Test public void testPubDev928() {
  // Create rebalanced dataset
  Key rebalancedKey = Key.make("rebalanced");
  NFSFileVec nfs = NFSFileVec.make(find_test_file("smalldata/logreg/prostate.csv"));
  Frame fr = ParseDataset.parse(Key.make(), nfs._key);
  RebalanceDataSet rb = new RebalanceDataSet(fr, rebalancedKey, (int)(fr.numRows()+1));
  H2O.submitTask(rb);
  rb.join();
  Frame rebalanced = DKV.get(rebalancedKey).get();

  // Assert that there is at least one 0-len chunk
  assertZeroLengthChunk("Rebalanced dataset should contain at least one 0-len chunk!", rebalanced.anyVec());

  DeepLearningModel dlModel = null;
  try {
    // Launch Deep Learning
    DeepLearningParameters dlParams = new DeepLearningParameters();
    dlParams._train = rebalancedKey;
    dlParams._epochs = 5;
    dlParams._response_column = "CAPSULE";

    dlModel = new DeepLearning(dlParams).trainModel().get();
  } finally {
    fr.delete();
    rebalanced.delete();
    if (dlModel != null) dlModel.delete();
  }
}
 

import water.fvec.Frame; //导入方法依赖的package包/类
FrameSynopsisV3(Frame fr) {
  Vec[] vecs = fr.vecs();

  frame_id = new FrameKeyV3(fr._key);
  _fr = fr;
  rows = fr.numRows();
  columns = vecs.length;
  byte_size = fr.byteSize();
  is_text = fr.numCols()==1 && vecs[0] instanceof ByteVec;
}
 

import water.fvec.Frame; //导入方法依赖的package包/类
/**
 * Sample rows from a frame.
 * Can be unlucky for small sampling fractions - will continue calling itself until at least 1 row is returned.
 * @param fr Input frame
 * @param rows Approximate number of rows to sample (across all chunks)
 * @param seed Seed for RNG
 * @return Sampled frame
 */
public static Frame sampleFrame(Frame fr, final long rows, final long seed) {
  if (fr == null) return null;
  final float fraction = rows > 0 ? (float)rows / fr.numRows() : 1.f;
  if (fraction >= 1.f) return fr;
  Key newKey = fr._key != null ? Key.make(fr._key.toString() + (fr._key.toString().contains("temporary") ? ".sample." : ".temporary.sample.") + PrettyPrint.formatPct(fraction).replace(" ","")) : null;

  Frame r = new MRTask() {
    @Override
    public void map(Chunk[] cs, NewChunk[] ncs) {
      final Random rng = getRNG(seed + cs[0].cidx());
      int count = 0;
      for (int r = 0; r < cs[0]._len; r++)
        if (rng.nextFloat() < fraction || (count == 0 && r == cs[0]._len-1) ) {
          count++;
          for (int i = 0; i < ncs.length; i++) {
            ncs[i].addNum(cs[i].atd(r));
          }
        }
    }
  }.doAll(fr.types(), fr).outputFrame(newKey, fr.names(), fr.domains());
  if (r.numRows() == 0) {
    Log.warn("You asked for " + rows + " rows (out of " + fr.numRows() + "), but you got none (seed=" + seed + ").");
    Log.warn("Let's try again. You've gotta ask yourself a question: \"Do I feel lucky?\"");
    return sampleFrame(fr, rows, seed+1);
  }
  return r;
}
 

import water.fvec.Frame; //导入方法依赖的package包/类
private Val array( Frame frx, Frame fry, Mode mode ) {
  Vec[] vecxs = frx.vecs();  int ncolx = vecxs.length;
  Vec[] vecys = fry.vecs();  int ncoly = vecys.length;
  double[] ymeans = new double[ncoly];
  for( int y=0; y<ncoly; y++ ) // All the Y means
    ymeans[y] = vecys[y].mean();

  // Launch tasks; each does all Ys vs one X
  CoVarTask[] cvts = new CoVarTask[ncolx];
  for( int x=0; x<ncolx; x++ )
    cvts[x] = new CoVarTask(vecxs[x].mean(),ymeans).dfork(new Frame(vecxs[x]).add(fry));
  // Short cut for the 1-row-1-col result: return a scalar
  if( ncolx==1 && ncoly==1 )
    return new ValNum(cvts[0].getResult()._covs[0]/(frx.numRows()-1));

  // Gather all the Ys-vs-X covariance arrays; divide by rows
  Vec[] res = new Vec[ncolx];
  Key<Vec>[] keys = Vec.VectorGroup.VG_LEN1.addVecs(ncolx);
  for( int x=0; x<ncolx; x++ )
    res[x] = Vec.makeVec(ArrayUtils.div(cvts[x].getResult()._covs, (frx.numRows()-1)), keys[x]);

  // CNC - For fun, uncomment this code to scale all values by their
  // respective std-devs, basically standardizing the results.  This gives
  // something similar to a r^2 correlation where 1.0 (max possible value)
  // indicates perfect linearity (maybe someting weaker: perfect equality?),
  // and -1 perfectly anti-linear (90% twist), and zero is remains
  // uncorrelated (with actual covariance, zero is also uncorrelated but
  // non-zero values are scaled by the columns' numeric range).
  //
  //for( int x=0; x<ncolx; x++ ) {
  //  double ds[] = ArrayUtils.div(cvts[x].getResult()._covs, (frx.numRows()-1));
  //  ArrayUtils.div(cvts[x].getResult()._covs, vecxs[x].sigma());
  //  for( int y=0; y<ncoly; y++ )
  //    ds[y] /= vecys[y].sigma();
  //  res[x] = Vec.makeVec(ds, keys[x]);
  //}
  return new ValFrame(new Frame(frx._names,res));
}
 

import water.fvec.Frame; //导入方法依赖的package包/类
Val exec_check( Env env, Env.StackHelp stk, Frame tst, AST ast, Frame xfr ) {
  Val val = ast.exec(env);
  if( val.isFrame() ) {
    Frame fr = stk.track(val).getFrame();
    if( tst.numCols() != fr.numCols() || tst.numRows() != fr.numRows() )
      throw new IllegalArgumentException("ifelse test frame and other frames must match dimensions, found "+tst+" and "+fr);
    xfr.add(fr);
  }
  return val;
}
 

import water.fvec.Frame; //导入方法依赖的package包/类
public MatrixMulTsk(H2OCountedCompleter cmp, Key progressKey, Frame x, Frame y) {
  super(cmp);
  if(x.numCols() != y.numRows())
    throw new IllegalArgumentException("dimensions do not match! x.numcols = " + x.numCols() + ", y.numRows = " + y.numRows());
  _x = x;
  _y = y;
  _progressKey = progressKey;
}
 

import water.fvec.Frame; //导入方法依赖的package包/类
double[][] initial_centers( KMeansModel model, final Vec[] vecs, final double[] means, final double[] mults, final int[] modes ) {

      // Categoricals use a different distance metric than numeric columns.
      model._output._categorical_column_count=0;
      _isCats = new String[vecs.length][];
      for( int v=0; v<vecs.length; v++ ) {
        _isCats[v] = vecs[v].isCategorical() ? new String[0] : null;
        if (_isCats[v] != null) model._output._categorical_column_count++;
      }
      
      Random rand = water.util.RandomUtils.getRNG(_parms._seed - 1);
      double centers[][];    // Cluster centers
      if( null != _parms._user_points ) { // User-specified starting points
        Frame user_points = _parms._user_points.get();
        int numCenters = (int)user_points.numRows();
        int numCols = model._output.nfeatures();
        centers = new double[numCenters][numCols];
        Vec[] centersVecs = user_points.vecs();
        // Get the centers and standardize them if requested
        for (int r=0; r<numCenters; r++) {
          for (int c=0; c<numCols; c++){
            centers[r][c] = centersVecs[c].at(r);
            centers[r][c] = data(centers[r][c], c, means, mults, modes);
          }
        }
      }
      else { // Random, Furthest, or PlusPlus initialization
        if (_parms._init == Initialization.Random) {
          // Initialize all cluster centers to random rows
          centers = new double[_parms._k][model._output.nfeatures()];
          for (double[] center : centers)
            randomRow(vecs, rand, center, means, mults, modes);
        } else {
          centers = new double[1][model._output.nfeatures()];
          // Initialize first cluster center to random row
          randomRow(vecs, rand, centers[0], means, mults, modes);

          model._output._iterations = 0;
          while (model._output._iterations < 5) {
            // Sum squares distances to cluster center
            SumSqr sqr = new SumSqr(centers, means, mults, modes, _isCats).doAll(vecs);

            // Sample with probability inverse to square distance
            Sampler sampler = new Sampler(centers, means, mults, modes, _isCats, sqr._sqr, _parms._k * 3, _parms._seed, hasWeightCol()).doAll(vecs);
            centers = ArrayUtils.append(centers, sampler._sampled);

            // Fill in sample centers into the model
            if (!isRunning()) return null; // Stopped/cancelled
            model._output._centers_raw = destandardize(centers, _isCats, means, mults);
            model._output._tot_withinss = sqr._sqr / _train.numRows();

            model._output._iterations++;     // One iteration done

            model.update(_key); // Make early version of model visible, but don't update progress using update(1)
          }
          // Recluster down to k cluster centers
          centers = recluster(centers, rand, _parms._k, _parms._init, _isCats);
          model._output._iterations = 0; // Reset iteration count
        }
      }
      return centers;
    }
 

import water.fvec.Frame; //导入方法依赖的package包/类
@Test
  public void testNfoldsOneVsRest() {
    Frame tfr = null;
    DRFModel drf1 = null;
    DRFModel drf2 = null;

    Scope.enter();
    try {
      tfr = parse_test_file("smalldata/junit/weights.csv");
      DKV.put(tfr);
      DRFModel.DRFParameters parms = new DRFModel.DRFParameters();
      parms._train = tfr._key;
      parms._response_column = "response";
      parms._seed = 9999;
      parms._min_rows = 2;
      parms._nfolds = (int) tfr.numRows();
      parms._fold_assignment = Model.Parameters.FoldAssignmentScheme.Modulo;
      parms._max_depth = 5;
      parms._ntrees = 5;

      DRF job1 = new DRF(parms);
      drf1 = job1.trainModel().get();

//            parms._nfolds = (int) tfr.numRows() + 1; //this is now an error
      DRF job2 = new DRF(parms);
      drf2 = job2.trainModel().get();

      ModelMetricsBinomial mm1 = (ModelMetricsBinomial)drf1._output._cross_validation_metrics;
      ModelMetricsBinomial mm2 = (ModelMetricsBinomial)drf2._output._cross_validation_metrics;
      assertEquals(mm1.auc()._auc, mm2.auc()._auc, 1e-12);
      assertEquals(mm1.mse(), mm2.mse(), 1e-12);
      assertEquals(mm1.r2(), mm2.r2(), 1e-12);
      assertEquals(mm1.logloss(), mm2.logloss(), 1e-12);

      //TODO: add check: the correct number of individual models were built. PUBDEV-1690

      job1.remove();
      job2.remove();
    } finally {
      if (tfr != null) tfr.remove();
      if (drf1 != null) {
        drf1.deleteCrossValidationModels();
        drf1.delete();
      }
      if (drf2 != null) {
        drf2.deleteCrossValidationModels();
        drf2.delete();
      }
      Scope.exit();
    }
  }
 

import water.fvec.Frame; //导入方法依赖的package包/类
RapidsFrameV3( Frame fr ) { key = new KeyV3.FrameKeyV3(fr._key); num_rows = fr.numRows(); num_cols = fr.numCols(); }