Java NewChunk类代码示例

import water.fvec.NewChunk; //导入依赖的package包/类
/**
 * Row-wise shuffle of a frame (only shuffles rows inside of each chunk)
 * @param fr Input frame
 * @return Shuffled frame
 */
public static Frame shuffleFramePerChunk(Frame fr, final long seed) {
  return new MRTask() {
    @Override
    public void map(Chunk[] cs, NewChunk[] ncs) {
      int[] idx = new int[cs[0]._len];
      for (int r=0; r<idx.length; ++r) idx[r] = r;
      ArrayUtils.shuffleArray(idx, getRNG(seed));
      for (long anIdx : idx) {
        for (int i = 0; i < ncs.length; i++) {
          ncs[i].addNum(cs[i].atd((int) anIdx));
        }
      }
    }
  }.doAll_numericResult(fr.numCols(), fr).outputFrame(fr.names(), fr.domains());
}
 

import water.fvec.NewChunk; //导入依赖的package包/类
/**
 * Compute the 1D discrete cosine transform for each row in the given Frame, and return a new Frame
 *
 * @param input   Frame containing numeric columns with data samples
 * @param N       Number of samples (must be less or equal than number of columns)
 * @param inverse Whether to compute the inverse
 * @return Frame containing 1D (inverse) DCT of each row (same dimensionality)
 */
public static Frame transform1D(Frame input, final int N, final boolean inverse) {
  initCheck(input, N, 1, 1);
  return new MRTask() {
    @Override
    public void map(Chunk[] cs, NewChunk[] ncs) {
      double[] a = new double[N];
      for (int row = 0; row < cs[0]._len; ++row) {
        // fill 1D array
        for (int i = 0; i < N; ++i)
          a[i] = cs[i].atd(row);

        // compute DCT for each row
        if (!inverse)
          new DoubleDCT_1D(N).forward(a, true);
        else
          new DoubleDCT_1D(N).inverse(a, true);

        // write result to NewChunk
        for (int i = 0; i < N; ++i)
          ncs[i].addNum(a[i]);
      }
    }
  }.doAll_numericResult(input.numCols(), input).outputFrame();
}
 

import water.fvec.NewChunk; //导入依赖的package包/类
@Override protected Frame transformImpl(Frame f) {
  final double[] fmeans = means;
  final double[] fmults = ArrayUtils.invert(sdevs);
  return new MRTask() {
    @Override public void map(Chunk[] cs, NewChunk[] ncs) {
      double[] in = new double[cs.length];
      for(int row=0; row<cs[0]._len; row++) {
        for(int col=0; col<cs.length; col++)
          in[col] = cs[col].atd(row);
        GenMunger.scaleInPlace(fmeans, fmults, in);
        for(int col=0; col<ncs.length; col++)
          ncs[col].addNum(in[col]);
      }
    }
  }.doAll_numericResult(f.numCols(),f).outputFrame(f.names(),f.domains());
}
 

import water.fvec.NewChunk; //导入依赖的package包/类
@Override Val apply( Env env, Env.StackHelp stk, AST asts[] ) {
  ArrayList<String> domain = new ArrayList<>();
  Futures fs = new Futures();
  AppendableVec av = new AppendableVec(Vec.VectorGroup.VG_LEN1.addVec(),Vec.T_CAT);
  NewChunk keys = new NewChunk(av,0);
  int r = 0;
  for( Key key : KeySnapshot.globalSnapshot().keys()) {
    keys.addCategorical(r++);
    domain.add(key.toString());
  }
  String[] key_domain = domain.toArray(new String[domain.size()]);
  av.setDomain(key_domain);
  keys.close(fs);
  Vec c0 = av.layout_and_close(fs);   // c0 is the row index vec
  fs.blockForPending();
  return new ValFrame(new Frame(Key.make("h2o_ls"), new String[]{"key"}, new Vec[]{c0}));
}
 

import water.fvec.NewChunk; //导入依赖的package包/类
private ValFrame fast_table( Vec v1, int ncols, String colname ) {
  if( ncols != 1 || !v1.isInt() ) return null;
  long spanl = (long)v1.max()-(long)v1.min()+1;
  if( spanl > 1000000 ) return null; // Cap at decent array size, for performance

  // First fast-pass counting
  final long cnts[] = new FastCnt((long)v1.min(),(int)spanl).doAll(v1)._cnts;

  // Second pass to build the result frame, skipping zeros
  Vec dataLayoutVec = Vec.makeCon(0, cnts.length);
  Frame fr = new MRTask() {
      @Override public void map(Chunk cs[], NewChunk nc0, NewChunk nc1) {
        final Chunk c = cs[0];
        for( int i = 0; i < c._len; ++i ) {
          int idx = (int) (i + c.start());
          if( cnts[idx] > 0 ) {
            nc0.addNum(idx);
            nc1.addNum(cnts[idx]);
          }
        }
      }
    }.doAll(new byte[]{Vec.T_NUM,Vec.T_NUM}, dataLayoutVec).outputFrame(new String[]{colname, "Count"},
                                          new String[][]{v1.domain(),null});
  dataLayoutVec.remove();
  return new ValFrame(fr);
}
 

import water.fvec.NewChunk; //导入依赖的package包/类
@Override Val apply( Env env, Env.StackHelp stk, AST asts[] ) {
  Val val = asts[1].exec(env);
  switch( val.type() ) {
  case Val.NUM: 
    double d = val.getNum();
    return new ValNum(Double.isNaN(d) ? d : op(new MutableDateTime(0),d));
  case Val.FRM: 
    Frame fr = stk.track(val).getFrame();
    if( fr.numCols() > 1 ) throw water.H2O.unimpl();
    return new ValFrame(new MRTask() {
        @Override public void map( Chunk chk, NewChunk cres ) {
          MutableDateTime mdt = new MutableDateTime(0,ParseTime.getTimezone());
          for( int i=0; i<chk._len; i++ )
            cres.addNum(chk.isNA(i) ? Double.NaN : op(mdt,chk.at8(i)));
        }
      }.doAll_numericResult(1,fr).outputFrame(fr._names, factors()));
  default: throw water.H2O.fail();
  }
}
 

import water.fvec.NewChunk; //导入依赖的package包/类
@Override public void map(Chunk [] chks) {
  Chunk zChunk = chks[chks.length-1];
  double [] res = MemoryManager.malloc8d(chks[0]._len);
  for(int i = 0; i < _y.length; ++i) {
    final double yVal = _y[i];
    final Chunk xChunk = chks[i];
    for (int k = xChunk.nextNZ(-1); k < res.length; k = xChunk.nextNZ(k))
      try { res[k] += yVal * xChunk.atd(k);} catch(Throwable t) {
        t.printStackTrace();
        throw new RuntimeException(t);
      }
  }
  Chunk modChunk = new NewChunk(res).setSparseRatio(2).compress();
  if(_progressKey != null)
    new UpdateProgress(modChunk.getBytes().length,modChunk.frozenType()).fork(_progressKey);
  DKV.put(zChunk.vec().chunkKey(zChunk.cidx()),modChunk,_fs);
}
 

import water.fvec.NewChunk; //导入依赖的package包/类
private void processRow(DataInfo.Row r, float [] res, double [] ps, NewChunk [] preds, int ncols) {
  if(_dinfo._responses != 0)res[0] = (float) r.response[0];
  if (r.predictors_bad) {
    Arrays.fill(ps,Double.NaN);
  } else if(r.weight == 0) {
    Arrays.fill(ps,0);
  } else {
    scoreRow(r, r.offset, ps);
    if (_computeMetrics && !r.response_bad)
      _mb.perRow(ps, res, r.weight, r.offset, _m);
  }
  if (_generatePredictions) {
    for (int c = 0; c < ncols; c++)  // Output predictions; sized for train only (excludes extra test classes)
      preds[c].addNum(ps[c]);
    if(_vcov != null) { // compute standard error on prediction
      preds[ncols].addNum(Math.sqrt(r.innerProduct(r.mtrxMul(_vcov, _tmp))));
    }
  }
}
 

import water.fvec.NewChunk; //导入依赖的package包/类
@Override
public Frame scoreExemplarMembers(Key<Frame> destination_key, final int exemplarIdx) {
  Vec booleanCol = new MRTask() {
    @Override
    public void map(Chunk c, NewChunk nc) {
      for (int i=0;i<c._len;++i)
        nc.addNum(c.at8(i)==_exemplars[exemplarIdx].gid ? 1 : 0,0);
    }
  }.doAll(Vec.T_NUM, new Frame(new Vec[]{_exemplar_assignment_vec_key.get()})).outputFrame().anyVec();

  Frame orig = _parms.train();
  Vec[] vecs = Arrays.copyOf(orig.vecs(), orig.vecs().length+1);
  vecs[vecs.length-1] = booleanCol;

  Frame ff = new Frame(orig.names(), orig.vecs());
  ff.add("predicate", booleanCol);
  Frame res = new Frame.DeepSelect().doAll(orig.types(),ff).outputFrame(destination_key, orig.names(), orig.domains());
  FrameUtils.shrinkDomainsToObservedSubset(res);
  DKV.put(res);
  assert(res.numRows()==_counts[exemplarIdx]);
  booleanCol.remove();
  return res;
}
 

import water.fvec.NewChunk; //导入依赖的package包/类
/**
 * Compute the 1D discrete cosine transform for each row in the given Frame, and return a new Frame
 *
 * @param input   Frame containing numeric columns with data samples
 * @param N       Number of samples (must be less or equal than number of columns)
 * @param inverse Whether to compute the inverse
 * @return Frame containing 1D (inverse) DCT of each row (same dimensionality)
 */
public static Frame transform1D(Frame input, final int N, final boolean inverse) {
  initCheck(input, N, 1, 1);
  return new MRTask() {
    @Override
    public void map(Chunk[] cs, NewChunk[] ncs) {
      double[] a = new double[N];
      for (int row = 0; row < cs[0]._len; ++row) {
        // fill 1D array
        for (int i = 0; i < N; ++i)
          a[i] = cs[i].atd(row);

        // compute DCT for each row
        if (!inverse)
          new DoubleDCT_1D(N).forward(a, true);
        else
          new DoubleDCT_1D(N).inverse(a, true);

        // write result to NewChunk
        for (int i = 0; i < N; ++i)
          ncs[i].addNum(a[i]);
      }
    }
  }.doAll(input.numCols(), Vec.T_NUM, input).outputFrame();
}
 

import water.fvec.NewChunk; //导入依赖的package包/类
@Override protected Frame transformImpl(Frame f) {
  final double[] fmeans = means;
  final double[] fmults = ArrayUtils.invert(sdevs);
  return new MRTask() {
    @Override public void map(Chunk[] cs, NewChunk[] ncs) {
      double[] in = new double[cs.length];
      for(int row=0; row<cs[0]._len; row++) {
        for(int col=0; col<cs.length; col++)
          in[col] = cs[col].atd(row);
        GenMunger.scaleInPlace(fmeans, fmults, in);
        for(int col=0; col<ncs.length; col++)
          ncs[col].addNum(in[col]);
      }
    }
  }.doAll(f.numCols(), Vec.T_NUM, f).outputFrame(f.names(), f.domains());
}
 

import water.fvec.NewChunk; //导入依赖的package包/类
@Override
public void map(Chunk c, NewChunk n) {
  OutputWriter w = OutputWriter.makeWriter(_gh, n, c.start());
  Pattern p = _gh.compilePattern();
  Matcher m = p.matcher("");
  BufferedString bs = new BufferedString();
  int rows = c._len;
  for (int r = 0; r < rows; r++) {
    if (c.isNA(r)) {
      w.addNA(r);
    } else {
      m.reset(c.atStr(bs, r).toString());
      w.addRow(r, m.find());
    }
  }
}
 

import water.fvec.NewChunk; //导入依赖的package包/类
@Override public void map(Chunk[] chks, NewChunk nc) {
  int nVecs = cm.length;
  assert chks.length == nVecs;
  Chronology chronology = ISOChronology.getInstanceUTC();
  int nChunkRows = chks[0]._len;
  int[] tpl = new int[tp.length];
  System.arraycopy(tp, 0, tpl, 0, tp.length);

  BYROW:
  for (int i = 0; i < nChunkRows; i++) {
    for (int j = 0; j < nVecs; j++) {
      double d = chks[j].atd(i);
      if (Double.isNaN(d)) {
        nc.addNum(Double.NaN);
        continue BYROW;
      }
      tpl[cm[j]] = (int) d;
    }
    try {
      double millis = chronology.getDateTimeMillis(tpl[0], tpl[1], tpl[2], tpl[3], tpl[4], tpl[5], tpl[6]);
      nc.addNum(millis);
    } catch (IllegalFieldValueException e) {
      nc.addNum(Double.NaN);
    }
  }
}
 

import water.fvec.NewChunk; //导入依赖的package包/类
@Override
public Val apply(Env env, Env.StackHelp stk, AstRoot asts[]) {
  Val val = asts[1].exec(env);
  switch (val.type()) {
    case Val.NUM:
      double d = val.getNum();
      return new ValNum(Double.isNaN(d) ? d : op(new MutableDateTime(0), d));
    case Val.FRM:
      Frame fr = stk.track(val).getFrame();
      if (fr.numCols() > 1) throw water.H2O.unimpl();
      return new ValFrame(new MRTask() {
        @Override
        public void map(Chunk chk, NewChunk cres) {
          MutableDateTime mdt = new MutableDateTime(0, ParseTime.getTimezone());
          for (int i = 0; i < chk._len; i++)
            cres.addNum(chk.isNA(i) ? Double.NaN : op(mdt, chk.at8(i)));
        }
      }.doAll(1, Vec.T_NUM, fr).outputFrame(fr._names, factors()));
    default:
      throw water.H2O.fail();
  }
}
 

import water.fvec.NewChunk; //导入依赖的package包/类
@Override
public ValFrame apply(Env env, Env.StackHelp stk, AstRoot asts[]) {
  double from = asts[1].exec(env).getNum();
  double to = asts[2].exec(env).getNum();
  double by = asts[3].exec(env).getNum();
  double delta = to - from;
  if (delta == 0 && to == 0)
    throw new IllegalArgumentException("Expected `to` and `from` to have nonzero difference.");
  else {
    double n = delta / by;
    if (n < 0) throw new IllegalArgumentException("wrong sign in 'by' argument");
    else if (n > Double.MAX_VALUE) throw new IllegalArgumentException("'by' argument is much too small");
    Futures fs = new Futures();
    AppendableVec av = new AppendableVec(Vec.newKey(), Vec.T_NUM);
    NewChunk nc = new NewChunk(av, 0);
    int len = (int) n + 1;
    for (int r = 0; r < len; r++) nc.addNum(from + r * by);
    // May need to adjust values = by > 0 ? min(values, to) : max(values, to)
    nc.close(0, fs);
    Vec vec = av.layout_and_close(fs);
    fs.blockForPending();
    return new ValFrame(new Frame(vec));
  }
}