Java ShuffleSplitFrame類代碼示例

import hex.splitframe.ShuffleSplitFrame; //導入依賴的package包/類
@Test public void testInteractionTrainTestSplitAdapt() {
  DataInfo dinfo=null, scoreInfo=null;
  Frame fr=null, expanded=null;
  Frame[] frSplits=null, expandSplits=null;
  Model.InteractionSpec interactions = Model.InteractionSpec.allPairwise(new String[]{"class", "sepal_len"});

  boolean useAll=false;
  boolean standardize=false;  // golden frame is standardized before splitting, while frame we want to check would be standardized post-split (not exactly what we want!)
  boolean skipMissing=true;
  try {
    fr = parse_test_file(Key.make("a.hex"), "smalldata/iris/iris_wheader.csv");
    fr.swap(3, 4);
    expanded = GLMModel.GLMOutput.expand(fr, interactions, useAll, standardize,skipMissing);   // here's the "golden" frame

    // now split fr and expanded
    long seed;
    frSplits = ShuffleSplitFrame.shuffleSplitFrame(fr, new Key[]{Key.make(), Key.make()}, new double[]{0.8, 0.2}, seed = new Random().nextLong());
    expandSplits = ShuffleSplitFrame.shuffleSplitFrame(expanded, new Key[]{Key.make(), Key.make()}, new double[]{0.8, 0.2}, seed);

    // check1: verify splits. expand frSplits with DataInfo and check against expandSplits
    checkSplits(frSplits,expandSplits,interactions,useAll,standardize);

    // now take the test frame from frSplits, and adapt it to a DataInfo built on the train frame
    dinfo = makeInfo(frSplits[0], interactions, useAll, standardize);
    GLMModel.GLMParameters parms = new GLMModel.GLMParameters();
    parms._response_column = "petal_wid";
    Model.adaptTestForTrain(frSplits[1],null,null,dinfo._adaptedFrame.names(),dinfo._adaptedFrame.domains(),parms,true,false,interactions,null,null, false);
    scoreInfo = dinfo.scoringInfo(dinfo._adaptedFrame._names,frSplits[1]);
    checkFrame(scoreInfo,expandSplits[1]);
  } finally {
    cleanup(fr,expanded);
    cleanup(frSplits);
    cleanup(expandSplits);
    cleanup(dinfo, scoreInfo);
  }
}
 

import hex.splitframe.ShuffleSplitFrame; //導入依賴的package包/類
@Test
public void Airlines() {
  Frame tr = null;
  DeepWaterModel m = null;
  Frame[] splits = null;
  try {
    DeepWaterParameters p = new DeepWaterParameters();
    File file = FileUtils.locateFile("smalldata/airlines/allyears2k_headers.zip");
    if (file != null) {
      p._response_column = "IsDepDelayed";
      p._ignored_columns = new String[]{"DepTime","ArrTime","Cancelled","CancellationCode","Diverted","CarrierDelay","WeatherDelay","NASDelay","SecurityDelay","LateAircraftDelay","IsArrDelayed"};
      NFSFileVec trainfv = NFSFileVec.make(file);
      tr = ParseDataset.parse(Key.make(), trainfv._key);
      for (String col : new String[]{p._response_column, "UniqueCarrier", "Origin", "Dest"}) {
        Vec v = tr.remove(col); tr.add(col, v.toCategoricalVec()); v.remove();
      }
      DKV.put(tr);

      double[] ratios = ard(0.5, 0.5);
      Key[] keys = aro(Key.make("test.hex"), Key.make("train.hex"));
      splits = ShuffleSplitFrame.shuffleSplitFrame(tr, keys, ratios, 42);

      p._backend = getBackend();
      p._train = keys[0];
      p._valid = keys[1];
      DeepWater j = new DeepWater(p);
      m = j.trainModel().get();
      Assert.assertTrue(((ModelMetricsBinomial)(m._output._validation_metrics)).auc() > 0.65);
    }
  } finally {
    if (tr!=null) tr.remove();
    if (m!=null) m.remove();
    if (splits!=null) for(Frame s: splits) s.remove();
  }
}
 

import hex.splitframe.ShuffleSplitFrame; //導入依賴的package包/類
@Test public void testInteractionTrainTestSplitAdaptAirlines() {
  DataInfo dinfo=null, scoreInfo=null;
  Frame frA=null, fr=null, expanded=null;
  Frame[] frSplits=null, expandSplits=null;
  Model.InteractionSpec interactions = Model.InteractionSpec.allPairwise(new String[]{"CRSDepTime", "Origin"});

  String[] keepColumns = new String[]{
          "Year",           "Month"     ,     "DayofMonth" ,    "DayOfWeek",
          "CRSDepTime" ,    "CRSArrTime"   , "UniqueCarrier" , "CRSElapsedTime",
          "Origin"     ,    "Dest"      ,     "Distance"  ,    "IsDepDelayed",

  };

  boolean useAll=false;
  boolean standardize=false;  // golden frame is standardized before splitting, while frame we want to check would be standardized post-split (not exactly what we want!)
  boolean skipMissing=false;
  try {
    frA = parse_test_file(Key.make("a.hex"), "smalldata/airlines/allyears2k_headers.zip");
    fr = frA.subframe(keepColumns);
    expanded = GLMModel.GLMOutput.expand(fr, interactions, useAll, standardize, skipMissing);   // here's the "golden" frame

    // now split fr and expanded
    long seed;
    frSplits = ShuffleSplitFrame.shuffleSplitFrame(fr, new Key[]{Key.make(), Key.make()}, new double[]{0.8, 0.2}, seed = new Random().nextLong());
    expandSplits = ShuffleSplitFrame.shuffleSplitFrame(expanded, new Key[]{Key.make(), Key.make()}, new double[]{0.8, 0.2}, seed);

    // check1: verify splits. expand frSplits with DataInfo and check against expandSplits
    checkSplits(frSplits,expandSplits,interactions,useAll,standardize,skipMissing);

    // now take the test frame from frSplits, and adapt it to a DataInfo built on the train frame
    dinfo = makeInfo(frSplits[0], interactions, useAll, standardize,skipMissing);
    GLMModel.GLMParameters parms = new GLMModel.GLMParameters();
    parms._response_column = "IsDepDelayed";
    Model.adaptTestForTrain(frSplits[1],null,null,dinfo._adaptedFrame.names(),dinfo._adaptedFrame.domains(),parms,true,false,interactions,null,null, false);

    scoreInfo = dinfo.scoringInfo(dinfo._adaptedFrame._names,frSplits[1]);
    checkFrame(scoreInfo,expandSplits[1], skipMissing);

  } finally {
    cleanup(fr,frA,expanded);
    cleanup(frSplits);
    cleanup(expandSplits);
    cleanup(dinfo, scoreInfo);
  }
}