當前位置: 首頁>>代碼示例>>Java>>正文

Java LocalFileSystem類代碼示例

本文整理匯總了Java中org.apache.hadoop.fs.LocalFileSystem類的典型用法代碼示例。如果您正苦於以下問題：Java LocalFileSystem類的具體用法？Java LocalFileSystem怎麽用？Java LocalFileSystem使用的例子？那麽, 這裏精選的類代碼示例或許可以為您提供幫助。


LocalFileSystem類屬於org.apache.hadoop.fs包，在下文中一共展示了LocalFileSystem類的15個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Java代碼示例。

示例1: incTrain

import org.apache.hadoop.fs.LocalFileSystem; //導入依賴的package包/類
public void incTrain() {
  setConf();
  String inputPath = "../data/exampledata/LRLocalExampleData/a9a.train";
  String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
  String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");
  String loadPath = LOCAL_FS + TMP_PATH + "/model";
  String savePath = LOCAL_FS + TMP_PATH + "/newmodel";
  String logPath = LOCAL_FS + TMP_PATH + "/log";

  // Set trainning data path
  conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, inputPath);
  // Set load model path
  conf.set(AngelConf.ANGEL_LOAD_MODEL_PATH, loadPath);
  // Set save model path
  conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, savePath);
  // Set log path
  conf.set(AngelConf.ANGEL_LOG_PATH, logPath);
  // Set actionType incremental train
  conf.set(AngelConf.ANGEL_ACTION_TYPE, MLConf.ANGEL_ML_INC_TRAIN());

  LRRunner runner = new LRRunner();
  runner.incTrain(conf);
}
開發者ID:Tencent，項目名稱:angel，代碼行數:24，代碼來源:SGDLRLocalExample.java

示例2: trainOnLocalCluster

import org.apache.hadoop.fs.LocalFileSystem; //導入依賴的package包/類
public void trainOnLocalCluster() throws Exception {
  setConf();
  String inputPath = "../data/exampledata/LRLocalExampleData/a9a.train";
  String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
  String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");
  String savePath = LOCAL_FS + TMP_PATH + "/model";
  String logPath = LOCAL_FS + TMP_PATH + "/log";

  // Set trainning data path
  conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, inputPath);
  // Set save model path
  conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, savePath);
  // Set log path
  conf.set(AngelConf.ANGEL_LOG_PATH, logPath);
  // Set actionType train
  conf.set(AngelConf.ANGEL_ACTION_TYPE, MLConf.ANGEL_ML_TRAIN());


  LRRunner runner = new LRRunner();
  runner.train(conf);
}
開發者ID:Tencent，項目名稱:angel，代碼行數:22，代碼來源:SGDLRLocalExample.java

示例3: open

import org.apache.hadoop.fs.LocalFileSystem; //導入依賴的package包/類
protected void open(Path dstPath, CompressionCodec codeC,
    CompressionType compType, Configuration conf, FileSystem hdfs)
        throws IOException {
  if(useRawLocalFileSystem) {
    if(hdfs instanceof LocalFileSystem) {
      hdfs = ((LocalFileSystem)hdfs).getRaw();
    } else {
      logger.warn("useRawLocalFileSystem is set to true but file system " +
          "is not of type LocalFileSystem: " + hdfs.getClass().getName());
    }
  }
  if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile
          (dstPath)) {
    outStream = hdfs.append(dstPath);
  } else {
    outStream = hdfs.create(dstPath);
  }
  writer = SequenceFile.createWriter(conf, outStream,
      serializer.getKeyClass(), serializer.getValueClass(), compType, codeC);

  registerCurrentStream(outStream, hdfs, dstPath);
}
開發者ID:Transwarp-DE，項目名稱:Transwarp-Sample-Code，代碼行數:23，代碼來源:HDFSSequenceFile.java

示例4: open

import org.apache.hadoop.fs.LocalFileSystem; //導入依賴的package包/類
protected void open(Path dstPath, CompressionCodec codeC,
    CompressionType compType, Configuration conf, FileSystem hdfs)
        throws IOException {
  if (useRawLocalFileSystem) {
    if (hdfs instanceof LocalFileSystem) {
      hdfs = ((LocalFileSystem)hdfs).getRaw();
    } else {
      logger.warn("useRawLocalFileSystem is set to true but file system " +
          "is not of type LocalFileSystem: " + hdfs.getClass().getName());
    }
  }
  if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile(dstPath)) {
    outStream = hdfs.append(dstPath);
  } else {
    outStream = hdfs.create(dstPath);
  }
  writer = SequenceFile.createWriter(conf, outStream,
      serializer.getKeyClass(), serializer.getValueClass(), compType, codeC);

  registerCurrentStream(outStream, hdfs, dstPath);
}
開發者ID:moueimei，項目名稱:flume-release-1.7.0，代碼行數:22，代碼來源:HDFSSequenceFile.java

示例5: predict

import org.apache.hadoop.fs.LocalFileSystem; //導入依賴的package包/類
public void predict() {
  setConf();
  String inputPath = "../data/exampledata/LRLocalExampleData/a9a.test";
  String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
  String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");
  String loadPath = LOCAL_FS + TMP_PATH + "/model";
  String savePath = LOCAL_FS + TMP_PATH + "/model";
  String logPath = LOCAL_FS + TMP_PATH + "/log";
  String predictPath = LOCAL_FS + TMP_PATH + "/predict";

  // Set trainning data path
  conf.set(AngelConf.ANGEL_PREDICT_DATA_PATH, inputPath);
  // Set load model path
  conf.set(AngelConf.ANGEL_LOAD_MODEL_PATH, loadPath);
  // Set predict result path
  conf.set(AngelConf.ANGEL_PREDICT_PATH, predictPath);
  // Set log path
  conf.set(AngelConf.ANGEL_LOG_PATH, logPath);
  // Set actionType prediction
  conf.set(AngelConf.ANGEL_ACTION_TYPE, MLConf.ANGEL_ML_PREDICT());

  LRRunner runner = new LRRunner();

  runner.predict(conf);
}
開發者ID:Tencent，項目名稱:angel，代碼行數:26，代碼來源:SGDLRLocalExample.java

示例6: trainOnLocalCluster

import org.apache.hadoop.fs.LocalFileSystem; //導入依賴的package包/類
public void trainOnLocalCluster() throws Exception {
  setConf();
  String inputPath = "../data/exampledata/LinearRegression";
  String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
  String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");
  String logPath = "./src/test/log";

  // Set trainning data path
  conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, inputPath);
  // Set save model path
  conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/model");
  // Set log path
  conf.set(AngelConf.ANGEL_LOG_PATH, logPath);
  // Set actionType train
  conf.set(AngelConf.ANGEL_ACTION_TYPE, MLConf.ANGEL_ML_TRAIN());


  LinearRegRunner runner = new LinearRegRunner();
  runner.train(conf);
}
開發者ID:Tencent，項目名稱:angel，代碼行數:21，代碼來源:LinearRegLocalExample.java

示例7: incTrain

import org.apache.hadoop.fs.LocalFileSystem; //導入依賴的package包/類
public void incTrain() {
  setConf();
  String inputPath = "../data/exampledata/LinearRegression/LinearReg100.train";
  String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
  String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");
  String logPath = "./src/test/log";

  // Set trainning data path
  conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, inputPath);
  // Set load model path
  conf.set(AngelConf.ANGEL_LOAD_MODEL_PATH, LOCAL_FS + TMP_PATH + "/model");
  // Set save model path
  conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/newmodel");
  // Set actionType incremental train
  conf.set(AngelConf.ANGEL_ACTION_TYPE, MLConf.ANGEL_ML_INC_TRAIN());
  // Set log path
  conf.set(AngelConf.ANGEL_LOG_PATH, logPath);

  LinearRegRunner runner = new LinearRegRunner();
  runner.incTrain(conf);
}
開發者ID:Tencent，項目名稱:angel，代碼行數:22，代碼來源:LinearRegLocalExample.java

示例8: predict

import org.apache.hadoop.fs.LocalFileSystem; //導入依賴的package包/類
public void predict() {
  setConf();
  String inputPath = "../data/exampledata/LinearRegression/LinearReg100.train";
  String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
  String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");

  // Set trainning data path
  conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, inputPath);
  // Set load model path
  conf.set(AngelConf.ANGEL_LOAD_MODEL_PATH, LOCAL_FS + TMP_PATH + "/model");
  // Set predict result path
  conf.set(AngelConf.ANGEL_PREDICT_PATH, LOCAL_FS + TMP_PATH + "/predict");
  // Set actionType prediction
  conf.set(AngelConf.ANGEL_ACTION_TYPE, MLConf.ANGEL_ML_PREDICT());
  LinearRegRunner runner = new LinearRegRunner();

  runner.predict(conf);
}
開發者ID:Tencent，項目名稱:angel，代碼行數:19，代碼來源:LinearRegLocalExample.java

示例9: trainOnLocalClusterTest

import org.apache.hadoop.fs.LocalFileSystem; //導入依賴的package包/類
@Test
public void trainOnLocalClusterTest() throws Exception {
  String inputPath = "./src/test/data/fm/food_fm_libsvm";
  String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
  String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");
  String savePath = LOCAL_FS + TMP_PATH + "/model";
  String logPath = LOCAL_FS + TMP_PATH + "/LRlog";

  // Set trainning data path
  conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, inputPath);
  // Set save model path
  conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, savePath);
  // Set log path
  conf.set(AngelConf.ANGEL_LOG_PATH, logPath);
  // Set actionType train
  conf.set(AngelConf.ANGEL_ACTION_TYPE, MLConf.ANGEL_ML_TRAIN());

  FMRunner runner = new FMRunner();
  runner.train(conf);
}
開發者ID:Tencent，項目名稱:angel，代碼行數:21，代碼來源:FMTest.java

示例10: FMClassificationTest

import org.apache.hadoop.fs.LocalFileSystem; //導入依賴的package包/類
@Test
public void FMClassificationTest() throws Exception {
  String inputPath = "./src/test/data/fm/a9a.train";
  String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
  String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");
  String savePath = LOCAL_FS + TMP_PATH + "/model";
  String logPath = LOCAL_FS + TMP_PATH + "/LRlog";

  // Set trainning data path
  conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, inputPath);
  // Set save model path
  conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, savePath);
  // Set log path
  conf.set(AngelConf.ANGEL_LOG_PATH, logPath);
  // Set actionType train
  conf.set(AngelConf.ANGEL_ACTION_TYPE, MLConf.ANGEL_ML_TRAIN());
  // Set learnType
  conf.set(MLConf.ML_FM_LEARN_TYPE(), "c");
  // Set feature number
  conf.set(MLConf.ML_FEATURE_NUM(), String.valueOf(124));

  FMRunner runner = new FMRunner();
  runner.train(conf);
}
開發者ID:Tencent，項目名稱:angel，代碼行數:25，代碼來源:FMTest.java

示例11: trainOnLocalClusterTest

import org.apache.hadoop.fs.LocalFileSystem; //導入依賴的package包/類
private void trainOnLocalClusterTest() throws Exception {
  try {
    String inputPath = "./src/test/data/LinearRegression/LinearReg100.train";
    String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
    String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");
    String logPath = "./src/test/log";

    // Set trainning data path
    conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, inputPath);
    // Set save model path
    conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/model");
    // Set log path
    conf.set(AngelConf.ANGEL_LOG_PATH, logPath);
    // Set actionType train
    conf.set(AngelConf.ANGEL_ACTION_TYPE, MLConf.ANGEL_ML_TRAIN());

    LinearRegRunner runner = new LinearRegRunner();
    runner.train(conf);
  } catch (Exception x) {
    LOG.error("run trainOnLocalClusterTest failed ", x);
    throw x;
  }
}
開發者ID:Tencent，項目名稱:angel，代碼行數:24，代碼來源:LinearRegTest.java

示例12: incTrainTest

import org.apache.hadoop.fs.LocalFileSystem; //導入依賴的package包/類
private void incTrainTest() throws Exception {
  try {
    String inputPath = "./src/test/data/LinearRegression/LinearReg100.train";
    String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
    String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");
    String logPath = "./src/test/log";

    // Set trainning data path
    conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, inputPath);
    // Set load model path
    conf.set(AngelConf.ANGEL_LOAD_MODEL_PATH, LOCAL_FS + TMP_PATH + "/model");
    // Set save model path
    conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/newmodel");
    // Set actionType incremental train
    conf.set(AngelConf.ANGEL_ACTION_TYPE, MLConf.ANGEL_ML_INC_TRAIN());
    // Set log path
    conf.set(AngelConf.ANGEL_LOG_PATH, logPath);

    LinearRegRunner runner = new LinearRegRunner();
    runner.incTrain(conf);
  } catch (Exception x) {
    LOG.error("run incTrainTest failed ", x);
    throw x;
  }
}
開發者ID:Tencent，項目名稱:angel，代碼行數:26，代碼來源:LinearRegTest.java

示例13: predictTest

import org.apache.hadoop.fs.LocalFileSystem; //導入依賴的package包/類
private void predictTest() throws Exception {
  try {
    String inputPath = "./src/test/data/LinearRegression/LinearReg100.train";
    String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
    String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");

    // Set trainning data path
    conf.set(AngelConf.ANGEL_PREDICT_DATA_PATH, inputPath);
    // Set load model path
    conf.set(AngelConf.ANGEL_LOAD_MODEL_PATH, LOCAL_FS + TMP_PATH + "/model");
    // Set predict result path
    conf.set(AngelConf.ANGEL_PREDICT_PATH, LOCAL_FS + TMP_PATH + "/predict");
    // Set log sava path
    conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/LOG/log");
    // Set actionType prediction
    conf.set(AngelConf.ANGEL_ACTION_TYPE, MLConf.ANGEL_ML_PREDICT());

    LinearRegRunner runner = new LinearRegRunner();

    runner.predict(conf);
  } catch (Exception x) {
    LOG.error("run predictTest failed ", x);
    throw x;
  }
}
開發者ID:Tencent，項目名稱:angel，代碼行數:26，代碼來源:LinearRegTest.java

示例14: _mkdirs

import org.apache.hadoop.fs.LocalFileSystem; //導入依賴的package包/類
private void _mkdirs(boolean exists, FsPermission before, FsPermission after)
    throws Throwable {
  File localDir = make(stub(File.class).returning(exists).from.exists());
  when(localDir.mkdir()).thenReturn(true);
  Path dir = mock(Path.class); // use default stubs
  LocalFileSystem fs = make(stub(LocalFileSystem.class)
      .returning(localDir).from.pathToFile(dir));
  FileStatus stat = make(stub(FileStatus.class)
      .returning(after).from.getPermission());
  when(fs.getFileStatus(dir)).thenReturn(stat);

  try {
    DiskChecker.mkdirsWithExistsAndPermissionCheck(fs, dir, before);

    if (!exists)
      verify(fs).setPermission(dir, before);
    else {
      verify(fs).getFileStatus(dir);
      verify(stat).getPermission();
    }
  }
  catch (DiskErrorException e) {
    if (before != after)
      assertTrue(e.getMessage().startsWith("Incorrect permission"));
  }
}
開發者ID:nucypher，項目名稱:hadoop-oss，代碼行數:27，代碼來源:TestDiskChecker.java

示例15: testCopyRecursive

import org.apache.hadoop.fs.LocalFileSystem; //導入依賴的package包/類
@Test
public void testCopyRecursive() throws Throwable {
  int expected = createTestFiles(sourceDir, 64);

  expectSuccess(
      "-s", sourceDir.toURI().toString(),
      "-d", destDir.toURI().toString(),
      "-t", "4",
      "-l", "3");

  LocalFileSystem local = FileSystem.getLocal(new Configuration());
  Set<String> entries = new TreeSet<>();
  RemoteIterator<LocatedFileStatus> iterator
      = local.listFiles(new Path(destDir.toURI()), true);
  int count = 0;
  while (iterator.hasNext()) {
    LocatedFileStatus next = iterator.next();
    entries.add(next.getPath().toUri().toString());
    LOG.info("Entry {} size = {}", next.getPath(), next.getLen());
    count++;
  }
  assertEquals("Mismatch in files found", expected, count);

}
開發者ID:steveloughran，項目名稱:cloudup，代碼行數:25，代碼來源:TestLocalCloudup.java


注：本文中的org.apache.hadoop.fs.LocalFileSystem類示例由純淨天空整理自Github/MSDocs等開源代碼及文檔管理平台，相關代碼片段篩選自各路編程大神貢獻的開源項目，源碼版權歸原作者所有，傳播和使用請參考對應項目的License；未經允許，請勿轉載。