当前位置: 首页>>代码示例>>Java>>正文

Java LocalFileSystem类代码示例

本文整理汇总了Java中org.apache.hadoop.fs.LocalFileSystem类的典型用法代码示例。如果您正苦于以下问题：Java LocalFileSystem类的具体用法？Java LocalFileSystem怎么用？Java LocalFileSystem使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。


LocalFileSystem类属于org.apache.hadoop.fs包，在下文中一共展示了LocalFileSystem类的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: incTrain

import org.apache.hadoop.fs.LocalFileSystem; //导入依赖的package包/类
public void incTrain() {
  setConf();
  String inputPath = "../data/exampledata/LRLocalExampleData/a9a.train";
  String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
  String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");
  String loadPath = LOCAL_FS + TMP_PATH + "/model";
  String savePath = LOCAL_FS + TMP_PATH + "/newmodel";
  String logPath = LOCAL_FS + TMP_PATH + "/log";

  // Set trainning data path
  conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, inputPath);
  // Set load model path
  conf.set(AngelConf.ANGEL_LOAD_MODEL_PATH, loadPath);
  // Set save model path
  conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, savePath);
  // Set log path
  conf.set(AngelConf.ANGEL_LOG_PATH, logPath);
  // Set actionType incremental train
  conf.set(AngelConf.ANGEL_ACTION_TYPE, MLConf.ANGEL_ML_INC_TRAIN());

  LRRunner runner = new LRRunner();
  runner.incTrain(conf);
}
开发者ID:Tencent，项目名称:angel，代码行数:24，代码来源:SGDLRLocalExample.java

示例2: trainOnLocalCluster

import org.apache.hadoop.fs.LocalFileSystem; //导入依赖的package包/类
public void trainOnLocalCluster() throws Exception {
  setConf();
  String inputPath = "../data/exampledata/LRLocalExampleData/a9a.train";
  String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
  String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");
  String savePath = LOCAL_FS + TMP_PATH + "/model";
  String logPath = LOCAL_FS + TMP_PATH + "/log";

  // Set trainning data path
  conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, inputPath);
  // Set save model path
  conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, savePath);
  // Set log path
  conf.set(AngelConf.ANGEL_LOG_PATH, logPath);
  // Set actionType train
  conf.set(AngelConf.ANGEL_ACTION_TYPE, MLConf.ANGEL_ML_TRAIN());


  LRRunner runner = new LRRunner();
  runner.train(conf);
}
开发者ID:Tencent，项目名称:angel，代码行数:22，代码来源:SGDLRLocalExample.java

示例3: open

import org.apache.hadoop.fs.LocalFileSystem; //导入依赖的package包/类
protected void open(Path dstPath, CompressionCodec codeC,
    CompressionType compType, Configuration conf, FileSystem hdfs)
        throws IOException {
  if(useRawLocalFileSystem) {
    if(hdfs instanceof LocalFileSystem) {
      hdfs = ((LocalFileSystem)hdfs).getRaw();
    } else {
      logger.warn("useRawLocalFileSystem is set to true but file system " +
          "is not of type LocalFileSystem: " + hdfs.getClass().getName());
    }
  }
  if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile
          (dstPath)) {
    outStream = hdfs.append(dstPath);
  } else {
    outStream = hdfs.create(dstPath);
  }
  writer = SequenceFile.createWriter(conf, outStream,
      serializer.getKeyClass(), serializer.getValueClass(), compType, codeC);

  registerCurrentStream(outStream, hdfs, dstPath);
}
开发者ID:Transwarp-DE，项目名称:Transwarp-Sample-Code，代码行数:23，代码来源:HDFSSequenceFile.java

示例4: open

import org.apache.hadoop.fs.LocalFileSystem; //导入依赖的package包/类
protected void open(Path dstPath, CompressionCodec codeC,
    CompressionType compType, Configuration conf, FileSystem hdfs)
        throws IOException {
  if (useRawLocalFileSystem) {
    if (hdfs instanceof LocalFileSystem) {
      hdfs = ((LocalFileSystem)hdfs).getRaw();
    } else {
      logger.warn("useRawLocalFileSystem is set to true but file system " +
          "is not of type LocalFileSystem: " + hdfs.getClass().getName());
    }
  }
  if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile(dstPath)) {
    outStream = hdfs.append(dstPath);
  } else {
    outStream = hdfs.create(dstPath);
  }
  writer = SequenceFile.createWriter(conf, outStream,
      serializer.getKeyClass(), serializer.getValueClass(), compType, codeC);

  registerCurrentStream(outStream, hdfs, dstPath);
}
开发者ID:moueimei，项目名称:flume-release-1.7.0，代码行数:22，代码来源:HDFSSequenceFile.java

示例5: predict

import org.apache.hadoop.fs.LocalFileSystem; //导入依赖的package包/类
public void predict() {
  setConf();
  String inputPath = "../data/exampledata/LRLocalExampleData/a9a.test";
  String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
  String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");
  String loadPath = LOCAL_FS + TMP_PATH + "/model";
  String savePath = LOCAL_FS + TMP_PATH + "/model";
  String logPath = LOCAL_FS + TMP_PATH + "/log";
  String predictPath = LOCAL_FS + TMP_PATH + "/predict";

  // Set trainning data path
  conf.set(AngelConf.ANGEL_PREDICT_DATA_PATH, inputPath);
  // Set load model path
  conf.set(AngelConf.ANGEL_LOAD_MODEL_PATH, loadPath);
  // Set predict result path
  conf.set(AngelConf.ANGEL_PREDICT_PATH, predictPath);
  // Set log path
  conf.set(AngelConf.ANGEL_LOG_PATH, logPath);
  // Set actionType prediction
  conf.set(AngelConf.ANGEL_ACTION_TYPE, MLConf.ANGEL_ML_PREDICT());

  LRRunner runner = new LRRunner();

  runner.predict(conf);
}
开发者ID:Tencent，项目名称:angel，代码行数:26，代码来源:SGDLRLocalExample.java

示例6: trainOnLocalCluster

import org.apache.hadoop.fs.LocalFileSystem; //导入依赖的package包/类
public void trainOnLocalCluster() throws Exception {
  setConf();
  String inputPath = "../data/exampledata/LinearRegression";
  String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
  String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");
  String logPath = "./src/test/log";

  // Set trainning data path
  conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, inputPath);
  // Set save model path
  conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/model");
  // Set log path
  conf.set(AngelConf.ANGEL_LOG_PATH, logPath);
  // Set actionType train
  conf.set(AngelConf.ANGEL_ACTION_TYPE, MLConf.ANGEL_ML_TRAIN());


  LinearRegRunner runner = new LinearRegRunner();
  runner.train(conf);
}
开发者ID:Tencent，项目名称:angel，代码行数:21，代码来源:LinearRegLocalExample.java

示例7: incTrain

import org.apache.hadoop.fs.LocalFileSystem; //导入依赖的package包/类
public void incTrain() {
  setConf();
  String inputPath = "../data/exampledata/LinearRegression/LinearReg100.train";
  String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
  String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");
  String logPath = "./src/test/log";

  // Set trainning data path
  conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, inputPath);
  // Set load model path
  conf.set(AngelConf.ANGEL_LOAD_MODEL_PATH, LOCAL_FS + TMP_PATH + "/model");
  // Set save model path
  conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/newmodel");
  // Set actionType incremental train
  conf.set(AngelConf.ANGEL_ACTION_TYPE, MLConf.ANGEL_ML_INC_TRAIN());
  // Set log path
  conf.set(AngelConf.ANGEL_LOG_PATH, logPath);

  LinearRegRunner runner = new LinearRegRunner();
  runner.incTrain(conf);
}
开发者ID:Tencent，项目名称:angel，代码行数:22，代码来源:LinearRegLocalExample.java

示例8: predict

import org.apache.hadoop.fs.LocalFileSystem; //导入依赖的package包/类
public void predict() {
  setConf();
  String inputPath = "../data/exampledata/LinearRegression/LinearReg100.train";
  String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
  String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");

  // Set trainning data path
  conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, inputPath);
  // Set load model path
  conf.set(AngelConf.ANGEL_LOAD_MODEL_PATH, LOCAL_FS + TMP_PATH + "/model");
  // Set predict result path
  conf.set(AngelConf.ANGEL_PREDICT_PATH, LOCAL_FS + TMP_PATH + "/predict");
  // Set actionType prediction
  conf.set(AngelConf.ANGEL_ACTION_TYPE, MLConf.ANGEL_ML_PREDICT());
  LinearRegRunner runner = new LinearRegRunner();

  runner.predict(conf);
}
开发者ID:Tencent，项目名称:angel，代码行数:19，代码来源:LinearRegLocalExample.java

示例9: trainOnLocalClusterTest

import org.apache.hadoop.fs.LocalFileSystem; //导入依赖的package包/类
@Test
public void trainOnLocalClusterTest() throws Exception {
  String inputPath = "./src/test/data/fm/food_fm_libsvm";
  String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
  String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");
  String savePath = LOCAL_FS + TMP_PATH + "/model";
  String logPath = LOCAL_FS + TMP_PATH + "/LRlog";

  // Set trainning data path
  conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, inputPath);
  // Set save model path
  conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, savePath);
  // Set log path
  conf.set(AngelConf.ANGEL_LOG_PATH, logPath);
  // Set actionType train
  conf.set(AngelConf.ANGEL_ACTION_TYPE, MLConf.ANGEL_ML_TRAIN());

  FMRunner runner = new FMRunner();
  runner.train(conf);
}
开发者ID:Tencent，项目名称:angel，代码行数:21，代码来源:FMTest.java

示例10: FMClassificationTest

import org.apache.hadoop.fs.LocalFileSystem; //导入依赖的package包/类
@Test
public void FMClassificationTest() throws Exception {
  String inputPath = "./src/test/data/fm/a9a.train";
  String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
  String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");
  String savePath = LOCAL_FS + TMP_PATH + "/model";
  String logPath = LOCAL_FS + TMP_PATH + "/LRlog";

  // Set trainning data path
  conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, inputPath);
  // Set save model path
  conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, savePath);
  // Set log path
  conf.set(AngelConf.ANGEL_LOG_PATH, logPath);
  // Set actionType train
  conf.set(AngelConf.ANGEL_ACTION_TYPE, MLConf.ANGEL_ML_TRAIN());
  // Set learnType
  conf.set(MLConf.ML_FM_LEARN_TYPE(), "c");
  // Set feature number
  conf.set(MLConf.ML_FEATURE_NUM(), String.valueOf(124));

  FMRunner runner = new FMRunner();
  runner.train(conf);
}
开发者ID:Tencent，项目名称:angel，代码行数:25，代码来源:FMTest.java

示例11: trainOnLocalClusterTest

import org.apache.hadoop.fs.LocalFileSystem; //导入依赖的package包/类
private void trainOnLocalClusterTest() throws Exception {
  try {
    String inputPath = "./src/test/data/LinearRegression/LinearReg100.train";
    String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
    String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");
    String logPath = "./src/test/log";

    // Set trainning data path
    conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, inputPath);
    // Set save model path
    conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/model");
    // Set log path
    conf.set(AngelConf.ANGEL_LOG_PATH, logPath);
    // Set actionType train
    conf.set(AngelConf.ANGEL_ACTION_TYPE, MLConf.ANGEL_ML_TRAIN());

    LinearRegRunner runner = new LinearRegRunner();
    runner.train(conf);
  } catch (Exception x) {
    LOG.error("run trainOnLocalClusterTest failed ", x);
    throw x;
  }
}
开发者ID:Tencent，项目名称:angel，代码行数:24，代码来源:LinearRegTest.java

示例12: incTrainTest

import org.apache.hadoop.fs.LocalFileSystem; //导入依赖的package包/类
private void incTrainTest() throws Exception {
  try {
    String inputPath = "./src/test/data/LinearRegression/LinearReg100.train";
    String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
    String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");
    String logPath = "./src/test/log";

    // Set trainning data path
    conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, inputPath);
    // Set load model path
    conf.set(AngelConf.ANGEL_LOAD_MODEL_PATH, LOCAL_FS + TMP_PATH + "/model");
    // Set save model path
    conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/newmodel");
    // Set actionType incremental train
    conf.set(AngelConf.ANGEL_ACTION_TYPE, MLConf.ANGEL_ML_INC_TRAIN());
    // Set log path
    conf.set(AngelConf.ANGEL_LOG_PATH, logPath);

    LinearRegRunner runner = new LinearRegRunner();
    runner.incTrain(conf);
  } catch (Exception x) {
    LOG.error("run incTrainTest failed ", x);
    throw x;
  }
}
开发者ID:Tencent，项目名称:angel，代码行数:26，代码来源:LinearRegTest.java

示例13: predictTest

import org.apache.hadoop.fs.LocalFileSystem; //导入依赖的package包/类
private void predictTest() throws Exception {
  try {
    String inputPath = "./src/test/data/LinearRegression/LinearReg100.train";
    String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
    String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");

    // Set trainning data path
    conf.set(AngelConf.ANGEL_PREDICT_DATA_PATH, inputPath);
    // Set load model path
    conf.set(AngelConf.ANGEL_LOAD_MODEL_PATH, LOCAL_FS + TMP_PATH + "/model");
    // Set predict result path
    conf.set(AngelConf.ANGEL_PREDICT_PATH, LOCAL_FS + TMP_PATH + "/predict");
    // Set log sava path
    conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/LOG/log");
    // Set actionType prediction
    conf.set(AngelConf.ANGEL_ACTION_TYPE, MLConf.ANGEL_ML_PREDICT());

    LinearRegRunner runner = new LinearRegRunner();

    runner.predict(conf);
  } catch (Exception x) {
    LOG.error("run predictTest failed ", x);
    throw x;
  }
}
开发者ID:Tencent，项目名称:angel，代码行数:26，代码来源:LinearRegTest.java

示例14: _mkdirs

import org.apache.hadoop.fs.LocalFileSystem; //导入依赖的package包/类
private void _mkdirs(boolean exists, FsPermission before, FsPermission after)
    throws Throwable {
  File localDir = make(stub(File.class).returning(exists).from.exists());
  when(localDir.mkdir()).thenReturn(true);
  Path dir = mock(Path.class); // use default stubs
  LocalFileSystem fs = make(stub(LocalFileSystem.class)
      .returning(localDir).from.pathToFile(dir));
  FileStatus stat = make(stub(FileStatus.class)
      .returning(after).from.getPermission());
  when(fs.getFileStatus(dir)).thenReturn(stat);

  try {
    DiskChecker.mkdirsWithExistsAndPermissionCheck(fs, dir, before);

    if (!exists)
      verify(fs).setPermission(dir, before);
    else {
      verify(fs).getFileStatus(dir);
      verify(stat).getPermission();
    }
  }
  catch (DiskErrorException e) {
    if (before != after)
      assertTrue(e.getMessage().startsWith("Incorrect permission"));
  }
}
开发者ID:nucypher，项目名称:hadoop-oss，代码行数:27，代码来源:TestDiskChecker.java

示例15: testCopyRecursive

import org.apache.hadoop.fs.LocalFileSystem; //导入依赖的package包/类
@Test
public void testCopyRecursive() throws Throwable {
  int expected = createTestFiles(sourceDir, 64);

  expectSuccess(
      "-s", sourceDir.toURI().toString(),
      "-d", destDir.toURI().toString(),
      "-t", "4",
      "-l", "3");

  LocalFileSystem local = FileSystem.getLocal(new Configuration());
  Set<String> entries = new TreeSet<>();
  RemoteIterator<LocatedFileStatus> iterator
      = local.listFiles(new Path(destDir.toURI()), true);
  int count = 0;
  while (iterator.hasNext()) {
    LocatedFileStatus next = iterator.next();
    entries.add(next.getPath().toUri().toString());
    LOG.info("Entry {} size = {}", next.getPath(), next.getLen());
    count++;
  }
  assertEquals("Mismatch in files found", expected, count);

}
开发者ID:steveloughran，项目名称:cloudup，代码行数:25，代码来源:TestLocalCloudup.java


注：本文中的org.apache.hadoop.fs.LocalFileSystem类示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。