Java LocalFileSystem.pathToFile方法代碼示例

import org.apache.hadoop.fs.LocalFileSystem; //導入方法依賴的package包/類
private void body(Context context) throws IOException {
    Text buf = new Text();
    LocalFileSystem fs = FileSystem.getLocal(context.getConfiguration());
    try (StageResourceDriver driver = new StageResourceDriver(context)) {
        for (Path path : driver.findCache("resource")) {
            File file = fs.pathToFile(path);
            List<String> lines = FileEditor.get(file);
            for (String line : lines) {
                buf.set(line);
                result.add(buf);
            }
        }
    }
}
 

import org.apache.hadoop.fs.LocalFileSystem; //導入方法依賴的package包/類
/**
 * The src file is on the local disk.  Add it to FS at
 * the given dst name.
 *
 * This version doesn't need to create a temporary file to calculate the md5.
 * Sadly this doesn't seem to be used by the shell cp :(
 *
 * delSrc indicates if the source should be removed
 * @param delSrc whether to delete the src
 * @param overwrite whether to overwrite an existing file
 * @param src path
 * @param dst path
 */
@Override
public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src, 
  Path dst) throws IOException {
  String key = pathToKey(dst);

  if (!overwrite && exists(dst)) {
    throw new IOException(dst + " already exists");
  }
  if (LOG.isDebugEnabled()) {
    LOG.debug("Copying local file from " + src + " to " + dst);
  }

  // Since we have a local file, we don't need to stream into a temporary file
  LocalFileSystem local = getLocal(getConf());
  File srcfile = local.pathToFile(src);

  final ObjectMetadata om = new ObjectMetadata();
  if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
    om.setServerSideEncryption(serverSideEncryptionAlgorithm);
  }
  PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, srcfile);
  putObjectRequest.setCannedAcl(cannedACL);
  putObjectRequest.setMetadata(om);

  ProgressListener progressListener = new ProgressListener() {
    public void progressChanged(ProgressEvent progressEvent) {
      switch (progressEvent.getEventCode()) {
        case ProgressEvent.PART_COMPLETED_EVENT_CODE:
          statistics.incrementWriteOps(1);
          break;
        default:
          break;
      }
    }
  };

  Upload up = transfers.upload(putObjectRequest);
  up.addProgressListener(progressListener);
  try {
    up.waitForUploadResult();
    statistics.incrementWriteOps(1);
  } catch (InterruptedException e) {
    throw new IOException("Got interrupted, cancelling");
  }

  // This will delete unnecessary fake parent directories
  finishedWrite(key);

  if (delSrc) {
    local.delete(src, false);
  }
}
 

import org.apache.hadoop.fs.LocalFileSystem; //導入方法依賴的package包/類
/**
 * The src file is on the local disk.  Add it to FS at
 * the given dst name.
 * <p/>
 * This version doesn't need to create a temporary file to calculate the md5.
 * Sadly this doesn't seem to be used by the shell cp :(
 * <p/>
 * delSrc indicates if the source should be removed
 *
 * @param delSrc    whether to delete the src
 * @param overwrite whether to overwrite an existing file
 * @param src       path
 * @param dst       path
 */
@Override
public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src,
                              Path dst) throws IOException {
  String key = pathToKey(dst);

  if (!overwrite && exists(dst)) {
    throw new IOException(dst + " already exists");
  }
  if (LOG.isDebugEnabled()) {
    LOG.debug("Copying local file from " + src + " to " + dst);
  }

  // Since we have a local file, we don't need to stream into a temporary file
  LocalFileSystem local = getLocal(getConf());
  File srcfile = local.pathToFile(src);

  final ObjectMetadata om = new ObjectMetadata();
  PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, srcfile);
  putObjectRequest.setMetadata(om);
  putObjectRequest.setProgressListener(new ProgressListener() {
    public void progressChanged(ProgressEvent progressEvent) {
      switch (progressEvent.getEventType()) {
        case TRANSFER_PART_COMPLETED_EVENT:
          statistics.incrementWriteOps(1);
          break;
        default:
          break;
      }
    }
  });

  try {
    client.putObject(putObjectRequest);
    statistics.incrementWriteOps(1);
  } catch (OSSException | ClientException e) {
    throw new IOException("Got interrupted, cancelling");
  }
  // This will delete unnecessary fake parent directories
  finishedWrite(key);
  if (delSrc) {
    local.delete(src, false);
  }
}
 

import org.apache.hadoop.fs.LocalFileSystem; //導入方法依賴的package包/類
/**
 * The src file is on the local disk.  Add it to FS at
 * the given dst name.
 *
 * This version doesn't need to create a temporary file to calculate the md5.
 * Sadly this doesn't seem to be used by the shell cp :(
 *
 * delSrc indicates if the source should be removed
 * @param delSrc whether to delete the src
 * @param overwrite whether to overwrite an existing file
 * @param src path
 * @param dst path
 */
@Override
public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src, 
  Path dst) throws IOException {
  String key = pathToKey(dst);

  if (!overwrite && exists(dst)) {
    throw new IOException(dst + " already exists");
  }
  if (LOG.isDebugEnabled()) {
    LOG.debug("Copying local file from " + src + " to " + dst);
  }

  // Since we have a local file, we don't need to stream into a temporary file
  LocalFileSystem local = getLocal(getConf());
  File srcfile = local.pathToFile(src);

  final ObjectMetadata om = new ObjectMetadata();
  if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
    om.setSSEAlgorithm(serverSideEncryptionAlgorithm);
  }
  PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, srcfile);
  putObjectRequest.setCannedAcl(cannedACL);
  putObjectRequest.setMetadata(om);

  ProgressListener progressListener = new ProgressListener() {
    public void progressChanged(ProgressEvent progressEvent) {
      switch (progressEvent.getEventType()) {
        case TRANSFER_PART_COMPLETED_EVENT:
          statistics.incrementWriteOps(1);
          break;
        default:
          break;
      }
    }
  };

  Upload up = transfers.upload(putObjectRequest);
  up.addProgressListener(progressListener);
  try {
    up.waitForUploadResult();
    statistics.incrementWriteOps(1);
  } catch (InterruptedException e) {
    throw new IOException("Got interrupted, cancelling");
  }

  // This will delete unnecessary fake parent directories
  finishedWrite(key);

  if (delSrc) {
    local.delete(src, false);
  }
}
 

import org.apache.hadoop.fs.LocalFileSystem; //導入方法依賴的package包/類
/**
 * Copies a file from the local system to the Manta object store.
 *
 * @param delSrc    whether to delete the source
 * @param overwrite whether to overwrite an existing file
 * @param src       file source path
 * @param dst       file destination
 */
@Override
public void copyFromLocalFile(final boolean delSrc, final boolean overwrite,
                              final Path src, final Path dst) throws IOException {
    String mantaPath = mantaPath(dst);

    LOG.debug("Copying local file [{}] to [{}]", src, dst);

    if (!overwrite) {
        try {
            MantaObject head = client.head(mantaPath);
            if (!head.isDirectory()) {
                throw new IOException("Can't copy file because destination "
                        + "already exists: " + dst);
            }
        } catch (MantaClientHttpResponseException e) {
            // 404 means we are good to go and not overwriting,
            // so we throw any error that is not a 404
            if (e.getStatusCode() != HttpStatus.SC_NOT_FOUND) {
                throw e;
            }

            // Make any missing parent paths
            Path parent = dst.getParent();
            LOG.debug("Creating parent directory: {}", parent);
            client.putDirectory(mantaPath(parent), true);
        }
    }

    LocalFileSystem localFs = getLocal(getConf());
    File localFile = localFs.pathToFile(src);

    /* We used the default copy implementation if it is a wildcard copy
     * because we don't support wildcard copy in the Manta SDK yet. */
    if (localFile.isDirectory()) {
        super.copyFromLocalFile(delSrc, overwrite, src, dst);
        return;
    }

    client.put(mantaPath, localFile);

    if (delSrc) {
        Files.delete(localFile.toPath());
    }
}
 

import org.apache.hadoop.fs.LocalFileSystem; //導入方法依賴的package包/類
/**
 * The src file is under FS, and the dst is on the local disk. Copy it from FS
 * control to the local dst name. delSrc indicates if the src will be removed
 * or not. useRawLocalFileSystem indicates whether to use RawLocalFileSystem
 * as local file system or not. RawLocalFileSystem is non crc file system.So,
 * It will not create any crc files at local.
 *
 * @param delSrc                whether to delete the src
 * @param src                   path
 * @param dst                   path
 * @param useRawLocalFileSystem whether to use RawLocalFileSystem as local file system or not.
 * @throws IOException - if any IO error
 */
@Override
public void copyToLocalFile(final boolean delSrc, final Path src,
                            final Path dst, final boolean useRawLocalFileSystem) throws IOException {
    /* If we can't get a reference to a File object, then we are better off
     * relying on the default implementation of this method.
     */
    if (useRawLocalFileSystem) {
        super.copyToLocalFile(delSrc, src, dst, useRawLocalFileSystem);
        return;
    }

    Configuration conf = getConf();
    LocalFileSystem local = getLocal(conf);
    File localFile = local.pathToFile(dst);
    String mantaPath = mantaPath(src);

    try {
        MantaObject head = client.head(mantaPath);

        /* We don't support wildcard copy yet, so we rely on the default
         * implementation of this method. */
        if (head.isDirectory()) {
            super.copyToLocalFile(delSrc, src, dst, useRawLocalFileSystem);
            return;
        }

    } catch (MantaClientHttpResponseException e) {
        if (e.getStatusCode() == HttpStatus.SC_NOT_FOUND) {
            throw new FileNotFoundException(mantaPath);
        }

        throw e;
    }

    try (MantaObjectInputStream in = client.getAsInputStream(mantaPath)) {
        Files.copy(in, localFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
    }

    if (delSrc) {
        client.delete(mantaPath);
    }
}
 

import org.apache.hadoop.fs.LocalFileSystem; //導入方法依賴的package包/類
/**
 * The src file is on the local disk.  Add it to FS at
 * the given dst name.
 *
 * This version doesn't need to create a temporary file to calculate the md5.
 * Sadly this doesn't seem to be used by the shell cp :(
 *
 * delSrc indicates if the source should be removed
 * @param delSrc whether to delete the src
 * @param overwrite whether to overwrite an existing file
 * @param src path
 * @param dst path
 */
@Override
public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src, 
  Path dst) throws IOException {
  String key = pathToKey(dst);

  if (!overwrite && exists(dst)) {
    throw new IOException(dst + " already exists");
  }
  if (LOG.isDebugEnabled()) {
    LOG.debug("Copying local file from " + src + " to " + dst);
  }

  // Since we have a local file, we don't need to stream into a temporary file
  LocalFileSystem local = getLocal(getConf());
  File srcfile = local.pathToFile(src);

  final ObjectMetadata om = new ObjectMetadata();
  if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
    om.setServerSideEncryption(serverSideEncryptionAlgorithm);
  }
  PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, srcfile);
  putObjectRequest.setCannedAcl(cannedACL);
  putObjectRequest.setMetadata(om);

  ProgressListener progressListener = new ProgressListener() {
    public void progressChanged(ProgressEvent progressEvent) {
      switch (progressEvent.getEventCode()) {
        case ProgressEvent.PART_COMPLETED_EVENT_CODE:
          statistics.incrementWriteOps(1);
          break;
      }
    }
  };

  Upload up = transfers.upload(putObjectRequest);
  up.addProgressListener(progressListener);
  try {
    up.waitForUploadResult();
    statistics.incrementWriteOps(1);
  } catch (InterruptedException e) {
    throw new IOException("Got interrupted, cancelling");
  }

  // This will delete unnecessary fake parent directories
  finishedWrite(key);

  if (delSrc) {
    local.delete(src, false);
  }
}
 

import org.apache.hadoop.fs.LocalFileSystem; //導入方法依賴的package包/類
/**
 * Create the directory or check permissions if it already exists.
 *
 * The semantics of mkdirsWithExistsAndPermissionCheck method is different
 * from the mkdirs method provided in the Sun's java.io.File class in the
 * following way:
 * While creating the non-existent parent directories, this method checks for
 * the existence of those directories if the mkdir fails at any point (since
 * that directory might have just been created by some other process).
 * If both mkdir() and the exists() check fails for any seemingly
 * non-existent directory, then we signal an error; Sun's mkdir would signal
 * an error (return false) if a directory it is attempting to create already
 * exists or the mkdir fails.
 *
 * @param localFS local filesystem
 * @param dir directory to be created or checked
 * @param expected expected permission
 * @throws IOException
 */
public static void mkdirsWithExistsAndPermissionCheck(
    LocalFileSystem localFS, Path dir, FsPermission expected)
    throws IOException {
  File directory = localFS.pathToFile(dir);
  boolean created = false;

  if (!directory.exists())
    created = mkdirsWithExistsCheck(directory);

  if (created || !localFS.getFileStatus(dir).getPermission().equals(expected))
      localFS.setPermission(dir, expected);
}
 

import org.apache.hadoop.fs.LocalFileSystem; //導入方法依賴的package包/類
/**
 * Create the directory or check permissions if it already exists.
 *
 * The semantics of mkdirsWithExistsAndPermissionCheck method is different
 * from the mkdirs method provided in the Sun's java.io.File class in the
 * following way:
 * While creating the non-existent parent directories, this method checks for
 * the existence of those directories if the mkdir fails at any point (since
 * that directory might have just been created by some other process).
 * If both mkdir() and the exists() check fails for any seemingly
 * non-existent directory, then we signal an error; Sun's mkdir would signal
 * an error (return false) if a directory it is attempting to create already
 * exists or the mkdir fails.
 *
 * @param localFS local filesystem
 * @param dir directory to be created or checked
 * @param expected expected permission
 * @throws IOException
 */
static void mkdirsWithExistsAndPermissionCheck(
    LocalFileSystem localFS, Path dir, FsPermission expected)
    throws IOException {
  File directory = localFS.pathToFile(dir);
  boolean created = false;

  if (!directory.exists())
    created = mkdirsWithExistsCheck(directory);

  if (created || !localFS.getFileStatus(dir).getPermission().equals(expected))
    localFS.setPermission(dir, expected);
}