Java IOUtils.fsync方法代码示例

import org.apache.lucene.util.IOUtils; //导入方法依赖的package包/类
/**
 * Moves the index folder found in <code>source</code> to <code>target</code>
 */
void upgrade(final Index index, final Path source, final Path target) throws IOException {
    boolean success = false;
    try {
        Files.move(source, target, StandardCopyOption.ATOMIC_MOVE);
        success = true;
    } catch (NoSuchFileException | FileNotFoundException exception) {
        // thrown when the source is non-existent because the folder was renamed
        // by another node (shared FS) after we checked if the target exists
        logger.error((Supplier<?>) () -> new ParameterizedMessage("multiple nodes trying to upgrade [{}] in parallel, retry " +
            "upgrading with single node", target), exception);
        throw exception;
    } finally {
        if (success) {
            logger.info("{} moved from [{}] to [{}]", index, source, target);
            logger.trace("{} syncing directory [{}]", index, target);
            IOUtils.fsync(target, true);
        }
    }
}
 

import org.apache.lucene.util.IOUtils; //导入方法依赖的package包/类
@Override
public void sync(Collection<String> names) throws IOException {
  ensureOpen();
  Set<String> toSync = new HashSet<>(names);
  toSync.retainAll(staleFiles);

  for (String name : toSync) {
    fsync(name);
  }
  
  // fsync the directory itsself, but only if there was any file fsynced before
  // (otherwise it can happen that the directory does not yet exist)!
  if (!toSync.isEmpty()) {
    IOUtils.fsync(directory, true);
  }
  
  staleFiles.removeAll(toSync);
}
 

import org.apache.lucene.util.IOUtils; //导入方法依赖的package包/类
/** Write a checkpoint file to the given location with the given generation */
public static void writeEmptyCheckpoint(Path filename, int translogLength, long translogGeneration) throws IOException {
    Checkpoint emptyCheckpoint =
            Checkpoint.emptyTranslogCheckpoint(translogLength, translogGeneration, SequenceNumbersService.UNASSIGNED_SEQ_NO);
    Checkpoint.write(FileChannel::open, filename, emptyCheckpoint,
        StandardOpenOption.WRITE, StandardOpenOption.READ, StandardOpenOption.CREATE_NEW);
    // fsync with metadata here to make sure.
    IOUtils.fsync(filename, false);
}
 

import org.apache.lucene.util.IOUtils; //导入方法依赖的package包/类
@Override
public void writeBlob(String blobName, InputStream inputStream, long blobSize) throws IOException {
    if (blobExists(blobName)) {
        throw new FileAlreadyExistsException("blob [" + blobName + "] already exists, cannot overwrite");
    }
    final Path file = path.resolve(blobName);
    try (OutputStream outputStream = Files.newOutputStream(file, StandardOpenOption.CREATE_NEW)) {
        Streams.copy(inputStream, outputStream, new byte[blobStore.bufferSizeInBytes()]);
    }
    IOUtils.fsync(file, false);
    IOUtils.fsync(path, true);
}
 

import org.apache.lucene.util.IOUtils; //导入方法依赖的package包/类
@Override
public void writeBlob(String blobName, InputStream inputStream, long blobSize) throws IOException {
    final Path file = path.resolve(blobName);
    try (OutputStream outputStream = Files.newOutputStream(file)) {
        Streams.copy(inputStream, outputStream, new byte[blobStore.bufferSizeInBytes()]);
    }
    IOUtils.fsync(file, false);
    IOUtils.fsync(path, true);
}
 

import org.apache.lucene.util.IOUtils; //导入方法依赖的package包/类
private void upgradeFiles(ShardId shard, ShardPath targetPath, final Path targetDir, String folderName, Path[] paths) throws IOException {
    List<Path> movedFiles = new ArrayList<>();
    for (Path path : paths) {
        if (path.equals(targetPath.getDataPath()) == false) {
            final Path sourceDir = path.resolve(folderName);
            if (Files.exists(sourceDir)) {
                logger.info("{} upgrading [{}] from [{}] to [{}]", shard, folderName, sourceDir, targetDir);
                try (DirectoryStream<Path> stream = Files.newDirectoryStream(sourceDir)) {
                    Files.createDirectories(targetDir);
                    for (Path file : stream) {
                        if (IndexWriter.WRITE_LOCK_NAME.equals(file.getFileName().toString()) || Files.isDirectory(file)) {
                            continue; // skip write.lock
                        }
                        logger.info("{} move file [{}] size: [{}]", shard, file.getFileName(), Files.size(file));
                        final Path targetFile = targetDir.resolve(file.getFileName());
                        /* We are pessimistic and do a copy first to the other path and then and atomic move to rename it such that
                           in the worst case the file exists twice but is never lost or half written.*/
                        final Path targetTempFile = Files.createTempFile(targetDir, "upgrade_", "_" + file.getFileName().toString());
                        Files.copy(file, targetTempFile, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING);
                        Files.move(targetTempFile, targetFile, StandardCopyOption.ATOMIC_MOVE); // we are on the same FS - this must work otherwise all bets are off
                        Files.delete(file);
                        movedFiles.add(targetFile);
                    }
                }
            }
        }
    }
    if (movedFiles.isEmpty() == false) {
        // fsync later it might be on disk already
        logger.info("{} fsync files", shard);
        for (Path moved : movedFiles) {
            logger.info("{} syncing [{}]", shard, moved.getFileName());
            IOUtils.fsync(moved, false);
        }
        logger.info("{} syncing directory [{}]", shard, targetDir);
        IOUtils.fsync(targetDir, true);
    }
}
 

import org.apache.lucene.util.IOUtils; //导入方法依赖的package包/类
@Override
protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
    boolean batch = options.has(batchMode);

    Path translogPath = getTranslogPath(options);
    Path idxLocation = translogPath.getParent().resolve("index");

    if (Files.exists(translogPath) == false || Files.isDirectory(translogPath) == false) {
        throw new ElasticsearchException("translog directory [" + translogPath + "], must exist and be a directory");
    }

    if (Files.exists(idxLocation) == false || Files.isDirectory(idxLocation) == false) {
        throw new ElasticsearchException("unable to find a shard at [" + idxLocation + "], which must exist and be a directory");
    }

    // Hold the lock open for the duration of the tool running
    try (Directory dir = FSDirectory.open(idxLocation, NativeFSLockFactory.INSTANCE);
            Lock writeLock = dir.obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
        Set<Path> translogFiles;
        try {
            terminal.println("Checking existing translog files");
            translogFiles = filesInDirectory(translogPath);
        } catch (IOException e) {
            terminal.println("encountered IOException while listing directory, aborting...");
            throw new ElasticsearchException("failed to find existing translog files", e);
        }

        // Warn about ES being stopped and files being deleted
        warnAboutDeletingFiles(terminal, translogFiles, batch);

        List<IndexCommit> commits;
        try {
            terminal.println("Reading translog UUID information from Lucene commit from shard at [" + idxLocation + "]");
            commits = DirectoryReader.listCommits(dir);
        } catch (IndexNotFoundException infe) {
            throw new ElasticsearchException("unable to find a valid shard at [" + idxLocation + "]", infe);
        }

        // Retrieve the generation and UUID from the existing data
        Map<String, String> commitData = commits.get(commits.size() - 1).getUserData();
        String translogGeneration = commitData.get(Translog.TRANSLOG_GENERATION_KEY);
        String translogUUID = commitData.get(Translog.TRANSLOG_UUID_KEY);
        if (translogGeneration == null || translogUUID == null) {
            throw new ElasticsearchException("shard must have a valid translog generation and UUID but got: [{}] and: [{}]",
                    translogGeneration, translogUUID);
        }
        terminal.println("Translog Generation: " + translogGeneration);
        terminal.println("Translog UUID      : " + translogUUID);

        Path tempEmptyCheckpoint = translogPath.resolve("temp-" + Translog.CHECKPOINT_FILE_NAME);
        Path realEmptyCheckpoint = translogPath.resolve(Translog.CHECKPOINT_FILE_NAME);
        Path tempEmptyTranslog = translogPath.resolve("temp-" + Translog.TRANSLOG_FILE_PREFIX +
                        translogGeneration + Translog.TRANSLOG_FILE_SUFFIX);
        Path realEmptyTranslog = translogPath.resolve(Translog.TRANSLOG_FILE_PREFIX +
                        translogGeneration + Translog.TRANSLOG_FILE_SUFFIX);

        // Write empty checkpoint and translog to empty files
        long gen = Long.parseLong(translogGeneration);
        int translogLen = writeEmptyTranslog(tempEmptyTranslog, translogUUID);
        writeEmptyCheckpoint(tempEmptyCheckpoint, translogLen, gen);

        terminal.println("Removing existing translog files");
        IOUtils.rm(translogFiles.toArray(new Path[]{}));

        terminal.println("Creating new empty checkpoint at [" + realEmptyCheckpoint + "]");
        Files.move(tempEmptyCheckpoint, realEmptyCheckpoint, StandardCopyOption.ATOMIC_MOVE);
        terminal.println("Creating new empty translog at [" + realEmptyTranslog + "]");
        Files.move(tempEmptyTranslog, realEmptyTranslog, StandardCopyOption.ATOMIC_MOVE);

        // Fsync the translog directory after rename
        IOUtils.fsync(translogPath, true);

    } catch (LockObtainFailedException lofe) {
        throw new ElasticsearchException("Failed to lock shard's directory at [" + idxLocation + "], is Elasticsearch still running?");
    }

    terminal.println("Done.");
}
 

import org.apache.lucene.util.IOUtils; //导入方法依赖的package包/类
/** recover all translog files found on disk */
private ArrayList<TranslogReader> recoverFromFiles(TranslogGeneration translogGeneration, Checkpoint checkpoint) throws IOException {
    boolean success = false;
    ArrayList<TranslogReader> foundTranslogs = new ArrayList<>();
    final Path tempFile = Files.createTempFile(location, TRANSLOG_FILE_PREFIX, TRANSLOG_FILE_SUFFIX); // a temp file to copy checkpoint to - note it must be in on the same FS otherwise atomic move won't work
    boolean tempFileRenamed = false;
    try (ReleasableLock lock = writeLock.acquire()) {
        logger.debug("open uncommitted translog checkpoint {}", checkpoint);
        final String checkpointTranslogFile = getFilename(checkpoint.generation);
        for (long i = translogGeneration.translogFileGeneration; i < checkpoint.generation; i++) {
            Path committedTranslogFile = location.resolve(getFilename(i));
            if (Files.exists(committedTranslogFile) == false) {
                throw new IllegalStateException("translog file doesn't exist with generation: " + i + " lastCommitted: " + lastCommittedTranslogFileGeneration + " checkpoint: " + checkpoint.generation + " - translog ids must be consecutive");
            }
            final TranslogReader reader = openReader(committedTranslogFile, Checkpoint.read(location.resolve(getCommitCheckpointFileName(i))));
            foundTranslogs.add(reader);
            logger.debug("recovered local translog from checkpoint {}", checkpoint);
        }
        foundTranslogs.add(openReader(location.resolve(checkpointTranslogFile), checkpoint));
        Path commitCheckpoint = location.resolve(getCommitCheckpointFileName(checkpoint.generation));
        if (Files.exists(commitCheckpoint)) {
            Checkpoint checkpointFromDisk = Checkpoint.read(commitCheckpoint);
            if (checkpoint.equals(checkpointFromDisk) == false) {
                throw new IllegalStateException("Checkpoint file " + commitCheckpoint.getFileName() + " already exists but has corrupted content expected: " + checkpoint + " but got: " + checkpointFromDisk);
            }
        } else {
            // we first copy this into the temp-file and then fsync it followed by an atomic move into the target file
            // that way if we hit a disk-full here we are still in an consistent state.
            Files.copy(location.resolve(CHECKPOINT_FILE_NAME), tempFile, StandardCopyOption.REPLACE_EXISTING);
            IOUtils.fsync(tempFile, false);
            Files.move(tempFile, commitCheckpoint, StandardCopyOption.ATOMIC_MOVE);
            tempFileRenamed = true;
            // we only fsync the directory the tempFile was already fsynced
            IOUtils.fsync(commitCheckpoint.getParent(), true);
        }
        success = true;
    } finally {
        if (success == false) {
            IOUtils.closeWhileHandlingException(foundTranslogs);
        }
        if (tempFileRenamed == false) {
            try {
                Files.delete(tempFile);
            } catch (IOException ex) {
                logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to delete temp file {}", tempFile), ex);
            }
        }
    }
    return foundTranslogs;
}
 

import org.apache.lucene.util.IOUtils; //导入方法依赖的package包/类
protected void fsync(String name) throws IOException {
  IOUtils.fsync(new File(directory, name), false);
}
 

import org.apache.lucene.util.IOUtils; //导入方法依赖的package包/类
/** recover all translog files found on disk */
private final ArrayList<ImmutableTranslogReader> recoverFromFiles(TranslogGeneration translogGeneration, Checkpoint checkpoint) throws IOException {
    boolean success = false;
    ArrayList<ImmutableTranslogReader> foundTranslogs = new ArrayList<>();
    final Path tempFile = Files.createTempFile(location, TRANSLOG_FILE_PREFIX, TRANSLOG_FILE_SUFFIX); // a temp file to copy checkpoint to - note it must be in on the same FS otherwise atomic move won't work
    boolean tempFileRenamed = false;
    try (ReleasableLock lock = writeLock.acquire()) {
        logger.debug("open uncommitted translog checkpoint {}", checkpoint);
        final String checkpointTranslogFile = getFilename(checkpoint.generation);
        for (long i = translogGeneration.translogFileGeneration; i < checkpoint.generation; i++) {
            Path committedTranslogFile = location.resolve(getFilename(i));
            if (Files.exists(committedTranslogFile) == false) {
                throw new IllegalStateException("translog file doesn't exist with generation: " + i + " lastCommitted: " + lastCommittedTranslogFileGeneration + " checkpoint: " + checkpoint.generation + " - translog ids must be consecutive");
            }
            final ImmutableTranslogReader reader = openReader(committedTranslogFile, Checkpoint.read(location.resolve(getCommitCheckpointFileName(i))));
            foundTranslogs.add(reader);
            logger.debug("recovered local translog from checkpoint {}", checkpoint);
        }
        foundTranslogs.add(openReader(location.resolve(checkpointTranslogFile), checkpoint));
        Path commitCheckpoint = location.resolve(getCommitCheckpointFileName(checkpoint.generation));
        if (Files.exists(commitCheckpoint)) {
            Checkpoint checkpointFromDisk = Checkpoint.read(commitCheckpoint);
            if (checkpoint.equals(checkpointFromDisk) == false) {
                throw new IllegalStateException("Checkpoint file " + commitCheckpoint.getFileName() + " already exists but has corrupted content expected: " + checkpoint + " but got: " + checkpointFromDisk);
            }
        } else {
            // we first copy this into the temp-file and then fsync it followed by an atomic move into the target file
            // that way if we hit a disk-full here we are still in an consistent state.
            Files.copy(location.resolve(CHECKPOINT_FILE_NAME), tempFile, StandardCopyOption.REPLACE_EXISTING);
            IOUtils.fsync(tempFile, false);
            Files.move(tempFile, commitCheckpoint, StandardCopyOption.ATOMIC_MOVE);
            tempFileRenamed = true;
            // we only fsync the directory the tempFile was already fsynced
            IOUtils.fsync(commitCheckpoint.getParent(), true);
        }
        success = true;
    } finally {
        if (success == false) {
            IOUtils.closeWhileHandlingException(foundTranslogs);
        }
        if (tempFileRenamed == false) {
            try {
                Files.delete(tempFile);
            } catch (IOException ex) {
                logger.warn("failed to delete temp file {}", ex, tempFile);
            }
        }
    }
    return foundTranslogs;
}