Java FilesDataUnit.Entry方法代码示例

import eu.unifiedviews.dataunit.files.FilesDataUnit; //导入方法依赖的package包/类
@Override
protected void innerExecute() throws DPUException {

    ContextUtils.sendShortInfo(ctx, "ExcelToCsv.message");

    try {
        sheetNameSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); // case-insensitive sheet names
        String sheetNames = config.getSheetNames();
        if (sheetNames != null && sheetNames.length() > 0) {
            sheetNameSet.addAll(Arrays.asList(sheetNames.split(Pattern.quote(":"))));
        }

        Set<FilesDataUnit.Entry> entries = FilesHelper.getFiles(input);
        for (FilesDataUnit.Entry entry : entries) {
            excelToCsv(entry);
        }
    } catch (DataUnitException | EncryptedDocumentException | InvalidFormatException | IOException ex) {
        throw ContextUtils.dpuException(ctx, ex, "ExcelToCsv.dpuFailed");
    }
}
 

import eu.unifiedviews.dataunit.files.FilesDataUnit; //导入方法依赖的package包/类
/**
 * Pack files in given iterator into zip file and add metadata.
 *
 * @param zipFile
 * @param filesIteration
 */
private void zipFiles(File zipFile, List<FilesDataUnit.Entry> filesIteration) throws DPUException {
    final byte[] buffer = new byte[8196];
    // Used to publish the error mesage only for the first time.
    boolean firstFailure = true;
    try (FileOutputStream fos = new FileOutputStream(zipFile); ZipOutputStream zos = new ZipOutputStream(fos)) {
        // Itarate over files and zip them.
        int counter = 0;
        for (FilesDataUnit.Entry entry : filesIteration) {
            LOG.info("Processing: {}/{}", counter++, filesIteration.size());
            if (ctx.canceled()) {
                break;
            }
            if (!addZipEntry(zos, buffer, entry)) {
                if (firstFailure) {
                    // TODO This needs rework, we fail but not instantly?
                    ContextUtils.sendError(ctx, "zipper.errors.zip.failed", "");
                }
                firstFailure = false;
            }
        }
    } catch (IOException ex) {
        throw new DPUException(ex);
    }
}
 

import eu.unifiedviews.dataunit.files.FilesDataUnit; //导入方法依赖的package包/类
private void assertInputOutputIdentical() throws Exception {
    List<FilesDataUnit.Entry> inputList = new ArrayList<FilesDataUnit.Entry>(FilesHelper.getFiles(input));
    List<FilesDataUnit.Entry> outputList = new ArrayList<FilesDataUnit.Entry>(FilesHelper.getFiles(outputValid));

    Collections.sort(inputList, new SymbolicNameComparator());
    Collections.sort(outputList, new SymbolicNameComparator());

    // iterate over files
    Iterator<FilesDataUnit.Entry> inputIterator = inputList.iterator();
    Iterator<FilesDataUnit.Entry> outputIterator = outputList.iterator();

    while (inputIterator.hasNext()) {
        String inputContents = IOUtils.toString(URI.create(inputIterator.next().getFileURIString()), "UTF-8");
        String outputContents = IOUtils.toString(URI.create(outputIterator.next().getFileURIString()), "UTF-8");

        // verify result
        assertEquals(inputContents, outputContents);
    }
}
 

import eu.unifiedviews.dataunit.files.FilesDataUnit; //导入方法依赖的package包/类
@Test
public void checkCorrectOutputFileExtension() throws Exception {
    addFileToInput("valid-for-xslt.xml");

    XSLTConfig_V1 config = new XSLTConfig_V1();
    config.setXslTemplate(resourceAsString("produce-list.xslt"));
    config.setOutputFileExtension(".extension");
    dpu.configure((new ConfigurationBuilder()).setDpuConfiguration(config).toString());

    env.run(dpu);

    VirtualPathHelper helper = VirtualPathHelpers.create(output);
    List<FilesDataUnit.Entry> outputList = new ArrayList<>(FilesHelper.getFiles(output));
    Iterator<FilesDataUnit.Entry> outputIterator = outputList.iterator();

    while (outputIterator.hasNext()) {
        String symbolicName = outputIterator.next().getSymbolicName();
        assertEquals("symbolicName-test.extension", helper.getVirtualPath(symbolicName));
    }
}
 

import eu.unifiedviews.dataunit.files.FilesDataUnit; //导入方法依赖的package包/类
/**
 * Check if file graph should be generated and if so, then generate new graph file.
 * 
 * @param rdfFile
 * @param graphName
 *            Name of the graph, that will be written into .graph file.
 * @throws DPUException
 */
private void generateGraphFile(final FilesDataUnit.Entry rdfFile, String graphName) throws DPUException {
    final FilesDataUnit.Entry graphFileEntry = faultTolerance.execute(new FaultTolerance.ActionReturn<FilesDataUnit.Entry>() {

        @Override
        public FilesDataUnit.Entry action() throws Exception {
            final String rdfFilePath = MetadataUtils.get(inRdfData, rdfFile, FilesVocabulary.UV_VIRTUAL_PATH);
            return FilesDataUnitUtils.createFile(outFilesData, rdfFilePath + ".graph");
        }
    });
    try {
        FileUtils.writeStringToFile(FaultToleranceUtils.asFile(faultTolerance, graphFileEntry), graphName);
    } catch (IOException ex) {
        throw ContextUtils.dpuException(ctx, ex, "rdfToFiles.error.graphFile");
    }
}
 

import eu.unifiedviews.dataunit.files.FilesDataUnit; //导入方法依赖的package包/类
@Test
public void rawPUTRequestTest() throws Exception {
    HttpRequestConfig_V1 config = createPutRawRequestConfig();

    URI sentResponse = this.getClass().getClassLoader().getResource(POST_RAW_RESPONSE_FILE).toURI();
    String sentResponseContent = readFile(new File(sentResponse));

    this.dpu.configure((new ConfigurationBuilder()).setDpuConfiguration(config).toString());
    HttpRequestExecutor executor = mockHttpExecutorWithResponses(config);
    this.dpu.setRequestExecutor(executor);
    this.env.run(this.dpu);

    FilesDataUnit.Entry entry = this.output.getIteration().next();
    File receivedResponse = new File(URI.create(entry.getFileURIString()));
    String receivedResponseContent = readFile(receivedResponse);

    assertEquals(sentResponseContent, receivedResponseContent);
}
 

import eu.unifiedviews.dataunit.files.FilesDataUnit; //导入方法依赖的package包/类
/**
 * Add single file into stream as zip entry.
 *
 * @param zos
 * @param buffer
 * @param entry
 * @return True if file has been added.
 * @throws DataUnitException
 */
private boolean addZipEntry(ZipOutputStream zos, byte[] buffer, final FilesDataUnit.Entry entry) throws DPUException {
    // Get virtual path.
    final String virtualPath = faultTolerance.execute(new FaultTolerance.ActionReturn<String>() {

        @Override
        public String action() throws Exception {
            return MetadataUtils.getFirst(inFilesData, entry, VirtualPathHelper.PREDICATE_VIRTUAL_PATH);
        }
    });
    if (virtualPath == null) {
        throw ContextUtils.dpuException(ctx, "zipper.error.missing.virtual.path", entry.toString());
    }
    // Add to the zip file.
    final File sourceFile = FaultToleranceUtils.asFile(faultTolerance, entry);
    LOG.info("Adding file '{}' from source '{}'", virtualPath, sourceFile);
    try (FileInputStream in = new FileInputStream(sourceFile)) {
        final ZipEntry ze = new ZipEntry(virtualPath);
        zos.putNextEntry(ze);
        // Copy data into zip file.
        int len;
        while ((len = in.read(buffer)) > 0) {
            zos.write(buffer, 0, len);
        }
    } catch (Exception ex) {
        LOG.error("Failed to add file: {}", entry.toString(), ex);
        return false;
    }
    return true;
}
 

import eu.unifiedviews.dataunit.files.FilesDataUnit; //导入方法依赖的package包/类
private void executeValidZipPasses() throws Exception {
    URI resource = this.getClass().getClassLoader().getResource(TXT_FILE).toURI();
    File inputFile = new File(resource);
    String fileContent = readFile(inputFile);

    FilesDataUnitUtils.addFile(input, inputFile, TXT_FILE);

    env.run(dpu);

    FilesDataUnit.Entry entry = output.getIteration().next();
    File zipArchive = new File(java.net.URI.create(entry.getFileURIString()));

    String unzippedFileContent = readZippedFile(TXT_FILE, zipArchive);
    assertEquals("Content of file before and after zipping should match!", fileContent, unzippedFileContent);
}
 

import eu.unifiedviews.dataunit.files.FilesDataUnit; //导入方法依赖的package包/类
private void assertOutputContainsFile(WritableFilesDataUnit output, String symbolicName) throws DataUnitException{
    List<FilesDataUnit.Entry> outputList = new ArrayList<FilesDataUnit.Entry>(FilesHelper.getFiles(output));
    boolean contains = false;
    for(Entry entry : outputList) {
        if(entry.getFileURIString().endsWith(symbolicName)){
            contains = true;
        }
    }
    assertEquals("Should return true.", true, contains);
}
 

import eu.unifiedviews.dataunit.files.FilesDataUnit; //导入方法依赖的package包/类
@Override
protected void innerExecute() throws DPUException {
    // Prepare output path.
    final FilesDataUnit.Entry output = faultTolerance.execute(new FaultTolerance.ActionReturn<FilesDataUnit.Entry>() {

        @Override
        public FilesDataUnit.Entry action() throws Exception {
            return FilesDataUnitUtils.createFile(outFilesData, config.getTargetPath());
        }
    });
    // Prepare query.
    final String queryAsString = faultTolerance.execute(new FaultTolerance.ActionReturn<String>() {

        @Override
        public String action() throws Exception {
            final List<RDFDataUnit.Entry> sources = DataUnitUtils.getEntries(inRdfData, RDFDataUnit.Entry.class);
            return config.getQuery().replaceFirst("(?i)WHERE", 
                    SparqlUtils.prepareClause("FROM", sources) + "WHERE ");
        }
    });
    // Execute query and import into csv.
    faultTolerance.execute(inRdfData, new FaultTolerance.ConnectionAction() {

        @Override
        public void action(RepositoryConnection connection) throws Exception {
            final File outFile = FilesDataUnitUtils.asFile(output);
            final SPARQLResultsCSVWriterFactory writerFactory = new SPARQLResultsCSVWriterFactory();
            // Create output file and write the result.
            try (OutputStream outputStream = new FileOutputStream(outFile)) {
                final TupleQueryResultWriter resultWriter = writerFactory.getWriter(outputStream);
                final TupleQuery query = connection.prepareTupleQuery(QueryLanguage.SPARQL, queryAsString);
                query.evaluate(resultWriter);
            } catch (IOException ex) {
                throw new DPUException(ex);
            }
        }
    });
}
 

import eu.unifiedviews.dataunit.files.FilesDataUnit; //导入方法依赖的package包/类
/**
 * Creates file data unit from HTTP response
 * 
 * @param response
 *            HTTP response
 * @param fileName
 *            File name of result file
 * @throws DPUException
 */
private void createFileFromResponse(CloseableHttpResponse response, final String fileName) throws DPUException {
    LOG.debug("Filename is: {}", fileName);
    // Prepare new output file record.
    final FilesDataUnit.Entry destinationFile = this.faultTolerance.execute(new FaultTolerance.ActionReturn<FilesDataUnit.Entry>() {

        @SuppressWarnings("unqualified-field-access")
        @Override
        public FilesDataUnit.Entry action() throws Exception {
            return FilesDataUnitUtils.createFile(HttpRequest.this.requestOutput, fileName);
        }
    });

    this.faultTolerance.execute(new FaultTolerance.Action() {

        @SuppressWarnings("unqualified-field-access")
        @Override
        public void action() throws Exception {
            final Resource resource = ResourceHelpers.getResource(HttpRequest.this.requestOutput, fileName);
            final Date now = new Date();
            resource.setCreated(now);
            resource.setLast_modified(now);
            ResourceHelpers.setResource(HttpRequest.this.requestOutput, fileName, resource);
        }
    }, "dpu.errors.resource");

    try {
        FileUtils.copyInputStreamToFile(response.getEntity().getContent(),
                FaultToleranceUtils.asFile(this.faultTolerance, destinationFile));
    } catch (IOException ex) {
        LOG.error("Failed to create file from response input stream", ex);
        throw ContextUtils.dpuException(this.ctx, ex, "dpu.errors.response.store");
    }
}
 

import eu.unifiedviews.dataunit.files.FilesDataUnit; //导入方法依赖的package包/类
@Override
protected void innerExecute() throws DPUException {
    final List<FilesDataUnit.Entry> files = FaultToleranceUtils.getEntries(faultTolerance, inFilesData, FilesDataUnit.Entry.class);
    final Iterator<FilesDataUnit.Entry> filesIteration = files.iterator();

    try {
        // for each input file
        while (!ctx.canceled() && filesIteration.hasNext()) {
            final File inputFile = FilesDataUnitUtils.asFile(filesIteration.next());
            LOG.debug("Adding file: {}", inputFile.getName());

            RelationalParser parser = null;
            switch (config.getParserType()) {
                case CSV:
                    parser = new CSVParser(ctx, config, outRelationalData);
                    break;
                case XLS:
                    parser = new XLSParser(ctx, config, outRelationalData);
                    break;
                case DBF:
                    parser = new DBFParser(ctx, config, outRelationalData);
                    break;
                default:
                    throw new DataUnitException("Unsupported ParserType!");
            }

            // parse file and add output to result table
            parser.parseFile(inputFile);
        }

        // finally add metadata
        final String tableName = config.getTableName().toUpperCase();
        outRelationalData.addExistingDatabaseTable(tableName, tableName);
        faultTolerance.execute(new FaultTolerance.Action() {
            @Override
            public void action() throws Exception {
                Resource resource = ResourceHelpers.getResource(outRelationalData, tableName);
                Date now = new Date();
                resource.setCreated(now);
                resource.setLast_modified(now);
                ResourceHelpers.setResource(outRelationalData, tableName, resource);
            }
        });
    } catch (DataUnitException e) {
        throw ContextUtils.dpuException(ctx, e, "errors.dpu.parse.failed");
    }
    ContextUtils.sendShortInfo(ctx, "parsing.finished");
}
 

import eu.unifiedviews.dataunit.files.FilesDataUnit; //导入方法依赖的package包/类
@Override
protected void innerExecute() throws DPUException {
    // Older version of unzipper does not have configuration at all.
    if (config == null) {
        config = new UnZipperConfig_V1();
    }
    // Prepare root output directory.
    final File baseTargetDirectory = faultTolerance.execute(new FaultTolerance.ActionReturn<File>() {

        @Override
        public File action() throws Exception {
            return new File(java.net.URI.create(outFilesData.getBaseFileURIString()));
        }
    }, "unzipper.errors.file.outputdir");
    // Get list of files to unzip.
    final List<FilesDataUnit.Entry> files = FaultToleranceUtils.getEntries(faultTolerance, inFilesData,
        FilesDataUnit.Entry.class);

    LOG.info(">> {}", files.size());

    int counter = 0;
    for (final FilesDataUnit.Entry fileEntry : files) {
        LOG.info("Processing: {}/{}", counter++, files.size());
        if (ctx.canceled()) {
            return;
        }
        final File sourceFile = FaultToleranceUtils.asFile(faultTolerance, fileEntry);
        // Get virtual path.
        final String zipRelativePath = faultTolerance.execute(new FaultTolerance.ActionReturn<String>() {

            @Override
            public String action() throws Exception {
                return MetadataUtils.getFirst(inFilesData, fileEntry, VirtualPathHelper.PREDICATE_VIRTUAL_PATH);
            }
        }, "unzipper.error.virtualpath.get.failed");
        if (zipRelativePath == null) {
            throw ContextUtils.dpuException(ctx, "unzipper.error.missing.virtual.path", fileEntry.toString());
        }
        // Unzip.
        final File targetDirectory = new File(baseTargetDirectory, zipRelativePath);
        unzip(sourceFile, targetDirectory);
        // Scan for new files.
        scanDirectory(targetDirectory, zipRelativePath);
    }
}
 

import eu.unifiedviews.dataunit.files.FilesDataUnit; //导入方法依赖的package包/类
/**
 * Add file to the DataUnit.
 * 
 * @param dataUnit
 * @param file
 *            File to add, must be under root.
 * @param symbolicName
 * @return
 * @throws eu.unifiedviews.dataunit.DataUnitException
 */
public static FilesDataUnit.Entry addFile(WritableFilesDataUnit dataUnit, File file, String symbolicName)
        throws DataUnitException {
    // Add existing file to DataUnit.
    dataUnit.addExistingFile(symbolicName, file.toURI().toString());
    // Set available metadata.
    MetadataUtils.add(dataUnit, symbolicName, FilesVocabulary.UV_VIRTUAL_PATH, symbolicName);
    // Return representing instance.
    return new InMemoryEntry(file.toURI().toString(), symbolicName);
}
 

import eu.unifiedviews.dataunit.files.FilesDataUnit; //导入方法依赖的package包/类
/**
 * Create file of under given path and return {@link File} to it. Also add {@link VirtualPathHelper#PREDICATE_VIRTUAL_PATH} metadata to the new file.
 * As this function create new connection is should not be used for greater number of files.
 * 
 * @param dataUnit
 * @param symbolicName
 * @return
 * @throws DataUnitException
 */
public static FilesDataUnit.Entry createFile(WritableFilesDataUnit dataUnit, String symbolicName)
        throws DataUnitException {
    final String fileUri = dataUnit.addNewFile(symbolicName);
    MetadataUtils.add(dataUnit, symbolicName, FilesVocabulary.UV_VIRTUAL_PATH, symbolicName);
    return new InMemoryEntry(fileUri, symbolicName);
}