Java TaskSource类代码示例

import org.embulk.config.TaskSource; //导入依赖的package包/类
@Test
public void testOpenForGeneratedArchives() throws Exception
{
    String[] testFormats = new String[]{
            ArchiveStreamFactory.AR,
            // ArchiveStreamFactory.ARJ, // ArchiveException: Archiver: arj not found.
            ArchiveStreamFactory.CPIO,
            // ArchiveStreamFactory.DUMP, // ArchiveException: Archiver: dump not found.
            ArchiveStreamFactory.JAR,
            // ArchiveStreamFactory.SEVEN_Z, // StreamingNotSupportedException: The 7z doesn't support streaming.
            ArchiveStreamFactory.TAR,
            ArchiveStreamFactory.ZIP,
    };

    for (String format : testFormats) {
        TaskSource mockTaskSource = new MockTaskSource(format);
        FileInput mockInput = new MockFileInput(
                getInputStreamAsBuffer(
                        getArchiveInputStream(format, "sample_1.csv", "sample_2.csv")));
        CommonsCompressDecoderPlugin plugin = new CommonsCompressDecoderPlugin();
        FileInput archiveFileInput = plugin.open(mockTaskSource, mockInput);
        verifyContents(archiveFileInput, "1,foo", "2,bar");
    }
}
 

import org.embulk.config.TaskSource; //导入依赖的package包/类
@Override
public TaskReport run(TaskSource taskSource,
        Schema schema, int taskIndex,
        PageOutput output)
{
    PluginTask task = taskSource.loadTask(PluginTask.class);
    Integer rows = task.getRows();
    final HashMap<Column, Map<String, Integer>> columnOptions = getColumnOptions(task);
    try (PageBuilder pagebuilder =
            new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
        IntStream.rangeClosed(
                taskIndex * rows + 1,
                taskIndex * rows + rows
        ).boxed().forEach(rowNumber -> {
            RandomjColumnVisitor visitor = new RandomjColumnVisitor(pagebuilder, task, rowNumber, columnOptions);
            schema.visitColumns(visitor);
            pagebuilder.addRecord();
        });
        pagebuilder.finish();
    }

    TaskReport taskReport = Exec.newTaskReport();
    taskReport.set("columns", schema.size());
    taskReport.set("rows", rows);
    return taskReport;
}
 

import org.embulk.config.TaskSource; //导入依赖的package包/类
@Override
public FileInput open(TaskSource taskSource, FileInput fileInput) {
  final PluginTask task = taskSource.loadTask(PluginTask.class);

  final FileInputInputStream files = new FileInputInputStream(fileInput);

  return new InputStreamFileInput(
      task.getBufferAllocator(),
      new InputStreamFileInput.Provider() {
        public InputStream openNext() throws IOException {
          if (!files.nextFile()) {
            return null;
          }
          LOGGER.debug("generate LZ4FrameInputStream for {}", files.toString());
          return new LZ4FrameInputStream(files);
        }

        public void close() throws IOException {
          files.close();
        }
      }
  );
}
 

import org.embulk.config.TaskSource; //导入依赖的package包/类
private void testQueryString(ConfigSource configSource, final Schema inputSchema, final String path, final AssertionWithPage assertion)
{
    final QueryStringFilterPlugin plugin = new QueryStringFilterPlugin();
    plugin.transaction(configSource, inputSchema, new FilterPlugin.Control()
    {
        @Override
        public void run(TaskSource taskSource, Schema outputSchema)
        {
            TestPageBuilderReader.MockPageOutput mockPageOutput = new TestPageBuilderReader.MockPageOutput();
            PageOutput pageOutput = plugin.open(taskSource, inputSchema, outputSchema, mockPageOutput);

            List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), inputSchema, "before", path, "after");
            for (Page page : pages) {
                pageOutput.add(page);
            }

            pageOutput.finish();
            pageOutput.close();

            PageReader pageReader = new PageReader(outputSchema);
            assertion.run(pageReader, mockPageOutput);
            ;
        }
    });
}
 

import org.embulk.config.TaskSource; //导入依赖的package包/类
@Override
public FileInput open(TaskSource taskSource, FileInput input)
{
    PluginTask task = taskSource.loadTask(PluginTask.class);
    return new CommonsCompressFileInput(
            task.getBufferAllocator(),
            new CommonsCompressProvider(task, new FileInputInputStream(input) {
                // NOTE: This is workaround code to avoid hanging issue.
                // This issue will be fixed after merging #112.
                // https://github.com/embulk/embulk/pull/112
                @Override
                public long skip(long len) {
                    long skipped = super.skip(len);
                    return skipped > 0 ? skipped : 0;
                }
            }));
}
 

import org.embulk.config.TaskSource; //导入依赖的package包/类
@Test
public void testOpenForGeneratedCompression() throws Exception
{
    String[] testFormats = new String[]{
            CompressorStreamFactory.BZIP2,
            CompressorStreamFactory.DEFLATE,
            CompressorStreamFactory.GZIP,
            // CompressorStreamFactory.LZMA, // CompressorException: Compressor: lzma not found.
            // CompressorStreamFactory.PACK200, // Failed to generate compressed file.
            // CompressorStreamFactory.SNAPPY_FRAMED, // CompressorException: Compressor: snappy-framed not found.
            // CompressorStreamFactory.SNAPPY_RAW, // CompressorException: Compressor: snappy-raw not found.
            // CompressorStreamFactory.XZ, // ClassNotFoundException: org.tukaani.xz.FilterOptions
            // CompressorStreamFactory.Z, // CompressorException: Compressor: z not found.
    };

    for (String format : testFormats) {
        TaskSource mockTaskSource = new MockTaskSource(format);
        FileInput mockInput = new MockFileInput(
                getInputStreamAsBuffer(
                        getCompressorInputStream(format, "sample_1.csv")));
        CommonsCompressDecoderPlugin plugin = new CommonsCompressDecoderPlugin();
        FileInput archiveFileInput = plugin.open(mockTaskSource, mockInput);
        verifyContents(archiveFileInput, "1,foo");
    }
}
 

import org.embulk.config.TaskSource; //导入依赖的package包/类
public Icu4jPageOutput(TaskSource taskSource, Schema inputSchema, Schema outputSchema, PageOutput output) {
    this.task = taskSource.loadTask(PluginTask.class);
    this.keyNameColumns = Lists.newArrayList();
    this.transliteratorsList = Lists.newArrayList();
    this.inputSchema = inputSchema;
    this.outputSchema = outputSchema;

    for (String keyName : task.getKeyNames()) {
        keyNameColumns.add(inputSchema.lookupColumn(keyName));
    }
    for (Map<String, String> setting : task.getSettings()) {
        List<Transliterator> tokenizers = Lists.newArrayList();
        for (String convertType : setting.get("transliterators").split(",")) {
            Transliterator transliterator = Transliterator.getInstance(convertType);
            tokenizers.add(transliterator);
        }
        transliteratorsList.add(tokenizers);
    }
    reader = new PageReader(inputSchema);
    builder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, output);
}
 

import org.embulk.config.TaskSource; //导入依赖的package包/类
public WootheePageOutput(TaskSource taskSource, Schema inputSchema, Schema outputSchema, PageOutput output) {
    this.task = taskSource.loadTask(PluginTask.class);
    this.outputColumns = outputSchema.getColumns();
    this.inputColumns = inputSchema.getColumns();
    Map<String, Column> inputColumnMap = Maps.newHashMap();
    this.wootheeColumnMap = Maps.newHashMap();
    for (Column column : outputColumns) {
        if (!inputColumns.contains(column)) {
            wootheeColumnMap.put(column.getName(), column);
        } else {
            inputColumnMap.put(column.getName(), column);
        }
    }
    this.reader = new PageReader(inputSchema);
    this.builder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, output);
    this.keyNameColumn = inputColumnMap.get(task.getKeyName());
}
 

import org.embulk.config.TaskSource; //导入依赖的package包/类
@Override
    public void cleanup(TaskSource taskSource,
            int taskCount,
            List<TaskReport> successTaskReports)
    {
        /*
          #37 causes permission failure while renaming remote file.
          https://github.com/embulk/embulk-output-sftp/issues/40
         */
//        SftpUtils sftpUtils = new SftpUtils(taskSource.loadTask(PluginTask.class));
//        for (TaskReport report : successTaskReports) {
//            List<Map<String, String>> moveFileList = report.get(List.class, "file_list");
//            for (Map<String, String> pairFiles : moveFileList) {
//                String temporaryFileName = pairFiles.get("temporary_filename");
//                String realFileName = pairFiles.get("real_filename");
//
//                sftpUtils.renameFile(temporaryFileName, realFileName);
//            }
//        }
//        sftpUtils.close();
    }
 

import org.embulk.config.TaskSource; //导入依赖的package包/类
@Test
public void testConfigureInjectedTask()
{
    String yaml = "" +
            "type: distinct\n" +
            "columns: [_c0, _c1]\n";

    ConfigSource config = loadConfigFromYaml(yaml);

    plugin.transaction(config, schema, new Control() {
        @Override
        public void run(TaskSource taskSource, Schema outputSchema)
        {
            PluginTask task = taskSource.loadTask(PluginTask.class);

            List<Column> columns = Lists.newArrayList();
            columns.add(new Column(0, "_c0", Types.STRING));
            columns.add(new Column(1, "_c1", Types.STRING));

            assertEquals(columns, task.getDistinctColumns());
        }
    });
}
 

import org.embulk.config.TaskSource; //导入依赖的package包/类
@Override
public ConfigDiff resume(TaskSource taskSource,
        Schema schema, int taskCount,
        InputPlugin.Control control)
{
    List<TaskReport> taskReportList =
            control.run(taskSource, schema, taskCount);

    // start_row_marker を ConfigDiff にセット
    ConfigDiff configDiff = Exec.newConfigDiff();
    for (TaskReport taskReport : taskReportList) {
        final String label = "start_row_marker";
        final String startRowMarker = taskReport.get(String.class, label, null);
        if (startRowMarker != null) {
            configDiff.set(label, startRowMarker);
        }
    }
    return configDiff;
}
 

import org.embulk.config.TaskSource; //导入依赖的package包/类
@Override
public void run(TaskSource taskSource, Schema schema,
        FileInput input, PageOutput output)
{
    PluginTask task = taskSource.loadTask(PluginTask.class);
    LineDecoder lineDecoder = new LineDecoder(input,task);
    PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output);
    String line = null;
    final String columnName = task.getColumnName();

    while( input.nextFile() ){
        while(true){
          line = lineDecoder.poll();

          if( line == null ){
              break;
          }

          pageBuilder.setString(0, line);
          pageBuilder.addRecord();
        }
    }
    pageBuilder.finish();
}
 

import org.embulk.config.TaskSource; //导入依赖的package包/类
@Override
public PageOutput open(TaskSource taskSource, Schema inputSchema,
        Schema outputSchema, PageOutput output)
{
    System.out.println("open!");
    System.out.print("taskSource: ");
    System.out.println(taskSource);
    System.out.print("inputSchema: ");
    System.out.println(inputSchema);
    System.out.print("outputSchema: ");
    System.out.println(outputSchema);
    System.out.print("output: ");
    System.out.println(output);
    PluginTask task = taskSource.loadTask(PluginTask.class);
    System.out.print("task: ");
    System.out.println(task);

    return new MyPageOutput(inputSchema, outputSchema, output);
}
 

import org.embulk.config.TaskSource; //导入依赖的package包/类
@Override
public ConfigDiff resume(TaskSource taskSource,
        Schema schema, int taskCount,
        InputPlugin.Control control)
{
    control.run(taskSource, schema, taskCount);
    return Exec.newConfigDiff();
}
 

import org.embulk.config.TaskSource; //导入依赖的package包/类
@Override
public void cleanup(TaskSource taskSource,
        Schema schema, int taskCount,
        List<TaskReport> successTaskReports)
{
    // throw new UnsupportedOperationException("randomj input plugin does not support cleanup");
}