Java PageReader类代码示例

import org.embulk.spi.PageReader; //导入依赖的package包/类
@Override
public byte[] serialize(Mode mode, PageReader reader, Schema schema, List<String> dataColumns, TimestampFormatter[] timestampFormatters) {
    S3PerRecordOutputColumnVisitor visitor;
    switch(mode) {
        case SINGLE_COLUMN:
            visitor = new MessagePackSingleColumnVisitor(reader, timestampFormatters);
            schema.lookupColumn(dataColumns.get(0)).visit(visitor);
            break;
        case MULTI_COLUMN:
            visitor = new MessagePackMultiColumnVisitor(reader, timestampFormatters);
            for (String columnName : dataColumns) {
                schema.lookupColumn(columnName).visit(visitor);
            }
            break;
        default:
            throw new RuntimeException("never reach here");
    }
    return visitor.getByteArray();
}
 

import org.embulk.spi.PageReader; //导入依赖的package包/类
@Test
public void testOpenSuccessfully()
{
    ConfigSource configSource = loadConfigSource("testOpen.yml");
    final Schema inputSchema = Schema.builder()
            .add("qb", STRING)
            .add("qs", STRING)
            .add("qa", STRING)
            .build();
    testQueryString(configSource, inputSchema, "/path?q1=1&q2=2#fragment", new AssertionWithPage()
    {
        @Override
        public void run(PageReader pageReader, TestPageBuilderReader.MockPageOutput pageOutput)
        {
            for (Page page : pageOutput.pages) {
                pageReader.setPage(page);

                assertThat(pageReader.getString(0), is("before"));
                assertThat(pageReader.getString(1), is("1"));
                assertEquals(2L, pageReader.getLong(2));
                assertThat(pageReader.getString(3), is("after"));
            }
        }
    });
}
 

import org.embulk.spi.PageReader; //导入依赖的package包/类
@Test
public void testOpenSuccessfullyWithHashbang()
{
    ConfigSource configSource = loadConfigSource("testOpen.yml");
    final Schema inputSchema = Schema.builder()
            .add("qb", STRING)
            .add("qs", STRING)
            .add("qa", STRING)
            .build();

    testQueryString(configSource, inputSchema, "/#!/path?q1=one&q2=2#fragment", new AssertionWithPage()
    {
        @Override
        public void run(PageReader pageReader, TestPageBuilderReader.MockPageOutput pageOutput)
        {
            for (Page page : pageOutput.pages) {
                pageReader.setPage(page);

                assertThat(pageReader.getString(0), is("before"));
                assertThat(pageReader.getString(1), is("one"));
                assertEquals(2L, pageReader.getLong(2));
                assertThat(pageReader.getString(3), is("after"));
            }
        }
    });
}
 

import org.embulk.spi.PageReader; //导入依赖的package包/类
private void testQueryString(ConfigSource configSource, final Schema inputSchema, final String path, final AssertionWithPage assertion)
{
    final QueryStringFilterPlugin plugin = new QueryStringFilterPlugin();
    plugin.transaction(configSource, inputSchema, new FilterPlugin.Control()
    {
        @Override
        public void run(TaskSource taskSource, Schema outputSchema)
        {
            TestPageBuilderReader.MockPageOutput mockPageOutput = new TestPageBuilderReader.MockPageOutput();
            PageOutput pageOutput = plugin.open(taskSource, inputSchema, outputSchema, mockPageOutput);

            List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), inputSchema, "before", path, "after");
            for (Page page : pages) {
                pageOutput.add(page);
            }

            pageOutput.finish();
            pageOutput.close();

            PageReader pageReader = new PageReader(outputSchema);
            assertion.run(pageReader, mockPageOutput);
            ;
        }
    });
}
 

import org.embulk.spi.PageReader; //导入依赖的package包/类
private List<Object[]> filter(PluginTask task, Schema inputSchema, Object ... objects)
{
    TestPageBuilderReader.MockPageOutput output = new TestPageBuilderReader.MockPageOutput();
    Schema outputSchema = BuildJsonFilterPlugin.buildOutputSchema(task, inputSchema);
    PageBuilder pageBuilder = new PageBuilder(runtime.getBufferAllocator(), outputSchema, output);
    PageReader pageReader = new PageReader(inputSchema);
    BuildJsonVisitorImpl visitor = new BuildJsonVisitorImpl(task, inputSchema, outputSchema, pageReader, pageBuilder);

    List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), inputSchema, objects);
    for (Page page : pages) {
        pageReader.setPage(page);

        while (pageReader.nextRecord()) {
            outputSchema.visitColumns(visitor);
            pageBuilder.addRecord();
        }
    }
    pageBuilder.finish();
    pageBuilder.close();
    return Pages.toObjects(outputSchema, output.pages);
}
 

import org.embulk.spi.PageReader; //导入依赖的package包/类
ColumnCaster(PluginTask task, Schema inputSchema, Schema outputSchema, PageReader pageReader, PageBuilder pageBuilder)
{
    this.task         = task;
    this.inputSchema  = inputSchema;
    this.outputSchema = outputSchema;
    this.pageReader   = pageReader;
    this.pageBuilder  = pageBuilder;

    buildTimestampParserMap();
    buildTimestampFormatterMap();
    buildFromTimestampUnitMap();
    buildToTimestampUnitMap();

    JsonCaster jsonCaster = new JsonCaster(task, timestampParserMap, timestampFormatterMap, fromTimestampUnitMap, toTimestampUnitMap);
    this.jsonVisitor = new JsonVisitor(task, jsonCaster);
}
 

import org.embulk.spi.PageReader; //导入依赖的package包/类
public Icu4jPageOutput(TaskSource taskSource, Schema inputSchema, Schema outputSchema, PageOutput output) {
    this.task = taskSource.loadTask(PluginTask.class);
    this.keyNameColumns = Lists.newArrayList();
    this.transliteratorsList = Lists.newArrayList();
    this.inputSchema = inputSchema;
    this.outputSchema = outputSchema;

    for (String keyName : task.getKeyNames()) {
        keyNameColumns.add(inputSchema.lookupColumn(keyName));
    }
    for (Map<String, String> setting : task.getSettings()) {
        List<Transliterator> tokenizers = Lists.newArrayList();
        for (String convertType : setting.get("transliterators").split(",")) {
            Transliterator transliterator = Transliterator.getInstance(convertType);
            tokenizers.add(transliterator);
        }
        transliteratorsList.add(tokenizers);
    }
    reader = new PageReader(inputSchema);
    builder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, output);
}
 

import org.embulk.spi.PageReader; //导入依赖的package包/类
public WootheePageOutput(TaskSource taskSource, Schema inputSchema, Schema outputSchema, PageOutput output) {
    this.task = taskSource.loadTask(PluginTask.class);
    this.outputColumns = outputSchema.getColumns();
    this.inputColumns = inputSchema.getColumns();
    Map<String, Column> inputColumnMap = Maps.newHashMap();
    this.wootheeColumnMap = Maps.newHashMap();
    for (Column column : outputColumns) {
        if (!inputColumns.contains(column)) {
            wootheeColumnMap.put(column.getName(), column);
        } else {
            inputColumnMap.put(column.getName(), column);
        }
    }
    this.reader = new PageReader(inputSchema);
    this.builder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, output);
    this.keyNameColumn = inputColumnMap.get(task.getKeyName());
}
 

import org.embulk.spi.PageReader; //导入依赖的package包/类
private List<Object[]> filter(PluginTask task, Schema inputSchema, Object ... objects)
{
    MockPageOutput output = new MockPageOutput();
    Schema outputSchema = ColumnFilterPlugin.buildOutputSchema(task, inputSchema);
    PageBuilder pageBuilder = new PageBuilder(runtime.getBufferAllocator(), outputSchema, output);
    PageReader pageReader = new PageReader(inputSchema);
    ColumnVisitorImpl visitor = new ColumnVisitorImpl(task, inputSchema, outputSchema, pageReader, pageBuilder);

    List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), inputSchema, objects);
    for (Page page : pages) {
        pageReader.setPage(page);

        while (pageReader.nextRecord()) {
            outputSchema.visitColumns(visitor);
            pageBuilder.addRecord();
        }
    }
    pageBuilder.finish();
    pageBuilder.close();
    return Pages.toObjects(outputSchema, output.pages);
}
 

import org.embulk.spi.PageReader; //导入依赖的package包/类
@Test
public void testOpen(final @Mocked PageReader reader, final @Mocked PageBuilder builder, final @Mocked Page page) throws Exception {
    new NonStrictExpectations() {{
        taskSource.loadTask(PluginTask.class); result = task;
        task.getDelimiter(); result = "";
        reader.nextRecord(); result = true; result = false;
    }};

    SpeedometerFilterPlugin plugin = new SpeedometerFilterPlugin();
    PageOutput output = plugin.open(taskSource, schema, schema, inPageOutput);
    output.add(page);

    new Verifications() {{
        taskSource.loadTask(PluginTask.class); times = 1;
        builder.addRecord(); times = 1;
        builder.finish(); times = 0;
        reader.nextRecord(); times = 2;
        reader.setPage(page); times = 1;
        schema.visitColumns(withInstanceOf(ColumnVisitor.class)); times = 1;
    }};
}
 

import org.embulk.spi.PageReader; //导入依赖的package包/类
@Test
public void testFinish(final @Mocked PageReader reader, final @Mocked PageBuilder builder, final @Mocked Page page) throws Exception {
    new NonStrictExpectations() {{
        taskSource.loadTask(PluginTask.class); result = task;
        task.getDelimiter(); result = "";
    }};

    SpeedometerFilterPlugin plugin = new SpeedometerFilterPlugin();
    PageOutput output = plugin.open(taskSource, schema, schema, inPageOutput);
    output.finish();

    new Verifications() {{
        taskSource.loadTask(PluginTask.class); times = 1;
        builder.finish(); times = 1;
    }};
}
 

import org.embulk.spi.PageReader; //导入依赖的package包/类
@Test
public void testClose(final @Mocked PageReader reader, final @Mocked PageBuilder builder, final @Mocked Page page) throws Exception {
    new NonStrictExpectations() {{
        taskSource.loadTask(PluginTask.class); result = task;
        task.getDelimiter(); result = "";
    }};

    SpeedometerFilterPlugin plugin = new SpeedometerFilterPlugin();
    PageOutput output = plugin.open(taskSource, schema, schema, inPageOutput);
    output.close();

    new Verifications() {{
        taskSource.loadTask(PluginTask.class); times = 1;
        builder.close(); times = 1;
        reader.close(); times = 1;
    }};
}
 

import org.embulk.spi.PageReader; //导入依赖的package包/类
JoinFilePageOutput(
        Schema inputSchema,
        Schema outputSchema,
        Column joinBaseColumn,
        HashMap<String, HashMap<String, String>> table,
        List<Column> joinColumns,
        HashMap<String, TimestampParser> timestampParserMap,
        PageOutput pageOutput
        )
{
    this.pageReader = new PageReader(inputSchema);
    this.pageBuilder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, pageOutput);
    this.joinBaseColumn = joinBaseColumn;
    this.table = table;
    this.joinColumns = joinColumns;
    this.timestampParserMap = timestampParserMap;
}
 

import org.embulk.spi.PageReader; //导入依赖的package包/类
private static String getCurrentJoinBaseColumnValue(PageReader pageReader, Column joinBaseColumn)
{
    if (pageReader.isNull(joinBaseColumn)) {
        return null;
    }

    if (Types.STRING.equals(joinBaseColumn.getType())) {
        return pageReader.getString(joinBaseColumn);
    }
    else if (Types.BOOLEAN.equals(joinBaseColumn.getType())) {
        return String.valueOf(pageReader.getBoolean(joinBaseColumn));
    }
    else if (Types.DOUBLE.equals(joinBaseColumn.getType())) {
        return String.valueOf(pageReader.getDouble(joinBaseColumn));
    }
    else if (Types.LONG.equals(joinBaseColumn.getType())) {
        return String.valueOf(pageReader.getLong(joinBaseColumn));
    }
    else if (Types.TIMESTAMP.equals(joinBaseColumn.getType())) {
        return String.valueOf(pageReader.getTimestamp(joinBaseColumn));
    }

    throw Throwables.propagate(new Throwable("Unsupported Column Type: " + joinBaseColumn.getType()));
}
 

import org.embulk.spi.PageReader; //导入依赖的package包/类
public GoogleTranslateApiPageOutput(TaskSource taskSource, Schema inputSchema, Schema outputSchema, PageOutput output) {
    this.task = taskSource.loadTask(PluginTask.class);
    this.outputSchema = outputSchema;
    this.inputColumns = inputSchema.getColumns();
    this.reader = new PageReader(inputSchema);
    this.builder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, output);
    this.keyNameColumns = Lists.newArrayList();
    for (String keyName : task.getKeyNames()) {
        this.keyNameColumns.add(outputSchema.lookupColumn(keyName));
    }
    this.translate = createTranslateService();

    this.srcLang = task.getSourceLang().isPresent() ? TranslateOption.sourceLanguage(task.getSourceLang().get()) : null;
    this.model = task.getModel().isPresent() ? TranslateOption.model(task.getModel().get()) : null;
}