Java CsvParser.parseNext方法代码示例

import com.univocity.parsers.csv.CsvParser; //导入方法依赖的package包/类
/**
 * Read stream of taxi trip data from file
 * and broadcast its value next.
 *
 * @throws InterruptedException
 */
public void readStream() throws InterruptedException {
    // setting up CSV parser
    CsvParser parser = setupParser();

    // read records one by one
    parser.beginParsing(getReader(this.fileName));

    String[] row;
    // read line by line
    while ((row = parser.parseNext()) != null) {
        // create a string from array separated by comma
        String trip = String.join(",", row);

        // sink values to trips broadcaster
        this.tripsProcessor.onNext(trip);
    }
    // close the channel as we are finished streaming data
    // this sends a complete signal which we can in turn observe
    this.tripsProcessor.onComplete();

    // finished parsing all the data from the csv file
    parser.stopParsing();
}
 

import com.univocity.parsers.csv.CsvParser; //导入方法依赖的package包/类
private void processSinglePathWithByteBuffer(BatchWriter writer, FileMapping mapping, Path p, CsvParser parser) throws IOException, MutationsRejectedException {
  final FileSystem fs = p.getFileSystem(conf);
  FSDataInputStream dis = fs.open(p, INPUT_BUFFER_SIZE);
  InputStreamReader reader = new InputStreamReader(dis, UTF_8);
  try {
    parser.beginParsing(reader);
    String[] line = null;
    while ((line = parser.parseNext()) != null) {
      writer.addMutation(parseLine(mapping, line));
    }
  } finally {
    if (null != reader) {
      reader.close();
    }
  }
}
 

import com.univocity.parsers.csv.CsvParser; //导入方法依赖的package包/类
@Before
public void init() {
	parser = new CsvParser(settings);
	parser.beginParsing(new StringReader(CSV));
	rowContext = PowerMockito.spy(new UnivocityParserRowContext(parser.getContext(), metadata.size()));
	parser.parseNext();
}
 

import com.univocity.parsers.csv.CsvParser; //导入方法依赖的package包/类
public List<BlockWithData> parseDataset(String pathToFile) {
    List<BlockWithData> blocksWithData = new ArrayList<>();
    CsvParser parser = dataParser.getParserForFile(pathToFile);
    String[] fields = parser.parseNext();

    if (fields != null) {
        List<String> fieldsNames = new ArrayList<>(Arrays.asList(fields));
        blocksWithData = blockBuilder.build(parser, fieldsNames);
    }
    return blocksWithData;
}
 

import com.univocity.parsers.csv.CsvParser; //导入方法依赖的package包/类
@Override
public List<BlockWithData> build(CsvParser parser, List<String> fieldsNames) {
    this.fieldsNames = fieldsNames;

    List<BlockWithData> blocksWithData = new ArrayList<>();
    List<Record> recordsForBlock = new ArrayList<>();
    String[] row = parser.parseNext();
    String currentBlockId;
    //while there are more rows in dataset
    currentBlockId = initForFirstRecordOnly(row, recordsForBlock);
    row = parser.parseNext();
    // we have already initialized the first record, therefore the index starts at 2
    int recordIndex = STARTING_RECORD_INDEX;
    while (row != null && currentBlockId != null) {
        Record record = this.createRecord(row, recordIndex);

        String blockId = getBlockIdFromRecord(record);
        if (blockId.equals(currentBlockId)) {
            recordsForBlock.add(record);
        } else {
            createBlockFromRecords(blocksWithData, recordsForBlock);
            recordsForBlock = new ArrayList<>();
            recordsForBlock.add(record);
            currentBlockId = blockId;
        }
        row = parser.parseNext();
        recordIndex++;
    }
    if (!recordsForBlock.isEmpty()) {
        createBlockFromRecords(blocksWithData, recordsForBlock);
    }

    return blocksWithData;
}
 

import com.univocity.parsers.csv.CsvParser; //导入方法依赖的package包/类
@Override
public void end(byte packID)
{
    isStartLoadData = false;
    this.packID = packID;
    //load in data空包 结束
    saveByteOrToFile(null, true);
    List<SQLExpr> columns = statement.getColumns();
    String tableName = statement.getTableName().getSimpleName();
    if (isHasStoreToFile)
    {
        parseFileByLine(tempFile, loadData.getCharset(), loadData.getLineTerminatedBy());
    } else
    {
        String content = new String(tempByteBuffer.toByteArray(), Charset.forName(loadData.getCharset()));

        // List<String> lines = Splitter.on(loadData.getLineTerminatedBy()).omitEmptyStrings().splitToList(content);
        CsvParserSettings settings = new CsvParserSettings();
        settings.setMaxColumns(65535);
        settings.setMaxCharsPerColumn(65535);
        settings.getFormat().setLineSeparator(loadData.getLineTerminatedBy());
        settings.getFormat().setDelimiter(loadData.getFieldTerminatedBy().charAt(0));
        if(loadData.getEnclose()!=null)
        {
            settings.getFormat().setQuote(loadData.getEnclose().charAt(0));
        }
        if(loadData.getEscape()!=null)
        {
        settings.getFormat().setQuoteEscape(loadData.getEscape().charAt(0));
        }
        settings.getFormat().setNormalizedNewline(loadData.getLineTerminatedBy().charAt(0));
        /*
         *  fix bug #1074 : LOAD DATA local INFILE导入的所有Boolean类型全部变成了false
         *  不可见字符将在CsvParser被当成whitespace过滤掉, 使用settings.trimValues(false)来避免被过滤掉
         *  TODO : 设置trimValues(false)之后, 会引起字段值前后的空白字符无法被过滤!
         */
        settings.trimValues(false);
        CsvParser parser = new CsvParser(settings);
        try
        {
            parser.beginParsing(new StringReader(content));
            String[] row = null;

            while ((row = parser.parseNext()) != null)
            {
                parseOneLine(columns, tableName, row, false, null);
            }
        } finally
        {
            parser.stopParsing();
        }


    }

    RouteResultset rrs = buildResultSet(routeResultMap);
    if (rrs != null)
    {
        flushDataToFile();
        serverConnection.getSession2().execute(rrs, ServerParse.LOAD_DATA_INFILE_SQL);
    }


    // sendOk(++packID);


}
 

import com.univocity.parsers.csv.CsvParser; //导入方法依赖的package包/类
@Override
public void end(byte packId) {
    isStartLoadData = false;
    this.packID = packId;
    //empty packet for end
    saveByteOrToFile(null, true);

    List<SQLExpr> columns = statement.getColumns();
    String tableSimpleName = statement.getTableName().getSimpleName();
    if (isHasStoreToFile) {
        parseFileByLine(tempFile, loadData.getCharset(), loadData.getLineTerminatedBy());
    } else {
        String content = new String(tempByteBuffer.toByteArray(), Charset.forName(loadData.getCharset()));

        // List<String> lines = Splitter.on(loadData.getLineTerminatedBy()).omitEmptyStrings().splitToList(content);
        CsvParserSettings settings = new CsvParserSettings();
        settings.setMaxColumns(65535);
        settings.setMaxCharsPerColumn(65535);
        settings.getFormat().setLineSeparator(loadData.getLineTerminatedBy());
        settings.getFormat().setDelimiter(loadData.getFieldTerminatedBy().charAt(0));
        if (loadData.getEnclose() != null) {
            settings.getFormat().setQuote(loadData.getEnclose().charAt(0));
        }
        if (loadData.getEscape() != null) {
            settings.getFormat().setQuoteEscape(loadData.getEscape().charAt(0));
        }
        settings.getFormat().setNormalizedNewline(loadData.getLineTerminatedBy().charAt(0));
        /*
         *  fix bug #1074 : LOAD DATA local INFILE导入的所有Boolean类型全部变成了false
         *  不可见字符将在CsvParser被当成whitespace过滤掉, 使用settings.trimValues(false)来避免被过滤掉
         *  FIXME : 设置trimValues(false)之后, 会引起字段值前后的空白字符无法被过滤!
         */
        settings.trimValues(false);

        CsvParser parser = new CsvParser(settings);
        try {
            parser.beginParsing(new StringReader(content));
            String[] row = null;

            int ignoreNumber = 0;
            if (statement.getIgnoreLinesNumber() != null && !"".equals(statement.getIgnoreLinesNumber().toString())) {
                ignoreNumber = Integer.parseInt(statement.getIgnoreLinesNumber().toString());
            }
            while ((row = parser.parseNext()) != null) {
                if (ignoreNumber == 0) {
                    parseOneLine(columns, tableSimpleName, row, true, loadData.getLineTerminatedBy());
                } else {
                    ignoreNumber--;
                }
            }
        } finally {
            parser.stopParsing();
        }
    }

    RouteResultset rrs = buildResultSet(routeResultMap);
    if (rrs != null) {
        flushDataToFile();
        serverConnection.getSession2().execute(rrs);
    }

    // sendOk(++packID);
}