Java CsvParser.beginParsing方法代码示例

import com.univocity.parsers.csv.CsvParser; //导入方法依赖的package包/类
/**
 * Read stream of taxi trip data from file
 * and broadcast its value next.
 *
 * @throws InterruptedException
 */
public void readStream() throws InterruptedException {
    // setting up CSV parser
    CsvParser parser = setupParser();

    // read records one by one
    parser.beginParsing(getReader(this.fileName));

    String[] row;
    // read line by line
    while ((row = parser.parseNext()) != null) {
        // create a string from array separated by comma
        String trip = String.join(",", row);

        // sink values to trips broadcaster
        this.tripsProcessor.onNext(trip);
    }
    // close the channel as we are finished streaming data
    // this sends a complete signal which we can in turn observe
    this.tripsProcessor.onComplete();

    // finished parsing all the data from the csv file
    parser.stopParsing();
}
 

import com.univocity.parsers.csv.CsvParser; //导入方法依赖的package包/类
private void processSinglePathWithByteBuffer(BatchWriter writer, FileMapping mapping, Path p, CsvParser parser) throws IOException, MutationsRejectedException {
  final FileSystem fs = p.getFileSystem(conf);
  FSDataInputStream dis = fs.open(p, INPUT_BUFFER_SIZE);
  InputStreamReader reader = new InputStreamReader(dis, UTF_8);
  try {
    parser.beginParsing(reader);
    String[] line = null;
    while ((line = parser.parseNext()) != null) {
      writer.addMutation(parseLine(mapping, line));
    }
  } finally {
    if (null != reader) {
      reader.close();
    }
  }
}
 

import com.univocity.parsers.csv.CsvParser; //导入方法依赖的package包/类
/**
 * Below method will be used to initialize the the parser
 *
 * @throws IOException
 */
public void initialize() throws IOException {
  CsvParserSettings parserSettings = new CsvParserSettings();
  parserSettings.getFormat().setDelimiter(csvParserVo.getDelimiter().charAt(0));
  parserSettings.getFormat().setComment(csvParserVo.getCommentCharacter().charAt(0));
  parserSettings.setLineSeparatorDetectionEnabled(true);
  parserSettings.setMaxColumns(
      getMaxColumnsForParsing(csvParserVo.getNumberOfColumns(), csvParserVo.getMaxColumns()));
  parserSettings.setNullValue("");
  parserSettings.setIgnoreLeadingWhitespaces(false);
  parserSettings.setIgnoreTrailingWhitespaces(false);
  parserSettings.setSkipEmptyLines(false);
  parserSettings.getFormat().setQuote(null == csvParserVo.getQuoteCharacter() ?
      '\"':csvParserVo.getQuoteCharacter().charAt(0));
  parserSettings.getFormat().setQuoteEscape(null == csvParserVo.getEscapeCharacter() ?
      '\\' :
      csvParserVo.getEscapeCharacter().charAt(0));
  blockCounter++;
  initializeReader();
  if (csvParserVo.getBlockDetailsList().get(blockCounter).getBlockOffset() == 0) {
    parserSettings.setHeaderExtractionEnabled(csvParserVo.isHeaderPresent());
  }
  parser = new CsvParser(parserSettings);
  parser.beginParsing(inputStreamReader);
}
 

import com.univocity.parsers.csv.CsvParser; //导入方法依赖的package包/类
@Override
public DataFrame load() throws Exception {
    String response;
    if (usePost) {
        response = postRequest();
    } else {
        response = getRequest();
    }

    CsvParserSettings settings = new CsvParserSettings();
    CsvParser csvParser = new CsvParser(settings);
    InputStream targetStream = new ByteArrayInputStream(response.getBytes(StandardCharsets.UTF_8.name()));
    InputStreamReader targetReader = new InputStreamReader(targetStream, "UTF-8");
    csvParser.beginParsing(targetReader);
    CSVDataFrameParser dfParser = new CSVDataFrameParser(csvParser, requiredColumns);
    dfParser.setColumnTypes(types);
    return dfParser.load();
}
 

import com.univocity.parsers.csv.CsvParser; //导入方法依赖的package包/类
@Test(dataProvider = "inputProvider")
public void testInternal(String input) {
    CsvParserSettings settings = new CsvParserSettings();
    settings.getFormat().setLineSeparator("\n");
    settings.setHeaderExtractionEnabled(true);

    CsvParser csvParser = new CsvParser(settings);

    csvParser.beginParsing(new StringReader(input));
    Record record = csvParser.parseNextRecord();
    RecordMetaData metaData = record.getMetaData();

    assertNotNull(metaData.headers());
    assertTrue(metaData.containsColumn("col1"));
    assertTrue(metaData.containsColumn("col2"));
}
 

import com.univocity.parsers.csv.CsvParser; //导入方法依赖的package包/类
@Before
public void init() {
	parser = new CsvParser(settings);
	parser.beginParsing(new StringReader(CSV));
	rowContext = PowerMockito.spy(new UnivocityParserRowContext(parser.getContext(), metadata.size()));
	parser.parseNext();
}
 

import com.univocity.parsers.csv.CsvParser; //导入方法依赖的package包/类
public CSVDataFrameParser(String fileName, List<String> requiredColumns) throws IOException {
    this.requiredColumns = requiredColumns;
    CsvParserSettings settings = new CsvParserSettings();
    settings.getFormat().setLineSeparator("\n");
    CsvParser csvParser = new CsvParser(settings);
    csvParser.beginParsing(getReader(fileName));
    this.parser = csvParser;
}
 

import com.univocity.parsers.csv.CsvParser; //导入方法依赖的package包/类
@Override
public void end(byte packID)
{
    isStartLoadData = false;
    this.packID = packID;
    //load in data空包 结束
    saveByteOrToFile(null, true);
    List<SQLExpr> columns = statement.getColumns();
    String tableName = statement.getTableName().getSimpleName();
    if (isHasStoreToFile)
    {
        parseFileByLine(tempFile, loadData.getCharset(), loadData.getLineTerminatedBy());
    } else
    {
        String content = new String(tempByteBuffer.toByteArray(), Charset.forName(loadData.getCharset()));

        // List<String> lines = Splitter.on(loadData.getLineTerminatedBy()).omitEmptyStrings().splitToList(content);
        CsvParserSettings settings = new CsvParserSettings();
        settings.setMaxColumns(65535);
        settings.setMaxCharsPerColumn(65535);
        settings.getFormat().setLineSeparator(loadData.getLineTerminatedBy());
        settings.getFormat().setDelimiter(loadData.getFieldTerminatedBy().charAt(0));
        if(loadData.getEnclose()!=null)
        {
            settings.getFormat().setQuote(loadData.getEnclose().charAt(0));
        }
        if(loadData.getEscape()!=null)
        {
        settings.getFormat().setQuoteEscape(loadData.getEscape().charAt(0));
        }
        settings.getFormat().setNormalizedNewline(loadData.getLineTerminatedBy().charAt(0));
        /*
         *  fix bug #1074 : LOAD DATA local INFILE导入的所有Boolean类型全部变成了false
         *  不可见字符将在CsvParser被当成whitespace过滤掉, 使用settings.trimValues(false)来避免被过滤掉
         *  TODO : 设置trimValues(false)之后, 会引起字段值前后的空白字符无法被过滤!
         */
        settings.trimValues(false);
        CsvParser parser = new CsvParser(settings);
        try
        {
            parser.beginParsing(new StringReader(content));
            String[] row = null;

            while ((row = parser.parseNext()) != null)
            {
                parseOneLine(columns, tableName, row, false, null);
            }
        } finally
        {
            parser.stopParsing();
        }


    }

    RouteResultset rrs = buildResultSet(routeResultMap);
    if (rrs != null)
    {
        flushDataToFile();
        serverConnection.getSession2().execute(rrs, ServerParse.LOAD_DATA_INFILE_SQL);
    }


    // sendOk(++packID);


}
 

import com.univocity.parsers.csv.CsvParser; //导入方法依赖的package包/类
@Override
public void end(byte packId) {
    isStartLoadData = false;
    this.packID = packId;
    //empty packet for end
    saveByteOrToFile(null, true);

    List<SQLExpr> columns = statement.getColumns();
    String tableSimpleName = statement.getTableName().getSimpleName();
    if (isHasStoreToFile) {
        parseFileByLine(tempFile, loadData.getCharset(), loadData.getLineTerminatedBy());
    } else {
        String content = new String(tempByteBuffer.toByteArray(), Charset.forName(loadData.getCharset()));

        // List<String> lines = Splitter.on(loadData.getLineTerminatedBy()).omitEmptyStrings().splitToList(content);
        CsvParserSettings settings = new CsvParserSettings();
        settings.setMaxColumns(65535);
        settings.setMaxCharsPerColumn(65535);
        settings.getFormat().setLineSeparator(loadData.getLineTerminatedBy());
        settings.getFormat().setDelimiter(loadData.getFieldTerminatedBy().charAt(0));
        if (loadData.getEnclose() != null) {
            settings.getFormat().setQuote(loadData.getEnclose().charAt(0));
        }
        if (loadData.getEscape() != null) {
            settings.getFormat().setQuoteEscape(loadData.getEscape().charAt(0));
        }
        settings.getFormat().setNormalizedNewline(loadData.getLineTerminatedBy().charAt(0));
        /*
         *  fix bug #1074 : LOAD DATA local INFILE导入的所有Boolean类型全部变成了false
         *  不可见字符将在CsvParser被当成whitespace过滤掉, 使用settings.trimValues(false)来避免被过滤掉
         *  FIXME : 设置trimValues(false)之后, 会引起字段值前后的空白字符无法被过滤!
         */
        settings.trimValues(false);

        CsvParser parser = new CsvParser(settings);
        try {
            parser.beginParsing(new StringReader(content));
            String[] row = null;

            int ignoreNumber = 0;
            if (statement.getIgnoreLinesNumber() != null && !"".equals(statement.getIgnoreLinesNumber().toString())) {
                ignoreNumber = Integer.parseInt(statement.getIgnoreLinesNumber().toString());
            }
            while ((row = parser.parseNext()) != null) {
                if (ignoreNumber == 0) {
                    parseOneLine(columns, tableSimpleName, row, true, loadData.getLineTerminatedBy());
                } else {
                    ignoreNumber--;
                }
            }
        } finally {
            parser.stopParsing();
        }
    }

    RouteResultset rrs = buildResultSet(routeResultMap);
    if (rrs != null) {
        flushDataToFile();
        serverConnection.getSession2().execute(rrs);
    }

    // sendOk(++packID);
}
 

import com.univocity.parsers.csv.CsvParser; //导入方法依赖的package包/类
@Test
public void example003RecordToMap() throws Exception {
	CsvParserSettings settings = new CsvParserSettings();
	//the file used in the example uses '\n' as the line separator sequence.
	//the line separator sequence is defined here to ensure systems such as MacOS and Windows
	//are able to process this file correctly (MacOS uses '\r'; and Windows uses '\r\n').
	settings.getFormat().setLineSeparator("\n");

	settings.setHeaderExtractionEnabled(true);

	CsvParser parser = new CsvParser(settings);

	//##CODE_START
	// this time, lest's parse on demand.
	parser.beginParsing(getReader("/examples/example.csv"));

	//null year should become 0000
	parser.getRecordMetadata().setDefaultValueOfColumns("0000", "Year");

	//decimal separator in prices will be replaced by comma
	parser.getRecordMetadata().convertFields(Conversions.replace("\\.00", ",00")).set("Price");

	//make will be uppercase
	parser.getRecordMetadata().convertFields(Conversions.toUpperCase()).set("make");

	//let's fill a map with values.
	LinkedHashMap<String, Object> values = new LinkedHashMap<String, Object>();

	//create instances of Record on demand
	Record record;
	while ((record = parser.parseNextRecord()) != null) {
		//we can get the original values of selected columns (by name or index) in a map.
		//Map<String, String> originalValues = record.toFieldMap();

		//to get the converted values as specified in the record metadata use the method ending with "ObjectMap"
		//Map<String, Object> convertedValues = record.toFieldObjectMap();

		//all map methods allow you to choose which columns to get data from. Here we select just the (originally parsed) year:
		Map<String, String> originalYearValues = record.toFieldMap("YEAR"); //the character case of the selection is kept regardless of what the headers contain.
		println(originalYearValues); //original values, no conversions applied.

		//instead of creating new maps every time, we can also reuse maps and invoke the "fill" methods
		record.fillFieldObjectMap(values);
		println(values);

	}

	//##CODE_END
	printAndValidate();
}
 

import com.univocity.parsers.csv.CsvParser; //导入方法依赖的package包/类
/**
 * This method generate a parser that starts an iterator-style parsing cycle that
 * does not rely in a {@link com.univocity.parsers.common.processor.RowProcessor}.
 * <p/>
 * To initialize a {@code CsvParser} with a default {@link com.univocity.parsers.common.processor.RowProcessor}
 * use {@link il.ac.technion.ie.experiments.parsers.DatasetParser#getParserForFile(String, com.univocity.parsers.csv.CsvParserSettings)} set the
 * parser in the {@link CsvParserSettings}
 *
 * @param pathToFile Full path in File System to a CSV file that contains records
 * @return an instance of {@link com.univocity.parsers.csv.CsvParser}
 */
public CsvParser getParserForFile(String pathToFile) {
    // The settings object provides many configuration options
    CsvParserSettings parserSettings = new CsvParserSettings();
    parserSettings.setLineSeparatorDetectionEnabled(true);
    CsvParser parser = createParser(parserSettings);
    // the 'parse' method will parse the file and delegate each parsed row to the RowProcessor you defined
    parser.beginParsing(getReader(pathToFile));

    return parser;
}