Java CsvParser类代码示例

import com.univocity.parsers.csv.CsvParser; //导入依赖的package包/类
/**
 * Read stream of taxi trip data from file
 * and broadcast its value next.
 *
 * @throws InterruptedException
 */
public void readStream() throws InterruptedException {
    // setting up CSV parser
    CsvParser parser = setupParser();

    // read records one by one
    parser.beginParsing(getReader(this.fileName));

    String[] row;
    // read line by line
    while ((row = parser.parseNext()) != null) {
        // create a string from array separated by comma
        String trip = String.join(",", row);

        // sink values to trips broadcaster
        this.tripsProcessor.onNext(trip);
    }
    // close the channel as we are finished streaming data
    // this sends a complete signal which we can in turn observe
    this.tripsProcessor.onComplete();

    // finished parsing all the data from the csv file
    parser.stopParsing();
}
 

import com.univocity.parsers.csv.CsvParser; //导入依赖的package包/类
@Test
public void everyGenderFemale() {
    String result = generateAnnotatedPersonCsv();
    CsvParser csvParser = new CsvParser(new CsvParserSettings());

    boolean firstRow = true;
    for (String[] row : csvParser.iterate(new StringReader(result))) {
        if (firstRow) {
            firstRow = false;
            continue;
        }

        if (!row[2].equals("Female")) {
            fail("Gender column is not female! Result: " + row[2]);
        }
    }
}
 

import com.univocity.parsers.csv.CsvParser; //导入依赖的package包/类
@Test
public void everyNameInList() {
    String result = generateAnnotatedPersonCsv();
    CsvParser csvParser = new CsvParser(new CsvParserSettings());

    List<String> names = AnnotatedPerson.NameGenerator.getNames();
    boolean firstRow = true;
    for (String[] row : csvParser.iterate(new StringReader(result))) {
        if (firstRow) {
            firstRow = false;
            continue;
        }

        if (!names.contains(row[0])) {
            fail("Name column value is not in list! Result: '" + row[0] + " 'List: " + names);
        }
    }
}
 

import com.univocity.parsers.csv.CsvParser; //导入依赖的package包/类
@Test
public void agesInRange() {
    String result = generateAnnotatedPersonCsv();
    CsvParser csvParser = new CsvParser(new CsvParserSettings());

    boolean firstRow = true;
    for (String[] row : csvParser.iterate(new StringReader(result))) {
        if (firstRow) {
            firstRow = false;
            continue;
        }

        int age = Integer.parseInt(row[1]);
        assertTrue(age + "",age <= maxAge);
        assertTrue(age + "",age >= minAge);

    }
}
 

import com.univocity.parsers.csv.CsvParser; //导入依赖的package包/类
private void processSinglePathWithByteBuffer(BatchWriter writer, FileMapping mapping, Path p, CsvParser parser) throws IOException, MutationsRejectedException {
  final FileSystem fs = p.getFileSystem(conf);
  FSDataInputStream dis = fs.open(p, INPUT_BUFFER_SIZE);
  InputStreamReader reader = new InputStreamReader(dis, UTF_8);
  try {
    parser.beginParsing(reader);
    String[] line = null;
    while ((line = parser.parseNext()) != null) {
      writer.addMutation(parseLine(mapping, line));
    }
  } finally {
    if (null != reader) {
      reader.close();
    }
  }
}
 

import com.univocity.parsers.csv.CsvParser; //导入依赖的package包/类
@Override
public void execute(IngestDocument ingestDocument) throws Exception {
    String content = ingestDocument.getFieldValue(field, String.class);

    if (Strings.hasLength(content)) {
        CsvParser parser = new CsvParser(this.csvSettings);
        String[] values = parser.parseLine(content);
        if (values.length != this.columns.size()) {
            // TODO should be error?
            throw new IllegalArgumentException("field[" + this.field + "] size ["
                + values.length + "] doesn't match header size [" + columns.size() + "].");
        }

        for (int i = 0; i < columns.size(); i++) {
            ingestDocument.setFieldValue(columns.get(i), values[i]);
        }
    } else {
        // TODO should we have ignoreMissing flag?
        throw new IllegalArgumentException("field[" + this.field + "] is empty string.");
    }

}
 

import com.univocity.parsers.csv.CsvParser; //导入依赖的package包/类
/**
 * Returns a DataFrame parsed from the stream specified stream
 * @param stream    the stream to parse
 * @return          the DataFrame parsed from stream
 * @throws IOException      if there stream read error
 */
private DataFrame<R,String> parse(CsvSourceOptions<R> options, InputStream stream) throws IOException {
    try (BufferedReader reader = new BufferedReader(new InputStreamReader(stream, options.getCharset().orElse(StandardCharsets.UTF_8)))) {
        final CsvRequestHandler handler = new CsvRequestHandler(options);
        final CsvParserSettings settings = new CsvParserSettings();
        settings.getFormat().setDelimiter(options.getDelimiter());
        settings.setHeaderExtractionEnabled(options.isHeader());
        settings.setLineSeparatorDetectionEnabled(true);
        settings.setRowProcessor(handler);
        settings.setIgnoreTrailingWhitespaces(true);
        settings.setIgnoreLeadingWhitespaces(true);
        settings.setMaxColumns(10000);
        settings.setReadInputOnSeparateThread(false);
        final CsvParser parser = new CsvParser(settings);
        parser.parse(reader);
        return handler.getFrame();
    }
}
 

import com.univocity.parsers.csv.CsvParser; //导入依赖的package包/类
private void loadZones(ZipInputStream zipis) throws UnsupportedEncodingException{
	CsvParserSettings settings = new CsvParserSettings();
	settings.setSkipEmptyLines(true);
	settings.setNumberOfRowsToSkip(1);
	settings.trimValues(true);
	CsvFormat format = new CsvFormat();
	format.setDelimiter(',');
	format.setLineSeparator("\n");
	format.setCharToEscapeQuoteEscaping('\0');
	format.setQuote('"');
	settings.setFormat(format);
	CsvParser parser = new CsvParser(settings);
	
	Map<String, TimeZone> idMapRead = new HashMap<>();
	List<String[]> lines = parser.parseAll(new InputStreamReader(zipis, "UTF-8"));
	
	for(String[] entry : lines){
		TimeZone timeZone = new TimeZone();
		timeZone.setId(Integer.parseInt(entry[0]));
		timeZone.setCountryIso(entry[1]);
		timeZone.setName(entry[2]);
		idMapRead.put(timeZone.getName().trim().toUpperCase(), timeZone);
	}
	this.idMap = idMapRead;
	logger.info("loaded "+idMapRead.size()+" time zones");
}
 

import com.univocity.parsers.csv.CsvParser; //导入依赖的package包/类
/**
 * Below method will be used to initialize the the parser
 *
 * @throws IOException
 */
public void initialize() throws IOException {
  CsvParserSettings parserSettings = new CsvParserSettings();
  parserSettings.getFormat().setDelimiter(csvParserVo.getDelimiter().charAt(0));
  parserSettings.getFormat().setComment(csvParserVo.getCommentCharacter().charAt(0));
  parserSettings.setLineSeparatorDetectionEnabled(true);
  parserSettings.setMaxColumns(
      getMaxColumnsForParsing(csvParserVo.getNumberOfColumns(), csvParserVo.getMaxColumns()));
  parserSettings.setNullValue("");
  parserSettings.setIgnoreLeadingWhitespaces(false);
  parserSettings.setIgnoreTrailingWhitespaces(false);
  parserSettings.setSkipEmptyLines(false);
  parserSettings.getFormat().setQuote(null == csvParserVo.getQuoteCharacter() ?
      '\"':csvParserVo.getQuoteCharacter().charAt(0));
  parserSettings.getFormat().setQuoteEscape(null == csvParserVo.getEscapeCharacter() ?
      '\\' :
      csvParserVo.getEscapeCharacter().charAt(0));
  blockCounter++;
  initializeReader();
  if (csvParserVo.getBlockDetailsList().get(blockCounter).getBlockOffset() == 0) {
    parserSettings.setHeaderExtractionEnabled(csvParserVo.isHeaderPresent());
  }
  parser = new CsvParser(parserSettings);
  parser.beginParsing(inputStreamReader);
}
 

import com.univocity.parsers.csv.CsvParser; //导入依赖的package包/类
public ListParser(Parser inParser, char inCollectionDelim, 
                  char inCollectionBegin, char inCollectionEnd) {
    parser = inParser;
    collectionDelim = inCollectionDelim;
    collectionBegin = inCollectionBegin;
    collectionEnd = inCollectionEnd;
    elements = new ArrayList<Object>();

    CsvParserSettings settings = new CsvParserSettings();
    settings.getFormat().setLineSeparator("\n");
    settings.getFormat().setDelimiter(collectionDelim);
    settings.getFormat().setQuote(collectionQuote);
    settings.getFormat().setQuoteEscape(collectionEscape);
    settings.getFormat().setCharToEscapeQuoteEscaping(collectionEscape);
    settings.setKeepQuotes(true);
    settings.setKeepEscapeSequences(true);
    
    csvp = new CsvParser(settings);
}
 

import com.univocity.parsers.csv.CsvParser; //导入依赖的package包/类
public MapParser(Parser inKeyParser, Parser inValueParser,
                 char inCollectionDelim, char inCollectionBegin, 
                 char inCollectionEnd, char inMapDelim) {
    keyParser = inKeyParser;
    valueParser = inValueParser;
    collectionDelim = inCollectionDelim;
    collectionBegin = inCollectionBegin;
    collectionEnd = inCollectionEnd;
    mapDelim = inMapDelim;
    elements = new HashMap<Object,Object>();

    CsvParserSettings settings = new CsvParserSettings();
    settings.getFormat().setLineSeparator("" + collectionDelim);
    settings.getFormat().setNormalizedNewline(collectionDelim);
    settings.getFormat().setDelimiter(mapDelim);
    settings.getFormat().setQuote(collectionQuote);
    settings.getFormat().setQuoteEscape(collectionEscape);
    settings.getFormat().setCharToEscapeQuoteEscaping(collectionEscape);
    settings.setKeepQuotes(true);
    settings.setKeepEscapeSequences(true);
    
    csvp = new CsvParser(settings);
}
 

import com.univocity.parsers.csv.CsvParser; //导入依赖的package包/类
public SetParser(Parser inParser, char inCollectionDelim, 
                 char inCollectionBegin, char inCollectionEnd) {
    parser = inParser;
    collectionDelim = inCollectionDelim;
    collectionBegin = inCollectionBegin;
    collectionEnd = inCollectionEnd;
    elements = new HashSet<Object>();

    CsvParserSettings settings = new CsvParserSettings();
    settings.getFormat().setLineSeparator("\n");
    settings.getFormat().setDelimiter(collectionDelim);
    settings.getFormat().setQuote(collectionQuote);
    settings.getFormat().setQuoteEscape(collectionEscape);
    settings.getFormat().setCharToEscapeQuoteEscaping(collectionEscape);
    settings.setKeepQuotes(true);
    settings.setKeepEscapeSequences(true);
    
    csvp = new CsvParser(settings);
}
 

import com.univocity.parsers.csv.CsvParser; //导入依赖的package包/类
private void processCqlSchema(String cqlSchema, Session session) throws ParseException {
    CsvParserSettings ks_settings = new CsvParserSettings();
    ks_settings.getFormat().setLineSeparator("\n");
    ks_settings.getFormat().setDelimiter('.');
    ks_settings.getFormat().setQuote('\"');
    ks_settings.getFormat().setQuoteEscape('\\');
    ks_settings.getFormat().setCharToEscapeQuoteEscaping('\\');
    ks_settings.setKeepQuotes(true);
    ks_settings.setKeepEscapeSequences(true);
    ks_settings.setMaxCharsPerColumn(-1);
    CsvParser ks_parser = new CsvParser(ks_settings);
    String[] ks_elements = ks_parser.parseLine(cqlSchema);
    keyspace = ks_elements[0];
    String table_string = cqlSchema.substring(keyspace.length() + 1);

    CsvParserSettings table_settings = new CsvParserSettings();
    table_settings.getFormat().setLineSeparator("\n");
    table_settings.getFormat().setDelimiter('(');
    table_settings.getFormat().setQuote('\"');
    table_settings.getFormat().setQuoteEscape('\\');
    table_settings.getFormat().setCharToEscapeQuoteEscaping('\\');
    table_settings.setKeepQuotes(true);
    table_settings.setKeepEscapeSequences(true);
    table_settings.setMaxCharsPerColumn(-1);
    CsvParser table_parser = new CsvParser(table_settings);
    String[] table_elements = table_parser.parseLine(table_string);
    tablename = table_elements[0];
    
    String schemaString = table_string.substring(tablename.length() + 1, 
                                                 table_string.length() - 1);

    sbl = schemaBits(schemaString, session);
}
 

import com.univocity.parsers.csv.CsvParser; //导入依赖的package包/类
@Override
public DataFrame load() throws Exception {
    String response;
    if (usePost) {
        response = postRequest();
    } else {
        response = getRequest();
    }

    CsvParserSettings settings = new CsvParserSettings();
    CsvParser csvParser = new CsvParser(settings);
    InputStream targetStream = new ByteArrayInputStream(response.getBytes(StandardCharsets.UTF_8.name()));
    InputStreamReader targetReader = new InputStreamReader(targetStream, "UTF-8");
    csvParser.beginParsing(targetReader);
    CSVDataFrameParser dfParser = new CSVDataFrameParser(csvParser, requiredColumns);
    dfParser.setColumnTypes(types);
    return dfParser.load();
}
 

import com.univocity.parsers.csv.CsvParser; //导入依赖的package包/类
@Test
public void example002RecordMetadata() throws Exception {

	CsvParserSettings settings = new CsvParserSettings();
	//the file used in the example uses '\n' as the line separator sequence.
	//the line separator sequence is defined here to ensure systems such as MacOS and Windows
	//are able to process this file correctly (MacOS uses '\r'; and Windows uses '\r\n').
	settings.getFormat().setLineSeparator("\n");

	settings.setHeaderExtractionEnabled(true);

	CsvParser parser = new CsvParser(settings);

	// parses all records in one go
	List<Record> allRecords = parser.parseAllRecords(getReader("/examples/example.csv"));

	//##CODE_START
	parser.getRecordMetadata().setTypeOfColumns(Long.class, "year", "price");
	parser.getRecordMetadata().setDefaultValueOfColumns("0", "Year");
	parser.getRecordMetadata().convertFields(Conversions.replace("\\.00", "")).set("Price");

	for (Record record : allRecords) {
		Long year = record.getLong("year");
		String model = record.getString("MODEL");
		Long price = record.getLong("Price");
		println(year + " " + model + ": $" + price);
	}

	//##CODE_END
	printAndValidate();
}