Java StrTokenizer.getTokenArray方法代码示例

import org.apache.commons.lang3.text.StrTokenizer; //导入方法依赖的package包/类
private long test(StrTokenizer tokenizer, File source) throws IOException {
  FileInputStream fis = new FileInputStream(source);
  InputStreamReader reader = new InputStreamReader(fis, "utf8");
  BufferedReader br = new BufferedReader(reader);

  // keep track of time while iterating
  long start = System.currentTimeMillis();
  String row = br.readLine();
  while (row != null) {
    tokenizer.reset(row);
    String[] columns = tokenizer.getTokenArray();
    row = br.readLine();
  }
  long dur = System.currentTimeMillis() - start;
  br.close();
  return dur;
}
 

import org.apache.commons.lang3.text.StrTokenizer; //导入方法依赖的package包/类
@Test
public void testCsvUnquoted() throws IOException {
  StrTokenizer tokenizer = new StrTokenizer();
  tokenizer.setDelimiterString(",");
  tokenizer.setEmptyTokenAsNull(true);
  tokenizer.setIgnoreEmptyTokens(false);

  tokenizer.reset("121,432423, 9099053,Frieda karla L.,DC.,Ahrens");
  String[] columns = tokenizer.getTokenArray();
  assertEquals("121", columns[0]);
  assertEquals("432423", columns[1]);
  assertEquals(" 9099053", columns[2]);
  assertEquals("Frieda karla L.", columns[3]);
  assertEquals("DC.", columns[4]);
  assertEquals("Ahrens", columns[5]);

  tokenizer.reset(",,,,zzz  ");
  columns = tokenizer.getTokenArray();
  assertNull(columns[0]);
  assertNull(columns[1]);
  assertNull(columns[2]);
  assertNull(columns[3]);
  assertEquals("zzz  ", columns[4]);
}
 

import org.apache.commons.lang3.text.StrTokenizer; //导入方法依赖的package包/类
static String[] _tokenizeString(String string) {
	final StrTokenizer _tokenizer = new StrTokenizer().
			setDelimiterMatcher(StrMatcher.trimMatcher()).
			setQuoteMatcher(StrMatcher.quoteMatcher()).
			setTrimmerMatcher(StrMatcher.trimMatcher()).
			setIgnoredMatcher(StrMatcher.quoteMatcher());
	_tokenizer.reset(string.toLowerCase());
	return _tokenizer.getTokenArray();
}
 

import org.apache.commons.lang3.text.StrTokenizer; //导入方法依赖的package包/类
public void parseKeyValue(String line) {
    int keySeparatorIndex = line.indexOf(keySeparator);
    String key;
    String valueString;
    if (keySeparatorIndex < 0) {
        if (keySeparatorOptional) {
            key = line.trim();
            valueString = "";
        } else {
            return;
        }
    } else {
        key = line.substring(0, keySeparatorIndex).trim();
        valueString = line.substring(
                keySeparatorIndex + keySeparator.length()
        ).trim();
    }

    String[] values;
    if (separator == null) {
        values = new String[]{valueString};
    } else {
        StrTokenizer tokenizer = createStrTokenizer(valueString);
        values = tokenizer.getTokenArray();
    }

    String[] result = new String[values.length + 1];
    result[0] = key;
    System.arraycopy(values, 0, result, 1, values.length);

    storeLine(result);
}
 

import org.apache.commons.lang3.text.StrTokenizer; //导入方法依赖的package包/类
@Test
public void testCsvQuoted() throws IOException {
  StrTokenizer tokenizer = new StrTokenizer();
  tokenizer.setDelimiterString(",");
  tokenizer.setQuoteChar('"');
  tokenizer.setEmptyTokenAsNull(true);
  tokenizer.setIgnoreEmptyTokens(false);

  tokenizer.reset("121,432423, 9099053,\"Frieda karla L.,DC.\",Ahrens");
  String[] columns = tokenizer.getTokenArray();
  assertEquals("121", columns[0]);
  assertEquals("432423", columns[1]);
  assertEquals(" 9099053", columns[2]);
  assertEquals("Frieda karla L.,DC.", columns[3]);
  assertEquals("Ahrens", columns[4]);

  tokenizer.reset("   ,4321");
  columns = tokenizer.getTokenArray();
  assertEquals("   ", columns[0]);
  assertEquals("4321", columns[1]);

  tokenizer.reset(" ,,,,zzz  ");
  columns = tokenizer.getTokenArray();
  assertEquals(" ", columns[0]);
  assertNull(columns[1]);
  assertNull(columns[2]);
  assertNull(columns[3]);
  assertEquals("zzz  ", columns[4]);

  tokenizer.reset(",,,,zzz  ");
  columns = tokenizer.getTokenArray();
  assertNull(columns[0]);
  assertNull(columns[1]);
  assertNull(columns[2]);
  assertNull(columns[3]);
  assertEquals("zzz  ", columns[4]);
}
 

import org.apache.commons.lang3.text.StrTokenizer; //导入方法依赖的package包/类
@Test
public void testPipes() throws IOException {
  StrTokenizer tokenizer = new StrTokenizer();
  tokenizer.setDelimiterChar('|');
  tokenizer.setQuoteChar('"');
  tokenizer.setEmptyTokenAsNull(true);
  tokenizer.setIgnoreEmptyTokens(false);

  tokenizer.reset("121|432423| 9099053|\"Frieda karla L.|DC.\"|Ahrens");
  String[] columns = tokenizer.getTokenArray();
  assertEquals("121", columns[0]);
  assertEquals("432423", columns[1]);
  assertEquals(" 9099053", columns[2]);
  assertEquals("Frieda karla L.|DC.", columns[3]);
  assertEquals("Ahrens", columns[4]);

  tokenizer.reset("   |4321");
  columns = tokenizer.getTokenArray();
  assertEquals("   ", columns[0]);
  assertEquals("4321", columns[1]);

  tokenizer.reset(" ||||zzz  ");
  columns = tokenizer.getTokenArray();
  assertEquals(" ", columns[0]);
  assertNull(columns[1]);
  assertNull(columns[2]);
  assertNull(columns[3]);
  assertEquals("zzz  ", columns[4]);

  tokenizer.reset("||||zzz  ");
  columns = tokenizer.getTokenArray();
  assertNull(columns[0]);
  assertNull(columns[1]);
  assertNull(columns[2]);
  assertNull(columns[3]);
  assertEquals("zzz  ", columns[4]);
}
 

import org.apache.commons.lang3.text.StrTokenizer; //导入方法依赖的package包/类
@Test
public void testTabQuoted() throws IOException {
  StrTokenizer tokenizer = new StrTokenizer();
  tokenizer.setDelimiterString("\t");
  tokenizer.setQuoteChar('"');
  tokenizer.setEmptyTokenAsNull(true);
  tokenizer.setIgnoreEmptyTokens(false);

  tokenizer.reset("121\t432423\t 9099053\t\"Frieda karla L.,DC.\"\tAhrens");
  String[] columns = tokenizer.getTokenArray();
  assertEquals("121", columns[0]);
  assertEquals("432423", columns[1]);
  assertEquals(" 9099053", columns[2]);
  assertEquals("Frieda karla L.,DC.", columns[3]);
  assertEquals("Ahrens", columns[4]);

  tokenizer.reset("   \t4321");
  columns = tokenizer.getTokenArray();
  assertEquals("   ", columns[0]);
  assertEquals("4321", columns[1]);

  tokenizer.reset(" \t\t\t\tzzz  ");
  columns = tokenizer.getTokenArray();
  assertEquals(" ", columns[0]);
  assertNull(columns[1]);
  assertNull(columns[2]);
  assertNull(columns[3]);
  assertEquals("zzz  ", columns[4]);

  tokenizer.reset("\t\t\t\tzzz  ");
  columns = tokenizer.getTokenArray();
  assertNull(columns[0]);
  assertNull(columns[1]);
  assertNull(columns[2]);
  assertNull(columns[3]);
  assertEquals("zzz  ", columns[4]);
}
 

import org.apache.commons.lang3.text.StrTokenizer; //导入方法依赖的package包/类
@Test
public void testTabUnquoted() throws IOException {
  StrTokenizer tokenizer = new StrTokenizer();
  tokenizer.setDelimiterString("\t");
  tokenizer.setEmptyTokenAsNull(true);
  tokenizer.setIgnoreEmptyTokens(false);

  tokenizer.reset("121\t432423\t 9099053\t\"Frieda karla L.,DC.\"\tAhrens");
  String[] columns = tokenizer.getTokenArray();
  assertEquals("121", columns[0]);
  assertEquals("432423", columns[1]);
  assertEquals(" 9099053", columns[2]);
  assertEquals("\"Frieda karla L.,DC.\"", columns[3]);
  assertEquals("Ahrens", columns[4]);

  tokenizer.reset("   \t4321");
  columns = tokenizer.getTokenArray();
  assertEquals("   ", columns[0]);
  assertEquals("4321", columns[1]);

  tokenizer.reset(" \t\t\t\tzzz  ");
  columns = tokenizer.getTokenArray();
  assertEquals(" ", columns[0]);
  assertNull(columns[1]);
  assertNull(columns[2]);
  assertNull(columns[3]);
  assertEquals("zzz  ", columns[4]);

  tokenizer.reset("\t\t\t\tzzz  ");
  columns = tokenizer.getTokenArray();
  assertNull(columns[0]);
  assertNull(columns[1]);
  assertNull(columns[2]);
  assertNull(columns[3]);
  assertEquals("zzz  ", columns[4]);
}
 

import org.apache.commons.lang3.text.StrTokenizer; //导入方法依赖的package包/类
private List<Column> initColumns(final StrTokenizer st, final String headerLine) {
	st.reset(headerLine);

	String[] headers = st.getTokenArray();
	List<Column> columns = newArrayListWithCapacity(headers.length);
	for (String header : headers) {
		columns.add(new Column(header));
	}
	return columns;
}
 

import org.apache.commons.lang3.text.StrTokenizer; //导入方法依赖的package包/类
public static Map<String, String> readAggregatedMap(final File executionsFile, final Charset charset) throws IOException {
	final StrTokenizer tokenizer = StrTokenizer.getCSVInstance();
	tokenizer.setDelimiterChar(';');

	Map<String, String> result = newHashMapWithExpectedSize(11);

	List<String> lines = Files.readLines(executionsFile, charset);
	String[] headers = null;

	for (String line : lines) {
		tokenizer.reset(line);
		String[] tokens = tokenizer.getTokenArray();

		if (headers == null) {
			headers = tokens;
		} else {

			String[] data = tokenizer.getTokenArray();

			String operation = data[0];
			for (int i = 1; i < headers.length; ++i) {
				result.put(operation + "." + headers[i], data[i]);
			}
		}
	}

	return result;
}
 

import org.apache.commons.lang3.text.StrTokenizer; //导入方法依赖的package包/类
private ListMultimap<ProcessKey, LoadProfileEvent> readLoadProfileEvents(final Element testplan) throws IOException {
	ListMultimap<ProcessKey, LoadProfileEvent> eventsByProcess = ArrayListMultimap.create();
	String loadProfile = testplan.elementTextTrim("loadProfile");

	// relative to testplan
	File loadProfileConfigFile = new File(new File(testplanFile.getParentFile(), "loadprofiles"), loadProfile);

	try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(loadProfileConfigFile), "UTF-8"))) {
		StrTokenizer st = StrTokenizer.getCSVInstance();
		st.setDelimiterChar(';');

		for (String line = null; (line = br.readLine()) != null;) {
			// ignore line that are blank, commented out, or represent markers
			if (isBlank(line) || startsWith(line, "#") || MARKER_PATTERN.matcher(line).matches()) {
				continue;
			}

			st.reset(line);
			String[] tokens = st.getTokenArray();

			long startTime = Long.parseLong(tokens[0]);
			String operation = tokens[1];
			String target = tokens[2];
			int daemonId = Integer.parseInt(tokens[3]);
			int processId = Integer.parseInt(tokens[4]);

			eventsByProcess.put(new ProcessKey(daemonId, processId), new LoadProfileEvent(startTime, operation, target,
					daemonId, processId));
		}
	}

	return eventsByProcess;
}
 

import org.apache.commons.lang3.text.StrTokenizer; //导入方法依赖的package包/类
public void parseTokenized(String line) {
    StrTokenizer tokenizer = createStrTokenizer(line);
    String[] tokens = tokenizer.getTokenArray();
    storeLine(tokens);
}
 

import org.apache.commons.lang3.text.StrTokenizer; //导入方法依赖的package包/类
/**
 * Processes the specified CSV file. For every line but the header line (which is required), the
 * specified command is executed.
 * 
 * @param reader
 *            the reader for loading the CSV data
 * @param delimiter
 *            the column separator
 * @param quoteChar
 *            the quote character ('\0' for no quoting)
 * @param command
 *            the command (i. e. a Groovy closure if used in a Groovy script) to be executed for
 *            every processed line
 */
public void processFile(final Reader reader, final String delimiter, final char quoteChar, final Runnable command) {
	try {
		List<String> inputLines = CharStreams.readLines(reader);

		StrTokenizer st = StrTokenizer.getCSVInstance();
		st.setDelimiterString(delimiter);
		if (quoteChar != '\0') {
			st.setQuoteChar(quoteChar);
		} else {
			st.setQuoteMatcher(StrMatcher.noneMatcher());
		}

		// extract header
		String headerLine = inputLines.remove(0);
		List<Column> columns = initColumns(st, headerLine);
		for (String line : inputLines) {
			st.reset(line);
			String[] colArray = st.getTokenArray();
			int len = colArray.length;
			checkState(len == columns.size(), "Mismatch between number of header columns and number of line columns.");

			DataSource dataSource = dataSourceProvider.get();
			Configuration config = configProvider.get();
			for (int i = 0; i < len; ++i) {
				String value = StringUtils.trimToEmpty(colArray[i]);

				String dataSetKey = columns.get(i).dataSetKey;
				String key = columns.get(i).key;
				if (dataSetKey != null) {
					if ("<auto>".equals(value)) {
						dataSource.resetFixedValue(dataSetKey, key);
					} else {
						log.debug("Setting data set entry for " + this + " to value=" + value);
						dataSource.setFixedValue(dataSetKey, key, value);
					}
				} else {
					log.debug("Setting property for " + this + " to value=" + value);
					config.put(key, value);
				}
			}

			command.run();
		}
	} catch (IOException ex) {
		throw new JFunkException("Error processing CSV data", ex);
	}
}
 

import org.apache.commons.lang3.text.StrTokenizer; //导入方法依赖的package包/类
/**
 * Reads a semicolon-delimited CSV file into a list. Each line in the result list will be
 * another list of {@link Number} objects. The file is expected to have two numberic columns
 * which are parsed using the specified number format.
 * 
 * @param file
 *            the file
 * @param charset
 *            the character set to read the file
 * @param numberFormat
 *            the number format for parsing the column values
 * @return the immutable result list
 */
public static List<SeriesPoint> readDataFile(final File file, final Charset charset, final NumberFormat numberFormat)
		throws IOException {
	final StrTokenizer tokenizer = StrTokenizer.getCSVInstance();
	tokenizer.setDelimiterChar(';');

	try (BufferedReader br = newReader(file, charset)) {
		boolean headerLine = true;
		List<SeriesPoint> result = newArrayListWithExpectedSize(200);

		for (String line; (line = br.readLine()) != null;) {
			try {
				if (headerLine) {
					headerLine = false;
				} else {
					tokenizer.reset(line);
					String[] tokens = tokenizer.getTokenArray();
					double x = numberFormat.parse(tokens[0]).doubleValue();
					double y = numberFormat.parse(tokens[1]).doubleValue();

					if (!result.isEmpty()) {
						// additional point for histogram
						SeriesPoint previousPoint = getLast(result);
						result.add(new SeriesPoint(x, previousPoint.getY()));
					}
					tokenizer.reset(line);
					result.add(new SeriesPoint(x, y));
				}
			} catch (ParseException ex) {
				throw new IOException("Error parsing number in file: " + file, ex);
			}
		}

		int size = result.size();
		if (size > 2) {
			// additional point at end for histogram
			SeriesPoint nextToLast = result.get(size - 3);
			SeriesPoint last = result.get(size - 1);
			double dX = last.getX().doubleValue() - nextToLast.getX().doubleValue();
			result.add(new SeriesPoint(last.getX().doubleValue() + dX, last.getY()));
		}
		return ImmutableList.copyOf(result);
	}
}