Java ParsingContext类代码示例

import com.univocity.parsers.common.ParsingContext; //导入依赖的package包/类
@Benchmark
public void mapCsv(Blackhole blackhole, CsvParam csvParam) throws IOException {
    CsvParserSettings settings = new CsvParserSettings();

    //turning off features enabled by default
    settings.setIgnoreLeadingWhitespaces(false);
    settings.setIgnoreTrailingWhitespaces(false);
    settings.setSkipEmptyLines(false);
    settings.setColumnReorderingEnabled(false);
    settings.setReadInputOnSeparateThread(false);

    settings.setRowProcessor(new BeanProcessor<City>(City.class) {
        @Override
        public void beanProcessed(City bean, ParsingContext context) {
            blackhole.consume(bean);
        }
    });

    com.univocity.parsers.csv.CsvParser parser = new com.univocity.parsers.csv.CsvParser(settings);
    try(Reader reader = csvParam.getReader()) {
        parser.parse(reader);
    }
}
 

import com.univocity.parsers.common.ParsingContext; //导入依赖的package包/类
@Benchmark
public void parseCsv(Blackhole blackhole, CsvParam csvParam) throws IOException {
    CsvParserSettings settings = new CsvParserSettings();

    //turning off features enabled by default
    settings.setIgnoreLeadingWhitespaces(false);
    settings.setIgnoreTrailingWhitespaces(false);
    settings.setSkipEmptyLines(false);
    settings.setColumnReorderingEnabled(false);
    settings.setReadInputOnSeparateThread(false);

    settings.setRowProcessor(new AbstractRowProcessor() {
        @Override
        public void rowProcessed(String[] row, ParsingContext context) {
            blackhole.consume(row);
        }
    });

    com.univocity.parsers.csv.CsvParser parser = new com.univocity.parsers.csv.CsvParser(settings);
    try(Reader reader = csvParam.getReader()) {
        parser.parse(reader);
    }
}
 

import com.univocity.parsers.common.ParsingContext; //导入依赖的package包/类
public static void main(String[] args) throws IOException {
    CsvParam csvParam = new CsvParam();
    csvParam.setUp();

    CsvParserSettings settings = new CsvParserSettings();

    //turning off features enabled by default
    settings.setIgnoreLeadingWhitespaces(false);
    settings.setIgnoreTrailingWhitespaces(false);
    settings.setSkipEmptyLines(false);
    settings.setColumnReorderingEnabled(false);
    settings.setReadInputOnSeparateThread(false);

    settings.setProcessor(new BeanProcessor<City>(City.class) {
        @Override
        public void beanProcessed(City bean, ParsingContext context) {
            System.out.println(bean);
        }
    });

    com.univocity.parsers.csv.CsvParser parser = new com.univocity.parsers.csv.CsvParser(settings);
    try(Reader reader = csvParam.getReader()) {
        parser.parse(reader);
    }
}
 

import com.univocity.parsers.common.ParsingContext; //导入依赖的package包/类
@Benchmark
public void parseCsv(Blackhole blackhole) throws IOException {
    CsvParserSettings settings = new CsvParserSettings();

    //turning off features enabled by default
    settings.setIgnoreLeadingWhitespaces(false);
    settings.setIgnoreTrailingWhitespaces(false);
    settings.setSkipEmptyLines(false);
    settings.setColumnReorderingEnabled(false);
    settings.setReadInputOnSeparateThread(true);

    settings.setRowProcessor(new AbstractRowProcessor() {
        @Override
        public void rowProcessed(String[] row, ParsingContext context) {
            blackhole.consume(row);
        }
    });

    com.univocity.parsers.csv.CsvParser parser = new com.univocity.parsers.csv.CsvParser(settings);
    try(Reader reader = CsvParam.getSingleThreadedReader(quotes, nbRows)) {
        parser.parse(reader);
    }
}
 

import com.univocity.parsers.common.ParsingContext; //导入依赖的package包/类
@Benchmark
public void mapCsv(Blackhole blackhole) throws IOException {
    CsvParserSettings settings = new CsvParserSettings();

    //turning off features enabled by default
    settings.setIgnoreLeadingWhitespaces(false);
    settings.setIgnoreTrailingWhitespaces(false);
    settings.setSkipEmptyLines(false);
    settings.setColumnReorderingEnabled(false);

    settings.setRowProcessor(new BeanProcessor<City>(City.class) {
        @Override
        public void beanProcessed(City bean, ParsingContext context) {
            blackhole.consume(bean);
        }
    });

    com.univocity.parsers.csv.CsvParser parser = new com.univocity.parsers.csv.CsvParser(settings);
    try(Reader reader = CsvParam.getSingleThreadedReader(quotes, nbRows)) {
        parser.parse(reader);
    }
}
 

import com.univocity.parsers.common.ParsingContext; //导入依赖的package包/类
@Benchmark
public void testUnivocityCsvMapper(final Blackhole blackhole) throws Exception {
	BeanProcessor<SmallBenchmarkObject> rowProcessor = new BeanProcessor<SmallBenchmarkObject>(SmallBenchmarkObject.class) {

		@Override
		public void beanProcessed(SmallBenchmarkObject bean,
				ParsingContext context) {
			blackhole.consume(bean);
		}
		
	};

    CsvParserSettings parserSettings = new CsvParserSettings();
    parserSettings.setRowProcessor(rowProcessor);
    parserSettings.setHeaderExtractionEnabled(true);

    com.univocity.parsers.csv.CsvParser parser = new com.univocity.parsers.csv.CsvParser(parserSettings);

    parser.parse(reader);
}
 

import com.univocity.parsers.common.ParsingContext; //导入依赖的package包/类
/**
 * Initializes data structures to capture parsed content
 * @param csvColCount   the number of columns in CSV content
 * @param context       the parsing context
 */
@SuppressWarnings("unchecked")
private void initBatch(int csvColCount, ParsingContext context) {
    final int colCount = initHeader(csvColCount, context);
    this.rowValues = new String[colCount];
    this.batch = new DataBatch<>(options, colCount);
    this.parsers = new Parser[colCount];
}
 

import com.univocity.parsers.common.ParsingContext; //导入依赖的package包/类
/**
 * Loads all user agents that should be in the user source category.
 */
private static void loadToolNamesForUserCategory()
{
  TsvParserSettings parserSettings = new TsvParserSettings();
  parserSettings.setLineSeparatorDetectionEnabled(true);
  parserSettings.setHeaderExtractionEnabled(true);
  parserSettings.setSkipEmptyLines(true);
  parserSettings.setReadInputOnSeparateThread(true);

  ObjectRowProcessor rowProcessor = new ObjectRowProcessor()
  {
    @Override
    public void rowProcessed(Object[] row, ParsingContext parsingContext)
    {
      if (row.length < 1) {
        logger.warn("Ignoring line without tab while parsing.");
        return;
      }
      if (row.length == 1) {
        sourceCategoryUserToolName.add(row[0].toString());
        return;
      }
      logger.warn("Line with row length " + row.length + " found. Is the formatting of toolNameForUserCategory.tsv correct?");
      return;
    }

  };

  parserSettings.setProcessor(rowProcessor);

  TsvParser parser = new TsvParser(parserSettings);

  try {
    parser.parse(new InputStreamReader(new FileInputStream("userAgentClassification/toolNameForUserCategory.tsv")));
  } catch (FileNotFoundException e) {
    logger.error("Could not open configuration file for standard prefixes.", e);
  }
}
 

import com.univocity.parsers.common.ParsingContext; //导入依赖的package包/类
/**
 * Loads the mapping of property to groups.
 */
private static void loadPropertyGroupMapping()
{
  TsvParserSettings parserSettings = new TsvParserSettings();
  parserSettings.setLineSeparatorDetectionEnabled(true);
  parserSettings.setHeaderExtractionEnabled(true);
  parserSettings.setSkipEmptyLines(true);
  parserSettings.setReadInputOnSeparateThread(true);

  ObjectRowProcessor rowProcessor = new ObjectRowProcessor()
  {
    @Override
    public void rowProcessed(Object[] row, ParsingContext parsingContext)
    {
      if (row.length <= 1) {
        logger.warn("Ignoring line without tab while parsing.");
        return;
      }
      if (row.length == 2) {
        if (row[1] == null) {
          return;
        }
        propertyGroupMapping.put(row[0].toString(), new HashSet<String>(Arrays.asList(row[1].toString().split(","))));
        return;
      }
      logger.warn("Line with row length " + row.length + " found. Is the formatting of propertyGroupMapping.tsv correct?");
      return;
    }

  };

  parserSettings.setProcessor(rowProcessor);

  TsvParser parser = new TsvParser(parserSettings);

  File file = new File("propertyClassification/propertyGroupMapping.tsv");

  parser.parse(file);
}
 

import com.univocity.parsers.common.ParsingContext; //导入依赖的package包/类
/**
 *
 * @param blackDuckIssue
 * @param context
 */
@Override
public void beanProcessed(BlackDuckIssue blackDuckIssue, ParsingContext context) {
    if (rowsProcessed == 0) {
        validateHeaders(context.headers());
    }
    buildVulnerability(blackDuckIssue, vulnerabilityHandler);
    rowsProcessed++;
}
 

import com.univocity.parsers.common.ParsingContext; //导入依赖的package包/类
public UnivocityParserRowContext(ParsingContext context, int size) {
	this.context = context;
	this.splitIndexes = new int[size];
	if (size > 0) {
		splitIndexes[0] = -1;
	}
}
 

import com.univocity.parsers.common.ParsingContext; //导入依赖的package包/类
/**
 * Called when a row is processed, it retrieve the headers if necessary.
 *
 * @param row     Processed row
 * @param context Parsing context
 */
@Override
public void rowProcessed(String[] row, ParsingContext context) {
    if (headers == null) {
        headers = context.headers();
    }
}
 

import com.univocity.parsers.common.ParsingContext; //导入依赖的package包/类
@Override
public void processStarted(ParsingContext context) {}
 

import com.univocity.parsers.common.ParsingContext; //导入依赖的package包/类
/**
 * Loads all pre-build query types.
 */
public static void loadPreBuildQueryTypes()
{

  try (DirectoryStream<Path> directoryStream =
           Files.newDirectoryStream(Paths.get("preBuildQueryTypeFiles"))) {
    for (Path filePath : directoryStream) {
      if (Files.isRegularFile(filePath)) {
        if (filePath.toString().endsWith(".preBuildQueryType")) {
          String queryString = new String(readAllBytes(filePath));
          OpenRDFQueryHandler queryHandler = new OpenRDFQueryHandler(QueryHandler.Validity.DEFAULT, -1L, -1, queryString, "preBuildQueryTypes", "", -1);
          if (queryHandler.getValidityStatus() != QueryHandler.Validity.VALID) {
            logger.info("The Pre-build query " + filePath + " is no valid SPARQL");
            continue;
          }
          ParsedQuery normalizedPreBuildQuery = queryHandler.getNormalizedQuery();
          String queryTypeName = filePath.toString().substring(filePath.toString().lastIndexOf("/") + 1, filePath.toString().lastIndexOf("."));
          if (normalizedPreBuildQuery != null) {
            String queryDump = normalizedPreBuildQuery.getTupleExpr().toString();
            byte[] md5 = DigestUtils.md5(queryDump);

            int index = Math.floorMod(queryDump.hashCode(), numberOfQueryTypeDiskMaps);
            queryTypes[index].put(md5, queryTypeName);
          } else {
            logger.info("Pre-build query " + queryTypeName + " could not be parsed.");
          }
        }
        if (filePath.toString().endsWith(".tsv")) {
          TsvParserSettings parserSettings = new TsvParserSettings();
          parserSettings.setLineSeparatorDetectionEnabled(true);
          parserSettings.setHeaderExtractionEnabled(true);
          parserSettings.setSkipEmptyLines(true);
          parserSettings.setReadInputOnSeparateThread(true);

          ObjectRowProcessor rowProcessor = new ObjectRowProcessor()
          {
            @Override
            public void rowProcessed(Object[] row, ParsingContext parsingContext)
            {
              if (row.length <= 1) {
                logger.warn("Ignoring line without tab while parsing.");
                return;
              }
              if (row.length == 5) {
                queryTypeToToolMapping.put(new Tuple2<>(row[0].toString(), row[1].toString()), new Tuple2<>(row[2].toString(), row[3].toString()));
                return;
              }
              logger.warn("Line with row length " + row.length + " found. Is the formatting of toolMapping.tsv correct?");
              return;
            }

          };

          parserSettings.setProcessor(rowProcessor);

          TsvParser parser = new TsvParser(parserSettings);

          parser.parse(filePath.toFile());
        }
      }

    }

  } catch (IOException e) {
    logger.error("Could not read from directory inputData/queryType/premadeQueryTypeFiles", e);
  }
}
 

import com.univocity.parsers.common.ParsingContext; //导入依赖的package包/类
/**
 * Read the file given by reader and hands the data to the outputHandler.
 *
 * @param outputHandler Handles the data that should be written.
 */
public final void parseTo(final OutputHandler outputHandler, int day)
{
  //read in queries from .tsv
  TsvParserSettings parserSettings = new TsvParserSettings();
  parserSettings.setLineSeparatorDetectionEnabled(true);
  parserSettings.setHeaderExtractionEnabled(true);
  parserSettings.setSkipEmptyLines(true);
  parserSettings.setReadInputOnSeparateThread(true);

  ObjectRowProcessor rowProcessor = new ObjectRowProcessor()
  {
    @Override
    public void rowProcessed(Object[] row, ParsingContext parsingContext)
    {
      if (row.length <= 1) {
        logger.warn("Ignoring line without tab while parsing.");
        return;
      }
      Tuple2<String, QueryHandler.Validity> queryTuple = decode(row[0].toString(), inputFile, parsingContext.currentLine());

      String queryString = queryTuple._1;
      QueryHandler.Validity validity = queryTuple._2;
      String userAgent = "null";
      if (row[2] != null) {
        userAgent = row[2].toString();
      }
      String timeStamp = "null";
      if (row[3] != null) {
        timeStamp = row[3].toString();
      }
      Long line = parsingContext.currentLine();

      try {
        outputHandler.writeLine(queryString, validity, userAgent, timeStamp, line, day, inputFile);
      } catch (NullPointerException e) {
        outputHandler.writeLine("", QueryHandler.Validity.INTERNAL_ERROR, userAgent, timeStamp, line, day, inputFile);
        logger.error("Unexpected Null Pointer Exception in writeLine.", e);
      }
    }
  };

  parserSettings.setProcessor(rowProcessor);

  TsvParser parser = new TsvParser(parserSettings);

  parser.parse(reader);

  outputHandler.closeFiles();

  if (preprocessedWriter != null) {
    preprocessedWriter.close();
  }
}