Java VectorStoreWriter类代码示例

import pitt.search.semanticvectors.VectorStoreWriter; //导入依赖的package包/类
public static void main(String[] args) throws IOException {
  FlagConfig flagConfig = null;
  try {
    flagConfig = FlagConfig.getFlagConfig(args);
    args = flagConfig.remainingArgs;
  } catch (IllegalArgumentException e) {
    System.err.println(usageMessage);
    throw e;
  }

  if (flagConfig.remainingArgs.length != 1) {
    throw new IllegalArgumentException("Wrong number of arguments after parsing command line flags.\n" + usageMessage);
  }

  VerbatimLogger.info("Building vector index of table in file: " + args[0] + "\n");
  BufferedReader fileReader = new BufferedReader(new FileReader(args[0]));
  String[] columnHeaders = fileReader.readLine().split(",");
  ArrayList<String[]> dataRows = new ArrayList<>();
  String dataLine;
  while((dataLine = fileReader.readLine()) != null) {
    String[] dataEntries = dataLine.split(",");
    if (dataEntries.length != columnHeaders.length) {
      throw new IllegalArgumentException(String.format(
          "Column headers have length %d and this row has length %d. This indicates a data error or a csv parsing error."
          + "\nColumn headers:%s\nData row: %s\n",
          columnHeaders.length, dataEntries.length,
          StringUtils.join(columnHeaders), StringUtils.join(dataEntries)));
    }
    dataRows.add(dataEntries);
  }
  fileReader.close();
  
  Table table = new Table(flagConfig, columnHeaders, dataRows);
  VectorStoreWriter.writeVectors(flagConfig.termvectorsfile(), flagConfig, table.getRowVectorStore());

  queryForSpecialValues(table);
  //queryForName(table, "J. Adams");
 // queryForName(table, "T. Roosevelt");
  
}
 

import pitt.search.semanticvectors.VectorStoreWriter; //导入依赖的package包/类
public void createNGrams(String fileOut, FlagConfig flagConfig, int numGrams )
{
	BeagleNGramVectors bngv;
	BeagleUtils utils = BeagleUtils.getInstance();

	long time;

	try
	{
		time = System.currentTimeMillis();

		bngv = new BeagleNGramVectors(
		    flagConfig, "index", 5, 2, new String[] {"contents"}, numGrams, "stoplist.txt");

		time = System.currentTimeMillis() - time;

		System.out.println("\nTime to process: " + time/1000 + " secs.");
		System.out.println("\nNumber of convolutions: " + utils.getNumConvolutions());

		VectorStoreWriter.writeVectors(
		    fileOut + "_" + flagConfig.dimension() + "_" + numGrams + ".bin", flagConfig, bngv);

		VectorStore indexVectors = bngv.getIndexVectors();
		VectorStoreWriter.writeVectors(
		    fileOut + "_" + flagConfig.dimension() + "_" + numGrams + "_index.bin", flagConfig, indexVectors);

		bngv = null;
		System.gc();
	}
	catch(Exception e)
	{
		e.printStackTrace();
	}
}
 

import pitt.search.semanticvectors.VectorStoreWriter; //导入依赖的package包/类
public static void main(String[] args) throws IOException {
  FlagConfig flagConfig = null;
  try {
    flagConfig = FlagConfig.getFlagConfig(args);
    args = flagConfig.remainingArgs;
  } catch (IllegalArgumentException e) {
    System.err.println(usageMessage);
    throw e;
  }
  
  VerbatimLogger.info("Building vector index of table in file: " + args[0] + "\n");
  BufferedReader fileReader = new BufferedReader(new FileReader(args[0]));
  String[] columnHeaders = fileReader.readLine().split(",");
  ArrayList<String[]> dataRows = new ArrayList<>();
  String dataLine;
  while((dataLine = fileReader.readLine()) != null) {
    String[] dataEntries = dataLine.split(",");
    if (dataEntries.length != columnHeaders.length) {
      throw new IllegalArgumentException(String.format(
          "Column headers have length %d and this row has length %d. This indicates a data error or a csv parsing error."
          + "\nColumn headers:%s\nData row: %s\n",
          columnHeaders.length, dataEntries.length,
          StringUtils.join(columnHeaders), StringUtils.join(dataEntries)));
    }
    dataRows.add(dataEntries);
  }
  fileReader.close();
  
  Table table = new Table(flagConfig, columnHeaders, dataRows);
  VectorStoreWriter.writeVectors(flagConfig.termvectorsfile(), flagConfig, table.getRowVectorStore());
  queryForSpecialValues(table);
}
 

import pitt.search.semanticvectors.VectorStoreWriter; //导入依赖的package包/类
/**
 * Main class of a utility method that takes as input:
 * (1) -queryvectorfile: the original vector store OR
 * (1.1) -elementalvectofile, -semanticvectorfile, -predicatevectorfile (for PSI queries)
 * (2) -startlistfile: the list of queries (e.g. terms), one per line, from which to construct a new vector store 
 * @param args
 * @throws IOException 
 */


public static void main(String[] args) throws IOException {
	
	FlagConfig flagConfig = FlagConfig.getFlagConfig(args);
	boolean PSIquery = true;
	
	VectorStoreRAM queryVectors = new VectorStoreRAM(flagConfig);
	VectorStoreRAM elementalVectors = new VectorStoreRAM(flagConfig);
	VectorStoreRAM predicateVectors = new VectorStoreRAM(flagConfig);
	VectorStoreRAM semanticVectors = new VectorStoreRAM(flagConfig);

	
	//if all PSI/ESP query files are at the default value, assume this is not a PSI/ESP query
	if (flagConfig.elementalvectorfile().equals("elementalvectors")
			&& flagConfig.semanticvectorfile().equals("semanticvectors")
			  && flagConfig.elementalpredicatevectorfile().equals("predicatevectors")
			) 
		{
			PSIquery = false;
			queryVectors.initFromFile(flagConfig.queryvectorfile());
		}
	else
	{
		elementalVectors.initFromFile(flagConfig.elementalvectorfile());
		semanticVectors.initFromFile(flagConfig.semanticvectorfile());
		predicateVectors.initFromFile(flagConfig.elementalpredicatevectorfile());
	}
	
	VectorStoreRAM outGoingVectors = new VectorStoreRAM(flagConfig);
	
	
	BufferedReader theReader = new BufferedReader(new FileReader(new File(flagConfig.startlistfile())));
	
	String inputString = theReader.readLine();
	while (inputString != null)
	{
		Vector vectorToAdd = null;
		
		if (!PSIquery) vectorToAdd = CompoundVectorBuilder.getQueryVectorFromString(queryVectors, null, flagConfig, inputString);
		else 		   vectorToAdd = CompoundVectorBuilder.getBoundProductQueryVectorFromString(flagConfig, elementalVectors, semanticVectors, predicateVectors, null, inputString);
		
		if (vectorToAdd == null || 
				vectorToAdd.isZeroVector() || 
					(flagConfig.vectortype().equals(VectorType.REAL) && (Float.isNaN(  ((RealVector) vectorToAdd).getCoordinates()[0]))) ||
					(flagConfig.vectortype().equals(VectorType.COMPLEX) && (Float.isNaN(  ((ComplexVector) vectorToAdd).getCoordinates()[0])))
						
				)
		{	VerbatimLogger.info("Could not represent "+inputString);}
		else
		{
			VerbatimLogger.info("Adding "+inputString.replaceAll(" ", "_"));
			outGoingVectors.putVector(inputString.replaceAll(" ", "_"),vectorToAdd);
		}  
		
		inputString = theReader.readLine();
	}
	
	theReader.close();
	
	VerbatimLogger.info("\nFinished adding vectors, proceeding to write out\n");
	
	VectorStoreWriter.writeVectorsInLuceneFormat(flagConfig.startlistfile().replaceAll("\\..*", "_subset.bin"), flagConfig, outGoingVectors);
	

}
 

import pitt.search.semanticvectors.VectorStoreWriter; //导入依赖的package包/类
/**
 * @param args
 */
public static void main(String[] args) {
	// TODO Auto-generated method stub

	try {
	FlagConfig flagConfig 			= FlagConfig.getFlagConfig(args);
	VectorStoreRAM objectVectors 	= new VectorStoreRAM(flagConfig);
	String[] argsRemaining			= flagConfig.remainingArgs;
	String incomingVecs				= argsRemaining[0];
	int newDimension				= Integer.parseInt(argsRemaining[1]);
	objectVectors.initFromFile(incomingVecs);
	
	if (newDimension > flagConfig.dimension())
		{
		
				System.out.println("Incoming file has dimensionality of " +flagConfig.dimension());
				System.out.println("New dimensionality must be less than incoming vector length, quitting");
				System.exit(0);	
		}
	
		String vectorFileName = incomingVecs.replaceAll("\\.bin", "")+"_"+newDimension+".bin";
	  	File vectorFile = new File(vectorFileName);
	    String parentPath = vectorFile.getParent();
	    if (parentPath == null) parentPath = "";
	    FSDirectory fsDirectory = FSDirectory.open(FileSystems.getDefault().getPath(parentPath));
	    IndexOutput outputStream = fsDirectory.createOutput(vectorFile.getName(), IOContext.DEFAULT);
	 	flagConfig.setDimension(newDimension);
		outputStream.writeString(VectorStoreWriter.generateHeaderString(flagConfig));
	    Enumeration<ObjectVector> vecEnum = objectVectors.getAllVectors();

	    // Write each vector.
	    while (vecEnum.hasMoreElements()) {
	      ObjectVector objectVector = vecEnum.nextElement();
	      outputStream.writeString(objectVector.getObject().toString());
	      objectVector.getVector().writeToLuceneStream(outputStream,flagConfig.dimension());
	    }
	    
	    
	    outputStream.close();
	    fsDirectory.close();
		
	    VerbatimLogger.info("wrote "+objectVectors.getNumVectors()+" vectors to file "+ vectorFileName);
	    VerbatimLogger.info("finished writing vectors.\n");
	 		
	} catch (IOException e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
		System.out.println("Usage: VectorStoreTruncater incomingFile.bin newDimensinoality");
	}
	
	
	
}