Java DocMaker类代码示例

import org.apache.lucene.benchmark.byTask.feeds.DocMaker; //导入依赖的package包/类
private PerfRunData createPerfRunData(File file, 
                                      boolean allowEmptyDocs,
                                      String docMakerName) throws Exception {
  Properties props = new Properties();
  props.setProperty("doc.maker", docMakerName);
  props.setProperty("line.file.out", file.getAbsolutePath());
  props.setProperty("directory", "RAMDirectory"); // no accidental FS dir.
  if (allowEmptyDocs) {
    props.setProperty("sufficient.fields", ",");
  }
  if (docMakerName.equals(LegalJustDateDocMaker.class.getName())) {
    props.setProperty("line.fields", DocMaker.DATE_FIELD);
    props.setProperty("sufficient.fields", DocMaker.DATE_FIELD);
  }
  Config config = new Config(props);
  return new PerfRunData(config);
}
 

import org.apache.lucene.benchmark.byTask.feeds.DocMaker; //导入依赖的package包/类
public ExtractWikipedia(DocMaker docMaker, File outputDir) {
  this.outputDir = outputDir;
  this.docMaker = docMaker;
  System.out.println("Deleting all files in " + outputDir);
  File[] files = outputDir.listFiles();
  for (int i = 0; i < files.length; i++) {
    files[i].delete();
  }
}
 

import org.apache.lucene.benchmark.byTask.feeds.DocMaker; //导入依赖的package包/类
public void extract() throws Exception {
  Document doc = null;
  System.out.println("Starting Extraction");
  long start = System.currentTimeMillis();
  try {
    while ((doc = docMaker.makeDocument()) != null) {
      create(doc.get(DocMaker.ID_FIELD), doc.get(DocMaker.TITLE_FIELD), doc
          .get(DocMaker.DATE_FIELD), doc.get(DocMaker.BODY_FIELD));
    }
  } catch (NoMoreDataException e) {
    //continue
  }
  long finish = System.currentTimeMillis();
  System.out.println("Extraction took " + (finish - start) + " ms");
}
 

import org.apache.lucene.benchmark.byTask.feeds.DocMaker; //导入依赖的package包/类
public static void main(String[] args) throws Exception {

    File wikipedia = null;
    File outputDir = new File("./enwiki");
    boolean keepImageOnlyDocs = true;
    for (int i = 0; i < args.length; i++) {
      String arg = args[i];
      if (arg.equals("--input") || arg.equals("-i")) {
        wikipedia = new File(args[i + 1]);
        i++;
      } else if (arg.equals("--output") || arg.equals("-o")) {
        outputDir = new File(args[i + 1]);
        i++;
      } else if (arg.equals("--discardImageOnlyDocs") || arg.equals("-d")) {
        keepImageOnlyDocs = false;
      }
    }
    
    Properties properties = new Properties();
    properties.setProperty("docs.file", wikipedia.getAbsolutePath());
    properties.setProperty("content.source.forever", "false");
    properties.setProperty("keep.image.only.docs", String.valueOf(keepImageOnlyDocs));
    Config config = new Config(properties);

    ContentSource source = new EnwikiContentSource();
    source.setConfig(config);
    
    DocMaker docMaker = new DocMaker();
    docMaker.setConfig(config, source);
    docMaker.resetInputs();
    if (wikipedia.exists()) {
      System.out.println("Extracting Wikipedia to: " + outputDir + " using EnwikiContentSource");
      outputDir.mkdirs();
      ExtractWikipedia extractor = new ExtractWikipedia(docMaker, outputDir);
      extractor.extract();
    } else {
      printUsage();
    }
  }
 

import org.apache.lucene.benchmark.byTask.feeds.DocMaker; //导入依赖的package包/类
@Override
protected PrintWriter lineFileOut(Document doc) {
  IndexableField titleField = doc.getField(DocMaker.TITLE_FIELD);
  if (titleField!=null && titleField.stringValue().startsWith("Category:")) {
    return categoryLineFileOut;
  }
  return super.lineFileOut(doc);
}
 

import org.apache.lucene.benchmark.byTask.feeds.DocMaker; //导入依赖的package包/类
@Override
public void setup() throws Exception {
  super.setup();
  DocMaker docMaker = getRunData().getDocMaker();
  if (docSize > 0) {
    doc = docMaker.makeDocument(docSize);
  } else {
    doc = docMaker.makeDocument();
  }
}
 

import org.apache.lucene.benchmark.byTask.feeds.DocMaker; //导入依赖的package包/类
@Override
public int doLogic() throws Exception {
  final String docID = doc.get(DocMaker.ID_FIELD);
  if (docID == null) {
    throw new IllegalStateException("document must define the docid field");
  }
  final IndexWriter iw = getRunData().getIndexWriter();
  iw.updateDocument(new Term(DocMaker.ID_FIELD, docID), doc);
  return 1;
}
 

import org.apache.lucene.benchmark.byTask.feeds.DocMaker; //导入依赖的package包/类
public PerfRunData (Config config) throws Exception {
  this.config = config;
  // analyzer (default is standard analyzer)
  analyzer = NewAnalyzerTask.createAnalyzer(config.get("analyzer",
      "org.apache.lucene.analysis.standard.StandardAnalyzer"));

  // content source
  String sourceClass = config.get("content.source", "org.apache.lucene.benchmark.byTask.feeds.SingleDocSource");
  contentSource = Class.forName(sourceClass).asSubclass(ContentSource.class).newInstance();
  contentSource.setConfig(config);

  // doc maker
  docMaker = Class.forName(config.get("doc.maker",
      "org.apache.lucene.benchmark.byTask.feeds.DocMaker")).asSubclass(DocMaker.class).newInstance();
  docMaker.setConfig(config, contentSource);
  // facet source
  facetSource = Class.forName(config.get("facet.source",
      "org.apache.lucene.benchmark.byTask.feeds.RandomFacetSource")).asSubclass(FacetSource.class).newInstance();
  facetSource.setConfig(config);
  // query makers
  readTaskQueryMaker = new HashMap<>();
  qmkrClass = Class.forName(config.get("query.maker","org.apache.lucene.benchmark.byTask.feeds.SimpleQueryMaker")).asSubclass(QueryMaker.class);

  // index stuff
  reinit(false);
  
  // statistic points
  points = new Points(config);
  
  if (Boolean.valueOf(config.get("log.queries","false")).booleanValue()) {
    System.out.println("------------> queries:");
    System.out.println(getQueryMaker(new SearchTask(this)).printQueries());
  }
}
 

import org.apache.lucene.benchmark.byTask.feeds.DocMaker; //导入依赖的package包/类
public PerfRunData (Config config) throws Exception {
  this.config = config;
  // analyzer (default is standard analyzer)
  analyzer = NewAnalyzerTask.createAnalyzer(config.get("analyzer",
      "org.apache.lucene.analysis.standard.StandardAnalyzer"));

  // content source
  String sourceClass = config.get("content.source", "org.apache.lucene.benchmark.byTask.feeds.SingleDocSource");
  contentSource = Class.forName(sourceClass).asSubclass(ContentSource.class).newInstance();
  contentSource.setConfig(config);

  // doc maker
  docMaker = Class.forName(config.get("doc.maker",
      "org.apache.lucene.benchmark.byTask.feeds.DocMaker")).asSubclass(DocMaker.class).newInstance();
  docMaker.setConfig(config, contentSource);
  // facet source
  facetSource = Class.forName(config.get("facet.source",
      "org.apache.lucene.benchmark.byTask.feeds.RandomFacetSource")).asSubclass(FacetSource.class).newInstance();
  facetSource.setConfig(config);
  // query makers
  readTaskQueryMaker = new HashMap<Class<? extends ReadTask>,QueryMaker>();
  qmkrClass = Class.forName(config.get("query.maker","org.apache.lucene.benchmark.byTask.feeds.SimpleQueryMaker")).asSubclass(QueryMaker.class);

  // index stuff
  reinit(false);
  
  // statistic points
  points = new Points(config);
  
  if (Boolean.valueOf(config.get("log.queries","false")).booleanValue()) {
    System.out.println("------------> queries:");
    System.out.println(getQueryMaker(new SearchTask(this)).printQueries());
  }
}