Java Job类代码示例

import org.galagosearch.tupleflow.execution.Job; //导入依赖的package包/类
private static void handleBuild(String[] args) throws Exception {
    if (args.length <= 1) {
        commandHelpBuild();
        return;
    }

    // handle --links and --stemming flags
    String[][] filtered = Utility.filterFlags(Utility.subarray(args, 2));

    String[] flags = filtered[0];
    String[] docs = filtered[1];

    Parameters p = new Parameters(flags);
    boolean useLinks = p.get("links", false);
    boolean stemming = p.get("stemming", true);

    BuildIndex build = new BuildIndex();
    Job job = build.getIndexJob(args[1], docs, useLinks, stemming);
    ErrorStore store = new ErrorStore();
    JobExecutor.runLocally(job, store);
    if (store.hasStatements()) {
        System.out.println(store.toString());
    }
}
 

import org.galagosearch.tupleflow.execution.Job; //导入依赖的package包/类
/**
 * Builds the entire job.
 */
public Job getZipfJob(String outputFile, String[] inputs) throws IOException {
    Job job = new Job();
    BuildIndex b = new BuildIndex();

    job.add(b.getSplitStage(inputs));
    job.add(getWordCountStage());
    job.add(getReduceCountsStage());
    job.add(getInvertByCountStage());
    job.add(getZipfReduceStage(outputFile));
    
    job.connect("inputSplit", "wordCount", ConnectionAssignmentType.Each);
    job.connect("wordCount", "reduceCounts", ConnectionAssignmentType.Each);
    job.connect("reduceCounts", "invertByCount", ConnectionAssignmentType.Each);
    job.connect("invertByCount", "zipfReduce", ConnectionAssignmentType.Combined);

    return job;
}
 

import org.galagosearch.tupleflow.execution.Job; //导入依赖的package包/类
private static void handleMakeCorpus(String[] args) throws Exception {
    if (args.length <= 2) {
        commandHelp(args[0]);
        return;
    }

    Job job = getDocumentConverter(args[1], Utility.subarray(args, 2));
    ErrorStore store = new ErrorStore();
    JobExecutor.runLocally(job, store);
    if (store.hasStatements()) {
        System.out.println(store.toString());
    }
}
 

import org.galagosearch.tupleflow.execution.Job; //导入依赖的package包/类
public void testCollectionLengthStage() throws IOException {
    BuildIndex buildIndex = new BuildIndex();
    Job job = new Job();
    job.add(buildIndex.getCollectionLengthStage());
    ErrorStore store = new ErrorStore();

    Verification.verify(job, store);
    assertEquals(0, store.getErrors().size());
    assertEquals(0, store.getWarnings().size());
}
 

import org.galagosearch.tupleflow.execution.Job; //导入依赖的package包/类
/**
 * Test of getSplitStage method, of class BuildIndex.
 */
public void testGetSplitStage() throws IOException {
    BuildIndex buildIndex = new BuildIndex();
    Job job = new Job();
    job.add(buildIndex.getSplitStage(new String[] {"/"}));
    ErrorStore store = new ErrorStore();

    Verification.verify(job, store);
    assertEquals(0, store.getErrors().size());
    assertEquals(0, store.getWarnings().size());
}
 

import org.galagosearch.tupleflow.execution.Job; //导入依赖的package包/类
/**
 * Test of getParseStage method, of class BuildIndex.
 */
public void testGetParseStage() {
    BuildIndex BuildIndex = new BuildIndex();
    Job job = new Job();
    job.add(BuildIndex.getParsePostingsStage());
    ErrorStore store = new ErrorStore();

    Verification.verify(job, store);
    assertEquals(0, store.getErrors().size());
    assertEquals(0, store.getWarnings().size());
}
 

import org.galagosearch.tupleflow.execution.Job; //导入依赖的package包/类
/**
 * Test of getLinkCombineStage method, of class BuildIndex.
 */
public void testGetLinkCombineStage() {
    BuildIndex BuildIndex = new BuildIndex();
    Job job = new Job();
    job.add(BuildIndex.getLinkCombineStage());
    ErrorStore store = new ErrorStore();

    Verification.verify(job, store);
    assertEquals("", store.toString());
}
 

import org.galagosearch.tupleflow.execution.Job; //导入依赖的package包/类
/**
 * Test of getWritePostingsStage method, of class BuildIndex.
 */
public void testGetWritePostingsStage() {
    BuildIndex BuildIndex = new BuildIndex();
    Job job = new Job();
    job.add(BuildIndex.getWritePostingsStage("postings", "input", "index"));
    ErrorStore store = new ErrorStore();

    Verification.verify(job, store);
    assertEquals(0, store.getErrors().size());
    assertEquals(0, store.getWarnings().size());
}
 

import org.galagosearch.tupleflow.execution.Job; //导入依赖的package包/类
/**
 * Test of getWriteExtentsStage method, of class BuildIndex.
 */
public void testGetWriteExtentsStage() {
    BuildIndex BuildIndex = new BuildIndex();
    Job job = new Job();
    job.add(BuildIndex.getWriteExtentsStage());
    ErrorStore store = new ErrorStore();

    Verification.verify(job, store);
    assertEquals(0, store.getErrors().size());
    assertEquals(0, store.getWarnings().size());
}
 

import org.galagosearch.tupleflow.execution.Job; //导入依赖的package包/类
/**
 * Test of getWriteDatesStage method, of class BuildIndex.
 */
public void testGetWriteDatesStage() {
    BuildIndex BuildIndex = new BuildIndex();
    Job job = new Job();
    job.add(BuildIndex.getWriteDatesStage());
    ErrorStore store = new ErrorStore();

    Verification.verify(job, store);
    assertEquals(0, store.getErrors().size());
    assertEquals(0, store.getWarnings().size());
}
 

import org.galagosearch.tupleflow.execution.Job; //导入依赖的package包/类
/**
 * Test of getWriteManifestStage method, of class BuildIndex.
 */
public void testGetWriteManifestStage() {
    BuildIndex BuildIndex = new BuildIndex();
    Job job = new Job();
    job.add(BuildIndex.getWriteManifestStage());
    ErrorStore store = new ErrorStore();

    Verification.verify(job, store);
    assertEquals(0, store.getErrors().size());
    assertEquals(0, store.getWarnings().size());
}
 

import org.galagosearch.tupleflow.execution.Job; //导入依赖的package包/类
/**
 * Test of getWriteDocumentLengthsStage method, of class BuildIndex.
 */
public void testGetWriteDocumentLengthsStage() {
    BuildIndex BuildIndex = new BuildIndex();
    Job job = new Job();
    job.add(BuildIndex.getWriteDocumentLengthsStage());
    ErrorStore store = new ErrorStore();

    Verification.verify(job, store);
    assertEquals(0, store.getErrors().size());
    assertEquals(0, store.getWarnings().size());
}
 

import org.galagosearch.tupleflow.execution.Job; //导入依赖的package包/类
/**
 * Test of getIndexJob method, of class BuildIndex.
 */
public void testGetIndexJob() throws IOException {
    BuildIndex buildIndex = new BuildIndex();
    Job job = buildIndex.getIndexJob("one", new String[] {"/"}, false, false);
    ErrorStore store = new ErrorStore();

    Verification.verify(job, store);
    assertEquals("", store.toString());
}
 

import org.galagosearch.tupleflow.execution.Job; //导入依赖的package包/类
public void testJobWithStemming() throws IOException {
    BuildIndex buildIndex = new BuildIndex();

    Job job = buildIndex.getIndexJob("one", new String[] {"/"}, false, true);
    ErrorStore store = new ErrorStore();

    Verification.verify(job, store);
    assertEquals("", store.toString());
}
 

import org.galagosearch.tupleflow.execution.Job; //导入依赖的package包/类
public void testJobWithLinks() throws IOException {
    BuildIndex buildIndex = new BuildIndex();        
    Job job = buildIndex.getIndexJob("one", new String[] {"/"}, true, false);
    ErrorStore store = new ErrorStore();

    Verification.verify(job, store);
    assertEquals("", store.toString());
}