Java GraphManager.enableBatching方法代码示例

import com.ociweb.pronghorn.stage.scheduling.GraphManager; //导入方法依赖的package包/类
public static File writeTapeToFileUsingPipe(Pipe<RawDataSchema> inputPipe, byte[] testData) throws IOException, FileNotFoundException {
      
    GraphManager gm = new GraphManager();
    File f2 = File.createTempFile("fileTapeWriteTest", "dat");
    f2.deleteOnExit();
    
    new ByteArrayProducerStage(gm, testData, inputPipe);
    new TapeWriteStage(gm, inputPipe, new RandomAccessFile(f2,"rw")); 
                
    GraphManager.enableBatching(gm);
    
    ThreadPerStageScheduler scheduler = new ThreadPerStageScheduler(gm);
    scheduler.startup();            
    scheduler.awaitTermination(3, TimeUnit.SECONDS);

    return f2;
}
 

import com.ociweb.pronghorn.stage.scheduling.GraphManager; //导入方法依赖的package包/类
private void runGraph(GraphManager gm, final int apps, final int iterations, PronghornStage stage) {
    boolean monitorPipes = false;
    if (monitorPipes) {
        MonitorConsoleStage.attach(gm);        
    } 
    GraphManager.enableBatching(gm);
    ThreadPerStageScheduler scheduler = new ThreadPerStageScheduler(gm);
    
    long start = System.currentTimeMillis();

    scheduler.startup();  
    
    if (monitorPipes) {
      try {
          Thread.sleep(1000);
      } catch (InterruptedException e) {
          Thread.currentThread().interrupt();
      }
    }
    
    gm.blockUntilStageBeginsShutdown(stage); //generator gets done very early and begins the shutdown before the route completes so this block is required.
    
    scheduler.awaitTermination(30, TimeUnit.SECONDS);
    
    long duration = System.currentTimeMillis()-start;
    long totalRequests = iterations*(long)apps;
    
    float requestPerMsSecond = totalRequests/(float)duration;
    System.out.println("totalRequests: "+totalRequests+" perMs:"+requestPerMsSecond);
}
 

import com.ociweb.pronghorn.stage.scheduling.GraphManager; //导入方法依赖的package包/类
private static long fileBlobReadTest() throws IOException {
    final ZeroCopyByteArrayOutputStream outputStream = new ZeroCopyByteArrayOutputStream(testSize);
    
    File tempFile = fileFullOfTestData();
    
    int largestBlock = testSize;
    
    long[] times = new long[200]; //guess on size
    int timeIdx = 0;
            
    
    int p = largestPipe;
    while (p>=2) {           
        
        int blockSize = largestBlock/p;
        
        GraphManager gm = new GraphManager();
        
        PipeConfig<RawDataSchema> config = new PipeConfig<RawDataSchema>(RawDataSchema.instance, p, blockSize);
        Pipe<RawDataSchema> loadedDataPipe = new Pipe<RawDataSchema>(config);
                                
        
        new FileBlobReadStage(gm, loadedDataPipe,tempFile.getAbsolutePath());            
        outputStream.reset();
        new ToOutputStreamStage(gm, loadedDataPipe, outputStream, false);
        
                    
        GraphManager.enableBatching(gm);//lower contention over head and tail            
      //  MonitorConsoleStage.attach(gm);
        
        ThreadPerStageScheduler scheduler = new ThreadPerStageScheduler(gm);
        
        long startTime = System.currentTimeMillis();
        scheduler.startup();        
        
        scheduler.awaitTermination(timeoutSeconds, TimeUnit.SECONDS);
        long duration = System.currentTimeMillis()-startTime;
        
        times[timeIdx++] = duration;
        
        validateReadData(outputStream);
        
        if (duration>(timeoutSeconds*1000)) {
            System.err.println("unable to get duration, timedout");
        } else {                
            int mbytesPerSecond = mBytesPerSecond(duration);                
            System.out.println("BLOB READ: pipeLength:"+p+" blockSize:"+(blockSize/1024)+"k  duration:"+duration+" totalBlobRingSize:"+loadedDataPipe.sizeOfBlobRing+" MB/S:"+mbytesPerSecond);               
        }
        if (p>200) {
            p-=100;
        } else {
            if (p>30) {
                p-=10;
            } else {            
                p-=2;
            }
        }
    }
    
    Arrays.sort(times,0, timeIdx);
    int middle = timeIdx/2;
    return times[middle];
    
}
 

import com.ociweb.pronghorn.stage.scheduling.GraphManager; //导入方法依赖的package包/类
private static long fileTapeReadTest() throws IOException {
           
    
    
    final ZeroCopyByteArrayOutputStream outputStream = new ZeroCopyByteArrayOutputStream(testSize);
    
    int largestBlock = testSize;
    
    long[] times = new long[200]; //guess on size
    int timeIdx = 0;
    
    
    int p = largestPipe;
    while (p>=2) {           
        
        int blockSize = largestBlock/p;
        int blobRingSize;
        long duration;
        {
            ////////////////////
            //must create a readable file of this size
            ////////////////////
            PipeConfig<RawDataSchema> config = new PipeConfig<RawDataSchema>(RawDataSchema.instance, p, blockSize);
            Pipe<RawDataSchema> inputPipe = new Pipe<RawDataSchema>(config);        
            File tapeFile = TapeRoundTripTest.writeTapeToFileUsingPipe(inputPipe, rawData);
                            
            
            GraphManager gm = new GraphManager();
            
            Pipe<RawDataSchema> loadedDataPipe = new Pipe<RawDataSchema>(config.grow2x());
            blobRingSize = loadedDataPipe.sizeOfBlobRing;
            new TapeReadStage(gm, new RandomAccessFile(tapeFile, "r"), loadedDataPipe); //TODO: Must check that loading pipe is large enough
            
            outputStream.reset();
            new ToOutputStreamStage(gm, loadedDataPipe, outputStream, false);
            
            GraphManager.enableBatching(gm);//lower contention over head and tail            
          //  MonitorConsoleStage.attach(gm);
            
            ThreadPerStageScheduler scheduler = new ThreadPerStageScheduler(gm);
            
            long startTime = System.currentTimeMillis();
            scheduler.startup();        
            
            scheduler.awaitTermination(timeoutSeconds, TimeUnit.SECONDS);
            duration = System.currentTimeMillis()-startTime;
        }
        validateReadData(outputStream);
        
        times[timeIdx++] = duration;
        
        if (duration>(timeoutSeconds*1000)) {
            System.err.println("unable to get duration, timedout");
        } else {                
            int mbytesPerSecond = mBytesPerSecond(duration);                
            System.out.println("TAPE READ: pipeLength:"+p+" blockSize:"+(blockSize/1024)+"k  duration:"+duration+" totalBlobRingSize:"+blobRingSize+" MB/S:"+mbytesPerSecond);               
        }
        if (p>200) {
            p-=100;
        } else {
            if (p>30) {
                p-=10;
            } else {            
                p-=2;
            }
        }
    }
    
    Arrays.sort(times,0, timeIdx);
    int middle = timeIdx/2;
    return times[middle];
    
}
 

import com.ociweb.pronghorn.stage.scheduling.GraphManager; //导入方法依赖的package包/类
private static long fileBlobWriteTest() throws IOException {
      
    
    long[] times = new long[200]; //guess on size
    int timeIdx = 0;
    
    
    int largestBlock = testSize;
    
    int p = largestPipe;
    while (p>=2) {           
        
        int blockSize = largestBlock/p;
        
        GraphManager gm = new GraphManager();
        
        PipeConfig<RawDataSchema> config = new PipeConfig<RawDataSchema>(RawDataSchema.instance, p, blockSize);
        Pipe<RawDataSchema> loadedDataPipe = new Pipe<RawDataSchema>(config);
        
        
        File tempFile = File.createTempFile("blobWrite", "speedTest");        
        tempFile.deleteOnExit();
        
        boolean append = false;
        
        PronghornStage s1 = new ByteArrayProducerStage(gm, rawData, loadedDataPipe);        
        PronghornStage s2 = new FileBlobWriteStage(gm, loadedDataPipe, append, tempFile.getAbsolutePath());  //NOTE: use rwd/rws to sync flush with every write (much slower)
        
        GraphManager.enableBatching(gm);//lower contention over head and tail
     //   MonitorConsoleStage.attach(gm);
        
        ThreadPerStageScheduler scheduler = new ThreadPerStageScheduler(gm);
        
        long startTime = System.currentTimeMillis();
        scheduler.startup();        
        
        scheduler.awaitTermination(timeoutSeconds, TimeUnit.SECONDS);
        long duration = System.currentTimeMillis()-startTime;
        
        if (tempFile.length() != testSize) {
            System.err.println("file produced is not the right length. "+tempFile.length()+" expected "+testSize);
        }
        
        times[timeIdx++] = duration;
        tempFile.delete();
        
        if (duration>(timeoutSeconds*1000)) {
            System.err.println("unable to get duration, timedout");
        } else {                
            int mbytesPerSecond = mBytesPerSecond(duration);                
            System.out.println("BLOB WRITE: pipeLength:"+p+" blockSize:"+(blockSize/1024)+"k  duration:"+duration+" totalBlobRingSize:"+loadedDataPipe.sizeOfBlobRing+" MB/S:"+mbytesPerSecond);               
        }
        if (p>200) {
            p-=100;
        } else {
            if (p>30) {
                p-=10;
            } else {            
                p-=2;
            }
        }
    }
    
    Arrays.sort(times,0, timeIdx);
    int middle = timeIdx/2;
    return times[middle];
    
}
 

import com.ociweb.pronghorn.stage.scheduling.GraphManager; //导入方法依赖的package包/类
private static long fileTapeWriteTest() throws IOException {
    
    long[] times = new long[200]; //guess on size
    int timeIdx = 0;
    
    
    int largestBlock = testSize;
    
    int p = largestPipe;
    while (p>=2) {           
        
        int blockSize = largestBlock/p;
        
        GraphManager gm = new GraphManager();
        
        PipeConfig<RawDataSchema> config = new PipeConfig<RawDataSchema>(RawDataSchema.instance, p, blockSize);
        Pipe<RawDataSchema> loadedDataPipe = new Pipe<RawDataSchema>(config);
        
        
        File tempFile = File.createTempFile("blobWrite", "speedTest");        
        tempFile.deleteOnExit();
        
        PronghornStage s1 = new ByteArrayProducerStage(gm, rawData, loadedDataPipe);        
        PronghornStage s2 = new TapeWriteStage(gm, loadedDataPipe, new RandomAccessFile(tempFile,"rw"));  //NOTE: use rwd/rws to sync flush with every write (much slower)
        
        GraphManager.enableBatching(gm);//lower contention over head and tail
      //  MonitorConsoleStage.attach(gm);
        
        ThreadPerStageScheduler scheduler = new ThreadPerStageScheduler(gm);
        
        long startTime = System.currentTimeMillis();
        scheduler.startup();        
        
        scheduler.awaitTermination(timeoutSeconds, TimeUnit.SECONDS);
        long duration = System.currentTimeMillis()-startTime;
        
        if (tempFile.length() <= testSize) {
            System.err.println("file produced is not the right length. "+tempFile.length()+" expected something larger than "+testSize);
        }
        tempFile.delete();
        
        times[timeIdx++] = duration;
        
        if (duration>(timeoutSeconds*1000)) {
            System.err.println("unable to get duration, timedout");
        } else {                
            int mbytesPerSecond = mBytesPerSecond(duration);                
            System.out.println("TAPE WRITE: pipeLength:"+p+" blockSize:"+(blockSize/1024)+"k  duration:"+duration+" totalBlobRingSize:"+loadedDataPipe.sizeOfBlobRing+" MB/S:"+mbytesPerSecond);               
        }
        if (p>200) {
            p-=100;
        } else {
            if (p>30) {
                p-=10;
            } else {            
                p-=2;
            }
        }
    }
    Arrays.sort(times,0, timeIdx);
    int middle = timeIdx/2;
    return times[middle];
}
 

import com.ociweb.pronghorn.stage.scheduling.GraphManager; //导入方法依赖的package包/类
@Test
public void fileBlobWriteTest() {
    
    try {
    
        GraphManager gm = new GraphManager();
        
        PipeConfig<RawDataSchema> config = new PipeConfig<RawDataSchema>(RawDataSchema.instance, 10, 65536);
        Pipe<RawDataSchema> inputPipe = new Pipe<RawDataSchema>(config);
        
        File f2 = File.createTempFile("roundTipTestB", "dat");
        f2.deleteOnExit();
        
        boolean append = false;
        new ByteArrayProducerStage(gm, rawData, inputPipe);
        FileBlobWriteStage lastStage = new FileBlobWriteStage(gm, inputPipe, append, f2.getAbsolutePath()); //TODO: need a FileBlobRead that can tail a file
                    
        GraphManager.enableBatching(gm);
        
        
        NonThreadScheduler scheduler= new NonThreadScheduler(gm);
        scheduler.startup();
        
        while (!GraphManager.isStageTerminated(gm, lastStage.stageId)) {
        	scheduler.run();
        }
        scheduler.shutdown();
     
                    
        confirmFileContentsMatchTestData(f2);
        
    
    } catch (Exception e) {
        e.printStackTrace();
        fail();
    }
}
 

import com.ociweb.pronghorn.stage.scheduling.GraphManager; //导入方法依赖的package包/类
@Ignore
    public void twoGeneratorsTest() {
        
        FieldReferenceOffsetManager from = buildFROM();        
        assertTrue(null!=from);
        
        PipeConfig busConfig = new PipeConfig(new MessageSchemaDynamic(from), 10, 64);
        
        GraphManager gm = new GraphManager();
        
        
//simple test with no split        
        PronghornStage generator1 = new TestGenerator(gm, seed, iterations, pipe(busConfig));        
        PronghornStage generator2 = new TestGenerator(gm, seed, iterations, pipe(busConfig));   
        
        PronghornStage validateResults = new TestValidator(gm, 
                                                getOutputPipe(gm, generator1), 
                                                getOutputPipe(gm, generator2));
               
        
        //start the timer       
        final long start = System.currentTimeMillis();
        
        GraphManager.enableBatching(gm);
        
        StageScheduler scheduler = new ThreadPerStageScheduler(GraphManager.cloneAll(gm));        
        scheduler.startup();        
        
        //blocks until all the submitted runnables have stopped
       

        //this timeout is set very large to support slow machines that may also run this test.
        boolean cleanExit = scheduler.awaitTermination(TIMEOUT_SECONDS, TimeUnit.SECONDS);
      
        long duration = System.currentTimeMillis()-start;
        
        
    }