Python test.PerformanceTestReporter类代码示例

    def dataCreationTest(self, totalMB, workers, threadsPerWorker, testName):
        s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory()

        #we wish we could actually test that we achieve saturation here but we can't yet.
        text = """size(Vector.range(%s, {_*_}))""" % (totalMB * 1024 * 1024 / 8)

        t0 = time.time()

        result,simulation = self.computeUsingSeveralWorkers(
                "1+1",
                s3,
                workers,
                timeout = 120,
                memoryLimitMb = 55 * 1024 / workers,
                threadCount = threadsPerWorker,
                returnSimulation = True,
                useInMemoryCache = False
                )

        try:
            t0 = time.time()
            result = simulation.compute(text, timeout=120)
            totalTimeToReturnResult = time.time() - t0

            PerformanceTestReporter.recordTest(testName, totalTimeToReturnResult, None)
        finally:
            simulation.teardown()

    def stringCreationAndSumTest(self, totalStrings, workers, threadsPerWorker, testName):
        s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory()

        #we wish we could actually test that we achieve saturation here but we can't yet.
        text = """Vector.range(%s, String).sum(size)""" % totalStrings

        t0 = time.time()

        _, simulation = \
            self.computeUsingSeveralWorkers(
                "1+1",
                s3,
                workers,
                timeout = 240,
                memoryLimitMb = 55 * 1024 / workers,
                threadCount = threadsPerWorker,
                returnSimulation = True,
                useInMemoryCache = False
                )

        try:
            t0 = time.time()
            result = simulation.compute(text, timeout=240)
            totalTimeToReturnResult = time.time() - t0

            PerformanceTestReporter.recordTest(testName, totalTimeToReturnResult, None)
        finally:
            simulation.teardown()

    def test_byteToStringAndBackInDifferentPatterns(self):
        s3 = ActualS3Interface.ActualS3InterfaceFactory()

        setupText = (
            """
            let ds = Vector.range(3000000000, {UInt8(_%100)});

            let dat = Vector.range(100, fun(block) {
                Vector.range(1000000, fun(o) { let base = block * 10000000 + o * 10; (base, base + 10) })
                });

            (ds, dat, dat.sum())
            """
            )

        setupResults, simulation = self.computeUsingSeveralWorkers(
                setupText,
                s3,
                1,
                memoryLimitMb=45 * 1024,
                threadCount=30,
                returnSimulation=True,
                ioTaskThreadOverride=8,
                useInMemoryCache=False,
                timeout=30,
                objectStore=self.createObjectStore(s3)
                )

        try:
            ds, dat, datSum = setupResults.asResult.result

            t0 = time.time()
            result = simulation.compute(
                "size(datSum ~~ { ds[_[0],_[1]].dataAsString }) == size(datSum)",
                timeout=120,
                ds=ds,
                dat=dat,
                datSum=datSum
                )
            PerformanceTestReporter.recordTest(
                "python.BigBox.DataAsString.FlatVector",
                time.time() - t0,
                None
                )

            t0 = time.time()
            result = simulation.compute(
                "size(dat ~~ {_ ~~ { ds[_[0],_[1]].dataAsString } }) == size(dat)",
                timeout=120,
                ds=ds,
                dat=dat,
                datSum=datSum
                )
            PerformanceTestReporter.recordTest(
                "python.BigBox.DataAsString.NestedVector",
                time.time() - t0,
                None
                )
        finally:
            simulation.teardown()

    def test_transposeToColumnMajor(self):
        s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory()

        nRows = 100000
        nColumns = 50

        result, simulation = InMemoryCumulusSimulation.computeUsingSeveralWorkers(
            self.transposeSetupScript(nRows, nColumns),
            s3, 1, timeout = 300, memoryLimitMb = 45 * 1024, threadCount = 30,
            returnSimulation = True, useInMemoryCache = False)

        try:
            self.assertTrue(result.isResult())

            rowMajor = result.asResult.result

            t0 = time.time()
            result = simulation.compute(
                self.transposeRowMajorToColumnMajorScript(nRows, nColumns),
                timeout = 500,
                rowMajor = rowMajor
                )
            totalTimeToReturnResult = time.time() - t0

            self.assertTrue(result.isResult())

            PerformanceTestReporter.recordTest(
                "algorithms.text.transposeRowMajorToColumnMajor.%srows_%scolumns" % (nRows, nColumns),
                totalTimeToReturnResult, None)

        finally:
            simulation.teardown()

    def downloadTaxiData(self,
                         filecount,
                         parse=False,
                         workers=1,
                         threadsPerWorker=30,
                         downloaderThreads=8):
        s3 = ActualS3Interface.ActualS3InterfaceFactory()
        
        bucketName = self.getTestDataBucket()

        result, simulation = self.computeUsingSeveralWorkers(
            "1+1",
            s3,
            workers,
            memoryLimitMb=45 * 1024 / workers,
            threadCount=threadsPerWorker,
            returnSimulation=True,
            ioTaskThreadOverride=downloaderThreads,
            useInMemoryCache=False,
            objectStore=self.createObjectStore(s3)
            )

        try:
            dsText = (
                """let ds = """ + "+".join([
                    'datasets.s3("%s", "taxi_month_%s.csv")' % (bucketName, ix) for ix in range(1, filecount+1)
                    ]) + ";"
                )

            text = dsText + "(ds, ds.sum(), size(ds))"

            downloadTimeStart = time.time()
            result = simulation.compute(text, timeout=240)
            self.assertTrue(result.isResult())
            downloadTimeEnd = time.time()
            ds, dsSum, bytecount = result.asResult.result

            if parse:
                parseTimeStart = time.time()
                result = simulation.compute("size(parsing.csv(ds))", timeout=240, ds=ds)
                parseTimeEnd = time.time()

                self.assertTrue(result.isResult())

                PerformanceTestReporter.recordTest(
                    "python.BigBox.LargeS3.ParseTaxidata." + str(filecount),
                    parseTimeEnd - parseTimeStart,
                    None
                    )
            else:
                bytecount = bytecount.pyval
                PerformanceTestReporter.recordTest(
                    "python.BigBox.LargeS3.TaxiSecondsPerGB." + str(filecount),
                    (downloadTimeEnd - downloadTimeStart) / (bytecount / 1024 / 1024.0 / 1024.0),
                    None
                    )
        finally:
            simulation.teardown()

 def testMemoryUpdate(self):
     measurementTime = self.measurementTime
     for cores in self.coreList:
         for allocSize in self.allocSizeList:
             PerformanceTestReporter.recordTest(
                 "python.BigBox.MemoryUpdate.SecondsPerGB.%sCore_%sMB" % (cores, allocSize),
                 self.measureMemoryUpdatePerformance(measurementTime, 1024 * 1024 * allocSize, cores),
                 None
                 )

 def testMmapAllocation(self):
     measurementTime = self.measurementTime
     for cores in self.coreList:
         for allocSize in self.allocSizeList:
             PerformanceTestReporter.recordTest(
                 "python.BigBox.MmapAlloc.SecondsPerGB.%sCore_%sMB" % (cores, allocSize),
                 self.measureMmapPerformance(measurementTime, 1024 * 1024 * allocSize, cores, False),
                 None
                 )

    def stringToInt64ParsingTest(self, threads, testName):
        s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory()

        #we wish we could actually test that we achieve saturation here but we can't yet.
        text = """
            let doALoop = fun(x) {
                //pass 's' through a vector so that the compiler can't tell what it is
                let s = ["2013"][0];

                let res = 0
                for ix in sequence(x) {
                    if (ix == 0)
                        s = s + String(ix)

                    res = res + Int64(s) + ix
                    }
                res
                };

            Vector.range(__thread_count__) ~~ {doALoop(20000000 + _)}
            """.replace("__thread_count__", str(threads))

        _, simulation = \
            self.computeUsingSeveralWorkers(
                "1+1",
                s3,
                1,
                timeout = 240,
                memoryLimitMb = 55 * 1024,
                threadCount = 30,
                returnSimulation = True,
                useInMemoryCache = False
                )

        t0 = time.time()

        _, simulation = \
            self.computeUsingSeveralWorkers(
                "1+1",
                s3,
                1,
                timeout = 240,
                memoryLimitMb = 55 * 1024,
                threadCount = 30,
                returnSimulation = True,
                useInMemoryCache = False
                )

        try:
            t0 = time.time()
            result = simulation.compute(text, timeout=240)
            totalTimeToReturnResult = time.time() - t0

            PerformanceTestReporter.recordTest(testName, totalTimeToReturnResult, None)
        finally:
            simulation.teardown()

    def test_cant_report_nonsensical_timing(self):
        tempDir = tempfile.mkdtemp()
        tempFile = os.path.join(tempDir, "data.json")

        with SetEnv(
                PerformanceTestReporter.TEST_DATA_LOCATION_ENVIRONMENT_VARIABLE, 
                tempFile
                ):
            with self.assertRaises(Exception):
                PerformanceTestReporter.recordTest("test1","not a float",None)

    def test_knows_is_reporting(self):
        with SetEnv(
                PerformanceTestReporter.TEST_DATA_LOCATION_ENVIRONMENT_VARIABLE, 
                None):
            self.assertFalse(PerformanceTestReporter.isCurrentlyTesting())

        with SetEnv(
                PerformanceTestReporter.TEST_DATA_LOCATION_ENVIRONMENT_VARIABLE, 
                "./testResults.json"):
            self.assertTrue(PerformanceTestReporter.isCurrentlyTesting())

    def gbmRegressionFittingTest(self, nRows, nColumns, depth, nThreads, maxBoosts):
        testName = self.getTestName(nRows, nColumns, depth, maxBoosts, nThreads)

        s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory()

        result, simulation = InMemoryCumulusSimulation.computeUsingSeveralWorkers(
                        self.dataGenerationScript(nRows, nColumns),
                        s3,
                        1,
                        timeout = 360,
                        memoryLimitMb = 30 * 1024,
                        threadCount = nThreads,
                        returnSimulation = True,
                        useInMemoryCache = False
                        )
        try:
            self.assertTrue(result.isResult())

            dfPredictors, dfResponse = result.asResult.result

            fitter = simulation.compute(
                self.regressionScript(depth, 1),
                timeout = 360,
                dfResponse = dfResponse,
                dfPredictors = dfPredictors
                ).asResult.result

            t0 = time.time()

            for nBoosts in range(1, maxBoosts):
                testName = self.getTestName(nRows, nColumns, depth, nBoosts, nThreads)

                predictions = simulation.compute(
                    "fitter.predictionsAndPseudoresiduals()",
                    timeout = 360,
                    fitter = fitter
                    ).asResult.result
                totalTimeToReturnResult = time.time() - t0

                PerformanceTestReporter.recordTest(
                    testName + "_predict", totalTimeToReturnResult, None)

                fitter = simulation.compute(
                    "fitter.nextGivenPredictions(predictions)",
                    timeout = 360,
                    fitter = fitter,
                    predictions = predictions
                    ).asResult.result
                totalTimeToReturnResult = time.time() - t0

                PerformanceTestReporter.recordTest(
                    testName, totalTimeToReturnResult, None)

        finally:
            simulation.teardown()

    def test_disk_scans(self):
        s3 = ActualS3Interface.ActualS3InterfaceFactory()
        objectStore = S3ObjectStore.S3ObjectStore(
            s3,
            Setup.config().userDataS3Bucket,
            prefix="test_object_cache/"
            )

        _, simulation = InMemoryCumulusSimulation.computeUsingSeveralWorkers(
            "1+1",
            s3,
            1,
            memoryLimitMb=1 * 1024,
            threadCount=30,
            returnSimulation=True,
            ioTaskThreadOverride=8,
            objectStore=objectStore,
            useInMemoryCache=False  #use an actual disk cache for this
            )

        try:
            gigabytes = 8

            t0 = time.time()

            resultVectors = []
            for ix in range(gigabytes):
                result = simulation.compute("Vector.range(125000000 + %s)" % ix, timeout=120)
                resultVectors.append(result.asResult.result)

            t1 = time.time()

            intResults = []
            for vec in resultVectors:
                result = simulation.compute("v.sum()", timeout = 120, v=vec)
                intResults.append(result.asResult.result.pyval)


            self.assertTrue(len(intResults) == gigabytes)

            PerformanceTestReporter.recordTest(
                "python.BigBox.Disk.Write.10GB",
                t1 - t0,
                None
                )

            PerformanceTestReporter.recordTest(
                "python.BigBox.Disk.WriteAndScan.%sGB" % gigabytes,
                time.time() - t0,
                None
                )
        finally:
            simulation.teardown()

    def largeDatasetJoinTest(self, mbOfData, columns, threads, machineCount, ratio = .5):
        s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory()

        t0 = time.time()

        result, simulation = InMemoryCumulusSimulation.computeUsingSeveralWorkers(
                        self.dataGenerationScript(mbOfData, columns),
                        s3,
                        machineCount,
                        timeout = 360,
                        memoryLimitMb = mbOfData / ratio / machineCount,
                        #channelThroughputMBPerSecond = 100.0,
                        threadCount = threads,
                        returnSimulation = True,
                        useInMemoryCache = False,
                        disableEventHandler = True
                        )

        try:
            self.assertTrue(result.isResult())

            data = result.asResult.result

            joinScript = """
                    let leftDF = dataframe.DataFrame(data[,size(data)/2])
                    let rightDF = dataframe.DataFrame(data[size(data)/2,])

                    size(leftDF.join(rightDF, on: "C0", how: `outer, chunkSize: 1000000, areSorted:true))
                    """

            t0 = time.time()
            result = simulation.compute(
                joinScript,
                timeout=1080,
                data=data
                )
            totalTimeToReturnResult = time.time() - t0

            logging.info("Total time to join: %s", totalTimeToReturnResult)

            self.assertTrue(result.isResult(), result)

            PerformanceTestReporter.recordTest(
                "algorithms.Join.inMemory_%sMB_%scols_%sthreads_%smachines" %
                    (mbOfData, columns,threads,machineCount),
                totalTimeToReturnResult,
                None
                )
        finally:
            dfResponse = None
            dfPredictors = None
            result = None
            simulation.teardown()

    def test_effectiveParallelism(self):
        s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory()

        #do a burn-in run
        self.computeUsingSeveralWorkers("""
                let v = Vector.range(5000000, { (1,_) } );

                let f = fun(ix) {
                    let res = 0
                    for x in sequence( (ix - 2000) >>> 0, ix )
                        res = res + size(v[x])
                    res
                    }

                Vector.range(size(v),  f).sum()

                """, s3, 2, wantsStats = True, timeout=240, memoryLimitMb=500
                )[1]

        t0 = time.time()

        stats = self.computeUsingSeveralWorkers("""
                let v = Vector.range(5000000, { (1,_) } );

                let f = fun(ix) {
                    let res = 0
                    for x in sequence( (ix - 2000) >>> 0, ix )
                        res = res + size(v[x])
                    res
                    }

                Vector.range(size(v),  f).sum()

                """, s3, 2, wantsStats = True, timeout=240, memoryLimitMb=500
                )[1]

        timeElapsed = time.time() - t0
        totalTime = stats.timeSpentInInterpreter + stats.timeSpentInCompiler
        effParallelism = totalTime / timeElapsed

        PerformanceTestReporter.recordTest(
            "python.cumulus.EffectiveParallelism.elapsed",
            timeElapsed,
            None
            )

        PerformanceTestReporter.recordTest(
            "python.cumulus.EffectiveParallelism.effectiveCores",
            effParallelism,
            {},
            units='count'
            )

    def test_throughputThrowsIfNonePassed(self):
        tempDir = tempfile.mkdtemp()
        tempFile = os.path.join(tempDir, "data.json")

        with SetEnv(
                PerformanceTestReporter.TEST_DATA_LOCATION_ENVIRONMENT_VARIABLE, 
                tempFile
                ):
            def testFunOfN(n):
                raise PerformanceTestReporter.TimedOutException("timed out!!")

            with self.assertRaises(AssertionError):
                PerformanceTestReporter.testThroughput(
                    "test1", testFunOfN = testFunOfN)