Python CallbackScheduler.singletonForTesting方法代码示例

# 需要导入模块: from ufora.native import CallbackScheduler [as 别名]
# 或者: from ufora.native.CallbackScheduler import singletonForTesting [as 别名]
    def __init__(self, numRandVals, numRelaxations, maxForRelax, maxForRand, testAxiomsPath, seed):
        object.__init__(self)
        self.callbackScheduler = CallbackScheduler.singletonForTesting()
        self.callbackSchedulerFactory = self.callbackScheduler.getFactory()
        
        self.numRandVals = numRandVals
        self.numRelaxations = numRelaxations
        self.maxForRelax = maxForRelax
        self.maxForRand = maxForRand
        self.seed = seed

        self.runtime = Runtime.getMainRuntime()
        self.axioms = self.runtime.getAxioms()
        self.typed_fora_compiler = self.runtime.getTypedForaCompiler()

        if testAxiomsPath is not None:
            pathToUse = testAxiomsPath
        else:
            pathToUse = UNIT_TEST_AXIOMS_PATH

        self.axiom_signatures_to_test = self.loadAxiomSignaturesFromFile(pathToUse)

        self.axiom_groups = []
        for i in range(self.axioms.axiomCount):
            self.axiom_groups.append(self.axioms.getAxiomGroupByIndex(i))

        self.symbol_strings = self.loadSymbolStrings()

        numpy.random.seed(seed)

# 需要导入模块: from ufora.native import CallbackScheduler [as 别名]
# 或者: from ufora.native.CallbackScheduler import singletonForTesting [as 别名]
def createWorker_(machineId,
                  viewFactory,
                  callbackSchedulerToUse,
                  threadCount,
                  memoryLimitMb,
                  cacheFunction,
                  pageSizeOverride,
                  disableEventHandler):
    if callbackSchedulerToUse is None:
        callbackSchedulerToUse = CallbackScheduler.singletonForTesting()

    vdm = ForaNative.VectorDataManager(
        callbackSchedulerToUse,
        pageSizeOverride if pageSizeOverride is not None else
        1 * 1024 * 1024 if memoryLimitMb < 1000 else
        5 * 1024 * 1024 if memoryLimitMb < 5000 else
        50 * 1024 * 1024
        )

    vdm.setMemoryLimit(
        int(memoryLimitMb * 1024 * 1024),
        min(int(memoryLimitMb * 1.25 * 1024 * 1024),
            int((memoryLimitMb + 1024 * 2) * 1024 * 1024))
        )

    vdm.setPersistentCacheIndex(
        CumulusNative.PersistentCacheIndex(
            viewFactory.createView(),
            callbackSchedulerToUse
            )
        )

    cache = cacheFunction()

    if disableEventHandler:
        eventHandler = CumulusNative.CumulusWorkerIgnoreEventHandler()
    else:
        eventHandler = CumulusNative.CumulusWorkerHoldEventsInMemoryEventHandler()

    return (
        CumulusNative.CumulusWorker(
            callbackSchedulerToUse,
            CumulusNative.CumulusWorkerConfiguration(
                machineId,
                threadCount,
                CumulusNative.CumulusCheckpointPolicy.None(),
                ExecutionContext.createContextConfiguration(),
                ""
                ),
            vdm,
            cache,
            eventHandler
            ),
        vdm,
        eventHandler
        )

# 需要导入模块: from ufora.native import CallbackScheduler [as 别名]
# 或者: from ufora.native.CallbackScheduler import singletonForTesting [as 别名]
def createClient_(clientId, callbackSchedulerToUse = None):
    if callbackSchedulerToUse is None:
        callbackSchedulerToUse = CallbackScheduler.singletonForTesting()

    vdm = ForaNative.VectorDataManager(callbackSchedulerToUse, 5 * 1024 * 1024)
    vdm.setMemoryLimit(100 * 1024 * 1024, 125 * 1024 * 1024)

    return (
        CumulusNative.CumulusClient(vdm, clientId, callbackSchedulerToUse),
        vdm
        )

# 需要导入模块: from ufora.native import CallbackScheduler [as 别名]
# 或者: from ufora.native.CallbackScheduler import singletonForTesting [as 别名]
    def setUp(self):
        self.callbackScheduler = CallbackScheduler.singletonForTesting()
        self.runtime = Runtime.getMainRuntime()
        self.axioms = self.runtime.getAxioms()
        self.compiler = self.runtime.getTypedForaCompiler()
        self.builtinsAsJOV = FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)

        pyforaPath = os.path.join(os.path.split(pyfora.__file__)[0], "fora/purePython")
        self.purePythonAsJOV = FORANative.JudgmentOnValue.Constant(FORA.importModule(pyforaPath).implVal_)
        
        self.instructionGraph = self.runtime.getInstructionGraph()
        self.reasoner = FORANative.SimpleForwardReasoner(self.compiler, self.instructionGraph, self.axioms)

# 需要导入模块: from ufora.native import CallbackScheduler [as 别名]
# 或者: from ufora.native.CallbackScheduler import singletonForTesting [as 别名]
    def setUp(self):
        self.callbackScheduler = CallbackScheduler.singletonForTesting()

        def createStorage(vdm):
            self.simpleOfflineCache = CumulusNative.SimpleOfflineCache(self.callbackScheduler, 1000000000)
            return self.simpleOfflineCache

        self.evaluator = LocalEvaluator.LocalEvaluator(
            createStorage,
            2000000,
            maxPageSizeInBytes = 100000
            )

        self.oldEvaluator = Evaluator.swapEvaluator(self.evaluator)

# 需要导入模块: from ufora.native import CallbackScheduler [as 别名]
# 或者: from ufora.native.CallbackScheduler import singletonForTesting [as 别名]
def createWorkersAndClients(
            workerCount,
            clientCount,
            viewFactory = None,
            memoryLimitMb = 100,
            threadCount = 2,
            callbackSchedulerToUse = None
            ):
    if callbackSchedulerToUse is None:
        callbackSchedulerToUse = CallbackScheduler.singletonForTesting()

    if viewFactory is None:
        viewFactory = createInMemorySharedStateViewFactory(callbackSchedulerToUse)

    workersVdmsAndEventHandlers = [
        createWorker(
            machineId(ix),
            viewFactory,
            memoryLimitMb = memoryLimitMb,
            threadCount=threadCount,
            callbackSchedulerToUse = callbackSchedulerToUse
            ) for ix in range(workerCount)
        ]

    clientsAndVdms = [
        createClient(
            clientId(ix),
            callbackSchedulerToUse = callbackSchedulerToUse
            )
        for ix in range(clientCount)
        ]

    for ix1 in range(len(workersVdmsAndEventHandlers)):
        workersVdmsAndEventHandlers[ix1][0].startComputations()

    for ix1 in range(len(workersVdmsAndEventHandlers)-1):
        for ix2 in range(ix1 + 1, len(workersVdmsAndEventHandlers)):
            worker1Channel1, worker2Channel1 = StringChannelNative.InMemoryStringChannel(callbackSchedulerToUse)
            worker1Channel2, worker2Channel2 = StringChannelNative.InMemoryStringChannel(callbackSchedulerToUse)
            workersVdmsAndEventHandlers[ix1][0].addMachine(machineId(ix2), [worker1Channel1, worker1Channel2], ForaNative.ImplValContainer(), callbackSchedulerToUse)
            workersVdmsAndEventHandlers[ix2][0].addMachine(machineId(ix1), [worker2Channel1, worker2Channel2], ForaNative.ImplValContainer(), callbackSchedulerToUse)

    for ix1 in range(len(workersVdmsAndEventHandlers)):
        for ix2 in range(len(clientsAndVdms)):
            workerChannel1, clientChannel1 = StringChannelNative.InMemoryStringChannel(callbackSchedulerToUse)
            workerChannel2, clientChannel2 = StringChannelNative.InMemoryStringChannel(callbackSchedulerToUse)
            workersVdmsAndEventHandlers[ix1][0].addCumulusClient(clientId(ix2), [workerChannel1, workerChannel2], ForaNative.ImplValContainer(), callbackSchedulerToUse)
            clientsAndVdms[ix2][0].addMachine(machineId(ix1), [clientChannel1, clientChannel2], ForaNative.ImplValContainer(), callbackSchedulerToUse)

    return workersVdmsAndEventHandlers, clientsAndVdms, viewFactory

# 需要导入模块: from ufora.native import CallbackScheduler [as 别名]
# 或者: from ufora.native.CallbackScheduler import singletonForTesting [as 别名]
    def __init__(self,
            inMemory,
            port = None,
            cachePathOverride = '',
            maxOpenFiles = 256,
            inMemChannelFactoryFactory = None,
            maxLogFileSizeMb = 10,
            pingInterval = None):

        self.inMemory = inMemory
        self.manager = None
        self.callbackScheduler = CallbackScheduler.singletonForTesting()

        if self.inMemory:
            self.manager = SharedStateService.KeyspaceManager(
                10001,
                1,
                cachePathOverride=cachePathOverride,
                pingInterval = IN_MEMORY_HARNESS_PING_INTERVAL if pingInterval is None else pingInterval,
                maxOpenFiles=maxOpenFiles,
                maxLogFileSizeMb=maxLogFileSizeMb
                )

            #although named otherwise InMemoryChannelFactory is actually a factory for a channelFactory
            # or a channelFactoryFactory

            channelFactoryFactory = inMemChannelFactoryFactory if inMemChannelFactoryFactory is not None \
                    else InMemoryChannelFactory.InMemoryChannelFactory

            logging.info(channelFactoryFactory)
            self.channelFactory = channelFactoryFactory(self.callbackScheduler, self.manager)
            self.viewFactory = ViewFactory.ViewFactory(self.channelFactory)
        else:
            class Settings(object):
                callbackScheduler = self.callbackScheduler

            assert port is not None

            self.service = SharedStateService.SharedStateService(
                    self.callbackScheduler,
                    cachePathOverride=cachePathOverride,
                    port=port
                    )

            self.service.startService()
            self.service.blockUntilListening()

            self.viewFactory = ViewFactory.ViewFactory.TcpViewFactory(self.callbackScheduler, "localhost", port)

# 需要导入模块: from ufora.native import CallbackScheduler [as 别名]
# 或者: from ufora.native.CallbackScheduler import singletonForTesting [as 别名]
    def setUp(self):
        self.callbackScheduler = CallbackScheduler.singletonForTesting()
        self.runtime = Runtime.getMainRuntime()
        self.axioms = self.runtime.getAxioms()
        self.native_runtime = self.runtime.getTypedForaCompiler()
        self.vals_to_test = self.loadValuesFromFile(os.path.join(os.path.split(__file__)[0],
                                                        "AxiomJOA_test.txt"))

        self.evaluator = LocalEvaluator.LocalEvaluator(
                            lambda vdm: CumulusNative.SimpleOfflineCache(self.callbackScheduler, 1000000000),
                            10000000,
                            maxPageSizeInBytes = 100000
                            )
        self.oldEvaluator = Evaluator.swapEvaluator(self.evaluator)

        self.knownModulesAsConstantJOVs = dict()
        self.knownModulesAsConstantJOVs["builtin"] = \
                FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)

# 需要导入模块: from ufora.native import CallbackScheduler [as 别名]
# 或者: from ufora.native.CallbackScheduler import singletonForTesting [as 别名]
def createInMemorySharedStateViewFactory(callbackSchedulerToUse = None):
    if callbackSchedulerToUse is None:
        callbackSchedulerToUse = CallbackScheduler.singletonForTesting()

    sharedStateManager = SharedStateService.KeyspaceManager(
        10001,
        1,
        cachePathOverride="",
        pingInterval = IN_MEMORY_CLUSTER_SS_PING_INTERVAL,
        maxOpenFiles=100
        )

    sharedStateChannelFactory = (
        InMemorySharedStateChannelFactory.InMemoryChannelFactory(
            callbackSchedulerToUse.getFactory().createScheduler("SharedState", 1),
            sharedStateManager
            )
        )

    return ViewFactory.ViewFactory(sharedStateChannelFactory)

# 需要导入模块: from ufora.native import CallbackScheduler [as 别名]
# 或者: from ufora.native.CallbackScheduler import singletonForTesting [as 别名]
def createWorker(machineId, viewFactory, callbackSchedulerToUse = None, threadCount = 2, memoryLimitMb = 100):
    if callbackSchedulerToUse is None:
        callbackSchedulerToUse = CallbackScheduler.singletonForTesting()

    vdm = ForaNative.VectorDataManager(callbackSchedulerToUse, 5 * 1024 * 1024)
    vdm.setMemoryLimit(
        int(memoryLimitMb * 1024 * 1024),
        min(int(memoryLimitMb * 1.25 * 1024 * 1024),
            int((memoryLimitMb + 1024 * 2) * 1024 * 1024))
        )

    vdm.setPersistentCacheIndex(
        CumulusNative.PersistentCacheIndex(
            viewFactory.createView(),
            callbackSchedulerToUse
            )
        )

    cache = CumulusNative.SimpleOfflineCache(callbackSchedulerToUse, 1000 * 1024 * 1024)

    eventHandler = CumulusNative.CumulusWorkerHoldEventsInMemoryEventHandler()

    return (
        CumulusNative.CumulusWorker(
            callbackSchedulerToUse,
            CumulusNative.CumulusWorkerConfiguration(
                machineId,
                threadCount,
                CumulusNative.CumulusCheckpointPolicy.None(),
                ExecutionContext.createContextConfiguration(),
                ""
                ),
            vdm,
            cache,
            eventHandler
            ),
        vdm,
        eventHandler
        )

# 需要导入模块: from ufora.native import CallbackScheduler [as 别名]
# 或者: from ufora.native.CallbackScheduler import singletonForTesting [as 别名]
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import unittest
import time
import ufora.FORA.python.FORA as FORA
import ufora.cumulus.test.InMemoryCumulusSimulation as InMemoryCumulusSimulation
import ufora.distributed.S3.InMemoryS3Interface as InMemoryS3Interface
import ufora.native.CallbackScheduler as CallbackScheduler
import ufora.test.PerformanceTestReporter as PerformanceTestReporter
import ufora.FORA.python.Runtime as Runtime

callbackScheduler = CallbackScheduler.singletonForTesting()

class BigboxStringPerformanceTest(unittest.TestCase):
    def computeUsingSeveralWorkers(self, *args, **kwds):
        return InMemoryCumulusSimulation.computeUsingSeveralWorkers(*args, **kwds)

    def stringCreationAndSumTest(self, totalStrings, workers, threadsPerWorker, testName):
        s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory()

        #we wish we could actually test that we achieve saturation here but we can't yet.
        text = """Vector.range(%s, String).sum(size)""" % totalStrings

        t0 = time.time()

        _, simulation = \
            self.computeUsingSeveralWorkers(

# 需要导入模块: from ufora.native import CallbackScheduler [as 别名]
# 或者: from ufora.native.CallbackScheduler import singletonForTesting [as 别名]
    def __init__(self,
                workerCount,
                clientCount,
                memoryPerWorkerMB,
                threadsPerWorker,
                s3Service,
                objectStore=None,
                callbackScheduler=None,
                sharedStateViewFactory=None,
                ioTaskThreadOverride=None,
                useInMemoryCache=True,
                channelThroughputMBPerSecond=None,
                pageSizeOverride=None,
                disableEventHandler=False,
                machineIdHashSeed=None
                ):
        self.useInMemoryCache = useInMemoryCache
        self.machineIdHashSeed = machineIdHashSeed

        if not self.useInMemoryCache:
            self.diskCacheCount = 0
            if os.getenv("CUMULUS_DATA_DIR") is None:
                self.diskCacheStorageDir = tempfile.mkdtemp()
            else:
                self.diskCacheStorageDir = os.path.join(
                    os.getenv("CUMULUS_DATA_DIR"),
                    str(uuid.uuid4())
                    )
        self.ioTaskThreadOverride = ioTaskThreadOverride
        self.workerCount = 0
        self.disableEventHandler = disableEventHandler
        self.clientCount = 0
        self.memoryPerWorkerMB = memoryPerWorkerMB
        self.threadsPerWorker = threadsPerWorker
        self.s3Service = s3Service
        self.objectStore = objectStore
        if self.objectStore is None:
            s3 = s3Service()
            if isinstance(s3, InMemoryS3Interface.InMemoryS3Interface):
                objectStoreBucket = "object_store_bucket"
                s3.setKeyValue(objectStoreBucket, 'dummyKey', 'dummyValue')
                s3.deleteKey(objectStoreBucket, 'dummyKey')
            else:
                objectStoreBucket = Setup.config().userDataS3Bucket
            self.objectStore = S3ObjectStore.S3ObjectStore(
                s3Service,
                objectStoreBucket,
                prefix="test/")
        self.callbackScheduler = callbackScheduler or CallbackScheduler.singletonForTesting()
        self.sharedStateViewFactory = (
            sharedStateViewFactory or createInMemorySharedStateViewFactory(self.callbackScheduler)
            )
        self.channelThroughputMBPerSecond = channelThroughputMBPerSecond
        self.resultVDM = ForaNative.VectorDataManager(self.callbackScheduler, 5 * 1024 * 1024)
        self.pageSizeOverride = pageSizeOverride

        self.rateLimitedChannelGroupsForEachListener = []
        self.workersVdmsAndEventHandlers = []
        self.machineIds = []
        self.machineIdsEverAllocated = 0
        self.clientsAndVdms = []
        self.loadingServices = []
        self.clientTeardownGates = []
        self.workerTeardownGates = []


        for ix in range(workerCount):
            self.addWorker()
        for ix in range(clientCount):
            self.addClient()

        if clientCount:
            self.listener = self.getClient(0).createListener()
        else:
            self.listener = None

# 需要导入模块: from ufora.native import CallbackScheduler [as 别名]
# 或者: from ufora.native.CallbackScheduler import singletonForTesting [as 别名]
 def setUp(self):
     self.runtime = Runtime.getMainRuntime()
     self.axioms = self.runtime.getAxioms()
     self.native_runtime = self.runtime.getTypedForaCompiler()
     self.vdm = FORANative.VectorDataManager(CallbackScheduler.singletonForTesting(), 10000000)

# 需要导入模块: from ufora.native import CallbackScheduler [as 别名]
# 或者: from ufora.native.CallbackScheduler import singletonForTesting [as 别名]
 def setUp(self):
     self.callbackScheduler = CallbackScheduler.singletonForTesting()
     self.runtime = Runtime.getMainRuntime()
     self.axioms = self.runtime.getAxioms()
     self.compiler = self.runtime.getTypedForaCompiler()
     self.builtinsAsJOV = FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)

# 需要导入模块: from ufora.native import CallbackScheduler [as 别名]
# 或者: from ufora.native.CallbackScheduler import singletonForTesting [as 别名]
 def setUp(self):
     self.callbackScheduler = CallbackScheduler.singletonForTesting()