Python Setup.config方法代码示例

# 需要导入模块: from ufora.config import Setup [as 别名]
# 或者: from ufora.config.Setup import config [as 别名]
    def __init__(self, callbackScheduler, cachePathOverride=None, port=None):
        self.callbackScheduler = callbackScheduler
        port = Setup.config().sharedStatePort
        logging.info("Initializing SharedStateService with port = %s", port)

        self.cachePath = cachePathOverride if cachePathOverride is not None else \
                         Setup.config().sharedStateCache

        if self.cachePath != '' and not os.path.exists(self.cachePath):
            os.makedirs(self.cachePath)

        CloudService.Service.__init__(self)
        self.socketServer = SimpleServer.SimpleServer(port)
        self.keyspaceManager = KeyspaceManager(
            0,
            1,
            pingInterval=120,
            cachePathOverride=cachePathOverride
            )


        self.socketServer._onConnect = self.onConnect
        self.socketServerThread = ManagedThread.ManagedThread(target=self.socketServer.start)
        self.logfilePruneThread = ManagedThread.ManagedThread(target=self.logFilePruner)

        self.stoppedFlag = threading.Event()

# 需要导入模块: from ufora.config import Setup [as 别名]
# 或者: from ufora.config.Setup import config [as 别名]
    def startSharedState(self):
        cacheDir = Setup.config().getConfigValue(
            "SHARED_STATE_CACHE",
            os.path.join(Setup.config().fakeAwsBaseDir, 'ss_cache')
            )

        logging.info("Starting shared state with cache dir '%s' and log file '%s'",
                     cacheDir,
                     self.sharedStateLogFile)

        with DirectoryScope.DirectoryScope(self.sharedStatePath):
            args = ['forever',
                    '--killSignal', 'SIGTERM',
                    '-l', self.sharedStateLogFile,
                    'start',
                    '-c', 'python', self.sharedStateMainline,
                    '--cacheDir', cacheDir,
                    '--logging', 'info'
                   ]

            def sharedStateStdout(msg):
                logging.info("SHARED STATE OUT> %s", msg)
            def sharedStateStderr(msg):
                logging.info("SHARED STATE ERR> %s", msg)

            startSharedState = SubprocessRunner.SubprocessRunner(
                args,
                sharedStateStdout,
                sharedStateStderr,
                dict(os.environ)
                )
            startSharedState.start()
            startSharedState.wait(60.0)
            startSharedState.stop()

# 需要导入模块: from ufora.config import Setup [as 别名]
# 或者: from ufora.config.Setup import config [as 别名]
def constructVDM(
        callbackScheduler,
        vectorRamCacheBytes = None,
        maxRamCacheBytes = None,
        maxVectorChunkSize = None
        ):
    if vectorRamCacheBytes is None:
        vectorRamCacheBytes = Setup.config().cumulusVectorRamCacheMB * 1024 * 1024

    if maxRamCacheBytes is None:
        maxRamCacheBytes = Setup.config().cumulusMaxRamCacheMB * 1024 * 1024

    if maxVectorChunkSize is None:
        maxVectorChunkSize = Setup.config().maxPageSizeInBytes

        if maxVectorChunkSize > vectorRamCacheBytes / 32:
            logging.info(
                "VDM constructor specified a chunk size of %s MB " +
                "and a memory size of %s MB. Reducing the chunk size because its too large",
                vectorRamCacheBytes / 1024.0 / 1024.0,
                maxVectorChunkSize / 1024.0 / 1024.0
                )

            maxVectorChunkSize = vectorRamCacheBytes / 32

    logging.info("Creating a VDM with %s MB of memory and %s max vector size",
        vectorRamCacheBytes / 1024.0 / 1024.0,
        maxVectorChunkSize / 1024.0 / 1024.0
        )

    VDM = FORANative.VectorDataManager(callbackScheduler, maxVectorChunkSize)
    VDM.setMemoryLimit(vectorRamCacheBytes, maxRamCacheBytes)

    return VDM

# 需要导入模块: from ufora.config import Setup [as 别名]
# 或者: from ufora.config.Setup import config [as 别名]
    def __init__(self):
        callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
        self.callbackScheduler = callbackSchedulerFactory.createScheduler("Simulator", 1)

        self.uforaPath = os.path.abspath(os.path.join(os.path.dirname(__file__), '../'))

        self.sharedStatePath = os.path.join(self.uforaPath, 'distributed/SharedState')
        self.sharedStateMainline = os.path.join(self.sharedStatePath, 'sharedStateMainline.py')

        self.gatewayServiceMainline = os.path.join(self.uforaPath, 'scripts/init/ufora-gateway.py')

        self.webPath = os.path.join(self.uforaPath, 'web/relay')
        self.relayScript = os.path.join(self.webPath, 'server.coffee')

        self.relayPort = Setup.config().relayPort
        self.relayHttpsPort = Setup.config().relayHttpsPort
        self.sharedStatePort = Setup.config().sharedStatePort
        self.restApiPort = Setup.config().restApiPort
        self.subscribableWebObjectsPort = Setup.config().subscribableWebObjectsPort

        #create an OutOfProcessDownloader so we can execute commands like 'forever'
        #from there, instead of forking from the main process (which can run out of memory)
        self.processPool = OutOfProcessDownloader.OutOfProcessDownloaderPool(1)


        self.desirePublisher = None
        self._connectionManager = None

# 需要导入模块: from ufora.config import Setup [as 别名]
# 或者: from ufora.config.Setup import config [as 别名]
    def createServiceAndServiceThread(self):
        config = Setup.config()
        config.cumulusMaxRamCacheMB = self.cumulusMaxRamCacheSizeOverride
        config.cumulusVectorRamCacheMB = self.cumulusVectorRamCacheSizeOverride
        config.cumulusServiceThreadCount = self.cumulusThreadCountOverride
        config.cumulusDiskCacheStorageSubdirectory = str(uuid.uuid4())

        ownAddress = str(uuid.uuid4())
        callbackScheduler = self.callbackSchedulerFactory.createScheduler(
            "InMemoryClusterChild",
            1)
        channelListener = self.createMultiChannelListener(
            callbackScheduler,
            [Setup.config().cumulusControlPort, Setup.config().cumulusDataPort],
            ownAddress)
        service = CumulusService.CumulusService(
            ownAddress=ownAddress,
            channelListener=channelListener,
            channelFactory=self.channelManager.createChannelFactory(),
            eventHandler=CumulusNative.CumulusWorkerHoldEventsInMemoryEventHandler(),
            callbackScheduler=callbackScheduler,
            diagnosticsDir=None,
            config=config,
            viewFactory=self.sharedStateViewFactory
            )
        service.startService(lambda: None)
        return service

# 需要导入模块: from ufora.config import Setup [as 别名]
# 或者: from ufora.config.Setup import config [as 别名]
    def __init__(self, vdm, offlineCache):
        Stoppable.Stoppable.__init__(self)
        self.dependencies_ = TwoWaySetMap.TwoWaySetMap()
        self.vdm_ = vdm
        self.offlineCache_ = offlineCache
        self.finishedValues_ = {}
        self.intermediates_ = {}
        self.lock_ = threading.RLock()
        self.completable_ = Queue.Queue()
        self.timesComputed = 0
        self.computingContexts_ = {}
        self.computingContexts_t0_ = {}
        self.isSplit_ = set()
        self.watchers_ = {}
        self.contexts_ = []

        self.inProcessDownloader = (
            OutOfProcessDownloader.OutOfProcessDownloaderPool(
                Setup.config().cumulusServiceThreadCount,
                actuallyRunOutOfProcess = False
                )
            )

        self.threads_ = []
        self.isActive = True
        #setup the primary cache object, and set its worker threads going
        for threadIx in range(Setup.config().cumulusServiceThreadCount):
            workerThread = ManagedThread.ManagedThread(target = self.threadWorker)
            workerThread.start()
            self.threads_.append(workerThread)

# 需要导入模块: from ufora.config import Setup [as 别名]
# 或者: from ufora.config.Setup import config [as 别名]
 def generateTestConfigFileBody_(self):
     return ("ROOT_DATA_DIR = %s\n"
             "BASE_PORT = %s\n"
             "FORA_MAX_MEM_MB = %s\n"
             ) % (
             Setup.config().rootDataDir,
             Setup.config().basePort,
             "10000" if multiprocessing.cpu_count() <= 8 else "60000"
             )

# 需要导入模块: from ufora.config import Setup [as 别名]
# 或者: from ufora.config.Setup import config [as 别名]
def runPythonUnitTests_(args, testFilter, testGroupName, testFiles):
    testArgs = ["dummy"]

    if args.testHarnessVerbose or args.list:
        testArgs.append('--nocaptureall')

    testArgs.append('--verbosity=0')

    if not args.list:
        print "Executing %s unit tests." % testGroupName

    Setup.config().configureLoggingForUserProgram()

    parser = PythonTestArgumentParser()
    filterActions = parser.parse_args(args.remainder)

    bsaRootDir = os.path.split(ufora.__file__)[0]

    testCasesToRun = []

    plugins = nose.plugins.manager.PluginManager([OutputCaptureNosePlugin()])

    config = nose.config.Config(plugins=plugins)
    config.configure(testArgs)
    for i in range(args.copies):
        testCases = UnitTestCommon.loadTestCases(config, testFiles, bsaRootDir, 'ufora')
        if filterActions:
            testCases = applyFilterActions(filterActions, testCases)

        testCasesToRun += testCases

    if testFilter is not None:
        testCasesToRun = testFilter(testCasesToRun)

    if args.list:
        for test in testCasesToRun:
            print test.id()

        os._exit(0)

    if args.random:
        import random
        random.shuffle(testCasesToRun)

    if args.pythreadcheck:
        results = {}
        for test in testCasesToRun:
            results[test] = runPyTestSuite(config, None, unittest.TestSuite([test]), testArgs)

        return True in results.values()
    else:
        testFiles = '.'
        return runPyTestSuite(config, None, testCasesToRun, testArgs)

# 需要导入模块: from ufora.config import Setup [as 别名]
# 或者: from ufora.config.Setup import config [as 别名]
def createService(args):
    callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
    callbackScheduler = callbackSchedulerFactory.createScheduler('ufora-worker', 1)
    channelListener = MultiChannelListener(callbackScheduler,
                                           [args.base_port, args.base_port + 1])

    sharedStateViewFactory = ViewFactory.ViewFactory.TcpViewFactory(
        callbackSchedulerFactory.createScheduler('SharedState', 1),
        args.manager_address,
        int(args.manager_port)
        )

    channelFactory = TcpChannelFactory.TcpStringChannelFactory(callbackScheduler)

    diagnostics_dir = os.getenv("UFORA_WORKER_DIAGNOSTICS_DIR")
    eventHandler = diagnostics_dir and createEventHandler(
        diagnostics_dir,
        callbackSchedulerFactory.createScheduler("ufora-worker-event-handler", 1)
        )

    own_address = args.own_address or get_own_ip()
    print "Listening on", own_address, "ports:", args.base_port, "and", args.base_port+1

    config = Setup.config()
    print "RAM cache of %d / %d MB and %d threads. Track tcmalloc: %s" % (
        config.cumulusVectorRamCacheMB,
        config.cumulusMaxRamCacheMB,
        config.cumulusServiceThreadCount,
        config.cumulusTrackTcmalloc
        )

    print "Ufora store at %s:%s" % (args.manager_address, args.manager_port)

    s3InterfaceFactory = ActualS3Interface.ActualS3InterfaceFactory()
    print "PythonIoTasks threads: %d. Out of process: %s" % (
        config.externalDatasetLoaderServiceThreads,
        s3InterfaceFactory.isCompatibleWithOutOfProcessDownloadPool
        )

    return CumulusService.CumulusService(
        own_address,
        channelListener,
        channelFactory,
        eventHandler,
        callbackScheduler,
        diagnostics_dir,
        Setup.config(),
        viewFactory=sharedStateViewFactory,
        s3InterfaceFactory=s3InterfaceFactory,
        objectStore=NullObjectStore.NullObjectStore()
        )

# 需要导入模块: from ufora.config import Setup [as 别名]
# 或者: from ufora.config.Setup import config [as 别名]
    def test_teardown_during_vector_load(self):
        vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)
        context = ExecutionContext.ExecutionContext(dataManager = vdm)

        context.evaluate(
                FORA.extractImplValContainer(
                    FORA.eval("fun() { let v = [1,2,3].paged; fun() { v[1] } }")
                    ),
                FORANative.symbol_Call
                )
        vdm.unloadAllPossible()

        pagedVecAccessFun = context.getFinishedResult().asResult.result

        context.teardown()

        context.evaluate(
            pagedVecAccessFun,
            FORANative.symbol_Call
            )

        self.assertFalse(context.isInterrupted())
        self.assertTrue(context.isVectorLoad())

        context.teardown()

# 需要导入模块: from ufora.config import Setup [as 别名]
# 或者: from ufora.config.Setup import config [as 别名]
    def test_serialize_while_holding_interior_vector(self):
        vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)
        context = ExecutionContext.ExecutionContext(dataManager = vdm, allowInterpreterTracing=False)

        context.evaluate(
            FORA.extractImplValContainer(
                FORA.eval("""
                    fun() {
                        let v = [[1].paged].paged;
                        let v2 = v[0]

                        `TriggerInterruptForTesting()

                        1+2+3+v+v2
                        }"""
                    )
                ),
            FORANative.symbol_Call
            )

        self.assertTrue(context.isInterrupted())

        serialized = context.serialize()

        context = None

# 需要导入模块: from ufora.config import Setup [as 别名]
# 或者: from ufora.config.Setup import config [as 别名]
    def test_extractPausedComputationDuringVectorLoad(self):
        self.runtime = Runtime.getMainRuntime()
        #self.dynamicOptimizer = self.runtime.dynamicOptimizer

        vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)

        context = ExecutionContext.ExecutionContext(
            dataManager = vdm,
            allowInterpreterTracing = False
            )

        context.evaluate(
            FORA.extractImplValContainer(FORA.eval("fun() { [1,2,3].paged }")),
            FORANative.ImplValContainer(FORANative.makeSymbol("Call"))
            )

        pagedVec = context.getFinishedResult().asResult.result

        context.placeInEvaluationState(
            FORANative.ImplValContainer(
                (pagedVec,
                FORANative.ImplValContainer(FORANative.makeSymbol("GetItem")),
                FORANative.ImplValContainer(0))
                )
            )

        vdm.unloadAllPossible()

        context.resume()

        self.assertTrue(context.isVectorLoad())

        computation = context.extractPausedComputation()

        self.assertEqual(len(computation.frames),1)

# 需要导入模块: from ufora.config import Setup [as 别名]
# 或者: from ufora.config.Setup import config [as 别名]
def defaultLocalEvaluator(remoteEvaluator=None, vdmOverride=None):
    return LocalEvaluator(
        lambda vdm: None,
        Setup.config().cumulusVectorRamCacheMB * 1024 * 1024,
        remoteEvaluator,
        vdmOverride=vdmOverride
        )

# 需要导入模块: from ufora.config import Setup [as 别名]
# 或者: from ufora.config.Setup import config [as 别名]
def createService(args):
    callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
    callbackScheduler = callbackSchedulerFactory.createScheduler('ufora-worker', 1)
    channelListener = MultiChannelListener(callbackScheduler,
                                           [args.base_port, args.base_port + 1])

    sharedStateViewFactory = ViewFactory.ViewFactory.TcpViewFactory(
        callbackSchedulerFactory.createScheduler('SharedState', 1),
        args.manager_address,
        int(args.manager_port)
        )

    channelFactory = TcpChannelFactory.TcpStringChannelFactory(callbackScheduler)

    diagnostics_dir = os.getenv("UFORA_WORKER_DIAGNOSTICS_DIR")
    eventHandler = diagnostics_dir and createEventHandler(
        diagnostics_dir,
        callbackSchedulerFactory.createScheduler("ufora-worker-event-handler", 1)
        )

    own_address = args.own_address or get_own_ip()
    print "Listening on", own_address, "ports:", args.base_port, "and", args.base_port+1

    return CumulusService.CumulusService(
        own_address,
        channelListener,
        channelFactory,
        eventHandler,
        callbackScheduler,
        diagnostics_dir,
        Setup.config(),
        viewFactory=sharedStateViewFactory
        )

# 需要导入模块: from ufora.config import Setup [as 别名]
# 或者: from ufora.config.Setup import config [as 别名]
    def __init__(self, relayHostname, relayHttpsPort = None, messageDelayInSeconds = None):
        """Initialize a PipeTransport.

        messageDelayInSeconds - if not None, then all messages will be delayed by this many
            seconds before being pumped into the receiving channel. This can simulate
            delays talking over the internet.
        """
        self.onMessageReceived = None
        self.onDisconnected = None
        self.inputLoopThread = None
        self.isShuttingDown = False
        self.proxyProcess = None
        self.isConnected = False
        self.messageDelayInSeconds = messageDelayInSeconds
        self.messagePumpThread = None
        self.messagePumpQueue = Queue.Queue()

        self.relayHostname = relayHostname
        if relayHttpsPort:
            self.relayHttpsPort = relayHttpsPort
        else:
            self.relayHttpsPort = Setup.config().relayHttpsPort

        self.proxyStdIn = None
        self.proxyStdOut = None
        self.proxyStdErr = None
        self.proxyOutputThread = None

        logging.info("PipeTransport created for host %s:%s", self.relayHostname, self.relayHttpsPort)