Python resource.ResourceAllocationManager类代码示例

 def config_ram(self, filename):
     """
     Configure the :class:`ResourceAllocationManager` instance from `filename`.
     Used to define resources needed for model execution.
     """
     self._logger.debug('config_ram %r', filename)
     from openmdao.main.resource import ResourceAllocationManager
     ResourceAllocationManager.configure(filename)

    def test_hostnames(self):
        logging.debug('')
        logging.debug('test_hostnames')

        hostnames = ResourceAllocationManager.get_hostnames({'n_cpus':1})
        self.assertEqual(hostnames[0], platform.node())
        
        hostnames = ResourceAllocationManager.get_hostnames({'no_such_resource':1})
        self.assertEqual(hostnames, None)

 def get_ram(self):
     """
     Returns the :class:`ResourceAllocationManager` instance.
     Used by :meth:`ResourceAllocationManager.add_remotes`.
     """
     from openmdao.main.resource import ResourceAllocationManager
     return ResourceAllocationManager._get_instance()

    def setUp(self):
        self.user = getpass.getuser()
        self.node = platform.node()
        self.name = self.node.replace('.', '_')
        self.python = find_python()
        self.cluster = None

        if sys.platform == 'win32' or self.user not in SSH_USERS:
            self.skip_ssh = True
        else:
            self.skip_ssh = False

        self.machines = []
        if self.node.startswith('gxterm'):
            # User environment assumed OK on this GRC cluster front-end.
            for i in range(1, 55):
                self.machines.append({'hostname':'gx%02d' % i,
                                      'python':self.python})
        else:
            self.machines.append({'hostname':self.node,
                                  'python':self.python})

        # Ensure we aren't held up by local host load problems.
        for allocator in ResourceAllocationManager.list_allocators():
            if allocator.name == 'LocalHost':
                self.local = allocator
                self.local.max_load = 10
                break
        else:
            raise RuntimeError('No LocalHost allocator!?')

def setup_cluster(encrypted=True):
    """ Use openmdao.testing.cluster.init_cluster, but fix 'max_load'. """
    name = init_cluster(encrypted, allow_shell=True)
    for allocator in ResourceAllocationManager.list_allocators():
        if allocator.name == 'LocalHost':
            allocator.max_load = 1.
    return name

def run_suite(resource_desc=None, name=None):
    """ Run suite of tests using `resource_desc` and resord under `name`. """
    resource_desc = resource_desc or {}
    name = name or ''
    print '\n%s' % name

    initial = 0.01
    limit = 20
    results = {}

    max_servers = ResourceAllocationManager.max_servers(resource_desc)
    print 'max servers', max_servers

    model = CID()
    model.driver.reload_model = False
    model.driver.sequential = False

    # Save to an egg to avoid analysis overhead during run_test().
    print '\nInitializing egg module analysis'
    template = Case(inputs=[('sleeper.delay', None, 0.01)])
    model.driver.iterator = Iterator(template)
    model.driver.recorders = [Recorder(model.driver.iterator, 1000)]
    start = time.time()
    egg_filename, required_distributions, orphan_modules = \
        model.save_to_egg('caseperf', '0')
    et = time.time() - start
    print '    done in %.2f' % et
    os.remove(egg_filename)

    print
    results = run_test(model, initial, limit, max_servers)
    record_results(results, name)

    def setUp(self):
        # Save existing RAM instance and force a rebuild.
        self.orig_ram = RAM._RAM
        RAM._RAM = None
        RAM.configure('')

        self.user = getpass.getuser()
        self.node = platform.node()
        self.name = self.node.replace('.', '_')
        self.python = find_python()
        self.cluster = None

        if sys.platform == 'win32' or self.user not in SSH_USERS:
            self.skip_ssh = True
        else:
            self.skip_ssh = False

        self.machines = []
        self.machines.append({'hostname': self.node,
                              'python': self.python})

        # Ensure we aren't held up by local host load problems.
        for allocator in RAM.list_allocators():
            if allocator.name == 'LocalHost':
                self.local = allocator
                self.local.max_load = 10
                break
        else:
            raise RuntimeError('No LocalHost allocator!?')

    def test_allocator(self):
        logging.debug('')
        logging.debug('test_allocator')

        # Since we're faking it with a remote LocalHost, we should match.
        local_servers = RAM.max_servers(dict(allocator='LocalHost'))
        max_servers = RAM.max_servers(dict(allocator=self.allocator.name))
        self.assertEqual(max_servers, local_servers)

        max_servers = RAM.max_servers(dict(allocator=self.allocator.name,
                                           localhost=True)) # Contradictory!
        self.assertEqual(max_servers, 0)

        server = self.allocator.deploy('test_server', {}, {})
        try:
            self.assertEqual(server.name, 'NAS_Allocator/test_server')
            self.assertEqual(server.host, socket.gethostname())
            self.assertTrue(server.pid > 0)
            retval = server.echo(123, 'twisty', 'narrow', 'passages')
            self.assertEqual(retval, (123, 'twisty', 'narrow', 'passages'))
            self.assertTrue(server.isdir('.'))
            self.assertEqual(sorted(server.listdir('.')),
                             ['openmdao_log.txt', 'stderr', 'stdout'])
        finally:
            self.allocator.release(server)

    def test_request(self):
        logging.debug('')
        logging.debug('test_request')

        assembly = Assembly()
        comp1 = assembly.add('comp1', ExtCode())
        comp2 = assembly.add('comp2', ExtCode())
        sub = assembly.add('sub', Assembly())
        comp3 = sub.add('comp3', ExtCode())

        comp1.resources = dict(min_cpus=10,
                               max_cpus=10,
                               resource_limits=dict(virtual_memory=100,
                                                    cpu_time=120),
                               rerunnable=True,
                               accounting_id='frobozz',
                               queue_name='debug',
                               job_category='MPI')

        comp2.resources = dict(max_cpus=2,
                               resource_limits=dict(wallclock_time=1000000))

        comp3.resources = dict(min_cpus=200,
                               resource_limits=dict(virtual_memory=20,
                                                    cpu_time=1000,
                                                    wallclock_time=500),
                               rerunnable=True,
                               accounting_id='frobozz',
                               queue_name='debug',
                               job_category='MPI')

        req = RAM.max_request(assembly)
        expected = dict(min_cpus=200,
                        max_cpus=200,
                        resource_limits=dict(virtual_memory=100,
                                             cpu_time=1000,
                                             wallclock_time=1000000))
        logging.debug('req: %r', req)
        logging.debug('exp: %r', expected)
        self.assertEqual(req, expected)

        req = RAM.total_request(assembly)
        expected = dict(min_cpus=200,
                        max_cpus=200,
                        resource_limits=dict(virtual_memory=100,
                                             cpu_time=1120,
                                             wallclock_time=1000500),
                        rerunnable=True,
                        accounting_id='frobozz',
                        queue_name='debug',
                        job_category='MPI')
        logging.debug('req: %r', req)
        logging.debug('exp: %r', expected)
        self.assertEqual(req, expected)

        comp3.resources['accounting_id'] = 'xyzzy'
        assert_raises(self, 'RAM.total_request(assembly)',
                      globals(), locals(), ValueError,
                      "Incompatible settings for 'accounting_id':"
                      " 'xyzzy' vs. 'frobozz'")

    def test_case_eval(self):
        logging.debug('')
        logging.debug('test_case_eval')

        # Run a fake job in style of CaseIteratorDriver.
        logging.debug('allocate server')
        server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
        echo = set_as_top(Echo(1))
        egg_info = echo.save_to_egg('EchoTest', '1', need_requirements=False)
        egg_filename = egg_info[0]
        try:
            logging.debug('transfer egg')
            filexfer(None, egg_filename, server, egg_filename, 'b')

            logging.debug('load model')
            tlo = server.load_model(egg_filename)

            logging.debug('set input')
            tlo.set('inp_0', 42)

            logging.debug('run')
            tlo.run()

            logging.debug('get output')
            output = tlo.get('out_0')
            self.assertEqual(output, 42)
        finally:
            os.remove(egg_filename)
            logging.debug('release')
            RAM.release(server)

    def setUp(self):
        self.orig_dir = os.getcwd()
        os.chdir(TestCase.directory)
        try:
            # Force use of fake 'ssh' and 'scp'.
            ssh = ("python", os.path.join(_TST_ROOT, "ssh.py"), _DMZ_ROOT)
            scp = ("python", os.path.join(_TST_ROOT, "scp.py"), _DMZ_ROOT)

            self.orig_ssh = protocol.configure_ssh(ssh)
            self.orig_scp = protocol.configure_scp(scp)

            # Avoid lots of polling log entries.
            if logging.getLogger().getEffectiveLevel() < logging.DEBUG:
                logging.getLogger().setLevel(logging.DEBUG)

            # Start RJE server.
            hostname = socket.gethostname()
            self.proc = start_server(hostname)

            # Create NAS_Allocator referring to server.
            logging.debug("create allocator")
            self.allocator = NAS_Allocator()
            parser = ConfigParser.ConfigParser()
            section = self.allocator.name
            parser.add_section(section)
            parser.set(section, "dmz_host", hostname)
            parser.set(section, "server_host", hostname)
            self.allocator.configure(parser)

            # Add allocator to RAM.
            RAM.add_allocator(self.allocator)
        except Exception:
            os.chdir(self.orig_dir)
            raise

    def test_case_eval(self):
        logging.debug("")
        logging.debug("test_case_eval")

        # Run a fake job in style of CaseIteratorDriver.
        logging.debug("allocate server")
        server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
        echo = set_as_top(Echo(1))
        egg_info = echo.save_to_egg("EchoTest", "1", need_requirements=False)
        egg_filename = egg_info[0]
        try:
            logging.debug("transfer egg")
            filexfer(None, egg_filename, server, egg_filename, "b")

            logging.debug("load model")
            tlo = server.load_model(egg_filename)

            logging.debug("set input")
            tlo.set("inp_0", 42)

            logging.debug("run")
            tlo.run()

            logging.debug("get output")
            output = tlo.get("out_0")
            self.assertEqual(output, 42)
        finally:
            os.remove(egg_filename)
            logging.debug("release")
            RAM.release(server)

    def setUp(self):

        nodes = []
        for i in range(12):
            nodes.append('g-0%02d'%i)

        # start the fake MPI_Allocator 
        self.cluster=MPI_Allocator(name='test',machines=nodes)
        # add it to to the RAM
        RAM.add_allocator(self.cluster)

    def test_hostnames(self):
        logging.debug('')
        logging.debug('test_hostnames')

        hostnames = RAM.get_hostnames({'min_cpus': 1})
        self.assertEqual(hostnames[0], platform.node())

        hostnames = RAM.get_hostnames({'allocator': 'LocalHost',
                                       'localhost': False})
        self.assertEqual(hostnames, None)

    def _service_loop(self, name, resource_desc, credentials, reply_q):
        """ Each server has an associated thread executing this. """
        set_credentials(credentials)

        server, server_info = RAM.allocate(resource_desc)
        # Just being defensive, this should never happen.
        if server is None:  # pragma no cover
            self._logger.error('Server allocation for %r failed :-(', name)
            reply_q.put((name, False, None))
            return
        else:
            # Clear egg re-use indicator.
            server_info['egg_file'] = None
            self._logger.debug('%r using %r', name, server_info['name'])
            if self._logger.level == logging.NOTSET:
                # By default avoid lots of protocol messages.
                server.set_log_level(logging.DEBUG)
            else:
                server.set_log_level(self._logger.level)

        request_q = Queue.Queue()

        try:
            with self._server_lock:
                sdata = self._servers[name]
                sdata.server = server
                sdata.info = server_info
                sdata.queue = request_q

            reply_q.put((name, True, None))  # ACK startup.

            while True:
                request = request_q.get()
                if request is None:
                    break
                try:
                    result = request[0](request[1])
                except Exception as req_exc:
                    self._logger.error('%r: %s caused %r', name,
                                       request[0], req_exc)
                    result = None
                else:
                    req_exc = None
                reply_q.put((name, result, req_exc))
        except Exception as exc:  # pragma no cover
            # This can easily happen if we take a long time to allocate and
            # we get 'cleaned-up' before we get started.
            if self._server_lock is not None:
                self._logger.error('%r: %r', name, exc)
        finally:
            self._logger.debug('%r releasing server', name)
            RAM.release(server)
            reply_q.put((name, True, None))  # ACK shutdown.