Python execution_engine_agent.ExecutionEngineAgentClient類代碼示例

        def heartbeat_callback(heartbeat, headers):

            eeagent_id = heartbeat["eeagent_id"]
            agent_client = SimpleResourceAgentClient(eeagent_id, name=eeagent_id, process=FakeProcess())
            ee_client = ExecutionEngineAgentClient(agent_client, timeout=2)

            try:
                ee_client.dump_state()
            except:
                log.exception("Heartbeat Failed!")
                beat_died[0] = True

    def setUp(self):
        self._start_container()
        self.container.start_rel_from_url('res/deploy/r2cei.yml')

        self.resource_id = "eeagent_1234"
        self._eea_name = "eeagent"

        self.supd_directory = tempfile.mkdtemp()

        self.agent_config = {
            'eeagent': {
                'heartbeat': 0,
                'slots': 100,
                'name': 'pyon_eeagent',
                'launch_type': {
                    'name': 'supd',
                    'pyon_directory': os.getcwd(),
                    'supd_directory': self.supd_directory,
                    'supdexe': 'bin/supervisord'
                },
            },
            'agent': {'resource_id': self.resource_id},
            'logging': {
                'loggers': {
                    'eeagent': {
                        'level': 'DEBUG',
                        'handlers': ['console']
                    }
                },
                'root': {
                    'handlers': ['console']
                }
            }
        }

        # Start eeagent.
        self._eea_pid = None

        self.container_client = ContainerAgentClient(
            node=self.container.node, name=self.container.name)
        self._eea_pid = self.container_client.spawn_process(
            name=self._eea_name,
            module="ion.agents.cei.execution_engine_agent",
            cls="ExecutionEngineAgent", config=self.agent_config)
        log.info('Agent pid=%s.', str(self._eea_pid))

        # Start a resource agent client to talk with the instrument agent.
        self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
        log.info('Got eea client %s.', str(self._eea_pyon_client))

        self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)

    def _start_eeagent(self):
        self.container_client = ContainerAgentClient(node=self.container.node,
            name=self.container.name)
        self.container = self.container_client._get_container_instance()

        # Start eeagent.
        self._eea_pid = self.container_client.spawn_process(name=self._eea_name,
            module="ion.agents.cei.execution_engine_agent",
            cls="ExecutionEngineAgent", config=self.agent_config)
        log.info('Agent pid=%s.', str(self._eea_pid))

        # Start a resource agent client to talk with the instrument agent.
        self._eea_pyon_client = ResourceAgentClient(self.resource_id, process=FakeProcess())
        log.info('Got eea client %s.', str(self._eea_pyon_client))

        self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)

    def setUp(self):
        self._start_container()
        self.container.start_rel_from_url("res/deploy/r2cei.yml")

        self.resource_id = "eeagent_1234"
        self._eea_name = "eeagent"

        self.supd_directory = tempfile.mkdtemp()

        self.agent_config = {
            "eeagent": {
                "heartbeat": 0,
                "slots": 100,
                "name": "pyon_eeagent",
                "launch_type": {
                    "name": "supd",
                    "pyon_directory": os.getcwd(),
                    "supd_directory": self.supd_directory,
                    "supdexe": "bin/supervisord",
                },
            },
            "agent": {"resource_id": self.resource_id},
            "logging": {
                "loggers": {"eeagent": {"level": "DEBUG", "handlers": ["console"]}},
                "root": {"handlers": ["console"]},
            },
        }

        # Start eeagent.
        self._eea_pid = None

        self.container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
        self._eea_pid = self.container_client.spawn_process(
            name=self._eea_name,
            module="ion.agents.cei.execution_engine_agent",
            cls="ExecutionEngineAgent",
            config=self.agent_config,
        )
        log.info("Agent pid=%s.", str(self._eea_pid))

        # Start a resource agent client to talk with the instrument agent.
        self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
        log.info("Got eea client %s.", str(self._eea_pyon_client))

        self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)

class ExecutionEngineAgentSupdIntTest(IonIntegrationTestCase):
    @needs_eeagent
    def setUp(self):
        self._start_container()
        self.container.start_rel_from_url("res/deploy/r2cei.yml")

        self.resource_id = "eeagent_1234"
        self._eea_name = "eeagent"

        self.supd_directory = tempfile.mkdtemp()

        self.agent_config = {
            "eeagent": {
                "heartbeat": 0,
                "slots": 100,
                "name": "pyon_eeagent",
                "launch_type": {
                    "name": "supd",
                    "pyon_directory": os.getcwd(),
                    "supd_directory": self.supd_directory,
                    "supdexe": "bin/supervisord",
                },
            },
            "agent": {"resource_id": self.resource_id},
            "logging": {
                "loggers": {"eeagent": {"level": "DEBUG", "handlers": ["console"]}},
                "root": {"handlers": ["console"]},
            },
        }

        # Start eeagent.
        self._eea_pid = None

        self.container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
        self._eea_pid = self.container_client.spawn_process(
            name=self._eea_name,
            module="ion.agents.cei.execution_engine_agent",
            cls="ExecutionEngineAgent",
            config=self.agent_config,
        )
        log.info("Agent pid=%s.", str(self._eea_pid))

        # Start a resource agent client to talk with the instrument agent.
        self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
        log.info("Got eea client %s.", str(self._eea_pyon_client))

        self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)

    def tearDown(self):
        self.container.terminate_process(self._eea_pid)
        shutil.rmtree(self.supd_directory)

    def wait_for_state(self, upid, desired_state, timeout=5):
        attempts = 0
        while timeout > attempts:
            state = self.eea_client.dump_state().result
            proc = get_proc_for_upid(state, upid)
            if proc.get("state") == desired_state:
                return
            gevent.sleep(1)
            attempts += 1

        assert False, "Process %s took too long to get to %s" % (upid, desired_state)

    @needs_eeagent
    def test_basics(self):
        true_u_pid = "test0"
        round = 0
        run_type = "supd"
        true_parameters = {"exec": "true", "argv": []}
        self.eea_client.launch_process(true_u_pid, round, run_type, true_parameters)

        self.wait_for_state(true_u_pid, [800, "EXITED"])

        cat_u_pid = "test1"
        round = 0
        run_type = "supd"
        cat_parameters = {"exec": "cat", "argv": []}
        self.eea_client.launch_process(cat_u_pid, round, run_type, cat_parameters)
        self.wait_for_state(cat_u_pid, [500, "RUNNING"])

        self.eea_client.terminate_process(cat_u_pid, round)
        self.wait_for_state(cat_u_pid, [700, "TERMINATED"])

class ExecutionEngineAgentPyonIntTest(IonIntegrationTestCase):

    _webserver = None

    @needs_eeagent
    def setUp(self):
        self._start_container()
        self.container.start_rel_from_url("res/deploy/r2cei.yml")

        self.resource_id = "eeagent_123456789"
        self._eea_name = "eeagent"

        self.persistence_directory = tempfile.mkdtemp()

        self.agent_config = {
            "eeagent": {
                "heartbeat": 1,
                "slots": 100,
                "name": "pyon_eeagent",
                "launch_type": {"name": "pyon", "persistence_directory": self.persistence_directory},
            },
            "agent": {"resource_id": self.resource_id},
            "logging": {
                "loggers": {"eeagent": {"level": "DEBUG", "handlers": ["console"]}},
                "root": {"handlers": ["console"]},
            },
        }

        self._start_eeagent()

    def _start_eeagent(self):
        self.container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
        self.container = self.container_client._get_container_instance()

        # Start eeagent.
        self._eea_pid = self.container_client.spawn_process(
            name=self._eea_name,
            module="ion.agents.cei.execution_engine_agent",
            cls="ExecutionEngineAgent",
            config=self.agent_config,
        )
        log.info("Agent pid=%s.", str(self._eea_pid))

        # Start a resource agent client to talk with the instrument agent.
        self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
        log.info("Got eea client %s.", str(self._eea_pyon_client))

        self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)

    def tearDown(self):
        self._stop_webserver()
        self.container.terminate_process(self._eea_pid)
        shutil.rmtree(self.persistence_directory)

    def _start_webserver(self, directory_to_serve, port=None):
        """ Start a webserver for testing code download
        Note: tries really hard to get a port, and if it can't use
        the suggested port, randomly picks another, and returns it
        """

        def log_message(self, format, *args):
            # swallow log massages
            pass

        class Server(HTTPServer):

            requests = 0

            def serve_forever(self):
                self._serving = 1
                while self._serving:
                    self.handle_request()
                    self.requests += 1

            def stop(self):
                self._serving = 0

        if port is None:
            port = 8008
        Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
        Handler.log_message = log_message

        for i in range(0, 100):
            try:
                self._webserver = Server(("localhost", port), Handler)
            except socket.error:
                print "port %s is in use, picking another" % port
                port = randint(8000, 10000)
                continue
            else:
                break

        self._web_glet = gevent.spawn(self._webserver.serve_forever)
        return port

    def _stop_webserver(self):
        if self._webserver is not None:
            self._web_glet.kill()

    def _enable_code_download(self, whitelist=None):
#.........這裏部分代碼省略.........

class ExecutionEngineAgentPyonSingleIntTest(IonIntegrationTestCase):
    @needs_eeagent
    def setUp(self):
        self._start_container()
        self.container.start_rel_from_url("res/deploy/r2cei.yml")

        self.resource_id = "eeagent_123456"
        self._eea_name = "eeagent"

        self.supd_directory = tempfile.mkdtemp()

        self.agent_config = {
            "eeagent": {
                "heartbeat": 0,
                "slots": 100,
                "name": "pyon_eeagent",
                "launch_type": {
                    "name": "pyon_single",
                    "pyon_directory": os.getcwd(),
                    "supd_directory": self.supd_directory,
                    "supdexe": "bin/supervisord",
                },
            },
            "agent": {"resource_id": self.resource_id},
            "logging": {
                "loggers": {"eeagent": {"level": "DEBUG", "handlers": ["console"]}},
                "root": {"handlers": ["console"]},
            },
        }

        # Start eeagent.
        self._eea_pid = None

        self.container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
        self._eea_pid = self.container_client.spawn_process(
            name=self._eea_name,
            module="ion.agents.cei.execution_engine_agent",
            cls="ExecutionEngineAgent",
            config=self.agent_config,
        )
        log.info("Agent pid=%s.", str(self._eea_pid))

        # Start a resource agent client to talk with the instrument agent.
        self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
        log.info("Got eea client %s.", str(self._eea_pyon_client))

        self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)

    def tearDown(self):
        self.container.terminate_process(self._eea_pid)
        shutil.rmtree(self.supd_directory)

    @needs_eeagent
    def test_basics(self):
        u_pid = "test0"
        round = 0
        run_type = "pyon_single"
        proc_name = "test_transform"
        module = "ion.agents.cei.test.test_eeagent"
        cls = "TestProcess"
        parameters = {"name": proc_name, "module": module, "cls": cls}
        self.eea_client.launch_process(u_pid, round, run_type, parameters)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, u_pid)

        self.assertEqual(proc.get("state"), [500, "RUNNING"])

        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, u_pid)
        self.assertEqual(proc.get("state"), [700, "TERMINATED"])

class ExecutionEngineAgentSupdIntTest(IonIntegrationTestCase):

    def setUp(self):
        self._start_container()
        self.container.start_rel_from_url('res/deploy/r2cei.yml')

        self.resource_id = "eeagent_1234"
        self._eea_name = "eeagent"

        self.supd_directory = tempfile.mkdtemp()

        self.agent_config = {
            'eeagent': {
              'heartbeat': 0,
              'slots': 100,
              'name': 'pyon_eeagent',
              'launch_type': {
                'name': 'supd',
                'pyon_directory': os.getcwd(),
                'supd_directory': self.supd_directory,
                'supdexe': 'bin/supervisord'
              },
            },
            'agent': {'resource_id': self.resource_id},
            'logging': {
            'loggers': {
              'eeagent': {
                'level': 'DEBUG',
                'handlers': ['console']
              }
            },
            'root': {
              'handlers': ['console']
            },
          }
        }

        # Start eeagent.
        self._eea_pid = None

        self.container_client = ContainerAgentClient(node=self.container.node,
            name=self.container.name)
        self._eea_pid = self.container_client.spawn_process(name=self._eea_name,
            module="ion.agents.cei.execution_engine_agent",
            cls="ExecutionEngineAgent", config=self.agent_config)
        log.info('Agent pid=%s.', str(self._eea_pid))

        # Start a resource agent client to talk with the instrument agent.
        self._eea_pyon_client = ResourceAgentClient(self.resource_id, process=FakeProcess())
        log.info('Got eea client %s.', str(self._eea_pyon_client))

        self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)

    def tearDown(self):
        self.container.terminate_process(self._eea_pid)
        shutil.rmtree(self.supd_directory)

    @needs_eeagent
    def test_basics(self):
        true_u_pid = "test0"
        round = 0
        run_type = "supd"
        true_parameters = {'exec': 'true', 'argv': []}
        self.eea_client.launch_process(true_u_pid, round, run_type, true_parameters)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, true_u_pid)

        self.assertEqual(proc.get('state'), [800, 'EXITED'])

        cat_u_pid = "test1"
        round = 0
        run_type = "supd"
        cat_parameters = {'exec': 'cat', 'argv': []}
        self.eea_client.launch_process(cat_u_pid, round, run_type, cat_parameters)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, cat_u_pid)
        self.assertEqual(proc.get('state'), [500, 'RUNNING'])

        self.eea_client.terminate_process(cat_u_pid, round)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, cat_u_pid)
        self.assertEqual(proc.get('state'), [700, 'TERMINATED'])

class ExecutionEngineAgentPyonIntTest(IonIntegrationTestCase):
    from ion.agents.cei.execution_engine_agent import ExecutionEngineAgentClient

    def setUp(self):
        self._start_container()
        self.container.start_rel_from_url('res/deploy/r2cei.yml')

        self.resource_id = "eeagent_123456789"
        self._eea_name = "eeagent"

        self.persistence_directory = tempfile.mkdtemp()

        self.agent_config = {
            'eeagent': {
              'heartbeat': 0,
              'slots': 100,
              'name': 'pyon_eeagent',
              'launch_type': {
                'name': 'pyon',
                'persistence_directory': self.persistence_directory,
              },
            },
            'agent': {'resource_id': self.resource_id},
            'logging': {
            'loggers': {
              'eeagent': {
                'level': 'DEBUG',
                'handlers': ['console']
              }
            },
            'root': {
              'handlers': ['console']
            },
          }
        }

        self._start_eeagent()

    def _start_eeagent(self):
        self.container_client = ContainerAgentClient(node=self.container.node,
            name=self.container.name)
        self.container = self.container_client._get_container_instance()

        # Start eeagent.
        self._eea_pid = self.container_client.spawn_process(name=self._eea_name,
            module="ion.agents.cei.execution_engine_agent",
            cls="ExecutionEngineAgent", config=self.agent_config)
        log.info('Agent pid=%s.', str(self._eea_pid))

        # Start a resource agent client to talk with the instrument agent.
        self._eea_pyon_client = ResourceAgentClient(self.resource_id, process=FakeProcess())
        log.info('Got eea client %s.', str(self._eea_pyon_client))

        self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)

    def tearDown(self):
        self.container.terminate_process(self._eea_pid)
        shutil.rmtree(self.persistence_directory)

    @needs_eeagent
    def test_basics(self):
        u_pid = "test0"
        round = 0
        run_type = "pyon"
        proc_name = 'test_x'
        module = 'ion.agents.cei.test.test_eeagent'
        cls = 'TestProcess'
        parameters = {'name': proc_name, 'module': module, 'cls': cls}
        self.eea_client.launch_process(u_pid, round, run_type, parameters)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, u_pid)

        self.assertIsNotNone(proc, "There is no state retrieved from eeagent")
        self.assertEqual(proc.get('state'), [500, 'RUNNING'])

        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, u_pid)

    @needs_eeagent
    def test_kill_and_revive(self):
        """test_kill_and_revive
        Ensure that when an eeagent dies, it pulls the processes it owned from
        persistence, and marks them as failed, so the PD can figure out what to
        do with them
        """
        u_pid = "test0"
        round = 0
        run_type = "pyon"
        proc_name = 'test_transform'
        module = 'ion.agents.cei.test.test_eeagent'
        cls = 'TestProcess'
        parameters = {'name': proc_name, 'module': module, 'cls': cls}
        self.eea_client.launch_process(u_pid, round, run_type, parameters)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, u_pid)

        self.assertIsNotNone(proc, "There is no state retrieved from eeagent")
        self.assertEqual(proc.get('state'), [500, 'RUNNING'])

#.........這裏部分代碼省略.........

class ExecutionEngineAgentPyonIntTest(IonIntegrationTestCase):

    _webserver = None

    @needs_eeagent
    def setUp(self):
        self._start_container()
        self.container.start_rel_from_url('res/deploy/r2cei.yml')

        self.resource_id = "eeagent_123456789"
        self._eea_name = "eeagent"

        self.persistence_directory = tempfile.mkdtemp()

        self.agent_config = {
            'eeagent': {
              'heartbeat': 0,
              'slots': 100,
              'name': 'pyon_eeagent',
              'launch_type': {
                'name': 'pyon',
                'persistence_directory': self.persistence_directory,
              }
            },
            'agent': {'resource_id': self.resource_id},
            'logging': {
            'loggers': {
              'eeagent': {
                'level': 'DEBUG',
                'handlers': ['console']
              }
            },
            'root': {
              'handlers': ['console']
            },
          }
        }

        self._start_eeagent()

    def _start_eeagent(self):
        self.container_client = ContainerAgentClient(node=self.container.node,
            name=self.container.name)
        self.container = self.container_client._get_container_instance()

        # Start eeagent.
        self._eea_pid = self.container_client.spawn_process(name=self._eea_name,
            module="ion.agents.cei.execution_engine_agent",
            cls="ExecutionEngineAgent", config=self.agent_config)
        log.info('Agent pid=%s.', str(self._eea_pid))

        # Start a resource agent client to talk with the instrument agent.
        self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
        log.info('Got eea client %s.', str(self._eea_pyon_client))

        self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)

    def tearDown(self):
        self._stop_webserver()
        self.container.terminate_process(self._eea_pid)
        shutil.rmtree(self.persistence_directory)

    def _start_webserver(self, directory_to_serve, port=None):
        """ Start a webserver for testing code download
        Note: tries really hard to get a port, and if it can't use
        the suggested port, randomly picks another, and returns it
        """
        def log_message(self, format, *args):
            #swallow log massages
            pass

        class Server(HTTPServer):

            requests = 0

            def serve_forever(self):
                self._serving = 1
                while self._serving:
                    self.handle_request()
                    self.requests += 1

            def stop(self):
                self._serving = 0

        if port is None:
            port = 8008
        self.old_cwd = os.getcwd()
        os.chdir(directory_to_serve)
        Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
        Handler.log_message = log_message

        for i in range(0, 100):
            try:
                self._webserver = Server(("localhost", port), Handler)
            except socket.error:
                print "port %s is in use, picking another" % port
                port = randint(8000, 10000)
                continue
            else:
                break
#.........這裏部分代碼省略.........