Python WorkerPool.check_results方法代码示例

# 需要导入模块: from gppylib.commands.base import WorkerPool [as 别名]
# 或者: from gppylib.commands.base.WorkerPool import check_results [as 别名]
def scp_file_to_hosts(host_list, filename, batch_default):
    pool = WorkerPool(numWorkers=min(len(host_list), batch_default))

    for hname in host_list:
        pool.addCommand(
            Scp("Copying table_filter_file to %s" % hname, srcFile=filename, dstFile=filename, dstHost=hname)
        )
    pool.join()
    pool.check_results()

# 需要导入模块: from gppylib.commands.base import WorkerPool [as 别名]
# 或者: from gppylib.commands.base.WorkerPool import check_results [as 别名]
def run_pool_command(host_list, cmd_str, batch_default, check_results=True):
    pool = WorkerPool(numWorkers = min(len(host_list), batch_default))

    for h in host_list:
        cmd = Command(h, cmd_str, ctxt=REMOTE, remoteHost = h)
        pool.addCommand(cmd)

    pool.join()
    if check_results:
        pool.check_results()

# 需要导入模块: from gppylib.commands.base import WorkerPool [as 别名]
# 或者: from gppylib.commands.base.WorkerPool import check_results [as 别名]
class GatherResults(Operation):
    def __init__(self, master_datadir, token, content, batch_default):
        self.master_datadir = master_datadir
        self.token = token
        self.content = content
        self.batch_default = batch_default
        self.pool = None
    def execute(self):
        logger.info('Gathering results of verification %s...' % self.token)
        to_gather = ValidateVerification(content = self.content,
                                         primaries_only = False).run()

        dest_base = os.path.join(self.master_datadir, 'pg_verify', self.token)
        if CheckDir(dest_base).run():
            # TODO: if end user has mucked around with artifacts on master, a regathering may
            # be needed; perhaps, a --force option to accompany --results?
            return
        MakeDir(dest_base).run()

        self.pool = WorkerPool(min(len(to_gather), self.batch_default))
        for seg in to_gather:
            host = seg.getSegmentHostName()
            content = seg.getSegmentContentId()
            role = seg.getSegmentRole()
            src = os.path.join(seg.getSegmentDataDirectory(), "pg_verify", "*%s*" % self.token)

            dest = os.path.join(dest_base, str(content), str(role))
            MakeDir(dest).run()
            cmd = Scp('consolidate', srcFile=src, srcHost=host, dstFile=dest)
            self.pool.addCommand(cmd)

        logger.info('Waiting for scp commands to complete...')
        self.pool.wait_and_printdots(len(to_gather))
        self.pool.check_results()

        dest = os.path.join(dest_base, 'verification_%s.fix' % self.token)
        with open(dest, 'w') as output:
            for seg in to_gather:
                content = seg.getSegmentContentId()
                role = seg.getSegmentRole()
                src = os.path.join(dest_base, str(content), str(role), 'verification_%s.fix' % self.token)
                with open(src, 'r') as input:
                    output.writelines(input.readlines())

# 需要导入模块: from gppylib.commands.base import WorkerPool [as 别名]
# 或者: from gppylib.commands.base.WorkerPool import check_results [as 别名]
class WorkerPoolTest(unittest.TestCase):
    def setUp(self):
        self.pool = WorkerPool(numWorkers=1, logger=mock.Mock())

    def tearDown(self):
        # All background threads must be stopped, or else the test runner will
        # hang waiting. Join the stopped threads to make sure we're completely
        # clean for the next test.
        self.pool.haltWork()
        self.pool.joinWorkers()

    def test_pool_must_have_some_workers(self):
        with self.assertRaises(Exception):
            WorkerPool(numWorkers=0)
        
    def test_pool_runs_added_command(self):
        cmd = mock.Mock(spec=Command)

        self.pool.addCommand(cmd)
        self.pool.join()

        cmd.run.assert_called_once_with()

    def test_completed_commands_are_retrievable(self):
        cmd = mock.Mock(spec=Command)

        self.pool.addCommand(cmd) # should quickly be completed
        self.pool.join()

        self.assertEqual(self.pool.getCompletedItems(), [cmd])

    def test_pool_is_not_marked_done_until_commands_finish(self):
        cmd = mock.Mock(spec=Command)

        # cmd.run() will block until this Event is set.
        event = threading.Event()
        def wait_for_event():
            event.wait()
        cmd.run.side_effect = wait_for_event

        self.assertTrue(self.pool.isDone())

        try:
            self.pool.addCommand(cmd)
            self.assertFalse(self.pool.isDone())

        finally:
            # Make sure that we unblock the thread even on a test failure.
            event.set()

        self.pool.join()

        self.assertTrue(self.pool.isDone())

    def test_pool_can_be_emptied_of_completed_commands(self):
        cmd = mock.Mock(spec=Command)

        self.pool.addCommand(cmd)
        self.pool.join()

        self.pool.empty_completed_items()
        self.assertEqual(self.pool.getCompletedItems(), [])

    def test_check_results_succeeds_when_no_items_fail(self):
        cmd = mock.Mock(spec=Command)

        # Command.get_results() returns a CommandResult.
        # CommandResult.wasSuccessful() should return True if the command
        # succeeds.
        result = cmd.get_results.return_value
        result.wasSuccessful.return_value = True

        self.pool.addCommand(cmd)
        self.pool.join()
        self.pool.check_results()

    def test_check_results_throws_exception_at_first_failure(self):
        cmd = mock.Mock(spec=Command)

        # Command.get_results() returns a CommandResult.
        # CommandResult.wasSuccessful() should return False to simulate a
        # failure.
        result = cmd.get_results.return_value
        result.wasSuccessful.return_value = False

        self.pool.addCommand(cmd)
        self.pool.join()

        with self.assertRaises(ExecutionError):
            self.pool.check_results()

    def test_join_with_timeout_returns_done_immediately_if_there_is_nothing_to_do(self):
        start = time.time()
        done = self.pool.join(10)
        delta = time.time() - start

        self.assertTrue(done)

        # "Returns immediately" is a difficult thing to test. Longer than two
        # seconds seems like a reasonable failure case, even on a heavily loaded
#.........这里部分代码省略.........

# 需要导入模块: from gppylib.commands.base import WorkerPool [as 别名]
# 或者: from gppylib.commands.base.WorkerPool import check_results [as 别名]
class WorkerPoolTest(unittest.TestCase):
    def setUp(self):
        self.pool = WorkerPool(numWorkers=1, logger=mock.Mock())

    def tearDown(self):
        # All background threads must be stopped, or else the test runner will
        # hang waiting. Join the stopped threads to make sure we're completely
        # clean for the next test.
        self.pool.haltWork()
        self.pool.joinWorkers()

    def test_pool_must_have_some_workers(self):
        with self.assertRaises(Exception):
            WorkerPool(numWorkers=0)
        
    def test_pool_runs_added_command(self):
        cmd = mock.Mock(spec=Command)

        self.pool.addCommand(cmd)
        self.pool.join()

        cmd.run.assert_called_once_with()

    def test_completed_commands_are_retrievable(self):
        cmd = mock.Mock(spec=Command)

        self.pool.addCommand(cmd) # should quickly be completed
        self.pool.join()

        self.assertEqual(self.pool.getCompletedItems(), [cmd])

    def test_pool_is_not_marked_done_until_commands_finish(self):
        cmd = mock.Mock(spec=Command)

        # cmd.run() will block until this Event is set.
        event = threading.Event()
        def wait_for_event():
            event.wait()
        cmd.run.side_effect = wait_for_event

        self.assertTrue(self.pool.isDone())

        try:
            self.pool.addCommand(cmd)
            self.assertFalse(self.pool.isDone())

        finally:
            # Make sure that we unblock the thread even on a test failure.
            event.set()

        self.pool.join()

        self.assertTrue(self.pool.isDone())

    def test_pool_can_be_emptied_of_completed_commands(self):
        cmd = mock.Mock(spec=Command)

        self.pool.addCommand(cmd)
        self.pool.join()

        self.pool.empty_completed_items()
        self.assertEqual(self.pool.getCompletedItems(), [])

    def test_check_results_succeeds_when_no_items_fail(self):
        cmd = mock.Mock(spec=Command)

        # Command.get_results() returns a CommandResult.
        # CommandResult.wasSuccessful() should return True if the command
        # succeeds.
        result = cmd.get_results.return_value
        result.wasSuccessful.return_value = True

        self.pool.addCommand(cmd)
        self.pool.join()
        self.pool.check_results()

    def test_check_results_throws_exception_at_first_failure(self):
        cmd = mock.Mock(spec=Command)

        # Command.get_results() returns a CommandResult.
        # CommandResult.wasSuccessful() should return False to simulate a
        # failure.
        result = cmd.get_results.return_value
        result.wasSuccessful.return_value = False

        self.pool.addCommand(cmd)
        self.pool.join()

        with self.assertRaises(ExecutionError):
            self.pool.check_results()

    def test_join_with_timeout_returns_done_immediately_if_there_is_nothing_to_do(self):
        start = time.time()
        done = self.pool.join(10)
        delta = time.time() - start

        self.assertTrue(done)

        # "Returns immediately" is a difficult thing to test. Longer than two
        # seconds seems like a reasonable failure case, even on a heavily loaded
#.........这里部分代码省略.........