Python WorkerPool.joinWorkers方法代码示例

# 需要导入模块: from gppylib.commands.base import WorkerPool [as 别名]
# 或者: from gppylib.commands.base.WorkerPool import joinWorkers [as 别名]
def restore_pg_hba_on_segment(gparr):
    """
    Restore the pg_hba.conf on all of the segments
    present in the array
    """
    logger.debug('Restoring pg_hba.conf file on segments...')

    host_to_seg_map = defaultdict(list)
    for seg in gparr.getDbList():
        if not seg.isSegmentMaster() and not seg.isSegmentStandby():
            host_to_seg_map[seg.getSegmentHostName()].append(seg.getSegmentDataDirectory())

    pool = WorkerPool(numWorkers=DEFAULT_BATCH_SIZE)

    try:
        for host, data_dirs_list in host_to_seg_map.items():
            pickled_data_dirs_list = base64.urlsafe_b64encode(pickle.dumps(data_dirs_list))
            cmdStr = "$GPHOME/lib/python/gppylib/operations/initstandby.py -d %s -r" % pickled_data_dirs_list
            cmd = Command('Restore the pg_hba.conf on remote hosts', cmdStr=cmdStr , ctxt=REMOTE, remoteHost=host)
            pool.addCommand(cmd)

        pool.join()

        for item in pool.getCompletedItems():
            result = item.get_results()
            if result.rc != 0:
                logger.error('Unable to restore pg_hba.conf %s' % str(result.stderr))
                logger.error('Please check the segment for more details')

    finally:
        pool.haltWork()
        pool.joinWorkers()
        pool = None

# 需要导入模块: from gppylib.commands.base import WorkerPool [as 别名]
# 或者: from gppylib.commands.base.WorkerPool import joinWorkers [as 别名]
class ConcurrentFilespaceMoveTestCase(unittest.TestCase):
    """ This test suite tests the scenario of running gpfilespace concurrently while
        trying to move the filespace. 
        The expected behavior is that only one of the processes succeeds and the 
        rest error out."""

    ALREADY_RUNNING_MSG = 'Another instance of gpfilespace is already running!'

    def setUp(self):
        self.pool = None
        self.pool = WorkerPool()

    def tearDown(self):
        if self.pool:
            self.pool.haltWork()
            self.pool.joinWorkers()
            self.pool.join()

    def get_move_filespace_cmd(self, filespace='myfspc', file_type=FileType.TEMPORARY_FILES):
        if file_type == FileType.TEMPORARY_FILES:
            file_type = 'movetempfiles'
        elif file_type == FileType.TRANSACTION_FILES:
            file_type = 'movetransfiles'

        return Command(name='move filespace', cmdStr='gpfilespace --%s %s' % (file_type, filespace))

    def run_concurrently(self, cmd_list):

        for cmd in cmd_list:
            self.pool.addCommand(cmd)
        self.pool.join()

    def check_concurrent_execution_result(self, execution_results):

        succeeded = 0
        for cmd in execution_results:
            results = cmd.get_results().stdout.strip()
            if self.ALREADY_RUNNING_MSG in results:
                continue
            succeeded += 1

        self.assertEqual(succeeded, 1)
            
    def test00_move_temp_filespace(self):

        cmd_list = [self.get_move_filespace_cmd(file_type=FileType.TEMPORARY_FILES) for i in range(2)]
        self.run_concurrently(cmd_list)
        self.check_concurrent_execution_result(self.pool.getCompletedItems())
            
    def test01_move_trans_filespace(self):

        cmd_list = [self.get_move_filespace_cmd(file_type=FileType.TRANSACTION_FILES) for i in range(2)]
        self.run_concurrently(cmd_list)
        self.check_concurrent_execution_result(self.pool.getCompletedItems())

    def test02_move_temp_and_trans_filespace(self):
        
        cmd_list = [self.get_move_filespace_cmd(file_type=FileType.TEMPORARY_FILES), self.get_move_filespace_cmd(file_type=FileType.TRANSACTION_FILES)]
        self.run_concurrently(cmd_list) 
        self.check_concurrent_execution_result(self.pool.getCompletedItems())

# 需要导入模块: from gppylib.commands.base import WorkerPool [as 别名]
# 或者: from gppylib.commands.base.WorkerPool import joinWorkers [as 别名]
    def execute(self):
        pool = WorkerPool()
        try:
            for seg in self.segments:
                datadir = seg.getSegmentDataDirectory()
                postmaster_pid_file = '%s/postmaster.pid' % datadir
                shared_mem = None
                if os.path.isfile(postmaster_pid_file):
                    with open(postmaster_pid_file) as fp:
                        shared_mem = fp.readlines()[-1].split()[-1].strip()
                if shared_mem:
                    cmd = Command('clean up shared memory', cmdStr="ipcrm -m %s" % shared_mem) 
                    pool.addCommand(cmd)
                pool.join()

            for item in pool.getCompletedItems():
                result = item.get_results()

                # This code is usually called after a GPDB segment has
                # been terminated.  In that case, it is possible that
                # the shared memory has already been freed by the
                # time we are called to clean up.  Due to this race
                # condition, it is possible to get an `ipcrm: invalid
                # id1` error from ipcrm.  We, therefore, ignore it.
                if result.rc != 0 and not result.stderr.startswith("ipcrm: invalid id"):
                    raise Exception('Unable to clean up shared memory for segment: (%s)' % (result.stderr))
        finally:
            pool.haltWork()
            pool.joinWorkers()
            pool = None

# 需要导入模块: from gppylib.commands.base import WorkerPool [as 别名]
# 或者: from gppylib.commands.base.WorkerPool import joinWorkers [as 别名]
    def validate_nic_down(self):
        """     
            Ping validation on the nics.
        """     

        pool = WorkerPool()

        try:    
            for nic, hostname in self.nic_to_address_map:
                address = self.nic_to_address_map[(nic, hostname)]
                cmd = Ping('ping validation', address, ctxt=REMOTE, remoteHost='localhost')
                pool.addCommand(cmd)
            pool.join()

            for cmd in pool.getCompletedItems():
                results = cmd.get_results()
                if results.rc == 0:
                    return False
        finally:
            pool.haltWork()
            pool.joinWorkers()
            pool.join()

        tinctest.logger.info("Successfully brought down nics ...")   
        return True

# 需要导入模块: from gppylib.commands.base import WorkerPool [as 别名]
# 或者: from gppylib.commands.base.WorkerPool import joinWorkers [as 别名]
 def _get_pgcontrol_data_from_segments(self, gpdb_list):
     pool = WorkerPool(numWorkers=self.workers)
     try:
         for gpdb in gpdb_list:  # iterate for all segments
             cmd = PgControlData(name='run pg_controldata', datadir=gpdb.getSegmentDataDirectory(),
                                 ctxt=REMOTE, remoteHost=gpdb.getSegmentHostName())
             cmd.gparray_gpdb = gpdb
             pool.addCommand(cmd)
         pool.join()
     finally:
         # Make sure that we halt the workers or else we'll hang
         pool.haltWork()
         pool.joinWorkers()
     return pool.getCompletedItems()

# 需要导入模块: from gppylib.commands.base import WorkerPool [as 别名]
# 或者: from gppylib.commands.base.WorkerPool import joinWorkers [as 别名]
def findFsDetails():
    global serverFSMap
    try:
        # find the mount points in parallel
        pool = WorkerPool()

        for hname in serverFSMap.keys():
            hname.strip()
            subCmd = "df -P %s" % (serverFSMap[hname])
            cmdStr = 'ssh -o PasswordAuthentication=no %s "%s"' % (hname, subCmd)
            pool.addCommand(Command(hname, cmdStr, REMOTE, hname))
        pool.join()
        items = pool.getCompletedItems()
        for i in items:
            if i.results.rc == 0:
                df_with_header = i.results.stdout.strip()
                df_list = df_with_header.splitlines()
                df_list.pop(0)
                fsList = serverFSMap[i.remoteHost].split()
                if len(df_list) != len(fsList):
                    print "Mismatch"
                    continue
                for df_vals in df_list:
                    df_val = df_vals.split()
                    fsDetailsMap[fsList.pop(0).strip()] = [i.remoteHost, df_val[0], df_val[5]]
            else:
                print("Failure in talking to host %s" % (i.remoteHost))

        pool.join()
        pool.haltWork()
        pool.joinWorkers()

    except Exception, e:
        print e.__str__()
        pool.join()
        pool.haltWork()
        pool.joinWorkers()

# 需要导入模块: from gppylib.commands.base import WorkerPool [as 别名]
# 或者: from gppylib.commands.base.WorkerPool import joinWorkers [as 别名]
    def bring_down_nic(self, nics, hostname):
        """
            Bring down nics based on the input nic names
        """ 
        if nics is None:
            return False

        pool = WorkerPool()

        try:    
            #get the ip address of the interface
            for nic in nics:
                cmd = Command(name='get the ip of the interface', cmdStr="/sbin/ifconfig %s | grep \'inet addr:\' | cut -d: -f2 | awk \'{ print $1}\'" % nic, ctxt=REMOTE, remoteHost=hostname)
                cmd.run(validateAfter=True)
                results = cmd.get_results()
                if results.rc != 0:
                    raise Exception('Unable to map interface to ipaddress') 

                self.nic_to_address_map[(nic, hostname)] = results.stdout.split()[0].strip()

            for nic in nics:
                tinctest.logger.info("Bringing down %s:%s ..." % (hostname, nic))   
                cmd = Command(name='bring NIC down', cmdStr='sudo /sbin/ifdown %s' % nic, ctxt=REMOTE, remoteHost=hostname)
                pool.addCommand(cmd)

            pool.join()
            for cmd in pool.getCompletedItems():
                results = cmd.get_results()
                if results.rc != 0:
                    return False
        finally:
            pool.haltWork()
            pool.joinWorkers()
            pool.join()

        return True

# 需要导入模块: from gppylib.commands.base import WorkerPool [as 别名]
# 或者: from gppylib.commands.base.WorkerPool import joinWorkers [as 别名]
class WorkerPoolTest(unittest.TestCase):
    def setUp(self):
        self.pool = WorkerPool(numWorkers=1, logger=mock.Mock())

    def tearDown(self):
        # All background threads must be stopped, or else the test runner will
        # hang waiting. Join the stopped threads to make sure we're completely
        # clean for the next test.
        self.pool.haltWork()
        self.pool.joinWorkers()

    def test_pool_must_have_some_workers(self):
        with self.assertRaises(Exception):
            WorkerPool(numWorkers=0)
        
    def test_pool_runs_added_command(self):
        cmd = mock.Mock(spec=Command)

        self.pool.addCommand(cmd)
        self.pool.join()

        cmd.run.assert_called_once_with()

    def test_completed_commands_are_retrievable(self):
        cmd = mock.Mock(spec=Command)

        self.pool.addCommand(cmd) # should quickly be completed
        self.pool.join()

        self.assertEqual(self.pool.getCompletedItems(), [cmd])

    def test_pool_is_not_marked_done_until_commands_finish(self):
        cmd = mock.Mock(spec=Command)

        # cmd.run() will block until this Event is set.
        event = threading.Event()
        def wait_for_event():
            event.wait()
        cmd.run.side_effect = wait_for_event

        self.assertTrue(self.pool.isDone())

        try:
            self.pool.addCommand(cmd)
            self.assertFalse(self.pool.isDone())

        finally:
            # Make sure that we unblock the thread even on a test failure.
            event.set()

        self.pool.join()

        self.assertTrue(self.pool.isDone())

    def test_pool_can_be_emptied_of_completed_commands(self):
        cmd = mock.Mock(spec=Command)

        self.pool.addCommand(cmd)
        self.pool.join()

        self.pool.empty_completed_items()
        self.assertEqual(self.pool.getCompletedItems(), [])

    def test_check_results_succeeds_when_no_items_fail(self):
        cmd = mock.Mock(spec=Command)

        # Command.get_results() returns a CommandResult.
        # CommandResult.wasSuccessful() should return True if the command
        # succeeds.
        result = cmd.get_results.return_value
        result.wasSuccessful.return_value = True

        self.pool.addCommand(cmd)
        self.pool.join()
        self.pool.check_results()

    def test_check_results_throws_exception_at_first_failure(self):
        cmd = mock.Mock(spec=Command)

        # Command.get_results() returns a CommandResult.
        # CommandResult.wasSuccessful() should return False to simulate a
        # failure.
        result = cmd.get_results.return_value
        result.wasSuccessful.return_value = False

        self.pool.addCommand(cmd)
        self.pool.join()

        with self.assertRaises(ExecutionError):
            self.pool.check_results()

    def test_join_with_timeout_returns_done_immediately_if_there_is_nothing_to_do(self):
        start = time.time()
        done = self.pool.join(10)
        delta = time.time() - start

        self.assertTrue(done)

        # "Returns immediately" is a difficult thing to test. Longer than two
        # seconds seems like a reasonable failure case, even on a heavily loaded
#.........这里部分代码省略.........

# 需要导入模块: from gppylib.commands.base import WorkerPool [as 别名]
# 或者: from gppylib.commands.base.WorkerPool import joinWorkers [as 别名]
class WorkerPoolTest(unittest.TestCase):
    def setUp(self):
        self.pool = WorkerPool(numWorkers=1, logger=mock.Mock())

    def tearDown(self):
        # All background threads must be stopped, or else the test runner will
        # hang waiting. Join the stopped threads to make sure we're completely
        # clean for the next test.
        self.pool.haltWork()
        self.pool.joinWorkers()

    def test_pool_must_have_some_workers(self):
        with self.assertRaises(Exception):
            WorkerPool(numWorkers=0)
        
    def test_pool_runs_added_command(self):
        cmd = mock.Mock(spec=Command)

        self.pool.addCommand(cmd)
        self.pool.join()

        cmd.run.assert_called_once_with()

    def test_completed_commands_are_retrievable(self):
        cmd = mock.Mock(spec=Command)

        self.pool.addCommand(cmd) # should quickly be completed
        self.pool.join()

        self.assertEqual(self.pool.getCompletedItems(), [cmd])

    def test_pool_is_not_marked_done_until_commands_finish(self):
        cmd = mock.Mock(spec=Command)

        # cmd.run() will block until this Event is set.
        event = threading.Event()
        def wait_for_event():
            event.wait()
        cmd.run.side_effect = wait_for_event

        self.assertTrue(self.pool.isDone())

        try:
            self.pool.addCommand(cmd)
            self.assertFalse(self.pool.isDone())

        finally:
            # Make sure that we unblock the thread even on a test failure.
            event.set()

        self.pool.join()

        self.assertTrue(self.pool.isDone())

    def test_pool_can_be_emptied_of_completed_commands(self):
        cmd = mock.Mock(spec=Command)

        self.pool.addCommand(cmd)
        self.pool.join()

        self.pool.empty_completed_items()
        self.assertEqual(self.pool.getCompletedItems(), [])

    def test_check_results_succeeds_when_no_items_fail(self):
        cmd = mock.Mock(spec=Command)

        # Command.get_results() returns a CommandResult.
        # CommandResult.wasSuccessful() should return True if the command
        # succeeds.
        result = cmd.get_results.return_value
        result.wasSuccessful.return_value = True

        self.pool.addCommand(cmd)
        self.pool.join()
        self.pool.check_results()

    def test_check_results_throws_exception_at_first_failure(self):
        cmd = mock.Mock(spec=Command)

        # Command.get_results() returns a CommandResult.
        # CommandResult.wasSuccessful() should return False to simulate a
        # failure.
        result = cmd.get_results.return_value
        result.wasSuccessful.return_value = False

        self.pool.addCommand(cmd)
        self.pool.join()

        with self.assertRaises(ExecutionError):
            self.pool.check_results()

    def test_join_with_timeout_returns_done_immediately_if_there_is_nothing_to_do(self):
        start = time.time()
        done = self.pool.join(10)
        delta = time.time() - start

        self.assertTrue(done)

        # "Returns immediately" is a difficult thing to test. Longer than two
        # seconds seems like a reasonable failure case, even on a heavily loaded
#.........这里部分代码省略.........