Python WorkerPool.wait_and_printdots方法代码示例

# 需要导入模块: from gppylib.commands.base import WorkerPool [as 别名]
# 或者: from gppylib.commands.base.WorkerPool import wait_and_printdots [as 别名]
    def execute(self):
        entry = ValidateVerificationEntry(token = self.token).run()
        if not entry['verdone']:   
            raise WrongStateError("Only finished verification tasks may be cleaned up.")

        path = os.path.join(get_masterdatadir(), 'pg_verify', self.token)
        Command('cleanup', 'rm -rf %s' % path).run(validateAfter=True)
        #RemoveTree(path).run()

        to_clean = ValidateVerification(content = entry['vercontent'],
                                        primaries_only = False).run()
        pool = WorkerPool(min(len(to_clean), self.batch_default))
        for seg in to_clean:
            host = seg.getSegmentHostName()
            path = os.path.join(seg.getSegmentDataDirectory(), 'pg_verify', "*%s*" % self.token)
            cmd = Command('cleanup', 'rm -f %s' % path, remoteHost=host)
            pool.addCommand(cmd)

        logger.info('Waiting for clean commands to complete...')
        pool.wait_and_printdots(len(to_clean))

        for cmd in pool.getCompletedItems():
            res = cmd.get_results()
            if not res.wasSuccessful():
                logger.error('Failed to send cleanup on %s' % cmd.host)
                logger.error('Error: %s' % res.stderr)
                raise CleanVerificationError()
        RemoveVerificationEntry(token = self.token).run()
        logger.info('Verification %s has been cleaned.' % self.token)

# 需要导入模块: from gppylib.commands.base import WorkerPool [as 别名]
# 或者: from gppylib.commands.base.WorkerPool import wait_and_printdots [as 别名]
class TriggerFilerepVerifyMessages(Operation):
    def __init__(self, content, token, batch_default, full=None, verify_file=None, verify_dir=None,
                       abort=None, suspend=None, resume=None, ignore_dir=None, ignore_file=None,
                       results=None, results_level=None):
        self.content = content
        self.token = token
        self.full = full
        self.verify_file = verify_file
        self.verify_dir = verify_dir
        self.abort = abort
        self.suspend = suspend
        self.resume = resume
        self.ignore_dir = ignore_dir
        self.ignore_file = ignore_file
        self.results = results
        self.results_level = results_level
        self.batch_default = batch_default
        self.pool = None
    def execute(self):
        """
        Sends arbitrary gp_primarymirror requests to the backend processes defined.
        """
        to_trigger = ValidateVerification(content = self.content).run()

        logger.info('Sending gp_primarymirror requests...')
        self.pool = WorkerPool(min(len(to_trigger), self.batch_default))

        for pseg in to_trigger:
            host, port = pseg.getSegmentHostName(), pseg.getSegmentPort()
            cmd = SendFilerepVerifyMessage(name = 'verify %s' % host, host = host, port = port,
                                           token = self.token,
                                           full = self.full,
                                           verify_file = self.verify_file,
                                           verify_dir = self.verify_dir,
                                           abort = self.abort,
                                           suspend = self.suspend,
                                           resume = self.resume,
                                           ignore_dir = self.ignore_dir,
                                           ignore_file = self.ignore_file,
                                           results = self.results,
                                           results_level = self.results_level)
            logger.debug("Sending request to %s:%d" % (host, port))
            self.pool.addCommand(cmd)

        logger.info('Waiting for gp_primarymirror commands to complete...')
        self.pool.wait_and_printdots(len(to_trigger))

        for cmd in self.pool.getCompletedItems():
            res = cmd.get_results()
            if not res.wasSuccessful():
                logger.error('Failed to send gp_primarymirror message to %s:%s' % (cmd.host, cmd.port))
                logger.error('Error: %s' % res.stderr)
                raise TriggerGpPrimaryMirrorFailure()
        logger.info('gp_primarymirror messages have been triggered succesfully.')

# 需要导入模块: from gppylib.commands.base import WorkerPool [as 别名]
# 或者: from gppylib.commands.base.WorkerPool import wait_and_printdots [as 别名]
class GatherResults(Operation):
    def __init__(self, master_datadir, token, content, batch_default):
        self.master_datadir = master_datadir
        self.token = token
        self.content = content
        self.batch_default = batch_default
        self.pool = None
    def execute(self):
        logger.info('Gathering results of verification %s...' % self.token)
        to_gather = ValidateVerification(content = self.content,
                                         primaries_only = False).run()

        dest_base = os.path.join(self.master_datadir, 'pg_verify', self.token)
        if CheckDir(dest_base).run():
            # TODO: if end user has mucked around with artifacts on master, a regathering may
            # be needed; perhaps, a --force option to accompany --results?
            return
        MakeDir(dest_base).run()

        self.pool = WorkerPool(min(len(to_gather), self.batch_default))
        for seg in to_gather:
            host = seg.getSegmentHostName()
            content = seg.getSegmentContentId()
            role = seg.getSegmentRole()
            src = os.path.join(seg.getSegmentDataDirectory(), "pg_verify", "*%s*" % self.token)

            dest = os.path.join(dest_base, str(content), str(role))
            MakeDir(dest).run()
            cmd = Scp('consolidate', srcFile=src, srcHost=host, dstFile=dest)
            self.pool.addCommand(cmd)

        logger.info('Waiting for scp commands to complete...')
        self.pool.wait_and_printdots(len(to_gather))
        self.pool.check_results()

        dest = os.path.join(dest_base, 'verification_%s.fix' % self.token)
        with open(dest, 'w') as output:
            for seg in to_gather:
                content = seg.getSegmentContentId()
                role = seg.getSegmentRole()
                src = os.path.join(dest_base, str(content), str(role), 'verification_%s.fix' % self.token)
                with open(src, 'r') as input:
                    output.writelines(input.readlines())

# 需要导入模块: from gppylib.commands.base import WorkerPool [as 别名]
# 或者: from gppylib.commands.base.WorkerPool import wait_and_printdots [as 别名]
    def rebalance(self):
        # Get the unbalanced primary segments grouped by hostname
        # These segments are what we will shutdown.
        logger.info("Getting unbalanced segments")
        unbalanced_primary_segs = GpArray.getSegmentsByHostName(self.gpArray.get_unbalanced_primary_segdbs())
        pool = WorkerPool()
        
        count = 0

        try:        
            # Disable ctrl-c
            signal.signal(signal.SIGINT,signal.SIG_IGN)
            
            logger.info("Stopping unbalanced primary segments...")
            for hostname in unbalanced_primary_segs.keys():
                cmd = GpSegStopCmd("stop unbalanced primary segs",
                                   self.gpEnv.getGpHome(),
                                   self.gpEnv.getGpVersion(),
                                   'fast',
                                   unbalanced_primary_segs[hostname],
                                   ctxt=REMOTE,
                                   remoteHost=hostname,
                                   timeout=600)
                pool.addCommand(cmd)
                count+=1
                
            pool.wait_and_printdots(count, False)
            
            failed_count = 0
            completed = pool.getCompletedItems()
            for res in completed:
                if not res.get_results().wasSuccessful():
                    failed_count+=1
                    
            if failed_count > 0:
                logger.warn("%d segments failed to stop.  A full rebalance of the")
                logger.warn("system is not possible at this time.  Please check the")
                logger.warn("log files, correct the problem, and run gprecoverseg -r")
                logger.warn("again.")
                logger.info("gprecoverseg will continue with a partial rebalance.")
            
            pool.empty_completed_items()
            # issue a distributed query to make sure we pick up the fault
            # that we just caused by shutting down segments
            conn = None
            try:
                logger.info("Triggering segment reconfiguration")
                dburl = dbconn.DbURL()
                conn = dbconn.connect(dburl)
                cmd = ReconfigDetectionSQLQueryCommand(conn)
                pool.addCommand(cmd)
                pool.wait_and_printdots(1, False)
            except Exception:
                # This exception is expected
                pass
            finally:
                if conn:
                    conn.close()

            # Final step is to issue a recoverseg operation to resync segments
            logger.info("Starting segment synchronization")
            cmd = GpRecoverseg("rebalance recoverseg")
            pool.addCommand(cmd)
            pool.wait_and_printdots(1, False)
        except Exception, ex:
            raise ex